| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Format::TCF.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: Datum parser|formatter: XML: CLARIN-D TCF (selected features only) | ||
| 6 | ## + uses DTA::CAB::Format::XmlTokWrap for output | ||
| 7 | |||
| 8 | package DTA::CAB::Format::TCF; | ||
| 9 | use DTA::CAB::Format::XmlCommon; | ||
| 10 | use DTA::CAB::Format::Raw; ##-- for tcf text tokenization | ||
| 11 | use DTA::CAB::Datum ':all'; | ||
| 12 | #use DTA::CAB::Utils ':temp'; | ||
| 13 | use XML::LibXML; | ||
| 14 | use IO::File; | ||
| 15 | use Carp; | ||
| 16 | use strict; | ||
| 17 | |||
| 18 | ##============================================================================== | ||
| 19 | ## Globals | ||
| 20 | ##============================================================================== | ||
| 21 | |||
| 22 | our @ISA = qw(DTA::CAB::Format::XmlCommon); | ||
| 23 | |||
| 24 | BEGIN { | ||
| 25 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, filenameRegex=>qr/\.(?i:(?:tcf[\.\-_]?xml)|(?:tcf))$/); | ||
| 26 | |||
| 27 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'text'}) | ||
| 28 | foreach (qw(tcf-text tcf+text)); | ||
| 29 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'text tokens sentences'}) | ||
| 30 | foreach (qw(tcf-tok tcf+tok)); | ||
| 31 | |||
| 32 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tokens sentences orthography'}) | ||
| 33 | foreach (qw(tcf-orth tcf+orth tcf-web)); ##-- for weblicht/dta | ||
| 34 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tokens sentences postags lemmas'}) | ||
| 35 | foreach (qw(tcf-pos tcf+pos)); ##-- for weblicht/d* | ||
| 36 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tokens sentences orthography postags lemmas'}) | ||
| 37 | foreach (qw(tcf tcf-xml tcfxml full-tcf xtcf)); | ||
| 38 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tokens sentences orthography postags lemmas names'}) | ||
| 39 | foreach (qw(tcf+ner tcf+names)); ##-- for including named entities parsed from teiws | ||
| 40 | |||
| 41 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tei text'}) | ||
| 42 | foreach (qw(tcf-tei-text tcf-tei+text tcf+tei+text)); | ||
| 43 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tei text tokens sentences'}) | ||
| 44 | foreach (qw(tcf-tei-tok tcf-tei+tok tcf+tei+tok)); | ||
| 45 | } | ||
| 46 | |||
| 47 | BEGIN { | ||
| 48 | *isa = \&UNIVERSAL::isa; | ||
| 49 | *can = \&UNIVERSAL::can; | ||
| 50 | } | ||
| 51 | |||
| 52 | ##============================================================================== | ||
| 53 | ## Constructors etc. | ||
| 54 | ##============================================================================== | ||
| 55 | |||
| 56 | ## $fmt = CLASS_OR_OBJ->new(%args) | ||
| 57 | ## + object structure: HASH ref | ||
| 58 | ## { | ||
| 59 | ## ##-- new in TCF | ||
| 60 | ## tcfbufr => \$buf, ##-- raw TCF buffer, for spliceback mode | ||
| 61 | ## textbufr => \$text, ##-- raw text buffer, for spliceback mode | ||
| 62 | ## tcflog => $level, ##-- debugging log-level (default: 'off') | ||
| 63 | ## spliceback => $bool, ##-- (output) if true (default), splice data back into 'tcfbufr' if available; otherwise create new TCF doc | ||
| 64 | ## tcflayers => $tcf_layer_names, ##-- layer names to include, space-separated list; known='tei text tokens sentences postags lemmas orthography' | ||
| 65 | ## tcftagset => $tagset, ##-- tagset name for POStags element (default='stts') | ||
| 66 | ## logsplice => $level, ##-- log level for spliceback messages (default:'none') | ||
| 67 | ## trimtext => $bool, ##-- if true (default), waste tokenizer hints will be trimmed from 'text' layer | ||
| 68 | ## | ||
| 69 | ## ##-- input: inherited from XmlCommon | ||
| 70 | ## xdoc => $xdoc, ##-- XML::LibXML::Document | ||
| 71 | ## xprs => $xprs, ##-- XML::LibXML parser | ||
| 72 | ## | ||
| 73 | ## ##-- output: inherited from XmlCommon | ||
| 74 | ## level => $level, ##-- output formatting level (OVERRIDE: default=1) | ||
| 75 | ## output => [$how,$arg] ##-- either ['fh',$fh], ['file',$filename], or ['str',\$buf] | ||
| 76 | ## } | ||
| 77 | sub new { | ||
| 78 | my $that = shift; | ||
| 79 | my $fmt = $that->SUPER::new( | ||
| 80 | ##-- local | ||
| 81 | #tcfbufr => undef, | ||
| 82 | tcflog => 'off', ##-- debugging log-level | ||
| 83 | #tcflayers => 'tei text tokens sentences orthography postags lemmas names', | ||
| 84 | tcflayers => 'tokens sentences orthography', | ||
| 85 | tcftagset => 'stts', | ||
| 86 | spliceback => 1, | ||
| 87 | logsplice => 'none', | ||
| 88 | trimtext => 1, | ||
| 89 | |||
| 90 | ##-- overrides (XmlTokWrap, XmlNative, XmlCommon) | ||
| 91 | ignoreKeys => { | ||
| 92 | tcfbufr=>undef, | ||
| 93 | textbufr=>undef, | ||
| 94 | tcfdoc=>undef, | ||
| 95 | }, | ||
| 96 | xprsopts => {keep_blanks=>0}, | ||
| 97 | level => 1, | ||
| 98 | |||
| 99 | ##-- user args | ||
| 100 | @_ | ||
| 101 | ); | ||
| 102 | |||
| 103 | return $fmt; | ||
| 104 | } | ||
| 105 | |||
| 106 | |||
| 107 | ##============================================================================= | ||
| 108 | ## Methods: Generic | ||
| 109 | |||
| 110 | ##============================================================================= | ||
| 111 | ## Methods: Input | ||
| 112 | ##============================================================================== | ||
| 113 | |||
| 114 | ##-------------------------------------------------------------- | ||
| 115 | ## Methods: Input: Generic API | ||
| 116 | |||
| 117 | ## $fmt = $fmt->close() | ||
| 118 | ## + close current input source, if any | ||
| 119 | ## + INHERITED from XmlCommon (calls flush()) | ||
| 120 | |||
| 121 | ## $fmt = $fmt->from(String|File|Fh) | ||
| 122 | ## + INHERITED from XmlCommon : populates $fmt->{xdoc} | ||
| 123 | |||
| 124 | ## $doc = $fmt->parseDocument() | ||
| 125 | ## + parse buffered XML::LibXML::Document from $fmt->{xdoc} | ||
| 126 | sub parseDocument { | ||
| 127 | my $fmt = shift; | ||
| 128 | $fmt->vlog($fmt->{tcflog}, "parseDocument()"); | ||
| 129 | |||
| 130 | ##-- parse: basic | ||
| 131 | my $doc = DTA::CAB::Document->new(); | ||
| 132 | my $xdoc = $fmt->{xdoc}; | ||
| 133 | my $xroot = $xdoc->documentElement; | ||
| 134 | $doc->{tcfdoc} = $xdoc if ($fmt->{spliceback}); | ||
| 135 | |||
| 136 | ##-- parse: metadata | ||
| 137 | if (defined(my $xmeta = [$xroot->getChildrenByLocalName("MetaData")]->[0])) { | ||
| 138 | my ($xsrc) = $xmeta->getChildrenByLocalName('source'); | ||
| 139 | $doc->{source} = $xsrc->textContent if (defined($xsrc)); | ||
| 140 | } | ||
| 141 | |||
| 142 | ##-- parse: corpus | ||
| 143 | my $xcorpus = [$xroot->getChildrenByLocalName('TextCorpus')]->[0] | ||
| 144 | or $fmt->logconfess("parseDocument(): no TextCorpus node found in XML document"); | ||
| 145 | |||
| 146 | ##-- parse: text (textbufr) | ||
| 147 | ## + annoying hack: we grep for elements here b/c libxml getChildrenByLocalName('text') also returns text-nodes! | ||
| 148 | my ($xtext) = grep {UNIVERSAL::isa($_,'XML::LibXML::Element')} $xcorpus->getChildrenByLocalName('text'); | ||
| 149 | if ($xtext) { | ||
| 150 | my $textbuf = $xtext->textContent; | ||
| 151 | $doc->{textbufr} = \$textbuf; | ||
| 152 | } | ||
| 153 | |||
| 154 | ##-- check for pre-tokenized input | ||
| 155 | if (defined(my $xtokens = [$xcorpus->getChildrenByLocalName('tokens')]->[0])) { | ||
| 156 | ##------------ pre-tokenized input | ||
| 157 | ##-- parse: tokens | ||
| 158 | my (@w,%id2w,$w); | ||
| 159 | foreach ($xtokens->getChildrenByLocalName('token')) { | ||
| 160 | push(@w, $w={text=>$_->textContent}); | ||
| 161 | if (!defined($w->{id}=$_->getAttribute('ID'))) { | ||
| 162 | $w->{id} = sprintf("w%x", $#w); | ||
| 163 | $_->setAttribute('ID'=>$w->{id}); | ||
| 164 | } | ||
| 165 | $id2w{$w->{id}} = $w; | ||
| 166 | } | ||
| 167 | |||
| 168 | ##-- parse: sentences | ||
| 169 | my ($s); | ||
| 170 | if (defined(my $xsents = [$xcorpus->getChildrenByLocalName('sentences')]->[0])) { | ||
| 171 | my $body = $doc->{body}; | ||
| 172 | foreach ($xsents->getChildrenByLocalName('sentence')) { | ||
| 173 | push(@$body, $s={}); | ||
| 174 | $s->{id} = $_->getAttribute('ID') if ($_->hasAttribute('ID')); | ||
| 175 | $s->{tokens} = [ @id2w{split(' ',$_->getAttribute('tokenIDs'))} ]; | ||
| 176 | } | ||
| 177 | } else { | ||
| 178 | $doc->{body} = {tokens=>\@w}; ##-- single-sentence | ||
| 179 | } | ||
| 180 | |||
| 181 | ##-- parse: POStags -> moot/tag | ||
| 182 | my ($id); | ||
| 183 | if (defined(my $xpostags = [$xcorpus->getChildrenByLocalName('POStags')]->[0])) { | ||
| 184 | foreach ($xpostags->getChildrenByLocalName('tag')) { | ||
| 185 | $id = $_->getAttribute('tokenIDs'); | ||
| 186 | $id2w{$id}{moot}{tag} = $_->textContent; | ||
| 187 | } | ||
| 188 | } | ||
| 189 | |||
| 190 | ##-- parse: lemmas -> moot/lemma | ||
| 191 | if (defined(my $xlemmas = [$xcorpus->getChildrenByLocalName('lemmas')]->[0])) { | ||
| 192 | foreach ($xlemmas->getChildrenByLocalName('lemma')) { | ||
| 193 | $id = $_->getAttribute('tokenIDs'); | ||
| 194 | $id2w{$id}{moot}{lemma} = $_->textContent; | ||
| 195 | } | ||
| 196 | } | ||
| 197 | |||
| 198 | ##-- parse: orthography -> moot/word | ||
| 199 | if (defined(my $xorths = [$xcorpus->getChildrenByLocalName('orthography')]->[0])) { | ||
| 200 | foreach (grep {($_->getAttribute('operation')//'') eq 'replace'} $xorths->getChildrenByLocalName('correction')) { | ||
| 201 | $id = $_->getAttribute('tokenIDs'); | ||
| 202 | $id2w{$id}{moot}{word} = $_->textContent; | ||
| 203 | } | ||
| 204 | } | ||
| 205 | } | ||
| 206 | elsif ($doc->{textbufr}) { | ||
| 207 | ##------------ un-tokenized input | ||
| 208 | my $rawfmt = DTA::CAB::Format::Raw->new(); | ||
| 209 | my $rawdoc = DTA::CAB::Format::Raw->new->parseString( ${$doc->{textbufr}} ); | ||
| 210 | $doc->{body} = $rawdoc->{body}; | ||
| 211 | } | ||
| 212 | else { | ||
| 213 | ##------------ no source | ||
| 214 | $fmt->logconfess("parseDocument(): no TextCorpus/text or TextCorpus/tokens node found in XML document"); | ||
| 215 | } | ||
| 216 | |||
| 217 | $fmt->vlog($fmt->{tcflog}, "parseDocument(): returning"); | ||
| 218 | return $doc; | ||
| 219 | } | ||
| 220 | |||
| 221 | ##============================================================================= | ||
| 222 | ## Methods: Output | ||
| 223 | ##============================================================================== | ||
| 224 | |||
| 225 | ##-------------------------------------------------------------- | ||
| 226 | ## Methods: Output: MIME & HTTP stuff | ||
| 227 | |||
| 228 | ## $short = $fmt->shortName() | ||
| 229 | ## + returns "official" short name for this format | ||
| 230 | ## + default just returns package suffix | ||
| 231 | sub shortName { | ||
| 232 | return 'tcf'; | ||
| 233 | } | ||
| 234 | |||
| 235 | ## $type = $fmt->mimeType() | ||
| 236 | ## + override returns text/xml | ||
| 237 | sub mimeType { return 'text/tcf+xml'; } | ||
| 238 | |||
| 239 | ## $ext = $fmt->defaultExtension() | ||
| 240 | ## + returns default filename extension for this format (default='.cab') | ||
| 241 | sub defaultExtension { return '.tcf.xml'; } | ||
| 242 | |||
| 243 | ##-------------------------------------------------------------- | ||
| 244 | ## Methods: Output: output selection | ||
| 245 | |||
| 246 | ## $fmt = $fmt->flush() | ||
| 247 | ## + flush any buffered output to selected output source | ||
| 248 | sub flush { | ||
| 249 | my $fmt = shift; | ||
| 250 | $fmt->vlog($fmt->{tcflog}, "flush()") if (Log::Log4perl->initialized); | ||
| 251 | $fmt->SUPER::flush(@_) || return undef; | ||
| 252 | delete @$fmt{qw(tcfbufr tcfdoc outbufr)}; | ||
| 253 | return $fmt; | ||
| 254 | } | ||
| 255 | |||
| 256 | ## $fmt = $fmt->to(String|File|Fh) | ||
| 257 | ## + INHERITED from XmlCommon : sets up $fmt->{output}=($outputHow,$outputArg) | ||
| 258 | |||
| 259 | ##-------------------------------------------------------------- | ||
| 260 | ## Methods: Output: Generic API | ||
| 261 | |||
| 262 | ## $fmt = $fmt->putDocument($doc) | ||
| 263 | ## + override respects local 'spliceback' and 'tcflayers' flags | ||
| 264 | sub putDocument { | ||
| 265 | my ($fmt,$doc) = @_; | ||
| 266 | $fmt->vlog($fmt->{tcflog}, "putDocument()"); | ||
| 267 | |||
| 268 | ##-- common vars | ||
| 269 | my $spliceback = $fmt->{spliceback}; | ||
| 270 | my $layers = $fmt->{tcflayers} // ''; | ||
| 271 | |||
| 272 | ##-- spliceback? | ||
| 273 | my ($xdoc); | ||
| 274 | if ($spliceback) { | ||
| 275 | $xdoc = $doc->{tcfdoc} // $fmt->{tcfdoc}; | ||
| 276 | if (!$xdoc) { | ||
| 277 | my $bufr = $doc->{tcfbufr} // $fmt->{tcfbufr}; | ||
| 278 | if (!$bufr || !$$bufr) { | ||
| 279 | $fmt->vlog($fmt->{logsplice}, "spliceback mode requested but no 'tcfdoc' or 'tcfbufr' document property - creating new document!"); | ||
| 280 | $spliceback = 0; | ||
| 281 | } | ||
| 282 | } | ||
| 283 | } | ||
| 284 | $xdoc //= XML::LibXML::Document->new("1.0","UTF-8"); | ||
| 285 | $fmt->{xdoc} = $xdoc; | ||
| 286 | |||
| 287 | ##-- document structure: root | ||
| 288 | my ($xroot); | ||
| 289 | if (!defined($xroot = $xdoc->documentElement)) { | ||
| 290 | $xdoc->setDocumentElement( $xroot = $xdoc->createElement('D-Spin') ); | ||
| 291 | $xroot->setNamespace('http://www.dspin.de/data'); | ||
| 292 | $xroot->setAttribute('version'=>'0.4'); | ||
| 293 | } | ||
| 294 | |||
| 295 | ##-- document structure: metadata | ||
| 296 | my ($xmeta) = $xroot->getChildrenByLocalName('MetaData'); | ||
| 297 | if (!defined($xmeta)) { | ||
| 298 | $xmeta = $xroot->addNewChild(undef,'MetaData'); | ||
| 299 | $xmeta->setNamespace('http://www.dspin.de/data/metadata'); | ||
| 300 | $xmeta->appendTextChild('source', $doc->{source}) if (defined($doc->{source})); | ||
| 301 | } | ||
| 302 | |||
| 303 | ##-- document structure: corpus | ||
| 304 | my ($xcorpus) = $xroot->getChildrenByLocalName('TextCorpus'); | ||
| 305 | if (!defined($xcorpus)) { | ||
| 306 | $xcorpus = $xroot->addNewChild(undef,'TextCorpus'); | ||
| 307 | $xcorpus->setNamespace('http://www.dspin.de/data/textcorpus'); | ||
| 308 | $xcorpus->setAttribute('lang'=>'de'); | ||
| 309 | } | ||
| 310 | |||
| 311 | ##-- document structure: TextCorpus/tei | ||
| 312 | if ($layers =~ /\btei\b/ && defined($doc->{teibufr})) { | ||
| 313 | my ($xtei) = $xcorpus->getChildrenByLocalName("textSource"); | ||
| 314 | my $xtype = $xtei ? ($xtei->getAttribute('type')//'') : ''; | ||
| 315 | undef ($xtei) if ($xtype !~ m{^(?:text|application)/tei\+xml\b} || $xtype =~ m{\btokenized=(?![0n])}); | ||
| 316 | if (!defined($xtei)) { | ||
| 317 | $xtei = $xcorpus->addNewChild(undef,'textSource'); | ||
| 318 | $xtei->setAttribute('type'=>'application/tei+xml'); | ||
| 319 | $xtei->appendText(${$doc->{teibufr}}); | ||
| 320 | } | ||
| 321 | } | ||
| 322 | |||
| 323 | ##-- document structure: TextCorpus/text | ||
| 324 | if ($layers =~ /\btext\b/) { | ||
| 325 | ##-- annoying hack: we grep for elements here b/c libxml getChildrenByLocalName('text') also returns text-nodes! | ||
| 326 | my ($xtext) = grep {UNIVERSAL::isa($_,'XML::LibXML::Element')} $xcorpus->getChildrenByLocalName('text'); | ||
| 327 | if (!defined($xtext)) { | ||
| 328 | $xtext = $xcorpus->addNewChild(undef,'text'); | ||
| 329 | if (defined($doc->{textbufr})) { | ||
| 330 | ##-- use doc-buffered text content | ||
| 331 | my $txt = ${$doc->{textbufr}}; | ||
| 332 | $txt =~ s/\s*\$WB\$\s*/ /sg; | ||
| 333 | $txt =~ s/\s*\$SB\$\s*/\n\n/sg; | ||
| 334 | $txt =~ s/%%[^%]*%%//sg; | ||
| 335 | $xtext->appendText($txt); | ||
| 336 | } | ||
| 337 | else { | ||
| 338 | ##-- generate dummy text content | ||
| 339 | $xtext->appendText(join(' ', map {$_->{text}} @{$_->{tokens}})."\n") foreach (@{$doc->{body}}); | ||
| 340 | } | ||
| 341 | } | ||
| 342 | } | ||
| 343 | |||
| 344 | ##-- document structure: corpus structure | ||
| 345 | my ($tokens,$sents,$lemmas,$postags,$orths,$names); | ||
| 346 | if ($layers =~ /\btokens\b/ && !$xcorpus->getChildrenByLocalName('tokens')) { | ||
| 347 | $tokens = $xcorpus->addNewChild(undef,'tokens'); | ||
| 348 | } | ||
| 349 | if ($layers =~ /\bsentences\b/ && !$xcorpus->getChildrenByLocalName('sentences')) { | ||
| 350 | $sents = $xcorpus->addNewChild(undef,'sentences'); | ||
| 351 | } | ||
| 352 | if ($layers =~ /\blemmas\b/ && !$xcorpus->getChildrenByLocalName('lemmas')) { | ||
| 353 | $lemmas = $xcorpus->addNewChild(undef,'lemmas'); | ||
| 354 | #$lemmas->setAttribute('type'=>'CAB'); | ||
| 355 | } | ||
| 356 | if ($layers =~ /\bpostags\b/ && !$xcorpus->getChildrenByLocalName('POStags')) { | ||
| 357 | $postags = $xcorpus->addNewChild(undef,'POStags'); | ||
| 358 | $postags->setAttribute('tagset'=>$fmt->{tcftagset}) if ($fmt->{tcftagset}); | ||
| 359 | #$postags->setAttribute('type'=>'CAB'); | ||
| 360 | } | ||
| 361 | if ($layers =~ /\borthography\b/ && !$xcorpus->getChildrenByLocalName('orthography')) { | ||
| 362 | $orths = $xcorpus->addNewChild(undef,'orthography'); | ||
| 363 | #$orths->setAttribute('type'=>'CAB'); | ||
| 364 | } | ||
| 365 | if ($layers =~ /\bnames\b/ && !$xcorpus->getChildrenByLocalName('namedEntities')) { | ||
| 366 | $names = $xcorpus->addNewChild(undef,'namedEntities'); | ||
| 367 | $names->setAttribute('type'=>'CoNLL2002'); | ||
| 368 | } | ||
| 369 | |||
| 370 | ##-- ensure ids | ||
| 371 | my $wi = 0; | ||
| 372 | my $si = 0; | ||
| 373 | my $ni = 0; | ||
| 374 | my (%nid2nod); ##-- ($nerEntityId => $tcfNamedEntityNode), for 'names' layer | ||
| 375 | my ($s,$w,$wid,@wids,$snod,$wnod,$nnod); | ||
| 376 | my ($pos,$lemma,$orth,$nea,$neid,$necls); | ||
| 377 | foreach $s (@{$doc->{body}}) { | ||
| 378 | ++$si; | ||
| 379 | @wids = qw(); | ||
| 380 | foreach $w (@{$s->{tokens}}) { | ||
| 381 | ++$wi; | ||
| 382 | $wid = $w->{id} // sprintf("w%x",$wi); | ||
| 383 | push(@wids,$wid); | ||
| 384 | |||
| 385 | ##-- generate token node: <token ID="t_0">Karin</token> | ||
| 386 | if ($tokens) { | ||
| 387 | $wnod = $tokens->addNewChild(undef,'token'); | ||
| 388 | $wnod->setAttribute(ID=>$wid); | ||
| 389 | $wnod->appendText($w->{text}); | ||
| 390 | } | ||
| 391 | |||
| 392 | ##-- generate token data: tag, lemma, orthography | ||
| 393 | $pos = $lemma = $orth = undef; | ||
| 394 | if ($w->{moot}) { | ||
| 395 | $pos = $w->{moot}{tag}; | ||
| 396 | $lemma = $w->{moot}{lemma}; | ||
| 397 | $orth = $w->{moot}{word}; | ||
| 398 | } | ||
| 399 | elsif ($w->{dmoot}) { | ||
| 400 | $orth = $w->{dmoot}{tag}; | ||
| 401 | } | ||
| 402 | $orth //= $w->{exlex} // ($w->{xlit} && $w->{xlit}{isLatinExt} ? $w->{xlit}{latin1Text} : undef); | ||
| 403 | |||
| 404 | if ($postags && defined($pos)) { | ||
| 405 | ##-- POStags: <tag ID="pt_0" tokenIDs="t_0">NE</tag> | ||
| 406 | $wnod = $postags->addNewChild(undef,'tag'); | ||
| 407 | $wnod->setAttribute(tokenIDs=>$wid); | ||
| 408 | $wnod->appendText($pos); | ||
| 409 | } | ||
| 410 | if ($lemmas && defined($lemma)) { | ||
| 411 | ##-- lemmas: <lemma ID="le_0" tokenIDs="t_0">Karin</lemma> | ||
| 412 | $wnod = $lemmas->addNewChild(undef,'lemma'); | ||
| 413 | $wnod->setAttribute(tokenIDs=>$wid); | ||
| 414 | $wnod->appendText($lemma); | ||
| 415 | } | ||
| 416 | if ($orths && defined($orth) && $orth ne $w->{text}) { | ||
| 417 | ##-- orthography: <correction operation="replace" tokenIDs="t_0">Karina</correction> | ||
| 418 | $wnod = $orths->addNewChild(undef,'correction'); | ||
| 419 | $wnod->setAttribute(tokenIDs=>$wid); | ||
| 420 | $wnod->setAttribute(operation=>'replace'); ##-- "norm" would be better, but isn't allowed | ||
| 421 | $wnod->appendText($orth); | ||
| 422 | } | ||
| 423 | if ($names && defined($w->{ner})) { | ||
| 424 | ##-- names: populate %nid2nod | ||
| 425 | foreach $nea (@{$w->{ner}}) { | ||
| 426 | $neid = $nea->{nid} || "n$wid"; | ||
| 427 | if (defined($nnod=$nid2nod{$neid})) { | ||
| 428 | ##-- append token to existing tcf namedEntities/entity node | ||
| 429 | $nnod->setAttribute('tokenIDs',$nnod->getAttribute('tokenIDs').' '.$wid); | ||
| 430 | } else { | ||
| 431 | ##-- create new tcf namedEntities/entity node | ||
| 432 | ++$ni; | ||
| 433 | $nnod = $nid2nod{$neid} = $names->addNewChild(undef,'entity'); | ||
| 434 | |||
| 435 | ##-- CoNLL2002 ner categories: PER, LOC, ORG, MISC | ||
| 436 | $necls = $nea->{func} || 'MISC'; | ||
| 437 | $necls = ($necls =~ /^(?:per|\@(?:First|Last|Title))/i ? 'PER' | ||
| 438 | : ($necls =~ /^place|loc|geo/i ? 'LOC' | ||
| 439 | : ($necls =~ /^org/i ? 'ORG' | ||
| 440 | : 'MISC'))); | ||
| 441 | |||
| 442 | $nnod->setAttribute('ID' => $neid); | ||
| 443 | $nnod->setAttribute('class' => $necls); | ||
| 444 | $nnod->setAttribute('tokenIDs' => $wid); | ||
| 445 | } | ||
| 446 | } | ||
| 447 | } | ||
| 448 | } | ||
| 449 | |||
| 450 | if ($sents) { | ||
| 451 | ##-- generate sentence node: <sentence ID="s_0" tokenIDs="t_0 t_1 t_2 t_3 t_4 t_5"></sentence> | ||
| 452 | $snod = $sents->addNewChild(undef,'sentence'); | ||
| 453 | $snod->setAttribute(ID=>($s->{id} // sprintf("s%x",$si))); | ||
| 454 | $snod->setAttribute(tokenIDs=>join(' ',@wids)); | ||
| 455 | } | ||
| 456 | } | ||
| 457 | |||
| 458 | $fmt->vlog($fmt->{tcflog}, "putDocument(): returning"); | ||
| 459 | return $fmt; | ||
| 460 | } | ||
| 461 | |||
| 462 | 1; ##-- be happy | ||
| 463 | |||
| 464 | __END__ |