CAB/Format/TCF.pm untested

filter: Statementstmt 0.0% 0 / 247 Branchbran 0.0% 0 / 94 Conditioncond 0.0% 0 / 81 MC/DCmcdc 0.0% 0 / 50 Subroutinesub 0.0% 0 / 16 Podpod 0.0% 0 / 9 totaltotal 0.0% 0 / 497 CC 48 SCAR 66.0
CC
48
Cov
0%
putDocument
1260.0
parseDocument
182.0
flush
6.0
CRAP 738.2
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Format::TCF.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: Datum parser|formatter: XML: CLARIN-D TCF (selected features only)
6 ## + uses DTA::CAB::Format::XmlTokWrap for output
7  
8 package DTA::CAB::Format::TCF;
9 use DTA::CAB::Format::XmlCommon;
10 use DTA::CAB::Format::Raw; ##-- for tcf text tokenization
11 use DTA::CAB::Datum ':all';
12 #use DTA::CAB::Utils ':temp';
13 use XML::LibXML;
14 use IO::File;
15 use Carp;
16 use strict;
17  
18 ##==============================================================================
19 ## Globals
20 ##==============================================================================
21  
22 our @ISA = qw(DTA::CAB::Format::XmlCommon);
23  
24 BEGIN {
25 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, filenameRegex=>qr/\.(?i:(?:tcf[\.\-_]?xml)|(?:tcf))$/);
26  
27 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'text'})
28 foreach (qw(tcf-text tcf+text));
29 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'text tokens sentences'})
30 foreach (qw(tcf-tok tcf+tok));
31  
32 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tokens sentences orthography'})
33 foreach (qw(tcf-orth tcf+orth tcf-web)); ##-- for weblicht/dta
34 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tokens sentences postags lemmas'})
35 foreach (qw(tcf-pos tcf+pos)); ##-- for weblicht/d*
36 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tokens sentences orthography postags lemmas'})
37 foreach (qw(tcf tcf-xml tcfxml full-tcf xtcf));
38 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tokens sentences orthography postags lemmas names'})
39 foreach (qw(tcf+ner tcf+names)); ##-- for including named entities parsed from teiws
40  
41 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tei text'})
42 foreach (qw(tcf-tei-text tcf-tei+text tcf+tei+text));
43 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{tcflayers=>'tei text tokens sentences'})
44 foreach (qw(tcf-tei-tok tcf-tei+tok tcf+tei+tok));
45 }
46  
47 BEGIN {
48 *isa = \&UNIVERSAL::isa;
49 *can = \&UNIVERSAL::can;
50 }
51  
52 ##==============================================================================
53 ## Constructors etc.
54 ##==============================================================================
55  
56 ## $fmt = CLASS_OR_OBJ->new(%args)
57 ## + object structure: HASH ref
58 ## {
59 ## ##-- new in TCF
60 ## tcfbufr => \$buf, ##-- raw TCF buffer, for spliceback mode
61 ## textbufr => \$text, ##-- raw text buffer, for spliceback mode
62 ## tcflog => $level, ##-- debugging log-level (default: 'off')
63 ## spliceback => $bool, ##-- (output) if true (default), splice data back into 'tcfbufr' if available; otherwise create new TCF doc
64 ## tcflayers => $tcf_layer_names, ##-- layer names to include, space-separated list; known='tei text tokens sentences postags lemmas orthography'
65 ## tcftagset => $tagset, ##-- tagset name for POStags element (default='stts')
66 ## logsplice => $level, ##-- log level for spliceback messages (default:'none')
67 ## trimtext => $bool, ##-- if true (default), waste tokenizer hints will be trimmed from 'text' layer
68 ##
69 ## ##-- input: inherited from XmlCommon
70 ## xdoc => $xdoc, ##-- XML::LibXML::Document
71 ## xprs => $xprs, ##-- XML::LibXML parser
72 ##
73 ## ##-- output: inherited from XmlCommon
74 ## level => $level, ##-- output formatting level (OVERRIDE: default=1)
75 ## output => [$how,$arg] ##-- either ['fh',$fh], ['file',$filename], or ['str',\$buf]
76 ## }
77 sub new {
78 my $that = shift;
79 my $fmt = $that->SUPER::new(
80 ##-- local
81 #tcfbufr => undef,
82 tcflog => 'off', ##-- debugging log-level
83 #tcflayers => 'tei text tokens sentences orthography postags lemmas names',
84 tcflayers => 'tokens sentences orthography',
85 tcftagset => 'stts',
86 spliceback => 1,
87 logsplice => 'none',
88 trimtext => 1,
89  
90 ##-- overrides (XmlTokWrap, XmlNative, XmlCommon)
91 ignoreKeys => {
92 tcfbufr=>undef,
93 textbufr=>undef,
94 tcfdoc=>undef,
95 },
96 xprsopts => {keep_blanks=>0},
97 level => 1,
98  
99 ##-- user args
100 @_
101 );
102  
103 return $fmt;
104 }
105  
106  
107 ##=============================================================================
108 ## Methods: Generic
109  
110 ##=============================================================================
111 ## Methods: Input
112 ##==============================================================================
113  
114 ##--------------------------------------------------------------
115 ## Methods: Input: Generic API
116  
117 ## $fmt = $fmt->close()
118 ## + close current input source, if any
119 ## + INHERITED from XmlCommon (calls flush())
120  
121 ## $fmt = $fmt->from(String|File|Fh)
122 ## + INHERITED from XmlCommon : populates $fmt->{xdoc}
123  
124 ## $doc = $fmt->parseDocument()
125 ## + parse buffered XML::LibXML::Document from $fmt->{xdoc}
126 sub parseDocument {
127 my $fmt = shift;
128 $fmt->vlog($fmt->{tcflog}, "parseDocument()");
129  
130 ##-- parse: basic
131 my $doc = DTA::CAB::Document->new();
132 my $xdoc = $fmt->{xdoc};
133 my $xroot = $xdoc->documentElement;
134 $doc->{tcfdoc} = $xdoc if ($fmt->{spliceback});
135  
136 ##-- parse: metadata
137 if (defined(my $xmeta = [$xroot->getChildrenByLocalName("MetaData")]->[0])) {
138 my ($xsrc) = $xmeta->getChildrenByLocalName('source');
139 $doc->{source} = $xsrc->textContent if (defined($xsrc));
140 }
141  
142 ##-- parse: corpus
143 my $xcorpus = [$xroot->getChildrenByLocalName('TextCorpus')]->[0]
144 or $fmt->logconfess("parseDocument(): no TextCorpus node found in XML document");
145  
146 ##-- parse: text (textbufr)
147 ## + annoying hack: we grep for elements here b/c libxml getChildrenByLocalName('text') also returns text-nodes!
148 my ($xtext) = grep {UNIVERSAL::isa($_,'XML::LibXML::Element')} $xcorpus->getChildrenByLocalName('text');
149 if ($xtext) {
150 my $textbuf = $xtext->textContent;
151 $doc->{textbufr} = \$textbuf;
152 }
153  
154 ##-- check for pre-tokenized input
155 if (defined(my $xtokens = [$xcorpus->getChildrenByLocalName('tokens')]->[0])) {
156 ##------------ pre-tokenized input
157 ##-- parse: tokens
158 my (@w,%id2w,$w);
159 foreach ($xtokens->getChildrenByLocalName('token')) {
160 push(@w, $w={text=>$_->textContent});
161 if (!defined($w->{id}=$_->getAttribute('ID'))) {
162 $w->{id} = sprintf("w%x", $#w);
163 $_->setAttribute('ID'=>$w->{id});
164 }
165 $id2w{$w->{id}} = $w;
166 }
167  
168 ##-- parse: sentences
169 my ($s);
170 if (defined(my $xsents = [$xcorpus->getChildrenByLocalName('sentences')]->[0])) {
171 my $body = $doc->{body};
172 foreach ($xsents->getChildrenByLocalName('sentence')) {
173 push(@$body, $s={});
174 $s->{id} = $_->getAttribute('ID') if ($_->hasAttribute('ID'));
175 $s->{tokens} = [ @id2w{split(' ',$_->getAttribute('tokenIDs'))} ];
176 }
177 } else {
178 $doc->{body} = {tokens=>\@w}; ##-- single-sentence
179 }
180  
181 ##-- parse: POStags -> moot/tag
182 my ($id);
183 if (defined(my $xpostags = [$xcorpus->getChildrenByLocalName('POStags')]->[0])) {
184 foreach ($xpostags->getChildrenByLocalName('tag')) {
185 $id = $_->getAttribute('tokenIDs');
186 $id2w{$id}{moot}{tag} = $_->textContent;
187 }
188 }
189  
190 ##-- parse: lemmas -> moot/lemma
191 if (defined(my $xlemmas = [$xcorpus->getChildrenByLocalName('lemmas')]->[0])) {
192 foreach ($xlemmas->getChildrenByLocalName('lemma')) {
193 $id = $_->getAttribute('tokenIDs');
194 $id2w{$id}{moot}{lemma} = $_->textContent;
195 }
196 }
197  
198 ##-- parse: orthography -> moot/word
199 if (defined(my $xorths = [$xcorpus->getChildrenByLocalName('orthography')]->[0])) {
200 foreach (grep {($_->getAttribute('operation')//'') eq 'replace'} $xorths->getChildrenByLocalName('correction')) {
201 $id = $_->getAttribute('tokenIDs');
202 $id2w{$id}{moot}{word} = $_->textContent;
203 }
204 }
205 }
206 elsif ($doc->{textbufr}) {
207 ##------------ un-tokenized input
208 my $rawfmt = DTA::CAB::Format::Raw->new();
209 my $rawdoc = DTA::CAB::Format::Raw->new->parseString( ${$doc->{textbufr}} );
210 $doc->{body} = $rawdoc->{body};
211 }
212 else {
213 ##------------ no source
214 $fmt->logconfess("parseDocument(): no TextCorpus/text or TextCorpus/tokens node found in XML document");
215 }
216  
217 $fmt->vlog($fmt->{tcflog}, "parseDocument(): returning");
218 return $doc;
219 }
220  
221 ##=============================================================================
222 ## Methods: Output
223 ##==============================================================================
224  
225 ##--------------------------------------------------------------
226 ## Methods: Output: MIME & HTTP stuff
227  
228 ## $short = $fmt->shortName()
229 ## + returns "official" short name for this format
230 ## + default just returns package suffix
231 sub shortName {
232 return 'tcf';
233 }
234  
235 ## $type = $fmt->mimeType()
236 ## + override returns text/xml
237 sub mimeType { return 'text/tcf+xml'; }
238  
239 ## $ext = $fmt->defaultExtension()
240 ## + returns default filename extension for this format (default='.cab')
241 sub defaultExtension { return '.tcf.xml'; }
242  
243 ##--------------------------------------------------------------
244 ## Methods: Output: output selection
245  
246 ## $fmt = $fmt->flush()
247 ## + flush any buffered output to selected output source
248 sub flush {
249 my $fmt = shift;
250 $fmt->vlog($fmt->{tcflog}, "flush()") if (Log::Log4perl->initialized);
251 $fmt->SUPER::flush(@_) || return undef;
252 delete @$fmt{qw(tcfbufr tcfdoc outbufr)};
253 return $fmt;
254 }
255  
256 ## $fmt = $fmt->to(String|File|Fh)
257 ## + INHERITED from XmlCommon : sets up $fmt->{output}=($outputHow,$outputArg)
258  
259 ##--------------------------------------------------------------
260 ## Methods: Output: Generic API
261  
262 ## $fmt = $fmt->putDocument($doc)
263 ## + override respects local 'spliceback' and 'tcflayers' flags
264 sub putDocument {
265 my ($fmt,$doc) = @_;
266 $fmt->vlog($fmt->{tcflog}, "putDocument()");
267  
268 ##-- common vars
269 my $spliceback = $fmt->{spliceback};
270 my $layers = $fmt->{tcflayers} // '';
271  
272 ##-- spliceback?
273 my ($xdoc);
274 if ($spliceback) {
275 $xdoc = $doc->{tcfdoc} // $fmt->{tcfdoc};
276 if (!$xdoc) {
277 my $bufr = $doc->{tcfbufr} // $fmt->{tcfbufr};
278 if (!$bufr || !$$bufr) {
279 $fmt->vlog($fmt->{logsplice}, "spliceback mode requested but no 'tcfdoc' or 'tcfbufr' document property - creating new document!");
280 $spliceback = 0;
281 }
282 }
283 }
284 $xdoc //= XML::LibXML::Document->new("1.0","UTF-8");
285 $fmt->{xdoc} = $xdoc;
286  
287 ##-- document structure: root
288 my ($xroot);
289 if (!defined($xroot = $xdoc->documentElement)) {
290 $xdoc->setDocumentElement( $xroot = $xdoc->createElement('D-Spin') );
291 $xroot->setNamespace('http://www.dspin.de/data');
292 $xroot->setAttribute('version'=>'0.4');
293 }
294  
295 ##-- document structure: metadata
296 my ($xmeta) = $xroot->getChildrenByLocalName('MetaData');
297 if (!defined($xmeta)) {
298 $xmeta = $xroot->addNewChild(undef,'MetaData');
299 $xmeta->setNamespace('http://www.dspin.de/data/metadata');
300 $xmeta->appendTextChild('source', $doc->{source}) if (defined($doc->{source}));
301 }
302  
303 ##-- document structure: corpus
304 my ($xcorpus) = $xroot->getChildrenByLocalName('TextCorpus');
305 if (!defined($xcorpus)) {
306 $xcorpus = $xroot->addNewChild(undef,'TextCorpus');
307 $xcorpus->setNamespace('http://www.dspin.de/data/textcorpus');
308 $xcorpus->setAttribute('lang'=>'de');
309 }
310  
311 ##-- document structure: TextCorpus/tei
312 if ($layers =~ /\btei\b/ && defined($doc->{teibufr})) {
313 my ($xtei) = $xcorpus->getChildrenByLocalName("textSource");
314 my $xtype = $xtei ? ($xtei->getAttribute('type')//'') : '';
315 undef ($xtei) if ($xtype !~ m{^(?:text|application)/tei\+xml\b} || $xtype =~ m{\btokenized=(?![0n])});
316 if (!defined($xtei)) {
317 $xtei = $xcorpus->addNewChild(undef,'textSource');
318 $xtei->setAttribute('type'=>'application/tei+xml');
319 $xtei->appendText(${$doc->{teibufr}});
320 }
321 }
322  
323 ##-- document structure: TextCorpus/text
324 if ($layers =~ /\btext\b/) {
325 ##-- annoying hack: we grep for elements here b/c libxml getChildrenByLocalName('text') also returns text-nodes!
326 my ($xtext) = grep {UNIVERSAL::isa($_,'XML::LibXML::Element')} $xcorpus->getChildrenByLocalName('text');
327 if (!defined($xtext)) {
328 $xtext = $xcorpus->addNewChild(undef,'text');
329 if (defined($doc->{textbufr})) {
330 ##-- use doc-buffered text content
331 my $txt = ${$doc->{textbufr}};
332 $txt =~ s/\s*\$WB\$\s*/ /sg;
333 $txt =~ s/\s*\$SB\$\s*/\n\n/sg;
334 $txt =~ s/%%[^%]*%%//sg;
335 $xtext->appendText($txt);
336 }
337 else {
338 ##-- generate dummy text content
339 $xtext->appendText(join(' ', map {$_->{text}} @{$_->{tokens}})."\n") foreach (@{$doc->{body}});
340 }
341 }
342 }
343  
344 ##-- document structure: corpus structure
345 my ($tokens,$sents,$lemmas,$postags,$orths,$names);
346 if ($layers =~ /\btokens\b/ && !$xcorpus->getChildrenByLocalName('tokens')) {
347 $tokens = $xcorpus->addNewChild(undef,'tokens');
348 }
349 if ($layers =~ /\bsentences\b/ && !$xcorpus->getChildrenByLocalName('sentences')) {
350 $sents = $xcorpus->addNewChild(undef,'sentences');
351 }
352 if ($layers =~ /\blemmas\b/ && !$xcorpus->getChildrenByLocalName('lemmas')) {
353 $lemmas = $xcorpus->addNewChild(undef,'lemmas');
354 #$lemmas->setAttribute('type'=>'CAB');
355 }
356 if ($layers =~ /\bpostags\b/ && !$xcorpus->getChildrenByLocalName('POStags')) {
357 $postags = $xcorpus->addNewChild(undef,'POStags');
358 $postags->setAttribute('tagset'=>$fmt->{tcftagset}) if ($fmt->{tcftagset});
359 #$postags->setAttribute('type'=>'CAB');
360 }
361 if ($layers =~ /\borthography\b/ && !$xcorpus->getChildrenByLocalName('orthography')) {
362 $orths = $xcorpus->addNewChild(undef,'orthography');
363 #$orths->setAttribute('type'=>'CAB');
364 }
365 if ($layers =~ /\bnames\b/ && !$xcorpus->getChildrenByLocalName('namedEntities')) {
366 $names = $xcorpus->addNewChild(undef,'namedEntities');
367 $names->setAttribute('type'=>'CoNLL2002');
368 }
369  
370 ##-- ensure ids
371 my $wi = 0;
372 my $si = 0;
373 my $ni = 0;
374 my (%nid2nod); ##-- ($nerEntityId => $tcfNamedEntityNode), for 'names' layer
375 my ($s,$w,$wid,@wids,$snod,$wnod,$nnod);
376 my ($pos,$lemma,$orth,$nea,$neid,$necls);
377 foreach $s (@{$doc->{body}}) {
378 ++$si;
379 @wids = qw();
380 foreach $w (@{$s->{tokens}}) {
381 ++$wi;
382 $wid = $w->{id} // sprintf("w%x",$wi);
383 push(@wids,$wid);
384  
385 ##-- generate token node: <token ID="t_0">Karin</token>
386 if ($tokens) {
387 $wnod = $tokens->addNewChild(undef,'token');
388 $wnod->setAttribute(ID=>$wid);
389 $wnod->appendText($w->{text});
390 }
391  
392 ##-- generate token data: tag, lemma, orthography
393 $pos = $lemma = $orth = undef;
394 if ($w->{moot}) {
395 $pos = $w->{moot}{tag};
396 $lemma = $w->{moot}{lemma};
397 $orth = $w->{moot}{word};
398 }
399 elsif ($w->{dmoot}) {
400 $orth = $w->{dmoot}{tag};
401 }
402 $orth //= $w->{exlex} // ($w->{xlit} && $w->{xlit}{isLatinExt} ? $w->{xlit}{latin1Text} : undef);
403  
404 if ($postags && defined($pos)) {
405 ##-- POStags: <tag ID="pt_0" tokenIDs="t_0">NE</tag>
406 $wnod = $postags->addNewChild(undef,'tag');
407 $wnod->setAttribute(tokenIDs=>$wid);
408 $wnod->appendText($pos);
409 }
410 if ($lemmas && defined($lemma)) {
411 ##-- lemmas: <lemma ID="le_0" tokenIDs="t_0">Karin</lemma>
412 $wnod = $lemmas->addNewChild(undef,'lemma');
413 $wnod->setAttribute(tokenIDs=>$wid);
414 $wnod->appendText($lemma);
415 }
416 if ($orths && defined($orth) && $orth ne $w->{text}) {
417 ##-- orthography: <correction operation="replace" tokenIDs="t_0">Karina</correction>
418 $wnod = $orths->addNewChild(undef,'correction');
419 $wnod->setAttribute(tokenIDs=>$wid);
420 $wnod->setAttribute(operation=>'replace'); ##-- "norm" would be better, but isn't allowed
421 $wnod->appendText($orth);
422 }
423 if ($names && defined($w->{ner})) {
424 ##-- names: populate %nid2nod
425 foreach $nea (@{$w->{ner}}) {
426 $neid = $nea->{nid} || "n$wid";
427 if (defined($nnod=$nid2nod{$neid})) {
428 ##-- append token to existing tcf namedEntities/entity node
429 $nnod->setAttribute('tokenIDs',$nnod->getAttribute('tokenIDs').' '.$wid);
430 } else {
431 ##-- create new tcf namedEntities/entity node
432 ++$ni;
433 $nnod = $nid2nod{$neid} = $names->addNewChild(undef,'entity');
434  
435 ##-- CoNLL2002 ner categories: PER, LOC, ORG, MISC
436 $necls = $nea->{func} || 'MISC';
437 $necls = ($necls =~ /^(?:per|\@(?:First|Last|Title))/i ? 'PER'
438 : ($necls =~ /^place|loc|geo/i ? 'LOC'
439 : ($necls =~ /^org/i ? 'ORG'
440 : 'MISC')));
441  
442 $nnod->setAttribute('ID' => $neid);
443 $nnod->setAttribute('class' => $necls);
444 $nnod->setAttribute('tokenIDs' => $wid);
445 }
446 }
447 }
448 }
449  
450 if ($sents) {
451 ##-- generate sentence node: <sentence ID="s_0" tokenIDs="t_0 t_1 t_2 t_3 t_4 t_5"></sentence>
452 $snod = $sents->addNewChild(undef,'sentence');
453 $snod->setAttribute(ID=>($s->{id} // sprintf("s%x",$si)));
454 $snod->setAttribute(tokenIDs=>join(' ',@wids));
455 }
456 }
457  
458 $fmt->vlog($fmt->{tcflog}, "putDocument(): returning");
459 return $fmt;
460 }
461  
462 1; ##-- be happy
463  
464 __END__