CAB/Format/XmlNative.pm untested

filter: Statementstmt 0.0% 0 / 203 Branchbran 0.0% 0 / 100 Conditioncond 0.0% 0 / 77 MC/DCmcdc 0.0% 0 / 43 Subroutinesub 0.0% 0 / 21 Podpod 0.0% 0 / 13 totaltotal 0.0% 0 / 457 CC 51 SCAR 53.5
CC
51
Cov
0%
parseNode
462.0
xmlNode
272.0
blockScanFoot
90.0
CRAP 211.5
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Format::XmlNative.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: Datum parser|formatter: XML (native)
6  
7 package DTA::CAB::Format::XmlNative;
8 use DTA::CAB::Format::XmlCommon;
9 use DTA::CAB::Datum ':all';
10 use XML::LibXML;
11 use IO::File;
12 use Carp;
13 use strict;
14  
15 #require 5.10.0; ##-- for ${^POSTMATCH} ; but this syntax makes services (perl v5.10.0) complain
16 require 5.010_000; ##-- same thing which doesn't make v5.10.0 complain
17  
18 ##==============================================================================
19 ## Globals
20 ##==============================================================================
21  
22 our @ISA = qw(DTA::CAB::Format::XmlCommon);
23  
24 BEGIN {
25 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, filenameRegex=>qr/\.(?i:xml\-native|xml\-dta\-cab|(?:dta[\-\._]cab[\-\._]xml))$/);
26 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>'cab-xml');
27 }
28  
29 BEGIN {
30 *isa = \&UNIVERSAL::isa;
31 *can = \&UNIVERSAL::can;
32 }
33  
34 ##==============================================================================
35 ## Constructors etc.
36 ##==============================================================================
37  
38 ## $fmt = CLASS_OR_OBJ->new(%args)
39 ## + object structure: HASH ref
40 ## {
41 ## ##-- input: inherited
42 ## xdoc => $xdoc, ##-- XML::LibXML::Document
43 ## xprs => $xprs, ##-- XML::LibXML parser
44 ##
45 ## ##-- input: new
46 ## parseXmlData => $bool, ##-- if unspecified or true, _xmldata key will be populated by parseNode() (default=undef->true)
47 ##
48 ## ##-- input+output: new
49 ## xml2key => \%xml2key, ##-- maps xml keys to internal keys
50 ## ignoreKeys => \%key2undef, ##-- keys to ignore for i/o
51 ##
52 ## ##-- output: new
53 ## arrayEltKeys => \%akey2ekey, ##-- maps array keys to element keys for output
54 ## arrayImplicitKeys => \%akey2undef, ##-- pseudo-hash of array keys NOT mapped to explicit elements
55 ## key2xml => \%key2xml, ##-- maps keys to XML-safe names
56 ##
57 ## ##-- output: inherited
58 ## #encoding => $inputEncoding, ##-- default: UTF-8; applies to output only!
59 ## level => $level, ##-- output formatting level (default=0)
60 ## }
61 sub new {
62 my $that = shift;
63 my $fmt = $that->SUPER::new(
64 ##-- defaults: output
65 #xmlns =>
66 #{
67 # 'cab' => 'http://www.deutschestextarchiv.de/cab/spec/1.0/XmlNative',
68 # '' => 'http://www.deutschestextarchiv.de/cab/spec/1.0/XmlNative',
69 #},
70  
71 key2xml => {
72 'xml:id' => 'id',
73 'xml:base' => 'base',
74 #'text' => 't', ##-- for TokWrap .t.xml
75 },
76 xml2key => {
77 'xml:id' => 'id',
78 'xml:base' => 'base',
79 't' => 'text', ##-- for TokWrap .t.xml
80 'cab:t' => 'text', ##-- for ddc-build/splice-cleaner.xsl output
81 'cab:text' => 'text', ##-- for ddc-build/splice-cleaner.xsl output
82 },
83  
84 arrayEltKeys => {
85 'body' => 's',
86 'tokens' => 'w',
87 'DEFAULT' => 'a',
88 },
89  
90 arrayImplicitKeys => {
91 body=>undef,
92 tokens=>undef,
93 'a'=>undef,
94 },
95 ignoreKeys => {
96 'teibufr'=>undef,
97 'textbufr'=>undef,
98 },
99 #parseXmlData => 1,
100  
101 ##-- user args
102 @_
103 );
104 $fmt->xmlparser->keep_blanks(0);
105 return $fmt;
106 }
107  
108 ##==============================================================================
109 ## Methods: I/O: Block-wise
110 ##==============================================================================
111  
112 ##--------------------------------------------------------------
113 ## Methods: I/O: Block-wise: Generic
114  
115 ## %blockOpts = $CLASS_OR_OBJECT->blockDefaults()
116 ## + returns default block options as for blockOptions()
117 ## + override returns as for $CLASS_OR_OBJECT->blockOptions('512k@s')
118 sub blockDefaults {
119 return ($_[0]->SUPER::blockDefaults(), bsize=>(512*1024), eob=>'s');
120 }
121  
122 ##--------------------------------------------------------------
123 ## Methods: I/O: Block-wise: Input
124  
125 ## \%head = blockScanHead(\$buf,$io,\%opts)
126 ## + gets header (${io}head) offset, length from (mmapped) \$buf
127 ## + %opts are as for blockScan()
128 sub blockScanHead {
129 my ($fmt,$bufr,$io,$opts) = @_;
130 my $elt = $opts->{xmlelt} || $opts->{eob} || 'w';
131 return $$bufr =~ m(\Q<$elt\E\b) ? [0,$-[0]] : [0,0];
132 }
133  
134 ## \%head = blockScanFoot(\$buf,$io,\%opts)
135 ## + gets footer (${io}foot) offset, length from (mmaped) \$buf
136 ## - override works from and may alter last body block in $opts->{${io}body}
137 ## - also uses $opts->{${io}fsize} (default=length($$buf)) to compute footer length
138 ## + %opts are as for blockScan()
139 sub blockScanFoot {
140 use bytes;
141 my ($fmt,$bufr,$io,$opts) = @_;
142 return [0,0] if (!$opts || !$opts->{"${io}body"} || !@{$opts->{"${io}body"}});
143 my $blk = $opts->{"${io}body"}[$#{$opts->{"${io}body"}}];
144 my $elt = $opts->{xmlelt} || $opts->{eob} || 'w';
145 pos($$bufr) = $blk->{"${io}off"} || 0; ##-- set to offset of final body block
146  
147 if (0) {
148 ##-- v0: use negative lookahead regex match: elegant but buggy under some perls
149 ## + causes segfault on in ddc/dta2012/build/cab_corpus for ddc/dta2012/build/xml_tok/campe_robinson02_1780.TEI-P5.chr.ddc.t.xml
150 ## + bug appears both on kaskade (debian-lenny, perl 5.10.0-19lenny5) and plato (debian-squeeze, perl 5.10.1-17squeeze3)
151 ## + only sefgaults under make (changing make -j , -blockSize , -njobs has no effect)
152 ## + backtrace:
153 ## #0 0x00002b26f788ef77 in ?? () from /usr/lib/libperl.so.5.10
154 ## #1 0x00002b26f7896fd0 in ?? () from /usr/lib/libperl.so.5.10
155 ## #2 0x00002b26f789ad29 in Perl_regexec_flags () from /usr/lib/libperl.so.5.10
156 ## #3 0x00002b26f7837e76 in Perl_pp_match () from /usr/lib/libperl.so.5.10
157 ## #4 0x00002b26f7831392 in Perl_runops_standard () from /usr/lib/libperl.so.5.10
158 ## #5 0x00002b26f782c5df in perl_run () from /usr/lib/libperl.so.5.10
159 ## #6 0x0000000000400d0c in main ()
160 if ($$bufr =~ m((?s:</\Q$elt\E>|<\Q$elt\E\b[^>]*/>)(?!.*(?s:</\Q$elt\E>|<\Q$elt\E\b[^>]*/>)))sg) {
161 my $end = $+[0];
162 $blk->{"${io}len"} = $end - ($blk->{"${io}off"} || 0);
163 return [$end, ($opts->{"${io}fsize"}||length($$bufr))-$end];
164 }
165 }
166 elsif (0) {
167 ##-- v1: !$useNegativeLookaheadRegex: use ${^POSTMATCH} safer but __much__ slower
168 my ($end);
169 while ($$bufr =~ m{</\Q$elt\E>|<\Q$elt\E\b[^>]*/>}msgp && defined($end=$+[0]) && ${^POSTMATCH} =~ m{</\Q$elt\E>|<\Q$elt\E\b[^>]*/>}ms) {
170 pos($$bufr) = $end = $end+$+[0];
171 }
172 if (defined($end)) {
173 $blk->{"${io}len"} = $end - ($blk->{"${io}off"} || 0);
174 return [$end, ($opts->{"${io}fsize"}||length($$bufr))-$end];
175 }
176 }
177 else {
178 ##-- v2: !$useNegativeLookaheadRegex : v0.1: use rindex()
179  
180 ##-- scan for literal end-of-element with rindex() for a first-stab
181 ## + doesn't find all valid element closers, just the likely ones
182 my $pos0 = pos($$bufr);
183 my $end = rindex(substr($$bufr,$pos0),"</$elt>"); ##-- usual case
184 $end = rindex(substr($$bufr,$pos0),"<$elt/>") if ($end<0); ##-- empty element
185  
186 ##-- now use slow regex scan
187 pos($$bufr) = $pos0+$end if ($end>=0);
188 while ($$bufr =~ m{</\Q$elt\E>|<\Q$elt\E\b[^>]*/>}msgp && defined($end=$+[0]) && ${^POSTMATCH} =~ m{</\Q$elt\E>|<\Q$elt\E\b[^>]*/>}ms) {
189 pos($$bufr) = $end = $end+$+[0];
190 }
191 if (defined($end) && $end>=0) {
192 $blk->{"${io}len"} = $end - ($blk->{"${io}off"} || 0);
193 return [$end, ($opts->{"${io}fsize"}||length($$bufr))-$end];
194 }
195 }
196 return [0,0];
197 }
198  
199 ## \@blocks = $fmt->blockScanBody(\$buf,\%opts)
200 ## + scans $filename for block boundaries according to \%opts
201 sub blockScanBody {
202 my ($fmt,$bufr,$opts) = @_;
203  
204 ##-- scan blocks into head, body, foot
205 my $bsize = $opts->{bsize};
206 my $fsize = $opts->{ifsize};
207 my $elt = $opts->{xmlelt} || $opts->{eob} || 'w';
208 my $eos = $elt eq 's' ? 1 : 0;
209 my $re_s = '(?s:<'.quotemeta($elt).'\b)';
210 my $blocks = [];
211  
212 ##-- hack workaround for slow m($re_qr)g on kira (ubuntu 16.04.1 LTS, perl 5.22.1)
213 #my $re = qr($re_s);
214 my $matchoff = eval qq{sub { \$\$bufr =~ m{$re_s}g ? \$-[0] : \$fsize }};
215  
216 my ($off0,$off1,$blk);
217 for ($off0=$opts->{ihead}[0]+$opts->{ihead}[1]; $off0 < $fsize; $off0=$off1) {
218 push(@$blocks, $blk={bsize=>$bsize, eob=>$elt, ioff=>$off0, eos=>$eos});
219 pos($$bufr) = ($off0+$bsize < $fsize ? $off0+$bsize : $fsize);
220 $off1 = $matchoff->();
221 $blk->{eos} = 1 if ($off1 >= $fsize); ##-- for tt
222 $blk->{ilen} = $off1-$off0;
223 }
224  
225 return $blocks;
226 }
227  
228 ##--------------------------------------------------------------
229 ## Methods: I/O: Block-wise: Output
230 ## + inherited from DTA::CAB::Format
231  
232  
233 ##=============================================================================
234 ## Methods: Input
235 ##==============================================================================
236  
237 ##--------------------------------------------------------------
238 ## Methods: Input: Local
239  
240 ## $obj = $fmt->parseNode($nod)
241 ## + returns a perl object represented by the XML::LibXML::Node $nod
242 ## + attempts to map xml to perl structure "sensibly"
243 ## + DTA::CAB::Datum nodes (documen,sentence,token) get some additional baggage:
244 ## _xmldata => $data, ##-- unparsed content (raw string)
245 sub parseNode {
246 my ($fmt,$top) = @_;
247 return undef if (!defined($top));
248  
249 my $xml2key = $fmt->{xml2key};
250 my $parseXmlData = !exists($fmt->{parseXmlData}) || $fmt->{parseXmlData};
251 my ($cd,$cs,$cw);
252 my ($nod,$cur,$name,$nxt);
253 my ($topval);
254  
255 my @stack = ([$top]);
256 while (@stack) {
257 ($nod,$cur) = @{pop @stack};
258 $name = $nod->nodeName;
259 $name = $xml2key->{$name} if (defined($xml2key->{$name}));
260 next if (exists($fmt->{ignoreKeys}{$name}));
261  
262 if (isa($nod,'XML::LibXML::Element')) {
263 ##-- Element
264 if ($name eq 'doc') {
265 ##-- Element: special: DTA::CAB::Document
266 $nxt = $cd = DTA::CAB::Document->new;
267 }
268 elsif ($name eq 's') {
269 ##-- Element: special: DTA::CAB::Sentence
270 #$nxt = $cs = DTA::CAB::Sentence->new;
271 $nxt = $cs = {tokens=>[]};
272 push(@{$cd->{body}},$cs);
273 }
274 elsif ($name eq 'w') {
275 ##-- Element: special: DTA::CAB::Token
276 #$nxt = $cw = DTA::CAB::Token->new;
277 $nxt = $cw = {text=>undef};
278 push(@{$cs->{tokens}},$cw);
279 }
280 elsif ($name eq 'a' && $nod->parentNode->nodeName eq 'w') {
281 ##-- Element: special: tokenizer analysis (toka)
282 push(@{$cw->{toka}}, $nod->textContent);
283 }
284 elsif ($name eq 'msafe') {
285 ##-- Element: special: msafe (backwards-compatible)
286 $cur->{msafe} = $nod->getAttribute('safe');
287 }
288 elsif ($nod->hasAttributes) {
289 ##-- Element: default: +attributes: HASH
290 $nxt = _pushValue($cur,$name,{});
291 }
292 elsif ($nod->hasChildNodes && $nod->parentNode->nodeName ne 'toka') {
293 ##-- Element: default: -attributes, +dtrs: ARRAY
294 $nxt = _pushValue($cur,$name,[]);
295 }
296 else {
297 ##-- Element: default: -attributes, -dtrs: append to _xmldata
298 $cur->{_xmldata} .= $nod->toString if ($parseXmlData && isa($cur,'HASH'));
299 }
300 ##-- Element: common: enqueue child nodes
301 push(@stack, map {[$_,$nxt]} reverse($nod->childNodes), $nod->attributes);
302  
303 ##-- Element: save top value
304 $topval = $nxt if (!defined($topval));
305 }
306 elsif (isa($nod,'XML::LibXML::Attr')) {
307 if ($name eq 'lang' && $nod->parentNode->nodeName eq 'w') {
308 ##-- attribute: special: lang
309 $cur->{$name} = [split(/ /, $nod->value)];
310 }
311 else {
312 ##-- Attribute (hash only)
313 $cur->{$name} = $nod->value if (isa($cur,'HASH'));
314 }
315 }
316 elsif (isa($nod,'XML::LibXML::Text')) {
317 ##-- Text
318 if (isa($cur,'HASH')) {
319 ##-- Text: to hash: append to _xmldata
320 $cur->{'_xmldata'} .= $nod->toString if ($parseXmlData);
321 }
322 elsif (isa($cur,'ARRAY')) {
323 ##-- Text: to array: append to array
324 push(@$cur,$nod->toString);
325 }
326 }
327 else {
328 $fmt->logwarn("parseNode() can't handle XML node of class ", ref($nod), " - skipping\n");
329 }
330 }##--/while (@queue)
331  
332 return $topval;
333 }
334  
335 ## $val = PACKAGE::_pushValue(\%hash, $key, $val); ##-- $hash{$key}=$val
336 ## $val = PACKAGE::_pushValue(\@array, $key, $val); ##-- push(@array,$val)
337 sub _pushValue {
338 return $_[0]{$_[1]}=$_[2] if (isa($_[0],'HASH'));
339 push(@{$_[0]},$_[2]);
340 return $_[2];
341 }
342  
343  
344 ##--------------------------------------------------------------
345 ## Methods: Input: Generic API
346  
347 ## $doc = $fmt->parseDocument()
348 ## + parses buffered XML::LibXML::Document
349 sub parseDocument {
350 my $fmt = shift;
351 if (!defined($fmt->{xdoc})) {
352 $fmt->logconfess("parseDocument(): no source document {xdoc} defined!");
353 return undef;
354 }
355 my $parsed = $fmt->parseNode($fmt->{xdoc}->documentElement);
356  
357 ##-- force document
358 return $fmt->forceDocument($parsed);
359 }
360  
361 ##=============================================================================
362 ## Methods: Output
363 ##==============================================================================
364  
365 ##--------------------------------------------------------------
366 ## Methods: Output: MIME & HTTP stuff
367  
368 ## $short = $fmt->shortName()
369 ## + returns "official" short name for this format
370 ## + default just returns package suffix
371 sub shortName {
372 return 'xml';
373 }
374  
375 ##--------------------------------------------------------------
376 ## Methods: Output: Local
377  
378 ## $nod = $fmt->xmlNode($thingy,$name)
379 ## + returns an xml node for $thingy using $name as key
380 sub xmlNode {
381 my ($fmt,$topval,$topkey) = @_;
382 my $topmom = XML::LibXML::Element->new('__root__');
383  
384 my $akey2ekey = $fmt->{arrayEltKeys};
385 my $key2xml = $fmt->{key2xml};
386  
387 my ($val,$key,$mom,$nod, $skey,$sval);
388 my @queue = ([$topval,$topkey,$topmom]); ## [$val,$key,$mom], ...
389 while (@queue) {
390 ($val,$key,$mom) = @{shift @queue};
391 $key = $key2xml->{$key} if (defined($key2xml->{$key}));
392  
393 if (exists($fmt->{ignoreKeys}{$key})) {
394 ;##-- ignored: skip it
395 }
396 elsif (!defined($val)) {
397 ;##-- undefined: skip it
398 }
399 elsif (!ref($val)) {
400 ##-- scalar: raw text
401 #$val = '' if (!defined($val));
402 if ($key eq '#text') {
403 $mom->appendText($val);
404 } else {
405 $mom->appendTextChild($key,$val);
406 }
407 }
408 elsif (can($val,'xmlNode') && UNIVERSAL::can($val,'xmlNode') ne \&defaultXmlNode) {
409 ##-- object: xml-aware (avoid circularities)
410 $nod = $val->xmlNode($key,$mom,$fmt);
411 $mom->appendChild($nod); ##-- fails if already added
412 }
413 elsif (isa($val,'HASH')) {
414 ##-- hash: map to element
415 $nod = $mom->addNewChild(undef,$key);
416 $nod->appendWellBalancedChunk($val->{_xmldata}) if (defined($val->{_xmldata}));
417 while (($skey,$sval)=each(%$val)) {
418 $skey = $key2xml->{$skey} if (defined($key2xml->{$skey}));
419 if ($skey eq '_xmldata' || !defined($sval)) {
420 next;
421 } elsif (!ref($sval)) {
422 $nod->setAttribute($skey,$sval);
423 } else {
424 push(@queue, [$sval,$skey,$nod]);
425 }
426 }
427 }
428 elsif (isa($val,'ARRAY')) {
429 if (exists($fmt->{arrayImplicitKeys}{$key})) {
430 ##-- array: implicit
431 $nod = $mom;
432 } elsif ($key eq 'lang') {
433 ##-- special: 'lang'
434 $mom->setAttribute('lang', join(' ', @$val));
435 next;
436 } else {
437 ##-- array: default: map to element
438 $nod = $mom->addNewChild(undef,$key);
439 }
440 ##-- array: append elements
441 $skey = $akey2ekey->{$key} || $akey2ekey->{DEFAULT};
442 push(@queue, [$_,$skey,$nod]) foreach (@$val);
443 }
444 else {
445 ##-- other: complain
446 $fmt->logcarp("xmlNode(): default node generator clause called for key='$key', value='$val'");
447 $nod = $mom->addNewChild(undef,$key);
448 }
449 }
450  
451 ##-- unbind & return
452 my $topnod = $topmom->firstChild();
453 $topnod->unbindNode if (defined($topnod));
454 return $topnod;
455 }
456  
457 ##--------------------------------------------------------------
458 ## Methods: Output: Generic API
459  
460 ## $fmt = $fmt->putDocument($doc)
461 sub putDocument {
462 my ($fmt,$doc) = @_;
463 my $xdoc = $fmt->xmlDocument();
464 my $docnod = $fmt->xmlNode($doc,'doc');
465 my ($root);
466 if (!defined($root=$xdoc->documentElement)) {
467 $xdoc->setDocumentElement($docnod);
468 } else {
469 $root->appendChild($_) foreach ($docnod->childNodes); ##-- hack
470 }
471 return $fmt;
472 }
473  
474 ##==============================================================================
475 ## Package: Xml (alias)
476 package DTA::CAB::Format::Xml;
477 our @ISA = qw(DTA::CAB::Format::XmlNative);
478  
479 1; ##-- be happy
480  
481 __END__