CAB/Format/TEIws.pm untested

filter: Statementstmt 0.0% 0 / 193 Branchbran 0.0% 0 / 50 Conditioncond 0.0% 0 / 72 MC/DCmcdc 0.0% 0 / 41 Subroutinesub 0.0% 0 / 26 Podpod 0.0% 0 / 15 totaltotal 0.0% 0 / 397 CC 26 SCAR 43.7
CC
26
Cov
0%
parseDocument
240.0
putDocument
42.0
fromString
20.0
CRAP 78.9
1 ## -*- Mode: CPerl; coding: utf-8 -*-
2 ##
3 ## File: DTA::CAB::Format::TEIws.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: Datum parser|formatter: XML: TEI: with //w and //s elements, as output by DTA::TokWrap
6 ## + uses DTA::CAB::Format::XmlTokWrap for output
7  
8 package DTA::CAB::Format::TEIws;
9 use DTA::CAB::Format::TEI;
10 use DTA::CAB::Datum ':all';
11 use DTA::CAB::Utils ':temp', ':libxml';
12 use DTA::TokWrap;
13 use DTA::TokWrap::Utils qw();
14 use File::Path;
15 use XML::LibXML;
16 use IO::File;
17 use Carp;
18 use utf8;
19 use strict;
20  
21 ##==============================================================================
22 ## Globals
23 ##==============================================================================
24  
25 our @ISA = qw(DTA::CAB::Format::XmlTokWrap);
26  
27 BEGIN {
28 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, filenameRegex=>qr/\.(?i:(?:spliced|tei[\.\-\+]?ws?|wst?)[\.\-]xml)$/);
29 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_)
30 foreach (qw(tei-ws tei+ws tei+w tei-w teiw wst-xml wstxml teiws-xml));
31  
32 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{'teinames'=>1})
33 foreach (qw(teiws-names teiws+names));
34  
35 my @lingnames = (qw(teiws-ling teiws-ling-xml),
36 qw(tei-ling-ws tei+ling+ws lteiws teilws teiwsl ltei-ws ltei+ws ltei+w ltei-w lteiw lwst-xml lwstxml lteiws-xml),
37 qw(tei-ws-ling tei+ws+ling teiws-ling-xml ling-tei-ws));
38 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{'att.linguistic'=>1})
39 foreach (@lingnames);
40  
41 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{'att.linguistic'=>1, teinames=>1})
42 foreach (map {("${_}+names","${_}-names")} @lingnames);
43 }
44  
45 BEGIN {
46 *isa = \&UNIVERSAL::isa;
47 *can = \&UNIVERSAL::can;
48 }
49  
50 ##==============================================================================
51 ## Constructors etc.
52 ##==============================================================================
53  
54 ## $fmt = CLASS_OR_OBJ->new(%args)
55 ## + object structure: HASH ref
56 ## {
57 ## ##-- new in Format::TEIws
58 ## spliceback => $bool, ##-- (output) if true (default), return .cws.cab.xml ; otherwise just .cab.t.xml [requires doc 'teibufr' attribute]
59 ## teinames => $bool, ##-- parse named-entity information from tei qw(persName placeName orgName name)? (default=false)
60 ## teiner => $attr, ##-- output attribute for tei name information (default='ner'; only used if 'teinames' is true)
61 ## teibufr => \$buf, ##-- tei+ws buffer, for spliceback mode
62 ## teidoc => $doc, ##-- tei+ws XML::LibXML::Document
63 ## spliceopts => \%opts, ##-- options for DTA::ToKWrap::Processor::idsplice::new()
64 ## att.linguistic => $bool, ##-- read/write TEI att.linguistic features?
65 ##
66 ## ##-- input: inherited from Format::XmlNative
67 ## xdoc => $xdoc, ##-- XML::LibXML::Document (tokwrap syntax)
68 ## xprs => $xprs, ##-- XML::LibXML parser
69 ## parseXmlData => $bool, ##-- if true, _xmldata key will be parsed (default OVERRIDE=false)
70 ##
71 ## ##-- output: inherited from Format::XmlTokWrap
72 ## arrayEltKeys => \%akey2ekey, ##-- maps array keys to element keys for output
73 ## arrayImplicitKeys => \%akey2undef, ##-- pseudo-hash of array keys NOT mapped to explicit elements
74 ## key2xml => \%key2xml, ##-- maps keys to XML-safe names
75 ## xml2key => \%xml2key, ##-- maps xml keys to internal keys
76 ## ##
77 ## ##-- output: inherited from Format::XmlNative
78 ## #encoding => $inputEncoding, ##-- default: UTF-8; applies to output only!
79 ## level => $level, ##-- output formatting level (default=0)
80 ##
81 ## ##-- common: safety
82 ## safe => $bool, ##-- if true (default), no "unsafe" token data will be generated (_xmlnod,etc.)
83 ## }
84 sub new {
85 my $that = shift;
86 my $fmt = $that->SUPER::new(
87 ##-- defaults
88 spliceback => 1,
89 spliceopts => {soIgnoreAttrs=>[qw(xb c teitext teixp)]},
90 teinames => undef,
91 'att.linguistic' => 0,
92  
93 ##-- user args
94 @_
95 );
96  
97 #$fmt->{ignoreKeys}{'c'} = undef;
98 $fmt->{parseXmlData} = 0 if (!exists($fmt->{parseXmlData})); ##-- ignore XML content
99  
100 if (0) {
101 ##-- DEBUG: also consider setting $DTA::CAB::Logger::defaultLogOpts{twLevel}='TRACE', e.g. with '-lo twLevel=TRACE' on the command-line
102 $fmt->{twopts}{$_} = 'DEBUG' foreach (qw(spliceInfo));
103 $DTA::TokWrap::Utils::TRACE_RUNCMD = 'debug';
104 }
105  
106 return $fmt;
107 }
108  
109 ##=============================================================================
110 ## Methods: Generic
111  
112 ##=============================================================================
113 ## Methods: Input
114  
115 ##--------------------------------------------------------------
116 ## Methods: Input: Generic API
117  
118 ## $fmt = $fmt->close()
119 ## + close current input source, if any
120 sub close {
121 my $fmt = shift;
122 delete $fmt->{teidoc};
123 return $fmt->SUPER::close(@_);
124 }
125  
126 ## $fmt = $fmt->fromString(\$string)
127 ## + select input from string $string
128 sub fromString {
129 my $fmt = shift;
130 my $str = ref($_[0]) ? $_[0] : \$_[0];
131 $fmt->close();
132  
133 ##-- prepare & save tei buffer
134 utf8::encode($$str) if (utf8::is_utf8($$str));
135 $$str =~ s|(<[^>]*)\sxmlns=|$1 XMLNS=|g; ##-- encode default namespaces so that XML::LibXML::Node::nodePath() works
136 $fmt->{teibufr} = $str if ($fmt->{spliceback});
137  
138 ##-- load source xml to $fmt->{teidoc}
139 $fmt->{teidoc} = $fmt->xmlparser()->parse_string($$str)
140 or $fmt->logconfess("XML::LibXML::parse_string() failed: $!");
141  
142 return $fmt;
143 }
144  
145 ## $fmt = $fmt->fromFile($filename_or_handle)
146 ## + calls $fmt->fromFh()
147 sub fromFile {
148 return $_[0]->DTA::CAB::Format::fromFile(@_[1..$#_]);
149 }
150  
151 ## $fmt = $fmt->fromFh($handle)
152 ## + just calls $fmt->fromString()
153 sub fromFh {
154 return $_[0]->DTA::CAB::Format::fromFh_str(@_[1..$#_]);
155 }
156  
157 ## $doc = $fmt->parseDocument()
158 ## + parses buffered XML::LibXML::Document
159 ## + override inserts $doc->{teibufr} attribute for spliceback mode
160 sub parseDocument {
161 my $fmt = shift;
162 if (!defined($fmt->{teidoc})) {
163 $fmt->logconfess("parseDocument(): no source document {teidoc} defined!");
164 return undef;
165 }
166 my $teidoc = $fmt->{teidoc} or return undef;
167 my $teiroot = $teidoc->documentElement or die("parseDocument(): no root element!");
168 my $teinames = $fmt->{teinames};
169 my $att_linguistic = $fmt->{'att.linguistic'};
170  
171 ##-- setup xpath context
172 my $xc = libxml_xpcontext($teiroot);
173  
174 ##-- hack for old libxml without unique_key() method (e.g. kaskade); but BROKEN there!
175 my $nod2key = (XML::LibXML::Node->can('unique_key') ##-- introduced in XML::LibXML v2.0100 2013-08-14
176 || sub {
177 ##-- workaround: use IDs if available
178 $_[0]->getAttribute('id')
179 || $_[0]->getAttribute('xml:id')
180 ##-- fallback (fails on kaskade: libxml-libxml-perl/2.0001+dfsg-1+deb7u1)
181 || (ref($_[0]) && UNIVERSAL::isa($_[0],'SCALAR') ? ${$_[0]} : "$_[0]");
182 }
183 );
184  
185 ##-- parse nodes
186 my (@sids);
187 my (%id2s,%id2w); ##-- %id2s->{$sid} = \%s ; %id2w->{$wid} = $w
188 my (%id2prev,%id2next);
189 my (%ukey2wid); ##-- ($libxml_wnod_unique_key => $wid); for name-parsing
190 my ($snod,$wnod,$sid,$wid,$s,$w, $si,$wi);
191 foreach $snod (@{$xc->findnodes('//*[local-name()="s"]',$teiroot)}) {
192 $sid = $snod->getAttribute('id') || $snod->getAttribute('xml:id') || ("teiws_s_".++$si);
193 $s = $id2s{$sid} = {(map {($_->name=>$_->value)} $snod->attributes), id=>$sid, teixp=>$snod->nodePath, wids=>[]};
194 $id2prev{$sid} = $snod->getAttribute('prev') if ($snod->hasAttribute('prev'));
195 $id2next{$sid} = $snod->getAttribute('next') if ($snod->hasAttribute('next'));
196 foreach $wnod (@{$xc->findnodes('.//*[local-name()="w"]',$snod)}) {
197 $wid = $wnod->getAttribute('id') || $wnod->getAttribute('xml:id') || ("teiws_w_".++$wi);
198 $w = $id2w{$wid} = $fmt->parseNode($wnod);
199 if ($att_linguistic) {
200 ##-- hack: parse TEI att.linguistic features
201 $w->{moot}{word} //= $w->{norm};
202 $w->{moot}{tag} //= $w->{tag};
203 $w->{moot}{lemma} //= $w->{lemma};
204 delete @$w{qw(norm tag lemma)};
205 }
206 @$w{qw(id teixp teitext)} = ($wid,$wnod->nodePath,join('', map {$_->nodeValue} grep {UNIVERSAL::isa($_,'XML::LibXML::Text')} $wnod->childNodes));
207 $id2prev{$wid} = $wnod->getAttribute('prev') if ($wnod->hasAttribute('prev'));
208 $id2next{$wid} = $wnod->getAttribute('next') if ($wnod->hasAttribute('next'));
209 $ukey2wid{$nod2key->($wnod)} = $wid if ($teinames);
210 push(@{$s->{wids}},$wid);
211 }
212 push(@sids,$sid);
213 }
214  
215 ##-- parse teinames?
216 if ($teinames) {
217 my $teiner = $fmt->{teiner} || 'ner';
218 my $ni = 0;
219 my ($nnod,$ntyp,$nid,$nref);
220 foreach $nnod (@{$xc->findnodes('//*[local-name()="persName" or local-name()="placeName" or local-name()="orgName" or local-name()="name"]')}) {
221 $nid = $nnod->getAttribute('id') || $nnod->getAttribute('xml:id') || ("teiws_ne_".++$ni);
222 $nref = $nnod->getAttribute('ref');
223 $ntyp = ($nnod->nodeName =~ /Name/ ? $nnod->nodeName : ($nnod->getAttribute('type') || 'MISC'));
224  
225 foreach $wnod (@{$xc->findnodes('.//*[local-name()="w"]',$nnod)}) {
226 next if (!defined($wid=$ukey2wid{$nod2key->($wnod)}) || !defined($w=$id2w{$wid}));
227 ##-- see DTA::CAB::Format::TT for syncope-style $w->{ner} conventions
228 push(@{$w->{$teiner}}, { nid=>$nid, func=>$ntyp, ($nref ? (ref=>$nref) : qw()) });
229 }
230 }
231 }
232  
233 ##-- construct output document, de-fragmenting as we go
234 my @body = qw();
235 my ($snxt,$wnxt,$tokens);
236 foreach $sid (grep {!exists $id2prev{$_}} @sids) {
237 push(@body, $s = $id2s{$sid});
238 while (($sid=$id2next{$sid})) {
239 $sid =~ s/^\#//;
240 $snxt = $id2s{$sid};
241 $s->{teixp} .= "|$snxt->{teixp}";
242 push(@{$s->{wids}},@{$snxt->{wids}});
243 }
244 $tokens = $s->{tokens} = [];
245 foreach $wid (grep {!exists $id2prev{$_}} @{$s->{wids}}) {
246 push(@$tokens, $w = $id2w{$wid});
247 while (($wid=$id2next{$wid})) {
248 $wid =~ s/^\#//;
249 $wnxt = $id2w{$wid};
250 $w->{teitext} .= $wnxt->{teitext};
251 $w->{teixp} .= "|$wnxt->{teixp}";
252 }
253 delete @$w{qw(prev next part ref XMLNS)};
254 if (!defined($w->{text})) {
255 ##-- hack for tei without //w/@t or //w/@text: heuristically analyze raw text content
256 ($w->{text}=$w->{teitext}) =~ s/[\-¬⁊‒–—]?\s+["'«»٬ۥ‘’‚‛“”„‟′″]*//g;
257 }
258 }
259 delete @$s{qw(wids prev next part ref XMLNS)};
260 }
261  
262 ##-- finalize document
263 my $doc = bless {(map {($_->name=>$_->value)} grep {$_->isa('XML::LibXML::Attr')} $teiroot->attributes), body=>\@body}, 'DTA::CAB::Document';
264 $doc->{teibufr} = $fmt->{teibufr} if ($fmt->{spliceback});
265  
266 return $doc;
267 }
268  
269 ##=============================================================================
270 ## Methods: Output
271 ##==============================================================================
272  
273 ##--------------------------------------------------------------
274 ## Methods: Output: MIME & HTTP stuff
275  
276 ## $short = $fmt->shortName()
277 ## + returns "official" short name for this format
278 ## + default just returns package suffix
279 sub shortName {
280 return 'teiws';
281 }
282  
283 ## $ext = $fmt->defaultExtension()
284 ## + returns default filename extension for this format (default='.cab')
285 sub defaultExtension { return '.tei+ws.xml'; }
286  
287 ##--------------------------------------------------------------
288 ## Methods: Output: output selection
289  
290 ## $fmt = $fmt->flush()
291 ## + flush any buffered output to selected output source
292 sub flush {
293 my $fmt = shift;
294 #$fmt->buf2fh(\$fmt->{outbuf}, $fmt->{fh}) if (defined($fmt->{outbuf}) && defined($fmt->{fh}));
295 #$fmt->SUPER::flush(@_); ##-- not here, since this writes literal {xdoc} to the output file!
296 $fmt->{fh}->flush() if (defined($fmt->{fh}));
297 delete @$fmt{qw(outbuf xdoc teidoc)};
298 return $fmt;
299 }
300  
301 ## $fmt = $fmt->toString(\$str)
302 ## $fmt = $fmt->toString(\$str,$formatLevel)
303 ## + select output to byte-string
304 ## + override reverts to DTA::CAB::Format::toString()
305 sub toString {
306 return $_[0]->DTA::CAB::Format::toString(@_[1..$#_]);
307 }
308  
309 ## $fmt_or_undef = $fmt->toFile($filename, $formatLevel)
310 ## + select output to $filename
311 ## + override reverts to DTA::CAB::Format::toFile()
312 sub toFile {
313 return $_[0]->DTA::CAB::Format::toFile(@_[1..$#_]);
314 }
315  
316 ## $fmt_or_undef = $fmt->toFh($fh,$formatLevel)
317 ## + select output to filehandle $fh
318 ## + override reverts to DTA::CAB::Format::toFh()
319 sub toFh {
320 return $_[0]->DTA::CAB::Format::toFh(@_[1..$#_]);
321 }
322  
323 ##--------------------------------------------------------------
324 ## Methods: Output: Generic API
325  
326 ## $fmt = $fmt->putDocument($doc)
327 ## + override respects local 'keepc' and 'spliceback' flags
328 sub putDocument {
329 my ($fmt,$doc) = @_;
330  
331 ##-- generate flat XML document in $fmt->{xdoc} and/or $sobuf
332 my ($rc,$sobuf);
333 if ($fmt->{'att.linguistic'}) {
334 ##-- att.linguistic hack: generate $sobuf
335 my $lfmt = DTA::CAB::Format::XmlLing->new(twcompat=>1);
336 $lfmt->toString(\$sobuf);
337 $rc = $lfmt->putDocument($doc) && $lfmt->flush();
338 } else {
339 ##-- usual case: call superclass (XmlTokWrap) method: populate $fmt->{xdoc}
340 $rc = $fmt->DTA::CAB::Format::XmlTokWrap::putDocument($doc);
341 }
342  
343 ##-- get original TEI-XML buffer
344 my $teibufr = $doc->{teibufr} || $fmt->{teibufr};
345 if (!$fmt->{spliceback} || !defined($teibufr) || !$$teibufr) {
346 $fmt->logwarn("spliceback mode requested but no 'teibufr' document property - using XmlTokWrap format") if ($fmt->{spliceback});
347 if (defined($sobuf)) {
348 $fmt->buf2fh(\$sobuf,$fmt->{fh});
349 } else {
350 $fmt->{xdoc}->toFH($fmt->{fh},($fmt->{level}||0));
351 }
352 return $rc;
353 }
354 $$teibufr =~ s|(<[^>]*)\sXMLNS=|$1 xmlns=|g; ##-- decode default namespaces (hack)
355  
356 ##-- splice in analysis data
357 my $splicer = DTA::TokWrap::Processor::idsplice->new(%{$fmt->{spliceopts}||{}});
358 $sobuf = $fmt->{xdoc}->toString(0) if (!defined($sobuf) && $fmt->{xdoc});
359 $splicer->splice_so(base=>$teibufr,so=>\$sobuf,out=>$fmt->{fh});
360  
361 return $fmt;
362 }
363  
364  
365  
366  
367  
368 1; ##-- be happy
369  
370 __END__