| 1 | ## -*- Mode: CPerl; coding: utf-8 -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Format::TEIws.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: Datum parser|formatter: XML: TEI: with //w and //s elements, as output by DTA::TokWrap | ||
| 6 | ## + uses DTA::CAB::Format::XmlTokWrap for output | ||
| 7 | |||
| 8 | package DTA::CAB::Format::TEIws; | ||
| 9 | use DTA::CAB::Format::TEI; | ||
| 10 | use DTA::CAB::Datum ':all'; | ||
| 11 | use DTA::CAB::Utils ':temp', ':libxml'; | ||
| 12 | use DTA::TokWrap; | ||
| 13 | use DTA::TokWrap::Utils qw(); | ||
| 14 | use File::Path; | ||
| 15 | use XML::LibXML; | ||
| 16 | use IO::File; | ||
| 17 | use Carp; | ||
| 18 | use utf8; | ||
| 19 | use strict; | ||
| 20 | |||
| 21 | ##============================================================================== | ||
| 22 | ## Globals | ||
| 23 | ##============================================================================== | ||
| 24 | |||
| 25 | our @ISA = qw(DTA::CAB::Format::XmlTokWrap); | ||
| 26 | |||
| 27 | BEGIN { | ||
| 28 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, filenameRegex=>qr/\.(?i:(?:spliced|tei[\.\-\+]?ws?|wst?)[\.\-]xml)$/); | ||
| 29 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_) | ||
| 30 | foreach (qw(tei-ws tei+ws tei+w tei-w teiw wst-xml wstxml teiws-xml)); | ||
| 31 | |||
| 32 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{'teinames'=>1}) | ||
| 33 | foreach (qw(teiws-names teiws+names)); | ||
| 34 | |||
| 35 | my @lingnames = (qw(teiws-ling teiws-ling-xml), | ||
| 36 | qw(tei-ling-ws tei+ling+ws lteiws teilws teiwsl ltei-ws ltei+ws ltei+w ltei-w lteiw lwst-xml lwstxml lteiws-xml), | ||
| 37 | qw(tei-ws-ling tei+ws+ling teiws-ling-xml ling-tei-ws)); | ||
| 38 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{'att.linguistic'=>1}) | ||
| 39 | foreach (@lingnames); | ||
| 40 | |||
| 41 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, opts=>{'att.linguistic'=>1, teinames=>1}) | ||
| 42 | foreach (map {("${_}+names","${_}-names")} @lingnames); | ||
| 43 | } | ||
| 44 | |||
| 45 | BEGIN { | ||
| 46 | *isa = \&UNIVERSAL::isa; | ||
| 47 | *can = \&UNIVERSAL::can; | ||
| 48 | } | ||
| 49 | |||
| 50 | ##============================================================================== | ||
| 51 | ## Constructors etc. | ||
| 52 | ##============================================================================== | ||
| 53 | |||
| 54 | ## $fmt = CLASS_OR_OBJ->new(%args) | ||
| 55 | ## + object structure: HASH ref | ||
| 56 | ## { | ||
| 57 | ## ##-- new in Format::TEIws | ||
| 58 | ## spliceback => $bool, ##-- (output) if true (default), return .cws.cab.xml ; otherwise just .cab.t.xml [requires doc 'teibufr' attribute] | ||
| 59 | ## teinames => $bool, ##-- parse named-entity information from tei qw(persName placeName orgName name)? (default=false) | ||
| 60 | ## teiner => $attr, ##-- output attribute for tei name information (default='ner'; only used if 'teinames' is true) | ||
| 61 | ## teibufr => \$buf, ##-- tei+ws buffer, for spliceback mode | ||
| 62 | ## teidoc => $doc, ##-- tei+ws XML::LibXML::Document | ||
| 63 | ## spliceopts => \%opts, ##-- options for DTA::ToKWrap::Processor::idsplice::new() | ||
| 64 | ## att.linguistic => $bool, ##-- read/write TEI att.linguistic features? | ||
| 65 | ## | ||
| 66 | ## ##-- input: inherited from Format::XmlNative | ||
| 67 | ## xdoc => $xdoc, ##-- XML::LibXML::Document (tokwrap syntax) | ||
| 68 | ## xprs => $xprs, ##-- XML::LibXML parser | ||
| 69 | ## parseXmlData => $bool, ##-- if true, _xmldata key will be parsed (default OVERRIDE=false) | ||
| 70 | ## | ||
| 71 | ## ##-- output: inherited from Format::XmlTokWrap | ||
| 72 | ## arrayEltKeys => \%akey2ekey, ##-- maps array keys to element keys for output | ||
| 73 | ## arrayImplicitKeys => \%akey2undef, ##-- pseudo-hash of array keys NOT mapped to explicit elements | ||
| 74 | ## key2xml => \%key2xml, ##-- maps keys to XML-safe names | ||
| 75 | ## xml2key => \%xml2key, ##-- maps xml keys to internal keys | ||
| 76 | ## ## | ||
| 77 | ## ##-- output: inherited from Format::XmlNative | ||
| 78 | ## #encoding => $inputEncoding, ##-- default: UTF-8; applies to output only! | ||
| 79 | ## level => $level, ##-- output formatting level (default=0) | ||
| 80 | ## | ||
| 81 | ## ##-- common: safety | ||
| 82 | ## safe => $bool, ##-- if true (default), no "unsafe" token data will be generated (_xmlnod,etc.) | ||
| 83 | ## } | ||
| 84 | sub new { | ||
| 85 | my $that = shift; | ||
| 86 | my $fmt = $that->SUPER::new( | ||
| 87 | ##-- defaults | ||
| 88 | spliceback => 1, | ||
| 89 | spliceopts => {soIgnoreAttrs=>[qw(xb c teitext teixp)]}, | ||
| 90 | teinames => undef, | ||
| 91 | 'att.linguistic' => 0, | ||
| 92 | |||
| 93 | ##-- user args | ||
| 94 | @_ | ||
| 95 | ); | ||
| 96 | |||
| 97 | #$fmt->{ignoreKeys}{'c'} = undef; | ||
| 98 | $fmt->{parseXmlData} = 0 if (!exists($fmt->{parseXmlData})); ##-- ignore XML content | ||
| 99 | |||
| 100 | if (0) { | ||
| 101 | ##-- DEBUG: also consider setting $DTA::CAB::Logger::defaultLogOpts{twLevel}='TRACE', e.g. with '-lo twLevel=TRACE' on the command-line | ||
| 102 | $fmt->{twopts}{$_} = 'DEBUG' foreach (qw(spliceInfo)); | ||
| 103 | $DTA::TokWrap::Utils::TRACE_RUNCMD = 'debug'; | ||
| 104 | } | ||
| 105 | |||
| 106 | return $fmt; | ||
| 107 | } | ||
| 108 | |||
| 109 | ##============================================================================= | ||
| 110 | ## Methods: Generic | ||
| 111 | |||
| 112 | ##============================================================================= | ||
| 113 | ## Methods: Input | ||
| 114 | |||
| 115 | ##-------------------------------------------------------------- | ||
| 116 | ## Methods: Input: Generic API | ||
| 117 | |||
| 118 | ## $fmt = $fmt->close() | ||
| 119 | ## + close current input source, if any | ||
| 120 | sub close { | ||
| 121 | my $fmt = shift; | ||
| 122 | delete $fmt->{teidoc}; | ||
| 123 | return $fmt->SUPER::close(@_); | ||
| 124 | } | ||
| 125 | |||
| 126 | ## $fmt = $fmt->fromString(\$string) | ||
| 127 | ## + select input from string $string | ||
| 128 | sub fromString { | ||
| 129 | my $fmt = shift; | ||
| 130 | my $str = ref($_[0]) ? $_[0] : \$_[0]; | ||
| 131 | $fmt->close(); | ||
| 132 | |||
| 133 | ##-- prepare & save tei buffer | ||
| 134 | utf8::encode($$str) if (utf8::is_utf8($$str)); | ||
| 135 | $$str =~ s|(<[^>]*)\sxmlns=|$1 XMLNS=|g; ##-- encode default namespaces so that XML::LibXML::Node::nodePath() works | ||
| 136 | $fmt->{teibufr} = $str if ($fmt->{spliceback}); | ||
| 137 | |||
| 138 | ##-- load source xml to $fmt->{teidoc} | ||
| 139 | $fmt->{teidoc} = $fmt->xmlparser()->parse_string($$str) | ||
| 140 | or $fmt->logconfess("XML::LibXML::parse_string() failed: $!"); | ||
| 141 | |||
| 142 | return $fmt; | ||
| 143 | } | ||
| 144 | |||
| 145 | ## $fmt = $fmt->fromFile($filename_or_handle) | ||
| 146 | ## + calls $fmt->fromFh() | ||
| 147 | sub fromFile { | ||
| 148 | return $_[0]->DTA::CAB::Format::fromFile(@_[1..$#_]); | ||
| 149 | } | ||
| 150 | |||
| 151 | ## $fmt = $fmt->fromFh($handle) | ||
| 152 | ## + just calls $fmt->fromString() | ||
| 153 | sub fromFh { | ||
| 154 | return $_[0]->DTA::CAB::Format::fromFh_str(@_[1..$#_]); | ||
| 155 | } | ||
| 156 | |||
| 157 | ## $doc = $fmt->parseDocument() | ||
| 158 | ## + parses buffered XML::LibXML::Document | ||
| 159 | ## + override inserts $doc->{teibufr} attribute for spliceback mode | ||
| 160 | sub parseDocument { | ||
| 161 | my $fmt = shift; | ||
| 162 | if (!defined($fmt->{teidoc})) { | ||
| 163 | $fmt->logconfess("parseDocument(): no source document {teidoc} defined!"); | ||
| 164 | return undef; | ||
| 165 | } | ||
| 166 | my $teidoc = $fmt->{teidoc} or return undef; | ||
| 167 | my $teiroot = $teidoc->documentElement or die("parseDocument(): no root element!"); | ||
| 168 | my $teinames = $fmt->{teinames}; | ||
| 169 | my $att_linguistic = $fmt->{'att.linguistic'}; | ||
| 170 | |||
| 171 | ##-- setup xpath context | ||
| 172 | my $xc = libxml_xpcontext($teiroot); | ||
| 173 | |||
| 174 | ##-- hack for old libxml without unique_key() method (e.g. kaskade); but BROKEN there! | ||
| 175 | my $nod2key = (XML::LibXML::Node->can('unique_key') ##-- introduced in XML::LibXML v2.0100 2013-08-14 | ||
| 176 | || sub { | ||
| 177 | ##-- workaround: use IDs if available | ||
| 178 | $_[0]->getAttribute('id') | ||
| 179 | || $_[0]->getAttribute('xml:id') | ||
| 180 | ##-- fallback (fails on kaskade: libxml-libxml-perl/2.0001+dfsg-1+deb7u1) | ||
| 181 | || (ref($_[0]) && UNIVERSAL::isa($_[0],'SCALAR') ? ${$_[0]} : "$_[0]"); | ||
| 182 | } | ||
| 183 | ); | ||
| 184 | |||
| 185 | ##-- parse nodes | ||
| 186 | my (@sids); | ||
| 187 | my (%id2s,%id2w); ##-- %id2s->{$sid} = \%s ; %id2w->{$wid} = $w | ||
| 188 | my (%id2prev,%id2next); | ||
| 189 | my (%ukey2wid); ##-- ($libxml_wnod_unique_key => $wid); for name-parsing | ||
| 190 | my ($snod,$wnod,$sid,$wid,$s,$w, $si,$wi); | ||
| 191 | foreach $snod (@{$xc->findnodes('//*[local-name()="s"]',$teiroot)}) { | ||
| 192 | $sid = $snod->getAttribute('id') || $snod->getAttribute('xml:id') || ("teiws_s_".++$si); | ||
| 193 | $s = $id2s{$sid} = {(map {($_->name=>$_->value)} $snod->attributes), id=>$sid, teixp=>$snod->nodePath, wids=>[]}; | ||
| 194 | $id2prev{$sid} = $snod->getAttribute('prev') if ($snod->hasAttribute('prev')); | ||
| 195 | $id2next{$sid} = $snod->getAttribute('next') if ($snod->hasAttribute('next')); | ||
| 196 | foreach $wnod (@{$xc->findnodes('.//*[local-name()="w"]',$snod)}) { | ||
| 197 | $wid = $wnod->getAttribute('id') || $wnod->getAttribute('xml:id') || ("teiws_w_".++$wi); | ||
| 198 | $w = $id2w{$wid} = $fmt->parseNode($wnod); | ||
| 199 | if ($att_linguistic) { | ||
| 200 | ##-- hack: parse TEI att.linguistic features | ||
| 201 | $w->{moot}{word} //= $w->{norm}; | ||
| 202 | $w->{moot}{tag} //= $w->{tag}; | ||
| 203 | $w->{moot}{lemma} //= $w->{lemma}; | ||
| 204 | delete @$w{qw(norm tag lemma)}; | ||
| 205 | } | ||
| 206 | @$w{qw(id teixp teitext)} = ($wid,$wnod->nodePath,join('', map {$_->nodeValue} grep {UNIVERSAL::isa($_,'XML::LibXML::Text')} $wnod->childNodes)); | ||
| 207 | $id2prev{$wid} = $wnod->getAttribute('prev') if ($wnod->hasAttribute('prev')); | ||
| 208 | $id2next{$wid} = $wnod->getAttribute('next') if ($wnod->hasAttribute('next')); | ||
| 209 | $ukey2wid{$nod2key->($wnod)} = $wid if ($teinames); | ||
| 210 | push(@{$s->{wids}},$wid); | ||
| 211 | } | ||
| 212 | push(@sids,$sid); | ||
| 213 | } | ||
| 214 | |||
| 215 | ##-- parse teinames? | ||
| 216 | if ($teinames) { | ||
| 217 | my $teiner = $fmt->{teiner} || 'ner'; | ||
| 218 | my $ni = 0; | ||
| 219 | my ($nnod,$ntyp,$nid,$nref); | ||
| 220 | foreach $nnod (@{$xc->findnodes('//*[local-name()="persName" or local-name()="placeName" or local-name()="orgName" or local-name()="name"]')}) { | ||
| 221 | $nid = $nnod->getAttribute('id') || $nnod->getAttribute('xml:id') || ("teiws_ne_".++$ni); | ||
| 222 | $nref = $nnod->getAttribute('ref'); | ||
| 223 | $ntyp = ($nnod->nodeName =~ /Name/ ? $nnod->nodeName : ($nnod->getAttribute('type') || 'MISC')); | ||
| 224 | |||
| 225 | foreach $wnod (@{$xc->findnodes('.//*[local-name()="w"]',$nnod)}) { | ||
| 226 | next if (!defined($wid=$ukey2wid{$nod2key->($wnod)}) || !defined($w=$id2w{$wid})); | ||
| 227 | ##-- see DTA::CAB::Format::TT for syncope-style $w->{ner} conventions | ||
| 228 | push(@{$w->{$teiner}}, { nid=>$nid, func=>$ntyp, ($nref ? (ref=>$nref) : qw()) }); | ||
| 229 | } | ||
| 230 | } | ||
| 231 | } | ||
| 232 | |||
| 233 | ##-- construct output document, de-fragmenting as we go | ||
| 234 | my @body = qw(); | ||
| 235 | my ($snxt,$wnxt,$tokens); | ||
| 236 | foreach $sid (grep {!exists $id2prev{$_}} @sids) { | ||
| 237 | push(@body, $s = $id2s{$sid}); | ||
| 238 | while (($sid=$id2next{$sid})) { | ||
| 239 | $sid =~ s/^\#//; | ||
| 240 | $snxt = $id2s{$sid}; | ||
| 241 | $s->{teixp} .= "|$snxt->{teixp}"; | ||
| 242 | push(@{$s->{wids}},@{$snxt->{wids}}); | ||
| 243 | } | ||
| 244 | $tokens = $s->{tokens} = []; | ||
| 245 | foreach $wid (grep {!exists $id2prev{$_}} @{$s->{wids}}) { | ||
| 246 | push(@$tokens, $w = $id2w{$wid}); | ||
| 247 | while (($wid=$id2next{$wid})) { | ||
| 248 | $wid =~ s/^\#//; | ||
| 249 | $wnxt = $id2w{$wid}; | ||
| 250 | $w->{teitext} .= $wnxt->{teitext}; | ||
| 251 | $w->{teixp} .= "|$wnxt->{teixp}"; | ||
| 252 | } | ||
| 253 | delete @$w{qw(prev next part ref XMLNS)}; | ||
| 254 | if (!defined($w->{text})) { | ||
| 255 | ##-- hack for tei without //w/@t or //w/@text: heuristically analyze raw text content | ||
| 256 | ($w->{text}=$w->{teitext}) =~ s/[\-¬⁊‒–—]?\s+["'«»٬ۥ‘’‚‛“”„‟′″]*//g; | ||
| 257 | } | ||
| 258 | } | ||
| 259 | delete @$s{qw(wids prev next part ref XMLNS)}; | ||
| 260 | } | ||
| 261 | |||
| 262 | ##-- finalize document | ||
| 263 | my $doc = bless {(map {($_->name=>$_->value)} grep {$_->isa('XML::LibXML::Attr')} $teiroot->attributes), body=>\@body}, 'DTA::CAB::Document'; | ||
| 264 | $doc->{teibufr} = $fmt->{teibufr} if ($fmt->{spliceback}); | ||
| 265 | |||
| 266 | return $doc; | ||
| 267 | } | ||
| 268 | |||
| 269 | ##============================================================================= | ||
| 270 | ## Methods: Output | ||
| 271 | ##============================================================================== | ||
| 272 | |||
| 273 | ##-------------------------------------------------------------- | ||
| 274 | ## Methods: Output: MIME & HTTP stuff | ||
| 275 | |||
| 276 | ## $short = $fmt->shortName() | ||
| 277 | ## + returns "official" short name for this format | ||
| 278 | ## + default just returns package suffix | ||
| 279 | sub shortName { | ||
| 280 | return 'teiws'; | ||
| 281 | } | ||
| 282 | |||
| 283 | ## $ext = $fmt->defaultExtension() | ||
| 284 | ## + returns default filename extension for this format (default='.cab') | ||
| 285 | sub defaultExtension { return '.tei+ws.xml'; } | ||
| 286 | |||
| 287 | ##-------------------------------------------------------------- | ||
| 288 | ## Methods: Output: output selection | ||
| 289 | |||
| 290 | ## $fmt = $fmt->flush() | ||
| 291 | ## + flush any buffered output to selected output source | ||
| 292 | sub flush { | ||
| 293 | my $fmt = shift; | ||
| 294 | #$fmt->buf2fh(\$fmt->{outbuf}, $fmt->{fh}) if (defined($fmt->{outbuf}) && defined($fmt->{fh})); | ||
| 295 | #$fmt->SUPER::flush(@_); ##-- not here, since this writes literal {xdoc} to the output file! | ||
| 296 | $fmt->{fh}->flush() if (defined($fmt->{fh})); | ||
| 297 | delete @$fmt{qw(outbuf xdoc teidoc)}; | ||
| 298 | return $fmt; | ||
| 299 | } | ||
| 300 | |||
| 301 | ## $fmt = $fmt->toString(\$str) | ||
| 302 | ## $fmt = $fmt->toString(\$str,$formatLevel) | ||
| 303 | ## + select output to byte-string | ||
| 304 | ## + override reverts to DTA::CAB::Format::toString() | ||
| 305 | sub toString { | ||
| 306 | return $_[0]->DTA::CAB::Format::toString(@_[1..$#_]); | ||
| 307 | } | ||
| 308 | |||
| 309 | ## $fmt_or_undef = $fmt->toFile($filename, $formatLevel) | ||
| 310 | ## + select output to $filename | ||
| 311 | ## + override reverts to DTA::CAB::Format::toFile() | ||
| 312 | sub toFile { | ||
| 313 | return $_[0]->DTA::CAB::Format::toFile(@_[1..$#_]); | ||
| 314 | } | ||
| 315 | |||
| 316 | ## $fmt_or_undef = $fmt->toFh($fh,$formatLevel) | ||
| 317 | ## + select output to filehandle $fh | ||
| 318 | ## + override reverts to DTA::CAB::Format::toFh() | ||
| 319 | sub toFh { | ||
| 320 | return $_[0]->DTA::CAB::Format::toFh(@_[1..$#_]); | ||
| 321 | } | ||
| 322 | |||
| 323 | ##-------------------------------------------------------------- | ||
| 324 | ## Methods: Output: Generic API | ||
| 325 | |||
| 326 | ## $fmt = $fmt->putDocument($doc) | ||
| 327 | ## + override respects local 'keepc' and 'spliceback' flags | ||
| 328 | sub putDocument { | ||
| 329 | my ($fmt,$doc) = @_; | ||
| 330 | |||
| 331 | ##-- generate flat XML document in $fmt->{xdoc} and/or $sobuf | ||
| 332 | my ($rc,$sobuf); | ||
| 333 | if ($fmt->{'att.linguistic'}) { | ||
| 334 | ##-- att.linguistic hack: generate $sobuf | ||
| 335 | my $lfmt = DTA::CAB::Format::XmlLing->new(twcompat=>1); | ||
| 336 | $lfmt->toString(\$sobuf); | ||
| 337 | $rc = $lfmt->putDocument($doc) && $lfmt->flush(); | ||
| 338 | } else { | ||
| 339 | ##-- usual case: call superclass (XmlTokWrap) method: populate $fmt->{xdoc} | ||
| 340 | $rc = $fmt->DTA::CAB::Format::XmlTokWrap::putDocument($doc); | ||
| 341 | } | ||
| 342 | |||
| 343 | ##-- get original TEI-XML buffer | ||
| 344 | my $teibufr = $doc->{teibufr} || $fmt->{teibufr}; | ||
| 345 | if (!$fmt->{spliceback} || !defined($teibufr) || !$$teibufr) { | ||
| 346 | $fmt->logwarn("spliceback mode requested but no 'teibufr' document property - using XmlTokWrap format") if ($fmt->{spliceback}); | ||
| 347 | if (defined($sobuf)) { | ||
| 348 | $fmt->buf2fh(\$sobuf,$fmt->{fh}); | ||
| 349 | } else { | ||
| 350 | $fmt->{xdoc}->toFH($fmt->{fh},($fmt->{level}||0)); | ||
| 351 | } | ||
| 352 | return $rc; | ||
| 353 | } | ||
| 354 | $$teibufr =~ s|(<[^>]*)\sXMLNS=|$1 xmlns=|g; ##-- decode default namespaces (hack) | ||
| 355 | |||
| 356 | ##-- splice in analysis data | ||
| 357 | my $splicer = DTA::TokWrap::Processor::idsplice->new(%{$fmt->{spliceopts}||{}}); | ||
| 358 | $sobuf = $fmt->{xdoc}->toString(0) if (!defined($sobuf) && $fmt->{xdoc}); | ||
| 359 | $splicer->splice_so(base=>$teibufr,so=>\$sobuf,out=>$fmt->{fh}); | ||
| 360 | |||
| 361 | return $fmt; | ||
| 362 | } | ||
| 363 | |||
| 364 | |||
| 365 | |||
| 366 | |||
| 367 | |||
| 368 | 1; ##-- be happy | ||
| 369 | |||
| 370 | __END__ |