| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Format::XmlTokWrap.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: Datum parser|formatter: XML (tokwrap), fast quick & dirty I/O for (.ddc).t.xml | ||
| 6 | |||
| 7 | package DTA::CAB::Format::XmlTokWrapFast; | ||
| 8 | use DTA::CAB::Format::XmlTokWrap; | ||
| 9 | use DTA::CAB::Datum ':all'; | ||
| 10 | use XML::Parser; | ||
| 11 | use IO::File; | ||
| 12 | use Carp; | ||
| 13 | use strict; | ||
| 14 | |||
| 15 | ##============================================================================== | ||
| 16 | ## Globals | ||
| 17 | ##============================================================================== | ||
| 18 | |||
| 19 | our @ISA = qw(DTA::CAB::Format::XmlTokWrap); | ||
| 20 | |||
| 21 | BEGIN { | ||
| 22 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, filenameRegex=>qr/(?:\.(?i:f[tuws](?:\.?)xml))$/); | ||
| 23 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_) foreach (qw(ftxml ft-xml ftwxml ftw-xml)); | ||
| 24 | } | ||
| 25 | |||
| 26 | BEGIN { | ||
| 27 | *isa = \&UNIVERSAL::isa; | ||
| 28 | *can = \&UNIVERSAL::can; | ||
| 29 | } | ||
| 30 | |||
| 31 | ##============================================================================== | ||
| 32 | ## Constructors etc. | ||
| 33 | ##============================================================================== | ||
| 34 | |||
| 35 | ## $fmt = CLASS_OR_OBJ->new(%args) | ||
| 36 | ## + object structure: HASH ref | ||
| 37 | ## { | ||
| 38 | ## ##-- input: new | ||
| 39 | ## doc => $doc, ##-- cached parsed DTA::CAB::Document | ||
| 40 | ## | ||
| 41 | ## ##-- input: inherited (but unused) | ||
| 42 | ## #xdoc => $xdoc, ##-- XML::LibXML::Document | ||
| 43 | ## #xprs => $xprs, ##-- override: XML::Parser parser | ||
| 44 | ## | ||
| 45 | ## ##-- output: inherited from DTA::CAB::Format | ||
| 46 | ## utf8 => $bool, ##-- always true | ||
| 47 | ## level => $level, ##-- output formatting level (default=0) | ||
| 48 | ## output_moot => $bool, ##-- include <moot> output element? (default=1) | ||
| 49 | ## output_ner => $bool, ##-- include <ner> output element? (default=0) | ||
| 50 | ## } | ||
| 51 | sub new { | ||
| 52 | my $that = shift; | ||
| 53 | my $fmt = $that->DTA::CAB::Format::XmlCommon::new | ||
| 54 | ( | ||
| 55 | xprs => undef, | ||
| 56 | doc => undef, | ||
| 57 | output_moot => 1, | ||
| 58 | output_ner => 0, | ||
| 59 | @_ | ||
| 60 | ); | ||
| 61 | return $fmt; | ||
| 62 | } | ||
| 63 | |||
| 64 | ## $xmlparser = $fmt->xmlparser() | ||
| 65 | ## + returns cached $fmt->{xprs} if available | ||
| 66 | ## + otherwise caches & returns new XML::Parser | ||
| 67 | sub xmlparser { | ||
| 68 | return $_[0]{xprs} if (defined($_[0]{xprs})); | ||
| 69 | #my $fmt = shift; | ||
| 70 | |||
| 71 | ##-------------------------------------- | ||
| 72 | ## closure variables | ||
| 73 | my ($doc,$body, $s,$stoks, $w,@stack,%attrs); | ||
| 74 | |||
| 75 | ##-------------------------------------- | ||
| 76 | ## parser | ||
| 77 | $_[0]{xprs} = XML::Parser->new | ||
| 78 | ( | ||
| 79 | ErrorContext => 1, | ||
| 80 | ProtocolEncoding => 'UTF-8', | ||
| 81 | #ParseParamEnt => '???', | ||
| 82 | Handlers => { | ||
| 83 | ##---------------- | ||
| 84 | ## undef = cb_init($expat) | ||
| 85 | Init => sub { | ||
| 86 | $body = []; | ||
| 87 | $doc = {body=>$body}; | ||
| 88 | @stack = qw(); | ||
| 89 | }, | ||
| 90 | |||
| 91 | ##---------------- | ||
| 92 | ## undef = cb_start($expat, $elt,%attrs) | ||
| 93 | Start => sub { | ||
| 94 | %attrs = @_[2..$#_]; | ||
| 95 | push(@stack,$_[1]); | ||
| 96 | |||
| 97 | if ($_[1] eq 'w') { | ||
| 98 | ##-- w | ||
| 99 | if (defined($attrs{t}) && !defined($attrs{text})) { | ||
| 100 | $attrs{text} = $attrs{t}; | ||
| 101 | delete($attrs{t}); | ||
| 102 | } | ||
| 103 | push(@$stoks, $w={%attrs}); | ||
| 104 | } elsif ($_[1] eq 'moot') { | ||
| 105 | ##-- w/moot : parse | ||
| 106 | $w->{moot} = {%attrs} if ($w); | ||
| 107 | } elsif ($_[1] eq 'a') { | ||
| 108 | ##-- w/a : do nothing | ||
| 109 | ; | ||
| 110 | } elsif ($_[1] eq 's') { | ||
| 111 | ##-- s | ||
| 112 | push(@$body, $s={%attrs,tokens=>($stoks=[])}); | ||
| 113 | } elsif (@stack==1) { | ||
| 114 | ##-- doc | ||
| 115 | if (defined($attrs{'xml:base'})) { | ||
| 116 | $attrs{'base'}=$attrs{'xml:base'}; | ||
| 117 | delete($attrs{'xml:base'}); | ||
| 118 | } | ||
| 119 | $doc = {%attrs, body=>$body}; | ||
| 120 | } | ||
| 121 | }, | ||
| 122 | |||
| 123 | ##---------------- | ||
| 124 | ## undef = cb_end($expat,$elt) | ||
| 125 | End => sub { | ||
| 126 | pop(@stack); | ||
| 127 | }, | ||
| 128 | |||
| 129 | ##---------------- | ||
| 130 | ## undef = cb_char($expat,$string) | ||
| 131 | Char => sub { | ||
| 132 | push(@{$w->{toka}}, $_[1]) if ($stack[$#stack] eq 'a'); | ||
| 133 | }, | ||
| 134 | |||
| 135 | ##---------------- | ||
| 136 | ## undef = cb_default($expat, $str) | ||
| 137 | #Default => $cb_default, | ||
| 138 | |||
| 139 | ##---------------- | ||
| 140 | ## $parse_rv = cb_final($expat) | ||
| 141 | Final => sub { | ||
| 142 | $body = $s = $stoks = $w = undef; | ||
| 143 | return bless($doc,'DTA::CAB::Document'); | ||
| 144 | }, | ||
| 145 | }, | ||
| 146 | ) | ||
| 147 | or $_[0]->logconfess("couldn't create XML::Parser"); | ||
| 148 | |||
| 149 | return $_[0]{xprs}; | ||
| 150 | } | ||
| 151 | |||
| 152 | ##============================================================================== | ||
| 153 | ## Methods: Persistence | ||
| 154 | ##============================================================================== | ||
| 155 | |||
| 156 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 157 | ## + returns list of keys not to be saved | ||
| 158 | ## + inherited | ||
| 159 | |||
| 160 | ##============================================================================= | ||
| 161 | ## Methods: I/O: generic | ||
| 162 | ##============================================================================== | ||
| 163 | |||
| 164 | ## $fmt = $fmt->close($savetmp=0) | ||
| 165 | ## + override calls $fmt->flush() and deletes @$fmt{qw(xdoc output)} | ||
| 166 | sub close { | ||
| 167 | return $_[0]->DTA::CAB::Format::close(@_[1..$#_]); | ||
| 168 | } | ||
| 169 | |||
| 170 | ## @layers = $fmt->iolayers() | ||
| 171 | ## + returns PerlIO layers to use for I/O handles | ||
| 172 | ## + override returns ':raw' | ||
| 173 | sub iolayers { | ||
| 174 | return qw(:raw); | ||
| 175 | } | ||
| 176 | |||
| 177 | ##============================================================================== | ||
| 178 | ## Methods: I/O: Block-wise | ||
| 179 | ##============================================================================== | ||
| 180 | |||
| 181 | ##-------------------------------------------------------------- | ||
| 182 | ## Methods: I/O: Block-wise: Generic | ||
| 183 | |||
| 184 | ## %blockOpts = $CLASS_OR_OBJECT->blockDefaults() | ||
| 185 | ## + returns default block options as for blockOptions() | ||
| 186 | ## + override returns as for $CLASS_OR_OBJECT->blockOptions('2m@s') | ||
| 187 | sub blockDefaults { | ||
| 188 | return ($_[0]->SUPER::blockDefaults(), bsize=>(2*1024*1024), eob=>'s'); | ||
| 189 | } | ||
| 190 | |||
| 191 | ##============================================================================= | ||
| 192 | ## Methods: Input | ||
| 193 | ##============================================================================== | ||
| 194 | |||
| 195 | ##-------------------------------------------------------------- | ||
| 196 | ## Methods: Input: Input selection | ||
| 197 | |||
| 198 | ## $fmt = $fmt->fromString(\$string) | ||
| 199 | ## + input from string | ||
| 200 | sub fromString { | ||
| 201 | my $fmt = shift; | ||
| 202 | $fmt->{doc} = $fmt->xmlparser->parse(ref($_[0]) ? ${$_[0]} : $_[0]) | ||
| 203 | or $fmt->logconfess("fromString(): XML::Parser::parse() failed: $!"); | ||
| 204 | return $fmt; | ||
| 205 | } | ||
| 206 | |||
| 207 | ## $fmt = $fmt->fromFile($filename) | ||
| 208 | ## + input from named file: override buffers XML document in $fmt->{xdoc} | ||
| 209 | sub fromFile { | ||
| 210 | my ($fmt,$file) = @_; | ||
| 211 | $fmt->{doc} = $fmt->xmlparser->parsefile($file) | ||
| 212 | or $fmt->logconfess("fromFile(): XML::Parser::parsefile($file) failed: $!"); | ||
| 213 | return $fmt; | ||
| 214 | } | ||
| 215 | |||
| 216 | ## $fmt = $fmt->fromFh($handle) | ||
| 217 | ## + input from filehandle: override buffers XML document in $fmt->{xdoc} | ||
| 218 | sub fromFh { | ||
| 219 | my ($fmt,$fh) = @_; | ||
| 220 | $fmt->{doc} = $fmt->xmlparser->parse($fh) | ||
| 221 | or $fmt->logconfess("fromFh(): XML::Parser::parse($fh) failed: $!"); | ||
| 222 | return $fmt; | ||
| 223 | } | ||
| 224 | |||
| 225 | ##-------------------------------------------------------------- | ||
| 226 | ## Methods: Input: Generic API | ||
| 227 | |||
| 228 | ## $doc = $fmt->parseDocument() | ||
| 229 | ## + parse document from currently selected input source | ||
| 230 | ## + override returns buffered $fmt->{doc} | ||
| 231 | sub parseDocument { | ||
| 232 | return $_[0]{doc}; | ||
| 233 | } | ||
| 234 | |||
| 235 | |||
| 236 | ##============================================================================= | ||
| 237 | ## Methods: Output | ||
| 238 | ##============================================================================== | ||
| 239 | |||
| 240 | ##-------------------------------------------------------------- | ||
| 241 | ## Methods: Output: MIME & HTTP stuff | ||
| 242 | |||
| 243 | ## $short = $fmt->shortName() | ||
| 244 | ## + returns "official" short name for this format | ||
| 245 | ## + default just returns package suffix | ||
| 246 | sub shortName { | ||
| 247 | return 'ftxml'; | ||
| 248 | } | ||
| 249 | |||
| 250 | ## $ext = $fmt->defaultExtension() | ||
| 251 | ## + returns default filename extension for this format (default='.ft.xml') | ||
| 252 | sub defaultExtension { return '.ft.xml'; } | ||
| 253 | |||
| 254 | ##-------------------------------------------------------------- | ||
| 255 | ## Methods: Output: output selection | ||
| 256 | |||
| 257 | ## $fmt = $fmt->flush() | ||
| 258 | ## + flush accumulated output | ||
| 259 | sub flush { | ||
| 260 | $_[0]->DTA::CAB::Format::flush(@_[1..$#_]); | ||
| 261 | } | ||
| 262 | |||
| 263 | ## $str = $fmt->toString() | ||
| 264 | ## $str = $fmt->toString($formatLevel) | ||
| 265 | ## + flush buffered output document to byte-string | ||
| 266 | sub toString { | ||
| 267 | $_[0]->DTA::CAB::Format::toString(@_[1..$#_]); | ||
| 268 | } | ||
| 269 | |||
| 270 | ## $fmt_or_undef = $fmt->toFile($filename_or_handle, $formatLevel) | ||
| 271 | ## + flush buffered output document to $filename_or_handle | ||
| 272 | ## + default implementation calls $fmt->toFh() | ||
| 273 | sub toFile { | ||
| 274 | $_[0]->DTA::CAB::Format::toFile(@_[1..$#_]); | ||
| 275 | } | ||
| 276 | |||
| 277 | ## $fmt_or_undef = $fmt->toFh($fh,$formatLevel) | ||
| 278 | ## + flush buffered output document to filehandle $fh | ||
| 279 | sub toFh { | ||
| 280 | $_[0]->DTA::CAB::Format::toFh(@_[1..$#_]); | ||
| 281 | } | ||
| 282 | |||
| 283 | ##-------------------------------------------------------------- | ||
| 284 | ## Methods: Output: quick and dirty | ||
| 285 | |||
| 286 | ## $fmt = $fmt->putDocument($doc) | ||
| 287 | sub putDocument { | ||
| 288 | my ($fmt,$doc) = @_; | ||
| 289 | |||
| 290 | ##-------------------- | ||
| 291 | ## local subs | ||
| 292 | |||
| 293 | my $nil = []; | ||
| 294 | |||
| 295 | ##-- $escaped = xmlesc($str) : xml escape (single string) | ||
| 296 | my ($_esc); | ||
| 297 | my $xmlescape = sub { | ||
| 298 | $_esc=shift; | ||
| 299 | $_esc=~s/([\&\'\"\<\>])/'&#'.ord($1).';'/ge; | ||
| 300 | return $_esc; | ||
| 301 | }; | ||
| 302 | |||
| 303 | ##-- $str = xmlattrs(%attrs) | ||
| 304 | my $xmlattrs = sub { | ||
| 305 | return join('', map {' '.$_[$_].'="' . $xmlescape->($_[$_+1]).'"'} grep {defined($_[$_+1])} map {$_*2} (0..$#{_}/2)); | ||
| 306 | }; | ||
| 307 | |||
| 308 | ##-- $str = xmlstart($name,%attrs) | ||
| 309 | my $xmlstart = sub { | ||
| 310 | return "<$_[0]" . $xmlattrs->(@_[1..$#_]) . ">"; | ||
| 311 | }; | ||
| 312 | |||
| 313 | ##-- $str = xmlempty($name,%attrs) | ||
| 314 | my $xmlempty = sub { | ||
| 315 | return "<$_[0]" . $xmlattrs->(@_[1..$#_]) . "/>"; | ||
| 316 | }; | ||
| 317 | |||
| 318 | ##-- $str = xmlelt($name,\@attrs,@content_strings) | ||
| 319 | my $xmlelt = sub { | ||
| 320 | return $xmlempty->($_[0], @{$_[1]||$nil}) if (@_ < 3); | ||
| 321 | return $xmlstart->($_[0], @{$_[1]||$nil}) . join('',@_[2..$#_]) . "</$_[0]>"; | ||
| 322 | }; | ||
| 323 | |||
| 324 | ##-- $str = fstelt($name,$aname,\@analyses) | ||
| 325 | my ($_fsta); | ||
| 326 | my $fstelt = sub { | ||
| 327 | return '' if (!$_[2]); | ||
| 328 | return $xmlelt->($_[0],$nil, | ||
| 329 | map { | ||
| 330 | $_fsta=$_; | ||
| 331 | $xmlempty->(($_[1]||'a'), map {($_=>$_fsta->{$_})} qw(lo lemma hi w)) | ||
| 332 | } @{$_[2]} | ||
| 333 | ); | ||
| 334 | }; | ||
| 335 | |||
| 336 | ##-- $str = mootelt($name,$aname,\%data) | ||
| 337 | my ($_moota); | ||
| 338 | my $mootelt = sub { | ||
| 339 | return '' if (!$_[2]); | ||
| 340 | return $xmlelt->($_[0],[word=>$_[2]{word},tag=>$_[2]{tag},lemma=>$_[2]{lemma}], | ||
| 341 | #$fstelt->('morph', 'a', $_[2]{morph}) | ||
| 342 | ## | ||
| 343 | #(!$_[2]{analyses} || !@{$_[2]{analyses}} ? qw() | ||
| 344 | # : $xmlelt->('analyses',$nil, | ||
| 345 | # map { | ||
| 346 | # $_moota=$_; | ||
| 347 | # $xmlempty->(($_[1]||'a'), map {($_=>$_moota->{$_})} qw(tag lemma prob details)) | ||
| 348 | # } @{$_[2]{analyses}})) | ||
| 349 | ); | ||
| 350 | }; | ||
| 351 | |||
| 352 | ##-- $str = nerelt($name,$aname,\@analyses) | ||
| 353 | my ($_nera); | ||
| 354 | my $nerelt = sub { | ||
| 355 | return '' if (!$_[2]); | ||
| 356 | return $xmlelt->($_[0],$nil, | ||
| 357 | map { | ||
| 358 | $_nera = $_; | ||
| 359 | $xmlempty->($_[1],map {defined($_nera->{$_}) ? ($_=>$_nera->{$_}) : qw()} qw(nid label labid cat func depth)) | ||
| 360 | } @{$_[2]} | ||
| 361 | ); | ||
| 362 | }; | ||
| 363 | |||
| 364 | ##-------------------- | ||
| 365 | ## output handle | ||
| 366 | my $fh = $fmt->{fh}; | ||
| 367 | binmode($fh,':utf8'); | ||
| 368 | $fh->print("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); | ||
| 369 | |||
| 370 | ##-------------------- | ||
| 371 | ## guts | ||
| 372 | my ($output_moot,$output_ner) = @$fmt{qw(output_moot output_ner)}; | ||
| 373 | my ($s,$w); | ||
| 374 | $fh->print("\n", $xmlstart->('doc', base=>$doc->{base})); | ||
| 375 | foreach $s (@{$doc->{body}}) { | ||
| 376 | $fh->print("\n ", $xmlstart->('s',id=>$s->{id}, (exists($s->{pn}) ? (pn=>$s->{pn}) : qw()))); | ||
| 377 | foreach $w (@{$s->{tokens}}) { | ||
| 378 | $fh->print("\n\t", | ||
| 379 | $xmlelt->('w', | ||
| 380 | [ | ||
| 381 | ##-- word attributes: literals | ||
| 382 | (t=>$w->{text}), | ||
| 383 | (map {$_=>$w->{$_}} qw(u id exlex pnd mapclass errid xc xr xp pb lb bb b xb msafe ws)), #c coff clen b boff blen | ||
| 384 | ], | ||
| 385 | ## | ||
| 386 | ##-- content: tokenizer analyses | ||
| 387 | #(map {$xmlelt->('a',$nil,$xmlescape->($w->{$_}))} @{$w->{toka}||$nil}), | ||
| 388 | #($w->{tokpp} && @{$w->{tokpp}} ? $xmlelt->('tokpp',$nil,map {xmlelt('a',$nil,$xmlescape->($_))} @{$w->{tokpp}}) : qw()), | ||
| 389 | ## | ||
| 390 | ##-- content: xlit | ||
| 391 | #($w->{xlit} ? $xmlempty->('xlit',%{$w->{xlit}}) : qw()), | ||
| 392 | ## | ||
| 393 | ##-- content: fsts | ||
| 394 | #(map {$fstelt->($_,'a',$w->{$_})} qw(lts morph rw mlatin eqpho eqrw)), | ||
| 395 | ## | ||
| 396 | ##-- content: moot | ||
| 397 | #$mootelt->('dmoot','a',$w->{dmoot}), | ||
| 398 | ($output_moot ? $mootelt->('moot','a',$w->{moot}) : qw()), | ||
| 399 | ## | ||
| 400 | ##-- content: ner | ||
| 401 | ($output_ner ? $nerelt->('ner','a',$w->{ner}) : qw()), | ||
| 402 | )); | ||
| 403 | } | ||
| 404 | $fh->print("\n </s>"); | ||
| 405 | } | ||
| 406 | $fh->print("\n</doc>\n"); | ||
| 407 | |||
| 408 | return $fmt; | ||
| 409 | } | ||
| 410 | |||
| 411 | |||
| 412 | 1; ##-- be happy | ||
| 413 | |||
| 414 | __END__ |