| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Format::XmlLing.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: Datum parser|formatter: XML (flat att.linguistic), fast quick & dirty "flat" XML formatter using TEI att.linguistic features | ||
| 6 | |||
| 7 | package DTA::CAB::Format::XmlLing; | ||
| 8 | use DTA::CAB::Format::XmlTokWrapFast; | ||
| 9 | use DTA::CAB::Datum ':all'; | ||
| 10 | use XML::Parser; | ||
| 11 | use IO::File; | ||
| 12 | use Carp; | ||
| 13 | use strict; | ||
| 14 | |||
| 15 | ##============================================================================== | ||
| 16 | ## Globals | ||
| 17 | ##============================================================================== | ||
| 18 | |||
| 19 | our @ISA = qw(DTA::CAB::Format::XmlTokWrapFast); | ||
| 20 | |||
| 21 | BEGIN { | ||
| 22 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, filenameRegex=>qr/(?:\.(?i:(?:ling|l[tuws])(?:\.?)xml))$/); | ||
| 23 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_) foreach (qw(ltxml lxml ling-xml lt-xml ltwxml ltw-xml)); | ||
| 24 | } | ||
| 25 | |||
| 26 | BEGIN { | ||
| 27 | *isa = \&UNIVERSAL::isa; | ||
| 28 | *can = \&UNIVERSAL::can; | ||
| 29 | } | ||
| 30 | |||
| 31 | ##============================================================================== | ||
| 32 | ## Constructors etc. | ||
| 33 | ##============================================================================== | ||
| 34 | |||
| 35 | ## $fmt = CLASS_OR_OBJ->new(%args) | ||
| 36 | ## + object structure: HASH ref | ||
| 37 | ## { | ||
| 38 | ## ##-- input: new | ||
| 39 | ## doc => $doc, ##-- cached parsed DTA::CAB::Document | ||
| 40 | ## | ||
| 41 | ## ##-- input: inherited (but unused) | ||
| 42 | ## #xdoc => $xdoc, ##-- XML::LibXML::Document | ||
| 43 | ## #xprs => $xprs, ##-- override: XML::Parser parser | ||
| 44 | ## | ||
| 45 | ## ##-- output: inherited from DTA::CAB::Format | ||
| 46 | ## utf8 => $bool, ##-- always true | ||
| 47 | ## level => $level, ##-- output formatting level (default=0) | ||
| 48 | ## | ||
| 49 | ## ##-- input/output: new | ||
| 50 | ## twcompat => $bool, ##-- read/write DTA::TokWrap-style attributes? (for use as CAB::Format::TEI txmlfmt sub-formatter; default=0) | ||
| 51 | ## } | ||
| 52 | sub new { | ||
| 53 | my $that = shift; | ||
| 54 | my $fmt = $that->DTA::CAB::Format::XmlCommon::new | ||
| 55 | ( | ||
| 56 | xprs => undef, | ||
| 57 | doc => undef, | ||
| 58 | twcompat => 0, | ||
| 59 | @_ | ||
| 60 | ); | ||
| 61 | return $fmt; | ||
| 62 | } | ||
| 63 | |||
| 64 | ## $xmlparser = $fmt->xmlparser() | ||
| 65 | ## + returns cached $fmt->{xprs} if available | ||
| 66 | ## + otherwise caches & returns new XML::Parser | ||
| 67 | sub xmlparser { | ||
| 68 | return $_[0]{xprs} if (defined($_[0]{xprs})); | ||
| 69 | #my $fmt = shift; | ||
| 70 | |||
| 71 | ##-------------------------------------- | ||
| 72 | ## closure variables | ||
| 73 | my ($doc,$body, $s,$stoks, $w,$wprev,@stack,%attrs); | ||
| 74 | my ($wpos,$xpos) = (-2,0); | ||
| 75 | my $twcompat = $_[0]{twcompat}; | ||
| 76 | |||
| 77 | ##-------------------------------------- | ||
| 78 | ## parser | ||
| 79 | $_[0]{xprs} = XML::Parser->new | ||
| 80 | ( | ||
| 81 | ErrorContext => 1, | ||
| 82 | ProtocolEncoding => 'UTF-8', | ||
| 83 | #ParseParamEnt => '???', | ||
| 84 | Handlers => { | ||
| 85 | ##---------------- | ||
| 86 | ## undef = cb_init($expat) | ||
| 87 | Init => sub { | ||
| 88 | $body = []; | ||
| 89 | $doc = {body=>$body}; | ||
| 90 | @stack = qw(); | ||
| 91 | }, | ||
| 92 | |||
| 93 | ##---------------- | ||
| 94 | ## undef = cb_start($expat, $elt,%attrs) | ||
| 95 | Start => sub { | ||
| 96 | %attrs = @_[2..$#_]; | ||
| 97 | push(@stack,$_[1]); | ||
| 98 | |||
| 99 | if ($_[1] eq 'w') { | ||
| 100 | ##-- w | ||
| 101 | $attrs{id} = $attrs{'xml:id'} if (defined($attrs{'xml:id'}) && !defined($attrs{'id'})); | ||
| 102 | $attrs{text} = $attrs{t} if ($twcompat && !defined($attrs{text}) && defined($attrs{t})); | ||
| 103 | $attrs{moot}{tag} = $attrs{pos} if (defined($attrs{pos})); | ||
| 104 | $attrs{moot}{lemma} = $attrs{lemma} if (defined($attrs{lemma})); | ||
| 105 | $attrs{moot}{word} = $attrs{norm} if (defined($attrs{norm})); | ||
| 106 | delete @attrs{qw(pos lemma norm xml:id)}; | ||
| 107 | push(@$stoks, $w={%attrs}); | ||
| 108 | } elsif ($_[1] eq 's') { | ||
| 109 | ##-- s | ||
| 110 | push(@$body, $s={%attrs,tokens=>($stoks=[])}); | ||
| 111 | } elsif (@stack==1) { | ||
| 112 | ##-- doc | ||
| 113 | if (defined($attrs{'xml:base'})) { | ||
| 114 | $attrs{'base'}=$attrs{'xml:base'}; | ||
| 115 | delete($attrs{'xml:base'}); | ||
| 116 | } | ||
| 117 | $doc = {%attrs, body=>$body}; | ||
| 118 | } | ||
| 119 | }, | ||
| 120 | |||
| 121 | ##---------------- | ||
| 122 | ## undef = cb_end($expat,$elt) | ||
| 123 | End => sub { | ||
| 124 | if ($stack[$#stack] eq 'w' && !defined($w->{text}) && defined($w->{textRaw})) { | ||
| 125 | $w->{text} = $w->{textRaw}; | ||
| 126 | $w->{text} =~ s/\s+/ /sg; | ||
| 127 | $w->{text} =~ s/^\s//s; | ||
| 128 | $w->{text} =~ s/\s$//s; | ||
| 129 | delete($w->{textRaw}) if ($w->{textRaw} eq $w->{text}); | ||
| 130 | } | ||
| 131 | pop(@stack); | ||
| 132 | }, | ||
| 133 | |||
| 134 | ##---------------- | ||
| 135 | ## undef = cb_char($expat,$string) | ||
| 136 | Char => sub { | ||
| 137 | $w->{textRaw} .= $_[1] if ($stack[$#stack] eq 'w' && !defined($w->{text})); | ||
| 138 | }, | ||
| 139 | |||
| 140 | ##---------------- | ||
| 141 | ## undef = cb_default($expat, $str) | ||
| 142 | #Default => $cb_default, | ||
| 143 | |||
| 144 | ##---------------- | ||
| 145 | ## $parse_rv = cb_final($expat) | ||
| 146 | Final => sub { | ||
| 147 | $body = $s = $stoks = $w = undef; | ||
| 148 | return bless($doc,'DTA::CAB::Document'); | ||
| 149 | }, | ||
| 150 | }, | ||
| 151 | ) | ||
| 152 | or $_[0]->logconfess("couldn't create XML::Parser"); | ||
| 153 | |||
| 154 | return $_[0]{xprs}; | ||
| 155 | } | ||
| 156 | |||
| 157 | ##============================================================================== | ||
| 158 | ## Methods: Persistence | ||
| 159 | ##============================================================================== | ||
| 160 | |||
| 161 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 162 | ## + returns list of keys not to be saved | ||
| 163 | ## + inherited from XmlTokWrap | ||
| 164 | |||
| 165 | ##============================================================================= | ||
| 166 | ## Methods: I/O: generic | ||
| 167 | ##============================================================================== | ||
| 168 | |||
| 169 | ## $fmt = $fmt->close($savetmp=0) | ||
| 170 | ## + override calls $fmt->flush() and deletes @$fmt{qw(xdoc output)} | ||
| 171 | ## + inherited from XmlTokWrapFast | ||
| 172 | |||
| 173 | ## @layers = $fmt->iolayers() | ||
| 174 | ## + returns PerlIO layers to use for I/O handles | ||
| 175 | ## + override returns ':raw' | ||
| 176 | ## + inherited from XmlTokWrapFast | ||
| 177 | |||
| 178 | ##============================================================================== | ||
| 179 | ## Methods: I/O: Block-wise | ||
| 180 | ##============================================================================== | ||
| 181 | |||
| 182 | ##-------------------------------------------------------------- | ||
| 183 | ## Methods: I/O: Block-wise: Generic | ||
| 184 | |||
| 185 | ## %blockOpts = $CLASS_OR_OBJECT->blockDefaults() | ||
| 186 | ## + returns default block options as for blockOptions() | ||
| 187 | ## + override returns as for $CLASS_OR_OBJECT->blockOptions('2m@s') | ||
| 188 | ## + inherited from XmlTokWrapFast | ||
| 189 | |||
| 190 | ##============================================================================= | ||
| 191 | ## Methods: Input | ||
| 192 | ##============================================================================== | ||
| 193 | |||
| 194 | ##-------------------------------------------------------------- | ||
| 195 | ## Methods: Input: Input selection | ||
| 196 | |||
| 197 | ## $fmt = $fmt->fromString(\$string) | ||
| 198 | ## + input from string | ||
| 199 | ## + inherited from XmlTokWrapFast | ||
| 200 | |||
| 201 | ## $fmt = $fmt->fromFile($filename) | ||
| 202 | ## + input from named file: override buffers XML document in $fmt->{xdoc} | ||
| 203 | ## + inherited from XmlTokWrapFast | ||
| 204 | |||
| 205 | ## $fmt = $fmt->fromFh($handle) | ||
| 206 | ## + input from filehandle: override buffers XML document in $fmt->{xdoc} | ||
| 207 | ## + inherited from XmlTokWrapFast | ||
| 208 | |||
| 209 | ##-------------------------------------------------------------- | ||
| 210 | ## Methods: Input: Generic API | ||
| 211 | |||
| 212 | ## $doc = $fmt->parseDocument() | ||
| 213 | ## + parse document from currently selected input source | ||
| 214 | ## + override returns buffered $fmt->{doc} | ||
| 215 | ## + inherited from XmlTokWrapFast | ||
| 216 | |||
| 217 | ##============================================================================= | ||
| 218 | ## Methods: Output | ||
| 219 | ##============================================================================== | ||
| 220 | |||
| 221 | ##-------------------------------------------------------------- | ||
| 222 | ## Methods: Output: MIME & HTTP stuff | ||
| 223 | |||
| 224 | ## $short = $fmt->shortName() | ||
| 225 | ## + returns "official" short name for this format | ||
| 226 | ## + default just returns package suffix | ||
| 227 | sub shortName { | ||
| 228 | return 'ltxml'; | ||
| 229 | } | ||
| 230 | |||
| 231 | ## $ext = $fmt->defaultExtension() | ||
| 232 | ## + returns default filename extension for this format (default='.ling.xml') | ||
| 233 | sub defaultExtension { return '.lt.xml'; } | ||
| 234 | |||
| 235 | ##-------------------------------------------------------------- | ||
| 236 | ## Methods: Output: output selection | ||
| 237 | |||
| 238 | ## $fmt = $fmt->flush() | ||
| 239 | ## + flush accumulated output | ||
| 240 | ## + inherited from XmlTokWrapFast | ||
| 241 | |||
| 242 | ## $str = $fmt->toString() | ||
| 243 | ## $str = $fmt->toString($formatLevel) | ||
| 244 | ## + flush buffered output document to byte-string | ||
| 245 | ## + inherited from XmlTokWrapFast | ||
| 246 | |||
| 247 | ## $fmt_or_undef = $fmt->toFile($filename_or_handle, $formatLevel) | ||
| 248 | ## + flush buffered output document to $filename_or_handle | ||
| 249 | ## + default implementation calls $fmt->toFh() | ||
| 250 | ## + inherited from XmlTokWrapFast | ||
| 251 | |||
| 252 | ## $fmt_or_undef = $fmt->toFh($fh,$formatLevel) | ||
| 253 | ## + flush buffered output document to filehandle $fh | ||
| 254 | ## + inherited from XmlTokWrapFast | ||
| 255 | |||
| 256 | ##-------------------------------------------------------------- | ||
| 257 | ## Methods: Output: quick and dirty | ||
| 258 | |||
| 259 | ## $fmt = $fmt->putDocument($doc) | ||
| 260 | sub putDocument { | ||
| 261 | my ($fmt,$doc) = @_; | ||
| 262 | |||
| 263 | ##-------------------- | ||
| 264 | ## local subs | ||
| 265 | |||
| 266 | my $nil = []; | ||
| 267 | |||
| 268 | ##-- $escaped = xmlesc($str) : xml escape (single string) | ||
| 269 | my ($_esc); | ||
| 270 | my $xmlescape = sub { | ||
| 271 | $_esc=shift; | ||
| 272 | $_esc=~s/([\&\'\"\<\>])/'&#'.ord($1).';'/ge; | ||
| 273 | return $_esc; | ||
| 274 | }; | ||
| 275 | |||
| 276 | ##-- $str = xmlattrs(%attrs) | ||
| 277 | my $xmlattrs = sub { | ||
| 278 | return join('', map {' '.$_[$_].'="' . $xmlescape->($_[$_+1]).'"'} grep {defined($_[$_+1])} map {$_*2} (0..$#{_}/2)); | ||
| 279 | }; | ||
| 280 | |||
| 281 | ##-- $str = xmlstart($name,%attrs) | ||
| 282 | my $xmlstart = sub { | ||
| 283 | return "<$_[0]" . $xmlattrs->(@_[1..$#_]) . ">"; | ||
| 284 | }; | ||
| 285 | |||
| 286 | ##-- $str = xmlempty($name,%attrs) | ||
| 287 | my $xmlempty = sub { | ||
| 288 | return "<$_[0]" . $xmlattrs->(@_[1..$#_]) . "/>"; | ||
| 289 | }; | ||
| 290 | |||
| 291 | ##-- $str = xmlelt($name,\@attrs,@content_strings) | ||
| 292 | my $xmlelt = sub { | ||
| 293 | return $xmlempty->($_[0], @{$_[1]||$nil}) if (@_ < 3); | ||
| 294 | return $xmlstart->($_[0], @{$_[1]||$nil}) . join('',@_[2..$#_]) . "</$_[0]>"; | ||
| 295 | }; | ||
| 296 | |||
| 297 | ##-- ($beg,$end) = twloc($w) | ||
| 298 | my (@loc); | ||
| 299 | my $twloc = sub { | ||
| 300 | return undef if (!$_[0] || !defined($_[0]{b})); | ||
| 301 | @loc = split(' ',$_[0]{b},2); | ||
| 302 | return [$loc[0],$loc[0]+$loc[1]]; | ||
| 303 | }; | ||
| 304 | |||
| 305 | ##-------------------- | ||
| 306 | ## output handle | ||
| 307 | my $fh = $fmt->{fh}; | ||
| 308 | binmode($fh,':utf8'); | ||
| 309 | $fh->print("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); | ||
| 310 | |||
| 311 | ##-------------------- | ||
| 312 | ## compatibility | ||
| 313 | my $twcompat = $fmt->{twcompat}; | ||
| 314 | my @twattrs = ($twcompat ? qw(b xb) : qw()); | ||
| 315 | |||
| 316 | ##-------------------- | ||
| 317 | ## guts | ||
| 318 | my ($s,$wi,$w, $wjoin,$loc,$loc_prev,$loc_next); | ||
| 319 | $fh->print("\n", $xmlstart->('text')); | ||
| 320 | foreach $s (@{$doc->{body}}) { | ||
| 321 | $fh->print("\n ", $xmlstart->('s','xml:id'=>($s->{id}||$s->{'xml:id'}))); | ||
| 322 | |||
| 323 | $loc_prev = undef; | ||
| 324 | $loc = $twloc->($s->{tokens}[0]); | ||
| 325 | |||
| 326 | for ($wi=0; $wi <= $#{$s->{tokens}}; ++$wi) { | ||
| 327 | $w = $s->{tokens}[$wi]; | ||
| 328 | |||
| 329 | ##-- compute att.linguistic 'join' attribute from TokWrap 'b' attribute | ||
| 330 | if (!defined($wjoin=$w->{join}) && $loc) { | ||
| 331 | $loc_next = $twloc->($s->{tokens}[$wi+1]); | ||
| 332 | $wjoin = ($loc_prev && $loc->[0] == $loc_prev->[1] | ||
| 333 | ? ($loc_next && $loc->[1] == $loc_next->[0] | ||
| 334 | ? 'both' | ||
| 335 | : 'left') | ||
| 336 | : ($loc_next && $loc->[1] == $loc_next->[0] | ||
| 337 | ? 'right' | ||
| 338 | : undef)); | ||
| 339 | $loc_prev = $loc; | ||
| 340 | $loc = $loc_next; | ||
| 341 | } else { | ||
| 342 | undef $wjoin; | ||
| 343 | } | ||
| 344 | |||
| 345 | $fh->print("\n\t", | ||
| 346 | $xmlelt->('w', | ||
| 347 | [ | ||
| 348 | ##-- word attributes: tokwrap (for use as CAB::Format::TEI txmlfmt sub-formatter) | ||
| 349 | (map {($_=>$w->{$_})} @twattrs), | ||
| 350 | ## | ||
| 351 | ##-- word attributes: id | ||
| 352 | ('xml:id'=>($w->{id}||$w->{'xml:id'})), | ||
| 353 | ## | ||
| 354 | ##-- word attributes: att.linguistic | ||
| 355 | ($w->{moot} | ||
| 356 | ? ('lemma'=>$w->{moot}{lemma}, | ||
| 357 | 'pos'=>$w->{moot}{tag}, | ||
| 358 | 'norm'=>$w->{moot}{word}) | ||
| 359 | : ($w->{xlit} | ||
| 360 | ? ('norm'=>$w->{xlit}{latin1Text}) | ||
| 361 | : qw())), | ||
| 362 | ('join'=>(defined($w->{join}) ? $w->{join} : $wjoin)), | ||
| 363 | ], | ||
| 364 | ## | ||
| 365 | ($twcompat ? qw() : (defined($w->{textRaw}) ? $w->{textRaw} : $xmlescape->($w->{text}))), | ||
| 366 | )); | ||
| 367 | } | ||
| 368 | $fh->print("\n </s>"); | ||
| 369 | } | ||
| 370 | $fh->print("\n</text>\n"); | ||
| 371 | |||
| 372 | return $fmt; | ||
| 373 | } | ||
| 374 | |||
| 375 | |||
| 376 | 1; ##-- be happy | ||
| 377 | |||
| 378 | __END__ |