blib/lib/Uplug/IO/XCESalign.pm untested

filter: Statementstmt 0.0% 0 / 352 Branchbran 0.0% 0 / 156 Conditioncond 0.0% 0 / 42 MC/DCmcdc 0.0% 0 / 26 Subroutinesub 0.0% 0 / 17 Podpod 0.0% 0 / 11 totaltotal 0.0% 0 / 604 CC 79 SCAR 46.6
CC
79
Cov
0%
open
182.0
select
156.0
FindAlignDocFile
132.0
CRAP 105.1
1 #-*-perl-*-
2 #####################################################################
3 # Copyright (C) 2004 Jörg Tiedemann <joerg@stp.ling.uu.se>
4 #
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18  
19 #####################################################################
20 #
21 # XCESalign
22 #
23 #####################################################################
24 # $Author$
25 # $Id$
26  
27  
28 package Uplug::IO::XCESalign;
29  
30 use strict;
31 use vars qw(@ISA);
32 use vars qw(%StreamOptions);
33 use Uplug::IO::XML;
34 use Uplug::IO::Any;
35 use Uplug::Data;
36 # use Uplug::Data::DOM;
37  
38 @ISA = qw( Uplug::IO::XML );
39  
40 # stream options and their default values!!
41  
42 %StreamOptions = ('DocRootTag' => 'cesAlign',
43 'root' => 'link',
44 'DataStructure' => 'complex',
45 'SkipDataHeader' => 1,
46 'SkipDataTail' => 1,
47 'DTDname' => 'cesAlign',
48 'DTDsystemID' => 'dtd/xcesAlign.dtd',
49 'DTDpublicID' => '-//CES//DTD XML cesAlign//EN'
50 );
51 $StreamOptions{DocRoot}{version}='1.0';
52  
53 sub new{
54 my $class=shift;
55 my $self=$class->SUPER::new($class);
56 foreach (keys %StreamOptions){
57 $self->setOption($_,$StreamOptions{$_});
58 }
59 # &Uplug::IO::AddHash2Hash($self->{StreamOptions},\%StreamOptions);
60 return $self;
61 }
62  
63  
64 sub open{
65 my $self = shift;
66 if ($self->SUPER::open(@_)){
67  
68 if ($self->{AccessMode} eq 'read'){ # if access-mode=read:
69 if (not $self->option('DocBodyTag')){ # set doc-body-tag
70 $self->setOption('DocBodyTag','(linkGrp|linkAlign)');
71 }
72 }
73 if (defined $self->{StreamOptions}->{fromDoc}){
74 $self->OpenAlignDocs($self->{StreamOptions});
75 }
76 elsif (defined $self->{StreamHeader}->{fromDoc}){
77 $self->OpenAlignDocs($self->{StreamHeader});
78 $self->{StreamOptions}->{fromDoc}=$self->{StreamHeader}->{fromDoc};
79 $self->{StreamOptions}->{toDoc}=$self->{StreamHeader}->{toDoc};
80 }
81 elsif (defined $self->{StreamOptions}->{DocRoot}->{fromDoc}){
82 $self->OpenAlignDocs($self->{StreamOptions}->{DocRoot});
83 $self->{StreamOptions}->{fromDoc}=
84 $self->{StreamOptions}->{DocRoot}->{fromDoc};
85 $self->{StreamOptions}->{toDoc}=
86 $self->{StreamOptions}->{DocRoot}->{toDoc};
87 }
88 elsif (defined $self->{StreamHeader}->{DocRoot}->{fromDoc}){
89 $self->OpenAlignDocs($self->{StreamHeader}->{DocRoot});
90 $self->{StreamOptions}->{fromDoc}=
91 $self->{StreamHeader}->{DocRoot}->{fromDoc};
92 $self->{StreamOptions}->{toDoc}=
93 $self->{StreamHeader}->{DocRoot}->{toDoc};
94 }
95  
96 if ($self->{AccessMode} ne 'read'){
97 if (($self->{fromDoc} ne 'exists') and (not ref($self->{source}))){
98 $self->{StreamOptions}->{fromDoc}=
99 $self->{StreamOptions}->{file}.'.src';
100 }
101 if (($self->{toDoc} ne 'exists') and (not ref($self->{target}))){
102 $self->{StreamOptions}->{toDoc}=
103 $self->{StreamOptions}->{file}.'.trg';
104 }
105 $self->OpenAlignDocs($self->{StreamOptions});
106  
107 # write linkGrp-tag only of we skip data headers!
108 # otherwise: linkGrp is expected to be in the data-headers
109 # of data written to cesAlign files!!
110  
111 if ($self->{StreamOptions}->{SkipDataHeader}){
112 $self->OpenTag('linkGrp',
113 {'targType' => 's',
114 'fromDoc' => $self->{StreamOptions}->{fromDoc},
115 'toDoc' => $self->{StreamOptions}->{toDoc}});
116 }
117 }
118  
119 if (ref($self->{XmlParser})){
120 $self->{XmlHandle}->{REMOVESPACES}=$self->option('REMOVESPACES');
121 $self->{XmlHandle}->{SubTreeRoot}=$self->option('root');
122 $self->{XmlHandle}->{DocRootTag}=$self->option('DocRootTag');
123 $self->{XmlHandle}->{DocBodyTag}=$self->option('DocBodyTag');
124 $self->CompileTagREs;
125 }
126  
127 return 1;
128 }
129 return 0;
130  
131 }
132  
133 sub close{
134 my $self=shift;
135 if (ref($self->{source})){
136 $self->{source}->close();
137 }
138 if (ref($self->{target})){
139 $self->{target}->close();
140 }
141 return $self->SUPER::close();
142 }
143  
144  
145 sub read{
146 my $self=shift;
147 my $data=shift;
148  
149 $data->init;
150 if (not ref($data->{link})){$data->{link}=Uplug::Data->new;}
151 else{$data->{link}->init;}
152  
153 my $ret=$self->SUPER::read($data->{link},@_);
154 if ($self->NewDocBody){
155 my $BodyAttr=$self->DocBodyAttr;
156 $self->addheader($BodyAttr);
157 if (not $self->OpenAlignDocs($BodyAttr)){
158 return 0;
159 }
160 delete $data->{sourceSent}; # delete previous data-objects
161 delete $data->{targetSent}; # (to get new XML::Parser instances)
162 }
163  
164 my $attr=$data->{link}->attribute();
165 # $data->addChild('align',undef,$attr);
166 $data->addNode('align',$attr);
167 my ($src,$trg);
168 my $xtargets=$data->{link}->attribute('xtargets');
169 if (defined $xtargets){
170 ($src,$trg)=split(/\s*\;\s*/,$xtargets);
171 }
172 else{return 0;}
173  
174 my @srcID=split(/\s/,$src);
175 my @trgID=split(/\s/,$trg);
176  
177 #----------- sentences ------------------------------
178  
179 &ReadSegments($self->{source},$data,\@srcID,'source');
180 &ReadSegments($self->{target},$data,\@trgID,'target');
181  
182 $self->Uplug::IO::read($data);
183  
184 # $data->{source}=$data->subData('source');
185 # $data->{target}=$data->subData('target');
186  
187 # $data->subTree($data->{source},'source');
188 # $data->subTree($data->{target},'target');
189  
190 return $ret;
191  
192 }
193  
194  
195  
196  
197 sub write{
198 my $self=shift;
199 my ($data)=@_;
200  
201 $self->Uplug::IO::write($data);
202  
203 my $ret;
204 my $source=$data->subData('source');
205 my $target=$data->subData('target');
206  
207 # my $source=Uplug::Data::DOM->new;
208 # my $source=Uplug::Data->new;
209 # $data->subTree($source,'source');
210 # my $target=Uplug::Data->new;
211 # my $target=Uplug::Data::DOM->new;
212 # $data->subTree($target,'target');
213  
214  
215 if (ref($data->{link})){
216 $ret=$self->SUPER::write($data->{link});
217 }
218 elsif (defined $data->attribute('xtargets')){
219 $ret=$self->SUPER::write($data);
220 }
221 else{
222 my $tag=$self->option('root');
223 my $link=Uplug::Data->new($tag);
224 my @src=$source->findNodes('s');
225 my @srcID=$source->attribute(\@src,'id');
226 foreach (0..$#src){
227 if (not defined $srcID[$_]){
228 $self->{srcID}++;
229 $srcID[$_]=$self->{srcID};
230 $source->setAttribute($src[$_],'id',$srcID[$_]);
231 }
232 $self->{srcID}=$srcID[$_];
233 }
234 my @trg=$target->findNodes('s');
235 my @trgID=$target->attribute(\@trg,'id');
236 foreach (0..$#trg){
237 if (not defined $trgID[$_]){
238 $self->{trgID}++;
239 $trgID[$_]=$self->{trgID};
240 $target->setAttribute($trg[$_],'id',$trgID[$_]);
241 }
242 $self->{trgID}=$trgID[$_];
243 }
244 my $s=join(' ',@srcID);
245 my $t=join(' ',@trgID);
246 $link->setAttribute('xtargets',"$s\;$t",$tag);
247  
248 my $id=$data->attribute('id');
249 if (not defined $id){
250 $self->{alignID}++;
251 $id=$self->{alignID};
252 }
253 $link->setAttribute('id',$id);
254 $ret=$self->SUPER::write($link);
255 }
256 if (($self->{fromDoc} ne 'exists') and (ref($self->{source}))){
257 # $source->moveAttribute('s','seg');
258 $self->{source}->write($source);
259 }
260 if (($self->{toDoc} ne 'exists') and (ref($self->{target}))){
261 # $target->moveAttribute('s','seg');
262 $self->{target}->write($target);
263 }
264 return $ret;
265 }
266  
267  
268  
269 #-----------------------------------------------------------------
270 # select: select data
271 #
272 # * read sequentially through the link-file and
273 # search bitext segments according to the matching pattern
274 #
275 # we cannot use select for searching in the link file because
276 # we need fromDoc and toDoc from the alignment tags .....!!!!
277  
278  
279 sub select{
280 my $self=shift;
281 my ($data,$pattern,$attr,$operator)=@_;
282  
283 $data->init;
284 if (not ref($data->{link})){$data->{link}=Uplug::Data->new;}
285 else{$data->{link}->init;}
286  
287 my %linkPattern=();
288 my %srcPattern=();
289 my %trgPattern=();
290 if (ref($pattern) eq 'HASH'){
291 %linkPattern=%{$pattern};
292 if (ref($pattern->{source}) eq 'HASH'){
293 %srcPattern=%{$pattern->{source}};
294 delete $linkPattern{source};
295 }
296 elsif (defined $pattern->{source}){
297 $srcPattern{'#text'}=$pattern->{source};
298 delete $linkPattern{source};
299 }
300 if (ref($pattern->{target}) eq 'HASH'){
301 %trgPattern=%{$pattern->{target}};
302 delete $linkPattern{target};
303 }
304 elsif (defined $pattern->{target}){
305 $trgPattern{'#text'}=$pattern->{target};
306 delete $linkPattern{target};
307 }
308 }
309  
310 if ($self->{ENDOFSTREAM}){
311 delete $self->{ENDOFSTREAM};
312 $self->reopen();
313 }
314  
315 while ($self->SUPER::read($data->{link})){
316  
317 print '';
318 if ($self->NewDocBody){
319 my $BodyAttr=$self->DocBodyAttr;
320 $self->addheader($BodyAttr);
321 if (not $self->OpenAlignDocs($BodyAttr)){
322 return 0;
323 }
324 delete $data->{sourceSent}; # delete previous data-objects
325 delete $data->{targetSent}; # (to get new XML::Parser instances)
326 }
327  
328 if (not $data->{link}->matchData(\%linkPattern,$operator)){next;}
329  
330  
331 my $attr=$data->{link}->attribute();
332 $data->addNode('align',$attr);
333 my ($src,$trg);
334 my $xtargets=$data->{link}->attribute('xtargets');
335  
336 if (defined $xtargets){
337 ($src,$trg)=split(/\s*\;\s*/,$xtargets);
338 }
339 else{return 0;}
340  
341 my @srcID=split(/\s/,$src);
342 my @trgID=split(/\s/,$trg);
343  
344 #----------- sentences ------------------------------
345  
346 # print STDERR "read src sentences ",join(":",@srcID),"\n";
347 # print STDERR "read trg sentences ",join(":",@trgID),"\n";
348  
349 &SearchSegments($self->{source},$data,\@srcID,'source');
350 &SearchSegments($self->{target},$data,\@trgID,'target');
351  
352 # &ReadSegments($self->{source},$data,\@srcID,'source');
353 # &ReadSegments($self->{target},$data,\@trgID,'target');
354  
355 $self->Uplug::IO::read($data);
356 return 1;
357 }
358  
359 $self->{ENDOFSTREAM}=1;
360 return 0;
361 }
362  
363  
364 # end of select
365 #-----------------------------------------------------------------
366  
367  
368  
369  
370 sub FindAlignDocFile{
371 my $self=shift;
372 my $doc=shift;
373 if ((-s $doc) or (-s "$doc.gz")){return $doc;} # found it --> ok!
374  
375 my $file=$self->files(); # no?
376 my $dir='./'; # check relative to the
377 if ($file=~/^(.*[\\\/])[^\\\/]+$/){$dir=$1;} # align-document
378 if ((-s $dir.$doc) or (-s "$dir$doc.gz")){ # (align-dir/filename)
379 # print STDERR "$doc --> $dir$doc\n";
380 if ($doc=~/^([\\\/\.]*[^\\\/\.]+)[\\\/]/){ # make symbolic links!
381 eval { symlink ($dir.$1,$1);1 }; # ... relative dir
382 } # ... or
383 else{eval { symlink ($dir.$doc,$doc);1 };} # ... file link
384 return $dir.$doc; # return path+file
385 }
386 my $docfile=$doc; # no?
387 if ($doc=~/^.*[\\\/]([^\\\/]+)$/){$docfile=$1;} # remove path in the
388 if ((-s $dir.$docfile) or (-s "$dir$docfile.gz")){ # filename and check
389 # print STDERR "$doc --> $dir$docfile\n";
390 if ($doc=~/^([\\\/\.]*[^\\\/\.]+)[\\\/]/){
391 eval { symlink ($dir.$1,$1);1 };
392 }
393 else{eval { symlink ($dir.$docfile,$docfile);1 };}
394 return $dir.$docfile; # found it --> return!
395 } # no? --> remove
396 if ($doc=~/^[^\\\/]+[\\\/]([^\\\/].*)$/){$docfile=$1;} # initial directory
397 if ((-s $dir.$docfile) or (-s "$dir$docfile.gz")){ # and check again!
398 # print STDERR "$doc --> $dir$docfile\n";
399 if ($doc=~/^([\\\/\.]*[^\\\/\.]+)[\\\/]/){
400 eval { symlink ($dir,$1);1 };
401 # print STDERR "symlink ($dir,$1)\n";
402 }
403 else{eval { symlink ($dir.$docfile,$docfile);1 };}
404 # print STDERR "symlink ($dir.$docfile,$docfile)\n";
405 return $dir.$docfile;
406 }
407 return $doc;
408 }
409  
410  
411  
412 sub OpenAlignDocs{
413 my $self=shift;
414 my $options=shift;
415  
416  
417 if (ref($options) ne 'HASH'){return 0;}
418 if (($self->{AccessMode} ne 'read') and
419 ((-s $options->{fromDoc}) or (-s "$options->{fromDoc}.gz") or
420 ($self->{StreamOptions}->{SkipSrcFile}))){
421 $self->{fromDoc}='exists';
422 }
423 else{
424 $options->{fromDoc}=$self->FindAlignDocFile($options->{fromDoc});
425 my %stream=('file' => $options->{fromDoc},
426 'format' => 'XML',
427 'root' => 's');
428  
429 ## make subtree index files (DBM hash) for the source file
430 ##
431 # if ($self->option('MAKESUBTREEINDEX')){ # commented out -> always!
432 $stream{MAKESUBTREEINDEX} = 1;
433 # }
434  
435  
436 if ($self->{AccessMode} ne 'read'){$stream{DocRootTag}='document';}
437 $self->{source}=Uplug::IO::Any->new(\%stream);
438 if (not $self->{source}->open($self->{AccessMode},\%stream)){
439 return 0;
440 }
441 }
442 if (($self->{AccessMode} ne 'read') and
443 ((-s $options->{toDoc}) or (-s "$options->{toDoc}.gz") or
444 ($self->{StreamOptions}->{SkipTrgFile}))){
445 $self->{toDoc}='exists';
446 }
447 else{
448 $options->{toDoc}=$self->FindAlignDocFile($options->{toDoc});
449 my %stream=('file' => $options->{toDoc},
450 'format' => 'XML',
451 'root' => 's');
452  
453 ## make subtree index files (DBM hash) for the source file
454 ##
455 # if ($self->option('MAKESUBTREEINDEX')){ # commented out -> always!
456 $stream{MAKESUBTREEINDEX} = 1;
457 # }
458  
459 if ($self->{AccessMode} ne 'read'){$stream{DocRootTag}='document';}
460 $self->{target}=Uplug::IO::Any->new(\%stream);
461 if (not $self->{target}->open($self->{AccessMode},\%stream)){
462 return 0;
463 }
464 }
465 return 1;
466 }
467  
468  
469  
470 #-------------------------------------------------------------------------
471 # ReadSegments: read segments from the aligned XML-files (sentences)
472 #
473 # * read sequentially through the data and add sentences with the
474 # requested IDs
475 # * if the ID's don't match --> use the select function for IO::XML
476 # (this is probably also just a sequential read -> might be a problem!)
477  
478 sub ReadSegments{
479  
480 my ($stream,$data,$IDs,$lang)=@_;
481  
482 if (not ref($stream)){return;}
483 if (not @{$IDs}){return;}
484  
485 #--------------------------------------------------------------
486 if (not ref($data->{$lang})){
487 $data->{$lang}=Uplug::Data::Lang->new(); # a new language object
488 }
489 if (ref($data->{$lang.'Sent'}) ne 'ARRAY'){ # the array of data objects
490 $data->{$lang.'Sent'}=[]; # (one for each sentence)
491 }
492 #--------------------------------------------------------------
493  
494 my $parent=$data->addNode($lang); # set root node = $lang
495 $data->{$lang}->setRoot($parent); # set root node of sub-lang-data
496  
497 my $count = 0; # make a new data object
498 if (not ref($data->{$lang.'Sent'}->[$count])){ # for reading the sentence
499 $data->{$lang.'Sent'}->[$count]=
500 Uplug::Data->new();
501 }
502 my $sent = $data->{$lang.'Sent'}->[$count]; # $sents points to it!
503  
504 while ($stream->read($sent)){ # read sequentially through the data
505 my $id=$sent->attribute('id'); # get the sentence ID
506 if (not grep ($_ eq $id,@{$IDs})){ # if s-ID is not in the requested:
507 foreach (0..$#{$IDs}){ # use the select-function
508  
509 if (not ref($data->{$lang.'Sent'}->[$_])){
510 $data->{$lang.'Sent'}->[$_]=Uplug::Data->new();
511 }
512 my $sent = $data->{$lang.'Sent'}->[$_];
513 if ($stream->select($sent,{id => $IDs->[$_]})){
514 my $node=$sent->root();
515 $data->addNode($parent,$node);
516 }
517 }
518 return;
519 }
520 my $node=$sent->root(); # otherwise: add the sentence
521 $data->addNode($parent,$node); # and continue to read if necessary
522 if ($id eq $IDs->[-1]){last;}
523  
524 $count ++; # still more sentences!
525 if (not ref($data->{$lang.'Sent'}->[$count])){ # make a new data-
526 $data->{$lang.'Sent'}->[$count]= # object if necessary
527 Uplug::Data->new();
528 }
529 $sent = $data->{$lang.'Sent'}->[$count]; # let $sent point to it
530 }
531 }
532  
533  
534  
535 #-------------------------------------------------------------------------
536 # SearchSegments: search sentences in the aligned XML files
537 #
538 # * similar to ReadSegments but uses the select function in IO::XML
539 # as standard (does not call data->read at all!)
540 # * 'select' is just reading sequentially through the data at the moment
541 # and could actually be used even for ReadSegments (??!)
542  
543  
544 sub SearchSegments{
545  
546 my ($stream,$data,$IDs,$lang)=@_;
547  
548 if (not ref($stream)){return;}
549 if (not @{$IDs}){return;}
550  
551 #--------------------------------------------------------------
552 if (not ref($data->{$lang})){
553 $data->{$lang}=Uplug::Data::Lang->new(); # a new language object
554 }
555 if (ref($data->{$lang.'Sent'}) ne 'ARRAY'){ # the array of data objects
556 $data->{$lang.'Sent'}=[]; # (one for each sentence)
557 }
558 #--------------------------------------------------------------
559  
560 my $parent=$data->addNode($lang); # set root node = $lang
561 $data->{$lang}->setRoot($parent); # set root node of sub-lang-data
562  
563 foreach (0..$#{$IDs}){
564 if (not ref($data->{$lang.'Sent'}->[$_])){
565 $data->{$lang.'Sent'}->[$_]=Uplug::Data->new();
566 }
567 my $sent = $data->{$lang.'Sent'}->[$_];
568 if ($stream->select($sent,{id => $IDs->[$_]})){
569 my $node=$sent->root();
570 $data->addNode($parent,$node);
571 }
572 }
573 }
574  
575  
576  
577 #-------------------------------------------------------------------------
578 # ReadSegments: old version of ReadSegments
579 #
580 # * reads sentences only if the IDs match OR the current ID is
581 # LOWER than the FIRST sentence ID in the list of requested onces
582 # * advantage: does not use the select function from IO::XML which is
583 # right just a sequential search through the file! this may cause the
584 # program to read through the whole file without finding anything, and
585 # this is slow
586 # (this problem appears if there is an requested ID which is LOWER than
587 # the one at the current file position, or if the requested ID does not
588 # exist in the file)
589 # * use this one instead of the one above by removing the 'Old' in the
590 # sub-name
591  
592  
593 sub ReadSegmentsOld{
594  
595 my ($stream,$data,$IDs,$lang)=@_;
596  
597 if (not ref($stream)){return;}
598 if (not @{$IDs}){return;}
599 my $i=0;
600  
601 #--------------------------------------------------------------
602 if (not ref($data->{$lang})){
603 $data->{$lang}=Uplug::Data::Lang->new(); # a new language object
604 }
605 if (not ref($data->{$lang.'Sent'})){
606 $data->{$lang.'Sent'}=Uplug::Data->new(); # a new object for reading
607 }
608 #--------------------------------------------------------------
609  
610 my $parent=$data->addNode($lang); # set root node = $lang
611 $data->{$lang}->setRoot($parent); # set root node of sub-lang-data
612 my $sent=$data->{$lang.'Sent'}; # sentences will be read into $sent
613  
614 my @start=split(/\./,$IDs->[0]); # split start ID (can be like 's3.2.5.2')
615 map(s/[^0-9]//,@start); # delete non-digits
616  
617 my $fail=0;
618 while ($stream->read($sent)){ # read sequentially through the data
619 my $id=$sent->attribute('id'); # get the sentence ID
620 if (not grep ($_ eq $id,@{$IDs})){ # if s-ID is not in the requested
621  
622 my @nr=split(/\./,$id); # split into ID-levels
623 map(s/[^0-9]//,@nr); # delete non-digits
624  
625 foreach my $l (0..$#start){ # compare each ID level
626 if ($nr[$l]<$start[$l]){ # if the current ID is lower:
627 last; # OK! continue reading! (we don't
628 } # have to check deeper levels!)
629 if ($nr[$l]>$start[$l]){ # if larger than the start-ID
630 $fail++;last; # --> fail and stop
631 }
632 }
633 if ($fail>1){last;} # allow to fail once (why ?!?!)
634 # $stream->close; # we could also re-open: takes too
635 # $stream->open('read'); # much time! ... ignore it for now!
636 next;
637 }
638 my $node=$sent->root();
639 $data->addNode($parent,$node);
640 if ($id eq $IDs->[-1]){last;}
641 $i++;
642 }
643 }
644  
645  
646  
647