blib/lib/Uplug/Data/Align.pm untested

filter: Statementstmt 0.0% 0 / 412 Branchbran 0.0% 0 / 152 Conditioncond 0.0% 0 / 27 MC/DCmcdc 0.0% 0 / 18 Subroutinesub 0.0% 0 / 45 Podpod 0.0% 0 / 41 totaltotal 0.0% 0 / 695 CC 77 SCAR 37.2
CC
77
Cov
0%
checkPairParameter
132.0
getBitextPhrases
90.0
getAlignPhrases
72.0
CRAP 41.1
1 ####################################################################
2 # Copyright (C) 2004 Jörg Tiedemann
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software
16 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 #
18 # $Author$
19 # $Id$
20 #
21 ###########################################################################
22 # Uplug::Data::Align
23 #
24 #
25 #
26 ###########################################################################
27  
28  
29 package Uplug::Data::Align;
30  
31 use strict;
32 use vars qw( @ISA );
33  
34 use Uplug::Data;
35 use Uplug::Data::Lang;
36  
37 @ISA = qw( Uplug::Data );
38  
39 sub init{
40 my $self=shift;
41 my $srclang=shift;
42 my $trglang=shift;
43  
44 if (not defined $self->{source}){
45 $self->{source}=Uplug::Data::Lang->new($srclang);
46 $self->setOption('SRCLANG',$srclang);
47 }
48 elsif ((defined $srclang) and ($self->{SRCLANG} ne $srclang)){
49 $self->{source}=Uplug::Data::Lang->init($srclang);
50 $self->setOption('SRCLANG',$srclang);
51 }
52 else{$self->{source}->init();}
53 if (not defined $self->{target}){
54 $self->{target}=Uplug::Data::Lang->new($trglang);
55 $self->setOption('TRGLANG',$trglang);
56 }
57 elsif ((defined $trglang) and ($self->{TRGLANG} ne $trglang)){
58 $self->{target}=Uplug::Data::Lang->new($trglang);
59 $self->setOption('TRGLANG',$trglang);
60 }
61 else{$self->{target}->init();}
62 # $self->{SRCSUBNEW}=1; # flag for creating new source sub-trees
63 # $self->{TRGSUBNEW}=1; # flag for creating new target sub-trees
64 return $self->SUPER::init(@_);
65 }
66  
67 sub clone{return Uplug::Data::Align->new();}
68  
69  
70 sub makeLangSubData{
71 my $self=shift;
72 $self->subData($self->{'source'},'source');
73 $self->subData($self->{'target'},'target');
74 }
75  
76 sub sourceData{return $_[0]->{source};}
77 sub targetData{return $_[0]->{target};}
78 sub linkData{return $_[0]->{link};}
79  
80  
81 sub getTokens{
82 my $self=shift;
83 my $lang=shift;
84 my $param=shift;
85 if (not defined $lang){$lang='source'};
86 if (not ref($self->{$lang})){return undef;}
87 # $self->subData($self->{$lang},$lang);
88 # $self->{$lang}=$self->subData($lang);
89 $self->makeParameter($param,$lang);
90 return $self->{$lang}->getTokens($param,@_);
91 }
92  
93 sub getSrcTokens{my $self=shift;return $self->getTokens('source',@_);}
94 sub getTrgTokens{my $self=shift;return $self->getTokens('target',@_);}
95  
96 sub getNgrams{
97 my $self=shift;
98 my $lang=shift;
99 my $param=shift;
100 if (not defined $lang){$lang='source'};
101 if (not ref($self->{$lang})){return undef;}
102 # $self->subData($self->{$lang},$lang);
103 # $self->{$lang}=$self->subData($lang);
104 $self->makeParameter($param,$lang);
105 return $self->{$lang}->getNgrams($param,@_);
106 }
107  
108 sub getSrcNgrams{my $self=shift;return $self->getNgrams('source',@_);}
109 sub getTrgNgrams{my $self=shift;return $self->getNgrams('target',@_);}
110  
111 sub getChunks{
112 my $self=shift;
113 my $lang=shift;
114 my $param=shift;
115 if (not defined $lang){$lang='source'};
116 if (not ref($self->{$lang})){return undef;}
117 # $self->subData($self->{$lang},$lang);
118 # $self->{$lang}=$self->subData($lang);
119 $self->makeParameter($param,$lang);
120 return $self->{$lang}->getChunks($param,@_);
121 }
122  
123 sub getSrcChunks{my $self=shift;return $self->getChunks('source',@_);}
124 sub getTrgChunks{my $self=shift;return $self->getChunks('target',@_);}
125  
126 sub getPhrases{
127 my $self=shift;
128 my $lang=shift;
129 my $param=shift;
130 if (not defined $lang){$lang='source'};
131 if (not ref($self->{$lang})){return undef;}
132 # $self->subData($self->{$lang},$lang);
133 # $self->{$lang}=$self->subData($lang);
134 $self->makeParameter($param,$lang);
135 return $self->{$lang}->getPhrases($param,@_);
136 }
137  
138 sub getSrcPhrases{my $self=shift;return $self->getPhrases('source',@_);}
139 sub getTrgPhrases{my $self=shift;return $self->getPhrases('target',@_);}
140  
141  
142 sub getPhrasePos{
143 my $self=shift;
144 my ($phraseNodes,$tokenNodes)=@_;
145 my @idx=();
146 foreach my $p (0..$#{$phraseNodes}){
147 my $lastIdx=0;
148 foreach my $t ($lastIdx..$#{$tokenNodes}){
149 if ($$phraseNodes[$p]==$$tokenNodes[$t]){
150 push(@idx,$t);
151 $lastIdx=$t+1;
152 }
153 }
154 }
155 return join ":",@idx;
156 }
157  
158  
159 sub getRelativePosition{
160 my $self=shift;
161 my ($srcPhr,$trgPhr)=@_;
162  
163 my $srcPos=$self->{source}->getPhrasePosition($srcPhr);
164 my $trgPos=$self->{target}->getPhrasePosition($trgPhr);
165  
166 if (not defined $srcPos){return 0;}
167 if (not defined $trgPos){return 0;}
168  
169 return $trgPos-$srcPos;
170 }
171  
172 sub getFeaturePairs{
173 my $self=shift;
174 return getAlignPhrases(@_);
175 }
176  
177 sub getAlignPhrases{
178 my $self=shift;
179 my ($param,$src,$trg,$token,$attr)=@_;
180 if (ref($param) ne 'HASH'){$param={};}
181  
182 #----------------------------------------------------------------------
183 my @srcTokNodes=(); # 1) get all tokens
184 my @trgTokNodes=();
185 my @srcTok=$self->getSrcTokens($param,\@srcTokNodes);
186 my @trgTok=$self->getTrgTokens($param,\@trgTokNodes);
187  
188 #----------------------------------------------------------------------
189 my $srcNodes=[]; # 2) get all possible phrases
190 my $trgNodes=[];
191 my @srcPhr=$self->getSrcPhrases($param,$srcNodes,
192 \@srcTokNodes,\@srcTok);
193 my @trgPhr=$self->getTrgPhrases($param,$trgNodes,
194 \@trgTokNodes,\@trgTok);
195  
196 #----------------------------------------------------------------------
197 my @srcIdx=(); # 3) get token positions for each phrase
198 my @trgIdx=();
199 foreach (0..$#srcPhr){
200 push (@srcIdx,$self->getPhrasePos($$srcNodes[$_],\@srcTokNodes));
201 }
202 foreach (0..$#trgPhr){
203 push (@trgIdx,$self->getPhrasePos($$trgNodes[$_],\@trgTokNodes));
204 }
205  
206 #----------------------------------------------------------------------
207  
208 $self->makeParameter($param,'source'); # get source feature
209 foreach (0..$#{$srcNodes}){
210 $$src{$srcIdx[$_]}=
211 $self->{source}->getPhraseFeature(\@{$$srcNodes[$_]},
212 $param);
213 if (defined $$param{'relative position'}){
214 my $srcPos=$self->{source}->getPhrasePosition($$srcNodes[$_]);
215 if ($$src{$srcIdx[$_]}=~/\S/){
216 $$src{$srcIdx[$_]}.=":pos($srcPos)";
217 }
218 else{
219 $$src{$srcIdx[$_]}="pos($srcPos)";
220 }
221 }
222 }
223 $self->makeParameter($param,'target'); # and generate feature
224 foreach (0..$#{$trgNodes}){
225 $$trg{$trgIdx[$_]}=
226 $self->{target}->getPhraseFeature(\@{$$trgNodes[$_]},
227 $param);
228 if (defined $$param{'relative position'}){
229 my $trgPos=$self->{target}->getPhrasePosition($$trgNodes[$_]);
230 if ($$trg{$trgIdx[$_]}=~/\S/){
231 $$trg{$trgIdx[$_]}.=":pos($trgPos)";
232 }
233 else{
234 $$trg{$trgIdx[$_]}="pos($trgPos)";
235 }
236 }
237 }
238 if (ref($token) eq 'HASH'){
239 @{$$token{source}}=@srcTok;
240 @{$$token{target}}=@trgTok;
241  
242 if (ref($attr) eq 'HASH'){
243 @{$$attr{source}}=$self->{source}->attribute(\@srcTokNodes);
244 @{$$attr{target}}=$self->{target}->attribute(\@trgTokNodes);
245 foreach (0..$#srcTokNodes){
246 $$attr{source}[$_]{content}=$self->content($srcTokNodes[$_]);
247 }
248 foreach (0..$#trgTokNodes){
249 $$attr{target}[$_]{content}=$self->content($trgTokNodes[$_]);
250 }
251 }
252 }
253 }
254  
255 sub getSrcTokenFeatures{
256 my $self=shift;
257 return $self->getTokenFeatures('source',@_);
258 }
259  
260 sub getTrgTokenFeatures{
261 my $self=shift;
262 return $self->getTokenFeatures('target',@_);
263 }
264  
265 sub getTokenFeatures{
266 my $self=shift;
267 my $lang=shift; # source / target
268 my ($param,$nodes)=@_;
269 if (ref($param) ne 'HASH'){$param={};}
270 if (ref($nodes) ne 'ARRAY'){$nodes=[];}
271 if (not ref($self->{$lang})){return undef;}
272  
273 #----------------------------------------------------------------------
274  
275 $self->makeParameter($param,$lang);
276 my @tok=$self->{$lang}->getTokens($param,$nodes);
277  
278 if (keys %{$param}){
279 foreach (0..$#{$nodes}){
280 $tok[$_]=$self->{$lang}->getPhraseFeature([$$nodes[$_]],$param);
281 }
282 }
283 return @tok;
284 }
285  
286  
287 sub getBitextPhrases{
288 my $self=shift;
289 my ($param,$src,$trg,$token,$attr)=@_;
290 if (ref($param) ne 'HASH'){$param={};}
291  
292  
293 #----------------------------------------------------------------------
294 my @srcTokNodes=(); # 1) get all tokens
295 my @trgTokNodes=();
296 my @srcTok=$self->getSrcTokens($$param{general},\@srcTokNodes);
297 my @trgTok=$self->getTrgTokens($$param{general},\@trgTokNodes);
298  
299 #----------------------------------------------------------------------
300 my $srcNodes=[]; # 2) get all possible phrases
301 my $trgNodes=[];
302 my @srcPhr=$self->getSrcPhrases($$param{general},$srcNodes,
303 \@srcTokNodes,\@srcTok);
304 my @trgPhr=$self->getTrgPhrases($$param{general},$trgNodes,
305 \@trgTokNodes,\@trgTok);
306  
307 #----------------------------------------------------------------------
308 my @srcIdx=(); # 3) get token positions for each phrase
309 my @trgIdx=();
310 foreach (0..$#srcPhr){
311 push (@srcIdx,$self->getPhrasePos($$srcNodes[$_],\@srcTokNodes));
312 }
313 foreach (0..$#trgPhr){
314 push (@trgIdx,$self->getPhrasePos($$trgNodes[$_],\@trgTokNodes));
315 }
316  
317 #----------------------------------------------------------------------
318  
319 foreach my $p (keys %{$param}){ # 4) generate phrase features
320 if ($p eq 'general'){ # a) general = phrase string
321 foreach (0..$#srcPhr){
322 $$src{$srcIdx[$_]}{$p}=$srcPhr[$_]; # the source phrase
323 }
324 foreach (0..$#trgPhr){
325 $$trg{$trgIdx[$_]}{$p}=$trgPhr[$_]; # the target phrase
326 }
327 next;
328 }
329 my $srcParam=$$param{$p}; # b) feature parameter
330 $self->makeParameter($srcParam,'source'); # get source feature
331 foreach (0..$#{$srcNodes}){
332 $$src{$srcIdx[$_]}{$p}=
333 $self->{source}->getPhraseFeature(\@{$$srcNodes[$_]},
334 $srcParam);
335 if ((ref($$param{$p}) eq 'HASH') and
336 (defined $$param{$p}{'relative position'})){
337 my $srcPos=$self->{source}->getPhrasePosition($$srcNodes[$_]);
338 if ($$src{$srcIdx[$_]}{$p}=~/\S/){
339 $$src{$srcIdx[$_]}{$p}.=":pos($srcPos)";
340 }
341 else{
342 $$src{$srcIdx[$_]}{$p}="pos($srcPos)";
343 }
344 }
345 }
346 my $trgParam=$$param{$p}; # get target features
347 $self->makeParameter($trgParam,'target'); # and generate feature
348 foreach (0..$#{$trgNodes}){
349 $$trg{$trgIdx[$_]}{$p}=
350 $self->{target}->getPhraseFeature(\@{$$trgNodes[$_]},
351 $trgParam);
352 if ((ref($$param{$p}) eq 'HASH') and
353 (defined $$param{$p}{'relative position'})){
354 my $trgPos=$self->{target}->getPhrasePosition($$trgNodes[$_]);
355 if ($$trg{$trgIdx[$_]}{$p}=~/\S/){
356 $$trg{$trgIdx[$_]}{$p}.=":pos($trgPos)";
357 }
358 else{
359 $$trg{$trgIdx[$_]}{$p}="pos($trgPos)";
360 }
361 }
362 }
363 }
364 if (ref($token) eq 'HASH'){
365 @{$$token{source}}=@srcTok;
366 @{$$token{target}}=@trgTok;
367  
368 if (ref($attr) eq 'HASH'){
369 @{$$attr{source}}=$self->{source}->attribute(\@srcTokNodes);
370 @{$$attr{target}}=$self->{target}->attribute(\@trgTokNodes);
371 foreach (0..$#srcTokNodes){
372 $$attr{source}[$_]{content}=$self->content($srcTokNodes[$_]);
373 }
374 foreach (0..$#trgTokNodes){
375 $$attr{target}[$_]{content}=$self->content($trgTokNodes[$_]);
376 }
377 }
378 }
379 }
380  
381  
382 sub getPhraseFeature{
383 my $self=shift;
384 my $lang=shift;
385 my $nodes=shift;
386 my $param=shift;
387 if (not defined $lang){$lang='source'};
388 if (not ref($self->{$lang})){return undef;}
389 # $self->subData($self->{$lang},$lang);
390 # $self->{$lang}=$self->subData($lang);
391 $self->makeParameter($param,$lang);
392 return $self->{$lang}->getPhraseFeature($nodes,$param,@_);
393 }
394  
395 sub getSrcPhraseFeature{my $s=shift;return $s->getPhraseFeature('source',@_);}
396 sub getTrgPhraseFeature{my $s=shift;return $s->getPhraseFeature('target',@_);}
397  
398 sub checkPairParameter{
399 my $self=shift;
400 my ($src,$trg,$param)=@_;
401 if ($$param{'minimal length (source)'}){
402 if (length($src)<$$param{'minimal length (source)'}){
403 # print STDERR "minimale length (source)\n";
404 return 0;
405 }
406 }
407 if ($$param{'minimal length (target)'}){
408 if (length($trg)<$$param{'minimal length (target)'}){
409 # print STDERR "minimale length (target)\n";
410 return 0;
411 }
412 }
413 if ($$param{'minimal length diff'}){
414 if ($self->lengthQuotient($src,$trg)<$$param{'minimal length diff'}){
415 # print STDERR "minimale length diff\n";
416 return 0;
417 }
418 }
419 if ($$param{'matching word class'}){
420 if (not $self->isSameType($src,$trg,$$param{'matching word class'})){
421 # print STDERR "matching word class\n";
422 return 0;
423 }
424 }
425 if ($$param{'stop words'}){
426 if (not $self->isSameType($src,$trg,$$param{'stop words'})){
427 # print STDERR "stop words\n";
428 return 0;
429 }
430 }
431 return 1;
432 }
433  
434  
435 sub isSameType{
436 my $self=shift;
437 my ($src,$trg,$check)=@_;
438  
439 if (($check eq 'open/closed') or ($check eq 'same')){
440 if ($self->{source}->isStopWord($src)){
441 return $self->{target}->isStopWord($trg);
442 }
443 return (not $self->{target}->isStopWord($trg));
444 }
445 elsif ($check eq 'exclude'){
446 if (not $self->{source}->isStopWord($src)){
447 return 1;
448 }
449 return (not $self->{target}->isStopWord($trg));
450 }
451 elsif(($check eq 'same_class') or ($check eq 'wordclass')){
452 return $self->isSameClass($src,$trg);
453 }
454 elsif(($check eq 'same_sub_class') or ($check eq 'subclass')){
455 return $self->isSameSubClass($src,$trg);
456 }
457 return 1;
458 }
459  
460  
461 sub isSameClass{
462 my $self=shift;
463 my ($src,$trg)=@_;
464  
465 my $cat='stop word class hash';
466 my $SrcData=$self->{source}->getLanguageData($cat);
467 my $TrgData=$self->{target}->getLanguageData($cat);
468  
469 if (ref($SrcData) ne 'HASH'){return 1;}
470 if (ref($TrgData) ne 'HASH'){return 1;}
471  
472 foreach my $c (%{$SrcData}){
473 if (defined $$SrcData{$c}{$src}){
474 if (defined $$TrgData{$c}){
475 if (defined $$TrgData{$c}{$trg}){
476 return 1;
477 }
478 }
479 return 0;
480 }
481  
482 }
483 foreach my $c (%{$TrgData}){
484 if (defined $$TrgData{$c}{$trg}){
485 return 0;
486 }
487  
488 }
489 return 1;
490 }
491  
492 sub isSameSubClass{
493 my $self=shift;
494 my ($src,$trg)=@_;
495  
496 my $cat='stop word subclass hash';
497 my $SrcData=$self->{source}->getLanguageData($cat);
498 my $TrgData=$self->{target}->getLanguageData($cat);
499  
500 if (ref($SrcData) ne 'HASH'){return 1;}
501 if (ref($TrgData) ne 'HASH'){return 1;}
502  
503 foreach my $x (%{$SrcData}){
504 foreach my $y (%{$$SrcData{$x}}){
505 if (defined $$SrcData{$x}{$y}{$src}){
506 if (defined $$TrgData{$x}){
507 if (defined $$TrgData{$x}{$y}){
508 if (defined $$TrgData{$x}{$y}{$trg}){
509 return 1;
510 }
511 }
512 }
513 return 0;
514 }
515 }
516 }
517 foreach my $x (keys %{$TrgData}){
518 foreach my $y (keys %{$$TrgData{$x}}){
519 if (defined $$TrgData{$x}{$y}{$trg}){
520 return 0;
521 }
522 }
523 }
524 return 1;
525 }
526  
527  
528  
529 sub lengthQuotient{
530 my $self=shift;
531 my ($src,$trg)=@_;
532 if (length($src)==0 or length($trg)==0) {return 0;}
533 if (length($src)>length($trg)) {return length($trg)/length($src);}
534 else {return length($src)/length($trg);}
535 }
536  
537  
538  
539 sub makeParameter{
540 my $self=shift;
541 my ($param,$lang)=@_;
542 if (ref($param) ne 'HASH'){return;}
543 foreach (keys %{$param}){
544 if (/^(.*) \($lang\)/){
545 $param->{$1}=$param->{$_};
546 }
547 }
548 }
549  
550  
551  
552  
553 sub rmLinkedToken{
554 my $data=shift;
555  
556 # my $srcData=Uplug::Data::Lang->new;
557 # my $trgData=Uplug::Data::Lang->new;
558 # $data->subData($srcData,'source');
559 # $data->subData($trgData,'target');
560 ## $data->{'source'}=$data->subData('source');
561 ## $data->{'target'}=$data->subData('target');
562  
563 my $srcData=$data->{source};
564 my $trgData=$data->{target};
565  
566 my $link=$data->{link};
567 my @nodes=$link->findNodes('wordLink');
568 my @xtrg=$link->attribute(\@nodes,'xtargets');
569  
570 foreach my $l (@xtrg){
571 if ($l=~/^(.*\S)\s?\;\s?(\S.*)$/){
572 my ($s,$t)=($1,$2);
573 $data->rmToken($s,$srcData);
574 $data->rmToken($t,$trgData);
575 }
576 }
577 }
578  
579 sub rmToken{
580 my $self=shift;
581 my ($span,$data)=@_;
582 my @token=split(/[\+\s]/,$span);
583 foreach (@token){
584 my ($node)=$data->findNodes('.*',{id => $_});
585 if (defined $node){
586 $node->getParentNode->removeChild($node);
587 $node->dispose();
588 }
589 }
590 }
591  
592 sub rmWordLinks{
593 my $data=shift;
594 # $data->{link}->delAttribute('wordLink');
595 if (ref($data->{link})){
596 $data->{link}->delNodes('wordLink');
597 }
598 }
599  
600 sub findLink{
601 my $self=shift;
602 my $link=shift;
603 my %attr=();
604 $attr{src}=$link->{source};
605 $attr{trg}=$link->{target};
606 my @nodes=$self->{link}->findNodes('wordLink',\%attr);
607 if (@nodes){
608 return @nodes;
609 }
610 return undef;
611 }
612  
613  
614 sub addWordLink{
615 my $data=shift;
616 my $link=shift;
617 my $OutData=$data->{link};
618  
619  
620 if (defined $data->findLink($link)){return;}
621  
622 my %attr=();
623  
624 if (defined $link->{score}){
625 $attr{certainty}=$link->{score};
626 }
627 $attr{lexPair}=$link->{link};
628 $attr{xtargets}="$link->{source};$link->{target}";
629 $attr{xtargets}=~tr/:/+/;
630  
631 if ($link->{src} and $link->{trg}){
632 $link->{src}=~tr/\&/\+/;
633 $link->{trg}=~tr/\&/\+/;
634 $attr{span}="$link->{src};$link->{trg}"
635 }
636 my $wordLink=$OutData->createNode('wordLink',\%attr);
637 $OutData->addNode($wordLink);
638  
639 # if (defined $link->{step}){$attr{step}=$link->{step};}
640 # $attr{'id'}=$id;
641 # $attr{'content'}="\n$src:$trg\n";
642  
643 }
644  
645  
646 sub toHTML{
647 my $self=shift;
648 my $html=$self->{source}->toHTML();
649 $html.=$self->{target}->toHTML();
650 }