blib/lib/Uplug/Align/Word.pm untested

filter: Statementstmt 0.0% 0 / 328 Branchbran 0.0% 0 / 86 Conditioncond 0.0% 0 / 6 MC/DCmcdc 0.0% 0 / 4 Subroutinesub 0.0% 0 / 44 Podpod 0.0% 0 / 40 totaltotal 0.0% 0 / 508 CC 44 SCAR 32.1
CC
44
Cov
0%
getLinkScores
110.0
findAlignment
56.0
align
30.0
CRAP 24.7
1 #-*-perl-*-
2 ####################################################################
3 # Copyright (C) 2004 Jörg Tiedemann <joerg@stp.ling.uu.se>
4 #
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 #
19 ###########################################################################
20 # Uplug::Align::Word
21 #
22 # (best first aligner)
23 #
24 ###########################################################################
25  
26  
27 package Uplug::Align::Word;
28  
29 use strict;
30 # use Time::HiRes qw(time);
31 use vars qw($DEBUG @ISA $CLUEWEIGHT );
32  
33 use Uplug::Data;
34 use Uplug::Data::Align;
35  
36 # @ISA = qw( );
37  
38 $CLUEWEIGHT = 0.05;
39 $DEBUG = 0;
40  
41 sub new{
42 my $class=shift;
43 my $stream=shift;
44 my $self={};
45 bless $self,$class;
46 if (ref($stream)){
47 $self->{input}=$stream;
48 }
49 while ($_=shift){ # set parameters!
50 $self->setParameter($_,shift);
51  
52 }
53 $self->{data}=Uplug::Data::Align->new($self->srcLanguage,$self->trgLanguage);
54 $self->{dataCount}=0;
55  
56 return $self;
57 }
58  
59 sub setLinkStreams{
60 my $self=shift;
61 my $streams=shift;
62 if (ref($streams) eq 'HASH'){
63 %{$self->{linkStreams}}=%{$streams};
64 }
65 }
66  
67 sub setInputStream{
68 my $self=shift;
69 my $stream=shift;
70 $self->{input}=$stream;
71 }
72 sub setInput{my $self=shift;return $self->setInputStream(@_);}
73  
74  
75 sub read{
76 my $self=shift;
77 my $id=shift;
78 $self->{dataCount}++;
79 if (defined $id){
80 return $self->{input}->select($self->{data},{id => $id});
81 }
82 return $self->{input}->read($self->{data});
83 }
84 sub readData{my $self=shift;return $self->read(@_);}
85  
86 sub dataId{
87 my $self=shift;
88  
89 if (defined $self->data->{link}){
90 return $self->data->{link}->attribute('id');
91 }
92 my $id=$self->data->attribute('id');
93 if (not defined $id){
94 return $self->{dataCount};
95 }
96 return $id;
97 }
98  
99 sub data{
100 my $self=shift;
101 return $self->{data};
102 }
103  
104  
105 sub align{
106 my $self=shift;
107 $self->{token}={};
108 $self->{tokenAttr}={};
109 $self->{srcToken}={};
110 $self->{trgToken}={};
111 # if ($self->getParameter('remove linked')){
112 # $self->{data}->rmLinkedToken;
113 # }
114  
115 my $time=time();
116  
117 if ($self->getParameter('non-aligned only')){ # --> align all tokens
118 $self->{data}->rmLinkedToken; # which haven't been
119 } # aligned yet
120 else{ # otherwise:
121 $self->{data}->rmWordLinks; # remove old links
122 }
123 $self->{data}->getBitextPhrases($self->{param},
124 $self->{srcToken},
125 $self->{trgToken},
126 $self->{token},
127 $self->{tokenAttr});
128  
129 $self->{nrSrcToken}+=$#{$self->{token}->{source}}+1;
130 $self->{nrTrgToken}+=$#{$self->{token}->{target}}+1;
131 ### DEBUG: store time for preparing data
132 $self->{prepare_time}+=time()-$time;$time=time() if ($DEBUG);
133 $self->getLinkScores();
134 ### DEBUG: store time for retrieving clue scores
135 $self->{get_scores_time}+=time()-$time;$time=time() if ($DEBUG);
136 $self->findAlignment();
137 ### DEBUG: store time for finding the best word alignment
138 $self->{align_time}+=time()-$time if ($DEBUG);
139  
140  
141 # if ($self->getParameter('remove word links')){
142 # $self->{data}->rmWordLinks;
143 # }
144 return $self->{links};
145 }
146  
147 sub getLinkScores{
148 my $self=shift;
149  
150 $self->{linkProbs}={};
151 $self->{links}={};
152 my $LinkProb=$self->{linkProbs};
153 # my $links=$self->{links};
154 my $links=$self->{linkStreams};
155 my $SrcTok=$self->{srcToken};
156 my $TrgTok=$self->{trgToken};
157 my $Param=$self->{param};
158 my $data=$self->{data};
159 my $MinScore=$self->scoreThreshold();
160  
161 foreach my $s (keys %{$SrcTok}){
162 foreach my $t (keys %{$TrgTok}){
163  
164 my ($src,$trg)=($$SrcTok{$s}{general},$$TrgTok{$t}{general});
165 if ($data->checkPairParameter($src,$trg,$$Param{general})){
166  
167 foreach (keys %{$links}){
168 my $weight=$self->defaultClueWeight();
169 if (ref($$Param{$_}) eq 'HASH'){
170 if (defined $$Param{$_}{'score weight'}){
171 $weight=$$Param{$_}{'score weight'};
172 }
173 }
174 my ($src,$trg)=($$SrcTok{$s}{$_},$$TrgTok{$t}{$_});
175 if ($src and $trg){
176 if ($data->checkPairParameter($src,$trg,$$Param{$_})){
177 my %search=('source' => $src,
178 'target' => $trg);
179 my $found=Uplug::Data->new;
180 if ($links->{$_}->select($found,\%search)){
181 if ($self->checkScore($found,$$Param{$_})){
182 my $p=$$LinkProb{"$s\x00\x00$t"};
183  
184 my $score=$found->attribute('score');
185  
186 # shouldn't be >1, but in case ...
187 #--------------------------------
188 if ($score>1){$score=1;}
189 #--------------------------------
190 #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
191 # default: weight=0.5 for each score
192 $score*=$weight;
193 #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
194  
195 $$LinkProb{"$s\x00\x00$t"}=
196 $p+$score-$p*$score;
197 }
198 }
199 }
200 }
201 }
202 }
203 }
204 }
205 foreach my $l (keys %{$LinkProb}){
206 if ($$LinkProb{$l}<$MinScore){
207 delete $$LinkProb{$l};
208 }
209 }
210 }
211  
212 sub findAlignment{
213 my $self=shift;
214 my $step=shift;
215  
216 $self->{links}={};
217 my $MinScore=$self->scoreThreshold();
218  
219 my $LinkProb=$self->{linkProbs};
220 my $Links=$self->{links};
221 my $Token=$self->{token};
222 my $TokenAttr=$self->{tokenAttr};
223 my $SrcTok=$self->{srcToken};
224 my $TrgTok=$self->{trgToken};
225 foreach (sort {$$LinkProb{$b} <=> $$LinkProb{$a}} keys %{$LinkProb}){
226 if ($$LinkProb{$_}<$MinScore){last;}
227 my ($s,$t)=split(/\x00\x00/,$_);
228 if (not defined $$SrcTok{$s}){next;}
229 if (not defined $$TrgTok{$t}){next;}
230  
231 my $link=$self->getLinkString($TokenAttr,$s,$t);
232  
233 $$Links{$s}{link}=$link;
234 $$Links{$s}{source}=$self->ngramIDs($s,$TokenAttr,'source');
235 $$Links{$s}{target}=$self->ngramIDs($t,$TokenAttr,'target');
236 $$Links{$s}{score}=$$LinkProb{$_};
237 if (defined $step){$$Links{$s}{step}=$step;}
238 my $span=$self->ngramSpans($s,$TokenAttr,'source');
239 if ($span){$$Links{$s}{src}=$span;}
240 $span=$self->ngramSpans($t,$TokenAttr,'target');
241 if ($span){$$Links{$s}{trg}=$span;}
242  
243 &RemoveOverlap($s,$SrcTok);
244 &RemoveOverlap($t,$TrgTok);
245 }
246 }
247  
248  
249  
250 sub getLinkString{
251 my $self=shift;
252 my ($TokenAttr,$s,$t)=@_;
253  
254 my $link='';
255 my @tok=split(/:/,$s);
256 foreach (@tok){
257 $link.=$$TokenAttr{source}[$_]{content}.' ';
258 }
259 chop $link;
260 $link.=';';
261 my @tok=split(/:/,$t);
262 foreach (@tok){
263 $link.=$$TokenAttr{target}[$_]{content}.' ';
264 }
265 chop $link;
266 return $link;
267 }
268  
269 sub ngramIDs{
270 my $self=shift;
271 my ($pos,$attr,$l)=@_;
272 my @arr=split(/\:/,$pos);
273 my @res=();
274 foreach (@arr){
275 if (defined $$attr{$l}[$_]{id}){
276 push (@res,$$attr{$l}[$_]{id});
277 }
278 else{
279 push (@res,$_);
280 }
281 }
282 return join ':',@res;
283 }
284  
285 sub ngramSpans{
286 my $self=shift;
287 my ($pos,$attrs,$l)=@_;
288 my @arr=split(/\:/,$pos);
289 my @spans=();
290 foreach (@arr){
291 push (@spans,$$attrs{$l}[$_]{span});
292 }
293 return join '&',@spans;
294 }
295  
296 sub makeRelPosFeature{
297 my $self=shift;
298 my ($src,$trg)=@_;
299 if ($src=~/pos\((\-?[0-9]+)\)/){
300 my $srcPos=$1;
301 $src=~s/pos\((\-?[0-9]+)\)/x/;
302 if ($trg=~/pos\((\-?[0-9]+)\)/){
303 my $relPos=$1-$srcPos;
304 $trg=~s/pos\((\-?[0-9]+)\)/$relPos/;
305 }
306 }
307 return ($src,$trg);
308 }
309  
310  
311 sub printBitextToken{
312 my $self=shift;
313 my ($SrcTok,$TrgTok)=@_;
314 print STDERR "\n";
315 print STDERR join ' ',@{$SrcTok};
316 print STDERR "\n";
317 print STDERR join ' ',@{$TrgTok};
318 print STDERR "\n\n";
319 }
320  
321 sub tokenID{
322 my $self=shift;
323 my ($lang,$idx)=@_;
324 if (ref($self->{tokenAttr}) eq 'HASH'){
325 if (ref($self->{tokenAttr}->{$lang}) eq 'ARRAY'){
326 if (ref($self->{tokenAttr}->{$lang}->[$idx]) eq 'HASH'){
327 return $self->{tokenAttr}->{$lang}->[$idx]->{id};
328 }
329 }
330 }
331 return undef;
332 }
333  
334 sub printBitextTokensWithID{
335 my $self=shift;
336  
337 print STDERR "\n\n====================================================\n";
338 print STDERR "bitext segment";
339 print STDERR "\n====================================================\n";
340  
341 $self->printTokensWithID('source');
342 $self->printTokensWithID('target');
343 }
344  
345 sub printTokensWithID{
346 my $self=shift;
347 my $lang=shift;
348  
349 if (ref($self->{token}->{$lang}) eq 'ARRAY'){
350 foreach (0..$#{$self->{token}->{$lang}}){
351 if (my $id=$self->tokenID($lang,$_)){
352 print STDERR "$id:";
353 }
354 my $token = $self->{token}->{$lang}->[$_];
355 $token=~s/^\s*//;
356 # $token=~s/\s*$//;
357 # print STDERR $self->{token}->{$lang}->[$_];
358 print STDERR $token;
359 print STDERR " ";
360 }
361 }
362 print STDERR "\n";
363 }
364  
365  
366  
367 sub printBitextLink{
368 my $self=shift;
369 my $id=shift;
370 my $link=shift;
371 my ($src,$trg)=split(/;/,$$link{link});
372 my $sidx = $$link{source};
373 my $tidx = $$link{target};
374 if (defined $$link{score}){
375 print STDERR join "\t",($id,$sidx,$tidx,$src,$trg,$$link{score});
376 print STDERR "\n";
377 # printf STDERR "%s %10s %-10s %s %s\t%s\n",
378 # $id,$sidx,$tidx,$src,$trg,$$link{score};
379 }
380 else{
381 print STDERR join "\t",($id,$sidx,$tidx,$src,$trg);
382 print STDERR "\n";
383 $id,$sidx,$tidx,$src,$trg;
384 }
385 }
386  
387  
388 sub getNrLinks{
389 my $self=shift;
390 if (ref($self->{linkProbs}) eq 'HASH'){
391 return scalar keys %{$self->{linkProbs}};
392 }
393 return undef;
394 }
395  
396 sub getParameter{
397 my $self=shift;
398 return $self->parameter(@_);
399 }
400 sub parameter{
401 my $self=shift;
402 my $name=shift;
403 return $self->{$name};
404 }
405 sub setParameter{
406 my $self=shift;
407 my ($attr,$val)=@_;
408 $self->{$attr}=$val;
409 }
410  
411 sub setLanguages{
412 my $self=shift;
413 my ($src,$trg)=@_;
414 $self->setSrcLanguage($src);
415 $self->setTrgLanguage($trg);
416 }
417  
418 sub srcLanguage{
419 my $self=shift;
420 return $self->{'language (source)'};
421 }
422 sub trgLanguage{
423 my $self=shift;
424 return $self->{'language (target)'};
425 }
426  
427 sub setSrcLanguage{
428 my $self=shift;
429 my ($src)=@_;
430 $self->{'language (source)'}=$src;
431 $self->{data}->{source}->setLanguage($src);
432 }
433 sub setTrgLanguage{
434 my $self=shift;
435 my ($trg)=@_;
436 $self->{'language (target)'}=$trg;
437 $self->{data}->{target}->setLanguage($trg);
438 }
439  
440 sub searchMethod{
441 my $self=shift;
442 return $self->{search};
443 }
444 sub scoreThreshold{
445 my $self=shift;
446 return $self->{'minimal score'};
447 }
448  
449 sub defaultClueWeight{
450 my $self=shift;
451 return $CLUEWEIGHT;
452 }
453  
454 sub setSearchMethod{
455 my $self=shift;
456 $self->{search}=shift;
457 }
458 sub setScoreThreshold{
459 my $self=shift;
460 $self->{'minimal score'}=shift;
461 }
462 sub setRmLinkedFlag{
463 my $self=shift;
464 $self->{'remove linked'}=shift;
465 }
466 sub setLinkParam{
467 my $self=shift;
468 my $linkType=shift;
469 my $param=shift;
470 if (ref($param) eq 'HASH'){
471 %{$self->{param}->{$linkType}}=%{$param};
472 # if ($linkType!~/^general$/){
473 # if ($linkType!~/^original$/){
474 # %{$self->{clueparam}->{$linkType}}=%{$param};
475 # }
476 # }
477 }
478 }
479  
480 sub setLinkParams{
481 my $self=shift;
482 my $param=shift;
483 if (ref($param) eq 'HASH'){
484 foreach (keys %{$param}){
485 $self->setLinkParam($_,$param->{$_});
486 }
487 }
488 }
489  
490  
491 sub checkScore{
492 my $self=shift;
493 my ($data,$param)=@_;
494 my $score=$data->attribute('score');
495 if (not defined $score){return 0;}
496 # if (defined $$param{normalize}){
497 ## ........
498 # }
499 if (defined $$param{'minimal score'}){
500 if ($score<$$param{'minimal score'}){
501 # print STDERR "score too low\n";
502 return 0;
503 }
504 }
505 # if ($score<$$param{general}{'minimal score'}){
506 # return 0;
507 # }
508 return 1;
509 }
510  
511  
512  
513  
514 sub linksToHtml{
515 my $self=shift;
516 my $links=$self->{links};
517 my $split=8;
518 my $count=0;
519 my $html;
520 if (ref($links) eq 'HASH'){
521 $html="<div class=\"alignments\"><table cellspacing=\"10\">";
522 foreach my $l (keys %{$links}){
523 if ((not $count % $split) || ($count==0)){
524 if ($count){$html.="</table></td>";}
525 $html.="<td valign=\"top\"><table cellspacing=1 border=0><tr>";
526 $html.="<th bgcolor=\"#c9c9c9\">source</th>";
527 $html.="<th bgcolor=\"#c9c9c9\">target</th>";
528 $html.="</tr>";
529 }
530 my ($src,$trg)=split(/;/,$$links{$l}{link});
531 $html.="<tr><td bgcolor=\"#EEEEEE\" align='center'>$src</td>";
532 $html.="<td bgcolor=\"#E4EEEE\" align='center'>$trg</td></tr>\n";
533 $count++;
534 }
535 $html.='</table></td></table></div>';
536 }
537 return $html;
538 }
539  
540  
541  
542  
543  
544  
545  
546  
547  
548  
549  
550  
551  
552 #--------------------------------------------------------------------------
553 # remove overlapping alignments
554 #
555  
556 sub RemoveOverlap{
557 my ($id,$token)=@_;
558 my $pat='('.$id.')';
559 $pat=~tr/:/|/;
560 foreach (keys %{$token}){
561 if (/(\A|\:)$pat(\:|\Z)/){
562 delete $token->{$_};
563 }
564 }
565 }
566  
567  
568  
569 1;
570  
571  
572 __END__