blib/lib/Uplug/Align/Word/Giza.pm untested

filter: Statementstmt 0.0% 0 / 589 Branchbran 0.0% 0 / 196 Conditioncond 0.0% 0 / 64 MC/DCmcdc 0.0% 0 / 37 Subroutinesub 0.0% 0 / 24 Podpod 0.0% 0 / 15 totaltotal 0.0% 0 / 925 CC 95 SCAR 52.3
CC
95
Cov
0%
add_adjacent_old
552.0
Giza2Uplug
182.0
Combined2Uplug
182.0
CRAP 187.5
1 #-*-perl-*-
2  
3 package Uplug::Align::Word::Giza;
4  
5  
6 use strict;
7  
8 use Cwd;
9 use Exporter;
10 use File::Copy;
11  
12 use Uplug::Data;
13 use Uplug::Data::Align;
14 use Uplug::IO::Any;
15 use Uplug::Config;
16  
17 use vars qw(@ISA @EXPORT $DEBUG $GIZAHOME);
18  
19 # @ISA = qw( Uplug::Align::Word Exporter );
20 @ISA = qw( Exporter);
21 $DEBUG = 0;
22  
23 @EXPORT = qw( &Bitext2Text &RunGiza &Combined2Uplug &Giza2Clue &Giza2Uplug );
24  
25 #BEGIN{
26 our $GIZAHOME="$ENV{UPLUGHOME}/ext/GIZA++";
27 if (not -d $GIZAHOME){$GIZAHOME="$ENV{HOME}/cvs/GIZA++-v2";}
28 if (not -d $GIZAHOME){$GIZAHOME="$ENV{HOME}/cvs/GIZA++";}
29 if (not -d $GIZAHOME){$GIZAHOME="/local/ling/GIZA++-v2";}
30 if (not -d $GIZAHOME){$GIZAHOME="/local/ling/GIZA++";}
31 #}
32  
33 # if (not -d $GIZAHOME){warn "cannot find GIZA++!";exit;}
34  
35  
36 #----------------------------------------------------------------------------
37 # Giza2Clue (new version): no external calls
38 # - looks for $dir/GIZA++.actual.ti.final (lexical prob's from GIZA)
39 # - creates data/runtime/giza.dbm
40 # - creates data/runtime/giza2.dbm (inverse alignments)
41  
42 sub Giza2Clue{
43 my $file=shift;
44 my $param=shift;
45 my $inverse=shift;
46 my $ClueDB=shift;
47 my $threshold=shift;
48  
49 if (-d $file){ # if file is a directory
50 $file="$file/GIZA++.actual.ti.final"; # look for the standard file in
51 } # this directory!
52  
53 my %dic;
54 if (ref($ClueDB) eq 'HASH'){
55 %dic=%{$ClueDB};
56 }
57 else{
58 %dic=('format' => 'dbm',
59 'write_mode' => 'overwrite',
60 'key' => ['source','target']);
61 my $cluedir='data/runtime';
62 if ($inverse){$dic{file}="$cluedir/giza2.dbm";}
63 else{$dic{file}="$cluedir/giza.dbm";}
64 }
65  
66 my %inStream=('file' => $file,
67 'format' => 'tab',
68 'field delimiter' => ' ');
69 if ($inverse){
70 $inStream{'columns'}=['source','target','value',],
71 }
72 else{
73 $inStream{'columns'}=['target','source','value',],
74 }
75  
76  
77 my %lex=();
78 my $data=Uplug::Data->new;
79  
80 my $in=Uplug::IO::Any->new(\%inStream);
81 $in->open('read',\%inStream);
82 my $count=0;
83 while ($in->read($data)){
84 $count++;
85 if (not ($count % 1000)){print STDERR '.';}
86 if (not ($count % 10000)){print STDERR "$count\n";}
87 my $src=$data->attribute('source');
88 my $trg=$data->attribute('target');
89 if ((not $src) or (not $trg)){next;}
90 my $value=$data->attribute('value');
91 if (not $value){$value=1;}
92 if ((defined $threshold) and ($value<$threshold)){next;}
93 $lex{$src}{$trg}=$value;
94 if (($src=~s/\_/ /gs) or ($trg=~s/\_/ /gs)){ # (for giza-clue:)
95 $lex{$src}{$trg}=$value; # '_' means ' '
96 }
97 }
98 my $header=$in->header;
99  
100 my $out=Uplug::IO::Any->new(\%dic);
101 $out->open('write',\%dic);
102 $out->addheader($header);
103 $out->addheader($param);
104 $out->writeheader();
105  
106 foreach my $s (keys %lex){
107 my $total;
108 foreach my $t (keys %{$lex{$s}}){
109 my $score=$lex{$s}{$t};
110 my $data=Uplug::Data->new;
111 $data->setAttribute('source',$s);
112 $data->setAttribute('target',$t);
113 $data->setAttribute('score',$score);
114 $out->write($data);
115 }
116 }
117  
118 $out->close;
119 $in->close;
120 }
121  
122 #----------------------------------------------------------------------------
123 # Giza2Uplug: convert GIZA's Viterbi alignment to Uplug format (XML)
124 # (slow and risky: GIZA's output must be complete and use a certain format)
125  
126 sub Giza2Uplug{
127 my $viterbi=shift;
128 my $bitext=shift;
129 my $param=shift;
130 my $links=shift;
131 my $inverse=shift;
132  
133 if (ref($links) ne 'HASH'){return 0;}
134 my $input=Uplug::IO::Any->new($bitext);
135 if (not ref($input)){return 0;}
136 if (not $input->open('read',$bitext)){return 0;}
137 my $BitextHeader=$input->header();
138 my $output=Uplug::IO::Any->new($links);
139 if (not ref($output)){return 0;}
140 $output->addheader($BitextHeader);
141 if (not $output->open('write',$links)){return 0;}
142 $output->setOption('SkipDataHeader',0); # don't skip headers and
143 $output->setOption('SkipDataTail',0); # footers (e.g. <linkGrp>)
144  
145  
146 #------------------------------------------------------------------------
147 if ($viterbi=~/\.gz$/){
148 open F,"gzip -cd <$viterbi |" ||
149 die "cannot open Viterbi alignment file $viterbi!";
150 }
151 else{
152 open F,"<$viterbi" ||
153 die "cannot open Viterbi alignment file $viterbi!";
154 }
155 #------------------------------------------------------------------------
156  
157 my $TokenLabel='w';
158 my $data=Uplug::Data::Align->new();
159 print STDERR "convert GIZA's Viterbi alignment to XML!\n";
160 my $count=0;
161  
162 while ($input->read($data)){
163  
164 $count++;
165 if (not ($count % 100)){
166 $|=1;print STDERR '.';$|=0;
167 }
168 if (not ($count % 1000)){
169 $|=1;print STDERR "$count\n";$|=0;
170 }
171  
172 #----------------------------------
173 # do the same as for Bitext2Text!!
174 # (to check for empty strings ...)
175 #
176 my @SrcNodes=();
177 my @TrgNodes=();
178 my ($srctxt,$trgtxt)=
179 &BitextStrings($data,$param,\@SrcNodes,\@TrgNodes);
180 if (($srctxt!~/\S/) or ($trgtxt!~/\S/)){next;}
181 #----------------------------------
182  
183 # my $SrcData=$data->sourceData();
184 # my $TrgData=$data->targetData();
185 #
186 # my @SrcNodes=$SrcData->findNodes($TokenLabel);
187 my @SrcIds=$data->attribute(\@SrcNodes,'id');
188 my @SrcSpans=$data->attribute(\@SrcNodes,'span');
189 # my @SrcTokens=$data->content(\@SrcNodes);
190 my @SrcTokens=$data->getSrcTokenFeatures($param,\@SrcNodes);
191  
192 # my @TrgNodes=$TrgData->findNodes($TokenLabel);
193 my @TrgIds=$data->attribute(\@TrgNodes,'id');
194 my @TrgSpans=$data->attribute(\@TrgNodes,'span');
195 # my @TrgTokens=$data->content(\@TrgNodes);
196 my @TrgTokens=$data->getTrgTokenFeatures($param,\@TrgNodes);
197  
198 if ((not @SrcNodes) or (not @TrgNodes)){next;}
199  
200 $_=<F>;
201 $_=<F>;
202 chomp;
203 my @src=split(/ /);
204 $_=<F>;
205 chomp;
206  
207 my %align=();
208 my $count=1;
209 while (/\s(\S.*?)\s\(\{\s(.*?)\}\)/g){ # strunta i NULL!!
210 if ($2){push (@{$align{$2}},$count);}
211 $count++;
212 }
213 foreach (sort keys %align){
214 my @s;my @t;
215 if ($inverse){
216 @t=@{$align{$_}};
217 @s=split(/\s/);
218 }
219 else{
220 @s=@{$align{$_}};
221 @t=split(/\s/);
222 }
223  
224 my @src=();my @trg=();
225 foreach (@s){push (@src,$SrcTokens[$_-1]);}
226 foreach (@t){push (@trg,$TrgTokens[$_-1]);}
227 my @srcId=();my @trgId=();
228 foreach (@s){push (@srcId,$SrcIds[$_-1]);}
229 foreach (@t){push (@trgId,$TrgIds[$_-1]);}
230 my @srcSpan=();my @trgSpan=();
231 foreach (@s){push (@srcSpan,$SrcSpans[$_-1]);}
232 foreach (@t){push (@trgSpan,$TrgSpans[$_-1]);}
233  
234 my %link=();
235 $link{link}=join ' ',@src;
236 $link{link}.=';';
237 $link{link}.=join ' ',@trg;
238 $link{source}=join '+',@srcId;
239 $link{target}=join '+',@trgId;
240 $link{src}=join '&',@srcSpan;
241 $link{trg}=join '&',@trgSpan;
242  
243 $data->addWordLink(\%link);
244 }
245  
246 $output->write($data);
247 }
248 $input->close;
249 $output->close;
250 }
251  
252  
253 #----------------------------------------------------------------------------
254 # Combined2Uplug: combine GIZA's Viterbi alignment and convert them to Uplug format (XML)
255 # (slow and risky: GIZA's output must be complete and must use a certain format)
256 #
257 # possible combinatins: union, intersection, refined
258 #
259  
260 sub Combined2Uplug{
261 my $giza0=shift;
262 my $giza1=shift;
263 my $combine=shift;
264 my $bitext=shift;
265 my $param=shift;
266 my $links=shift;
267  
268 if (ref($links) ne 'HASH'){return 0;}
269 my $input=Uplug::IO::Any->new($bitext);
270 if (not ref($input)){return 0;}
271 if (not $input->open('read',$bitext)){return 0;}
272 my $output; # open output later (after reading the first record of
273 # the input stream --> this gives the complete input header)
274  
275 #------------------------------------------------------------------------
276 if ($giza0=~/\.gz$/){open F0,"gzip -cd <$giza0 |";}
277 else{open F0,"<$giza0";}
278 # ($]>=5.008){binmode(F0, ":encoding(utf-8)");}
279  
280 if ($giza1=~/\.gz$/){open F1,"gzip -cd <$giza1 |";}
281 else{open F1,"<$giza1";}
282 # ($]>=5.008){binmode(F1, ":encoding(utf-8)");}
283 #------------------------------------------------------------------------
284  
285 my $TokenLabel='w';
286 my $data=Uplug::Data::Align->new();
287 print STDERR "combine GIZA's Viterbi alignments and convert to XML!\n";
288 my $count=0;
289  
290 while ($input->read($data)){
291  
292 if (not $output){ # output is not opened yet:
293 my $BitextHeader=$input->header(); # - get the input header
294 $output=Uplug::IO::Any->new($links); # - create an output stream
295 if (not ref($output)){return 0;} # (or die)
296 $output->addheader($BitextHeader); # - add input header
297 if (not $output->open('write',$links)){ # - open the output stream
298 return 0; # (or die)
299 }
300 $output->setOption('SkipDataHeader',0); # don't skip headers and
301 $output->setOption('SkipDataTail',0); # footers (e.g. <linkGrp>)
302 }
303  
304 $count++;
305 if (not ($count % 100)){
306 $|=1;print STDERR '.';$|=0;
307 }
308 if (not ($count % 1000)){
309 $|=1;print STDERR "$count\n";$|=0;
310 }
311  
312 #----------------------------------
313 # do the same as for Bitext2Text!!
314 # (to check for empty strings ...)
315 #
316 my @SrcNodes=();
317 my @TrgNodes=();
318 my ($srctxt,$trgtxt)=
319 &BitextStrings($data,$param,\@SrcNodes,\@TrgNodes);
320 if (($srctxt!~/\S/) or ($trgtxt!~/\S/)){next;}
321 #----------------------------------
322  
323 # my @SrcNodes=$SrcData->findNodes($TokenLabel);
324 my @SrcIds=$data->attribute(\@SrcNodes,'id');
325 my @SrcSpans=$data->attribute(\@SrcNodes,'span');
326 # my @SrcTokens=$data->content(\@SrcNodes);
327 my @SrcTokens=$data->getSrcTokenFeatures($param,\@SrcNodes);
328  
329 # my @TrgNodes=$TrgData->findNodes($TokenLabel);
330 my @TrgIds=$data->attribute(\@TrgNodes,'id');
331 my @TrgSpans=$data->attribute(\@TrgNodes,'span');
332 # my @TrgTokens=$data->content(\@TrgNodes);
333 my @TrgTokens=$data->getTrgTokenFeatures($param,\@TrgNodes);
334  
335 if ((not @SrcNodes) or (not @TrgNodes)){next;}
336  
337 $_=<F1>;$_=<F1>;chomp; # read source->target viterbi alignment
338 my @src=split(/ /);
339 $_=<F1>;chomp;
340 my %srclinks=();
341 my $count=1;
342 while (/\s(\S.*?)\s\(\{\s(.*?)\}\)/g){ # strunta i NULL!!
343 my @s=split(/\s/,$2);
344 foreach (@s){$srclinks{$_}{$count}=1;}
345 $count++;
346 }
347  
348  
349 $_=<F0>;$_=<F0>;chomp; # read source->target viterbi alignment
350 my @trg=split(/ /);
351 $_=<F0>;chomp;
352 my %trglinks=();
353 my $count=1;
354 while (/\s(\S.*?)\s\(\{\s(.*?)\}\)/g){ # strunta i NULL!!
355 my @t=split(/\s/,$2);
356 foreach (@t){$trglinks{$_}{$count}=1;}
357 $count++;
358 }
359  
360 my (%CombinedSrc,%CombinedTrg);
361 &CombineLinks(\%srclinks,\%trglinks,$combine,\%CombinedSrc,\%CombinedTrg);
362 my @cluster=&LinkClusters(\%CombinedSrc,\%CombinedTrg);
363  
364 foreach my $c (@cluster){
365 # my @s=sort {$a <=> $b} keys %{$cluster[$_]{src}};
366 # my @t=sort {$a <=> $b} keys %{$cluster[$_]{trg}};
367  
368 my @s=@{$$c{src}};
369 my @t=@{$$c{trg}};
370  
371 my @src=();my @trg=();
372 foreach (@s){push (@src,$SrcTokens[$_-1]);}
373 foreach (@t){push (@trg,$TrgTokens[$_-1]);}
374 my @srcId=();my @trgId=();
375 foreach (@s){push (@srcId,$SrcIds[$_-1]);}
376 foreach (@t){push (@trgId,$TrgIds[$_-1]);}
377 my @srcSpan=();my @trgSpan=();
378 foreach (@s){push (@srcSpan,$SrcSpans[$_-1]);}
379 foreach (@t){push (@trgSpan,$TrgSpans[$_-1]);}
380  
381 my %link=();
382 $link{link}=join ' ',@src;
383 $link{link}.=';';
384 $link{link}.=join ' ',@trg;
385 $link{source}=join '+',@srcId;
386 $link{target}=join '+',@trgId;
387 $link{src}=join '&',@srcSpan;
388 $link{trg}=join '&',@trgSpan;
389  
390 $data->addWordLink(\%link);
391 }
392  
393 $output->write($data);
394 }
395 $input->close;
396 $output->close;
397 }
398  
399  
400 sub LinkClusters{
401 my ($src,$trg)=@_;
402 my @cluster=();
403 while (keys %{$src}){
404 my ($s,$links)=each %{$src}; # get the next source token
405 if ((ref($$src{$s}) ne 'HASH') or
406 (not keys %{$$src{$s}})){ # if no links exist:
407 delete $$src{$s}; # delete and next!
408 next;
409 }
410 push (@cluster,{src=>[],trg=>[]}); # create a new link cluster
411 push (@{$cluster[-1]{src}},$s); # and save it in the cluster
412 &AddLinks($cluster[-1],$src,$trg,$s, # add all tokens aligned to the
413 'src','trg'); # source token to the cluster
414 } # (and recursively the ones
415  
416 foreach my $c (@cluster){
417 @{$$c{src}}=sort {$a <=> $b} @{$$c{src}};
418 @{$$c{trg}}=sort {$a <=> $b} @{$$c{trg}};
419 }
420 return @cluster;
421 } # linked to them, see AddLinks)
422  
423 sub AddLinks{
424 my ($cluster,$src,$trg,$s,$key1,$key2)=@_;
425 foreach my $t (keys %{$$src{$s}}){ # add all linked tokens to the
426 delete $$src{$s}{$t}; # cluster and delete the links
427 delete $$trg{$t}{$s}; # in the link-hashs
428 push (@{$$cluster{$key2}},$t);
429 &AddLinks($cluster,$trg,$src,$t,$key2,$key1); # add tokens aligned to the
430 } # linked token to the cluster
431 delete $$src{$s}; # delete the source token link hash
432 }
433  
434  
435  
436 sub CombineLinks{
437 my ($src,$trg,$method,$srclinks,$trglinks)=@_;
438 # my %srclinks;
439 # my %trglinks;
440 if ($method eq 'union'){
441 foreach my $s (keys %{$src}){
442 foreach my $t (keys %{$$src{$s}}){
443 $$srclinks{$s}{$t}=1;
444 $$trglinks{$t}{$s}=1;
445 }
446 }
447 foreach my $t (keys %{$trg}){
448 foreach my $s (keys %{$$trg{$t}}){
449 $$srclinks{$s}{$t}=1;
450 $$trglinks{$t}{$s}=1;
451 }
452 }
453 }
454 elsif ($method eq 'intersection'){
455 foreach my $s (keys %{$src}){
456 foreach my $t (keys %{$$src{$s}}){
457 if (exists $$trg{$t}{$s}){
458 $$srclinks{$s}{$t}=1;
459 $$trglinks{$t}{$s}=1;
460 }
461 }
462 }
463 }
464 if ($method eq 'refined'){ # refined combination:
465 my %links=();
466 foreach my $s (keys %{$src}){ # 1) start with intersection
467 foreach my $t (keys %{$$src{$s}}){
468 if (exists $$trg{$t}{$s}){
469 $$srclinks{$s}{$t}=1;
470 $$trglinks{$t}{$s}=1;
471 }
472 else{
473 $links{"$s:$t"}=1; # keep union of links
474 }
475 }
476 }
477 foreach my $t (keys %{$trg}){
478 foreach my $s (keys %{$$trg{$t}}){
479 if (not exists $$src{$s}{$t}){
480 $links{"$s:$t"}=1;
481 }
482 }
483 }
484 add_unaligned(\%links,$srclinks,$trglinks); # 2) add unaligned pairs
485 add_adjacent(\%links,$srclinks,$trglinks); # 3) add adjacent links
486 }
487 # $src=\%srclinks;
488 # $trg=\%trglinks;
489 }
490  
491  
492 sub is_diagonal{
493 my ($s,$t,$srclinks,$trglinks)=@_;
494 if (defined $$srclinks{$s-1}){
495 return 1 if (defined $$trglinks{$t-1});
496 return 1 if (defined $$trglinks{$t+1});
497 }
498 if (defined $$srclinks{$s+1}){
499 return 1 if (defined $$trglinks{$t-1});
500 return 1 if (defined $$trglinks{$t+1});
501 }
502 return 0;
503 }
504  
505  
506  
507 sub is_adjacent{
508 my ($s,$t,$srclinks,$trglinks)=@_;
509 if (exists $$srclinks{$s}){
510 return 1 if (exists $$srclinks{$s}{$t-1});
511 return 1 if (exists $$srclinks{$s}{$t+1});
512 }
513 elsif (exists $$trglinks{$t}){
514 return 1 if (exists $$trglinks{$t}{$s-1});
515 return 1 if (exists $$trglinks{$t}{$s+1});
516 }
517 return 0;
518 }
519  
520  
521 sub add_adjacent{
522 my $links=shift;
523 my $srclinks=shift;
524 my $trglinks=shift;
525  
526 my $add=1;
527 while (%{$links} && $add){
528 $add=0;
529 foreach my $l (keys %{$links}){
530 my ($l) = each %{$links};
531 my ($s,$t) = split(/:/,$l);
532 next if (exists $$srclinks{$s} && exists $$trglinks{$t});
533 if (is_adjacent($s,$t,$srclinks,$trglinks)){
534 $$srclinks{$s}{$t}=1;
535 $$trglinks{$t}{$s}=1;
536 delete $$links{$l};
537 $add++;
538 }
539 }
540 }
541 }
542  
543  
544  
545 sub is_unaligned{
546 my ($s,$t,$srclinks,$trglinks)=@_;
547 if (not exists $$srclinks{$s}){
548 return 1 if (not exists $$trglinks{$t});
549 }
550 return 0;
551 }
552  
553 sub add_unaligned{
554 my $links=shift;
555 my $srclinks=shift;
556 my $trglinks=shift;
557 foreach my $l (keys %{$links}){
558 my ($s,$t) = split(/:/,$l);
559 if (is_unaligned($s,$t,$srclinks,$trglinks)){
560 $$srclinks{$s}{$t}=1;
561 $$trglinks{$t}{$s}=1;
562 delete $$links{$l};
563 }
564 }
565 }
566  
567  
568  
569  
570 sub add_adjacent_old{
571 my $links=shift;
572 my $srclinks=shift;
573 my $trglinks=shift;
574 foreach my $s (0..$#{$links}){
575 foreach my $t (0..$#{$$links[$s]}){
576 next if (not $$links[$s][$t]);
577 if ((not defined $$srclinks{$s}) and
578 (not defined $$trglinks{$t})){ # - if both are not aligned yet:
579 $$srclinks{$s}{$t}=1; # add the link
580 $$trglinks{$t}{$s}=1;
581 }
582 elsif ((defined $$srclinks{$s-1}) or
583 (defined $$srclinks{$s+1})){
584 if (($$srclinks{$s-1}{$t}) or # if the link is adjacent to
585 ($$srclinks{$s+1}{$t})){ # another one horizontally:
586 if ($$srclinks{$s}{$t+1}){next;} # do not accept if it is also
587 if ($$srclinks{$s}{$t-1}){next;} # adjacent to other links vertically
588 if ($$srclinks{$s-1}{$t}){ # do not accept if the adjacent
589 if ($$srclinks{$s-1}{$t-1}){next;} # link is also adjacent to other
590 if ($$srclinks{$s-1}{$t+1}){next;} # links vertically
591 }
592 if ($$srclinks{$s+1}{$t}){ # the same for the other
593 if ($$srclinks{$s+1}{$t-1}){next;} # adjacency direction
594 if ($$srclinks{$s+1}{$t+1}){next;}
595 }
596 $$srclinks{$s}{$t}=1; # everything ok: add the link
597 $$trglinks{$t}{$s}=1;
598 }
599 }
600 elsif ((defined $$trglinks{$t-1}) or
601 (defined $$trglinks{$t+1})){
602 if (($$srclinks{$s}{$t-1}) or # if the link is adjacent to
603 ($$srclinks{$s}{$t+1})){ # another one vertically:
604 if ($$srclinks{$s+1}{$t}){next;} # do not accept if it is also
605 if ($$srclinks{$s-1}{$t}){next;} # adjacent to other links horizontally
606 if ($$srclinks{$s}{$t-1}){ # do not accept if the adjacent
607 if ($$srclinks{$s-1}{$t-1}){next;} # link is also adjacent to other
608 if ($$srclinks{$s+1}{$t-1}){next;} # links horizontally
609 }
610 if ($$srclinks{$s}{$t+1}){ # the same for the other
611 if ($$srclinks{$s-1}{$t+1}){next;} # adjacency direction
612 if ($$srclinks{$s+1}{$t+1}){next;}
613 }
614 $$srclinks{$s}{$t}=1; # everything ok: add the link
615 $$trglinks{$t}{$s}=1;
616 }
617 }
618 }
619 }
620 }
621  
622  
623 #----------------------------------------------------------------------------
624 # RunGiza: run GIZA++ using external scripts
625 # (GIZA must be installed in the given directory)
626  
627 sub RunGiza{
628 my $src=shift;
629 my $trg=shift;
630 my $viterbi=shift;
631  
632 if (not -d $GIZAHOME){
633 $GIZAHOME="$ENV{UPLUGHOME}/ext/GIZA++";
634 if (not -d $GIZAHOME){$GIZAHOME="$ENV{HOME}/cvs/GIZA++-v2";}
635 if (not -d $GIZAHOME){$GIZAHOME="$ENV{HOME}/cvs/GIZA++";}
636 if (not -d $GIZAHOME){$GIZAHOME="/local/ling/GIZA++-v2";}
637 if (not -d $GIZAHOME){$GIZAHOME="/local/ling/GIZA++";}
638 if (not -d $GIZAHOME){
639 warn "cannot find GIZA++ in $GIZAHOME! Set ENV{UPLUGHOME} or Uplug::Align::Word::Giza::GIZAHOME!";
640 return 0;
641 }
642 }
643 if (my $sig=system "$GIZAHOME/plain2snt.out $src $trg"){
644 die "got signal $? from plain2snt!\n";
645 }
646 my $command="PATH=\$\{PATH\}:$GIZAHOME;";
647 my $snt="$src$trg\.snt";
648 if (not -e $snt){$snt="$src\_$trg\.snt";}
649 if (not -e $snt){die "cannot find alignment-file: $snt!\n";}
650 $command.="$GIZAHOME/trainGIZA++.sh $src\.vcb $trg\.vcb $snt";
651 if (my $sig=system $command){
652 die "got signal $? from trainGIZA++.sh!\n";
653 }
654 if ($viterbi){copy ('GIZA++.A3.final',$viterbi);}
655 else{copy ('GIZA++.A3.final','viterbi');}
656 }
657  
658 #----------------------------------------------------------------------------
659 # Bitext2Text: convert bitexts from Uplug format (XML) to GIZA's format
660 # (this is much too slow ....)
661  
662 sub Bitext2Text{
663 my $bitext=shift;
664 my $srcfile=shift;
665 my $trgfile=shift;
666 my $param=shift;
667  
668 my %SrcStream=('format'=>'text','file'=>$srcfile);
669 my %TrgStream=('format'=>'text','file'=>$trgfile);
670  
671 my $input=Uplug::IO::Any->new($bitext);
672 my $source=Uplug::IO::Any->new(\%SrcStream);
673 my $target=Uplug::IO::Any->new(\%TrgStream);
674 $input->open('read',$bitext)
675 || warn "cannot open the bitext" && return 0;
676 $source->open('write',\%SrcStream)
677 || warn "cannot write to $srcfile" && return 0;
678 $target->open('write',\%TrgStream)
679 || warn "cannot write to $trgfile" && return 0;
680  
681 #-------------------------------------------------------------------------
682  
683 my $data=Uplug::Data::Align->new();
684  
685 print STDERR "convert bitext to plain text!\n";
686 my $count=0;
687 while ($input->read($data)){
688 $count++;
689 if (not ($count % 100)){
690 $|=1;print STDERR '.';$|=0;
691 }
692 if (not ($count % 1000)){
693 $|=1;print STDERR "$count\n";$|=0;
694 }
695  
696 my ($srctxt,$trgtxt)=&BitextStrings($data,$param);
697  
698 if (($srctxt=~/\S/) and ($trgtxt=~/\S/)){
699 $source->write($srctxt);
700 $target->write($trgtxt);
701 }
702 }
703 # $BitextHeader=$input->header;
704 $input->close;
705 $source->close;
706 $target->close;
707 return $input->header;
708 }
709  
710 #----------------------------------------------------------------------------
711 # get the actual strings from the bitext (using feature-parameters)
712 # (feature specifications as in coocfreq.pl)
713  
714 sub BitextStrings{
715 my $data=shift;
716 my $param=shift;
717 my ($srcnodes,$trgnodes)=@_;
718  
719 my @srctok=$data->getSrcTokenFeatures($param,$srcnodes);
720 my @trgtok=$data->getTrgTokenFeatures($param,$trgnodes);
721  
722 map($_=~s/^\s+//sg,@srctok); # delete initial white-space
723 map($_=~s/^\s+//sg,@trgtok);
724 map($_=~s/(\S)\s+$/$1/sg,@srctok); # delete final white-space
725 map($_=~s/(\S)\s+$/$1/sg,@trgtok);
726  
727 map($_=~s/\n/ /sg,@srctok); # otherwise: convert to space
728 map($_=~s/\n/ /sg,@trgtok);
729 map($_=~s/\s/\_/sg,@srctok); # and replace space with underline
730 map($_=~s/\s/\_/sg,@trgtok); # (to avoid extra tokens)
731
732 my $srctxt=join(' ',@srctok);
733 my $trgtxt=join(' ',@trgtok);
734  
735 $srctxt=~tr/\n/ /;
736 $trgtxt=~tr/\n/ /;
737 return ($srctxt,$trgtxt);
738 }
739  
740 #----------------------------------------------------------------------------
741  
742  
743  
744 =pod
745  
746 =head1 Synopsis
747  
748 use lib '/path/to/uplug';
749 use Uplug::Align::Word::Giza;
750  
751 $ENV{UPLUGHOME}='/path/to/uplug';
752  
753  
754 my %bitext = ('file' => 'svenprf.xces',
755 'format' => 'xces align');
756  
757  
758 &Bitext2Text(\%bitext,'src','trg',{}); # convert to plain text
759 &RunGiza('src','trg','viterbi.src-trg'); # run GIZA++ (src-->trg)
760 &RunGiza('trg','src','viterbi.trg-src'); # run GIZA++ (trg-->src)
761  
762 my %dbm = (file=>'clues.dbm',
763 format=>'dbm',
764 key => ['source','target']);
765  
766 &Giza2Clue('.', # directory where GIZA was running (=current)
767 {}, # parameter (= clue dbm header)
768 1, # =1 --> inverse (trg-src)
769 \%dbm); # clue DBM (giza.dbm if not specified)
770  
771 my $combine = 'intersection' # combine heuristics (union|refined)
772 my %out = ('file' => $combined, # save the result in this file
773 'format' => 'xces align');
774  
775 &Combined2Uplug('viterbi.src-trg', # name of first viterbi alignment
776 'viterbi.trg-src', # name of second viterbi alignment
777 $combine, # type of combination heuristics
778 \%bitext, # bitext
779 {}, # token parameters
780 \%out); # output stream
781  
782  
783 =head1 GIZA++ directories
784  
785 You have to tell the program where it can find the GIZA++ executables.
786 You can either set the UPLUGHOME environment variable or the GIZAHOME variable.
787  
788 $ENV{UPLUGHOME}='/path/to/uplug';
789 $Uplug::Align::Word::Giza::GIZAHOME="/path/to/uplug/ext/GIZA++";
790  
791 =cut
792  
793  
794  
795 1;