| 1 | #-*-perl-*- | ||
| 2 | |||
| 3 | package Uplug::Align::Word::Giza; | ||
| 4 | |||
| 5 | |||
| 6 | use strict; | ||
| 7 | |||
| 8 | use Cwd; | ||
| 9 | use Exporter; | ||
| 10 | use File::Copy; | ||
| 11 | |||
| 12 | use Uplug::Data; | ||
| 13 | use Uplug::Data::Align; | ||
| 14 | use Uplug::IO::Any; | ||
| 15 | use Uplug::Config; | ||
| 16 | |||
| 17 | use vars qw(@ISA @EXPORT $DEBUG $GIZAHOME); | ||
| 18 | |||
| 19 | # @ISA = qw( Uplug::Align::Word Exporter ); | ||
| 20 | @ISA = qw( Exporter); | ||
| 21 | $DEBUG = 0; | ||
| 22 | |||
| 23 | @EXPORT = qw( &Bitext2Text &RunGiza &Combined2Uplug &Giza2Clue &Giza2Uplug ); | ||
| 24 | |||
| 25 | #BEGIN{ | ||
| 26 | our $GIZAHOME="$ENV{UPLUGHOME}/ext/GIZA++"; | ||
| 27 | if (not -d $GIZAHOME){$GIZAHOME="$ENV{HOME}/cvs/GIZA++-v2";} | ||
| 28 | if (not -d $GIZAHOME){$GIZAHOME="$ENV{HOME}/cvs/GIZA++";} | ||
| 29 | if (not -d $GIZAHOME){$GIZAHOME="/local/ling/GIZA++-v2";} | ||
| 30 | if (not -d $GIZAHOME){$GIZAHOME="/local/ling/GIZA++";} | ||
| 31 | #} | ||
| 32 | |||
| 33 | # if (not -d $GIZAHOME){warn "cannot find GIZA++!";exit;} | ||
| 34 | |||
| 35 | |||
| 36 | #---------------------------------------------------------------------------- | ||
| 37 | # Giza2Clue (new version): no external calls | ||
| 38 | # - looks for $dir/GIZA++.actual.ti.final (lexical prob's from GIZA) | ||
| 39 | # - creates data/runtime/giza.dbm | ||
| 40 | # - creates data/runtime/giza2.dbm (inverse alignments) | ||
| 41 | |||
| 42 | sub Giza2Clue{ | ||
| 43 | my $file=shift; | ||
| 44 | my $param=shift; | ||
| 45 | my $inverse=shift; | ||
| 46 | my $ClueDB=shift; | ||
| 47 | my $threshold=shift; | ||
| 48 | |||
| 49 | if (-d $file){ # if file is a directory | ||
| 50 | $file="$file/GIZA++.actual.ti.final"; # look for the standard file in | ||
| 51 | } # this directory! | ||
| 52 | |||
| 53 | my %dic; | ||
| 54 | if (ref($ClueDB) eq 'HASH'){ | ||
| 55 | %dic=%{$ClueDB}; | ||
| 56 | } | ||
| 57 | else{ | ||
| 58 | %dic=('format' => 'dbm', | ||
| 59 | 'write_mode' => 'overwrite', | ||
| 60 | 'key' => ['source','target']); | ||
| 61 | my $cluedir='data/runtime'; | ||
| 62 | if ($inverse){$dic{file}="$cluedir/giza2.dbm";} | ||
| 63 | else{$dic{file}="$cluedir/giza.dbm";} | ||
| 64 | } | ||
| 65 | |||
| 66 | my %inStream=('file' => $file, | ||
| 67 | 'format' => 'tab', | ||
| 68 | 'field delimiter' => ' '); | ||
| 69 | if ($inverse){ | ||
| 70 | $inStream{'columns'}=['source','target','value',], | ||
| 71 | } | ||
| 72 | else{ | ||
| 73 | $inStream{'columns'}=['target','source','value',], | ||
| 74 | } | ||
| 75 | |||
| 76 | |||
| 77 | my %lex=(); | ||
| 78 | my $data=Uplug::Data->new; | ||
| 79 | |||
| 80 | my $in=Uplug::IO::Any->new(\%inStream); | ||
| 81 | $in->open('read',\%inStream); | ||
| 82 | my $count=0; | ||
| 83 | while ($in->read($data)){ | ||
| 84 | $count++; | ||
| 85 | if (not ($count % 1000)){print STDERR '.';} | ||
| 86 | if (not ($count % 10000)){print STDERR "$count\n";} | ||
| 87 | my $src=$data->attribute('source'); | ||
| 88 | my $trg=$data->attribute('target'); | ||
| 89 | if ((not $src) or (not $trg)){next;} | ||
| 90 | my $value=$data->attribute('value'); | ||
| 91 | if (not $value){$value=1;} | ||
| 92 | if ((defined $threshold) and ($value<$threshold)){next;} | ||
| 93 | $lex{$src}{$trg}=$value; | ||
| 94 | if (($src=~s/\_/ /gs) or ($trg=~s/\_/ /gs)){ # (for giza-clue:) | ||
| 95 | $lex{$src}{$trg}=$value; # '_' means ' ' | ||
| 96 | } | ||
| 97 | } | ||
| 98 | my $header=$in->header; | ||
| 99 | |||
| 100 | my $out=Uplug::IO::Any->new(\%dic); | ||
| 101 | $out->open('write',\%dic); | ||
| 102 | $out->addheader($header); | ||
| 103 | $out->addheader($param); | ||
| 104 | $out->writeheader(); | ||
| 105 | |||
| 106 | foreach my $s (keys %lex){ | ||
| 107 | my $total; | ||
| 108 | foreach my $t (keys %{$lex{$s}}){ | ||
| 109 | my $score=$lex{$s}{$t}; | ||
| 110 | my $data=Uplug::Data->new; | ||
| 111 | $data->setAttribute('source',$s); | ||
| 112 | $data->setAttribute('target',$t); | ||
| 113 | $data->setAttribute('score',$score); | ||
| 114 | $out->write($data); | ||
| 115 | } | ||
| 116 | } | ||
| 117 | |||
| 118 | $out->close; | ||
| 119 | $in->close; | ||
| 120 | } | ||
| 121 | |||
| 122 | #---------------------------------------------------------------------------- | ||
| 123 | # Giza2Uplug: convert GIZA's Viterbi alignment to Uplug format (XML) | ||
| 124 | # (slow and risky: GIZA's output must be complete and use a certain format) | ||
| 125 | |||
| 126 | sub Giza2Uplug{ | ||
| 127 | my $viterbi=shift; | ||
| 128 | my $bitext=shift; | ||
| 129 | my $param=shift; | ||
| 130 | my $links=shift; | ||
| 131 | my $inverse=shift; | ||
| 132 | |||
| 133 | if (ref($links) ne 'HASH'){return 0;} | ||
| 134 | my $input=Uplug::IO::Any->new($bitext); | ||
| 135 | if (not ref($input)){return 0;} | ||
| 136 | if (not $input->open('read',$bitext)){return 0;} | ||
| 137 | my $BitextHeader=$input->header(); | ||
| 138 | my $output=Uplug::IO::Any->new($links); | ||
| 139 | if (not ref($output)){return 0;} | ||
| 140 | $output->addheader($BitextHeader); | ||
| 141 | if (not $output->open('write',$links)){return 0;} | ||
| 142 | $output->setOption('SkipDataHeader',0); # don't skip headers and | ||
| 143 | $output->setOption('SkipDataTail',0); # footers (e.g. <linkGrp>) | ||
| 144 | |||
| 145 | |||
| 146 | #------------------------------------------------------------------------ | ||
| 147 | if ($viterbi=~/\.gz$/){ | ||
| 148 | open F,"gzip -cd <$viterbi |" || | ||
| 149 | die "cannot open Viterbi alignment file $viterbi!"; | ||
| 150 | } | ||
| 151 | else{ | ||
| 152 | open F,"<$viterbi" || | ||
| 153 | die "cannot open Viterbi alignment file $viterbi!"; | ||
| 154 | } | ||
| 155 | #------------------------------------------------------------------------ | ||
| 156 | |||
| 157 | my $TokenLabel='w'; | ||
| 158 | my $data=Uplug::Data::Align->new(); | ||
| 159 | print STDERR "convert GIZA's Viterbi alignment to XML!\n"; | ||
| 160 | my $count=0; | ||
| 161 | |||
| 162 | while ($input->read($data)){ | ||
| 163 | |||
| 164 | $count++; | ||
| 165 | if (not ($count % 100)){ | ||
| 166 | $|=1;print STDERR '.';$|=0; | ||
| 167 | } | ||
| 168 | if (not ($count % 1000)){ | ||
| 169 | $|=1;print STDERR "$count\n";$|=0; | ||
| 170 | } | ||
| 171 | |||
| 172 | #---------------------------------- | ||
| 173 | # do the same as for Bitext2Text!! | ||
| 174 | # (to check for empty strings ...) | ||
| 175 | # | ||
| 176 | my @SrcNodes=(); | ||
| 177 | my @TrgNodes=(); | ||
| 178 | my ($srctxt,$trgtxt)= | ||
| 179 | &BitextStrings($data,$param,\@SrcNodes,\@TrgNodes); | ||
| 180 | if (($srctxt!~/\S/) or ($trgtxt!~/\S/)){next;} | ||
| 181 | #---------------------------------- | ||
| 182 | |||
| 183 | # my $SrcData=$data->sourceData(); | ||
| 184 | # my $TrgData=$data->targetData(); | ||
| 185 | # | ||
| 186 | # my @SrcNodes=$SrcData->findNodes($TokenLabel); | ||
| 187 | my @SrcIds=$data->attribute(\@SrcNodes,'id'); | ||
| 188 | my @SrcSpans=$data->attribute(\@SrcNodes,'span'); | ||
| 189 | # my @SrcTokens=$data->content(\@SrcNodes); | ||
| 190 | my @SrcTokens=$data->getSrcTokenFeatures($param,\@SrcNodes); | ||
| 191 | |||
| 192 | # my @TrgNodes=$TrgData->findNodes($TokenLabel); | ||
| 193 | my @TrgIds=$data->attribute(\@TrgNodes,'id'); | ||
| 194 | my @TrgSpans=$data->attribute(\@TrgNodes,'span'); | ||
| 195 | # my @TrgTokens=$data->content(\@TrgNodes); | ||
| 196 | my @TrgTokens=$data->getTrgTokenFeatures($param,\@TrgNodes); | ||
| 197 | |||
| 198 | if ((not @SrcNodes) or (not @TrgNodes)){next;} | ||
| 199 | |||
| 200 | $_=<F>; | ||
| 201 | $_=<F>; | ||
| 202 | chomp; | ||
| 203 | my @src=split(/ /); | ||
| 204 | $_=<F>; | ||
| 205 | chomp; | ||
| 206 | |||
| 207 | my %align=(); | ||
| 208 | my $count=1; | ||
| 209 | while (/\s(\S.*?)\s\(\{\s(.*?)\}\)/g){ # strunta i NULL!! | ||
| 210 | if ($2){push (@{$align{$2}},$count);} | ||
| 211 | $count++; | ||
| 212 | } | ||
| 213 | foreach (sort keys %align){ | ||
| 214 | my @s;my @t; | ||
| 215 | if ($inverse){ | ||
| 216 | @t=@{$align{$_}}; | ||
| 217 | @s=split(/\s/); | ||
| 218 | } | ||
| 219 | else{ | ||
| 220 | @s=@{$align{$_}}; | ||
| 221 | @t=split(/\s/); | ||
| 222 | } | ||
| 223 | |||
| 224 | my @src=();my @trg=(); | ||
| 225 | foreach (@s){push (@src,$SrcTokens[$_-1]);} | ||
| 226 | foreach (@t){push (@trg,$TrgTokens[$_-1]);} | ||
| 227 | my @srcId=();my @trgId=(); | ||
| 228 | foreach (@s){push (@srcId,$SrcIds[$_-1]);} | ||
| 229 | foreach (@t){push (@trgId,$TrgIds[$_-1]);} | ||
| 230 | my @srcSpan=();my @trgSpan=(); | ||
| 231 | foreach (@s){push (@srcSpan,$SrcSpans[$_-1]);} | ||
| 232 | foreach (@t){push (@trgSpan,$TrgSpans[$_-1]);} | ||
| 233 | |||
| 234 | my %link=(); | ||
| 235 | $link{link}=join ' ',@src; | ||
| 236 | $link{link}.=';'; | ||
| 237 | $link{link}.=join ' ',@trg; | ||
| 238 | $link{source}=join '+',@srcId; | ||
| 239 | $link{target}=join '+',@trgId; | ||
| 240 | $link{src}=join '&',@srcSpan; | ||
| 241 | $link{trg}=join '&',@trgSpan; | ||
| 242 | |||
| 243 | $data->addWordLink(\%link); | ||
| 244 | } | ||
| 245 | |||
| 246 | $output->write($data); | ||
| 247 | } | ||
| 248 | $input->close; | ||
| 249 | $output->close; | ||
| 250 | } | ||
| 251 | |||
| 252 | |||
| 253 | #---------------------------------------------------------------------------- | ||
| 254 | # Combined2Uplug: combine GIZA's Viterbi alignment and convert them to Uplug format (XML) | ||
| 255 | # (slow and risky: GIZA's output must be complete and must use a certain format) | ||
| 256 | # | ||
| 257 | # possible combinatins: union, intersection, refined | ||
| 258 | # | ||
| 259 | |||
| 260 | sub Combined2Uplug{ | ||
| 261 | my $giza0=shift; | ||
| 262 | my $giza1=shift; | ||
| 263 | my $combine=shift; | ||
| 264 | my $bitext=shift; | ||
| 265 | my $param=shift; | ||
| 266 | my $links=shift; | ||
| 267 | |||
| 268 | if (ref($links) ne 'HASH'){return 0;} | ||
| 269 | my $input=Uplug::IO::Any->new($bitext); | ||
| 270 | if (not ref($input)){return 0;} | ||
| 271 | if (not $input->open('read',$bitext)){return 0;} | ||
| 272 | my $output; # open output later (after reading the first record of | ||
| 273 | # the input stream --> this gives the complete input header) | ||
| 274 | |||
| 275 | #------------------------------------------------------------------------ | ||
| 276 | if ($giza0=~/\.gz$/){open F0,"gzip -cd <$giza0 |";} | ||
| 277 | else{open F0,"<$giza0";} | ||
| 278 | # ($]>=5.008){binmode(F0, ":encoding(utf-8)");} | ||
| 279 | |||
| 280 | if ($giza1=~/\.gz$/){open F1,"gzip -cd <$giza1 |";} | ||
| 281 | else{open F1,"<$giza1";} | ||
| 282 | # ($]>=5.008){binmode(F1, ":encoding(utf-8)");} | ||
| 283 | #------------------------------------------------------------------------ | ||
| 284 | |||
| 285 | my $TokenLabel='w'; | ||
| 286 | my $data=Uplug::Data::Align->new(); | ||
| 287 | print STDERR "combine GIZA's Viterbi alignments and convert to XML!\n"; | ||
| 288 | my $count=0; | ||
| 289 | |||
| 290 | while ($input->read($data)){ | ||
| 291 | |||
| 292 | if (not $output){ # output is not opened yet: | ||
| 293 | my $BitextHeader=$input->header(); # - get the input header | ||
| 294 | $output=Uplug::IO::Any->new($links); # - create an output stream | ||
| 295 | if (not ref($output)){return 0;} # (or die) | ||
| 296 | $output->addheader($BitextHeader); # - add input header | ||
| 297 | if (not $output->open('write',$links)){ # - open the output stream | ||
| 298 | return 0; # (or die) | ||
| 299 | } | ||
| 300 | $output->setOption('SkipDataHeader',0); # don't skip headers and | ||
| 301 | $output->setOption('SkipDataTail',0); # footers (e.g. <linkGrp>) | ||
| 302 | } | ||
| 303 | |||
| 304 | $count++; | ||
| 305 | if (not ($count % 100)){ | ||
| 306 | $|=1;print STDERR '.';$|=0; | ||
| 307 | } | ||
| 308 | if (not ($count % 1000)){ | ||
| 309 | $|=1;print STDERR "$count\n";$|=0; | ||
| 310 | } | ||
| 311 | |||
| 312 | #---------------------------------- | ||
| 313 | # do the same as for Bitext2Text!! | ||
| 314 | # (to check for empty strings ...) | ||
| 315 | # | ||
| 316 | my @SrcNodes=(); | ||
| 317 | my @TrgNodes=(); | ||
| 318 | my ($srctxt,$trgtxt)= | ||
| 319 | &BitextStrings($data,$param,\@SrcNodes,\@TrgNodes); | ||
| 320 | if (($srctxt!~/\S/) or ($trgtxt!~/\S/)){next;} | ||
| 321 | #---------------------------------- | ||
| 322 | |||
| 323 | # my @SrcNodes=$SrcData->findNodes($TokenLabel); | ||
| 324 | my @SrcIds=$data->attribute(\@SrcNodes,'id'); | ||
| 325 | my @SrcSpans=$data->attribute(\@SrcNodes,'span'); | ||
| 326 | # my @SrcTokens=$data->content(\@SrcNodes); | ||
| 327 | my @SrcTokens=$data->getSrcTokenFeatures($param,\@SrcNodes); | ||
| 328 | |||
| 329 | # my @TrgNodes=$TrgData->findNodes($TokenLabel); | ||
| 330 | my @TrgIds=$data->attribute(\@TrgNodes,'id'); | ||
| 331 | my @TrgSpans=$data->attribute(\@TrgNodes,'span'); | ||
| 332 | # my @TrgTokens=$data->content(\@TrgNodes); | ||
| 333 | my @TrgTokens=$data->getTrgTokenFeatures($param,\@TrgNodes); | ||
| 334 | |||
| 335 | if ((not @SrcNodes) or (not @TrgNodes)){next;} | ||
| 336 | |||
| 337 | $_=<F1>;$_=<F1>;chomp; # read source->target viterbi alignment | ||
| 338 | my @src=split(/ /); | ||
| 339 | $_=<F1>;chomp; | ||
| 340 | my %srclinks=(); | ||
| 341 | my $count=1; | ||
| 342 | while (/\s(\S.*?)\s\(\{\s(.*?)\}\)/g){ # strunta i NULL!! | ||
| 343 | my @s=split(/\s/,$2); | ||
| 344 | foreach (@s){$srclinks{$_}{$count}=1;} | ||
| 345 | $count++; | ||
| 346 | } | ||
| 347 | |||
| 348 | |||
| 349 | $_=<F0>;$_=<F0>;chomp; # read source->target viterbi alignment | ||
| 350 | my @trg=split(/ /); | ||
| 351 | $_=<F0>;chomp; | ||
| 352 | my %trglinks=(); | ||
| 353 | my $count=1; | ||
| 354 | while (/\s(\S.*?)\s\(\{\s(.*?)\}\)/g){ # strunta i NULL!! | ||
| 355 | my @t=split(/\s/,$2); | ||
| 356 | foreach (@t){$trglinks{$_}{$count}=1;} | ||
| 357 | $count++; | ||
| 358 | } | ||
| 359 | |||
| 360 | my (%CombinedSrc,%CombinedTrg); | ||
| 361 | &CombineLinks(\%srclinks,\%trglinks,$combine,\%CombinedSrc,\%CombinedTrg); | ||
| 362 | my @cluster=&LinkClusters(\%CombinedSrc,\%CombinedTrg); | ||
| 363 | |||
| 364 | foreach my $c (@cluster){ | ||
| 365 | # my @s=sort {$a <=> $b} keys %{$cluster[$_]{src}}; | ||
| 366 | # my @t=sort {$a <=> $b} keys %{$cluster[$_]{trg}}; | ||
| 367 | |||
| 368 | my @s=@{$$c{src}}; | ||
| 369 | my @t=@{$$c{trg}}; | ||
| 370 | |||
| 371 | my @src=();my @trg=(); | ||
| 372 | foreach (@s){push (@src,$SrcTokens[$_-1]);} | ||
| 373 | foreach (@t){push (@trg,$TrgTokens[$_-1]);} | ||
| 374 | my @srcId=();my @trgId=(); | ||
| 375 | foreach (@s){push (@srcId,$SrcIds[$_-1]);} | ||
| 376 | foreach (@t){push (@trgId,$TrgIds[$_-1]);} | ||
| 377 | my @srcSpan=();my @trgSpan=(); | ||
| 378 | foreach (@s){push (@srcSpan,$SrcSpans[$_-1]);} | ||
| 379 | foreach (@t){push (@trgSpan,$TrgSpans[$_-1]);} | ||
| 380 | |||
| 381 | my %link=(); | ||
| 382 | $link{link}=join ' ',@src; | ||
| 383 | $link{link}.=';'; | ||
| 384 | $link{link}.=join ' ',@trg; | ||
| 385 | $link{source}=join '+',@srcId; | ||
| 386 | $link{target}=join '+',@trgId; | ||
| 387 | $link{src}=join '&',@srcSpan; | ||
| 388 | $link{trg}=join '&',@trgSpan; | ||
| 389 | |||
| 390 | $data->addWordLink(\%link); | ||
| 391 | } | ||
| 392 | |||
| 393 | $output->write($data); | ||
| 394 | } | ||
| 395 | $input->close; | ||
| 396 | $output->close; | ||
| 397 | } | ||
| 398 | |||
| 399 | |||
| 400 | sub LinkClusters{ | ||
| 401 | my ($src,$trg)=@_; | ||
| 402 | my @cluster=(); | ||
| 403 | while (keys %{$src}){ | ||
| 404 | my ($s,$links)=each %{$src}; # get the next source token | ||
| 405 | if ((ref($$src{$s}) ne 'HASH') or | ||
| 406 | (not keys %{$$src{$s}})){ # if no links exist: | ||
| 407 | delete $$src{$s}; # delete and next! | ||
| 408 | next; | ||
| 409 | } | ||
| 410 | push (@cluster,{src=>[],trg=>[]}); # create a new link cluster | ||
| 411 | push (@{$cluster[-1]{src}},$s); # and save it in the cluster | ||
| 412 | &AddLinks($cluster[-1],$src,$trg,$s, # add all tokens aligned to the | ||
| 413 | 'src','trg'); # source token to the cluster | ||
| 414 | } # (and recursively the ones | ||
| 415 | |||
| 416 | foreach my $c (@cluster){ | ||
| 417 | @{$$c{src}}=sort {$a <=> $b} @{$$c{src}}; | ||
| 418 | @{$$c{trg}}=sort {$a <=> $b} @{$$c{trg}}; | ||
| 419 | } | ||
| 420 | return @cluster; | ||
| 421 | } # linked to them, see AddLinks) | ||
| 422 | |||
| 423 | sub AddLinks{ | ||
| 424 | my ($cluster,$src,$trg,$s,$key1,$key2)=@_; | ||
| 425 | foreach my $t (keys %{$$src{$s}}){ # add all linked tokens to the | ||
| 426 | delete $$src{$s}{$t}; # cluster and delete the links | ||
| 427 | delete $$trg{$t}{$s}; # in the link-hashs | ||
| 428 | push (@{$$cluster{$key2}},$t); | ||
| 429 | &AddLinks($cluster,$trg,$src,$t,$key2,$key1); # add tokens aligned to the | ||
| 430 | } # linked token to the cluster | ||
| 431 | delete $$src{$s}; # delete the source token link hash | ||
| 432 | } | ||
| 433 | |||
| 434 | |||
| 435 | |||
| 436 | sub CombineLinks{ | ||
| 437 | my ($src,$trg,$method,$srclinks,$trglinks)=@_; | ||
| 438 | # my %srclinks; | ||
| 439 | # my %trglinks; | ||
| 440 | if ($method eq 'union'){ | ||
| 441 | foreach my $s (keys %{$src}){ | ||
| 442 | foreach my $t (keys %{$$src{$s}}){ | ||
| 443 | $$srclinks{$s}{$t}=1; | ||
| 444 | $$trglinks{$t}{$s}=1; | ||
| 445 | } | ||
| 446 | } | ||
| 447 | foreach my $t (keys %{$trg}){ | ||
| 448 | foreach my $s (keys %{$$trg{$t}}){ | ||
| 449 | $$srclinks{$s}{$t}=1; | ||
| 450 | $$trglinks{$t}{$s}=1; | ||
| 451 | } | ||
| 452 | } | ||
| 453 | } | ||
| 454 | elsif ($method eq 'intersection'){ | ||
| 455 | foreach my $s (keys %{$src}){ | ||
| 456 | foreach my $t (keys %{$$src{$s}}){ | ||
| 457 | if (exists $$trg{$t}{$s}){ | ||
| 458 | $$srclinks{$s}{$t}=1; | ||
| 459 | $$trglinks{$t}{$s}=1; | ||
| 460 | } | ||
| 461 | } | ||
| 462 | } | ||
| 463 | } | ||
| 464 | if ($method eq 'refined'){ # refined combination: | ||
| 465 | my %links=(); | ||
| 466 | foreach my $s (keys %{$src}){ # 1) start with intersection | ||
| 467 | foreach my $t (keys %{$$src{$s}}){ | ||
| 468 | if (exists $$trg{$t}{$s}){ | ||
| 469 | $$srclinks{$s}{$t}=1; | ||
| 470 | $$trglinks{$t}{$s}=1; | ||
| 471 | } | ||
| 472 | else{ | ||
| 473 | $links{"$s:$t"}=1; # keep union of links | ||
| 474 | } | ||
| 475 | } | ||
| 476 | } | ||
| 477 | foreach my $t (keys %{$trg}){ | ||
| 478 | foreach my $s (keys %{$$trg{$t}}){ | ||
| 479 | if (not exists $$src{$s}{$t}){ | ||
| 480 | $links{"$s:$t"}=1; | ||
| 481 | } | ||
| 482 | } | ||
| 483 | } | ||
| 484 | add_unaligned(\%links,$srclinks,$trglinks); # 2) add unaligned pairs | ||
| 485 | add_adjacent(\%links,$srclinks,$trglinks); # 3) add adjacent links | ||
| 486 | } | ||
| 487 | # $src=\%srclinks; | ||
| 488 | # $trg=\%trglinks; | ||
| 489 | } | ||
| 490 | |||
| 491 | |||
| 492 | sub is_diagonal{ | ||
| 493 | my ($s,$t,$srclinks,$trglinks)=@_; | ||
| 494 | if (defined $$srclinks{$s-1}){ | ||
| 495 | return 1 if (defined $$trglinks{$t-1}); | ||
| 496 | return 1 if (defined $$trglinks{$t+1}); | ||
| 497 | } | ||
| 498 | if (defined $$srclinks{$s+1}){ | ||
| 499 | return 1 if (defined $$trglinks{$t-1}); | ||
| 500 | return 1 if (defined $$trglinks{$t+1}); | ||
| 501 | } | ||
| 502 | return 0; | ||
| 503 | } | ||
| 504 | |||
| 505 | |||
| 506 | |||
| 507 | sub is_adjacent{ | ||
| 508 | my ($s,$t,$srclinks,$trglinks)=@_; | ||
| 509 | if (exists $$srclinks{$s}){ | ||
| 510 | return 1 if (exists $$srclinks{$s}{$t-1}); | ||
| 511 | return 1 if (exists $$srclinks{$s}{$t+1}); | ||
| 512 | } | ||
| 513 | elsif (exists $$trglinks{$t}){ | ||
| 514 | return 1 if (exists $$trglinks{$t}{$s-1}); | ||
| 515 | return 1 if (exists $$trglinks{$t}{$s+1}); | ||
| 516 | } | ||
| 517 | return 0; | ||
| 518 | } | ||
| 519 | |||
| 520 | |||
| 521 | sub add_adjacent{ | ||
| 522 | my $links=shift; | ||
| 523 | my $srclinks=shift; | ||
| 524 | my $trglinks=shift; | ||
| 525 | |||
| 526 | my $add=1; | ||
| 527 | while (%{$links} && $add){ | ||
| 528 | $add=0; | ||
| 529 | foreach my $l (keys %{$links}){ | ||
| 530 | my ($l) = each %{$links}; | ||
| 531 | my ($s,$t) = split(/:/,$l); | ||
| 532 | next if (exists $$srclinks{$s} && exists $$trglinks{$t}); | ||
| 533 | if (is_adjacent($s,$t,$srclinks,$trglinks)){ | ||
| 534 | $$srclinks{$s}{$t}=1; | ||
| 535 | $$trglinks{$t}{$s}=1; | ||
| 536 | delete $$links{$l}; | ||
| 537 | $add++; | ||
| 538 | } | ||
| 539 | } | ||
| 540 | } | ||
| 541 | } | ||
| 542 | |||
| 543 | |||
| 544 | |||
| 545 | sub is_unaligned{ | ||
| 546 | my ($s,$t,$srclinks,$trglinks)=@_; | ||
| 547 | if (not exists $$srclinks{$s}){ | ||
| 548 | return 1 if (not exists $$trglinks{$t}); | ||
| 549 | } | ||
| 550 | return 0; | ||
| 551 | } | ||
| 552 | |||
| 553 | sub add_unaligned{ | ||
| 554 | my $links=shift; | ||
| 555 | my $srclinks=shift; | ||
| 556 | my $trglinks=shift; | ||
| 557 | foreach my $l (keys %{$links}){ | ||
| 558 | my ($s,$t) = split(/:/,$l); | ||
| 559 | if (is_unaligned($s,$t,$srclinks,$trglinks)){ | ||
| 560 | $$srclinks{$s}{$t}=1; | ||
| 561 | $$trglinks{$t}{$s}=1; | ||
| 562 | delete $$links{$l}; | ||
| 563 | } | ||
| 564 | } | ||
| 565 | } | ||
| 566 | |||
| 567 | |||
| 568 | |||
| 569 | |||
| 570 | sub add_adjacent_old{ | ||
| 571 | my $links=shift; | ||
| 572 | my $srclinks=shift; | ||
| 573 | my $trglinks=shift; | ||
| 574 | foreach my $s (0..$#{$links}){ | ||
| 575 | foreach my $t (0..$#{$$links[$s]}){ | ||
| 576 | next if (not $$links[$s][$t]); | ||
| 577 | if ((not defined $$srclinks{$s}) and | ||
| 578 | (not defined $$trglinks{$t})){ # - if both are not aligned yet: | ||
| 579 | $$srclinks{$s}{$t}=1; # add the link | ||
| 580 | $$trglinks{$t}{$s}=1; | ||
| 581 | } | ||
| 582 | elsif ((defined $$srclinks{$s-1}) or | ||
| 583 | (defined $$srclinks{$s+1})){ | ||
| 584 | if (($$srclinks{$s-1}{$t}) or # if the link is adjacent to | ||
| 585 | ($$srclinks{$s+1}{$t})){ # another one horizontally: | ||
| 586 | if ($$srclinks{$s}{$t+1}){next;} # do not accept if it is also | ||
| 587 | if ($$srclinks{$s}{$t-1}){next;} # adjacent to other links vertically | ||
| 588 | if ($$srclinks{$s-1}{$t}){ # do not accept if the adjacent | ||
| 589 | if ($$srclinks{$s-1}{$t-1}){next;} # link is also adjacent to other | ||
| 590 | if ($$srclinks{$s-1}{$t+1}){next;} # links vertically | ||
| 591 | } | ||
| 592 | if ($$srclinks{$s+1}{$t}){ # the same for the other | ||
| 593 | if ($$srclinks{$s+1}{$t-1}){next;} # adjacency direction | ||
| 594 | if ($$srclinks{$s+1}{$t+1}){next;} | ||
| 595 | } | ||
| 596 | $$srclinks{$s}{$t}=1; # everything ok: add the link | ||
| 597 | $$trglinks{$t}{$s}=1; | ||
| 598 | } | ||
| 599 | } | ||
| 600 | elsif ((defined $$trglinks{$t-1}) or | ||
| 601 | (defined $$trglinks{$t+1})){ | ||
| 602 | if (($$srclinks{$s}{$t-1}) or # if the link is adjacent to | ||
| 603 | ($$srclinks{$s}{$t+1})){ # another one vertically: | ||
| 604 | if ($$srclinks{$s+1}{$t}){next;} # do not accept if it is also | ||
| 605 | if ($$srclinks{$s-1}{$t}){next;} # adjacent to other links horizontally | ||
| 606 | if ($$srclinks{$s}{$t-1}){ # do not accept if the adjacent | ||
| 607 | if ($$srclinks{$s-1}{$t-1}){next;} # link is also adjacent to other | ||
| 608 | if ($$srclinks{$s+1}{$t-1}){next;} # links horizontally | ||
| 609 | } | ||
| 610 | if ($$srclinks{$s}{$t+1}){ # the same for the other | ||
| 611 | if ($$srclinks{$s-1}{$t+1}){next;} # adjacency direction | ||
| 612 | if ($$srclinks{$s+1}{$t+1}){next;} | ||
| 613 | } | ||
| 614 | $$srclinks{$s}{$t}=1; # everything ok: add the link | ||
| 615 | $$trglinks{$t}{$s}=1; | ||
| 616 | } | ||
| 617 | } | ||
| 618 | } | ||
| 619 | } | ||
| 620 | } | ||
| 621 | |||
| 622 | |||
| 623 | #---------------------------------------------------------------------------- | ||
| 624 | # RunGiza: run GIZA++ using external scripts | ||
| 625 | # (GIZA must be installed in the given directory) | ||
| 626 | |||
| 627 | sub RunGiza{ | ||
| 628 | my $src=shift; | ||
| 629 | my $trg=shift; | ||
| 630 | my $viterbi=shift; | ||
| 631 | |||
| 632 | if (not -d $GIZAHOME){ | ||
| 633 | $GIZAHOME="$ENV{UPLUGHOME}/ext/GIZA++"; | ||
| 634 | if (not -d $GIZAHOME){$GIZAHOME="$ENV{HOME}/cvs/GIZA++-v2";} | ||
| 635 | if (not -d $GIZAHOME){$GIZAHOME="$ENV{HOME}/cvs/GIZA++";} | ||
| 636 | if (not -d $GIZAHOME){$GIZAHOME="/local/ling/GIZA++-v2";} | ||
| 637 | if (not -d $GIZAHOME){$GIZAHOME="/local/ling/GIZA++";} | ||
| 638 | if (not -d $GIZAHOME){ | ||
| 639 | warn "cannot find GIZA++ in $GIZAHOME! Set ENV{UPLUGHOME} or Uplug::Align::Word::Giza::GIZAHOME!"; | ||
| 640 | return 0; | ||
| 641 | } | ||
| 642 | } | ||
| 643 | if (my $sig=system "$GIZAHOME/plain2snt.out $src $trg"){ | ||
| 644 | die "got signal $? from plain2snt!\n"; | ||
| 645 | } | ||
| 646 | my $command="PATH=\$\{PATH\}:$GIZAHOME;"; | ||
| 647 | my $snt="$src$trg\.snt"; | ||
| 648 | if (not -e $snt){$snt="$src\_$trg\.snt";} | ||
| 649 | if (not -e $snt){die "cannot find alignment-file: $snt!\n";} | ||
| 650 | $command.="$GIZAHOME/trainGIZA++.sh $src\.vcb $trg\.vcb $snt"; | ||
| 651 | if (my $sig=system $command){ | ||
| 652 | die "got signal $? from trainGIZA++.sh!\n"; | ||
| 653 | } | ||
| 654 | if ($viterbi){copy ('GIZA++.A3.final',$viterbi);} | ||
| 655 | else{copy ('GIZA++.A3.final','viterbi');} | ||
| 656 | } | ||
| 657 | |||
| 658 | #---------------------------------------------------------------------------- | ||
| 659 | # Bitext2Text: convert bitexts from Uplug format (XML) to GIZA's format | ||
| 660 | # (this is much too slow ....) | ||
| 661 | |||
| 662 | sub Bitext2Text{ | ||
| 663 | my $bitext=shift; | ||
| 664 | my $srcfile=shift; | ||
| 665 | my $trgfile=shift; | ||
| 666 | my $param=shift; | ||
| 667 | |||
| 668 | my %SrcStream=('format'=>'text','file'=>$srcfile); | ||
| 669 | my %TrgStream=('format'=>'text','file'=>$trgfile); | ||
| 670 | |||
| 671 | my $input=Uplug::IO::Any->new($bitext); | ||
| 672 | my $source=Uplug::IO::Any->new(\%SrcStream); | ||
| 673 | my $target=Uplug::IO::Any->new(\%TrgStream); | ||
| 674 | $input->open('read',$bitext) | ||
| 675 | || warn "cannot open the bitext" && return 0; | ||
| 676 | $source->open('write',\%SrcStream) | ||
| 677 | || warn "cannot write to $srcfile" && return 0; | ||
| 678 | $target->open('write',\%TrgStream) | ||
| 679 | || warn "cannot write to $trgfile" && return 0; | ||
| 680 | |||
| 681 | #------------------------------------------------------------------------- | ||
| 682 | |||
| 683 | my $data=Uplug::Data::Align->new(); | ||
| 684 | |||
| 685 | print STDERR "convert bitext to plain text!\n"; | ||
| 686 | my $count=0; | ||
| 687 | while ($input->read($data)){ | ||
| 688 | $count++; | ||
| 689 | if (not ($count % 100)){ | ||
| 690 | $|=1;print STDERR '.';$|=0; | ||
| 691 | } | ||
| 692 | if (not ($count % 1000)){ | ||
| 693 | $|=1;print STDERR "$count\n";$|=0; | ||
| 694 | } | ||
| 695 | |||
| 696 | my ($srctxt,$trgtxt)=&BitextStrings($data,$param); | ||
| 697 | |||
| 698 | if (($srctxt=~/\S/) and ($trgtxt=~/\S/)){ | ||
| 699 | $source->write($srctxt); | ||
| 700 | $target->write($trgtxt); | ||
| 701 | } | ||
| 702 | } | ||
| 703 | # $BitextHeader=$input->header; | ||
| 704 | $input->close; | ||
| 705 | $source->close; | ||
| 706 | $target->close; | ||
| 707 | return $input->header; | ||
| 708 | } | ||
| 709 | |||
| 710 | #---------------------------------------------------------------------------- | ||
| 711 | # get the actual strings from the bitext (using feature-parameters) | ||
| 712 | # (feature specifications as in coocfreq.pl) | ||
| 713 | |||
| 714 | sub BitextStrings{ | ||
| 715 | my $data=shift; | ||
| 716 | my $param=shift; | ||
| 717 | my ($srcnodes,$trgnodes)=@_; | ||
| 718 | |||
| 719 | my @srctok=$data->getSrcTokenFeatures($param,$srcnodes); | ||
| 720 | my @trgtok=$data->getTrgTokenFeatures($param,$trgnodes); | ||
| 721 | |||
| 722 | map($_=~s/^\s+//sg,@srctok); # delete initial white-space | ||
| 723 | map($_=~s/^\s+//sg,@trgtok); | ||
| 724 | map($_=~s/(\S)\s+$/$1/sg,@srctok); # delete final white-space | ||
| 725 | map($_=~s/(\S)\s+$/$1/sg,@trgtok); | ||
| 726 | |||
| 727 | map($_=~s/\n/ /sg,@srctok); # otherwise: convert to space | ||
| 728 | map($_=~s/\n/ /sg,@trgtok); | ||
| 729 | map($_=~s/\s/\_/sg,@srctok); # and replace space with underline | ||
| 730 | map($_=~s/\s/\_/sg,@trgtok); # (to avoid extra tokens) | ||
| 731 | |||
| 732 | my $srctxt=join(' ',@srctok); | ||
| 733 | my $trgtxt=join(' ',@trgtok); | ||
| 734 | |||
| 735 | $srctxt=~tr/\n/ /; | ||
| 736 | $trgtxt=~tr/\n/ /; | ||
| 737 | return ($srctxt,$trgtxt); | ||
| 738 | } | ||
| 739 | |||
| 740 | #---------------------------------------------------------------------------- | ||
| 741 | |||
| 742 | |||
| 743 | |||
| 744 | =pod | ||
| 745 | |||
| 746 | =head1 Synopsis | ||
| 747 | |||
| 748 | use lib '/path/to/uplug'; | ||
| 749 | use Uplug::Align::Word::Giza; | ||
| 750 | |||
| 751 | $ENV{UPLUGHOME}='/path/to/uplug'; | ||
| 752 | |||
| 753 | |||
| 754 | my %bitext = ('file' => 'svenprf.xces', | ||
| 755 | 'format' => 'xces align'); | ||
| 756 | |||
| 757 | |||
| 758 | &Bitext2Text(\%bitext,'src','trg',{}); # convert to plain text | ||
| 759 | &RunGiza('src','trg','viterbi.src-trg'); # run GIZA++ (src-->trg) | ||
| 760 | &RunGiza('trg','src','viterbi.trg-src'); # run GIZA++ (trg-->src) | ||
| 761 | |||
| 762 | my %dbm = (file=>'clues.dbm', | ||
| 763 | format=>'dbm', | ||
| 764 | key => ['source','target']); | ||
| 765 | |||
| 766 | &Giza2Clue('.', # directory where GIZA was running (=current) | ||
| 767 | {}, # parameter (= clue dbm header) | ||
| 768 | 1, # =1 --> inverse (trg-src) | ||
| 769 | \%dbm); # clue DBM (giza.dbm if not specified) | ||
| 770 | |||
| 771 | my $combine = 'intersection' # combine heuristics (union|refined) | ||
| 772 | my %out = ('file' => $combined, # save the result in this file | ||
| 773 | 'format' => 'xces align'); | ||
| 774 | |||
| 775 | &Combined2Uplug('viterbi.src-trg', # name of first viterbi alignment | ||
| 776 | 'viterbi.trg-src', # name of second viterbi alignment | ||
| 777 | $combine, # type of combination heuristics | ||
| 778 | \%bitext, # bitext | ||
| 779 | {}, # token parameters | ||
| 780 | \%out); # output stream | ||
| 781 | |||
| 782 | |||
| 783 | =head1 GIZA++ directories | ||
| 784 | |||
| 785 | You have to tell the program where it can find the GIZA++ executables. | ||
| 786 | You can either set the UPLUGHOME environment variable or the GIZAHOME variable. | ||
| 787 | |||
| 788 | $ENV{UPLUGHOME}='/path/to/uplug'; | ||
| 789 | $Uplug::Align::Word::Giza::GIZAHOME="/path/to/uplug/ext/GIZA++"; | ||
| 790 | |||
| 791 | =cut | ||
| 792 | |||
| 793 | |||
| 794 | |||
| 795 | 1; |