| 1 | #################################################################### | ||
| 2 | # Copyright (C) 2004 Jörg Tiedemann | ||
| 3 | # | ||
| 4 | # This program is free software; you can redistribute it and/or modify | ||
| 5 | # it under the terms of the GNU General Public License as published by | ||
| 6 | # the Free Software Foundation; either version 2 of the License, or | ||
| 7 | # (at your option) any later version. | ||
| 8 | # | ||
| 9 | # This program is distributed in the hope that it will be useful, | ||
| 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 12 | # GNU General Public License for more details. | ||
| 13 | # | ||
| 14 | # You should have received a copy of the GNU General Public License | ||
| 15 | # along with this program; if not, write to the Free Software | ||
| 16 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
| 17 | # | ||
| 18 | # $Author$ | ||
| 19 | # $Id$ | ||
| 20 | # | ||
| 21 | ########################################################################### | ||
| 22 | # Uplug::Data::Align | ||
| 23 | # | ||
| 24 | # | ||
| 25 | # | ||
| 26 | ########################################################################### | ||
| 27 | |||
| 28 | |||
| 29 | package Uplug::Data::Align; | ||
| 30 | |||
| 31 | use strict; | ||
| 32 | use vars qw( @ISA ); | ||
| 33 | |||
| 34 | use Uplug::Data; | ||
| 35 | use Uplug::Data::Lang; | ||
| 36 | |||
| 37 | @ISA = qw( Uplug::Data ); | ||
| 38 | |||
| 39 | sub init{ | ||
| 40 | my $self=shift; | ||
| 41 | my $srclang=shift; | ||
| 42 | my $trglang=shift; | ||
| 43 | |||
| 44 | if (not defined $self->{source}){ | ||
| 45 | $self->{source}=Uplug::Data::Lang->new($srclang); | ||
| 46 | $self->setOption('SRCLANG',$srclang); | ||
| 47 | } | ||
| 48 | elsif ((defined $srclang) and ($self->{SRCLANG} ne $srclang)){ | ||
| 49 | $self->{source}=Uplug::Data::Lang->init($srclang); | ||
| 50 | $self->setOption('SRCLANG',$srclang); | ||
| 51 | } | ||
| 52 | else{$self->{source}->init();} | ||
| 53 | if (not defined $self->{target}){ | ||
| 54 | $self->{target}=Uplug::Data::Lang->new($trglang); | ||
| 55 | $self->setOption('TRGLANG',$trglang); | ||
| 56 | } | ||
| 57 | elsif ((defined $trglang) and ($self->{TRGLANG} ne $trglang)){ | ||
| 58 | $self->{target}=Uplug::Data::Lang->new($trglang); | ||
| 59 | $self->setOption('TRGLANG',$trglang); | ||
| 60 | } | ||
| 61 | else{$self->{target}->init();} | ||
| 62 | # $self->{SRCSUBNEW}=1; # flag for creating new source sub-trees | ||
| 63 | # $self->{TRGSUBNEW}=1; # flag for creating new target sub-trees | ||
| 64 | return $self->SUPER::init(@_); | ||
| 65 | } | ||
| 66 | |||
| 67 | sub clone{return Uplug::Data::Align->new();} | ||
| 68 | |||
| 69 | |||
| 70 | sub makeLangSubData{ | ||
| 71 | my $self=shift; | ||
| 72 | $self->subData($self->{'source'},'source'); | ||
| 73 | $self->subData($self->{'target'},'target'); | ||
| 74 | } | ||
| 75 | |||
| 76 | sub sourceData{return $_[0]->{source};} | ||
| 77 | sub targetData{return $_[0]->{target};} | ||
| 78 | sub linkData{return $_[0]->{link};} | ||
| 79 | |||
| 80 | |||
| 81 | sub getTokens{ | ||
| 82 | my $self=shift; | ||
| 83 | my $lang=shift; | ||
| 84 | my $param=shift; | ||
| 85 | if (not defined $lang){$lang='source'}; | ||
| 86 | if (not ref($self->{$lang})){return undef;} | ||
| 87 | # $self->subData($self->{$lang},$lang); | ||
| 88 | # $self->{$lang}=$self->subData($lang); | ||
| 89 | $self->makeParameter($param,$lang); | ||
| 90 | return $self->{$lang}->getTokens($param,@_); | ||
| 91 | } | ||
| 92 | |||
| 93 | sub getSrcTokens{my $self=shift;return $self->getTokens('source',@_);} | ||
| 94 | sub getTrgTokens{my $self=shift;return $self->getTokens('target',@_);} | ||
| 95 | |||
| 96 | sub getNgrams{ | ||
| 97 | my $self=shift; | ||
| 98 | my $lang=shift; | ||
| 99 | my $param=shift; | ||
| 100 | if (not defined $lang){$lang='source'}; | ||
| 101 | if (not ref($self->{$lang})){return undef;} | ||
| 102 | # $self->subData($self->{$lang},$lang); | ||
| 103 | # $self->{$lang}=$self->subData($lang); | ||
| 104 | $self->makeParameter($param,$lang); | ||
| 105 | return $self->{$lang}->getNgrams($param,@_); | ||
| 106 | } | ||
| 107 | |||
| 108 | sub getSrcNgrams{my $self=shift;return $self->getNgrams('source',@_);} | ||
| 109 | sub getTrgNgrams{my $self=shift;return $self->getNgrams('target',@_);} | ||
| 110 | |||
| 111 | sub getChunks{ | ||
| 112 | my $self=shift; | ||
| 113 | my $lang=shift; | ||
| 114 | my $param=shift; | ||
| 115 | if (not defined $lang){$lang='source'}; | ||
| 116 | if (not ref($self->{$lang})){return undef;} | ||
| 117 | # $self->subData($self->{$lang},$lang); | ||
| 118 | # $self->{$lang}=$self->subData($lang); | ||
| 119 | $self->makeParameter($param,$lang); | ||
| 120 | return $self->{$lang}->getChunks($param,@_); | ||
| 121 | } | ||
| 122 | |||
| 123 | sub getSrcChunks{my $self=shift;return $self->getChunks('source',@_);} | ||
| 124 | sub getTrgChunks{my $self=shift;return $self->getChunks('target',@_);} | ||
| 125 | |||
| 126 | sub getPhrases{ | ||
| 127 | my $self=shift; | ||
| 128 | my $lang=shift; | ||
| 129 | my $param=shift; | ||
| 130 | if (not defined $lang){$lang='source'}; | ||
| 131 | if (not ref($self->{$lang})){return undef;} | ||
| 132 | # $self->subData($self->{$lang},$lang); | ||
| 133 | # $self->{$lang}=$self->subData($lang); | ||
| 134 | $self->makeParameter($param,$lang); | ||
| 135 | return $self->{$lang}->getPhrases($param,@_); | ||
| 136 | } | ||
| 137 | |||
| 138 | sub getSrcPhrases{my $self=shift;return $self->getPhrases('source',@_);} | ||
| 139 | sub getTrgPhrases{my $self=shift;return $self->getPhrases('target',@_);} | ||
| 140 | |||
| 141 | |||
| 142 | sub getPhrasePos{ | ||
| 143 | my $self=shift; | ||
| 144 | my ($phraseNodes,$tokenNodes)=@_; | ||
| 145 | my @idx=(); | ||
| 146 | foreach my $p (0..$#{$phraseNodes}){ | ||
| 147 | my $lastIdx=0; | ||
| 148 | foreach my $t ($lastIdx..$#{$tokenNodes}){ | ||
| 149 | if ($$phraseNodes[$p]==$$tokenNodes[$t]){ | ||
| 150 | push(@idx,$t); | ||
| 151 | $lastIdx=$t+1; | ||
| 152 | } | ||
| 153 | } | ||
| 154 | } | ||
| 155 | return join ":",@idx; | ||
| 156 | } | ||
| 157 | |||
| 158 | |||
| 159 | sub getRelativePosition{ | ||
| 160 | my $self=shift; | ||
| 161 | my ($srcPhr,$trgPhr)=@_; | ||
| 162 | |||
| 163 | my $srcPos=$self->{source}->getPhrasePosition($srcPhr); | ||
| 164 | my $trgPos=$self->{target}->getPhrasePosition($trgPhr); | ||
| 165 | |||
| 166 | if (not defined $srcPos){return 0;} | ||
| 167 | if (not defined $trgPos){return 0;} | ||
| 168 | |||
| 169 | return $trgPos-$srcPos; | ||
| 170 | } | ||
| 171 | |||
| 172 | sub getFeaturePairs{ | ||
| 173 | my $self=shift; | ||
| 174 | return getAlignPhrases(@_); | ||
| 175 | } | ||
| 176 | |||
| 177 | sub getAlignPhrases{ | ||
| 178 | my $self=shift; | ||
| 179 | my ($param,$src,$trg,$token,$attr)=@_; | ||
| 180 | if (ref($param) ne 'HASH'){$param={};} | ||
| 181 | |||
| 182 | #---------------------------------------------------------------------- | ||
| 183 | my @srcTokNodes=(); # 1) get all tokens | ||
| 184 | my @trgTokNodes=(); | ||
| 185 | my @srcTok=$self->getSrcTokens($param,\@srcTokNodes); | ||
| 186 | my @trgTok=$self->getTrgTokens($param,\@trgTokNodes); | ||
| 187 | |||
| 188 | #---------------------------------------------------------------------- | ||
| 189 | my $srcNodes=[]; # 2) get all possible phrases | ||
| 190 | my $trgNodes=[]; | ||
| 191 | my @srcPhr=$self->getSrcPhrases($param,$srcNodes, | ||
| 192 | \@srcTokNodes,\@srcTok); | ||
| 193 | my @trgPhr=$self->getTrgPhrases($param,$trgNodes, | ||
| 194 | \@trgTokNodes,\@trgTok); | ||
| 195 | |||
| 196 | #---------------------------------------------------------------------- | ||
| 197 | my @srcIdx=(); # 3) get token positions for each phrase | ||
| 198 | my @trgIdx=(); | ||
| 199 | foreach (0..$#srcPhr){ | ||
| 200 | push (@srcIdx,$self->getPhrasePos($$srcNodes[$_],\@srcTokNodes)); | ||
| 201 | } | ||
| 202 | foreach (0..$#trgPhr){ | ||
| 203 | push (@trgIdx,$self->getPhrasePos($$trgNodes[$_],\@trgTokNodes)); | ||
| 204 | } | ||
| 205 | |||
| 206 | #---------------------------------------------------------------------- | ||
| 207 | |||
| 208 | $self->makeParameter($param,'source'); # get source feature | ||
| 209 | foreach (0..$#{$srcNodes}){ | ||
| 210 | $$src{$srcIdx[$_]}= | ||
| 211 | $self->{source}->getPhraseFeature(\@{$$srcNodes[$_]}, | ||
| 212 | $param); | ||
| 213 | if (defined $$param{'relative position'}){ | ||
| 214 | my $srcPos=$self->{source}->getPhrasePosition($$srcNodes[$_]); | ||
| 215 | if ($$src{$srcIdx[$_]}=~/\S/){ | ||
| 216 | $$src{$srcIdx[$_]}.=":pos($srcPos)"; | ||
| 217 | } | ||
| 218 | else{ | ||
| 219 | $$src{$srcIdx[$_]}="pos($srcPos)"; | ||
| 220 | } | ||
| 221 | } | ||
| 222 | } | ||
| 223 | $self->makeParameter($param,'target'); # and generate feature | ||
| 224 | foreach (0..$#{$trgNodes}){ | ||
| 225 | $$trg{$trgIdx[$_]}= | ||
| 226 | $self->{target}->getPhraseFeature(\@{$$trgNodes[$_]}, | ||
| 227 | $param); | ||
| 228 | if (defined $$param{'relative position'}){ | ||
| 229 | my $trgPos=$self->{target}->getPhrasePosition($$trgNodes[$_]); | ||
| 230 | if ($$trg{$trgIdx[$_]}=~/\S/){ | ||
| 231 | $$trg{$trgIdx[$_]}.=":pos($trgPos)"; | ||
| 232 | } | ||
| 233 | else{ | ||
| 234 | $$trg{$trgIdx[$_]}="pos($trgPos)"; | ||
| 235 | } | ||
| 236 | } | ||
| 237 | } | ||
| 238 | if (ref($token) eq 'HASH'){ | ||
| 239 | @{$$token{source}}=@srcTok; | ||
| 240 | @{$$token{target}}=@trgTok; | ||
| 241 | |||
| 242 | if (ref($attr) eq 'HASH'){ | ||
| 243 | @{$$attr{source}}=$self->{source}->attribute(\@srcTokNodes); | ||
| 244 | @{$$attr{target}}=$self->{target}->attribute(\@trgTokNodes); | ||
| 245 | foreach (0..$#srcTokNodes){ | ||
| 246 | $$attr{source}[$_]{content}=$self->content($srcTokNodes[$_]); | ||
| 247 | } | ||
| 248 | foreach (0..$#trgTokNodes){ | ||
| 249 | $$attr{target}[$_]{content}=$self->content($trgTokNodes[$_]); | ||
| 250 | } | ||
| 251 | } | ||
| 252 | } | ||
| 253 | } | ||
| 254 | |||
| 255 | sub getSrcTokenFeatures{ | ||
| 256 | my $self=shift; | ||
| 257 | return $self->getTokenFeatures('source',@_); | ||
| 258 | } | ||
| 259 | |||
| 260 | sub getTrgTokenFeatures{ | ||
| 261 | my $self=shift; | ||
| 262 | return $self->getTokenFeatures('target',@_); | ||
| 263 | } | ||
| 264 | |||
| 265 | sub getTokenFeatures{ | ||
| 266 | my $self=shift; | ||
| 267 | my $lang=shift; # source / target | ||
| 268 | my ($param,$nodes)=@_; | ||
| 269 | if (ref($param) ne 'HASH'){$param={};} | ||
| 270 | if (ref($nodes) ne 'ARRAY'){$nodes=[];} | ||
| 271 | if (not ref($self->{$lang})){return undef;} | ||
| 272 | |||
| 273 | #---------------------------------------------------------------------- | ||
| 274 | |||
| 275 | $self->makeParameter($param,$lang); | ||
| 276 | my @tok=$self->{$lang}->getTokens($param,$nodes); | ||
| 277 | |||
| 278 | if (keys %{$param}){ | ||
| 279 | foreach (0..$#{$nodes}){ | ||
| 280 | $tok[$_]=$self->{$lang}->getPhraseFeature([$$nodes[$_]],$param); | ||
| 281 | } | ||
| 282 | } | ||
| 283 | return @tok; | ||
| 284 | } | ||
| 285 | |||
| 286 | |||
| 287 | sub getBitextPhrases{ | ||
| 288 | my $self=shift; | ||
| 289 | my ($param,$src,$trg,$token,$attr)=@_; | ||
| 290 | if (ref($param) ne 'HASH'){$param={};} | ||
| 291 | |||
| 292 | |||
| 293 | #---------------------------------------------------------------------- | ||
| 294 | my @srcTokNodes=(); # 1) get all tokens | ||
| 295 | my @trgTokNodes=(); | ||
| 296 | my @srcTok=$self->getSrcTokens($$param{general},\@srcTokNodes); | ||
| 297 | my @trgTok=$self->getTrgTokens($$param{general},\@trgTokNodes); | ||
| 298 | |||
| 299 | #---------------------------------------------------------------------- | ||
| 300 | my $srcNodes=[]; # 2) get all possible phrases | ||
| 301 | my $trgNodes=[]; | ||
| 302 | my @srcPhr=$self->getSrcPhrases($$param{general},$srcNodes, | ||
| 303 | \@srcTokNodes,\@srcTok); | ||
| 304 | my @trgPhr=$self->getTrgPhrases($$param{general},$trgNodes, | ||
| 305 | \@trgTokNodes,\@trgTok); | ||
| 306 | |||
| 307 | #---------------------------------------------------------------------- | ||
| 308 | my @srcIdx=(); # 3) get token positions for each phrase | ||
| 309 | my @trgIdx=(); | ||
| 310 | foreach (0..$#srcPhr){ | ||
| 311 | push (@srcIdx,$self->getPhrasePos($$srcNodes[$_],\@srcTokNodes)); | ||
| 312 | } | ||
| 313 | foreach (0..$#trgPhr){ | ||
| 314 | push (@trgIdx,$self->getPhrasePos($$trgNodes[$_],\@trgTokNodes)); | ||
| 315 | } | ||
| 316 | |||
| 317 | #---------------------------------------------------------------------- | ||
| 318 | |||
| 319 | foreach my $p (keys %{$param}){ # 4) generate phrase features | ||
| 320 | if ($p eq 'general'){ # a) general = phrase string | ||
| 321 | foreach (0..$#srcPhr){ | ||
| 322 | $$src{$srcIdx[$_]}{$p}=$srcPhr[$_]; # the source phrase | ||
| 323 | } | ||
| 324 | foreach (0..$#trgPhr){ | ||
| 325 | $$trg{$trgIdx[$_]}{$p}=$trgPhr[$_]; # the target phrase | ||
| 326 | } | ||
| 327 | next; | ||
| 328 | } | ||
| 329 | my $srcParam=$$param{$p}; # b) feature parameter | ||
| 330 | $self->makeParameter($srcParam,'source'); # get source feature | ||
| 331 | foreach (0..$#{$srcNodes}){ | ||
| 332 | $$src{$srcIdx[$_]}{$p}= | ||
| 333 | $self->{source}->getPhraseFeature(\@{$$srcNodes[$_]}, | ||
| 334 | $srcParam); | ||
| 335 | if ((ref($$param{$p}) eq 'HASH') and | ||
| 336 | (defined $$param{$p}{'relative position'})){ | ||
| 337 | my $srcPos=$self->{source}->getPhrasePosition($$srcNodes[$_]); | ||
| 338 | if ($$src{$srcIdx[$_]}{$p}=~/\S/){ | ||
| 339 | $$src{$srcIdx[$_]}{$p}.=":pos($srcPos)"; | ||
| 340 | } | ||
| 341 | else{ | ||
| 342 | $$src{$srcIdx[$_]}{$p}="pos($srcPos)"; | ||
| 343 | } | ||
| 344 | } | ||
| 345 | } | ||
| 346 | my $trgParam=$$param{$p}; # get target features | ||
| 347 | $self->makeParameter($trgParam,'target'); # and generate feature | ||
| 348 | foreach (0..$#{$trgNodes}){ | ||
| 349 | $$trg{$trgIdx[$_]}{$p}= | ||
| 350 | $self->{target}->getPhraseFeature(\@{$$trgNodes[$_]}, | ||
| 351 | $trgParam); | ||
| 352 | if ((ref($$param{$p}) eq 'HASH') and | ||
| 353 | (defined $$param{$p}{'relative position'})){ | ||
| 354 | my $trgPos=$self->{target}->getPhrasePosition($$trgNodes[$_]); | ||
| 355 | if ($$trg{$trgIdx[$_]}{$p}=~/\S/){ | ||
| 356 | $$trg{$trgIdx[$_]}{$p}.=":pos($trgPos)"; | ||
| 357 | } | ||
| 358 | else{ | ||
| 359 | $$trg{$trgIdx[$_]}{$p}="pos($trgPos)"; | ||
| 360 | } | ||
| 361 | } | ||
| 362 | } | ||
| 363 | } | ||
| 364 | if (ref($token) eq 'HASH'){ | ||
| 365 | @{$$token{source}}=@srcTok; | ||
| 366 | @{$$token{target}}=@trgTok; | ||
| 367 | |||
| 368 | if (ref($attr) eq 'HASH'){ | ||
| 369 | @{$$attr{source}}=$self->{source}->attribute(\@srcTokNodes); | ||
| 370 | @{$$attr{target}}=$self->{target}->attribute(\@trgTokNodes); | ||
| 371 | foreach (0..$#srcTokNodes){ | ||
| 372 | $$attr{source}[$_]{content}=$self->content($srcTokNodes[$_]); | ||
| 373 | } | ||
| 374 | foreach (0..$#trgTokNodes){ | ||
| 375 | $$attr{target}[$_]{content}=$self->content($trgTokNodes[$_]); | ||
| 376 | } | ||
| 377 | } | ||
| 378 | } | ||
| 379 | } | ||
| 380 | |||
| 381 | |||
| 382 | sub getPhraseFeature{ | ||
| 383 | my $self=shift; | ||
| 384 | my $lang=shift; | ||
| 385 | my $nodes=shift; | ||
| 386 | my $param=shift; | ||
| 387 | if (not defined $lang){$lang='source'}; | ||
| 388 | if (not ref($self->{$lang})){return undef;} | ||
| 389 | # $self->subData($self->{$lang},$lang); | ||
| 390 | # $self->{$lang}=$self->subData($lang); | ||
| 391 | $self->makeParameter($param,$lang); | ||
| 392 | return $self->{$lang}->getPhraseFeature($nodes,$param,@_); | ||
| 393 | } | ||
| 394 | |||
| 395 | sub getSrcPhraseFeature{my $s=shift;return $s->getPhraseFeature('source',@_);} | ||
| 396 | sub getTrgPhraseFeature{my $s=shift;return $s->getPhraseFeature('target',@_);} | ||
| 397 | |||
| 398 | sub checkPairParameter{ | ||
| 399 | my $self=shift; | ||
| 400 | my ($src,$trg,$param)=@_; | ||
| 401 | if ($$param{'minimal length (source)'}){ | ||
| 402 | if (length($src)<$$param{'minimal length (source)'}){ | ||
| 403 | # print STDERR "minimale length (source)\n"; | ||
| 404 | return 0; | ||
| 405 | } | ||
| 406 | } | ||
| 407 | if ($$param{'minimal length (target)'}){ | ||
| 408 | if (length($trg)<$$param{'minimal length (target)'}){ | ||
| 409 | # print STDERR "minimale length (target)\n"; | ||
| 410 | return 0; | ||
| 411 | } | ||
| 412 | } | ||
| 413 | if ($$param{'minimal length diff'}){ | ||
| 414 | if ($self->lengthQuotient($src,$trg)<$$param{'minimal length diff'}){ | ||
| 415 | # print STDERR "minimale length diff\n"; | ||
| 416 | return 0; | ||
| 417 | } | ||
| 418 | } | ||
| 419 | if ($$param{'matching word class'}){ | ||
| 420 | if (not $self->isSameType($src,$trg,$$param{'matching word class'})){ | ||
| 421 | # print STDERR "matching word class\n"; | ||
| 422 | return 0; | ||
| 423 | } | ||
| 424 | } | ||
| 425 | if ($$param{'stop words'}){ | ||
| 426 | if (not $self->isSameType($src,$trg,$$param{'stop words'})){ | ||
| 427 | # print STDERR "stop words\n"; | ||
| 428 | return 0; | ||
| 429 | } | ||
| 430 | } | ||
| 431 | return 1; | ||
| 432 | } | ||
| 433 | |||
| 434 | |||
| 435 | sub isSameType{ | ||
| 436 | my $self=shift; | ||
| 437 | my ($src,$trg,$check)=@_; | ||
| 438 | |||
| 439 | if (($check eq 'open/closed') or ($check eq 'same')){ | ||
| 440 | if ($self->{source}->isStopWord($src)){ | ||
| 441 | return $self->{target}->isStopWord($trg); | ||
| 442 | } | ||
| 443 | return (not $self->{target}->isStopWord($trg)); | ||
| 444 | } | ||
| 445 | elsif ($check eq 'exclude'){ | ||
| 446 | if (not $self->{source}->isStopWord($src)){ | ||
| 447 | return 1; | ||
| 448 | } | ||
| 449 | return (not $self->{target}->isStopWord($trg)); | ||
| 450 | } | ||
| 451 | elsif(($check eq 'same_class') or ($check eq 'wordclass')){ | ||
| 452 | return $self->isSameClass($src,$trg); | ||
| 453 | } | ||
| 454 | elsif(($check eq 'same_sub_class') or ($check eq 'subclass')){ | ||
| 455 | return $self->isSameSubClass($src,$trg); | ||
| 456 | } | ||
| 457 | return 1; | ||
| 458 | } | ||
| 459 | |||
| 460 | |||
| 461 | sub isSameClass{ | ||
| 462 | my $self=shift; | ||
| 463 | my ($src,$trg)=@_; | ||
| 464 | |||
| 465 | my $cat='stop word class hash'; | ||
| 466 | my $SrcData=$self->{source}->getLanguageData($cat); | ||
| 467 | my $TrgData=$self->{target}->getLanguageData($cat); | ||
| 468 | |||
| 469 | if (ref($SrcData) ne 'HASH'){return 1;} | ||
| 470 | if (ref($TrgData) ne 'HASH'){return 1;} | ||
| 471 | |||
| 472 | foreach my $c (%{$SrcData}){ | ||
| 473 | if (defined $$SrcData{$c}{$src}){ | ||
| 474 | if (defined $$TrgData{$c}){ | ||
| 475 | if (defined $$TrgData{$c}{$trg}){ | ||
| 476 | return 1; | ||
| 477 | } | ||
| 478 | } | ||
| 479 | return 0; | ||
| 480 | } | ||
| 481 | |||
| 482 | } | ||
| 483 | foreach my $c (%{$TrgData}){ | ||
| 484 | if (defined $$TrgData{$c}{$trg}){ | ||
| 485 | return 0; | ||
| 486 | } | ||
| 487 | |||
| 488 | } | ||
| 489 | return 1; | ||
| 490 | } | ||
| 491 | |||
| 492 | sub isSameSubClass{ | ||
| 493 | my $self=shift; | ||
| 494 | my ($src,$trg)=@_; | ||
| 495 | |||
| 496 | my $cat='stop word subclass hash'; | ||
| 497 | my $SrcData=$self->{source}->getLanguageData($cat); | ||
| 498 | my $TrgData=$self->{target}->getLanguageData($cat); | ||
| 499 | |||
| 500 | if (ref($SrcData) ne 'HASH'){return 1;} | ||
| 501 | if (ref($TrgData) ne 'HASH'){return 1;} | ||
| 502 | |||
| 503 | foreach my $x (%{$SrcData}){ | ||
| 504 | foreach my $y (%{$$SrcData{$x}}){ | ||
| 505 | if (defined $$SrcData{$x}{$y}{$src}){ | ||
| 506 | if (defined $$TrgData{$x}){ | ||
| 507 | if (defined $$TrgData{$x}{$y}){ | ||
| 508 | if (defined $$TrgData{$x}{$y}{$trg}){ | ||
| 509 | return 1; | ||
| 510 | } | ||
| 511 | } | ||
| 512 | } | ||
| 513 | return 0; | ||
| 514 | } | ||
| 515 | } | ||
| 516 | } | ||
| 517 | foreach my $x (keys %{$TrgData}){ | ||
| 518 | foreach my $y (keys %{$$TrgData{$x}}){ | ||
| 519 | if (defined $$TrgData{$x}{$y}{$trg}){ | ||
| 520 | return 0; | ||
| 521 | } | ||
| 522 | } | ||
| 523 | } | ||
| 524 | return 1; | ||
| 525 | } | ||
| 526 | |||
| 527 | |||
| 528 | |||
| 529 | sub lengthQuotient{ | ||
| 530 | my $self=shift; | ||
| 531 | my ($src,$trg)=@_; | ||
| 532 | if (length($src)==0 or length($trg)==0) {return 0;} | ||
| 533 | if (length($src)>length($trg)) {return length($trg)/length($src);} | ||
| 534 | else {return length($src)/length($trg);} | ||
| 535 | } | ||
| 536 | |||
| 537 | |||
| 538 | |||
| 539 | sub makeParameter{ | ||
| 540 | my $self=shift; | ||
| 541 | my ($param,$lang)=@_; | ||
| 542 | if (ref($param) ne 'HASH'){return;} | ||
| 543 | foreach (keys %{$param}){ | ||
| 544 | if (/^(.*) \($lang\)/){ | ||
| 545 | $param->{$1}=$param->{$_}; | ||
| 546 | } | ||
| 547 | } | ||
| 548 | } | ||
| 549 | |||
| 550 | |||
| 551 | |||
| 552 | |||
| 553 | sub rmLinkedToken{ | ||
| 554 | my $data=shift; | ||
| 555 | |||
| 556 | # my $srcData=Uplug::Data::Lang->new; | ||
| 557 | # my $trgData=Uplug::Data::Lang->new; | ||
| 558 | # $data->subData($srcData,'source'); | ||
| 559 | # $data->subData($trgData,'target'); | ||
| 560 | ## $data->{'source'}=$data->subData('source'); | ||
| 561 | ## $data->{'target'}=$data->subData('target'); | ||
| 562 | |||
| 563 | my $srcData=$data->{source}; | ||
| 564 | my $trgData=$data->{target}; | ||
| 565 | |||
| 566 | my $link=$data->{link}; | ||
| 567 | my @nodes=$link->findNodes('wordLink'); | ||
| 568 | my @xtrg=$link->attribute(\@nodes,'xtargets'); | ||
| 569 | |||
| 570 | foreach my $l (@xtrg){ | ||
| 571 | if ($l=~/^(.*\S)\s?\;\s?(\S.*)$/){ | ||
| 572 | my ($s,$t)=($1,$2); | ||
| 573 | $data->rmToken($s,$srcData); | ||
| 574 | $data->rmToken($t,$trgData); | ||
| 575 | } | ||
| 576 | } | ||
| 577 | } | ||
| 578 | |||
| 579 | sub rmToken{ | ||
| 580 | my $self=shift; | ||
| 581 | my ($span,$data)=@_; | ||
| 582 | my @token=split(/[\+\s]/,$span); | ||
| 583 | foreach (@token){ | ||
| 584 | my ($node)=$data->findNodes('.*',{id => $_}); | ||
| 585 | if (defined $node){ | ||
| 586 | $node->getParentNode->removeChild($node); | ||
| 587 | $node->dispose(); | ||
| 588 | } | ||
| 589 | } | ||
| 590 | } | ||
| 591 | |||
| 592 | sub rmWordLinks{ | ||
| 593 | my $data=shift; | ||
| 594 | # $data->{link}->delAttribute('wordLink'); | ||
| 595 | if (ref($data->{link})){ | ||
| 596 | $data->{link}->delNodes('wordLink'); | ||
| 597 | } | ||
| 598 | } | ||
| 599 | |||
| 600 | sub findLink{ | ||
| 601 | my $self=shift; | ||
| 602 | my $link=shift; | ||
| 603 | my %attr=(); | ||
| 604 | $attr{src}=$link->{source}; | ||
| 605 | $attr{trg}=$link->{target}; | ||
| 606 | my @nodes=$self->{link}->findNodes('wordLink',\%attr); | ||
| 607 | if (@nodes){ | ||
| 608 | return @nodes; | ||
| 609 | } | ||
| 610 | return undef; | ||
| 611 | } | ||
| 612 | |||
| 613 | |||
| 614 | sub addWordLink{ | ||
| 615 | my $data=shift; | ||
| 616 | my $link=shift; | ||
| 617 | my $OutData=$data->{link}; | ||
| 618 | |||
| 619 | |||
| 620 | if (defined $data->findLink($link)){return;} | ||
| 621 | |||
| 622 | my %attr=(); | ||
| 623 | |||
| 624 | if (defined $link->{score}){ | ||
| 625 | $attr{certainty}=$link->{score}; | ||
| 626 | } | ||
| 627 | $attr{lexPair}=$link->{link}; | ||
| 628 | $attr{xtargets}="$link->{source};$link->{target}"; | ||
| 629 | $attr{xtargets}=~tr/:/+/; | ||
| 630 | |||
| 631 | if ($link->{src} and $link->{trg}){ | ||
| 632 | $link->{src}=~tr/\&/\+/; | ||
| 633 | $link->{trg}=~tr/\&/\+/; | ||
| 634 | $attr{span}="$link->{src};$link->{trg}" | ||
| 635 | } | ||
| 636 | my $wordLink=$OutData->createNode('wordLink',\%attr); | ||
| 637 | $OutData->addNode($wordLink); | ||
| 638 | |||
| 639 | # if (defined $link->{step}){$attr{step}=$link->{step};} | ||
| 640 | # $attr{'id'}=$id; | ||
| 641 | # $attr{'content'}="\n$src:$trg\n"; | ||
| 642 | |||
| 643 | } | ||
| 644 | |||
| 645 | |||
| 646 | sub toHTML{ | ||
| 647 | my $self=shift; | ||
| 648 | my $html=$self->{source}->toHTML(); | ||
| 649 | $html.=$self->{target}->toHTML(); | ||
| 650 | } |