blib/lib/Uplug/Data/Lang.pm untested

filter: Statementstmt 0.0% 0 / 675 Branchbran 0.0% 0 / 292 Conditioncond 0.0% 0 / 30 MC/DCmcdc 0.0% 0 / 20 Subroutinesub 0.0% 0 / 52 Podpod 0.0% 0 / 48 totaltotal 0.0% 0 / 1117 CC 147 SCAR 41.5
CC
147
Cov
0%
getNgrams
182.0
getPhraseFeature
182.0
initLanguageData
132.0
CRAP 63.4
1 #####################################################################
2 # Copyright (C) 2004 Jörg Tiedemann <joerg@stp.ling.uu.se>
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software
16 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 #
18 # $Author$
19 # $Id$
20 #
21 ###########################################################################
22 #
23 #
24 #
25 #
26 ###########################################################################
27  
28  
29 package Uplug::Data::Lang;
30  
31 use strict;
32 use vars qw( @ISA
33 $DEFAULTLANGUAGE $DEFAULTLANGUAGEFILE
34 $DEFAULTSPLITPATTERN $DEFAULTDELIMITER );
35  
36 use Uplug::Data;
37 use Uplug::Config;
38  
39 #----------------------
40 # Uplug::Data::Lang is derived from
41 # either Uplug::Data::XML
42 # either Uplug::Data::Tree
43 # or Uplug::Data::DOM (otherwise)
44 # depending on $Uplug::Data::DEFAULTDATATYPE
45 #----------------------
46  
47 @ISA=qw( Uplug::Data );
48  
49 $DEFAULTLANGUAGE = 'default';
50 $DEFAULTLANGUAGEFILE = 'default.ini';
51 $DEFAULTSPLITPATTERN = '\s+';
52 $DEFAULTDELIMITER=$Uplug::Data::DEFAULTDELIMITER;
53  
54  
55  
56 sub init{
57 my $self=shift;
58 my $language=shift;
59 if ((defined $language) or (not ref($self->{LanguageData}))){
60 $self->{LanguageData}={};
61 if ((defined $language) and ($self->{language} ne $language)){
62 $self->loadLanguageFile($language.'.ini');
63 }
64 # if (not $language){$language=$DEFAULTLANGUAGE;}
65 # if ($self->{language} ne $language){
66 # $self->{language}=$language;
67 # $self->loadLanguageFile($language.'.ini');
68 # }
69 }
70 return $self->SUPER::init(@_);
71 }
72  
73 sub clone{return Uplug::Data::Lang->new();}
74  
75 sub loadLanguageFile{
76 my $self=shift;
77 my $file=shift;
78 my $lang=$self->getLanguage();
79 $self->{LanguageData}=&ReadConfig($file);
80 # &LoadIniData($self->{LanguageData},$file);
81 if (defined $self->{LanguageData}->{$lang}){ # we don't need
82 $self->{LanguageData}=$self->{LanguageData}->{$lang}; # a root node!
83 }
84 return $self->initLanguageData();
85 }
86  
87 sub setLanguage{
88 my $self=shift;
89 my $lang=shift;
90 if ($lang ne $self->getLanguage()){
91 $self->{language}=$lang;
92 return $self->loadLanguageFile($lang.'.ini');
93 }
94 return 1;
95 }
96  
97 sub getLanguage{return $_[0]->{language};}
98 sub language{return $_[0]->getLanguage();}
99  
100 sub getLanguageData{
101 my $self=shift;
102 my ($cat,$subcat,$attr)=@_;
103 my $data=$self->{LanguageData};
104 if ((defined $cat) and (defined $data->{$cat})){
105 if ((defined $subcat) and (defined $data->{$cat}->{$subcat})){
106 if ((defined $attr) and
107 (defined $data->{$cat}->{$subcat}->{$attr})){
108 return $data->{$cat}->{$subcat}->{$attr};
109 }
110 elsif (defined $attr){return undef;}
111 return $data->{$cat}->{$subcat};
112 }
113 elsif (defined $subcat){return undef;}
114 return $data->{$cat};
115 }
116 elsif (defined $cat){return undef;}
117 return $data;
118 }
119 sub languageData{return $_[0]->getLanguageData(@_);}
120  
121  
122  
123 #----------------------------------------------------------------------------
124 # initLanguageData:
125 #
126 # initialize language specific data hashs!
127 #
128  
129  
130 sub initLanguageData{
131 my $self=shift;
132  
133 #----------------------------------------------------------------------------
134 # make skip token hashs (from skip token arrays)
135 #----------------------------------------------------------------------------
136  
137 my @skip=('skip phrase before',
138 'skip phrase after',
139 'skip phrase at',
140 'skip token',
141 'non-phrase-starter',
142 'non-phrase-ender');
143  
144 my $data=$self->getLanguageData();
145  
146 if (defined $$data{'phrases'}){
147 foreach (@skip){
148 if (defined $$data{'phrases'}{$_}){
149 if (ref($$data{'phrases'}{$_}) eq 'ARRAY'){
150 my @SkipWords=@{$$data{'phrases'}{$_}};
151 %{$$data{'phrases'}{"$_ hash"}}=();
152 &ArrayToHash(\@SkipWords,$$data{'phrases'}{"$_ hash"});
153 }
154 }
155 }
156 }
157  
158 #----------------------------------------------------------------------------
159 # make skip token string type arrays
160 #----------------------------------------------------------------------------
161  
162 if (defined $$data{'phrases'}){
163 foreach (@skip){
164 if (defined $$data{'phrases'}{"$_ string type"}){
165 if (ref($$data{'phrases'}{"$_ string type"}) ne 'ARRAY'){
166 my @arr=$$data{'phrases'}{"$_ string type"};
167 delete $$data{'phrases'}{"$_ string type"};
168 @{$$data{'phrases'}{"$_ string type"}}=@arr;
169 }
170 }
171  
172 }
173 }
174  
175 #----------------------------------------------------------------------------
176 # make stop word hashs (from stop word arrays)
177 #----------------------------------------------------------------------------
178  
179 if (defined $$data{'stop words'}){
180 if (defined $$data{'stop words'}{'wordform'}){
181 my @words=@{$$data{'stop words'}{'wordform'}};
182 &ArrayToHash(\@words,$$data{'stop word hash'},$data);
183 }
184 if (defined $$data{'stop words'}{'classes'}){
185 my ($cat,$subcat);
186 foreach $cat (@{$$data{'stop words'}{'classes'}}){
187 if (defined $$data{$cat}){
188 foreach $subcat (keys %{$$data{$cat}}){
189 foreach (@{$$data{$cat}{$subcat}}){
190 $$data{'stop word hash'}{$_}=1;
191 $$data{'stop word class hash'}{$cat}{$_}=1;
192 $$data{'stop word subclass hash'}{$cat}{$subcat}{$_}=1;
193 }
194 }
195 }
196 }
197 }
198 }
199  
200 #----------------------------------------------------------------------------
201 # define some special character classes
202 #----------------------------------------------------------------------------
203  
204 my $cat='character specifications';
205  
206 $$data{$cat}{'alphabetic'}=$$data{$cat}{'letter'}.$$data{$cat}{'hyphen'};
207 $$data{$cat}{'alphanumeric'}=
208 $$data{$cat}{'letter'}.
209 $$data{$cat}{'hyphen'}.
210 $$data{$cat}{'digit'}.
211 $$data{$cat}{'numeric symbol'};
212 $$data{$cat}{'numeric'}=
213 $$data{$cat}{'digit'}.$$data{$cat}{'numeric symbol'};
214  
215 #------- compatibility with old spelling errors ....... -------------------
216  
217 $$data{$cat}{'alpha'}=$$data{$cat}{'alphabetic'};
218 $$data{$cat}{'vowels'}=$$data{$cat}{'vowel'};
219 $$data{$cat}{'consonants'}=$$data{$cat}{'consonant'};
220 $$data{$cat}{'vocals'}=$$data{$cat}{'vowel'};
221 }
222  
223  
224 #####################################################################
225 # check stop word classes
226 #####################################################################
227  
228  
229  
230 sub isStopWord{
231 my $self=shift;
232 my ($str)=@_;
233 my $cat='stop word hash';
234 my $data=$self->getLanguageData($cat);
235 if (ref($data) eq 'HASH'){
236 if (defined $$data{$str}){
237 return 1;
238 }
239 }
240 return 0;
241 }
242  
243  
244 #####################################################################
245 # string processing functions
246 #####################################################################
247  
248  
249 # split into VC sequences ....
250  
251 sub splitIntoVC{
252 my $self=shift;
253 my $str=shift;
254  
255 my $cat='character specifications';
256 my $Vowel=$self->getLanguageData($cat,'vowel');
257 my $Consonant=$self->getLanguageData($cat,'consonant');
258 if (not $Vowel){return ($str);}
259 if (not $Consonant){return ($str);}
260  
261 my @arr=();
262 while ($str ne ''){
263 if ($str=~/^([$Vowel]+)([^$Vowel].*)$/){
264 push (@arr,$1);
265 $str=$2;
266 }
267 elsif ($str=~/^([$Consonant]+)([^$Consonant].*)$/){
268 push (@arr,$1);
269 $str=$2;
270 }
271 elsif ($str=~/^([^$Vowel$Consonant]+)(.*)$/){
272 push (@arr,$1);
273 $str=$2;
274 }
275 else{
276 push @arr,$str;
277 $str='';
278 }
279 }
280 return @arr;
281 }
282  
283  
284  
285 #-----------------------------------------------------------------------------
286 # getTokens
287 #
288 # get all tokens which match the parameters in %{$param}
289 #
290  
291 sub getTokens{
292 my $self=shift;
293 my $param=shift;
294 my $accepted=shift;
295  
296 if (ref($accepted) ne 'ARRAY'){
297 $accepted=[];
298 }
299 my @nodes;
300 my @tokens=();
301  
302 if (not defined $$param{'token label'}){
303 @nodes=$self->contentElements();
304 }
305 else{
306 @nodes=$self->findNodes($$param{'token label'});
307 }
308 my @accepted=();
309 foreach my $n (@nodes){
310 if (defined (my $t=$self->checkTokenParameter($n,$param))){
311 push(@{$accepted},$n);
312 push(@tokens,$t);
313 }
314 }
315 return @tokens;
316 }
317  
318  
319 #---------------------------------------------------------------
320 # GetChunks
321 #
322 # get all chunks from data which have been labeled with $label
323 #
324  
325  
326 sub getChunks{
327 my $self=shift;
328 my $param=shift;
329 my ($phraseNodes,$tokenNodes,$tokens,$del)=@_;
330  
331 my @phrases=();
332 if (not defined $del){$del=$DEFAULTDELIMITER;}
333 if (not ref($tokenNodes)){
334 $tokens=[];
335 $tokenNodes=[];
336 @{$tokens}=$self->getTokens($param,$tokenNodes);
337 }
338  
339 #----------------------------------------------------------------------
340 # check certain markup for additional phrases
341 # (chunks etc.)
342  
343 if (defined $$param{'chunks'}){
344 my @chunks=$self->findNodes($$param{'chunks'});
345 foreach my $c (@chunks){
346 if (not ref($c)){next;}
347 my @nodes;
348 if (defined $$param{'token label'}){
349 # @nodes=$self->findNodes($$param{'token label'},undef,$c);
350 @nodes=$c->getElementsByTagName($$param{'token label'});
351 }
352 else{@nodes=$c->getElementsByTagName('*');}
353 # else{@nodes=$self->findNodes('*',undef,$c);}
354  
355 my @chunk=(); # find accepted tokens
356 my @chunkNodes=(); # in the chunk
357 foreach my $x (@nodes){
358 foreach my $y (0..$#{$tokenNodes}){
359 if ($x==$$tokenNodes[$y]){
360 push(@chunkNodes,$$tokenNodes[$y]);
361 if (ref($tokens) eq 'ARRAY'){
362 push(@chunk,$$tokens[$y]);
363 }
364 }
365 }
366 }
367  
368 if (@chunkNodes){
369 if (ref($phraseNodes)){
370 my $idx=$#{$phraseNodes}+1;
371 @{$$phraseNodes[$idx]}=@chunkNodes;
372 }
373 if (@chunk){
374 push (@phrases,join $del,@chunk);
375 }
376 else{
377 my $idx=$#phrases+1;
378 push (@phrases,'#chunk'.$idx);
379 }
380 # $phrases[$idx]="chunk:".$phrases[$idx]
381 }
382 }
383 }
384 return @phrases;
385 }
386  
387  
388 #---------------------------------------------------------------
389 # GetNgrams
390 #
391 # get all ngrams from data
392 #
393  
394  
395 sub getNgrams{
396 my $self=shift;
397 my $param=shift;
398 my ($phraseNodes,$tokenNodes,$tokens,$del)=@_;
399  
400 my $minTokenLength=1;
401 my $maxNgramLength=1;
402 if (defined $$param{'minimal length'}){
403 $minTokenLength=$$param{'minimal length'};
404 }
405 if (defined $$param{'maximal ngram length'}){
406 $maxNgramLength=$$param{'maximal ngram length'};
407 }
408 if ($maxNgramLength<2){return;}
409  
410 #----------------------------------------------------------------------
411 # 1) get tokens if necessary
412 #----------------------------------------------------------------------
413  
414 my @phrases=();
415 if (not defined $del){$del=$DEFAULTDELIMITER;}
416 if (not ref($tokenNodes)){
417 $tokens=[];
418 $tokenNodes=[];
419 @{$tokens}=$self->getTokens($param,$tokenNodes);
420 }
421 if (not ref($tokens)){$tokens=[];}
422 foreach (0..$#{$tokenNodes}){
423 if (not defined $$tokens[$_]){
424 $$tokens[$_]=$self->content($$tokenNodes[$_]);
425 }
426 }
427  
428 #----------------------------------------------------------------------
429 # 2) compute all N-grams (N>1)
430 #----------------------------------------------------------------------
431  
432 my @Ngram=();
433 my @NgramNodes=();
434 my @words=@{$tokens}; # save tokens in a new array
435 my @nodes=@{$tokenNodes}; # and corresponding nodes, too
436 push (@Ngram,shift(@words)); # (because we're shifting all
437 push (@NgramNodes,shift(@nodes)); # tokens from the array)
438  
439 while (@words){
440  
441 my $t=shift(@words);
442 my $n=shift(@nodes);
443  
444 # print STDERR "length: $minTokenLength\n";
445 # if (length($t)<$minTokenLength){
446 # print STDERR "skip $t\n";
447 # next;
448 # }
449 # if ((length($t)<$minTokenLength) or ($self->skipToken($t))){
450  
451 if ($self->skipToken($t)){
452 $self->addNgrams(\@Ngram,\@NgramNodes,\@phrases,$phraseNodes,$del);
453 # print STDERR "skip $t\n";
454 @Ngram=();
455 @NgramNodes=();
456 next;
457 }
458 elsif ($self->skipPhraseAt($t)){
459 $self->addNgrams(\@Ngram,\@NgramNodes,\@phrases,$phraseNodes,$del);
460 # print STDERR "skip at $t\n";
461 @Ngram=();
462 @NgramNodes=();
463 next;
464 }
465 elsif ($self->skipPhraseBefore($t)){
466 $self->addNgrams(\@Ngram,\@NgramNodes,\@phrases,$phraseNodes,$del);
467 @Ngram=();
468 @NgramNodes=();
469 # print STDERR "skip before $t\n";
470 }
471  
472 push (@Ngram,$t);
473 push (@NgramNodes,$n);
474  
475 if (scalar @Ngram == $maxNgramLength){
476 $self->addNgrams(\@Ngram,\@NgramNodes,\@phrases,$phraseNodes,$del);
477 shift (@Ngram);
478 shift (@NgramNodes);
479 # print STDERR "length = max\n";
480 }
481 if ($self->skipPhraseAfter($t)){
482 $self->addNgrams(\@Ngram,\@NgramNodes,\@phrases,$phraseNodes,$del);
483 @Ngram=();
484 @NgramNodes=();
485 # print STDERR "skip after $t\n";
486 }
487 }
488 $self->addNgrams(\@Ngram,\@NgramNodes,\@phrases,$phraseNodes,$del);
489 return @phrases;
490 }
491  
492  
493 #-----------------------------------------------------------
494 # $OBJ->getPhrases($param,\@phraseNodes)
495 #
496 # get phrase candidates
497 # @phraseNodes: list of lists of data-nodes
498 #
499 #
500  
501  
502 sub getPhrases{
503 my $self=shift;
504 my $param=shift;
505 my ($phraseNodes,$tokenNodes,$tokens,$del)=@_;
506  
507 if (not defined $del){$del=$DEFAULTDELIMITER;}
508 if (not ref($tokenNodes)){
509 $tokens=[];
510 $tokenNodes=[];
511 @{$tokens}=$self->getTokens($param,$tokenNodes);
512 }
513  
514 my @phrases=$self->getChunks($param,$phraseNodes,$tokenNodes,$tokens,$del);
515 my @ngrams=$self->getNgrams($param,$phraseNodes,$tokenNodes,$tokens,$del);
516 push (@phrases,@ngrams);
517  
518 #----------------------------------------------------------------------
519 # add all single tokens that match the required string type
520 #----------------------------------------------------------------------
521  
522 # my $minTokenLength=1;
523 # if (defined $$param{'minimal length'}){
524 # $minTokenLength=$$param{'minimal length'};
525 # }
526 foreach (0..$#{$tokenNodes}){
527 if (not ref($tokens) or $self->skipToken($$tokens[$_])){next;}
528 # if (length($$tokens[$_])<$minTokenLength){next;}
529 my $idx=$#phrases+1;
530 if (ref($phraseNodes)){
531 @{$$phraseNodes[$idx]}=($$tokenNodes[$_]);
532 }
533 if (ref($tokens)){
534 push (@phrases,$$tokens[$_]);
535 }
536 else{
537 push (@phrases,'#token'.$idx);
538 }
539 }
540  
541 @phrases=$self->removeIdenticalPhrases($phraseNodes,\@phrases);
542 return @phrases;
543 }
544  
545  
546  
547 sub getPhrasePosition{
548 my $self=shift;
549 my ($phr)=@_;
550 if (ref($phr) ne 'ARRAY'){return 0;}
551 my $tok=$phr->[0];
552 if (not ref($tok)){return 0;}
553 my $tokID=$self->attribute($tok,'id');
554 if ($tokID=~/(\A|[^0-9])([0-9]+)$/){
555 return $2;
556 }
557 return undef;
558 }
559  
560  
561 sub getPhraseContent{
562 my $self=shift;
563 my $nodes=shift;
564 my $param=shift;
565 my ($del)=@_;
566  
567 if (not defined $del){$del=$DEFAULTDELIMITER;}
568 my @phrase=();
569 foreach my $n (@{$nodes}){
570 if (my $t=$self->checkTokenParameter($n,$param)){
571 push(@phrase,$t);
572 }
573 }
574 return join $del,@phrase;
575 }
576  
577 sub getPhraseFeature{
578 my $self=shift;
579 my ($phraseNodes,$param)=@_;
580  
581 if (ref($phraseNodes) ne 'ARRAY'){return undef;}
582 if (not @{$phraseNodes}){return '';}
583  
584  
585 if (not ref($param)){return $self->getPhraseContent($phraseNodes,$param);}
586 if (ref($param->{features}) ne 'HASH'){
587 return $self->getPhraseContent($phraseNodes,$param);
588 }
589  
590 my $FeatureString='';
591 foreach my $f (sort keys %{$param->{features}}){ # for all features:
592  
593 my %NodeHash=(); # feature node names
594 my @FeatNodes=(); # array of unique feature nodes
595 my @AllFeatNodes=(); # array of all feature nodes (for suffix/prefix)
596 my $attr=$f; # initialize feature-attribute
597 my $path=undef; # initialize feature-node path
598 # print STDERR "$f\n";
599 if ($f=~/^(.*)\:([^:]+)$/){ # first part is the path
600 ($path,$attr)=($1,$2); # second is the attribute
601 }
602  
603 foreach my $t (@{$phraseNodes}){ # for all token nodes:
604 my $node=$self->getFeatureNode($t,$path); # find the feature node
605 push(@AllFeatNodes,$node); # save feature nodes
606 if (not ref($node)){next;}
607 if (not defined $NodeHash{$node}){
608 $NodeHash{$node}=1; # save unique
609 push(@FeatNodes,$node); # feature nodes
610 }
611 }
612  
613 #---------------------------------------------------------------
614 # check if the left neighbour of the first token has the same
615 # feature node as the first token itself ---> add prefix ')'
616 # check if the right neighbour of the last token has the same
617 # feature node as the last token itself ---> add suffix '('
618 #
619 # e.g., if the feature node is a chunk-node
620 # and the neighbours of the current phrase belong to the same chunk
621  
622 my $prefix;
623 my $suffix;
624 my $node=$self->getFeatureNode($$phraseNodes[0],'left:'.$path);
625 if ($node and ($node eq $AllFeatNodes[0])){
626 $prefix=')';
627 }
628 my $node=$self->getFeatureNode($$phraseNodes[-1],'right:'.$path);
629 if ($node and ($node eq $AllFeatNodes[-1])){
630 $suffix='(';
631 }
632  
633 #---------------------------------------------------------------
634  
635  
636 my $feature='';
637 my $pattern=$param->{features}->{$f}; # substitution pattern
638 my $re=undef; # regular expression
639 my $subst=undef; # substitution
640 if ($pattern=~/(.*)\/(.*)/){ # subst.-pattern found:
641 $re=$1; # set variables
642 $subst=$2;
643 }
644  
645 foreach my $n (@FeatNodes){ # for all feature nodes:
646 my $value; # get feature string
647 if ($attr eq '#text'){
648 $value=$self->content($n);
649 }
650 else{
651 $value=$self->attribute($n,$attr);
652 }
653 if (defined $re){
654 # eval { $value=~s/$re/$subst/; }
655 eval "\$value=~s/$re/$subst/;"; # change thos ?!?
656 }
657 $feature.=$value.' ';
658 }
659 chop $feature;
660 $FeatureString.=$prefix.$feature.$suffix.':'; # put everything together
661 }
662 chop $FeatureString;
663 return $FeatureString;
664 }
665  
666  
667 sub getFeatureNode{
668 my $self=shift;
669 return $self->moveTo(@_);
670 }
671  
672  
673 #-----------------------------------------------------------
674 #
675  
676 sub removeIdenticalPhrases{
677 my $self=shift;
678 my ($nodes,$phrases)=@_;
679  
680 if ((not ref($nodes)) and (ref($phrases))){ # easy! just check strings!
681 my %hash=();
682 foreach (@{$phrases}){$hash{$_}=1;}
683 return keys %hash;
684 }
685 if (ref($nodes) ne 'ARRAY'){return undef;}
686 my @accNodes=();
687 my @accPhrases=();
688 my %hash;
689 foreach my $p (0..$#{$nodes}){
690 my $key=join "\x00",@{$$nodes[$p]};
691 if (not defined $hash{$key}){
692 $hash{$key}=1;
693 push(@accNodes,$$nodes[$p]);
694 if (ref($phrases)){
695 push(@accPhrases,$$phrases[$p]);
696 }
697 }
698 # else{print STDERR "remove phrase $$phrases[$p]\n";}
699 }
700 @{$nodes}=@accNodes;
701 return @accPhrases;
702 }
703  
704  
705 #---------------------------------
706 # add all Ngrams and sub-Ngrams
707 #
708  
709 sub addNgrams{
710 my $self=shift;
711 my $tokens=shift;
712 my $nodes=shift;
713  
714 if ($#{$tokens}<1){return;} # at least bigram!
715 $self->addNgram($tokens,$nodes,@_); # add this Ngram
716  
717 my $t=shift @{$tokens}; # recursively add N-1_grams
718 my $n=shift @{$nodes}; # * without the initial token
719 $self->addNgrams($tokens,$nodes,@_);
720 unshift(@{$tokens},$t);
721 unshift(@{$nodes},$n);
722 $t=pop @{$tokens}; # * without the final token
723 $n=pop @{$nodes};
724 $self->addNgrams($tokens,$nodes,@_);
725 push(@{$tokens},$t); # always restore
726 push(@{$nodes},$n); # the original arrays!
727 }
728  
729 #---------------------------------
730 # add Ngram
731 #
732  
733 sub addNgram{
734 my $self=shift;
735 my ($tokens,$nodes,$ngrams,$ngramNodes,$del)=@_;
736  
737 if ($#{$tokens}<1){return;} # at least bigram
738 if (not $self->isNonStarter($$tokens[0])){ # no non-starter
739 if (not $self->isNonEnder($$tokens[-1])){ # no non-ender
740 if (not defined $del){$del=$DEFAULTDELIMITER;}
741 if (ref($ngramNodes) eq 'ARRAY'){
742 my $idx=$#{$ngramNodes}+1;
743 @{$$ngramNodes[$idx]}=@{$nodes};
744 }
745 if (ref($ngrams) eq 'ARRAY'){
746 push(@{$ngrams},join $del,@{$tokens});
747 }
748 }
749 }
750 }
751  
752  
753  
754  
755  
756 sub checkTokenParameter{
757 my $self=shift;
758 my $node=shift;
759 my $param=shift;
760  
761 if (not ref($node)){return undef;}
762 my $token=$self->content($node);
763 if (ref($param) ne 'HASH'){return $token;}
764  
765 if (defined $$param{'use attribute'}){
766 my $attr=$self->attribute($node,$$param{'use attribute'});
767 if (defined $attr){$token=$attr;}
768 # else{$token='_undef';}
769 }
770 if ($$param{'grep token'}){
771 if (not $self->isStringType($token,$$param{'grep token'})){
772 return undef;
773 }
774 }
775 if (defined $$param{'minimal length'}){
776 if (length($token)<$$param{'minimal length'}){
777 return undef;
778 }
779 }
780 if ($$param{'lower case'}){
781 return $self->lowerCase($token);
782 }
783 return $token;
784 }
785  
786  
787  
788 sub makeLowInitial{
789 my $self=shift;
790 my $string=shift(@_);
791 my $LowerCaseLetter=$self->getLanguageData('character specifications',
792 'lower case letter');
793 if (not $LowerCaseLetter){return $string;}
794 if ($string=~/^(.)[$LowerCaseLetter]/){
795 my $low=$self->lowerCase($1);
796 $string=~s/^./$low/;
797 }
798 elsif ($string=~/^.$/){
799 $string=$self->LowerCase($string);
800 }
801 return $string;
802 }
803  
804 # convert string to low case characters
805  
806 sub lowerCase {
807 my $self=shift;
808 my $string=shift(@_);
809 my $LowerCaseLetter=
810 $self->getLanguageData('character specifications','lower case letter');
811 my $UpperCaseLetter=
812 $self->getLanguageData('character specifications','upper case letter');
813  
814 if ((not $UpperCaseLetter) or (not $LowerCaseLetter)){
815 return lc($string);
816 }
817  
818 eval "\$string\=\~tr/$UpperCaseLetter/$LowerCaseLetter/;";
819 return $string;
820 }
821  
822 # get number of alphabetic characters in string
823  
824 sub containsAlpha {
825 my $self=shift;
826 my $string=shift(@_);
827 my $Letter=$self->getLanguageData('character specifications','letter');
828 my $result=eval("\$string\=\~tr/$Letter//");
829 return $result;
830 }
831  
832 sub containsNumeric {
833 my $self=shift;
834 my $string=shift(@_);
835 my $Numeric=$self->getLanguageData('character specifications','numeric');
836 if (not $Numeric){return 0;}
837 my $pattern="\[$Numeric\]";
838 return $string=~/$pattern/;
839 }
840  
841 sub isVowel {
842 my $self=shift;
843 my $string=shift;
844 my $Vowel=$self->getLanguageData('character specifications','vowel');
845 my $pattern="\[$Vowel\]\+";
846 if (not $Vowel){return 0;}
847 return $string=~/^$pattern$/;
848 }
849  
850 sub isConsonant {
851 my $self=shift;
852 my $string=shift;
853 my $Consonant=
854 $self->getLanguageData('character specifications','consonant');
855 my $pattern="\[$Consonant\]\+";
856 if (not $Consonant){return 0;}
857 return $string=~/^$pattern$/;
858 }
859  
860 sub isLetter {
861 my $self=shift;
862 my $string=shift;
863 my $Letter=$self->getLanguageData('character specifications','letter');
864 if (not $Letter){return 0;}
865 my $pattern="\[$Letter\]";
866 return $string=~/^$pattern$/;
867 }
868  
869 sub isLetterSeq {
870 my $self=shift;
871 my $string=shift;
872 my $Letter=$self->getLanguageData('character specifications','letter');
873 if (not $Letter){return 0;}
874 my $pattern="\[$Letter\]\+";
875 return $string=~/^$pattern$/;
876 }
877  
878 sub isAlphabetic {
879 my $self=shift;
880 my $string=shift;
881 my $Alphabetic=
882 $self->getLanguageData('character specifications','alphabetic');
883 if (not $Alphabetic){return 0;}
884 my $pattern="\[$Alphabetic\]\+";
885 return $string=~/^$pattern$/;
886 }
887  
888 sub isAlphanumeric {
889 my $self=shift;
890 my $string=shift;
891 my $Alphanumeric=
892 $self->getLanguageData('character specifications','alphanumeric');
893 if (not $Alphanumeric){return 0;}
894 my $pattern="\[$Alphanumeric\]\+";
895 return $string=~/^$pattern$/;
896 }
897  
898 sub isNumeric {
899 my $self=shift;
900 my $string=shift;
901 my $Numeric=
902 $self->getLanguageData('character specifications','numeric');
903 if (not $Numeric){return 0;}
904 my $pattern="\[$Numeric\]\+";
905 return $string=~/^$pattern$/;
906 }
907  
908 sub isNumber {
909 my $self=shift;
910 my $string=shift;
911 my $Digit=
912 $self->getLanguageData('character specifications','digit');
913 if (not $Digit){return 0;}
914 my $pattern="\[$Digit\]\+";
915 return $string=~/^$pattern$/;
916 }
917  
918 sub isPunctuation {
919 my $self=shift;
920 my $string=shift;
921 my $Punctuation=
922 $self->getLanguageData('character specifications','punctuation');
923 if (not $Punctuation){return 0;}
924 my $pattern="\[$Punctuation\]\+";
925 return $string=~/^$pattern$/;
926 }
927  
928  
929  
930  
931 sub isStringType{
932 my $self=shift;
933 my ($string,$type)=@_;
934 if ($type eq 'all'){
935 return 1;
936 }
937 my $CharacterSpec=$self->getLanguageData('character specifications');
938  
939 my $pattern;
940 my $mod;
941 if ($type=~/^not[\_\s](.*)$/){
942 $mod='not';
943 $type=$1;
944 }
945 if ($type=~/^contains[\_\s](.*)$/){
946 $mod.='contains';
947 $type=$1;
948 }
949 if (defined $$CharacterSpec{$type}){
950 $pattern=$$CharacterSpec{$type};
951 if (not $pattern){return 1;}
952 if ($mod eq 'not'){
953 return $string!~/^[$pattern]+$/;
954 }
955 elsif ($mod eq 'contains'){
956 return $string=~/[$pattern]/;
957 }
958 elsif ($mod eq 'notcontains'){
959 return $string!~/[$pattern]/;
960 }
961 else{
962 return $string=~/^[$pattern]+$/;
963 }
964 }
965 return 1;
966 }
967  
968 sub grepStringType{
969 my $self=shift;
970 my ($list,$type)=@_;
971 return grep ($self->isStringType($_,$type),@{$list});
972 }
973  
974 sub grepStringTypeElements{
975 my $self=shift;
976 my ($list,$type)=@_;
977 return grep ($self->IsStringType($$list[$_],$type),(0..$#{$list}));
978 }
979  
980  
981  
982 sub isNonStarter{
983 my $self=shift;
984 my $token=shift;
985  
986 my $skip=$self->getLanguageData('phrases','non-phrase-starter hash');
987 if (ref($skip) eq 'HASH'){
988 if (exists $$skip{$token}){
989 return 1;
990 }
991 }
992 my $skip=
993 $self->getLanguageData('phrases','non-phrase-starter string type');
994 if (ref($skip) eq 'ARRAY'){
995 foreach (@{$skip}){
996 if (&IsStringType($token,$_)){
997 return 1;
998 }
999 }
1000 }
1001 return 0;
1002 }
1003  
1004 sub isNonEnder{
1005 my $self=shift;
1006 my $token=shift;
1007  
1008 my $skip=$self->getLanguageData('phrases','non-phrase-ender hash');
1009 if (ref($skip) eq 'HASH'){
1010 if (exists $$skip{$token}){
1011 return 1;
1012 }
1013 }
1014 my $skip=$self->getLanguageData('phrases','non-phrase-ender string type');
1015 if (ref($skip) eq 'ARRAY'){
1016 foreach (@{$skip}){
1017 if (&IsStringType($token,$_)){
1018 return 1;
1019 }
1020 }
1021 }
1022 return 0;
1023 }
1024  
1025 sub skipToken{
1026 my $self=shift;
1027 my $token=shift;
1028  
1029 my $skip=$self->getLanguageData('phrases','skip token hash');
1030 if (ref($skip) eq 'HASH'){
1031 if (exists $$skip{$token}){
1032 return 1;
1033 }
1034 }
1035 my $skip=$self->getLanguageData('phrases','skip token string type');
1036 if (ref($skip) eq 'ARRAY'){
1037 foreach (@{$skip}){
1038 if ($self->isStringType($token,$_)){
1039 return 1;
1040 }
1041 }
1042 }
1043 return 0;
1044 }
1045  
1046 sub skipPhraseAt{
1047 my $self=shift;
1048 my $token=shift;
1049  
1050 my $skip=$self->getLanguageData('phrases','skip phrase at hash');
1051 if (ref($skip) eq 'HASH'){
1052 if (exists $$skip{$token}){
1053 return 1;
1054 }
1055 }
1056 my $skip=$self->getLanguageData('phrases','skip phrase at string type');
1057 if (ref($skip) eq 'ARRAY'){
1058 foreach (@{$skip}){
1059 if ($self->isStringType($token,$_)){
1060 # print STDERR "skip at $_ type\n";
1061 return 1;
1062 }
1063 }
1064 }
1065 return 0;
1066 }
1067  
1068  
1069 sub skipPhraseBefore{
1070 my $self=shift;
1071 my $token=shift;
1072  
1073 my $skip=$self->getLanguageData('phrases','skip phrase before hash');
1074 if (ref($skip) eq 'HASH'){
1075 if (exists $$skip{$token}){
1076 return 1;
1077 }
1078 }
1079 my $skip=
1080 $self->getLanguageData('phrases','skip phrase before string type');
1081 if (ref($skip) eq 'ARRAY'){
1082 foreach (@{$skip}){
1083 if ($self->isStringType($token,$_)){
1084 return 1;
1085 }
1086 }
1087 }
1088 return 0;
1089 }
1090  
1091 sub skipPhraseAfter{
1092 my $self=shift;
1093 my $token=shift;
1094  
1095 my $skip=$self->getLanguageData('phrases','skip phrase after hash');
1096 if (ref($skip) eq 'HASH'){
1097 if (exists $$skip{$token}){
1098 return 1;
1099 }
1100 }
1101 my $skip=$self->getLanguageData('phrases','skip phrase after string type');
1102 if (ref($skip) eq 'ARRAY'){
1103 foreach (@{$skip}){
1104 if ($self->isStringType($token,$_)){
1105 return 1;
1106 }
1107 }
1108 }
1109 return 0;
1110 }
1111  
1112  
1113  
1114  
1115 #-----------------------------------------------------------------------------
1116 # auxiliary functions ....
1117  
1118  
1119 sub ArrayToHash{
1120 my ($Array,$Hash)=@_;
1121 if (ref($Array) eq 'ARRAY'){
1122 foreach (@{$Array}){
1123 $$Hash{$_}=1;
1124 }
1125 }
1126 }
1127  
1128  
1129 sub GetGeneralParam{
1130 my $Param=shift;
1131 if (defined $$Param{general}){
1132 if (ref($$Param{general}) eq 'HASH'){
1133 return $$Param{general};
1134 }
1135 }
1136 return $Param;
1137 }
1138  
1139 sub GetNgramLengthParam{
1140 my $Param=shift;
1141 my $length=1;
1142  
1143 if (defined $$Param{'maximal ngram length'}){
1144 $length=$$Param{'maximal ngram length'};
1145 }
1146 foreach (keys %{$Param}){
1147 if (ref($$Param{$_}) ne 'HASH'){next;}
1148 if ($$Param{$_}{'maximal ngram length'}>$length){
1149 $length=$$Param{$_}{'maximal ngram length'};
1150 }
1151 }
1152 return $length;
1153 }
1154  
1155 1;