MetaTrans.pm

filter: Statementstmt 86.6% 13 / 15 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 5 / 5 Podpod n/a totaltotal 90.0% 18 / 20 CC 1 SCAR 0.6
CC
1
Cov
87%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.1
« MetaTrans/WordbookCz.pm ↑ Summary
1 =head1 NAME
2  
3 MetaTrans - Class for creating multilingual meta-translators
4  
5 =head1 SYNOPSIS
6  
7 use MetaTrans;
8  
9 my $mt = new MetaTrans;
10  
11 # plug-ins we want to use
12 my @plugin_classes = (
13 'MetaTrans::UltralinguaNet',
14 'MetaTrans::SlovnikCz',
15 'MetaTrans::SeznamCz',
16 );
17  
18 foreach my $plugin_class (@plugin_classes)
19 {
20 # load module
21 eval "require $plugin_class";
22  
23 # instantiate
24 my $plugin = new $plugin_class;
25  
26 # plug the plug-in in :)
27 $mt->add_translators($plugin);
28 }
29  
30 # plug-ins which support English to Czech translation
31 @translators = $mt->get_translators_for_direction('eng', 'cze');
32  
33 # if we have at least one we will perform a translation of 'dog'
34 if (@translators > 0)
35 {
36 $mt->run_translators('dog', 'eng', 'cze');
37 my @translations;
38 while (my $translation = $mt->get_translation)
39 { push @translations, $translation; }
40  
41 # we want the output to be sorted
42 my @sorted_translations = MetaTrans::sort_translations(@translations);
43 print join("\n", @sorted_translations) . "\n";
44 }
45  
46 You are also encouraged to trying the Perl/Tk frontend. Simply run
47  
48 metatrans
49  
50 =head1 DESCRIPTION
51  
52 The C<MetaTrans> class provides an interface for making multilingual
53 translations using multiple data sources (translators). Its design
54 is especially suitable for extracting data from online translators
55 like L<http://www.ultralingua.net/>.
56  
57 To do something useful a C<MetaTrans> object must be provided with
58 plug-ins for extracting data from every source to be used. By now
59 creating a plug-in from a scratch might be a bit complicated for
60 some ugly hacks had to be made in the originally clean design of
61 C<MetaTrans> to make it working in Perl/Tk applications. Hopefully
62 this is going to change in some of the future releases.
63  
64 Currently the only recommended way for creating C<MetaTrans> plug-ins
65 is by derriving from the C<MetaTrans::Base> class. See
66 L<MetaTrans::Base> for information on how to do so.
67  
68 =cut
69  
70 package MetaTrans;
71  
72 2 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
73 2 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
74 2 ▶use vars qw($VERSION @ISA @EXPORT_OK);
Subroutine BEGIN: called CC=1 SCAR=0.0
75 2 ▶use Exporter;
Subroutine BEGIN: called CC=1 SCAR=0.0
76 0 ▶use MetaTrans::Base qw(:match_funcs);
Subroutine BEGIN: called CC=1 SCAR=2.6
77  
78 use Carp;
79 use Encode;
80 use IO::Select;
81 use Proc::SyncExec qw(sync_fhpopen_noshell sync_popen_noshell);
82  
83 $VERSION = do { my @r = (q$Revision: 1.5 $ =~ /\d+/g); sprintf "%d."."%02d", @r };
84 @ISA = qw(Exporter);
85 @EXPORT_OK = qw(sort_translations);
86  
87 =head1 CONSTRUCTOR METHODS
88  
89 =over 4
90  
91 =item MetaTrans->new(@translators)
92  
93 This method constructs a new MetaTrans object and returns it. Translators
94 array argument may be provided to plug in desired translators.
95  
96 =cut
97  
98 sub new
99 {
100 my $class = shift;
101 my @translators = @_;
102  
103 my $self = bless {}, $class;
104 $self->add_translators(@translators);
105
106 return $self;
107 }
108  
109 =back
110  
111 =cut
112  
113 =head1 METHODS
114  
115 =cut
116  
117 =over 4
118  
119 =item $mt->add_translators(@translators)
120  
121 Plug in one or more translators.
122  
123 =cut
124  
125 sub add_translators
126 {
127 my $self = shift;
128 my @translators = @_;
129  
130 foreach my $trans (@translators)
131 {
132 push @{$self->{translators}}, $trans;
133 $self->enable_translator($trans);
134 }
135 }
136  
137 =item $mt->get_translators
138  
139 Return an array of all plug-ins being used.
140  
141 =cut
142  
143 sub get_translators
144 {
145 my $self = shift;
146 return @{$self->{translators}};
147 }
148  
149 =item $mt->enable_translator($trans)
150  
151 Enable the translator. The argument is an object.
152  
153 =cut
154  
155 sub enable_translator
156 {
157 my $self = shift;
158 my $trans = shift;
159  
160 ${$self->{enabled}}{$trans} = 1;
161 }
162  
163 =item $mt->disable_translator($trans)
164  
165 Disable the translator. The argument is an object.
166  
167 =cut
168  
169 sub disable_translator
170 {
171 my $self = shift;
172 my $trans = shift;
173  
174 ${$self->{enabled}}{$trans} = 0;
175 }
176  
177 =item $mt->toggle_enabled_translator($trans)
178  
179 Togle translator's enabled/disabled status. The argument is an object.
180  
181 =cut
182  
183 sub toggle_enabled_translator
184 {
185 my $self = shift;
186 my $trans = shift;
187  
188 ${$self->{enabled}}{$trans} = not ${$self->{enabled}}{$trans};
189 }
190  
191 =item $mt->is_enabled_translator($trans)
192  
193 Returns true value if the translator is enabled, false otherwise.
194 The argument is an object.
195  
196 =cut
197  
198 sub is_enabled_translator
199 {
200 my $self = shift;
201 my $trans = shift;
202  
203 return ${$self->{enabled}}{$trans};
204 }
205  
206 =item $mt->get_translators_state($trans)
207  
208 Returns current state of the translator. Possible values are
209  
210 VALUE MEANING
211 --------- --------------------------------------------------------
212 "ok" successfully finished a translation (initial state, too)
213 "busy" working on a translation
214 "timeout" a timeout occured when querying an online translator
215 "error" unknown error occured when queryign an online translator
216  
217 =cut
218  
219 sub get_translators_state
220 {
221 my $self = shift;
222 my $trans = shift;
223  
224 return "ok" unless exists ${$self->{state}}{$trans};
225 return ${$self->{state}}{$trans};
226 }
227  
228 =item $mt->get_all_src_lang_codes
229  
230 Returns a list of language codes, which some of the enabled plug-ins are
231 able to translate from.
232  
233 The method calls the C<get_all_src_lang_codes> method for all enabled
234 plug-ins (see L<MetaTrans::Base>) and unions results.
235  
236 =cut
237  
238 sub get_all_src_lang_codes
239 {
240 my $self = shift;
241 my @codes;
242 my %codes_hash;
243
244 foreach my $trans (@{$self->{translators}})
245 {
246 next unless $self->is_enabled_translator($trans);
247 foreach my $code ($trans->get_all_src_lang_codes)
248 {
249 push @codes, $code
250 unless $codes_hash{$code};
251 $codes_hash{$code} = 1;
252 }
253 }
254  
255 return @codes;
256 }
257  
258 =item $mt->get_dest_lang_codes_for_src_lang_code($src_lang_code)
259  
260 Returns a list of language codes, which some of the enabled plug-ins are
261 able to translate to from the language with $src_lang_code.
262  
263 The method calls the C<get_dest_lang_codes_for_src_lang_codes> method for
264 all enabled plug-ins (see L<MetaTrans::Base>) and unions results.
265  
266 =cut
267  
268 sub get_dest_lang_codes_for_src_lang_code
269 {
270 my $self = shift;
271 my $src_lang_code = shift;
272 my @codes;
273 my %codes_hash;
274  
275 foreach my $trans (@{$self->{translators}})
276 {
277 next unless $self->is_enabled_translator($trans);
278 foreach my $code
279 ($trans->get_dest_lang_codes_for_src_lang_code($src_lang_code))
280 {
281 push @codes, $code
282 unless $codes_hash{$code};
283 $codes_hash{$code} = 1;
284 }
285 }
286  
287 return @codes;
288 }
289  
290 =item $mt->get_translators_for_direction($src_lang_code, $dest_lang_code)
291  
292 Retuns an array of enabled tranlators, which support the translation direction
293 from language with C<$src_lang_code> to language with C<$dest_lang_code>.
294  
295 =cut
296  
297 sub get_translators_for_direction
298 {
299 my $self = shift;
300 my $src_lang_code = shift;
301 my $dest_lang_code = shift;
302 my @result;
303  
304 foreach my $trans (@{$self->{translators}})
305 {
306 next unless $self->is_enabled_translator($trans);
307 push @result, $trans
308 if $trans->is_supported_dir($src_lang_code, $dest_lang_code);
309 }
310  
311 return @result;
312 }
313  
314 =item $mt->run_translators($expression, $src_lang_code, $dest_lang_code,
315 %options)
316  
317 Perform a translation of C<$expression> from C<$src_lang_code> language to
318 C<$dest_lang_code> language simultaneously on all enabled translators
319 (plug-ins), which support this translation direction. The method returns
320 true value on success, false on error. Use C<get_translation> method for
321 retrieving the results of particular translations.
322  
323 The method sets the state of all plug-ins to C<"busy">. See C<get_state>
324 method.
325  
326 There are two ways of performing parallel run. If C<$options{tk_safe}> is
327 undefined or set to false value, then a child process is forked for every
328 translator to be used and C<translate> method is called. This is generally
329 cleaner and more effective way of doing so then the one mentioned bellow.
330 However, this causes trouble if the module is used in Perl/Tk applications.
331  
332 If C<$options{tk_safe}> is set to a true value, then a brand new child
333 process is created for every plug-in to be used. For this plug-ins are
334 required to implement C<get_trans_command> method, which is expected to
335 return a string containing a command, which can be run from a shell and
336 provides appropriate functionality for the translation to be performed.
337 This is an ugly hack necessary for making C<MetaTrans> work in Perl/Tk
338 applications. Hopefully this will be fixed in some of the future releases.
339 See also L<MetaTrans::Base> for more information on this.
340  
341 Generally, if the plug-ins are only to be run with C<$options{tk_safe}> set to
342 false, they are not required to implement the C<get_trans_command> method.
343 Reversely, if the plug-ins are only to be run with C<$options{tk_safe}>
344 set to true, the are not required to implement the C<translate> method.
345 Plug-ins derrived from C<MetaTrans::Base> implement both methods.
346  
347 =cut
348  
349 sub run_translators
350 {
351 my $self = shift;
352 my $expression = shift;
353 my $src_lang_code = shift;
354 my $dest_lang_code = shift;
355 my %options = @_;
356  
357 my @translators = $self->get_translators_for_direction(
358 $src_lang_code, $dest_lang_code);
359 if (@translators == 0)
360 {
361 Carp::cluck "no translators available for direction: " .
362 "'${src_lang_code}2${dest_lang_code}'";
363 return undef;
364 }
365  
366 $self->{running} = 0;
367 undef $self->{pids};
368 $self->{select} = new IO::Select();
369 my @fhs;
370 my $i = 0;
371  
372 foreach my $translator (@translators)
373 {
374 my $pid;
375 if ($options{tk_safe})
376 {
377 # tk-safe fork
378 my $translator_id = $self->_get_trans_id($translator);
379 my @command = $translator->get_trans_command($expression,
380 $src_lang_code, $dest_lang_code, "/$translator_id");
381  
382 ($fhs[$i], $pid) = sync_popen_noshell('r', @command);
383 unless($pid)
384 {
385 carp("can't run '@command', make sure that runtrans is ".
386 "in your \$PATH variable");
387 return undef;
388 }
389 }
390 else
391 {
392 # non-tk-safe fork
393 do
394 {
395 $pid = open($fhs[$i], '-|');
396 unless (defined $pid)
397 {
398 warn "cannot fork: $!, still trying...";
399 sleep 2;
400 }
401 }
402 until defined $pid;
403 }
404  
405 ${$self->{state}}{$translator} = "busy";
406  
407 if ($pid)
408 {
409 # parent
410 push @{$self->{pids}}, $pid;
411 $self->{select}->add($fhs[$i]);
412 $self->{running}++;
413 }
414 else
415 {
416 #child (non-tk-safe fork only)
417 $self->_run_process($translator, $expression, $src_lang_code,
418 $dest_lang_code);
419 }
420 }
421 continue
422 { $i++; }
423  
424 return 1;
425 }
426  
427 =item $mt->get_translation(%options)
428  
429 Returns a translation returned by one of the running plug-ins (translators)
430 as a string of following form:
431  
432 expression = translation
433  
434 The method blocks until there is a translation is available (until some of
435 the running plug-ins is ready to provide an output). The order, in which
436 the translations are returned depends on the order, in which the translators
437 return their result and is therefore non-deterministic.
438  
439 The behaviour of the method depends on the C<$options{return_translators}>
440 option. If undefined or set to a false value then every call returns one
441 translation, C<undef> value is returned to indicate the end.
442  
443 If C<$options{return_value}> is set to true value, the every call returns a
444 (translation, translator) pair in an array, where the translator is the one,
445 which returned the translation. (C<undef>, translator) pair is returned to
446 indicate that the translator finished running and. C<undef> value is returned
447 to indicate that no more translations are available.
448  
449 The method also sets states of particular translators. See C<get_state> method.
450  
451 =cut
452  
453 sub get_translation
454 {
455 my $self = shift;
456 my %options = @_;
457  
458 return undef
459 if $self->{running} == 0;
460  
461 while (1)
462 {
463 my @ready;
464 do { @ready = $self->{select}->can_read(0.1); } until @ready > 0;
465  
466 my $fh = shift @ready;
467 chomp(my $translation = <$fh>);
468  
469 $translation =~ s|/([0-9]+)$||;
470 my $translator_id = $1;
471  
472 if ($translation =~ /^(ok|error|timeout)$/)
473 {
474 my $translator = $self->_get_trans_by_id($translator_id);
475 ${$self->{state}}{$translator} = $translation;
476 $translation = '';
477  
478 $self->{running}--;
479 $self->{select}->remove($fh);
480 $fh->close;
481 }
482  
483 return ($translation, $self->_get_trans_by_id($translator_id))
484 if $options{return_translators};
485  
486 # return translations only
487 return undef
488 if $self->{running} == 0;
489 return $translation
490 unless $translation eq '';
491 }
492 }
493  
494 =item $mt->is_translation_available($timeout)
495  
496 A non-blocking call, which returns a true value if next translation is already
497 available. Otherwise it blocks for at most C<$timeout> seconds and then returns
498 false if a translation is still unavailable. However, if the C<$timeout> is
499 undefined then the method always blocks and never returns false value.
500  
501 It is useful if you want to do something while waiting for the next
502 translation. Example:
503  
504 LOOP: while (1)
505 {
506 # check every second
507 until ($mt->is_translation_available(1.0))
508 {
509 last LOOP
510 if &something_happened;
511 }
512  
513 my $translation = $mt->get_translation;
514  
515 # ... do something with $translation ...
516 }
517  
518 Note: To be more exact, the C<is_translation_available> returns a true value if
519 the C<get_translation_method> has something to say. This must not necessairly
520 be a next translation, but also an C<undef> value or (<undef>, translator)
521 pair.
522  
523 =cut
524  
525 sub is_translation_available
526 {
527 my $self = shift;
528 my $timeout = shift;
529  
530 return 1
531 if $self->{running} == 0;
532  
533 my @handles = $self->{select}->handles;
534 return 1
535 if @handles = 0;
536  
537 my @ready = $self->{select}->can_read($timeout);
538 return (@ready > 0);
539 }
540  
541 =item $mt->stop_translators
542  
543 Stop all running plug-ins. This simply kills all running child processes.
544 The correspondent translators will end in the C<"busy"> state.
545  
546 =cut
547  
548 sub stop_translators
549 {
550 my $self = shift;
551  
552 kill(9, @{$self->{pids}});
553 foreach my $fh ($self->{select}->handles)
554 { $fh->close; }
555 }
556  
557 =back
558  
559 Following methods set correspondent attributes of all plug-ins being used
560 to specified values. See C<ATTRIBUTES> section of L<MetaTrans::Base> for
561 more information.
562  
563 =over 4
564  
565 =item $mt->set_timeout($timeout)
566  
567 =item $mt->set_matching($type)
568  
569 =item $mt->set_match_at_bounds($bool)
570  
571 =back
572  
573 =cut
574  
575 sub set_timeout
576 {
577 my $self = shift;
578 my $timeout = shift;
579  
580 foreach my $trans (@{$self->{translators}})
581 { $trans->timeout($timeout); }
582 }
583  
584 sub set_matching
585 {
586 my $self = shift;
587 my $matching = shift;
588  
589 foreach my $trans (@{$self->{translators}})
590 { $trans->matching($matching); }
591 }
592  
593 sub set_match_at_bounds
594 {
595 my $self = shift;
596 my $at_bounds = shift;
597  
598 foreach my $trans (@{$self->{translators}})
599 { $trans->match_at_bounds($at_bounds); }
600 }
601  
602 =head1 FUNCTIONS
603  
604 =over 4
605  
606 =item sort_translations($expression, @translations)
607  
608 Returns an array of translations sorted by relevance to the C<$expression>.
609 In addition, any duplicate information is removed.
610  
611 =back
612  
613 =cut
614  
615 sub sort_translations
616 {
617 my $expr = shift;
618 my @translations = @_;
619  
620 # sort
621 my @trans_sorted = sort {
622 &_translation_order_index($expr, $b) <=> &_translation_order_index($expr, $a)
623 ||
624 decode_utf8($a) cmp decode_utf8($b)
625 } @translations;
626  
627 # make unique
628 my @result;
629 my @same;
630 my $last_same;
631  
632 while (1)
633 {
634 my $trans = shift @trans_sorted;
635  
636 if (@same == 0 || $trans && &_eq_stripped($trans, $last_same))
637 {
638 $last_same = $trans;
639 push @same, $last_same;
640 next unless @trans_sorted == 0;
641 }
642  
643 # if the translations are the same when stripping the grammar info
644 # then only the longest one is kept
645 my $longest = '';
646 foreach (@same)
647 {
648 $longest = $_
649 if length($_) > length($longest);
650 }
651 push @result, $longest;
652  
653 $last_same = $trans;
654 @same = ($last_same);
655  
656 last unless $trans;
657 }
658  
659 return @result;
660 }
661  
662 ################################################################################
663 # private methods #
664 ################################################################################
665  
666 # runs a translation in a child process (tk-safe fork)
667 sub _run_process
668 {
669 my $self = shift;
670 my $translator = shift;
671 my $expression = shift;
672 my $src_lang_code = shift;
673 my $dest_lang_code = shift;
674  
675 my $translator_id = $self->_get_trans_id($translator);
676 my @translations = $translator->translate($expression,
677 $src_lang_code, $dest_lang_code);
678
679 if (@translations && $translations[0] !~ /=/)
680 {
681 print $translations[0] . "\n";
682 exit;
683 }
684
685 foreach my $trans (@translations)
686 { print "$trans/$translator_id\n"; }
687  
688 print "ok/$translator_id\n";
689 exit;
690 }
691  
692 # returns a number indicating relevance of a translation ($trans) to the
693 # searched expression ($expr); this is used for ordering the translations
694 sub _translation_order_index
695 {
696 my $expr = shift;
697 my $trans = shift;
698 my $index = 0;
699  
700 my ($trans_left) = split / = /, $trans;
701  
702 $index += 1
703 if is_match_words($expr, $trans_left, 1);
704 $index *= 10;
705  
706 $index += 1
707 if is_match_words($expr, $trans_left, 0);
708 $index *= 10;
709  
710 $index += _words_matched($expr, $trans_left, 1);
711 $index *= 10;
712  
713 $index += _words_matched($expr, $trans_left, 0);
714 $index *= 100;
715  
716 my @words = split /\W+/, MetaTrans::Base::strip_grammar_info($trans_left);
717 $index -= @words;
718 $index *= 100;
719  
720 $index += 1
721 if is_match_at_start($expr, $trans_left, 0);
722 #$index *= 100;
723  
724 return $index;
725 }
726  
727 # returns true value if the two translations with grammar information stripped
728 # are equal
729 sub _eq_stripped
730 {
731 my $trans1 = shift;
732 my $trans2 = shift;
733  
734 $trans1 = Encode::decode_utf8($trans1)
735 unless Encode::is_utf8($trans1);
736 $trans2 = Encode::decode_utf8($trans2)
737 unless Encode::is_utf8($trans2);
738  
739 my ($left1, $right1) = split(/ = /, $trans1);
740 my ($left2, $right2) = split(/ = /, $trans2);
741  
742 $left1 = MetaTrans::Base::strip_grammar_info($left1);
743 $left2 = MetaTrans::Base::strip_grammar_info($left2);
744  
745 return $left1 eq $left2 && $right1 eq $right2;
746 }
747  
748 # returns the plugin with an internal ID ($id),
749 # or undef
750 sub _get_trans_by_id
751 {
752 my $self = shift;
753 my $id = shift;
754  
755 return ${$self->{translators}}[$id];
756 }
757  
758 # returns an internal ID of a plugin,
759 # or -1 if this plugin is not used
760 sub _get_trans_id
761 {
762 my $self = shift;
763 my $trans = shift;
764  
765 my @translators = @{$self->{translators}};
766 for my $i (0 .. $#translators)
767 {
768 return $i
769 if $trans eq $translators[$i];
770 }
771  
772 return -1;
773 }
774  
775 # returns number of words of $in_expr matched in $found_expr
776 sub _words_matched
777 {
778 my $in_expr = shift;
779 my $found_expr = shift;
780 my $at_bounds = shift;
781  
782 my $in_stripped = MetaTrans::Base::strip_grammar_info($in_expr);
783 my $found_stripped = MetaTrans::Base::strip_grammar_info($found_expr);
784  
785 my $count = 0;
786 while ($in_stripped =~ /(\w+)/g)
787 {
788 my $word = $1;
789 if ($at_bounds)
790 {
791 $count++
792 if $found_stripped =~ /\b$word\b/;
793 }
794 else
795 {
796 $count++
797 if $found_stripped =~ /$word/;
798 }
799 }
800  
801 return $count;
802 }
803  
804 1;
805  
806 __END__
« MetaTrans/WordbookCz.pm ↑ Summary