blib/lib/Uplug/Web.pm untested

filter: Statementstmt 0.0% 0 / 1494 Branchbran 0.0% 0 / 378 Conditioncond 0.0% 0 / 39 MC/DCmcdc 0.0% 0 / 25 Subroutinesub 0.0% 0 / 75 Podpod 0.0% 0 / 57 totaltotal 0.0% 0 / 2068 CC 188 SCAR 45.8
CC
188
Cov
0%
ShowCorpusInfo
552.0
moveSentLinks
156.0
changeLinks
132.0
CRAP 97.0
1 #-*-perl-*-
2 #
3 # Copyright (C) 2004 Jörg Tiedemann <joerg@stp.ling.uu.se>
4 #
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18  
19  
20 package Uplug::Web;
21  
22 use strict;
23 use CGI qw/:standard escapeHTML escape/;
24 use Uplug::Web::Corpus;
25 use Uplug::Web::Process;
26 use Uplug::Web::Process::Lock;
27 use Uplug::Web::User;
28 use Uplug::Web::CWB;
29 use XML::Parser;
30 use File::Copy;
31 use Encode;
32  
33 our $CWBREG=$ENV{UPLUGCWB}.'/reg/';
34 our %ISO639; # ISO639 - 2-letter-language-codes
35  
36 my $MAXVIEWLINES=40; # view max X lines from text files
37 my $MAXVIEWDATA=10; # view max X data records
38 my $MAXFLOCKWAIT=10; # wait max X seconds for flock
39  
40 binmode(STDOUT, ":utf8"); # set UTF8 for STDOUT
41 #binmode(STDIN, ":utf8"); # set UTF8 for STDIN
42  
43 my %DataAccess=
44 (admin => {corpus => ['info','view','send','add','preprocess','align',
45 'index','query','remove'],
46 doc => ['info','view','send','remove'],
47 user => ['info','edit','remove']},
48 user => {corpus => ['info','view','send','add','preprocess','align',
49 'index','query','remove'],
50 doc => ['info','view','send','remove'],
51 'Uplug::Web::Data' => ['xml'],
52 'Uplug::Web::Bitext' => ['text','xml','wordalign'],
53 'Uplug::Web::BitextLinks' => ['text','xml','matrix','edit'],
54 user => ['info','edit']},
55 all => {doc => ['info','view','send'],
56 corpus => ['info','view','send'],
57 'Uplug::Web::Data' => ['xml'],
58 'Uplug::Web::Bitext' => ['text','xml'],
59 'Uplug::Web::BitextLinks' => ['text','xml','matrix'],
60 user => ['info']});
61  
62  
63 $ENV{UPLUGCONFIG} = 'systems';
64  
65  
66 sub AccessMode{
67 my $priv=shift;
68 my $type=shift;
69 return $DataAccess{$priv}{$type};
70 }
71  
72 sub ShowUserInfo{
73 my $query=shift;
74 my $user=shift;
75 my $UserData=shift;
76 my $name=shift;
77 my $admin=shift;
78  
79 my @rows=();
80 foreach my $u (keys %{$UserData}){
81  
82 my $priv='all'; # access priviliges: public
83 if ($admin){$priv='admin';} # administrator
84 elsif ($u eq $user){$priv='user';} # registered user
85  
86 my $url=&AddUrlParam($query,'n',$u);
87 if (keys %{$$UserData{$u}} > 1){
88 push (@rows,
89 &th([$u]).
90 &td(&ActionLinks($url,&AccessMode($priv,'user'))));
91 foreach (keys %{$$UserData{$u}}){
92 if ($_ eq 'Password'){$$UserData{$u}{$_}='*******';}
93 push (@rows,td([$_,$$UserData{$u}{$_}]));
94 }
95 }
96 else{
97 push (@rows,
98 &th([$u]).
99 &td(&ActionLinks($url,&AccessMode($priv,'user'))));
100 }
101 }
102 return &table({},caption(''),&Tr(\@rows));
103 }
104  
105  
106 #--------------------------------------------------------------------
107  
108 sub RemoveCorpus{
109 my $user=shift;
110 my $corpus=shift;
111 my $param=shift;
112  
113 if ($$param{'really'}){
114 return &Uplug::Web::Corpus::RemoveCorpus($user,$corpus);
115 }
116 elsif (not defined $$param{'really'}){
117 my $str= &start_multipart_form;
118 $str.="Are you really sure to remove the entire '$corpus' corpus? ";
119 $str.=&checkbox(-name=>'really',
120 -value=>1,
121 -label=>'yes!');
122 $str.= &p();
123 $str.= &submit(-name => 'submit');
124 $str.= &endform;
125 return $str;
126 }
127 return "Corpus $corpus has not been removed!";
128 }
129  
130  
131 sub AddCorpus{
132 my $user=shift;
133 my $param=shift;
134 my $query=shift;
135  
136 my $name=$$param{name};
137 my $priv=$$param{priv};
138  
139 #------------------------------------------------------------
140  
141 if (not $name){return &AddCorpusForm();}
142 else{
143 my ($ret,$msg)=&Uplug::Web::Corpus::AddCorpus($user,$name,$priv);
144 return &h3($msg);
145 }
146 }
147  
148 sub AddDocument{
149 my $user=shift;
150 my $corpus=shift;
151 my $file=shift;
152 my $param=shift;
153 my $query=shift;
154  
155 my $name=$$param{name};
156 my $lang=$$param{lang};
157 my $enc=$$param{enc};
158  
159 #------------------------------------------------------------
160  
161 my $html;
162 if ($corpus and $name and $file){
163 my ($ret,$msg)=&Uplug::Web::Corpus::AddDocument($user,$corpus,
164 $name,$file,
165 $lang,$enc);
166 $html=&h3($msg);
167 }
168 return &AddDocumentForm($user,$corpus).$html;
169 }
170  
171  
172  
173  
174 sub AddCorpusForm{
175 my @rows;
176 push (@rows,
177 &td(["Corpus name: ",
178 &textfield(-name=>'name',
179 -size=>25,
180 -maxlength=>50)]).
181 &td([&checkbox(-name=>'priv',
182 # -checked=>'checked',
183 -value=>1,
184 -label=>'private')]));
185  
186 my $str="Add a corpus to your repository!".&p();
187 $str.= "Specify a unique name for your corpus ";
188 $str.= "with not more than 10 characters!".&br;
189 $str.= "Use ASCII characters only for the name of the corpus using the following character set: ";
190 $str.= "[a-z,A-Z,0-9,_]!".&br();
191  
192 $str.= &start_multipart_form;
193 $str.= &table({},caption(''),&Tr(\@rows));
194 $str.= &p();
195 $str.= &submit(-name => 'submit');
196 $str.= &endform;
197 }
198  
199  
200  
201 sub AddDocumentForm{
202 my $user=shift;
203 my $corpus=shift;
204  
205 my @rows;
206 my $corpora=&Uplug::Web::Corpus::Corpora($user);
207  
208 push (@rows,&td(['Select a corpus: ',
209 &popup_menu(-name=> 'c',
210 -values => [sort keys %{$corpora}])]));
211 push (@rows,&td(["Document name: ",
212 &textfield(-name=>'name',
213 -size=>25,
214 -maxlength=>50).
215 &Uplug::Web::iso639_menu('lang','en')]));
216  
217 push (@rows,&td(["Upload file: ",
218 &filefield(-name=>'file',
219 -size=>25,
220 -maxlength=>50)]));
221  
222 push (@rows,&td(['Encoding',&Uplug::Web::encodings_menu('enc','utf8')]));
223  
224 my $str="Add a document to your repository!".&p();
225 $str.= "Select a corpus and specify a unique name for your document ";
226 $str.= "with not more than 15 characters!".&br;
227 $str.= "Use ASCII characters only for the name of the document using the following character set: ";
228 $str.= "[a-z,A-Z,0-9,_,.]!".&br();
229 $str.= "Each document may be translated into different languages!".&p();
230 $str.= "The file must be a plain text file! Additional markup is not recognized and will be used as text!".&br();
231 $str.= "Make sure that the specified character encoding matches the encoding of your corpus file".&br();
232 $str.= "Check for example ".&a({-href => 'http://czyborra.com/charsets/iso8859.html'},'this').' page for more information about character encoding'.&br();
233  
234  
235 $str.= &start_multipart_form;
236 $str.= &table({},caption(''),&Tr(\@rows));
237 $str.= &p();
238 $str.= &submit(-name => 'submit');
239 $str.= &endform;
240 }
241  
242  
243  
244  
245  
246  
247  
248  
249  
250 sub CorpusIndexerForm{
251 my ($user)=@_;
252 my %corpora=();
253 &Uplug::Web::Corpus::GetCorpusData(\%corpora,$user);
254 my $form= &startform();
255 $form.='Select a corpus to be indexed by the Corpus Work Bench (CWB)'.&p();
256 $form.=&popup_menu(-name=> 'corpus',
257 -values => [sort keys %corpora]);
258 $form.=&br().&encodings_menu('srcenc','iso-8859-1');
259 $form.='character encoding in the index (source language if bitext)'.&br();
260 $form.=&encodings_menu('trgenc','iso-8859-1');
261 $form.='character encoding in the index (target, only for bitexts)'.&br();
262 $form.=&p();
263 $form.= &submit(-name => 'action',-value => 'add');
264 $form.= &endform;
265 return $form;
266 }
267  
268 sub LostPassword{
269 my ($email)=@_;
270 my $html=&h3("Lost password").&hr();
271 if ($email){return $html.=&Uplug::Web::User::SendUserPassword($email);}
272 my $form= &startform();
273 $form.='Type your e-mail adress: ';
274 $form.=&textfield(-name => 'e',-default=>'user@uplug.se');
275 $form.= &submit(-name => 'action',-value => 'send');
276 $form.= &endform;
277 return $html.$form;
278 }
279  
280 sub CorpusQueryForm{
281 my ($owner,$corpus)=@_;
282 return &Uplug::Web::CWB::Query('?a=corpus;t=query',$owner,$corpus);
283 }
284  
285  
286 sub CorpusQuery{
287  
288 my ($user,$owner,$corpus,$lang,$cqp,$aligned,$style)=@_;
289  
290 my $registry=$CWBREG.$user.'/'.$corpus;
291 $WebCqp::Query::Registry = $registry;
292  
293 my $query;
294 eval { $query = new WebCqp::Query("$lang"); };
295 if ($@){print "--$@--$!--$?--";}
296  
297 $query->on_error(sub{grep {print "$_".&br()} @_});
298 my @corpora=($lang);
299 if (ref($aligned) eq 'ARRAY'){
300 $query->alignments(sort @{$aligned});
301 push (@corpora,@{$aligned});
302 }
303 $query->context('1 s', '1 s');
304 if ($cqp!~/^[\"\[]/){$cqp='"'.$cqp.'"';}
305 my @result = $query->query($cqp);
306 my $nr_result = @result;
307  
308 my $html="Query string: \"$cqp\"".&br();
309 $html.="<b>$nr_result</b> hits found<br>--------".&p();
310  
311 my @rows=();
312 if ($style eq 'vertical'){
313 push (@rows,&th(['',@corpora]));
314 }
315  
316 my $nr;my $i;
317 for ($i = 0; $i < $nr_result; $i++) {
318 $nr = $i + 1;
319 my $m = $result[$i];
320 my $pos = $m->{'cpos'};
321 my $ord = $m->{'kwic'}->{'match'};
322 my $res_r = $m->{'kwic'}->{'right'};
323 my $res_l = $m->{'kwic'}->{'left'};
324  
325 push (@rows,&td({-valign=>'top'},
326 [$pos,"$res_l <b>$ord</b> $res_r"]));
327 if (ref($aligned) eq 'ARRAY'){
328 foreach (@{$aligned}){
329 if ($style eq 'vertical'){
330 $rows[-1].=&td({-valign=>'top'},$m->{$_});
331 }
332 else{
333 push (@rows,&td({-valign=>'top'},[$_,$m->{$_}]));
334 }
335 }
336 }
337 }
338 $html.=&table({-width=>'100%'},caption(''),&Tr(\@rows));
339 return $html;
340 }
341  
342  
343 sub ShowRemovedCorpusInfo{
344 my ($task,$owner,$corpus,$docbase,$doc,$priv)=@_;
345 $Uplug::Web::Corpus::CorpusDir.='/.recycled';
346 my $html=&ShowCorpusInfo($task,$owner,$corpus,$docbase,$doc,$priv,1);
347 $Uplug::Web::Corpus::CorpusDir=$ENV{UPLUGDATA};
348 return $html;
349 }
350  
351  
352  
353  
354 ############################################################################
355 #
356 # ShowCorpusInfo:
357 # * list all corpora of a certain owner
358 # * list info about documents for selected corpora
359 #
360 ############################################################################
361  
362 sub ShowCorpusInfo{
363 my $task=shift;
364 my $owner=shift;
365 my $corpus=shift;
366 my $docbase=shift;
367 my $doc=shift;
368 my $priv=shift; # =1 -> private corpora
369 my $notasks=shift; # =1 -> skip task-list
370  
371 my $CorpusNames=&Uplug::Web::Corpus::Corpora($owner);
372 if (not defined $corpus){($corpus)=each %{$CorpusNames};}
373 if (not defined $task){$task='view';}
374  
375 #--------------------------------------------------------
376 # read info about corpus documents if a corpus is selected
377  
378 my %docs=();
379 my $CorpusDocs={};
380 if (defined $$CorpusNames{$corpus}){
381 $CorpusDocs=&Uplug::Web::Corpus::CorpusDocuments($owner,$corpus);
382 foreach my $c (sort keys %{$CorpusDocs}){
383 my $d=$$CorpusDocs{$c}{corpus};
384 $d=~s/\sword//;
385 my $l=$$CorpusDocs{$c}{language};
386 if ($l=~/^(.+)\-(.+)$/){ # alignments:
387 my $s=$1; # source language
388 my $t=$2; # target language
389 if ($c=~/word\s\(/){ # word alignments:
390 $docs{$d}{$s}{$t}=$c; # save source->target
391 }
392 else{ # sentence alignment:
393 $docs{$d}{$t}{$s}=$c; # save target->source
394 }
395 }
396 else{
397 $docs{$d}{$l}{'_doc'}=$c;
398 }
399 }
400 }
401 #--------------------------------------------------------
402  
403 my $query=&AddUrlParam('','a','corpus');
404 $query=&AddUrlParam($query,'o',$owner);
405  
406 my $link=$query;
407 if (not defined $docbase){($docbase)=each %docs;}
408 if (defined $corpus){$link=&AddUrlParam($link,'c',$corpus);}
409 if (defined $docbase){$link=&AddUrlParam($link,'b',$docbase);}
410  
411 my $html;
412 $query=&AddUrlParam($query,'t',$task);
413 $html.=&p();
414  
415 my $count;
416 foreach my $c (sort keys %{$CorpusNames}){
417 $count++;
418  
419 $query=&AddUrlParam($query,'c',$c);
420 if ($c eq $corpus){$html.="Corpus $count: ".&b($c).&p();}
421 else{$html.="Corpus $count: ".&a({-href=>$query},&b($c)).&br();}
422  
423 ################################################################
424 # stop here if this is not the selected corpus
425 #
426  
427 if ($c ne $corpus){next;}
428  
429 ################################################################
430 # add links for the different access modes
431 # and corpus tasks
432 ################################################################
433  
434 if (not $notasks){
435 # $html.=&TaskLinks({url=>$query,selected=>$task},
436 # &AccessMode($priv,'corpus'));
437  
438 $html.='modes: ';
439 if (($corpus eq $c) and (not keys %docs)){ # no docs in the corpus:
440 if ($priv ne 'all'){ # show only 'add'-mode!
441 $html.=&TaskLinks({url=>$query,
442 selected=>$task},'add');
443 }
444 next; # ... and go to the next
445 }
446 if ($priv ne 'all'){
447 $html.=&TaskLinks({url=>$query,selected=>$task},
448 'add','view','info','send','remove');
449 }
450 else{
451 $html.=&TaskLinks({url=>$query,selected=>$task},
452 'add','view','info','send');
453 }
454 $html.=&br().'tasks: ';
455 if ($priv ne 'all'){
456 $html.=&TaskLinks({url=>$query,selected=>$task},
457 'preprocess','align','index','query');
458 }
459 else{
460 $html.=&TaskLinks({url=>$query,selected=>$task},
461 'index','query');
462 }
463 }
464 $html.=&p();
465  
466 #
467 # end of the task bars
468 ##################################################################
469  
470 if ($task eq 'remove'){
471 $html.=&p().&b('ATTENTION! ');
472 $html.='Documents will be removed immediately';
473 $html.=' when clicking on the links below!';
474 }
475  
476 #-----------------------------------------------------------
477 # add document links
478  
479 my @rows=();
480 foreach my $d (sort keys %docs){
481 my $link=&AddUrlParam($query,'b',$d); # add base name
482 if ($d eq $docbase){push(@rows,&th([$d]));}
483 else{
484 push(@rows,&th([&a({-href=>$link},$d)]));
485 }
486 my @lang=sort keys %{$docs{$d}};
487 foreach my $l (@lang){
488 $link=&AddUrlParam($link,'d',$docs{$d}{$l}{'_doc'});
489 $rows[-1].=&td({-align=>'center'},
490 ['['.&a({-href=>$link},$l).']']);
491 }
492 if ($docbase ne $d){next;}
493 if (@lang<2){next;} # no matrix for only one language!
494  
495 #-------------------------------------------------------
496 # create the multilingual document matrix
497 # for the selected document
498  
499 foreach my $s (@lang){
500 $link=&AddUrlParam($link,'d',$docs{$d}{$s}{'_doc'});
501 push (@rows,&td({-align=>'right'},
502 ['['.&a({-href=>$link},$s).']']));
503 foreach my $t (@lang){
504 if (defined $docs{$d}{$s}{$t}){
505 $link=&AddUrlParam($link,'d',$docs{$d}{$s}{$t});
506 if ($docs{$d}{$s}{$t}=~/\sword\s\(/){
507 $rows[-1].=&th([&a({-href=>$link},'word')]);
508 }
509 else{
510 $rows[-1].=&th([&a({-href=>$link},'sent')]);
511 }
512 }
513 else{
514 $rows[-1].=&th({-align=>'center'},['-']);
515 }
516 }
517 }
518 }
519 $html.=&table({},caption(''),&Tr(\@rows)).&p();
520  
521 #--------------------------------------------------------
522 # the next part is for the info-mode:
523 # show extra info for the selected document!
524  
525 if ((defined $doc) and ($task eq 'info')){
526 my @rows=();
527 push (@rows,&th(['info:',$doc]));
528 foreach (keys %{$$CorpusDocs{$doc}}){
529 push (@rows,&td([$_,$$CorpusDocs{$doc}{$_}]));
530 }
531 $html.=&table({},caption(''),&Tr(\@rows)).&p();
532 }
533 }
534 return $html;
535 }
536  
537  
538 #
539 # end of ShowCorpusInfo
540 #
541 ############################################################################
542  
543  
544  
545 #-------------------------------------------------------------------------
546 # view corpus data
547 # * text files (even XML)
548 # * bitexts (sentence aligned)
549 # * bitexts (sentence and word aligned)
550 # uses object classes further down in this file!
551  
552 sub ViewCorpus{
553 my ($owner, # owner of the corpus
554 $corpus, # corpus name
555 $doc, # document name
556 $url, # current query URL
557 $pos, # last position in the document
558 $style, # display style
559 $params)=@_; # other parameters (pointer to hash)
560  
561 my $DocConfig=&Uplug::Web::Corpus::GetCorpusInfo($owner,$corpus,$doc);
562  
563 if (not defined $$DocConfig{file}){return undef;}
564 my $file=$$DocConfig{file};
565 my $data;
566 my $html;
567  
568 my $priv='user'; # default: user privileges
569 if ($owner eq 'pub'){$priv='all';} # for public data: restricted access!
570 if ($$DocConfig{format}=~/align/){ # for bitexts:
571  
572 #---------------------------------------------------------------------
573 if ($$DocConfig{status}=~/word/){ # word alignments
574 $data=new Uplug::Web::BitextLinks($priv,$file); # -> BitextLinks
575 }
576 #---------------------------------------------------------------------
577 elsif ($style eq 'wordalign'){ # style=wordalign
578 $style='edit'; # - make align-
579 $doc=~s/(\s\(..\-..\))$/ word$1/; # modus for
580 $html=&b("Align words in document '$doc'!"); # sentence-
581 if (not -e "$file.links"){ # aligned bitexts
582 copy($file,"$file.links"); # - create a word-
583 # open F,">$file.links.lock";close F; # align file and
584 # chmod 0664,"$file.links.lock"; # a lock-file
585 # chmod 0664,"$file.links"; # - set permissions
586 system "g+w $file.links"; # - set permissions
587 $$DocConfig{file}="$file.links"; # - add info to the
588 $$DocConfig{status}='word'; # user configfile
589 my $DocBase=$$DocConfig{corpus}.' word'; # (without lang)
590 &Uplug::Web::Corpus::ChangeCorpusInfo($owner,
591 $corpus,
592 $DocBase,
593 $DocConfig);
594 $html.=&br().&i("(added document ($doc) to corpus '$corpus')");
595 }
596 $html.=&p(); # - view the word-
597 $file.='.links'; # align file
598 $url=&AddUrlParam($url,'d',$doc); # in edit mode!
599 $url=&AddUrlParam($url,'s',$style);
600 $data=new Uplug::Web::BitextLinks($priv,$file);
601 }
602 #---------------------------------------------------------------------
603 else{$data=new Uplug::Web::Bitext($priv,$file);} # sentence align
604 }
605 #-------------------------------------------------------------------------
606 else{$data=new Uplug::Web::Data($priv,$file);} # all other data:
607 return $html.$data->view($url,$style,$pos,$params); # view as text
608 }
609  
610 # end of ViewCorpus
611 #-----------------------------------------------------------------------------
612  
613  
614  
615 ###################################################################
616  
617 sub SelectCorpusForm{
618 my $user=shift;
619 my $corpora=&Uplug::Web::Corpus::Corpora($user);
620 my $str = &start_multipart_form;
621 $str.='corpora: ';
622 $str.=&popup_menu(-name=> 'c',-values => [sort keys %{$corpora}]);
623 $str.= &submit(-name => 'select');
624 $str.= &endform;
625 return $str;
626 }
627  
628  
629 #------------------------------------------------------------
630 # sentence-align all bitexts in a given corpus
631  
632 sub AlignAllDocuments{
633 my ($user,$corpus)=@_;
634 my $config='./systems/align/sent';
635  
636 my %docs;
637 my %aligned;
638 my $CorpusData=&Uplug::Web::Corpus::CorpusDocuments($user,$corpus);
639 my $html;
640 foreach my $c (keys %{$CorpusData}){
641 if ($$CorpusData{$c}{language}=~/^(.+)\-(.+)$/){ # aligned corpora
642 $aligned{$$CorpusData{$c}{corpus}}{$1}{$2}=1; # (save lang pairs)
643 next;
644 }
645 if ($$CorpusData{$c}{status}!~/(tag|tok|chunk)/){ # check status
646 $html.="$c is not tokenized yet! Run pre-processing first!".&br();
647 next;
648 }
649 $docs{$$CorpusData{$c}{corpus}}{$$CorpusData{$c}{language}}=$c;
650 }
651 my $count=0;
652 foreach my $c (keys %docs){
653 my @lang=sort keys %{$docs{$c}};
654 while (@lang){
655 my $s=shift(@lang);
656 foreach my $t (@lang){
657 if ($aligned{$c}{$s}{$t}){next;} # skip if already aligned!
658 if ($aligned{$c}{$t}{$s}){next;} # (even if source->target)
659 my %para=('input:source text:stream name' => $docs{$c}{$s},
660 'input:target text:stream name' => $docs{$c}{$t});
661 &Uplug::Web::Process::MakeUplugProcess($user,$corpus,
662 $config,\%para);
663 $count++;
664 }
665 }
666 }
667 return $html.
668 &h3("$count sentence alignment process(es) added to the queue!");
669 }
670  
671  
672 #------------------------------------------------------------
673 # sentence-align all bitexts in a given corpus
674  
675 sub PreprocessAllDocuments{
676 my ($user,$corpus)=@_;
677  
678 my $CorpusData=&Uplug::Web::Corpus::CorpusDocuments($user,$corpus);
679 my $count;
680 foreach my $c (keys %{$CorpusData}){
681 if ($$CorpusData{$c}{status} eq 'text'){
682 $count++;
683 my $config=&GetPreprocessConfig($$CorpusData{$c}{language});
684 my %para=('input:text:stream name' => $c);
685 &Uplug::Web::Process::MakeUplugProcess($user,$corpus,
686 $config,\%para);
687 }
688 }
689 return &h3("$count pre-processing job(s) added to the queue!");
690 }
691  
692 sub GetPreprocessConfig{
693 my $lang=shift;
694 my $configbase='systems/pre';
695 my $LangName=$ISO639{$lang};
696 $LangName=~tr/A-Z/a-z/;
697 if (-e "$ENV{UPLUGHOME}/$configbase/$LangName"){
698 return "$configbase/$LangName";
699 }
700 return $configbase.'/basic';
701 }
702  
703  
704  
705  
706 sub Process{
707 my $query=shift;
708 my ($user,$corpus,$name,$params)=@_;
709  
710 if (not defined $corpus){ # if no corpus selected:
711 return &SelectCorpusForm($user); # return corpus-select-form
712 }
713 if (not defined $name){$name='main';} # default-module = main
714  
715 #-----------------------------------------------
716 # 'submit'
717 # create a uplug-process in the process-queue
718  
719 if ((ref($params) eq 'HASH') and (defined $$params{submit})){
720 my $proc=&Uplug::Web::Process::MakeUplugProcess($user,$corpus,
721 $name,$params);
722 my $html="job $proc added to queue!<hr />";
723 $html.=&UplugSystemForm($query,$user,$corpus,$name);
724 return $html;
725 }
726  
727 #-----------------------------------------------
728 # 'save'
729 # save configuration
730  
731 elsif ((ref($params) eq 'HASH') and (defined $$params{save})){
732 if (&Uplug::Web::Process::SaveUplugSettings($user,$name,$params)){
733 return
734 # &h3($name).
735 &UplugSystemForm($query,$user,$corpus,$name).
736 &p().&b('settings saved!');
737 }
738 }
739  
740 #-----------------------------------------------
741 # 'reset'
742 # restore default parameters
743  
744 elsif ((ref($params) eq 'HASH') and (defined $$params{reset})){
745 &Uplug::Web::Process::ResetUplugSettings($user,$name,$params);
746 }
747 return &UplugSystemForm($query,$user,$corpus,$name);
748 }
749  
750  
751 ###################################################################
752  
753  
754  
755 sub ProcessTable{
756 my $query=shift;
757 my $user=shift;
758 my $type=shift;
759 my $process=shift;
760 my @actions=@_;
761  
762 my $url=&Uplug::Web::AddUrlParam($query,'y',$type);
763 my $html;
764 if (defined $user){$html=&h3($type);}
765 else{$html=&h3($type.' '.&ActionLinks($url,'clear'));}
766 my @proc=&Uplug::Web::Process::GetProcesses($type);
767 if (not @proc){
768 return "No process in $type-processes-stack!".&br();
769 }
770 my @rows;
771 my $count=0;
772 foreach (@proc){
773 $count++;
774 chomp;
775 my ($u,$p,@c)=split(/\:/);
776 if ((defined $user) and ($user ne $u)){next;}
777 $url=&Uplug::Web::AddUrlParam($url,'p',$p);
778 $url=&Uplug::Web::AddUrlParam($url,'u',$u);
779 push (@rows,
780 &td([$count.')']).
781 &th([$u]).
782 &td([$p.&Uplug::Web::ActionLinks($url,@actions)]));
783 if ($process eq $p){
784 push (@rows,&td(['',@c]));
785 }
786 }
787 return $html.&table({},caption(''),&Tr(\@rows));
788 }
789  
790 sub ShowProcessInfo{
791 my $query=shift;
792 my $user=shift;
793 my $process=shift;
794 my $admin=shift;
795 my $html;
796  
797 if (not $admin){
798 $html.= &ProcessTable($query,$user,'todo',$process,
799 'view','remove');
800 $html.= &ProcessTable($query,$user,'queued',$process,
801 'view');
802 $html.= &ProcessTable($query,$user,'working',$process,
803 'view','logfile');
804 $html.= &ProcessTable($query,$user,'done',$process,
805 'view','remove','logfile');
806 $html.= &ProcessTable($query,$user,'failed',$process,
807 'view','remove','restart','logfile');
808 return $html;
809 }
810  
811 #-----------------------------------
812 # administrator view!!!
813 #
814 $html.= &ProcessTable($query,undef,'todo',$process,'view','remove');
815 $html.= &ProcessTable($query,$user,'queued',$process,'view','remove');
816 $html.= &ProcessTable($query,undef,'working',$process,
817 'view','remove','logfile');
818 $html.= &ProcessTable($query,undef,'done',$process,
819 'view','remove','restart','logfile');
820 $html.= &ProcessTable($query,undef,'failed',$process,
821 'view','remove','restart','logfile');
822 return $html;
823 }
824  
825  
826  
827 ###################################################################
828  
829  
830  
831  
832  
833 sub MakeSubmoduleLinks{
834 my $url=shift;
835 my $user=shift;
836 my $module=shift;
837 my @mod=&Uplug::Web::Process::GetSubmodules($user,$module);
838  
839 my @links=();
840 while (@mod){
841 my $m=shift(@mod);
842 $m=~s/^(\S+)\s.*$/$1/;
843 my $n=shift(@mod);
844 my $query=&AddUrlParam($url,'m',$m);
845 push (@links,&a({-href => $query},$n));
846 }
847 return wantarray ? @links : join &br(),@links;
848 }
849  
850  
851 sub UplugSystemForm{
852 my $url=shift;
853 my $user=shift;
854 my $corpus=shift;
855 my $name=shift;
856  
857 my %config;
858 if (not &Uplug::Web::Process::GetConfiguration(\%config,$user,$name)){
859 print "Cannot find $name!",&p();
860 return undef;
861 }
862  
863 my $shortcuts=$config{shortcuts};
864 if (ref($config{arguments}) eq 'HASH'){
865 $shortcuts=$config{arguments}{shortcuts};
866 }
867  
868 my $back=&a({-href=>'javascript:history.go(-1)'},'back');
869 my $html.=$config{description}.&p();
870 # $html=~s/\n/\<br\>/gs;
871 # $html=~s/ /\&nbsp\;/gs;
872 if (ref($config{module}) eq 'HASH'){
873 if (defined $config{module}{name}){
874 $html=&h3($config{module}{name}).$html;
875 }
876 else{$html=&h3($name).$html;}
877 $html.=&MakeSubmoduleLinks($url,$user,$config{module}).&p();
878 }
879  
880 my $form= &startform;
881 my ($widgets,$msg)=
882 &MakeWidgetForm($user,$corpus,$config{widgets},\%config,$shortcuts);
883 if (not $widgets){return $back.&p().$msg.$html;}
884 $form.= $widgets;
885 $form.= &p();
886 $form.= &submit(-name => 'reset',-value => 'reset');
887 $form.= &submit(-name => 'save',-value => 'save settings');
888 if (ref($config{widgets}{input}) eq 'HASH'){
889 if (keys %{$config{widgets}{input}}){
890 $form.= &submit(-name => 'submit',-value => 'add job');
891 }
892 }
893 $form.= &endform;
894 return $back.$html.$form;
895 }
896  
897 sub MakeWidgetForm{
898 my $user=shift;
899 my $corpus=shift;
900 my $config=shift;
901 my $defaults=shift;
902 my $shortcuts=shift;
903 my $menu=shift;
904  
905 if (ref($config) ne 'HASH'){return;}
906  
907 my @rows=();
908 foreach my $p (sort keys %{$config}){
909 my $name=$p;
910 if (defined $menu){$name="$menu:$p";}
911 my $def;
912 if (ref($defaults) eq 'HASH'){$def=$$defaults{$p};}
913 if (ref($$config{$p}) eq 'HASH'){
914 push (@rows,&th([$p]));
915 my ($form,$msg)=&MakeWidgetForm($user,
916 $corpus,
917 $$config{$p}, # sub-menu config
918 $def, # default value
919 $shortcuts, # shortcuts hash
920 $name); # sub-menu name
921 if (not $form){return (undef,$msg);}
922 push (@rows,&td([$form]));
923 }
924 else{
925 if (ref($shortcuts) eq 'HASH'){
926 my ($short)=grep ($$shortcuts{$_} eq $name,keys %{$shortcuts});
927 if ($short){$name='-'.$short;}
928 }
929 my ($widget,$msg)=&MakeWidget($user,$corpus,
930 $name,$$config{$p},$def);
931 if (not $widget){return (undef,$msg);}
932 push (@rows,&td([$p,$widget]));
933 }
934 }
935 if (not @rows){return (undef,'empty?!');}
936 return (&table({},&Tr(\@rows)));
937 }
938  
939  
940 sub MakeWidget{
941 my $user=shift;
942 my $corpus=shift;
943 my $name=shift;
944 my $config=shift;
945 my $default=shift;
946  
947 if ($config=~/stream\s*\((.*)\)/){
948 my %para=split(/\s*[\,\=]\s*/,$1);
949 # my @streams=&Uplug::Web::Corpus::GetCorpusStreams($user,%para);
950 my @streams=
951 &Uplug::Web::Corpus::MatchingDocuments($user,$corpus,%para);
952 if (not @streams){
953 my $msg="No appropriate document found in this corpus ($corpus)!";
954 $msg.=&p();
955 foreach (keys %para){
956 $msg.="$_=$para{$_}".&br;
957 }
958 return (undef,$msg);
959 }
960 return (&td([&popup_menu(-name=> $name,
961 -default => $default,
962 -values => [sort @streams])]));
963 }
964 elsif($config=~/optionmenu\s*\((.*)\)/){
965 my @options=split(/\s*\,\s*/,$1);
966 return (&td([&popup_menu(-name=> $name,
967 -default => $default,
968 -values => [sort @options])]));
969 }
970 elsif($config=~/scale\s*\((.*)\)/){
971 my ($start,$end,$steps,$bigsteps)=split(/\s*\,\s*/,$1);
972 my @options;my $i;
973 for ($i=$start;$i<=$end;$i+=$bigsteps){
974 push (@options,$i);
975 }
976 if ((defined $default) and (not grep ($_==$default,@options))){
977 push (@options,$default);
978 }
979 return (&td([&popup_menu(-name=> $name,
980 -default => $default,
981 -values => [sort {$a <=> $b} @options])]));
982 }
983 elsif($config=~/checkbox/){
984 # return (&checkbox(-name=>$name,
985 # -value=>1,
986 # -checked=>$default,
987 # -label=>''));
988 return (join ('&nbsp;',&radio_group(-name=>$name,
989 -values=>['1','0'],
990 -default=>$default,
991 -labels=> {'1' => 'on',
992 '0' => 'off'})));
993 }
994 return (&td([&textfield(-name => $name,-default=>$default)]));
995 }
996  
997  
998  
999  
1000  
1001  
1002  
1003  
1004 sub DelUrlParam{
1005 my ($url,$name)=@_;
1006 while($url=~s/([\;\?])$name\=[^\;]*(\;|\Z)/$1/){1;}
1007 return $url;
1008 }
1009  
1010  
1011  
1012  
1013 sub AddUrlParam{
1014 my ($url,$name,$val)=@_;
1015 $val=escape($val);
1016 if ($url!~/[\?\;]$name\=/){
1017 if ($url!~/\?/){$url.="\?$name=$val";}
1018 else{$url.="\;$name=$val";}
1019 }
1020 else{$url=~s/$name\=[^\;]*(\;|\Z)/$name=$val$1/;}
1021 return $url;
1022 }
1023  
1024 sub ActionLinks{
1025 my $url=shift;
1026 my @action;
1027 if (ref($_[0]) eq 'ARRAY'){@action=@{$_[0]};}
1028 else{@action=@_;}
1029  
1030 my @links;
1031 foreach (@action){
1032 push (@links,'['.&a({-href => &AddUrlParam($url,'t',$_)},$_).']');
1033 }
1034 return wantarray ? @links : join '',@links;
1035 }
1036  
1037  
1038 sub TaskLinks{
1039  
1040 my %para=();
1041 my $url;
1042 my $key='t'; # default url-attr: t
1043  
1044 if (ref($_[0]) eq 'HASH'){ # first arg = HASH ref?
1045 %para=%{$_[0]};shift; # -> parameter hash
1046 $url=$para{url}; # get URL parameter
1047 }
1048 else{ # otherwise:
1049 $url=shift; # URL=first arg
1050 $key=shift; # URL-attr=second arg
1051 }
1052  
1053 if (exists $para{key}){$key=$para{key};} # check key parameter
1054 my $selected=$para{selected}; # selected 'task'
1055  
1056 my @tasks; # tasks:
1057 if (ref($_[0]) eq 'ARRAY'){@tasks=@{$_[0]};} # array reference
1058 else{@tasks=@_;} # or all remaining args
1059  
1060 my @links;
1061 foreach my $t (@tasks){
1062 if ($t eq $selected){
1063 push (@links,'['.$t.']');
1064 }
1065 elsif (defined $key){
1066 push (@links,'['.&a({-href=>&AddUrlParam($url,$key,$t)},$t).']');
1067 }
1068 else{
1069 push (@links,'['.&a({-href=>$url},$t).']');
1070 }
1071 }
1072 return wantarray ? @links : join '',@links;
1073 }
1074  
1075  
1076  
1077  
1078  
1079  
1080 # http://ftp.ics.uci.edu/pub/ietf/http/related/iso639.txt
1081 # Technical contents of ISO 639:1988 (E/F)
1082 # "Code for the representation of names of languages".
1083 #
1084 # Typed by Keld.Simonsen@dkuug.dk 1990-11-30 <ftp://dkuug.dk/i18n/ISO_639>
1085 # Minor corrections, 1992-09-08 by Keld Simonsen
1086 # Sundanese corrected, 1992-11-11 by Keld Simonsen
1087 # Telugu corrected, 1995-08-24 by Keld Simonsen
1088 # Hebrew, Indonesian, Yiddish corrected 1995-10-10 by Michael Everson
1089 # Inuktitut, Uighur, Zhuang added 1995-10-10 by Michael Everson
1090 # Sinhalese corrected, 1995-10-10 by Michael Everson
1091 # Faeroese corrected to Faroese, 1995-11-18 by Keld Simonsen
1092 # Sangro corrected to Sangho, 1996-07-28 by Keld Simonsen
1093 #
1094 # Two-letter lower-case symbols are used.
1095 # The Registration Authority for ISO 639 is Infoterm, Osterreichisches
1096 # Normungsinstitut (ON), Postfach 130, A-1021 Vienna, Austria.
1097  
1098 %ISO639=( qw(
1099 aa Afar
1100 ab Abkhazian
1101 af Afrikaans
1102 am Amharic
1103 ar Arabic
1104 as Assamese
1105 ay Aymara
1106 az Azerbaijani
1107  
1108 ba Bashkir
1109 be Byelorussian
1110 bg Bulgarian
1111 bh Bihari
1112 bi Bislama
1113 bn Bengali;Bangla
1114 bo Tibetan
1115 br Breton
1116  
1117 ca Catalan
1118 co Corsican
1119 cs Czech
1120 cy Welsh
1121  
1122 da Danish
1123 de German
1124 dz Bhutani
1125  
1126 el Greek
1127 en English
1128 eo Esperanto
1129 es Spanish
1130 et Estonian
1131 eu Basque
1132  
1133 fa Persian
1134 fi Finnish
1135 fj Fiji
1136 fo Faroese
1137 fr French
1138 fy Frisian
1139  
1140 ga Irish
1141 gd Scots_Gaelic
1142 gl Galician
1143 gn Guarani
1144 gu Gujarati
1145  
1146 ha Hausa
1147 he Hebrew
1148 hi Hindi
1149 hr Croatian
1150 hu Hungarian
1151 hy Armenian
1152  
1153 ia Interlingua
1154 id Indonesian
1155 ie Interlingue
1156 ik Inupiak
1157 is Icelandic
1158 it Italian
1159 iu Inuktitut
1160  
1161 ja Japanese
1162 jw Javanese
1163  
1164 ka Georgian
1165 kk Kazakh
1166 kl Greenlandic
1167 km Cambodian
1168 kn Kannada
1169 ko Korean
1170 ks Kashmiri
1171 ku Kurdish
1172 ky Kirghiz
1173  
1174 la Latin
1175 ln Lingala
1176 lo Laothian
1177 lt Lithuanian
1178 lv Latvian,Lettish
1179  
1180 mg Malagasy
1181 mi Maori
1182 mk Macedonian
1183 ml Malayalam
1184 mn Mongolian
1185 mo Moldavian
1186 mr Marathi
1187 ms Malay
1188 mt Maltese
1189 my Burmese
1190  
1191 na Nauru
1192 ne Nepali
1193 nl Dutch
1194 no Norwegian
1195  
1196 oc Occitan
1197 om (Afan)Oromo
1198 or Oriya
1199  
1200 pa Punjabi
1201 pl Polish
1202 ps Pashto,Pushto
1203 pt Portuguese
1204  
1205 qu Quechua
1206  
1207 rm Rhaeto-Romance
1208 rn Kirundi
1209 ro Romanian
1210 ru Russian
1211 rw Kinyarwanda
1212  
1213 sa Sanskrit
1214 sd Sindhi
1215 sg Sangho
1216 sh Serbo-Croatian
1217 si Sinhalese
1218 sk Slovak
1219 sl Slovenian
1220 sm Samoan
1221 sn Shona
1222 so Somali
1223 sq Albanian
1224 sr Serbian
1225 ss Siswati
1226 st Sesotho
1227 su Sundanese
1228 sv Swedish
1229 sw Swahili
1230  
1231 ta Tamil
1232 te Telugu
1233 tg Tajik
1234 th Thai
1235 ti Tigrinya
1236 tk Turkmen
1237 tl Tagalog
1238 tn Setswana
1239 to Tonga
1240 tr Turkish
1241 ts Tsonga
1242 tt Tatar
1243 tw Twi
1244  
1245 ug Uighur
1246 uk Ukrainian
1247 ur Urdu
1248 uz Uzbek
1249  
1250 vi Vietnamese
1251 vo Volapuk
1252  
1253 wo Wolof
1254  
1255 xh Xhosa
1256  
1257 yi Yiddish
1258 yo Yoruba
1259  
1260 za Zhuang
1261 zh Chinese
1262 zu Zulu
1263 ));
1264  
1265  
1266 sub iso639_menu{
1267 my $name=shift;
1268 my $default=shift;
1269 return &popup_menu(-name=>'lang',
1270 -values => [sort {$ISO639{$a} cmp $ISO639{$b}}
1271 keys %ISO639],
1272 -labels => \%ISO639,
1273 -default => $default);
1274 }
1275  
1276  
1277  
1278 my %Encodings=();
1279  
1280 if ($]>=5.008){
1281 eval { require Encode; };
1282 if (not $@){
1283 my @enc=Encode->encodings(":all");
1284 foreach (@enc){$Encodings{$_}=$_;}
1285 }
1286 }
1287 else{
1288 %Encodings=('utf8' => 'Unicode UTF8',
1289 'iso-8859-1' => 'iso-8859-1 (latin 1)',
1290 'iso-8859-2' => 'iso-8859-2 (latin 2)');
1291 }
1292  
1293  
1294  
1295 sub encodings_menu{
1296 my $name=shift;
1297 my $default=shift;
1298 return &popup_menu(-name=>$name,
1299 -values => [sort {$Encodings{$a} cmp $Encodings{$b}}
1300 keys %Encodings],
1301 -labels => \%Encodings,
1302 -default => $default);
1303 }
1304  
1305  
1306  
1307  
1308  
1309  
1310  
1311  
1312  
1313  
1314 #############################################################################
1315 #############################################################################
1316 #############################################################################
1317 #
1318 # classes for reading datafiles
1319 #
1320 # Uplug::Web::Data ......... read text files (e.g. XML as plain text)
1321 # Uplug::Web::Bitext ....... read sentence aligned bitexts
1322 # Uplug::Web::BitextLinks .. read word aligned bitexts
1323 #
1324  
1325 package Uplug::Web::Data;
1326  
1327 use CGI qw/:standard escapeHTML escape/;
1328  
1329 sub new{
1330 my $class=shift;
1331 my $priv=shift;
1332 my $file=shift;
1333 my $self={};
1334 bless $self,$class;
1335 $self->{FILE}=$file;
1336 $self->{PRIV}=$priv;
1337 $self->{STYLES}=&Uplug::Web::AccessMode($priv,$class);
1338 # print @{$self->{STYLES}};
1339 # print $self->{STYLES};
1340 return $self;
1341 }
1342  
1343 sub view{
1344 my $self=shift;
1345 my ($url,$style,$pos)=@_;
1346  
1347 if (not defined $style){$style='xml';}
1348  
1349 my $file=$self->{FILE};
1350 open F,"< $file";
1351 binmode(F,":utf8");
1352 # binmode(F);
1353 if (defined $pos){
1354 seek (F,$pos,0);
1355 }
1356 $self->{STYLE}=$style;
1357 $self->{POS}=$pos;
1358  
1359 my $html='';
1360 my $skip=0;
1361 my $count=0;
1362 while (<F>){
1363 # if ($skip<$pos){$skip++;next;}
1364 $html.=&escapeHTML($_);
1365 # $html=~s/\n/\<br\>/gs;
1366 # $html=~s/\s/\&nbsp\;\&nbsp\;/gs;
1367 $count++;
1368 if ($count>$MAXVIEWLINES){last;}
1369 }
1370 $html=&pre($html);
1371 $self->{NEXT}=tell(F);
1372 $self->{COUNT}=$count;
1373 close F;
1374  
1375 my $links=$self->nextLinks($url);
1376 return $links.$html.$links;
1377  
1378 }
1379  
1380 sub edit{ # no edit defined!
1381 my $self=shift; # (just view the corpus)
1382 return $self->view(@_);
1383 }
1384  
1385  
1386  
1387 sub nextLinks{
1388 my $self=shift;
1389 my $url=shift;
1390  
1391 my $count=$self->{COUNT};
1392 my $style=$self->{STYLE};
1393 my $pos=$self->{POS};
1394 if (not $pos){$pos=0;}
1395 my $NextPos=$self->{NEXT};
1396 my ($start,$prev,$next,$styles);
1397 if (ref($self->{STYLES}) eq 'ARRAY'){
1398 foreach (@{$self->{STYLES}}){
1399 if ($style eq $_){next;}
1400 my $link=&Uplug::Web::AddUrlParam($url,'s',$_);
1401 $styles.=' ['.&a({-href => $link},$_).']';
1402 }
1403 }
1404 if ($styles){$styles='display style: '.$styles.&br();}
1405 if ($pos>0){
1406 my $link=&Uplug::Web::AddUrlParam($url,'x',0);
1407 $link=&Uplug::Web::DelUrlParam($link,'sx');
1408 $link=&Uplug::Web::DelUrlParam($link,'tx');
1409 $start='['.&a({-href => $link},'start').']';
1410 if ($style ne 'edit'){
1411 $prev='['.&a({-href=>'javascript:history.go(-1)'},'previous').']';
1412 }
1413 }
1414 if ($count>=$MAXVIEWDATA){
1415 my $link=&Uplug::Web::AddUrlParam($url,'x',$NextPos);
1416 param('ax',$pos);
1417 my $link=&Uplug::Web::AddUrlParam($link,'ax',$pos);
1418 if ($self->{'FROMDOC-POS'}>0){
1419 $link=&Uplug::Web::AddUrlParam($link,'sx',$self->{'FROMDOC-POS'});
1420 }
1421 if ($self->{'TODOC-POS'}>0){
1422 $link=&Uplug::Web::AddUrlParam($link,'tx',$self->{'TODOC-POS'});
1423 }
1424 $next='['.&a({-href => $link},'next').']';
1425 }
1426 # return $styles.&p().$start.&br().$prev.&br().$next.&p();
1427 return $styles.&p().$start.$prev.$next.&p();
1428 }
1429  
1430  
1431  
1432 #############################################################################
1433 #############################################################################
1434 #
1435 # sentence aligned bitext
1436 #
1437  
1438  
1439 package Uplug::Web::Bitext;
1440  
1441 use CGI qw/:standard escapeHTML escape/;
1442 use vars qw(@ISA);
1443 @ISA = qw( Uplug::Web::Data );
1444  
1445 use Uplug::Web::Process::Lock;
1446  
1447  
1448 sub new{
1449 my $class=shift;
1450 my $self=$class->SUPER::new(@_);
1451 # $self->{STYLES}=['text','xml'];
1452 $self->{ROOT}='link';
1453 return $self;
1454 }
1455  
1456  
1457 sub view{
1458 my $self=shift;
1459 my ($url,$style,$pos,$param)=@_;
1460  
1461 my $html;
1462 if (not defined $style){$style='xml';}
1463 $self->{STYLE}=$style;
1464 $self->{POS}=$pos;
1465 $self->{URL}=$url;
1466 if (ref($param) eq 'HASH'){
1467 $self->{'FROMDOC-POS'}=$$param{sx};
1468 $self->{'TODOC-POS'}=$$param{tx};
1469 # print join '<br>',%{$param};
1470 $url=&Uplug::Web::AddUrlParam($url,'sx',$self->{'FROMDOC-POS'});
1471 $url=&Uplug::Web::AddUrlParam($url,'tx',$self->{'TODOC-POS'});
1472 $self->{URL}=$url;
1473 if ($$param{mn}){
1474 $self->moveSentLinks($param);
1475 }
1476 }
1477 my $count;my $fromDoc;my $toDoc;
1478 if (not $self->readLinks($pos)){return undef;}
1479 my $seg=$self->readSentLinks($style);
1480 $html=$self->nextLinks($url);
1481 $html.=$seg.&p();
1482 $html.=$self->nextLinks($url);
1483 return $html;
1484 }
1485  
1486 sub moveSentLinks{
1487 my $self=shift;
1488 my $param=shift;
1489 my $file=$self->{FILE};
1490 if (not -e $file){return 0;}
1491  
1492 ##
1493 ## file locking with flock
1494 ##
1495 # my $LOCK=$file.'.lock'; # lock the lock-file
1496 # open LCK,"+<$file\.lock";
1497 # my $sec=0;
1498 ## print "lock file<br>";
1499 # while (not flock(LCK,2)){
1500 # $sec++;sleep(1);
1501 # if ($sec>$MAXFLOCKWAIT){
1502 # close LCK;
1503 # return 0;
1504 # }
1505 # }
1506  
1507 ##
1508 ## file locking with nflock
1509 ##
1510  
1511 if (not &nflock($file,$MAXFLOCKWAIT)){
1512 print STDERR "# Uplug::Web - can't get exclusive lock for $file!\n";
1513 return 0;
1514 }
1515  
1516 open F,"< $file";
1517 binmode(F);
1518  
1519 my @pos=sort {$a <=> $b} ($$param{mox},$$param{mnx});
1520 my %before;
1521 my %link;
1522 my $after;
1523  
1524 local $/='>'; # read up to the next '>'
1525  
1526 # print "read pos 0<br>";
1527 read(F,$before{$pos[0]},$pos[0]);
1528 $link{$pos[0]}=<F>;
1529 if (($pos[1]-tell(F))>0){
1530 read(F,$before{$pos[1]},$pos[1]-tell(F));
1531 }
1532 # print "read pos 1<br>";
1533 $link{$pos[1]}=<F>;
1534 # print "read rest<br>";
1535 while (<F>){$after.=$_;} # read up to end-of-file
1536 close F; # and close the files!
1537  
1538 if ($link{$$param{mox}}=~/xtargets=\"(.*?)\;(.*?)\"/){
1539 my ($src,$trg)=($1,$2);
1540 if ($$param{ms} eq 'src'){
1541 my @w=split(/\s/,$src);
1542 @w=grep ($_ ne $$param{mid},@w);
1543 $src=join (' ',@w);
1544 }
1545 if ($$param{ms} eq 'trg'){
1546 my @w=split(/\s/,$trg);
1547 @w=grep ($_ ne $$param{mid},@w);
1548 $trg=join ('s',@w);
1549 }
1550 $link{$$param{mox}}=~s/(xtargets=\").*?(\")/$1$src;$trg$2/s;
1551 }
1552  
1553  
1554 if ($link{$$param{mnx}}=~/xtargets=\"(.*?)\;(.*?)\"/){
1555 my ($src,$trg)=($1,$2);
1556 if ($$param{ms} eq 'src'){
1557 my @w=split(/\s/,$src);
1558 if ($$param{mox}>$$param{mnx}){push (@w,$$param{mid});}
1559 else{unshift (@w,$$param{mid});}
1560 $src=join (' ',@w);
1561 }
1562 if ($$param{ms} eq 'trg'){
1563 my @w=split(/\s/,$trg);
1564 if ($$param{mox}>$$param{mnx}){push (@w,$$param{mid});}
1565 else{unshift (@w,$$param{mid});}
1566 $trg=join (' ',@w);
1567 }
1568 $link{$$param{mnx}}=~s/(xtargets=\").*?(\")/$1$src;$trg$2/s;
1569 }
1570  
1571 # print &pre(escapeHTML($before{$pos[0]})),'<hr>';
1572 # print &pre(escapeHTML($link{$pos[0]})),'<hr>';
1573 # print &pre(escapeHTML($before{$pos[1]})),'<hr>';
1574 # print &pre(escapeHTML($link{$pos[1]})),'<hr>';
1575 # print escapeHTML($after),'<hr>';
1576 open F,"> $file";
1577 binmode(F);
1578 print F $before{$pos[0]};
1579 print F $link{$pos[0]};
1580 print F $before{$pos[1]};
1581 print F $link{$pos[1]};
1582 print F $after;
1583 close F;
1584  
1585 ##
1586 ## unlocking file (when locked with flock)
1587 ##
1588 # close LCK;
1589  
1590 ##
1591 ## unlocking file (when locked with nflock)
1592 ##
1593  
1594 &nunflock($file);
1595  
1596 }
1597  
1598  
1599 sub readLinks{
1600 my $self=shift;
1601 my ($pos)=@_;
1602  
1603 my $file=$self->{FILE};
1604 open F,"< $file";
1605 binmode(F);
1606 local $/='>';
1607  
1608 my $parser=new XML::Parser(Handlers => {Start => \&XmlStart,
1609 End => \&XmlEnd,
1610 Default => \&XmlChar});
1611 my $handle=$parser->parse_start;
1612 $self->{SENTLINKS}=[];
1613 $self->{ID}=[];
1614 $self->{FILEPOS}=[];
1615 $self->{ENDPOS}=[];
1616 $self->{WORDLINKS}=[];
1617 $handle->{ROOT}=$self->{ROOT};
1618 &ParseBitextHeader(*F,$handle); # read bitext header (get src/trg-doc)
1619 if ($pos>0){seek (F,$pos,0);} # go to last file position
1620  
1621 my $count;
1622 # push (@{$self->{FILEPOS}},tell(F)); # save file position!
1623 while (&ParseXml(*F,$handle)){
1624 $count++;
1625  
1626 push (@{$self->{ID}},$handle->{DATA}->{id}); # save ALIGN-ID's
1627 # push (@{$self->{FILEPOS}},tell(F)); # save file position!
1628 push (@{$self->{FILEPOS}},$handle->{STARTPOS});# save file position!
1629 if ($count>$MAXVIEWDATA){last;} # 1 segment look-ahead
1630 $self->{NEXT}=tell(F);
1631 push (@{$self->{SENTLINKS}},$handle->{DATA}->{xtargets}); # sent-links
1632  
1633 if (ref($handle->{SUBDATA}) ne 'HASH'){next;}
1634 if (ref($handle->{SUBDATA}->{wordLink}) ne 'ARRAY'){next;}
1635 my $i=$#{$self->{SENTLINKS}};
1636 foreach (0..$#{$handle->{SUBDATA}->{wordLink}}){
1637 if (ref($handle->{SUBDATA}->{wordLink}->[$_]) ne 'HASH'){next;}
1638 my $key=$handle->{SUBDATA}->{wordLink}->[$_]->{xtargets};
1639 %{$self->{WORDLINKS}->[$i]->{$key}}=
1640 %{$handle->{SUBDATA}->{wordLink}->[$_]};
1641 }
1642 }
1643 if (not $handle->{FROMDOC}){print "no source document found!";return 0;}
1644 if (not $handle->{TODOC}){print "no target document found!";return 0;}
1645 close F;
1646 $self->{FROMDOC}=$handle->{FROMDOC};
1647 $self->{TODOC}=$handle->{TODOC};
1648 $self->{COUNT}=$count;
1649 return 1;
1650 }
1651  
1652  
1653  
1654 sub readSentLinks{
1655 my $self=shift;
1656 my $style=shift;
1657 my @rows=();
1658 my $LastID;
1659 my $LastPos;
1660 foreach (@{$self->{SENTLINKS}}){
1661 $self->{PREV_ALIGN_ID}=$LastID;
1662 $self->{THIS_ALIGN_ID}=shift(@{$self->{ID}});
1663 $self->{NEXT_ALIGN_ID}=$self->{ID}->[0];
1664 $self->{PREV_ALIGN_POS}=$LastPos;
1665 $self->{THIS_ALIGN_POS}=shift(@{$self->{FILEPOS}});
1666 $self->{NEXT_ALIGN_POS}=$self->{FILEPOS}->[0];
1667 push (@rows,$self->readBitextSegment($_,$style));
1668 $LastID=$self->{THIS_ALIGN_ID};
1669 $LastPos=$self->{THIS_ALIGN_POS};
1670 }
1671 $self->{'FROMDOC-POS'}=tell $self->{'FROMDOCHANDLE'};
1672 $self->{'TODOC-POS'}=tell $self->{'TODOCHANDLE'};
1673 return &table({},caption(''),&Tr(\@rows));
1674 }
1675  
1676  
1677 sub readBitextSegment{
1678 my $self=shift;
1679 my $link=shift;
1680 my $style=shift;
1681  
1682 my ($src,$trg)=split(/\s*\;\s*/,$link);
1683 my @s=split(/\s+/,$src);
1684 my @t=split(/\s+/,$trg);
1685  
1686 my $src='';my $trg='';
1687 my $url=$self->{URL};
1688 my $url=&Uplug::Web::AddUrlParam($url,'ms','src'); # src sentence
1689 foreach (@s){$src.=$self->sentLinkHeader($url,$_);}
1690 my $url=&Uplug::Web::AddUrlParam($url,'ms','trg'); # trg sentence
1691 foreach (@t){$trg.=$self->sentLinkHeader($url,$_);}
1692  
1693 my $srctext=$self->readSegment('FROMDOC',\@s,$style);
1694 my $trgtext=$self->readSegment('TODOC',\@t,$style);
1695  
1696 my @rows=();
1697 push (@rows,&th([$src,$trg]));
1698 push (@rows,&td([$srctext,$trgtext]));
1699 return @rows;
1700 }
1701  
1702 sub sentLinkHeader{
1703 my $self=shift;
1704 my ($url,$sent)=@_;
1705  
1706 my $old=$self->{THIS_ALIGN_ID}; # old bitex segment (this one)
1707 my $up=$self->{PREV_ALIGN_ID}; # the one before
1708 my $down=$self->{NEXT_ALIGN_ID}; # the one before
1709  
1710 $url=&Uplug::Web::AddUrlParam($url,'mid',$sent); # sentence ID
1711 $url=&Uplug::Web::AddUrlParam($url,'mo',$old); # old ID (bitext segment)
1712 $url=&Uplug::Web::AddUrlParam($url,'mox',$self->{THIS_ALIGN_POS});
1713 my $html;
1714 if ($up and ($self->{PRIV} ne 'all')){
1715 $url=&Uplug::Web::AddUrlParam($url,'mn',$up); # new ID (up)
1716 $url=&Uplug::Web::AddUrlParam($url,'mnx',$self->{PREV_ALIGN_POS});
1717 $html=&a({-href=>$url},'&uArr;');
1718 }
1719 $html.=$sent;
1720 if ($down and ($self->{PRIV} ne 'all')){
1721 $url=&Uplug::Web::AddUrlParam($url,'mn',$down); # new ID (down)
1722 $url=&Uplug::Web::AddUrlParam($url,'mnx',$self->{NEXT_ALIGN_POS});
1723 $html.=&a({-href=>$url},'&dArr;');
1724 }
1725 return $html.'&nbsp;';
1726 }
1727  
1728 sub readSegment{
1729 my $self=shift;
1730 my $doc=shift;
1731 my $ids=shift;
1732 my $style=shift;
1733  
1734 if (not ref($self->{$doc.'HANDLE'})){$self->openDocument($doc);}
1735 my $fh=$self->{$doc.'HANDLE'};
1736 my $parser=$self->{$doc.'PARSER'};
1737  
1738 my $text='';
1739 delete $parser->{SUBDATA};
1740 my $failed=0;
1741 my $startpos=tell($fh);
1742 while (@{$ids} and &ParseXml($fh,$parser,1)){
1743 # print "try to find $$ids[0] ($self->{$doc},$startpos) ...";
1744 if ($$ids[0] eq $parser->{DATA}->{id}){
1745 if ($style eq 'text'){$text.=$parser->{HTMLTXT};}
1746 else{$text.=$parser->{HTML};}
1747 shift(@{$ids});
1748 # print " found!";
1749 }
1750 else{
1751 # print " next ...";
1752 $failed++; # wrong segment-ID
1753 delete $parser->{SUBDATA}; # delete segment
1754 if ($failed>5){ # skip no more than 5 segments
1755 if ($startpos==0){last;} # started at the beginning->stop
1756 $startpos-=500; # enough! go 500 bytes back
1757 if ($startpos<0){$startpos=0;} # (but not more than filestart)
1758 seek($fh,$startpos,0); # move the pointer
1759 $failed=0; # reset fail counter
1760 # print "start=$startpos ...";
1761 }
1762 }
1763 # print "<br>";
1764 if (not @{$ids}){last;}
1765 }
1766 if ($style ne 'text'){$text=&pre($text);}
1767 return $text;
1768 }
1769  
1770 #-------------------------------------------------
1771 # open XML-documents
1772 # * open the file
1773 # * create a XML-parser-object
1774 # * read the XML-header and the XML-root-tag
1775  
1776 sub openDocument{
1777 my $self=shift;
1778 my $doc=shift;
1779  
1780 open $self->{$doc.'HANDLE'},"< $self->{$doc}";
1781 binmode($self->{$doc.'HANDLE'});
1782 $self->{$doc.'EXPAT'}=new XML::Parser(Handlers => {Start => \&XmlStart,
1783 End => \&XmlEnd,
1784 Default => \&XmlChar});
1785 $self->{$doc.'PARSER'}=$self->{$doc.'EXPAT'}->parse_start();
1786 $self->{$doc.'PARSER'}->{ROOT}='s';
1787 $self->{$doc.'PARSER'}->{SUBROOT}='w';
1788  
1789 my $fh=$self->{$doc.'HANDLE'}; # the file handle
1790 my $parser=$self->{$doc.'PARSER'}; # the XML-parser
1791 local $/='>'; # set input boundary to '>'
1792 my $xml=<$fh>; # simply read the XML header
1793 eval { $parser->parse_more($xml); }; # parse XML-header
1794 $xml=<$fh>; # read the root-tag of the document
1795 eval { $parser->parse_more($xml); }; # ... and parse it
1796  
1797 if ($self->{$doc.'-POS'}>0){ # go to the last file-position
1798 # print "seek $doc-pos: $self->{$doc.'-POS'}<hr>";
1799 seek ($fh,$self->{$doc.'-POS'},0); # (if > 0)
1800 }
1801 }
1802  
1803  
1804 #--------------------------------------------
1805 # parse the header of a bitext file
1806 # * parse XML header and
1807 # * look for source and target documents
1808  
1809 sub ParseBitextHeader{
1810 my ($fh,$p)=@_;
1811  
1812 delete $p->{DATA};
1813 delete $p->{OPEN};
1814 delete $p->{SUBOPEN};
1815 delete $p->{COMPLETE};
1816 delete $p->{INSIDE};
1817 delete $p->{BEFORE};
1818 delete $p->{INSIDETXT};
1819 delete $p->{BEFORETXT};
1820 delete $p->{HTML};
1821 delete $p->{HTMLTXT};
1822 delete $p->{FROMDOC};
1823 delete $p->{TODOC};
1824  
1825 while (<$fh>){
1826 eval { $p->parse_more($_); };
1827 if ($@){print "problems when parsing ($@)!\n";return 0;}
1828 if ($p->{FROMDOC} and $p->{TODOC}){return 1;}
1829 }
1830 return 0;
1831 }
1832  
1833  
1834 #------------------------------------------------
1835 # parse XML until a complete ROOT-subtree is found
1836  
1837 sub ParseXml{
1838 my ($fh,$p,$keepSubData)=@_;
1839  
1840 delete $p->{DATA};
1841 delete $p->{OPEN};
1842 delete $p->{SUBOPEN};
1843 delete $p->{COMPLETE};
1844 delete $p->{INSIDE};
1845 delete $p->{BEFORE};
1846 delete $p->{INSIDETXT};
1847 delete $p->{BEFORETXT};
1848 delete $p->{HTML};
1849 delete $p->{HTMLTXT};
1850 if (not $keepSubData){delete $p->{SUBDATA};}
1851  
1852 local $/='>'; # set input boundary to '>'
1853 my $pos=tell $fh; # save the current file position
1854 while (<$fh>){ # go to next ROOT-tag
1855 if (/\<$p->{ROOT}(\s|\>)/){last;} # (avoid not welformed XML
1856 $pos=tell $fh; # when jumping within the file)
1857 }
1858 seek ($fh,$pos,0);
1859 $p->{STARTPOS}=$pos;
1860  
1861 while (<$fh>){ # read from the file
1862 eval { $p->parse_more($_); }; # and parse the XML-string
1863 if ($@){
1864 s/</&lt;/g;s/</&gt;/g;
1865 print "problems when parsing ($@)! XML-string: $_\n";
1866 return 0;
1867 }
1868 if ($p->{COMPLETE}){return 1;}
1869 }
1870 return 0;
1871 }
1872  
1873  
1874 #-----------------------------------------------------------------------
1875 # XML-parser subroutines
1876 # XmlStart .... XML-start-tag
1877 # XmlEnd ...... XML-end-tag
1878 # XmlChar ..... everything else
1879  
1880 sub XmlStart{
1881 my ($p,$e,%attr)=@_;
1882 my $text=$p->recognized_string();
1883 if ($e eq $p->{ROOT}){
1884 $p->{OPEN}=1;
1885 %{$p->{DATA}}=%attr;
1886 }
1887 if ($p->{OPEN}){
1888 $p->{INSIDE}.=$text;
1889 $p->{HTML}.=&escapeHTML($text);
1890 if ($e ne $p->{ROOT}){
1891 if (not ref($p->{SUBDATA})){$p->{SUBDATA}={};}
1892 if (not ref($p->{SUBDATA}->{$e})){$p->{SUBDATA}->{$e}=[];}
1893 my $i=@{$p->{SUBDATA}->{$e}};
1894 %{$p->{SUBDATA}->{$e}->[$i]}=%attr;
1895 if ($e eq $p->{SUBROOT}){
1896 $p->{SUBOPEN}=1;
1897 $p->{SUBDATA}->{$e}->[$i]->{'#text'}='';
1898 }
1899 }
1900 }
1901 else{$p->{BEFORE}.=$p->recognized_string();}
1902 if (defined $attr{fromDoc}){
1903 $p->{FROMDOC}=$attr{fromDoc};
1904 }
1905 if (defined $attr{toDoc}){
1906 $p->{TODOC}=$attr{toDoc};
1907 }
1908 }
1909  
1910 sub XmlEnd{
1911 my ($p,$e)=@_;
1912 my $text=$p->recognized_string();
1913 if ($p->{OPEN}){
1914 $p->{INSIDE}.=$p->recognized_string();
1915 $p->{HTML}.=&escapeHTML($text);
1916 }
1917 if ($e eq $p->{ROOT}){
1918 delete $p->{OPEN};
1919 $p->{COMPLETE}=1;
1920 }
1921 elsif ($e eq $p->{SUBROOT}){
1922 delete $p->{SUBOPEN};
1923 }
1924 }
1925  
1926 sub XmlChar{
1927 my ($p)=@_;
1928 my $text=$p->recognized_string();
1929  
1930 if ($p->{OPEN}){
1931 $p->{INSIDE}.=$text;
1932 $p->{INSIDETXT}.=$text;
1933 $p->{HTML}.=&escapeHTML($text);
1934 $p->{HTMLTXT}.=&escapeHTML($text);
1935 if ($p->{SUBOPEN}){
1936 $p->{SUBDATA}->{$p->{SUBROOT}}->[-1]->{'#text'}.=$text;
1937 }
1938 }
1939 else{
1940 $p->{BEFORETXT}.=$text;
1941 $p->{BEFORE}.=$text;
1942 }
1943 }
1944  
1945  
1946  
1947  
1948 #############################################################################
1949 #############################################################################
1950 #
1951 # word-aligned bitexts
1952 #
1953  
1954  
1955 package Uplug::Web::BitextLinks;
1956  
1957 use CGI qw/:standard escapeHTML escape/;
1958 use vars qw(@ISA);
1959 @ISA = qw( Uplug::Web::Bitext );
1960  
1961 use Uplug::Web::Process::Lock;
1962  
1963 sub new{
1964 my $class=shift;
1965 my $self=$class->SUPER::new(@_);
1966 # $self->{STYLES}=['text','xml','matrix','edit'];
1967 $self->{ROOT}='link';
1968 return $self;
1969 }
1970  
1971  
1972  
1973 sub view{
1974 my $self=shift;
1975 my ($url,$style,$pos,$params)=@_;
1976  
1977 $self->{URL}=$url;
1978 if (ref($params) eq 'HASH'){
1979 $self->{'FROMDOC-POS'}=$$params{sx};
1980 $self->{'TODOC-POS'}=$$params{tx};
1981 $url=&Uplug::Web::AddUrlParam($url,'sx',$self->{'FROMDOC-POS'});
1982 $url=&Uplug::Web::AddUrlParam($url,'tx',$self->{'TODOC-POS'});
1983 # &param('sx',$$params{sx});
1984 # &param('sx',$$params{tx});
1985 $self->{URL}=$url;
1986 if ($$params{edit} eq 'change'){
1987 # print join('<br>',%{$params});
1988 $self->changeLinks($params);
1989 $url=Uplug::Web::DelUrlParam($url,'start');
1990 $url=Uplug::Web::DelUrlParam($url,'end');
1991 $url=Uplug::Web::DelUrlParam($url,'seg');
1992 $url=Uplug::Web::DelUrlParam($url,'links');
1993 $url=Uplug::Web::DelUrlParam($url,'edit');
1994 }
1995 }
1996  
1997 if (not $style){$style='text';}
1998 if ($style eq 'xml'){
1999 return $self->Uplug::Web::Data::view($url,$style,$pos);
2000 }
2001 if ($style eq 'edit'){$MAXVIEWDATA=1;}
2002 return $self->SUPER::view($url,$style,$pos);
2003 }
2004  
2005  
2006 sub changeLinks{
2007 my $self=shift;
2008 my $param=shift;
2009  
2010 my $file=$self->{FILE};
2011 if (not -e $file){return 0;}
2012  
2013  
2014 ##
2015 ## file locking with flock
2016 ##
2017 # my $LOCK=$file.'.lock'; # lock the lock-file
2018 # open LCK,"+<$file\.lock";
2019 # my $sec=0;
2020 # while (not flock(LCK,2)){
2021 # $sec++;sleep(1);
2022 # if ($sec>$MAXFLOCKWAIT){
2023 # close LCK;
2024 # return 0;
2025 # }
2026 # }
2027  
2028 ##
2029 ## file locking with nflock
2030 ##
2031  
2032 if (not &nflock($file,$MAXFLOCKWAIT)){
2033 print STDERR "# Uplug::Web - can't get exclusive lock for $file!\n";
2034 return 0;
2035 }
2036  
2037  
2038 open F,"< $file";
2039 # binmode(F,":utf8");
2040 binmode(F);
2041 my ($before,$old,$after);
2042 if ($$param{start}){
2043 read(F,$before,$$param{start});
2044 if (tell(F)!=$$param{start}){
2045 print "something wrong (file positions)!".&br();
2046 return 0;
2047 }
2048 # my $wrong=tell(F)-$$param{start};
2049 # if ($wrong<0){my $new;read(F,$new,-$wrong);$before.=$new;}
2050 # if ($wrong>0){for (0..$wrong){$before=~s/.$//;}}
2051 # seek(F,$$param{start},0);
2052 }
2053 read(F,$old,$$param{end}-$$param{start});
2054 # my $remove=tell(F)-$$param{end}-2;
2055 # if ($remove){for (0..$remove){$after=~s/.$//;}}
2056 # print "remove: $remove<hr>";
2057 seek(F,$$param{end},0);
2058 while (<F>){$after.=$_;}
2059 close F;
2060  
2061 #--------------------------------------------------------------------------
2062 my %src; # source token
2063 my %trg; # target token
2064 my @lex; # lexical pairs (lexPair attribute)
2065 my @links=&param('links'); # all 1:1 word links
2066 # @links= map {$_->[0] } # sort link ID's by the last digit
2067 # sort { # (=sorted by target ID's)
2068 # my @af=@$a[1..$#$a]; # this map-sort-map is taken from
2069 # my @bf=@$b[1..$#$b]; # Programming Perl
2070 # $af[-1] <=> $bf[-1]; # (the "Schwartzian Transform")
2071 # }
2072 # map { [$_,split /./] } @links;
2073 my %srclink=(); # save link-cluster for src-tokens
2074 my %trglink=(); # save link-cluster for src-tokens
2075 my @xtrg=(); # list of link clusters
2076 foreach (@links){ # for each 1:1 word link
2077 my ($s,$t)=split(/:/); # split src and trg ID
2078 if (not defined $src{$s}){
2079 $src{$s}=param("S$s"); # get the source token
2080 $src{$s}=Encode::decode('utf-8', # (decode UTF-8)
2081 $src{$s});
2082 }
2083 if (not defined $trg{$t}){
2084 $trg{$t}=param("T$t"); # get the target token
2085 $trg{$t}=Encode::decode('utf-8',
2086 $trg{$t});
2087 }
2088 my $i;
2089 if (not defined $srclink{$s}){ # src tokens is not part of any xtrg:
2090 if (not defined $trglink{$t}){# neither is the trg token
2091 $i=$#xtrg+1; # --> just create a new xtrg
2092 @{$xtrg[$i]}=($s,$t); # (easy!)
2093 @{$lex[$i]}=($src{$s},
2094 $trg{$t});
2095 }
2096 else{ # trg token is part of another xtrg
2097 $i=$trglink{$t}; # --> add source token to this xtrg
2098 $xtrg[$i][0].="+$s";
2099 $lex[$i][0].=" $src{$s}";
2100 }
2101 }
2102 elsif (not defined $trglink{$t}){ # src token is part of another xtrg
2103 $i=$srclink{$s}; # AND trg token is not:
2104 $xtrg[$i][1].="+$t"; # --> add target token to this xtrg
2105 $lex[$i][1].=" $trg{$t}";
2106 }
2107 else{ # the hardest case: both are part of
2108 $i=$srclink{$s}; # other xtrg's:
2109 my $j=$trglink{$t}; # join the two xtrgs!
2110 if ($i!=$j){
2111 $xtrg[$i][0].="+$xtrg[$j][0]";
2112 $xtrg[$i][1].="+$xtrg[$j][1]";
2113 splice(@xtrg,$j,1);
2114 $lex[$i][0].=" $lex[$j][0]";
2115 $lex[$i][1].=" $lex[$j][1]";
2116 splice(@lex,$j,1);
2117 }
2118 }
2119 $srclink{$s}=$i; # save the index of the xtrg both
2120 $trglink{$t}=$i; # tokens are part of
2121 }
2122 #--------------------------------------------------------------------------
2123  
2124 $old=~s/^(.*?\<link\s[^>]*?)\/?(\>).*$/$1$2/s; # keep only <link ...>
2125 $old.="\n";
2126 foreach (0..$#xtrg){
2127 $old.=" <wordLink ";
2128 $lex[$_][0] = _escapeLiteral($lex[$_][0]);
2129 $lex[$_][1] = _escapeLiteral($lex[$_][1]);
2130 $old.="lexPair=\"$lex[$_][0];$lex[$_][1]\" ";
2131 $old.="xtargets=\"$xtrg[$_][0];$xtrg[$_][1]\" />\n";
2132 }
2133 $old.='</link>';
2134  
2135 open F,"> $file";
2136 binmode(F);
2137 # binmode(F,":utf8");
2138 print F $before;
2139 $self->{POS}=tell(F);
2140 $old=Encode::encode('utf-8',$old);
2141 print F $old;
2142 $self->{NEXT}=tell(F);
2143 print F $after;
2144 close F;
2145  
2146 ##
2147 ## unlocking file (when locked with flock)
2148 ##
2149 # close LCK;
2150  
2151 ##
2152 ## unlocking file (when locked with nflock)
2153 ##
2154  
2155 &nunflock($file);
2156  
2157 param('start',$self->{POS});
2158 param('end',$self->{NEXT});
2159 }
2160  
2161  
2162 sub readSentLinks{
2163 my $self=shift;
2164 my $style=shift;
2165  
2166 my @rows=();
2167 my $LastID;
2168 my $LastPos;
2169 foreach my $l (0..$#{$self->{SENTLINKS}}){
2170 $self->{PREV_ALIGN_ID}=$LastID;
2171 $self->{THIS_ALIGN_ID}=shift(@{$self->{ID}});
2172 $self->{NEXT_ALIGN_ID}=$self->{ID}->[0];
2173 $self->{PREV_ALIGN_POS}=$LastPos;
2174 $self->{THIS_ALIGN_POS}=shift(@{$self->{FILEPOS}});
2175 $self->{NEXT_ALIGN_POS}=$self->{FILEPOS}->[0];
2176 push (@rows,$self->readBitextSegment($self->{SENTLINKS}->[$l],
2177 $self->{WORDLINKS}->[$l],
2178 $style));
2179 if ($style eq 'text'){
2180 push (@rows,$self->viewWordLinks($self->{WORDLINKS}->[$l]));
2181 }
2182 $LastID=$self->{THIS_ALIGN_ID};
2183 $LastPos=$self->{THIS_ALIGN_POS};
2184 }
2185 $self->{'FROMDOC-POS'}=tell $self->{'FROMDOCHANDLE'};
2186 $self->{'TODOC-POS'}=tell $self->{'TODOCHANDLE'};
2187 if ($style=~/(matrix|edit)/){return join '<hr>',@rows;}
2188 else{return &table({},caption(''),&Tr(\@rows));}
2189 }
2190  
2191 sub viewWordLinks{
2192 my $self=shift;
2193 my $links=shift;
2194 if (ref($links) ne 'HASH'){$links={};}
2195 my @rows=();
2196 foreach (sort {$$links{$b}{certainty} <=> $$links{$a}{certainty}}
2197 keys %{$links}){
2198 if (ref($$links{$_}) ne 'HASH'){next;}
2199 my ($src,$trg)=split(';',$$links{$_}{lexPair});
2200 my $score=sprintf "%1.5f",$$links{$_}{certainty};
2201 push (@rows,
2202 &td({-align => 'right'},[$src.'&nbsp;&nbsp;']).
2203 &td(['&nbsp;&nbsp;'.$trg,$score]))
2204 }
2205 return @rows;
2206 }
2207  
2208  
2209 sub readBitextSegment{
2210 my $self=shift;
2211 my $sentLink=shift;
2212 my $wordLinks=shift;
2213 my $style=shift;
2214  
2215 my ($src,$trg)=split(/\s*\;\s*/,$sentLink);
2216 my @s=split(/\s+/,$src);
2217 my @t=split(/\s+/,$trg);
2218  
2219 my $srctext=$self->readSegment('FROMDOC',\@s,$style);
2220 my $trgtext=$self->readSegment('TODOC',\@t,$style);
2221  
2222 if ($style eq 'matrix'){
2223 return $self->linkMatrix($wordLinks,$sentLink);
2224 }
2225 if ($style eq 'edit'){
2226 return $self->linkMatrix($wordLinks,$sentLink,1);
2227 }
2228 else{
2229 my @rows=();
2230 push (@rows,&th([$src,$trg]));
2231 push (@rows,&td([$srctext,$trgtext]));
2232 return @rows;
2233 }
2234 }
2235  
2236  
2237  
2238  
2239 sub linkMatrix{
2240 my $self=shift;
2241 my ($links,$id,$form)=@_;
2242  
2243 my $src=$self->{FROMDOCPARSER};
2244 my $trg=$self->{TODOCPARSER};
2245 if (ref($links) ne 'HASH'){$links={};}
2246  
2247 my $srcTok=$src->{SUBDATA}->{$src->{SUBROOT}};
2248 my $trgTok=$trg->{SUBDATA}->{$trg->{SUBROOT}};
2249  
2250 my %matrix=();
2251 foreach (keys %{$links}){
2252 my ($srcX,$trgX)=split(/\s*\;\s*/,$_);
2253 my @src=split(/\+/,$srcX);
2254 my @trg=split(/\+/,$trgX);
2255 foreach my $s (@src){
2256 foreach my $t (@trg){
2257 $matrix{$s}{$t}=1;
2258 }
2259 }
2260 }
2261 my $html='';
2262 if ($form){
2263 $html.=&startform();
2264 $html.=hidden(-name=>'start',-default=>[$self->{POS}]); # start and
2265 $html.=hidden(-name=>'end',-default=>[$self->{NEXT}]); # end position
2266 $html.=hidden(-name=>'seg',-default=>[$id]); # seg ID
2267 $html.=hidden(-name=>'sx',-default=>[$self->{'FROMDOC-POS'}]);
2268 $html.=hidden(-name=>'tx',-default=>[$self->{'TODOC-POS'}]);
2269 }
2270 my @rows=();
2271 push (@rows,&th([$id]));
2272 foreach my $t (0..$#{$trgTok}){
2273 $rows[0].=&td($trgTok->[$t]->{'#text'});
2274 if ($form){
2275 my $key='T'.$trgTok->[$t]->{id}; # save target tokens
2276 my $val=$trgTok->[$t]->{'#text'};
2277 param($key,$val);
2278 $html.=hidden(-name=>$key,-default=>[$val]);
2279 }
2280 }
2281 foreach my $s (0..$#{$srcTok}){
2282 my $row=&td($srcTok->[$s]->{'#text'});
2283 if ($form){
2284 my $key='S'.$srcTok->[$s]->{id};
2285 my $val=$srcTok->[$s]->{'#text'};
2286 param($key,$val);
2287 $html.=hidden(-name=>$key,-default=>[$val]);
2288 }
2289 foreach my $t (0..$#{$trgTok}){
2290 my $value="$srcTok->[$s]->{id}:$trgTok->[$t]->{id}";
2291 my $cell='';
2292 if ($matrix{$srcTok->[$s]->{id}}{$trgTok->[$t]->{id}}){
2293 if ($form){
2294 $cell=&checkbox(-name=>'links',-checked=>'checked',
2295 -value=>$value,-label=>'');
2296 # $cell.=&checkbox(-name=>'fuzzy',-checked=>'checked',
2297 # -value=>$value,-label=>'');
2298 }
2299 $row.=&td({},[$cell]);
2300 }
2301 else{
2302 if ($form){
2303 $cell=&checkbox(-name=>'links',-value=>$value,-label=>'');
2304 # $cell.=&checkbox(-name=>'fuzzy',-value=>$value,-label=>'');
2305 }
2306 $row.=&th({},[$cell]);
2307 }
2308 }
2309 $row.=&td($srcTok->[$s]->{'#text'});
2310 push (@rows,$row);
2311 }
2312 push (@rows,&th(['']));
2313 foreach my $t (0..$#{$trgTok}){
2314 $rows[-1].=&td($trgTok->[$t]->{'#text'});
2315 }
2316 $html.='<div class="matrix">';
2317 $html.=&table({},caption(''),&Tr(\@rows));
2318 $html.="</div>\n";
2319 if ($form){
2320 $html.=&p().&submit(-name => 'edit',-value => 'change');
2321 $html.=&endform();
2322 }
2323 return $html;
2324 }
2325  
2326 #
2327 # Private: escape an attribute value literal.
2328 # (taken from XML::Writer)
2329 #
2330 sub _escapeLiteral {
2331 my $data = $_[0];
2332 if ($data =~ /[\&\<\>\"]/) {
2333 $data =~ s/\&/\&amp\;/g;
2334 $data =~ s/\</\&lt\;/g;
2335 $data =~ s/\>/\&gt\;/g;
2336 $data =~ s/\"/\&quot\;/g;
2337 }
2338 return $data;
2339 }
2340  
2341  
2342 1;
2343  
2344 __END__