blib/lib/Uplug/Web/Process.pm untested

filter: Statementstmt 0.0% 0 / 346 Branchbran 0.0% 0 / 104 Conditioncond 0.0% 0 / 3 MC/DCmcdc 0.0% 0 / 2 Subroutinesub 0.0% 0 / 33 Podpod 0.0% 0 / 22 totaltotal 0.0% 0 / 510 CC 51 SCAR 39.3
CC
51
Cov
0%
PrepareProcess
182.0
ExpandParameter
110.0
GetSubmodules
30.0
CRAP 51.1
1 # Copyright (C) 2004 Jörg Tiedemann <joerg@stp.ling.uu.se>
2 #
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
7 #
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
12 #
13 # You should have received a copy of the GNU General Public License
14 # along with this program; if not, write to the Free Software
15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16  
17 package Uplug::Web::Process;
18  
19 use strict;
20 use IO::File;
21 use POSIX qw(tmpnam);
22 use File::Copy;
23 use File::Basename;
24 use ExtUtils::Command;
25  
26 use Uplug::Web;
27 use Uplug::Web::Process::Stack;
28 use Uplug::Web::User;
29 use Uplug::Web::Corpus;
30 use Uplug::Config;
31  
32  
33 our $UPLUG=$ENV{UPLUGHOME}.'/uplug';
34  
35 #-------------------------------------------------------------
36 # RUN: script for running jobs
37 # CLUEDBMFILES: file-base for word alignment clue databases
38  
39 my $RUN=$ENV{UPLUGHOME}.'/web/bin/run.pl';
40 my $INDEX=$ENV{UPLUGHOME}.'/web/bin/index.pl';
41 my @CLUEDBMFILES=('giza','dice','mi','tscore','str',
42 'chunk','chunktri','chunktripos','chunktriposi',
43 'lex','lexpos','pos','pos_coarse','position',
44 'posposi','postri','postriposi');
45 #-------------------------------------------------------------
46  
47 my $ProcHome = $ENV{UPLUGDATA}.'/.process';
48  
49 my $todoFile = $ProcHome.'/.todo';
50 my $queuedFile = $ProcHome.'/.queued';
51 my $workingFile = $ProcHome.'/.working';
52 my $doneFile = $ProcHome.'/.done';
53 my $failedFile = $ProcHome.'/.failed';
54 my $serverlogFile = $ProcHome.'/.serverlog';
55  
56 my $NewProcDir=0;
57 if (not -e $ProcHome){ # if the Process directory does not exist:
58 mkdir $ProcHome; # - create it
59 $NewProcDir=1; # - and set a flag that it has been created
60 }
61  
62 #if (not -e "$ProcHome/.lockdir"){ # Lock.pm changed:
63 # mkdir "$ProcHome/.lockdir"; # don't use a special .lockdir
64 # system "chmod g+w $ProcHome/.lockdir"; # for locking anymore!
65 #} # (lock-files will be in ProcHome)
66  
67 my $todo=Uplug::Web::Process::Stack->new($todoFile); # new stack-objects
68 my $queued=Uplug::Web::Process::Stack->new($queuedFile); # (this will even
69 my $working=Uplug::Web::Process::Stack->new($workingFile); # create new stack
70 my $done=Uplug::Web::Process::Stack->new($doneFile); # files if they
71 my $failed=Uplug::Web::Process::Stack->new($failedFile); # do not exist!)
72  
73 if ($NewProcDir){ # if the process dir is new:
74 system "chmod g+w $ProcHome"; # add write mode (has to be after
75 } # creating stack files above!)
76  
77 sub AvailableUplugSystems{
78 return &GetApplications(@_);
79 }
80  
81 sub GetSubmodules{
82 my $user=shift;
83 my $module=shift;
84 if (ref($module) ne 'HASH'){
85 my %config=();
86 &GetConfiguration(\%config,$user,$module);
87 $module=$config{module};
88 }
89 if (ref($module) ne 'HASH'){return undef;}
90 if (ref($$module{submodules}) ne 'ARRAY'){return undef;}
91 my $names=$$module{submodules};
92 if (ref($$module{'submodule names'}) eq 'ARRAY'){
93 $names=$$module{'submodule names'};
94 }
95 my @mod=();
96 foreach (0..$#{$$module{submodules}}){
97 push (@mod,$$module{submodules}[$_]);
98 push (@mod,$$names[$_]);
99 }
100 return @mod;
101 }
102  
103  
104 sub UplugSystemIni{
105 my $data=shift;
106 my $user=shift;
107 my $name=shift;
108  
109 if (not -e $name){chdir &Uplug::Web::Corpus::GetCorpusDir($user);}
110 &Uplug::Config::ReadNamed('UserDataStreams.ini'); # reload named streams
111 # (they might have changed since
112 # last time reading them!!)
113 my $config=&ReadConfig($name);
114 %{$data}=%{$config};
115 return 1;
116 }
117  
118 sub GetConfiguration{
119 my $data=shift;
120 my $user=shift;
121 my $name=shift;
122  
123 chdir &Uplug::Web::Corpus::GetCorpusDir($user);
124 if (! -e $name){
125 if (-e $ENV{UPLUGHOME}.'/'.$name){
126 my $dir = dirname($name);
127 if (! -d $dir){system "mkdir -p $dir";}
128 copy ($ENV{UPLUGHOME}.'/'.$name,$name);
129 }
130 }
131 my $config=&ReadConfig($name);
132 %{$data}=%{$config};
133 return 1;
134 }
135  
136  
137 sub SaveUplugSettings{
138 my $user=shift;
139 my $configfile=shift;
140 my $para=shift;
141 # print join '<br>',%{$para};
142 if (not defined $configfile){return 0;}
143  
144 my $UserDir=&Uplug::Web::Corpus::GetCorpusDir($user);
145 chdir $UserDir;
146  
147 if (ref($para) eq 'ARRAY'){
148 my $config=&ReadConfig($configfile,@{$para});
149 return &WriteConfig($configfile,$config);
150 }
151 elsif (ref($para) eq 'HASH'){
152 my $config=&ReadConfig($configfile,%{$para});
153 return &WriteConfig($configfile,$config);
154 }
155 }
156  
157 sub ResetUplugSettings{
158 my $user=shift;
159 my $configfile=shift;
160 my $para=shift;
161 if (not defined $configfile){return 0;}
162  
163 my $UserDir=&Uplug::Web::Corpus::GetCorpusDir($user);
164 chdir $UserDir;
165 if (-f $configfile){unlink $configfile;}
166 }
167  
168  
169  
170 sub MakeIndexerProcess{
171 my ($user,$corpus,$srcenc,$trgenc)=@_;
172 my $command="$INDEX '$user' '$corpus' '$srcenc' '$trgenc'";
173 $todo->push($user,$corpus,$command);
174 }
175  
176 sub ViewLogfile{
177 my $user=shift;
178 my $process=shift;
179 my $UserDir=&Uplug::Web::Corpus::GetCorpusDir($user);
180 my $ProcessDir="$UserDir/$process";
181 if (-e "$ProcessDir/uplugweb.log"){
182 open F,"<$ProcessDir/uplugweb.log";
183 my @cont=<F>;
184 close F;
185 return join '',@cont;
186 }
187 return "There is no logfile for this process!";
188 }
189  
190 sub MakeUplugProcess{
191 my $user=shift;
192 my $corpus=shift;
193 my $configfile=shift;
194 my $para=shift;
195  
196 my $UserDir=&Uplug::Web::Corpus::GetCorpusDir($user);
197 my $DocConfig=&Uplug::Web::Corpus::DocumentConfigFile($user,$corpus);
198 my $ThisProcDir=$UserDir;
199 my $pid=$$;
200 my $time=time();
201 while (-d "$ThisProcDir/$time\_$pid"){$pid++;}
202 my $process=$time.'_'.$pid;
203 $ThisProcDir.='/'.$process;
204  
205 mkdir $ThisProcDir,0755;
206 mkdir "$ThisProcDir/ini",0755;
207 system "cp -R $UserDir/systems $ThisProcDir/"; # copy config files
208 copy ($DocConfig, # copy document config-
209 "$ThisProcDir/ini/UserDataStreams.ini"); # file to process-dir
210 open F,">$ThisProcDir/uplugweb.log";close F; # create a log-file
211 # chmod 0664,"$ThisProcDir/uplugweb.log"; # make it writable
212 system "chmod g+w $ThisProcDir/uplugweb.log"; # make it writable
213 # system "cp -R $UserDir/ini $ThisProcDir/"; # copy ini files
214 # system "chmod g+w $ThisProcDir";
215 chdir $ThisProcDir;
216  
217 open F,">/tmp/uplug-paras";
218 print F "save paras i $configfile\n";
219 close F;
220  
221 if (not defined $configfile){return 0;}
222  
223 &Uplug::Config::ReadNamed('UserDataStreams.ini');
224 my $config=&ReadConfig($configfile,%{$para});
225 &PrepareProcess($user,$corpus,$process,$configfile,$config);
226 open F,">>/tmp/uplug-paras";
227 print F "save paras i $configfile\n";
228 close F;
229 &WriteConfig($configfile,$config);
230  
231 $todo->push($user,$process,$corpus,$configfile);
232 return $process;
233 }
234  
235 ##---------------------------------------
236 ## PrepareProcess:
237 ## do some special preperations for certain Uplug processes
238 ## e.g. create output files, set permissions, change configurations
239 ## set postprocessing tasks for client processes (run.pl)
240 ## - config->postprocessing->STDOUT = file-to-save-stdout-in
241 ## - config->postprocessing->corpus = hash of new corpus streams
242 ## (to be set in user's corpus config)
243 ## this is highly specific and dependent on uplug modules!
244 ##
245  
246 sub PrepareProcess{
247 my ($user,$corpus,$process,$configfile,$config)=@_;
248  
249 ## 2) touch output files and give write permissions
250  
251 my $output;
252  
253 #---------------------------------
254 # sentence alignment
255 #
256 if ($configfile=~/align\/sent$/){
257  
258 my ($src,$srclang,$srcname);
259 if (defined $$config{input}{'source text'}{'stream name'}){
260 $src=$$config{input}{'source text'}{'stream name'};
261 ($srcname,$srclang)=&Uplug::Web::Corpus::SplitCorpusName($src);
262 }
263 else{
264 $srclang = $$config{input}{'source text'}{'language'};
265 $srcname = $$config{input}{'source text'}{'corpus'};
266 $src = $srcname.' ('.$srclang.')';
267 $src = &Uplug::Web::Corpus::GetCorpusName($srcname,$srclang);
268 }
269  
270 my ($trg,$trglang,$trgname);
271 if (defined $$config{input}{'target text'}{'stream name'}){
272 $trg=$$config{input}{'target text'}{'stream name'};
273 ($trgname,$trglang)=&Uplug::Web::Corpus::SplitCorpusName($trg);
274 }
275 else{
276 $trglang = $$config{input}{'target text'}{'language'};
277 $trgname = $$config{input}{'target text'}{'corpus'};
278 $trg = $trgname.' ('.$trglang.')';
279 $trg = &Uplug::Web::Corpus::GetCorpusName($trgname,$trglang);
280 }
281  
282  
283 my $srccorpus=&Uplug::Web::Corpus::GetCorpusInfo($user,$corpus,$src);
284 my $trgcorpus=&Uplug::Web::Corpus::GetCorpusInfo($user,$corpus,$trg);
285 my $dir=&Uplug::Web::Corpus::GetCorpusDir($user,$trgname);
286  
287 my $lang=$srclang.'-'.$trglang;
288 my $dir=&Uplug::Web::Corpus::GetCorpusDir($user,$corpus,$lang);
289 $output=$dir.'/'.$trgname;
290 if (not -e $output){open F,">$output";close F;}
291 # chmod 0664,$output;
292 system "chmod g+w $output";
293 # my $lockfile=$output.'.lock';
294 # if (not -e $lockfile){open F,">$lockfile";close F;}
295 # chmod 0664,$lockfile;
296  
297 $$config{output}{bitext}{corpus}=$srcname;
298 if ($srcname ne $trgname){
299 $$config{output}{bitext}{corpus}.='-'.$trgname;
300 }
301 $$config{output}{bitext}{fromDoc}=$$srccorpus{file};
302 $$config{output}{bitext}{toDoc}=$$trgcorpus{file};
303 $$config{output}{bitext}{file}=$output;
304 $$config{output}{bitext}{language}=$lang;
305 }
306  
307 #------------------------
308 # word alignment
309  
310 elsif ($configfile=~/align\/word\/(..\-..|basic|default|advanced|giza)$/){
311  
312 my ($bitext,$lang,$name);
313 if (defined $$config{input}{'bitext'}{'stream name'}){
314 $bitext=$$config{input}{'bitext'}{'stream name'};
315 ($name,$lang)=&Uplug::Web::Corpus::SplitCorpusName($bitext);
316 }
317 else{
318 $lang = $$config{input}{'bitext'}{'language'};
319 $name = $$config{input}{'bitext'}{'corpus'};
320 $bitext = &Uplug::Web::Corpus::GetCorpusName($name,$lang);
321 }
322  
323  
324 my $corp=&Uplug::Web::Corpus::GetCorpusInfo($user,$corpus,$bitext);
325 my $dir=&Uplug::Web::Corpus::GetCorpusDir($user,$corpus,$lang);
326 $output=$dir.'/'.$name.'.links';
327 if (not -e $output){open F,">$output";close F;}
328 # chmod 0664,$output;
329 system "chmod g+w $output";
330 # my $lockfile=$output.'.lock';
331 # if (not -e $lockfile){open F,">$lockfile";close F;}
332 # chmod 0664,$lockfile;
333 my $dir=&Uplug::Web::Corpus::GetCorpusDir($user);
334 if (not -d "$dir/data"){mkdir "$dir/data",0755;}
335 if (not -d "$dir/data/runtime"){
336 mkdir "$dir/data/runtime",0775;
337 system "chmod g+w $dir/data/runtime";
338 }
339 $dir.='/data/runtime/';
340 # foreach (@CLUEDBMFILES){
341 # if (not -e "$dir$_.dbm"){open F,">$dir$_.dbm";close F;}
342 # chmod 0664,"$dir$_.dbm";
343 # if (not -e "$dir$_.dbm.head"){open F,">$dir$_.dbm.head";close F;}
344 # chmod 0664,"$dir$_.dbm.head";
345 # }
346  
347 $$config{output}{'bitext links'}{corpus}=$name.' word';
348 if (-e $$corp{fromDoc}){
349 $$config{output}{'bitext links'}{fromDoc}=$$corp{fromDoc};
350 }
351 if (-e $$corp{toDoc}){
352 $$config{output}{'bitext links'}{toDoc}=$$corp{toDoc};
353 }
354 $$config{output}{'bitext links'}{file}=$output;
355 $$config{output}{'bitext links'}{language}=$lang;
356 }
357 }
358  
359 ## end of PrepareProcess
360 ##--------------------------------------------------------------------------
361  
362  
363 sub GetOutputFiles{
364 my $user=shift;
365 my $configfile=shift;
366  
367 my %config=();
368 my @files;
369 &UplugSystemIni(\%config,$user,$configfile);
370 if (ref($config{output}) ne 'HASH'){return undef;}
371 foreach (keys %{$config{output}}){
372 if (ref($config{output}{$_}) eq 'HASH'){
373 if (defined $config{output}{$_}{file}){
374 push (@files,$config{output}{$_}{file});
375 }
376 }
377 }
378 return @files;
379 }
380  
381  
382  
383  
384  
385 sub ExpandParameter{
386 my $config=shift;
387 my $para=shift;
388 if (ref($para) ne 'HASH'){return;}
389 if (ref($config) ne 'HASH'){return;}
390  
391 foreach (keys %{$para}){
392 my $key=$_;
393 if (/^\-(.*)$/){
394 if (defined $$config{arguments}{shortcuts}{$1}){
395 $key=$$config{arguments}{shortcuts}{$1};
396 }
397 else{next;}
398 }
399 # print "$key=$$para{$_}".'<br>';
400 my @arr=split(/\:/,$key);
401 if (defined $arr[4]){
402 $$config{$arr[0]}{$arr[1]}{$arr[2]}{$arr[3]}{$arr[4]}=$$para{$_};
403 }
404 elsif (defined $arr[3]){
405 $$config{$arr[0]}{$arr[1]}{$arr[2]}{$arr[3]}=$$para{$_};
406 }
407 elsif (defined $arr[2]){
408 $$config{$arr[0]}{$arr[1]}{$arr[2]}=$$para{$_};
409 }
410 elsif (defined $arr[1]){
411 $$config{$arr[0]}{$arr[1]}=$$para{$_};
412 }
413 elsif (defined $arr[0]){
414 $$config{$arr[0]}=$$para{$_};
415 }
416 }
417 }
418  
419  
420  
421 ###########################################################################
422  
423  
424 sub GetProcesses{
425 my $type=shift;
426 my $stack=&GetJobStack($type);
427 return $stack->read();
428 }
429  
430 sub GetNextProcess{
431 my $type=shift;
432 my $stack=&GetJobStack($type);
433 return $stack->shift();
434 }
435  
436 sub AddProcess{
437 my $type=shift;
438 my $stack=&GetJobStack($type);
439 return $stack->push(@_);
440 }
441  
442  
443  
444 sub RemoveProcess{
445 my $type=shift;
446 my $user=shift;
447 my $process=shift;
448  
449 my $stack=&GetJobStack($type);
450 $stack->remove($user,$process);
451  
452 my $UserDir=&Uplug::Web::Corpus::GetCorpusDir($user);
453 if (-d "$UserDir/$process"){
454 system "rm -fr $UserDir/$process";
455 }
456 }
457  
458  
459 sub RestartProcess{
460 my $type=shift;
461 my $user=shift;
462 my $process=shift;
463 &MoveJob($user,$process,$type,'todo');
464 }
465  
466 sub MoveJob{
467 my $user=shift;
468 my $process=shift;
469 my $from=shift;
470 my $to=shift;
471  
472 my $fromstack=&GetJobStack($from);
473 my $tostack=&GetJobStack($to);
474  
475 $fromstack->open('lock'); # open the stack to lock it!
476 if (my @data=$fromstack->find($user,$process)){# look for the process ID
477 $fromstack->remove($user,$process); # remove it from fromstack
478 $tostack->push(@data); # push it into the tostack
479 $fromstack->close(); # release the file lock
480 return 1; # and return true
481 } # process not found:
482 $fromstack->close(); # release lock
483 return 0; # and return false
484 }
485  
486  
487 sub MoveJobTo{
488 my $user=shift;
489 my $process=shift;
490 my $to=shift;
491  
492 foreach ('todo','queued','working','done','failed'){
493 if (&MoveJob($user,$process,$_,$to)){return 1;}
494 }
495 return 0;
496 }
497  
498 sub FindJobStack{
499 my $user=shift;
500 my $process=shift;
501 my $type=shift;
502  
503 if (not defined $type){
504 foreach ('todo','queued','working','done','failed'){
505 if (&FindJobStack($user,$process,$_)){return $_;}
506 }
507 }
508 my $stack=&GetJobStack($type);
509 if ($stack->find($user,$process)){return $type;}
510 return undef;
511 }
512  
513 sub GetJobStack{
514 my $type=shift;
515 my $stack=$todo;
516 if ($type eq 'queued'){$stack=$queued;}
517 if ($type eq 'working'){$stack=$working;}
518 if ($type eq 'done'){$stack=$done;}
519 if ($type eq 'failed'){$stack=$failed;}
520 return $stack;
521 }
522  
523  
524 #sub RunProcess{
525 # my $type=shift;
526 # my $user=shift;
527 # my $process=shift;
528 #
529 # my $stack=&GetJobStack($type);
530 # my @data=$stack->find($user,$process);
531 # &MoveJob($user,$process,'type','queued');
532 # if (@data){`$data[2]`;}
533 #}
534  
535  
536 sub ClearStack{
537 my $type=shift;
538 my $user=shift;
539 my $limit=shift;
540  
541 my @data=&GetProcesses($type);
542 if (defined $user){
543 my $pattern=quotemeta($user);
544 @data=grep ($_=~/$user/,@data);
545 }
546 while (@data and (@data>$limit)){
547 my ($u,$p)=split(/\:/,$data[0]);
548 &RemoveProcess($type,$u,$p);
549 shift @data;
550 }
551 }
552  
553  
554  
555 #########################################
556 # return a true value
557  
558 1;