| 1 | # Copyright (C) 2004 Jörg Tiedemann <joerg@stp.ling.uu.se> | ||
| 2 | # | ||
| 3 | # This program is free software; you can redistribute it and/or modify | ||
| 4 | # it under the terms of the GNU General Public License as published by | ||
| 5 | # the Free Software Foundation; either version 2 of the License, or | ||
| 6 | # (at your option) any later version. | ||
| 7 | # | ||
| 8 | # This program is distributed in the hope that it will be useful, | ||
| 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 11 | # GNU General Public License for more details. | ||
| 12 | # | ||
| 13 | # You should have received a copy of the GNU General Public License | ||
| 14 | # along with this program; if not, write to the Free Software | ||
| 15 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
| 16 | |||
| 17 | package Uplug::Web::Corpus; | ||
| 18 | |||
| 19 | use strict; | ||
| 20 | use IO::File; | ||
| 21 | use POSIX qw(tmpnam); | ||
| 22 | use File::Copy; | ||
| 23 | use ExtUtils::Command; | ||
| 24 | use File::Basename; | ||
| 25 | |||
| 26 | use Uplug::Web; | ||
| 27 | use Uplug::Web::Config; | ||
| 28 | use Uplug::Web::Process; | ||
| 29 | use Uplug::Web::Process::Stack; | ||
| 30 | use Uplug::Web::User; | ||
| 31 | use Uplug::Config; | ||
| 32 | |||
| 33 | |||
| 34 | |||
| 35 | our $INDEXER=$ENV{UPLUGHOME}.'/web/bin/uplug-indexer.pl'; | ||
| 36 | our $RECODE=$ENV{RECODE}; | ||
| 37 | |||
| 38 | our $CorpusDir=$ENV{UPLUGDATA}; | ||
| 39 | |||
| 40 | my $MAXFLOCKWAIT=3; | ||
| 41 | |||
| 42 | my $CorpusIndexFile=$ENV{UPLUGDATA}.'/.index'; | ||
| 43 | my $CorpusIndex=Uplug::Web::Process::Stack->new($CorpusIndexFile); | ||
| 44 | |||
| 45 | sub GetIndexedCorpora{ | ||
| 46 | my $data=shift; | ||
| 47 | if (ref($data) ne 'HASH'){return $CorpusIndex->read();} | ||
| 48 | my @corpora=$CorpusIndex->read(); | ||
| 49 | foreach (@corpora){ | ||
| 50 | my ($user,$name,$lang,$alg,$enc)=split(/\:/,$_); | ||
| 51 | $$data{$user}{$name}{$lang}{encoding}=$enc; | ||
| 52 | if ($alg){ | ||
| 53 | push (@{$$data{$user}{$name}{$lang}{align}},$alg); | ||
| 54 | } | ||
| 55 | } | ||
| 56 | return keys %{$data}; | ||
| 57 | } | ||
| 58 | |||
| 59 | |||
| 60 | sub IndexCorpus{ | ||
| 61 | my $owner=shift; | ||
| 62 | my $corpus=shift; | ||
| 63 | |||
| 64 | my $CorpusDir=&GetCorpusDir($owner,$corpus); | ||
| 65 | my $CWBREG="$ENV{UPLUGCWB}/reg/$owner/$corpus"; | ||
| 66 | my $CWBDAT="$ENV{UPLUGCWB}/dat/$owner/$corpus"; | ||
| 67 | |||
| 68 | if (not -d $ENV{UPLUGCWB}){mkdir $ENV{UPLUGCWB};} | ||
| 69 | if (not -d "$ENV{UPLUGCWB}/reg"){mkdir "$ENV{UPLUGCWB}/reg";} | ||
| 70 | if (not -d "$ENV{UPLUGCWB}/dat"){mkdir "$ENV{UPLUGCWB}/dat";} | ||
| 71 | if (not -d "$ENV{UPLUGCWB}/reg/$owner"){mkdir "$ENV{UPLUGCWB}/reg/$owner";} | ||
| 72 | if (not -d "$ENV{UPLUGCWB}/dat/$owner"){mkdir "$ENV{UPLUGCWB}/dat/$owner";} | ||
| 73 | |||
| 74 | if (not -d "$ENV{UPLUGCWB}/reg/$owner/$corpus"){ | ||
| 75 | mkdir "$ENV{UPLUGCWB}/reg/$owner/$corpus"; | ||
| 76 | system "chmod g+w $ENV{UPLUGCWB}/reg/$owner/$corpus"; | ||
| 77 | } | ||
| 78 | if (not -d "$ENV{UPLUGCWB}/dat/$owner/$corpus"){ | ||
| 79 | mkdir "$ENV{UPLUGCWB}/dat/$owner/$corpus"; | ||
| 80 | system "chmod g+w $ENV{UPLUGCWB}/dat/$owner/$corpus"; | ||
| 81 | } | ||
| 82 | |||
| 83 | my $process=time().'_'.$$; | ||
| 84 | my $command="$INDEXER $CWBREG $CWBDAT $CorpusDir"; | ||
| 85 | &Uplug::Web::Process::AddProcess('todo',$owner,$process,'$bash',$command); | ||
| 86 | # print "$command<hr>"; | ||
| 87 | |||
| 88 | } | ||
| 89 | |||
| 90 | |||
| 91 | |||
| 92 | sub AddCorpusToIndex{ | ||
| 93 | my $user=shift; | ||
| 94 | my $corpus=shift; | ||
| 95 | my $srcenc=shift; | ||
| 96 | my $trgenc=shift; | ||
| 97 | my $alg=shift; | ||
| 98 | my $info=&GetCorpusInfo($user,$corpus); | ||
| 99 | if ($$info{format}=~/align/){ | ||
| 100 | my ($src,$trg)=split(/\-/,$$info{language}); | ||
| 101 | &AddCorpusToIndex($user, | ||
| 102 | &GetCorpusName($$info{corpus},$src), | ||
| 103 | $srcenc,$trgenc, | ||
| 104 | $trg); | ||
| 105 | &AddCorpusToIndex($user, | ||
| 106 | &GetCorpusName($$info{corpus},$trg), | ||
| 107 | $trgenc,$srcenc, | ||
| 108 | $src); | ||
| 109 | } | ||
| 110 | else{ | ||
| 111 | $CorpusIndex->remove($user,$$info{corpus},$$info{language},$alg); | ||
| 112 | $CorpusIndex->push($user,$$info{corpus},$$info{language},$alg,$srcenc); | ||
| 113 | } | ||
| 114 | } | ||
| 115 | |||
| 116 | |||
| 117 | |||
| 118 | |||
| 119 | sub GetCorpusDataFileOld{ | ||
| 120 | my $user=shift; | ||
| 121 | return "$CorpusDir/$user/ini/uplugUserStreams.ini"; | ||
| 122 | } | ||
| 123 | |||
| 124 | sub GetCorpusDataFile{ | ||
| 125 | my $user=shift; | ||
| 126 | my $corpus=shift; | ||
| 127 | return "$CorpusDir/$user/$corpus/.documents"; | ||
| 128 | } | ||
| 129 | |||
| 130 | |||
| 131 | sub GetCorpusDir{ | ||
| 132 | my $user=shift; | ||
| 133 | my $corpus=shift; | ||
| 134 | my $lang=shift; | ||
| 135 | if (not defined $user){return $CorpusDir;} | ||
| 136 | if (not defined $corpus){return "$CorpusDir/$user";} | ||
| 137 | if (not -d "$CorpusDir/$user/$corpus"){mkdir "$CorpusDir/$user/$corpus";} | ||
| 138 | if (not defined $lang){return "$CorpusDir/$user/$corpus";} | ||
| 139 | if (not -d "$CorpusDir/$user/$corpus/$lang"){ | ||
| 140 | mkdir "$CorpusDir/$user/$corpus/$lang"; | ||
| 141 | } | ||
| 142 | return "$CorpusDir/$user/$corpus/$lang"; | ||
| 143 | } | ||
| 144 | |||
| 145 | sub GetRecycleDir{ | ||
| 146 | my $user=shift; | ||
| 147 | my $corpus=shift; | ||
| 148 | my $lang=shift; | ||
| 149 | |||
| 150 | if (not -d "$CorpusDir/.recycled"){ | ||
| 151 | mkdir "$CorpusDir/.recycled",0755; | ||
| 152 | } | ||
| 153 | if (not defined $user){return "$CorpusDir/.recycled";} | ||
| 154 | if (not -d "$CorpusDir/.recycled/$user"){ | ||
| 155 | mkdir "$CorpusDir/.recycled/$user",0755; | ||
| 156 | } | ||
| 157 | if (not defined $corpus){return "$CorpusDir/.recycled/$user";} | ||
| 158 | if (not -d "$CorpusDir/.recycled/$user/$corpus"){ | ||
| 159 | mkdir "$CorpusDir/.recycled/$user/$corpus",0755; | ||
| 160 | } | ||
| 161 | if (defined $lang){ | ||
| 162 | if (not -d "$CorpusDir/.recycled/$user/$corpus/$lang"){ | ||
| 163 | mkdir "$CorpusDir/.recycled/$user/$corpus/$lang",0755; | ||
| 164 | } | ||
| 165 | return "$CorpusDir/.recycled/$user/$corpus/$lang"; | ||
| 166 | } | ||
| 167 | return "$CorpusDir/.recycled/$user/$corpus"; | ||
| 168 | } | ||
| 169 | |||
| 170 | sub GetCorpusStreams{ | ||
| 171 | my $user=shift; | ||
| 172 | my %para=@_; | ||
| 173 | my %CorpusData=(); | ||
| 174 | &GetCorpusData(\%CorpusData,$user); | ||
| 175 | my @streams=(); | ||
| 176 | foreach my $c (keys %CorpusData){ | ||
| 177 | my $match=1; | ||
| 178 | foreach (keys %para){ | ||
| 179 | if ($CorpusData{$c}{$_}!~/$para{$_}/){$match=0;last;} | ||
| 180 | } | ||
| 181 | if ($match){push (@streams,$c);} | ||
| 182 | } | ||
| 183 | return @streams; | ||
| 184 | } | ||
| 185 | |||
| 186 | |||
| 187 | #------------------------------------------------------------------ | ||
| 188 | # MatchingDocuments | ||
| 189 | # find all documents within a corpus with matching attributes | ||
| 190 | # (%para=attribute-value pairs to be matched) | ||
| 191 | |||
| 192 | sub MatchingDocuments{ | ||
| 193 | my $user=shift; | ||
| 194 | my $corpus=shift; | ||
| 195 | my %para=@_; | ||
| 196 | |||
| 197 | my $docs=&CorpusDocuments($user,$corpus); | ||
| 198 | my @ok=(); | ||
| 199 | foreach my $c (keys %{$docs}){ | ||
| 200 | my $match=1; | ||
| 201 | foreach (keys %para){ | ||
| 202 | if ($$docs{$c}{$_}!~/$para{$_}/){$match=0;last;} | ||
| 203 | } | ||
| 204 | if ($match){push (@ok,$c);} | ||
| 205 | } | ||
| 206 | return @ok; | ||
| 207 | } | ||
| 208 | |||
| 209 | #------------------------------------------------------------------ | ||
| 210 | |||
| 211 | |||
| 212 | sub GetCorpusData{ | ||
| 213 | |||
| 214 | my $CorpusData=shift; | ||
| 215 | my $user=shift; | ||
| 216 | |||
| 217 | my $CorpusInfoFile=&GetCorpusDataFile($user); | ||
| 218 | if (ref($CorpusData) ne 'HASH'){return 0;} | ||
| 219 | if (not -e $CorpusInfoFile){return 0;} | ||
| 220 | &LoadIniData($CorpusData,$CorpusInfoFile); | ||
| 221 | return keys %{$CorpusData}; | ||
| 222 | } | ||
| 223 | |||
| 224 | |||
| 225 | |||
| 226 | sub RestoreDocument{ | ||
| 227 | my ($owner,$corpus,$doc)=@_; | ||
| 228 | |||
| 229 | $CorpusDir.='/.recycled'; # set recycle-dir | ||
| 230 | my $ConfigFile=&CorporaConfigFile($owner); | ||
| 231 | my $corpora=Uplug::Web::Config->new($ConfigFile); # user corpora | ||
| 232 | my $ConfigFile=&DocumentConfigFile($owner,$corpus); # corpus configfile | ||
| 233 | my $documents=Uplug::Web::Config->new($ConfigFile); # corpus documents | ||
| 234 | my $config=$documents->read(); | ||
| 235 | $CorpusDir=$ENV{UPLUGDATA}; # restore data-dir | ||
| 236 | |||
| 237 | if (defined $$config{$doc}){ | ||
| 238 | my $lang=$$config{$doc}{language}; | ||
| 239 | my $file=$$config{$doc}{file}; | ||
| 240 | my $RecycleDir=&GetRecycleDir($owner,$corpus,$lang); | ||
| 241 | my $RemovedFile=$RecycleDir.'/'.&basename($file); | ||
| 242 | if (-e $RemovedFile){ | ||
| 243 | move ($RemovedFile,$file); | ||
| 244 | } | ||
| 245 | |||
| 246 | my $ConfigFile=&DocumentConfigFile($owner,$corpus); # add the restored | ||
| 247 | my $ResDoc=Uplug::Web::Config->new($ConfigFile); # document to the | ||
| 248 | my $ResConfig=$ResDoc->read(); # corpus config | ||
| 249 | $$ResConfig{$doc}=$$config{$doc}; # file | ||
| 250 | $ResDoc->write($ResConfig); # write configfile | ||
| 251 | |||
| 252 | delete $$config{$doc}; # delete doc-data | ||
| 253 | $documents->write($config); # write configfile | ||
| 254 | |||
| 255 | if (not keys %{$config}){ # if no more removed documents | ||
| 256 | my $corpconf=$corpora->read(); # in this corpus: read the | ||
| 257 | delete $$corpconf{$corpus}; # config file and delete the | ||
| 258 | $corpora->write($corpconf); # corpus and save | ||
| 259 | } | ||
| 260 | |||
| 261 | } | ||
| 262 | $documents->close(); | ||
| 263 | } | ||
| 264 | |||
| 265 | |||
| 266 | sub RemoveDocument{ | ||
| 267 | my ($owner,$corpus,$doc)=@_; | ||
| 268 | |||
| 269 | my $ConfigFile=&DocumentConfigFile($owner,$corpus); | ||
| 270 | my $documents=Uplug::Web::Config->new($ConfigFile); | ||
| 271 | my $config=$documents->read(); | ||
| 272 | |||
| 273 | if (defined $$config{$doc}){ | ||
| 274 | my $lang=$$config{$doc}{language}; | ||
| 275 | my $file=$$config{$doc}{file}; | ||
| 276 | # my $corpus=$$config{$doc}{corpus}; | ||
| 277 | my $RecycleDir=&GetRecycleDir($owner,$corpus,$lang); | ||
| 278 | if (-e $file){ | ||
| 279 | move ($file,"$RecycleDir/"); | ||
| 280 | } | ||
| 281 | |||
| 282 | $CorpusDir.='/.recycled'; # set recycle-dir | ||
| 283 | my $ConfigFile=&DocumentConfigFile($owner,$corpus); # config-filename | ||
| 284 | my $RemDoc=Uplug::Web::Config->new($ConfigFile); # open config-file | ||
| 285 | my $RemConfig=$RemDoc->read(); # read it | ||
| 286 | $$RemConfig{$doc}=$$config{$doc}; # save doc-data | ||
| 287 | $RemDoc->write($RemConfig); # write configfile | ||
| 288 | |||
| 289 | my $ConfigFile=&CorporaConfigFile($owner); # config-filename | ||
| 290 | my $RemCorpora=Uplug::Web::Config->new($ConfigFile); # open config-file | ||
| 291 | my $RemConfig=$RemCorpora->read(); # read it | ||
| 292 | $$RemConfig{$corpus}=1; # set corpus | ||
| 293 | $RemCorpora->write($RemConfig); # write configfile | ||
| 294 | |||
| 295 | delete $$config{$doc}; # delete doc-data | ||
| 296 | $documents->write($config); # write configfile | ||
| 297 | $CorpusDir=$ENV{UPLUGDATA}; # restore data-dir | ||
| 298 | } | ||
| 299 | $documents->close(); | ||
| 300 | } | ||
| 301 | |||
| 302 | sub RemoveCorpus{ | ||
| 303 | my ($owner,$corpus)=@_; | ||
| 304 | |||
| 305 | my $ConfigFile=&CorporaConfigFile($owner); | ||
| 306 | my $corpora=Uplug::Web::Config->new($ConfigFile); | ||
| 307 | my $config=$corpora->read(); | ||
| 308 | |||
| 309 | if (defined $$config{$corpus}){ | ||
| 310 | my $RecycleDir=&GetRecycleDir($owner); | ||
| 311 | my $DataDir=&GetCorpusDir($owner,$corpus); | ||
| 312 | |||
| 313 | if (-d "$RecycleDir/$corpus"){ # quite a hack ... | ||
| 314 | system "rm -fr $RecycleDir/$corpus"; # and maybe dangerous!! | ||
| 315 | } | ||
| 316 | if (-e $DataDir){ | ||
| 317 | system "mv $DataDir $RecycleDir/"; # requires UNIX!! | ||
| 318 | } | ||
| 319 | |||
| 320 | $CorpusDir.='/.recycled'; # set recycle-dir | ||
| 321 | my $ConfigFile=&CorporaConfigFile($owner); # config-filename | ||
| 322 | my $RemCorpora=Uplug::Web::Config->new($ConfigFile); # open config-file | ||
| 323 | my $RemConfig=$RemCorpora->read(); # read it | ||
| 324 | $$RemConfig{$corpus}=$$config{$corpus}; # set corpus data | ||
| 325 | $RemCorpora->write($RemConfig); # write configfile | ||
| 326 | |||
| 327 | delete $$config{$corpus}; # delete corpus | ||
| 328 | $corpora->write($config); # from configfile | ||
| 329 | $CorpusDir=$ENV{UPLUGDATA}; # restore data-dir | ||
| 330 | } | ||
| 331 | $corpora->close(); | ||
| 332 | |||
| 333 | my $ConfigFile=&CorporaConfigFile('pub'); # delete from public | ||
| 334 | my $corpora=Uplug::Web::Config->new($ConfigFile); # corpora list | ||
| 335 | my $config=$corpora->read(); | ||
| 336 | if (defined $$config{"../$owner/$corpus"}){ | ||
| 337 | delete $$config{"../$owner/$corpus"}; | ||
| 338 | $corpora->write($config); | ||
| 339 | } | ||
| 340 | $corpora->close(); | ||
| 341 | |||
| 342 | } | ||
| 343 | |||
| 344 | |||
| 345 | |||
| 346 | sub RemoveCorpusOld{ | ||
| 347 | my ($user,$owner,$name)=@_; | ||
| 348 | |||
| 349 | if ($owner ne $user){print "Cannot remove corpus $name!";return 0;} | ||
| 350 | |||
| 351 | my $CorpusInfoFile=&GetCorpusDataFile($owner,$name); | ||
| 352 | my %CorpusData; | ||
| 353 | &LoadIniData(\%CorpusData,$CorpusInfoFile); | ||
| 354 | if (defined $CorpusData{$name}){ | ||
| 355 | my $lang=$CorpusData{$name}{language}; | ||
| 356 | my $file=$CorpusData{$name}{file}; | ||
| 357 | my $corpus=$CorpusData{$name}{corpus}; | ||
| 358 | my $RecycleDir=&GetRecycleDir($owner,$corpus); | ||
| 359 | if (-e $file){ | ||
| 360 | move ($file,"$RecycleDir/"); | ||
| 361 | } | ||
| 362 | delete $CorpusData{$name}; | ||
| 363 | &WriteIniFile($CorpusInfoFile,\%CorpusData); | ||
| 364 | } | ||
| 365 | |||
| 366 | } | ||
| 367 | |||
| 368 | |||
| 369 | sub GetCorpusName{ | ||
| 370 | my ($name,$lang)=@_; | ||
| 371 | return "$name ($lang)"; | ||
| 372 | } | ||
| 373 | |||
| 374 | sub SplitCorpusName{ | ||
| 375 | my ($name)=@_; | ||
| 376 | if ($name=~/^(.*)\s\((.*)\)/){ | ||
| 377 | return ($1,$2); | ||
| 378 | } | ||
| 379 | return undef; | ||
| 380 | } | ||
| 381 | |||
| 382 | |||
| 383 | sub GetCorpusInfo{ | ||
| 384 | my $user=shift; | ||
| 385 | my $corpus=shift; | ||
| 386 | my $doc=shift; | ||
| 387 | |||
| 388 | my $documents=&CorpusDocuments($user,$corpus); | ||
| 389 | if (ref($$documents{$doc}) eq 'HASH'){return $$documents{$doc};} | ||
| 390 | return {}; | ||
| 391 | } | ||
| 392 | |||
| 393 | sub GetCorpusInfoOld{ | ||
| 394 | my $user=shift; | ||
| 395 | my $CorpusName=shift; | ||
| 396 | |||
| 397 | my $CorpusInfoFile=&GetCorpusDataFile($user,$CorpusName); | ||
| 398 | my %CorpusData; | ||
| 399 | &LoadIniData(\%CorpusData,$CorpusInfoFile); | ||
| 400 | if (ref($CorpusData{$CorpusName}) eq 'HASH'){ | ||
| 401 | return %{$CorpusData{$CorpusName}} | ||
| 402 | } | ||
| 403 | return undef; | ||
| 404 | } | ||
| 405 | |||
| 406 | |||
| 407 | |||
| 408 | #sub ReadCorpus{ | ||
| 409 | # my $user=shift; | ||
| 410 | # my $name=shift; | ||
| 411 | # my $start=shift; | ||
| 412 | # my $nr=shift; | ||
| 413 | # | ||
| 414 | # my %stream=&Uplug::Web::Corpus::GetCorpusInfo($user,$name); | ||
| 415 | # if (not keys %stream){ | ||
| 416 | # print "Cannot find corpus data for $name\n"; | ||
| 417 | # } | ||
| 418 | # my $corpus=new Uplug::IO::Any(\%stream); | ||
| 419 | # if (not $corpus->open('read',\%stream)){ | ||
| 420 | # print "Cannot open $name\n"; | ||
| 421 | # } | ||
| 422 | # my $html; | ||
| 423 | # my @rows; | ||
| 424 | # my $data=Uplug::Data::DOM->new(); | ||
| 425 | # my $count; | ||
| 426 | # my $skipped; | ||
| 427 | # while ($corpus->read($data)){ | ||
| 428 | # if ($skipped<$start){$skipped++;next;} | ||
| 429 | # $count++; | ||
| 430 | # if ($count>$nr){last;} | ||
| 431 | # push(@rows,$data->toHtml()); | ||
| 432 | # } | ||
| 433 | # $corpus->close(); | ||
| 434 | # return @rows; | ||
| 435 | #} | ||
| 436 | # | ||
| 437 | |||
| 438 | |||
| 439 | sub SendCorpus{ | ||
| 440 | my $to=shift; | ||
| 441 | my $owner=shift; | ||
| 442 | my $corpus=shift; | ||
| 443 | my $doc=shift; | ||
| 444 | |||
| 445 | my $data=&GetCorpusInfo($owner,$corpus,$doc); | ||
| 446 | |||
| 447 | if (defined $$data{file}){ | ||
| 448 | &Uplug::Web::User::SendFile($to,'UplugWeb - '.$corpus,$$data{file}); | ||
| 449 | return 1; | ||
| 450 | } | ||
| 451 | return 0; | ||
| 452 | } | ||
| 453 | |||
| 454 | sub CorpusIsPrivate{ | ||
| 455 | my $owner=shift; | ||
| 456 | my $corpus=shift; | ||
| 457 | my $CorpusConfig=Uplug::Web::Config->new("$CorpusDir/$owner/.corpora"); | ||
| 458 | my $corpora=$CorpusConfig->read(); | ||
| 459 | return $$CorpusConfig{$corpus}; | ||
| 460 | } | ||
| 461 | |||
| 462 | sub CorpusIsPublic{ | ||
| 463 | return not &CorpusIsPrivate(@_); | ||
| 464 | } | ||
| 465 | |||
| 466 | |||
| 467 | sub CorporaConfigFile{ | ||
| 468 | my $owner=shift; | ||
| 469 | if (not -d "$CorpusDir/$owner"){mkdir "$CorpusDir/$owner";} | ||
| 470 | return "$CorpusDir/$owner/.corpora"; | ||
| 471 | } | ||
| 472 | |||
| 473 | sub DocumentConfigFile{ | ||
| 474 | my $owner=shift; | ||
| 475 | my $corpus=shift; | ||
| 476 | if (not -d "$CorpusDir/$owner/$corpus"){mkdir "$CorpusDir/$owner/$corpus";} | ||
| 477 | return "$CorpusDir/$owner/$corpus/.documents"; | ||
| 478 | } | ||
| 479 | |||
| 480 | sub Corpora{ | ||
| 481 | my $owner=shift; | ||
| 482 | my $ConfigFile=&CorporaConfigFile($owner); | ||
| 483 | my $CorpusConfig=Uplug::Web::Config->new($ConfigFile); | ||
| 484 | return $CorpusConfig->read(); | ||
| 485 | } | ||
| 486 | |||
| 487 | |||
| 488 | sub CorpusDocuments{ | ||
| 489 | my $owner=shift; | ||
| 490 | my $corpus=shift; | ||
| 491 | my $ConfigFile=&DocumentConfigFile($owner,$corpus); | ||
| 492 | my $documents=Uplug::Web::Config->new($ConfigFile); | ||
| 493 | return $documents->read(); | ||
| 494 | } | ||
| 495 | |||
| 496 | |||
| 497 | |||
| 498 | sub AddCorpus{ | ||
| 499 | my $user=shift; | ||
| 500 | my $corpus=shift; | ||
| 501 | my $priv=shift; # =1 --> private corpus (don't store in public) | ||
| 502 | |||
| 503 | if ((defined $corpus) and ($corpus!~/^[a-zA-Z\_0-9]{1,10}$/)){ | ||
| 504 | return (0,"Corpus name $corpus is not valid!"); | ||
| 505 | } | ||
| 506 | |||
| 507 | my $UserCorpusFile=&CorporaConfigFile($user); | ||
| 508 | # "$CorpusDir/$user/.corpora"; | ||
| 509 | my $UserCorpora=Uplug::Web::Config->new($UserCorpusFile); | ||
| 510 | my $corpora=$UserCorpora->read(); | ||
| 511 | |||
| 512 | if (defined $$corpora{$corpus}){ | ||
| 513 | return (0,"A corpus with the name '$corpus' exists already!"); | ||
| 514 | } | ||
| 515 | |||
| 516 | $$corpora{$corpus}=1; | ||
| 517 | if (not $UserCorpora->write($corpora)){ | ||
| 518 | return (0,"Could not add corpus info to $UserCorpusFile!"); | ||
| 519 | } | ||
| 520 | $UserCorpora->close(); | ||
| 521 | |||
| 522 | if (not mkdir "$CorpusDir/$user/$corpus"){ | ||
| 523 | return (0,"Could not create corpus directory for '$corpus'!"); | ||
| 524 | } | ||
| 525 | |||
| 526 | if (not $priv){ | ||
| 527 | my $PublicCorpusFile=&CorporaConfigFile('pub'); | ||
| 528 | my $PublicCorpora=Uplug::Web::Config->new($PublicCorpusFile); | ||
| 529 | my $public=$PublicCorpora->read(); | ||
| 530 | $$public{"../$user/$corpus"}=1; | ||
| 531 | if (not $PublicCorpora->write($public)){ | ||
| 532 | return (0,"Could not add corpus info to $PublicCorpusFile!"); | ||
| 533 | } | ||
| 534 | $PublicCorpora->close(); | ||
| 535 | } | ||
| 536 | return (1,"Corpus '$corpus' sucessfully added!"); | ||
| 537 | } | ||
| 538 | |||
| 539 | sub AddDocument{ | ||
| 540 | my ($user,$corpus,$name,$fh,$lang,$enc)=@_; | ||
| 541 | |||
| 542 | if ((defined $name) and ($name!~/^[a-zA-Z\_\.0-9]{1,15}$/)){ | ||
| 543 | return (0,"Invalid document name '$name'! (use: [a-zA-Z_.]{1,15})"); | ||
| 544 | } | ||
| 545 | |||
| 546 | my $documents=&CorpusDocuments($user,$corpus); | ||
| 547 | my $CorpusName=&GetCorpusName($corpus,$lang); | ||
| 548 | if (defined $$documents{$CorpusName}){ | ||
| 549 | return (0,"A document with the name '$CorpusName' exists already!"); | ||
| 550 | } | ||
| 551 | |||
| 552 | my $dir="$CorpusDir/$user/$corpus/$lang"; | ||
| 553 | if (not -e $dir){ | ||
| 554 | if (not mkdir $dir){ | ||
| 555 | return (0,"Could not create $lang language directory for '$corpus'!"); | ||
| 556 | } | ||
| 557 | } | ||
| 558 | my $file="$dir/$name"; | ||
| 559 | # my $tmpfile=&GetTempFileName; | ||
| 560 | # open OUT, '>:encoding(utf8)',$tmpfile; | ||
| 561 | open OUT, '>:encoding(utf8)',$file; | ||
| 562 | binmode($fh);require Encode; | ||
| 563 | |||
| 564 | #---------------------------------- | ||
| 565 | # read data and save them in tempfile | ||
| 566 | # | ||
| 567 | while (<$fh>){ | ||
| 568 | eval {$_=&Encode::decode($enc,$_,1); }; | ||
| 569 | if ($@){print $@;return undef;} | ||
| 570 | print OUT $_; | ||
| 571 | } | ||
| 572 | close OUT; | ||
| 573 | |||
| 574 | # move($tmpfile,$file); # create the corpus file | ||
| 575 | # my $lckfile="$file.lock"; | ||
| 576 | # open F,">$lckfile";close F; # create a lock file | ||
| 577 | chmod 0664,$file; | ||
| 578 | # chmod 0664,$lckfile; | ||
| 579 | # unlink $tmpfile; | ||
| 580 | |||
| 581 | &AddCorpusInfo($user,$corpus,$name,$lang,'text', | ||
| 582 | {file => $file,format => 'text'}); | ||
| 583 | return (1,"Document $fh successfully added to corpus $corpus!"); | ||
| 584 | } | ||
| 585 | |||
| 586 | |||
| 587 | sub AddCorpusInfo{ | ||
| 588 | |||
| 589 | my $owner=shift; | ||
| 590 | my $corpus=shift; | ||
| 591 | my $name=shift; | ||
| 592 | my $lang=shift; | ||
| 593 | my $status=shift; | ||
| 594 | my $para=shift; | ||
| 595 | |||
| 596 | my $CorpusFile="$CorpusDir/$owner/$corpus/.documents"; | ||
| 597 | my $UserCorpora=Uplug::Web::Config->new($CorpusFile); | ||
| 598 | my $corpora=$UserCorpora->read(); | ||
| 599 | |||
| 600 | my $CorpusName=&GetCorpusName($name,$lang); | ||
| 601 | %{$$corpora{$CorpusName}}=('language' => $lang, | ||
| 602 | 'corpus' => $name, | ||
| 603 | 'status' => $status); | ||
| 604 | if (ref($para) eq 'HASH'){ | ||
| 605 | foreach (keys %{$para}){ | ||
| 606 | $$corpora{$CorpusName}{$_}=$$para{$_}; | ||
| 607 | } | ||
| 608 | } | ||
| 609 | if (not $UserCorpora->write($corpora)){ | ||
| 610 | return (0,"Could not add corpus info to $CorpusFile!"); | ||
| 611 | } | ||
| 612 | $UserCorpora->close(); | ||
| 613 | } | ||
| 614 | |||
| 615 | |||
| 616 | sub ChangeCorpusInfo{ | ||
| 617 | |||
| 618 | my $owner=shift; | ||
| 619 | my $corpus=shift; | ||
| 620 | my $doc=shift; # either EXISTING doc-name or doc-base-name without lang! | ||
| 621 | my $para=shift; | ||
| 622 | |||
| 623 | my $CorpusFile="$CorpusDir/$owner/$corpus/.documents"; | ||
| 624 | my $UserCorpora=Uplug::Web::Config->new($CorpusFile); | ||
| 625 | my $corpora=$UserCorpora->read(); | ||
| 626 | |||
| 627 | if (not defined $$corpora{$doc}){ | ||
| 628 | if ((ref($para) eq 'HASH') and (defined $$para{language})){ | ||
| 629 | $doc=&GetCorpusName($doc,$$para{language}); | ||
| 630 | } | ||
| 631 | } | ||
| 632 | if (ref($para) eq 'HASH'){ | ||
| 633 | foreach (keys %{$para}){ | ||
| 634 | $$corpora{$doc}{$_}=$$para{$_}; | ||
| 635 | } | ||
| 636 | } | ||
| 637 | if (not $UserCorpora->write($corpora)){ | ||
| 638 | return (0,"Could not add corpus info to $CorpusFile!"); | ||
| 639 | } | ||
| 640 | $UserCorpora->close(); | ||
| 641 | } | ||
| 642 | |||
| 643 | sub ChangeCorpusStatus{ | ||
| 644 | my $owner=shift; | ||
| 645 | my $corpus=shift; | ||
| 646 | my $doc=shift; | ||
| 647 | my $status=shift; | ||
| 648 | |||
| 649 | my $CorpusFile="$CorpusDir/$owner/$corpus/.documents"; | ||
| 650 | my $UserCorpora=Uplug::Web::Config->new($CorpusFile); | ||
| 651 | my $corpora=$UserCorpora->read(); | ||
| 652 | |||
| 653 | if (not defined $$corpora{$doc}){return undef;} | ||
| 654 | my $old=$$corpora{$doc}{status}; | ||
| 655 | $$corpora{$doc}{status}=$status; | ||
| 656 | $UserCorpora->close(); | ||
| 657 | return $old; | ||
| 658 | } | ||
| 659 | |||
| 660 | sub ChangeCorpusInfoOld{ | ||
| 661 | |||
| 662 | my $user=shift; | ||
| 663 | my $CorpusName=shift; | ||
| 664 | my $para=shift; | ||
| 665 | |||
| 666 | my $CorpusInfoFile=&GetCorpusDataFile($user,$CorpusName); | ||
| 667 | my %CorpusData; | ||
| 668 | &LoadIniData(\%CorpusData,$CorpusInfoFile); | ||
| 669 | if (not defined $CorpusData{$CorpusName}){ | ||
| 670 | if ((ref($para) eq 'HASH') and (defined $$para{language})){ | ||
| 671 | $CorpusName=&GetCorpusName($CorpusName,$$para{language}); | ||
| 672 | } | ||
| 673 | } | ||
| 674 | if (ref($para) eq 'HASH'){ | ||
| 675 | foreach (keys %{$para}){ | ||
| 676 | $CorpusData{$CorpusName}{$_}=$$para{$_}; | ||
| 677 | } | ||
| 678 | } | ||
| 679 | &WriteIniFile($CorpusInfoFile,\%CorpusData); | ||
| 680 | } | ||
| 681 | |||
| 682 | sub ChangeCorpusStatusOld{ | ||
| 683 | |||
| 684 | my $user=shift; | ||
| 685 | my $CorpusName=shift; | ||
| 686 | my $status=shift; | ||
| 687 | |||
| 688 | my $CorpusInfoFile=&GetCorpusDataFile($user,$CorpusName); | ||
| 689 | my %CorpusData; | ||
| 690 | &LoadIniData(\%CorpusData,$CorpusInfoFile); | ||
| 691 | if (not defined $CorpusData{$CorpusName}){return undef;} | ||
| 692 | my $old=$CorpusData{$CorpusName}{status}; | ||
| 693 | $CorpusData{$CorpusName}{status}=$status; | ||
| 694 | &WriteIniFile($CorpusInfoFile,\%CorpusData); | ||
| 695 | return $old; | ||
| 696 | } | ||
| 697 | |||
| 698 | |||
| 699 | sub GetTempFileName{ | ||
| 700 | my $fh; | ||
| 701 | my $file; | ||
| 702 | do {$file=tmpnam();} | ||
| 703 | until ($fh=IO::File->new($file,O_RDWR|O_CREAT|O_EXCL)); | ||
| 704 | $fh->close; | ||
| 705 | return $file; | ||
| 706 | } | ||
| 707 | |||
| 708 | |||
| 709 | |||
| 710 | sub ChangeWordLinks{ | ||
| 711 | my $file=shift; | ||
| 712 | my $links=shift; | ||
| 713 | my $params=shift; | ||
| 714 | |||
| 715 | my $sentLink=$params->{seg}; | ||
| 716 | print "change links is not implemented yet!<br>"; | ||
| 717 | print join '+',@{$links}; | ||
| 718 | print '<hr>'; | ||
| 719 | |||
| 720 | |||
| 721 | # if (not -e $file){return 0;} | ||
| 722 | # if ($file=~/\.gz$/){open F,"$GUNZIP < $file |";} | ||
| 723 | # else{open F,"< $file";} | ||
| 724 | # my $sec=0; | ||
| 725 | # while (not flock(F,2)){ | ||
| 726 | # $sec++;sleep(1); | ||
| 727 | # if ($sec>$MAXFLOCKWAIT){ | ||
| 728 | # close F; | ||
| 729 | # return 0; | ||
| 730 | # } | ||
| 731 | # } | ||
| 732 | # local $/='<link '; | ||
| 733 | # my @align=<F>; | ||
| 734 | # print join '<hr>',@align; | ||
| 735 | # | ||
| 736 | # close F; | ||
| 737 | |||
| 738 | } |