| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## File: Lingua::TT::DBx::Document.pm | ||
| 3 | ## Author: Bryan Jurish <TT/IO.pm> | ||
| 4 | ## Descript: TT I/O: Document via Berkely DB | ||
| 5 | |||
| 6 | |||
| 7 | package Lingua::TT::DBx::Document; | ||
| 8 | use Lingua::TT::Document; | ||
| 9 | use Lingua::TT::DBx::Field; | ||
| 10 | use Lingua::TT::DBFile; | ||
| 11 | use Lingua::TT::DBFile::Hash; | ||
| 12 | use Lingua::TT::DBFile::BTree; | ||
| 13 | use Lingua::TT::DBFile::Array; | ||
| 14 | use Lingua::TT::DBFile::PackedArray; | ||
| 15 | use File::Path qw(mkpath); | ||
| 16 | use File::Basename qw(basename dirname); | ||
| 17 | use DB_File; | ||
| 18 | use Fcntl; | ||
| 19 | use Encode qw(encode decode); | ||
| 20 | use Carp; | ||
| 21 | use strict; | ||
| 22 | |||
| 23 | ##============================================================================== | ||
| 24 | ## Globals & Constants | ||
| 25 | |||
| 26 | our @ISA = qw(Lingua::TT::Document); | ||
| 27 | |||
| 28 | our $DEFAULT_FIELDS = ['ttline']; | ||
| 29 | |||
| 30 | ##============================================================================== | ||
| 31 | ## Constructors etc. | ||
| 32 | |||
| 33 | |||
| 34 | ## $dbdoc = CLASS_OR_OBJECT->new(%opts) | ||
| 35 | ## + %opts, %$dbdoc: | ||
| 36 | ## ##-- options | ||
| 37 | ## dir => $dirname, ##-- db dirname | ||
| 38 | ## fields => \@fields, ##-- data fields (default: none) | ||
| 39 | ## ## | ||
| 40 | ## ##-- open() options: | ||
| 41 | ## nocreate => $bool, ##-- don't implicitly create $dir (default=0) | ||
| 42 | ## truncate => $bool, ##-- implicitly truncate files in $dir (implies nocreate=>0) | ||
| 43 | ## flags => $flags, ##-- Fcntl O_* flags for DB_File (default: O_RDWR|...) | ||
| 44 | sub new { | ||
| 45 | my $that = shift; | ||
| 46 | my $dbdoc = bless({ | ||
| 47 | ##-- options | ||
| 48 | dir => undef, | ||
| 49 | fields => undef, | ||
| 50 | |||
| 51 | ##-- open() options | ||
| 52 | flags => O_RDWR, | ||
| 53 | |||
| 54 | ##-- user args | ||
| 55 | @_, | ||
| 56 | }, ref($that)||$that); | ||
| 57 | $dbdoc->fields(@{$dbdoc->{fields}||$DEFAULT_FIELDS}); | ||
| 58 | return $dbdoc->open($dbdoc->{dir}) if (defined($dbdoc->{dir})); | ||
| 59 | return $dbdoc; | ||
| 60 | } | ||
| 61 | |||
| 62 | ## $doc = CLASS_OR_OBJECT->newFromString($str) | ||
| 63 | ## + should be equivalent to CLASS_OR_OBJECT->new()->fromString($str) | ||
| 64 | ## + INHERITED from Lingua::TT::Document | ||
| 65 | |||
| 66 | ## \&dummysub = _dummy($SUBNAME) | ||
| 67 | sub _dummy { | ||
| 68 | my $subname = shift; | ||
| 69 | return sub { | ||
| 70 | croak( __PACKAGE__ . "::$subname() not yet implemented!" ); | ||
| 71 | }; | ||
| 72 | } | ||
| 73 | |||
| 74 | ## $doc2 = $doc->copy($depth) | ||
| 75 | ## + creates a copy of $doc | ||
| 76 | ## + if $depth is 0, only a shallow copy is created (sentences & tokens are shared) | ||
| 77 | ## + if $depth is >=1 (or <0), sentences are copied as well (tokens are still shared) | ||
| 78 | ## + if $depth is >=2 (or <0), tokens are copied as well | ||
| 79 | BEGIN { *copy = _dummy('copy'); } | ||
| 80 | |||
| 81 | |||
| 82 | ##============================================================================== | ||
| 83 | ## Methods: Fields | ||
| 84 | |||
| 85 | ## \@fields = $dbdoc->fields(@FIELD_SPECS) | ||
| 86 | ## + compiles $dbdoc->{fields} from @FIELD_SPECS | ||
| 87 | sub fields { | ||
| 88 | my ($dbdoc,@fields) = @_; | ||
| 89 | my ($fspec,$f); | ||
| 90 | my $fields = $dbdoc->{fields} = []; | ||
| 91 | foreach $fspec (@fields) { | ||
| 92 | if (UNIVERSAL::isa($fspec,'Lingua::TT::DBx::Field')) { | ||
| 93 | $fspec = $fspec->new() if (!ref($fspec)); | ||
| 94 | push(@$fields,$fspec); | ||
| 95 | } elsif (UNIVERSAL::isa(('Lingua::TT::DBx::Field::'.lc($fspec)),'Lingua::TT::DBx::Field')) { | ||
| 96 | push(@$fields,('Lingua::TT::DBx::Field::'.lc($fspec))->new); | ||
| 97 | } elsif (!ref($fspec)) { | ||
| 98 | my ($name,$packfmt) = $fspec =~ /^.*\:.$/ ? ($1,$2) : ($fspec,'L'); | ||
| 99 | push(@$fields, Lingua::TT::DBx::Field->new(name=>$name,packfmt=>$packfmt)); | ||
| 100 | } elsif (UNIVERSAL::isa($fspec,'HASH')) { | ||
| 101 | push(@$fields,Lingua::TT::DBx::Field->new(%$fspec)); | ||
| 102 | } elsif (UNIVERSAL::isa($fspec,'ARRAY')) { | ||
| 103 | push(@$fields,Lingua::TT::DBx::Field->new(@$fspec)); | ||
| 104 | } elsif (UNIVERSAL::isa($fspec,'CODE')) { | ||
| 105 | push(@$fields,$fspec->($dbdoc)); | ||
| 106 | } else { | ||
| 107 | warn(ref($dbdoc)."::fields(): cannot parse field spec $fspec - skipping"); | ||
| 108 | } | ||
| 109 | } | ||
| 110 | $_->{dir} = $dbdoc->{dir} foreach (@{$dbdoc->{fields}}); | ||
| 111 | return $dbdoc->{fields}; | ||
| 112 | } | ||
| 113 | |||
| 114 | ##============================================================================== | ||
| 115 | ## Methods: I/O: open, close, sync | ||
| 116 | |||
| 117 | ## $bool = $dbdoc->opened() | ||
| 118 | sub opened { | ||
| 119 | my $dbdoc = shift; | ||
| 120 | return defined($dbdoc->{dir}) && !grep {!$_ || !$_->opened} @{$dbdoc->{fields}}, @$dbdoc{qw(docs sents cmts)}; | ||
| 121 | } | ||
| 122 | |||
| 123 | ## $bool = $dbdoc->sync() | ||
| 124 | ## + syncs all files | ||
| 125 | sub sync { | ||
| 126 | my $dbdoc = shift; | ||
| 127 | my $rc = $dbdoc->saveLocalFiles(); | ||
| 128 | foreach (@{$dbdoc->{fields}}, @$dbdoc{qw(docs sents cmts)}) { | ||
| 129 | $rc &&= $_->sync(); | ||
| 130 | } | ||
| 131 | return $rc; | ||
| 132 | } | ||
| 133 | |||
| 134 | ## $bool = $dbdoc->close(%opts) | ||
| 135 | ## + %opts: | ||
| 136 | ## nosync => $bool, ##-- if true, no implicit sync() will be performed (default=0) | ||
| 137 | sub close { | ||
| 138 | my ($dbdoc,%opts) = @_; | ||
| 139 | my $rc = $opts{nosync} ? 1 : $dbdoc->sync(); | ||
| 140 | foreach (@{$dbdoc->{fields}}, @$dbdoc{qw(docs sents cmts)}) { | ||
| 141 | $rc &&= $_->close(%opts); | ||
| 142 | } | ||
| 143 | return $rc; | ||
| 144 | } | ||
| 145 | |||
| 146 | ## $dbdoc = $dbdoc->open($dir,%opts) | ||
| 147 | ## + %opts: | ||
| 148 | ## nocreate => $bool, ##-- don't implicitly create $dir (default=0) | ||
| 149 | ## truncate => $bool, ##-- implicitly truncate files in $dir (implies nocreate=>0) | ||
| 150 | ## mode => $mode, ##-- Fcntl mode for DB_File (default: O_RDWR|...) | ||
| 151 | sub open { | ||
| 152 | my ($dbdoc,$dir,%opts) = @_; | ||
| 153 | $dir =~ s/\/+$//; | ||
| 154 | $dbdoc->close if ($dbdoc->opened); | ||
| 155 | @$dbdoc{keys %opts} = values(%opts); | ||
| 156 | $dbdoc->{dir} = $dir; | ||
| 157 | |||
| 158 | $dbdoc->{flags} = O_RDWR if (!defined($dbdoc->{flags})); | ||
| 159 | |||
| 160 | ##-- implicit create? | ||
| 161 | if (!$dbdoc->{nocreate}) { | ||
| 162 | $dbdoc->{flags} |= O_CREAT; | ||
| 163 | -d $dir || File::Path::mkpath($dir) | ||
| 164 | or confess(ref($dbdoc)."::open(+create): mkpath() failed for '$dir': $!"); | ||
| 165 | } | ||
| 166 | |||
| 167 | ##-- load local files (only if not truncating) | ||
| 168 | $dbdoc->loadLocalFiles if (!$dbdoc->{truncate}); | ||
| 169 | |||
| 170 | ##-- set basic field data dir | ||
| 171 | @$_{qw(dir flags)} = @$dbdoc{qw(dir flags)} foreach (@{$dbdoc->{fields}}); | ||
| 172 | |||
| 173 | ##-- truncate field datafiles before open? | ||
| 174 | if ($dbdoc->{truncate}) { | ||
| 175 | $dbdoc->{flags} |= O_TRUNC; | ||
| 176 | foreach (@{$dbdoc->{fields}}) { | ||
| 177 | next if ($dbdoc->{nocreate} && !$_->fileExists); | ||
| 178 | $_->truncate(); | ||
| 179 | } | ||
| 180 | } | ||
| 181 | ##-- open local data files | ||
| 182 | ## + docs: ARRAY: $file_i => join("\t", $doc_offset_nsents, $doc_filename)."\n" | ||
| 183 | ## + sents: ARRAY: $sent_i => pack('L', $sent_offset_ntoks) | ||
| 184 | ## + cmts: BTREE: $tok_i => join("\n", @comment_lines_before_tok_i) | ||
| 185 | $dbdoc->{docs} = Lingua::TT::DBFile::Array->new(file=>"$dir/docs.db", | ||
| 186 | flags=>$dbdoc->{flags} | ||
| 187 | ) | ||
| 188 | or confess(ref($dbdoc)."::open(): open failed for '$dir/docs.db': $!"); | ||
| 189 | $dbdoc->{sents} = Lingua::TT::DBFile::PackedArray->new(file=>"$dir/sents.db", | ||
| 190 | flags=>$dbdoc->{flags}, | ||
| 191 | packfmt=>'L') | ||
| 192 | or confess(ref($dbdoc)."::open(): open failed for '$dir/sents.db': $!"); | ||
| 193 | $dbdoc->{cmts} = Lingua::TT::DBFile::BTree->new(file=>"$dir/cmts.db", | ||
| 194 | flags=>$dbdoc->{flags}, | ||
| 195 | dbopts=>{compare=>sub { $_[0] <=> $_[1] }}, | ||
| 196 | ) | ||
| 197 | or confess(ref($dbdoc)."::open(): open failed for '$dir/cmts.db': $!"); | ||
| 198 | |||
| 199 | ##-- open field data files | ||
| 200 | foreach (@{$dbdoc->{fields}}) { | ||
| 201 | $_->open(flags=>$dbdoc->{flags}) | ||
| 202 | or confess(ref($dbdoc)."::open(): open failed for field name '$_->{name}': $!"); | ||
| 203 | } | ||
| 204 | |||
| 205 | ##-- compile field closures | ||
| 206 | $_->compileClosures() foreach (@{$dbdoc->{fields}}); | ||
| 207 | |||
| 208 | return $dbdoc; | ||
| 209 | } | ||
| 210 | |||
| 211 | ##============================================================================== | ||
| 212 | ## Methods: I/O: utils | ||
| 213 | |||
| 214 | ## $dbdoc = $dbdoc->truncate() | ||
| 215 | sub truncate { | ||
| 216 | my $dbdoc = shift; | ||
| 217 | my $dir = $dbdoc->{dir}; | ||
| 218 | return if (!defined($dir)); | ||
| 219 | CORE::truncate("$dir/config.pm",0); | ||
| 220 | $_->truncate() foreach (@{$dbdoc->{fields}}, @$dbdoc{qw(docs sents cmts)}); | ||
| 221 | return $dbdoc; | ||
| 222 | } | ||
| 223 | |||
| 224 | ## $dbdoc = $dbdoc->loadLocalFiles() | ||
| 225 | sub loadLocalFiles { | ||
| 226 | my $dbdoc = shift; | ||
| 227 | my $dir = $dbdoc->{dir}||'.'; | ||
| 228 | |||
| 229 | ##-- load: config | ||
| 230 | $dbdoc->loadPerlFile("$dir/config.plm") | ||
| 231 | or confess(ref($dbdoc)."::loadLocalFiles(): load failed for '$dir/config.plm': $!"); | ||
| 232 | |||
| 233 | return $dbdoc; | ||
| 234 | } | ||
| 235 | |||
| 236 | ## $dbdoc = $dbdoc->saveLocalFiles(); | ||
| 237 | sub saveLocalFiles { | ||
| 238 | my $dbdoc = shift; | ||
| 239 | my $dir = $dbdoc->{dir} || '.'; | ||
| 240 | |||
| 241 | ##-- save: config | ||
| 242 | $dbdoc->savePerlFile("$dir/config.plm") | ||
| 243 | or confess(ref($dbdoc)."::saveLocalFiles(): save failed for '$dir/config.plm': $!"); | ||
| 244 | |||
| 245 | return $dbdoc; | ||
| 246 | } | ||
| 247 | |||
| 248 | ## @filenames = $dbdoc->datafiles() | ||
| 249 | sub datafiles { | ||
| 250 | my $dbdoc = shift; | ||
| 251 | my $dir = $dbdoc->{dir}; | ||
| 252 | return ("$dir/config.plm", | ||
| 253 | "$dir/docs.db", | ||
| 254 | "$dir/sents.db", | ||
| 255 | "$dir/cmts.db", | ||
| 256 | (map {$_->datafiles} @{$dbdoc->{fields}}), | ||
| 257 | ); | ||
| 258 | } | ||
| 259 | |||
| 260 | ##============================================================================== | ||
| 261 | ## Methods: I/O: TT::Persistent interface | ||
| 262 | |||
| 263 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 264 | sub noSaveKeys { | ||
| 265 | return (qw(fields dir truncate nocreate flags docs sents cmts)); | ||
| 266 | } | ||
| 267 | |||
| 268 | ## $ref = $dbdoc->savePerlRef() | ||
| 269 | sub savePerlRef { | ||
| 270 | my $obj = shift; | ||
| 271 | my $ref = $obj->SUPER::savePerlRef(); | ||
| 272 | $ref->{fields} = [map {$_->savePerlRef} @{$obj->{fields}}]; | ||
| 273 | delete(@$_{qw(dir file flags)}) foreach (@{$ref->{fields}}); | ||
| 274 | return $ref; | ||
| 275 | } | ||
| 276 | |||
| 277 | ## $ref = $dbdoc->loadPerlRef() | ||
| 278 | sub loadPerlRef { | ||
| 279 | my ($that,$ref) = @_; | ||
| 280 | my $obj = $that->SUPER::loadPerlRef($ref); | ||
| 281 | foreach (@{$obj->{fields}}) { | ||
| 282 | $_->{dir} = $obj->{dir} if (defined($obj->{dir})); | ||
| 283 | $_->{flags} = $obj->{flags} if (defined($obj->{flags})); | ||
| 284 | #$_->{file} = $_->filename; | ||
| 285 | } | ||
| 286 | return $obj; | ||
| 287 | } | ||
| 288 | |||
| 289 | ##============================================================================== | ||
| 290 | ## Methods: Access: add TT Data | ||
| 291 | |||
| 292 | ## $offset = $dbdoc->offset() | ||
| 293 | ## + gets current (max) token offset | ||
| 294 | sub offset { | ||
| 295 | return $_[0]{fields}[0]{dbf}{tied}->length; | ||
| 296 | } | ||
| 297 | |||
| 298 | ## $dbdoc = $dbdoc->putLine($tokstr) | ||
| 299 | sub putLine { | ||
| 300 | $_[0]->putToken(Lingua::TT::Token->newFromString($_[1])); | ||
| 301 | return $_[0]; | ||
| 302 | } | ||
| 303 | |||
| 304 | ## $dbdoc = $dbdoc->putToken(\@tok) | ||
| 305 | sub putToken { | ||
| 306 | if ($_[1][0] =~ /^\s*\%\%/) { | ||
| 307 | ##-- comment | ||
| 308 | my $off = $_[0]->offset; | ||
| 309 | if (!exists($_[0]{cmts}{data}{$off})) { | ||
| 310 | $_[0]{cmts}{data}{$off} = join("\t", @{$_[1]})."\n"; | ||
| 311 | } else { | ||
| 312 | $_[0]{cmts}{data}{$off} .= join("\t", @{$_[1]})."\n"; | ||
| 313 | } | ||
| 314 | } else { | ||
| 315 | ##-- vanilla | ||
| 316 | $_->putToken($_[1]) foreach (@{$_[0]{fields}}); | ||
| 317 | } | ||
| 318 | return $_[0]; | ||
| 319 | } | ||
| 320 | |||
| 321 | ## $dbdoc = $dbdoc->putSentence($sent) | ||
| 322 | sub putSentence { | ||
| 323 | my $off0 = $_[0]{sents}{data}[$#{$_[0]{sents}{data}}]; | ||
| 324 | my $off1 = pack('L',$_[0]->offset); | ||
| 325 | push(@{$_[0]{sents}{data}}, $off1) if (!defined($off0) || $off1 ne $off0); | ||
| 326 | $_[0]->putToken($_) foreach (@{$_[1]}); | ||
| 327 | push(@{$_[0]{sents}{data}}, pack('L',$_[0]->offset)) if (@{$_[1]}); | ||
| 328 | return $_[0]; | ||
| 329 | } | ||
| 330 | |||
| 331 | ## $dbdoc = $dbdoc->putDocument($doc) | ||
| 332 | sub putDocument { | ||
| 333 | $_[0]->putSentence($_) foreach (@{$_[1]}); | ||
| 334 | return $_[0]; | ||
| 335 | |||
| 336 | } | ||
| 337 | |||
| 338 | ## $dbdoc = $dbdoc->putIO($io) | ||
| 339 | sub putIO { | ||
| 340 | $_[0]->putDocument($_[1]->getDocument); | ||
| 341 | } | ||
| 342 | |||
| 343 | |||
| 344 | ##!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! | ||
| 345 | ## OLD TT::Document API | ||
| 346 | ##!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! | ||
| 347 | __END__ |