TT/DBx/Document.pm untested

filter: Statementstmt 0.0% 0 / 191 Branchbran 0.0% 0 / 46 Conditioncond 0.0% 0 / 45 MC/DCmcdc 0.0% 0 / 26 Subroutinesub 0.0% 0 / 35 Podpod 0.0% 0 / 21 totaltotal 0.0% 0 / 364 CC 24 SCAR 31.8
CC
24
Cov
0%
fields
90.0
open
56.0
loadPerlRef
12.0
CRAP 24.1
1 ## -*- Mode: CPerl -*-
2 ## File: Lingua::TT::DBx::Document.pm
3 ## Author: Bryan Jurish <TT/IO.pm>
4 ## Descript: TT I/O: Document via Berkely DB
5  
6  
7 package Lingua::TT::DBx::Document;
8 use Lingua::TT::Document;
9 use Lingua::TT::DBx::Field;
10 use Lingua::TT::DBFile;
11 use Lingua::TT::DBFile::Hash;
12 use Lingua::TT::DBFile::BTree;
13 use Lingua::TT::DBFile::Array;
14 use Lingua::TT::DBFile::PackedArray;
15 use File::Path qw(mkpath);
16 use File::Basename qw(basename dirname);
17 use DB_File;
18 use Fcntl;
19 use Encode qw(encode decode);
20 use Carp;
21 use strict;
22  
23 ##==============================================================================
24 ## Globals & Constants
25  
26 our @ISA = qw(Lingua::TT::Document);
27  
28 our $DEFAULT_FIELDS = ['ttline'];
29  
30 ##==============================================================================
31 ## Constructors etc.
32  
33  
34 ## $dbdoc = CLASS_OR_OBJECT->new(%opts)
35 ## + %opts, %$dbdoc:
36 ## ##-- options
37 ## dir => $dirname, ##-- db dirname
38 ## fields => \@fields, ##-- data fields (default: none)
39 ## ##
40 ## ##-- open() options:
41 ## nocreate => $bool, ##-- don't implicitly create $dir (default=0)
42 ## truncate => $bool, ##-- implicitly truncate files in $dir (implies nocreate=>0)
43 ## flags => $flags, ##-- Fcntl O_* flags for DB_File (default: O_RDWR|...)
44 sub new {
45 my $that = shift;
46 my $dbdoc = bless({
47 ##-- options
48 dir => undef,
49 fields => undef,
50  
51 ##-- open() options
52 flags => O_RDWR,
53  
54 ##-- user args
55 @_,
56 }, ref($that)||$that);
57 $dbdoc->fields(@{$dbdoc->{fields}||$DEFAULT_FIELDS});
58 return $dbdoc->open($dbdoc->{dir}) if (defined($dbdoc->{dir}));
59 return $dbdoc;
60 }
61  
62 ## $doc = CLASS_OR_OBJECT->newFromString($str)
63 ## + should be equivalent to CLASS_OR_OBJECT->new()->fromString($str)
64 ## + INHERITED from Lingua::TT::Document
65  
66 ## \&dummysub = _dummy($SUBNAME)
67 sub _dummy {
68 my $subname = shift;
69 return sub {
70 croak( __PACKAGE__ . "::$subname() not yet implemented!" );
71 };
72 }
73  
74 ## $doc2 = $doc->copy($depth)
75 ## + creates a copy of $doc
76 ## + if $depth is 0, only a shallow copy is created (sentences & tokens are shared)
77 ## + if $depth is >=1 (or <0), sentences are copied as well (tokens are still shared)
78 ## + if $depth is >=2 (or <0), tokens are copied as well
79 BEGIN { *copy = _dummy('copy'); }
80  
81  
82 ##==============================================================================
83 ## Methods: Fields
84  
85 ## \@fields = $dbdoc->fields(@FIELD_SPECS)
86 ## + compiles $dbdoc->{fields} from @FIELD_SPECS
87 sub fields {
88 my ($dbdoc,@fields) = @_;
89 my ($fspec,$f);
90 my $fields = $dbdoc->{fields} = [];
91 foreach $fspec (@fields) {
92 if (UNIVERSAL::isa($fspec,'Lingua::TT::DBx::Field')) {
93 $fspec = $fspec->new() if (!ref($fspec));
94 push(@$fields,$fspec);
95 } elsif (UNIVERSAL::isa(('Lingua::TT::DBx::Field::'.lc($fspec)),'Lingua::TT::DBx::Field')) {
96 push(@$fields,('Lingua::TT::DBx::Field::'.lc($fspec))->new);
97 } elsif (!ref($fspec)) {
98 my ($name,$packfmt) = $fspec =~ /^.*\:.$/ ? ($1,$2) : ($fspec,'L');
99 push(@$fields, Lingua::TT::DBx::Field->new(name=>$name,packfmt=>$packfmt));
100 } elsif (UNIVERSAL::isa($fspec,'HASH')) {
101 push(@$fields,Lingua::TT::DBx::Field->new(%$fspec));
102 } elsif (UNIVERSAL::isa($fspec,'ARRAY')) {
103 push(@$fields,Lingua::TT::DBx::Field->new(@$fspec));
104 } elsif (UNIVERSAL::isa($fspec,'CODE')) {
105 push(@$fields,$fspec->($dbdoc));
106 } else {
107 warn(ref($dbdoc)."::fields(): cannot parse field spec $fspec - skipping");
108 }
109 }
110 $_->{dir} = $dbdoc->{dir} foreach (@{$dbdoc->{fields}});
111 return $dbdoc->{fields};
112 }
113  
114 ##==============================================================================
115 ## Methods: I/O: open, close, sync
116  
117 ## $bool = $dbdoc->opened()
118 sub opened {
119 my $dbdoc = shift;
120 return defined($dbdoc->{dir}) && !grep {!$_ || !$_->opened} @{$dbdoc->{fields}}, @$dbdoc{qw(docs sents cmts)};
121 }
122  
123 ## $bool = $dbdoc->sync()
124 ## + syncs all files
125 sub sync {
126 my $dbdoc = shift;
127 my $rc = $dbdoc->saveLocalFiles();
128 foreach (@{$dbdoc->{fields}}, @$dbdoc{qw(docs sents cmts)}) {
129 $rc &&= $_->sync();
130 }
131 return $rc;
132 }
133  
134 ## $bool = $dbdoc->close(%opts)
135 ## + %opts:
136 ## nosync => $bool, ##-- if true, no implicit sync() will be performed (default=0)
137 sub close {
138 my ($dbdoc,%opts) = @_;
139 my $rc = $opts{nosync} ? 1 : $dbdoc->sync();
140 foreach (@{$dbdoc->{fields}}, @$dbdoc{qw(docs sents cmts)}) {
141 $rc &&= $_->close(%opts);
142 }
143 return $rc;
144 }
145  
146 ## $dbdoc = $dbdoc->open($dir,%opts)
147 ## + %opts:
148 ## nocreate => $bool, ##-- don't implicitly create $dir (default=0)
149 ## truncate => $bool, ##-- implicitly truncate files in $dir (implies nocreate=>0)
150 ## mode => $mode, ##-- Fcntl mode for DB_File (default: O_RDWR|...)
151 sub open {
152 my ($dbdoc,$dir,%opts) = @_;
153 $dir =~ s/\/+$//;
154 $dbdoc->close if ($dbdoc->opened);
155 @$dbdoc{keys %opts} = values(%opts);
156 $dbdoc->{dir} = $dir;
157  
158 $dbdoc->{flags} = O_RDWR if (!defined($dbdoc->{flags}));
159  
160 ##-- implicit create?
161 if (!$dbdoc->{nocreate}) {
162 $dbdoc->{flags} |= O_CREAT;
163 -d $dir || File::Path::mkpath($dir)
164 or confess(ref($dbdoc)."::open(+create): mkpath() failed for '$dir': $!");
165 }
166  
167 ##-- load local files (only if not truncating)
168 $dbdoc->loadLocalFiles if (!$dbdoc->{truncate});
169  
170 ##-- set basic field data dir
171 @$_{qw(dir flags)} = @$dbdoc{qw(dir flags)} foreach (@{$dbdoc->{fields}});
172  
173 ##-- truncate field datafiles before open?
174 if ($dbdoc->{truncate}) {
175 $dbdoc->{flags} |= O_TRUNC;
176 foreach (@{$dbdoc->{fields}}) {
177 next if ($dbdoc->{nocreate} && !$_->fileExists);
178 $_->truncate();
179 }
180 }
181 ##-- open local data files
182 ## + docs: ARRAY: $file_i => join("\t", $doc_offset_nsents, $doc_filename)."\n"
183 ## + sents: ARRAY: $sent_i => pack('L', $sent_offset_ntoks)
184 ## + cmts: BTREE: $tok_i => join("\n", @comment_lines_before_tok_i)
185 $dbdoc->{docs} = Lingua::TT::DBFile::Array->new(file=>"$dir/docs.db",
186 flags=>$dbdoc->{flags}
187 )
188 or confess(ref($dbdoc)."::open(): open failed for '$dir/docs.db': $!");
189 $dbdoc->{sents} = Lingua::TT::DBFile::PackedArray->new(file=>"$dir/sents.db",
190 flags=>$dbdoc->{flags},
191 packfmt=>'L')
192 or confess(ref($dbdoc)."::open(): open failed for '$dir/sents.db': $!");
193 $dbdoc->{cmts} = Lingua::TT::DBFile::BTree->new(file=>"$dir/cmts.db",
194 flags=>$dbdoc->{flags},
195 dbopts=>{compare=>sub { $_[0] <=> $_[1] }},
196 )
197 or confess(ref($dbdoc)."::open(): open failed for '$dir/cmts.db': $!");
198  
199 ##-- open field data files
200 foreach (@{$dbdoc->{fields}}) {
201 $_->open(flags=>$dbdoc->{flags})
202 or confess(ref($dbdoc)."::open(): open failed for field name '$_->{name}': $!");
203 }
204  
205 ##-- compile field closures
206 $_->compileClosures() foreach (@{$dbdoc->{fields}});
207  
208 return $dbdoc;
209 }
210  
211 ##==============================================================================
212 ## Methods: I/O: utils
213  
214 ## $dbdoc = $dbdoc->truncate()
215 sub truncate {
216 my $dbdoc = shift;
217 my $dir = $dbdoc->{dir};
218 return if (!defined($dir));
219 CORE::truncate("$dir/config.pm",0);
220 $_->truncate() foreach (@{$dbdoc->{fields}}, @$dbdoc{qw(docs sents cmts)});
221 return $dbdoc;
222 }
223  
224 ## $dbdoc = $dbdoc->loadLocalFiles()
225 sub loadLocalFiles {
226 my $dbdoc = shift;
227 my $dir = $dbdoc->{dir}||'.';
228  
229 ##-- load: config
230 $dbdoc->loadPerlFile("$dir/config.plm")
231 or confess(ref($dbdoc)."::loadLocalFiles(): load failed for '$dir/config.plm': $!");
232  
233 return $dbdoc;
234 }
235  
236 ## $dbdoc = $dbdoc->saveLocalFiles();
237 sub saveLocalFiles {
238 my $dbdoc = shift;
239 my $dir = $dbdoc->{dir} || '.';
240  
241 ##-- save: config
242 $dbdoc->savePerlFile("$dir/config.plm")
243 or confess(ref($dbdoc)."::saveLocalFiles(): save failed for '$dir/config.plm': $!");
244  
245 return $dbdoc;
246 }
247  
248 ## @filenames = $dbdoc->datafiles()
249 sub datafiles {
250 my $dbdoc = shift;
251 my $dir = $dbdoc->{dir};
252 return ("$dir/config.plm",
253 "$dir/docs.db",
254 "$dir/sents.db",
255 "$dir/cmts.db",
256 (map {$_->datafiles} @{$dbdoc->{fields}}),
257 );
258 }
259  
260 ##==============================================================================
261 ## Methods: I/O: TT::Persistent interface
262  
263 ## @keys = $class_or_obj->noSaveKeys()
264 sub noSaveKeys {
265 return (qw(fields dir truncate nocreate flags docs sents cmts));
266 }
267  
268 ## $ref = $dbdoc->savePerlRef()
269 sub savePerlRef {
270 my $obj = shift;
271 my $ref = $obj->SUPER::savePerlRef();
272 $ref->{fields} = [map {$_->savePerlRef} @{$obj->{fields}}];
273 delete(@$_{qw(dir file flags)}) foreach (@{$ref->{fields}});
274 return $ref;
275 }
276  
277 ## $ref = $dbdoc->loadPerlRef()
278 sub loadPerlRef {
279 my ($that,$ref) = @_;
280 my $obj = $that->SUPER::loadPerlRef($ref);
281 foreach (@{$obj->{fields}}) {
282 $_->{dir} = $obj->{dir} if (defined($obj->{dir}));
283 $_->{flags} = $obj->{flags} if (defined($obj->{flags}));
284 #$_->{file} = $_->filename;
285 }
286 return $obj;
287 }
288  
289 ##==============================================================================
290 ## Methods: Access: add TT Data
291  
292 ## $offset = $dbdoc->offset()
293 ## + gets current (max) token offset
294 sub offset {
295 return $_[0]{fields}[0]{dbf}{tied}->length;
296 }
297  
298 ## $dbdoc = $dbdoc->putLine($tokstr)
299 sub putLine {
300 $_[0]->putToken(Lingua::TT::Token->newFromString($_[1]));
301 return $_[0];
302 }
303  
304 ## $dbdoc = $dbdoc->putToken(\@tok)
305 sub putToken {
306 if ($_[1][0] =~ /^\s*\%\%/) {
307 ##-- comment
308 my $off = $_[0]->offset;
309 if (!exists($_[0]{cmts}{data}{$off})) {
310 $_[0]{cmts}{data}{$off} = join("\t", @{$_[1]})."\n";
311 } else {
312 $_[0]{cmts}{data}{$off} .= join("\t", @{$_[1]})."\n";
313 }
314 } else {
315 ##-- vanilla
316 $_->putToken($_[1]) foreach (@{$_[0]{fields}});
317 }
318 return $_[0];
319 }
320  
321 ## $dbdoc = $dbdoc->putSentence($sent)
322 sub putSentence {
323 my $off0 = $_[0]{sents}{data}[$#{$_[0]{sents}{data}}];
324 my $off1 = pack('L',$_[0]->offset);
325 push(@{$_[0]{sents}{data}}, $off1) if (!defined($off0) || $off1 ne $off0);
326 $_[0]->putToken($_) foreach (@{$_[1]});
327 push(@{$_[0]{sents}{data}}, pack('L',$_[0]->offset)) if (@{$_[1]});
328 return $_[0];
329 }
330  
331 ## $dbdoc = $dbdoc->putDocument($doc)
332 sub putDocument {
333 $_[0]->putSentence($_) foreach (@{$_[1]});
334 return $_[0];
335  
336 }
337  
338 ## $dbdoc = $dbdoc->putIO($io)
339 sub putIO {
340 $_[0]->putDocument($_[1]->getDocument);
341 }
342  
343  
344 ##!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
345 ## OLD TT::Document API
346 ##!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
347 __END__