Bio/DB/Flat/BDB.pm

filter: Statementstmt 66.6% 4 / 6 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 2 / 2 Podpod n/a totaltotal 75.0% 6 / 8 CC 1 SCAR 1.4
CC
1
Cov
67%
BEGIN
1.3
BEGIN
1.0
CRAP 1.1
« Bio/Factory/ObjectFactoryI.pm ↑ Summary Bio/OntologyIO/InterProParser.pm »
1 #
2 #
3 # BioPerl module for Bio::DB::Flat::BDB
4 #
5 # Please direct questions and support issues to <bioperl-l@bioperl.org>
6 #
7 # Cared for by Lincoln Stein <lstein@cshl.org>
8 #
9 # You may distribute this module under the same terms as perl itself
10  
11 # POD documentation - main docs before the code
12  
13 =head1 NAME
14  
15 Bio::DB::Flat::BDB - Interface for BioHackathon standard BDB-indexed flat file
16  
17 =head1 SYNOPSIS
18  
19 #You should not be using this module directly.
20  
21 See L<Bio::DB::Flat>.
22  
23 =head1 DESCRIPTION
24  
25 This object provides the basic mechanism to associate positions in
26 files with primary and secondary name spaces. Unlike
27 Bio::Index::Abstract (see L<Bio::Index::Abstract>), this is specialized
28 to work with the BerkeleyDB-indexed "common" flat file format worked
29 out at the 2002 BioHackathon.
30  
31 This object is the guts to the mechanism, which will be used by the
32 specific objects inheriting from it.
33  
34 =head1 FEEDBACK
35  
36 =head2 Mailing Lists
37  
38 User feedback is an integral part of the evolution of this and other
39 Bioperl modules. Send your comments and suggestions preferably to one
40 of the Bioperl mailing lists. Your participation is much appreciated.
41  
42 bioperl-l@bioperl.org - General discussion
43 http://bioperl.org/wiki/Mailing_lists - About the mailing lists
44  
45 =head2 Support
46  
47 Please direct usage questions or support issues to the mailing list:
48  
49 I<bioperl-l@bioperl.org>
50  
51 rather than to the module maintainer directly. Many experienced and
52 reponsive experts will be able look at the problem and quickly
53 address it. Please include a thorough description of the problem
54 with code and data examples if at all possible.
55  
56 =head2 Reporting Bugs
57  
58 Report bugs to the Bioperl bug tracking system to help us keep track
59 the bugs and their resolution. Bug reports can be submitted via
60 email or the web:
61  
62 https://github.com/bioperl/bioperl-live/issues
63  
64 =head1 AUTHOR - Lincoln Stein
65  
66 Email - lstein@cshl.org
67  
68 =head1 SEE ALSO
69  
70 L<Bio::DB::Flat>,
71  
72 =head1 APPENDIX
73  
74 The rest of the documentation details each of the object methods. Internal
75 methods are usually preceded with an "_" (underscore).
76  
77 =cut
78  
79  
80 # Let the code begin...
81  
82 package Bio::DB::Flat::BDB;
83 $Bio::DB::Flat::BDB::VERSION = '1.7.8';
84 5 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
85 0 ▶use DB_File;
Subroutine BEGIN: called CC=1 SCAR=2.6
86 use IO::File;
87 use Fcntl qw(O_CREAT O_RDWR O_RDONLY);
88 use File::Spec;
89 use Bio::SeqIO;
90 use Bio::DB::RandomAccessI;
91 use Bio::Root::Root;
92 use Bio::Root::IO;
93  
94 use base qw(Bio::DB::Flat);
95  
96 sub _initialize {
97 my $self = shift;
98 my ($max_open) = $self->_rearrange(['MAXOPEN'],@_);
99 $self->{bdb_maxopen} = $max_open || 32;
100 }
101  
102 # return a filehandle seeked to the appropriate place
103 # this only works with the primary namespace
104 sub _get_stream {
105 my ($self,$id) = @_;
106 my ($filepath,$offset,$length) = $self->_lookup_primary($id)
107 or $self->throw("Unable to find a record for $id in the flat file index");
108 my $fh = $self->_fhcache($filepath)
109 or $self->throw("couldn't open $filepath: $!");
110 seek($fh,$offset,0) or $self->throw("can't seek on $filepath: $!");
111 $fh;
112 }
113  
114 # return records corresponding to the indicated index
115 # if there are multiple hits will return a list in list context,
116 # otherwise will throw an exception
117 sub fetch_raw {
118 my ($self,$id,$namespace) = @_;
119  
120 # secondary lookup
121 if (defined $namespace && $namespace ne $self->primary_namespace) {
122 my @hits = $self->_lookup_secondary($namespace,$id);
123 $self->throw("Multiple records correspond to $namespace=>$id but function called in a scalar context")
124 unless wantarray;
125 return map {$self->_read_record(@$_)} @hits;
126 }
127  
128 # primary lookup
129 my @args = $self->_lookup_primary($id)
130 or $self->throw("Unable to find a record for $id in the flat file index");
131 return $self->_read_record(@args);
132 }
133  
134 # create real live Bio::Seq object
135 sub get_Seq_by_id {
136 my $self = shift;
137 my $id = shift;
138 my $fh = eval {$self->_get_stream($id)} or return;
139 my $seqio =
140 $self->{bdb_cached_parsers}{fileno $fh} ||= Bio::SeqIO->new( -Format => $self->file_format,
141 -fh => $fh);
142 return $seqio->next_seq;
143 }
144  
145 # fetch array of Bio::Seq objects
146 sub get_Seq_by_acc {
147 my $self = shift;
148 unshift @_,'ACC' if @_==1;
149 my ($ns,$key) = @_;
150 my @primary_ids = $self->expand_ids($ns => $key);
151 $self->throw("more than one sequences correspond to this accession")
152 if @primary_ids > 1 && ! wantarray;
153 my @rc = map {$self->get_Seq_by_id($_)} @primary_ids;
154 return wantarray ? @rc : $rc[0];
155 }
156  
157 # fetch array of Bio::Seq objects
158 sub get_Seq_by_version {
159 my $self = shift;
160 unshift @_,'VERSION' if @_==1;
161 my ($ns,$key) = @_;
162 my @primary_ids = $self->expand_ids($ns => $key);
163 $self->throw("more than one sequences correspond to this accession")
164 if @primary_ids > 1 && !wantarray;
165 my @rc = map {$self->get_Seq_by_id($_)} @primary_ids;
166 return wantarray ? @rc : $rc[0];
167 }
168  
169 =head2 get_PrimarySeq_stream
170  
171 Title : get_PrimarySeq_stream
172 Usage : $stream = get_PrimarySeq_stream
173 Function: Makes a Bio::DB::SeqStreamI compliant object
174 which provides a single method, next_primary_seq
175 Returns : Bio::DB::SeqStreamI
176 Args : none
177  
178  
179 =cut
180  
181 sub get_PrimarySeq_stream {
182 my $self = shift;
183 my @files = $self->files || 0;
184 my $out = Bio::SeqIO::MultiFile->new( -format => $self->file_format ,
185 -files => \@files);
186 return $out;
187 }
188  
189 sub get_all_primary_ids {
190 my $self = shift;
191 my $db = $self->primary_db;
192 return keys %$db;
193 }
194  
195 =head2 get_all_primary_ids
196  
197 Title : get_all_primary_ids
198 Usage : @ids = $seqdb->get_all_primary_ids()
199 Function: gives an array of all the primary_ids of the
200 sequence objects in the database.
201 Example :
202 Returns : an array of strings
203 Args : none
204  
205 =cut
206  
207 # this will perform an ID lookup on a (possibly secondary)
208 # id, returning all the corresponding ids
209 sub expand_ids {
210 my $self = shift;
211 my ($ns,$key) = @_;
212 return $key unless defined $ns;
213 return $key if $ns eq $self->primary_namespace;
214 my $db = $self->secondary_db($ns)
215 or $self->throw("invalid secondary namespace $ns");
216 my $record = $db->{$key} or return; # nothing there
217 return $self->unpack_secondary($record);
218 }
219  
220 # build index from files listed
221 sub build_index {
222 my $self = shift;
223 my @files = @_;
224 my $count = 0;
225 for my $file (@files) {
226 $file = File::Spec->rel2abs($file)
227 unless File::Spec->file_name_is_absolute($file);
228 $count += $self->_index_file($file);
229 }
230 $self->write_config;
231 $count;
232 }
233  
234 sub _index_file {
235 my $self = shift;
236 my $file = shift;
237  
238 my $fileno = $self->_path2fileno($file);
239 defined $fileno or $self->throw("could not create a file number for $file");
240  
241 my $fh = $self->_fhcache($file) or $self->throw("could not open $file for indexing: $!");
242 my $offset = 0;
243 my $count = 0;
244  
245 while (!eof($fh)) {
246 my ($ids,$adjustment) = $self->parse_one_record($fh) or next;
247 $adjustment ||= 0; # prevent uninit variable warning
248 my $pos = tell($fh) + $adjustment;
249 $self->_store_index($ids,$file,$offset,$pos-$offset);
250 $offset = $pos;
251 $count++;
252 }
253 $count;
254 }
255  
256 =head2 To Be Implemented in Subclasses
257  
258 The following methods MUST be implemented by subclasses.
259  
260 =cut
261  
262 =head2 May Be Overridden in Subclasses
263  
264 The following methods MAY be overridden by subclasses.
265  
266 =cut
267  
268 sub default_primary_namespace {
269 return "ACC";
270 }
271  
272 sub default_secondary_namespaces {
273 return;
274 }
275  
276 sub _read_record {
277 my $self = shift;
278 my ($filepath,$offset,$length) = @_;
279 my $fh = $self->_fhcache($filepath)
280 or $self->throw("couldn't open $filepath: $!");
281 seek($fh,$offset,0) or $self->throw("can't seek on $filepath: $!");
282 my $record;
283 read($fh,$record,$length) or $self->throw("can't read $filepath: $!");
284 $record
285 }
286  
287 # return a list in the form ($filepath,$offset,$length)
288 sub _lookup_primary {
289 my $self = shift;
290 my $primary = shift;
291 my $db = $self->primary_db
292 or $self->throw("no primary namespace database is open");
293  
294 my $record = $db->{$primary} or return; # nothing here
295  
296 my($fileid,$offset,$length) = $self->unpack_primary($record);
297 my $filepath = $self->_fileno2path($fileid)
298 or $self->throw("no file path entry for fileid $fileid");
299 return ($filepath,$offset,$length);
300 }
301  
302 # return a list of array refs in the form [$filepath,$offset,$length]
303 sub _lookup_secondary {
304 my $self = shift;
305 my ($namespace,$secondary) = @_;
306 my @primary = $self->expand_ids($namespace=>$secondary);
307 return map {[$self->_lookup_primary($_)]} @primary;
308 }
309  
310 # store indexing information into a primary & secondary record
311 # $namespaces is one of:
312 # 1. a scalar corresponding to the primary name
313 # 2. a hashref corresponding to namespace=>id identifiers
314 # it is valid for secondary id to be an arrayref
315 sub _store_index {
316 my $self = shift;
317 my ($keys,$filepath,$offset,$length) = @_;
318 my ($primary,%secondary);
319  
320 if (ref $keys eq 'HASH') {
321 my %valid_secondary = map {$_=>1} $self->secondary_namespaces;
322 while (my($ns,$value) = each %$keys) {
323 if ($ns eq $self->primary_namespace) {
324 $primary = $value;
325 } else {
326 $valid_secondary{$ns} or $self->throw("invalid secondary namespace $ns");
327 push @{$secondary{$ns}},$value;
328 }
329 }
330 $primary or $self->throw("no primary namespace ID provided");
331 } else {
332 $primary = $keys;
333 }
334  
335 $self->throw("invalid primary ID; must be a scalar")
336 if ref($primary) =~ /^(ARRAY|HASH)$/; # but allow stringified objects
337  
338 $self->_store_primary($primary,$filepath,$offset,$length);
339 for my $ns (keys %secondary) {
340 my @ids = ref $secondary{$ns} ? @{$secondary{$ns}} : $secondary{$ns};
341 $self->_store_secondary($ns,$_,$primary) foreach @ids;
342 }
343  
344 1;
345 }
346  
347 # store primary index
348 sub _store_primary {
349 my $self = shift;
350 my ($id,$filepath,$offset,$length) = @_;
351  
352 my $db = $self->primary_db
353 or $self->throw("no primary namespace database is open");
354 my $fileno = $self->_path2fileno($filepath);
355 defined $fileno or $self->throw("could not create a file number for $filepath");
356  
357 my $record = $self->pack_primary($fileno,$offset,$length);
358 $db->{$id} = $record or return; # nothing here
359 1;
360 }
361  
362 # store a primary index name under a secondary index
363 sub _store_secondary {
364 my $self = shift;
365 my ($secondary_ns,$secondary_id,$primary_id) = @_;
366  
367 my $db = $self->secondary_db($secondary_ns)
368 or $self->throw("invalid secondary namespace $secondary_ns");
369  
370 # first get whatever secondary ids are already stored there
371 my @primary = $self->unpack_secondary($db->{$secondary_id});
372 # uniqueify
373 my %unique = map {$_=>undef} @primary,$primary_id;
374  
375 my $record = $self->pack_secondary(keys %unique);
376 $db->{$secondary_id} = $record;
377 }
378  
379 # get output file handle
380 sub _outfh {
381 my $self = shift;
382 #### XXXXX FINISH #####
383 # my $
384 }
385  
386 # unpack a primary record into fileid,offset,length
387 sub unpack_primary {
388 my $self = shift;
389 my $index_record = shift;
390 return split "\t",$index_record;
391 }
392  
393 # unpack a secondary record into a list of primary ids
394 sub unpack_secondary {
395 my $self = shift;
396 my $index_record = shift or return;
397 return split "\t",$index_record;
398 }
399  
400 # pack a list of fileid,offset,length into a primary id record
401 sub pack_primary {
402 my $self = shift;
403 my ($fileid,$offset,$length) = @_;
404 return join "\t",($fileid,$offset,$length);
405 }
406  
407 # pack a list of primary ids into a secondary id record
408 sub pack_secondary {
409 my $self = shift;
410 my @secondaries = @_;
411 return join "\t",@secondaries;
412 }
413  
414 sub primary_db {
415 my $self = shift;
416 # lazy opening
417 $self->_open_bdb unless exists $self->{bdb_primary_db};
418 return $self->{bdb_primary_db};
419 }
420  
421 sub secondary_db {
422 my $self = shift;
423 my $secondary_namespace = shift
424 or $self->throw("usage: secondary_db(\$secondary_namespace)");
425 $self->_open_bdb unless exists $self->{bdb_primary_db};
426 return $self->{bdb_secondary_db}{$secondary_namespace};
427 }
428  
429 sub _open_bdb {
430 my $self = shift;
431  
432 my $flags = $self->write_flag ? O_CREAT|O_RDWR : O_RDONLY;
433  
434 my $primary_db = {};
435 tie(%$primary_db,'DB_File',$self->_catfile($self->_primary_db_name),$flags,0666,$DB_BTREE)
436 or $self->throw("Could not open primary index file: $! (did you remember to use -write_flag=>1?)");
437 $self->{bdb_primary_db} = $primary_db;
438  
439 for my $secondary ($self->secondary_namespaces) {
440 my $secondary_db = {};
441 tie(%$secondary_db,'DB_File',$self->_catfile($self->_secondary_db_name($secondary)),$flags,0666,$DB_BTREE)
442 or $self->throw("Could not open primary index file");
443 $self->{bdb_secondary_db}{$secondary} = $secondary_db;
444 }
445  
446 1;
447 }
448  
449 sub _primary_db_name {
450 my $self = shift;
451 my $pns = $self->primary_namespace or $self->throw('no primary namespace defined');
452 return "key_$pns";
453 }
454  
455 sub _secondary_db_name {
456 my $self = shift;
457 my $sns = shift;
458 return "id_$sns";
459 }
460  
461 sub _fhcache {
462 my $self = shift;
463 my $path = shift;
464 my $write = shift;
465  
466 if (!$self->{bdb_fhcache}{$path}) {
467 $self->{bdb_curopen} ||= 0;
468 if ($self->{bdb_curopen} >= $self->{bdb_maxopen}) {
469 my @lru = sort {$self->{bdb_cacheseq}{$a} <=> $self->{bdb_cacheseq}{$b};} keys %{$self->{bdb_fhcache}};
470 splice(@lru, $self->{bdb_maxopen} / 3);
471 $self->{bdb_curopen} -= @lru;
472 for (@lru) { delete $self->{bdb_fhcache}{$_} }
473 }
474 if ($write) {
475 my $modifier = $self->{bdb_fhcache_seenit}{$path}++ ? '>' : '>>';
476 $self->{bdb_fhcache}{$path} = IO::File->new("${modifier}${path}") or return;
477 } else {
478 $self->{bdb_fhcache}{$path} = IO::File->new($path) or return;
479 }
480 $self->{bdb_curopen}++;
481 }
482 $self->{bdb_cacheseq}{$path}++;
483 $self->{bdb_fhcache}{$path}
484 }
485  
486 1;
« Bio/Factory/ObjectFactoryI.pm ↑ Summary Bio/OntologyIO/InterProParser.pm »