Phylo/Parsers/Dwca.pm untested

filter: Statementstmt 0.0% 0 / 119 Branchbran 0.0% 0 / 22 Conditioncond 0.0% 0 / 6 MC/DCmcdc n/a Subroutinesub 0.0% 0 / 14 Podpod 0.0% 0 / 5 totaltotal 0.0% 0 / 166 CC 12 SCAR 24.4
CC
12
Cov
0%
_process_record
30.0
_process_meta_xml
12.0
_process_header
12.0
CRAP 11.5
1 package Bio::Phylo::Parsers::Dwca;
2 use strict;
3 use base 'Bio::Phylo::Parsers::Abstract';
4 use File::Temp qw[tempfile];
5 use Bio::Phylo::Factory;
6 use Bio::Phylo::Util::Logger qw[:levels];
7 use Bio::Phylo::Util::Exceptions qw[throw];
8 use Bio::Phylo::Util::Dependency qw[XML::Twig];
9 use Bio::Phylo::Util::Dependency qw[Archive::Zip];
10 use Bio::Phylo::Util::CONSTANT qw[/looks_like/ :namespaces :objecttypes];
11  
12 # because we use the dependency management module we don't import
13 # the status codes and constants, hence we call them explicitly here
14 my $AZ_OK = Archive::Zip::AZ_OK();
15 my $AZ_STREAM_END = Archive::Zip::AZ_STREAM_END();
16 my $COMPRESSION_STORED = Archive::Zip::COMPRESSION_STORED();
17  
18 =head1 NAME
19  
20 Bio::Phylo::Parsers::Dwca - Parser used by Bio::Phylo::IO, no serviceable parts inside
21  
22 =head1 DESCRIPTION
23  
24 This module parses standard Darwin Core Archive files as produced by GBIF
25 (see: L<http://www.gbif.org/resource/80639>). The end result is a L<Bio::Phylo::Taxa>
26 object that has as many taxon objects in it as there are distinct C<dwc:scientificName>s
27 in the archive. For example, if the archive is for a single species there will thus be
28 only one taxon object in the produced result. Each taxon object is annotated with as many
29 C<dwc:Occurrence> records, instantiated as L<Bio::Phylo::NeXML::Meta> annotations, as
30 there are occurrences for that species in the archive. In turn, nested in each of these
31 annotations are the predicates and objects for that record.
32  
33 Using this module, the contents of a Darwin Core Archive can be easily accessed and, for
34 example, prepared as input for MAXENT. Here is an example to show how this is done:
35  
36 use Bio::Phylo::IO 'parse';
37 use Bio::Phylo::Util::CONSTANT ':objecttypes';
38  
39 # a set of fossil occurrences of th feral horse, Equus ferus Boddaert, 1785
40 # this corresponds with data set doi:10.15468/dl.yyyhyn
41 my $url = 'http://api.gbif.org/v1/occurrence/download/request/0074675-160910150852091.zip';
42  
43 # like every Bio::Phylo::IO module, we can parse directly from a web location
44 my $proj = parse(
45 '-format' => 'dwca',
46 '-url' => $url,
47 '-as_project' => 1,
48 );
49  
50 # write a CSV file with MAXENT header
51 print "Species,Latitude,Longitude\n";
52 for my $t ( @{ $proj->get_items(_TAXON_) } ) {
53 my $name = $t->get_name;
54 for my $m ( @{ $t->get_meta } ) {
55 my $lat = $m->get_meta_object('dwc:decimalLatitude');
56 my $lon = $m->get_meta_object('dwc:decimalLongitude');
57 print "\"$name\",$lat,$lon\n";
58 }
59 }
60  
61 =cut
62  
63 sub _parse {
64 my $self = shift;
65 my $fh = $self->_handle;
66 my $fac = $self->_factory;
67 my $log = $self->_logger;
68
69 # have to make a seekable, local copy which we clean up afterwards
70 my ( $h, $tempfile ) = tempfile();
71 print $h $_ while <$fh>;
72 close $h;
73 my ( $core, $zip ) = _process_meta_xml( $tempfile );
74
75 # start building return value
76 my $taxa = $fac->create_taxa(
77 '-namespaces' => {
78 'dwc' => _NS_DWC_,
79 'dcterms' => _NS_DCTERMS_,
80 'gbif' => _NS_GBIF_,
81 }
82 );
83
84 # iterate over the file locations
85 for my $file ( map { $_->text } $core->first_child('files')->children('location') ) {
86
87 # process an occurrences file
88 $log->info("going to read file $file");
89 my @header;
90 my $record = 1;
91 my $fdel = $core->att('fieldsTerminatedBy');
92 my $ldel = $core->att('linesTerminatedBy');
93 LINE: for my $line ( split /$ldel/, _read_zip_member( $zip => $file ) ) {
94 my @fields = split /$fdel/, $line;
95 if ( not @header and $core->att('ignoreHeaderLines') == 1 ) {
96 @header = $self->_process_header( \@fields, $core, $taxa );
97 next LINE;
98 }
99 $log->info("processing record " . $record++);
100 $self->_process_record( \@fields, \@header, $taxa );
101 }
102 }
103 unlink $tempfile;
104 return $taxa;
105 }
106  
107 sub _process_meta_xml {
108 my ( $infh ) = @_;
109  
110 # test reading
111 my $zip = Archive::Zip->new;
112 if ( $zip->read($infh) != $AZ_OK ) {
113 throw 'FileError' => "$infh can't be read as ZIP file";
114 }
115
116 # extract to string, parse in memory, validate type
117 my $xml = XML::Twig->new;
118 $xml->parse( _read_zip_member( $zip => 'meta.xml' ) );
119 my $core = $xml->root->first_child('core');
120 if ( $core->att('rowType') ne _NS_DWC_ . 'Occurrence' ) {
121 throw 'FileError' => "$infh does not contain occurrences as core data";
122 }
123 return $core, $zip;
124 }
125  
126 sub _process_record {
127 my ( $self, $fields, $header, $taxa ) = @_;
128  
129 # process the line
130 my $occ;
131 FIELD: for my $i ( 0 .. $#{ $fields } ) {
132 next FIELD if $fields->[$i] =~ /^$/;
133
134 # create and populate the container meta object
135 my $pre = $header->[$i]->{'prefix'};
136 my $ns = $header->[$i]->{'namespace'};
137 my $p = $pre . ':' . $header->[$i]->{'predicate'};
138 if ( $occ ) {
139 $occ->add_meta($self->_factory->create_meta('-triple'=>{$p=>$fields->[$i]}));
140 }
141 else {
142 $occ = $self->_factory->create_meta(
143 '-triple' => {
144 'dwc:Occurrence' => $self->_factory->create_meta(
145 '-triple' => { $p => $fields->[$i] }
146 )
147 }
148 );
149 }
150
151 # fetch or create the taxon object
152 if ( $p eq 'dwc:scientificName' ) {
153 my $n = $fields->[$i];
154 my $t = $taxa->get_by_name($n);
155 if ( not $t ) {
156 $self->_logger->info("creating taxon $n");
157 $t = $self->_factory->create_taxon( '-name' => $n );
158 $taxa->insert($t);
159 }
160 $t->add_meta($occ);
161 }
162 }
163 }
164  
165 sub _process_header {
166 my ( $self, $fields, $core, $taxa ) = @_;
167 my @header = @$fields;
168 my $nsi = 1;
169 $self->_logger->info("processing ".scalar(@header)." header columns");
170
171 # process the header fields
172 for my $field ( $core->children('field') ) {
173
174 # split the term in namespace and predicate
175 my $term = $field->att('term');
176 if ( $term =~ m/^(.+\/)([^\/]+)$/ ) {
177 my ( $namespace, $predicate ) = ( $1, $2 );
178
179 # generate namespace prefix
180 my $p = $taxa->get_prefix_for_namespace($namespace);
181 if ( not $p ) {
182 $p = 'ns' . $nsi++;
183 $taxa->set_namespaces( $p => $namespace );
184 $self->_logger->info("created prefix $p for namespace $namespace");
185 }
186
187 # store namespace, predicate and prefix
188 my $i = $field->att('index');
189 $header[$i] = {
190 'namespace' => $namespace,
191 'predicate' => $predicate,
192 'prefix' => $p,
193 };
194 }
195 }
196 return @header;
197 }
198  
199 sub _read_zip_member {
200 my ( $zip, $member_name ) = @_;
201
202 # instantiate the named member object
203 my $member = $zip->memberNamed( $member_name );
204 $member->desiredCompressionMethod( $COMPRESSION_STORED );
205
206 # rewind to the start of the member
207 my $status = $member->rewindData();
208 if ( $status != $AZ_OK ) {
209 throw 'FileError' => "Can't rewind $member_name: $status";
210 }
211
212 # read buffered
213 my $contents;
214 while ( ! $member->readIsDone() ) {
215 my ( $buffer_ref, $status ) = $member->readChunk();
216 if ( $status != $AZ_OK && $status != $AZ_STREAM_END ) {
217 throw 'FileError' => "Can't read chunk from $member_name: $status";
218 }
219 $contents .= $$buffer_ref
220 }
221 $member->endRead();
222 return $contents;
223 }
224  
225 =head1 SEE ALSO
226  
227 There is a mailing list at L<https://groups.google.com/forum/#!forum/bio-phylo>
228 for any user or developer questions and discussions.
229  
230 =over
231  
232 =item L<Bio::Phylo::IO>
233  
234 The dwca parser is called by the L<Bio::Phylo::IO|Bio::Phylo::IO> object.
235 Look there to learn how to parse data in general
236  
237 =item L<Bio::Phylo::Manual>
238  
239 Also see the manual: L<Bio::Phylo::Manual> and L<http://rutgervos.blogspot.com>
240  
241 =back
242  
243 =head1 CITATION
244  
245 If you use Bio::Phylo in published research, please cite it:
246  
247 B<Rutger A Vos>, B<Jason Caravas>, B<Klaas Hartmann>, B<Mark A Jensen>
248 and B<Chase Miller>, 2011. Bio::Phylo - phyloinformatic analysis using Perl.
249 I<BMC Bioinformatics> B<12>:63.
250 L<http://dx.doi.org/10.1186/1471-2105-12-63>
251  
252 =cut
253  
254 1;