Phylo/Parsers/Dwca.pm untested

filter: Statementstmt 0.0% 0 / 122 Branchbran 0.0% 0 / 22 Conditioncond 0.0% 0 / 6 MC/DCmcdc 0.0% 0 / 4 Subroutinesub 0.0% 0 / 15 Podpod 0.0% 0 / 5 totaltotal 0.0% 0 / 174 CC 12 SCAR 24.1
CC
12
Cov
0%
_process_record
30.0
_process_meta_xml
12.0
_process_header
12.0
CRAP 11.2
1 package Bio::Phylo::Parsers::Dwca;
2 use strict;
3 use warnings;
4 use base 'Bio::Phylo::Parsers::Abstract';
5 use File::Temp qw[tempfile];
6 use Bio::Phylo::Factory;
7 use Bio::Phylo::Util::Logger qw[:levels];
8 use Bio::Phylo::Util::Exceptions qw[throw];
9 use Bio::Phylo::Util::Dependency qw[XML::Twig];
10 use Bio::Phylo::Util::Dependency qw[Archive::Zip];
11 use Bio::Phylo::Util::CONSTANT qw[/looks_like/ :namespaces :objecttypes];
12  
13 # because we use the dependency management module we don't import
14 # the status codes and constants, hence we call them explicitly here
15 my $AZ_OK = Archive::Zip::AZ_OK();
16 my $AZ_STREAM_END = Archive::Zip::AZ_STREAM_END();
17 my $COMPRESSION_STORED = Archive::Zip::COMPRESSION_STORED();
18  
19 =head1 NAME
20  
21 Bio::Phylo::Parsers::Dwca - Parser used by Bio::Phylo::IO, no serviceable parts inside
22  
23 =head1 DESCRIPTION
24  
25 This module parses standard Darwin Core Archive files as produced by GBIF
26 (see: L<http://www.gbif.org/resource/80639>). The end result is a L<Bio::Phylo::Taxa>
27 object that has as many taxon objects in it as there are distinct C<dwc:scientificName>s
28 in the archive. For example, if the archive is for a single species there will thus be
29 only one taxon object in the produced result. Each taxon object is annotated with as many
30 C<dwc:Occurrence> records, instantiated as L<Bio::Phylo::NeXML::Meta> annotations, as
31 there are occurrences for that species in the archive. In turn, nested in each of these
32 annotations are the predicates and objects for that record.
33  
34 Using this module, the contents of a Darwin Core Archive can be easily accessed and, for
35 example, prepared as input for MAXENT. Here is an example to show how this is done:
36  
37 use Bio::Phylo::IO 'parse';
38 use Bio::Phylo::Util::CONSTANT ':objecttypes';
39  
40 # a set of fossil occurrences of th feral horse, Equus ferus Boddaert, 1785
41 # this corresponds with data set doi:10.15468/dl.yyyhyn
42 my $url = 'http://api.gbif.org/v1/occurrence/download/request/0074675-160910150852091.zip';
43  
44 # like every Bio::Phylo::IO module, we can parse directly from a web location
45 my $proj = parse(
46 '-format' => 'dwca',
47 '-url' => $url,
48 '-as_project' => 1,
49 );
50  
51 # write a CSV file with MAXENT header
52 print "Species,Latitude,Longitude\n";
53 for my $t ( @{ $proj->get_items(_TAXON_) } ) {
54 my $name = $t->get_name;
55 for my $m ( @{ $t->get_meta } ) {
56 my $lat = $m->get_meta_object('dwc:decimalLatitude');
57 my $lon = $m->get_meta_object('dwc:decimalLongitude');
58 print "\"$name\",$lat,$lon\n";
59 }
60 }
61  
62 =cut
63  
64 sub _parse {
65 my $self = shift;
66 my $fh = $self->_handle;
67 my $fac = $self->_factory;
68 my $log = $self->_logger;
69
70 # have to make a seekable, local copy which we clean up afterwards
71 my ( $h, $tempfile ) = tempfile();
72 print $h $_ while <$fh>;
73 close $h;
74 my ( $core, $zip ) = _process_meta_xml( $tempfile );
75
76 # start building return value
77 my $taxa = $fac->create_taxa(
78 '-namespaces' => {
79 'dwc' => _NS_DWC_,
80 'dcterms' => _NS_DCTERMS_,
81 'gbif' => _NS_GBIF_,
82 }
83 );
84
85 # iterate over the file locations
86 for my $file ( map { $_->text } $core->first_child('files')->children('location') ) {
87
88 # process an occurrences file
89 $log->info("going to read file $file");
90 my @header;
91 my $record = 1;
92 my $fdel = $core->att('fieldsTerminatedBy');
93 my $ldel = $core->att('linesTerminatedBy');
94 LINE: for my $line ( split /$ldel/, _read_zip_member( $zip => $file ) ) {
95 my @fields = split /$fdel/, $line;
96 if ( not @header and $core->att('ignoreHeaderLines') == 1 ) {
97 @header = $self->_process_header( \@fields, $core, $taxa );
98 next LINE;
99 }
100 $log->info("processing record " . $record++);
101 $self->_process_record( \@fields, \@header, $taxa );
102 }
103 }
104 unlink $tempfile;
105 return $taxa;
106 }
107  
108 sub _process_meta_xml {
109 my ( $infh ) = @_;
110  
111 # test reading
112 my $zip = Archive::Zip->new;
113 if ( $zip->read($infh) != $AZ_OK ) {
114 throw 'FileError' => "$infh can't be read as ZIP file";
115 }
116
117 # extract to string, parse in memory, validate type
118 my $xml = XML::Twig->new;
119 $xml->parse( _read_zip_member( $zip => 'meta.xml' ) );
120 my $core = $xml->root->first_child('core');
121 if ( $core->att('rowType') ne _NS_DWC_ . 'Occurrence' ) {
122 throw 'FileError' => "$infh does not contain occurrences as core data";
123 }
124 return $core, $zip;
125 }
126  
127 sub _process_record {
128 my ( $self, $fields, $header, $taxa ) = @_;
129  
130 # process the line
131 my $occ;
132 FIELD: for my $i ( 0 .. $#{ $fields } ) {
133 next FIELD if $fields->[$i] =~ /^$/;
134
135 # create and populate the container meta object
136 my $pre = $header->[$i]->{'prefix'};
137 my $ns = $header->[$i]->{'namespace'};
138 my $p = $pre . ':' . $header->[$i]->{'predicate'};
139 if ( $occ ) {
140 $occ->add_meta($self->_factory->create_meta('-triple'=>{$p=>$fields->[$i]}));
141 }
142 else {
143 $occ = $self->_factory->create_meta(
144 '-triple' => {
145 'dwc:Occurrence' => $self->_factory->create_meta(
146 '-triple' => { $p => $fields->[$i] }
147 )
148 }
149 );
150 }
151
152 # fetch or create the taxon object
153 if ( $p eq 'dwc:scientificName' ) {
154 my $n = $fields->[$i];
155 my $t = $taxa->get_by_name($n);
156 if ( not $t ) {
157 $self->_logger->info("creating taxon $n");
158 $t = $self->_factory->create_taxon( '-name' => $n );
159 $taxa->insert($t);
160 }
161 $t->add_meta($occ);
162 }
163 }
164 }
165  
166 sub _process_header {
167 my ( $self, $fields, $core, $taxa ) = @_;
168 my @header = @$fields;
169 my $nsi = 1;
170 $self->_logger->info("processing ".scalar(@header)." header columns");
171
172 # process the header fields
173 for my $field ( $core->children('field') ) {
174
175 # split the term in namespace and predicate
176 my $term = $field->att('term');
177 if ( $term =~ m/^(.+\/)([^\/]+)$/ ) {
178 my ( $namespace, $predicate ) = ( $1, $2 );
179
180 # generate namespace prefix
181 my $p = $taxa->get_prefix_for_namespace($namespace);
182 if ( not $p ) {
183 $p = 'ns' . $nsi++;
184 $taxa->set_namespaces( $p => $namespace );
185 $self->_logger->info("created prefix $p for namespace $namespace");
186 }
187
188 # store namespace, predicate and prefix
189 my $i = $field->att('index');
190 $header[$i] = {
191 'namespace' => $namespace,
192 'predicate' => $predicate,
193 'prefix' => $p,
194 };
195 }
196 }
197 return @header;
198 }
199  
200 sub _read_zip_member {
201 my ( $zip, $member_name ) = @_;
202
203 # instantiate the named member object
204 my $member = $zip->memberNamed( $member_name );
205 $member->desiredCompressionMethod( $COMPRESSION_STORED );
206
207 # rewind to the start of the member
208 my $status = $member->rewindData();
209 if ( $status != $AZ_OK ) {
210 throw 'FileError' => "Can't rewind $member_name: $status";
211 }
212
213 # read buffered
214 my $contents;
215 while ( ! $member->readIsDone() ) {
216 my ( $buffer_ref, $status ) = $member->readChunk();
217 if ( $status != $AZ_OK && $status != $AZ_STREAM_END ) {
218 throw 'FileError' => "Can't read chunk from $member_name: $status";
219 }
220 $contents .= $$buffer_ref
221 }
222 $member->endRead();
223 return $contents;
224 }
225  
226 =head1 SEE ALSO
227  
228 There is a mailing list at L<https://groups.google.com/forum/#!forum/bio-phylo>
229 for any user or developer questions and discussions.
230  
231 =over
232  
233 =item L<Bio::Phylo::IO>
234  
235 The dwca parser is called by the L<Bio::Phylo::IO|Bio::Phylo::IO> object.
236 Look there to learn how to parse data in general
237  
238 =item L<Bio::Phylo::Manual>
239  
240 Also see the manual: L<Bio::Phylo::Manual> and L<http://rutgervos.blogspot.com>
241  
242 =back
243  
244 =head1 CITATION
245  
246 If you use Bio::Phylo in published research, please cite it:
247  
248 B<Rutger A Vos>, B<Jason Caravas>, B<Klaas Hartmann>, B<Mark A Jensen>
249 and B<Chase Miller>, 2011. Bio::Phylo - phyloinformatic analysis using Perl.
250 I<BMC Bioinformatics> B<12>:63.
251 L<http://dx.doi.org/10.1186/1471-2105-12-63>
252  
253 =cut
254  
255 1;