| 1 | package Bio::Phylo::Parsers::Dwca; | ||
| 2 | use strict; | ||
| 3 | use warnings; | ||
| 4 | use base 'Bio::Phylo::Parsers::Abstract'; | ||
| 5 | use File::Temp qw[tempfile]; | ||
| 6 | use Bio::Phylo::Factory; | ||
| 7 | use Bio::Phylo::Util::Logger qw[:levels]; | ||
| 8 | use Bio::Phylo::Util::Exceptions qw[throw]; | ||
| 9 | use Bio::Phylo::Util::Dependency qw[XML::Twig]; | ||
| 10 | use Bio::Phylo::Util::Dependency qw[Archive::Zip]; | ||
| 11 | use Bio::Phylo::Util::CONSTANT qw[/looks_like/ :namespaces :objecttypes]; | ||
| 12 | |||
| 13 | # because we use the dependency management module we don't import | ||
| 14 | # the status codes and constants, hence we call them explicitly here | ||
| 15 | my $AZ_OK = Archive::Zip::AZ_OK(); | ||
| 16 | my $AZ_STREAM_END = Archive::Zip::AZ_STREAM_END(); | ||
| 17 | my $COMPRESSION_STORED = Archive::Zip::COMPRESSION_STORED(); | ||
| 18 | |||
| 19 | =head1 NAME | ||
| 20 | |||
| 21 | Bio::Phylo::Parsers::Dwca - Parser used by Bio::Phylo::IO, no serviceable parts inside | ||
| 22 | |||
| 23 | =head1 DESCRIPTION | ||
| 24 | |||
| 25 | This module parses standard Darwin Core Archive files as produced by GBIF | ||
| 26 | (see: L<http://www.gbif.org/resource/80639>). The end result is a L<Bio::Phylo::Taxa> | ||
| 27 | object that has as many taxon objects in it as there are distinct C<dwc:scientificName>s | ||
| 28 | in the archive. For example, if the archive is for a single species there will thus be | ||
| 29 | only one taxon object in the produced result. Each taxon object is annotated with as many | ||
| 30 | C<dwc:Occurrence> records, instantiated as L<Bio::Phylo::NeXML::Meta> annotations, as | ||
| 31 | there are occurrences for that species in the archive. In turn, nested in each of these | ||
| 32 | annotations are the predicates and objects for that record. | ||
| 33 | |||
| 34 | Using this module, the contents of a Darwin Core Archive can be easily accessed and, for | ||
| 35 | example, prepared as input for MAXENT. Here is an example to show how this is done: | ||
| 36 | |||
| 37 | use Bio::Phylo::IO 'parse'; | ||
| 38 | use Bio::Phylo::Util::CONSTANT ':objecttypes'; | ||
| 39 | |||
| 40 | # a set of fossil occurrences of th feral horse, Equus ferus Boddaert, 1785 | ||
| 41 | # this corresponds with data set doi:10.15468/dl.yyyhyn | ||
| 42 | my $url = 'http://api.gbif.org/v1/occurrence/download/request/0074675-160910150852091.zip'; | ||
| 43 | |||
| 44 | # like every Bio::Phylo::IO module, we can parse directly from a web location | ||
| 45 | my $proj = parse( | ||
| 46 | '-format' => 'dwca', | ||
| 47 | '-url' => $url, | ||
| 48 | '-as_project' => 1, | ||
| 49 | ); | ||
| 50 | |||
| 51 | # write a CSV file with MAXENT header | ||
| 52 | print "Species,Latitude,Longitude\n"; | ||
| 53 | for my $t ( @{ $proj->get_items(_TAXON_) } ) { | ||
| 54 | my $name = $t->get_name; | ||
| 55 | for my $m ( @{ $t->get_meta } ) { | ||
| 56 | my $lat = $m->get_meta_object('dwc:decimalLatitude'); | ||
| 57 | my $lon = $m->get_meta_object('dwc:decimalLongitude'); | ||
| 58 | print "\"$name\",$lat,$lon\n"; | ||
| 59 | } | ||
| 60 | } | ||
| 61 | |||
| 62 | =cut | ||
| 63 | |||
| 64 | sub _parse { | ||
| 65 | my $self = shift; | ||
| 66 | my $fh = $self->_handle; | ||
| 67 | my $fac = $self->_factory; | ||
| 68 | my $log = $self->_logger; | ||
| 69 | |||
| 70 | # have to make a seekable, local copy which we clean up afterwards | ||
| 71 | my ( $h, $tempfile ) = tempfile(); | ||
| 72 | print $h $_ while <$fh>; | ||
| 73 | close $h; | ||
| 74 | my ( $core, $zip ) = _process_meta_xml( $tempfile ); | ||
| 75 | |||
| 76 | # start building return value | ||
| 77 | my $taxa = $fac->create_taxa( | ||
| 78 | '-namespaces' => { | ||
| 79 | 'dwc' => _NS_DWC_, | ||
| 80 | 'dcterms' => _NS_DCTERMS_, | ||
| 81 | 'gbif' => _NS_GBIF_, | ||
| 82 | } | ||
| 83 | ); | ||
| 84 | |||
| 85 | # iterate over the file locations | ||
| 86 | for my $file ( map { $_->text } $core->first_child('files')->children('location') ) { | ||
| 87 | |||
| 88 | # process an occurrences file | ||
| 89 | $log->info("going to read file $file"); | ||
| 90 | my @header; | ||
| 91 | my $record = 1; | ||
| 92 | my $fdel = $core->att('fieldsTerminatedBy'); | ||
| 93 | my $ldel = $core->att('linesTerminatedBy'); | ||
| 94 | LINE: for my $line ( split /$ldel/, _read_zip_member( $zip => $file ) ) { | ||
| 95 | my @fields = split /$fdel/, $line; | ||
| 96 | if ( not @header and $core->att('ignoreHeaderLines') == 1 ) { | ||
| 97 | @header = $self->_process_header( \@fields, $core, $taxa ); | ||
| 98 | next LINE; | ||
| 99 | } | ||
| 100 | $log->info("processing record " . $record++); | ||
| 101 | $self->_process_record( \@fields, \@header, $taxa ); | ||
| 102 | } | ||
| 103 | } | ||
| 104 | unlink $tempfile; | ||
| 105 | return $taxa; | ||
| 106 | } | ||
| 107 | |||
| 108 | sub _process_meta_xml { | ||
| 109 | my ( $infh ) = @_; | ||
| 110 | |||
| 111 | # test reading | ||
| 112 | my $zip = Archive::Zip->new; | ||
| 113 | if ( $zip->read($infh) != $AZ_OK ) { | ||
| 114 | throw 'FileError' => "$infh can't be read as ZIP file"; | ||
| 115 | } | ||
| 116 | |||
| 117 | # extract to string, parse in memory, validate type | ||
| 118 | my $xml = XML::Twig->new; | ||
| 119 | $xml->parse( _read_zip_member( $zip => 'meta.xml' ) ); | ||
| 120 | my $core = $xml->root->first_child('core'); | ||
| 121 | if ( $core->att('rowType') ne _NS_DWC_ . 'Occurrence' ) { | ||
| 122 | throw 'FileError' => "$infh does not contain occurrences as core data"; | ||
| 123 | } | ||
| 124 | return $core, $zip; | ||
| 125 | } | ||
| 126 | |||
| 127 | sub _process_record { | ||
| 128 | my ( $self, $fields, $header, $taxa ) = @_; | ||
| 129 | |||
| 130 | # process the line | ||
| 131 | my $occ; | ||
| 132 | FIELD: for my $i ( 0 .. $#{ $fields } ) { | ||
| 133 | next FIELD if $fields->[$i] =~ /^$/; | ||
| 134 | |||
| 135 | # create and populate the container meta object | ||
| 136 | my $pre = $header->[$i]->{'prefix'}; | ||
| 137 | my $ns = $header->[$i]->{'namespace'}; | ||
| 138 | my $p = $pre . ':' . $header->[$i]->{'predicate'}; | ||
| 139 | if ( $occ ) { | ||
| 140 | $occ->add_meta($self->_factory->create_meta('-triple'=>{$p=>$fields->[$i]})); | ||
| 141 | } | ||
| 142 | else { | ||
| 143 | $occ = $self->_factory->create_meta( | ||
| 144 | '-triple' => { | ||
| 145 | 'dwc:Occurrence' => $self->_factory->create_meta( | ||
| 146 | '-triple' => { $p => $fields->[$i] } | ||
| 147 | ) | ||
| 148 | } | ||
| 149 | ); | ||
| 150 | } | ||
| 151 | |||
| 152 | # fetch or create the taxon object | ||
| 153 | if ( $p eq 'dwc:scientificName' ) { | ||
| 154 | my $n = $fields->[$i]; | ||
| 155 | my $t = $taxa->get_by_name($n); | ||
| 156 | if ( not $t ) { | ||
| 157 | $self->_logger->info("creating taxon $n"); | ||
| 158 | $t = $self->_factory->create_taxon( '-name' => $n ); | ||
| 159 | $taxa->insert($t); | ||
| 160 | } | ||
| 161 | $t->add_meta($occ); | ||
| 162 | } | ||
| 163 | } | ||
| 164 | } | ||
| 165 | |||
| 166 | sub _process_header { | ||
| 167 | my ( $self, $fields, $core, $taxa ) = @_; | ||
| 168 | my @header = @$fields; | ||
| 169 | my $nsi = 1; | ||
| 170 | $self->_logger->info("processing ".scalar(@header)." header columns"); | ||
| 171 | |||
| 172 | # process the header fields | ||
| 173 | for my $field ( $core->children('field') ) { | ||
| 174 | |||
| 175 | # split the term in namespace and predicate | ||
| 176 | my $term = $field->att('term'); | ||
| 177 | if ( $term =~ m/^(.+\/)([^\/]+)$/ ) { | ||
| 178 | my ( $namespace, $predicate ) = ( $1, $2 ); | ||
| 179 | |||
| 180 | # generate namespace prefix | ||
| 181 | my $p = $taxa->get_prefix_for_namespace($namespace); | ||
| 182 | if ( not $p ) { | ||
| 183 | $p = 'ns' . $nsi++; | ||
| 184 | $taxa->set_namespaces( $p => $namespace ); | ||
| 185 | $self->_logger->info("created prefix $p for namespace $namespace"); | ||
| 186 | } | ||
| 187 | |||
| 188 | # store namespace, predicate and prefix | ||
| 189 | my $i = $field->att('index'); | ||
| 190 | $header[$i] = { | ||
| 191 | 'namespace' => $namespace, | ||
| 192 | 'predicate' => $predicate, | ||
| 193 | 'prefix' => $p, | ||
| 194 | }; | ||
| 195 | } | ||
| 196 | } | ||
| 197 | return @header; | ||
| 198 | } | ||
| 199 | |||
| 200 | sub _read_zip_member { | ||
| 201 | my ( $zip, $member_name ) = @_; | ||
| 202 | |||
| 203 | # instantiate the named member object | ||
| 204 | my $member = $zip->memberNamed( $member_name ); | ||
| 205 | $member->desiredCompressionMethod( $COMPRESSION_STORED ); | ||
| 206 | |||
| 207 | # rewind to the start of the member | ||
| 208 | my $status = $member->rewindData(); | ||
| 209 | if ( $status != $AZ_OK ) { | ||
| 210 | throw 'FileError' => "Can't rewind $member_name: $status"; | ||
| 211 | } | ||
| 212 | |||
| 213 | # read buffered | ||
| 214 | my $contents; | ||
| 215 | while ( ! $member->readIsDone() ) { | ||
| 216 | my ( $buffer_ref, $status ) = $member->readChunk(); | ||
| 217 | if ( $status != $AZ_OK && $status != $AZ_STREAM_END ) { | ||
| 218 | throw 'FileError' => "Can't read chunk from $member_name: $status"; | ||
| 219 | } | ||
| 220 | $contents .= $$buffer_ref | ||
| 221 | } | ||
| 222 | $member->endRead(); | ||
| 223 | return $contents; | ||
| 224 | } | ||
| 225 | |||
| 226 | =head1 SEE ALSO | ||
| 227 | |||
| 228 | There is a mailing list at L<https://groups.google.com/forum/#!forum/bio-phylo> | ||
| 229 | for any user or developer questions and discussions. | ||
| 230 | |||
| 231 | =over | ||
| 232 | |||
| 233 | =item L<Bio::Phylo::IO> | ||
| 234 | |||
| 235 | The dwca parser is called by the L<Bio::Phylo::IO|Bio::Phylo::IO> object. | ||
| 236 | Look there to learn how to parse data in general | ||
| 237 | |||
| 238 | =item L<Bio::Phylo::Manual> | ||
| 239 | |||
| 240 | Also see the manual: L<Bio::Phylo::Manual> and L<http://rutgervos.blogspot.com> | ||
| 241 | |||
| 242 | =back | ||
| 243 | |||
| 244 | =head1 CITATION | ||
| 245 | |||
| 246 | If you use Bio::Phylo in published research, please cite it: | ||
| 247 | |||
| 248 | B<Rutger A Vos>, B<Jason Caravas>, B<Klaas Hartmann>, B<Mark A Jensen> | ||
| 249 | and B<Chase Miller>, 2011. Bio::Phylo - phyloinformatic analysis using Perl. | ||
| 250 | I<BMC Bioinformatics> B<12>:63. | ||
| 251 | L<http://dx.doi.org/10.1186/1471-2105-12-63> | ||
| 252 | |||
| 253 | =cut | ||
| 254 | |||
| 255 | 1; |