| 1 | package Bio::Phylo::CIPRES; | ||
| 2 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 3 | 1 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 4 | 1 | ▶ | use URI; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 5 | 1 | ▶ | use Carp; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 6 | 0 | ▶ | use XML::Twig; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 7 | use Data::Dumper; | ||
| 8 | use LWP::UserAgent; | ||
| 9 | use YAML qw(LoadFile); | ||
| 10 | use Bio::Phylo::Util::Logger ':simple'; | ||
| 11 | use Bio::Phylo::Util::Exceptions 'throw'; | ||
| 12 | |||
| 13 | =head1 NAME | ||
| 14 | |||
| 15 | Bio::Phylo::CIPRES - Reusable components for CIPRES REST API access | ||
| 16 | |||
| 17 | =head1 SYNOPSIS | ||
| 18 | |||
| 19 | my $cipres = Bio::Phylo::CIPRES->new( | ||
| 20 | 'infile' => 'infile.fasta', # input data file | ||
| 21 | 'tool' => 'MAFFT_XSEDE', # tool to run | ||
| 22 | 'param' => { 'vparam.runtime_' => 7.5 }, # extra parameters, e.g. max runtime | ||
| 23 | 'outfile' => { 'output.mafft' => 'out.fa' }, # name of output data to fetch | ||
| 24 | 'yml' => 'config.yml', # client credentials | ||
| 25 | ); | ||
| 26 | |||
| 27 | my $url = $cipres->run; | ||
| 28 | |||
| 29 | $cipres->clean_job( $url ); | ||
| 30 | |||
| 31 | =head1 DESCRIPTION | ||
| 32 | |||
| 33 | The CyberInfrastructure for Phylogenetic RESearch (L<CIPRES|http://www.phylo.org>) is a | ||
| 34 | portal that provides access to phylogenetic analyses tools that can be run on the XSEDE | ||
| 35 | HPC infrastructure. The portal has a web browser (point and click) interface, but also | ||
| 36 | a web service interface that can be interacted with using RESTful commands. The basic | ||
| 37 | workflow is as follows: | ||
| 38 | |||
| 39 | =over | ||
| 40 | |||
| 41 | =item * B<Launch a job> | ||
| 42 | This is done by issuing an HTTP POST request that includes: 1) HTTP authentication (i.e. | ||
| 43 | a user name and password that is registered to the realm), 2) uploading input data, 3) | ||
| 44 | configuration options for the job. The result value is an XML document that reports the | ||
| 45 | status. If all goes well, this will report that the job was launched successfully, and it | ||
| 46 | gives a URL to visit to check up on the status. | ||
| 47 | |||
| 48 | =item * B<Check job status> | ||
| 49 | The job URL is visited periodically (at most once every 60 seconds, as per CIPRES policy). | ||
| 50 | This is done using an authenticated GET request where the return value is an XML document | ||
| 51 | that reports whether the job has finished. Once it is finished, the document will include | ||
| 52 | a link to another document that lists the output data, which will be named (e.g. | ||
| 53 | C<output.mafft>) and which will be identifiable by a URL from whence the data can be | ||
| 54 | retrieved. | ||
| 55 | |||
| 56 | =item * B<Get results> | ||
| 57 | Upon completion the results are fetched from their respective URLs. Under simple cases | ||
| 58 | this will be just a single file (e.g. an alignment), but there could be multiple file | ||
| 59 | types, as well as output from STDERR and STDOUT and various job status files that the | ||
| 60 | server generated internally. | ||
| 61 | |||
| 62 | =back | ||
| 63 | |||
| 64 | This module hides the complexity of this interaction, so that entire analyses can be | ||
| 65 | run using only the commands shown in the synopsis section. The general idea is that | ||
| 66 | you can reuse this functionality in other modules and scripts. It is also does the heavy | ||
| 67 | lifting for the L<cipresrun> executable that allows you to run analyses from terminal | ||
| 68 | interfaces. | ||
| 69 | |||
| 70 | =cut | ||
| 71 | |||
| 72 | # global constants | ||
| 73 | our $AUTOLOAD; | ||
| 74 | use version; our $VERSION = qv("v0.2.1"); | ||
| 75 | my $REALM = "Cipres Authentication"; | ||
| 76 | my $PORT = 443; | ||
| 77 | |||
| 78 | =head1 METHODS | ||
| 79 | |||
| 80 | =head2 new() | ||
| 81 | |||
| 82 | The constructor takes the arguments as shown in the SYNOPSIS section. The arguments are | ||
| 83 | a direct translation of the named arguments (not option flags) that are passed on the | ||
| 84 | command line to the L<cipresrun> program. The value of the C<outfile> argument, and that | ||
| 85 | of the C<param> argument, is both a hash reference. | ||
| 86 | |||
| 87 | =cut | ||
| 88 | |||
| 89 | sub new { | ||
| 90 | my $class = shift; | ||
| 91 | my @fields = qw[infile tool param outfile url user pass cipres_appkey]; | ||
| 92 | my $self = bless { map { $_ => undef } @fields }, $class; | ||
| 93 | my %args = @_; | ||
| 94 | while( my ( $property, $value ) = each %args ) { | ||
| 95 | $self->$property($value); | ||
| 96 | } | ||
| 97 | return $self; | ||
| 98 | } | ||
| 99 | |||
| 100 | =head2 run() | ||
| 101 | |||
| 102 | Runs the entire analysis using the configuration as provided to the constructor. Returns | ||
| 103 | key value pairs where each key is an C<outfile> and each file is the data as text. | ||
| 104 | |||
| 105 | =cut | ||
| 106 | |||
| 107 | sub run { | ||
| 108 | my $self = shift; | ||
| 109 | my $url = $self->launch_job; | ||
| 110 | while(1) { | ||
| 111 | sleep(60); | ||
| 112 | my $status = $self->check_status($url); | ||
| 113 | if ( $status->{'completed'} eq 'true' ) { | ||
| 114 | $self->get_results( $status->{'outfiles'} ); | ||
| 115 | return $url; | ||
| 116 | } | ||
| 117 | } | ||
| 118 | } | ||
| 119 | |||
| 120 | =head2 launch_job() | ||
| 121 | |||
| 122 | Is called by C<run()>. Launches the analysis and returns the status URL at which progress | ||
| 123 | can be inspected. | ||
| 124 | |||
| 125 | =cut | ||
| 126 | |||
| 127 | sub launch_job { | ||
| 128 | my $self = shift; | ||
| 129 | my $ua = $self->ua; | ||
| 130 | my $url = $self->url; | ||
| 131 | my $load = $self->payload; | ||
| 132 | my @head = $self->headers(1); | ||
| 133 | my $res = $ua->post( $url . '/job/' . $self->user, $load, @head ); | ||
| 134 | if ( $res->is_success ) { | ||
| 135 | |||
| 136 | # run submission, parse result | ||
| 137 | my $status_url; | ||
| 138 | my $result = $res->decoded_content; | ||
| 139 | DEBUG $result; | ||
| 140 | XML::Twig->new( | ||
| 141 | 'twig_handlers' => { | ||
| 142 | 'jobstatus/selfUri/url' => sub { $status_url = $_->text } | ||
| 143 | } | ||
| 144 | )->parse($result); | ||
| 145 | INFO "Job launched at $status_url"; | ||
| 146 | return $status_url; | ||
| 147 | } | ||
| 148 | else { | ||
| 149 | throw 'NetworkError' => $res->status_line; | ||
| 150 | } | ||
| 151 | } | ||
| 152 | |||
| 153 | =head2 check_status() | ||
| 154 | |||
| 155 | Is called by C<run()>. Consults the status URL. Returns a hash reference whose values | ||
| 156 | specify whether the job is done, and if so, where the results can be fetched. | ||
| 157 | |||
| 158 | =cut | ||
| 159 | |||
| 160 | sub check_status { | ||
| 161 | my ( $self, $url ) = @_; | ||
| 162 | INFO "Going to check status for $url"; | ||
| 163 | my $ua = $self->ua; | ||
| 164 | my @head = $self->headers(0); | ||
| 165 | my $res = $ua->get( $url, @head ); | ||
| 166 | if ( $res->is_success ) { | ||
| 167 | |||
| 168 | # post request, fetch result | ||
| 169 | my ( $status, $outfiles ); | ||
| 170 | my $result = $res->decoded_content; | ||
| 171 | DEBUG $result; | ||
| 172 | XML::Twig->new( | ||
| 173 | 'twig_handlers' => { | ||
| 174 | 'jobstatus/resultsUri/url' => sub { $outfiles = $_->text }, | ||
| 175 | 'jobstatus/terminalStage' => sub { $status = $_->text } | ||
| 176 | } | ||
| 177 | )->parse($result); | ||
| 178 | my $time = localtime(); | ||
| 179 | INFO "[$time] completed: $status"; | ||
| 180 | return { 'completed' => $status, 'outfiles' => $outfiles }; | ||
| 181 | } | ||
| 182 | else { | ||
| 183 | throw 'NetworkError' => $res->status_line; | ||
| 184 | } | ||
| 185 | } | ||
| 186 | |||
| 187 | =head2 get_results() | ||
| 188 | |||
| 189 | Is called by C<run()>. Returns the named result data as a hash. | ||
| 190 | |||
| 191 | =cut | ||
| 192 | |||
| 193 | sub get_results { | ||
| 194 | my ( $self, $url ) = @_; | ||
| 195 | my %out = %{ $self->outfile }; | ||
| 196 | my $ua = $self->ua; | ||
| 197 | my @head = $self->headers(0); | ||
| 198 | my $res = $ua->get( $url, @head ); | ||
| 199 | my %out_url; | ||
| 200 | if ( $res->is_success ) { | ||
| 201 | my $result = $res->decoded_content; | ||
| 202 | DEBUG $result; | ||
| 203 | XML::Twig->new( | ||
| 204 | 'twig_handlers' => { | ||
| 205 | 'results/jobfiles/jobfile' => sub { | ||
| 206 | my $node = $_; | ||
| 207 | my $name = $node->findvalue('filename'); | ||
| 208 | if ( $out{ $name } ) { | ||
| 209 | $out_url{ $name } = $node->findvalue('downloadUri/url'); | ||
| 210 | } | ||
| 211 | DEBUG $node->toString; | ||
| 212 | } | ||
| 213 | } | ||
| 214 | )->parse($result); | ||
| 215 | for my $name ( keys %out ) { | ||
| 216 | my $location = $out_url{ $name }; | ||
| 217 | $res = $ua->get( $location, @head ); | ||
| 218 | if ( $res->is_success ) { | ||
| 219 | open my $fh, '>', $out{ $name } or die $!; | ||
| 220 | print $fh $res->decoded_content; | ||
| 221 | } | ||
| 222 | else { | ||
| 223 | throw 'NetworkError' => $res->status_line; | ||
| 224 | } | ||
| 225 | } | ||
| 226 | } | ||
| 227 | else { | ||
| 228 | throw 'NetworkError' => $res->status_line; | ||
| 229 | } | ||
| 230 | } | ||
| 231 | |||
| 232 | =head2 yml() | ||
| 233 | |||
| 234 | Given the location the config.yml file, populates properties of the object with the | ||
| 235 | right parameter values to authenticate the client with the CIPRES server. | ||
| 236 | |||
| 237 | =cut | ||
| 238 | |||
| 239 | sub yml { | ||
| 240 | my ( $self, $yml ) = @_; | ||
| 241 | INFO "Reading YAML file $yml"; | ||
| 242 | my $info = LoadFile($yml); | ||
| 243 | DEBUG "Parsed " . Dumper( $info ); | ||
| 244 | $self->user( $info->{'CRA_USER'} ); | ||
| 245 | $self->pass( $info->{'PASSWORD'} ); | ||
| 246 | $self->url( $info->{'URL'} ); | ||
| 247 | $self->cipres_appkey( $info->{'KEY'} ); | ||
| 248 | } | ||
| 249 | |||
| 250 | =head2 ua() | ||
| 251 | |||
| 252 | Instantiates an authenticated L<LWP::UserAgent> object. | ||
| 253 | |||
| 254 | =cut | ||
| 255 | |||
| 256 | sub ua { | ||
| 257 | my $self = shift; | ||
| 258 | my $host = URI->new( $self->url )->host(); | ||
| 259 | my $user = $self->user; | ||
| 260 | my $pass = $self->pass; | ||
| 261 | my $ua = LWP::UserAgent->new; | ||
| 262 | DEBUG "Instantiating UserAgent $host:$PORT / $REALM / $user:****"; | ||
| 263 | $ua->ssl_opts( 'verify_hostname' => 0 ); | ||
| 264 | $ua->credentials( | ||
| 265 | $host . ':' . $PORT, | ||
| 266 | $REALM, | ||
| 267 | $user => $pass | ||
| 268 | ); | ||
| 269 | return $ua; | ||
| 270 | } | ||
| 271 | |||
| 272 | =head2 payload() | ||
| 273 | |||
| 274 | Constructs the HTTP POST payload for launching jobs. Returns an array reference of | ||
| 275 | key/value pairs. | ||
| 276 | |||
| 277 | =cut | ||
| 278 | |||
| 279 | sub payload { | ||
| 280 | my $self = shift; | ||
| 281 | DEBUG "Composing payload for ".$self->tool." with infile ".$self->infile; | ||
| 282 | my $load = [ | ||
| 283 | 'tool' => $self->tool, | ||
| 284 | 'input.infile_' => [ $self->infile ], | ||
| 285 | 'metadata.statusEmail' => 'true', | ||
| 286 | %{ $self->param } | ||
| 287 | ]; | ||
| 288 | DEBUG Dumper($load); | ||
| 289 | return $load; | ||
| 290 | } | ||
| 291 | |||
| 292 | =head2 headers() | ||
| 293 | |||
| 294 | Constructs the HTTP headers to identify the client app and, optionally, to tell the | ||
| 295 | server that multipart/form-data is being attached as a payload. | ||
| 296 | |||
| 297 | =cut | ||
| 298 | |||
| 299 | sub headers { | ||
| 300 | my ( $self, $form ) = @_; | ||
| 301 | if ( $form ) { | ||
| 302 | DEBUG "Composing POST / form-data headers"; | ||
| 303 | return ( | ||
| 304 | 'Content_Type' => 'form-data', | ||
| 305 | 'cipres-appkey' => $self->cipres_appkey, | ||
| 306 | ); | ||
| 307 | } | ||
| 308 | else { | ||
| 309 | DEBUG "Composing GET headers"; | ||
| 310 | return ( 'cipres-appkey' => $self->cipres_appkey ); | ||
| 311 | } | ||
| 312 | } | ||
| 313 | |||
| 314 | =head2 clean_job() | ||
| 315 | |||
| 316 | Cleans up the job on the server that is identified by the provided input URL (i.e. the | ||
| 317 | status URL, which is the return value of the run() method). | ||
| 318 | |||
| 319 | =cut | ||
| 320 | |||
| 321 | sub clean_job { | ||
| 322 | my ( $self, $url ) = @_; | ||
| 323 | my $ua = $self->ua; | ||
| 324 | my @head = $self->headers(0); | ||
| 325 | INFO "Going to clean job $url"; | ||
| 326 | $ua->delete( $url, @head ); | ||
| 327 | } | ||
| 328 | |||
| 329 | sub AUTOLOAD { | ||
| 330 | my ( $self, $arg ) = @_; | ||
| 331 | my $property = $AUTOLOAD; | ||
| 332 | $property =~ s/.*://; | ||
| 333 | if ( exists $self->{$property} ) { | ||
| 334 | if ( $arg ) { | ||
| 335 | $self->{$property} = $arg; | ||
| 336 | return $self; | ||
| 337 | } | ||
| 338 | else { | ||
| 339 | return $self->{$property}; | ||
| 340 | } | ||
| 341 | } | ||
| 342 | else { | ||
| 343 | my $template = 'Can\'t locate object method "%s" via package "%s"'; | ||
| 344 | croak sprintf $template, $property, __PACKAGE__; | ||
| 345 | } | ||
| 346 | } | ||
| 347 | |||
| 348 | sub DESTROY { | ||
| 349 | # maybe kill and delete process on server? | ||
| 350 | } | ||
| 351 | |||
| 352 | =head1 SEE ALSO | ||
| 353 | |||
| 354 | Also consult the documentation for L<cipresrun>, which shows the usage of this module | ||
| 355 | from the command line. | ||
| 356 | |||
| 357 | =head1 LICENSE | ||
| 358 | |||
| 359 | MIT License | ||
| 360 | |||
| 361 | Copyright (c) 2020 Naturalis Biodiversity Center | ||
| 362 | |||
| 363 | Permission is hereby granted, free of charge, to any person obtaining a copy | ||
| 364 | of this software and associated documentation files (the "Software"), to deal | ||
| 365 | in the Software without restriction, including without limitation the rights | ||
| 366 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
| 367 | copies of the Software, and to permit persons to whom the Software is | ||
| 368 | furnished to do so, subject to the following conditions: | ||
| 369 | |||
| 370 | The above copyright notice and this permission notice shall be included in all | ||
| 371 | copies or substantial portions of the Software. | ||
| 372 | |||
| 373 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 374 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 375 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
| 376 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
| 377 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
| 378 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
| 379 | SOFTWARE. | ||
| 380 | |||
| 381 | =cut | ||
| 382 | |||
| 383 | 1; | ||