blib/lib/Parallel/Downloader.pm

filter: Statementstmt 93.7% 105 / 112 Branchbran 62.5% 10 / 16 Conditioncond 50.0% 1 / 2 MC/DCmcdc 0.0% 0 / 1 Subroutinesub 96.3% 26 / 27 Podpod 100% 2 / 2 totaltotal 90.0% 144 / 160 CC 14 SCAR 9.9
CC
14
Cov
89%
_requests_interleaved_by_host
7.0
_sanitize_worker_max
4.5
run
3.0
CRAP 2.7
↑ Summary
1 2 ▶use strictures;
Subroutine BEGIN: called CC=1 SCAR=0.0
2  
3 package Parallel::Downloader;
4  
5 2 our $VERSION = '0.132071'; # VERSION
6  
7 # ABSTRACT: simply download multiple files at once
8  
9 #
10 # This file is part of Parallel-Downloader
11 #
12 #
13 # Christian Walde has dedicated the work to the Commons by waiving all of his
14 # or her rights to the work worldwide under copyright law and all related or
15 # neighboring legal rights he or she had in the work, to the extent allowable by
16 # law.
17 #
18 # Works under CC0 do not require attribution. When citing the work, you should
19 # not imply endorsement by the author.
20 #
21  
22  
23 2 ▶use Moo;
Subroutine BEGIN: called CC=1 SCAR=0.0
24 2 ▶use MooX::Types::MooseLike::Base qw( Bool Int HashRef CodeRef ArrayRef );
Subroutine BEGIN: called CC=1 SCAR=0.0
25  
26 2 ▶sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
27 2 has requests => ( is => 'ro', isa => ArrayRef, required => 1 );
28 2 ▶ has workers => ( is => 'ro', isa => Int, default => sub { 10 } );
Subroutine __ANON__: called CC=1 SCAR=0.0
29 2 ▶ has conns_per_host => ( is => 'ro', isa => Int, default => sub { 4 } );
Subroutine __ANON__: called CC=1 SCAR=0.0
30 2 ▶ has aehttp_args => ( is => 'ro', isa => HashRef, default => sub { {} } );
Subroutine __ANON__: called CC=1 SCAR=0.0
31 2 ▶ has debug => ( is => 'ro', isa => Bool, default => sub { 0 } );
Subroutine __ANON__: called CC=1 SCAR=0.0
32 2 ▶ has logger => ( is => 'ro', isa => CodeRef, default => sub { \&_default_log } );
Subroutine __ANON__: called CC=1 SCAR=0.0
33 2 ▶ has build_response => ( is => 'ro', isa => CodeRef, default => sub { \&_default_build_response } );
Subroutine __ANON__: called CC=1 SCAR=0.0
34 2 ▶ has sorted => ( is => 'ro', isa => Bool, default => sub { 1 } );
Subroutine __ANON__: called CC=1 SCAR=0.0
35  
36 2 has _consumables => ( is => 'lazy', isa => ArrayRef, builder => '_requests_interleaved_by_host' );
37  
38 2 ▶ has _responses => ( is => 'ro', isa => ArrayRef, default => sub { [] } );
Subroutine __ANON__: called CC=1 SCAR=0.0
39 2 ▶ has _cv => ( is => 'ro', isa => sub { $_[0]->isa( 'AnyEvent::CondVar' ) }, default => sub { AnyEvent->condvar } );
Subroutine __ANON__: called CC=1 SCAR=0.0
Subroutine __ANON__: called CC=1 SCAR=0.0
40 }
41 2 ->();
42  
43 2 ▶use AnyEvent::HTTP;
Subroutine BEGIN: called CC=1 SCAR=0.0
44 2 ▶use Sub::Exporter::Simple 'async_download';
Subroutine BEGIN: called CC=1 SCAR=0.0
45  
46  
47 1 ▶sub async_download {
Subroutine async_download: called CC=1 SCAR=0.0
Pod documented
48 1 return __PACKAGE__->new( @_ )->run;
49 }
50  
51 1 ▶sub _requests_interleaved_by_host {
Subroutine _requests_interleaved_by_host: called CC=7 SCAR=19.5
52 1 my ( $self, $requests ) = @_;
53  
54 1 my %hosts;
55 1 for ( @{ $self->requests } ) {
56 3 my $host_name = $_->uri->host;
57 3 ▶ my $host = $hosts{$host_name} ||= [];
Condition50%1/2
$hosts{$host_name} ||= []
l!l
03
MC/DC0%0/1
$hosts{$host_name} ||= []
$hosts{$host_name}
Truth table50%1/2
A
A= $hosts{$host_name}
Aresult
11
00
58 3 push @{$host}, $_;
59 }
60  
61 1 my @interleaved_list;
62 1 ▶ while ( keys %hosts ) {
Branch: 1 branch
BranchTrueFalseTotal
if (keys %hosts) 1 1 2
63 1 push @interleaved_list, shift @{$_} for values %hosts;
64 1 for ( keys %hosts ) {
65 3 ▶ next if @{ $hosts{$_} };
Branch: 1 branch
BranchTrueFalseTotal
if @{$hosts{$_};} 0 3 3
66 3 delete $hosts{$_};
67 }
68 }
69  
70 1 return \@interleaved_list;
71 }
72  
73  
74 1 ▶sub run {
Subroutine run: called CC=3 SCAR=11.0
Pod documented
75 1 my ( $self ) = @_;
76  
77 1 local $AnyEvent::HTTP::MAX_PER_HOST = $self->conns_per_host;
78  
79 1 for ( 1 .. $self->_sanitize_worker_max ) {
80 3 $self->_cv->begin;
81 3 $self->_log( msg => "$_ started", type => "WorkerStart", worker_id => $_ );
82 3 $self->_add_request( $_ );
83 }
84  
85 1 $self->_cv->recv;
86  
87 0 ▶ return @{ $self->_responses } if !$self->sorted;
Branch: 1 branch
BranchTrueFalseTotal
unless $self->sorted 0 1 1
88  
89 1 my %unsorted = map { 0 + $_->[2] => $_ } @{ $self->_responses };
90 1 my @sorted = map { $unsorted{ 0 + $_ } } @{ $self->requests };
91  
92 1 return @sorted;
93 }
94  
95 6 ▶sub _add_request {
Subroutine _add_request: called CC=2 SCAR=6.9
96 6 my ( $self, $worker_id ) = @_;
97  
98 6 my $req = shift @{ $self->_consumables };
99 6 ▶ return $self->_end_worker( $worker_id ) if !$req;
Branch: 1 branch
BranchTrueFalseTotal
unless $req 3 3 6
100  
101 3 my $post_download_sub = $self->_make_post_download_sub( $worker_id, $req );
102  
103 http_request(
104 $req->method,
105 $req->uri->as_string,
106 body => $req->content,
107 headers => $req->{_headers},
108 3 %{ $self->aehttp_args },
109 $post_download_sub
110 );
111  
112 3 my $host_name = $req->uri->host;
113 3 $self->_log(
114 msg => "$worker_id accepted new request for $host_name",
115 type => "WorkerRequestAdd",
116 worker_id => $worker_id,
117 req => $req
118 );
119  
120 3 return;
121 }
122  
123 3 ▶sub _make_post_download_sub {
Subroutine _make_post_download_sub: called CC=1 SCAR=0.0
124 3 my ( $self, $worker_id, $req ) = @_;
125  
126 3 ▶ my $post_download_sub = sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
127 3 push @{ $self->_responses }, $self->build_response->( @_, $req );
128  
129 3 my $host_name = $req->uri->host;
130 3 $self->_log(
131 msg => "$worker_id completed a request for $host_name",
132 type => "WorkerRequestEnd",
133 worker_id => $worker_id,
134 req => $req
135 );
136  
137 3 $self->_add_request( $worker_id );
138 3 return;
139 3 };
140  
141 3 return $post_download_sub;
142 }
143  
144 3 ▶sub _default_build_response {
Subroutine _default_build_response: called CC=1 SCAR=0.0
145 3 my ( $body, $hdr, $req ) = @_;
146 3 return [ $body, $hdr, $req ];
147 }
148  
149 3 ▶sub _end_worker {
Subroutine _end_worker: called CC=1 SCAR=0.0
150 3 my ( $self, $worker_id ) = @_;
151 3 $self->_log( msg => "$worker_id ended", type => "WorkerEnd", worker_id => $worker_id );
152 3 $self->_cv->end;
153 3 return;
154 }
155  
156 1 ▶sub _sanitize_worker_max {
Subroutine _sanitize_worker_max: called CC=4 SCAR=15.0
157 1 my ( $self ) = @_;
158  
159 1 ▶ die "max should be 0 or more" if $self->workers < 0;
Branch: 1 branch
BranchTrueFalseTotal
if $self->workers < 0 0 1 1
160  
161 1 my $request_count = @{ $self->requests };
162  
163 1 ▶ return $request_count if !$self->workers; # 0 = as many parallel as possible
Branch: 1 branch
BranchTrueFalseTotal
unless $self->workers 0 1 1
164 1 ▶ return $request_count if $self->workers > $request_count; # not more than the request count
Branch: 1 branch
BranchTrueFalseTotal
if $self->workers > $request_count 1 0 1
165  
166 0 return $self->workers;
167 }
168  
169 12 ▶sub _log {
Subroutine _log: called CC=2 SCAR=9.2
170 12 my ( $self, %msg ) = @_;
171 12 ▶ return if !$self->debug;
Branch: 1 branch
BranchTrueFalseTotal
unless $self->debug 12 0 12
172 0 $self->logger->( $self, \%msg );
173 0 return;
174 }
175  
176 0 ▶sub _default_log {
Subroutine _default_log: not called CC=1 SCAR=6.9
177 0 my ( $self, $msg ) = @_;
178 0 print "$msg->{msg}\n";
179 0 return;
180 }
181  
182 2 1;
183  
184 __END__
↑ Summary