blib/lib/Mojolicious/Command/bulkget.pm

filter: Statementstmt 98.1% 52 / 53 Branchbran 54.5% 12 / 22 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 11 / 11 Podpod 100% 1 / 1 totaltotal 87.3% 76 / 87 CC 12 SCAR 12.8
CC
12
Cov
85%
run
6.2
_handle_result
5.5
_makepromise
3.0
CRAP 3.6
↑ Summary
1 package Mojolicious::Command::bulkget;
2 1 ▶use Mojo::Base 'Mojolicious::Command';
Subroutine BEGIN: called CC=1 SCAR=0.0
3 1 ▶use Mojo::UserAgent;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 1 ▶use Mojo::Promise;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 1 ▶use Mojo::File 'path';
Subroutine BEGIN: called CC=1 SCAR=0.0
6 1 ▶use Mojo::Util qw(getopt);
Subroutine BEGIN: called CC=1 SCAR=0.0
7  
8 1 our $VERSION = '0.03';
9  
10 1 my $MAXREQ = 20;
11  
12 1 has description => 'Perform bulk get requests';
13 1 ▶has usage => sub { shift->extract_usage . "\n" };
Subroutine __ANON__: called CC=1 SCAR=0.0
14  
15 2 ▶sub run {
Subroutine run: called CC=6 SCAR=18.3
Pod documented
16 2 my ($self, @args) = @_;
17 2 getopt \@args,
18 'v|verbose' => \my $verbose;
19 2 my ($urlbase, $outdir, $suffixesfile) = @args;
20 2 ▶ die $self->usage . "No URL" if !$urlbase;
Branch: 1 branch
BranchTrueFalseTotal
unless $urlbase 1 1 2
21 1 ▶ die $self->usage . "$outdir: $!" if ! -d $outdir;
Branch: 1 branch
BranchTrueFalseTotal
unless -d $outdir 0 1 1
22 1 ▶ die $self->usage . "$suffixesfile: $!" if ! -f $suffixesfile;
Branch: 1 branch
BranchTrueFalseTotal
unless -f $suffixesfile 0 1 1
23 1 my $ua = Mojo::UserAgent->new;
24 # Detect proxy for absolute URLs
25 1 ▶ $urlbase !~ m!^/! ? $ua->proxy->detect : $ua->server->app($self->app);
Branch: 1 branch
BranchTrueFalseTotal
!($urlbase =~ m[^/]u) ? : 0 1 1
26 1 my $outpath = path($outdir);
27 1 my @suffixes = _getsuffixes($suffixesfile, $outpath);
28 1 my @promises = map _makepromise($urlbase, $ua, \@suffixes, $outpath, $verbose), (1..$MAXREQ);
29 1 ▶ Mojo::Promise->all(@promises)->wait if @promises;
Branch: 1 branch
BranchTrueFalseTotal
if @promises 1 0 1
30 }
31  
32 23 ▶sub _makepromise {
Subroutine _makepromise: called CC=3 SCAR=11.0
33 23 my ($urlbase, $ua, $suffixes, $outpath, $verbose) = @_;
34 23 my $s = shift @$suffixes;
35 23 ▶ return if !defined $s;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $s 20 3 23
36 3 my $url = $urlbase . $s;
37 3 ▶ warn "getting $url\n" if $verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $verbose 0 3 3
38 3 ▶ $ua->get_p($url)->then(sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
39 3 my ($tx) = @_;
40 3 _handle_result($outpath, $tx, $s, $verbose);
41 3 _makepromise($urlbase, $ua, $suffixes, $outpath, $verbose);
42 3 });
43 }
44  
45 3 ▶sub _handle_result {
Subroutine _handle_result: called CC=4 SCAR=17.1
46 3 my ($outpath, $tx, $s, $verbose) = @_;
47 3 ▶ if ($tx->res->is_success) {
Branch: 1 branch
BranchTrueFalseTotal
if ($tx->res->is_success) { } 3 0 3
48 3 ▶ warn "got $s\n" if $verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $verbose 0 3 3
49 3 $outpath->child($s)->spurt($tx->res->body);
50 } else {
51 0 ▶ warn "error $s\n" if $verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $verbose 0 0 0
52 }
53 }
54  
55 1 ▶sub _getsuffixes {
Subroutine _getsuffixes: called CC=2 SCAR=7.0
56 1 my ($suffixesfile, $outpath) = @_;
57 1 ▶ open my $fh, '<', $suffixesfile or die $!;
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, "<", $suffixesfile or die $! 1 0 1
58 1 grep { !-f $outpath->child($_); } map { chomp; $_ } <$fh>;
59 }
60  
61 1 1;
62  
63 =encoding utf8
64  
65 =head1 NAME
66  
67 Mojolicious::Command::bulkget - Perform bulk get requests
68  
69 =begin markdown
70  
71 # PROJECT STATUS
72  
73 | OS | Build status |
74 |:-------:|--------------:|
75 | Linux | [![Build Status](https://travis-ci.org/mohawk2/Mojolicious-Command-bulkget.svg?branch=master)](https://travis-ci.org/mohawk2/Mojolicious-Command-bulkget) |
76  
77 [![CPAN version](https://badge.fury.io/pl/Mojolicious-Command-bulkget.svg)](https://metacpan.org/pod/Mojolicious::Command::bulkget)
78  
79 =end markdown
80  
81 =head1 SYNOPSIS
82  
83 Usage: APPLICATION bulkget urlbase outdir suffixesfile
84  
85 # suffixes contains lines with 1, 2, 3
86 # fetches /pets/1, /pets/2, ...
87 # stores results in outputdir/1, outputdir/2, ...
88 mojo bulkget http://example.com/pets/ outputdir suffixes
89  
90 Options:
91 -v, --verbose Print progress information
92  
93 =head1 DESCRIPTION
94  
95 L<Mojolicious::Command::bulkget> is a command line interface for
96 bulk-fetching URLs.
97  
98 Each line of the "suffixes" file is a suffix. It gets appended to the URL
99 "base", then a non-blocking request is made. Only 20 requests will be
100 active at the same time. When ready, the result is stored in the output
101 directory with the suffix as the filename.
102  
103 This command uses the relatively new Mojolicious feature, Promises. The
104 code may be considered worth examining for lessons on what to do, and/or
105 what not to do.
106  
107 =head1 ATTRIBUTES
108  
109 =head2 description
110  
111 $str = $self->description;
112  
113 =head2 usage
114  
115 $str = $self->usage;
116  
117 =head1 METHODS
118  
119 =head2 run
120  
121 $get->run(@ARGV);
122  
123 Run this command.
124  
125 =head1 AUTHOR
126  
127 Ed J
128  
129 Based heavily on L<Mojolicious::Command::openapi>.
130  
131 =head1 COPYRIGHT AND LICENSE
132  
133 This is free software; you can redistribute it and/or modify it under
134 the same terms as the Perl 5 programming language system itself.
135  
136 =cut
↑ Summary