Indexer/Queue.pm

filter: Statementstmt 29.6% 35 / 118 Branchbran 0.0% 0 / 26 Conditioncond 0.0% 0 / 20 MC/DCmcdc 0.0% 0 / 11 Subroutinesub 54.5% 12 / 22 Podpod 60.0% 3 / 5 totaltotal 24.7% 50 / 202 CC 25 SCAR 33.4
CC
25
Cov
21%
get_queue
72.0
_get_file_list
42.0
_copy_file
42.0
CRAP 28.2
« App/BackPAN/Indexer.pm ↑ Summary Indexer/Reporter/AsYAML.pm »
1 package MyCPAN::Indexer::Queue;
2 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
4  
5 1 ▶use parent qw(MyCPAN::Indexer::Component);
Subroutine BEGIN: called CC=1 SCAR=0.0
6 1 ▶use vars qw($VERSION $logger);
Subroutine BEGIN: called CC=1 SCAR=0.0
7 1 $VERSION = '1.282';
8  
9 1 ▶use File::Basename;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 1 ▶use File::Find;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 1 ▶use File::Find::Closures qw( find_by_regex );
Subroutine BEGIN: called CC=1 SCAR=0.0
12 1 ▶use File::Path qw( mkpath );
Subroutine BEGIN: called CC=1 SCAR=0.0
13 1 ▶use File::Spec::Functions qw( catfile rel2abs );
Subroutine BEGIN: called CC=1 SCAR=0.0
14 1 ▶use Log::Log4perl;
Subroutine BEGIN: called CC=1 SCAR=0.0
15  
16 1 ▶BEGIN {
Subroutine BEGIN: called CC=1 SCAR=0.0
17 1 $logger = Log::Log4perl->get_logger( 'Queue' );
18 }
19  
20 =encoding utf8
21  
22 =head1 NAME
23  
24 MyCPAN::Indexer::Queue - Find distributions to index
25  
26 =head1 SYNOPSIS
27  
28 Use this in backpan_indexer.pl by specifying it as the queue class:
29  
30 # in backpan_indexer.config
31 queue_class MyCPAN::Indexer::Queue
32  
33 =head1 DESCRIPTION
34  
35 This class returns a list of Perl distributions for the BackPAN
36 indexer to process.
37  
38 =head2 Methods
39  
40 =over 4
41  
42 =item component_type
43  
44 This is a queue type.
45  
46 =cut
47  
48 0 ▶sub component_type { $_[0]->queue_type }
Subroutine component_type: not called CC=1 SCAR=6.9
Pod documented
49  
50 =item get_queue
51  
52 C<get_queue> sets the key C<queue> in C<$Notes> hash reference. It
53 finds all of the tarballs or zip archives in under the directories
54 named in C<backpan_dir> and C<merge_dirs> in the configuration.
55  
56 It specifically skips files that end in C<.txt.gz> or C<.data.gz>
57 since PAUSE creates those meta files near the actual module
58 installations.
59  
60 If the C<organize_dists> configuration value is true, it also copies
61 any distributions it finds into a PAUSE-like structure using the
62 value of the C<pause_id> configuration to create the path.
63  
64 This queue component tries to skip any distributions that already have
65 a report to make the list of distributions to examine much shorter. It
66 relies on the
67  
68 =cut
69  
70 sub get_queue
71 0 ▶ {
Subroutine get_queue: not called CC=8 SCAR=42.8
Pod documented
72 0 my( $self ) = @_;
73  
74 0 ▶ my @dirs =
Condition0%0/2
$self->get_config->merge_dirs || ''
l!l
00
MC/DC0%0/1
$self->get_config->merge_dirs || ''
$self->get_config->merge_dirs
Truth table0%0/2
A
A= $self->get_config->merge_dirs
Aresult
11
00
75 (
76 $self->get_config->backpan_dir,
77 split /\x00/, $self->get_config->merge_dirs || ''
78 )
79 ;
80  
81 0 foreach my $dir ( @dirs )
82 {
83 0 ▶ $logger->error( "Distribution source directory does not exist: [$dir]" )
Branch: 1 branch
BranchTrueFalseTotal
unless -e $dir 0 0 0
84 unless -e $dir;
85 }
86  
87 0 @dirs = grep { -d $_ } @dirs;
88 0 ▶ $logger->logdie( "No directories to index!" ) unless @dirs;
Branch: 1 branch
BranchTrueFalseTotal
unless @dirs 0 0 0
89  
90 0 my $queue = $self->_get_file_list( @dirs );
91  
92 0 ▶ if( $self->get_config->organize_dists )
Branch: 1 branch
BranchTrueFalseTotal
if ($self->get_config->organize_dists) 0 0 0
93 {
94 0 $self->_setup_organize_dists( $dirs[0] );
95  
96 # I really hate this following line. It's sure to
97 # break on something
98 0 my $regex = catfile( qw( authors id (.) .. .+? ), '' );
99  
100 0 foreach my $i ( 0 .. $#$queue )
101 {
102 0 my $file = $queue->[$i];
103 0 $logger->debug( "Processing $file" );
104  
105 0 ▶ next if $file =~ m|$regex|;
Branch: 1 branch
BranchTrueFalseTotal
if $file =~ /$regex/ 0 0 0
106 0 $logger->debug( "Copying $file into PAUSE structure" );
107  
108 0 $queue->[$i] = $self->_copy_file( $file, $dirs[0] );
109 }
110 }
111  
112 0 $self->set_note( 'queue', $queue );
113  
114 0 1;
115 }
116  
117 sub _get_file_list
118 0 ▶ {
Subroutine _get_file_list: not called CC=6 SCAR=37.4
119 0 my( $self, @dirs ) = @_;
120  
121 0 $logger->debug( "Taking dists from [@dirs]" );
122 0 my( $wanted, $reporter ) =
123 File::Find::Closures::find_by_regex( qr/\.(?:(?:tar\.|t)gz|zip)$/ );
124  
125 0 find( $wanted, @dirs );
126  
127 0 my $dist_count = () = $reporter->();
128 0 $logger->info( "Found $dist_count distributions to possibly index" );
129  
130 my $files_to_examine = [
131 0 grep { ! $self->report_exists_already( $_ ) }
132 0 map { rel2abs($_) }
133 0 ▶ grep { ! /.(data|txt).gz$/ and ! /02packages/ }
Condition0%0/3
/.(data|txt).gz$/ || /02packages/
l!l&&r!l&&!r
000
MC/DC0%0/2
/.(data|txt).gz$/ || /02packages/
/.(data|txt).gz$/ /02packages/
Truth table0%0/3
A || B
A= /.(data|txt).gz$/B= /02packages/
ABresult
1X1
011
000
134 $reporter->()
135 ];
136  
137 {
138 0 my $examine_count = () = @$files_to_examine;
139 0 $logger->info( "Found $examine_count distributions to actually index" );
140 0 ▶ my $success_reports = $self->success_report_count || 0;
Condition0%0/2
$self->success_report_count || 0
l!l
00
MC/DC0%0/1
$self->success_report_count || 0
$self->success_report_count
Truth table0%0/2
A
A= $self->success_report_count
Aresult
11
00
141 0 ▶ my $error_reports = $self->error_report_count || 0;
Condition0%0/2
$self->error_report_count || 0
l!l
00
MC/DC0%0/1
$self->error_report_count || 0
$self->error_report_count
Truth table0%0/2
A
A= $self->error_report_count
Aresult
11
00
142  
143 0 ▶ my $success_percent = sprintf "%d", 100 * eval { $success_reports / $dist_count } || 0;
Condition0%0/2
100 * eval { do { $success_reports / $dist_count } } || 0
l!l
00
MC/DC0%0/1
100 * eval { do { $success_reports / $dist_count } } || 0
100 * eval { do { $success_reports / $dist_count } }
Truth table0%0/2
A
A= 100 * eval { do { $success_reports / $dist_count } }
Aresult
11
00
144 0 ▶ my $error_percent = sprintf "%d", 100 * eval { $error_reports / $dist_count } || 0;
Condition0%0/2
100 * eval { do { $error_reports / $dist_count } } || 0
l!l
00
MC/DC0%0/1
100 * eval { do { $error_reports / $dist_count } } || 0
100 * eval { do { $error_reports / $dist_count } }
Truth table0%0/2
A
A= 100 * eval { do { $error_reports / $dist_count } }
Aresult
11
00
145  
146 0 $logger->info( "Found $success_reports previous success reports ($success_percent%)" );
147 0 $logger->info( "Found $error_reports previous error reports ($error_percent%)" );
148 }
149  
150 0 return $files_to_examine;
151 }
152  
153 =item report_exists_already( DIST )
154  
155 This method goes through this process to decide what to return:
156  
157 =over 4
158  
159 =item Return false if the C<fresh_start> configuration is true
160 (so existing reports don't matter).
161  
162 =item Return true if there is a successful report already.
163  
164 =item Return false if C<retry_errors> is true.
165  
166 =item Return true if there is already an error report.
167  
168 =item Return false as the default case.
169  
170 =back
171  
172 =cut
173  
174 1 ▶BEGIN {
Subroutine BEGIN: called CC=1 SCAR=0.0
175 1 my $success_reports;
176 1 my $error_reports;
177  
178 sub report_exists_already
179 0 ▶ {
Subroutine report_exists_already: not called CC=5 SCAR=34.0
Pod documented
180 0 my( $self, $dist ) = @_;
181  
182 0 ▶ return 0 if $self->get_config->fresh_start;
Branch: 1 branch
BranchTrueFalseTotal
if $self->get_config->fresh_start 0 0 0
183  
184 0 my $reporter = $self->get_coordinator->get_component( 'reporter' );
185  
186 0 my $success_report = $reporter->get_successful_report_path( $dist );
187 0 ▶ do { $success_reports++; return 1 } if -e $success_report;
Branch: 1 branch
BranchTrueFalseTotal
if (-e $success_report) 0 0 0
188  
189 0 ▶ return 0 if $self->get_config->retry_errors;
Branch: 1 branch
BranchTrueFalseTotal
if $self->get_config->retry_errors 0 0 0
190 0 my $error_report = $reporter->get_error_report_path( $dist );
191 0 ▶ do { $error_reports++; return 1 } if -e $error_report;
Branch: 1 branch
BranchTrueFalseTotal
if (-e $error_report) 0 0 0
192  
193 0 return 0;
194 }
195  
196 0 ▶sub success_report_count { $success_reports }
Subroutine success_report_count: not called CC=1 SCAR=6.9
Pod undocumented
197  
198 0 ▶sub error_report_count { $error_reports }
Subroutine error_report_count: not called CC=1 SCAR=6.9
Pod undocumented
199 }
200  
201 sub _setup_organize_dists
202 0 ▶ {
Subroutine _setup_organize_dists: not called CC=3 SCAR=24.8
203 0 my( $self, $base_dir ) = @_;
204  
205 0 ▶ my $pause_id = eval { $self->get_config->pause_id } || 'MYCPAN';
Condition0%0/2
eval { do { $self->get_config->pause_id } } || 'MYCPAN'
l!l
00
MC/DC0%0/1
eval { do { $self->get_config->pause_id } } || 'MYCPAN'
eval { do { $self->get_config->pause_id } }
Truth table0%0/2
A
A= eval { do { $self->get_config->pause_id } }
Aresult
11
00
206  
207 0 eval { mkpath
208 catfile( $base_dir, $self->_path_parts( $pause_id ) ),
209 { mode => 0775 }
210 };
211 0 ▶ $logger->error( "Could not create PAUSE author path for [$pause_id]: $@" )
Branch: 1 branch
BranchTrueFalseTotal
if $@ 0 0 0
212 if $@;
213  
214 0 1;
215 }
216  
217 sub _path_parts
218 0 ▶ {
Subroutine _path_parts: not called CC=1 SCAR=6.9
219 0 catfile (
220 qw(authors id),
221 substr( $_[1], 0, 1 ),
222 substr( $_[1], 0, 2 ),
223 $_[1]
224 );
225 }
226  
227 # if there is an error with the rename, return the original file name
228 sub _copy_file
229 0 ▶ {
Subroutine _copy_file: not called CC=6 SCAR=37.4
230 0 require File::Copy;
231  
232 0 my( $self, $file, $base_dir ) = @_;
233  
234 0 ▶ my $pause_id = eval { $self->get_config->pause_id } || 'MYCPAN';
Condition0%0/2
eval { do { $self->get_config->pause_id } } || 'MYCPAN'
l!l
00
MC/DC0%0/1
eval { do { $self->get_config->pause_id } } || 'MYCPAN'
eval { do { $self->get_config->pause_id } }
Truth table0%0/2
A
A= eval { do { $self->get_config->pause_id } }
Aresult
11
00
235  
236 0 my $basename = basename( $file );
237 0 $logger->debug( "Need to copy file $basename into $pause_id" );
238  
239 0 my $new_name = rel2abs(
240 catfile( $base_dir, $self->_path_parts( $pause_id ), $basename )
241 );
242  
243 0 ▶ if( -e $new_name and
Branch: 1 branch
BranchTrueFalseTotal
if (-e $new_name and $self->_file_md5($new_name) eq $self->_file_md5($file)) 0 0 0
Condition0%0/3
-e $new_name and $self->_file_md5($new_name) eq $self->_file_md5($file)
!ll&&!rl&&r
000
MC/DC0%0/2
-e $new_name and $self->_file_md5($new_name) eq $self->_file_md5($file)
-e $new_name $self->_file_md5($new_name) eq $self->_file_md5($file)
Truth table0%0/3
A and B
A= -e $new_nameB= $self->_file_md5($new_name) eq $self->_file_md5($file)
ABresult
0X0
100
111
244 $self->_file_md5( $new_name ) eq $self->_file_md5( $file ) )
245 {
246 0 $logger->debug( "Files [$file] and [$new_name] are the same. Not copying" );
247 }
248  
249 0 my $rc = File::Copy::copy( $file => $new_name );
250 0 ▶ $logger->error( "Could not copy [$file] to [$new_name]: $!" )
Branch: 1 branch
BranchTrueFalseTotal
unless $rc 0 0 0
251 unless $rc;
252  
253 0 ▶ return $rc ? $new_name : $file;
Branch: 1 branch
BranchTrueFalseTotal
$rc ? : 0 0 0
254 }
255  
256 sub _file_md5
257 0 ▶ {
Subroutine _file_md5: not called CC=2 SCAR=17.9
258 0 my( $self, $file ) = @_;
259  
260 0 require Digest::MD5;
261  
262 0 ▶ open my( $fh ), '<', $file or return '';
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, "<", $file or return "" 0 0 0
263  
264 0 my $ctx = Digest::MD5->new;
265 0 $ctx->addfile($fh);
266 0 $ctx->hexdigest;
267 }
268  
269 1 1;
270  
271 =back
272  
273 =head1 SEE ALSO
274  
275 MyCPAN::Indexer, MyCPAN::Indexer::Tutorial
276  
277 =head1 SOURCE AVAILABILITY
278  
279 This code is in Github:
280  
281 git://github.com/briandfoy/mycpan-indexer.git
282  
283 =head1 AUTHOR
284  
285 brian d foy, C<< <bdfoy@cpan.org> >>
286  
287 =head1 COPYRIGHT AND LICENSE
288  
289 Copyright © 2008-2018, brian d foy <bdfoy@cpan.org>. All rights reserved.
290  
291 You may redistribute this under the terms of the Artistic License 2.0.
292  
293 =cut
« App/BackPAN/Indexer.pm ↑ Summary Indexer/Reporter/AsYAML.pm »