Surveyor/MakeCpan.pm

filter: Statementstmt 81.0% 137 / 169 Branchbran 36.0% 18 / 50 Conditioncond 29.4% 5 / 17 MC/DCmcdc 10.0% 1 / 10 Subroutinesub 100% 19 / 19 Podpod 33.3% 2 / 6 totaltotal 67.1% 182 / 271 CC 40 SCAR 26.7
CC
40
Cov
68%
_readpkgs
26.8
add_release
25.9
close
21.3
CRAP 14.5
« Surveyor/Inquiry.pm ↑ Summary Surveyor/DB_File.pm »
1 package Dist::Surveyor::MakeCpan;
2  
3 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 1 ▶use Carp; # core
Subroutine BEGIN: called CC=1 SCAR=0.0
6 1 ▶use Data::Dumper; # core
Subroutine BEGIN: called CC=1 SCAR=0.0
7 1 ▶use File::Path; # core
Subroutine BEGIN: called CC=1 SCAR=0.0
8 1 ▶use CPAN::DistnameInfo;
Subroutine BEGIN: called CC=1 SCAR=0.0
9 1 ▶use File::Basename qw{dirname}; # core
Subroutine BEGIN: called CC=1 SCAR=0.0
10 1 ▶use HTTP::Tiny;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 1 ▶use Dist::Surveyor::Inquiry;
Subroutine BEGIN: called CC=1 SCAR=0.0
12 1 ▶use List::Util qw(max); # core
Subroutine BEGIN: called CC=1 SCAR=0.0
13  
14 1 our $VERSION = '0.023';
15  
16 1 our $verbose;
17 1 *verbose = \$::VERBOSE;
18  
19 1 ▶sub new {
Subroutine new: called CC=2 SCAR=6.9
Pod undocumented
20 1 my ($class, $cpan_dir, $progname, $irregularities) = @_;
21  
22 1 require Compress::Zlib;
23 1 mkpath("$cpan_dir/modules");
24  
25 # --- write extra data files that may be useful XXX may change
26 # XXX these don't all (yet?) merge with existing data
27 1 my $survey_datadump_dir = "$cpan_dir/$progname";
28 1 mkpath($survey_datadump_dir);
29  
30 # Write list of releases, like default stdout
31 1 open my $rel_fh, ">", "$survey_datadump_dir/releases.txt";
32  
33 # dump the primary result data for additional info and debugging
34 1 ▶ my $gzwrite = Compress::Zlib::gzopen("$survey_datadump_dir/_data_dump.perl.gz", 'wb')
Branch: 1 branch
BranchTrueFalseTotal
my $gzwrite = &Compress::Zlib::gzopen("$survey_datadump_dir/_data_dump.perl.gz", "wb") or croak("Cannot open $survey_datadump_dir/_data_dump.perl.gz for writing: " . $Compress::Zlib::gzerrno) 1 0 1
35 or croak "Cannot open $survey_datadump_dir/_data_dump.perl.gz for writing: " . $Compress::Zlib::gzerrno;
36 1 $gzwrite->gzwrite("[\n");
37  
38  
39 1 my $self = {
40 errors => 0,
41 cpan_dir => $cpan_dir,
42 irregularities => $irregularities,
43 pkg_ver_rel => {}, # for 02packages
44 progname => $progname,
45 rel_fh => $rel_fh,
46 gzwrite => $gzwrite,
47 };
48 1 return bless $self, $class;
49 }
50  
51 1 ▶sub close {
Subroutine close: called CC=13 SCAR=30.6
Pod documented
52 1 my $self = shift;
53  
54 # --- write 02packages file
55  
56 1 my $pkg_lines = _readpkgs($self->{cpan_dir});
57 1 my %packages;
58 1 for my $line (@$pkg_lines, map { $_->{line} } values %{ $self->{pkg_ver_rel} }) {
59 2 my ($pkg) = split(/\s+/, $line, 2);
60 2 ▶ if ($packages{$pkg} and $packages{$pkg} ne $line) {
Branch: 1 branch
BranchTrueFalseTotal
if ($packages{$pkg} and $packages{$pkg} ne $line) 0 2 2
Condition33%1/3
$packages{$pkg} and $packages{$pkg} ne $line
!ll&&!rl&&r
200
MC/DC0%0/2
$packages{$pkg} and $packages{$pkg} ne $line
$packages{$pkg} $packages{$pkg} ne $line
Truth table33%1/3
A and B
A= $packages{$pkg}B= $packages{$pkg} ne $line
ABresult
0X0
100
111
61 0 ▶ warn "Old $packages{$pkg}\nNew $line\n" if $verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $Dist::Surveyor::MakeCpan::verbose 0 0 0
62 }
63 2 $packages{$pkg} = $line;
64 };
65 1 _writepkgs($self->{cpan_dir}, [ sort { lc $a cmp lc $b } values %packages ] );
66  
67  
68  
69 # Write list of token packages - each should match only one release.
70 # This makes it _much_ faster to do installs via cpanm because it
71 # can skip the modules it knows are installed (whereas using a list of
72 # distros it has to reinstall _all_ of them every time).
73 # XXX maybe add as a separate option: "--mainpkgs mainpkgs.lst"
74 1 my %dist_packages;
75 1 ▶ while ( my ($pkg, $line) = each %packages) {
Branch: 1 branch
BranchTrueFalseTotal
if (() = each %packages) 2 1 3
76 2 my $distpath = (split /\s+/, $line)[2];
77 2 $dist_packages{$distpath}{$pkg}++;
78 }
79 1 my %token_package;
80 1 my %token_package_pri = ( # alter install order for some modules
81 'Module::Build' => 100, # should be near first
82 Moose => 50,
83  
84 # install distros that use Module::Install late so their dependencies
85 # have already been resolved (else they try to fetch them directly,
86 # bypassing our cpanm --mirror-only goal)
87 'Olson::Abbreviations' => -90,
88  
89 # distros with special needs
90 'Term::ReadKey' => -100, # tests hang if run in background
91 );
92 1 for my $distpath (sort keys %dist_packages) {
93 1 my $dp = $dist_packages{$distpath};
94 1 my $di = CPAN::DistnameInfo->new($distpath);
95 #warn Dumper([ $distpath, $di->dist, $di]);
96 1 (my $token_pkg = $di->dist) =~ s/-/::/g;
97 1 ▶ if (!$dp->{$token_pkg}) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($dp->{$token_pkg}) 0 1 1
98 0 ▶ if (my $keypkg = $self->{irregularities}->{$di->dist}) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $keypkg = $self->{'irregularities'}{$di->dist}) { } 0 0 0
99 0 $token_pkg = $keypkg;
100 }
101 else {
102 # XXX not good - may pick a dummy test package
103 0 ▶ $token_pkg = (grep { $_ } keys %$dp)[0] || $token_pkg;
Condition0%0/3
(grep {$_;} keys %$dp)[0] || $token_pkg
l!l&&r!l&&!r
000
MC/DC0%0/2
(grep {$_;} keys %$dp)[0] || $token_pkg
(grep {$_;} keys %$dp)[0] $token_pkg
Truth table0%0/3
A || B
A= (grep {$_;} keys %$dp)[0]B= $token_pkg
ABresult
1X1
011
000
104 0 warn "Picked $token_pkg as token package for ".$di->distvname."\n";
105 }
106 }
107 1 ▶ $token_package{$token_pkg} = $token_package_pri{$token_pkg} || 0;
Condition50%1/2
$token_package_pri{$token_pkg} || 0
l!l
01
MC/DC0%0/1
$token_package_pri{$token_pkg} || 0
$token_package_pri{$token_pkg}
Truth table50%1/2
A
A= $token_package_pri{$token_pkg}
Aresult
11
00
108 }
109  
110 0 ▶ my @main_pkgs = sort { $token_package{$b} <=> $token_package{$a} or $a cmp $b } keys %token_package;
Condition0%0/3
$token_package{$b} <=> $token_package{$a} || $a cmp $b
l!l&&r!l&&!r
000
MC/DC0%0/2
$token_package{$b} <=> $token_package{$a} || $a cmp $b
$token_package{$b} <=> $token_package{$a} $a cmp $b
Truth table0%0/3
A || B
A= $token_package{$b} <=> $token_package{$a}B= $a cmp $b
ABresult
1X1
011
000
111 1 open my $key_pkg_fh, ">", join('/', $self->{cpan_dir}, $self->{progname}, "token_packages.txt");
112 1 print $key_pkg_fh "$_\n" for @main_pkgs;
113 1 close $key_pkg_fh;
114  
115 1 close $self->{rel_fh};
116  
117 1 $self->{gzwrite}->gzwrite("]\n");
118 1 $self->{gzwrite}->gzclose;
119  
120 1 warn $self->{cpan_dir}." updated.\n";
121 1 return $self->{errors};
122 }
123  
124 1 ▶sub add_release {
Subroutine add_release: called CC=13 SCAR=32.5
Pod documented
125 1 my ($self, $ri) = @_;
126  
127 # --- get the file
128  
129 1 my $main_url = $ri->{download_url};
130 1 my $di = distname_info_from_url($main_url);
131 1 my $pathfile = "authors/id/".$di->pathname;
132 1 my $destfile = $self->{cpan_dir}."/$pathfile";
133 1 mkpath(dirname($destfile));
134  
135 1 my @urls = ($main_url);
136 1 for my $mirror ('http://backpan.perl.org') {
137 1 push @urls, "$mirror/$pathfile";
138 }
139  
140 1 my $mirror_status;
141 1 my $ua = HTTP::Tiny->new(agent => "dist_surveyor/$VERSION");
142 1 for my $url (@urls) {
143 1 $mirror_status = $ua->mirror($url, $destfile);
144 1 ▶ last if $mirror_status->{success};
Branch: 1 branch
BranchTrueFalseTotal
if $mirror_status->{"success"} 1 0 1
145 }
146 1 ▶ if (!$mirror_status->{success}) {
Branch: 1 branch
BranchTrueFalseTotal
if (not $mirror_status->{'success'}) { } 0 1 1
147 0 ▶ my $err = $mirror_status->{status} == 599 ? $mirror_status->{content} : $mirror_status->{status};
Branch: 1 branch
BranchTrueFalseTotal
$mirror_status->{'status'} == 599 ? : 0 0 0
148 0 my $msg = "Error $err mirroring $main_url";
149 0 ▶ if (-f $destfile) {
Branch: 1 branch
BranchTrueFalseTotal
if (-f $destfile) { } 0 0 0
150 0 warn "$msg - using existing file\n";
151 }
152 else {
153 # better to keep going and add the packages to the index
154 # than abort at this stage due to network/mirror problems
155 # the user can drop the files in later
156 0 warn "$msg - continuing, ADD FILE MANUALLY!\n";
157 0 $self->{errors}++;
158 }
159 }
160 else {
161 1 ▶ warn "$mirror_status->{status} $main_url\n" if $verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $Dist::Surveyor::MakeCpan::verbose 0 1 1
162 }
163  
164  
165 1 my $mods_in_rel = get_module_versions_in_release($ri->{author}, $ri->{name});
166  
167 1 ▶ if (!keys %$mods_in_rel) { # XXX hack for common::sense
Branch: 1 branch
BranchTrueFalseTotal
unless (keys %$mods_in_rel) 0 1 1
168 0 (my $dist_as_pkg = $ri->{distribution}) =~ s/-/::/g;
169 0 warn "$ri->{author}/$ri->{name} has no modules! Adding fake module $dist_as_pkg ".$di->version."\n";
170 0 $mods_in_rel->{$dist_as_pkg} = {
171 name => $dist_as_pkg,
172 version => $di->version,
173 version_obj => version->parse($di->version),
174 };
175 }
176  
177  
178 # --- accumulate package info for 02packages file
179  
180 1 for my $pkg (sort keys %$mods_in_rel ) {
181 # pi => { name=>, version=>, version_obj=> }
182 2 my $pi = $mods_in_rel->{$pkg};
183  
184 # for selecting which dist a package belongs to
185 # XXX should factor in authorization status
186 2 my $p_r_match_score = p_r_match_score($pkg, $ri);
187  
188 2 ▶ if (my $pvr = $self->{pkg_ver_rel}->{$pkg}) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $pvr = $self->{"pkg_ver_rel"}{$pkg}) 0 2 2
189 # already seen same package name in different distribution
190 0 ▶ if ($p_r_match_score < $pvr->{p_r_match_score}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($p_r_match_score < $pvr->{"p_r_match_score"}) 0 0 0
191 0 warn "$pkg seen in $pvr->{ri}{name} so ignoring one in $ri->{name}\n";
192 0 next;
193 }
194 0 warn "$pkg seen in $pvr->{ri}{name} - now overridden by $ri->{name}\n";
195 }
196  
197 2 my $line = _fmtmodule($pkg, $di->pathname, $pi->{version});
198 2 $self->{pkg_ver_rel}->{$pkg} = { line => $line, pi => $pi, ri => $ri, p_r_match_score => $p_r_match_score };
199 }
200  
201 1 ▶ printf { $self->{rel_fh} } "%s\n", ( exists $ri->{url} ? $ri->{url} : "?url" );
Branch: 1 branch
BranchTrueFalseTotal
exists $ri->{'url'} ? : 1 0 1
202  
203 1 $self->{gzwrite}->gzwrite(Dumper($ri));
204 1 $self->{gzwrite}->gzwrite(",");
205  
206 }
207  
208 2 ▶sub p_r_match_score {
Subroutine p_r_match_score: called CC=5 SCAR=17.2
Pod undocumented
209 2 my ($pkg_name, $ri) = @_;
210 2 my @p = split /\W/, $pkg_name;
211 2 my @r = split /\W/, $ri->{name};
212 2 for my $i (0..max(scalar @p, scalar @r)) {
213 4 ▶ return $i if not defined $p[$i]
Branch: 1 branch
BranchTrueFalseTotal
if not defined $p[$i] && defined $r[$i] or $p[$i] ne $r[$i] 2 2 4
Condition50%3/6
not defined $p[$i] && defined $r[$i] or $p[$i] ne $r[$i]
l!l&&r!l&&!r
022
defined $p[$i] && defined $r[$i]
!ll&&!rl&&r
004
MC/DC33%1/3
not defined $p[$i] && defined $r[$i] or $p[$i] ne $r[$i]
defined $p[$i] defined $r[$i] $p[$i] ne $r[$i]
Truth table50%2/4
not(A && B) or C
A= defined $p[$i]B= defined $r[$i]C= $p[$i] ne $r[$i]
ABCresult
0XX1
10X1
1111
1100
214 or not defined $r[$i]
215 or $p[$i] ne $r[$i]
216 }
217 0 die; # unreached
218 }
219  
220 # copied from CPAN::Mini::Inject and hacked
221  
222 1 ▶sub _readpkgs {
Subroutine _readpkgs: called CC=6 SCAR=32.9
223 1 my ($cpandir) = @_;
224  
225 1 my $packages_file = $cpandir.'/modules/02packages.details.txt.gz';
226 1 ▶ return [] if not -f $packages_file;
Branch: 1 branch
BranchTrueFalseTotal
unless -f $packages_file 1 0 1
227  
228 0 ▶ my $gzread = Compress::Zlib::gzopen($packages_file, 'rb')
Branch: 1 branch
BranchTrueFalseTotal
my $gzread = &Compress::Zlib::gzopen($packages_file, "rb") or croak("Cannot open ${packages_file}: " . $Compress::Zlib::gzerrno . "\n") 0 0 0
229 or croak "Cannot open $packages_file: " . $Compress::Zlib::gzerrno . "\n";
230  
231 0 my $inheader = 1;
232 0 my @packages;
233 my $package;
234  
235 0 ▶ while ( $gzread->gzreadline( $package ) ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($gzread->gzreadline($package)) 0 0 0
236 0 ▶ if ( $inheader ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($inheader) 0 0 0
237 0 ▶ $inheader = 0 unless $package =~ /\S/;
Branch: 1 branch
BranchTrueFalseTotal
unless $package =~ /\S/ 0 0 0
238 0 next;
239 }
240 0 chomp $package;
241 0 push @packages, $package;
242 }
243  
244 0 $gzread->gzclose;
245  
246 0 return \@packages;
247 }
248  
249 1 ▶sub _writepkgs {
Subroutine _writepkgs: called CC=3 SCAR=11.0
250 1 my ($cpandir, $pkgs) = @_;
251  
252 1 my $packages_file = $cpandir.'/modules/02packages.details.txt.gz';
253 1 ▶ my $gzwrite = Compress::Zlib::gzopen($packages_file, 'wb')
Branch: 1 branch
BranchTrueFalseTotal
my $gzwrite = &Compress::Zlib::gzopen($packages_file, "wb") or croak("Cannot open $packages_file for writing: " . $Compress::Zlib::gzerrno) 1 0 1
254 or croak "Cannot open $packages_file for writing: " . $Compress::Zlib::gzerrno;
255
256 1 $gzwrite->gzwrite( "File: 02packages.details.txt\n" );
257 1 $gzwrite->gzwrite(
258 "URL: http://www.perl.com/CPAN/modules/02packages.details.txt\n"
259 );
260 1 $gzwrite->gzwrite(
261 'Description: Package names found in directory $CPAN/authors/id/'
262 . "\n" );
263 1 $gzwrite->gzwrite( "Columns: package name, version, path\n" );
264 1 $gzwrite->gzwrite(
265 "Intended-For: Automated fetch routines, namespace documentation.\n"
266 );
267 1 $gzwrite->gzwrite( "Written-By: $0 0.001\n" ); # XXX TODO
268 1 $gzwrite->gzwrite( "Line-Count: " . scalar( @$pkgs ) . "\n" );
269 # Last-Updated: Sat, 19 Mar 2005 19:49:10 GMT
270 1 my @date = split( /\s+/, scalar( gmtime ) );
271 1 $gzwrite->gzwrite( "Last-Updated: $date[0], $date[2] $date[1] $date[4] $date[3] GMT\n\n" );
272
273 1 $gzwrite->gzwrite( "$_\n" ) for ( @$pkgs );
274
275 1 $gzwrite->gzclose;
276 }
277  
278 1 ▶sub distname_info_from_url {
Subroutine distname_info_from_url: called CC=2 SCAR=7.0
Pod undocumented
279 1 my ($url) = @_;
280 1 ▶ $url =~ s{.* \b authors/id/ }{}x
Branch: 1 branch
BranchTrueFalseTotal
$url =~ s[.* \b authors/id/ ][]x or warn "No authors/ in '${url}'\n" 1 0 1
281 or warn "No authors/ in '$url'\n";
282 1 my $di = CPAN::DistnameInfo->new($url);
283 1 return $di;
284 }
285  
286 2 ▶sub _fmtmodule {
Subroutine _fmtmodule: called CC=3 SCAR=11.3
287 2 my ( $module, $file, $version ) = @_;
288 2 ▶ $version = "undef" if not defined $version;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $version 0 2 2
289 2 my $fw = 38 - length $version;
290 2 ▶ $fw = length $module if $fw < length $module;
Branch: 1 branch
BranchTrueFalseTotal
if $fw < length $module 0 2 2
291 2 return sprintf "%-${fw}s %s %s", $module, $version, $file;
292 }
293  
294 1 ▶sub errors {
Subroutine errors: called CC=1 SCAR=0.0
Pod undocumented
295 1 my $self = shift;
296 1 return $self->{errors};
297 }
298  
299 1 1;
300  
301 =head1 NAME
302  
303 Dist::Surveyor::MakeCpan - Create a Mini-CPAN for the surveyed modules
304  
305 =head1 SYNOPSIS
306  
307 use Dist::Surveyor::MakeCpan;
308 my $cpan = Dist::Surveyor::MakeCpan->new(
309 $cpan_dir, $progname, $irregularities);
310 foreach my $rel (@releases) {
311 $cpan->add_release($rel);
312 }
313 $cpan->close();
314 say "There where ", $cpan->errors(), " errors";
315  
316 =head1 DESCRIPTION
317  
318 Create a mini-CPAN for the surveyed modules, so you will be able to re-install
319 the same setup in a new computer.
320  
321 =head1 CONSTRUCTOR
322  
323 my $cpan = Dist::Surveyor::MakeCpan->new(
324 $cpan_dir, $progname, $irregularities, $verbose);
325  
326 =over
327  
328 =item $cpan_dir
329  
330 The directory where the mini-cpan will be created
331  
332 =item $progname
333  
334 The name of the running program - will be used to create a subdirectory
335 inside $cpan_dir, that will contain debug information.
336  
337 =item $irregularities
338  
339 A hashref with a list of irregular named releases. i.e. 'libwww-perl' => 'LWP'.
340  
341 =back
342  
343 =head1 METHODS
344  
345 =head2 $cpan->add_release($rel)
346  
347 Add one release to the mini-cpan. the $rel should be a hashref,
348 and contain the following fields:
349  
350 $rel = {
351 download_url => 'http://cpan.metacpan.org/authors/id/S/SE/SEMUELF/Dist-Surveyor-0.009.tar.gz',
352 url => 'authors/id/S/SE/SEMUELF/Dist-Surveyor-0.009.tar.gz',
353 author => 'SEMUELF',
354 name => 'Dist-Surveyor-0.009',
355 distribution => 'Dist-Surveyor',
356 }
357  
358 =head2 $cpan->close()
359  
360 Close the mini-CPAN, and close all the debug data dump files.
361  
362 =head1 License, Copyright
363  
364 Please see L<Dist::Surveyor> for details
365  
366 =cut
« Surveyor/Inquiry.pm ↑ Summary Surveyor/DB_File.pm »