blib/lib/CPAN/Digger.pm

filter: Statementstmt 94.4% 34 / 36 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 12 / 12 Podpod n/a totaltotal 95.8% 46 / 48 CC 1 SCAR 0.2
CC
1
Cov
94%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« blib/lib/CPAN/Digger/CLI.pm ↑ Summary
1 package CPAN::Digger;
2 5 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 5 ▶use warnings FATAL => 'all';
Subroutine BEGIN: called CC=1 SCAR=0.0
4  
5 our $VERSION = '1.04';
6  
7 5 ▶use Capture::Tiny qw(capture);
Subroutine BEGIN: called CC=1 SCAR=0.0
8 5 ▶use Cwd qw(getcwd);
Subroutine BEGIN: called CC=1 SCAR=0.0
9 5 ▶use Data::Dumper qw(Dumper);
Subroutine BEGIN: called CC=1 SCAR=0.0
10 5 ▶use Exporter qw(import);
Subroutine BEGIN: called CC=1 SCAR=0.0
11 5 ▶use File::Spec ();
Subroutine BEGIN: called CC=1 SCAR=0.0
12 5 ▶use File::Temp qw(tempdir);
Subroutine BEGIN: called CC=1 SCAR=0.0
13 5 ▶use Log::Log4perl ();
Subroutine BEGIN: called CC=1 SCAR=0.0
14 5 ▶use LWP::UserAgent ();
Subroutine BEGIN: called CC=1 SCAR=0.0
15 5 ▶use MetaCPAN::Client ();
Subroutine BEGIN: called CC=1 SCAR=0.0
16 0 ▶use DateTime ();
Subroutine BEGIN: called CC=1 SCAR=2.6
17 use Template ();
18  
19 my @ci_names = qw(travis github_actions circleci appveyor azure_pipeline gitlab_pipeline);
20  
21  
22 use CPAN::Digger::DB qw(get_fields);
23  
24 my $tempdir = tempdir( CLEANUP => ($ENV{KEEP_TEMPDIR} ? 0 : 1) );
25  
26 my %known_licenses = map {$_ => 1} qw(apache_2_0 artistic_2 bsd gpl_2 gpl_3 lgpl_2_1 lgpl_3_0 perl_5); # open_source, unknown
27  
28 sub new {
29 my ($class, %args) = @_;
30 my $self = bless {}, $class;
31 for my $key (keys %args) {
32 $self->{$key} = $args{$key};
33 }
34 $self->{log} = uc $self->{log};
35 $self->{check_vcs} = delete $self->{vcs};
36 $self->{total} = 0;
37  
38 my $dt = DateTime->now;
39 $self->{end_date} = $dt->ymd;
40 if ($self->{days}) {
41 $self->{start_date} = $dt->add( days => -$self->{days} )->ymd;
42 }
43  
44 $self->{db} = CPAN::Digger::DB->new(db => $self->{db});
45  
46 return $self;
47 }
48  
49 sub read_dashboards {
50 my ($self) = @_;
51 my @pathes = ('dashboard', '/home/gabor/cpan/dashboard');
52 for my $path (@pathes) {
53 if (-e $path) {
54 $self->{dashboards} = { map { m{.*/([^/]+)\.json$}; $1 => 1 } glob "$path/authors/*.json" };
55 }
56 }
57 }
58  
59 sub get_vcs {
60 my ($repository) = @_;
61 if ($repository) {
62 # $html .= sprintf qq{<a href="%s">%s %s</a><br>\n}, $repository->{$k}, $k, $repository->{$k};
63 # Try to get the web link
64 my $url = $repository->{web};
65 if (not $url) {
66 $url = $repository->{url};
67 $url =~ s{^git://}{https://};
68 $url =~ s{\.git$}{};
69 }
70 my $name = "repository";
71 if ($url =~ m{^https?://github.com/}) {
72 $name = 'GitHub';
73 }
74 if ($url =~ m{^https?://gitlab.com/}) {
75 $name = 'GitLab';
76 }
77 return $url, $name;
78 }
79 }
80  
81 sub get_data {
82 my ($self, $item) = @_;
83  
84 my $logger = Log::Log4perl->get_logger();
85 my %data = (
86 distribution => $item->distribution,
87 version => $item->version,
88 author => $item->author,
89 date => $item->date,
90 );
91 #die Dumper $item;
92  
93 $logger->debug('dist: ', $item->distribution);
94 $logger->debug(' ', $item->author);
95 my @licenses = @{ $item->license };
96 $data{licenses} = join ' ', @licenses;
97 $logger->debug(' ', $data{licenses});
98 for my $license (@licenses) {
99 if ($license eq 'unknown') {
100 $logger->error("Unknown license '$license'");
101 } elsif (not exists $known_licenses{$license}) {
102 $logger->warn("Unknown license '$license'. Probably CPAN::Digger needs to be updated");
103 }
104 }
105 # if there are not licenses =>
106 # if there is a license called "unknonws"
107 # check against a known list of licenses (grow it later, or look it up somewhere?)
108 my %resources = %{ $item->resources };
109 #say ' ', join ' ', keys %resources;
110 if ($resources{repository}) {
111 my ($vcs_url, $vcs_name) = get_vcs($resources{repository});
112 if ($vcs_url) {
113 $data{vcs_url} = $vcs_url;
114 $data{vcs_name} = $vcs_name;
115 $logger->debug(" $vcs_name: $vcs_url");
116 }
117 } else {
118 $logger->error('No repository for ', $item->distribution);
119 }
120 return %data;
121 }
122  
123  
124 sub analyze_vcs {
125 my ($data) = @_;
126 my $logger = Log::Log4perl->get_logger();
127  
128 my $vcs_url = $data->{vcs_url};
129 my $repo_name = (split '\/', $vcs_url)[-1];
130 $logger->info("Analyze GitHub repo '$vcs_url' in directory $repo_name");
131  
132 my $ua = LWP::UserAgent->new(timeout => 5);
133 my $response = $ua->get($vcs_url);
134 my $status_line = $response->status_line;
135 if ($status_line eq '404 Not Found') {
136 $logger->error("Repository '$vcs_url' Received 404 Not Found. Please update the link in the META file");
137 return;
138 }
139 if ($response->code != 200) {
140 $logger->error("Repository '$vcs_url' got a response of '$status_line'. Please report this to the maintainer of CPAN::Digger.");
141 return;
142 }
143 if ($response->redirects) {
144 $logger->error("Repository '$vcs_url' is being redirected. Please update the link in the META file");
145 }
146  
147 my $git = 'git';
148  
149 my @cmd = ($git, "clone", "--depth", "1", $data->{vcs_url});
150 my $cwd = getcwd();
151 chdir($tempdir);
152 my ($out, $err, $exit_code) = capture {
153 system(@cmd);
154 };
155 chdir($cwd);
156 my $repo = "$tempdir/$repo_name";
157 $logger->debug("REPO path '$repo'");
158  
159 if ($exit_code != 0) {
160 # TODO capture stderr and include in the log
161 $logger->error("Failed to clone $vcs_url");
162 return;
163 }
164  
165 if ($data->{vcs_name} eq 'GitHub') {
166 analyze_github($data, $repo);
167 }
168 if ($data->{vcs_name} eq 'GitLab') {
169 analyze_gitlab($data, $repo);
170 }
171  
172 for my $ci (@ci_names) {
173 $logger->debug("Is CI '$ci'?");
174 if ($data->{$ci}) {
175 $logger->debug("CI '$ci' found!");
176 $data->{has_ci} = 1;
177 }
178 }
179 }
180  
181 sub analyze_gitlab {
182 my ($data, $repo) = @_;
183  
184 $data->{gitlab_pipeline} = -e "$repo/.gitlab-ci.yml";
185 }
186  
187 sub analyze_github {
188 my ($data, $repo) = @_;
189  
190 $data->{travis} = -e "$repo/.travis.yml";
191 my @ga = glob("$repo/.github/workflows/*");
192 $data->{github_actions} = (scalar(@ga) ? 1 : 0);
193 $data->{circleci} = -e "$repo/.circleci";
194 $data->{appveyor} = (-e "$repo/.appveyor.yml") || (-e "$repo/appveyor.yml");
195 $data->{azure_pipeline} = -e "$repo/azure-pipelines.yml";
196 }
197  
198 sub html {
199 my ($self) = @_;
200  
201 return if not $self->{html};
202 if (not -d $self->{html}) {
203 mkdir $self->{html};
204 }
205  
206 $self->read_dashboards;
207  
208 my @distros = @{ $self->{db}->db_get_every_distro() };
209 my %stats = (
210 total => scalar @distros,
211 has_vcs => 0,
212 vcs => {},
213 has_ci => 0,
214 ci => {},
215 );
216 for my $ci (@ci_names) {
217 $stats{ci}{$ci} = 0;
218 }
219 for my $dist (@distros) {
220 $dist->{dashboard} = $self->{dashboards}{ $dist->{author} };
221 if ($dist->{vcs_name}) {
222 $stats{has_vcs}++;
223 $stats{vcs}{ $dist->{vcs_name} }++;
224 }
225 if ($dist->{has_ci}) {
226 $stats{has_ci}++;
227 for my $ci (@ci_names) {
228 $stats{ci}{$ci}++ if $dist->{$ci};
229 }
230 }
231 }
232 if ($stats{total}) {
233 $stats{has_vcs_percentage} = int(100 * $stats{has_vcs} / $stats{total});
234 $stats{has_ci_percentage} = int(100 * $stats{has_ci} / $stats{total});
235 }
236  
237  
238 my $tt = Template->new({
239 INCLUDE_PATH => './templates',
240 INTERPOLATE => 1,
241 WRAPPER => 'wrapper.tt',
242 }) or die "$Template::ERROR\n";
243  
244 my $report;
245 $tt->process('main.tt', {
246 distros => \@distros,
247 version => $VERSION,
248 timestamp => DateTime->now,
249 stats => \%stats,
250 }, \$report) or die $tt->error(), "\n";
251 my $html_file = File::Spec->catfile($self->{html}, 'index.html');
252 open(my $fh, '>', $html_file) or die "Could not open '$html_file'";
253 print $fh $report;
254 close $fh;
255 }
256  
257  
258 sub report {
259 my ($self) = @_;
260  
261 return if not $self->{report};
262  
263 print "Report\n";
264 print "------------\n";
265 my @distros = @{ $self->{db}->db_get_every_distro() };
266 if ($self->{limit} and @distros > $self->{limit}) {
267 @distros = @distros[0 .. $self->{limit}-1];
268 }
269 for my $distro (@distros) {
270 #die Dumper $distro;
271 printf "%s %-40s %-7s", $distro->{date}, $distro->{distribution}, ($distro->{vcs_url} ? '' : 'NO VCS');
272 if ($self->{check_vcs}) {
273 printf "%-7s", ($distro->{has_ci} ? '' : 'NO CI');
274 }
275 print "\n";
276 }
277  
278 if ($self->{days}) {
279 my $distros = $self->{db}->get_distro_count($self->{start_date}, $self->{end_date});
280 my $authors = $self->{db}->get_author_count($self->{start_date}, $self->{end_date});
281 my $vcs_count = $self->{db}->get_vcs_count($self->{start_date}, $self->{end_date});
282 my $ci_count = $self->{db}->get_ci_count($self->{start_date}, $self->{end_date});
283 printf
284 "Last week there were a total of %s uploads to CPAN of %s distinct distributions by %s different authors. Number of distributions with link to VCS: %s. Number of distros with CI: %s.\n",
285 $self->{total}, $distros, $authors, $vcs_count,
286 $ci_count;
287 }
288 }
289  
290 sub collect {
291 my ($self) = @_;
292  
293 my @all_the_distributions;
294  
295 my $log_level = $self->{log}; # TODO: shall we validate?
296 Log::Log4perl->easy_init({
297 level => $log_level,
298 layout => '%d{yyyy-MM-dd HH:mm:ss} - %p - %m%n',
299 });
300  
301 my $logger = Log::Log4perl->get_logger();
302 $logger->info('Starting');
303 $logger->info("Tempdir: $tempdir");
304 $logger->info("Recent: $self->{recent}") if $self->{recent};
305 $logger->info("Author: $self->{author}") if $self->{author};
306  
307 my $mcpan = MetaCPAN::Client->new();
308 my $rset;
309 if ($self->{author}) {
310 my $author = $mcpan->author($self->{author});
311 #print $author;
312 $rset = $author->releases;
313 } else {
314 $rset = $mcpan->recent($self->{recent});
315 }
316 $logger->info("MetaCPAN::Client::ResultSet received with a total of $rset->{total} items");
317 my %distros;
318 my @fields = get_fields();
319 while ( my $item = $rset->next ) {
320 if ($self->{days}) {
321 next if $item->date lt $self->{start_date};
322 next if $self->{end_date} le $item->date;
323 }
324  
325 $self->{total}++;
326  
327 next if $distros{ $item->distribution }; # We have already deal with this in this session
328 $distros{ $item->distribution } = 1;
329  
330 my $row = $self->{db}->db_get_distro($item->distribution);
331 next if $row and $row->{version} eq $item->version; # we already have this in the database (shall we call last?)
332 my %data = $self->get_data($item);
333 #die Dumper \%data;
334 $self->{db}->db_insert_into(@data{@fields});
335 push @all_the_distributions, \%data;
336 }
337  
338 if ($self->{author}) {
339 @all_the_distributions = reverse sort {$a->{date} cmp $b->{date}} @all_the_distributions;
340 if ($self->{limit} and @all_the_distributions > $self->{limit}) {
341 @all_the_distributions = @all_the_distributions[0 .. $self->{limit}-1];
342 }
343 }
344  
345 # Check on the VCS
346 if ($self->{check_vcs}) {
347 $logger->info("Starting to check GitHub");
348 for my $data (@all_the_distributions) {
349 my $distribution = $data->{distribution};
350 my $data_ref = $self->{db}->db_get_distro($distribution);
351 next if not $data_ref->{vcs_name};
352  
353 analyze_vcs($data_ref);
354  
355 my %data = %$data_ref;
356 $self->{db}->db_update($distribution, @data{@fields});
357 sleep $self->{sleep} if $self->{sleep};
358 }
359 }
360  
361  
362 $self->report;
363 $self->html;
364 }
365  
366  
367 42;
368  
369  
370 =head1 NAME
371  
372 CPAN::Digger - To dig CPAN
373  
374 =head1 SYNOPSIS
375  
376 cpan-digger
377  
378 =head1 DESCRIPTION
379  
380 This is a command line program to collect some meta information about CPAN modules.
381  
382  
383 =head1 COPYRIGHT AND LICENSE
384  
385 Copyright (C) 2020 by L<Gabor Szabo|https://szabgab.com/>
386  
387 This library is free software; you can redistribute it and/or modify
388 it under the same terms as Perl itself, either Perl version 5.26.1 or,
389 at your option, any later version of Perl 5 you may have available.
390  
391 =cut
392  
« blib/lib/CPAN/Digger/CLI.pm ↑ Summary