| 1 | package CPAN::Digger; | ||
| 2 | 5 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 3 | 5 | ▶ | use warnings FATAL => 'all'; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 4 | |||
| 5 | our $VERSION = '1.04'; | ||
| 6 | |||
| 7 | 5 | ▶ | use Capture::Tiny qw(capture); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 8 | 5 | ▶ | use Cwd qw(getcwd); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 9 | 5 | ▶ | use Data::Dumper qw(Dumper); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 10 | 5 | ▶ | use Exporter qw(import); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 11 | 5 | ▶ | use File::Spec (); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 12 | 5 | ▶ | use File::Temp qw(tempdir); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 13 | 5 | ▶ | use Log::Log4perl (); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 14 | 5 | ▶ | use LWP::UserAgent (); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 15 | 5 | ▶ | use MetaCPAN::Client (); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 16 | 0 | ▶ | use DateTime (); |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 17 | use Template (); | ||
| 18 | |||
| 19 | my @ci_names = qw(travis github_actions circleci appveyor azure_pipeline gitlab_pipeline); | ||
| 20 | |||
| 21 | |||
| 22 | use CPAN::Digger::DB qw(get_fields); | ||
| 23 | |||
| 24 | my $tempdir = tempdir( CLEANUP => ($ENV{KEEP_TEMPDIR} ? 0 : 1) ); | ||
| 25 | |||
| 26 | my %known_licenses = map {$_ => 1} qw(apache_2_0 artistic_2 bsd gpl_2 gpl_3 lgpl_2_1 lgpl_3_0 perl_5); # open_source, unknown | ||
| 27 | |||
| 28 | sub new { | ||
| 29 | my ($class, %args) = @_; | ||
| 30 | my $self = bless {}, $class; | ||
| 31 | for my $key (keys %args) { | ||
| 32 | $self->{$key} = $args{$key}; | ||
| 33 | } | ||
| 34 | $self->{log} = uc $self->{log}; | ||
| 35 | $self->{check_vcs} = delete $self->{vcs}; | ||
| 36 | $self->{total} = 0; | ||
| 37 | |||
| 38 | my $dt = DateTime->now; | ||
| 39 | $self->{end_date} = $dt->ymd; | ||
| 40 | if ($self->{days}) { | ||
| 41 | $self->{start_date} = $dt->add( days => -$self->{days} )->ymd; | ||
| 42 | } | ||
| 43 | |||
| 44 | $self->{db} = CPAN::Digger::DB->new(db => $self->{db}); | ||
| 45 | |||
| 46 | return $self; | ||
| 47 | } | ||
| 48 | |||
| 49 | sub read_dashboards { | ||
| 50 | my ($self) = @_; | ||
| 51 | my @pathes = ('dashboard', '/home/gabor/cpan/dashboard'); | ||
| 52 | for my $path (@pathes) { | ||
| 53 | if (-e $path) { | ||
| 54 | $self->{dashboards} = { map { m{.*/([^/]+)\.json$}; $1 => 1 } glob "$path/authors/*.json" }; | ||
| 55 | } | ||
| 56 | } | ||
| 57 | } | ||
| 58 | |||
| 59 | sub get_vcs { | ||
| 60 | my ($repository) = @_; | ||
| 61 | if ($repository) { | ||
| 62 | # $html .= sprintf qq{<a href="%s">%s %s</a><br>\n}, $repository->{$k}, $k, $repository->{$k}; | ||
| 63 | # Try to get the web link | ||
| 64 | my $url = $repository->{web}; | ||
| 65 | if (not $url) { | ||
| 66 | $url = $repository->{url}; | ||
| 67 | $url =~ s{^git://}{https://}; | ||
| 68 | $url =~ s{\.git$}{}; | ||
| 69 | } | ||
| 70 | my $name = "repository"; | ||
| 71 | if ($url =~ m{^https?://github.com/}) { | ||
| 72 | $name = 'GitHub'; | ||
| 73 | } | ||
| 74 | if ($url =~ m{^https?://gitlab.com/}) { | ||
| 75 | $name = 'GitLab'; | ||
| 76 | } | ||
| 77 | return $url, $name; | ||
| 78 | } | ||
| 79 | } | ||
| 80 | |||
| 81 | sub get_data { | ||
| 82 | my ($self, $item) = @_; | ||
| 83 | |||
| 84 | my $logger = Log::Log4perl->get_logger(); | ||
| 85 | my %data = ( | ||
| 86 | distribution => $item->distribution, | ||
| 87 | version => $item->version, | ||
| 88 | author => $item->author, | ||
| 89 | date => $item->date, | ||
| 90 | ); | ||
| 91 | #die Dumper $item; | ||
| 92 | |||
| 93 | $logger->debug('dist: ', $item->distribution); | ||
| 94 | $logger->debug(' ', $item->author); | ||
| 95 | my @licenses = @{ $item->license }; | ||
| 96 | $data{licenses} = join ' ', @licenses; | ||
| 97 | $logger->debug(' ', $data{licenses}); | ||
| 98 | for my $license (@licenses) { | ||
| 99 | if ($license eq 'unknown') { | ||
| 100 | $logger->error("Unknown license '$license'"); | ||
| 101 | } elsif (not exists $known_licenses{$license}) { | ||
| 102 | $logger->warn("Unknown license '$license'. Probably CPAN::Digger needs to be updated"); | ||
| 103 | } | ||
| 104 | } | ||
| 105 | # if there are not licenses => | ||
| 106 | # if there is a license called "unknonws" | ||
| 107 | # check against a known list of licenses (grow it later, or look it up somewhere?) | ||
| 108 | my %resources = %{ $item->resources }; | ||
| 109 | #say ' ', join ' ', keys %resources; | ||
| 110 | if ($resources{repository}) { | ||
| 111 | my ($vcs_url, $vcs_name) = get_vcs($resources{repository}); | ||
| 112 | if ($vcs_url) { | ||
| 113 | $data{vcs_url} = $vcs_url; | ||
| 114 | $data{vcs_name} = $vcs_name; | ||
| 115 | $logger->debug(" $vcs_name: $vcs_url"); | ||
| 116 | } | ||
| 117 | } else { | ||
| 118 | $logger->error('No repository for ', $item->distribution); | ||
| 119 | } | ||
| 120 | return %data; | ||
| 121 | } | ||
| 122 | |||
| 123 | |||
| 124 | sub analyze_vcs { | ||
| 125 | my ($data) = @_; | ||
| 126 | my $logger = Log::Log4perl->get_logger(); | ||
| 127 | |||
| 128 | my $vcs_url = $data->{vcs_url}; | ||
| 129 | my $repo_name = (split '\/', $vcs_url)[-1]; | ||
| 130 | $logger->info("Analyze GitHub repo '$vcs_url' in directory $repo_name"); | ||
| 131 | |||
| 132 | my $ua = LWP::UserAgent->new(timeout => 5); | ||
| 133 | my $response = $ua->get($vcs_url); | ||
| 134 | my $status_line = $response->status_line; | ||
| 135 | if ($status_line eq '404 Not Found') { | ||
| 136 | $logger->error("Repository '$vcs_url' Received 404 Not Found. Please update the link in the META file"); | ||
| 137 | return; | ||
| 138 | } | ||
| 139 | if ($response->code != 200) { | ||
| 140 | $logger->error("Repository '$vcs_url' got a response of '$status_line'. Please report this to the maintainer of CPAN::Digger."); | ||
| 141 | return; | ||
| 142 | } | ||
| 143 | if ($response->redirects) { | ||
| 144 | $logger->error("Repository '$vcs_url' is being redirected. Please update the link in the META file"); | ||
| 145 | } | ||
| 146 | |||
| 147 | my $git = 'git'; | ||
| 148 | |||
| 149 | my @cmd = ($git, "clone", "--depth", "1", $data->{vcs_url}); | ||
| 150 | my $cwd = getcwd(); | ||
| 151 | chdir($tempdir); | ||
| 152 | my ($out, $err, $exit_code) = capture { | ||
| 153 | system(@cmd); | ||
| 154 | }; | ||
| 155 | chdir($cwd); | ||
| 156 | my $repo = "$tempdir/$repo_name"; | ||
| 157 | $logger->debug("REPO path '$repo'"); | ||
| 158 | |||
| 159 | if ($exit_code != 0) { | ||
| 160 | # TODO capture stderr and include in the log | ||
| 161 | $logger->error("Failed to clone $vcs_url"); | ||
| 162 | return; | ||
| 163 | } | ||
| 164 | |||
| 165 | if ($data->{vcs_name} eq 'GitHub') { | ||
| 166 | analyze_github($data, $repo); | ||
| 167 | } | ||
| 168 | if ($data->{vcs_name} eq 'GitLab') { | ||
| 169 | analyze_gitlab($data, $repo); | ||
| 170 | } | ||
| 171 | |||
| 172 | for my $ci (@ci_names) { | ||
| 173 | $logger->debug("Is CI '$ci'?"); | ||
| 174 | if ($data->{$ci}) { | ||
| 175 | $logger->debug("CI '$ci' found!"); | ||
| 176 | $data->{has_ci} = 1; | ||
| 177 | } | ||
| 178 | } | ||
| 179 | } | ||
| 180 | |||
| 181 | sub analyze_gitlab { | ||
| 182 | my ($data, $repo) = @_; | ||
| 183 | |||
| 184 | $data->{gitlab_pipeline} = -e "$repo/.gitlab-ci.yml"; | ||
| 185 | } | ||
| 186 | |||
| 187 | sub analyze_github { | ||
| 188 | my ($data, $repo) = @_; | ||
| 189 | |||
| 190 | $data->{travis} = -e "$repo/.travis.yml"; | ||
| 191 | my @ga = glob("$repo/.github/workflows/*"); | ||
| 192 | $data->{github_actions} = (scalar(@ga) ? 1 : 0); | ||
| 193 | $data->{circleci} = -e "$repo/.circleci"; | ||
| 194 | $data->{appveyor} = (-e "$repo/.appveyor.yml") || (-e "$repo/appveyor.yml"); | ||
| 195 | $data->{azure_pipeline} = -e "$repo/azure-pipelines.yml"; | ||
| 196 | } | ||
| 197 | |||
| 198 | sub html { | ||
| 199 | my ($self) = @_; | ||
| 200 | |||
| 201 | return if not $self->{html}; | ||
| 202 | if (not -d $self->{html}) { | ||
| 203 | mkdir $self->{html}; | ||
| 204 | } | ||
| 205 | |||
| 206 | $self->read_dashboards; | ||
| 207 | |||
| 208 | my @distros = @{ $self->{db}->db_get_every_distro() }; | ||
| 209 | my %stats = ( | ||
| 210 | total => scalar @distros, | ||
| 211 | has_vcs => 0, | ||
| 212 | vcs => {}, | ||
| 213 | has_ci => 0, | ||
| 214 | ci => {}, | ||
| 215 | ); | ||
| 216 | for my $ci (@ci_names) { | ||
| 217 | $stats{ci}{$ci} = 0; | ||
| 218 | } | ||
| 219 | for my $dist (@distros) { | ||
| 220 | $dist->{dashboard} = $self->{dashboards}{ $dist->{author} }; | ||
| 221 | if ($dist->{vcs_name}) { | ||
| 222 | $stats{has_vcs}++; | ||
| 223 | $stats{vcs}{ $dist->{vcs_name} }++; | ||
| 224 | } | ||
| 225 | if ($dist->{has_ci}) { | ||
| 226 | $stats{has_ci}++; | ||
| 227 | for my $ci (@ci_names) { | ||
| 228 | $stats{ci}{$ci}++ if $dist->{$ci}; | ||
| 229 | } | ||
| 230 | } | ||
| 231 | } | ||
| 232 | if ($stats{total}) { | ||
| 233 | $stats{has_vcs_percentage} = int(100 * $stats{has_vcs} / $stats{total}); | ||
| 234 | $stats{has_ci_percentage} = int(100 * $stats{has_ci} / $stats{total}); | ||
| 235 | } | ||
| 236 | |||
| 237 | |||
| 238 | my $tt = Template->new({ | ||
| 239 | INCLUDE_PATH => './templates', | ||
| 240 | INTERPOLATE => 1, | ||
| 241 | WRAPPER => 'wrapper.tt', | ||
| 242 | }) or die "$Template::ERROR\n"; | ||
| 243 | |||
| 244 | my $report; | ||
| 245 | $tt->process('main.tt', { | ||
| 246 | distros => \@distros, | ||
| 247 | version => $VERSION, | ||
| 248 | timestamp => DateTime->now, | ||
| 249 | stats => \%stats, | ||
| 250 | }, \$report) or die $tt->error(), "\n"; | ||
| 251 | my $html_file = File::Spec->catfile($self->{html}, 'index.html'); | ||
| 252 | open(my $fh, '>', $html_file) or die "Could not open '$html_file'"; | ||
| 253 | print $fh $report; | ||
| 254 | close $fh; | ||
| 255 | } | ||
| 256 | |||
| 257 | |||
| 258 | sub report { | ||
| 259 | my ($self) = @_; | ||
| 260 | |||
| 261 | return if not $self->{report}; | ||
| 262 | |||
| 263 | print "Report\n"; | ||
| 264 | print "------------\n"; | ||
| 265 | my @distros = @{ $self->{db}->db_get_every_distro() }; | ||
| 266 | if ($self->{limit} and @distros > $self->{limit}) { | ||
| 267 | @distros = @distros[0 .. $self->{limit}-1]; | ||
| 268 | } | ||
| 269 | for my $distro (@distros) { | ||
| 270 | #die Dumper $distro; | ||
| 271 | printf "%s %-40s %-7s", $distro->{date}, $distro->{distribution}, ($distro->{vcs_url} ? '' : 'NO VCS'); | ||
| 272 | if ($self->{check_vcs}) { | ||
| 273 | printf "%-7s", ($distro->{has_ci} ? '' : 'NO CI'); | ||
| 274 | } | ||
| 275 | print "\n"; | ||
| 276 | } | ||
| 277 | |||
| 278 | if ($self->{days}) { | ||
| 279 | my $distros = $self->{db}->get_distro_count($self->{start_date}, $self->{end_date}); | ||
| 280 | my $authors = $self->{db}->get_author_count($self->{start_date}, $self->{end_date}); | ||
| 281 | my $vcs_count = $self->{db}->get_vcs_count($self->{start_date}, $self->{end_date}); | ||
| 282 | my $ci_count = $self->{db}->get_ci_count($self->{start_date}, $self->{end_date}); | ||
| 283 | printf | ||
| 284 | "Last week there were a total of %s uploads to CPAN of %s distinct distributions by %s different authors. Number of distributions with link to VCS: %s. Number of distros with CI: %s.\n", | ||
| 285 | $self->{total}, $distros, $authors, $vcs_count, | ||
| 286 | $ci_count; | ||
| 287 | } | ||
| 288 | } | ||
| 289 | |||
| 290 | sub collect { | ||
| 291 | my ($self) = @_; | ||
| 292 | |||
| 293 | my @all_the_distributions; | ||
| 294 | |||
| 295 | my $log_level = $self->{log}; # TODO: shall we validate? | ||
| 296 | Log::Log4perl->easy_init({ | ||
| 297 | level => $log_level, | ||
| 298 | layout => '%d{yyyy-MM-dd HH:mm:ss} - %p - %m%n', | ||
| 299 | }); | ||
| 300 | |||
| 301 | my $logger = Log::Log4perl->get_logger(); | ||
| 302 | $logger->info('Starting'); | ||
| 303 | $logger->info("Tempdir: $tempdir"); | ||
| 304 | $logger->info("Recent: $self->{recent}") if $self->{recent}; | ||
| 305 | $logger->info("Author: $self->{author}") if $self->{author}; | ||
| 306 | |||
| 307 | my $mcpan = MetaCPAN::Client->new(); | ||
| 308 | my $rset; | ||
| 309 | if ($self->{author}) { | ||
| 310 | my $author = $mcpan->author($self->{author}); | ||
| 311 | #print $author; | ||
| 312 | $rset = $author->releases; | ||
| 313 | } else { | ||
| 314 | $rset = $mcpan->recent($self->{recent}); | ||
| 315 | } | ||
| 316 | $logger->info("MetaCPAN::Client::ResultSet received with a total of $rset->{total} items"); | ||
| 317 | my %distros; | ||
| 318 | my @fields = get_fields(); | ||
| 319 | while ( my $item = $rset->next ) { | ||
| 320 | if ($self->{days}) { | ||
| 321 | next if $item->date lt $self->{start_date}; | ||
| 322 | next if $self->{end_date} le $item->date; | ||
| 323 | } | ||
| 324 | |||
| 325 | $self->{total}++; | ||
| 326 | |||
| 327 | next if $distros{ $item->distribution }; # We have already deal with this in this session | ||
| 328 | $distros{ $item->distribution } = 1; | ||
| 329 | |||
| 330 | my $row = $self->{db}->db_get_distro($item->distribution); | ||
| 331 | next if $row and $row->{version} eq $item->version; # we already have this in the database (shall we call last?) | ||
| 332 | my %data = $self->get_data($item); | ||
| 333 | #die Dumper \%data; | ||
| 334 | $self->{db}->db_insert_into(@data{@fields}); | ||
| 335 | push @all_the_distributions, \%data; | ||
| 336 | } | ||
| 337 | |||
| 338 | if ($self->{author}) { | ||
| 339 | @all_the_distributions = reverse sort {$a->{date} cmp $b->{date}} @all_the_distributions; | ||
| 340 | if ($self->{limit} and @all_the_distributions > $self->{limit}) { | ||
| 341 | @all_the_distributions = @all_the_distributions[0 .. $self->{limit}-1]; | ||
| 342 | } | ||
| 343 | } | ||
| 344 | |||
| 345 | # Check on the VCS | ||
| 346 | if ($self->{check_vcs}) { | ||
| 347 | $logger->info("Starting to check GitHub"); | ||
| 348 | for my $data (@all_the_distributions) { | ||
| 349 | my $distribution = $data->{distribution}; | ||
| 350 | my $data_ref = $self->{db}->db_get_distro($distribution); | ||
| 351 | next if not $data_ref->{vcs_name}; | ||
| 352 | |||
| 353 | analyze_vcs($data_ref); | ||
| 354 | |||
| 355 | my %data = %$data_ref; | ||
| 356 | $self->{db}->db_update($distribution, @data{@fields}); | ||
| 357 | sleep $self->{sleep} if $self->{sleep}; | ||
| 358 | } | ||
| 359 | } | ||
| 360 | |||
| 361 | |||
| 362 | $self->report; | ||
| 363 | $self->html; | ||
| 364 | } | ||
| 365 | |||
| 366 | |||
| 367 | 42; | ||
| 368 | |||
| 369 | |||
| 370 | =head1 NAME | ||
| 371 | |||
| 372 | CPAN::Digger - To dig CPAN | ||
| 373 | |||
| 374 | =head1 SYNOPSIS | ||
| 375 | |||
| 376 | cpan-digger | ||
| 377 | |||
| 378 | =head1 DESCRIPTION | ||
| 379 | |||
| 380 | This is a command line program to collect some meta information about CPAN modules. | ||
| 381 | |||
| 382 | |||
| 383 | =head1 COPYRIGHT AND LICENSE | ||
| 384 | |||
| 385 | Copyright (C) 2020 by L<Gabor Szabo|https://szabgab.com/> | ||
| 386 | |||
| 387 | This library is free software; you can redistribute it and/or modify | ||
| 388 | it under the same terms as Perl itself, either Perl version 5.26.1 or, | ||
| 389 | at your option, any later version of Perl 5 you may have available. | ||
| 390 | |||
| 391 | =cut | ||
| 392 | |||