| 1 | use 5.006; | ||
| 2 | use strict; | ||
| 3 | use warnings; | ||
| 4 | package CPAN::Mini::Devel::Recent; | ||
| 5 | # ABSTRACT: Create CPAN::Mini mirror with recent developer releases | ||
| 6 | |||
| 7 | our $VERSION = '0.07'; | ||
| 8 | |||
| 9 | use Config; | ||
| 10 | use CPAN::Mini 0.567; | ||
| 11 | use CPAN 1.92 (); | ||
| 12 | use CPAN::Tarzip; | ||
| 13 | use CPAN::HandleConfig; | ||
| 14 | use File::Temp 0.20; | ||
| 15 | use File::Spec 3.27; | ||
| 16 | use File::Path (); | ||
| 17 | use File::Basename qw/basename/; | ||
| 18 | use YAML::Syck 1.07 qw/LoadFile/; | ||
| 19 | |||
| 20 | our @ISA = qw/CPAN::Mini/; | ||
| 21 | |||
| 22 | #--------------------------------------------------------------------------# | ||
| 23 | # globals | ||
| 24 | #--------------------------------------------------------------------------# | ||
| 25 | |||
| 26 | BEGIN { | ||
| 27 | *DEBUG = $ENV{CPAN_MINI_DEBUG} ? sub {1} : sub {0}; | ||
| 28 | } | ||
| 29 | |||
| 30 | my $tmp_dir = File::Temp->newdir( 'CPAN-Mini-Devel-Recent-XXXXXXX', | ||
| 31 | DIR => File::Spec->tmpdir, | ||
| 32 | ); | ||
| 33 | |||
| 34 | #--------------------------------------------------------------------------# | ||
| 35 | # Extend index methods to mirror RECENT* files | ||
| 36 | #--------------------------------------------------------------------------# | ||
| 37 | |||
| 38 | my @recent_files = qw( | ||
| 39 | authors/RECENT-1h.yaml | ||
| 40 | authors/RECENT-6h.yaml | ||
| 41 | authors/RECENT-1d.yaml | ||
| 42 | authors/RECENT-1W.yaml | ||
| 43 | authors/RECENT-1M.yaml | ||
| 44 | authors/RECENT-1Q.yaml | ||
| 45 | authors/RECENT-1Y.yaml | ||
| 46 | authors/RECENT-Z.yaml | ||
| 47 | ); | ||
| 48 | |||
| 49 | sub _fixed_mirrors { | ||
| 50 | my $self = shift; | ||
| 51 | return (@recent_files, $self->SUPER::_fixed_mirrors); | ||
| 52 | } | ||
| 53 | |||
| 54 | #--------------------------------------------------------------------------# | ||
| 55 | # Replace _get_mirror_list to add developer versions | ||
| 56 | #--------------------------------------------------------------------------# | ||
| 57 | |||
| 58 | sub _get_mirror_list { | ||
| 59 | my $self = shift; | ||
| 60 | |||
| 61 | ## CPAN::Mini::Devel::Recent addition using find-ls.gz | ||
| 62 | my @recent = map {File::Spec->catfile($self->{scratch}, $_)} @recent_files; | ||
| 63 | |||
| 64 | my $packages = File::Spec->catfile( | ||
| 65 | $self->{scratch}, | ||
| 66 | qw(modules 02packages.details.txt.gz) | ||
| 67 | ); | ||
| 68 | |||
| 69 | return $self->_parse_module_index( $packages, @recent ); | ||
| 70 | } | ||
| 71 | |||
| 72 | #--------------------------------------------------------------------------# | ||
| 73 | # private variables and functions | ||
| 74 | #--------------------------------------------------------------------------# | ||
| 75 | |||
| 76 | my $module_index_re = qr{ | ||
| 77 | ^\s href="\.\./authors/id/./../ # skip prelude | ||
| 78 | ([^"]+) # capture to next dquote mark | ||
| 79 | .+? </a> # skip to end of hyperlink | ||
| 80 | \s+ # skip spaces | ||
| 81 | \S+ # skip size | ||
| 82 | \s+ # skip spaces | ||
| 83 | (\S+) # capture day | ||
| 84 | \s+ # skip spaces | ||
| 85 | (\S+) # capture month | ||
| 86 | \s+ # skip spaces | ||
| 87 | (\S+) # capture year | ||
| 88 | }xms; | ||
| 89 | |||
| 90 | my %months = ( | ||
| 91 | Jan => '01', Feb => '02', Mar => '03', Apr => '04', May => '05', | ||
| 92 | Jun => '06', Jul => '07', Aug => '08', Sep => '09', Oct => '10', | ||
| 93 | Nov => '11', Dec => '12' | ||
| 94 | ); | ||
| 95 | |||
| 96 | # standard regexes | ||
| 97 | # note on archive suffixes -- .pm.gz shows up in 02packagesf | ||
| 98 | my %re = ( | ||
| 99 | perls => qr{(?: | ||
| 100 | /(?:emb|syb|bio)?perl-\d | ||
| 101 | | /(?:parrot|ponie|kurila|Perl6-Pugs)-\d | ||
| 102 | | /perl-?5\.004 | ||
| 103 | | /perl_mlb\.zip | ||
| 104 | | /Per6-Pugs | ||
| 105 | )}xi, | ||
| 106 | archive => qr{\.(?:tar\.(?:bz2|gz|Z)|t(?:gz|bz)|(?<!ppm\.)zip|pm.gz)$}i, | ||
| 107 | target_dir => qr{^id/\w/\w\w/}, | ||
| 108 | leading_initials => qr{(.)/\1./}, | ||
| 109 | ); | ||
| 110 | |||
| 111 | # match version and suffix | ||
| 112 | $re{version_suffix} = qr{([-._]v?[0-9].*)?($re{archive})}; | ||
| 113 | |||
| 114 | # split into "AUTHOR/Name" and "Version" | ||
| 115 | $re{split_them} = qr{^(.+?)$re{version_suffix}$}; | ||
| 116 | |||
| 117 | # matches "AUTHOR/tarball.suffix" or AUTHOR/modules/tarball.suffix | ||
| 118 | # and not other "AUTHOR/subdir/whatever" | ||
| 119 | |||
| 120 | # Just get AUTHOR/tarball.suffix from whatever file name is passed in | ||
| 121 | sub _get_base_id { | ||
| 122 | my $file = shift; | ||
| 123 | my $base_id = $file; | ||
| 124 | $base_id =~ s{$re{target_dir}}{}; | ||
| 125 | return $base_id; | ||
| 126 | } | ||
| 127 | |||
| 128 | sub _base_name { | ||
| 129 | my ($base_id) = @_; | ||
| 130 | my $base_file = basename $base_id; | ||
| 131 | my ($base_name, $base_version) = $base_file =~ $re{split_them}; | ||
| 132 | return $base_name; | ||
| 133 | } | ||
| 134 | |||
| 135 | |||
| 136 | #--------------------------------------------------------------------------# | ||
| 137 | # _parse_module_index | ||
| 138 | # | ||
| 139 | # parse index and return array_ref of distributions in reverse date order | ||
| 140 | #--------------------------------------------------------------------------#- | ||
| 141 | |||
| 142 | sub _parse_module_index { | ||
| 143 | my ($self, $packages, @recent ) = @_; | ||
| 144 | |||
| 145 | # first walk the packages list | ||
| 146 | # and build an index | ||
| 147 | |||
| 148 | my (%valid_distros, %mirror); | ||
| 149 | my (%latest, %latest_dev); | ||
| 150 | |||
| 151 | my $gz = Compress::Zlib::gzopen($packages, "rb") | ||
| 152 | or die "Cannot open package list: $Compress::Zlib::gzerrno"; | ||
| 153 | |||
| 154 | $self->trace( "Scanning 02packages.details ...\n" ); | ||
| 155 | my $inheader = 1; | ||
| 156 | while ($gz->gzreadline($_) > 0) { | ||
| 157 | if ($inheader) { | ||
| 158 | $inheader = 0 unless /\S/; | ||
| 159 | next; | ||
| 160 | } | ||
| 161 | |||
| 162 | my ($module, $version, $path) = split; | ||
| 163 | |||
| 164 | # shouldn't happen, but 02packages has in the past indexed a PPM file | ||
| 165 | next unless $path =~ $re{archive}; | ||
| 166 | |||
| 167 | next if $self->_filter_module({ | ||
| 168 | module => $module, | ||
| 169 | version => $version, | ||
| 170 | path => $path, | ||
| 171 | }); | ||
| 172 | |||
| 173 | |||
| 174 | my $base_id = _get_base_id("id/$path") | ||
| 175 | or $self->trace("Error getting base_id of '$path'\n"); | ||
| 176 | $valid_distros{$base_id}++; | ||
| 177 | my $base_name = _base_name( $base_id ) | ||
| 178 | or $self->trace("Error getting base_name of '$base_id'\n"); | ||
| 179 | if ($base_name) { | ||
| 180 | $latest{$base_name} = { | ||
| 181 | datetime => 0, | ||
| 182 | base_id => $base_id | ||
| 183 | }; | ||
| 184 | } | ||
| 185 | } | ||
| 186 | |||
| 187 | # next walk the recent list | ||
| 188 | $self->trace( "Scanning RECENT files for dev releases ...\n" ); | ||
| 189 | |||
| 190 | my %living; | ||
| 191 | |||
| 192 | for my $rr ( reverse @recent ) { | ||
| 193 | my $yaml = LoadFile( $rr ); | ||
| 194 | my ($add, $del) = (0,0); | ||
| 195 | for my $rec ( sort {$a->{epoch} <=> $b->{epoch}} @{$yaml->{recent}} ) { | ||
| 196 | if ( $rec->{type} eq 'new' ) { | ||
| 197 | $living{$rec->{path}} = $rec; | ||
| 198 | $add++; | ||
| 199 | } | ||
| 200 | elsif ( $rec->{type} eq 'delete' ) { | ||
| 201 | delete $living{$rec->{path}}; | ||
| 202 | $del++; | ||
| 203 | } | ||
| 204 | } | ||
| 205 | } | ||
| 206 | |||
| 207 | my $match = 0; | ||
| 208 | for my $rec ( sort {$a->{epoch} <=> $b->{epoch}} values %living ) { | ||
| 209 | my %stat; | ||
| 210 | @stat{qw/name datetime type/}=@{$rec}{qw/path epoch type/}; | ||
| 211 | |||
| 212 | # skip things that aren't a tarball | ||
| 213 | next unless $stat{name} =~ $re{archive}; | ||
| 214 | |||
| 215 | # skip if not AUTHOR/tarball | ||
| 216 | # skip perls | ||
| 217 | my $base_id = _get_base_id($stat{name}) | ||
| 218 | or $self->trace("Error getting base_id of '$stat{name}'\n"); | ||
| 219 | next unless $base_id; | ||
| 220 | |||
| 221 | next if $base_id =~ $re{perls}; | ||
| 222 | |||
| 223 | my $base_name = _base_name( $base_id ) | ||
| 224 | or $self->trace("# Error getting base_name from '$base_id'"); | ||
| 225 | |||
| 226 | # if $base_id matches 02packages, then it is the latest version | ||
| 227 | # and we definitely want it; also update datetime from the initial | ||
| 228 | # assumption of 0 | ||
| 229 | if ( $valid_distros{$base_id} ) { | ||
| 230 | $mirror{$base_id} = $stat{datetime}; | ||
| 231 | next unless $base_name; | ||
| 232 | $latest{$base_name}{base_id} = $base_id; | ||
| 233 | if ( $stat{datetime} > $latest{$base_name}{datetime} ) { | ||
| 234 | $latest{$base_name}{datetime} = $stat{datetime}; | ||
| 235 | } | ||
| 236 | $match++; | ||
| 237 | } | ||
| 238 | # if not in the packages file, we only want it if it resembles | ||
| 239 | # something in the package file and we only the most recent one | ||
| 240 | else { | ||
| 241 | # skip if couldn't parse out the name without version number | ||
| 242 | next unless defined $base_name; | ||
| 243 | |||
| 244 | # skip unless there's a matching base from the packages file | ||
| 245 | next unless exists $latest{$base_name}; | ||
| 246 | |||
| 247 | # keep only the latest | ||
| 248 | $latest_dev{$base_name} ||= { datetime => 0 }; | ||
| 249 | if ( $stat{datetime} > $latest_dev{$base_name}{datetime} ) { | ||
| 250 | $latest_dev{$base_name} = { | ||
| 251 | datetime => $stat{datetime}, | ||
| 252 | base_id => $base_id | ||
| 253 | }; | ||
| 254 | } | ||
| 255 | } | ||
| 256 | } | ||
| 257 | |||
| 258 | # pick up anything from packages we didn't already get (shouldn't happen) | ||
| 259 | my $missing = 0; | ||
| 260 | for my $base_name ( keys %latest ) { | ||
| 261 | my $base_id = $latest{$base_name}{base_id}; | ||
| 262 | next if exists $mirror{$base_id}; | ||
| 263 | $mirror{$base_id} = $latest{$base_name}{datetime}; | ||
| 264 | $missing++; | ||
| 265 | } | ||
| 266 | |||
| 267 | # for dev versions, it must be newer than the latest version of | ||
| 268 | # the same base name from the packages file | ||
| 269 | |||
| 270 | for my $base_name ( keys %latest_dev ) { | ||
| 271 | if ( ! $latest{$base_name} ) { | ||
| 272 | $self->trace( "Shouldn't be missing '$base_name' matching '$latest_dev{$base_name}{base_id}'\n" ); | ||
| 273 | next; | ||
| 274 | } | ||
| 275 | next if $latest{$base_name}{datetime} > $latest_dev{$base_name}{datetime}; | ||
| 276 | my $base_id = $latest_dev{$base_name}{base_id}; | ||
| 277 | $mirror{ $base_id } = $latest_dev{$base_name}{datetime}; | ||
| 278 | } | ||
| 279 | |||
| 280 | my $mirror_list = | ||
| 281 | [ sort map { s{^(((.).).+)$}{authors/id/$3/$2/$1}; $_ } keys %mirror ]; ## no critic | ||
| 282 | |||
| 283 | return $mirror_list; | ||
| 284 | } | ||
| 285 | |||
| 286 | 1; #modules must return true | ||
| 287 | |||
| 288 | __END__ |