| 1 | package urpm::get_pkgs; | ||
| 2 | |||
| 3 | |||
| 4 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 5 | 0 | ▶ | use urpm::msg; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 6 | use urpm::sys; | ||
| 7 | use urpm::util qw(basename put_in_hash); | ||
| 8 | use urpm::media; | ||
| 9 | use urpm 'file_from_local_url'; | ||
| 10 | # perl_checker: require urpm::select | ||
| 11 | |||
| 12 | |||
| 13 | =head1 NAME | ||
| 14 | |||
| 15 | urpm::get_pkgs - Package retrieving routines for urpmi | ||
| 16 | |||
| 17 | =head1 SYNOPSIS | ||
| 18 | |||
| 19 | =head1 DESCRIPTION | ||
| 20 | |||
| 21 | =over | ||
| 22 | |||
| 23 | =cut | ||
| 24 | |||
| 25 | sub clean_all_cache { | ||
| 26 | my ($urpm) = @_; | ||
| 27 | #- clean download directory, do it here even if this is not the best moment. | ||
| 28 | $urpm->{log}(N("cleaning %s and %s", "$urpm->{cachedir}/partial", "$urpm->{cachedir}/rpms")); | ||
| 29 | urpm::sys::empty_dir("$urpm->{cachedir}/partial"); | ||
| 30 | urpm::sys::empty_dir("$urpm->{cachedir}/rpms"); | ||
| 31 | } | ||
| 32 | |||
| 33 | sub cachedir_rpms { | ||
| 34 | my ($urpm) = @_; | ||
| 35 | |||
| 36 | #- examine the local repository, which is trusted (no gpg or pgp signature check but md5 is now done). | ||
| 37 | my %fn2file; | ||
| 38 | foreach my $filepath (glob("$urpm->{cachedir}/rpms/*")) { | ||
| 39 | next if -d $filepath; | ||
| 40 | |||
| 41 | if (! -s $filepath) { | ||
| 42 | unlink $filepath; #- this file should be removed or is already empty. | ||
| 43 | } else { | ||
| 44 | my $filename = basename($filepath); | ||
| 45 | my ($fullname) = $filename =~ /(.*)\.rpm$/ or next; | ||
| 46 | $fn2file{$fullname} = $filepath; | ||
| 47 | } | ||
| 48 | } | ||
| 49 | \%fn2file; | ||
| 50 | } | ||
| 51 | |||
| 52 | #- select sources for selected packages, | ||
| 53 | #- according to keys of the packages hash. | ||
| 54 | #- returns a list of lists containing the source description for each rpm, | ||
| 55 | #- matching the exact number of registered media; ignored media being | ||
| 56 | #- associated to a null list. | ||
| 57 | sub _selected2local_and_ids { | ||
| 58 | my ($urpm, $packages, %options) = @_; | ||
| 59 | my (%protected_files, %local_sources, %fullname2id); | ||
| 60 | |||
| 61 | #- build association hash to retrieve id and examine all list files. | ||
| 62 | foreach (keys %$packages) { | ||
| 63 | foreach my $id (split /\|/, $_) { | ||
| 64 | if ($urpm->{source}{$_}) { | ||
| 65 | my $file = $local_sources{$id} = $urpm->{source}{$id}; | ||
| 66 | $protected_files{$file} = undef; | ||
| 67 | } else { | ||
| 68 | $fullname2id{$urpm->{depslist}[$id]->fullname} = $id; | ||
| 69 | } | ||
| 70 | } | ||
| 71 | } | ||
| 72 | |||
| 73 | #- examine the local repository, which is trusted (no gpg or pgp signature check but md5 is now done). | ||
| 74 | my $cachedir_rpms = cachedir_rpms($urpm); | ||
| 75 | |||
| 76 | foreach my $fullname (keys %$cachedir_rpms) { | ||
| 77 | my $filepath = $cachedir_rpms->{$fullname}; | ||
| 78 | |||
| 79 | if (my $id = delete $fullname2id{$fullname}) { | ||
| 80 | $local_sources{$id} = $filepath; | ||
| 81 | } else { | ||
| 82 | $options{clean_other} && ! exists $protected_files{$filepath} and unlink $filepath; | ||
| 83 | } | ||
| 84 | } | ||
| 85 | |||
| 86 | my %id2ids; | ||
| 87 | foreach my $id (values %fullname2id) { | ||
| 88 | my $pkg = $urpm->{depslist}[$id]; | ||
| 89 | my $fullname = $pkg->fullname; | ||
| 90 | my @pkg_ids = $pkg->arch eq 'src' ? do { | ||
| 91 | # packages_by_name can't be used here since $urpm->{provides} doesn't have src.rpm | ||
| 92 | # so a full search is needed | ||
| 93 | my %requested; | ||
| 94 | urpm::select::search_packages($urpm, \%requested, [$pkg->name], src => 1); | ||
| 95 | map { split /\|/ } keys %requested; | ||
| 96 | } : do { | ||
| 97 | map { $_->id } grep { | ||
| 98 | $_->filename !~ /\.delta\.rpm$/ || $urpm->is_delta_installable($_, $urpm->{root}); | ||
| 99 | } grep { $fullname eq $_->fullname } $urpm->packages_by_name($pkg->name); | ||
| 100 | }; | ||
| 101 | |||
| 102 | $id2ids{$id} = \@pkg_ids; | ||
| 103 | } | ||
| 104 | |||
| 105 | (\%local_sources, \%id2ids); | ||
| 106 | } | ||
| 107 | |||
| 108 | sub selected2local_and_blists { | ||
| 109 | my ($urpm, $selected, %options) = @_; | ||
| 110 | |||
| 111 | my ($local_sources, $id2ids) = _selected2local_and_ids($urpm, $selected, %options); | ||
| 112 | |||
| 113 | # id_map is a remapping of id. | ||
| 114 | # it is needed because @list must be [ { id => pkg } ] where id is one the selected id, | ||
| 115 | # not really the real package id | ||
| 116 | my %id_map; | ||
| 117 | foreach my $id (keys %$id2ids) { | ||
| 118 | $id_map{$_} = $id foreach @{$id2ids->{$id}}; | ||
| 119 | } | ||
| 120 | |||
| 121 | my @remaining_ids = sort { $a <=> $b } keys %id_map; | ||
| 122 | |||
| 123 | my @blists = map { | ||
| 124 | my $medium = $_; | ||
| 125 | my %pkgs; | ||
| 126 | if (urpm::media::is_valid_medium($medium) && !$medium->{ignore}) { | ||
| 127 | while (@remaining_ids) { | ||
| 128 | my $id = $remaining_ids[0]; | ||
| 129 | $medium->{start} <= $id && $id <= $medium->{end} or last; | ||
| 130 | shift @remaining_ids; | ||
| 131 | |||
| 132 | my $pkg = $urpm->{depslist}[$id]; | ||
| 133 | $pkgs{$id_map{$id}} = $pkg; | ||
| 134 | } | ||
| 135 | } | ||
| 136 | %pkgs ? { medium => $medium, pkgs => \%pkgs } : @{[]}; | ||
| 137 | } (@{$urpm->{media} || []}); | ||
| 138 | |||
| 139 | if (@remaining_ids) { | ||
| 140 | $urpm->{error}(N("package %s is not found.", $urpm->{depslist}[$_]->fullname)) foreach @remaining_ids; | ||
| 141 | return; | ||
| 142 | } | ||
| 143 | |||
| 144 | ($local_sources, \@blists); | ||
| 145 | } | ||
| 146 | |||
| 147 | #- side-effects: none | ||
| 148 | sub _create_old_list_from_blists { | ||
| 149 | my ($media, $blists) = @_; | ||
| 150 | |||
| 151 | [ map { | ||
| 152 | my $medium = $_; | ||
| 153 | my ($blist) = grep { $_->{medium} == $medium } @$blists; | ||
| 154 | |||
| 155 | { map { $_ => urpm::blist_pkg_to_url($blist, $blist->{pkgs}{$_}) } keys %{$blist->{pkgs}} } | ||
| 156 | } @$media ]; | ||
| 157 | } | ||
| 158 | |||
| 159 | sub verify_partial_rpm_and_move { | ||
| 160 | my ($urpm, $cachedir, $filename) = @_; | ||
| 161 | |||
| 162 | URPM::verify_rpm("$cachedir/partial/$filename", nosignatures => 1) or do { | ||
| 163 | unlink "$cachedir/partial/$filename"; | ||
| 164 | return; | ||
| 165 | }; | ||
| 166 | #- it seems the the file has been downloaded correctly and has been checked to be valid. | ||
| 167 | unlink "$cachedir/rpms/$filename"; | ||
| 168 | urpm::sys::move_or_die($urpm, "$cachedir/partial/$filename", "$cachedir/rpms/$filename"); | ||
| 169 | "$cachedir/rpms/$filename"; | ||
| 170 | } | ||
| 171 | |||
| 172 | |||
| 173 | =item get_distant_media_filesize($blists, $sources) | ||
| 174 | |||
| 175 | Get the filesize of packages to download from remote media. | ||
| 176 | |||
| 177 | =cut | ||
| 178 | |||
| 179 | sub get_distant_media_filesize { | ||
| 180 | my ($blists, $sources) = @_; | ||
| 181 | |||
| 182 | my $filesize; | ||
| 183 | #- get back all ftp and http accessible rpm files into the local cache | ||
| 184 | foreach my $blist (@$blists) { | ||
| 185 | #- examine all files to know what can be indexed on multiple media. | ||
| 186 | while (my ($id, $pkg) = each %{$blist->{pkgs}}) { | ||
| 187 | #- the given URL is trusted, so the file can safely be ignored. | ||
| 188 | defined $sources->{$id} and next; | ||
| 189 | if (!urpm::is_local_medium($blist->{medium})) { | ||
| 190 | if (my $n = $pkg->filesize) { | ||
| 191 | $filesize += $n; | ||
| 192 | } | ||
| 193 | } | ||
| 194 | } | ||
| 195 | } | ||
| 196 | $filesize; | ||
| 197 | } | ||
| 198 | |||
| 199 | =item download_packages_of_distant_media($urpm, $blists, $sources, $error_sources, %options) | ||
| 200 | |||
| 201 | Download packages listed in $blists, and put the result in $sources or | ||
| 202 | $error_sources | ||
| 203 | |||
| 204 | Options: quiet, callback, | ||
| 205 | |||
| 206 | =cut | ||
| 207 | |||
| 208 | sub download_packages_of_distant_media { | ||
| 209 | my ($urpm, $blists, $sources, $error_sources, %options) = @_; | ||
| 210 | |||
| 211 | my %errors; | ||
| 212 | my %new_sources; | ||
| 213 | |||
| 214 | #- get back all ftp and http accessible rpm files into the local cache | ||
| 215 | foreach my $blist (@$blists) { | ||
| 216 | my %blist_distant = (%$blist, pkgs => {}); | ||
| 217 | |||
| 218 | #- examine all files to know what can be indexed on multiple media. | ||
| 219 | while (my ($id, $pkg) = each %{$blist->{pkgs}}) { | ||
| 220 | #- the given URL is trusted, so the file can safely be ignored. | ||
| 221 | if (defined $sources->{$id}) { | ||
| 222 | $new_sources{$id} = [ $pkg->id, $sources->{$id} ]; | ||
| 223 | delete $sources->{$id}; | ||
| 224 | next; | ||
| 225 | } | ||
| 226 | |||
| 227 | exists $new_sources{$id} and next; | ||
| 228 | if (urpm::is_local_medium($blist->{medium})) { | ||
| 229 | my $local_file = file_from_local_url(urpm::blist_pkg_to_url($blist, $pkg)); | ||
| 230 | if (-r $local_file) { | ||
| 231 | $new_sources{$id} = [ $pkg->id, $local_file ]; | ||
| 232 | } else { | ||
| 233 | $errors{$id} = [ $local_file, 'missing' ]; | ||
| 234 | } | ||
| 235 | } else { | ||
| 236 | $blist_distant{pkgs}{$id} = $pkg; | ||
| 237 | } | ||
| 238 | } | ||
| 239 | |||
| 240 | if (%{$blist_distant{pkgs}}) { | ||
| 241 | my ($remote_sources, $remote_errors) = _download_packages_of_distant_media($urpm, \%blist_distant, %options); | ||
| 242 | put_in_hash(\%new_sources, $remote_sources); | ||
| 243 | put_in_hash(\%errors, $remote_errors); | ||
| 244 | } | ||
| 245 | } | ||
| 246 | |||
| 247 | #- clean failed download which have succeeded. | ||
| 248 | delete @errors{keys %$sources, keys %new_sources}; | ||
| 249 | |||
| 250 | foreach (values %new_sources) { | ||
| 251 | my ($id, $local_file) = @$_; | ||
| 252 | $sources->{$id} = $local_file; | ||
| 253 | } | ||
| 254 | |||
| 255 | push @$error_sources, values %errors; | ||
| 256 | |||
| 257 | 1; | ||
| 258 | } | ||
| 259 | |||
| 260 | # download packages listed in $blist, | ||
| 261 | # and put the result in $sources or $errors | ||
| 262 | sub _download_packages_of_distant_media { | ||
| 263 | my ($urpm, $blist, %options) = @_; | ||
| 264 | |||
| 265 | my $cachedir = urpm::valid_cachedir($urpm); | ||
| 266 | my (%sources, %errors); | ||
| 267 | |||
| 268 | $urpm->{log}(N("retrieving rpm files from medium \"%s\"...", $blist->{medium}{name})); | ||
| 269 | if (urpm::download::sync_rel($urpm, $blist->{medium}, [ urpm::blist_to_filenames($blist) ], | ||
| 270 | dir => "$cachedir/partial", quiet => $options{quiet}, | ||
| 271 | is_versioned => 1, | ||
| 272 | resume => $urpm->{options}{resume}, | ||
| 273 | ask_retry => $options{ask_retry}, | ||
| 274 | callback => $options{callback})) { | ||
| 275 | $urpm->{log}(N("...retrieving done")); | ||
| 276 | } else { | ||
| 277 | $urpm->{error}(N("...retrieving failed: %s", $@)); | ||
| 278 | } | ||
| 279 | |||
| 280 | #- clean files that have not been downloaded, but keep in mind | ||
| 281 | #- there have been problems downloading them at least once, this | ||
| 282 | #- is necessary to keep track of failing downloads in order to | ||
| 283 | #- present the error to the user. | ||
| 284 | foreach my $id (keys %{$blist->{pkgs}}) { | ||
| 285 | my $pkg = $blist->{pkgs}{$id}; | ||
| 286 | my $filename = $pkg->filename; | ||
| 287 | my $url = urpm::blist_pkg_to_url($blist, $pkg); | ||
| 288 | if ($filename && -s "$cachedir/partial/$filename") { | ||
| 289 | if (my $rpm = verify_partial_rpm_and_move($urpm, $cachedir, $filename)) { | ||
| 290 | $sources{$id} = [ $pkg->id, $rpm ]; | ||
| 291 | } else { | ||
| 292 | $errors{$id} = [ $url, 'bad' ]; | ||
| 293 | } | ||
| 294 | } else { | ||
| 295 | $errors{$id} = [ $url, 'missing' ]; | ||
| 296 | } | ||
| 297 | } | ||
| 298 | (\%sources, \%errors); | ||
| 299 | } | ||
| 300 | |||
| 301 | 1; | ||
| 302 | |||
| 303 | |||
| 304 | =back | ||
| 305 | |||
| 306 | =head1 COPYRIGHT | ||
| 307 | |||
| 308 | Copyright (C) 2005 MandrakeSoft SA | ||
| 309 | |||
| 310 | Copyright (C) 2005-2010 Mandriva SA | ||
| 311 | |||
| 312 | Copyright (C) 2011-2020 Mageia | ||
| 313 | |||
| 314 | =cut | ||