| 1 | package urpm::mirrors; | ||
| 2 | |||
| 3 | |||
| 4 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 5 | 1 | ▶ | use urpm::util qw(cat_ find output_safe reduce_pathname); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 6 | 0 | ▶ | use urpm::msg; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 7 | use urpm::download; | ||
| 8 | |||
| 9 | |||
| 10 | =head1 NAME | ||
| 11 | |||
| 12 | urpm::mirrors - Mirrors routines for urpmi | ||
| 13 | |||
| 14 | =head1 SYNOPSIS | ||
| 15 | |||
| 16 | =head1 DESCRIPTION | ||
| 17 | |||
| 18 | =over | ||
| 19 | |||
| 20 | =item try($urpm, $medium, $try) | ||
| 21 | |||
| 22 | $medium fields used: mirrorlist, with-dir | ||
| 23 | |||
| 24 | =cut | ||
| 25 | |||
| 26 | #- side-effects: $medium->{url} | ||
| 27 | #- + those of _pick_one ($urpm->{mirrors_cache}) | ||
| 28 | sub try { | ||
| 29 | my ($urpm, $medium, $try) = @_; | ||
| 30 | |||
| 31 | for (my $nb = 1; $nb < $urpm->{options}{'max-round-robin-tries'}; $nb++) { | ||
| 32 | my $url = _pick_one($urpm, $medium->{mirrorlist}, $nb == 1, '') or return; | ||
| 33 | $urpm->{info}(N("trying again with mirror %s", $url)) if $nb > 1; | ||
| 34 | $medium->{url} = _add__with_dir($url, $medium->{'with-dir'}); | ||
| 35 | $try->() and return 1; | ||
| 36 | black_list($urpm, $medium->{mirrorlist}, $url); | ||
| 37 | } | ||
| 38 | 0; | ||
| 39 | } | ||
| 40 | |||
| 41 | =item try_probe($urpm, $medium, $try) | ||
| 42 | |||
| 43 | Similar to try() above, but failure is "normal" (useful when we lookup | ||
| 44 | a file) | ||
| 45 | |||
| 46 | $medium fields used: mirrorlist, with-dir | ||
| 47 | |||
| 48 | =cut | ||
| 49 | |||
| 50 | #- side-effects: $medium->{url} | ||
| 51 | #- + those of list_urls ($urpm->{mirrors_cache}) | ||
| 52 | sub try_probe { | ||
| 53 | my ($urpm, $medium, $try) = @_; | ||
| 54 | |||
| 55 | my $nb = 0; | ||
| 56 | foreach my $mirror (map { @$_ } list_urls($urpm, $medium, '')) { | ||
| 57 | $nb++ < $urpm->{options}{'max-round-robin-probes'} or last; | ||
| 58 | my $url = $mirror->{url}; | ||
| 59 | $nb > 1 ? $urpm->{info}(N("trying again with mirror %s", $url)) | ||
| 60 | : $urpm->{log}("using mirror $url"); | ||
| 61 | $medium->{url} = _add__with_dir($url, $medium->{'with-dir'}); | ||
| 62 | $try->() and return 1; | ||
| 63 | } | ||
| 64 | 0; | ||
| 65 | } | ||
| 66 | |||
| 67 | #- side-effects: none | ||
| 68 | sub _add__with_dir { | ||
| 69 | my ($url, $with_dir) = @_; | ||
| 70 | reduce_pathname($url . ($with_dir ? "/$with_dir" : '')); | ||
| 71 | } | ||
| 72 | |||
| 73 | #- side-effects: $medium->{url} | ||
| 74 | #- + those of _pick_one ($urpm->{mirrors_cache}) | ||
| 75 | sub pick_one { | ||
| 76 | my ($urpm, $medium, $allow_cache_update) = @_; | ||
| 77 | |||
| 78 | my $url = _pick_one($urpm, $medium->{mirrorlist}, 'must_succeed', $allow_cache_update); | ||
| 79 | $medium->{url} = _add__with_dir($url, $medium->{'with-dir'}); | ||
| 80 | } | ||
| 81 | |||
| 82 | #- side-effects: | ||
| 83 | #- + those of _pick_one_ ($urpm->{mirrors_cache}) | ||
| 84 | sub list_urls { | ||
| 85 | my ($urpm, $medium, $allow_cache_update) = @_; | ||
| 86 | |||
| 87 | my @l = split(' ', $medium->{mirrorlist}); | ||
| 88 | map { | ||
| 89 | my $cache = _pick_one_($urpm, $_, $allow_cache_update, $_ ne $l[-1]); | ||
| 90 | $cache ? $cache->{list} : []; | ||
| 91 | } @l; | ||
| 92 | } | ||
| 93 | |||
| 94 | #- side-effects: $urpm->{mirrors_cache} | ||
| 95 | sub _pick_one { | ||
| 96 | my ($urpm, $mirrorlists, $must_succeed, $allow_cache_update) = @_; | ||
| 97 | |||
| 98 | my @l = split(' ', $mirrorlists); | ||
| 99 | foreach my $mirrorlist (@l) { | ||
| 100 | if (my $cache = _pick_one_($urpm, $mirrorlist, $allow_cache_update, $mirrorlist ne $l[-1])) { | ||
| 101 | |||
| 102 | if ($cache->{nb_uses}++) { | ||
| 103 | $urpm->{debug} and $urpm->{debug}("using mirror $cache->{chosen}"); | ||
| 104 | } else { | ||
| 105 | $urpm->{log}("using mirror $cache->{chosen}"); | ||
| 106 | } | ||
| 107 | |||
| 108 | return $cache->{chosen}; | ||
| 109 | } | ||
| 110 | } | ||
| 111 | $must_succeed and $urpm->{fatal}(10, N("Could not find a mirror from mirrorlist %s", $mirrorlists)); | ||
| 112 | undef; | ||
| 113 | } | ||
| 114 | |||
| 115 | #- side-effects: $urpm->{mirrors_cache} | ||
| 116 | sub _pick_one_ { | ||
| 117 | my ($urpm, $mirrorlist, $allow_cache_update, $set_network_mtime) = @_; | ||
| 118 | |||
| 119 | my $cache = _cache__may_clean_if_outdated($urpm, $mirrorlist, $allow_cache_update); | ||
| 120 | |||
| 121 | if (!$cache->{chosen}) { | ||
| 122 | if (!$cache->{list}) { | ||
| 123 | if (_is_only_one_mirror($mirrorlist)) { | ||
| 124 | $cache->{list} = [ { url => $mirrorlist } ]; | ||
| 125 | } else { | ||
| 126 | $cache->{list} = [ _list($urpm, $mirrorlist) ]; | ||
| 127 | } | ||
| 128 | $cache->{time} = time(); | ||
| 129 | |||
| 130 | # the cache will be deemed outdated if network_mtime is more recent than the cache's | ||
| 131 | $cache->{network_mtime} = _network_mtime() if $set_network_mtime; | ||
| 132 | $cache->{product_id_mtime} = _product_id_mtime(); | ||
| 133 | } | ||
| 134 | |||
| 135 | if (-x '/usr/bin/rsync') { | ||
| 136 | $cache->{chosen} = $cache->{list}[0]{url}; | ||
| 137 | } else { | ||
| 138 | my $m = find { $_->{url} !~ m!^rsync://! } @{$cache->{list}}; | ||
| 139 | $cache->{chosen} = $m->{url}; | ||
| 140 | } | ||
| 141 | $cache->{chosen} or return; | ||
| 142 | _save_cache($urpm); | ||
| 143 | } | ||
| 144 | $cache; | ||
| 145 | } | ||
| 146 | #- side-effects: $urpm->{mirrors_cache} | ||
| 147 | sub black_list { | ||
| 148 | my ($urpm, $mirrorlists, $url) = @_; | ||
| 149 | foreach my $mirrorlist (split ' ', $mirrorlists) { | ||
| 150 | my $cache = _cache($urpm, $mirrorlist); | ||
| 151 | |||
| 152 | if ($cache->{list}) { | ||
| 153 | @{$cache->{list}} = grep { $_->{url} ne $url } @{$cache->{list}}; | ||
| 154 | } | ||
| 155 | delete $cache->{chosen}; | ||
| 156 | } | ||
| 157 | } | ||
| 158 | |||
| 159 | sub _trigger_cache_update { | ||
| 160 | my ($urpm, $cache, $o_is_upgrade) = @_; | ||
| 161 | |||
| 162 | my $reason = $o_is_upgrade ? "reason=upgrade" : "reason=update"; | ||
| 163 | $urpm->{log}("URPMI_ADDMEDIA_REASON $reason"); | ||
| 164 | $ENV{URPMI_ADDMEDIA_REASON} = $reason; | ||
| 165 | %$cache = (); | ||
| 166 | } | ||
| 167 | |||
| 168 | #- side-effects: | ||
| 169 | #- + those of _cache ($urpm->{mirrors_cache}) | ||
| 170 | sub _cache__may_clean_if_outdated { | ||
| 171 | my ($urpm, $mirrorlist, $allow_cache_update) = @_; | ||
| 172 | |||
| 173 | my $cache = _cache($urpm, $mirrorlist); | ||
| 174 | |||
| 175 | if ($allow_cache_update) { | ||
| 176 | if ($cache->{network_mtime} && _network_mtime() > $cache->{network_mtime}) { | ||
| 177 | $urpm->{log}("not using cached mirror list $mirrorlist since network configuration changed"); | ||
| 178 | _trigger_cache_update($urpm, $cache); | ||
| 179 | } elsif ($cache->{time} && | ||
| 180 | time() > $cache->{time} + 24*60*60 * $urpm->{options}{'days-between-mirrorlist-update'}) { | ||
| 181 | $urpm->{log}("not using outdated cached mirror list $mirrorlist"); | ||
| 182 | _trigger_cache_update($urpm, $cache); | ||
| 183 | } elsif (!$cache->{product_id_mtime}) { | ||
| 184 | $urpm->{log}("cached mirror list uses an old format, invalidating it"); | ||
| 185 | _trigger_cache_update($urpm, $cache, 1); | ||
| 186 | } elsif ($cache->{product_id_mtime} && _product_id_mtime() != $cache->{product_id_mtime}) { | ||
| 187 | $urpm->{log}("not using cached mirror list $mirrorlist since product id file changed"); | ||
| 188 | _trigger_cache_update($urpm, $cache, 1); | ||
| 189 | } | ||
| 190 | } | ||
| 191 | $cache; | ||
| 192 | } | ||
| 193 | |||
| 194 | #- side-effects: $urpm->{mirrors_cache} | ||
| 195 | sub _cache { | ||
| 196 | my ($urpm, $mirrorlist) = @_; | ||
| 197 | my $full_cache = $urpm->{mirrors_cache} ||= _load_cache($urpm); | ||
| 198 | $full_cache->{$mirrorlist} ||= {}; | ||
| 199 | } | ||
| 200 | sub cache_file { | ||
| 201 | my ($urpm) = @_; | ||
| 202 | "$urpm->{cachedir}/mirrors.cache"; | ||
| 203 | } | ||
| 204 | sub _load_cache { | ||
| 205 | my ($urpm) = @_; | ||
| 206 | my $cache; | ||
| 207 | if (-e cache_file($urpm)) { | ||
| 208 | $urpm->{debug} and $urpm->{debug}("loading mirrors cache"); | ||
| 209 | $cache = eval(cat_(cache_file($urpm))); | ||
| 210 | $@ and $urpm->{error}("failed to read " . cache_file($urpm) . ": $@"); | ||
| 211 | $_->{nb_uses} = 0 foreach values %$cache; | ||
| 212 | } | ||
| 213 | if ($ENV{URPMI_ADDMEDIA_PRODUCT_VERSION} && delete $cache->{'$MIRRORLIST'}) { | ||
| 214 | $urpm->{log}('not using cached mirror list $MIRRORLIST since URPMI_ADDMEDIA_PRODUCT_VERSION is set'); | ||
| 215 | } | ||
| 216 | $cache || {}; | ||
| 217 | } | ||
| 218 | sub _save_cache { | ||
| 219 | my ($urpm) = @_; | ||
| 220 | require Data::Dumper; | ||
| 221 | my $s = Data::Dumper::Dumper($urpm->{mirrors_cache}); | ||
| 222 | $s =~ s/.*?=//; # get rid of $VAR1 = | ||
| 223 | output_safe(cache_file($urpm), $s); | ||
| 224 | } | ||
| 225 | |||
| 226 | #- side-effects: none | ||
| 227 | sub _list { | ||
| 228 | my ($urpm, $mirrorlist) = @_; | ||
| 229 | |||
| 230 | my @mirrors = _mirrors_filtered($urpm, _expand($mirrorlist)); | ||
| 231 | add_proximity_and_sort($urpm, \@mirrors); | ||
| 232 | @mirrors; | ||
| 233 | } | ||
| 234 | |||
| 235 | sub _expand { | ||
| 236 | my ($mirrorlist) = @_; | ||
| 237 | |||
| 238 | # expand the variables | ||
| 239 | |||
| 240 | if ($mirrorlist eq '$MIRRORLIST') { | ||
| 241 | _MIRRORLIST(); | ||
| 242 | } else { | ||
| 243 | require urpm::cfg; | ||
| 244 | urpm::cfg::expand_line($mirrorlist); | ||
| 245 | } | ||
| 246 | } | ||
| 247 | |||
| 248 | #- side-effects: $mirrors | ||
| 249 | sub add_proximity_and_sort { | ||
| 250 | my ($urpm, $mirrors) = @_; | ||
| 251 | |||
| 252 | my ($latitude, $longitude, $country_code); | ||
| 253 | |||
| 254 | require Time::ZoneInfo; | ||
| 255 | if (my $zone = Time::ZoneInfo->current_zone) { | ||
| 256 | if (my $zones = Time::ZoneInfo->new) { | ||
| 257 | if (($latitude, $longitude) = $zones->latitude_longitude_decimal($zone)) { | ||
| 258 | $country_code = $zones->country($zone); | ||
| 259 | $urpm->{log}(N("found geolocalisation %s %.2f %.2f from timezone %s", $country_code, $latitude, $longitude, $zone)); | ||
| 260 | } | ||
| 261 | } | ||
| 262 | } | ||
| 263 | defined $latitude && defined $longitude or return; | ||
| 264 | |||
| 265 | foreach (@$mirrors) { | ||
| 266 | $_->{latitude} || $_->{longitude} or next; | ||
| 267 | my $PI = 3.14159265358979; | ||
| 268 | my $x = $latitude - $_->{latitude}; | ||
| 269 | my $y = ($longitude - $_->{longitude}) * cos($_->{latitude} / 180 * $PI); | ||
| 270 | $_->{proximity} = sqrt($x * $x + $y * $y); | ||
| 271 | } | ||
| 272 | my ($best) = sort { $a->{proximity} <=> $b->{proximity} } @$mirrors; | ||
| 273 | |||
| 274 | foreach (@$mirrors) { | ||
| 275 | $_->{proximity_corrected} = $_->{proximity} * _random_correction(); | ||
| 276 | $_->{proximity_corrected} *= _between_country_correction($country_code, $_->{country}) if $best; | ||
| 277 | $_->{proximity_corrected} *= _between_continent_correction($best->{continent}, $_->{continent}) if $best; | ||
| 278 | $_->{proximity_corrected} *= _protocol_correction($_->{url}); | ||
| 279 | } | ||
| 280 | |||
| 281 | @$mirrors = sort { $a->{proximity_corrected} <=> $b->{proximity_corrected} } @$mirrors; | ||
| 282 | |||
| 283 | #foreach (@$mirrors) { | ||
| 284 | # print $_->{proximity_corrected} . " " . $_->{proximity} . " " . $_->{url} . "\n"; | ||
| 285 | #} | ||
| 286 | } | ||
| 287 | |||
| 288 | # add +/- 5% random | ||
| 289 | sub _random_correction() { | ||
| 290 | my $correction = 0.05; | ||
| 291 | 1 + (rand() - 0.5) * $correction * 2; | ||
| 292 | } | ||
| 293 | |||
| 294 | sub _between_country_correction { | ||
| 295 | my ($here, $mirror) = @_; | ||
| 296 | $here && $mirror or return 1; | ||
| 297 | $here eq $mirror ? 0.5 : 1; | ||
| 298 | } | ||
| 299 | |||
| 300 | sub _between_continent_correction { | ||
| 301 | my ($here, $mirror) = @_; | ||
| 302 | $here && $mirror or return 1; | ||
| 303 | $here eq $mirror ? 0.5 : # favor same continent | ||
| 304 | $here eq 'SA' && $mirror eq 'NA' ? 0.9 : # favor going "South America" -> "North America" | ||
| 305 | 1; | ||
| 306 | } | ||
| 307 | |||
| 308 | sub _protocol_correction { | ||
| 309 | my ($url) = @_; | ||
| 310 | # favor encrypted protocols, then http | ||
| 311 | ( $url =~ m!https://! ) and return 0.7; | ||
| 312 | ( $url =~ m!ftps://! ) and return 0.8; | ||
| 313 | ( $url =~ m!http://! ) and return 0.9; | ||
| 314 | 1; | ||
| 315 | } | ||
| 316 | |||
| 317 | sub _mirrors_raw { | ||
| 318 | my ($urpm, $url) = @_; | ||
| 319 | |||
| 320 | $urpm->{log}(N("getting mirror list from %s", $url)); | ||
| 321 | my @l = urpm::download::get_content($urpm, $url, disable_metalink => 1) or $urpm->{error}("mirror list not found"); | ||
| 322 | @l; | ||
| 323 | } | ||
| 324 | |||
| 325 | sub _mirrors_filtered { | ||
| 326 | my ($urpm, $mirrorlist) = @_; | ||
| 327 | |||
| 328 | grep { | ||
| 329 | $_->{type} eq 'distrib'; # type=updates seems to be history, and type=iso is not interesting here | ||
| 330 | } map { chomp; parse_LDAP_namespace_structure($_) } _mirrors_raw($urpm, $mirrorlist); | ||
| 331 | } | ||
| 332 | |||
| 333 | sub _MIRRORLIST() { | ||
| 334 | my $product_id = parse_LDAP_namespace_structure(cat_('/etc/product.id')); | ||
| 335 | _mageia_mirrorlist($product_id); | ||
| 336 | } | ||
| 337 | sub _mageia_mirrorlist { | ||
| 338 | my ($product_id, $o_arch) = @_; | ||
| 339 | |||
| 340 | #- contact the following URL to retrieve the list of mirrors. | ||
| 341 | #- https://wiki.mageia.org/en/Product_id | ||
| 342 | my $_product_type = lc($product_id->{type}); $product_id =~ s/\s//g; | ||
| 343 | my $arch = $o_arch || $product_id->{arch}; | ||
| 344 | |||
| 345 | my @para = grep { $_ } $ENV{URPMI_ADDMEDIA_REASON}; | ||
| 346 | my $product_version = $ENV{URPMI_ADDMEDIA_PRODUCT_VERSION} || $product_id->{version}; | ||
| 347 | |||
| 348 | #"https://mirrors.mageia.org/api/$product_type.$product_version.$arch.list" | ||
| 349 | "https://mirrors.mageia.org/api/mageia.$product_version.$arch.list" | ||
| 350 | . (@para ? '?' . join('&', @para) : ''); | ||
| 351 | } | ||
| 352 | |||
| 353 | #- heuristic to detect wether it is really a mirrorlist or a simple mirror url: | ||
| 354 | sub _is_only_one_mirror { | ||
| 355 | my ($mirrorlist) = @_; | ||
| 356 | _expand($mirrorlist) !~ /\.list(\?|$)/; | ||
| 357 | } | ||
| 358 | |||
| 359 | sub _network_mtime() { (stat('/etc/resolv.conf'))[9] } | ||
| 360 | sub _product_id_mtime() { (stat('/etc/product.id'))[9] } | ||
| 361 | |||
| 362 | sub parse_LDAP_namespace_structure { | ||
| 363 | my ($s) = @_; | ||
| 364 | my %h = map { /(.*?)=(.*)/ ? ($1 => $2) : @{[]} } split(',', $s); | ||
| 365 | \%h; | ||
| 366 | } | ||
| 367 | |||
| 368 | 1; | ||
| 369 | |||
| 370 | |||
| 371 | =back | ||
| 372 | |||
| 373 | =head1 COPYRIGHT | ||
| 374 | |||
| 375 | Copyright (C) 2005 MandrakeSoft SA | ||
| 376 | |||
| 377 | Copyright (C) 2005-2010 Mandriva SA | ||
| 378 | |||
| 379 | Copyright (C) 2011-2020 Mageia | ||
| 380 | |||
| 381 | =cut | ||