| 1 | package urpm::download; | ||
| 2 | |||
| 3 | |||
| 4 | 2 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 5 | 0 | ▶ | use urpm::msg; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 6 | use urpm::util qw(cat_ basename dirname file_size max member output_safe reduce_pathname); | ||
| 7 | use bytes (); | ||
| 8 | use Cwd; | ||
| 9 | use Exporter; | ||
| 10 | # perl_checker: require urpm | ||
| 11 | |||
| 12 | # help perl_checker: | ||
| 13 | sub getcwd { goto &Cwd::getcwd } | ||
| 14 | |||
| 15 | our @ISA = 'Exporter'; | ||
| 16 | our @EXPORT = qw(get_proxy | ||
| 17 | propagate_sync_callback | ||
| 18 | sync_file sync_rsync sync_ssh | ||
| 19 | set_proxy_config dump_proxy_config | ||
| 20 | ); | ||
| 21 | |||
| 22 | #- proxy config file. | ||
| 23 | our $PROXY_CFG = '/etc/urpmi/proxy.cfg'; | ||
| 24 | my $proxy_config; | ||
| 25 | |||
| 26 | #- Timeout for curl connection and wget operations | ||
| 27 | our $CONNECT_TIMEOUT = 60; #- (in seconds) | ||
| 28 | |||
| 29 | |||
| 30 | =head1 NAME | ||
| 31 | |||
| 32 | urpm::download - download routines for the urpm* tools | ||
| 33 | |||
| 34 | =head1 SYNOPSIS | ||
| 35 | |||
| 36 | =head1 DESCRIPTION | ||
| 37 | |||
| 38 | =over | ||
| 39 | |||
| 40 | =cut | ||
| 41 | |||
| 42 | |||
| 43 | sub ftp_http_downloaders() { qw(wget curl prozilla aria2) } | ||
| 44 | |||
| 45 | sub available_ftp_http_downloaders() { | ||
| 46 | my %binaries = ( | ||
| 47 | wget => 'wget', | ||
| 48 | curl => 'curl', | ||
| 49 | prozilla => 'proz', | ||
| 50 | aria2 => 'aria2c', | ||
| 51 | ); | ||
| 52 | grep { -x "/usr/bin/$binaries{$_}" || -x "/bin/$binaries{$_}" } ftp_http_downloaders(); | ||
| 53 | } | ||
| 54 | |||
| 55 | sub metalink_downloaders() { qw(aria2) } | ||
| 56 | |||
| 57 | sub available_metalink_downloaders() { | ||
| 58 | my %binaries = ( | ||
| 59 | aria2 => 'aria2c', | ||
| 60 | ); | ||
| 61 | grep { -x "/usr/bin/$binaries{$_}" || -x "/bin/$binaries{$_}" } metalink_downloaders(); | ||
| 62 | } | ||
| 63 | |||
| 64 | sub use_metalink { | ||
| 65 | my ($urpm, $medium) = @_; | ||
| 66 | |||
| 67 | $medium->{allow_metalink} //= do { | ||
| 68 | my $use_metalink = 1; | ||
| 69 | preferred_downloader($urpm, $medium, \$use_metalink); | ||
| 70 | $use_metalink; | ||
| 71 | }; | ||
| 72 | } | ||
| 73 | |||
| 74 | my %warned; | ||
| 75 | sub preferred_downloader { | ||
| 76 | my ($urpm, $medium, $use_metalink) = @_; | ||
| 77 | |||
| 78 | my @available = available_ftp_http_downloaders(); | ||
| 79 | my @metalink_downloaders = available_metalink_downloaders(); | ||
| 80 | my $metalink_disabled = !$$use_metalink && $medium->{disable_metalink}; | ||
| 81 | |||
| 82 | if ($$use_metalink && !$metalink_disabled) { | ||
| 83 | #- If metalink is used, only aria2 is available as other downloaders doesn't support metalink | ||
| 84 | unshift @available, @metalink_downloaders; | ||
| 85 | } | ||
| 86 | |||
| 87 | #- first downloader of @available is the default one | ||
| 88 | my $preferred = $available[0]; | ||
| 89 | my $requested_downloader = requested_ftp_http_downloader($urpm, $medium); | ||
| 90 | if ($requested_downloader) { | ||
| 91 | if (member($requested_downloader, @available)) { | ||
| 92 | #- use user default downloader if provided and available | ||
| 93 | $preferred = $requested_downloader; | ||
| 94 | } elsif ($warned{webfetch_not_available}++ == 0) { | ||
| 95 | $urpm->{log}(N("%s is not available, falling back on %s", $requested_downloader, $preferred)); | ||
| 96 | } | ||
| 97 | } | ||
| 98 | |||
| 99 | if ($$use_metalink && !member($preferred, @metalink_downloaders)) { | ||
| 100 | $warned{not_using_metalink}++ or | ||
| 101 | $urpm->{log}($requested_downloader eq $preferred ? | ||
| 102 | "not using metalink since requested downloader does not handle it" : | ||
| 103 | "not using metalink since no downloaders handling metalink are available"); | ||
| 104 | $$use_metalink = 0; | ||
| 105 | } | ||
| 106 | $preferred; | ||
| 107 | } | ||
| 108 | |||
| 109 | sub parse_http_proxy { | ||
| 110 | $_[0] =~ m!^(?:http://)?([^:/]+(:\d+)?)/*$!; | ||
| 111 | } | ||
| 112 | |||
| 113 | #- parses proxy.cfg (private) | ||
| 114 | sub load_proxy_config () { | ||
| 115 | return if defined $proxy_config; | ||
| 116 | $proxy_config = {}; | ||
| 117 | foreach (cat_($PROXY_CFG)) { | ||
| 118 | chomp; s/#.*$//; s/^\s*//; s/\s*$//; | ||
| 119 | if (/^(?:(.*):\s*)?(ftp_proxy|http_proxy)\s*=\s*(.*)$/) { | ||
| 120 | $proxy_config->{$1 || ''}{$2} = $3; | ||
| 121 | next; | ||
| 122 | } | ||
| 123 | if (/^(?:(.*):\s*)?proxy_user\s*=\s*([^:]*)(?::(.*))?$/) { | ||
| 124 | $proxy_config->{$1 || ''}{user} = $2; | ||
| 125 | $proxy_config->{$1 || ''}{pwd} = $3 if defined $3; | ||
| 126 | next; | ||
| 127 | } | ||
| 128 | if (/^(?:(.*):\s*)?proxy_user_ask/) { | ||
| 129 | $proxy_config->{$1 || ''}{ask} = 1; | ||
| 130 | next; | ||
| 131 | } | ||
| 132 | } | ||
| 133 | } | ||
| 134 | |||
| 135 | #- writes proxy.cfg | ||
| 136 | sub dump_proxy_config () { | ||
| 137 | $proxy_config or return 0; #- hasn't been read yet | ||
| 138 | |||
| 139 | my $has_password; | ||
| 140 | |||
| 141 | open my $f, '>', $PROXY_CFG or return 0; | ||
| 142 | foreach ('', sort grep { !/^(|cmd_line)$/ } keys %$proxy_config) { | ||
| 143 | my $m = $_ eq '' ? '' : "$_:"; | ||
| 144 | my $p = $proxy_config->{$_}; | ||
| 145 | foreach (qw(http_proxy ftp_proxy)) { | ||
| 146 | if (defined $p->{$_} && $p->{$_} ne '') { | ||
| 147 | print $f "$m$_=$p->{$_}\n"; | ||
| 148 | $has_password ||= hide_password($p->{$_}) ne $p->{$_}; | ||
| 149 | } | ||
| 150 | } | ||
| 151 | if ($p->{ask}) { | ||
| 152 | print $f "${m}proxy_user_ask\n"; | ||
| 153 | } elsif (defined $p->{user} && $p->{user} ne '') { | ||
| 154 | print $f "${m}proxy_user=$p->{user}:$p->{pwd}\n"; | ||
| 155 | $has_password ||= $p->{pwd}; | ||
| 156 | } | ||
| 157 | } | ||
| 158 | close $f; | ||
| 159 | chmod 0600, $PROXY_CFG if $has_password; | ||
| 160 | return 1; | ||
| 161 | } | ||
| 162 | |||
| 163 | #- deletes the proxy configuration for the specified media | ||
| 164 | sub remove_proxy_media { | ||
| 165 | defined $proxy_config and delete $proxy_config->{$_[0] || ''}; | ||
| 166 | } | ||
| 167 | |||
| 168 | sub get_proxy_ { | ||
| 169 | my ($urpm) = @_; | ||
| 170 | |||
| 171 | -e $PROXY_CFG && !-r $PROXY_CFG and $urpm->{error}(N("can not read proxy settings (not enough rights to read %s)", $PROXY_CFG)); | ||
| 172 | |||
| 173 | get_proxy($urpm); | ||
| 174 | } | ||
| 175 | |||
| 176 | =item get_proxy($media) | ||
| 177 | |||
| 178 | Reads and loads the proxy.cfg file ; | ||
| 179 | Returns the global proxy settings (without arguments) or the | ||
| 180 | proxy settings for the specified media (with a media name as argument) | ||
| 181 | |||
| 182 | =cut | ||
| 183 | |||
| 184 | sub get_proxy (;$) { | ||
| 185 | my ($o_media) = @_; $o_media ||= ''; | ||
| 186 | load_proxy_config(); | ||
| 187 | my $p = $proxy_config->{cmd_line} | ||
| 188 | || $proxy_config->{$o_media} | ||
| 189 | || $proxy_config->{''} | ||
| 190 | || { | ||
| 191 | http_proxy => undef, | ||
| 192 | ftp_proxy => undef, | ||
| 193 | user => undef, | ||
| 194 | pwd => undef, | ||
| 195 | }; | ||
| 196 | if ($p->{ask} && ($p->{http_proxy} || $p->{ftp_proxy}) && !$p->{user}) { | ||
| 197 | our $PROMPT_PROXY; | ||
| 198 | unless (defined $PROMPT_PROXY) { | ||
| 199 | require urpm::prompt; | ||
| 200 | $PROMPT_PROXY = urpm::prompt->new( | ||
| 201 | N("Please enter your credentials for accessing proxy\n"), | ||
| 202 | [ N("User name:"), N("Password:") ], | ||
| 203 | undef, | ||
| 204 | [ 0, 1 ], | ||
| 205 | ); | ||
| 206 | } | ||
| 207 | ($p->{user}, $p->{pwd}) = $PROMPT_PROXY->prompt; | ||
| 208 | } | ||
| 209 | $p; | ||
| 210 | } | ||
| 211 | |||
| 212 | #- copies the settings for proxies from the command line to media named $media | ||
| 213 | #- and writes the proxy.cfg file (used when adding new media) | ||
| 214 | sub copy_cmd_line_proxy { | ||
| 215 | my ($media) = @_; | ||
| 216 | return unless $media; | ||
| 217 | load_proxy_config(); | ||
| 218 | if (defined $proxy_config->{cmd_line}) { | ||
| 219 | $proxy_config->{$media} = $proxy_config->{cmd_line}; | ||
| 220 | dump_proxy_config(); | ||
| 221 | } else { | ||
| 222 | #- use default if available | ||
| 223 | $proxy_config->{$media} = $proxy_config->{''}; | ||
| 224 | } | ||
| 225 | } | ||
| 226 | |||
| 227 | =item set_cmdline_proxy(%h) | ||
| 228 | |||
| 229 | Overrides the config file proxy settings with values passed via command-line | ||
| 230 | |||
| 231 | =cut | ||
| 232 | |||
| 233 | sub set_cmdline_proxy { | ||
| 234 | my (%h) = @_; | ||
| 235 | load_proxy_config(); | ||
| 236 | $proxy_config->{cmd_line} ||= { | ||
| 237 | http_proxy => undef, | ||
| 238 | ftp_proxy => undef, | ||
| 239 | user => undef, | ||
| 240 | pwd => undef, | ||
| 241 | }; | ||
| 242 | $proxy_config->{cmd_line}{$_} = $h{$_} foreach keys %h; | ||
| 243 | } | ||
| 244 | |||
| 245 | =item set_proxy_config($key, $value, $o_media) | ||
| 246 | |||
| 247 | Changes permanently the proxy settings | ||
| 248 | |||
| 249 | =cut | ||
| 250 | |||
| 251 | sub set_proxy_config { | ||
| 252 | my ($key, $value, $o_media) = @_; | ||
| 253 | $proxy_config->{$o_media || ''}{$key} = $value; | ||
| 254 | } | ||
| 255 | |||
| 256 | #- set up the environment for proxy usage for the appropriate tool. | ||
| 257 | #- returns an array of command-line arguments for wget or curl. | ||
| 258 | sub set_proxy { | ||
| 259 | my ($proxy) = @_; | ||
| 260 | |||
| 261 | my $p = $proxy->{proxy}; | ||
| 262 | defined $p->{http_proxy} || defined $p->{ftp_proxy} or return; | ||
| 263 | |||
| 264 | my @res; | ||
| 265 | if ($proxy->{type} =~ /\bwget\b/) { | ||
| 266 | if (defined $p->{http_proxy}) { | ||
| 267 | $ENV{http_proxy} = $p->{http_proxy} =~ /^http:/ | ||
| 268 | ? $p->{http_proxy} : "http://$p->{http_proxy}"; | ||
| 269 | } | ||
| 270 | $ENV{ftp_proxy} = $p->{ftp_proxy} if defined $p->{ftp_proxy}; | ||
| 271 | @res = ("--proxy-user=$p->{user}", "--proxy-passwd=$p->{pwd}") | ||
| 272 | if defined $p->{user} && defined $p->{pwd}; | ||
| 273 | } elsif ($proxy->{type} =~ /\bcurl\b/) { | ||
| 274 | push @res, ('-x', $p->{http_proxy}) if defined $p->{http_proxy}; | ||
| 275 | push @res, ('-x', $p->{ftp_proxy}) if defined $p->{ftp_proxy}; | ||
| 276 | push @res, ('-U', "$p->{user}:$p->{pwd}") | ||
| 277 | if defined $p->{user} && defined $p->{pwd}; | ||
| 278 | push @res, '-H', 'Pragma:' if @res; | ||
| 279 | } elsif ($proxy->{type} =~ /\baria2\b/) { | ||
| 280 | if (my ($http_proxy) = $p->{http_proxy} && parse_http_proxy($p->{http_proxy})) { | ||
| 281 | my $allproxy = $p->{user}; | ||
| 282 | $allproxy .= ":" . $p->{pwd} if $p->{pwd}; | ||
| 283 | $allproxy .= "@" if $p->{user}; | ||
| 284 | $allproxy .= $http_proxy; | ||
| 285 | @res = ("--all-proxy=http://$allproxy"); | ||
| 286 | } | ||
| 287 | } else { | ||
| 288 | die N("Unknown webfetch `%s' !!!\n", $proxy->{type}); | ||
| 289 | } | ||
| 290 | @res; | ||
| 291 | } | ||
| 292 | |||
| 293 | sub _error_msg { | ||
| 294 | my ($name) = @_; | ||
| 295 | |||
| 296 | my $msg = $? & 127 ? N("%s failed: exited with signal %d", $name, $? & 127) : | ||
| 297 | N("%s failed: exited with %d", $name, $? >> 8); | ||
| 298 | "$msg\n"; | ||
| 299 | } | ||
| 300 | |||
| 301 | sub _error { | ||
| 302 | my ($name) = @_; | ||
| 303 | die _error_msg($name); | ||
| 304 | } | ||
| 305 | |||
| 306 | sub hide_password { | ||
| 307 | my ($url) = @_; | ||
| 308 | $url =~ s|([^:]*://[^/:\@]*:)[^/:\@]*(\@.*)|$1xxxx$2|; #- if needed... | ||
| 309 | $url; | ||
| 310 | } | ||
| 311 | |||
| 312 | sub propagate_sync_callback { | ||
| 313 | my $options = shift; | ||
| 314 | if (ref($options) && $options->{callback}) { | ||
| 315 | my $mode = shift; | ||
| 316 | if ($mode =~ /^(?:start|progress|end)$/) { | ||
| 317 | my $file = shift; | ||
| 318 | return $options->{callback}($mode, hide_password($file), @_); | ||
| 319 | } else { | ||
| 320 | return $options->{callback}($mode, @_); | ||
| 321 | } | ||
| 322 | } | ||
| 323 | } | ||
| 324 | |||
| 325 | sub sync_file { | ||
| 326 | my $options = shift; | ||
| 327 | foreach (@_) { | ||
| 328 | propagate_sync_callback($options, 'start', $_); | ||
| 329 | require urpm::util; | ||
| 330 | urpm::util::copy($_, ref($options) ? $options->{dir} : $options) | ||
| 331 | or die N("copy failed"); | ||
| 332 | propagate_sync_callback($options, 'end', $_); | ||
| 333 | } | ||
| 334 | } | ||
| 335 | |||
| 336 | sub sync_wget { | ||
| 337 | -x "/usr/bin/wget" or die N("wget is missing\n"); | ||
| 338 | my $options = shift; | ||
| 339 | $options = { dir => $options } if !ref $options; | ||
| 340 | #- force download to be done in cachedir to avoid polluting cwd. | ||
| 341 | (my $cwd) = getcwd() =~ /(.*)/; | ||
| 342 | chdir $options->{dir}; | ||
| 343 | my ($buf, $total, $file) = ('', undef, undef); | ||
| 344 | my $wget_command = join(" ", map { "'$_'" } | ||
| 345 | #- construction of the wget command-line | ||
| 346 | "/usr/bin/wget", | ||
| 347 | ($options->{'limit-rate'} ? "--limit-rate=$options->{'limit-rate'}" : @{[]}), | ||
| 348 | ($options->{resume} ? "--continue" : "--force-clobber"), | ||
| 349 | "--no-timestamping", #- timestamping conflicts with --force-clobber | ||
| 350 | ($options->{proxy} ? set_proxy({ type => "wget", proxy => $options->{proxy} }) : @{[]}), | ||
| 351 | ($options->{retry} ? ('-t', $options->{retry}) : @{[]}), | ||
| 352 | ($options->{callback} ? ("--progress=bar:force", "-o", "-") : | ||
| 353 | $options->{quiet} ? "-q" : @{[]}), | ||
| 354 | "--retr-symlinks", | ||
| 355 | ($options->{"no-certificate-check"} ? "--no-check-certificate" : @{[]}), | ||
| 356 | "--timeout=$CONNECT_TIMEOUT", | ||
| 357 | (defined $options->{'wget-options'} ? split /\s+/, $options->{'wget-options'} : @{[]}), | ||
| 358 | '-P', $options->{dir}, | ||
| 359 | @_ | ||
| 360 | ) . " |"; | ||
| 361 | $options->{debug} and $options->{debug}($wget_command); | ||
| 362 | local $ENV{LC_ALL} = 'C'; | ||
| 363 | my $wget_pid = open(my $wget, $wget_command); | ||
| 364 | local $/ = \1; #- read input by only one char, this is slow but very nice (and it works!). | ||
| 365 | local $_; | ||
| 366 | while (<$wget>) { | ||
| 367 | $buf .= $_; | ||
| 368 | if ($_ eq "\r" || $_ eq "\n") { | ||
| 369 | if ($options->{callback}) { | ||
| 370 | if ($buf =~ /^--(\d\d\d\d-\d\d-\d\d )?\d\d:\d\d:\d\d--\s+(\S.*)\n/ms) { | ||
| 371 | my $file_ = $2; | ||
| 372 | if ($file && $file ne $file_) { | ||
| 373 | propagate_sync_callback($options, 'end', $file); | ||
| 374 | undef $file; | ||
| 375 | } | ||
| 376 | ! defined $file and propagate_sync_callback($options, 'start', $file = $file_); | ||
| 377 | } elsif (defined $file && ! defined $total && ($buf =~ /==>\s+RETR/ || $buf =~ /200 OK$/)) { | ||
| 378 | $total = ''; | ||
| 379 | } elsif ($buf =~ /^Length:\s*(\d\S*)/) { | ||
| 380 | $total = $1; | ||
| 381 | } elsif (defined $total && $buf =~ m!^\s*(\d+)%.*\s+(\S+/s)\s+((ETA|eta)\s+(.*?)\s*)?[\r\n]$!ms) { | ||
| 382 | my ($percent, $speed, $eta) = ($1, $2, $5); | ||
| 383 | if (propagate_sync_callback($options, 'progress', $file, $percent, $total, $eta, $speed) eq 'canceled') { | ||
| 384 | kill 15, $wget_pid; | ||
| 385 | close $wget; | ||
| 386 | return; | ||
| 387 | } | ||
| 388 | if ($_ eq "\n") { | ||
| 389 | propagate_sync_callback($options, 'end', $file); | ||
| 390 | ($total, $file) = (undef, undef); | ||
| 391 | } | ||
| 392 | } | ||
| 393 | } else { | ||
| 394 | $options->{quiet} or print STDERR $buf; | ||
| 395 | } | ||
| 396 | $buf = ''; | ||
| 397 | } | ||
| 398 | } | ||
| 399 | $file and propagate_sync_callback($options, 'end', $file); | ||
| 400 | chdir $cwd; | ||
| 401 | close $wget or _error('wget'); | ||
| 402 | } | ||
| 403 | |||
| 404 | sub sync_curl { | ||
| 405 | -x "/usr/bin/curl" or die N("curl is missing\n"); | ||
| 406 | my $options = shift; | ||
| 407 | $options = { dir => $options } if !ref $options; | ||
| 408 | if (defined $options->{'limit-rate'} && $options->{'limit-rate'} =~ /\d$/) { | ||
| 409 | #- use bytes by default | ||
| 410 | $options->{'limit-rate'} .= 'B'; | ||
| 411 | } | ||
| 412 | #- force download to be done in cachedir to avoid polluting cwd, | ||
| 413 | #- however for curl, this is mandatory. | ||
| 414 | (my $cwd) = getcwd() =~ /(.*)/; | ||
| 415 | chdir($options->{dir}); | ||
| 416 | my (@ftp_files, @other_files); | ||
| 417 | foreach (@_) { | ||
| 418 | my ($proto, $nick, $rest) = m,^(http|ftp)://([^:/]+):(.*),,; | ||
| 419 | if ($nick) { #- escape @ in user names | ||
| 420 | $nick =~ s/@/%40/; | ||
| 421 | $_ = "$proto://$nick:$rest"; | ||
| 422 | } | ||
| 423 | if (m|^ftp://.*/([^/]*)$| && file_size($1) > 8192) { #- manage time stamp for large file only | ||
| 424 | push @ftp_files, $_; | ||
| 425 | } else { | ||
| 426 | push @other_files, $_; | ||
| 427 | } | ||
| 428 | } | ||
| 429 | if (@ftp_files) { | ||
| 430 | my ($cur_ftp_file, %ftp_files_info); | ||
| 431 | local $_; | ||
| 432 | |||
| 433 | eval { require Date::Manip }; | ||
| 434 | |||
| 435 | #- prepare to get back size and time stamp of each file. | ||
| 436 | my $cmd = join(" ", map { "'$_'" } "/usr/bin/curl", | ||
| 437 | "-q", # don't read .curlrc; some toggle options might interfer | ||
| 438 | ($options->{'limit-rate'} ? ("--limit-rate", $options->{'limit-rate'}) : @{[]}), | ||
| 439 | ($options->{proxy} ? set_proxy({ type => "curl", proxy => $options->{proxy} }) : @{[]}), | ||
| 440 | ($options->{retry} ? ('--retry', $options->{retry}) : @{[]}), | ||
| 441 | "--stderr", "-", # redirect everything to stdout | ||
| 442 | "--disable-epsv", | ||
| 443 | "--connect-timeout", $CONNECT_TIMEOUT, | ||
| 444 | "-s", "-I", | ||
| 445 | "--anyauth", | ||
| 446 | (defined $options->{'curl-options'} ? split /\s+/, $options->{'curl-options'} : @{[]}), | ||
| 447 | @ftp_files); | ||
| 448 | $options->{debug} and $options->{debug}($cmd); | ||
| 449 | open my $curl, "$cmd |"; | ||
| 450 | while (<$curl>) { | ||
| 451 | if (/Content-Length:\s*(\d+)/) { | ||
| 452 | !$cur_ftp_file || exists($ftp_files_info{$cur_ftp_file}{size}) | ||
| 453 | and $cur_ftp_file = shift @ftp_files; | ||
| 454 | $ftp_files_info{$cur_ftp_file}{size} = $1; | ||
| 455 | } | ||
| 456 | if (/Last-Modified:\s*(.*)/) { | ||
| 457 | !$cur_ftp_file || exists($ftp_files_info{$cur_ftp_file}{time}) | ||
| 458 | and $cur_ftp_file = shift @ftp_files; | ||
| 459 | eval { | ||
| 460 | $ftp_files_info{$cur_ftp_file}{time} = Date::Manip::ParseDate($1); | ||
| 461 | }; | ||
| 462 | } | ||
| 463 | } | ||
| 464 | close $curl or _error('curl'); | ||
| 465 | |||
| 466 | #- now analyse size and time stamp according to what already exists here. | ||
| 467 | if (@ftp_files) { | ||
| 468 | #- re-insert back shifted element of ftp_files, because curl output above | ||
| 469 | #- has not been parsed correctly, so in doubt download them all. | ||
| 470 | push @ftp_files, keys %ftp_files_info; | ||
| 471 | } else { | ||
| 472 | #- for that, it should be clear ftp_files is empty... | ||
| 473 | #- elsewhere, the above work was useless. | ||
| 474 | foreach (keys %ftp_files_info) { | ||
| 475 | my ($lfile) = m|/([^/]*)$| or next; #- strange if we can't parse it correctly. | ||
| 476 | my $ltime = eval { Date::Manip::ParseDate(scalar gmtime((stat $1)[9])) }; | ||
| 477 | $ltime && -s $lfile == $ftp_files_info{$_}{size} && $ftp_files_info{$_}{time} eq $ltime | ||
| 478 | or push @ftp_files, $_; | ||
| 479 | } | ||
| 480 | } | ||
| 481 | } | ||
| 482 | #- http files (and other files) are correctly managed by curl wrt conditional download. | ||
| 483 | #- options for ftp files, -R (-O <file>)* | ||
| 484 | #- options for http files, -R (-O <file>)* | ||
| 485 | my $result; | ||
| 486 | if (my @all_files = ( | ||
| 487 | (map { (_curl_output_option($_), $_) } @ftp_files), | ||
| 488 | (map { m|/| ? (_curl_output_option($_), $_) : @{[]} } @other_files))) | ||
| 489 | { | ||
| 490 | my @l = (@ftp_files, @other_files); | ||
| 491 | my $cmd = join(" ", map { "'$_'" } "/usr/bin/curl", | ||
| 492 | "-q", # don't read .curlrc; some toggle options might interfer | ||
| 493 | ($options->{'limit-rate'} ? ("--limit-rate", $options->{'limit-rate'}) : @{[]}), | ||
| 494 | ($options->{resume} ? ("--continue-at", "-") : @{[]}), | ||
| 495 | ($options->{proxy} ? set_proxy({ type => "curl", proxy => $options->{proxy} }) : @{[]}), | ||
| 496 | ($options->{retry} ? ('--retry', $options->{retry}) : @{[]}), | ||
| 497 | ($options->{quiet} ? "-s" : @{[]}), | ||
| 498 | ($options->{"no-certificate-check"} ? "-k" : @{[]}), | ||
| 499 | "--location-trusted", | ||
| 500 | "-R", | ||
| 501 | "-f", | ||
| 502 | "--disable-epsv", | ||
| 503 | "--connect-timeout", $CONNECT_TIMEOUT, | ||
| 504 | "--anyauth", | ||
| 505 | (defined $options->{'curl-options'} ? split /\s+/, $options->{'curl-options'} : @{[]}), | ||
| 506 | "--stderr", "-", # redirect everything to stdout | ||
| 507 | @all_files); | ||
| 508 | $options->{debug} and $options->{debug}($cmd); | ||
| 509 | $result = _curl_action($cmd, $options, @l); | ||
| 510 | } | ||
| 511 | chdir $cwd; | ||
| 512 | $result; | ||
| 513 | } | ||
| 514 | |||
| 515 | sub _curl_output_option { | ||
| 516 | my ($remote_file) = @_; | ||
| 517 | #- curl 7.86 changed the -O option to remove any trailing query part from the output | ||
| 518 | #- file name. So if we have a query part, use the -o option instead. If not, use the | ||
| 519 | #- -O option to avoid bloating the command line. | ||
| 520 | my $local_file = basename($remote_file); | ||
| 521 | $local_file =~ /\?/ ? ('-o', $local_file) : '-O'; | ||
| 522 | } | ||
| 523 | |||
| 524 | sub _curl_action { | ||
| 525 | my ($cmd, $options, @l) = @_; | ||
| 526 | |||
| 527 | my ($buf, $file); $buf = ''; | ||
| 528 | my $curl_pid = open(my $curl, "$cmd |"); | ||
| 529 | local $/ = \1; #- read input by only one char, this is slow but very nice (and it works!). | ||
| 530 | local $_; | ||
| 531 | while (<$curl>) { | ||
| 532 | $buf .= $_; | ||
| 533 | if ($_ eq "\r" || $_ eq "\n") { | ||
| 534 | if ($options->{callback}) { | ||
| 535 | unless (defined $file) { | ||
| 536 | $file = shift @l; | ||
| 537 | propagate_sync_callback($options, 'start', $file); | ||
| 538 | } | ||
| 539 | if (my ($percent, $total, $eta, $speed) = $buf =~ /^\s*(\d+)\s+(\S+)[^\r\n]*\s+(\S+)\s+(\S+)\s*[\r\n]$/ms) { | ||
| 540 | $speed =~ s/^-//; | ||
| 541 | if (propagate_sync_callback($options, 'progress', $file, $percent, $total, $eta, $speed) eq 'canceled') { | ||
| 542 | kill 15, $curl_pid; | ||
| 543 | close $curl; | ||
| 544 | die N("curl failed: download canceled\n"); | ||
| 545 | } | ||
| 546 | #- this checks that download has actually started | ||
| 547 | if ($_ eq "\n" | ||
| 548 | && !($speed == 0 && $percent == 100 && index($eta, '--') >= 0) #- work around bug 13685 | ||
| 549 | ) { | ||
| 550 | propagate_sync_callback($options, 'end', $file); | ||
| 551 | $file = undef; | ||
| 552 | } | ||
| 553 | } elsif ($buf =~ /^curl:/) { #- likely to be an error reported by curl | ||
| 554 | local $/ = "\n"; | ||
| 555 | chomp $buf; | ||
| 556 | propagate_sync_callback($options, 'error', $file, $buf); | ||
| 557 | } | ||
| 558 | } else { | ||
| 559 | $options->{quiet} or print STDERR $buf; | ||
| 560 | } | ||
| 561 | $buf = ''; | ||
| 562 | } | ||
| 563 | } | ||
| 564 | close $curl or _error('curl'); | ||
| 565 | } | ||
| 566 | |||
| 567 | sub _calc_limit_rate { | ||
| 568 | my $limit_rate = $_[0]; | ||
| 569 | for ($limit_rate) { | ||
| 570 | /^(\d+)$/ and $limit_rate = int $1/1024, last; | ||
| 571 | /^(\d+)[kK]$/ and $limit_rate = $1, last; | ||
| 572 | /^(\d+)[mM]$/ and $limit_rate = 1024*$1, last; | ||
| 573 | /^(\d+)[gG]$/ and $limit_rate = 1024*1024*$1, last; | ||
| 574 | } | ||
| 575 | $limit_rate; | ||
| 576 | } | ||
| 577 | |||
| 578 | sub sync_rsync { | ||
| 579 | -x "/usr/bin/rsync" or die N("rsync is missing\n"); | ||
| 580 | my $options = shift; | ||
| 581 | $options = { dir => $options } if !ref $options; | ||
| 582 | #- force download to be done in cachedir to avoid polluting cwd. | ||
| 583 | (my $cwd) = getcwd() =~ /(.*)/; | ||
| 584 | chdir($options->{dir}); | ||
| 585 | my $limit_rate = _calc_limit_rate($options->{'limit-rate'}); | ||
| 586 | foreach (@_) { | ||
| 587 | my $count = 10; #- retry count on error (if file exists). | ||
| 588 | my $basename = basename($_); | ||
| 589 | my $file = m!^rsync://([^/]*::.*)! ? $1 : $_; | ||
| 590 | propagate_sync_callback($options, 'start', $file); | ||
| 591 | do { | ||
| 592 | local $_; | ||
| 593 | my $buf = ''; | ||
| 594 | my $cmd = join(" ", "/usr/bin/rsync", | ||
| 595 | ($limit_rate ? "--bwlimit=$limit_rate" : @{[]}), | ||
| 596 | ($options->{quiet} ? qw(-q) : qw(--progress -v --no-human-readable)), | ||
| 597 | ($options->{compress} ? qw(-z) : @{[]}), | ||
| 598 | ($options->{ssh} ? qq(-e $options->{ssh}) : | ||
| 599 | ("--timeout=$CONNECT_TIMEOUT", | ||
| 600 | "--contimeout=$CONNECT_TIMEOUT")), | ||
| 601 | qw(--partial --no-whole-file --no-motd --copy-links), | ||
| 602 | (defined $options->{'rsync-options'} ? split /\s+/, $options->{'rsync-options'} : @{[]}), | ||
| 603 | "'$file' '$options->{dir}' 2>&1"); | ||
| 604 | $options->{debug} and $options->{debug}($cmd); | ||
| 605 | open(my $rsync, "$cmd |"); | ||
| 606 | local $/ = \1; #- read input by only one char, this is slow but very nice (and it works!). | ||
| 607 | local $_; | ||
| 608 | while (<$rsync>) { | ||
| 609 | $buf .= $_; | ||
| 610 | if ($_ eq "\r" || $_ eq "\n") { | ||
| 611 | if ($options->{callback}) { | ||
| 612 | if (my ($percent, $speed) = $buf =~ /^\s*\d+\s+(\d+)%\s+(\S+)\s+/) { | ||
| 613 | propagate_sync_callback($options, 'progress', $file, $percent, undef, undef, $speed); | ||
| 614 | } else { | ||
| 615 | $options->{debug} and $options->{debug}($buf); | ||
| 616 | } | ||
| 617 | } else { | ||
| 618 | $options->{quiet} or print STDERR $buf; | ||
| 619 | $options->{debug} and $options->{debug}($buf); | ||
| 620 | } | ||
| 621 | $buf = ''; | ||
| 622 | } | ||
| 623 | } | ||
| 624 | close $rsync; | ||
| 625 | } while ($? != 0 && --$count > 0 && -e $options->{dir} . "/$basename"); | ||
| 626 | propagate_sync_callback($options, 'end', $file); | ||
| 627 | } | ||
| 628 | chdir $cwd; | ||
| 629 | $? == 0 or _error('rsync'); | ||
| 630 | } | ||
| 631 | |||
| 632 | our $SSH_PATH; | ||
| 633 | sub _init_ssh_path() { | ||
| 634 | foreach (qw(/usr/bin/ssh /bin/ssh)) { | ||
| 635 | -x $_ and $SSH_PATH = $_; | ||
| 636 | next; | ||
| 637 | } | ||
| 638 | } | ||
| 639 | |||
| 640 | #- Don't generate a tmp dir name, so when we restart urpmi, the old ssh | ||
| 641 | #- connection can be reused | ||
| 642 | our $SSH_CONTROL_DIR = $ENV{TMP} || $ENV{TMPDIR} || '/tmp'; | ||
| 643 | our $SSH_CONTROL_OPTION; | ||
| 644 | |||
| 645 | sub sync_ssh { | ||
| 646 | $SSH_PATH or _init_ssh_path(); | ||
| 647 | $SSH_PATH or die N("ssh is missing\n"); | ||
| 648 | my $options = shift; | ||
| 649 | $options = { dir => $options } if !ref $options; | ||
| 650 | unless ($options->{'rsync-options'} =~ /(?:-e|--rsh)\b/) { | ||
| 651 | my ($server, $user) = ('', getpwuid($<)); | ||
| 652 | $_[0] =~ /((?:\w|\.)*):/ and $server = $1; | ||
| 653 | $_[0] =~ /((?:\w|-)*)@/ and $user = $1; | ||
| 654 | $SSH_CONTROL_OPTION = "-o 'ControlPath $SSH_CONTROL_DIR/ssh-urpmi-$$-%h_%p_%r' -o 'ControlMaster auto'"; | ||
| 655 | if (start_ssh_master($server, $user)) { | ||
| 656 | $options->{ssh} = qq("$SSH_PATH $SSH_CONTROL_OPTION"); | ||
| 657 | } else { | ||
| 658 | #- can't start master, use single connection | ||
| 659 | $options->{ssh} = $SSH_PATH; | ||
| 660 | } | ||
| 661 | } | ||
| 662 | sync_rsync($options, @_); | ||
| 663 | } | ||
| 664 | |||
| 665 | sub sync_prozilla { | ||
| 666 | -x "/usr/bin/proz" or die N("prozilla is missing\n"); | ||
| 667 | my $options = shift; | ||
| 668 | $options = { dir => $options } if !ref $options; | ||
| 669 | #- force download to be done in cachedir to avoid polluting cwd. | ||
| 670 | (my $cwd) = getcwd() =~ /(.*)/; | ||
| 671 | chdir $options->{dir}; | ||
| 672 | my $proz_command = join(" ", map { "'$_'" } | ||
| 673 | "/usr/bin/proz", | ||
| 674 | "--no-curses", | ||
| 675 | (defined $options->{'prozilla-options'} ? split /\s+/, $options->{'prozilla-options'} : @{[]}), | ||
| 676 | @_ | ||
| 677 | ); | ||
| 678 | my $ret = system($proz_command); | ||
| 679 | chdir $cwd; | ||
| 680 | if ($ret) { | ||
| 681 | if ($? == -1) { | ||
| 682 | die N("Couldn't execute prozilla\n"); | ||
| 683 | } else { | ||
| 684 | _error('prozilla'); | ||
| 685 | } | ||
| 686 | } | ||
| 687 | } | ||
| 688 | |||
| 689 | sub sync_aria2 { | ||
| 690 | my ($urpm, $medium, $rel_files, $options) = @_; | ||
| 691 | |||
| 692 | -x "/usr/bin/aria2c" or die N("aria2 is missing\n"); | ||
| 693 | |||
| 694 | #- force download to be done in cachedir to avoid polluting cwd. | ||
| 695 | (my $cwd) = getcwd() =~ /(.*)/; | ||
| 696 | chdir $options->{dir}; | ||
| 697 | |||
| 698 | my $stat_file = ($< ? $ENV{HOME} : '/root') . '/.aria2-adaptive-stats'; | ||
| 699 | |||
| 700 | my $aria2c_command = join(" ", map { "'$_'" } | ||
| 701 | "/usr/bin/aria2c", $options->{debug} ? ('--log', "$options->{dir}/.aria2.log") : @{[]}, | ||
| 702 | '--auto-file-renaming=false', | ||
| 703 | '--ftp-pasv', | ||
| 704 | '--summary-interval=1', | ||
| 705 | '--follow-metalink=mem', | ||
| 706 | $medium->{mirrorlist} ? ( | ||
| 707 | '--metalink-enable-unique-protocol=true', # do not try to connect to the same server using the same protocol | ||
| 708 | '--metalink-preferred-protocol=http', # try http as first protocol as they're stateless and | ||
| 709 | # will put less strain on ie. the ftp servers which connections | ||
| 710 | # are statefull for, causing unhappy mirror admins complaining | ||
| 711 | # about increase of connections, increasing resource usage. | ||
| 712 | '--max-tries=5', # nb: not using $options->{retry} | ||
| 713 | '--lowest-speed-limit=20K', "--timeout", 3, | ||
| 714 | '--split=3', # maximum number of servers to use for one download | ||
| 715 | '--uri-selector=adaptive', "--server-stat-if=$stat_file", "--server-stat-of=$stat_file", | ||
| 716 | $options->{is_versioned} ? @{[]} : '--max-file-not-found=9', # number of not found errors on different servers before aborting file download | ||
| 717 | '--connect-timeout=6', # $CONNECT_TIMEOUT, | ||
| 718 | ) : @{[]}, | ||
| 719 | '-Z', '-j1', | ||
| 720 | ($options->{'limit-rate'} ? "--max-download-limit=" . $options->{'limit-rate'} : @{[]}), | ||
| 721 | ($options->{resume} ? "--continue" : "--allow-overwrite=true"), | ||
| 722 | ($options->{proxy} ? set_proxy({ type => "aria2", proxy => $options->{proxy} }) : @{[]}), | ||
| 723 | ($options->{"no-certificate-check"} ? "--check-certificate=false" : @{[]}), | ||
| 724 | (defined $options->{'aria2-options'} ? split /\s+/, $options->{'aria2-options'} : @{[]}), | ||
| 725 | _create_metalink_($urpm, $medium, $rel_files, $options)); | ||
| 726 | |||
| 727 | $options->{debug} and $options->{debug}($aria2c_command); | ||
| 728 | |||
| 729 | local $ENV{LC_ALL} = 'C'; | ||
| 730 | my $aria2_pid = open(my $aria2, "$aria2c_command |"); | ||
| 731 | |||
| 732 | _parse_aria2_output($options, $aria2, $aria2_pid, $medium, $rel_files); | ||
| 733 | |||
| 734 | chdir $cwd; | ||
| 735 | if (!close $aria2) { | ||
| 736 | my $raw_msg = _error_msg('aria2'); | ||
| 737 | my $want_retry; | ||
| 738 | if (!$options->{is_retry} & $options->{is_versioned}) { | ||
| 739 | $want_retry = 1; | ||
| 740 | } else { | ||
| 741 | my $msg = N("Failed to download %s", $rel_files->[0]); | ||
| 742 | $want_retry = $options->{ask_retry} && $options->{ask_retry}($raw_msg, $msg); | ||
| 743 | } | ||
| 744 | if ($want_retry) { | ||
| 745 | $options->{is_retry}++; | ||
| 746 | $options->{debug} and $options->{debug}("retrying ($options->{is_retry})"); | ||
| 747 | goto &sync_aria2; | ||
| 748 | } | ||
| 749 | die $raw_msg; | ||
| 750 | } | ||
| 751 | } | ||
| 752 | |||
| 753 | sub _parse_aria2_output { | ||
| 754 | my ($options, $aria2, $aria2_pid, $medium, $rel_files) = @_; | ||
| 755 | |||
| 756 | my ($buf, $_total, $file) = ('', undef, undef); | ||
| 757 | |||
| 758 | local $/ = \1; #- read input by only one char, this is slow but very nice (and it works!). | ||
| 759 | local $_; | ||
| 760 | |||
| 761 | while (<$aria2>) { | ||
| 762 | if ($_ eq "\r" || $_ eq "\n") { | ||
| 763 | $options->{debug}("aria2c: $buf") if $options->{debug}; | ||
| 764 | if ($options->{callback}) { | ||
| 765 | if (!defined($file) && @$rel_files) { | ||
| 766 | $file = $medium->{mirrorlist} ? | ||
| 767 | $medium->{mirrorlist} . ': ' . $medium->{'with-dir'} . "/$rel_files->[0]" : | ||
| 768 | "$medium->{url}/$rel_files->[0]"; | ||
| 769 | propagate_sync_callback($options, 'start', $file) | ||
| 770 | if !$options->{is_retry}; | ||
| 771 | } | ||
| 772 | |||
| 773 | # aria2c 1.16 and beyond: | ||
| 774 | # parses aria2c: [#2c8dae 496KiB/830KiB(59%) CN:1 DL:84KiB ETA:3s] | ||
| 775 | # | ||
| 776 | # using multiline mode and comments for better readability: | ||
| 777 | # | ||
| 778 | if ($buf =~ m! | ||
| 779 | ^\[\#[\dA-Fa-f]+ # match #2c8dae | ||
| 780 | \s+ | ||
| 781 | ([\d\.]+\w*) # Match 496KiB | ||
| 782 | / | ||
| 783 | ([\d\.]+\w*) # Match 830KiB | ||
| 784 | \s* \( (\d+) % \) # Match (59%) | ||
| 785 | \s+ | ||
| 786 | CN:(\S+) # Match CN:1 | ||
| 787 | \s+ | ||
| 788 | DL:(\S+) # Match DL:84KiB | ||
| 789 | \s+ | ||
| 790 | ETA:(\w+) | ||
| 791 | \]$ | ||
| 792 | !msx | ||
| 793 | ) | ||
| 794 | { | ||
| 795 | my ($total, $percent, $speed, $eta) = ($2, $3, $5, $6); | ||
| 796 | #- $1 = current downloaded size, $4 = connections | ||
| 797 | if (propagate_sync_callback($options, 'progress', $file, $percent, $total, $eta, $speed) eq 'canceled') { | ||
| 798 | kill 15, $aria2_pid; | ||
| 799 | close $aria2; | ||
| 800 | return; | ||
| 801 | } | ||
| 802 | } | ||
| 803 | if ($buf =~ m!Download\scomplete:\s/!) { | ||
| 804 | propagate_sync_callback($options, 'end', $file); | ||
| 805 | shift @$rel_files; | ||
| 806 | delete $options->{is_retry}; | ||
| 807 | $file = undef; | ||
| 808 | } elsif ($buf =~ /ERR\|(.*)/) { | ||
| 809 | propagate_sync_callback($options, 'error', $file, $1); | ||
| 810 | } | ||
| 811 | } else { | ||
| 812 | $options->{quiet} or print STDERR "$buf\n"; | ||
| 813 | } | ||
| 814 | $buf = ''; | ||
| 815 | } else { | ||
| 816 | $buf .= $_; | ||
| 817 | } | ||
| 818 | } | ||
| 819 | } | ||
| 820 | |||
| 821 | sub start_ssh_master { | ||
| 822 | my ($server, $user) = @_; | ||
| 823 | $server or return 0; | ||
| 824 | if (!check_ssh_master($server, $user)) { | ||
| 825 | system(qq($SSH_PATH -f -N $SSH_CONTROL_OPTION -M $user\@$server)); | ||
| 826 | return ! $?; | ||
| 827 | } | ||
| 828 | return 1; | ||
| 829 | } | ||
| 830 | |||
| 831 | sub check_ssh_master { | ||
| 832 | my ($server, $user) = @_; | ||
| 833 | system(qq($SSH_PATH -q -f -N $SSH_CONTROL_OPTION $user\@$server -O check)); | ||
| 834 | return ! $?; | ||
| 835 | } | ||
| 836 | |||
| 837 | END { | ||
| 838 | #- remove ssh persistent connections | ||
| 839 | foreach my $socket (glob "$SSH_CONTROL_DIR/ssh-urpmi-$$-*") { | ||
| 840 | my ($server, $login) = $socket =~ /ssh-urpmi-\d+-([^_]+)_\d+_(.*)$/ or next; | ||
| 841 | system($SSH_PATH, '-q', '-f', '-N', '-o', "ControlPath $socket", '-O', 'exit', "$login\@$server"); | ||
| 842 | } | ||
| 843 | } | ||
| 844 | |||
| 845 | #- get the width of the terminal | ||
| 846 | my $wchar = 79; | ||
| 847 | if (-t *STDOUT) { | ||
| 848 | eval { | ||
| 849 | require Term::ReadKey; | ||
| 850 | ($wchar) = Term::ReadKey::GetTerminalSize(); | ||
| 851 | --$wchar; | ||
| 852 | }; | ||
| 853 | } | ||
| 854 | |||
| 855 | sub progress_text { | ||
| 856 | my ($mode, $percent, $total, $eta, $speed) = @_; | ||
| 857 | $mode eq 'progress' ? | ||
| 858 | (defined $total && defined $eta ? | ||
| 859 | N(" %s%% of %s completed, ETA = %s, speed = %s", $percent, $total, $eta, $speed) : | ||
| 860 | N(" %s%% completed, speed = %s", $percent, $speed)) : ''; | ||
| 861 | } | ||
| 862 | |||
| 863 | =item sync_logger($mode, $file, $percent, $_total, $_eta, $_speed) | ||
| 864 | |||
| 865 | Default logger (callback) suitable for sync operation on STDERR only. | ||
| 866 | |||
| 867 | =cut | ||
| 868 | |||
| 869 | sub sync_logger { | ||
| 870 | my ($mode, $file, $percent, $total, $eta, $speed) = @_; | ||
| 871 | if ($mode eq 'start') { | ||
| 872 | print STDERR " $file\n"; | ||
| 873 | } elsif ($mode eq 'progress') { | ||
| 874 | my $text = progress_text($mode, $percent, $total, $eta, $speed); | ||
| 875 | if (length($text) > $wchar) { $text = substr($text, 0, $wchar) } | ||
| 876 | if (bytes::length($text) < $wchar) { | ||
| 877 | # clearing more than needed in case the terminal is not handling utf8 and we have a utf8 string | ||
| 878 | print STDERR $text, " " x ($wchar - bytes::length($text)), "\r"; | ||
| 879 | } else { | ||
| 880 | # clearing all the line first since we can't really know the "length" of the string | ||
| 881 | print STDERR " " x $wchar, "\r", $text, "\r"; | ||
| 882 | } | ||
| 883 | } elsif ($mode eq 'end') { | ||
| 884 | print STDERR " " x $wchar, "\r"; | ||
| 885 | } elsif ($mode eq 'error') { | ||
| 886 | #- error is 3rd argument, saved in $percent | ||
| 887 | print STDERR N("...retrieving failed: %s", $percent), "\n"; | ||
| 888 | } | ||
| 889 | } | ||
| 890 | |||
| 891 | =item requested_ftp_http_downloader($urpm, $medium) | ||
| 892 | |||
| 893 | Return the downloader program to use (whether it pas provided on the | ||
| 894 | command line or in the config file). | ||
| 895 | |||
| 896 | =cut | ||
| 897 | |||
| 898 | sub requested_ftp_http_downloader { | ||
| 899 | my ($urpm, $medium) = @_; | ||
| 900 | |||
| 901 | $urpm->{options}{downloader} || #- cmd-line switch | ||
| 902 | $medium && $medium->{downloader} || | ||
| 903 | $urpm->{global_config}{downloader} || ""; | ||
| 904 | } | ||
| 905 | |||
| 906 | sub parse_url_with_login { | ||
| 907 | my ($url) = @_; | ||
| 908 | $url =~ m!([^:]*)://([^/:]*)(:([^/:\@]*))?\@([^/]*)(.*)! && $1 ne 'ssh' && | ||
| 909 | { proto => $1, login => $2, password => $4, machine => $5, dir => $6 }; | ||
| 910 | } | ||
| 911 | sub url_obscuring_password { | ||
| 912 | my ($url) = @_; | ||
| 913 | my $u = parse_url_with_login($url); | ||
| 914 | if ($u && $u->{password}) { | ||
| 915 | sprintf('%s://xxx:xxx@%s%s', $u->{proto}, $u->{machine}, $u->{dir}); | ||
| 916 | } else { | ||
| 917 | $url; | ||
| 918 | } | ||
| 919 | } | ||
| 920 | |||
| 921 | #- $medium can be undef | ||
| 922 | sub _all_options { | ||
| 923 | my ($urpm, $medium, $options) = @_; | ||
| 924 | |||
| 925 | my %all_options = ( | ||
| 926 | dir => "$urpm->{cachedir}/partial", | ||
| 927 | proxy => get_proxy_($urpm), | ||
| 928 | metalink => $medium->{mirrorlist}, | ||
| 929 | $medium->{"disable-certificate-check"} ? "no-certificate-check" : @{[]}, | ||
| 930 | $urpm->{debug} ? (debug => $urpm->{debug}) : @{[]}, | ||
| 931 | %$options, | ||
| 932 | ); | ||
| 933 | foreach my $cpt (qw(compress limit-rate retry wget-options curl-options rsync-options prozilla-options aria2-options metalink)) { | ||
| 934 | $all_options{$cpt} = $urpm->{options}{$cpt} if defined $urpm->{options}{$cpt}; | ||
| 935 | } | ||
| 936 | \%all_options; | ||
| 937 | } | ||
| 938 | |||
| 939 | sub sync_rel { | ||
| 940 | my ($urpm, $medium, $rel_files, %options) = @_; | ||
| 941 | |||
| 942 | my @files = map { reduce_pathname("$medium->{url}/$_") } @$rel_files; | ||
| 943 | |||
| 944 | my $files_text = join(' ', (use_metalink($urpm, $medium) ? ($medium->{mirrorlist}, $medium->{'with-dir'}) : url_obscuring_password($medium->{url})), @$rel_files); | ||
| 945 | $urpm->{debug} and $urpm->{debug}(N("retrieving %s", $files_text)); | ||
| 946 | |||
| 947 | my $all_options = _all_options($urpm, $medium, \%options); | ||
| 948 | my @result_files = map { $all_options->{dir} . '/' . basename($_) } @$rel_files; | ||
| 949 | unlink @result_files if $all_options->{preclean}; | ||
| 950 | |||
| 951 | #- The files must be world-readable, else mgaapplet and urpm* commands run as | ||
| 952 | #- a normal user won't be able to read them. We enforce umask here in the case | ||
| 953 | #- where the msec security level is set to 'secure' (which means umask 077) | ||
| 954 | #- or where we are run from a gdm-x-session (mga#24636) | ||
| 955 | my $old_umask = umask 0022; | ||
| 956 | |||
| 957 | (my $cwd) = getcwd() =~ /(.*)/; | ||
| 958 | eval { _sync_webfetch_raw($urpm, $medium, $rel_files, \@files, $all_options) }; | ||
| 959 | my $err = $@; | ||
| 960 | chdir $cwd; | ||
| 961 | umask $old_umask; | ||
| 962 | if (!$err) { | ||
| 963 | $urpm->{log}(N("retrieved %s", $files_text)); | ||
| 964 | \@result_files; | ||
| 965 | } else { | ||
| 966 | $urpm->{log}("error: $err"); | ||
| 967 | # don't leave partial download | ||
| 968 | unlink @result_files; | ||
| 969 | undef; | ||
| 970 | } | ||
| 971 | } | ||
| 972 | |||
| 973 | sub sync_rel_one { | ||
| 974 | my ($urpm, $medium, $rel_file, %options) = @_; | ||
| 975 | |||
| 976 | my $files = sync_rel($urpm, $medium, [$rel_file], %options) or return; | ||
| 977 | $files->[0]; | ||
| 978 | } | ||
| 979 | |||
| 980 | =item sync_url($urpm, $url, %options) | ||
| 981 | |||
| 982 | Retrieve a file from the network and return the local cached file path. | ||
| 983 | |||
| 984 | =cut | ||
| 985 | |||
| 986 | sub sync_url { | ||
| 987 | my ($urpm, $url, %options) = @_; | ||
| 988 | sync_rel_one($urpm, { url => dirname($url), disable_metalink => $options{disable_metalink} }, basename($url), %options); | ||
| 989 | } | ||
| 990 | |||
| 991 | sub sync_rel_to { | ||
| 992 | my ($urpm, $medium, $rel_file, $dest_file, %options) = @_; | ||
| 993 | |||
| 994 | my $files = sync_rel($urpm, $medium, [$rel_file], %options) or return undef; | ||
| 995 | my $result_file = $files->[0]; | ||
| 996 | $result_file ne $dest_file or rename($result_file, $dest_file) or return; | ||
| 997 | $result_file; | ||
| 998 | } | ||
| 999 | |||
| 1000 | =item get_content($urpm, $url, %o_options) | ||
| 1001 | |||
| 1002 | Retrieve a file and return its content. | ||
| 1003 | |||
| 1004 | =cut | ||
| 1005 | |||
| 1006 | sub get_content { | ||
| 1007 | my ($urpm, $url, %o_options) = @_; | ||
| 1008 | |||
| 1009 | my $file = sync_url($urpm, $url, %o_options, quiet => 1, preclean => 1) or return; | ||
| 1010 | |||
| 1011 | my @l = cat_($file); | ||
| 1012 | unlink $file; | ||
| 1013 | |||
| 1014 | wantarray() ? @l : join('', @l); | ||
| 1015 | } | ||
| 1016 | |||
| 1017 | |||
| 1018 | #- syncing algorithms. | ||
| 1019 | #- | ||
| 1020 | #- nb: $files is constructed from $rel_files using $medium | ||
| 1021 | sub _sync_webfetch_raw { | ||
| 1022 | my ($urpm, $medium, $rel_files, $files, $options) = @_; | ||
| 1023 | |||
| 1024 | #- currently ftp and http protocols are managed by curl or wget, | ||
| 1025 | #- ssh and rsync protocols are managed by rsync *AND* ssh. | ||
| 1026 | my $proto = urpm::protocol_from_url($medium->{url}) or die N("unknown protocol defined for %s", $medium->{url}); | ||
| 1027 | |||
| 1028 | if ($proto eq 'file') { | ||
| 1029 | my @l = map { urpm::file_from_local_url($_) } @$files; | ||
| 1030 | eval { sync_file($options, @l) }; | ||
| 1031 | $urpm->{fatal}(10, $@) if $@; | ||
| 1032 | } elsif ($proto eq 'rsync') { | ||
| 1033 | sync_rsync($options, @$files); | ||
| 1034 | } elsif (member($proto, 'ftp', 'http', 'https') || $options->{metalink}) { | ||
| 1035 | |||
| 1036 | my $preferred = preferred_downloader($urpm, $medium, \$options->{metalink}); | ||
| 1037 | if ($preferred eq 'aria2') { | ||
| 1038 | sync_aria2($urpm, $medium, $rel_files, $options); | ||
| 1039 | } else { | ||
| 1040 | my $sync = $urpm::download::{"sync_$preferred"} or die N("no webfetch found, supported webfetch are: %s\n", join(", ", ftp_http_downloaders())); | ||
| 1041 | |||
| 1042 | my @l = @$files; | ||
| 1043 | while (@l) { | ||
| 1044 | my $half_MAX_ARG = 131072 / 2; | ||
| 1045 | # restrict the number of elements so that it fits on cmdline of curl/wget/proz/aria2c | ||
| 1046 | my $n = 0; | ||
| 1047 | for (my $len = 0; $n < @l && $len < $half_MAX_ARG; $len += length($l[$n++])) {} | ||
| 1048 | $sync->($options, splice(@l, 0, $n)); | ||
| 1049 | } | ||
| 1050 | } | ||
| 1051 | } elsif ($proto eq 'ssh') { | ||
| 1052 | my @ssh_files = map { m!^ssh://([^/]*)(.*)! ? "$1:$2" : @{[]} } @$files; | ||
| 1053 | sync_ssh($options, @ssh_files); | ||
| 1054 | } else { | ||
| 1055 | die N("unable to handle protocol: %s", $proto); | ||
| 1056 | } | ||
| 1057 | } | ||
| 1058 | |||
| 1059 | sub _take_n_elem { | ||
| 1060 | my ($n, @l) = @_; | ||
| 1061 | @l < $n ? @l : @l[0 .. $n-1]; | ||
| 1062 | } | ||
| 1063 | |||
| 1064 | sub _create_one_metalink_line { | ||
| 1065 | my ($medium, $mirror, $rel_file, $counter) = @_; | ||
| 1066 | |||
| 1067 | my $type = urpm::protocol_from_url($mirror->{url}); | ||
| 1068 | |||
| 1069 | # If more than 100 mirrors, give all the remaining mirrors a priority of 0 | ||
| 1070 | my $preference = max(0, 100 - $counter); | ||
| 1071 | |||
| 1072 | my @options = (qq(type="$type"), qq(preference="$preference")); | ||
| 1073 | # Not supported in metalinks | ||
| 1074 | #if (@$list[$i]->{bw}) { | ||
| 1075 | # push @options, 'bandwidth="' . @$list[$i]->{bw} . '"'; | ||
| 1076 | # } | ||
| 1077 | # Supported in metalinks, but no longer used in mirror list..? | ||
| 1078 | if ($mirror->{connections}) { | ||
| 1079 | push @options, qq(maxconnections="$mirror->{connections}"); | ||
| 1080 | } | ||
| 1081 | push @options, 'location="' . lc($mirror->{zone}) . '"'; | ||
| 1082 | my $base = urpm::mirrors::_add__with_dir($mirror->{url}, $medium->{'with-dir'}); | ||
| 1083 | sprintf('<url %s>%s/%s</url>', join(' ', @options), $base, $rel_file); | ||
| 1084 | } | ||
| 1085 | |||
| 1086 | sub _create_metalink_ { | ||
| 1087 | my ($urpm, $medium, $rel_files, $options) = @_; | ||
| 1088 | # Don't create a metalink when downloading mirror list | ||
| 1089 | $medium or return; | ||
| 1090 | |||
| 1091 | # only use the 8 best mirrors, then we let aria2 choose | ||
| 1092 | require urpm::mirrors; | ||
| 1093 | my @mirrors = $medium->{mirrorlist} ? (map { | ||
| 1094 | # aria2 doesn't handle rsync | ||
| 1095 | my @l = grep { urpm::protocol_from_url($_->{url}) ne 'rsync' } @$_; | ||
| 1096 | _take_n_elem(8, @l); | ||
| 1097 | } urpm::mirrors::list_urls($urpm, $medium, '')) : { url => $medium->{url} }; | ||
| 1098 | |||
| 1099 | my $metalinkfile = "$urpm->{cachedir}/$options->{media}.metalink"; | ||
| 1100 | # Even if not required by metalink spec, this line is needed at top of | ||
| 1101 | # metalink file, otherwise aria2 won't be able to autodetect it.. | ||
| 1102 | my @metalink = ( | ||
| 1103 | '<?xml version="1.0" encoding="utf-8"?>', | ||
| 1104 | '<metalink version="3.0" generator="URPMI" xmlns="http://www.metalinker.org/">', | ||
| 1105 | '<files>', | ||
| 1106 | ); | ||
| 1107 | |||
| 1108 | foreach my $rel_file (@$rel_files) { | ||
| 1109 | my $i = 0; | ||
| 1110 | my @lines = map { | ||
| 1111 | $i++; | ||
| 1112 | _create_one_metalink_line($medium, $_, $rel_file, $i); | ||
| 1113 | } @mirrors; | ||
| 1114 | |||
| 1115 | push @metalink, map { "\t$_" } | ||
| 1116 | sprintf('<file name="%s"><resources>', basename($rel_file)), | ||
| 1117 | (map { "\t$_" } @lines), | ||
| 1118 | '</resources></file>'; | ||
| 1119 | } | ||
| 1120 | push @metalink, '</files>', '</metalink>'; | ||
| 1121 | |||
| 1122 | output_safe($metalinkfile, join('', map { "$_\n" } @metalink)); | ||
| 1123 | $metalinkfile; | ||
| 1124 | } | ||
| 1125 | |||
| 1126 | 1; | ||
| 1127 | |||
| 1128 | |||
| 1129 | =back | ||
| 1130 | |||
| 1131 | =head1 COPYRIGHT | ||
| 1132 | |||
| 1133 | Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005 MandrakeSoft SA | ||
| 1134 | |||
| 1135 | Copyright (C) 2005-2010 Mandriva SA | ||
| 1136 | |||
| 1137 | Copyright (C) 2011-2020 Mageia | ||
| 1138 | |||
| 1139 | =cut | ||