urpm/download.pm

filter: Statementstmt 66.6% 4 / 6 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 2 / 2 Podpod n/a totaltotal 75.0% 6 / 8 CC 1 SCAR 1.4
CC
1
Cov
67%
BEGIN
1.3
BEGIN
1.0
CRAP 1.1
« urpm/cdrom.pm ↑ Summary urpm/get_pkgs.pm »
1 package urpm::download;
2  
3  
4 2 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 0 ▶use urpm::msg;
Subroutine BEGIN: called CC=1 SCAR=2.6
6 use urpm::util qw(cat_ basename dirname file_size max member output_safe reduce_pathname);
7 use bytes ();
8 use Cwd;
9 use Exporter;
10 # perl_checker: require urpm
11  
12 # help perl_checker:
13 sub getcwd { goto &Cwd::getcwd }
14  
15 our @ISA = 'Exporter';
16 our @EXPORT = qw(get_proxy
17 propagate_sync_callback
18 sync_file sync_rsync sync_ssh
19 set_proxy_config dump_proxy_config
20 );
21  
22 #- proxy config file.
23 our $PROXY_CFG = '/etc/urpmi/proxy.cfg';
24 my $proxy_config;
25  
26 #- Timeout for curl connection and wget operations
27 our $CONNECT_TIMEOUT = 60; #- (in seconds)
28  
29  
30 =head1 NAME
31  
32 urpm::download - download routines for the urpm* tools
33  
34 =head1 SYNOPSIS
35  
36 =head1 DESCRIPTION
37  
38 =over
39  
40 =cut
41  
42  
43 sub ftp_http_downloaders() { qw(wget curl prozilla aria2) }
44  
45 sub available_ftp_http_downloaders() {
46 my %binaries = (
47 wget => 'wget',
48 curl => 'curl',
49 prozilla => 'proz',
50 aria2 => 'aria2c',
51 );
52 grep { -x "/usr/bin/$binaries{$_}" || -x "/bin/$binaries{$_}" } ftp_http_downloaders();
53 }
54  
55 sub metalink_downloaders() { qw(aria2) }
56  
57 sub available_metalink_downloaders() {
58 my %binaries = (
59 aria2 => 'aria2c',
60 );
61 grep { -x "/usr/bin/$binaries{$_}" || -x "/bin/$binaries{$_}" } metalink_downloaders();
62 }
63  
64 sub use_metalink {
65 my ($urpm, $medium) = @_;
66  
67 $medium->{allow_metalink} //= do {
68 my $use_metalink = 1;
69 preferred_downloader($urpm, $medium, \$use_metalink);
70 $use_metalink;
71 };
72 }
73  
74 my %warned;
75 sub preferred_downloader {
76 my ($urpm, $medium, $use_metalink) = @_;
77  
78 my @available = available_ftp_http_downloaders();
79 my @metalink_downloaders = available_metalink_downloaders();
80 my $metalink_disabled = !$$use_metalink && $medium->{disable_metalink};
81  
82 if ($$use_metalink && !$metalink_disabled) {
83 #- If metalink is used, only aria2 is available as other downloaders doesn't support metalink
84 unshift @available, @metalink_downloaders;
85 }
86
87 #- first downloader of @available is the default one
88 my $preferred = $available[0];
89 my $requested_downloader = requested_ftp_http_downloader($urpm, $medium);
90 if ($requested_downloader) {
91 if (member($requested_downloader, @available)) {
92 #- use user default downloader if provided and available
93 $preferred = $requested_downloader;
94 } elsif ($warned{webfetch_not_available}++ == 0) {
95 $urpm->{log}(N("%s is not available, falling back on %s", $requested_downloader, $preferred));
96 }
97 }
98  
99 if ($$use_metalink && !member($preferred, @metalink_downloaders)) {
100 $warned{not_using_metalink}++ or
101 $urpm->{log}($requested_downloader eq $preferred ?
102 "not using metalink since requested downloader does not handle it" :
103 "not using metalink since no downloaders handling metalink are available");
104 $$use_metalink = 0;
105 }
106 $preferred;
107 }
108  
109 sub parse_http_proxy {
110 $_[0] =~ m!^(?:http://)?([^:/]+(:\d+)?)/*$!;
111 }
112  
113 #- parses proxy.cfg (private)
114 sub load_proxy_config () {
115 return if defined $proxy_config;
116 $proxy_config = {};
117 foreach (cat_($PROXY_CFG)) {
118 chomp; s/#.*$//; s/^\s*//; s/\s*$//;
119 if (/^(?:(.*):\s*)?(ftp_proxy|http_proxy)\s*=\s*(.*)$/) {
120 $proxy_config->{$1 || ''}{$2} = $3;
121 next;
122 }
123 if (/^(?:(.*):\s*)?proxy_user\s*=\s*([^:]*)(?::(.*))?$/) {
124 $proxy_config->{$1 || ''}{user} = $2;
125 $proxy_config->{$1 || ''}{pwd} = $3 if defined $3;
126 next;
127 }
128 if (/^(?:(.*):\s*)?proxy_user_ask/) {
129 $proxy_config->{$1 || ''}{ask} = 1;
130 next;
131 }
132 }
133 }
134  
135 #- writes proxy.cfg
136 sub dump_proxy_config () {
137 $proxy_config or return 0; #- hasn't been read yet
138  
139 my $has_password;
140  
141 open my $f, '>', $PROXY_CFG or return 0;
142 foreach ('', sort grep { !/^(|cmd_line)$/ } keys %$proxy_config) {
143 my $m = $_ eq '' ? '' : "$_:";
144 my $p = $proxy_config->{$_};
145 foreach (qw(http_proxy ftp_proxy)) {
146 if (defined $p->{$_} && $p->{$_} ne '') {
147 print $f "$m$_=$p->{$_}\n";
148 $has_password ||= hide_password($p->{$_}) ne $p->{$_};
149 }
150 }
151 if ($p->{ask}) {
152 print $f "${m}proxy_user_ask\n";
153 } elsif (defined $p->{user} && $p->{user} ne '') {
154 print $f "${m}proxy_user=$p->{user}:$p->{pwd}\n";
155 $has_password ||= $p->{pwd};
156 }
157 }
158 close $f;
159 chmod 0600, $PROXY_CFG if $has_password;
160 return 1;
161 }
162  
163 #- deletes the proxy configuration for the specified media
164 sub remove_proxy_media {
165 defined $proxy_config and delete $proxy_config->{$_[0] || ''};
166 }
167  
168 sub get_proxy_ {
169 my ($urpm) = @_;
170  
171 -e $PROXY_CFG && !-r $PROXY_CFG and $urpm->{error}(N("can not read proxy settings (not enough rights to read %s)", $PROXY_CFG));
172  
173 get_proxy($urpm);
174 }
175  
176 =item get_proxy($media)
177  
178 Reads and loads the proxy.cfg file ;
179 Returns the global proxy settings (without arguments) or the
180 proxy settings for the specified media (with a media name as argument)
181  
182 =cut
183  
184 sub get_proxy (;$) {
185 my ($o_media) = @_; $o_media ||= '';
186 load_proxy_config();
187 my $p = $proxy_config->{cmd_line}
188 || $proxy_config->{$o_media}
189 || $proxy_config->{''}
190 || {
191 http_proxy => undef,
192 ftp_proxy => undef,
193 user => undef,
194 pwd => undef,
195 };
196 if ($p->{ask} && ($p->{http_proxy} || $p->{ftp_proxy}) && !$p->{user}) {
197 our $PROMPT_PROXY;
198 unless (defined $PROMPT_PROXY) {
199 require urpm::prompt;
200 $PROMPT_PROXY = urpm::prompt->new(
201 N("Please enter your credentials for accessing proxy\n"),
202 [ N("User name:"), N("Password:") ],
203 undef,
204 [ 0, 1 ],
205 );
206 }
207 ($p->{user}, $p->{pwd}) = $PROMPT_PROXY->prompt;
208 }
209 $p;
210 }
211  
212 #- copies the settings for proxies from the command line to media named $media
213 #- and writes the proxy.cfg file (used when adding new media)
214 sub copy_cmd_line_proxy {
215 my ($media) = @_;
216 return unless $media;
217 load_proxy_config();
218 if (defined $proxy_config->{cmd_line}) {
219 $proxy_config->{$media} = $proxy_config->{cmd_line};
220 dump_proxy_config();
221 } else {
222 #- use default if available
223 $proxy_config->{$media} = $proxy_config->{''};
224 }
225 }
226  
227 =item set_cmdline_proxy(%h)
228  
229 Overrides the config file proxy settings with values passed via command-line
230  
231 =cut
232  
233 sub set_cmdline_proxy {
234 my (%h) = @_;
235 load_proxy_config();
236 $proxy_config->{cmd_line} ||= {
237 http_proxy => undef,
238 ftp_proxy => undef,
239 user => undef,
240 pwd => undef,
241 };
242 $proxy_config->{cmd_line}{$_} = $h{$_} foreach keys %h;
243 }
244  
245 =item set_proxy_config($key, $value, $o_media)
246  
247 Changes permanently the proxy settings
248  
249 =cut
250  
251 sub set_proxy_config {
252 my ($key, $value, $o_media) = @_;
253 $proxy_config->{$o_media || ''}{$key} = $value;
254 }
255  
256 #- set up the environment for proxy usage for the appropriate tool.
257 #- returns an array of command-line arguments for wget or curl.
258 sub set_proxy {
259 my ($proxy) = @_;
260  
261 my $p = $proxy->{proxy};
262 defined $p->{http_proxy} || defined $p->{ftp_proxy} or return;
263  
264 my @res;
265 if ($proxy->{type} =~ /\bwget\b/) {
266 if (defined $p->{http_proxy}) {
267 $ENV{http_proxy} = $p->{http_proxy} =~ /^http:/
268 ? $p->{http_proxy} : "http://$p->{http_proxy}";
269 }
270 $ENV{ftp_proxy} = $p->{ftp_proxy} if defined $p->{ftp_proxy};
271 @res = ("--proxy-user=$p->{user}", "--proxy-passwd=$p->{pwd}")
272 if defined $p->{user} && defined $p->{pwd};
273 } elsif ($proxy->{type} =~ /\bcurl\b/) {
274 push @res, ('-x', $p->{http_proxy}) if defined $p->{http_proxy};
275 push @res, ('-x', $p->{ftp_proxy}) if defined $p->{ftp_proxy};
276 push @res, ('-U', "$p->{user}:$p->{pwd}")
277 if defined $p->{user} && defined $p->{pwd};
278 push @res, '-H', 'Pragma:' if @res;
279 } elsif ($proxy->{type} =~ /\baria2\b/) {
280 if (my ($http_proxy) = $p->{http_proxy} && parse_http_proxy($p->{http_proxy})) {
281 my $allproxy = $p->{user};
282 $allproxy .= ":" . $p->{pwd} if $p->{pwd};
283 $allproxy .= "@" if $p->{user};
284 $allproxy .= $http_proxy;
285 @res = ("--all-proxy=http://$allproxy");
286 }
287 } else {
288 die N("Unknown webfetch `%s' !!!\n", $proxy->{type});
289 }
290 @res;
291 }
292  
293 sub _error_msg {
294 my ($name) = @_;
295  
296 my $msg = $? & 127 ? N("%s failed: exited with signal %d", $name, $? & 127) :
297 N("%s failed: exited with %d", $name, $? >> 8);
298 "$msg\n";
299 }
300  
301 sub _error {
302 my ($name) = @_;
303 die _error_msg($name);
304 }
305  
306 sub hide_password {
307 my ($url) = @_;
308 $url =~ s|([^:]*://[^/:\@]*:)[^/:\@]*(\@.*)|$1xxxx$2|; #- if needed...
309 $url;
310 }
311  
312 sub propagate_sync_callback {
313 my $options = shift;
314 if (ref($options) && $options->{callback}) {
315 my $mode = shift;
316 if ($mode =~ /^(?:start|progress|end)$/) {
317 my $file = shift;
318 return $options->{callback}($mode, hide_password($file), @_);
319 } else {
320 return $options->{callback}($mode, @_);
321 }
322 }
323 }
324  
325 sub sync_file {
326 my $options = shift;
327 foreach (@_) {
328 propagate_sync_callback($options, 'start', $_);
329 require urpm::util;
330 urpm::util::copy($_, ref($options) ? $options->{dir} : $options)
331 or die N("copy failed");
332 propagate_sync_callback($options, 'end', $_);
333 }
334 }
335  
336 sub sync_wget {
337 -x "/usr/bin/wget" or die N("wget is missing\n");
338 my $options = shift;
339 $options = { dir => $options } if !ref $options;
340 #- force download to be done in cachedir to avoid polluting cwd.
341 (my $cwd) = getcwd() =~ /(.*)/;
342 chdir $options->{dir};
343 my ($buf, $total, $file) = ('', undef, undef);
344 my $wget_command = join(" ", map { "'$_'" }
345 #- construction of the wget command-line
346 "/usr/bin/wget",
347 ($options->{'limit-rate'} ? "--limit-rate=$options->{'limit-rate'}" : @{[]}),
348 ($options->{resume} ? "--continue" : "--force-clobber"),
349 "--no-timestamping", #- timestamping conflicts with --force-clobber
350 ($options->{proxy} ? set_proxy({ type => "wget", proxy => $options->{proxy} }) : @{[]}),
351 ($options->{retry} ? ('-t', $options->{retry}) : @{[]}),
352 ($options->{callback} ? ("--progress=bar:force", "-o", "-") :
353 $options->{quiet} ? "-q" : @{[]}),
354 "--retr-symlinks",
355 ($options->{"no-certificate-check"} ? "--no-check-certificate" : @{[]}),
356 "--timeout=$CONNECT_TIMEOUT",
357 (defined $options->{'wget-options'} ? split /\s+/, $options->{'wget-options'} : @{[]}),
358 '-P', $options->{dir},
359 @_
360 ) . " |";
361 $options->{debug} and $options->{debug}($wget_command);
362 local $ENV{LC_ALL} = 'C';
363 my $wget_pid = open(my $wget, $wget_command);
364 local $/ = \1; #- read input by only one char, this is slow but very nice (and it works!).
365 local $_;
366 while (<$wget>) {
367 $buf .= $_;
368 if ($_ eq "\r" || $_ eq "\n") {
369 if ($options->{callback}) {
370 if ($buf =~ /^--(\d\d\d\d-\d\d-\d\d )?\d\d:\d\d:\d\d--\s+(\S.*)\n/ms) {
371 my $file_ = $2;
372 if ($file && $file ne $file_) {
373 propagate_sync_callback($options, 'end', $file);
374 undef $file;
375 }
376 ! defined $file and propagate_sync_callback($options, 'start', $file = $file_);
377 } elsif (defined $file && ! defined $total && ($buf =~ /==>\s+RETR/ || $buf =~ /200 OK$/)) {
378 $total = '';
379 } elsif ($buf =~ /^Length:\s*(\d\S*)/) {
380 $total = $1;
381 } elsif (defined $total && $buf =~ m!^\s*(\d+)%.*\s+(\S+/s)\s+((ETA|eta)\s+(.*?)\s*)?[\r\n]$!ms) {
382 my ($percent, $speed, $eta) = ($1, $2, $5);
383 if (propagate_sync_callback($options, 'progress', $file, $percent, $total, $eta, $speed) eq 'canceled') {
384 kill 15, $wget_pid;
385 close $wget;
386 return;
387 }
388 if ($_ eq "\n") {
389 propagate_sync_callback($options, 'end', $file);
390 ($total, $file) = (undef, undef);
391 }
392 }
393 } else {
394 $options->{quiet} or print STDERR $buf;
395 }
396 $buf = '';
397 }
398 }
399 $file and propagate_sync_callback($options, 'end', $file);
400 chdir $cwd;
401 close $wget or _error('wget');
402 }
403  
404 sub sync_curl {
405 -x "/usr/bin/curl" or die N("curl is missing\n");
406 my $options = shift;
407 $options = { dir => $options } if !ref $options;
408 if (defined $options->{'limit-rate'} && $options->{'limit-rate'} =~ /\d$/) {
409 #- use bytes by default
410 $options->{'limit-rate'} .= 'B';
411 }
412 #- force download to be done in cachedir to avoid polluting cwd,
413 #- however for curl, this is mandatory.
414 (my $cwd) = getcwd() =~ /(.*)/;
415 chdir($options->{dir});
416 my (@ftp_files, @other_files);
417 foreach (@_) {
418 my ($proto, $nick, $rest) = m,^(http|ftp)://([^:/]+):(.*),,;
419 if ($nick) { #- escape @ in user names
420 $nick =~ s/@/%40/;
421 $_ = "$proto://$nick:$rest";
422 }
423 if (m|^ftp://.*/([^/]*)$| && file_size($1) > 8192) { #- manage time stamp for large file only
424 push @ftp_files, $_;
425 } else {
426 push @other_files, $_;
427 }
428 }
429 if (@ftp_files) {
430 my ($cur_ftp_file, %ftp_files_info);
431 local $_;
432  
433 eval { require Date::Manip };
434  
435 #- prepare to get back size and time stamp of each file.
436 my $cmd = join(" ", map { "'$_'" } "/usr/bin/curl",
437 "-q", # don't read .curlrc; some toggle options might interfer
438 ($options->{'limit-rate'} ? ("--limit-rate", $options->{'limit-rate'}) : @{[]}),
439 ($options->{proxy} ? set_proxy({ type => "curl", proxy => $options->{proxy} }) : @{[]}),
440 ($options->{retry} ? ('--retry', $options->{retry}) : @{[]}),
441 "--stderr", "-", # redirect everything to stdout
442 "--disable-epsv",
443 "--connect-timeout", $CONNECT_TIMEOUT,
444 "-s", "-I",
445 "--anyauth",
446 (defined $options->{'curl-options'} ? split /\s+/, $options->{'curl-options'} : @{[]}),
447 @ftp_files);
448 $options->{debug} and $options->{debug}($cmd);
449 open my $curl, "$cmd |";
450 while (<$curl>) {
451 if (/Content-Length:\s*(\d+)/) {
452 !$cur_ftp_file || exists($ftp_files_info{$cur_ftp_file}{size})
453 and $cur_ftp_file = shift @ftp_files;
454 $ftp_files_info{$cur_ftp_file}{size} = $1;
455 }
456 if (/Last-Modified:\s*(.*)/) {
457 !$cur_ftp_file || exists($ftp_files_info{$cur_ftp_file}{time})
458 and $cur_ftp_file = shift @ftp_files;
459 eval {
460 $ftp_files_info{$cur_ftp_file}{time} = Date::Manip::ParseDate($1);
461 };
462 }
463 }
464 close $curl or _error('curl');
465  
466 #- now analyse size and time stamp according to what already exists here.
467 if (@ftp_files) {
468 #- re-insert back shifted element of ftp_files, because curl output above
469 #- has not been parsed correctly, so in doubt download them all.
470 push @ftp_files, keys %ftp_files_info;
471 } else {
472 #- for that, it should be clear ftp_files is empty...
473 #- elsewhere, the above work was useless.
474 foreach (keys %ftp_files_info) {
475 my ($lfile) = m|/([^/]*)$| or next; #- strange if we can't parse it correctly.
476 my $ltime = eval { Date::Manip::ParseDate(scalar gmtime((stat $1)[9])) };
477 $ltime && -s $lfile == $ftp_files_info{$_}{size} && $ftp_files_info{$_}{time} eq $ltime
478 or push @ftp_files, $_;
479 }
480 }
481 }
482 #- http files (and other files) are correctly managed by curl wrt conditional download.
483 #- options for ftp files, -R (-O <file>)*
484 #- options for http files, -R (-O <file>)*
485 my $result;
486 if (my @all_files = (
487 (map { (_curl_output_option($_), $_) } @ftp_files),
488 (map { m|/| ? (_curl_output_option($_), $_) : @{[]} } @other_files)))
489 {
490 my @l = (@ftp_files, @other_files);
491 my $cmd = join(" ", map { "'$_'" } "/usr/bin/curl",
492 "-q", # don't read .curlrc; some toggle options might interfer
493 ($options->{'limit-rate'} ? ("--limit-rate", $options->{'limit-rate'}) : @{[]}),
494 ($options->{resume} ? ("--continue-at", "-") : @{[]}),
495 ($options->{proxy} ? set_proxy({ type => "curl", proxy => $options->{proxy} }) : @{[]}),
496 ($options->{retry} ? ('--retry', $options->{retry}) : @{[]}),
497 ($options->{quiet} ? "-s" : @{[]}),
498 ($options->{"no-certificate-check"} ? "-k" : @{[]}),
499 "--location-trusted",
500 "-R",
501 "-f",
502 "--disable-epsv",
503 "--connect-timeout", $CONNECT_TIMEOUT,
504 "--anyauth",
505 (defined $options->{'curl-options'} ? split /\s+/, $options->{'curl-options'} : @{[]}),
506 "--stderr", "-", # redirect everything to stdout
507 @all_files);
508 $options->{debug} and $options->{debug}($cmd);
509 $result = _curl_action($cmd, $options, @l);
510 }
511 chdir $cwd;
512 $result;
513 }
514  
515 sub _curl_output_option {
516 my ($remote_file) = @_;
517 #- curl 7.86 changed the -O option to remove any trailing query part from the output
518 #- file name. So if we have a query part, use the -o option instead. If not, use the
519 #- -O option to avoid bloating the command line.
520 my $local_file = basename($remote_file);
521 $local_file =~ /\?/ ? ('-o', $local_file) : '-O';
522 }
523  
524 sub _curl_action {
525 my ($cmd, $options, @l) = @_;
526
527 my ($buf, $file); $buf = '';
528 my $curl_pid = open(my $curl, "$cmd |");
529 local $/ = \1; #- read input by only one char, this is slow but very nice (and it works!).
530 local $_;
531 while (<$curl>) {
532 $buf .= $_;
533 if ($_ eq "\r" || $_ eq "\n") {
534 if ($options->{callback}) {
535 unless (defined $file) {
536 $file = shift @l;
537 propagate_sync_callback($options, 'start', $file);
538 }
539 if (my ($percent, $total, $eta, $speed) = $buf =~ /^\s*(\d+)\s+(\S+)[^\r\n]*\s+(\S+)\s+(\S+)\s*[\r\n]$/ms) {
540 $speed =~ s/^-//;
541 if (propagate_sync_callback($options, 'progress', $file, $percent, $total, $eta, $speed) eq 'canceled') {
542 kill 15, $curl_pid;
543 close $curl;
544 die N("curl failed: download canceled\n");
545 }
546 #- this checks that download has actually started
547 if ($_ eq "\n"
548 && !($speed == 0 && $percent == 100 && index($eta, '--') >= 0) #- work around bug 13685
549 ) {
550 propagate_sync_callback($options, 'end', $file);
551 $file = undef;
552 }
553 } elsif ($buf =~ /^curl:/) { #- likely to be an error reported by curl
554 local $/ = "\n";
555 chomp $buf;
556 propagate_sync_callback($options, 'error', $file, $buf);
557 }
558 } else {
559 $options->{quiet} or print STDERR $buf;
560 }
561 $buf = '';
562 }
563 }
564 close $curl or _error('curl');
565 }
566  
567 sub _calc_limit_rate {
568 my $limit_rate = $_[0];
569 for ($limit_rate) {
570 /^(\d+)$/ and $limit_rate = int $1/1024, last;
571 /^(\d+)[kK]$/ and $limit_rate = $1, last;
572 /^(\d+)[mM]$/ and $limit_rate = 1024*$1, last;
573 /^(\d+)[gG]$/ and $limit_rate = 1024*1024*$1, last;
574 }
575 $limit_rate;
576 }
577  
578 sub sync_rsync {
579 -x "/usr/bin/rsync" or die N("rsync is missing\n");
580 my $options = shift;
581 $options = { dir => $options } if !ref $options;
582 #- force download to be done in cachedir to avoid polluting cwd.
583 (my $cwd) = getcwd() =~ /(.*)/;
584 chdir($options->{dir});
585 my $limit_rate = _calc_limit_rate($options->{'limit-rate'});
586 foreach (@_) {
587 my $count = 10; #- retry count on error (if file exists).
588 my $basename = basename($_);
589 my $file = m!^rsync://([^/]*::.*)! ? $1 : $_;
590 propagate_sync_callback($options, 'start', $file);
591 do {
592 local $_;
593 my $buf = '';
594 my $cmd = join(" ", "/usr/bin/rsync",
595 ($limit_rate ? "--bwlimit=$limit_rate" : @{[]}),
596 ($options->{quiet} ? qw(-q) : qw(--progress -v --no-human-readable)),
597 ($options->{compress} ? qw(-z) : @{[]}),
598 ($options->{ssh} ? qq(-e $options->{ssh}) :
599 ("--timeout=$CONNECT_TIMEOUT",
600 "--contimeout=$CONNECT_TIMEOUT")),
601 qw(--partial --no-whole-file --no-motd --copy-links),
602 (defined $options->{'rsync-options'} ? split /\s+/, $options->{'rsync-options'} : @{[]}),
603 "'$file' '$options->{dir}' 2>&1");
604 $options->{debug} and $options->{debug}($cmd);
605 open(my $rsync, "$cmd |");
606 local $/ = \1; #- read input by only one char, this is slow but very nice (and it works!).
607 local $_;
608 while (<$rsync>) {
609 $buf .= $_;
610 if ($_ eq "\r" || $_ eq "\n") {
611 if ($options->{callback}) {
612 if (my ($percent, $speed) = $buf =~ /^\s*\d+\s+(\d+)%\s+(\S+)\s+/) {
613 propagate_sync_callback($options, 'progress', $file, $percent, undef, undef, $speed);
614 } else {
615 $options->{debug} and $options->{debug}($buf);
616 }
617 } else {
618 $options->{quiet} or print STDERR $buf;
619 $options->{debug} and $options->{debug}($buf);
620 }
621 $buf = '';
622 }
623 }
624 close $rsync;
625 } while ($? != 0 && --$count > 0 && -e $options->{dir} . "/$basename");
626 propagate_sync_callback($options, 'end', $file);
627 }
628 chdir $cwd;
629 $? == 0 or _error('rsync');
630 }
631  
632 our $SSH_PATH;
633 sub _init_ssh_path() {
634 foreach (qw(/usr/bin/ssh /bin/ssh)) {
635 -x $_ and $SSH_PATH = $_;
636 next;
637 }
638 }
639  
640 #- Don't generate a tmp dir name, so when we restart urpmi, the old ssh
641 #- connection can be reused
642 our $SSH_CONTROL_DIR = $ENV{TMP} || $ENV{TMPDIR} || '/tmp';
643 our $SSH_CONTROL_OPTION;
644  
645 sub sync_ssh {
646 $SSH_PATH or _init_ssh_path();
647 $SSH_PATH or die N("ssh is missing\n");
648 my $options = shift;
649 $options = { dir => $options } if !ref $options;
650 unless ($options->{'rsync-options'} =~ /(?:-e|--rsh)\b/) {
651 my ($server, $user) = ('', getpwuid($<));
652 $_[0] =~ /((?:\w|\.)*):/ and $server = $1;
653 $_[0] =~ /((?:\w|-)*)@/ and $user = $1;
654 $SSH_CONTROL_OPTION = "-o 'ControlPath $SSH_CONTROL_DIR/ssh-urpmi-$$-%h_%p_%r' -o 'ControlMaster auto'";
655 if (start_ssh_master($server, $user)) {
656 $options->{ssh} = qq("$SSH_PATH $SSH_CONTROL_OPTION");
657 } else {
658 #- can't start master, use single connection
659 $options->{ssh} = $SSH_PATH;
660 }
661 }
662 sync_rsync($options, @_);
663 }
664  
665 sub sync_prozilla {
666 -x "/usr/bin/proz" or die N("prozilla is missing\n");
667 my $options = shift;
668 $options = { dir => $options } if !ref $options;
669 #- force download to be done in cachedir to avoid polluting cwd.
670 (my $cwd) = getcwd() =~ /(.*)/;
671 chdir $options->{dir};
672 my $proz_command = join(" ", map { "'$_'" }
673 "/usr/bin/proz",
674 "--no-curses",
675 (defined $options->{'prozilla-options'} ? split /\s+/, $options->{'prozilla-options'} : @{[]}),
676 @_
677 );
678 my $ret = system($proz_command);
679 chdir $cwd;
680 if ($ret) {
681 if ($? == -1) {
682 die N("Couldn't execute prozilla\n");
683 } else {
684 _error('prozilla');
685 }
686 }
687 }
688  
689 sub sync_aria2 {
690 my ($urpm, $medium, $rel_files, $options) = @_;
691  
692 -x "/usr/bin/aria2c" or die N("aria2 is missing\n");
693  
694 #- force download to be done in cachedir to avoid polluting cwd.
695 (my $cwd) = getcwd() =~ /(.*)/;
696 chdir $options->{dir};
697  
698 my $stat_file = ($< ? $ENV{HOME} : '/root') . '/.aria2-adaptive-stats';
699  
700 my $aria2c_command = join(" ", map { "'$_'" }
701 "/usr/bin/aria2c", $options->{debug} ? ('--log', "$options->{dir}/.aria2.log") : @{[]},
702 '--auto-file-renaming=false',
703 '--ftp-pasv',
704 '--summary-interval=1',
705 '--follow-metalink=mem',
706 $medium->{mirrorlist} ? (
707 '--metalink-enable-unique-protocol=true', # do not try to connect to the same server using the same protocol
708 '--metalink-preferred-protocol=http', # try http as first protocol as they're stateless and
709 # will put less strain on ie. the ftp servers which connections
710 # are statefull for, causing unhappy mirror admins complaining
711 # about increase of connections, increasing resource usage.
712 '--max-tries=5', # nb: not using $options->{retry}
713 '--lowest-speed-limit=20K', "--timeout", 3,
714 '--split=3', # maximum number of servers to use for one download
715 '--uri-selector=adaptive', "--server-stat-if=$stat_file", "--server-stat-of=$stat_file",
716 $options->{is_versioned} ? @{[]} : '--max-file-not-found=9', # number of not found errors on different servers before aborting file download
717 '--connect-timeout=6', # $CONNECT_TIMEOUT,
718 ) : @{[]},
719 '-Z', '-j1',
720 ($options->{'limit-rate'} ? "--max-download-limit=" . $options->{'limit-rate'} : @{[]}),
721 ($options->{resume} ? "--continue" : "--allow-overwrite=true"),
722 ($options->{proxy} ? set_proxy({ type => "aria2", proxy => $options->{proxy} }) : @{[]}),
723 ($options->{"no-certificate-check"} ? "--check-certificate=false" : @{[]}),
724 (defined $options->{'aria2-options'} ? split /\s+/, $options->{'aria2-options'} : @{[]}),
725 _create_metalink_($urpm, $medium, $rel_files, $options));
726  
727 $options->{debug} and $options->{debug}($aria2c_command);
728  
729 local $ENV{LC_ALL} = 'C';
730 my $aria2_pid = open(my $aria2, "$aria2c_command |");
731  
732 _parse_aria2_output($options, $aria2, $aria2_pid, $medium, $rel_files);
733  
734 chdir $cwd;
735 if (!close $aria2) {
736 my $raw_msg = _error_msg('aria2');
737 my $want_retry;
738 if (!$options->{is_retry} & $options->{is_versioned}) {
739 $want_retry = 1;
740 } else {
741 my $msg = N("Failed to download %s", $rel_files->[0]);
742 $want_retry = $options->{ask_retry} && $options->{ask_retry}($raw_msg, $msg);
743 }
744 if ($want_retry) {
745 $options->{is_retry}++;
746 $options->{debug} and $options->{debug}("retrying ($options->{is_retry})");
747 goto &sync_aria2;
748 }
749 die $raw_msg;
750 }
751 }
752  
753 sub _parse_aria2_output {
754 my ($options, $aria2, $aria2_pid, $medium, $rel_files) = @_;
755  
756 my ($buf, $_total, $file) = ('', undef, undef);
757  
758 local $/ = \1; #- read input by only one char, this is slow but very nice (and it works!).
759 local $_;
760  
761 while (<$aria2>) {
762 if ($_ eq "\r" || $_ eq "\n") {
763 $options->{debug}("aria2c: $buf") if $options->{debug};
764 if ($options->{callback}) {
765 if (!defined($file) && @$rel_files) {
766 $file = $medium->{mirrorlist} ?
767 $medium->{mirrorlist} . ': ' . $medium->{'with-dir'} . "/$rel_files->[0]" :
768 "$medium->{url}/$rel_files->[0]";
769 propagate_sync_callback($options, 'start', $file)
770 if !$options->{is_retry};
771 }
772  
773 # aria2c 1.16 and beyond:
774 # parses aria2c: [#2c8dae 496KiB/830KiB(59%) CN:1 DL:84KiB ETA:3s]
775 #
776 # using multiline mode and comments for better readability:
777 #
778 if ($buf =~ m!
779 ^\[\#[\dA-Fa-f]+ # match #2c8dae
780 \s+
781 ([\d\.]+\w*) # Match 496KiB
782 /
783 ([\d\.]+\w*) # Match 830KiB
784 \s* \( (\d+) % \) # Match (59%)
785 \s+
786 CN:(\S+) # Match CN:1
787 \s+
788 DL:(\S+) # Match DL:84KiB
789 \s+
790 ETA:(\w+)
791 \]$
792 !msx
793 )
794 {
795 my ($total, $percent, $speed, $eta) = ($2, $3, $5, $6);
796 #- $1 = current downloaded size, $4 = connections
797 if (propagate_sync_callback($options, 'progress', $file, $percent, $total, $eta, $speed) eq 'canceled') {
798 kill 15, $aria2_pid;
799 close $aria2;
800 return;
801 }
802 }
803 if ($buf =~ m!Download\scomplete:\s/!) {
804 propagate_sync_callback($options, 'end', $file);
805 shift @$rel_files;
806 delete $options->{is_retry};
807 $file = undef;
808 } elsif ($buf =~ /ERR\|(.*)/) {
809 propagate_sync_callback($options, 'error', $file, $1);
810 }
811 } else {
812 $options->{quiet} or print STDERR "$buf\n";
813 }
814 $buf = '';
815 } else {
816 $buf .= $_;
817 }
818 }
819 }
820  
821 sub start_ssh_master {
822 my ($server, $user) = @_;
823 $server or return 0;
824 if (!check_ssh_master($server, $user)) {
825 system(qq($SSH_PATH -f -N $SSH_CONTROL_OPTION -M $user\@$server));
826 return ! $?;
827 }
828 return 1;
829 }
830  
831 sub check_ssh_master {
832 my ($server, $user) = @_;
833 system(qq($SSH_PATH -q -f -N $SSH_CONTROL_OPTION $user\@$server -O check));
834 return ! $?;
835 }
836  
837 END {
838 #- remove ssh persistent connections
839 foreach my $socket (glob "$SSH_CONTROL_DIR/ssh-urpmi-$$-*") {
840 my ($server, $login) = $socket =~ /ssh-urpmi-\d+-([^_]+)_\d+_(.*)$/ or next;
841 system($SSH_PATH, '-q', '-f', '-N', '-o', "ControlPath $socket", '-O', 'exit', "$login\@$server");
842 }
843 }
844  
845 #- get the width of the terminal
846 my $wchar = 79;
847 if (-t *STDOUT) {
848 eval {
849 require Term::ReadKey;
850 ($wchar) = Term::ReadKey::GetTerminalSize();
851 --$wchar;
852 };
853 }
854  
855 sub progress_text {
856 my ($mode, $percent, $total, $eta, $speed) = @_;
857 $mode eq 'progress' ?
858 (defined $total && defined $eta ?
859 N(" %s%% of %s completed, ETA = %s, speed = %s", $percent, $total, $eta, $speed) :
860 N(" %s%% completed, speed = %s", $percent, $speed)) : '';
861 }
862  
863 =item sync_logger($mode, $file, $percent, $_total, $_eta, $_speed)
864  
865 Default logger (callback) suitable for sync operation on STDERR only.
866  
867 =cut
868  
869 sub sync_logger {
870 my ($mode, $file, $percent, $total, $eta, $speed) = @_;
871 if ($mode eq 'start') {
872 print STDERR " $file\n";
873 } elsif ($mode eq 'progress') {
874 my $text = progress_text($mode, $percent, $total, $eta, $speed);
875 if (length($text) > $wchar) { $text = substr($text, 0, $wchar) }
876 if (bytes::length($text) < $wchar) {
877 # clearing more than needed in case the terminal is not handling utf8 and we have a utf8 string
878 print STDERR $text, " " x ($wchar - bytes::length($text)), "\r";
879 } else {
880 # clearing all the line first since we can't really know the "length" of the string
881 print STDERR " " x $wchar, "\r", $text, "\r";
882 }
883 } elsif ($mode eq 'end') {
884 print STDERR " " x $wchar, "\r";
885 } elsif ($mode eq 'error') {
886 #- error is 3rd argument, saved in $percent
887 print STDERR N("...retrieving failed: %s", $percent), "\n";
888 }
889 }
890  
891 =item requested_ftp_http_downloader($urpm, $medium)
892  
893 Return the downloader program to use (whether it pas provided on the
894 command line or in the config file).
895  
896 =cut
897  
898 sub requested_ftp_http_downloader {
899 my ($urpm, $medium) = @_;
900  
901 $urpm->{options}{downloader} || #- cmd-line switch
902 $medium && $medium->{downloader} ||
903 $urpm->{global_config}{downloader} || "";
904 }
905  
906 sub parse_url_with_login {
907 my ($url) = @_;
908 $url =~ m!([^:]*)://([^/:]*)(:([^/:\@]*))?\@([^/]*)(.*)! && $1 ne 'ssh' &&
909 { proto => $1, login => $2, password => $4, machine => $5, dir => $6 };
910 }
911 sub url_obscuring_password {
912 my ($url) = @_;
913 my $u = parse_url_with_login($url);
914 if ($u && $u->{password}) {
915 sprintf('%s://xxx:xxx@%s%s', $u->{proto}, $u->{machine}, $u->{dir});
916 } else {
917 $url;
918 }
919 }
920  
921 #- $medium can be undef
922 sub _all_options {
923 my ($urpm, $medium, $options) = @_;
924  
925 my %all_options = (
926 dir => "$urpm->{cachedir}/partial",
927 proxy => get_proxy_($urpm),
928 metalink => $medium->{mirrorlist},
929 $medium->{"disable-certificate-check"} ? "no-certificate-check" : @{[]},
930 $urpm->{debug} ? (debug => $urpm->{debug}) : @{[]},
931 %$options,
932 );
933 foreach my $cpt (qw(compress limit-rate retry wget-options curl-options rsync-options prozilla-options aria2-options metalink)) {
934 $all_options{$cpt} = $urpm->{options}{$cpt} if defined $urpm->{options}{$cpt};
935 }
936 \%all_options;
937 }
938  
939 sub sync_rel {
940 my ($urpm, $medium, $rel_files, %options) = @_;
941  
942 my @files = map { reduce_pathname("$medium->{url}/$_") } @$rel_files;
943  
944 my $files_text = join(' ', (use_metalink($urpm, $medium) ? ($medium->{mirrorlist}, $medium->{'with-dir'}) : url_obscuring_password($medium->{url})), @$rel_files);
945 $urpm->{debug} and $urpm->{debug}(N("retrieving %s", $files_text));
946  
947 my $all_options = _all_options($urpm, $medium, \%options);
948 my @result_files = map { $all_options->{dir} . '/' . basename($_) } @$rel_files;
949 unlink @result_files if $all_options->{preclean};
950  
951 #- The files must be world-readable, else mgaapplet and urpm* commands run as
952 #- a normal user won't be able to read them. We enforce umask here in the case
953 #- where the msec security level is set to 'secure' (which means umask 077)
954 #- or where we are run from a gdm-x-session (mga#24636)
955 my $old_umask = umask 0022;
956  
957 (my $cwd) = getcwd() =~ /(.*)/;
958 eval { _sync_webfetch_raw($urpm, $medium, $rel_files, \@files, $all_options) };
959 my $err = $@;
960 chdir $cwd;
961 umask $old_umask;
962 if (!$err) {
963 $urpm->{log}(N("retrieved %s", $files_text));
964 \@result_files;
965 } else {
966 $urpm->{log}("error: $err");
967 # don't leave partial download
968 unlink @result_files;
969 undef;
970 }
971 }
972  
973 sub sync_rel_one {
974 my ($urpm, $medium, $rel_file, %options) = @_;
975  
976 my $files = sync_rel($urpm, $medium, [$rel_file], %options) or return;
977 $files->[0];
978 }
979  
980 =item sync_url($urpm, $url, %options)
981  
982 Retrieve a file from the network and return the local cached file path.
983  
984 =cut
985  
986 sub sync_url {
987 my ($urpm, $url, %options) = @_;
988 sync_rel_one($urpm, { url => dirname($url), disable_metalink => $options{disable_metalink} }, basename($url), %options);
989 }
990  
991 sub sync_rel_to {
992 my ($urpm, $medium, $rel_file, $dest_file, %options) = @_;
993  
994 my $files = sync_rel($urpm, $medium, [$rel_file], %options) or return undef;
995 my $result_file = $files->[0];
996 $result_file ne $dest_file or rename($result_file, $dest_file) or return;
997 $result_file;
998 }
999  
1000 =item get_content($urpm, $url, %o_options)
1001  
1002 Retrieve a file and return its content.
1003  
1004 =cut
1005  
1006 sub get_content {
1007 my ($urpm, $url, %o_options) = @_;
1008  
1009 my $file = sync_url($urpm, $url, %o_options, quiet => 1, preclean => 1) or return;
1010  
1011 my @l = cat_($file);
1012 unlink $file;
1013  
1014 wantarray() ? @l : join('', @l);
1015 }
1016
1017  
1018 #- syncing algorithms.
1019 #-
1020 #- nb: $files is constructed from $rel_files using $medium
1021 sub _sync_webfetch_raw {
1022 my ($urpm, $medium, $rel_files, $files, $options) = @_;
1023  
1024 #- currently ftp and http protocols are managed by curl or wget,
1025 #- ssh and rsync protocols are managed by rsync *AND* ssh.
1026 my $proto = urpm::protocol_from_url($medium->{url}) or die N("unknown protocol defined for %s", $medium->{url});
1027  
1028 if ($proto eq 'file') {
1029 my @l = map { urpm::file_from_local_url($_) } @$files;
1030 eval { sync_file($options, @l) };
1031 $urpm->{fatal}(10, $@) if $@;
1032 } elsif ($proto eq 'rsync') {
1033 sync_rsync($options, @$files);
1034 } elsif (member($proto, 'ftp', 'http', 'https') || $options->{metalink}) {
1035  
1036 my $preferred = preferred_downloader($urpm, $medium, \$options->{metalink});
1037 if ($preferred eq 'aria2') {
1038 sync_aria2($urpm, $medium, $rel_files, $options);
1039 } else {
1040 my $sync = $urpm::download::{"sync_$preferred"} or die N("no webfetch found, supported webfetch are: %s\n", join(", ", ftp_http_downloaders()));
1041  
1042 my @l = @$files;
1043 while (@l) {
1044 my $half_MAX_ARG = 131072 / 2;
1045 # restrict the number of elements so that it fits on cmdline of curl/wget/proz/aria2c
1046 my $n = 0;
1047 for (my $len = 0; $n < @l && $len < $half_MAX_ARG; $len += length($l[$n++])) {}
1048 $sync->($options, splice(@l, 0, $n));
1049 }
1050 }
1051 } elsif ($proto eq 'ssh') {
1052 my @ssh_files = map { m!^ssh://([^/]*)(.*)! ? "$1:$2" : @{[]} } @$files;
1053 sync_ssh($options, @ssh_files);
1054 } else {
1055 die N("unable to handle protocol: %s", $proto);
1056 }
1057 }
1058  
1059 sub _take_n_elem {
1060 my ($n, @l) = @_;
1061 @l < $n ? @l : @l[0 .. $n-1];
1062 }
1063  
1064 sub _create_one_metalink_line {
1065 my ($medium, $mirror, $rel_file, $counter) = @_;
1066  
1067 my $type = urpm::protocol_from_url($mirror->{url});
1068  
1069 # If more than 100 mirrors, give all the remaining mirrors a priority of 0
1070 my $preference = max(0, 100 - $counter);
1071  
1072 my @options = (qq(type="$type"), qq(preference="$preference"));
1073 # Not supported in metalinks
1074 #if (@$list[$i]->{bw}) {
1075 # push @options, 'bandwidth="' . @$list[$i]->{bw} . '"';
1076 # }
1077 # Supported in metalinks, but no longer used in mirror list..?
1078 if ($mirror->{connections}) {
1079 push @options, qq(maxconnections="$mirror->{connections}");
1080 }
1081 push @options, 'location="' . lc($mirror->{zone}) . '"';
1082 my $base = urpm::mirrors::_add__with_dir($mirror->{url}, $medium->{'with-dir'});
1083 sprintf('<url %s>%s/%s</url>', join(' ', @options), $base, $rel_file);
1084 }
1085  
1086 sub _create_metalink_ {
1087 my ($urpm, $medium, $rel_files, $options) = @_;
1088 # Don't create a metalink when downloading mirror list
1089 $medium or return;
1090  
1091 # only use the 8 best mirrors, then we let aria2 choose
1092 require urpm::mirrors;
1093 my @mirrors = $medium->{mirrorlist} ? (map {
1094 # aria2 doesn't handle rsync
1095 my @l = grep { urpm::protocol_from_url($_->{url}) ne 'rsync' } @$_;
1096 _take_n_elem(8, @l);
1097 } urpm::mirrors::list_urls($urpm, $medium, '')) : { url => $medium->{url} };
1098
1099 my $metalinkfile = "$urpm->{cachedir}/$options->{media}.metalink";
1100 # Even if not required by metalink spec, this line is needed at top of
1101 # metalink file, otherwise aria2 won't be able to autodetect it..
1102 my @metalink = (
1103 '<?xml version="1.0" encoding="utf-8"?>',
1104 '<metalink version="3.0" generator="URPMI" xmlns="http://www.metalinker.org/">',
1105 '<files>',
1106 );
1107  
1108 foreach my $rel_file (@$rel_files) {
1109 my $i = 0;
1110 my @lines = map {
1111 $i++;
1112 _create_one_metalink_line($medium, $_, $rel_file, $i);
1113 } @mirrors;
1114  
1115 push @metalink, map { "\t$_" }
1116 sprintf('<file name="%s"><resources>', basename($rel_file)),
1117 (map { "\t$_" } @lines),
1118 '</resources></file>';
1119 }
1120 push @metalink, '</files>', '</metalink>';
1121
1122 output_safe($metalinkfile, join('', map { "$_\n" } @metalink));
1123 $metalinkfile;
1124 }
1125  
1126 1;
1127  
1128  
1129 =back
1130  
1131 =head1 COPYRIGHT
1132  
1133 Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005 MandrakeSoft SA
1134  
1135 Copyright (C) 2005-2010 Mandriva SA
1136  
1137 Copyright (C) 2011-2020 Mageia
1138  
1139 =cut
« urpm/cdrom.pm ↑ Summary urpm/get_pkgs.pm »