urpm/parallel.pm

filter: Statementstmt 66.6% 4 / 6 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 2 / 2 Podpod n/a totaltotal 75.0% 6 / 8 CC 1 SCAR 1.4
CC
1
Cov
67%
BEGIN
1.3
BEGIN
1.0
CRAP 1.1
« urpm/media.pm ↑ Summary urpm/removable.pm »
1 package urpm::parallel;
2  
3 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 0 ▶use urpm;
Subroutine BEGIN: called CC=1 SCAR=2.6
5 use urpm::util qw(any basename cat_);
6 use urpm::msg;
7  
8 =head1 NAME
9  
10 urpm::parallel - Run an urpmi command in parallel on a number of hosts
11  
12 =head1 SYNOPSIS
13  
14 This enables to run urpmi command on several computers at once.
15  
16 Two backends are available: L<urpm::parallel_ka_run> and L <urpm::parallel_ssh>
17  
18 =head1 DESCRIPTION
19  
20 =over
21  
22 =cut
23  
24  
25 sub configure {
26 my ($urpm, $alias) = @_;
27 my @parallel_options;
28 #- read parallel configuration
29 foreach (cat_("/etc/urpmi/parallel.cfg")) {
30 chomp; s/#.*$//; s/^\s*//; s/\s*$//;
31 /\s*([^:]*):(.*)/ or $urpm->{error}(N("unable to parse \"%s\" in file [%s]", $_, "/etc/urpmi/parallel.cfg")), next;
32 $1 eq $alias and push @parallel_options, $2;
33 }
34 #- if a configuration option has been found, use it; else fatal error.
35 my $parallel_handler;
36 if (@parallel_options) {
37 foreach my $dir (grep { -d $_ } map { "$_/urpm" } @INC) {
38 foreach my $pm (grep { -f $_ } glob("$dir/parallel_*.pm")) {
39 #- load parallel modules
40 $urpm->{log}->(N("examining parallel handler in file [%s]", $pm));
41 # perl_checker: require urpm::parallel_ka_run
42 # perl_checker: require urpm::parallel_ssh
43 eval { require $pm; $parallel_handler = $urpm->handle_parallel_options(join("\n", @parallel_options)) };
44 $parallel_handler and last;
45 }
46 $parallel_handler and last;
47 }
48 }
49 if ($parallel_handler) {
50 if ($parallel_handler->{nodes}) {
51 $urpm->{log}->(N("found parallel handler for nodes: %s", join(', ', keys %{$parallel_handler->{nodes}})));
52 }
53 $urpm->{parallel_handler} = $parallel_handler;
54 } else {
55 $urpm->{fatal}(1, N("unable to use parallel option \"%s\"", $alias));
56 }
57 }
58  
59 sub resolve_dependencies {
60 my ($urpm, $state, $requested, %options) = @_;
61  
62 #- build the global synthesis file first.
63 my $file = "$urpm->{cachedir}/partial/parallel.cz";
64 unlink $file;
65 foreach (@{$urpm->{media}}) {
66 urpm::media::is_valid_medium($_) or next;
67 my $f = urpm::media::any_synthesis($urpm, $_);
68 system "cat '$f' >> '$file'";
69 }
70 #- let each node determine what is requested, according to handler given.
71 $urpm->{parallel_handler}->parallel_resolve_dependencies($file, $urpm, $state, $requested, %options);
72 }
73  
74 =item remove($urpm, $remove, %options)
75  
76 remove packages from node as remembered according to resolving done.
77  
78 =cut
79  
80 sub remove {
81 my ($urpm, $remove, %options) = @_;
82 my $state = {};
83 my $callback = sub { $urpm->{fatal}(1, "internal distributed remove fatal error") };
84 $urpm->{parallel_handler}->parallel_find_remove($urpm, $state, $remove, %options,
85 callback_notfound => undef,
86 callback_fuzzy => $callback,
87 callback_base => $callback,
88 );
89 }
90  
91 =item parallel_find_remove($parallel, $urpm, $state, $l, %options)
92  
93 parallel find_packages_to_remove
94  
95 =cut
96  
97 sub parallel_find_remove {
98 my ($parallel, $urpm, $state, $l, %options) = @_;
99  
100 my ($test, $pkgs) = _find_remove_pre($urpm, $state, %options);
101 $pkgs and return @$pkgs;
102  
103 my (%bad_nodes, %base_to_remove, %notfound);
104  
105 #- now try an iteration of urpme.
106 $parallel->urpm_popen($urpm, 'urpme', "--auto $test" . join(' ', map { "'$_'" } @$l) . ' 2>&1', sub {
107 my ($node, $s) = @_;
108  
109 _parse_urpme_output($urpm, $state, $node, $s,
110 \%notfound, \%base_to_remove, \%bad_nodes, %options);
111 });
112  
113 #- check base, which has been delayed until there.
114 if ($options{callback_base} && %base_to_remove) {
115 $options{callback_base}->($urpm, keys %base_to_remove) or return ();
116 }
117  
118 #- build error list contains all the error returned by each node.
119 $urpm->{error_remove} = [ map {
120 my $msg = N("on node %s", $_);
121 map { "$msg, $_" } @{$bad_nodes{$_}};
122 } keys %bad_nodes ];
123  
124 #- if at least one node has the package, it should be seen as unknown...
125 delete @notfound{map { /^(.*)-[^-]*-[^-]*$/ } keys %{$state->{rejected}}};
126 if (%notfound) {
127 $options{callback_notfound} && $options{callback_notfound}->($urpm, keys %notfound)
128 or delete $state->{rejected};
129 }
130  
131 keys %{$state->{rejected}};
132 }
133  
134  
135 =item parallel_register_rpms($parallel, $urpm, @files)
136  
137 parallel copy
138  
139 =cut
140  
141 sub parallel_register_rpms {
142 my ($parallel, $urpm, @files) = @_;
143  
144 $parallel->copy_to_dir($urpm, @files, "$urpm->{cachedir}/rpms");
145  
146 #- keep trace of direct files.
147 $parallel->{line} .=
148 join(' ',
149 map { "'$_'" }
150 map { "$urpm->{cachedir}/rpms/" . basename($_) } @files);
151 }
152  
153 sub _find_remove_pre {
154 my ($urpm, $state, %options) = @_;
155  
156 #- keep in mind if the previous selection is still active, it avoids
157 #- to re-start urpme --test on each node.
158 if ($options{find_packages_to_remove}) {
159 delete $state->{rejected};
160 delete $urpm->{error_remove};
161 '--test ';
162 } elsif (@{$urpm->{error_remove} || []}) {
163 undef, $urpm->{error_remove};
164 } elsif ($options{test}) {
165 #- no need to restart what has been started before.
166 undef, [ keys %{$state->{rejected}} ];
167 } else {
168 '--force ';
169 }
170 }
171  
172 sub _parse_urpme_output {
173 my ($urpm, $state, $node, $s, $notfound, $base_to_remove, $bad_nodes, %options) = @_;
174  
175 if ($s =~ /^\s*$/) {
176 } elsif ($s =~ /unknown packages?:? (.*)/) {
177 #- remember unknown packages from the node, because it should not be a fatal error
178 #- if other nodes have it.
179 $notfound->{$_} = undef foreach split ", ", $1;
180 } elsif ($s =~ /The following packages contain ([^:]*): (.*)/) {
181 $options{callback_fuzzy} && $options{callback_fuzzy}->($urpm, $1, split(" ", $2))
182 or delete($state->{rejected}), return 'stop_parse';
183 } elsif ($s =~ /removing package (.*) will break your system/) {
184 $base_to_remove->{$1} = undef;
185 } elsif ($s =~ /^(removing|testing removal of) (.*)/) {
186 foreach my $fn (split ' ', $2) {
187 $state->{rejected}{$fn}{removed} = 1;
188 $state->{rejected}{$fn}{nodes}{$node} = undef;
189 }
190 } elsif ($s =~ /Remov(?:al|ing) failed/) {
191 $bad_nodes->{$node} = [];
192 } elsif (exists $bad_nodes->{$node}) {
193 $s =~ /^\s+(.+)/ and push @{$bad_nodes->{$node}}, $1;
194 }
195 return;
196 }
197  
198 sub _parse_urpmq_output {
199 my ($urpm, $state, $node, $s, $cont, $chosen, %options) = @_;
200  
201 chomp $s;
202  
203 if (my ($action, $what) = $s =~ /^\@([^\@]*)\@(.*)/) {
204 if ($action eq 'removing') {
205 $state->{rejected}{$what}{removed} = 1;
206 $state->{rejected}{$what}{nodes}{$node} = undef;
207 }
208 } elsif ($s =~ /\|/) {
209 #- distant urpmq returned a choices, check if it has already been chosen
210 #- or continue iteration to make sure no more choices are left.
211 $$cont ||= 1; #- invalid transitory state (still choices is strange here if next sentence is not executed).
212 unless (any { exists $chosen->{$_} } split /\|/, $s) {
213 my $choice = $options{callback_choices}->($urpm, undef, $state, [ map { $urpm->search($_) } split /\|/, $s ]);
214 if ($choice) {
215 $chosen->{scalar $choice->fullname} = $choice;
216 #- it has not yet been chosen so need to ask user.
217 $$cont = 2;
218 } else {
219 #- no choices resolved, so forget it (no choices means no choices at all).
220 $$cont = 0;
221 }
222 }
223 } else {
224 my $pkg = $urpm->search($s) or return; #TODO
225 $state->{selected}{$pkg->id}{$node} = $s;
226 }
227 }
228  
229 =item parallel_resolve_dependencies($parallel, $synthesis, $urpm, $state, $requested, %options)
230  
231 parallel resolve_dependencies on each node
232  
233 =cut
234  
235 sub parallel_resolve_dependencies {
236 my ($parallel, $synthesis, $urpm, $state, $requested, %options) = @_;
237  
238 #- first propagate the synthesis file to all machines
239 $parallel->propagate_file($urpm, $synthesis);
240  
241 $parallel->{synthesis} = $synthesis;
242  
243 my $line = _simple_resolve_dependencies($parallel, $urpm, $state, $requested, %options);
244  
245 #- execute urpmq to determine packages to install.
246 my ($cont, %chosen);
247 do {
248 $cont = 0; #- prepare to stop iteration.
249 #- the following state should be cleaned for each iteration.
250 delete $state->{selected};
251 #- now try an iteration of urpmq.
252 my @errors = $parallel->urpm_popen($urpm, 'urpmq', "--synthesis $synthesis -fmc $line " . join(' ', keys %chosen), sub {
253 my ($node, $s) = @_;
254 _parse_urpmq_output($urpm, $state, $node, $s, \$cont, \%chosen, %options);
255 undef;
256 });
257 @errors and $urpm->{fatal}(1, join("\n", @errors));
258 #- check for internal error of resolution.
259 $cont == 1 and die "internal distant urpmq error on choice not taken";
260 } while $cont;
261  
262 #- keep trace of what has been chosen finally (if any).
263 $parallel->{line} = join(' ', $line, keys %chosen);
264 }
265  
266 =item _simple_resolve_dependencies($parallel, $urpm, $state, $requested, %options)
267  
268 Compute command line of urpm? tools
269  
270 =cut
271  
272 sub _simple_resolve_dependencies {
273 my ($parallel, $urpm, $state, $requested, %options) = @_;
274  
275 my @pkgs;
276 foreach (keys %$requested) {
277 if (/\|/) {
278 #- taken from URPM::Resolve to filter out choices, not complete though.
279 my @packages = $urpm->find_candidate_packages_($_);
280 foreach (@packages) {
281 my ($best_requested, $best);
282 foreach (@$_) {
283 exists $state->{selected}{$_->id} and $best_requested = $_, last;
284 if ($best_requested) {
285 if ($best_requested && $best_requested != $_) {
286 $_->compare_pkg($best_requested) > 0 and $best_requested = $_;
287 } else {
288 $best_requested = $_;
289 }
290 } elsif ($best && $best != $_) {
291 $_->compare_pkg($best) > 0 and $best = $_;
292 } else {
293 $best = $_;
294 }
295 }
296 $_ = $best_requested || $best;
297 }
298 #- simplified choice resolution.
299 my $choice = $options{callback_choices}->($urpm, undef, $state, \@packages);
300 if ($choice) {
301 push @pkgs, $choice;
302 }
303 } else {
304 my $pkg = $urpm->{depslist}[$_] or next;
305 push @pkgs, $pkg;
306 }
307 }
308 #- local packages have already been added.
309 @pkgs = grep { !$urpm->{source}{$_->id} } @pkgs;
310  
311 $parallel->{line} .
312 ($options{auto_select} ? ' --auto-select' : '') .
313 ($options{keep} ? ' --keep' : '') .
314 join(' ', map { scalar $_->fullname } @pkgs);
315 }
316  
317 sub parallel_install {
318 my ($parallel, $urpm, undef, $install, $upgrade, %options) = @_;
319  
320 $parallel->copy_to_dir($urpm, values %$install, values %$upgrade, "$urpm->{cachedir}/rpms");
321  
322 my (%bad_nodes, @good_nodes);
323 $parallel->urpm_popen($urpm, 'urpmi', "--pre-clean --test --no-verify-rpm --auto --synthesis $parallel->{synthesis} $parallel->{line}", sub {
324 my ($node, $s) = @_;
325 $s =~ /^\s*$/ and return;
326 $bad_nodes{$node} .= "$s\n";
327 $s =~ /Installation failed/ and $bad_nodes{$node} = '';
328 $s =~ /Installation is possible|Packages are up to date/ and push @good_nodes, $node;
329 undef;
330 });
331 delete $bad_nodes{$_} foreach @good_nodes;
332  
333 foreach (keys %{$parallel->{nodes}}) {
334 exists $bad_nodes{$_} or next;
335 $urpm->{error}(N("Installation failed on node %s", $_) . ":\n" . $bad_nodes{$_});
336 }
337 %bad_nodes and return;
338  
339 if ($options{test}) {
340 $urpm->{error}(N("Installation is possible"));
341 1;
342 } else {
343 my $line = $parallel->{line} . ($options{excludepath} ? " --excludepath '$options{excludepath}'" : "");
344 #- continue installation.
345 $parallel->run_urpm_command($urpm, 'urpmi', "--no-verify-rpm --auto --synthesis $parallel->{synthesis} $line");
346 }
347 }
348  
349 1;
350  
351 =back
352  
353 =head1 COPYRIGHT
354  
355 Copyright (C) 2005 MandrakeSoft SA
356  
357 Copyright (C) 2005-2010 Mandriva SA
358  
359 Copyright (C) 2011-2020 Mageia
360  
361 =cut
« urpm/media.pm ↑ Summary urpm/removable.pm »