blib/lib/App/FuguVM/Mirror.pm untested

filter: Statementstmt 0.0% 0 / 236 Branchbran 0.0% 0 / 100 Conditioncond 0.0% 0 / 28 MC/DCmcdc 0.0% 0 / 16 Subroutinesub 0.0% 0 / 31 Podpod 0.0% 0 / 21 totaltotal 0.0% 0 / 432 CC 51 SCAR 35.7
CC
51
Cov
0%
verify_cache
90.0
ensure
72.0
manifest
56.0
CRAP 35.6
1 # ex:ts=8 sw=4:
2 # $OpenBSD$
3 #
4 # Copyright (c) 2026 Dick Olsson <hi@senzilla.io>
5 #
6 # Permission to use, copy, modify, and distribute this software for any
7 # purpose with or without fee is hereby granted, provided that the above
8 # copyright notice and this permission notice appear in all copies.
9 #
10 # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17  
18 use v5.36;
19  
20 package App::FuguVM::Mirror;
21 our $VERSION = '0.3.1';
22  
23 use File::Temp ();
24 use Fugu::Curl 0.5.0;
25 use Fugu::File;
26 use Fugu::Log;
27 use Fugu::Signify 0.5.0;
28  
29 # App::FuguVM::Mirror - the OpenBSD mirror of one guest.
30 #
31 # The module is the one home of the mirror facts: the host, the URLs
32 # of a version and an architecture, the downloader, the release key,
33 # and the verification. Fugu::Curl downloads each file, Fugu::Signify
34 # proves the signed SHA256 manifest, and this module decides which
35 # manifest signs which file.
36 # The proof runs under the signify engine of that module. A release
37 # of OpenBSD signs its SHA256 in the embedded form, which the perl
38 # engine of Fugu::Signify does not read, so the command makes the
39 # proof.
40 #
41 # A file must verify before it enters the cache, because the cache is
42 # what a later run reads. A verification failure therefore leaves no
43 # file in the cache.
44  
45 # The host of the two scopes. No consumer asks for a second release
46 # host, and a second one would multiply the trees under cache_dir.
47 # The distfile tree is not a scope of this module: the guest fetches
48 # a distfile itself, from a mirror host that carries the tree.
49 use constant { CDN_HOST => 'cdn.openbsd.org', };
50  
51 # The share directory of the release keys. The commit that adds a key
52 # is a trust decision, so a human adds each file.
53 use constant { KEYS_SHARE_DIR => 'share/fuguvm/signify', };
54  
55 # The whole-fetch bound of one download, in seconds. A set file of
56 # OpenBSD is hundreds of megabytes, and the default of Fugu::Curl is
57 # too short for it. A stalled connection must not hold the tool
58 # forever.
59 use constant { FETCH_TIMEOUT => 3600, };
60  
61 # $class->new(%args):
62 # Build the mirror of one version and one architecture.
63 #
64 # %args:
65 # cache => $cache # Required: an App::FuguVM::Proxy::Cache
66 # version => '7.8' # Required: the OpenBSD version
67 # arch => 'arm64' # Required: the architecture name
68 # keys_dir => $dir # Optional: the signify_dir directive
69 # verify => 1 # Optional: the default is 1
70 # log => $log # Optional: Fugu::Log->default
71 #
72 # The method dies when cache, version or arch is absent. Each
73 # one is a programming error.
74 #
75 # The method builds the one downloader of the mirror. Fugu::Curl
76 # resolves its command in its own constructor, and it runs no
77 # process there, so one mirror resolves the command one time.
78 #
79 # The method makes no file and no directory, so it fails for a
80 # programming error alone. The first fetch makes the private
81 # download directory of the mirror.
82 sub new ( $class, %args )
83 {
84 for my $required (qw(cache version arch)) {
85 die "App::FuguVM::Mirror needs $required\n"
86 if !defined $args{$required};
87 }
88  
89 return bless {
90 cache => $args{cache},
91 version => $args{version},
92 arch => $args{arch},
93 keys_dir => $args{keys_dir},
94 verify => $args{verify} // 1,
95 log => $args{log} // Fugu::Log->default,
96 download => Fugu::Curl->new( timeout => FETCH_TIMEOUT ),
97 tmpdir => undef,
98 manifest => {},
99 error => undef,
100 }, $class;
101 }
102  
103 # $self->version:
104 # Return the OpenBSD version of this mirror. The mirror is the
105 # one home of the version, so a consumer asks it and holds no
106 # copy.
107 sub version ($self)
108 {
109 return $self->{version};
110 }
111  
112 # $self->url($file):
113 # Return the URL of a file of the architecture directory.
114 sub url ( $self, $file )
115 {
116 return 'https://' . CDN_HOST
117 . "/pub/OpenBSD/$self->{version}/$self->{arch}/$file";
118 }
119  
120 # $self->source_url($file):
121 # Return the URL of a file of the version directory. The source
122 # tarballs and their manifest live there.
123 sub source_url ( $self, $file )
124 {
125 return 'https://' . CDN_HOST . "/pub/OpenBSD/$self->{version}/$file";
126 }
127  
128 # $self->error:
129 # Return the reason of the most recent failure, or undef. Every
130 # public method clears the reason before it starts.
131 sub error ($self)
132 {
133 return $self->{error};
134 }
135  
136 # $self->key_path:
137 # Return the public key file of the version, or undef with the
138 # reason in error. The version names the file: 7.8 gives
139 # openbsd-78-base.pub. The resolution order is the keys_dir
140 # argument, then /etc/signify, then the share tree. An OpenBSD
141 # host holds the authentic key from its own installation, which
142 # is a better anchor than a copy in a repository.
143 sub key_path ($self)
144 {
145 $self->{error} = undef;
146  
147 ( my $short = $self->{version} ) =~ s/\.//g;
148 my $file = "openbsd-$short-base.pub";
149  
150 my @tried;
151 for my $dir ( $self->{keys_dir}, '/etc/signify' ) {
152 next if !defined $dir;
153 push @tried, $dir;
154 my $path = "$dir/$file";
155 return $path if -f $path;
156 }
157  
158 my $shared = Fugu::File->share_path(
159 KEYS_SHARE_DIR . "/$file",
160 from => __FILE__,
161 dist => 'App-FuguVM'
162 );
163 return $shared if defined $shared;
164  
165 push @tried, KEYS_SHARE_DIR;
166 $self->{error} = "no public key $file in " . join( ', ', @tried );
167  
168 return;
169 }
170  
171 # $self->_tmpdir:
172 # The private download directory of the mirror, at mode 0700.
173 # The method makes it on the first call, and it holds the
174 # directory for the life of the mirror. It returns undef with
175 # the reason in error when the host refuses the directory.
176 #
177 # Fugu::Curl writes a sibling file next to its destination, with
178 # a name that another process can guess, and it opens that file
179 # without O_EXCL. A download in the shared temporary directory
180 # therefore takes a file that another user made first.
181 #
182 # File::Temp->newdir croaks when the temporary directory of the
183 # host is absent or unwritable. That is an environment failure,
184 # and the eval below catches the death of the library. A caller
185 # then reads the reason in error, as it reads every other
186 # failure of fetch.
187 sub _tmpdir ($self)
188 {
189 return $self->{tmpdir} if defined $self->{tmpdir};
190  
191 my $dir = eval { File::Temp->newdir };
192 if ( !defined $dir ) {
193 my $reason = $@ || 'unknown reason';
194 $reason =~ s/\s+\z//;
195 $self->{error} = "cannot make a download directory: $reason";
196 return;
197 }
198  
199 $self->{tmpdir} = $dir;
200  
201 return $dir;
202 }
203  
204 # $self->fetch($url):
205 # Download the URL to a temporary file of the private download
206 # directory, through the downloader of the mirror. Return the
207 # File::Temp object, or undef with the reason in error. This
208 # method is the one home of the download call.
209 #
210 # A caller must read filename of the object, and must never read
211 # the object as a filehandle. Fugu::Curl renames its own file
212 # onto the path, so the filehandle of the object holds the old
213 # inode.
214 #
215 # The method reports the error of the downloader as it stands.
216 # That error names the URL after a failed fetch, and it names no
217 # URL when the constructor of the downloader found no command.
218 sub fetch ( $self, $url )
219 {
220 $self->{error} = undef;
221  
222 my $download = $self->{download};
223 if ( !$download->is_available ) {
224 $self->{error} = $download->error;
225 return;
226 }
227  
228 my $tmpdir = $self->_tmpdir;
229 return if !defined $tmpdir;
230  
231 my $tmp = File::Temp->new( DIR => $tmpdir );
232  
233 # The downloader writes no progress, and a download of a
234 # hundred megabytes is a wait that an operator wants to see.
235 # The URL is what the operator waits for.
236 $self->{log}->info( 'Downloading %s', $url );
237  
238 if ( !$download->fetch( $url, $tmp->filename ) ) {
239 $self->{error} = $download->error;
240 return;
241 }
242  
243 if ( !-s $tmp->filename ) {
244 $self->{error} = "download wrote no bytes: $url";
245 return;
246 }
247  
248 return $tmp;
249 }
250  
251 # $self->manifest($scope):
252 # Make sure that the SHA256 and the SHA256.sig of the scope are
253 # cached, and prove the signature under the release key. Return
254 # the cached manifest path, or undef with the reason in error.
255 # The scope is 'release' for the architecture directory and
256 # 'source' for the version directory. The module caches the
257 # proven path for the length of the object.
258 sub manifest ( $self, $scope )
259 {
260 $self->{error} = undef;
261  
262 return $self->{manifest}{$scope}
263 if defined $self->{manifest}{$scope};
264  
265 if ( !$self->{verify} ) {
266 $self->{error} =
267 'verification is off, so no manifest is proven';
268 return;
269 }
270  
271 my $key = $self->key_path;
272 return if !defined $key;
273  
274 my $manifest = $self->_cached_manifest($scope);
275 return if !defined $manifest;
276 my $signature = $self->_ensure_unverified( $scope, 'SHA256.sig' );
277 return if !defined $signature;
278  
279 # The call passes one key, and not a key set. The release
280 # directory of a numbered release carries one signature, under
281 # the base key of that release, so a second key would accept a
282 # file that the version does not own.
283 #
284 # The engine is signify, and not the perl default. A release of
285 # OpenBSD signs its SHA256 in the embedded form: the signature
286 # file carries the manifest after the two signature lines. The
287 # perl engine reads a file of two lines only, and it refuses
288 # such a pair, so the command makes the proof.
289 my $signify = Fugu::Signify->new( engine => 'signify' );
290 my $proven = $signify->verify(
291 keys => [$key],
292 file => $manifest,
293 signature => $signature,
294 );
295 if ( !defined $proven ) {
296 $self->{error} = $self->_signify_error($signify);
297 return;
298 }
299  
300 $self->{manifest}{$scope} = $manifest;
301  
302 return $manifest;
303 }
304  
305 # $self->manifest_names($scope):
306 # Return the file names that the manifest of the scope holds, as
307 # a sorted array reference, or undef with the reason in error.
308 # With verification on, the names come from the proven manifest.
309 # With verification off, the manifest is a routing table only,
310 # and the method reads it without a proof.
311 sub manifest_names ( $self, $scope )
312 {
313 $self->{error} = undef;
314  
315 my $digests = $self->_manifest_digests($scope);
316 return if !defined $digests;
317  
318 return [ sort keys %$digests ];
319 }
320  
321 # $self->_manifest_digests($scope):
322 # Return the name-to-digest map of the manifest of the scope, or
323 # undef with the reason in error. With verification on, the map
324 # comes from the proven manifest.
325 #
326 # The parser accepts a duplicate line whose digest is identical:
327 # the real SHA256 of an OpenBSD release directory repeats the
328 # install image lines, and a repeated identical fact changes
329 # nothing. Fugu LIB-SIGNIFY refuses such a manifest whole, so
330 # this module checks each digest itself, under the signature
331 # that the manifest proof made. A duplicate name with a
332 # different digest is a broken manifest, and the parser refuses
333 # it, like a line that does not parse.
334 sub _manifest_digests ( $self, $scope )
335 {
336 my $manifest =
337 $self->{verify}
338 ? $self->manifest($scope)
339 : $self->_cached_manifest($scope);
340 return if !defined $manifest;
341  
342 my $bytes = Fugu::File->read($manifest);
343 if ( !defined $bytes ) {
344 $self->{error} = "cannot read $manifest";
345 return;
346 }
347  
348 my %digest;
349 for my $line ( split /\n/, $bytes ) {
350 my ( $name, $hex ) =
351 $line =~ /^SHA256 \((.+)\) = ([0-9A-Fa-f]{64})$/;
352 if ( !defined $name ) {
353 $self->{error} =
354 "$manifest: cannot parse manifest line: $line";
355 return;
356 }
357 $hex = lc $hex;
358 if ( exists $digest{$name} && $digest{$name} ne $hex ) {
359 $self->{error} = "$manifest: duplicate name $name"
360 . ' with two digests';
361 return;
362 }
363 $digest{$name} = $hex;
364 }
365  
366 if ( !%digest ) {
367 $self->{error} = "$manifest: the manifest is empty";
368 return;
369 }
370  
371 return \%digest;
372 }
373  
374 # $self->verify_file($scope, $name, $path):
375 # Verify one local file against the proven manifest of the
376 # scope, under its manifest name. Return 1, or undef with the
377 # reason in error. The manifest proof runs the signature check;
378 # the digest check runs here, so a manifest with a repeated
379 # identical line still verifies.
380 sub verify_file ( $self, $scope, $name, $path )
381 {
382 $self->{error} = undef;
383  
384 if ( !$self->{verify} ) {
385 $self->{error} = 'verification is off, so no proof is made';
386 return;
387 }
388  
389 my $digests = $self->_manifest_digests($scope);
390 return if !defined $digests;
391  
392 my $expected = $digests->{$name};
393 if ( !defined $expected ) {
394 $self->{error} =
395 "the manifest of the $scope scope does not hold $name";
396 return;
397 }
398  
399 my $computed = _digest($path);
400 if ( !defined $computed ) {
401 $self->{error} = "cannot digest $path";
402 return;
403 }
404  
405 if ( $computed ne $expected ) {
406 $self->{error} = "$name: digest mismatch:"
407 . " expected $expected, computed $computed";
408 return;
409 }
410  
411 return 1;
412 }
413  
414 # _digest($path):
415 # The lowercase hex SHA256 digest of a file, or undef when the
416 # file does not open. addfile reads in blocks, so a file set of
417 # hundreds of megabytes never enters memory whole.
418 sub _digest ($path)
419 {
420 require Digest::SHA;
421  
422 open my $fh, '<', $path or return;
423 binmode $fh;
424  
425 my $sha = Digest::SHA->new(256);
426 $sha->addfile($fh);
427 close $fh;
428  
429 return lc $sha->hexdigest;
430 }
431  
432 # $self->ensure($scope, $file):
433 # Return the cached path of a file of the scope. On a cache
434 # miss, prove the manifest, fetch the file, verify it, and store
435 # it. Return undef with the reason in error on any failure. The
436 # file verifies before it enters the cache, so a verification
437 # failure leaves no file in the cache.
438 sub ensure ( $self, $scope, $file )
439 {
440 $self->{error} = undef;
441  
442 my $url = $self->_scope_url( $scope, $file );
443 my $cached = $self->{cache}->lookup($url);
444 return $cached if defined $cached;
445  
446 if ( !$self->{verify} ) {
447  
448 # GST-MIRROR-3 demands this warning, and MODE_QUIET of
449 # Fugu::Log drops every level. The line therefore takes
450 # the process default logger, which no file of lib and
451 # bin makes quiet, so --quiet never drops it.
452 Fugu::Log->default->warning(
453 'Verification is off: %s enters the cache unproven',
454 $file );
455 }
456 else {
457 return if !defined $self->manifest($scope);
458 }
459  
460 my $tmp = $self->fetch($url);
461 return if !defined $tmp;
462  
463 if ( $self->{verify} ) {
464 return
465 if !
466 defined $self->verify_file( $scope, $file, $tmp->filename );
467 }
468  
469 my $path = $self->{cache}->store_from_file( $url, $tmp->filename );
470 if ( !defined $path ) {
471 $self->{error} = "cannot store $file in the cache";
472 return;
473 }
474  
475 return $path;
476 }
477  
478 # $self->verify_cache:
479 # Verify every cached file of the two scopes. Return
480 # { ok, failed, unknown }, each one a list of { scope, name }
481 # entries sorted by name, or undef with the reason in error.
482 #
483 # A cached file that the proven manifest of its scope names is
484 # ok or failed. A cached file that no proven manifest names is
485 # unknown: index.txt, a package, and every file of a scope whose
486 # manifest is absent or unproven, are all in that class. The
487 # method never skips a cached file. The SHA256 and SHA256.sig
488 # pairs stay out of the three lists while their proof holds,
489 # because that proof covers them.
490 #
491 # A cached manifest pair that fails its own proof is a failure:
492 # the method removes the pair and reports its SHA256 under
493 # failed, so a poisoned pair cannot pin every later run. A pair
494 # that the method cannot fetch leaves the files of its scope
495 # unknown.
496 #
497 # The manifest proof makes one signify(1) run for each scope,
498 # and each digest check runs in this module, so a tree of a
499 # hundred files costs one signify(1) run for each scope.
500 sub verify_cache ($self)
501 {
502 $self->{error} = undef;
503  
504 my %report = ( ok => [], failed => [], unknown => [] );
505  
506 for my $scope (qw(release source)) {
507 my @entries =
508 grep { $_->[0] ne 'SHA256' && $_->[0] ne 'SHA256.sig' }
509 @{ $self->_scope_files($scope) };
510 my $cached_manifest = $self->{cache}
511 ->lookup( $self->_scope_url( $scope, 'SHA256' ) );
512 next if !@entries && !defined $cached_manifest;
513  
514 my $manifest = $self->manifest($scope);
515 if ( !defined $manifest ) {
516 my $reason = $self->{error};
517  
518 # A key that does not resolve proves nothing,
519 # and nothing failed a proof. Stop.
520 return if !defined $self->key_path;
521 $self->{error} = $reason;
522  
523 # A cached pair that failed its proof must not
524 # stay: it would pin every later run.
525 if ( defined $cached_manifest ) {
526 unlink $cached_manifest;
527 my $sig = $self->{cache}->cache_path(
528 $self->_scope_url(
529 $scope, 'SHA256.sig'
530 ) );
531 unlink $sig if defined $sig && -f $sig;
532 push @{ $report{failed} },
533 { scope => $scope, name => 'SHA256' };
534 }
535  
536 # No proven manifest names these files.
537 push @{ $report{unknown} },
538 map { { scope => $scope, name => $_->[0] } }
539 @entries;
540 next;
541 }
542  
543 my $digests = $self->_manifest_digests($scope);
544 return if !defined $digests;
545  
546 # The manifest proof above ran the one signify(1) call
547 # of the scope. Each digest check runs here.
548 for my $entry (@entries) {
549 my ( $name, $path ) = @$entry;
550 my $expected = $digests->{$name};
551 if ( !defined $expected ) {
552 push @{ $report{unknown} },
553 { scope => $scope, name => $name };
554 next;
555 }
556  
557 my $computed = _digest($path);
558 my $state =
559 defined $computed && $computed eq $expected
560 ? 'ok'
561 : 'failed';
562 push @{ $report{$state} },
563 { scope => $scope, name => $name };
564 }
565 }
566  
567 @{ $report{$_} } = sort { $a->{name} cmp $b->{name} } @{ $report{$_} }
568 for keys %report;
569  
570 # The removal above is a report, not a failure of this method.
571 $self->{error} = undef;
572  
573 return \%report;
574 }
575  
576 # $self->cached_path($scope, $file):
577 # Return the cache path of a file of the scope, cached or not.
578 # The verify verb of the tool removes a failed file through this
579 # answer.
580 sub cached_path ( $self, $scope, $file )
581 {
582 return $self->{cache}->cache_path( $self->_scope_url( $scope, $file ) );
583 }
584  
585 # $self->_scope_url($scope, $file):
586 # Return the URL of a file of the scope. An unknown scope is a
587 # programming error.
588 sub _scope_url ( $self, $scope, $file )
589 {
590 return $self->url($file) if $scope eq 'release';
591 return $self->source_url($file) if $scope eq 'source';
592  
593 die "unknown mirror scope: $scope\n";
594 }
595  
596 # $self->_scope_files($scope):
597 # Return every cached file of the scope directory, as
598 # [ [ $name, $path ], ... ]. The release scope reads the
599 # architecture directory, and the source scope reads the files
600 # directly under the version directory.
601 sub _scope_files ( $self, $scope )
602 {
603 my $dir = $self->cached_path( $scope, '' );
604 $dir =~ s{/\z}{};
605  
606 my @files;
607 return \@files if !-d $dir;
608  
609 opendir my $dh, $dir or return \@files;
610 my @names = sort grep { -f "$dir/$_" } readdir $dh;
611 closedir $dh;
612  
613 @files = map { [ $_, "$dir/$_" ] } @names;
614  
615 return \@files;
616 }
617  
618 # $self->_cached_manifest($scope):
619 # Return the cached SHA256 of the scope, and fetch it first when
620 # the cache misses. The manifest needs no proof of its own to
621 # enter the cache: its signature proves it on every read.
622 sub _cached_manifest ( $self, $scope )
623 {
624 return $self->_ensure_unverified( $scope, 'SHA256' );
625 }
626  
627 # $self->_ensure_unverified($scope, $file):
628 # Return the cached path of a manifest file, and fetch it first
629 # when the cache misses. Only the SHA256 pair takes this path:
630 # the signature check proves both, so a store before the proof
631 # is safe.
632 sub _ensure_unverified ( $self, $scope, $file )
633 {
634 my $url = $self->_scope_url( $scope, $file );
635 my $cached = $self->{cache}->lookup($url);
636 return $cached if defined $cached;
637  
638 my $tmp = $self->fetch($url);
639 return if !defined $tmp;
640  
641 my $path = $self->{cache}->store_from_file( $url, $tmp->filename );
642 if ( !defined $path ) {
643 $self->{error} = "cannot store $file in the cache";
644 return;
645 }
646  
647 return $path;
648 }
649  
650 # $self->_signify_error($signify):
651 # Return the reason of a failed signify call. An absent command
652 # is an install problem, so that reason also names the packages.
653 sub _signify_error ( $self, $signify )
654 {
655 my $reason = $signify->error // 'signify failed';
656 $reason .=
657 ' (install the package signify-openbsd on Linux,'
658 . ' or signify-osx on Darwin)'
659 if $signify->command_absent;
660  
661 return $reason;
662 }
663  
664 1;