blib/lib/App/FuguVM/Proxy.pm untested

filter: Statementstmt 0.0% 0 / 115 Branchbran 0.0% 0 / 30 Conditioncond 0.0% 0 / 17 MC/DCmcdc 0.0% 0 / 10 Subroutinesub 0.0% 0 / 18 Podpod 0.0% 0 / 11 totaltotal 0.0% 0 / 201 CC 16 SCAR 24.0
CC
16
Cov
0%
prune
30.0
guest_url
12.0
_is_openbsd_content
12.0
CRAP 11.0
1 # ex:ts=8 sw=4:
2 # $OpenBSD$
3 #
4 # Copyright (c) 2024 Dick Olsson <hi@senzilla.io>
5 #
6 # Permission to use, copy, modify, and distribute this software for any
7 # purpose with or without fee is hereby granted, provided that the above
8 # copyright notice and this permission notice appear in all copies.
9 #
10 # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17  
18 use v5.36;
19  
20 package App::FuguVM::Proxy;
21 our $VERSION = '0.3.1';
22  
23 use Fugu::Log;
24 use Fugu::Proxy;
25  
26 our @ISA = ('Fugu::Proxy');
27  
28 # App::FuguVM::Proxy - the OpenBSD mirror policy over Fugu::Proxy.
29 #
30 # The generic proxy holds the serve loop, the cache and the metadata.
31 # This file holds only what is true of an OpenBSD mirror and of a QEMU
32 # guest: which URL is worth keeping, how to prune a release that is no
33 # longer in use, the address that the guest reaches the host by, and
34 # the port range.
35  
36 # QEMU user-mode networking puts the host at this address. A guest
37 # that fetches through the proxy names it, and no other address
38 # reaches back out of the SLIRP network.
39 use constant HOST_GATEWAY => '10.0.2.2';
40  
41 # $class->run_child($port, $cache_dir):
42 # The entry point of the spawned child. The child builds its own
43 # cache, warms the metadata, and serves until a SIGTERM.
44 #
45 # The spawn passes a fixed argument list, so the distfile cap
46 # reaches the child through the environment, which it inherits
47 # from fuguvm. An absent variable means 0, and 0 turns the
48 # distfile cache off.
49 sub run_child ( $class, $port, $cache_dir )
50 {
51 my $log = Fugu::Log->new( mode => 'stderr', level => 'debug' );
52  
53 my $limit = $ENV{FUGUVM_DISTFILE_LIMIT} // 0;
54 $limit = 0 if $limit !~ /\A[0-9]+\z/;
55  
56 my $self = bless {
57 cache => App::FuguVM::Proxy::Cache->new( $cache_dir, $limit ),
58 meta => Fugu::Proxy::Meta->new,
59 log => $log,
60 }, $class;
61  
62 $log->info( 'Proxy starting on port %d', $port );
63 $log->info( 'Cache directory: %s', $cache_dir );
64  
65 $self->warm;
66  
67 return $self->serve($port);
68 }
69  
70 # $self->guest_url:
71 # Return the proxy URL as the guest reaches it, through the QEMU
72 # gateway. The loopback address serves the same proxy, but a
73 # guest cannot reach it. The method returns undef while the
74 # proxy does not run: a port that a crash left behind is not a
75 # reachable proxy.
76 sub guest_url ($self)
77 {
78 return if !$self->is_running;
79  
80 my $port = $self->port;
81 return if !defined $port;
82  
83 return 'http://' . HOST_GATEWAY . ":$port";
84 }
85  
86 package App::FuguVM::Proxy::Cache;
87 our $VERSION = '0.3.1';
88  
89 use Fugu::Log;
90 use Fugu::Proxy;
91  
92 our @ISA = ('Fugu::Proxy::Cache');
93  
94 # App::FuguVM::Proxy::Cache - which parts of an OpenBSD mirror to keep.
95 #
96 # The patterns are the release tree, the packages, the syspatch sets,
97 # the source tarballs and the files that an installer reads. Every one
98 # of them is version-scoped, so nothing here outlives the release it
99 # belongs to. That is what makes prune safe.
100 #
101 # The distfile tree is the one exception: a distfile belongs to a
102 # port and not to a release, so no version scopes it and prune leaves
103 # it. The distfile cap of trim_distfiles is its bound instead, and
104 # the tree is cacheable only while the cap is above zero.
105  
106 my @CACHEABLE = (
107 qr{/pub/OpenBSD/\d+\.\d+/\w+/.*\.(tgz|img|gz)$}, # File sets
108 qr{/pub/OpenBSD/syspatch/.*\.tgz$}, # Patches
109 qr{/pub/OpenBSD/\d+\.\d+/packages/\w+/.*\.tgz$}, # Packages
110 qr{/pub/OpenBSD/\d+\.\d+/\w+/SHA256(\.sig)?$}, # Checksums
111 qr{/pub/OpenBSD/\d+\.\d+/\w+/miniroot\d+\.img$}, # Miniroot images
112 qr{/pub/OpenBSD/\d+\.\d+/\w+/bsd(\.mp|\.rd)?$}, # Kernel files
113 qr{/pub/OpenBSD/\d+\.\d+/\w+/BUILDINFO$}, # Build info
114 qr{/pub/OpenBSD/\d+\.\d+/\w+/.*\.txt$}, # Text files (index, etc)
115  
116 # The source tarballs of a release, by name. A wildcard at the
117 # version level would admit any file that a mirror puts there.
118 qr{/pub/OpenBSD/\d+\.\d+/(ports|src|sys|xenocara)\.tar\.gz$},
119  
120 # The manifest of the version directory, which signs the four
121 # source tarballs.
122 qr{/pub/OpenBSD/\d+\.\d+/SHA256(\.sig)?$},
123 );
124  
125 # The distfile pattern is conditional on the cap. It requires a file
126 # name at the end, so a directory listing with a trailing solidus
127 # stays outside the cache. It filters no extension, because a
128 # distfile carries every extension and sometimes none.
129 my $DISTFILE = qr{/pub/OpenBSD/distfiles/(?:[^/]+/)*[^/]+$};
130  
131 # $class->new($cache_dir, $limit):
132 # Build the cache. The optional limit is the distfile cap in
133 # bytes, and 0 turns the distfile cache off.
134 sub new ( $class, $cache_dir, $limit = 0 )
135 {
136 my $self = $class->SUPER::new(
137 dir => $cache_dir,
138 cacheable => sub ($url) { _is_openbsd_content( $url, $limit ) },
139 types => {
140  
141 # A kernel has no extension, so the generic
142 # table cannot name it
143 qr{/bsd(\.mp|\.rd)?$} => 'application/octet-stream',
144 },
145 );
146 $self->{distfile_limit} = $limit;
147  
148 return $self;
149 }
150  
151 # _is_openbsd_content($url, $limit):
152 # Report if the URL is part of an OpenBSD release tree, or a
153 # distfile while the cap is above zero. A cache that grows with
154 # no bound in a home directory is the failure that the default
155 # of 0 avoids.
156 sub _is_openbsd_content ( $url, $limit = 0 )
157 {
158 for my $pattern (@CACHEABLE) {
159 return 1 if $url =~ $pattern;
160 }
161  
162 return 1 if $limit > 0 && $url =~ $DISTFILE;
163  
164 return 0;
165 }
166  
167 # $self->store($url, $content):
168 # Store one response body, and hold the distfile tree under the
169 # cap after a distfile store. A set store must not walk the
170 # distfile tree.
171 sub store ( $self, $url, $content )
172 {
173 my $path = $self->SUPER::store( $url, $content );
174 return if !defined $path;
175  
176 $self->trim_distfiles if $url =~ $DISTFILE;
177  
178 return $path;
179 }
180  
181 # $self->distfile_limit:
182 # Return the distfile cap in bytes.
183 sub distfile_limit ($self)
184 {
185 return $self->{distfile_limit};
186 }
187  
188 # $self->distfile_size:
189 # Return the bytes of the distfile tree, over every cached host.
190 sub distfile_size ($self)
191 {
192 my $total = 0;
193 $total += $self->dir_size($_) for $self->_distfile_roots;
194  
195 return $total;
196 }
197  
198 # $self->trim_distfiles:
199 # Hold the distfile tree under the cap. The eviction removes the
200 # oldest file first, by modification time. The modification time
201 # is the store time of the file, because store writes each file
202 # one time; the access time is not usable, because a host can
203 # mount its home directory with noatime. A cap of 0 removes the
204 # whole tree, because the operator turned the cache off. The
205 # method removes files and leaves the directories, because a
206 # later fetch refills them. It returns the removed files as
207 # [ { path, size } ].
208 sub trim_distfiles ($self)
209 {
210 my @files;
211 for my $root ( $self->_distfile_roots ) {
212 $self->walk(
213 $root,
214 sub ($path) {
215 my @stat = stat $path;
216 return if !@stat;
217 push @files,
218 {
219 path => $path,
220 size => $stat[7],
221 mtime => $stat[9],
222 };
223 } );
224 }
225  
226 my $total = 0;
227 $total += $_->{size} for @files;
228  
229 my @removed;
230 for my $file ( sort { $a->{mtime} <=> $b->{mtime} } @files ) {
231 last if $total <= $self->{distfile_limit};
232  
233 unlink $file->{path} or do {
234 Fugu::Log->default->warning( 'Cannot remove %s',
235 $file->{path} );
236 next;
237 };
238  
239 $total -= $file->{size};
240 push @removed, { path => $file->{path}, size => $file->{size} };
241 }
242  
243 return \@removed;
244 }
245  
246 # $self->_distfile_roots:
247 # Return the distfile tree of each cached host that holds one.
248 sub _distfile_roots ($self)
249 {
250 my $root = $self->root;
251  
252 opendir my $dh, $root or return ();
253 my @hosts = sort grep { !/\A\.\.?\z/ } readdir $dh;
254 closedir $dh;
255  
256 return grep { -d $_ }
257 map { "$root/$_/pub/OpenBSD/distfiles" } @hosts;
258 }
259  
260 # $self->prune(@keep):
261 # Remove the cached download tree of every OpenBSD version other
262 # than @keep. The method returns [ { version, path, size } ] for
263 # each removed tree.
264 #
265 # Nothing else bounds this cache. 'fuguvm cache clear --stale'
266 # prunes installed images. Those images live beside these
267 # downloads under the same cache_dir, but no common key connects
268 # them. Thus a version bump left the full file sets of the
269 # previous version here for good. Nothing read them again,
270 # because every pattern that is_cacheable admits is
271 # version-scoped. And every copy of the directory that a
272 # continuous-integration cache made still carried them.
273 #
274 # The method removes whole directories, not matching files.
275 # Removal of the files alone leaves the empty version tree
276 # behind, and such a copy walks the tree.
277 #
278 # The method does not touch a directory whose name is not a
279 # version. In practice there are none. The patterns put
280 # everything under pub/OpenBSD/<version>/ or
281 # pub/OpenBSD/syspatch/<version>/. And a cache under $HOME is the
282 # wrong place to delete on a guess.
283 sub prune ( $self, @keep )
284 {
285 my $root = $self->root;
286 return [] if !-d $root;
287  
288 my %keep = map { $_ => 1 } @keep;
289 my @removed;
290  
291 for my $version_root ( $self->_version_roots ) {
292 opendir my $dh, $version_root or next;
293 my @versions = sort grep { /\A[0-9]+\.[0-9]+\z/ } readdir $dh;
294 closedir $dh;
295  
296 for my $version (@versions) {
297 next if $keep{$version};
298  
299 my $dir = "$version_root/$version";
300 next if !-d $dir;
301  
302 # Measure the size before removal. Thus the caller
303 # can say what the prune freed.
304 my $size = $self->dir_size($dir);
305  
306 require File::Path;
307 File::Path::remove_tree( $dir,
308 { error => \my $errors } );
309 if ( $errors && @$errors ) {
310 Fugu::Log->default->warning( 'Cannot remove %s',
311 $dir );
312 next;
313 }
314  
315 push @removed,
316 {
317 version => $version,
318 path => $dir,
319 size => $size,
320 };
321 }
322 }
323  
324 return \@removed;
325 }
326  
327 # $self->_version_roots:
328 # Return every directory whose immediate children are OpenBSD
329 # version numbers, across all cached hosts. Release trees hang
330 # off pub/OpenBSD. Syspatch sets sit one level deeper. Both have
331 # the name of a version, and neither outlives it.
332 sub _version_roots ($self)
333 {
334 my $root = $self->root;
335  
336 opendir my $dh, $root or return ();
337 my @hosts = sort grep { !/\A\.\.?\z/ } readdir $dh;
338 closedir $dh;
339  
340 my @roots;
341 for my $host (@hosts) {
342 my $release = "$root/$host/pub/OpenBSD";
343 next if !-d $release;
344  
345 push @roots, $release;
346 push @roots, "$release/syspatch" if -d "$release/syspatch";
347 }
348  
349 return @roots;
350 }
351  
352 1;