| 1 | # ex:ts=8 sw=4: | ||
| 2 | # $OpenBSD$ | ||
| 3 | # | ||
| 4 | # Copyright (c) 2024 Dick Olsson <hi@senzilla.io> | ||
| 5 | # | ||
| 6 | # Permission to use, copy, modify, and distribute this software for any | ||
| 7 | # purpose with or without fee is hereby granted, provided that the above | ||
| 8 | # copyright notice and this permission notice appear in all copies. | ||
| 9 | # | ||
| 10 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES | ||
| 11 | # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF | ||
| 12 | # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR | ||
| 13 | # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES | ||
| 14 | # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN | ||
| 15 | # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF | ||
| 16 | # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. | ||
| 17 | |||
| 18 | use v5.36; | ||
| 19 | |||
| 20 | package App::FuguVM::Proxy; | ||
| 21 | our $VERSION = '0.3.1'; | ||
| 22 | |||
| 23 | use Fugu::Log; | ||
| 24 | use Fugu::Proxy; | ||
| 25 | |||
| 26 | our @ISA = ('Fugu::Proxy'); | ||
| 27 | |||
| 28 | # App::FuguVM::Proxy - the OpenBSD mirror policy over Fugu::Proxy. | ||
| 29 | # | ||
| 30 | # The generic proxy holds the serve loop, the cache and the metadata. | ||
| 31 | # This file holds only what is true of an OpenBSD mirror and of a QEMU | ||
| 32 | # guest: which URL is worth keeping, how to prune a release that is no | ||
| 33 | # longer in use, the address that the guest reaches the host by, and | ||
| 34 | # the port range. | ||
| 35 | |||
| 36 | # QEMU user-mode networking puts the host at this address. A guest | ||
| 37 | # that fetches through the proxy names it, and no other address | ||
| 38 | # reaches back out of the SLIRP network. | ||
| 39 | use constant HOST_GATEWAY => '10.0.2.2'; | ||
| 40 | |||
| 41 | # $class->run_child($port, $cache_dir): | ||
| 42 | # The entry point of the spawned child. The child builds its own | ||
| 43 | # cache, warms the metadata, and serves until a SIGTERM. | ||
| 44 | # | ||
| 45 | # The spawn passes a fixed argument list, so the distfile cap | ||
| 46 | # reaches the child through the environment, which it inherits | ||
| 47 | # from fuguvm. An absent variable means 0, and 0 turns the | ||
| 48 | # distfile cache off. | ||
| 49 | sub run_child ( $class, $port, $cache_dir ) | ||
| 50 | { | ||
| 51 | my $log = Fugu::Log->new( mode => 'stderr', level => 'debug' ); | ||
| 52 | |||
| 53 | my $limit = $ENV{FUGUVM_DISTFILE_LIMIT} // 0; | ||
| 54 | $limit = 0 if $limit !~ /\A[0-9]+\z/; | ||
| 55 | |||
| 56 | my $self = bless { | ||
| 57 | cache => App::FuguVM::Proxy::Cache->new( $cache_dir, $limit ), | ||
| 58 | meta => Fugu::Proxy::Meta->new, | ||
| 59 | log => $log, | ||
| 60 | }, $class; | ||
| 61 | |||
| 62 | $log->info( 'Proxy starting on port %d', $port ); | ||
| 63 | $log->info( 'Cache directory: %s', $cache_dir ); | ||
| 64 | |||
| 65 | $self->warm; | ||
| 66 | |||
| 67 | return $self->serve($port); | ||
| 68 | } | ||
| 69 | |||
| 70 | # $self->guest_url: | ||
| 71 | # Return the proxy URL as the guest reaches it, through the QEMU | ||
| 72 | # gateway. The loopback address serves the same proxy, but a | ||
| 73 | # guest cannot reach it. The method returns undef while the | ||
| 74 | # proxy does not run: a port that a crash left behind is not a | ||
| 75 | # reachable proxy. | ||
| 76 | sub guest_url ($self) | ||
| 77 | { | ||
| 78 | return if !$self->is_running; | ||
| 79 | |||
| 80 | my $port = $self->port; | ||
| 81 | return if !defined $port; | ||
| 82 | |||
| 83 | return 'http://' . HOST_GATEWAY . ":$port"; | ||
| 84 | } | ||
| 85 | |||
| 86 | package App::FuguVM::Proxy::Cache; | ||
| 87 | our $VERSION = '0.3.1'; | ||
| 88 | |||
| 89 | use Fugu::Log; | ||
| 90 | use Fugu::Proxy; | ||
| 91 | |||
| 92 | our @ISA = ('Fugu::Proxy::Cache'); | ||
| 93 | |||
| 94 | # App::FuguVM::Proxy::Cache - which parts of an OpenBSD mirror to keep. | ||
| 95 | # | ||
| 96 | # The patterns are the release tree, the packages, the syspatch sets, | ||
| 97 | # the source tarballs and the files that an installer reads. Every one | ||
| 98 | # of them is version-scoped, so nothing here outlives the release it | ||
| 99 | # belongs to. That is what makes prune safe. | ||
| 100 | # | ||
| 101 | # The distfile tree is the one exception: a distfile belongs to a | ||
| 102 | # port and not to a release, so no version scopes it and prune leaves | ||
| 103 | # it. The distfile cap of trim_distfiles is its bound instead, and | ||
| 104 | # the tree is cacheable only while the cap is above zero. | ||
| 105 | |||
| 106 | my @CACHEABLE = ( | ||
| 107 | qr{/pub/OpenBSD/\d+\.\d+/\w+/.*\.(tgz|img|gz)$}, # File sets | ||
| 108 | qr{/pub/OpenBSD/syspatch/.*\.tgz$}, # Patches | ||
| 109 | qr{/pub/OpenBSD/\d+\.\d+/packages/\w+/.*\.tgz$}, # Packages | ||
| 110 | qr{/pub/OpenBSD/\d+\.\d+/\w+/SHA256(\.sig)?$}, # Checksums | ||
| 111 | qr{/pub/OpenBSD/\d+\.\d+/\w+/miniroot\d+\.img$}, # Miniroot images | ||
| 112 | qr{/pub/OpenBSD/\d+\.\d+/\w+/bsd(\.mp|\.rd)?$}, # Kernel files | ||
| 113 | qr{/pub/OpenBSD/\d+\.\d+/\w+/BUILDINFO$}, # Build info | ||
| 114 | qr{/pub/OpenBSD/\d+\.\d+/\w+/.*\.txt$}, # Text files (index, etc) | ||
| 115 | |||
| 116 | # The source tarballs of a release, by name. A wildcard at the | ||
| 117 | # version level would admit any file that a mirror puts there. | ||
| 118 | qr{/pub/OpenBSD/\d+\.\d+/(ports|src|sys|xenocara)\.tar\.gz$}, | ||
| 119 | |||
| 120 | # The manifest of the version directory, which signs the four | ||
| 121 | # source tarballs. | ||
| 122 | qr{/pub/OpenBSD/\d+\.\d+/SHA256(\.sig)?$}, | ||
| 123 | ); | ||
| 124 | |||
| 125 | # The distfile pattern is conditional on the cap. It requires a file | ||
| 126 | # name at the end, so a directory listing with a trailing solidus | ||
| 127 | # stays outside the cache. It filters no extension, because a | ||
| 128 | # distfile carries every extension and sometimes none. | ||
| 129 | my $DISTFILE = qr{/pub/OpenBSD/distfiles/(?:[^/]+/)*[^/]+$}; | ||
| 130 | |||
| 131 | # $class->new($cache_dir, $limit): | ||
| 132 | # Build the cache. The optional limit is the distfile cap in | ||
| 133 | # bytes, and 0 turns the distfile cache off. | ||
| 134 | sub new ( $class, $cache_dir, $limit = 0 ) | ||
| 135 | { | ||
| 136 | my $self = $class->SUPER::new( | ||
| 137 | dir => $cache_dir, | ||
| 138 | cacheable => sub ($url) { _is_openbsd_content( $url, $limit ) }, | ||
| 139 | types => { | ||
| 140 | |||
| 141 | # A kernel has no extension, so the generic | ||
| 142 | # table cannot name it | ||
| 143 | qr{/bsd(\.mp|\.rd)?$} => 'application/octet-stream', | ||
| 144 | }, | ||
| 145 | ); | ||
| 146 | $self->{distfile_limit} = $limit; | ||
| 147 | |||
| 148 | return $self; | ||
| 149 | } | ||
| 150 | |||
| 151 | # _is_openbsd_content($url, $limit): | ||
| 152 | # Report if the URL is part of an OpenBSD release tree, or a | ||
| 153 | # distfile while the cap is above zero. A cache that grows with | ||
| 154 | # no bound in a home directory is the failure that the default | ||
| 155 | # of 0 avoids. | ||
| 156 | sub _is_openbsd_content ( $url, $limit = 0 ) | ||
| 157 | { | ||
| 158 | for my $pattern (@CACHEABLE) { | ||
| 159 | return 1 if $url =~ $pattern; | ||
| 160 | } | ||
| 161 | |||
| 162 | return 1 if $limit > 0 && $url =~ $DISTFILE; | ||
| 163 | |||
| 164 | return 0; | ||
| 165 | } | ||
| 166 | |||
| 167 | # $self->store($url, $content): | ||
| 168 | # Store one response body, and hold the distfile tree under the | ||
| 169 | # cap after a distfile store. A set store must not walk the | ||
| 170 | # distfile tree. | ||
| 171 | sub store ( $self, $url, $content ) | ||
| 172 | { | ||
| 173 | my $path = $self->SUPER::store( $url, $content ); | ||
| 174 | return if !defined $path; | ||
| 175 | |||
| 176 | $self->trim_distfiles if $url =~ $DISTFILE; | ||
| 177 | |||
| 178 | return $path; | ||
| 179 | } | ||
| 180 | |||
| 181 | # $self->distfile_limit: | ||
| 182 | # Return the distfile cap in bytes. | ||
| 183 | sub distfile_limit ($self) | ||
| 184 | { | ||
| 185 | return $self->{distfile_limit}; | ||
| 186 | } | ||
| 187 | |||
| 188 | # $self->distfile_size: | ||
| 189 | # Return the bytes of the distfile tree, over every cached host. | ||
| 190 | sub distfile_size ($self) | ||
| 191 | { | ||
| 192 | my $total = 0; | ||
| 193 | $total += $self->dir_size($_) for $self->_distfile_roots; | ||
| 194 | |||
| 195 | return $total; | ||
| 196 | } | ||
| 197 | |||
| 198 | # $self->trim_distfiles: | ||
| 199 | # Hold the distfile tree under the cap. The eviction removes the | ||
| 200 | # oldest file first, by modification time. The modification time | ||
| 201 | # is the store time of the file, because store writes each file | ||
| 202 | # one time; the access time is not usable, because a host can | ||
| 203 | # mount its home directory with noatime. A cap of 0 removes the | ||
| 204 | # whole tree, because the operator turned the cache off. The | ||
| 205 | # method removes files and leaves the directories, because a | ||
| 206 | # later fetch refills them. It returns the removed files as | ||
| 207 | # [ { path, size } ]. | ||
| 208 | sub trim_distfiles ($self) | ||
| 209 | { | ||
| 210 | my @files; | ||
| 211 | for my $root ( $self->_distfile_roots ) { | ||
| 212 | $self->walk( | ||
| 213 | $root, | ||
| 214 | sub ($path) { | ||
| 215 | my @stat = stat $path; | ||
| 216 | return if !@stat; | ||
| 217 | push @files, | ||
| 218 | { | ||
| 219 | path => $path, | ||
| 220 | size => $stat[7], | ||
| 221 | mtime => $stat[9], | ||
| 222 | }; | ||
| 223 | } ); | ||
| 224 | } | ||
| 225 | |||
| 226 | my $total = 0; | ||
| 227 | $total += $_->{size} for @files; | ||
| 228 | |||
| 229 | my @removed; | ||
| 230 | for my $file ( sort { $a->{mtime} <=> $b->{mtime} } @files ) { | ||
| 231 | last if $total <= $self->{distfile_limit}; | ||
| 232 | |||
| 233 | unlink $file->{path} or do { | ||
| 234 | Fugu::Log->default->warning( 'Cannot remove %s', | ||
| 235 | $file->{path} ); | ||
| 236 | next; | ||
| 237 | }; | ||
| 238 | |||
| 239 | $total -= $file->{size}; | ||
| 240 | push @removed, { path => $file->{path}, size => $file->{size} }; | ||
| 241 | } | ||
| 242 | |||
| 243 | return \@removed; | ||
| 244 | } | ||
| 245 | |||
| 246 | # $self->_distfile_roots: | ||
| 247 | # Return the distfile tree of each cached host that holds one. | ||
| 248 | sub _distfile_roots ($self) | ||
| 249 | { | ||
| 250 | my $root = $self->root; | ||
| 251 | |||
| 252 | opendir my $dh, $root or return (); | ||
| 253 | my @hosts = sort grep { !/\A\.\.?\z/ } readdir $dh; | ||
| 254 | closedir $dh; | ||
| 255 | |||
| 256 | return grep { -d $_ } | ||
| 257 | map { "$root/$_/pub/OpenBSD/distfiles" } @hosts; | ||
| 258 | } | ||
| 259 | |||
| 260 | # $self->prune(@keep): | ||
| 261 | # Remove the cached download tree of every OpenBSD version other | ||
| 262 | # than @keep. The method returns [ { version, path, size } ] for | ||
| 263 | # each removed tree. | ||
| 264 | # | ||
| 265 | # Nothing else bounds this cache. 'fuguvm cache clear --stale' | ||
| 266 | # prunes installed images. Those images live beside these | ||
| 267 | # downloads under the same cache_dir, but no common key connects | ||
| 268 | # them. Thus a version bump left the full file sets of the | ||
| 269 | # previous version here for good. Nothing read them again, | ||
| 270 | # because every pattern that is_cacheable admits is | ||
| 271 | # version-scoped. And every copy of the directory that a | ||
| 272 | # continuous-integration cache made still carried them. | ||
| 273 | # | ||
| 274 | # The method removes whole directories, not matching files. | ||
| 275 | # Removal of the files alone leaves the empty version tree | ||
| 276 | # behind, and such a copy walks the tree. | ||
| 277 | # | ||
| 278 | # The method does not touch a directory whose name is not a | ||
| 279 | # version. In practice there are none. The patterns put | ||
| 280 | # everything under pub/OpenBSD/<version>/ or | ||
| 281 | # pub/OpenBSD/syspatch/<version>/. And a cache under $HOME is the | ||
| 282 | # wrong place to delete on a guess. | ||
| 283 | sub prune ( $self, @keep ) | ||
| 284 | { | ||
| 285 | my $root = $self->root; | ||
| 286 | return [] if !-d $root; | ||
| 287 | |||
| 288 | my %keep = map { $_ => 1 } @keep; | ||
| 289 | my @removed; | ||
| 290 | |||
| 291 | for my $version_root ( $self->_version_roots ) { | ||
| 292 | opendir my $dh, $version_root or next; | ||
| 293 | my @versions = sort grep { /\A[0-9]+\.[0-9]+\z/ } readdir $dh; | ||
| 294 | closedir $dh; | ||
| 295 | |||
| 296 | for my $version (@versions) { | ||
| 297 | next if $keep{$version}; | ||
| 298 | |||
| 299 | my $dir = "$version_root/$version"; | ||
| 300 | next if !-d $dir; | ||
| 301 | |||
| 302 | # Measure the size before removal. Thus the caller | ||
| 303 | # can say what the prune freed. | ||
| 304 | my $size = $self->dir_size($dir); | ||
| 305 | |||
| 306 | require File::Path; | ||
| 307 | File::Path::remove_tree( $dir, | ||
| 308 | { error => \my $errors } ); | ||
| 309 | if ( $errors && @$errors ) { | ||
| 310 | Fugu::Log->default->warning( 'Cannot remove %s', | ||
| 311 | $dir ); | ||
| 312 | next; | ||
| 313 | } | ||
| 314 | |||
| 315 | push @removed, | ||
| 316 | { | ||
| 317 | version => $version, | ||
| 318 | path => $dir, | ||
| 319 | size => $size, | ||
| 320 | }; | ||
| 321 | } | ||
| 322 | } | ||
| 323 | |||
| 324 | return \@removed; | ||
| 325 | } | ||
| 326 | |||
| 327 | # $self->_version_roots: | ||
| 328 | # Return every directory whose immediate children are OpenBSD | ||
| 329 | # version numbers, across all cached hosts. Release trees hang | ||
| 330 | # off pub/OpenBSD. Syspatch sets sit one level deeper. Both have | ||
| 331 | # the name of a version, and neither outlives it. | ||
| 332 | sub _version_roots ($self) | ||
| 333 | { | ||
| 334 | my $root = $self->root; | ||
| 335 | |||
| 336 | opendir my $dh, $root or return (); | ||
| 337 | my @hosts = sort grep { !/\A\.\.?\z/ } readdir $dh; | ||
| 338 | closedir $dh; | ||
| 339 | |||
| 340 | my @roots; | ||
| 341 | for my $host (@hosts) { | ||
| 342 | my $release = "$root/$host/pub/OpenBSD"; | ||
| 343 | next if !-d $release; | ||
| 344 | |||
| 345 | push @roots, $release; | ||
| 346 | push @roots, "$release/syspatch" if -d "$release/syspatch"; | ||
| 347 | } | ||
| 348 | |||
| 349 | return @roots; | ||
| 350 | } | ||
| 351 | |||
| 352 | 1; |