blib/lib/Web/VWF.pm untested

filter: Statementstmt 0.0% 0 / 737 Branchbran 0.0% 0 / 286 Conditioncond 0.0% 0 / 233 MC/DCmcdc 0.0% 0 / 125 Subroutinesub 0.0% 0 / 70 Podpod 0.0% 0 / 28 totaltotal 0.0% 0 / 1479 CC 143 SCAR 65.2
CC
143
Cov
0%
_doit
2652.0
_init
156.0
_config_entries
156.0
CRAP 679.8
1 package Web::VWF;
2  
3 # njh@nigelhorne.com
4  
5 use strict;
6 use warnings;
7 # use diagnostics;
8  
9 use version; our $VERSION = version->declare('0.003.0');
10  
11 # TODO: Try Log::Any::Adapter::Abstraction instead of Log::Any
12  
13 BEGIN {
14 # Sanitize environment variables
15 delete @ENV{qw(IFS CDPATH ENV BASH_ENV)};
16 $ENV{'PATH'} = '/usr/local/bin:/bin:/usr/bin'; # For insecurity
17 }
18  
19 no lib '.';
20  
21 use Log::WarnDie 0.09;
22 use Carp::Always;
23 use CGI::ACL 0.06; # For deny_cloud
24 use CGI::Carp qw(fatalsToBrowser);
25 use CGI::Info 0.94; # Gets all messages
26 use CGI::Lingua 0.61;
27 use CHI;
28 use Class::Simple;
29 use Config::Abstraction;
30 use Database::Abstraction;
31 # use Devel::Confess;
32 use File::Basename;
33 # use CGI::Alert $ENV{'SERVER_ADMIN'} || 'you@example.com';
34 use FCGI;
35 use FCGI::Buffer;
36 use File::HomeDir;
37 use HTTP::Status;
38 use Log::Abstraction;
39 use Error qw(:try);
40 use File::Spec;
41 use Params::Get;
42 use POSIX qw(strftime);
43 use Readonly;
44 use Module::Runtime qw(require_module); # Safe dynamic module loading; avoids string eval
45 use Timer::Simple;
46 use Time::HiRes;
47  
48 # FIXME: Sometimes gives Insecure dependency in require while running with -T switch in Module/Runtime.pm
49 # use Taint::Runtime qw($TAINT taint_env);
50 use autodie qw(:all);
51  
52 # Where to find the site's own page modules and configuration. The framework
53 # modules themselves are installed by HTTP-VWF, so this only needs to point at
54 # the site's own lib directory when the site has page subclasses of its own.
55 # use lib '/usr/lib/example.com/lib';
56 # use lib '../lib';
57 use lib CGI::Info::script_dir() . '/../lib';
58 # use lib File::HomeDir->my_home() . '/lib/perl5';
59  
60 use Web::VWF::Allow;
61 use Web::VWF::Blacklist;
62 use Web::VWF::Config;
63 use Web::VWF::Utils;
64  
65 # Where the server keeps the configuration of the sites it serves: one directory
66 # per site, named after the site, read when the site has no conf/ of its own
67 # beside cgi-bin/. A site whose host keeps them elsewhere changes this line and
68 # nothing else - it is passed to Web::VWF->new() below, and to the page modules,
69 # so that both read the site's configuration from the same place.
70 Readonly my $SERVER_CONF_DIR => '/usr/lib/vwf';
71  
72 # $TAINT = 1;
73 # taint_env();
74  
75 # Soft rate-limit: present a CAPTCHA challenge when a client exceeds this
76 # many requests within $TIME_WINDOW. Overridable via the config file.
77 Readonly my $MAX_REQUESTS => 100;
78 Readonly my $TIME_WINDOW => '60s'; # CHI-style duration string
79  
80 Readonly my @suffixlist => ('.pl', '.fcgi');
81  
82 # Set up by _init() on the first call to run(), not when the module is loaded.
83 # Loading a module should do no work: reading configuration here meant that
84 # anything that executed the page script while it was still compiling - such as
85 # a configuration reader that took the script for its own configuration file -
86 # ran the framework again inside itself, with any errors silently swallowed.
87 my $info;
88 my $script_name;
89 my $tmpdir;
90  
91 # $vwflog holds the path to the CSV access log; resolved from config on first request.
92 my $vwflog;
93  
94 my $lingua_cache;
95 my $buffercache;
96  
97 # Also set by _init()
98 my $script_dir;
99 my $env_prefix;
100 my $config_abstraction;
101 my $logger;
102  
103 # True while run() is executing, so that a nested call can be refused
104 our $RUNNING;
105  
106 # my $pagename = "Web::VWF::Display::$script_name";
107 # eval "require $pagename";
108  
109 # Loaded later, only when needed
110 # use Web::VWF::Display::index;
111 # use Web::VWF::Display::upload;
112 # use Web::VWF::Display::editor;
113 # use Web::VWF::Display::meta_data;
114  
115 # -- What a site declares, and what it gets if it declares nothing -----------
116 # The directory its databases are in, the data modules it loads, the languages
117 # it offers and the addresses it allows or trusts are all site settings, read
118 # from the domain's configuration file - see "CONFIGURATION" in Web::VWF's
119 # documentation. What is below is what this script used to hardcode, so a site
120 # that says nothing behaves exactly as it did before. The settings are resolved
121 # by _site_setup() on the first request, because the configuration cannot be read
122 # until the domain is known.
123 Readonly my $DEFAULT_DATABASE_DIR => 'data'; # beside cgi-bin/
124 Readonly my @DEFAULT_DATA_MODULES => qw(Web::VWF::Data::index);
125 Readonly my @DEFAULT_SUPPORTED_LANGUAGES => qw(en-gb);
126 # One site's own address, kept so that a site that came from this script and says
127 # nothing keeps working; a site that wants a different list sets allow_ips.
128 #
129 # It is not called allowed_ips because that is a CGI::ACL constructor argument,
130 # and CGI::ACL's new() runs the site's configuration through
131 # Object::Configure::configure(), which hands a key of that name straight to it
132 # as a string where a hash is expected, breaking the ACL outright.
133 Readonly my @DEFAULT_ALLOW_IPS => qw(108.44.193.70 127.0.0.1);
134 # Loopback and private addresses are exempt from rate limiting.
135 Readonly my @DEFAULT_RATE_LIMIT_TRUSTED_IPS => qw(127.0.0.1 192.168.1.1);
136  
137 # The access log is the one data module this script builds itself, because which
138 # log to read comes from the vwflog setting; its class is loaded here so that
139 # doit() can construct it without a compile-time use.
140 require_module('Web::VWF::Data::vwf_log');
141 Web::VWF::Data::vwf_log->import() if(Web::VWF::Data::vwf_log->can('import'));
142  
143 # Resolved from the configuration by _site_setup(): the directory holding the
144 # site's databases, and the data handles it declared, by the name each is passed
145 # to a page under (index, say).
146 my $database_dir;
147 my %data;
148  
149 # The lists from the configuration, likewise.
150 my @supported_languages;
151 my @rate_limit_trusted_ips;
152  
153 # $config is populated lazily inside doit() so that per-domain configuration
154 # is loaded after the FCGI request has been accepted and the domain is known.
155 my $config;
156  
157 # The pages that may be loaded; worked out in doit() since it needs $config
158 my @valid_pages;
159  
160 # What each valid page says it is for, keyed by page name; only the pages the
161 # site described in its configuration have an entry. Used by the 300 response.
162 my %page_description;
163  
164 # Where a page's module is, for the pages that have a <directory> of their own
165 # or inherit <display><directory>; any other page's module is found in @INC.
166 my %page_directory;
167  
168 # FastCGI signal handling: the FCGI process lives across many requests so we
169 # cannot exit immediately on SIGTERM/SIGUSR1 - we set a flag and exit cleanly
170 # after the current request finishes. See http://fastcgi.com/docs/faq.html#PerlSignals
171 my $requestcount = 0;
172 my $handling_request = 0; # 1 while inside doit(), 0 between requests
173 my $exit_requested = 0; # set to 1 by sig_handler; checked after each request
174  
175 # In-memory set of IPs that have been permanently blacklisted this process
176 # lifetime (e.g. for sending SQL-injection strings).
177 my %blacklisted_ip;
178  
179 # Per-IP request counter cache; created lazily on first request.
180 my $rate_limit_cache;
181  
182 # The CGI::ACL object is built on the first request, once $config is loaded,
183 # because the country blacklist (Web::VWF::Blacklist) can come from the config file.
184 my $acl;
185  
186 =head1 NAME
187  
188 Web::VWF - Versatile Web Framework - easily build a website to support different languages and client platforms
189  
190 =head1 VERSION
191  
192 Version 0.003.0
193  
194 =head1 SYNOPSIS
195  
196 A site's page script, F<cgi-bin/page.fcgi>, is all of this:
197  
198 use FindBin;
199 use Web::VWF;
200  
201 Web::VWF->new({ directory => "$FindBin::Bin/../conf" })->run();
202  
203 =head1 DESCRIPTION
204  
205 Web::VWF is L<VWF|https://github.com/nigelhorne/vwf>, the Versatile Web
206 Framework, packaged as a CPAN distribution. It is an MVC framework for
207 building responsive websites that adapt to the client's language, to the kind
208 of client (web, mobile, robot, search engine or AI crawler) and to whichever
209 database back end the site uses.
210  
211 This module is the distribution's front door: it holds the version number, this
212 overview, and the one thing a site builds by hand - L</new>, the site's
213 configuration and the directory the server keeps it in. Working the framework
214 does these modules:
215  
216 =over 4
217  
218 =item L<Web::VWF::Display>
219  
220 The page renderer. Given the configuration, the browser's language and the
221 kind of client it is, it finds the right template for a page and renders it.
222 A site subclasses it for each of its pages.
223  
224 =item L<Web::VWF::Config>
225  
226 Reads the site's XML configuration.
227  
228 =item L<Web::VWF::Allow>
229  
230 Access control and rate limiting.
231  
232 =item L<Web::VWF::Blacklist>
233  
234 The country blacklist, shared by the page script, L<Web::VWF::Allow> and
235 L<Web::VWF::Display>.
236  
237 =item L<Web::VWF::Utils>
238  
239 Cache initialisation and the Haversine distance calculation.
240  
241 =item L<Web::VWF::CAPTCHA>
242  
243 reCAPTCHA integration and the bypass that follows a solved challenge.
244  
245 =item L<Web::VWF::Mail>
246  
247 Sends mail through sendmail(1), signed with DKIM when the site has a key.
248  
249 =item L<Web::VWF::Throttle>
250  
251 A YAML backend for Data::Throttler that also works on Windows.
252  
253 =item L<Web::VWF::Data::E<lt>sourceE<gt>>
254  
255 The data access layer: CSV, XML and SQLite, through Database::Abstraction.
256  
257 =back
258  
259 F<example/> in this distribution is a small working site to copy and adapt:
260 F<example/cgi-bin/page.fcgi> is the page script, F<example/conf/default> is the
261 configuration (placeholders only - put real keys in your own copy), and
262 F<example/templates/> holds the templates. Run it from the command line with
263 C<CONFIG_DIR=$(pwd)/example/conf root_dir=$(pwd)/example perl -Ilib example/cgi-bin/page.fcgi page=index>.
264  
265 A site's templates sit under a path built from the page's class name,
266 so the file to change when the page module is C<Web::VWF::Display::index> is
267 F<templates/web/Web/VWF/Display/index.tmpl> - see L<Web::VWF::Display>
268 for the rest of the search order.
269  
270 A site that used to carry a renamed copy of VWF of its own deletes that copy,
271 installs this distribution, renames C<VWF::> to C<Web::VWF::> in its own code,
272 and moves its templates from C<.../VWF/Display/> to C<.../Web/VWF/Display/>.
273  
274 =head1 CONFIGURATION
275  
276 A site is configured by files, usually XML, in its F<conf/> directory (see
277 L<Web::VWF::Config> for where they are looked for, in which order, and how an
278 environment variable of the same name as a top-level key overrides that key).
279 The file named F<default> is read first, then one named for the site's domain
280 name, then C<$CONFIG_FILE>. F<example/conf/default> is a complete working
281 example.
282  
283 Every setting is optional. What a site does not say gets the default given
284 below, which is what the framework did before the setting existed.
285  
286 =head2 Environment variables
287  
288 =over 4
289  
290 =item C<CONFIG_DIR>
291  
292 The one directory configuration is read from. When it is set nothing else is
293 searched. It must exist.
294  
295 =item C<CONFIG_FILE>
296  
297 An extra configuration file, read after C<default> and the domain's file.
298  
299 =item C<root_dir>
300  
301 The site's own directory: where the templates are. It also overrides the C<root_dir> of every cache driver,
302 and the one in the configuration. The server's directory of sites, for a
303 server that has one directory for each, is the C<directory> argument of
304 L</new> and defaults to this variable and then to F</usr/lib/vwf>.
305  
306 =item C<CACHEDIR>, C<CACHE_DIR>
307  
308 Where the file-based caches (C<File> and C<BerkeleyDB>) are kept, so that they
309 need not be inside the site: C<CACHEDIR> is used if it is set and not empty,
310 else C<CACHE_DIR>. Either beats C<root_dir> above, and the directory is
311 created if it does not exist.
312  
313 =back
314  
315 =head2 Pages: C<display>
316  
317 <display>
318 <directory>/path/to/modules</directory>
319 <page>
320 <name>states</name>
321 <description>Displays information about the U.S. states</description>
322 <directory>/path/to/this/page</directory>
323 </page>
324 </display>
325  
326 One C<page> for each page that C<?page=> may name. The C<name> is required and
327 may only be word characters. The C<description> is shown in the C<300 Multiple
328 Choices> response, which lists the pages when no C<page> is given. A page is
329 the class C<Web::VWF::Display::E<lt>nameE<gt>> and its template.
330  
331 C<directory> says where the page modules are. In C<display> it is the default
332 for all of them, in a C<page> it is that page's alone and wins. It is where the
333 F<E<lt>nameE<gt>.pm> file is, not a directory of templates. A relative one is
334 taken from the directory the page script is in. A page with no directory at
335 all is found in C<@INC>.
336  
337 With no C<page> at all, C<display>'s C<directory> is searched and every F<.pm>
338 in it, except F<captcha.pm>, is a page. With no C<display> either, that is
339 F<../lib/Web/VWF/Display> beside the script.
340  
341 The old C<pages> is read when there is no C<display>, as is its old form of one
342 string of names, C<< <pages>index, meta_data</pages> >>.
343  
344 =head2 Data: C<databases>
345  
346 <databases>
347 <directory>/path/to/modules</directory>
348 <database>
349 <name>states</name>
350 <description>List of data about the U.S. states</description>
351 <directory>/path/to/this/module</directory>
352 </database>
353 </databases>
354  
355 One C<database> for each data module a page may use. A page is given each as
356 a variable named for the last part of its class, lower-cased: the module
357 C<Web::VWF::Data::states> is C<$states> in the page. A C<name> with no C<::> in it is
358 a C<Web::VWF::Data::> module; one with C<::>, such as C<Bands::Data::bands>, is
359 used as written. The C<description> is only logged.
360  
361 C<directory> is where the module's F<.pm> file is, in the same way as for
362 C<display>, named for the last part of the class. It is not where the
363 database's own files are: that is C<database_dir>.
364  
365 What is loaded, in order of preference:
366  
367 =over 4
368  
369 =item 1. the C<database> elements;
370  
371 =item 2. the old C<data_modules>, one string of names, C<index, states> (an empty
372 one means none);
373  
374 =item 3. with only a C<databases> C<directory>, every F<.pm> in it;
375  
376 =item 4. the default, C<Web::VWF::Data::index>.
377  
378 =back
379  
380 A module that cannot be loaded is logged and skipped; the pages that do not
381 need it still work.
382  
383 This is not called C<data> because the data modules are given the configuration's top-level
384 keys as their constructor arguments, and a key called C<data> replaces the rows
385 of every database.
386  
387 =head2 Other settings
388  
389 =over 4
390  
391 =item C<database_dir>
392  
393 The directory of the database files (CSV, XML, SQLite) read by the data
394 modules. Default: F<../data> beside the script. Pages are told it as
395 C<database_dir> and C<databasedir>.
396  
397 =item C<supported_languages>
398  
399 The languages the site has templates for. Default: C<en-gb>.
400  
401 =item C<allow_ips>
402  
403 Addresses always allowed through the access control. Default: C<108.44.193.70>
404 and C<127.0.0.1>; a site should set its own. It is not called C<allowed_ips>,
405 which would be taken as a constructor argument by L<CGI::ACL>.
406  
407 =item C<rate_limit_trusted_ips>
408  
409 Addresses never rate limited. Default: C<127.0.0.1> and C<192.168.1.1>.
410  
411 =item C<blacklist_countries>
412  
413 Two-letter country codes to refuse. Default: C<BY MD RU CN BR UY TR MA VE SA
414 CY CO MX IN RS PK UA XH>. Setting it replaces the list.
415  
416 =item C<vwflog>
417  
418 The path of the access log, a CSV file that the C<meta_data> page reads.
419 Default: F<vwf.log> in the log directory L<CGI::Info> chooses.
420  
421 =item C<root_dir>
422  
423 The site's own directory, as the environment variable of the same name, which
424 wins. When it is set, the output of pages other than the home page and mail
425 sending is also saved, for a day, in F<save_to> inside it, by L<FCGI::Buffer>.
426  
427 =item C<twitter>
428  
429 A Twitter handle, which gives the pages a tweet button.
430  
431 =item C<SiteTitle>, C<contact>, and any other key
432  
433 All of the configuration is available to the templates, so a site may put its
434 own settings in it. C<SiteTitle> and C<contact> in the example are only
435 that.
436  
437 =item C<syslog>
438  
439 Passed to L<CGI::Info> to log to syslog. C<server> is accepted for C<host>.
440  
441 =item C<DKIMSelector>, C<DKIMKey>
442  
443 The selector and the private key file with which the mail the contact page
444 sends is signed. Without both it is sent unsigned. See L<Web::VWF::Mail>.
445  
446 =back
447  
448 List-valued settings (C<supported_languages>, C<allow_ips>,
449 C<rate_limit_trusted_ips>, C<blacklist_countries>) may be written as one string
450 separated by commas or spaces, or as an element with one child for each value.
451  
452 =head2 C<recaptcha>
453  
454 <recaptcha>
455 <enabled>1</enabled>
456 <site_key>...</site_key>
457 <secret_key>...</secret_key>
458 </recaptcha>
459  
460 Google reCAPTCHA, from L<https://www.google.com/recaptcha/admin>. When it is
461 enabled a client past the soft rate limit is shown a challenge instead of
462 being refused. When it is not, the hard limit refuses with a plain-text C<429>.
463  
464 =head2 C<security>
465  
466 =over 4
467  
468 =item C<security.rate_limiting.max_requests>
469  
470 Requests from one address in the C<time_window> after which the CAPTCHA is
471 shown. Default: 100.
472  
473 =item C<security.rate_limiting.max_requests_hard>
474  
475 After which the request is refused. Default: one and a half times
476 C<max_requests>.
477  
478 =item C<security.rate_limiting.time_window>
479  
480 A duration such as C<60s> or C<5 minutes>. Default: C<60s>.
481  
482 =item C<security.rate_limiting.captcha_bypass_duration>
483  
484 How long a solved CAPTCHA exempts the client from the limit. Default: C<300s>.
485  
486 =item C<security.ids_threshold>
487  
488 The L<CGI::IDS> impact above which a request is treated as an attack.
489 Default: 50.
490  
491 =item C<security.csrf.enable>
492  
493 Set to C<0> to turn off the CSRF cookie that pages with a form are given.
494 Default: 1.
495  
496 =item C<security.csrf.secret>
497  
498 The secret the CSRF tokens are signed with. Without one a random secret is
499 made for each process and the tokens do not outlive it. See
500 L<Web::VWF::Display>.
501  
502 =back
503  
504 =head2 C<throttle>
505  
506 The per-address throttle used by L<Web::VWF::Allow> and by
507 L<Web::VWF::Display>, which share one file.
508  
509 =over 4
510  
511 =item C<throttle.max_items>
512  
513 Requests allowed in the interval. Default: 30.
514  
515 =item C<throttle.interval>
516  
517 The interval, in seconds. Default: 90.
518  
519 =item C<throttle.file>
520  
521 The file the counts are kept in. Default: F<throttle> in the temporary
522 directory.
523  
524 =back
525  
526 =head2 Caches: C<disc_cache> and C<memory_cache>
527  
528 <disc_cache>
529 <driver>File</driver>
530 <root_dir>/var/tmp/cache</root_dir>
531 </disc_cache>
532  
533 The L<CHI> caches. C<driver> is one of C<Memory>, C<BerkeleyDB>, C<DBI>,
534 C<Redis>, C<Memcached>, C<SharedMem>, C<File> or C<Null>. Default for
535 C<disc_cache>: C<BerkeleyDB>, or C<File> where that is not installed; for
536 C<memory_cache>: C<Memory>. The other keys depend on the driver:
537  
538 =over 4
539  
540 =item C<root_dir>
541  
542 For C<File> and C<BerkeleyDB>. Overridden by C<$CACHEDIR>, C<$CACHE_DIR> and
543 C<$root_dir>, in that order. It must exist and be writable unless one of the
544 environment variables made it.
545  
546 =item C<server> (or C<host>) and C<port>
547  
548 For C<Memcached>: C<host:port> pairs separated by commas, or a host with the
549 port in C<port>. Default: C<127.0.0.1:11211>.
550  
551 =item C<connect>, C<create_table>
552  
553 For C<DBI>: the DSN, which is required, and whether to create the table
554 (default: yes).
555  
556 =item C<reconnect>, C<every>, C<encoding>, C<redis_options>
557  
558 For C<Redis>, which also takes C<server> (or C<host>) and C<port> as above.
559  
560 =item C<shm_key>, C<shm_size>, C<max_size>
561  
562 For C<SharedMem>; C<shm_key> is required.
563  
564 =item C<global>, C<datastore>
565  
566 For C<Memory>; not both.
567  
568 =back
569  
570 =head1 SUBROUTINES/METHODS
571  
572 =head2 new
573  
574 The configuration of a site, and the directory the server keeps it in.
575  
576 B<Parameters:>
577  
578 =over 4
579  
580 =item * C<directory> - The directory the server keeps the configuration of the
581 sites it serves in: one directory per site, named after the site, so that a site
582 reached as F<example.com> is configured from F</usr/lib/vwf/example.com>. Any
583 other root can be named here - F</etc/vwf>, F</usr/local/etc/vwf> - and the
584 default is F</usr/lib/vwf>. Optional.
585  
586 =item * C<info>, C<logger>, C<config_directory>, C<config_file>, C<config> -
587 handed to L<Web::VWF::Config>, which describes them.
588  
589 =back
590  
591 =head3 API SPECIFICATION
592  
593 =head4 INPUT
594  
595 {
596 directory => { type => 'string', optional => 1 },
597 info => { type => 'object', optional => 1 },
598 logger => { type => 'object', optional => 1 },
599 config_directory => { type => 'string', optional => 1 },
600 config_file => { type => 'string', optional => 1 },
601 config => { type => 'hashref', optional => 1 },
602 }
603  
604 C<new> croaks with C<directory must be a string> when C<directory> is a reference.
605  
606 =head4 OUTPUT
607  
608 { type => 'object', isa => 'Web::VWF' }
609  
610 =cut
611  
612 sub new
613 {
614 my $proto = shift;
615 my $params = Params::Get::get_params(undef, @_) // {};
616  
617 my $class = ref($proto) || $proto;
618  
619 if(defined($params->{'directory'}) && ref($params->{'directory'})) {
620 Carp::croak('directory must be a string');
621 }
622  
623 return bless {
624 directory => $params->{'directory'},
625 params => $params,
626 }, $class;
627 }
628  
629 =head2 run
630  
631 Serve requests. It accepts FastCGI requests one after another and sends each
632 one to the page named by C<?page=>, as described in L</DESCRIPTION>; an error in
633 one request is logged and does not end the worker.
634  
635 The loop ends, and C<run> returns, when the web server asks the worker to exit,
636 after a TERM or USR1 signal, or when the page script has been changed since the
637 worker started, so that the next request starts a worker with the new code.
638  
639 When C<REMOTE_ADDR> is not set, which is how the script is run from a command
640 line, there is no web server: C<run> handles one request, built from the
641 command-line arguments such as C<page=index>, turns caching off, takes the
642 language from C<LANG> and prints debug logging to STDOUT among the response.
643  
644 C<run> refuses to run again from inside itself, and to run while the script is
645 still being compiled, which happens when something reads the page script as if
646 it were a configuration file. It prints a message on STDERR and returns.
647  
648 =head3 API SPECIFICATION
649  
650 =head4 INPUT
651  
652 {}
653  
654 =head4 OUTPUT
655  
656 { type => 'undef' }
657  
658 =cut
659  
660 =head2 _directory
661  
662 Internal: not part of the public interface. The directory the server keeps this site's configuration in: what C<new()> was
663 given, or F</usr/lib/vwf>. L<Web::VWF::Config> adds the site's own name to it.
664  
665 =head3 API SPECIFICATION
666  
667 =head4 INPUT
668  
669 {}
670  
671 =head4 OUTPUT
672  
673 { type => 'string' }
674  
675 =cut
676  
677 sub _directory
678 {
679 my $self = shift;
680  
681 return $self->{'directory'} // $ENV{root_dir} // $Web::VWF::Config::DEFAULT_DIRECTORY;
682 }
683  
684 =head2 _config
685  
686 Internal: not part of the public interface. The site's configuration, read once and kept: a site that has no F<conf/> of its
687 own beside F<cgi-bin/> is configured from the server's directory for it, and
688 L<Web::VWF::Config> describes the rest of the search.
689  
690 =head3 API SPECIFICATION
691  
692 =head4 INPUT
693  
694 {}
695  
696 =head4 OUTPUT
697  
698 { type => 'object', isa => 'Web::VWF::Config' }
699  
700 =cut
701  
702 sub _config
703 {
704 my $self = shift;
705  
706 return $self->{'config'} ||= Web::VWF::Config->new({
707 %{$self->{'params'}},
708 directory => $self->_directory(),
709 });
710 }
711  
712 # Deferred shutdown handler: set the exit flag and, if we are between requests,
713 # flush caches and exit immediately. If we are mid-request, the flag is checked
714 # after doit() returns so the in-flight response is sent cleanly.
715 sub sig_handler {
716 $exit_requested = 1;
717 $logger->trace('In sig_handler') if($logger);
718 if(!$handling_request) {
719 local $SIG{TERM} = local $SIG{USR1} = 'IGNORE';
720 $logger->info('Shutting down') if($logger);
721 # Flush the page-level response cache to disk before we exit.
722 if($buffercache) {
723 $buffercache->purge();
724 }
725 CHI->stats->flush();
726 # Detach the dispatcher so that Log::WarnDie does not try to log
727 # after the logger object has been destroyed.
728 Log::WarnDie->dispatcher(undef);
729 exit(0);
730 }
731 }
732  
733  
734 # Everything the framework needs before the first request: the bootstrap
735 # CGI::Info, the script's name and temporary directory, the configuration and the
736 # logger. Called once, from run(). The handlers for warnings and fatal errors
737 # are installed first, so that a failure while setting up is reported rather
738 # than lost.
739 sub _init
740 {
741 my $self = shift;
742  
743 return if($self->{initialised});
744 $self->{initialised} = 1;
745  
746 # USR1 and TERM both trigger a clean shutdown; PIPE is silenced because a
747 # broken client connection mid-response should not kill the worker.
748 $SIG{USR1} = \&sig_handler;
749 $SIG{TERM} = \&sig_handler;
750 $SIG{PIPE} = 'IGNORE';
751  
752 # Catch all Perl warnings and mirror them to the per-script stderr log file
753 # as well as to the structured logger.
754 $SIG{__WARN__} = sub {
755 my $msg = join '', @_;
756 if($tmpdir && open(my $fout, '>>', File::Spec->catfile($tmpdir, "$script_name.stderr"))) {
757 print {$fout} (($info ? $info->domain_name() : undef) // '(unknown host)'), ": $msg";
758 close $fout;
759 }
760 $logger->warn($msg) if($logger);
761 };
762  
763 # Catch fatal errors. The $^S guard prevents this handler from interfering
764 # with Error.pm's try/catch blocks which use eval internally.
765 $SIG{__DIE__} = sub {
766 return if $^S; # inside an eval - let the caller handle it
767 my $msg = join '', @_;
768 # Detach Log::WarnDie first so a logger error cannot cause infinite recursion.
769 Log::WarnDie->dispatcher(undef);
770 if($tmpdir && open(my $fout, '>>', File::Spec->catfile($tmpdir, "$script_name.stderr"))) {
771 print {$fout} (($info ? $info->domain_name() : undef) // '(unknown host)'), ": $msg";
772 close $fout;
773 }
774 $logger->fatal($msg) if($logger);
775 CORE::die @_; # re-throw so the original die location is preserved
776 };
777  
778 # Bootstrap CGI::Info before the FCGI accept loop so we can derive the script
779 # name and temp directory for logging even before the first real request.
780 $info = CGI::Info->new();
781 $script_name = $self->{script_name} = basename($info->script_name(), @suffixlist);
782 $tmpdir = $info->tmpdir();
783  
784 # When running under a real HTTP server, redirect STDERR to a per-script log
785 # file in the temp directory so that die/warn messages are captured.
786 if($ENV{'HTTP_USER_AGENT'}) {
787 close STDERR;
788 open(STDERR, '>>', File::Spec->catfile($tmpdir, "$self->{script_name}.stderr"));
789 }
790  
791 # Register the log filter that suppresses known-harmless warnings.
792 Log::WarnDie->filter(\&filter);
793  
794 # Derive the environment-variable prefix from the hostname (e.g. MY_SITE_)
795 # so that per-domain config overrides can be passed as environment variables.
796 $script_dir = $info->script_dir();
797 $env_prefix = uc($info->host_name()) . '_';
798 $env_prefix =~ tr/\./_/; # dots in hostnames become underscores in env var names
799 # Config::Abstraction returns nothing, rather than an object, when no file
800 # matches the domain: a vhost that has not been given a config file yet, the
801 # command line test in the POD, or CI, where the hostname is a container id.
802 # That is not fatal, so the logger is built with no configuration and the
803 # framework carries on with its defaults - dying here would take the site down
804 # over a configuration file that is only there to describe it. The reason is
805 # reported once rather than on every request.
806 $config_abstraction = Config::Abstraction->new(env_prefix => $env_prefix, flatten => 0, config_file => $info->domain_name(), config_dirs => ["$script_dir/../conf/", "$script_dir/../../conf"]);
807 $logger = Log::Abstraction->new($config_abstraction ? $config_abstraction->all() : {});
808 Log::WarnDie->dispatcher($logger);
809  
810 unless($config_abstraction) {
811 $logger->warn('No configuration file for ', ($info->domain_name() // '(unknown host)'), " in $script_dir/../conf/ or $script_dir/../../conf; using the defaults");
812 }
813 }
814  
815 sub run
816 {
817 my $self = shift;
818  
819 # A nested call means something has executed the page script again while
820 # it was running or still compiling - for example a configuration reader
821 # that took the script for its own configuration file and evaluated it.
822 # Refuse, loudly, on STDERR: a die here would be swallowed by that reader's
823 # eval, and carrying on would consume the server's requests.
824 if($RUNNING || (${^GLOBAL_PHASE} ne 'RUN')) {
825 print STDERR __PACKAGE__, '::run() called re-entrantly or during compilation (phase ',
826 ${^GLOBAL_PHASE}, '); is the script being read as its own configuration file?', "\n";
827 return;
828 }
829 local $RUNNING = 1;
830  
831 $self->_init();
832  
833 # my $request = FCGI::Request($stdin, $stdout, $stderr);
834 my $request = FCGI::Request();
835  
836 # --- Main FCGI request loop ---------------------------------------------------
837 # Accept() blocks until the web server sends a new request. It returns a
838 # negative value when the server wants this worker to exit (e.g. graceful
839 # shutdown), which breaks the loop naturally.
840 while($handling_request = ($request->Accept() >= 0)) {
841 # REMOTE_ADDR is absent when running from the command line for testing.
842 # Switch to a verbose debug logger and single-shot mode in that case.
843 unless($ENV{'REMOTE_ADDR'}) {
844 my $timer = Timer::Simple->new();
845  
846 # Disable all caching so every run reflects the current templates.
847 $ENV{'NO_CACHE'} = 1;
848  
849 # Map the LANG shell variable to the HTTP Accept-Language header so
850 # that language detection works the same way it does in production.
851 if((!defined($ENV{'HTTP_ACCEPT_LANGUAGE'})) && defined($ENV{'LANG'})) {
852 my $lang = $ENV{'LANG'};
853 $lang =~ s/\..*$//; # strip encoding suffix (e.g. .UTF-8)
854 $lang =~ tr/_/-/; # POSIX uses _, HTTP uses -
855 $ENV{'HTTP_ACCEPT_LANGUAGE'} = lc($lang);
856 }
857  
858 Database::Abstraction::init({ logger => $logger });
859  
860 # Replace the production logger with a simple STDOUT printer.
861 $logger = Log::Abstraction->new(logger => sub { print join(', ', @{$_[0]->{'message'}}), "\n" }, level => 'debug');
862 Log::WarnDie->dispatcher($logger);
863 $info->set_logger($logger);
864 $self->_set_data_loggers($logger);
865  
866 # Enable full stack traces in Error.pm for command-line debugging.
867 $Error::Debug = 1;
868 try {
869 $self->_doit(debug => 1);
870 } catch Error with {
871 my $msg = shift;
872 warn "$msg\n", $msg->stacktrace();
873 $logger->error($msg);
874 };
875  
876 # Report wall-clock time taken for this command-line invocation.
877 my @elapsed_time = $timer->hms();
878 my $timetaken = int($elapsed_time[2] * 1000);
879 $logger->info("$self->{script_name} completed in ${timetaken}ms");
880  
881 last; # command-line mode is single-shot; exit the accept loop
882 }
883  
884 # -- Live HTTP request --------------------------------------------------
885 $requestcount++;
886 $logger->info("Request $requestcount: ", $ENV{'REMOTE_ADDR'});
887  
888 # Propagate the current logger to all data-access objects that may emit
889 # log messages during this request.
890 $info->set_logger($logger);
891 $self->_set_data_loggers($logger);
892  
893 # Dispatch the request. Any unhandled Error.pm exception is caught here
894 # so a single bad request cannot kill the FCGI worker process.
895 try {
896 $self->_doit(debug => 0);
897 } catch Error with {
898 my $msg = shift;
899 $logger->error("$msg: ", $msg->stacktrace());
900 # Invalidate the response cache on error so a stale error page is
901 # never served to subsequent clients.
902 if($buffercache) {
903 $buffercache->clear();
904 $buffercache = undef;
905 }
906 };
907  
908 $request->Finish();
909  
910 $handling_request = 0;
911 if($exit_requested) {
912 last;
913 }
914 if($ENV{SCRIPT_FILENAME}) {
915 if(-M $ENV{SCRIPT_FILENAME} < 0) {
916 last;
917 }
918 }
919 }
920 }
921  
922 # --- doit() - per-request handler --------------------------------------------
923 # Called once per FCGI iteration. Handles security checks, language detection,
924 # template dispatch, response caching, and access logging.
925 sub _doit
926 {
927 my $self = shift;
928  
929 # Record the wall-clock start time so we can report request duration later.
930 my $request_start = Timer::Simple->new();
931  
932 # CGI::Info caches state between calls; reset it so this request gets a
933 # fresh view of the environment rather than stale values from the last one.
934 CGI::Info->reset();
935  
936 my %params = (ref($_[0]) eq 'HASH') ? %{$_[0]} : @_;
937  
938 # $config is built once per process and reused. We cannot build it before
939 # the first Accept() because the domain name is not known until then.
940 # So don't pass $info in since it was created before the connection
941 $config ||= Web::VWF::Config->new({
942 directory => $self->_directory(),
943 logger => $logger,
944 debug => $params{'debug'},
945 # A throwaway CGI::Lingua is used here only to satisfy Web::VWF::Config;
946 # the real, cache-backed instance is created further below. It has to be
947 # given the default languages rather than the site's configured ones:
948 # this call is what reads the configuration, so the site's list is not
949 # known yet.
950 lingua => CGI::Lingua->new({ supported => [ @DEFAULT_SUPPORTED_LANGUAGES ], info => $info, logger => $logger })
951 });
952  
953 # The site's own settings - its database directory, data modules, languages
954 # and trusted addresses - are read from that configuration, once per process.
955 $self->_site_setup($config);
956  
957 # deny_cloud() blocks all known cloud provider address ranges (AWS, GCP,
958 # Azure) which generate almost no legitimate human traffic but are a
959 # common origin for automated scanning. Geo-blocking is a blunt
960 # instrument but effective at reducing noise. It is also why an address a
961 # site allows here is still refused if its reverse DNS says it is in a cloud.
962 $acl ||= CGI::ACL->new()
963 ->deny_cloud()
964 ->deny_country(country => Web::VWF::Blacklist->new(countries => $config->{'blacklist_countries'})->countries());
965  
966 foreach my $ip ($self->_config_list($config, 'allow_ips', @DEFAULT_ALLOW_IPS)) {
967 $acl->allow_ip($ip);
968 }
969  
970 unless(@valid_pages) {
971 @valid_pages = $self->_valid_pages($config);
972 %page_description = $self->_page_descriptions($config);
973 %page_directory = $self->_page_directories($config, @valid_pages);
974 }
975  
976 # These caches are created lazily on the first request and reused across the
977 # FCGI accept loop to amortise the cost of cache initialisation.
978 # The CGI::Info disc cache stores parsed request data across FCGI restarts.
979 $self->{info_cache} ||= create_disc_cache(config => $config, logger => $logger, namespace => 'CGI::Info');
980  
981 my $options = {
982 cache => $self->{info_cache},
983 logger => $logger
984 };
985  
986 # If syslog is configured, normalise the 'server' key to 'host' (the
987 # Sys::Syslog convention) and pass the stanza to CGI::Info.
988 my $syslog;
989 if($syslog = $config->syslog()) {
990 if($syslog->{'server'}) {
991 $syslog->{'host'} = delete $syslog->{'server'};
992 }
993 $options->{'syslog'} = $syslog;
994 }
995 # Rebuild $info now that we have a cache and the domain is known.
996 $info = CGI::Info->new($options);
997 $logger->debug('In doit - domain is ', $info->domain_name());
998  
999 # Lazily initialise the in-memory rate-limit counter store.
1000 $rate_limit_cache ||= create_memory_cache(config => $config, logger => $logger, namespace => 'rate_limit');
1001  
1002 # Use the real remote IP as the rate-limit key; fall back to 'unknown'
1003 # only in the unlikely event that REMOTE_ADDR is absent.
1004 my $client_ip = $ENV{'REMOTE_ADDR'} || 'unknown';
1005  
1006 # A successful CAPTCHA solve stores a short-lived bypass token in the cache.
1007 # While the token is present the client is exempt from rate-limit checks.
1008 my $captcha_bypass_key = "$self->{script_name}:captcha_bypass:$client_ip";
1009 my $has_captcha_bypass = $rate_limit_cache->get($captcha_bypass_key);
1010  
1011 # Read the current request count for this IP from the sliding-window cache.
1012 my $request_count = $rate_limit_cache->get("$self->{script_name}:rate_limit:$client_ip") || 0;
1013  
1014 # Allow the thresholds to be tuned via the config file; fall back to the
1015 # compile-time constants if the stanza is missing.
1016 my $max_requests = $config->{'security'}->{'rate_limiting'}->{'max_requests'} || $MAX_REQUESTS;
1017 my $max_requests_hard = $config->{'security'}->{'rate_limiting'}->{'max_requests_hard'} || ($max_requests * 1.5);
1018  
1019 # The rate-limit window, as a CHI duration for the counter's TTL, and in
1020 # seconds for the Retry-After header (RFC 9110 section 10.2.3)
1021 my $time_window = $config->{'security'}->{'rate_limiting'}->{'time_window'} || $TIME_WINDOW;
1022 my $retry_after = duration_seconds($time_window);
1023  
1024 # Check if this is a CAPTCHA verification attempt
1025 if ($info->param('g-recaptcha-response')) {
1026 unless(Web::VWF::CAPTCHA->can('new')) {
1027 require Web::VWF::CAPTCHA;
1028 Web::VWF::CAPTCHA->import();
1029 }
1030  
1031 my $recaptcha_config = $config->recaptcha();
1032 if ($recaptcha_config && $recaptcha_config->{enabled}) {
1033 my $captcha = Web::VWF::CAPTCHA->new(
1034 site_key => $recaptcha_config->{site_key},
1035 secret_key => $recaptcha_config->{secret_key},
1036 logger => $logger
1037 );
1038  
1039 if ($captcha->verify($info->param('g-recaptcha-response'), $client_ip)) {
1040 # CAPTCHA verified - grant bypass
1041 my $bypass_duration = $config->{'security'}->{'rate_limiting'}->{'captcha_bypass_duration'} || '300s';
1042 $rate_limit_cache->set($captcha_bypass_key, 1, $bypass_duration);
1043 $rate_limit_cache->set("$self->{script_name}:rate_limit:$client_ip", 0, '60s'); # Reset counter
1044  
1045 $logger->info("CAPTCHA verified for $client_ip - rate limit bypass granted");
1046 $has_captcha_bypass = 1;
1047  
1048 # Redirect the client back to the page they were trying to reach.
1049 # SECURITY - CRLF / header-injection defence:
1050 # Both the page name and SCRIPT_NAME are interpolated into the
1051 # Location header. A raw %0d%0a sequence in either value would
1052 # let an attacker inject arbitrary HTTP response headers
1053 # (response-splitting / header-injection).
1054 # Strip $redirect_page to a strict allowlist (word chars and
1055 # hyphens only), and remove any embedded CR or LF from the
1056 # server-supplied SCRIPT_NAME before building the header.
1057 my $redirect_page = $info->param('page') || 'index';
1058 $redirect_page =~ s/[^A-Za-z0-9_-]//g;
1059  
1060 # Also sanitize the server variable - it is attacker-influenced
1061 # in some reverse-proxy configurations.
1062 my $script = $ENV{SCRIPT_NAME} // '/cgi-bin/page.fcgi';
1063 $script =~ s/[\r\n]//g;
1064  
1065 $info->status(302);
1066 print "Status: 302 Found\r\n",
1067 "Location: ${script}?page=${redirect_page}\r\n\r\n";
1068 return;
1069 } else {
1070 $logger->warn("CAPTCHA verification failed for $client_ip");
1071 # Fall through to show CAPTCHA again
1072 }
1073 }
1074 }
1075  
1076 $vwflog ||= $config->vwflog() || File::Spec->catfile($info->logdir(), 'vwf.log');
1077 # Reader for the CSV access log, used by the meta_data page.
1078 # Created from the same path that vwflog() writes to, which needs $config.
1079 $self->{vwf_log} ||= Web::VWF::Data::vwf_log->new({
1080 directory => dirname($vwflog),
1081 filename => basename($vwflog),
1082 no_entry => 1,
1083 logger => $logger,
1084 });
1085 my $log = Class::Simple->new();
1086  
1087 # Stores things for a month or longer
1088 $lingua_cache ||= create_disc_cache(config => $config, logger => $logger, namespace => 'CGI::Lingua');
1089  
1090 # Language negotiation, offering the languages this site offers
1091 my $lingua = CGI::Lingua->new({
1092 supported => \@supported_languages,
1093 cache => $lingua_cache,
1094 info => $info,
1095 logger => $logger,
1096 debug => $params{'debug'},
1097 syslog => $syslog,
1098 });
1099  
1100 my $cachedir = Web::VWF::Utils::env_cache_dir() || $params{'cachedir'} || $config->{disc_cache}->{root_dir} || File::Spec->catfile($tmpdir, 'cache');
1101  
1102 # Rate limit by IP (unless bypassed)
1103 unless($has_captcha_bypass || grep { $_ eq $client_ip } @rate_limit_trusted_ips) {
1104 if ($request_count >= $max_requests_hard) {
1105 # Hard limit exceeded - show CAPTCHA with warning
1106 my $recaptcha_config = $config->recaptcha();
1107  
1108 if ($recaptcha_config && $recaptcha_config->{enabled}) {
1109 $logger->warn("Hard rate limit exceeded for $client_ip ($request_count requests)");
1110 $info->status(429);
1111  
1112 unless(Web::VWF::Display::captcha->can('new')) {
1113 require Web::VWF::Display::captcha;
1114 Web::VWF::Display::captcha->import();
1115 }
1116 my $display = Web::VWF::Display::captcha->new({
1117 cachedir => $cachedir,
1118 info => $info,
1119 logger => $logger,
1120 lingua => $lingua,
1121 config => $config,
1122 });
1123  
1124 print 'Status: 429 ', HTTP::Status::status_message(429), "\n";
1125 print $display->as_string({
1126 'Retry-After' => $retry_after,
1127 Retry_After => $retry_after,
1128 hard_block => 1,
1129 request_count => $request_count,
1130 });
1131  
1132 $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Hard rate limit - CAPTCHA shown', $log, $request_start);
1133 return;
1134 }
1135  
1136 # No CAPTCHA to offer, so block until the window expires
1137 $logger->warn("Hard rate limit exceeded for $client_ip ($request_count requests)");
1138 send_error(status => 429, body => "Too many requests - try again later\n", headers => "Retry-After: $retry_after\n");
1139 $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Hard rate limit', $log, $request_start);
1140 return;
1141 } elsif ($request_count >= $max_requests) {
1142 # Soft limit exceeded - show CAPTCHA
1143 my $recaptcha_config = $config->recaptcha();
1144  
1145 if ($recaptcha_config && $recaptcha_config->{enabled}) {
1146 $logger->info("Soft rate limit exceeded for $client_ip ($request_count requests) - CAPTCHA challenge issued");
1147 $info->status(429);
1148  
1149 unless(Web::VWF::Display::captcha->can('new')) {
1150 require Web::VWF::Display::captcha;
1151 Web::VWF::Display::captcha->import();
1152 }
1153 my $display = Web::VWF::Display::captcha->new({
1154 cachedir => $cachedir,
1155 info => $info,
1156 logger => $logger,
1157 lingua => $lingua,
1158 config => $config,
1159 });
1160  
1161 if($display) {
1162 print $display->as_string({
1163 'Retry-After' => $retry_after,
1164 Retry_After => $retry_after,
1165 hard_block => 0,
1166 request_count => $request_count,
1167 });
1168  
1169 $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Soft rate limit - CAPTCHA shown', $log, $request_start);
1170 } else {
1171 my $error = $@;
1172 if($error) {
1173 $logger->error($error);
1174 } else {
1175 $logger->error("Unable to show captcha page - request count = $request_count");
1176 }
1177 $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Soft rate limit - CAPTCHA not shown', $log, $request_start);
1178 }
1179 return;
1180 }
1181 }
1182 }
1183  
1184 # Commit the incremented request count back to the sliding-window cache.
1185 # The TTL is the rate-limit window; the counter expires automatically.
1186 $rate_limit_cache->set("$self->{script_name}:rate_limit:$client_ip", $request_count + 1, $time_window);
1187  
1188 # A request with no page parameter cannot be dispatched; send a 300
1189 # response listing the known valid pages instead.
1190 if(!defined($info->param('page'))) {
1191 $logger->info('No page given in ', $info->as_string());
1192 choose();
1193 return;
1194 }
1195  
1196 # Multi-layer access control
1197 # Three independent checks are run in order of cheapness:
1198 # 1. CGI::ACL - cloud ranges, country blocks, explicit IP allow/deny.
1199 # 2. blacklisted() - in-process IP set built from prior SQL-injection attempts.
1200 # 3. Web::VWF::Allow - DShield feed, user-agent blacklist, throttler, IDS.
1201 if(my $remote_addr = $ENV{'REMOTE_ADDR'}) {
1202 my $reason;
1203 if($acl->all_denied(lingua => $lingua)) {
1204 $reason = 'Denied by CGI::ACL';
1205 } elsif(blacklisted($info)) {
1206 $reason = 'Blacklisted for attempting to break in';
1207 } else {
1208 # Web::VWF::Allow may throw an Error object when it blocks a request.
1209 # Catch it here so the 403 response is sent rather than propagating.
1210 try {
1211 unless(Web::VWF::Allow::allow({
1212 info => $info,
1213 lingua => $lingua,
1214 logger => $logger,
1215 cache => $rate_limit_cache,
1216 config => $config,
1217 })) {
1218 $reason = 'Blocked by Web::VWF::Allow';
1219 }
1220 } catch Error with {
1221 $reason = shift;
1222 };
1223 }
1224 if($reason) {
1225 # Return a minimal plain-text 403 - no template rendering so that
1226 # a blocked attacker receives no information about site structure.
1227 send_error(status => 403, body => "Access Denied\n");
1228 $logger->info("$remote_addr: access denied: $reason");
1229 $info->status(403);
1230 $self->_vwflog($vwflog, $info, $lingua, $syslog, $reason, $log, $request_start);
1231 return;
1232 }
1233 }
1234  
1235 # -- FCGI::Buffer setup ------------------------------------------------
1236 # FCGI::Buffer post-processes the output: compresses it, generates an ETag
1237 # and Last-Modified header, and serves a 304 Not Modified when appropriate.
1238 my $args = {
1239 generate_etag => 1,
1240 generate_last_modified => 1,
1241 compress_content => 1,
1242 generate_304 => 1,
1243 info => $info,
1244 optimise_content => 1,
1245 logger => $logger,
1246 # lint_content runs HTML::Tidy on the output; enable in debug mode or
1247 # when explicitly requested via the lint_content query parameter.
1248 lint_content => $info->param('lint_content') // $params{'debug'},
1249 lingua => $lingua
1250 };
1251  
1252 if((!$info->is_search_engine()) && $config->root_dir() &&
1253 ($info->param('page') ne 'home') &&
1254 ((!defined($info->param('action'))) || ($info->param('action') ne 'send'))) {
1255 $args->{'save_to'} = {
1256 directory => File::Spec->catfile($config->root_dir(), 'save_to'),
1257 ttl => 3600 * 24,
1258 create_table => 1
1259 };
1260 }
1261  
1262 my $fb = FCGI::Buffer->new()->init($args);
1263  
1264 if($fb->can_cache()) {
1265 my $cache_key = join('/',
1266 'FCGI::Buffer',
1267 $info->browser_type(),
1268 $info->domain_name(),
1269 $self->{script_name},
1270 $info->as_string(),
1271 $lingua->language()
1272 );
1273 $cache_key =~ s/;string=%5Bobject%20HTMLInputElement%5D//;
1274 $cache_key =~ s/;string=\[object HTMLInputElement\]//;
1275 if($info->param('location')) {
1276 $cache_key .= '/' . $info->param('location');
1277 }
1278 $logger->debug("enabling cache key $cache_key");
1279  
1280 $buffercache ||= create_disc_cache(config => $config, logger => $logger, namespace => $self->{script_name}, root_dir => $cachedir);
1281 $fb->init(
1282 cache_key => $cache_key,
1283 cache => $buffercache,
1284 # generate_304 => 0,
1285 cache_duration => '1 day',
1286 );
1287 if($fb->is_cached()) {
1288 return;
1289 }
1290 }
1291  
1292 # $display holds the instantiated Web::VWF::Display subclass when the page is
1293 # found; $invalidpage is set when the page name is invalid or unloadable.
1294 my $display;
1295 my $invalidpage;
1296  
1297 # Arguments forwarded to every Web::VWF::Display subclass constructor. The
1298 # server's directory is among them because a page reads its configuration for
1299 # itself, and has to look where this script did.
1300 $args = {
1301 cachedir => $cachedir,
1302 info => $info,
1303 logger => $logger,
1304 lingua => $lingua,
1305 config => $config,
1306 server_directory => $SERVER_CONF_DIR,
1307 log => $log
1308 };
1309  
1310 # -- Page dispatch ------------------------------------------------------
1311 # The outer block eval catches any exception thrown during module loading
1312 # or display-object construction, setting $@ for inspection below.
1313 eval {
1314 my $page = $info->param('page');
1315  
1316 # Strip URL fragment identifiers - the server never needs them.
1317 $page =~ s/#.*$//;
1318  
1319 # Reject backslashes: no legitimate page name contains one, and
1320 # they have historically been used to probe Windows path traversal.
1321 $page =~ s/\\//g;
1322  
1323 if($page =~ /\//) {
1324 # A slash in the page name indicates an attempt to traverse
1325 # directories (e.g. page=/etc/passwd or page=http://evil.com).
1326 $logger->info("Blocking '/' in $page");
1327 $info->status(403);
1328 $log->status(403);
1329 $invalidpage = 1;
1330 } else {
1331 # Strip every non-word character so that the page name can only
1332 # contain [A-Za-z0-9_] - safe to use as a Perl package suffix.
1333 $page =~ s/\W//g;
1334 $page =~ s/\s//g;
1335 }
1336  
1337 if($invalidpage) {
1338 # Already rejected above
1339 } elsif(!grep { $_ eq $page } @valid_pages) {
1340 # Only pages on the allow-list are loaded, so that no other
1341 # Web::VWF::Display::* module that happens to be on @INC (e.g. under
1342 # ~/lib/perl5) can be reached from the query string.
1343 $logger->info("Unknown page $page");
1344 $invalidpage = 1;
1345 if($info->status() == 200) {
1346 $info->status(404);
1347 }
1348 } else {
1349 my $display_module = "Web::VWF::Display::$page";
1350  
1351 $logger->debug("doit(): Loading module $display_module from ", $page_directory{$page} // '@INC');
1352 unless($display_module->can('new')) {
1353 # String-eval elimination:
1354 # The original code used eval "require $display_module" which is a
1355 # string eval on a user-derived value. Although $page has been
1356 # stripped of \W characters, a Unicode or locale edge-case could
1357 # allow a bypass. Module::Runtime::require_module() loads a module
1358 # by name using a block eval internally, with no string-eval surface.
1359 #
1360 # A page with a directory is loaded from its file there instead
1361 # of from @INC. Its name has been checked against the valid
1362 # pages, which are \w+ only, so the path cannot leave the
1363 # directory.
1364 if(my $dir = $page_directory{$page}) {
1365 my $file = File::Spec->catfile($dir, "$page.pm");
1366 eval { -f $file or die "No such file $file\n"; require $file };
1367 } else {
1368 eval { require_module($display_module) };
1369 }
1370 $display_module->import() unless $@;
1371 }
1372 if($@) {
1373 $logger->debug("Failed to load module $display_module: $@");
1374 $logger->info("Unknown page $page");
1375 $invalidpage = 1;
1376 if($info->status() == 200) {
1377 $info->status(404);
1378 }
1379 } else {
1380 # use Class::Inspector;
1381 # my $methods = Class::Inspector->methods($display_module);
1382 # print "$display_module exports ", join(', ', @{$methods}), "\n";
1383 $display = do {
1384 eval { $display_module->new($args) };
1385 };
1386 if(!defined($display)) {
1387 if($@) {
1388 $logger->warn("$display_module->new(): $@");
1389 } else {
1390 $logger->notice("Can't instantiate page $page");
1391 }
1392 $invalidpage = 1;
1393 if($info->status() == 200) {
1394 $info->status(404);
1395 }
1396 } elsif(!$display->can('as_string')) {
1397 $logger->warn("Problem understanding $page");
1398 undef $display;
1399 }
1400 }
1401 }
1402 };
1403  
1404 my $error = $@;
1405 if($error) {
1406 if($info->status() == 429) {
1407 $logger->notice($error);
1408 } else {
1409 $logger->error($error);
1410 }
1411 $display = undef;
1412 }
1413  
1414 if(defined($display)) {
1415 # Pass in handles to the databases: the site's own data handles, under
1416 # the name each was declared with, and the access log, which the
1417 # framework reads from the log it writes.
1418 print $display->as_string({
1419 cachedir => $cachedir,
1420 config => $config,
1421 databasedir => $database_dir,
1422 database_dir => $database_dir,
1423 %data,
1424 vwf_log => $self->{vwf_log},
1425 });
1426 $self->_vwflog($vwflog, $info, $lingua, $syslog, '', $log, $request_start);
1427 } elsif($invalidpage) {
1428 if($info->status() == 429) {
1429 # Throttled by Web::VWF::Display
1430 my $interval = $config->{'throttle'}->{'interval'} // 90;
1431 send_error(status => 429, body => "Too many requests - try again later\n", headers => "Retry-After: $interval\n", buffered => 1);
1432 $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Throttled', $log, $request_start);
1433 } else {
1434 choose();
1435 $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Unknown page', $log, $request_start);
1436 }
1437 return;
1438 } else {
1439 $logger->debug('disabling cache');
1440 $fb->init(
1441 cache => undef,
1442 );
1443 # Handle errors gracefully
1444 my ($status, $body);
1445 if($error eq 'Unknown page to display') {
1446 ($status, $body) = (400, "I don't know what you want me to display.\n");
1447 } elsif($error =~ /Can\'t locate .* in \@INC/) {
1448 $logger->error($error);
1449 ($status, $body) = (500, "Software error - contact the webmaster\n");
1450 } elsif(($info->status() == 200) || ($info->status() == 403)) {
1451 # No permission to show this page
1452 ($status, $body) = (403, "Access Denied\n");
1453 } else {
1454 $status = $info->status();
1455 $body = "Page unavailable - something is wrong at your end, please fix and try again\n";
1456 }
1457 send_error(status => $status, body => $body, buffered => 1);
1458 $log->status($status);
1459 $self->_vwflog($vwflog, $info, $lingua, $syslog, $error ? $error : 'Access denied', $log, $request_start);
1460 throw Error::Simple($error ? $error : $info->as_string());
1461 }
1462 }
1463  
1464 # Send a 300 Multiple Choices response listing the pages this site serves.
1465 # Called when the ?page= parameter is absent or the requested page is unknown.
1466 sub choose
1467 {
1468 $logger->info('Called with no page to display');
1469  
1470 my $status = $info->status();
1471  
1472 # If the status is already non-200 (e.g. 404 set by the caller), relay
1473 # that status rather than overriding it with a 300. That only happens
1474 # for an invalid page, after FCGI::Buffer has been set up, and it sends
1475 # the Status line from $info->status() itself.
1476 return if($status != 200);
1477  
1478 print "Status: 300 Multiple Choices\n",
1479 "Content-type: text/plain\n";
1480  
1481 $info->status(300);
1482  
1483 # Include a Last-Modified header based on the script's mtime so that
1484 # caches and conditional-GET clients can revalidate efficiently.
1485 if(my $path = $info->script_path()) {
1486 require HTTP::Date;
1487 HTTP::Date->import();
1488  
1489 my @statb = stat($path);
1490 my $mtime = $statb[9];
1491 print 'Last-Modified: ', HTTP::Date::time2str($mtime), "\n";
1492 }
1493  
1494 print "\n";
1495  
1496 # RFC 7231 section 4.3.2: a HEAD response must not include a body.
1497 unless($ENV{'REQUEST_METHOD'} && ($ENV{'REQUEST_METHOD'} eq 'HEAD')) {
1498 # The same list that doit() allows, so the two cannot differ
1499 print map {
1500 "/cgi-bin/page.fcgi?page=$_" . (length($page_description{$_} // '') ? " - $page_description{$_}" : '') . "\n"
1501 } @valid_pages;
1502 }
1503 }
1504  
1505 # Send a short plain-text error response, and record its status in $info.
1506 # status - the HTTP status code
1507 # body - the text to send (not sent for HEAD)
1508 # headers - optional extra header lines, each ending in "\n", e.g. Retry-After
1509 # buffered - true once FCGI::Buffer is capturing the output. It sends the
1510 # Status line from $info->status(), so it mustn't be printed here too.
1511 sub send_error
1512 {
1513 my %args = (ref($_[0]) eq 'HASH') ? %{$_[0]} : @_;
1514 my $status = $args{'status'};
1515  
1516 $info->status($status);
1517  
1518 print 'Status: ', $status, ' ', HTTP::Status::status_message($status), "\n" unless($args{'buffered'});
1519 print $args{'headers'} // '',
1520 "Content-type: text/plain\n",
1521 "Pragma: no-cache\n\n";
1522  
1523 # RFC 9110 section 9.3.2: a HEAD response must not include a body.
1524 my $is_head = $ENV{'REQUEST_METHOD'} && ($ENV{'REQUEST_METHOD'} eq 'HEAD');
1525 print $args{'body'} if(defined($args{'body'}) && !$is_head);
1526  
1527 return 1;
1528 }
1529  
1530 # A list-valued setting: what the configuration file says about it, or the
1531 # caller's defaults when it says nothing about it. A list can be written as one
1532 # string of values separated by commas or spaces, which is what the plain form
1533 # of a configuration file gives, or as an element with one child per value,
1534 # which is what an XML file gives; both are read, so an XML file can hold a list
1535 # in whichever of the two ways reads better. The values are returned as they
1536 # were written and in the order they were written, with repetitions dropped, and
1537 # a setting that is present but empty means no values at all - so that a site can
1538 # say "none" as well as saying nothing.
1539 sub _config_list
1540 {
1541 # The invocant is not used: the setting and its defaults are all this needs.
1542 my (undef, $config, $key, @defaults) = @_;
1543  
1544 my $value = $config->{$key};
1545 return @defaults unless(defined($value));
1546  
1547 my @values;
1548 if(ref($value) eq 'HASH') {
1549 # One child per value; the names of the children are not used
1550 foreach my $child (values %{$value}) {
1551 push(@values, ref($child) eq 'ARRAY' ? @{$child} : $child);
1552 }
1553 } elsif(ref($value) eq 'ARRAY') {
1554 @values = @{$value};
1555 } else {
1556 @values = split /[\s,]+/, $value;
1557 }
1558  
1559 my %seen;
1560 return grep { defined($_) && length($_) && !$seen{$_}++ } @values;
1561 }
1562  
1563 # The <display> setting, named after the Web::VWF::Display directory the pages
1564 # live in. A site written before the rename used <pages>, which is still read
1565 # when there is no <display>.
1566 sub _display_setting
1567 {
1568 my $config = shift;
1569  
1570 return $config->{'display'} // $config->{'pages'};
1571 }
1572  
1573 # The pages the site's configuration file declares, as a list of
1574 # { name => ..., description => ... } in the order they were written:
1575 # <display>
1576 # <page><name>states</name><description>About the U.S. states</description></page>
1577 # <directory>/path/to/pages</directory>
1578 # </display>
1579 # A <directory> in <display> is where the page modules are looked for, and the
1580 # list of pages when there are no <page>s (see _valid_pages); one in a <page>
1581 # is where that page's module is, and beats the one in <display>. One <page> is read as a hash and several as an array, so both are handled.
1582 # The old form, a single string of names (<pages>index, meta_data</pages>), is
1583 # still read, with no descriptions, so existing sites keep working. A page
1584 # without a usable name is dropped, and so is a repeated one, since the same
1585 # file can be loaded more than once when the configuration is merged.
1586 sub _config_pages
1587 {
1588 my (undef, $config) = @_;
1589  
1590 return _config_entries(_display_setting($config), 'page');
1591 }
1592  
1593 # The common reader for a setting that is a list of named things, each with an
1594 # optional description and directory: the elements called $item inside $setting
1595 # (a hash), or the array or string of names of the old form. Returns
1596 # { name, description, directory } hashes, in order, without repetitions.
1597 sub _config_entries
1598 {
1599 my ($setting, $item) = @_;
1600  
1601 return unless(defined($setting));
1602  
1603 my @entries;
1604 if(ref($setting) eq 'HASH') {
1605 my $found = $setting->{$item};
1606 return unless(defined($found));
1607 @entries = (ref($found) eq 'ARRAY') ? @{$found} : ($found);
1608 } elsif(ref($setting) eq 'ARRAY') {
1609 @entries = @{$setting};
1610 } else {
1611 @entries = split /[\s,]+/, $setting;
1612 }
1613  
1614 my (%seen, @found);
1615 foreach my $entry (@entries) {
1616 next unless(defined($entry));
1617 my ($name, $description, $directory) = (ref($entry) eq 'HASH') ? @{$entry}{qw(name description directory)} : ($entry, undef, undef);
1618 next unless(defined($name) && !ref($name) && length($name));
1619 next if($seen{$name}++);
1620 push @found, {
1621 name => $name,
1622 description => (defined($description) && !ref($description)) ? $description : '',
1623 directory => (defined($directory) && !ref($directory)) ? $directory : '',
1624 };
1625 }
1626 return @found;
1627 }
1628  
1629 # The pages that ?page= may load. Sites declare them in the config file (see
1630 # _config_pages); otherwise every *.pm in the page directory is allowed, except
1631 # captcha, which only the rate limiter shows. The page directory is
1632 # <display><directory>, or ../lib/Web/VWF/Display next to this script.
1633 sub _valid_pages
1634 {
1635 my $self = shift;
1636 my $config = shift;
1637  
1638 my @pages = map { $_->{'name'} } $self->_config_pages($config);
1639 return grep { /\A\w+\z/ } @pages if(@pages);
1640  
1641 # <display><directory> overrides where the page modules are looked for.
1642 # Only the default has the framework's own hard-coded layout.
1643 my $dir = $self->_pages_directory($config);
1644 unless(defined($dir)) {
1645 # FIXME: Remove the hard coding
1646 $dir = File::Spec->catdir($script_dir, File::Spec->updir(), 'lib', 'Web', 'VWF', 'Display');
1647 }
1648 # The list is built first rather than returned from inside sort(): a sort
1649 # that is returned directly takes the caller's context, and sort in scalar
1650 # context has no defined meaning.
1651 my @found = sort(grep { $_ ne 'captcha' } map { basename($_, '.pm') } glob(File::Spec->catfile($dir, '*.pm')));
1652 return @found;
1653 }
1654  
1655 # A directory from the configuration, made absolute: a relative one is taken
1656 # from the script's directory, as the default is. Undef if there isn't one.
1657 sub _resolve_directory
1658 {
1659 my $directory = shift;
1660  
1661 # An explicit undef, not a bare return: this is used inside map, where an
1662 # empty list would unbalance the hash being built
1663 return undef unless(defined($directory) && !ref($directory) && length($directory)); ## no critic (ProhibitExplicitReturnUndef)
1664 return File::Spec->file_name_is_absolute($directory) ? $directory : File::Spec->catdir($script_dir, $directory);
1665 }
1666  
1667 # <display><directory>, resolved, or undef
1668 sub _pages_directory
1669 {
1670 my (undef, $config) = @_;
1671  
1672 my $display = _display_setting($config);
1673 return unless(ref($display) eq 'HASH');
1674 return _resolve_directory($display->{'directory'});
1675 }
1676  
1677 # name => directory for each of the given pages whose module is not simply in
1678 # @INC: the page's own <directory>, else <display><directory> if there is one.
1679 sub _page_directories
1680 {
1681 my ($self, $config, @pages) = @_;
1682  
1683 my $default = $self->_pages_directory($config);
1684 my %own = map { $_->{'name'} => _resolve_directory($_->{'directory'}) } $self->_config_pages($config);
1685  
1686 my %directory;
1687 foreach my $page (@pages) {
1688 my $dir = $own{$page} // $default;
1689 $directory{$page} = $dir if(defined($dir));
1690 }
1691 return %directory;
1692 }
1693  
1694 # name => description for each page the configuration describes
1695 sub _page_descriptions
1696 {
1697 my ($self, $config) = @_;
1698  
1699 return map { $_->{'name'} => $_->{'description'} } grep { length($_->{'description'}) } $self->_config_pages($config);
1700 }
1701  
1702 # The data modules the site declares, as a list of
1703 # { name, module, description, directory } in the order they were written:
1704 # <databases>
1705 # <directory>/path/to/modules</directory>
1706 # <database>
1707 # <name>states</name>
1708 # <description>List of data about the U.S. states</description>
1709 # <directory>/path/to/states/module</directory>
1710 # </database>
1711 # </databases>
1712 # The container is <databases>, not <data>: the data modules take their
1713 # constructor arguments from the configuration's top-level keys, and a key called
1714 # data would replace the rows of every database with the setting itself.
1715 # name is what the page is given (lower-cased last part), module the class, and
1716 # directory where its .pm file is, or undef to find the class in @INC. A
1717 # <directory> in a <database> beats the one in <databases>. The directories are those
1718 # of the modules, not of the databases' files: those are <database_dir>.
1719 # What is declared, in order of preference:
1720 # 1. the <database> elements;
1721 # 2. the old <data_modules>index, states</data_modules> string, which is also
1722 # how a site says "none" by leaving it empty;
1723 # 3. with a <databases><directory> and no <database>, every *.pm in that directory;
1724 # 4. the default, Web::VWF::Data::index.
1725 # A name with no :: is one of this distribution's own, so index is
1726 # Web::VWF::Data::index; a name that has :: is used as written, which is how a
1727 # site names its own modules. The file looked for in a directory is named for
1728 # the last part of the class, whichever way the class was written.
1729 sub _data_modules
1730 {
1731 my ($self, $config) = @_;
1732  
1733 my $databases = $config->{'databases'};
1734 my $default_dir = (ref($databases) eq 'HASH') ? _resolve_directory($databases->{'directory'}) : undef;
1735  
1736 my @entries = _config_entries($databases, 'database');
1737 unless(@entries) {
1738 if(defined($config->{'data_modules'})) {
1739 @entries = map { +{ name => $_, description => '', directory => '' } } $self->_config_list($config, 'data_modules');
1740 } elsif(defined($default_dir)) {
1741 @entries = map { +{ name => $_, description => '', directory => '' } } sort(map { basename($_, '.pm') } glob(File::Spec->catfile($default_dir, '*.pm')));
1742 } else {
1743 @entries = map { +{ name => $_, description => '', directory => '' } } @DEFAULT_DATA_MODULES;
1744 }
1745 }
1746  
1747 my @modules;
1748 foreach my $entry (@entries) {
1749 my $module = ($entry->{'name'} =~ /::/) ? $entry->{'name'} : "Web::VWF::Data::$entry->{'name'}";
1750 my $last = (split(/::/, $module))[-1];
1751 my $dir = _resolve_directory($entry->{'directory'}) // $default_dir;
1752 push @modules, {
1753 name => lc($last),
1754 module => $module,
1755 description => $entry->{'description'},
1756 file => defined($dir) ? File::Spec->catfile($dir, "$last.pm") : undef,
1757 };
1758 }
1759 return @modules;
1760 }
1761  
1762 # Read the settings this site declares, once per process, and act on them: open
1763 # the database directory it names, load the data modules it names and give the
1764 # pages the languages and addresses it names. Everything left unsaid keeps the
1765 # default it had when these values were written into this script.
1766 sub _site_setup
1767 {
1768 my $self = shift;
1769 my $config = shift;
1770  
1771 return if($self->{site_ready});
1772 # The configuration, the database directory, the
1773 # data handles and these lists last for the life of the process, not for one
1774 # request.
1775  
1776 $self->{site_ready} = 1;
1777  
1778 # Where the site's own databases are. Database::Abstraction reads them, and
1779 # the pages are told the directory as databasedir and database_dir.
1780 $database_dir = $config->{'database_dir'}
1781 || File::Spec->catdir($script_dir, File::Spec->updir(), $DEFAULT_DATABASE_DIR);
1782 $logger->debug('Reading databases from ', $database_dir);
1783  
1784 Database::Abstraction::init({
1785 cache => CHI->new(driver => 'Memory', datastore => {}),
1786 cache_duration => '1 day',
1787 directory => $database_dir,
1788 logger => $logger
1789 });
1790  
1791 # The languages the site offers. CGI::Lingua negotiates within this list, so
1792 # listing a language is what makes that language's templates reachable.
1793 @supported_languages = $self->_config_list($config, 'supported_languages', @DEFAULT_SUPPORTED_LANGUAGES);
1794  
1795 # Addresses exempt from rate limiting
1796 @rate_limit_trusted_ips = $self->_config_list($config, 'rate_limit_trusted_ips', @DEFAULT_RATE_LIMIT_TRUSTED_IPS);
1797  
1798 # The site's data modules (see _data_modules). Each is loaded, constructed
1799 # with ->new() and given to every page under the last part of its name,
1800 # lower-cased: Web::VWF::Data::index arrives at a page as index, and a
1801 # site's own Bands::Data::bands as bands.
1802 foreach my $database ($self->_data_modules($config)) {
1803 my $module = $database->{'module'};
1804 my $handle;
1805 eval {
1806 if(defined($database->{'file'})) {
1807 # From the directory the site gave, not from @INC
1808 -f $database->{'file'} or die "No such file $database->{'file'}\n";
1809 require $database->{'file'};
1810 } else {
1811 require_module($module);
1812 }
1813 $module->import() if($module->can('import'));
1814 $handle = $module->new();
1815 };
1816 if(!defined($handle)) {
1817 # A module the site named that cannot be loaded is reported and
1818 # skipped rather than taking the site down: it is a setting now, so
1819 # getting it wrong should not stop the pages that do not need it.
1820 $logger->error("Can't load data module $module: $@");
1821 next;
1822 }
1823  
1824 my $name = $database->{'name'};
1825 $handle->set_logger($logger) if($handle->can('set_logger'));
1826 $data{$name} = $handle;
1827 $logger->debug("Data module $module is available to pages as \$$name", length($database->{'description'}) ? ": $database->{'description'}" : '');
1828 }
1829  
1830 return 1;
1831 }
1832  
1833 # Give the current logger to every data handle. This runs before each request,
1834 # since command-line mode replaces the logger; the handles themselves are made
1835 # once, by _site_setup().
1836 sub _set_data_loggers
1837 {
1838 my $self = shift;
1839 my $logger = shift;
1840  
1841 foreach my $handle (values %data) {
1842 $handle->set_logger($logger) if($handle->can('set_logger'));
1843 }
1844 $self->{vwf_log}->set_logger($logger) if($self->{vwf_log});
1845 }
1846  
1847 # Convert a CHI-style duration ('60s', '2 minutes', '90') to whole seconds
1848 sub duration_seconds
1849 {
1850 my $duration = shift;
1851  
1852 return $duration if($duration =~ /\A\d+\z/);
1853  
1854 require Time::Duration::Parse;
1855  
1856 my $seconds = eval { Time::Duration::Parse::parse_duration($duration) };
1857 if(!defined($seconds)) {
1858 $logger->warn("Can't parse duration '$duration', using 60 seconds");
1859 return 60;
1860 }
1861 return int($seconds);
1862 }
1863  
1864 # Check whether the current request looks like a SQL injection attempt.
1865 # IPs that trigger a match are added to the in-process %blacklisted_ip set so
1866 # that subsequent requests from the same IP are rejected without re-scanning.
1867 sub blacklisted
1868 {
1869 if(my $remote = $ENV{'REMOTE_ADDR'}) {
1870 my $info = shift;
1871  
1872 # Fast path: this IP was already blacklisted earlier in this process
1873 # lifetime; no need to re-scan the request string.
1874 if($blacklisted_ip{$remote}) {
1875 $info->status(403);
1876 return 1;
1877 }
1878  
1879 if(my $string = $info->as_string()) {
1880 # Hard length cap: skip pattern matching entirely on absurdly long
1881 # inputs. Even bounded quantifiers like {1,200} iterate many times
1882 # across a 1 MB string, so cap before we run any regex.
1883 return 0 if length($string) > 8192;
1884  
1885 # SECURITY - ReDoS defence:
1886 # The original patterns used greedy .+ between SQL keywords, which
1887 # causes catastrophic (exponential) backtracking when an attacker
1888 # sends a string that contains the opening keyword but not the
1889 # closing one (e.g. thousands of chars after SELECT with no AND).
1890 # All .+ quantifiers are replaced with the bounded class [^;]{0,N}:
1891 # - the semicolon is a natural SQL statement terminator so it is
1892 # a safe anchor that real SQL injection never crosses, and
1893 # - the explicit upper bound caps backtracking to O(N) steps.
1894 # Word-boundary assertions (\b) also eliminate false positives on
1895 # ordinary words that happen to contain the substring.
1896 if( ($string =~ /SELECT\b[^;]{0,500}\bAND\b/i)
1897 || ($string =~ /ORDER\s+BY\s/i)
1898 || ($string =~ /\bOR\s+NOT\b/i)
1899 || ($string =~ /\bAND\s+\d+=\d+/i)
1900 || ($string =~ /\bTHEN\b[^;]{0,200}\bELSE\b[^;]{0,200}\bEND\b/i)
1901 || ($string =~ /\bAND\b[^;]{0,200}\bSELECT\b/i)
1902 || ($string =~ /\sAND\s[^;]{0,100}\sAND\s/i)
1903 || ($string =~ /\bAND\s+CASE\s+WHEN\b/i)) {
1904 $blacklisted_ip{$remote} = 1;
1905 $info->status(403);
1906 return 1;
1907 }
1908 }
1909 }
1910 return 0;
1911 }
1912  
1913 # False positives we don't need in the logs
1914 sub filter
1915 {
1916 # return 0 if($_[0] =~ /Can't locate Net\/OAuth\/V1_0A\/ProtectedResourceRequest.pm in /);
1917 # return 0 if($_[0] =~ /Can't locate auto\/NetAddr\/IP\/InetBase\/AF_INET6.al in /);
1918 # return 0 if($_[0] =~ /S_IFFIFO is not a valid Fcntl macro at /);
1919  
1920 return 0 if $_[0] =~ /Can't locate (Net\/OAuth\/V1_0A\/ProtectedResourceRequest\.pm|auto\/NetAddr\/IP\/InetBase\/AF_INET6\.al) in |S_IFFIFO is not a valid Fcntl macro at /;
1921 return 1;
1922 }
1923  
1924 # Escape a single value for safe inclusion in a double-quoted CSV field.
1925 # Follows RFC 4180 section 2.7 and additionally neutralises spreadsheet formulas.
1926 sub _csv_escape
1927 {
1928 my $v = shift // '';
1929  
1930 # RFC 4180: a double-quote inside a quoted field is represented by two
1931 # double-quote characters. Without this, one embedded " would break the
1932 # column boundary and corrupt every subsequent field on the row.
1933 $v =~ s/"/""/g;
1934  
1935 # SECURITY - CSV formula injection defence:
1936 # Spreadsheet applications (Excel, LibreOffice Calc) interpret cell values
1937 # that begin with = + - @ TAB or CR as formulas. An attacker who controls
1938 # a logged field (e.g. the page parameter) could inject =cmd|'/C calc'!A0.
1939 # Prefix such values with a single-quote to force literal interpretation.
1940 $v =~ s/^([=+\-@\t\r])/'$1/;
1941  
1942 return $v;
1943 }
1944  
1945 # Write one access record to vwf.log (CSV format) and optionally to syslog.
1946 # All user-influenced fields are escaped through _csv_escape before output.
1947 sub _vwflog
1948 {
1949 my $self = shift;
1950  
1951 my ($vwflog, $info, $lingua, $syslog, $message, $log, $request_start) = @_;
1952  
1953 # Calculate request duration in milliseconds if a start timer was supplied.
1954 # $request_start is a Timer::Simple, so ask it for the seconds that have
1955 # passed: its numeric overload yields elapsed(), so subtracting it from the
1956 # clock gave an epoch's worth of seconds (about 1.8e12 ms per request). A
1957 # plain epoch is still accepted for callers that pass one.
1958 my $duration_ms = '';
1959 if($request_start) {
1960 my $seconds = ref($request_start) ? $request_start->elapsed() : Time::HiRes::time() - $request_start;
1961 $duration_ms = int($seconds * 1000);
1962 }
1963  
1964 # Determine which template was rendered for this request (may be empty on error).
1965 my $template;
1966 if($log) {
1967 $template = $log->template();
1968 }
1969 if(!defined($template)) {
1970 $template = '';
1971 }
1972 $message ||= '';
1973  
1974 # Create the log file with a header row on first use.
1975 if(!-e $vwflog) {
1976 open(my $fout, '>', $vwflog);
1977 print $fout '"domain_name","time","IP","country","type","language","http_code","template","args","messages","error","duration_ms"',
1978 "\n";
1979 close $fout;
1980 }
1981  
1982 # Collect any warn/notice-level messages emitted during this request.
1983 my $warnings;
1984 if(my $messages = $info->messages()) {
1985 $warnings = join('; ',
1986 grep defined, map { (($_->{'level'} eq 'warn') || ($_->{'level'} eq 'notice')) ? $_->{'message'} : undef } @{$messages}
1987 )
1988 }
1989 $warnings ||= '';
1990  
1991 my $country = $lingua->country() || 'unknown';
1992  
1993 # Open the log in append mode. If the open fails we skip logging silently
1994 # so that a disk-full or permissions error does not crash the live request.
1995 if(open(my $fout, '>>', $vwflog)) {
1996 # SECURITY - CSV injection defence:
1997 # Every user-visible field is passed through _csv_escape so that
1998 # embedded double-quotes cannot break the CSV column structure, and
1999 # leading formula characters cannot trigger code execution when the
2000 # file is opened in a spreadsheet application.
2001 print $fout
2002 '"', _csv_escape($info->domain_name()), '",',
2003 '"', strftime('%F %T', localtime), '",',
2004 '"', _csv_escape($ENV{REMOTE_ADDR} // ''), '",',
2005 '"', _csv_escape($country), '",',
2006 '"', _csv_escape($info->browser_type()), '",',
2007 '"', _csv_escape($lingua->language() // ''), '",',
2008 $info->status(), ',',
2009 '"', _csv_escape($template), '",',
2010 '"', _csv_escape($info->as_string(raw => 1)), '",',
2011 '"', _csv_escape($warnings), '",',
2012 '"', _csv_escape($message), '",',
2013 $duration_ms,
2014 "\n";
2015 close($fout);
2016 }
2017  
2018 # Optionally mirror the record to syslog (configured via the syslog stanza).
2019 if($syslog) {
2020 unless(Sys::Syslog->can('openlog')) {
2021 require Sys::Syslog;
2022 Sys::Syslog->import();
2023 }
2024  
2025 # Configure the socket transport if a hash of options was provided.
2026 if(ref($syslog) eq 'HASH') {
2027 Sys::Syslog::setlogsock($syslog);
2028 }
2029 Sys::Syslog::openlog($self->{script_name}, 'cons,pid', 'user');
2030 # Use positional %s/%d format args so that special characters in the
2031 # values cannot be interpreted as syslog format directives.
2032 Sys::Syslog::syslog('info|local0', '%s %s %s %s %s %d %s %s %d %s %s',
2033 $info->domain_name() || '',
2034 $ENV{REMOTE_ADDR} || '',
2035 $country,
2036 $info->browser_type() || '',
2037 $lingua->language() || '',
2038 $info->status() || '',
2039 $template || '',
2040 $info->as_string(raw => 1) || '',
2041 $duration_ms,
2042 $warnings,
2043 $message
2044 );
2045 Sys::Syslog::closelog();
2046 }
2047 }
2048  
2049 sub DESTROY
2050 {
2051 if(defined($^V) && ($^V ge 'v5.14.0')) {
2052 return if ${^GLOBAL_PHASE} eq 'DESTRUCT'; # >= 5.14.0 only
2053 }
2054 my $self = shift;
2055  
2056 # Clean up resources before shutdown
2057 $logger->info('Shutting down') if(defined($logger));
2058 if($buffercache && !$exit_requested) {
2059 $buffercache->purge();
2060 }
2061 if($rate_limit_cache) {
2062 # Memcached can't purge().
2063 # I don't like this hardwired code, it would be better if I could find a way to determine if a driver can run purge()
2064 $rate_limit_cache->purge() if($rate_limit_cache->short_driver_name() ne 'Memcached');
2065 }
2066 if(my $info_cache = $self->{info_cache}) {
2067 $info_cache->purge();
2068 }
2069 if($lingua_cache) {
2070 $lingua_cache->purge();
2071 }
2072 CHI->stats->flush();
2073 Log::WarnDie->dispatcher(undef);
2074 }
2075  
2076 =head1 AUTHOR
2077  
2078 Nigel Horne, C<< <njh at nigelhorne.com> >>
2079  
2080 =head1 LICENCE AND COPYRIGHT
2081  
2082 Copyright 2010-2026 Nigel Horne.
2083  
2084 Usage is subject to the GPL2 licence terms.
2085 If you use it,
2086 please let me know.
2087  
2088 =cut
2089  
2090 1;