| 1 | package Web::VWF; | ||
| 2 | |||
| 3 | # njh@nigelhorne.com | ||
| 4 | |||
| 5 | use strict; | ||
| 6 | use warnings; | ||
| 7 | # use diagnostics; | ||
| 8 | |||
| 9 | use version; our $VERSION = version->declare('0.003.0'); | ||
| 10 | |||
| 11 | # TODO: Try Log::Any::Adapter::Abstraction instead of Log::Any | ||
| 12 | |||
| 13 | BEGIN { | ||
| 14 | # Sanitize environment variables | ||
| 15 | delete @ENV{qw(IFS CDPATH ENV BASH_ENV)}; | ||
| 16 | $ENV{'PATH'} = '/usr/local/bin:/bin:/usr/bin'; # For insecurity | ||
| 17 | } | ||
| 18 | |||
| 19 | no lib '.'; | ||
| 20 | |||
| 21 | use Log::WarnDie 0.09; | ||
| 22 | use Carp::Always; | ||
| 23 | use CGI::ACL 0.06; # For deny_cloud | ||
| 24 | use CGI::Carp qw(fatalsToBrowser); | ||
| 25 | use CGI::Info 0.94; # Gets all messages | ||
| 26 | use CGI::Lingua 0.61; | ||
| 27 | use CHI; | ||
| 28 | use Class::Simple; | ||
| 29 | use Config::Abstraction; | ||
| 30 | use Database::Abstraction; | ||
| 31 | # use Devel::Confess; | ||
| 32 | use File::Basename; | ||
| 33 | # use CGI::Alert $ENV{'SERVER_ADMIN'} || 'you@example.com'; | ||
| 34 | use FCGI; | ||
| 35 | use FCGI::Buffer; | ||
| 36 | use File::HomeDir; | ||
| 37 | use HTTP::Status; | ||
| 38 | use Log::Abstraction; | ||
| 39 | use Error qw(:try); | ||
| 40 | use File::Spec; | ||
| 41 | use Params::Get; | ||
| 42 | use POSIX qw(strftime); | ||
| 43 | use Readonly; | ||
| 44 | use Module::Runtime qw(require_module); # Safe dynamic module loading; avoids string eval | ||
| 45 | use Timer::Simple; | ||
| 46 | use Time::HiRes; | ||
| 47 | |||
| 48 | # FIXME: Sometimes gives Insecure dependency in require while running with -T switch in Module/Runtime.pm | ||
| 49 | # use Taint::Runtime qw($TAINT taint_env); | ||
| 50 | use autodie qw(:all); | ||
| 51 | |||
| 52 | # Where to find the site's own page modules and configuration. The framework | ||
| 53 | # modules themselves are installed by HTTP-VWF, so this only needs to point at | ||
| 54 | # the site's own lib directory when the site has page subclasses of its own. | ||
| 55 | # use lib '/usr/lib/example.com/lib'; | ||
| 56 | # use lib '../lib'; | ||
| 57 | use lib CGI::Info::script_dir() . '/../lib'; | ||
| 58 | # use lib File::HomeDir->my_home() . '/lib/perl5'; | ||
| 59 | |||
| 60 | use Web::VWF::Allow; | ||
| 61 | use Web::VWF::Blacklist; | ||
| 62 | use Web::VWF::Config; | ||
| 63 | use Web::VWF::Utils; | ||
| 64 | |||
| 65 | # Where the server keeps the configuration of the sites it serves: one directory | ||
| 66 | # per site, named after the site, read when the site has no conf/ of its own | ||
| 67 | # beside cgi-bin/. A site whose host keeps them elsewhere changes this line and | ||
| 68 | # nothing else - it is passed to Web::VWF->new() below, and to the page modules, | ||
| 69 | # so that both read the site's configuration from the same place. | ||
| 70 | Readonly my $SERVER_CONF_DIR => '/usr/lib/vwf'; | ||
| 71 | |||
| 72 | # $TAINT = 1; | ||
| 73 | # taint_env(); | ||
| 74 | |||
| 75 | # Soft rate-limit: present a CAPTCHA challenge when a client exceeds this | ||
| 76 | # many requests within $TIME_WINDOW. Overridable via the config file. | ||
| 77 | Readonly my $MAX_REQUESTS => 100; | ||
| 78 | Readonly my $TIME_WINDOW => '60s'; # CHI-style duration string | ||
| 79 | |||
| 80 | Readonly my @suffixlist => ('.pl', '.fcgi'); | ||
| 81 | |||
| 82 | # Set up by _init() on the first call to run(), not when the module is loaded. | ||
| 83 | # Loading a module should do no work: reading configuration here meant that | ||
| 84 | # anything that executed the page script while it was still compiling - such as | ||
| 85 | # a configuration reader that took the script for its own configuration file - | ||
| 86 | # ran the framework again inside itself, with any errors silently swallowed. | ||
| 87 | my $info; | ||
| 88 | my $script_name; | ||
| 89 | my $tmpdir; | ||
| 90 | |||
| 91 | # $vwflog holds the path to the CSV access log; resolved from config on first request. | ||
| 92 | my $vwflog; | ||
| 93 | |||
| 94 | my $lingua_cache; | ||
| 95 | my $buffercache; | ||
| 96 | |||
| 97 | # Also set by _init() | ||
| 98 | my $script_dir; | ||
| 99 | my $env_prefix; | ||
| 100 | my $config_abstraction; | ||
| 101 | my $logger; | ||
| 102 | |||
| 103 | # True while run() is executing, so that a nested call can be refused | ||
| 104 | our $RUNNING; | ||
| 105 | |||
| 106 | # my $pagename = "Web::VWF::Display::$script_name"; | ||
| 107 | # eval "require $pagename"; | ||
| 108 | |||
| 109 | # Loaded later, only when needed | ||
| 110 | # use Web::VWF::Display::index; | ||
| 111 | # use Web::VWF::Display::upload; | ||
| 112 | # use Web::VWF::Display::editor; | ||
| 113 | # use Web::VWF::Display::meta_data; | ||
| 114 | |||
| 115 | # -- What a site declares, and what it gets if it declares nothing ----------- | ||
| 116 | # The directory its databases are in, the data modules it loads, the languages | ||
| 117 | # it offers and the addresses it allows or trusts are all site settings, read | ||
| 118 | # from the domain's configuration file - see "CONFIGURATION" in Web::VWF's | ||
| 119 | # documentation. What is below is what this script used to hardcode, so a site | ||
| 120 | # that says nothing behaves exactly as it did before. The settings are resolved | ||
| 121 | # by _site_setup() on the first request, because the configuration cannot be read | ||
| 122 | # until the domain is known. | ||
| 123 | Readonly my $DEFAULT_DATABASE_DIR => 'data'; # beside cgi-bin/ | ||
| 124 | Readonly my @DEFAULT_DATA_MODULES => qw(Web::VWF::Data::index); | ||
| 125 | Readonly my @DEFAULT_SUPPORTED_LANGUAGES => qw(en-gb); | ||
| 126 | # One site's own address, kept so that a site that came from this script and says | ||
| 127 | # nothing keeps working; a site that wants a different list sets allow_ips. | ||
| 128 | # | ||
| 129 | # It is not called allowed_ips because that is a CGI::ACL constructor argument, | ||
| 130 | # and CGI::ACL's new() runs the site's configuration through | ||
| 131 | # Object::Configure::configure(), which hands a key of that name straight to it | ||
| 132 | # as a string where a hash is expected, breaking the ACL outright. | ||
| 133 | Readonly my @DEFAULT_ALLOW_IPS => qw(108.44.193.70 127.0.0.1); | ||
| 134 | # Loopback and private addresses are exempt from rate limiting. | ||
| 135 | Readonly my @DEFAULT_RATE_LIMIT_TRUSTED_IPS => qw(127.0.0.1 192.168.1.1); | ||
| 136 | |||
| 137 | # The access log is the one data module this script builds itself, because which | ||
| 138 | # log to read comes from the vwflog setting; its class is loaded here so that | ||
| 139 | # doit() can construct it without a compile-time use. | ||
| 140 | require_module('Web::VWF::Data::vwf_log'); | ||
| 141 | Web::VWF::Data::vwf_log->import() if(Web::VWF::Data::vwf_log->can('import')); | ||
| 142 | |||
| 143 | # Resolved from the configuration by _site_setup(): the directory holding the | ||
| 144 | # site's databases, and the data handles it declared, by the name each is passed | ||
| 145 | # to a page under (index, say). | ||
| 146 | my $database_dir; | ||
| 147 | my %data; | ||
| 148 | |||
| 149 | # The lists from the configuration, likewise. | ||
| 150 | my @supported_languages; | ||
| 151 | my @rate_limit_trusted_ips; | ||
| 152 | |||
| 153 | # $config is populated lazily inside doit() so that per-domain configuration | ||
| 154 | # is loaded after the FCGI request has been accepted and the domain is known. | ||
| 155 | my $config; | ||
| 156 | |||
| 157 | # The pages that may be loaded; worked out in doit() since it needs $config | ||
| 158 | my @valid_pages; | ||
| 159 | |||
| 160 | # What each valid page says it is for, keyed by page name; only the pages the | ||
| 161 | # site described in its configuration have an entry. Used by the 300 response. | ||
| 162 | my %page_description; | ||
| 163 | |||
| 164 | # Where a page's module is, for the pages that have a <directory> of their own | ||
| 165 | # or inherit <display><directory>; any other page's module is found in @INC. | ||
| 166 | my %page_directory; | ||
| 167 | |||
| 168 | # FastCGI signal handling: the FCGI process lives across many requests so we | ||
| 169 | # cannot exit immediately on SIGTERM/SIGUSR1 - we set a flag and exit cleanly | ||
| 170 | # after the current request finishes. See http://fastcgi.com/docs/faq.html#PerlSignals | ||
| 171 | my $requestcount = 0; | ||
| 172 | my $handling_request = 0; # 1 while inside doit(), 0 between requests | ||
| 173 | my $exit_requested = 0; # set to 1 by sig_handler; checked after each request | ||
| 174 | |||
| 175 | # In-memory set of IPs that have been permanently blacklisted this process | ||
| 176 | # lifetime (e.g. for sending SQL-injection strings). | ||
| 177 | my %blacklisted_ip; | ||
| 178 | |||
| 179 | # Per-IP request counter cache; created lazily on first request. | ||
| 180 | my $rate_limit_cache; | ||
| 181 | |||
| 182 | # The CGI::ACL object is built on the first request, once $config is loaded, | ||
| 183 | # because the country blacklist (Web::VWF::Blacklist) can come from the config file. | ||
| 184 | my $acl; | ||
| 185 | |||
| 186 | =head1 NAME | ||
| 187 | |||
| 188 | Web::VWF - Versatile Web Framework - easily build a website to support different languages and client platforms | ||
| 189 | |||
| 190 | =head1 VERSION | ||
| 191 | |||
| 192 | Version 0.003.0 | ||
| 193 | |||
| 194 | =head1 SYNOPSIS | ||
| 195 | |||
| 196 | A site's page script, F<cgi-bin/page.fcgi>, is all of this: | ||
| 197 | |||
| 198 | use FindBin; | ||
| 199 | use Web::VWF; | ||
| 200 | |||
| 201 | Web::VWF->new({ directory => "$FindBin::Bin/../conf" })->run(); | ||
| 202 | |||
| 203 | =head1 DESCRIPTION | ||
| 204 | |||
| 205 | Web::VWF is L<VWF|https://github.com/nigelhorne/vwf>, the Versatile Web | ||
| 206 | Framework, packaged as a CPAN distribution. It is an MVC framework for | ||
| 207 | building responsive websites that adapt to the client's language, to the kind | ||
| 208 | of client (web, mobile, robot, search engine or AI crawler) and to whichever | ||
| 209 | database back end the site uses. | ||
| 210 | |||
| 211 | This module is the distribution's front door: it holds the version number, this | ||
| 212 | overview, and the one thing a site builds by hand - L</new>, the site's | ||
| 213 | configuration and the directory the server keeps it in. Working the framework | ||
| 214 | does these modules: | ||
| 215 | |||
| 216 | =over 4 | ||
| 217 | |||
| 218 | =item L<Web::VWF::Display> | ||
| 219 | |||
| 220 | The page renderer. Given the configuration, the browser's language and the | ||
| 221 | kind of client it is, it finds the right template for a page and renders it. | ||
| 222 | A site subclasses it for each of its pages. | ||
| 223 | |||
| 224 | =item L<Web::VWF::Config> | ||
| 225 | |||
| 226 | Reads the site's XML configuration. | ||
| 227 | |||
| 228 | =item L<Web::VWF::Allow> | ||
| 229 | |||
| 230 | Access control and rate limiting. | ||
| 231 | |||
| 232 | =item L<Web::VWF::Blacklist> | ||
| 233 | |||
| 234 | The country blacklist, shared by the page script, L<Web::VWF::Allow> and | ||
| 235 | L<Web::VWF::Display>. | ||
| 236 | |||
| 237 | =item L<Web::VWF::Utils> | ||
| 238 | |||
| 239 | Cache initialisation and the Haversine distance calculation. | ||
| 240 | |||
| 241 | =item L<Web::VWF::CAPTCHA> | ||
| 242 | |||
| 243 | reCAPTCHA integration and the bypass that follows a solved challenge. | ||
| 244 | |||
| 245 | =item L<Web::VWF::Mail> | ||
| 246 | |||
| 247 | Sends mail through sendmail(1), signed with DKIM when the site has a key. | ||
| 248 | |||
| 249 | =item L<Web::VWF::Throttle> | ||
| 250 | |||
| 251 | A YAML backend for Data::Throttler that also works on Windows. | ||
| 252 | |||
| 253 | =item L<Web::VWF::Data::E<lt>sourceE<gt>> | ||
| 254 | |||
| 255 | The data access layer: CSV, XML and SQLite, through Database::Abstraction. | ||
| 256 | |||
| 257 | =back | ||
| 258 | |||
| 259 | F<example/> in this distribution is a small working site to copy and adapt: | ||
| 260 | F<example/cgi-bin/page.fcgi> is the page script, F<example/conf/default> is the | ||
| 261 | configuration (placeholders only - put real keys in your own copy), and | ||
| 262 | F<example/templates/> holds the templates. Run it from the command line with | ||
| 263 | C<CONFIG_DIR=$(pwd)/example/conf root_dir=$(pwd)/example perl -Ilib example/cgi-bin/page.fcgi page=index>. | ||
| 264 | |||
| 265 | A site's templates sit under a path built from the page's class name, | ||
| 266 | so the file to change when the page module is C<Web::VWF::Display::index> is | ||
| 267 | F<templates/web/Web/VWF/Display/index.tmpl> - see L<Web::VWF::Display> | ||
| 268 | for the rest of the search order. | ||
| 269 | |||
| 270 | A site that used to carry a renamed copy of VWF of its own deletes that copy, | ||
| 271 | installs this distribution, renames C<VWF::> to C<Web::VWF::> in its own code, | ||
| 272 | and moves its templates from C<.../VWF/Display/> to C<.../Web/VWF/Display/>. | ||
| 273 | |||
| 274 | =head1 CONFIGURATION | ||
| 275 | |||
| 276 | A site is configured by files, usually XML, in its F<conf/> directory (see | ||
| 277 | L<Web::VWF::Config> for where they are looked for, in which order, and how an | ||
| 278 | environment variable of the same name as a top-level key overrides that key). | ||
| 279 | The file named F<default> is read first, then one named for the site's domain | ||
| 280 | name, then C<$CONFIG_FILE>. F<example/conf/default> is a complete working | ||
| 281 | example. | ||
| 282 | |||
| 283 | Every setting is optional. What a site does not say gets the default given | ||
| 284 | below, which is what the framework did before the setting existed. | ||
| 285 | |||
| 286 | =head2 Environment variables | ||
| 287 | |||
| 288 | =over 4 | ||
| 289 | |||
| 290 | =item C<CONFIG_DIR> | ||
| 291 | |||
| 292 | The one directory configuration is read from. When it is set nothing else is | ||
| 293 | searched. It must exist. | ||
| 294 | |||
| 295 | =item C<CONFIG_FILE> | ||
| 296 | |||
| 297 | An extra configuration file, read after C<default> and the domain's file. | ||
| 298 | |||
| 299 | =item C<root_dir> | ||
| 300 | |||
| 301 | The site's own directory: where the templates are. It also overrides the C<root_dir> of every cache driver, | ||
| 302 | and the one in the configuration. The server's directory of sites, for a | ||
| 303 | server that has one directory for each, is the C<directory> argument of | ||
| 304 | L</new> and defaults to this variable and then to F</usr/lib/vwf>. | ||
| 305 | |||
| 306 | =item C<CACHEDIR>, C<CACHE_DIR> | ||
| 307 | |||
| 308 | Where the file-based caches (C<File> and C<BerkeleyDB>) are kept, so that they | ||
| 309 | need not be inside the site: C<CACHEDIR> is used if it is set and not empty, | ||
| 310 | else C<CACHE_DIR>. Either beats C<root_dir> above, and the directory is | ||
| 311 | created if it does not exist. | ||
| 312 | |||
| 313 | =back | ||
| 314 | |||
| 315 | =head2 Pages: C<display> | ||
| 316 | |||
| 317 | <display> | ||
| 318 | <directory>/path/to/modules</directory> | ||
| 319 | <page> | ||
| 320 | <name>states</name> | ||
| 321 | <description>Displays information about the U.S. states</description> | ||
| 322 | <directory>/path/to/this/page</directory> | ||
| 323 | </page> | ||
| 324 | </display> | ||
| 325 | |||
| 326 | One C<page> for each page that C<?page=> may name. The C<name> is required and | ||
| 327 | may only be word characters. The C<description> is shown in the C<300 Multiple | ||
| 328 | Choices> response, which lists the pages when no C<page> is given. A page is | ||
| 329 | the class C<Web::VWF::Display::E<lt>nameE<gt>> and its template. | ||
| 330 | |||
| 331 | C<directory> says where the page modules are. In C<display> it is the default | ||
| 332 | for all of them, in a C<page> it is that page's alone and wins. It is where the | ||
| 333 | F<E<lt>nameE<gt>.pm> file is, not a directory of templates. A relative one is | ||
| 334 | taken from the directory the page script is in. A page with no directory at | ||
| 335 | all is found in C<@INC>. | ||
| 336 | |||
| 337 | With no C<page> at all, C<display>'s C<directory> is searched and every F<.pm> | ||
| 338 | in it, except F<captcha.pm>, is a page. With no C<display> either, that is | ||
| 339 | F<../lib/Web/VWF/Display> beside the script. | ||
| 340 | |||
| 341 | The old C<pages> is read when there is no C<display>, as is its old form of one | ||
| 342 | string of names, C<< <pages>index, meta_data</pages> >>. | ||
| 343 | |||
| 344 | =head2 Data: C<databases> | ||
| 345 | |||
| 346 | <databases> | ||
| 347 | <directory>/path/to/modules</directory> | ||
| 348 | <database> | ||
| 349 | <name>states</name> | ||
| 350 | <description>List of data about the U.S. states</description> | ||
| 351 | <directory>/path/to/this/module</directory> | ||
| 352 | </database> | ||
| 353 | </databases> | ||
| 354 | |||
| 355 | One C<database> for each data module a page may use. A page is given each as | ||
| 356 | a variable named for the last part of its class, lower-cased: the module | ||
| 357 | C<Web::VWF::Data::states> is C<$states> in the page. A C<name> with no C<::> in it is | ||
| 358 | a C<Web::VWF::Data::> module; one with C<::>, such as C<Bands::Data::bands>, is | ||
| 359 | used as written. The C<description> is only logged. | ||
| 360 | |||
| 361 | C<directory> is where the module's F<.pm> file is, in the same way as for | ||
| 362 | C<display>, named for the last part of the class. It is not where the | ||
| 363 | database's own files are: that is C<database_dir>. | ||
| 364 | |||
| 365 | What is loaded, in order of preference: | ||
| 366 | |||
| 367 | =over 4 | ||
| 368 | |||
| 369 | =item 1. the C<database> elements; | ||
| 370 | |||
| 371 | =item 2. the old C<data_modules>, one string of names, C<index, states> (an empty | ||
| 372 | one means none); | ||
| 373 | |||
| 374 | =item 3. with only a C<databases> C<directory>, every F<.pm> in it; | ||
| 375 | |||
| 376 | =item 4. the default, C<Web::VWF::Data::index>. | ||
| 377 | |||
| 378 | =back | ||
| 379 | |||
| 380 | A module that cannot be loaded is logged and skipped; the pages that do not | ||
| 381 | need it still work. | ||
| 382 | |||
| 383 | This is not called C<data> because the data modules are given the configuration's top-level | ||
| 384 | keys as their constructor arguments, and a key called C<data> replaces the rows | ||
| 385 | of every database. | ||
| 386 | |||
| 387 | =head2 Other settings | ||
| 388 | |||
| 389 | =over 4 | ||
| 390 | |||
| 391 | =item C<database_dir> | ||
| 392 | |||
| 393 | The directory of the database files (CSV, XML, SQLite) read by the data | ||
| 394 | modules. Default: F<../data> beside the script. Pages are told it as | ||
| 395 | C<database_dir> and C<databasedir>. | ||
| 396 | |||
| 397 | =item C<supported_languages> | ||
| 398 | |||
| 399 | The languages the site has templates for. Default: C<en-gb>. | ||
| 400 | |||
| 401 | =item C<allow_ips> | ||
| 402 | |||
| 403 | Addresses always allowed through the access control. Default: C<108.44.193.70> | ||
| 404 | and C<127.0.0.1>; a site should set its own. It is not called C<allowed_ips>, | ||
| 405 | which would be taken as a constructor argument by L<CGI::ACL>. | ||
| 406 | |||
| 407 | =item C<rate_limit_trusted_ips> | ||
| 408 | |||
| 409 | Addresses never rate limited. Default: C<127.0.0.1> and C<192.168.1.1>. | ||
| 410 | |||
| 411 | =item C<blacklist_countries> | ||
| 412 | |||
| 413 | Two-letter country codes to refuse. Default: C<BY MD RU CN BR UY TR MA VE SA | ||
| 414 | CY CO MX IN RS PK UA XH>. Setting it replaces the list. | ||
| 415 | |||
| 416 | =item C<vwflog> | ||
| 417 | |||
| 418 | The path of the access log, a CSV file that the C<meta_data> page reads. | ||
| 419 | Default: F<vwf.log> in the log directory L<CGI::Info> chooses. | ||
| 420 | |||
| 421 | =item C<root_dir> | ||
| 422 | |||
| 423 | The site's own directory, as the environment variable of the same name, which | ||
| 424 | wins. When it is set, the output of pages other than the home page and mail | ||
| 425 | sending is also saved, for a day, in F<save_to> inside it, by L<FCGI::Buffer>. | ||
| 426 | |||
| 427 | =item C<twitter> | ||
| 428 | |||
| 429 | A Twitter handle, which gives the pages a tweet button. | ||
| 430 | |||
| 431 | =item C<SiteTitle>, C<contact>, and any other key | ||
| 432 | |||
| 433 | All of the configuration is available to the templates, so a site may put its | ||
| 434 | own settings in it. C<SiteTitle> and C<contact> in the example are only | ||
| 435 | that. | ||
| 436 | |||
| 437 | =item C<syslog> | ||
| 438 | |||
| 439 | Passed to L<CGI::Info> to log to syslog. C<server> is accepted for C<host>. | ||
| 440 | |||
| 441 | =item C<DKIMSelector>, C<DKIMKey> | ||
| 442 | |||
| 443 | The selector and the private key file with which the mail the contact page | ||
| 444 | sends is signed. Without both it is sent unsigned. See L<Web::VWF::Mail>. | ||
| 445 | |||
| 446 | =back | ||
| 447 | |||
| 448 | List-valued settings (C<supported_languages>, C<allow_ips>, | ||
| 449 | C<rate_limit_trusted_ips>, C<blacklist_countries>) may be written as one string | ||
| 450 | separated by commas or spaces, or as an element with one child for each value. | ||
| 451 | |||
| 452 | =head2 C<recaptcha> | ||
| 453 | |||
| 454 | <recaptcha> | ||
| 455 | <enabled>1</enabled> | ||
| 456 | <site_key>...</site_key> | ||
| 457 | <secret_key>...</secret_key> | ||
| 458 | </recaptcha> | ||
| 459 | |||
| 460 | Google reCAPTCHA, from L<https://www.google.com/recaptcha/admin>. When it is | ||
| 461 | enabled a client past the soft rate limit is shown a challenge instead of | ||
| 462 | being refused. When it is not, the hard limit refuses with a plain-text C<429>. | ||
| 463 | |||
| 464 | =head2 C<security> | ||
| 465 | |||
| 466 | =over 4 | ||
| 467 | |||
| 468 | =item C<security.rate_limiting.max_requests> | ||
| 469 | |||
| 470 | Requests from one address in the C<time_window> after which the CAPTCHA is | ||
| 471 | shown. Default: 100. | ||
| 472 | |||
| 473 | =item C<security.rate_limiting.max_requests_hard> | ||
| 474 | |||
| 475 | After which the request is refused. Default: one and a half times | ||
| 476 | C<max_requests>. | ||
| 477 | |||
| 478 | =item C<security.rate_limiting.time_window> | ||
| 479 | |||
| 480 | A duration such as C<60s> or C<5 minutes>. Default: C<60s>. | ||
| 481 | |||
| 482 | =item C<security.rate_limiting.captcha_bypass_duration> | ||
| 483 | |||
| 484 | How long a solved CAPTCHA exempts the client from the limit. Default: C<300s>. | ||
| 485 | |||
| 486 | =item C<security.ids_threshold> | ||
| 487 | |||
| 488 | The L<CGI::IDS> impact above which a request is treated as an attack. | ||
| 489 | Default: 50. | ||
| 490 | |||
| 491 | =item C<security.csrf.enable> | ||
| 492 | |||
| 493 | Set to C<0> to turn off the CSRF cookie that pages with a form are given. | ||
| 494 | Default: 1. | ||
| 495 | |||
| 496 | =item C<security.csrf.secret> | ||
| 497 | |||
| 498 | The secret the CSRF tokens are signed with. Without one a random secret is | ||
| 499 | made for each process and the tokens do not outlive it. See | ||
| 500 | L<Web::VWF::Display>. | ||
| 501 | |||
| 502 | =back | ||
| 503 | |||
| 504 | =head2 C<throttle> | ||
| 505 | |||
| 506 | The per-address throttle used by L<Web::VWF::Allow> and by | ||
| 507 | L<Web::VWF::Display>, which share one file. | ||
| 508 | |||
| 509 | =over 4 | ||
| 510 | |||
| 511 | =item C<throttle.max_items> | ||
| 512 | |||
| 513 | Requests allowed in the interval. Default: 30. | ||
| 514 | |||
| 515 | =item C<throttle.interval> | ||
| 516 | |||
| 517 | The interval, in seconds. Default: 90. | ||
| 518 | |||
| 519 | =item C<throttle.file> | ||
| 520 | |||
| 521 | The file the counts are kept in. Default: F<throttle> in the temporary | ||
| 522 | directory. | ||
| 523 | |||
| 524 | =back | ||
| 525 | |||
| 526 | =head2 Caches: C<disc_cache> and C<memory_cache> | ||
| 527 | |||
| 528 | <disc_cache> | ||
| 529 | <driver>File</driver> | ||
| 530 | <root_dir>/var/tmp/cache</root_dir> | ||
| 531 | </disc_cache> | ||
| 532 | |||
| 533 | The L<CHI> caches. C<driver> is one of C<Memory>, C<BerkeleyDB>, C<DBI>, | ||
| 534 | C<Redis>, C<Memcached>, C<SharedMem>, C<File> or C<Null>. Default for | ||
| 535 | C<disc_cache>: C<BerkeleyDB>, or C<File> where that is not installed; for | ||
| 536 | C<memory_cache>: C<Memory>. The other keys depend on the driver: | ||
| 537 | |||
| 538 | =over 4 | ||
| 539 | |||
| 540 | =item C<root_dir> | ||
| 541 | |||
| 542 | For C<File> and C<BerkeleyDB>. Overridden by C<$CACHEDIR>, C<$CACHE_DIR> and | ||
| 543 | C<$root_dir>, in that order. It must exist and be writable unless one of the | ||
| 544 | environment variables made it. | ||
| 545 | |||
| 546 | =item C<server> (or C<host>) and C<port> | ||
| 547 | |||
| 548 | For C<Memcached>: C<host:port> pairs separated by commas, or a host with the | ||
| 549 | port in C<port>. Default: C<127.0.0.1:11211>. | ||
| 550 | |||
| 551 | =item C<connect>, C<create_table> | ||
| 552 | |||
| 553 | For C<DBI>: the DSN, which is required, and whether to create the table | ||
| 554 | (default: yes). | ||
| 555 | |||
| 556 | =item C<reconnect>, C<every>, C<encoding>, C<redis_options> | ||
| 557 | |||
| 558 | For C<Redis>, which also takes C<server> (or C<host>) and C<port> as above. | ||
| 559 | |||
| 560 | =item C<shm_key>, C<shm_size>, C<max_size> | ||
| 561 | |||
| 562 | For C<SharedMem>; C<shm_key> is required. | ||
| 563 | |||
| 564 | =item C<global>, C<datastore> | ||
| 565 | |||
| 566 | For C<Memory>; not both. | ||
| 567 | |||
| 568 | =back | ||
| 569 | |||
| 570 | =head1 SUBROUTINES/METHODS | ||
| 571 | |||
| 572 | =head2 new | ||
| 573 | |||
| 574 | The configuration of a site, and the directory the server keeps it in. | ||
| 575 | |||
| 576 | B<Parameters:> | ||
| 577 | |||
| 578 | =over 4 | ||
| 579 | |||
| 580 | =item * C<directory> - The directory the server keeps the configuration of the | ||
| 581 | sites it serves in: one directory per site, named after the site, so that a site | ||
| 582 | reached as F<example.com> is configured from F</usr/lib/vwf/example.com>. Any | ||
| 583 | other root can be named here - F</etc/vwf>, F</usr/local/etc/vwf> - and the | ||
| 584 | default is F</usr/lib/vwf>. Optional. | ||
| 585 | |||
| 586 | =item * C<info>, C<logger>, C<config_directory>, C<config_file>, C<config> - | ||
| 587 | handed to L<Web::VWF::Config>, which describes them. | ||
| 588 | |||
| 589 | =back | ||
| 590 | |||
| 591 | =head3 API SPECIFICATION | ||
| 592 | |||
| 593 | =head4 INPUT | ||
| 594 | |||
| 595 | { | ||
| 596 | directory => { type => 'string', optional => 1 }, | ||
| 597 | info => { type => 'object', optional => 1 }, | ||
| 598 | logger => { type => 'object', optional => 1 }, | ||
| 599 | config_directory => { type => 'string', optional => 1 }, | ||
| 600 | config_file => { type => 'string', optional => 1 }, | ||
| 601 | config => { type => 'hashref', optional => 1 }, | ||
| 602 | } | ||
| 603 | |||
| 604 | C<new> croaks with C<directory must be a string> when C<directory> is a reference. | ||
| 605 | |||
| 606 | =head4 OUTPUT | ||
| 607 | |||
| 608 | { type => 'object', isa => 'Web::VWF' } | ||
| 609 | |||
| 610 | =cut | ||
| 611 | |||
| 612 | sub new | ||
| 613 | { | ||
| 614 | my $proto = shift; | ||
| 615 | my $params = Params::Get::get_params(undef, @_) // {}; | ||
| 616 | |||
| 617 | my $class = ref($proto) || $proto; | ||
| 618 | |||
| 619 | if(defined($params->{'directory'}) && ref($params->{'directory'})) { | ||
| 620 | Carp::croak('directory must be a string'); | ||
| 621 | } | ||
| 622 | |||
| 623 | return bless { | ||
| 624 | directory => $params->{'directory'}, | ||
| 625 | params => $params, | ||
| 626 | }, $class; | ||
| 627 | } | ||
| 628 | |||
| 629 | =head2 run | ||
| 630 | |||
| 631 | Serve requests. It accepts FastCGI requests one after another and sends each | ||
| 632 | one to the page named by C<?page=>, as described in L</DESCRIPTION>; an error in | ||
| 633 | one request is logged and does not end the worker. | ||
| 634 | |||
| 635 | The loop ends, and C<run> returns, when the web server asks the worker to exit, | ||
| 636 | after a TERM or USR1 signal, or when the page script has been changed since the | ||
| 637 | worker started, so that the next request starts a worker with the new code. | ||
| 638 | |||
| 639 | When C<REMOTE_ADDR> is not set, which is how the script is run from a command | ||
| 640 | line, there is no web server: C<run> handles one request, built from the | ||
| 641 | command-line arguments such as C<page=index>, turns caching off, takes the | ||
| 642 | language from C<LANG> and prints debug logging to STDOUT among the response. | ||
| 643 | |||
| 644 | C<run> refuses to run again from inside itself, and to run while the script is | ||
| 645 | still being compiled, which happens when something reads the page script as if | ||
| 646 | it were a configuration file. It prints a message on STDERR and returns. | ||
| 647 | |||
| 648 | =head3 API SPECIFICATION | ||
| 649 | |||
| 650 | =head4 INPUT | ||
| 651 | |||
| 652 | {} | ||
| 653 | |||
| 654 | =head4 OUTPUT | ||
| 655 | |||
| 656 | { type => 'undef' } | ||
| 657 | |||
| 658 | =cut | ||
| 659 | |||
| 660 | =head2 _directory | ||
| 661 | |||
| 662 | Internal: not part of the public interface. The directory the server keeps this site's configuration in: what C<new()> was | ||
| 663 | given, or F</usr/lib/vwf>. L<Web::VWF::Config> adds the site's own name to it. | ||
| 664 | |||
| 665 | =head3 API SPECIFICATION | ||
| 666 | |||
| 667 | =head4 INPUT | ||
| 668 | |||
| 669 | {} | ||
| 670 | |||
| 671 | =head4 OUTPUT | ||
| 672 | |||
| 673 | { type => 'string' } | ||
| 674 | |||
| 675 | =cut | ||
| 676 | |||
| 677 | sub _directory | ||
| 678 | { | ||
| 679 | my $self = shift; | ||
| 680 | |||
| 681 | return $self->{'directory'} // $ENV{root_dir} // $Web::VWF::Config::DEFAULT_DIRECTORY; | ||
| 682 | } | ||
| 683 | |||
| 684 | =head2 _config | ||
| 685 | |||
| 686 | Internal: not part of the public interface. The site's configuration, read once and kept: a site that has no F<conf/> of its | ||
| 687 | own beside F<cgi-bin/> is configured from the server's directory for it, and | ||
| 688 | L<Web::VWF::Config> describes the rest of the search. | ||
| 689 | |||
| 690 | =head3 API SPECIFICATION | ||
| 691 | |||
| 692 | =head4 INPUT | ||
| 693 | |||
| 694 | {} | ||
| 695 | |||
| 696 | =head4 OUTPUT | ||
| 697 | |||
| 698 | { type => 'object', isa => 'Web::VWF::Config' } | ||
| 699 | |||
| 700 | =cut | ||
| 701 | |||
| 702 | sub _config | ||
| 703 | { | ||
| 704 | my $self = shift; | ||
| 705 | |||
| 706 | return $self->{'config'} ||= Web::VWF::Config->new({ | ||
| 707 | %{$self->{'params'}}, | ||
| 708 | directory => $self->_directory(), | ||
| 709 | }); | ||
| 710 | } | ||
| 711 | |||
| 712 | # Deferred shutdown handler: set the exit flag and, if we are between requests, | ||
| 713 | # flush caches and exit immediately. If we are mid-request, the flag is checked | ||
| 714 | # after doit() returns so the in-flight response is sent cleanly. | ||
| 715 | sub sig_handler { | ||
| 716 | $exit_requested = 1; | ||
| 717 | $logger->trace('In sig_handler') if($logger); | ||
| 718 | if(!$handling_request) { | ||
| 719 | local $SIG{TERM} = local $SIG{USR1} = 'IGNORE'; | ||
| 720 | $logger->info('Shutting down') if($logger); | ||
| 721 | # Flush the page-level response cache to disk before we exit. | ||
| 722 | if($buffercache) { | ||
| 723 | $buffercache->purge(); | ||
| 724 | } | ||
| 725 | CHI->stats->flush(); | ||
| 726 | # Detach the dispatcher so that Log::WarnDie does not try to log | ||
| 727 | # after the logger object has been destroyed. | ||
| 728 | Log::WarnDie->dispatcher(undef); | ||
| 729 | exit(0); | ||
| 730 | } | ||
| 731 | } | ||
| 732 | |||
| 733 | |||
| 734 | # Everything the framework needs before the first request: the bootstrap | ||
| 735 | # CGI::Info, the script's name and temporary directory, the configuration and the | ||
| 736 | # logger. Called once, from run(). The handlers for warnings and fatal errors | ||
| 737 | # are installed first, so that a failure while setting up is reported rather | ||
| 738 | # than lost. | ||
| 739 | sub _init | ||
| 740 | { | ||
| 741 | my $self = shift; | ||
| 742 | |||
| 743 | return if($self->{initialised}); | ||
| 744 | $self->{initialised} = 1; | ||
| 745 | |||
| 746 | # USR1 and TERM both trigger a clean shutdown; PIPE is silenced because a | ||
| 747 | # broken client connection mid-response should not kill the worker. | ||
| 748 | $SIG{USR1} = \&sig_handler; | ||
| 749 | $SIG{TERM} = \&sig_handler; | ||
| 750 | $SIG{PIPE} = 'IGNORE'; | ||
| 751 | |||
| 752 | # Catch all Perl warnings and mirror them to the per-script stderr log file | ||
| 753 | # as well as to the structured logger. | ||
| 754 | $SIG{__WARN__} = sub { | ||
| 755 | my $msg = join '', @_; | ||
| 756 | if($tmpdir && open(my $fout, '>>', File::Spec->catfile($tmpdir, "$script_name.stderr"))) { | ||
| 757 | print {$fout} (($info ? $info->domain_name() : undef) // '(unknown host)'), ": $msg"; | ||
| 758 | close $fout; | ||
| 759 | } | ||
| 760 | $logger->warn($msg) if($logger); | ||
| 761 | }; | ||
| 762 | |||
| 763 | # Catch fatal errors. The $^S guard prevents this handler from interfering | ||
| 764 | # with Error.pm's try/catch blocks which use eval internally. | ||
| 765 | $SIG{__DIE__} = sub { | ||
| 766 | return if $^S; # inside an eval - let the caller handle it | ||
| 767 | my $msg = join '', @_; | ||
| 768 | # Detach Log::WarnDie first so a logger error cannot cause infinite recursion. | ||
| 769 | Log::WarnDie->dispatcher(undef); | ||
| 770 | if($tmpdir && open(my $fout, '>>', File::Spec->catfile($tmpdir, "$script_name.stderr"))) { | ||
| 771 | print {$fout} (($info ? $info->domain_name() : undef) // '(unknown host)'), ": $msg"; | ||
| 772 | close $fout; | ||
| 773 | } | ||
| 774 | $logger->fatal($msg) if($logger); | ||
| 775 | CORE::die @_; # re-throw so the original die location is preserved | ||
| 776 | }; | ||
| 777 | |||
| 778 | # Bootstrap CGI::Info before the FCGI accept loop so we can derive the script | ||
| 779 | # name and temp directory for logging even before the first real request. | ||
| 780 | $info = CGI::Info->new(); | ||
| 781 | $script_name = $self->{script_name} = basename($info->script_name(), @suffixlist); | ||
| 782 | $tmpdir = $info->tmpdir(); | ||
| 783 | |||
| 784 | # When running under a real HTTP server, redirect STDERR to a per-script log | ||
| 785 | # file in the temp directory so that die/warn messages are captured. | ||
| 786 | if($ENV{'HTTP_USER_AGENT'}) { | ||
| 787 | close STDERR; | ||
| 788 | open(STDERR, '>>', File::Spec->catfile($tmpdir, "$self->{script_name}.stderr")); | ||
| 789 | } | ||
| 790 | |||
| 791 | # Register the log filter that suppresses known-harmless warnings. | ||
| 792 | Log::WarnDie->filter(\&filter); | ||
| 793 | |||
| 794 | # Derive the environment-variable prefix from the hostname (e.g. MY_SITE_) | ||
| 795 | # so that per-domain config overrides can be passed as environment variables. | ||
| 796 | $script_dir = $info->script_dir(); | ||
| 797 | $env_prefix = uc($info->host_name()) . '_'; | ||
| 798 | $env_prefix =~ tr/\./_/; # dots in hostnames become underscores in env var names | ||
| 799 | # Config::Abstraction returns nothing, rather than an object, when no file | ||
| 800 | # matches the domain: a vhost that has not been given a config file yet, the | ||
| 801 | # command line test in the POD, or CI, where the hostname is a container id. | ||
| 802 | # That is not fatal, so the logger is built with no configuration and the | ||
| 803 | # framework carries on with its defaults - dying here would take the site down | ||
| 804 | # over a configuration file that is only there to describe it. The reason is | ||
| 805 | # reported once rather than on every request. | ||
| 806 | $config_abstraction = Config::Abstraction->new(env_prefix => $env_prefix, flatten => 0, config_file => $info->domain_name(), config_dirs => ["$script_dir/../conf/", "$script_dir/../../conf"]); | ||
| 807 | $logger = Log::Abstraction->new($config_abstraction ? $config_abstraction->all() : {}); | ||
| 808 | Log::WarnDie->dispatcher($logger); | ||
| 809 | |||
| 810 | unless($config_abstraction) { | ||
| 811 | $logger->warn('No configuration file for ', ($info->domain_name() // '(unknown host)'), " in $script_dir/../conf/ or $script_dir/../../conf; using the defaults"); | ||
| 812 | } | ||
| 813 | } | ||
| 814 | |||
| 815 | sub run | ||
| 816 | { | ||
| 817 | my $self = shift; | ||
| 818 | |||
| 819 | # A nested call means something has executed the page script again while | ||
| 820 | # it was running or still compiling - for example a configuration reader | ||
| 821 | # that took the script for its own configuration file and evaluated it. | ||
| 822 | # Refuse, loudly, on STDERR: a die here would be swallowed by that reader's | ||
| 823 | # eval, and carrying on would consume the server's requests. | ||
| 824 | if($RUNNING || (${^GLOBAL_PHASE} ne 'RUN')) { | ||
| 825 | print STDERR __PACKAGE__, '::run() called re-entrantly or during compilation (phase ', | ||
| 826 | ${^GLOBAL_PHASE}, '); is the script being read as its own configuration file?', "\n"; | ||
| 827 | return; | ||
| 828 | } | ||
| 829 | local $RUNNING = 1; | ||
| 830 | |||
| 831 | $self->_init(); | ||
| 832 | |||
| 833 | # my $request = FCGI::Request($stdin, $stdout, $stderr); | ||
| 834 | my $request = FCGI::Request(); | ||
| 835 | |||
| 836 | # --- Main FCGI request loop --------------------------------------------------- | ||
| 837 | # Accept() blocks until the web server sends a new request. It returns a | ||
| 838 | # negative value when the server wants this worker to exit (e.g. graceful | ||
| 839 | # shutdown), which breaks the loop naturally. | ||
| 840 | while($handling_request = ($request->Accept() >= 0)) { | ||
| 841 | # REMOTE_ADDR is absent when running from the command line for testing. | ||
| 842 | # Switch to a verbose debug logger and single-shot mode in that case. | ||
| 843 | unless($ENV{'REMOTE_ADDR'}) { | ||
| 844 | my $timer = Timer::Simple->new(); | ||
| 845 | |||
| 846 | # Disable all caching so every run reflects the current templates. | ||
| 847 | $ENV{'NO_CACHE'} = 1; | ||
| 848 | |||
| 849 | # Map the LANG shell variable to the HTTP Accept-Language header so | ||
| 850 | # that language detection works the same way it does in production. | ||
| 851 | if((!defined($ENV{'HTTP_ACCEPT_LANGUAGE'})) && defined($ENV{'LANG'})) { | ||
| 852 | my $lang = $ENV{'LANG'}; | ||
| 853 | $lang =~ s/\..*$//; # strip encoding suffix (e.g. .UTF-8) | ||
| 854 | $lang =~ tr/_/-/; # POSIX uses _, HTTP uses - | ||
| 855 | $ENV{'HTTP_ACCEPT_LANGUAGE'} = lc($lang); | ||
| 856 | } | ||
| 857 | |||
| 858 | Database::Abstraction::init({ logger => $logger }); | ||
| 859 | |||
| 860 | # Replace the production logger with a simple STDOUT printer. | ||
| 861 | $logger = Log::Abstraction->new(logger => sub { print join(', ', @{$_[0]->{'message'}}), "\n" }, level => 'debug'); | ||
| 862 | Log::WarnDie->dispatcher($logger); | ||
| 863 | $info->set_logger($logger); | ||
| 864 | $self->_set_data_loggers($logger); | ||
| 865 | |||
| 866 | # Enable full stack traces in Error.pm for command-line debugging. | ||
| 867 | $Error::Debug = 1; | ||
| 868 | try { | ||
| 869 | $self->_doit(debug => 1); | ||
| 870 | } catch Error with { | ||
| 871 | my $msg = shift; | ||
| 872 | warn "$msg\n", $msg->stacktrace(); | ||
| 873 | $logger->error($msg); | ||
| 874 | }; | ||
| 875 | |||
| 876 | # Report wall-clock time taken for this command-line invocation. | ||
| 877 | my @elapsed_time = $timer->hms(); | ||
| 878 | my $timetaken = int($elapsed_time[2] * 1000); | ||
| 879 | $logger->info("$self->{script_name} completed in ${timetaken}ms"); | ||
| 880 | |||
| 881 | last; # command-line mode is single-shot; exit the accept loop | ||
| 882 | } | ||
| 883 | |||
| 884 | # -- Live HTTP request -------------------------------------------------- | ||
| 885 | $requestcount++; | ||
| 886 | $logger->info("Request $requestcount: ", $ENV{'REMOTE_ADDR'}); | ||
| 887 | |||
| 888 | # Propagate the current logger to all data-access objects that may emit | ||
| 889 | # log messages during this request. | ||
| 890 | $info->set_logger($logger); | ||
| 891 | $self->_set_data_loggers($logger); | ||
| 892 | |||
| 893 | # Dispatch the request. Any unhandled Error.pm exception is caught here | ||
| 894 | # so a single bad request cannot kill the FCGI worker process. | ||
| 895 | try { | ||
| 896 | $self->_doit(debug => 0); | ||
| 897 | } catch Error with { | ||
| 898 | my $msg = shift; | ||
| 899 | $logger->error("$msg: ", $msg->stacktrace()); | ||
| 900 | # Invalidate the response cache on error so a stale error page is | ||
| 901 | # never served to subsequent clients. | ||
| 902 | if($buffercache) { | ||
| 903 | $buffercache->clear(); | ||
| 904 | $buffercache = undef; | ||
| 905 | } | ||
| 906 | }; | ||
| 907 | |||
| 908 | $request->Finish(); | ||
| 909 | |||
| 910 | $handling_request = 0; | ||
| 911 | if($exit_requested) { | ||
| 912 | last; | ||
| 913 | } | ||
| 914 | if($ENV{SCRIPT_FILENAME}) { | ||
| 915 | if(-M $ENV{SCRIPT_FILENAME} < 0) { | ||
| 916 | last; | ||
| 917 | } | ||
| 918 | } | ||
| 919 | } | ||
| 920 | } | ||
| 921 | |||
| 922 | # --- doit() - per-request handler -------------------------------------------- | ||
| 923 | # Called once per FCGI iteration. Handles security checks, language detection, | ||
| 924 | # template dispatch, response caching, and access logging. | ||
| 925 | sub _doit | ||
| 926 | { | ||
| 927 | my $self = shift; | ||
| 928 | |||
| 929 | # Record the wall-clock start time so we can report request duration later. | ||
| 930 | my $request_start = Timer::Simple->new(); | ||
| 931 | |||
| 932 | # CGI::Info caches state between calls; reset it so this request gets a | ||
| 933 | # fresh view of the environment rather than stale values from the last one. | ||
| 934 | CGI::Info->reset(); | ||
| 935 | |||
| 936 | my %params = (ref($_[0]) eq 'HASH') ? %{$_[0]} : @_; | ||
| 937 | |||
| 938 | # $config is built once per process and reused. We cannot build it before | ||
| 939 | # the first Accept() because the domain name is not known until then. | ||
| 940 | # So don't pass $info in since it was created before the connection | ||
| 941 | $config ||= Web::VWF::Config->new({ | ||
| 942 | directory => $self->_directory(), | ||
| 943 | logger => $logger, | ||
| 944 | debug => $params{'debug'}, | ||
| 945 | # A throwaway CGI::Lingua is used here only to satisfy Web::VWF::Config; | ||
| 946 | # the real, cache-backed instance is created further below. It has to be | ||
| 947 | # given the default languages rather than the site's configured ones: | ||
| 948 | # this call is what reads the configuration, so the site's list is not | ||
| 949 | # known yet. | ||
| 950 | lingua => CGI::Lingua->new({ supported => [ @DEFAULT_SUPPORTED_LANGUAGES ], info => $info, logger => $logger }) | ||
| 951 | }); | ||
| 952 | |||
| 953 | # The site's own settings - its database directory, data modules, languages | ||
| 954 | # and trusted addresses - are read from that configuration, once per process. | ||
| 955 | $self->_site_setup($config); | ||
| 956 | |||
| 957 | # deny_cloud() blocks all known cloud provider address ranges (AWS, GCP, | ||
| 958 | # Azure) which generate almost no legitimate human traffic but are a | ||
| 959 | # common origin for automated scanning. Geo-blocking is a blunt | ||
| 960 | # instrument but effective at reducing noise. It is also why an address a | ||
| 961 | # site allows here is still refused if its reverse DNS says it is in a cloud. | ||
| 962 | $acl ||= CGI::ACL->new() | ||
| 963 | ->deny_cloud() | ||
| 964 | ->deny_country(country => Web::VWF::Blacklist->new(countries => $config->{'blacklist_countries'})->countries()); | ||
| 965 | |||
| 966 | foreach my $ip ($self->_config_list($config, 'allow_ips', @DEFAULT_ALLOW_IPS)) { | ||
| 967 | $acl->allow_ip($ip); | ||
| 968 | } | ||
| 969 | |||
| 970 | unless(@valid_pages) { | ||
| 971 | @valid_pages = $self->_valid_pages($config); | ||
| 972 | %page_description = $self->_page_descriptions($config); | ||
| 973 | %page_directory = $self->_page_directories($config, @valid_pages); | ||
| 974 | } | ||
| 975 | |||
| 976 | # These caches are created lazily on the first request and reused across the | ||
| 977 | # FCGI accept loop to amortise the cost of cache initialisation. | ||
| 978 | # The CGI::Info disc cache stores parsed request data across FCGI restarts. | ||
| 979 | $self->{info_cache} ||= create_disc_cache(config => $config, logger => $logger, namespace => 'CGI::Info'); | ||
| 980 | |||
| 981 | my $options = { | ||
| 982 | cache => $self->{info_cache}, | ||
| 983 | logger => $logger | ||
| 984 | }; | ||
| 985 | |||
| 986 | # If syslog is configured, normalise the 'server' key to 'host' (the | ||
| 987 | # Sys::Syslog convention) and pass the stanza to CGI::Info. | ||
| 988 | my $syslog; | ||
| 989 | if($syslog = $config->syslog()) { | ||
| 990 | if($syslog->{'server'}) { | ||
| 991 | $syslog->{'host'} = delete $syslog->{'server'}; | ||
| 992 | } | ||
| 993 | $options->{'syslog'} = $syslog; | ||
| 994 | } | ||
| 995 | # Rebuild $info now that we have a cache and the domain is known. | ||
| 996 | $info = CGI::Info->new($options); | ||
| 997 | $logger->debug('In doit - domain is ', $info->domain_name()); | ||
| 998 | |||
| 999 | # Lazily initialise the in-memory rate-limit counter store. | ||
| 1000 | $rate_limit_cache ||= create_memory_cache(config => $config, logger => $logger, namespace => 'rate_limit'); | ||
| 1001 | |||
| 1002 | # Use the real remote IP as the rate-limit key; fall back to 'unknown' | ||
| 1003 | # only in the unlikely event that REMOTE_ADDR is absent. | ||
| 1004 | my $client_ip = $ENV{'REMOTE_ADDR'} || 'unknown'; | ||
| 1005 | |||
| 1006 | # A successful CAPTCHA solve stores a short-lived bypass token in the cache. | ||
| 1007 | # While the token is present the client is exempt from rate-limit checks. | ||
| 1008 | my $captcha_bypass_key = "$self->{script_name}:captcha_bypass:$client_ip"; | ||
| 1009 | my $has_captcha_bypass = $rate_limit_cache->get($captcha_bypass_key); | ||
| 1010 | |||
| 1011 | # Read the current request count for this IP from the sliding-window cache. | ||
| 1012 | my $request_count = $rate_limit_cache->get("$self->{script_name}:rate_limit:$client_ip") || 0; | ||
| 1013 | |||
| 1014 | # Allow the thresholds to be tuned via the config file; fall back to the | ||
| 1015 | # compile-time constants if the stanza is missing. | ||
| 1016 | my $max_requests = $config->{'security'}->{'rate_limiting'}->{'max_requests'} || $MAX_REQUESTS; | ||
| 1017 | my $max_requests_hard = $config->{'security'}->{'rate_limiting'}->{'max_requests_hard'} || ($max_requests * 1.5); | ||
| 1018 | |||
| 1019 | # The rate-limit window, as a CHI duration for the counter's TTL, and in | ||
| 1020 | # seconds for the Retry-After header (RFC 9110 section 10.2.3) | ||
| 1021 | my $time_window = $config->{'security'}->{'rate_limiting'}->{'time_window'} || $TIME_WINDOW; | ||
| 1022 | my $retry_after = duration_seconds($time_window); | ||
| 1023 | |||
| 1024 | # Check if this is a CAPTCHA verification attempt | ||
| 1025 | if ($info->param('g-recaptcha-response')) { | ||
| 1026 | unless(Web::VWF::CAPTCHA->can('new')) { | ||
| 1027 | require Web::VWF::CAPTCHA; | ||
| 1028 | Web::VWF::CAPTCHA->import(); | ||
| 1029 | } | ||
| 1030 | |||
| 1031 | my $recaptcha_config = $config->recaptcha(); | ||
| 1032 | if ($recaptcha_config && $recaptcha_config->{enabled}) { | ||
| 1033 | my $captcha = Web::VWF::CAPTCHA->new( | ||
| 1034 | site_key => $recaptcha_config->{site_key}, | ||
| 1035 | secret_key => $recaptcha_config->{secret_key}, | ||
| 1036 | logger => $logger | ||
| 1037 | ); | ||
| 1038 | |||
| 1039 | if ($captcha->verify($info->param('g-recaptcha-response'), $client_ip)) { | ||
| 1040 | # CAPTCHA verified - grant bypass | ||
| 1041 | my $bypass_duration = $config->{'security'}->{'rate_limiting'}->{'captcha_bypass_duration'} || '300s'; | ||
| 1042 | $rate_limit_cache->set($captcha_bypass_key, 1, $bypass_duration); | ||
| 1043 | $rate_limit_cache->set("$self->{script_name}:rate_limit:$client_ip", 0, '60s'); # Reset counter | ||
| 1044 | |||
| 1045 | $logger->info("CAPTCHA verified for $client_ip - rate limit bypass granted"); | ||
| 1046 | $has_captcha_bypass = 1; | ||
| 1047 | |||
| 1048 | # Redirect the client back to the page they were trying to reach. | ||
| 1049 | # SECURITY - CRLF / header-injection defence: | ||
| 1050 | # Both the page name and SCRIPT_NAME are interpolated into the | ||
| 1051 | # Location header. A raw %0d%0a sequence in either value would | ||
| 1052 | # let an attacker inject arbitrary HTTP response headers | ||
| 1053 | # (response-splitting / header-injection). | ||
| 1054 | # Strip $redirect_page to a strict allowlist (word chars and | ||
| 1055 | # hyphens only), and remove any embedded CR or LF from the | ||
| 1056 | # server-supplied SCRIPT_NAME before building the header. | ||
| 1057 | my $redirect_page = $info->param('page') || 'index'; | ||
| 1058 | $redirect_page =~ s/[^A-Za-z0-9_-]//g; | ||
| 1059 | |||
| 1060 | # Also sanitize the server variable - it is attacker-influenced | ||
| 1061 | # in some reverse-proxy configurations. | ||
| 1062 | my $script = $ENV{SCRIPT_NAME} // '/cgi-bin/page.fcgi'; | ||
| 1063 | $script =~ s/[\r\n]//g; | ||
| 1064 | |||
| 1065 | $info->status(302); | ||
| 1066 | print "Status: 302 Found\r\n", | ||
| 1067 | "Location: ${script}?page=${redirect_page}\r\n\r\n"; | ||
| 1068 | return; | ||
| 1069 | } else { | ||
| 1070 | $logger->warn("CAPTCHA verification failed for $client_ip"); | ||
| 1071 | # Fall through to show CAPTCHA again | ||
| 1072 | } | ||
| 1073 | } | ||
| 1074 | } | ||
| 1075 | |||
| 1076 | $vwflog ||= $config->vwflog() || File::Spec->catfile($info->logdir(), 'vwf.log'); | ||
| 1077 | # Reader for the CSV access log, used by the meta_data page. | ||
| 1078 | # Created from the same path that vwflog() writes to, which needs $config. | ||
| 1079 | $self->{vwf_log} ||= Web::VWF::Data::vwf_log->new({ | ||
| 1080 | directory => dirname($vwflog), | ||
| 1081 | filename => basename($vwflog), | ||
| 1082 | no_entry => 1, | ||
| 1083 | logger => $logger, | ||
| 1084 | }); | ||
| 1085 | my $log = Class::Simple->new(); | ||
| 1086 | |||
| 1087 | # Stores things for a month or longer | ||
| 1088 | $lingua_cache ||= create_disc_cache(config => $config, logger => $logger, namespace => 'CGI::Lingua'); | ||
| 1089 | |||
| 1090 | # Language negotiation, offering the languages this site offers | ||
| 1091 | my $lingua = CGI::Lingua->new({ | ||
| 1092 | supported => \@supported_languages, | ||
| 1093 | cache => $lingua_cache, | ||
| 1094 | info => $info, | ||
| 1095 | logger => $logger, | ||
| 1096 | debug => $params{'debug'}, | ||
| 1097 | syslog => $syslog, | ||
| 1098 | }); | ||
| 1099 | |||
| 1100 | my $cachedir = Web::VWF::Utils::env_cache_dir() || $params{'cachedir'} || $config->{disc_cache}->{root_dir} || File::Spec->catfile($tmpdir, 'cache'); | ||
| 1101 | |||
| 1102 | # Rate limit by IP (unless bypassed) | ||
| 1103 | unless($has_captcha_bypass || grep { $_ eq $client_ip } @rate_limit_trusted_ips) { | ||
| 1104 | if ($request_count >= $max_requests_hard) { | ||
| 1105 | # Hard limit exceeded - show CAPTCHA with warning | ||
| 1106 | my $recaptcha_config = $config->recaptcha(); | ||
| 1107 | |||
| 1108 | if ($recaptcha_config && $recaptcha_config->{enabled}) { | ||
| 1109 | $logger->warn("Hard rate limit exceeded for $client_ip ($request_count requests)"); | ||
| 1110 | $info->status(429); | ||
| 1111 | |||
| 1112 | unless(Web::VWF::Display::captcha->can('new')) { | ||
| 1113 | require Web::VWF::Display::captcha; | ||
| 1114 | Web::VWF::Display::captcha->import(); | ||
| 1115 | } | ||
| 1116 | my $display = Web::VWF::Display::captcha->new({ | ||
| 1117 | cachedir => $cachedir, | ||
| 1118 | info => $info, | ||
| 1119 | logger => $logger, | ||
| 1120 | lingua => $lingua, | ||
| 1121 | config => $config, | ||
| 1122 | }); | ||
| 1123 | |||
| 1124 | print 'Status: 429 ', HTTP::Status::status_message(429), "\n"; | ||
| 1125 | print $display->as_string({ | ||
| 1126 | 'Retry-After' => $retry_after, | ||
| 1127 | Retry_After => $retry_after, | ||
| 1128 | hard_block => 1, | ||
| 1129 | request_count => $request_count, | ||
| 1130 | }); | ||
| 1131 | |||
| 1132 | $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Hard rate limit - CAPTCHA shown', $log, $request_start); | ||
| 1133 | return; | ||
| 1134 | } | ||
| 1135 | |||
| 1136 | # No CAPTCHA to offer, so block until the window expires | ||
| 1137 | $logger->warn("Hard rate limit exceeded for $client_ip ($request_count requests)"); | ||
| 1138 | send_error(status => 429, body => "Too many requests - try again later\n", headers => "Retry-After: $retry_after\n"); | ||
| 1139 | $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Hard rate limit', $log, $request_start); | ||
| 1140 | return; | ||
| 1141 | } elsif ($request_count >= $max_requests) { | ||
| 1142 | # Soft limit exceeded - show CAPTCHA | ||
| 1143 | my $recaptcha_config = $config->recaptcha(); | ||
| 1144 | |||
| 1145 | if ($recaptcha_config && $recaptcha_config->{enabled}) { | ||
| 1146 | $logger->info("Soft rate limit exceeded for $client_ip ($request_count requests) - CAPTCHA challenge issued"); | ||
| 1147 | $info->status(429); | ||
| 1148 | |||
| 1149 | unless(Web::VWF::Display::captcha->can('new')) { | ||
| 1150 | require Web::VWF::Display::captcha; | ||
| 1151 | Web::VWF::Display::captcha->import(); | ||
| 1152 | } | ||
| 1153 | my $display = Web::VWF::Display::captcha->new({ | ||
| 1154 | cachedir => $cachedir, | ||
| 1155 | info => $info, | ||
| 1156 | logger => $logger, | ||
| 1157 | lingua => $lingua, | ||
| 1158 | config => $config, | ||
| 1159 | }); | ||
| 1160 | |||
| 1161 | if($display) { | ||
| 1162 | print $display->as_string({ | ||
| 1163 | 'Retry-After' => $retry_after, | ||
| 1164 | Retry_After => $retry_after, | ||
| 1165 | hard_block => 0, | ||
| 1166 | request_count => $request_count, | ||
| 1167 | }); | ||
| 1168 | |||
| 1169 | $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Soft rate limit - CAPTCHA shown', $log, $request_start); | ||
| 1170 | } else { | ||
| 1171 | my $error = $@; | ||
| 1172 | if($error) { | ||
| 1173 | $logger->error($error); | ||
| 1174 | } else { | ||
| 1175 | $logger->error("Unable to show captcha page - request count = $request_count"); | ||
| 1176 | } | ||
| 1177 | $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Soft rate limit - CAPTCHA not shown', $log, $request_start); | ||
| 1178 | } | ||
| 1179 | return; | ||
| 1180 | } | ||
| 1181 | } | ||
| 1182 | } | ||
| 1183 | |||
| 1184 | # Commit the incremented request count back to the sliding-window cache. | ||
| 1185 | # The TTL is the rate-limit window; the counter expires automatically. | ||
| 1186 | $rate_limit_cache->set("$self->{script_name}:rate_limit:$client_ip", $request_count + 1, $time_window); | ||
| 1187 | |||
| 1188 | # A request with no page parameter cannot be dispatched; send a 300 | ||
| 1189 | # response listing the known valid pages instead. | ||
| 1190 | if(!defined($info->param('page'))) { | ||
| 1191 | $logger->info('No page given in ', $info->as_string()); | ||
| 1192 | choose(); | ||
| 1193 | return; | ||
| 1194 | } | ||
| 1195 | |||
| 1196 | # Multi-layer access control | ||
| 1197 | # Three independent checks are run in order of cheapness: | ||
| 1198 | # 1. CGI::ACL - cloud ranges, country blocks, explicit IP allow/deny. | ||
| 1199 | # 2. blacklisted() - in-process IP set built from prior SQL-injection attempts. | ||
| 1200 | # 3. Web::VWF::Allow - DShield feed, user-agent blacklist, throttler, IDS. | ||
| 1201 | if(my $remote_addr = $ENV{'REMOTE_ADDR'}) { | ||
| 1202 | my $reason; | ||
| 1203 | if($acl->all_denied(lingua => $lingua)) { | ||
| 1204 | $reason = 'Denied by CGI::ACL'; | ||
| 1205 | } elsif(blacklisted($info)) { | ||
| 1206 | $reason = 'Blacklisted for attempting to break in'; | ||
| 1207 | } else { | ||
| 1208 | # Web::VWF::Allow may throw an Error object when it blocks a request. | ||
| 1209 | # Catch it here so the 403 response is sent rather than propagating. | ||
| 1210 | try { | ||
| 1211 | unless(Web::VWF::Allow::allow({ | ||
| 1212 | info => $info, | ||
| 1213 | lingua => $lingua, | ||
| 1214 | logger => $logger, | ||
| 1215 | cache => $rate_limit_cache, | ||
| 1216 | config => $config, | ||
| 1217 | })) { | ||
| 1218 | $reason = 'Blocked by Web::VWF::Allow'; | ||
| 1219 | } | ||
| 1220 | } catch Error with { | ||
| 1221 | $reason = shift; | ||
| 1222 | }; | ||
| 1223 | } | ||
| 1224 | if($reason) { | ||
| 1225 | # Return a minimal plain-text 403 - no template rendering so that | ||
| 1226 | # a blocked attacker receives no information about site structure. | ||
| 1227 | send_error(status => 403, body => "Access Denied\n"); | ||
| 1228 | $logger->info("$remote_addr: access denied: $reason"); | ||
| 1229 | $info->status(403); | ||
| 1230 | $self->_vwflog($vwflog, $info, $lingua, $syslog, $reason, $log, $request_start); | ||
| 1231 | return; | ||
| 1232 | } | ||
| 1233 | } | ||
| 1234 | |||
| 1235 | # -- FCGI::Buffer setup ------------------------------------------------ | ||
| 1236 | # FCGI::Buffer post-processes the output: compresses it, generates an ETag | ||
| 1237 | # and Last-Modified header, and serves a 304 Not Modified when appropriate. | ||
| 1238 | my $args = { | ||
| 1239 | generate_etag => 1, | ||
| 1240 | generate_last_modified => 1, | ||
| 1241 | compress_content => 1, | ||
| 1242 | generate_304 => 1, | ||
| 1243 | info => $info, | ||
| 1244 | optimise_content => 1, | ||
| 1245 | logger => $logger, | ||
| 1246 | # lint_content runs HTML::Tidy on the output; enable in debug mode or | ||
| 1247 | # when explicitly requested via the lint_content query parameter. | ||
| 1248 | lint_content => $info->param('lint_content') // $params{'debug'}, | ||
| 1249 | lingua => $lingua | ||
| 1250 | }; | ||
| 1251 | |||
| 1252 | if((!$info->is_search_engine()) && $config->root_dir() && | ||
| 1253 | ($info->param('page') ne 'home') && | ||
| 1254 | ((!defined($info->param('action'))) || ($info->param('action') ne 'send'))) { | ||
| 1255 | $args->{'save_to'} = { | ||
| 1256 | directory => File::Spec->catfile($config->root_dir(), 'save_to'), | ||
| 1257 | ttl => 3600 * 24, | ||
| 1258 | create_table => 1 | ||
| 1259 | }; | ||
| 1260 | } | ||
| 1261 | |||
| 1262 | my $fb = FCGI::Buffer->new()->init($args); | ||
| 1263 | |||
| 1264 | if($fb->can_cache()) { | ||
| 1265 | my $cache_key = join('/', | ||
| 1266 | 'FCGI::Buffer', | ||
| 1267 | $info->browser_type(), | ||
| 1268 | $info->domain_name(), | ||
| 1269 | $self->{script_name}, | ||
| 1270 | $info->as_string(), | ||
| 1271 | $lingua->language() | ||
| 1272 | ); | ||
| 1273 | $cache_key =~ s/;string=%5Bobject%20HTMLInputElement%5D//; | ||
| 1274 | $cache_key =~ s/;string=\[object HTMLInputElement\]//; | ||
| 1275 | if($info->param('location')) { | ||
| 1276 | $cache_key .= '/' . $info->param('location'); | ||
| 1277 | } | ||
| 1278 | $logger->debug("enabling cache key $cache_key"); | ||
| 1279 | |||
| 1280 | $buffercache ||= create_disc_cache(config => $config, logger => $logger, namespace => $self->{script_name}, root_dir => $cachedir); | ||
| 1281 | $fb->init( | ||
| 1282 | cache_key => $cache_key, | ||
| 1283 | cache => $buffercache, | ||
| 1284 | # generate_304 => 0, | ||
| 1285 | cache_duration => '1 day', | ||
| 1286 | ); | ||
| 1287 | if($fb->is_cached()) { | ||
| 1288 | return; | ||
| 1289 | } | ||
| 1290 | } | ||
| 1291 | |||
| 1292 | # $display holds the instantiated Web::VWF::Display subclass when the page is | ||
| 1293 | # found; $invalidpage is set when the page name is invalid or unloadable. | ||
| 1294 | my $display; | ||
| 1295 | my $invalidpage; | ||
| 1296 | |||
| 1297 | # Arguments forwarded to every Web::VWF::Display subclass constructor. The | ||
| 1298 | # server's directory is among them because a page reads its configuration for | ||
| 1299 | # itself, and has to look where this script did. | ||
| 1300 | $args = { | ||
| 1301 | cachedir => $cachedir, | ||
| 1302 | info => $info, | ||
| 1303 | logger => $logger, | ||
| 1304 | lingua => $lingua, | ||
| 1305 | config => $config, | ||
| 1306 | server_directory => $SERVER_CONF_DIR, | ||
| 1307 | log => $log | ||
| 1308 | }; | ||
| 1309 | |||
| 1310 | # -- Page dispatch ------------------------------------------------------ | ||
| 1311 | # The outer block eval catches any exception thrown during module loading | ||
| 1312 | # or display-object construction, setting $@ for inspection below. | ||
| 1313 | eval { | ||
| 1314 | my $page = $info->param('page'); | ||
| 1315 | |||
| 1316 | # Strip URL fragment identifiers - the server never needs them. | ||
| 1317 | $page =~ s/#.*$//; | ||
| 1318 | |||
| 1319 | # Reject backslashes: no legitimate page name contains one, and | ||
| 1320 | # they have historically been used to probe Windows path traversal. | ||
| 1321 | $page =~ s/\\//g; | ||
| 1322 | |||
| 1323 | if($page =~ /\//) { | ||
| 1324 | # A slash in the page name indicates an attempt to traverse | ||
| 1325 | # directories (e.g. page=/etc/passwd or page=http://evil.com). | ||
| 1326 | $logger->info("Blocking '/' in $page"); | ||
| 1327 | $info->status(403); | ||
| 1328 | $log->status(403); | ||
| 1329 | $invalidpage = 1; | ||
| 1330 | } else { | ||
| 1331 | # Strip every non-word character so that the page name can only | ||
| 1332 | # contain [A-Za-z0-9_] - safe to use as a Perl package suffix. | ||
| 1333 | $page =~ s/\W//g; | ||
| 1334 | $page =~ s/\s//g; | ||
| 1335 | } | ||
| 1336 | |||
| 1337 | if($invalidpage) { | ||
| 1338 | # Already rejected above | ||
| 1339 | } elsif(!grep { $_ eq $page } @valid_pages) { | ||
| 1340 | # Only pages on the allow-list are loaded, so that no other | ||
| 1341 | # Web::VWF::Display::* module that happens to be on @INC (e.g. under | ||
| 1342 | # ~/lib/perl5) can be reached from the query string. | ||
| 1343 | $logger->info("Unknown page $page"); | ||
| 1344 | $invalidpage = 1; | ||
| 1345 | if($info->status() == 200) { | ||
| 1346 | $info->status(404); | ||
| 1347 | } | ||
| 1348 | } else { | ||
| 1349 | my $display_module = "Web::VWF::Display::$page"; | ||
| 1350 | |||
| 1351 | $logger->debug("doit(): Loading module $display_module from ", $page_directory{$page} // '@INC'); | ||
| 1352 | unless($display_module->can('new')) { | ||
| 1353 | # String-eval elimination: | ||
| 1354 | # The original code used eval "require $display_module" which is a | ||
| 1355 | # string eval on a user-derived value. Although $page has been | ||
| 1356 | # stripped of \W characters, a Unicode or locale edge-case could | ||
| 1357 | # allow a bypass. Module::Runtime::require_module() loads a module | ||
| 1358 | # by name using a block eval internally, with no string-eval surface. | ||
| 1359 | # | ||
| 1360 | # A page with a directory is loaded from its file there instead | ||
| 1361 | # of from @INC. Its name has been checked against the valid | ||
| 1362 | # pages, which are \w+ only, so the path cannot leave the | ||
| 1363 | # directory. | ||
| 1364 | if(my $dir = $page_directory{$page}) { | ||
| 1365 | my $file = File::Spec->catfile($dir, "$page.pm"); | ||
| 1366 | eval { -f $file or die "No such file $file\n"; require $file }; | ||
| 1367 | } else { | ||
| 1368 | eval { require_module($display_module) }; | ||
| 1369 | } | ||
| 1370 | $display_module->import() unless $@; | ||
| 1371 | } | ||
| 1372 | if($@) { | ||
| 1373 | $logger->debug("Failed to load module $display_module: $@"); | ||
| 1374 | $logger->info("Unknown page $page"); | ||
| 1375 | $invalidpage = 1; | ||
| 1376 | if($info->status() == 200) { | ||
| 1377 | $info->status(404); | ||
| 1378 | } | ||
| 1379 | } else { | ||
| 1380 | # use Class::Inspector; | ||
| 1381 | # my $methods = Class::Inspector->methods($display_module); | ||
| 1382 | # print "$display_module exports ", join(', ', @{$methods}), "\n"; | ||
| 1383 | $display = do { | ||
| 1384 | eval { $display_module->new($args) }; | ||
| 1385 | }; | ||
| 1386 | if(!defined($display)) { | ||
| 1387 | if($@) { | ||
| 1388 | $logger->warn("$display_module->new(): $@"); | ||
| 1389 | } else { | ||
| 1390 | $logger->notice("Can't instantiate page $page"); | ||
| 1391 | } | ||
| 1392 | $invalidpage = 1; | ||
| 1393 | if($info->status() == 200) { | ||
| 1394 | $info->status(404); | ||
| 1395 | } | ||
| 1396 | } elsif(!$display->can('as_string')) { | ||
| 1397 | $logger->warn("Problem understanding $page"); | ||
| 1398 | undef $display; | ||
| 1399 | } | ||
| 1400 | } | ||
| 1401 | } | ||
| 1402 | }; | ||
| 1403 | |||
| 1404 | my $error = $@; | ||
| 1405 | if($error) { | ||
| 1406 | if($info->status() == 429) { | ||
| 1407 | $logger->notice($error); | ||
| 1408 | } else { | ||
| 1409 | $logger->error($error); | ||
| 1410 | } | ||
| 1411 | $display = undef; | ||
| 1412 | } | ||
| 1413 | |||
| 1414 | if(defined($display)) { | ||
| 1415 | # Pass in handles to the databases: the site's own data handles, under | ||
| 1416 | # the name each was declared with, and the access log, which the | ||
| 1417 | # framework reads from the log it writes. | ||
| 1418 | print $display->as_string({ | ||
| 1419 | cachedir => $cachedir, | ||
| 1420 | config => $config, | ||
| 1421 | databasedir => $database_dir, | ||
| 1422 | database_dir => $database_dir, | ||
| 1423 | %data, | ||
| 1424 | vwf_log => $self->{vwf_log}, | ||
| 1425 | }); | ||
| 1426 | $self->_vwflog($vwflog, $info, $lingua, $syslog, '', $log, $request_start); | ||
| 1427 | } elsif($invalidpage) { | ||
| 1428 | if($info->status() == 429) { | ||
| 1429 | # Throttled by Web::VWF::Display | ||
| 1430 | my $interval = $config->{'throttle'}->{'interval'} // 90; | ||
| 1431 | send_error(status => 429, body => "Too many requests - try again later\n", headers => "Retry-After: $interval\n", buffered => 1); | ||
| 1432 | $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Throttled', $log, $request_start); | ||
| 1433 | } else { | ||
| 1434 | choose(); | ||
| 1435 | $self->_vwflog($vwflog, $info, $lingua, $syslog, 'Unknown page', $log, $request_start); | ||
| 1436 | } | ||
| 1437 | return; | ||
| 1438 | } else { | ||
| 1439 | $logger->debug('disabling cache'); | ||
| 1440 | $fb->init( | ||
| 1441 | cache => undef, | ||
| 1442 | ); | ||
| 1443 | # Handle errors gracefully | ||
| 1444 | my ($status, $body); | ||
| 1445 | if($error eq 'Unknown page to display') { | ||
| 1446 | ($status, $body) = (400, "I don't know what you want me to display.\n"); | ||
| 1447 | } elsif($error =~ /Can\'t locate .* in \@INC/) { | ||
| 1448 | $logger->error($error); | ||
| 1449 | ($status, $body) = (500, "Software error - contact the webmaster\n"); | ||
| 1450 | } elsif(($info->status() == 200) || ($info->status() == 403)) { | ||
| 1451 | # No permission to show this page | ||
| 1452 | ($status, $body) = (403, "Access Denied\n"); | ||
| 1453 | } else { | ||
| 1454 | $status = $info->status(); | ||
| 1455 | $body = "Page unavailable - something is wrong at your end, please fix and try again\n"; | ||
| 1456 | } | ||
| 1457 | send_error(status => $status, body => $body, buffered => 1); | ||
| 1458 | $log->status($status); | ||
| 1459 | $self->_vwflog($vwflog, $info, $lingua, $syslog, $error ? $error : 'Access denied', $log, $request_start); | ||
| 1460 | throw Error::Simple($error ? $error : $info->as_string()); | ||
| 1461 | } | ||
| 1462 | } | ||
| 1463 | |||
| 1464 | # Send a 300 Multiple Choices response listing the pages this site serves. | ||
| 1465 | # Called when the ?page= parameter is absent or the requested page is unknown. | ||
| 1466 | sub choose | ||
| 1467 | { | ||
| 1468 | $logger->info('Called with no page to display'); | ||
| 1469 | |||
| 1470 | my $status = $info->status(); | ||
| 1471 | |||
| 1472 | # If the status is already non-200 (e.g. 404 set by the caller), relay | ||
| 1473 | # that status rather than overriding it with a 300. That only happens | ||
| 1474 | # for an invalid page, after FCGI::Buffer has been set up, and it sends | ||
| 1475 | # the Status line from $info->status() itself. | ||
| 1476 | return if($status != 200); | ||
| 1477 | |||
| 1478 | print "Status: 300 Multiple Choices\n", | ||
| 1479 | "Content-type: text/plain\n"; | ||
| 1480 | |||
| 1481 | $info->status(300); | ||
| 1482 | |||
| 1483 | # Include a Last-Modified header based on the script's mtime so that | ||
| 1484 | # caches and conditional-GET clients can revalidate efficiently. | ||
| 1485 | if(my $path = $info->script_path()) { | ||
| 1486 | require HTTP::Date; | ||
| 1487 | HTTP::Date->import(); | ||
| 1488 | |||
| 1489 | my @statb = stat($path); | ||
| 1490 | my $mtime = $statb[9]; | ||
| 1491 | print 'Last-Modified: ', HTTP::Date::time2str($mtime), "\n"; | ||
| 1492 | } | ||
| 1493 | |||
| 1494 | print "\n"; | ||
| 1495 | |||
| 1496 | # RFC 7231 section 4.3.2: a HEAD response must not include a body. | ||
| 1497 | unless($ENV{'REQUEST_METHOD'} && ($ENV{'REQUEST_METHOD'} eq 'HEAD')) { | ||
| 1498 | # The same list that doit() allows, so the two cannot differ | ||
| 1499 | print map { | ||
| 1500 | "/cgi-bin/page.fcgi?page=$_" . (length($page_description{$_} // '') ? " - $page_description{$_}" : '') . "\n" | ||
| 1501 | } @valid_pages; | ||
| 1502 | } | ||
| 1503 | } | ||
| 1504 | |||
| 1505 | # Send a short plain-text error response, and record its status in $info. | ||
| 1506 | # status - the HTTP status code | ||
| 1507 | # body - the text to send (not sent for HEAD) | ||
| 1508 | # headers - optional extra header lines, each ending in "\n", e.g. Retry-After | ||
| 1509 | # buffered - true once FCGI::Buffer is capturing the output. It sends the | ||
| 1510 | # Status line from $info->status(), so it mustn't be printed here too. | ||
| 1511 | sub send_error | ||
| 1512 | { | ||
| 1513 | my %args = (ref($_[0]) eq 'HASH') ? %{$_[0]} : @_; | ||
| 1514 | my $status = $args{'status'}; | ||
| 1515 | |||
| 1516 | $info->status($status); | ||
| 1517 | |||
| 1518 | print 'Status: ', $status, ' ', HTTP::Status::status_message($status), "\n" unless($args{'buffered'}); | ||
| 1519 | print $args{'headers'} // '', | ||
| 1520 | "Content-type: text/plain\n", | ||
| 1521 | "Pragma: no-cache\n\n"; | ||
| 1522 | |||
| 1523 | # RFC 9110 section 9.3.2: a HEAD response must not include a body. | ||
| 1524 | my $is_head = $ENV{'REQUEST_METHOD'} && ($ENV{'REQUEST_METHOD'} eq 'HEAD'); | ||
| 1525 | print $args{'body'} if(defined($args{'body'}) && !$is_head); | ||
| 1526 | |||
| 1527 | return 1; | ||
| 1528 | } | ||
| 1529 | |||
| 1530 | # A list-valued setting: what the configuration file says about it, or the | ||
| 1531 | # caller's defaults when it says nothing about it. A list can be written as one | ||
| 1532 | # string of values separated by commas or spaces, which is what the plain form | ||
| 1533 | # of a configuration file gives, or as an element with one child per value, | ||
| 1534 | # which is what an XML file gives; both are read, so an XML file can hold a list | ||
| 1535 | # in whichever of the two ways reads better. The values are returned as they | ||
| 1536 | # were written and in the order they were written, with repetitions dropped, and | ||
| 1537 | # a setting that is present but empty means no values at all - so that a site can | ||
| 1538 | # say "none" as well as saying nothing. | ||
| 1539 | sub _config_list | ||
| 1540 | { | ||
| 1541 | # The invocant is not used: the setting and its defaults are all this needs. | ||
| 1542 | my (undef, $config, $key, @defaults) = @_; | ||
| 1543 | |||
| 1544 | my $value = $config->{$key}; | ||
| 1545 | return @defaults unless(defined($value)); | ||
| 1546 | |||
| 1547 | my @values; | ||
| 1548 | if(ref($value) eq 'HASH') { | ||
| 1549 | # One child per value; the names of the children are not used | ||
| 1550 | foreach my $child (values %{$value}) { | ||
| 1551 | push(@values, ref($child) eq 'ARRAY' ? @{$child} : $child); | ||
| 1552 | } | ||
| 1553 | } elsif(ref($value) eq 'ARRAY') { | ||
| 1554 | @values = @{$value}; | ||
| 1555 | } else { | ||
| 1556 | @values = split /[\s,]+/, $value; | ||
| 1557 | } | ||
| 1558 | |||
| 1559 | my %seen; | ||
| 1560 | return grep { defined($_) && length($_) && !$seen{$_}++ } @values; | ||
| 1561 | } | ||
| 1562 | |||
| 1563 | # The <display> setting, named after the Web::VWF::Display directory the pages | ||
| 1564 | # live in. A site written before the rename used <pages>, which is still read | ||
| 1565 | # when there is no <display>. | ||
| 1566 | sub _display_setting | ||
| 1567 | { | ||
| 1568 | my $config = shift; | ||
| 1569 | |||
| 1570 | return $config->{'display'} // $config->{'pages'}; | ||
| 1571 | } | ||
| 1572 | |||
| 1573 | # The pages the site's configuration file declares, as a list of | ||
| 1574 | # { name => ..., description => ... } in the order they were written: | ||
| 1575 | # <display> | ||
| 1576 | # <page><name>states</name><description>About the U.S. states</description></page> | ||
| 1577 | # <directory>/path/to/pages</directory> | ||
| 1578 | # </display> | ||
| 1579 | # A <directory> in <display> is where the page modules are looked for, and the | ||
| 1580 | # list of pages when there are no <page>s (see _valid_pages); one in a <page> | ||
| 1581 | # is where that page's module is, and beats the one in <display>. One <page> is read as a hash and several as an array, so both are handled. | ||
| 1582 | # The old form, a single string of names (<pages>index, meta_data</pages>), is | ||
| 1583 | # still read, with no descriptions, so existing sites keep working. A page | ||
| 1584 | # without a usable name is dropped, and so is a repeated one, since the same | ||
| 1585 | # file can be loaded more than once when the configuration is merged. | ||
| 1586 | sub _config_pages | ||
| 1587 | { | ||
| 1588 | my (undef, $config) = @_; | ||
| 1589 | |||
| 1590 | return _config_entries(_display_setting($config), 'page'); | ||
| 1591 | } | ||
| 1592 | |||
| 1593 | # The common reader for a setting that is a list of named things, each with an | ||
| 1594 | # optional description and directory: the elements called $item inside $setting | ||
| 1595 | # (a hash), or the array or string of names of the old form. Returns | ||
| 1596 | # { name, description, directory } hashes, in order, without repetitions. | ||
| 1597 | sub _config_entries | ||
| 1598 | { | ||
| 1599 | my ($setting, $item) = @_; | ||
| 1600 | |||
| 1601 | return unless(defined($setting)); | ||
| 1602 | |||
| 1603 | my @entries; | ||
| 1604 | if(ref($setting) eq 'HASH') { | ||
| 1605 | my $found = $setting->{$item}; | ||
| 1606 | return unless(defined($found)); | ||
| 1607 | @entries = (ref($found) eq 'ARRAY') ? @{$found} : ($found); | ||
| 1608 | } elsif(ref($setting) eq 'ARRAY') { | ||
| 1609 | @entries = @{$setting}; | ||
| 1610 | } else { | ||
| 1611 | @entries = split /[\s,]+/, $setting; | ||
| 1612 | } | ||
| 1613 | |||
| 1614 | my (%seen, @found); | ||
| 1615 | foreach my $entry (@entries) { | ||
| 1616 | next unless(defined($entry)); | ||
| 1617 | my ($name, $description, $directory) = (ref($entry) eq 'HASH') ? @{$entry}{qw(name description directory)} : ($entry, undef, undef); | ||
| 1618 | next unless(defined($name) && !ref($name) && length($name)); | ||
| 1619 | next if($seen{$name}++); | ||
| 1620 | push @found, { | ||
| 1621 | name => $name, | ||
| 1622 | description => (defined($description) && !ref($description)) ? $description : '', | ||
| 1623 | directory => (defined($directory) && !ref($directory)) ? $directory : '', | ||
| 1624 | }; | ||
| 1625 | } | ||
| 1626 | return @found; | ||
| 1627 | } | ||
| 1628 | |||
| 1629 | # The pages that ?page= may load. Sites declare them in the config file (see | ||
| 1630 | # _config_pages); otherwise every *.pm in the page directory is allowed, except | ||
| 1631 | # captcha, which only the rate limiter shows. The page directory is | ||
| 1632 | # <display><directory>, or ../lib/Web/VWF/Display next to this script. | ||
| 1633 | sub _valid_pages | ||
| 1634 | { | ||
| 1635 | my $self = shift; | ||
| 1636 | my $config = shift; | ||
| 1637 | |||
| 1638 | my @pages = map { $_->{'name'} } $self->_config_pages($config); | ||
| 1639 | return grep { /\A\w+\z/ } @pages if(@pages); | ||
| 1640 | |||
| 1641 | # <display><directory> overrides where the page modules are looked for. | ||
| 1642 | # Only the default has the framework's own hard-coded layout. | ||
| 1643 | my $dir = $self->_pages_directory($config); | ||
| 1644 | unless(defined($dir)) { | ||
| 1645 | # FIXME: Remove the hard coding | ||
| 1646 | $dir = File::Spec->catdir($script_dir, File::Spec->updir(), 'lib', 'Web', 'VWF', 'Display'); | ||
| 1647 | } | ||
| 1648 | # The list is built first rather than returned from inside sort(): a sort | ||
| 1649 | # that is returned directly takes the caller's context, and sort in scalar | ||
| 1650 | # context has no defined meaning. | ||
| 1651 | my @found = sort(grep { $_ ne 'captcha' } map { basename($_, '.pm') } glob(File::Spec->catfile($dir, '*.pm'))); | ||
| 1652 | return @found; | ||
| 1653 | } | ||
| 1654 | |||
| 1655 | # A directory from the configuration, made absolute: a relative one is taken | ||
| 1656 | # from the script's directory, as the default is. Undef if there isn't one. | ||
| 1657 | sub _resolve_directory | ||
| 1658 | { | ||
| 1659 | my $directory = shift; | ||
| 1660 | |||
| 1661 | # An explicit undef, not a bare return: this is used inside map, where an | ||
| 1662 | # empty list would unbalance the hash being built | ||
| 1663 | return undef unless(defined($directory) && !ref($directory) && length($directory)); ## no critic (ProhibitExplicitReturnUndef) | ||
| 1664 | return File::Spec->file_name_is_absolute($directory) ? $directory : File::Spec->catdir($script_dir, $directory); | ||
| 1665 | } | ||
| 1666 | |||
| 1667 | # <display><directory>, resolved, or undef | ||
| 1668 | sub _pages_directory | ||
| 1669 | { | ||
| 1670 | my (undef, $config) = @_; | ||
| 1671 | |||
| 1672 | my $display = _display_setting($config); | ||
| 1673 | return unless(ref($display) eq 'HASH'); | ||
| 1674 | return _resolve_directory($display->{'directory'}); | ||
| 1675 | } | ||
| 1676 | |||
| 1677 | # name => directory for each of the given pages whose module is not simply in | ||
| 1678 | # @INC: the page's own <directory>, else <display><directory> if there is one. | ||
| 1679 | sub _page_directories | ||
| 1680 | { | ||
| 1681 | my ($self, $config, @pages) = @_; | ||
| 1682 | |||
| 1683 | my $default = $self->_pages_directory($config); | ||
| 1684 | my %own = map { $_->{'name'} => _resolve_directory($_->{'directory'}) } $self->_config_pages($config); | ||
| 1685 | |||
| 1686 | my %directory; | ||
| 1687 | foreach my $page (@pages) { | ||
| 1688 | my $dir = $own{$page} // $default; | ||
| 1689 | $directory{$page} = $dir if(defined($dir)); | ||
| 1690 | } | ||
| 1691 | return %directory; | ||
| 1692 | } | ||
| 1693 | |||
| 1694 | # name => description for each page the configuration describes | ||
| 1695 | sub _page_descriptions | ||
| 1696 | { | ||
| 1697 | my ($self, $config) = @_; | ||
| 1698 | |||
| 1699 | return map { $_->{'name'} => $_->{'description'} } grep { length($_->{'description'}) } $self->_config_pages($config); | ||
| 1700 | } | ||
| 1701 | |||
| 1702 | # The data modules the site declares, as a list of | ||
| 1703 | # { name, module, description, directory } in the order they were written: | ||
| 1704 | # <databases> | ||
| 1705 | # <directory>/path/to/modules</directory> | ||
| 1706 | # <database> | ||
| 1707 | # <name>states</name> | ||
| 1708 | # <description>List of data about the U.S. states</description> | ||
| 1709 | # <directory>/path/to/states/module</directory> | ||
| 1710 | # </database> | ||
| 1711 | # </databases> | ||
| 1712 | # The container is <databases>, not <data>: the data modules take their | ||
| 1713 | # constructor arguments from the configuration's top-level keys, and a key called | ||
| 1714 | # data would replace the rows of every database with the setting itself. | ||
| 1715 | # name is what the page is given (lower-cased last part), module the class, and | ||
| 1716 | # directory where its .pm file is, or undef to find the class in @INC. A | ||
| 1717 | # <directory> in a <database> beats the one in <databases>. The directories are those | ||
| 1718 | # of the modules, not of the databases' files: those are <database_dir>. | ||
| 1719 | # What is declared, in order of preference: | ||
| 1720 | # 1. the <database> elements; | ||
| 1721 | # 2. the old <data_modules>index, states</data_modules> string, which is also | ||
| 1722 | # how a site says "none" by leaving it empty; | ||
| 1723 | # 3. with a <databases><directory> and no <database>, every *.pm in that directory; | ||
| 1724 | # 4. the default, Web::VWF::Data::index. | ||
| 1725 | # A name with no :: is one of this distribution's own, so index is | ||
| 1726 | # Web::VWF::Data::index; a name that has :: is used as written, which is how a | ||
| 1727 | # site names its own modules. The file looked for in a directory is named for | ||
| 1728 | # the last part of the class, whichever way the class was written. | ||
| 1729 | sub _data_modules | ||
| 1730 | { | ||
| 1731 | my ($self, $config) = @_; | ||
| 1732 | |||
| 1733 | my $databases = $config->{'databases'}; | ||
| 1734 | my $default_dir = (ref($databases) eq 'HASH') ? _resolve_directory($databases->{'directory'}) : undef; | ||
| 1735 | |||
| 1736 | my @entries = _config_entries($databases, 'database'); | ||
| 1737 | unless(@entries) { | ||
| 1738 | if(defined($config->{'data_modules'})) { | ||
| 1739 | @entries = map { +{ name => $_, description => '', directory => '' } } $self->_config_list($config, 'data_modules'); | ||
| 1740 | } elsif(defined($default_dir)) { | ||
| 1741 | @entries = map { +{ name => $_, description => '', directory => '' } } sort(map { basename($_, '.pm') } glob(File::Spec->catfile($default_dir, '*.pm'))); | ||
| 1742 | } else { | ||
| 1743 | @entries = map { +{ name => $_, description => '', directory => '' } } @DEFAULT_DATA_MODULES; | ||
| 1744 | } | ||
| 1745 | } | ||
| 1746 | |||
| 1747 | my @modules; | ||
| 1748 | foreach my $entry (@entries) { | ||
| 1749 | my $module = ($entry->{'name'} =~ /::/) ? $entry->{'name'} : "Web::VWF::Data::$entry->{'name'}"; | ||
| 1750 | my $last = (split(/::/, $module))[-1]; | ||
| 1751 | my $dir = _resolve_directory($entry->{'directory'}) // $default_dir; | ||
| 1752 | push @modules, { | ||
| 1753 | name => lc($last), | ||
| 1754 | module => $module, | ||
| 1755 | description => $entry->{'description'}, | ||
| 1756 | file => defined($dir) ? File::Spec->catfile($dir, "$last.pm") : undef, | ||
| 1757 | }; | ||
| 1758 | } | ||
| 1759 | return @modules; | ||
| 1760 | } | ||
| 1761 | |||
| 1762 | # Read the settings this site declares, once per process, and act on them: open | ||
| 1763 | # the database directory it names, load the data modules it names and give the | ||
| 1764 | # pages the languages and addresses it names. Everything left unsaid keeps the | ||
| 1765 | # default it had when these values were written into this script. | ||
| 1766 | sub _site_setup | ||
| 1767 | { | ||
| 1768 | my $self = shift; | ||
| 1769 | my $config = shift; | ||
| 1770 | |||
| 1771 | return if($self->{site_ready}); | ||
| 1772 | # The configuration, the database directory, the | ||
| 1773 | # data handles and these lists last for the life of the process, not for one | ||
| 1774 | # request. | ||
| 1775 | |||
| 1776 | $self->{site_ready} = 1; | ||
| 1777 | |||
| 1778 | # Where the site's own databases are. Database::Abstraction reads them, and | ||
| 1779 | # the pages are told the directory as databasedir and database_dir. | ||
| 1780 | $database_dir = $config->{'database_dir'} | ||
| 1781 | || File::Spec->catdir($script_dir, File::Spec->updir(), $DEFAULT_DATABASE_DIR); | ||
| 1782 | $logger->debug('Reading databases from ', $database_dir); | ||
| 1783 | |||
| 1784 | Database::Abstraction::init({ | ||
| 1785 | cache => CHI->new(driver => 'Memory', datastore => {}), | ||
| 1786 | cache_duration => '1 day', | ||
| 1787 | directory => $database_dir, | ||
| 1788 | logger => $logger | ||
| 1789 | }); | ||
| 1790 | |||
| 1791 | # The languages the site offers. CGI::Lingua negotiates within this list, so | ||
| 1792 | # listing a language is what makes that language's templates reachable. | ||
| 1793 | @supported_languages = $self->_config_list($config, 'supported_languages', @DEFAULT_SUPPORTED_LANGUAGES); | ||
| 1794 | |||
| 1795 | # Addresses exempt from rate limiting | ||
| 1796 | @rate_limit_trusted_ips = $self->_config_list($config, 'rate_limit_trusted_ips', @DEFAULT_RATE_LIMIT_TRUSTED_IPS); | ||
| 1797 | |||
| 1798 | # The site's data modules (see _data_modules). Each is loaded, constructed | ||
| 1799 | # with ->new() and given to every page under the last part of its name, | ||
| 1800 | # lower-cased: Web::VWF::Data::index arrives at a page as index, and a | ||
| 1801 | # site's own Bands::Data::bands as bands. | ||
| 1802 | foreach my $database ($self->_data_modules($config)) { | ||
| 1803 | my $module = $database->{'module'}; | ||
| 1804 | my $handle; | ||
| 1805 | eval { | ||
| 1806 | if(defined($database->{'file'})) { | ||
| 1807 | # From the directory the site gave, not from @INC | ||
| 1808 | -f $database->{'file'} or die "No such file $database->{'file'}\n"; | ||
| 1809 | require $database->{'file'}; | ||
| 1810 | } else { | ||
| 1811 | require_module($module); | ||
| 1812 | } | ||
| 1813 | $module->import() if($module->can('import')); | ||
| 1814 | $handle = $module->new(); | ||
| 1815 | }; | ||
| 1816 | if(!defined($handle)) { | ||
| 1817 | # A module the site named that cannot be loaded is reported and | ||
| 1818 | # skipped rather than taking the site down: it is a setting now, so | ||
| 1819 | # getting it wrong should not stop the pages that do not need it. | ||
| 1820 | $logger->error("Can't load data module $module: $@"); | ||
| 1821 | next; | ||
| 1822 | } | ||
| 1823 | |||
| 1824 | my $name = $database->{'name'}; | ||
| 1825 | $handle->set_logger($logger) if($handle->can('set_logger')); | ||
| 1826 | $data{$name} = $handle; | ||
| 1827 | $logger->debug("Data module $module is available to pages as \$$name", length($database->{'description'}) ? ": $database->{'description'}" : ''); | ||
| 1828 | } | ||
| 1829 | |||
| 1830 | return 1; | ||
| 1831 | } | ||
| 1832 | |||
| 1833 | # Give the current logger to every data handle. This runs before each request, | ||
| 1834 | # since command-line mode replaces the logger; the handles themselves are made | ||
| 1835 | # once, by _site_setup(). | ||
| 1836 | sub _set_data_loggers | ||
| 1837 | { | ||
| 1838 | my $self = shift; | ||
| 1839 | my $logger = shift; | ||
| 1840 | |||
| 1841 | foreach my $handle (values %data) { | ||
| 1842 | $handle->set_logger($logger) if($handle->can('set_logger')); | ||
| 1843 | } | ||
| 1844 | $self->{vwf_log}->set_logger($logger) if($self->{vwf_log}); | ||
| 1845 | } | ||
| 1846 | |||
| 1847 | # Convert a CHI-style duration ('60s', '2 minutes', '90') to whole seconds | ||
| 1848 | sub duration_seconds | ||
| 1849 | { | ||
| 1850 | my $duration = shift; | ||
| 1851 | |||
| 1852 | return $duration if($duration =~ /\A\d+\z/); | ||
| 1853 | |||
| 1854 | require Time::Duration::Parse; | ||
| 1855 | |||
| 1856 | my $seconds = eval { Time::Duration::Parse::parse_duration($duration) }; | ||
| 1857 | if(!defined($seconds)) { | ||
| 1858 | $logger->warn("Can't parse duration '$duration', using 60 seconds"); | ||
| 1859 | return 60; | ||
| 1860 | } | ||
| 1861 | return int($seconds); | ||
| 1862 | } | ||
| 1863 | |||
| 1864 | # Check whether the current request looks like a SQL injection attempt. | ||
| 1865 | # IPs that trigger a match are added to the in-process %blacklisted_ip set so | ||
| 1866 | # that subsequent requests from the same IP are rejected without re-scanning. | ||
| 1867 | sub blacklisted | ||
| 1868 | { | ||
| 1869 | if(my $remote = $ENV{'REMOTE_ADDR'}) { | ||
| 1870 | my $info = shift; | ||
| 1871 | |||
| 1872 | # Fast path: this IP was already blacklisted earlier in this process | ||
| 1873 | # lifetime; no need to re-scan the request string. | ||
| 1874 | if($blacklisted_ip{$remote}) { | ||
| 1875 | $info->status(403); | ||
| 1876 | return 1; | ||
| 1877 | } | ||
| 1878 | |||
| 1879 | if(my $string = $info->as_string()) { | ||
| 1880 | # Hard length cap: skip pattern matching entirely on absurdly long | ||
| 1881 | # inputs. Even bounded quantifiers like {1,200} iterate many times | ||
| 1882 | # across a 1 MB string, so cap before we run any regex. | ||
| 1883 | return 0 if length($string) > 8192; | ||
| 1884 | |||
| 1885 | # SECURITY - ReDoS defence: | ||
| 1886 | # The original patterns used greedy .+ between SQL keywords, which | ||
| 1887 | # causes catastrophic (exponential) backtracking when an attacker | ||
| 1888 | # sends a string that contains the opening keyword but not the | ||
| 1889 | # closing one (e.g. thousands of chars after SELECT with no AND). | ||
| 1890 | # All .+ quantifiers are replaced with the bounded class [^;]{0,N}: | ||
| 1891 | # - the semicolon is a natural SQL statement terminator so it is | ||
| 1892 | # a safe anchor that real SQL injection never crosses, and | ||
| 1893 | # - the explicit upper bound caps backtracking to O(N) steps. | ||
| 1894 | # Word-boundary assertions (\b) also eliminate false positives on | ||
| 1895 | # ordinary words that happen to contain the substring. | ||
| 1896 | if( ($string =~ /SELECT\b[^;]{0,500}\bAND\b/i) | ||
| 1897 | || ($string =~ /ORDER\s+BY\s/i) | ||
| 1898 | || ($string =~ /\bOR\s+NOT\b/i) | ||
| 1899 | || ($string =~ /\bAND\s+\d+=\d+/i) | ||
| 1900 | || ($string =~ /\bTHEN\b[^;]{0,200}\bELSE\b[^;]{0,200}\bEND\b/i) | ||
| 1901 | || ($string =~ /\bAND\b[^;]{0,200}\bSELECT\b/i) | ||
| 1902 | || ($string =~ /\sAND\s[^;]{0,100}\sAND\s/i) | ||
| 1903 | || ($string =~ /\bAND\s+CASE\s+WHEN\b/i)) { | ||
| 1904 | $blacklisted_ip{$remote} = 1; | ||
| 1905 | $info->status(403); | ||
| 1906 | return 1; | ||
| 1907 | } | ||
| 1908 | } | ||
| 1909 | } | ||
| 1910 | return 0; | ||
| 1911 | } | ||
| 1912 | |||
| 1913 | # False positives we don't need in the logs | ||
| 1914 | sub filter | ||
| 1915 | { | ||
| 1916 | # return 0 if($_[0] =~ /Can't locate Net\/OAuth\/V1_0A\/ProtectedResourceRequest.pm in /); | ||
| 1917 | # return 0 if($_[0] =~ /Can't locate auto\/NetAddr\/IP\/InetBase\/AF_INET6.al in /); | ||
| 1918 | # return 0 if($_[0] =~ /S_IFFIFO is not a valid Fcntl macro at /); | ||
| 1919 | |||
| 1920 | return 0 if $_[0] =~ /Can't locate (Net\/OAuth\/V1_0A\/ProtectedResourceRequest\.pm|auto\/NetAddr\/IP\/InetBase\/AF_INET6\.al) in |S_IFFIFO is not a valid Fcntl macro at /; | ||
| 1921 | return 1; | ||
| 1922 | } | ||
| 1923 | |||
| 1924 | # Escape a single value for safe inclusion in a double-quoted CSV field. | ||
| 1925 | # Follows RFC 4180 section 2.7 and additionally neutralises spreadsheet formulas. | ||
| 1926 | sub _csv_escape | ||
| 1927 | { | ||
| 1928 | my $v = shift // ''; | ||
| 1929 | |||
| 1930 | # RFC 4180: a double-quote inside a quoted field is represented by two | ||
| 1931 | # double-quote characters. Without this, one embedded " would break the | ||
| 1932 | # column boundary and corrupt every subsequent field on the row. | ||
| 1933 | $v =~ s/"/""/g; | ||
| 1934 | |||
| 1935 | # SECURITY - CSV formula injection defence: | ||
| 1936 | # Spreadsheet applications (Excel, LibreOffice Calc) interpret cell values | ||
| 1937 | # that begin with = + - @ TAB or CR as formulas. An attacker who controls | ||
| 1938 | # a logged field (e.g. the page parameter) could inject =cmd|'/C calc'!A0. | ||
| 1939 | # Prefix such values with a single-quote to force literal interpretation. | ||
| 1940 | $v =~ s/^([=+\-@\t\r])/'$1/; | ||
| 1941 | |||
| 1942 | return $v; | ||
| 1943 | } | ||
| 1944 | |||
| 1945 | # Write one access record to vwf.log (CSV format) and optionally to syslog. | ||
| 1946 | # All user-influenced fields are escaped through _csv_escape before output. | ||
| 1947 | sub _vwflog | ||
| 1948 | { | ||
| 1949 | my $self = shift; | ||
| 1950 | |||
| 1951 | my ($vwflog, $info, $lingua, $syslog, $message, $log, $request_start) = @_; | ||
| 1952 | |||
| 1953 | # Calculate request duration in milliseconds if a start timer was supplied. | ||
| 1954 | # $request_start is a Timer::Simple, so ask it for the seconds that have | ||
| 1955 | # passed: its numeric overload yields elapsed(), so subtracting it from the | ||
| 1956 | # clock gave an epoch's worth of seconds (about 1.8e12 ms per request). A | ||
| 1957 | # plain epoch is still accepted for callers that pass one. | ||
| 1958 | my $duration_ms = ''; | ||
| 1959 | if($request_start) { | ||
| 1960 | my $seconds = ref($request_start) ? $request_start->elapsed() : Time::HiRes::time() - $request_start; | ||
| 1961 | $duration_ms = int($seconds * 1000); | ||
| 1962 | } | ||
| 1963 | |||
| 1964 | # Determine which template was rendered for this request (may be empty on error). | ||
| 1965 | my $template; | ||
| 1966 | if($log) { | ||
| 1967 | $template = $log->template(); | ||
| 1968 | } | ||
| 1969 | if(!defined($template)) { | ||
| 1970 | $template = ''; | ||
| 1971 | } | ||
| 1972 | $message ||= ''; | ||
| 1973 | |||
| 1974 | # Create the log file with a header row on first use. | ||
| 1975 | if(!-e $vwflog) { | ||
| 1976 | open(my $fout, '>', $vwflog); | ||
| 1977 | print $fout '"domain_name","time","IP","country","type","language","http_code","template","args","messages","error","duration_ms"', | ||
| 1978 | "\n"; | ||
| 1979 | close $fout; | ||
| 1980 | } | ||
| 1981 | |||
| 1982 | # Collect any warn/notice-level messages emitted during this request. | ||
| 1983 | my $warnings; | ||
| 1984 | if(my $messages = $info->messages()) { | ||
| 1985 | $warnings = join('; ', | ||
| 1986 | grep defined, map { (($_->{'level'} eq 'warn') || ($_->{'level'} eq 'notice')) ? $_->{'message'} : undef } @{$messages} | ||
| 1987 | ) | ||
| 1988 | } | ||
| 1989 | $warnings ||= ''; | ||
| 1990 | |||
| 1991 | my $country = $lingua->country() || 'unknown'; | ||
| 1992 | |||
| 1993 | # Open the log in append mode. If the open fails we skip logging silently | ||
| 1994 | # so that a disk-full or permissions error does not crash the live request. | ||
| 1995 | if(open(my $fout, '>>', $vwflog)) { | ||
| 1996 | # SECURITY - CSV injection defence: | ||
| 1997 | # Every user-visible field is passed through _csv_escape so that | ||
| 1998 | # embedded double-quotes cannot break the CSV column structure, and | ||
| 1999 | # leading formula characters cannot trigger code execution when the | ||
| 2000 | # file is opened in a spreadsheet application. | ||
| 2001 | print $fout | ||
| 2002 | '"', _csv_escape($info->domain_name()), '",', | ||
| 2003 | '"', strftime('%F %T', localtime), '",', | ||
| 2004 | '"', _csv_escape($ENV{REMOTE_ADDR} // ''), '",', | ||
| 2005 | '"', _csv_escape($country), '",', | ||
| 2006 | '"', _csv_escape($info->browser_type()), '",', | ||
| 2007 | '"', _csv_escape($lingua->language() // ''), '",', | ||
| 2008 | $info->status(), ',', | ||
| 2009 | '"', _csv_escape($template), '",', | ||
| 2010 | '"', _csv_escape($info->as_string(raw => 1)), '",', | ||
| 2011 | '"', _csv_escape($warnings), '",', | ||
| 2012 | '"', _csv_escape($message), '",', | ||
| 2013 | $duration_ms, | ||
| 2014 | "\n"; | ||
| 2015 | close($fout); | ||
| 2016 | } | ||
| 2017 | |||
| 2018 | # Optionally mirror the record to syslog (configured via the syslog stanza). | ||
| 2019 | if($syslog) { | ||
| 2020 | unless(Sys::Syslog->can('openlog')) { | ||
| 2021 | require Sys::Syslog; | ||
| 2022 | Sys::Syslog->import(); | ||
| 2023 | } | ||
| 2024 | |||
| 2025 | # Configure the socket transport if a hash of options was provided. | ||
| 2026 | if(ref($syslog) eq 'HASH') { | ||
| 2027 | Sys::Syslog::setlogsock($syslog); | ||
| 2028 | } | ||
| 2029 | Sys::Syslog::openlog($self->{script_name}, 'cons,pid', 'user'); | ||
| 2030 | # Use positional %s/%d format args so that special characters in the | ||
| 2031 | # values cannot be interpreted as syslog format directives. | ||
| 2032 | Sys::Syslog::syslog('info|local0', '%s %s %s %s %s %d %s %s %d %s %s', | ||
| 2033 | $info->domain_name() || '', | ||
| 2034 | $ENV{REMOTE_ADDR} || '', | ||
| 2035 | $country, | ||
| 2036 | $info->browser_type() || '', | ||
| 2037 | $lingua->language() || '', | ||
| 2038 | $info->status() || '', | ||
| 2039 | $template || '', | ||
| 2040 | $info->as_string(raw => 1) || '', | ||
| 2041 | $duration_ms, | ||
| 2042 | $warnings, | ||
| 2043 | $message | ||
| 2044 | ); | ||
| 2045 | Sys::Syslog::closelog(); | ||
| 2046 | } | ||
| 2047 | } | ||
| 2048 | |||
| 2049 | sub DESTROY | ||
| 2050 | { | ||
| 2051 | if(defined($^V) && ($^V ge 'v5.14.0')) { | ||
| 2052 | return if ${^GLOBAL_PHASE} eq 'DESTRUCT'; # >= 5.14.0 only | ||
| 2053 | } | ||
| 2054 | my $self = shift; | ||
| 2055 | |||
| 2056 | # Clean up resources before shutdown | ||
| 2057 | $logger->info('Shutting down') if(defined($logger)); | ||
| 2058 | if($buffercache && !$exit_requested) { | ||
| 2059 | $buffercache->purge(); | ||
| 2060 | } | ||
| 2061 | if($rate_limit_cache) { | ||
| 2062 | # Memcached can't purge(). | ||
| 2063 | # I don't like this hardwired code, it would be better if I could find a way to determine if a driver can run purge() | ||
| 2064 | $rate_limit_cache->purge() if($rate_limit_cache->short_driver_name() ne 'Memcached'); | ||
| 2065 | } | ||
| 2066 | if(my $info_cache = $self->{info_cache}) { | ||
| 2067 | $info_cache->purge(); | ||
| 2068 | } | ||
| 2069 | if($lingua_cache) { | ||
| 2070 | $lingua_cache->purge(); | ||
| 2071 | } | ||
| 2072 | CHI->stats->flush(); | ||
| 2073 | Log::WarnDie->dispatcher(undef); | ||
| 2074 | } | ||
| 2075 | |||
| 2076 | =head1 AUTHOR | ||
| 2077 | |||
| 2078 | Nigel Horne, C<< <njh at nigelhorne.com> >> | ||
| 2079 | |||
| 2080 | =head1 LICENCE AND COPYRIGHT | ||
| 2081 | |||
| 2082 | Copyright 2010-2026 Nigel Horne. | ||
| 2083 | |||
| 2084 | Usage is subject to the GPL2 licence terms. | ||
| 2085 | If you use it, | ||
| 2086 | please let me know. | ||
| 2087 | |||
| 2088 | =cut | ||
| 2089 | |||
| 2090 | 1; |