| 1 | package Web::VWF::Allow; | ||
| 2 | |||
| 3 | use Web::VWF::Blacklist; | ||
| 4 | use Web::VWF::Utils; | ||
| 5 | |||
| 6 | # Decide if we're going to allow this client to view the website | ||
| 7 | # Usage: | ||
| 8 | # unless(Web::VWF::Allow::allow({info => $info, lingua => $lingua})) { | ||
| 9 | |||
| 10 | use strict; | ||
| 11 | use warnings; | ||
| 12 | |||
| 13 | use version; our $VERSION = version->declare('0.003.0'); | ||
| 14 | use Carp; | ||
| 15 | use Error; | ||
| 16 | use File::Spec; | ||
| 17 | |||
| 18 | use constant DSHIELD_BASE => 'https://secure.dshield.org/api/sources/attacks/100/'; | ||
| 19 | |||
| 20 | my %blacklist_agents = ( | ||
| 21 | 'Barkrowler' => 'Barkrowler', | ||
| 22 | 'masscan' => 'Masscan', | ||
| 23 | 'WBSearchBot' => 'Warebay', | ||
| 24 | 'MJ12' => 'Majestic', | ||
| 25 | 'Mozilla/4.0 (compatible; Vagabondo/4.0; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)' => 'wise-guys', | ||
| 26 | 'Mozilla/5.0 zgrab/0.x' => 'zgrab', | ||
| 27 | 'Mozilla/5.0 (compatible; IODC-Odysseus Survey 21796-100-051215155936-107; +https://iodc.co.uk)' => 'iodc', | ||
| 28 | 'Mozilla/5.0 (compatible; adscanner/)' => 'adscanner', | ||
| 29 | 'Mozilla/5.0 (compatible; SemrushBot/6~bl; +http://www.semrush.com/bot.html)' => 'SemrushBot', | ||
| 30 | 'ZoominfoBot (zoominfobot at zoominfo dot com)' => 'zoominfobot', | ||
| 31 | ); | ||
| 32 | |||
| 33 | my %status; | ||
| 34 | |||
| 35 | my $STATUS_TTL = 300; # 5 minutes; keeps memory bounded and lets throttle windows expire | ||
| 36 | |||
| 37 | sub _set_status { | ||
| 38 | my ($addr, $val) = @_; | ||
| 39 | $status{$addr} = [$val, time() + $STATUS_TTL]; | ||
| 40 | } | ||
| 41 | |||
| 42 | sub allow { | ||
| 43 | my $addr = $ENV{'REMOTE_ADDR'}; | ||
| 44 | |||
| 45 | if(!defined($addr)) { | ||
| 46 | # Not running as a CGI | ||
| 47 | return 1; | ||
| 48 | } | ||
| 49 | my %args = (ref($_[0]) eq 'HASH') ? %{$_[0]} : @_; | ||
| 50 | |||
| 51 | my $logger = $args{'logger'}; | ||
| 52 | |||
| 53 | if(defined($status{$addr})) { | ||
| 54 | my ($val, $expires) = @{$status{$addr}}; | ||
| 55 | if(time() < $expires) { | ||
| 56 | if($logger) { | ||
| 57 | $logger->debug("$addr: cached value $val"); | ||
| 58 | } | ||
| 59 | return $val; | ||
| 60 | } | ||
| 61 | delete $status{$addr}; | ||
| 62 | } | ||
| 63 | |||
| 64 | if($logger) { | ||
| 65 | $logger->trace('In ', __PACKAGE__); | ||
| 66 | } | ||
| 67 | |||
| 68 | if($ENV{'HTTP_USER_AGENT'}) { | ||
| 69 | if(my $blocked = $blacklist_agents{$ENV{'HTTP_USER_AGENT'}}) { | ||
| 70 | if($logger) { | ||
| 71 | $logger->info("$blocked blacklisted"); | ||
| 72 | } | ||
| 73 | _set_status($addr, 0); | ||
| 74 | throw Error::Simple("$addr: $blocked is blacklisted", 1); | ||
| 75 | } | ||
| 76 | } | ||
| 77 | |||
| 78 | my $info = $args{'info'}; | ||
| 79 | if(!defined($info)) { | ||
| 80 | if($logger) { | ||
| 81 | $logger->warn('Info not given'); | ||
| 82 | } else { | ||
| 83 | carp('Info not given'); | ||
| 84 | } | ||
| 85 | _set_status($addr, 1); | ||
| 86 | return 1; | ||
| 87 | } | ||
| 88 | |||
| 89 | unless($info->is_search_engine()) { | ||
| 90 | require Data::Throttler; | ||
| 91 | Data::Throttler->import(); | ||
| 92 | |||
| 93 | my $interval = 90; | ||
| 94 | my $max_items = 30; | ||
| 95 | my $db_file = File::Spec->catfile($info->tmpdir(), 'throttle'); | ||
| 96 | if(my $config = $args{config}) { | ||
| 97 | if(my $throttle = $config->{throttle}) { | ||
| 98 | $db_file = $throttle->{'file'} // File::Spec->catdir($info->tmpdir(), 'throttle'); | ||
| 99 | $max_items = $throttle->{'max_items'} // 30; # Allow 30 requests | ||
| 100 | $interval = $throttle->{'interval'} // 90; # Per 90 second window | ||
| 101 | } | ||
| 102 | } | ||
| 103 | # Handle YAML Errors | ||
| 104 | eval { | ||
| 105 | my $throttler = Data::Throttler->new( | ||
| 106 | max_items => $max_items, | ||
| 107 | interval => $interval, | ||
| 108 | backend => 'YAML', | ||
| 109 | backend_options => { | ||
| 110 | db_file => $db_file | ||
| 111 | } | ||
| 112 | ); | ||
| 113 | |||
| 114 | unless($throttler->try_push(key => $addr)) { | ||
| 115 | $info->status(429); | ||
| 116 | if($logger) { | ||
| 117 | $logger->warn("$addr has been throttled"); | ||
| 118 | } | ||
| 119 | _set_status($addr, 0); | ||
| 120 | throw Error::Simple("$addr has been throttled"); | ||
| 121 | } | ||
| 122 | }; | ||
| 123 | if($@) { | ||
| 124 | if(ref($@) && $@->isa('Error')) { | ||
| 125 | # Deliberate throttle block: re-throw so the caller sees the denial. | ||
| 126 | # Do NOT unlink the DB here -- that would erase the throttle window. | ||
| 127 | $@->throw(); | ||
| 128 | } | ||
| 129 | # Genuine YAML/IO error from Data::Throttler -- remove the corrupt file | ||
| 130 | # and fall through so this request is served (fail open). | ||
| 131 | if($logger) { | ||
| 132 | $logger->info("removing corrupt throttle DB $db_file: $@"); | ||
| 133 | } | ||
| 134 | unlink($db_file); | ||
| 135 | } | ||
| 136 | |||
| 137 | unless(($addr =~ /^192\.168\./) || $info->baidu()) { | ||
| 138 | my $lingua = $args{'lingua'}; | ||
| 139 | my $countries = $args{'config'} ? $args{'config'}->{'blacklist_countries'} : undef; | ||
| 140 | if(defined($lingua) && Web::VWF::Blacklist->new(countries => $countries)->is_blocked($lingua->country())) { | ||
| 141 | if($logger) { | ||
| 142 | $logger->warn("$addr blocked connexion from ", $lingua->country()); | ||
| 143 | } | ||
| 144 | _set_status($addr, 0); | ||
| 145 | $info->status(403); | ||
| 146 | throw Error::Simple("$addr: blocked connexion from " . $lingua->country(), 0); | ||
| 147 | } | ||
| 148 | } | ||
| 149 | |||
| 150 | if(defined($ENV{'REQUEST_METHOD'}) && ($ENV{'REQUEST_METHOD'} eq 'GET')) { | ||
| 151 | my $params = $info->params(); | ||
| 152 | if(defined($params) && scalar(%{$params})) { | ||
| 153 | require CGI::IDS; | ||
| 154 | CGI::IDS->import(); | ||
| 155 | |||
| 156 | my $ids = CGI::IDS->new(); | ||
| 157 | $ids->set_scan_keys(scan_keys => 1); | ||
| 158 | delete($params->{'fbclid'}); # Facebook key is OK | ||
| 159 | my $impact = $ids->detect_attacks(request => $params); | ||
| 160 | if($impact > 0) { | ||
| 161 | if($logger) { | ||
| 162 | $logger->warn("$addr: IDS blocked connexion for ", $info->as_string(), " impact = $impact"); | ||
| 163 | require Data::Dumper; | ||
| 164 | $logger->warn(Data::Dumper->new([$ids->get_attacks()])->Dump()); | ||
| 165 | } | ||
| 166 | if($impact > 30) { | ||
| 167 | _set_status($addr, 0); | ||
| 168 | $info->status(403); | ||
| 169 | throw Error::Simple("$addr: IDS blocked connexion for " . $info->as_string()); | ||
| 170 | } | ||
| 171 | } | ||
| 172 | foreach my $v (values %{$params}) { | ||
| 173 | if($v eq '/etc/passwd') { | ||
| 174 | if($logger) { | ||
| 175 | $logger->warn("$addr: blocked connexion attempt for /etc/passwd from ", $info->as_string()); | ||
| 176 | } | ||
| 177 | _set_status($addr, 0); | ||
| 178 | $info->status(403); | ||
| 179 | return 0; | ||
| 180 | } | ||
| 181 | } | ||
| 182 | } | ||
| 183 | } | ||
| 184 | |||
| 185 | if(my $referer = $ENV{'HTTP_REFERER'}) { | ||
| 186 | $referer =~ tr/ /+/; # FIXME - this shouldn't be happening | ||
| 187 | |||
| 188 | if(($referer =~ /^http:\/\/keywords-monitoring-your-success\.com\/try\.php/) || | ||
| 189 | ($referer =~ /^http:\/\/www\.tcsindustry\.com\//) || | ||
| 190 | ($referer =~ /^http:\/\/free-video-tool\.com\//)) { | ||
| 191 | if($logger) { | ||
| 192 | $logger->warn("$addr: Blocked trawler"); | ||
| 193 | } | ||
| 194 | _set_status($addr, 0); | ||
| 195 | throw Error::Simple("$addr: Blocked trawler"); | ||
| 196 | } | ||
| 197 | # Protect against Shellshocker | ||
| 198 | require Data::Validate::URI; | ||
| 199 | Data::Validate::URI->import(); | ||
| 200 | |||
| 201 | unless(Data::Validate::URI->new()->is_uri($referer)) { | ||
| 202 | if($logger) { | ||
| 203 | $logger->warn("$addr: Blocked shellshocker for $referer"); | ||
| 204 | } | ||
| 205 | _set_status($addr, 0); | ||
| 206 | throw Error::Simple("$addr: Blocked shellshocker for $referer"); | ||
| 207 | } | ||
| 208 | } | ||
| 209 | |||
| 210 | if(my $script_uri = $ENV{'SCRIPT_URI'}) { | ||
| 211 | if(($script_uri =~ /\/shell\.php$/) || | ||
| 212 | ($script_uri =~ /\/cmd\.php$/)) { | ||
| 213 | if($logger) { | ||
| 214 | $logger->warn("$addr: Blocked attacker from $addr"); | ||
| 215 | } | ||
| 216 | _set_status($addr, 0); | ||
| 217 | throw Error::Simple("$addr: Blocked attacker for $addr"); | ||
| 218 | } | ||
| 219 | } | ||
| 220 | } | ||
| 221 | |||
| 222 | require DateTime; | ||
| 223 | DateTime->import(); | ||
| 224 | |||
| 225 | my @ips; | ||
| 226 | my $today = DateTime->today()->ymd(); | ||
| 227 | my $readfromcache; | ||
| 228 | |||
| 229 | my $cache = $args{'cache'}; | ||
| 230 | if(!defined($cache)) { | ||
| 231 | throw Error::Simple('Either cache or config must be given') unless($args{config}); | ||
| 232 | $cache = Web::VWF::Utils::create_memory_cache(config => $args{'config'}, namespace => __PACKAGE__, logger => $logger); | ||
| 233 | } | ||
| 234 | if(defined($cache)) { | ||
| 235 | my $cachecontent = $cache->get($today); | ||
| 236 | if($cachecontent) { | ||
| 237 | if($logger) { | ||
| 238 | $logger->debug("read from cache $cachecontent"); | ||
| 239 | } | ||
| 240 | @ips = split(/,/, $cachecontent); | ||
| 241 | if(scalar(@ips)) { | ||
| 242 | $readfromcache = 1; | ||
| 243 | } else { | ||
| 244 | if($logger) { | ||
| 245 | $logger->info("DShield cache for $today is empty, deleting to force reread"); | ||
| 246 | } | ||
| 247 | $cache->remove($today); | ||
| 248 | } | ||
| 249 | } elsif($logger) { | ||
| 250 | $logger->debug("Can't find $today in the cache"); | ||
| 251 | } | ||
| 252 | } elsif($logger) { | ||
| 253 | $logger->warn('Couldn\'t create the DShield cache'); | ||
| 254 | } | ||
| 255 | |||
| 256 | unless(scalar(@ips)) { | ||
| 257 | require LWP::Simple::WithCache; | ||
| 258 | LWP::Simple::WithCache->import(); | ||
| 259 | require XML::LibXML; | ||
| 260 | XML::LibXML->import(); | ||
| 261 | |||
| 262 | if($logger) { | ||
| 263 | $logger->trace('Downloading DShield signatures'); | ||
| 264 | } | ||
| 265 | |||
| 266 | my $xml; | ||
| 267 | eval { | ||
| 268 | my $dshield_url = DSHIELD_BASE . $today; | ||
| 269 | if(my $string = LWP::Simple::WithCache::get($dshield_url)) { | ||
| 270 | $xml = XML::LibXML->load_xml(string => $string); | ||
| 271 | } elsif($logger) { | ||
| 272 | $logger->warn("Couldn't download $dshield_url"); | ||
| 273 | return 1; | ||
| 274 | } else { | ||
| 275 | warn "Couldn't download $dshield_url\n"; | ||
| 276 | return 1; | ||
| 277 | } | ||
| 278 | }; | ||
| 279 | unless($@ || !defined($xml)) { | ||
| 280 | foreach my $source ($xml->findnodes('/sources/data')) { | ||
| 281 | my $lastseen = $source->findnodes('./lastseen')->to_literal(); | ||
| 282 | next if($readfromcache && ($lastseen ne $today)); # FIXME: Should be today or yesterday to avoid midnight rush | ||
| 283 | my $ip = $source->findnodes('./ip')->to_literal(); | ||
| 284 | $ip =~ s/0*(\d+)/$1/g; # Perl interprets numbers leading with 0 as octal | ||
| 285 | push @ips, $ip; | ||
| 286 | } | ||
| 287 | if(defined($cache) && scalar(@ips) && !$readfromcache) { | ||
| 288 | my $cachecontent = join(',', @ips); | ||
| 289 | if($logger) { | ||
| 290 | $logger->info("Setting DShield cache for $today to $cachecontent"); | ||
| 291 | } | ||
| 292 | $cache->set($today, $cachecontent, '1 day'); | ||
| 293 | } | ||
| 294 | } | ||
| 295 | } | ||
| 296 | |||
| 297 | # FIXME: Doesn't realise 1.2.3.4 is the same as 001.002.003.004 | ||
| 298 | if(grep($_ eq $addr, @ips)) { | ||
| 299 | if($logger) { | ||
| 300 | $logger->warn("Dshield blocked connexion from $addr"); | ||
| 301 | } | ||
| 302 | _set_status($addr, 0); | ||
| 303 | throw Error::Simple("Dshield blocked connexion from $addr"); | ||
| 304 | } | ||
| 305 | |||
| 306 | if($info->get_cookie(cookie_name => 'mycustomtrackid')) { | ||
| 307 | if($logger) { | ||
| 308 | $logger->warn('Blocking possible jqic'); | ||
| 309 | } | ||
| 310 | _set_status($addr, 0); | ||
| 311 | throw Error::Simple('Blocking possible jqic'); | ||
| 312 | } | ||
| 313 | |||
| 314 | if($logger) { | ||
| 315 | $logger->trace("Allowing connexion from $addr"); | ||
| 316 | } | ||
| 317 | |||
| 318 | _set_status($addr, 1); | ||
| 319 | return 1; | ||
| 320 | } | ||
| 321 | |||
| 322 | 1; | ||
| 323 | |||
| 324 | =encoding utf8 | ||
| 325 | |||
| 326 | =head1 NAME | ||
| 327 | |||
| 328 | Web::VWF::Allow - The last access check a request must pass | ||
| 329 | |||
| 330 | =head1 VERSION | ||
| 331 | |||
| 332 | Version 0.003.0 | ||
| 333 | |||
| 334 | =head1 SYNOPSIS | ||
| 335 | |||
| 336 | unless(Web::VWF::Allow::allow({ info => $info, lingua => $lingua, config => $config, logger => $logger })) { | ||
| 337 | # refused | ||
| 338 | } | ||
| 339 | |||
| 340 | =head1 DESCRIPTION | ||
| 341 | |||
| 342 | Decides whether a client may be shown the site. It checks, in this order, the | ||
| 343 | user agent against a list of unwelcome robots, the per-address L<Data::Throttler> | ||
| 344 | limit (the C<throttle> settings, see L<Web::VWF/CONFIGURATION>), the country | ||
| 345 | against the C<blacklist_countries> setting, L<CGI::IDS>, a few known attack | ||
| 346 | signatures in the Referer and URL, the DShield list of attacking addresses, and a | ||
| 347 | cookie set by one kind of scanner. | ||
| 348 | |||
| 349 | Search engines are not throttled. The verdict for an address is remembered for | ||
| 350 | five minutes. | ||
| 351 | |||
| 352 | =head1 FUNCTIONS | ||
| 353 | |||
| 354 | =head2 allow | ||
| 355 | |||
| 356 | Takes a hash or a hash reference: C<info> (a L<CGI::Info>, required in practice), | ||
| 357 | C<lingua> (a L<CGI::Lingua>), C<logger>, C<config> and C<cache>. One of | ||
| 358 | C<cache> and C<config> is needed for the DShield list. | ||
| 359 | |||
| 360 | Returns true when the client is allowed, which is always so when C<REMOTE_ADDR> | ||
| 361 | is not set (the script is being run from a command line), and when C<info> is | ||
| 362 | not given, which is logged. A client that is refused makes it throw an | ||
| 363 | L<Error::Simple>, and the next calls for the same address in the following five | ||
| 364 | minutes return false instead of throwing. C<info-E<gt>status> is set to 429 for | ||
| 365 | a throttled address. | ||
| 366 | |||
| 367 | =head3 API SPECIFICATION | ||
| 368 | |||
| 369 | =head4 INPUT | ||
| 370 | |||
| 371 | { | ||
| 372 | info => { type => 'object', optional => 1 }, | ||
| 373 | lingua => { type => 'object', optional => 1 }, | ||
| 374 | logger => { type => 'object', optional => 1 }, | ||
| 375 | config => { type => 'object', optional => 1 }, | ||
| 376 | cache => { type => 'object', optional => 1 }, | ||
| 377 | } | ||
| 378 | |||
| 379 | =head4 OUTPUT | ||
| 380 | |||
| 381 | { type => 'boolean' } | ||
| 382 | |||
| 383 | and throws L<Error::Simple> when the client is refused. | ||
| 384 | |||
| 385 | =head1 AUTHOR | ||
| 386 | |||
| 387 | Nigel Horne, C<< <njh at nigelhorne.com> >> | ||
| 388 | |||
| 389 | =head1 LICENCE AND COPYRIGHT | ||
| 390 | |||
| 391 | Copyright 2010-2026 Nigel Horne. | ||
| 392 | |||
| 393 | Usage is subject to the GPL2 licence terms. | ||
| 394 | If you use it, | ||
| 395 | please let me know. | ||
| 396 | |||
| 397 | =cut |