| 1 | # Web page download and cache. | ||
| 2 | |||
| 3 | # Copyright 2007, 2008, 2009, 2010, 2011, 2017, 2024 Kevin Ryde | ||
| 4 | |||
| 5 | # This file is part of Chart. | ||
| 6 | # | ||
| 7 | # Chart is free software; you can redistribute it and/or modify it under the | ||
| 8 | # terms of the GNU General Public License as published by the Free Software | ||
| 9 | # Foundation; either version 3, or (at your option) any later version. | ||
| 10 | # | ||
| 11 | # Chart is distributed in the hope that it will be useful, but WITHOUT ANY | ||
| 12 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
| 13 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | ||
| 14 | # details. | ||
| 15 | # | ||
| 16 | # You should have received a copy of the GNU General Public License along | ||
| 17 | # with Chart. If not, see <http://www.gnu.org/licenses/>. | ||
| 18 | |||
| 19 | package App::Chart::Pagebits; | ||
| 20 | use 5.010; | ||
| 21 | use strict; | ||
| 22 | use warnings; | ||
| 23 | use Carp; | ||
| 24 | use Data::Dumper; | ||
| 25 | |||
| 26 | use App::Chart; | ||
| 27 | use App::Chart::Database; | ||
| 28 | use App::Chart::Download; | ||
| 29 | |||
| 30 | sub get { | ||
| 31 | my %options = @_; | ||
| 32 | |||
| 33 | $options{'name'} or croak "Pagebits missing 'name'\n"; | ||
| 34 | my $url = $options{'url'} || croak "Pagebits missing 'url'\n"; | ||
| 35 | my $method = $options{'method'} || 'GET'; | ||
| 36 | my $data = $options{'data'}; | ||
| 37 | my $key = $options{'key'} || croak "Pagebits missing 'key'\n"; | ||
| 38 | my $parse = $options{'parse'} || croak "Pagebits missing 'parse'\n"; | ||
| 39 | my $freq_days = $options{'freq_days'} | ||
| 40 | || croak "Pagebits missing 'freq_days'\n"; | ||
| 41 | |||
| 42 | my $str = App::Chart::Database->read_extra ('', $key); | ||
| 43 | my $h = eval ($str || '{}'); | ||
| 44 | if (! App::Chart::Download::timestamp_within ($h->{'timestamp'}, | ||
| 45 | $freq_days * 86400)) { | ||
| 46 | App::Chart::Download::status ($options{'name'}); | ||
| 47 | my $resp = App::Chart::Download->get | ||
| 48 | ($url, | ||
| 49 | method => $method, | ||
| 50 | data => $data, | ||
| 51 | etag => $h->{'ETag'}, | ||
| 52 | last_modified => $h->{'Last-Modified'}, | ||
| 53 | allow_404 => $options{'allow_404'}, | ||
| 54 | user_agent => $options{'user_agent'}, | ||
| 55 | ); | ||
| 56 | |||
| 57 | if ($options{'allow_404'} && $resp->code == 404) { | ||
| 58 | return undef; | ||
| 59 | } | ||
| 60 | if ($resp->is_success) { | ||
| 61 | my $content = $resp->decoded_content (raise_error=>1); | ||
| 62 | $h = $parse->($content, $resp); | ||
| 63 | $h->{'ETag'} = scalar $resp->header('ETag'); | ||
| 64 | $h->{'Last-Modified'} = $resp->last_modified; | ||
| 65 | } | ||
| 66 | $h->{'timestamp'} = App::Chart::Download::timestamp_now(); | ||
| 67 | |||
| 68 | my $dumper = Data::Dumper->new ([$h], ['var']); | ||
| 69 | $dumper->Indent(1); | ||
| 70 | $dumper->Terse(1); | ||
| 71 | $dumper->Sortkeys(1); | ||
| 72 | $str = $dumper->Dump; | ||
| 73 | App::Chart::Database->write_extra ('', $key, $str); | ||
| 74 | } | ||
| 75 | |||
| 76 | return $h; | ||
| 77 | } | ||
| 78 | |||
| 79 | 1; | ||
| 80 | __END__ |