| 1 | # Download functions. | ||
| 2 | |||
| 3 | # Copyright 2007, 2008, 2009, 2010, 2011, 2013, 2015, 2016, 2017, 2018, 2020, 2023, 2024 Kevin Ryde | ||
| 4 | |||
| 5 | # This file is part of Chart. | ||
| 6 | # | ||
| 7 | # Chart is free software; you can redistribute it and/or modify it under the | ||
| 8 | # terms of the GNU General Public License as published by the Free Software | ||
| 9 | # Foundation; either version 3, or (at your option) any later version. | ||
| 10 | # | ||
| 11 | # Chart is distributed in the hope that it will be useful, but WITHOUT ANY | ||
| 12 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
| 13 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | ||
| 14 | # details. | ||
| 15 | # | ||
| 16 | # You should have received a copy of the GNU General Public License along | ||
| 17 | # with Chart. If not, see <http://www.gnu.org/licenses/>. | ||
| 18 | |||
| 19 | package App::Chart::Download; | ||
| 20 | 2 | ▶ | use 5.010; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 21 | 2 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 22 | 2 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 23 | 2 | ▶ | use Carp 'carp','croak'; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 24 | 2 | ▶ | use Date::Calc; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 25 | 2 | ▶ | use List::Util qw(min max); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 26 | 2 | ▶ | use List::MoreUtils; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 27 | 2 | ▶ | use Regexp::Common 'whitespace'; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 28 | 2 | ▶ | use Locale::TextDomain ('App-Chart'); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 29 | |||
| 30 | 2 | ▶ | use PerlIO::via::EscStatus; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 31 | 2 | ▶ | use Tie::TZ; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 32 | |||
| 33 | 0 | ▶ | use App::Chart; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 34 | use App::Chart::Database; | ||
| 35 | use App::Chart::DBI; | ||
| 36 | use App::Chart::TZ; | ||
| 37 | |||
| 38 | # uncomment this to run the ### lines | ||
| 39 | # use Smart::Comments; | ||
| 40 | |||
| 41 | use constant DEBUG => 0; | ||
| 42 | |||
| 43 | #------------------------------------------------------------------------------ | ||
| 44 | |||
| 45 | sub http_code_is_allowed { | ||
| 46 | my ($aref, $code) = @_; | ||
| 47 | return (grep {$code eq $_} @{$aref // []}) > 0; | ||
| 48 | } | ||
| 49 | |||
| 50 | sub get { | ||
| 51 | my ($class, $url, %options) = @_; | ||
| 52 | ### Download get(): $url | ||
| 53 | |||
| 54 | # URI object becomes string | ||
| 55 | $url = "$url"; | ||
| 56 | |||
| 57 | my $ua = $options{'ua'} || do { require App::Chart::UserAgent; | ||
| 58 | App::Chart::UserAgent->instance }; | ||
| 59 | $ua->cookie_jar ($options{'cookie_jar'}); # undef for none | ||
| 60 | ### cookie_jar object: ($options{'cookie_jar'} // "")."" | ||
| 61 | |||
| 62 | require HTTP::Request; | ||
| 63 | my $method = $options{'method'} || 'GET'; | ||
| 64 | my @headers = (Referer => $options{'referer'}); | ||
| 65 | my $data = $options{'data'}; | ||
| 66 | if (defined $data) { | ||
| 67 | push @headers, 'Content-Type' => 'application/x-www-form-urlencoded'; | ||
| 68 | } | ||
| 69 | my $req = HTTP::Request->new ($method, $url, \@headers, $data); | ||
| 70 | |||
| 71 | # possible override | ||
| 72 | if (my $user_agent = $options{'user_agent'}) { | ||
| 73 | $req->user_agent($user_agent); | ||
| 74 | } | ||
| 75 | |||
| 76 | my $etag = $options{'etag'}; | ||
| 77 | my $lastmod = $options{'last_modified'}; | ||
| 78 | |||
| 79 | if (my $key = $options{'url_tags_key'}) { | ||
| 80 | my $symbol = $options{'symbol'}; | ||
| 81 | my $prev_url = App::Chart::Database->read_extra($symbol,"$key-URL"); | ||
| 82 | if (defined $prev_url && $url eq $prev_url) { | ||
| 83 | $etag = App::Chart::Database->read_extra($symbol,"$key-ETag"); | ||
| 84 | $lastmod = App::Chart::Database->read_extra($symbol,"$key-Last-Modified"); | ||
| 85 | } | ||
| 86 | } | ||
| 87 | |||
| 88 | if ($etag) { $req->header ('If-None-Match' => $etag); } | ||
| 89 | if ($lastmod) { $req->header ('If-Modified-Since' => $lastmod); } | ||
| 90 | |||
| 91 | my $resp = $ua->request ($req); | ||
| 92 | |||
| 93 | # internal message from LWP when a keep-alive has missed the boat | ||
| 94 | if ($resp->status_line =~ /500 Server closed connection/i) { | ||
| 95 | substatus (__('retry')); | ||
| 96 | $resp = $ua->request ($req); | ||
| 97 | } | ||
| 98 | |||
| 99 | my $code = $resp->code; | ||
| 100 | if ($resp->is_success | ||
| 101 | || ($options{'allow_401'} && $code == 401) | ||
| 102 | || ($options{'allow_404'} && $code == 404) | ||
| 103 | || (($etag || $lastmod) && $code == 304) | ||
| 104 | || http_code_is_allowed($options{'allow_http_codes'}, $code)) { | ||
| 105 | substatus (__('processing')); | ||
| 106 | return $resp; | ||
| 107 | } else { | ||
| 108 | my $error = $resp->status_line . "\n"; | ||
| 109 | if ($code == 500) { | ||
| 110 | $error .= $resp->decoded_content; | ||
| 111 | unless ($error =~ /\n$/s) { $error .= "\n"; } | ||
| 112 | } | ||
| 113 | croak "Cannot download $url\n" . $error; | ||
| 114 | } | ||
| 115 | } | ||
| 116 | |||
| 117 | #------------------------------------------------------------------------------ | ||
| 118 | |||
| 119 | my $last_status = ''; # without substatus addition | ||
| 120 | |||
| 121 | sub download_message { | ||
| 122 | print join (' ',@_),"\n"; | ||
| 123 | } | ||
| 124 | sub verbose_message { | ||
| 125 | if ($App::Chart::option{'verbose'}) { | ||
| 126 | print join (' ',@_),"\n"; | ||
| 127 | } | ||
| 128 | } | ||
| 129 | |||
| 130 | sub status { | ||
| 131 | my $str = join (' ', @_); | ||
| 132 | $last_status = $str; | ||
| 133 | PerlIO::via::EscStatus::print_status ($str); | ||
| 134 | } | ||
| 135 | sub substatus { | ||
| 136 | my ($str) = @_; | ||
| 137 | if ($str) { | ||
| 138 | PerlIO::via::EscStatus::print_status ($last_status, ' [', $str, ']'); | ||
| 139 | } | ||
| 140 | } | ||
| 141 | |||
| 142 | #------------------------------------------------------------------------------ | ||
| 143 | |||
| 144 | sub split_lines { | ||
| 145 | my ($str) = @_; | ||
| 146 | my @lines = split (/[\r\n]+/, $str); # LF or CRLF | ||
| 147 | foreach (@lines) { $_ =~ s/[ \t]+$// } # trailing whitespace | ||
| 148 | return grep {$_ ne ''} @lines; # no blanks | ||
| 149 | } | ||
| 150 | |||
| 151 | #------------------------------------------------------------------------------ | ||
| 152 | |||
| 153 | # Return $str with trailing "0"s removed. | ||
| 154 | # Don't go below $want_deicmals digits after the decimal point. | ||
| 155 | sub trim_decimals { | ||
| 156 | my ($str, $want_decimals) = @_; | ||
| 157 | if ($str && $str =~ /(.*\.[0-9]{$want_decimals}[0-9]*?)0+$/) { | ||
| 158 | return $1; | ||
| 159 | } else { | ||
| 160 | return $str; | ||
| 161 | } | ||
| 162 | } | ||
| 163 | |||
| 164 | #------------------------------------------------------------------------------ | ||
| 165 | |||
| 166 | sub str_is_zero { | ||
| 167 | my ($str) = @_; | ||
| 168 | return ($str =~ /^0+(\.0*)?$|^0*(\.0+)$/ ? 1 : 0); | ||
| 169 | } | ||
| 170 | |||
| 171 | |||
| 172 | #------------------------------------------------------------------------------ | ||
| 173 | |||
| 174 | sub cents_to_dollars { | ||
| 175 | my ($str) = @_; | ||
| 176 | $str =~ /^([^.]*)(\.(.*))?$/ | ||
| 177 | or croak "cents_to_dollars(): bad string: \"$str\""; | ||
| 178 | my $int = $1; | ||
| 179 | my $frac = (defined $3 ? $3 : ''); | ||
| 180 | if (length ($int) < 3) { | ||
| 181 | $int = sprintf ('%03s', $int); | ||
| 182 | } | ||
| 183 | return substr ($int, 0, length($int)-2) . '.' . | ||
| 184 | substr ($int, length($int)-2) . $frac; | ||
| 185 | } | ||
| 186 | |||
| 187 | |||
| 188 | #------------------------------------------------------------------------------ | ||
| 189 | |||
| 190 | sub month_to_nearest_year { | ||
| 191 | my ($target_month) = @_; | ||
| 192 | my ($year, $month, $day) = Date::Calc::Today(); | ||
| 193 | $month --; # 0=January | ||
| 194 | $target_month --; | ||
| 195 | |||
| 196 | my $diff = $target_month - $month; | ||
| 197 | $diff += 5; | ||
| 198 | $diff %= 12; | ||
| 199 | $diff -= 5; # now range -5 to +6 | ||
| 200 | |||
| 201 | # applying $diff makes $month == $target_month modulo 12, | ||
| 202 | # but $month<0 is last year, 0 to 11 this year, >= 12 next year | ||
| 203 | $month += $diff; | ||
| 204 | return $year + ($month < 0 ? -1 : $month < 12 ? 0 : 1); | ||
| 205 | } | ||
| 206 | |||
| 207 | |||
| 208 | #------------------------------------------------------------------------------ | ||
| 209 | |||
| 210 | sub Decode_Date_EU_to_iso { | ||
| 211 | my ($str, $noerror) = @_; | ||
| 212 | my ($year, $month, $day) = Date::Calc::Decode_Date_EU ($str); | ||
| 213 | unless (defined $year && defined $month && defined $day) { | ||
| 214 | if ($noerror) { | ||
| 215 | return undef; | ||
| 216 | } else { | ||
| 217 | croak "Decode_Date_EU_to_iso: unrecognised date \"$str\"\n"; | ||
| 218 | } | ||
| 219 | } | ||
| 220 | return App::Chart::ymd_to_iso ($year, $month, $day); | ||
| 221 | } | ||
| 222 | |||
| 223 | sub Decode_Date_US_to_iso { | ||
| 224 | my ($str) = @_; | ||
| 225 | my ($year, $month, $day) = Date::Calc::Decode_Date_US ($str); | ||
| 226 | unless (defined $year && defined $month && defined $day) { | ||
| 227 | croak "Decode_Date_US_to_iso: unrecognised date \"$str\"\n"; | ||
| 228 | } | ||
| 229 | return App::Chart::ymd_to_iso ($year, $month, $day); | ||
| 230 | } | ||
| 231 | |||
| 232 | #------------------------------------------------------------------------------ | ||
| 233 | |||
| 234 | sub Decode_Date_YMD { | ||
| 235 | my ($str) = @_; | ||
| 236 | ($str =~ m{^ # 6 or 8 digits are yyyymmdd or yymmdd | ||
| 237 | ["'[:space:]]* | ||
| 238 | (\d{2,4}) # $1 year | ||
| 239 | ((\d{2})|([A-Za-z]+)) # $3 numeric month, $4 alpha month | ||
| 240 | (\d{2}) # $5 day | ||
| 241 | ["'[:space:]]* | ||
| 242 | $}x) | ||
| 243 | or | ||
| 244 | ($str =~ m{^ | ||
| 245 | ["'[:space:]]* | ||
| 246 | (\d{2,4}) # $1 year | ||
| 247 | [-_/:.[:space:]]* | ||
| 248 | ((\d{1,2})|([A-Za-z]+)) # $3 numeric month, $4 alpha month | ||
| 249 | [-_/:.[:space:]]* | ||
| 250 | (\d{1,2}) # $5 day | ||
| 251 | ["'[:space:]]* | ||
| 252 | $}x) | ||
| 253 | or return; | ||
| 254 | |||
| 255 | my $year = $1; | ||
| 256 | my $num_month = $3; | ||
| 257 | my $alpha_month = $4, | ||
| 258 | my $day = $5; | ||
| 259 | my $month = $num_month || Date::Calc::Decode_Month ($alpha_month); | ||
| 260 | $year = Date::Calc::Moving_Window ($year); | ||
| 261 | return ($year, $month, $day); | ||
| 262 | } | ||
| 263 | |||
| 264 | sub Decode_Date_YMD_to_iso { | ||
| 265 | my ($str) = @_; | ||
| 266 | my ($year, $month, $day) = Decode_Date_YMD ($str); | ||
| 267 | if (! defined $year || ! defined $month || ! defined $day | ||
| 268 | || ! Date::Calc::check_date ($year, $month, $day)) { | ||
| 269 | croak "Decode_Date_YMD_to_iso: invalid date \"$str\"\n"; | ||
| 270 | } | ||
| 271 | return App::Chart::ymd_to_iso ($year, $month, $day); | ||
| 272 | } | ||
| 273 | |||
| 274 | |||
| 275 | #------------------------------------------------------------------------------ | ||
| 276 | |||
| 277 | sub date_parse_to_iso { | ||
| 278 | my ($str) = @_; | ||
| 279 | require Date::Parse; | ||
| 280 | my ($ss,$mm,$hh,$day,$month,$year,$zone) = Date::Parse::strptime ($str); | ||
| 281 | if (! defined ($day) || ! defined ($month) || ! defined ($year)) { | ||
| 282 | croak "date_parse_to_iso: unrecognised date \"$str\"\n"; | ||
| 283 | } | ||
| 284 | return App::Chart::ymd_to_iso ($year + 1900, $month + 1, $day); | ||
| 285 | } | ||
| 286 | |||
| 287 | #------------------------------------------------------------------------------ | ||
| 288 | |||
| 289 | # $h is a hash of share prices | ||
| 290 | # data => [ { symbol=>$str, | ||
| 291 | # date => $str, | ||
| 292 | # open => $price, | ||
| 293 | # high => $price, | ||
| 294 | # low => $price, | ||
| 295 | # close => $price, | ||
| 296 | # volume => $number, | ||
| 297 | # openint => $number, | ||
| 298 | # }, | ||
| 299 | # ... | ||
| 300 | # ] | ||
| 301 | # dividends => [ { symbol => $str, | ||
| 302 | # ex_date => $date, | ||
| 303 | # record_date => $date, | ||
| 304 | # pay_date => $date, | ||
| 305 | # type => $str, | ||
| 306 | # amount => $price, | ||
| 307 | # imputation => $price, | ||
| 308 | # qualifier => $str, | ||
| 309 | # note => $str, | ||
| 310 | # }, | ||
| 311 | # ... | ||
| 312 | # ], | ||
| 313 | # write it as daily data in the database | ||
| 314 | sub write_daily_group { | ||
| 315 | my ($h) = @_; | ||
| 316 | |||
| 317 | crunch_h ($h); | ||
| 318 | |||
| 319 | my $prefer_decimals = $h->{'prefer_decimals'}; | ||
| 320 | my $database_symbols_hash = App::Chart::Database::database_symbols_hash(); | ||
| 321 | |||
| 322 | my %decimals; | ||
| 323 | my %data_changed; | ||
| 324 | |||
| 325 | substatus (__('writing database')); | ||
| 326 | my $dbh = App::Chart::DBI->instance; | ||
| 327 | App::Chart::Database::call_with_transaction | ||
| 328 | ($dbh, sub { | ||
| 329 | if ($h->{'cost_key'}) { | ||
| 330 | require App::Chart::DownloadCost; | ||
| 331 | App::Chart::DownloadCost::cost_store_h ($h); | ||
| 332 | } | ||
| 333 | |||
| 334 | my %symbol_hash; | ||
| 335 | { | ||
| 336 | my $sth = $dbh->prepare_cached | ||
| 337 | ('INSERT OR REPLACE INTO daily | ||
| 338 | (symbol, date, open, high, low, close, volume, openint) | ||
| 339 | VALUES (?,?,?,?,?,?,?,?)'); | ||
| 340 | |||
| 341 | foreach my $data (@{$h->{'data'}}) { | ||
| 342 | my $symbol = $data->{'symbol'}; | ||
| 343 | next unless exists $database_symbols_hash->{$symbol}; | ||
| 344 | |||
| 345 | if (defined $prefer_decimals) { | ||
| 346 | $decimals{$symbol} = $prefer_decimals; | ||
| 347 | } | ||
| 348 | $data_changed{$symbol} = 1; | ||
| 349 | |||
| 350 | $sth->execute($symbol, | ||
| 351 | $data->{'date'}, | ||
| 352 | $data->{'open'}, | ||
| 353 | $data->{'high'}, | ||
| 354 | $data->{'low'}, | ||
| 355 | $data->{'close'}, | ||
| 356 | $data->{'volume'}, | ||
| 357 | $data->{'openint'}); | ||
| 358 | $sth->finish; | ||
| 359 | } | ||
| 360 | } | ||
| 361 | |||
| 362 | foreach my $dividend (@{$h->{'dividends'}}) { | ||
| 363 | my $symbol = $dividend->{'symbol'} | ||
| 364 | or croak "write_daily_group: missing symbol in dividend record"; | ||
| 365 | next unless exists $database_symbols_hash->{$symbol}; | ||
| 366 | |||
| 367 | my $ex_date = $dividend->{'ex_date'}; | ||
| 368 | my $record_date = $dividend->{'record_date'}; | ||
| 369 | my $pay_date = $dividend->{'pay_date'}; | ||
| 370 | |||
| 371 | my $type = $dividend->{'type'} || ''; | ||
| 372 | my $amount = $dividend->{'amount'}; | ||
| 373 | my $imputation = $dividend->{'imputation'}; | ||
| 374 | my $qualifier = crunch_empty_undef ($dividend->{'qualifier'}); | ||
| 375 | my $note = crunch_empty_undef ($dividend->{'note'}); | ||
| 376 | |||
| 377 | my $old_qualifier = App::Chart::DBI->read_single | ||
| 378 | ('SELECT qualifier FROM dividend WHERE symbol=? AND ex_date=? AND type=?', $symbol, $ex_date, $type); | ||
| 379 | |||
| 380 | my $sth = $dbh->prepare_cached | ||
| 381 | ('INSERT OR REPLACE INTO dividend | ||
| 382 | (symbol, ex_date, record_date, pay_date, | ||
| 383 | type, amount, imputation, qualifier, note) | ||
| 384 | VALUES (?,?,?,?, ?,?,?,?,?)'); | ||
| 385 | |||
| 386 | $sth->execute ($symbol, $ex_date, $record_date, $pay_date, | ||
| 387 | $type, $amount, $imputation, $qualifier, $note); | ||
| 388 | $sth->finish; | ||
| 389 | $data_changed{$symbol} = 1; | ||
| 390 | } | ||
| 391 | |||
| 392 | foreach my $split (@{$h->{'splits'}}) { | ||
| 393 | my $symbol = $split->{'symbol'} | ||
| 394 | or croak "write_daily_group: missing symbol in split record"; | ||
| 395 | next unless exists $database_symbols_hash->{$symbol}; | ||
| 396 | |||
| 397 | my $date = $split->{'date'}; | ||
| 398 | my $new = crunch_number ($split->{'new'}); | ||
| 399 | my $old = crunch_number ($split->{'old'}); | ||
| 400 | my $note = crunch_empty_undef ($split->{'note'}); | ||
| 401 | |||
| 402 | my $sth = $dbh->prepare_cached | ||
| 403 | ('INSERT OR REPLACE INTO split (symbol, date, new, old, note) | ||
| 404 | VALUES (?,?,?,?,?)'); | ||
| 405 | $sth->execute ($symbol, $date, $new, $old, $note); | ||
| 406 | $sth->finish; | ||
| 407 | $data_changed{$symbol} = 1; | ||
| 408 | } | ||
| 409 | |||
| 410 | if (my $names = $h->{'names'}) { | ||
| 411 | while (my ($symbol, $name) = each %$names) { | ||
| 412 | if (defined $name) { | ||
| 413 | $data_changed{$symbol} |= set_symbol_name ($symbol, $name); | ||
| 414 | } | ||
| 415 | } | ||
| 416 | } | ||
| 417 | if (my $currencies = $h->{'currencies'}) { | ||
| 418 | while (my ($symbol, $currency) = each %$currencies) { | ||
| 419 | if (defined $currency) { | ||
| 420 | $data_changed{$symbol} |= set_currency ($symbol, $currency); | ||
| 421 | } | ||
| 422 | } | ||
| 423 | } | ||
| 424 | if (my $isins = $h->{'isins'}) { | ||
| 425 | while (my ($symbol, $isin) = each %$isins) { | ||
| 426 | if (defined $isin) { | ||
| 427 | $data_changed{$symbol} |= set_isin ($symbol, $isin); | ||
| 428 | } | ||
| 429 | } | ||
| 430 | } | ||
| 431 | if (my $exchanges = $h->{'exchanges'}) { | ||
| 432 | while (my ($symbol, $exchange) = each %$exchanges) { | ||
| 433 | if (defined $exchange) { | ||
| 434 | $data_changed{$symbol} |= set_exchange ($symbol, $exchange); | ||
| 435 | } | ||
| 436 | } | ||
| 437 | } | ||
| 438 | while (my ($symbol, $decimals) = each %decimals) { | ||
| 439 | if (defined $decimals) { | ||
| 440 | $data_changed{$symbol} |= set_decimals ($symbol, $decimals); | ||
| 441 | } | ||
| 442 | } | ||
| 443 | |||
| 444 | my $symbol_list = [ keys %data_changed ]; | ||
| 445 | |||
| 446 | if (my $key = $h->{'url_tags_key'}) { | ||
| 447 | my $resp = $h->{'resp'}; | ||
| 448 | foreach my $symbol (@$symbol_list) { | ||
| 449 | my $url = (defined $resp ? $resp->request->uri : undef); | ||
| 450 | my $etag = (defined $resp ? scalar $resp->header('ETag') : undef); | ||
| 451 | my $last_modified =(defined $resp ? $resp->last_modified : undef); | ||
| 452 | |||
| 453 | App::Chart::Database->write_extra | ||
| 454 | ($symbol, "$key-URL", "$url"); # stringize URI object | ||
| 455 | App::Chart::Database->write_extra | ||
| 456 | ($symbol, "$key-ETag", $etag); | ||
| 457 | App::Chart::Database->write_extra | ||
| 458 | ($symbol, "$key-Last-Modified", $last_modified); | ||
| 459 | } | ||
| 460 | } | ||
| 461 | |||
| 462 | if (my $key = $h->{'copyright_key'}) { | ||
| 463 | foreach my $symbol (@$symbol_list) { | ||
| 464 | App::Chart::Database->write_extra | ||
| 465 | ($symbol, $key, $h->{'copyright'}); | ||
| 466 | } | ||
| 467 | } | ||
| 468 | |||
| 469 | my $timestamp = timestamp_now(); | ||
| 470 | |||
| 471 | if (my $key = $h->{'recheck_key'}) { | ||
| 472 | my @recheck_list; | ||
| 473 | if (my $l = $h->{'recheck_list'}) { | ||
| 474 | @recheck_list = @$l; | ||
| 475 | } | ||
| 476 | if (my $pred = $h->{'recheck_pred'}) { | ||
| 477 | push @recheck_list, | ||
| 478 | grep { $pred->match($_) } keys %$database_symbols_hash; | ||
| 479 | } | ||
| 480 | if (DEBUG) { print "Recheck write ",join(' ',@recheck_list),"\n"; } | ||
| 481 | foreach my $symbol (@recheck_list) { | ||
| 482 | App::Chart::Database->write_extra ($symbol, $key, $timestamp); | ||
| 483 | } | ||
| 484 | } | ||
| 485 | |||
| 486 | if ($h->{'last_download'}) { | ||
| 487 | foreach my $symbol (keys %data_changed) { | ||
| 488 | App::Chart::Database->write_extra ($symbol, 'last-download', | ||
| 489 | $timestamp); | ||
| 490 | } | ||
| 491 | consider_historical ($symbol_list); | ||
| 492 | } | ||
| 493 | consider_latest_from_daily ($h, $database_symbols_hash); | ||
| 494 | }); | ||
| 495 | |||
| 496 | ### data_changed: %data_changed | ||
| 497 | App::Chart::chart_dirbroadcast()->send ('data-changed', \%data_changed); | ||
| 498 | } | ||
| 499 | |||
| 500 | sub consider_historical { | ||
| 501 | my ($symbol_list) = @_; | ||
| 502 | |||
| 503 | my $all_list; | ||
| 504 | my $historical_list; | ||
| 505 | |||
| 506 | foreach my $symbol (@$symbol_list) { | ||
| 507 | if (App::Chart::Database->symbol_is_historical ($symbol)) { | ||
| 508 | next; # already marked | ||
| 509 | } | ||
| 510 | my $reason = want_historical($symbol) // next; | ||
| 511 | download_message ($reason); | ||
| 512 | |||
| 513 | my $dbh = App::Chart::DBI->instance; | ||
| 514 | $dbh->do ('UPDATE info SET historical=1 WHERE symbol=?', | ||
| 515 | {}, $symbol); | ||
| 516 | |||
| 517 | require App::Chart::Gtk2::Symlist; | ||
| 518 | $all_list ||= App::Chart::Gtk2::Symlist::All->instance; | ||
| 519 | $historical_list ||= App::Chart::Gtk2::Symlist::Historical->instance; | ||
| 520 | |||
| 521 | $all_list->delete_symbol ($symbol); | ||
| 522 | $historical_list->insert_symbol ($symbol); | ||
| 523 | } | ||
| 524 | } | ||
| 525 | |||
| 526 | # return true if $symbol should be marked historical, meaning it has had no | ||
| 527 | # new daily data for a long time | ||
| 528 | # | ||
| 529 | sub want_historical { | ||
| 530 | my ($symbol) = @_; | ||
| 531 | my $last_download_timestamp | ||
| 532 | = App::Chart::Database->read_extra ($symbol, 'last-download') | ||
| 533 | // return undef; # no download attempted, not historical | ||
| 534 | |||
| 535 | my $date = App::Chart::DBI->read_single | ||
| 536 | ('SELECT date FROM daily WHERE (symbol=?) AND (close NOTNULL) | ||
| 537 | ORDER BY date DESC LIMIT 1', | ||
| 538 | $symbol); | ||
| 539 | if (! defined $date) { | ||
| 540 | return __x('{symbol} no data at all, marked historical', | ||
| 541 | symbol => $symbol); | ||
| 542 | } | ||
| 543 | my $days = iso_timestamp_days_ago ($date, $last_download_timestamp); | ||
| 544 | if ($days > 21) { | ||
| 545 | return __x('{symbol} no data for {days} days, marked historical', | ||
| 546 | days => $days, | ||
| 547 | symbol => $symbol); | ||
| 548 | } | ||
| 549 | return undef; | ||
| 550 | } | ||
| 551 | |||
| 552 | sub iso_timestamp_days_ago { | ||
| 553 | my ($iso, $prev_timestamp) = @_; | ||
| 554 | my ($prev_year,$prev_month,$prev_day) = timestamp_to_ymdhms($prev_timestamp); | ||
| 555 | return Date::Calc::Delta_Days (App::Chart::iso_to_ymd($iso), | ||
| 556 | $prev_year, $prev_month, $prev_day); | ||
| 557 | } | ||
| 558 | |||
| 559 | sub consider_latest_from_daily { | ||
| 560 | my ($h) = @_; | ||
| 561 | my $dbh = App::Chart::DBI->instance; | ||
| 562 | my %latest_changed; | ||
| 563 | my $timestamp; | ||
| 564 | |||
| 565 | my $database_symbols_hash = App::Chart::Database::database_symbols_hash(); | ||
| 566 | |||
| 567 | # find the newest and second newest data record of each symbol | ||
| 568 | my %data_newest; | ||
| 569 | my %data_second; | ||
| 570 | foreach my $data (@{$h->{'data'}}) { | ||
| 571 | my $symbol = $data->{'symbol'}; | ||
| 572 | if ($data->{'date'} ge ($data_newest{$symbol}->{'date'} // '')) { | ||
| 573 | $data_second{$symbol} = $data_newest{$symbol}; | ||
| 574 | $data_newest{$symbol} = $data; | ||
| 575 | } elsif ($data->{'date'} ge ($data_second{$symbol}->{'date'} // '')) { | ||
| 576 | $data_second{$symbol} = $data; | ||
| 577 | } | ||
| 578 | } | ||
| 579 | |||
| 580 | foreach my $symbol (keys %data_newest) { | ||
| 581 | my $newest = $data_newest{$symbol}; | ||
| 582 | my $date = $newest->{'date'}; | ||
| 583 | |||
| 584 | # For symbols in the database, if newest daily is >= latest quote then | ||
| 585 | # delete that quote in order to prefer the daily data in the database. | ||
| 586 | if (exists $database_symbols_hash->{$symbol}) { | ||
| 587 | my $latest_delete_sth = $dbh->prepare_cached | ||
| 588 | ('DELETE FROM latest WHERE symbol=? AND quote_date < ?'); | ||
| 589 | if ($latest_delete_sth->execute ($symbol, $date)) { | ||
| 590 | $latest_changed{$symbol} = 1; | ||
| 591 | } | ||
| 592 | next; | ||
| 593 | } | ||
| 594 | |||
| 595 | # For symbols not in the database, if the newest daily is >= latest quote | ||
| 596 | # then replace that quote with the daily. | ||
| 597 | # | ||
| 598 | # Times in the latest record are not considered, so it's possible a | ||
| 599 | # quote taken after close of trading will be deleted or overwritten. | ||
| 600 | # Would want something in the latest to say it's after the close ... | ||
| 601 | |||
| 602 | my $latest_get_sth = $dbh->prepare_cached | ||
| 603 | ('SELECT last_date, name FROM latest WHERE symbol=?'); | ||
| 604 | my ($last_date, $name, $dividend) | ||
| 605 | = $dbh->selectrow_array ($latest_get_sth, undef, $symbol); | ||
| 606 | if (defined $last_date && $last_date gt $date) { next; } | ||
| 607 | |||
| 608 | # "name" from the daily, or retain name from existing latest record. | ||
| 609 | $name = $h->{'names'}->{$symbol} // $name; | ||
| 610 | |||
| 611 | # "dividend" from existing latest record retained, but only if same date. | ||
| 612 | unless (defined $last_date && $last_date eq $date) { | ||
| 613 | undef $dividend; | ||
| 614 | } | ||
| 615 | |||
| 616 | # change by difference from second newest daily, if have one | ||
| 617 | my $change = undef; | ||
| 618 | if (defined(my $second_close = $data_second{$symbol}->{'close'})) { | ||
| 619 | $change = App::Chart::decimal_sub ($newest->{'close'}, $second_close); | ||
| 620 | } | ||
| 621 | |||
| 622 | $timestamp ||= timestamp_now(); | ||
| 623 | |||
| 624 | my $latest_set_sth = $dbh->prepare_cached | ||
| 625 | ('INSERT OR REPLACE INTO latest | ||
| 626 | (symbol, name, currency, exchange, dividend, | ||
| 627 | last_date, open, high, low, last, change, volume, | ||
| 628 | source, fetch_timestamp) | ||
| 629 | VALUES (?,?,?,?,?, ?,?,?,?,?,?,?, ?,?)'); | ||
| 630 | $latest_set_sth->execute | ||
| 631 | ($symbol, | ||
| 632 | $name, | ||
| 633 | $h->{'currencies'}->{$symbol}, | ||
| 634 | $h->{'exchanges'}->{$symbol}, | ||
| 635 | $dividend, | ||
| 636 | # | ||
| 637 | $newest->{'date'}, | ||
| 638 | $newest->{'open'}, | ||
| 639 | $newest->{'high'}, | ||
| 640 | $newest->{'low'}, | ||
| 641 | $newest->{'close'}, | ||
| 642 | $change, | ||
| 643 | $newest->{'volume'}, | ||
| 644 | # | ||
| 645 | $h->{'source'}, | ||
| 646 | $timestamp); | ||
| 647 | $latest_changed{$symbol} = 1; | ||
| 648 | } | ||
| 649 | |||
| 650 | foreach my $dividend (@{$h->{'dividends'}}) { | ||
| 651 | my $latest_dividend_sth = $dbh->prepare_cached | ||
| 652 | ('UPDATE latest SET dividend=? WHERE symbol=? AND last_date=?'); | ||
| 653 | my $symbol = $dividend->{'symbol'}; | ||
| 654 | if ($latest_dividend_sth->execute ($dividend->{'amount'}, | ||
| 655 | $symbol, | ||
| 656 | $dividend->{'ex_date'})) { | ||
| 657 | $latest_changed{$symbol} = 1; | ||
| 658 | } | ||
| 659 | } | ||
| 660 | |||
| 661 | App::Chart::chart_dirbroadcast()->send ('latest-changed', \%latest_changed); | ||
| 662 | |||
| 663 | require App::Chart::Annotation; | ||
| 664 | foreach my $symbol (keys %latest_changed) { | ||
| 665 | App::Chart::Annotation::Alert::update_alert ($symbol); | ||
| 666 | } | ||
| 667 | } | ||
| 668 | |||
| 669 | |||
| 670 | #------------------------------------------------------------------------------ | ||
| 671 | |||
| 672 | sub write_latest_group { | ||
| 673 | my ($h) = @_; | ||
| 674 | ### write_latest_group(): $h | ||
| 675 | |||
| 676 | crunch_h ($h); | ||
| 677 | ### crunched: $h | ||
| 678 | |||
| 679 | my $fetch_timestamp = timestamp_now(); | ||
| 680 | my $prefer_decimals = $h->{'prefer_decimals'}; | ||
| 681 | my $source = $h->{'source'} | ||
| 682 | or croak 'missing "source" for latest records'; | ||
| 683 | my %latest; | ||
| 684 | |||
| 685 | my $dbh = App::Chart::DBI->instance; | ||
| 686 | App::Chart::Database::call_with_transaction | ||
| 687 | ($dbh, sub { | ||
| 688 | |||
| 689 | my $sth = $dbh->prepare_cached | ||
| 690 | ('INSERT OR REPLACE INTO latest | ||
| 691 | (symbol, name, month, exchange, currency, | ||
| 692 | quote_date, quote_time, bid, offer, | ||
| 693 | last_date, last_time, open, high, low, last, change, volume, | ||
| 694 | note, error, dividend, copyright, source, | ||
| 695 | fetch_timestamp, url, etag, last_modified) | ||
| 696 | VALUES (?,?,?,?,?, ?,?,?,?, ?,?,?,?,?,?,?,?, ?,?,?,?,?, ?,?,?,?)'); | ||
| 697 | |||
| 698 | my $resp = $h->{'resp'}; | ||
| 699 | my $etag = (defined $resp ? scalar $resp->header('ETag') : undef); | ||
| 700 | my $last_modified = (defined $resp ? $resp->last_modified : undef); | ||
| 701 | |||
| 702 | foreach my $data (@{$h->{'data'}}) { | ||
| 703 | my $symbol = $data->{'symbol'}; | ||
| 704 | my $this_date = $data->{'date'}; | ||
| 705 | if ($latest{$symbol}) { | ||
| 706 | my $got_date = $latest{$symbol}->{'date'}; | ||
| 707 | if (! defined $got_date || ! defined $this_date) { | ||
| 708 | carp "write_latest_group: $source: two records for '$symbol', but no 'date' field"; | ||
| 709 | if (DEBUG || 1) { | ||
| 710 | require Data::Dumper; | ||
| 711 | print Data::Dumper->Dump([$data,$latest{$symbol}], | ||
| 712 | ['data','latest-so-far']); | ||
| 713 | } | ||
| 714 | next; | ||
| 715 | } | ||
| 716 | if ($got_date ge $this_date) { next; } | ||
| 717 | } | ||
| 718 | $latest{$symbol} = $data; | ||
| 719 | } | ||
| 720 | |||
| 721 | my $error = $h->{'error'}; | ||
| 722 | if (! defined $error && defined $resp && ! $resp->is_success) { | ||
| 723 | $error = $resp->status_line; | ||
| 724 | } | ||
| 725 | |||
| 726 | foreach my $data (values %latest) { | ||
| 727 | my $symbol = $data->{'symbol'}; | ||
| 728 | |||
| 729 | my $bid = $data->{'bid'}; | ||
| 730 | my $offer = $data->{'offer'}; | ||
| 731 | |||
| 732 | # disallow 0 for prices | ||
| 733 | if ($bid && $bid == 0) { $bid = undef; } | ||
| 734 | if ($offer && $offer == 0) { $offer = undef; } | ||
| 735 | |||
| 736 | my $quote_date = crunch_date ($data->{'quote_date'}); | ||
| 737 | my $quote_time = crunch_time ($data->{'quote_time'}); | ||
| 738 | if ($quote_time && ! $quote_date) { | ||
| 739 | croak "quote_time without quote_date for $symbol"; | ||
| 740 | } | ||
| 741 | # default quote date/time to now | ||
| 742 | if (($bid || $offer) && ! $quote_date) { | ||
| 743 | my $symbol_timezone = App::Chart::TZ->for_symbol ($symbol); | ||
| 744 | ($quote_date, $quote_time) | ||
| 745 | = $symbol_timezone->iso_date_time | ||
| 746 | (time() - 60 * ($data->{'quote_delay_minutes'} || 0)); | ||
| 747 | } | ||
| 748 | |||
| 749 | my $last_date = crunch_date ($data->{'last_date'} || $data->{'date'}); | ||
| 750 | my $last_time = crunch_time ($data->{'last_time'}); | ||
| 751 | |||
| 752 | my $open = $data->{'open'}; | ||
| 753 | my $high = $data->{'high'}; | ||
| 754 | my $low = $data->{'low'}; | ||
| 755 | my $last = $data->{'last'} || $data->{'close'}; | ||
| 756 | my $change = $data->{'change'}; | ||
| 757 | my $prev = crunch_price ($data->{'prev'}, $prefer_decimals); | ||
| 758 | my $volume = $data->{'volume'}; | ||
| 759 | |||
| 760 | if (! defined $last) { | ||
| 761 | # if there's no last price then try to use the prev | ||
| 762 | $open = $high = $low = undef; | ||
| 763 | $last = $prev; | ||
| 764 | $prev = undef; | ||
| 765 | $change = undef; | ||
| 766 | $last_date = undef; | ||
| 767 | $last_time = undef; | ||
| 768 | |||
| 769 | } elsif (! defined $change) { | ||
| 770 | # if no change given then try to calculate it from last and prev | ||
| 771 | |||
| 772 | if (! defined $prev) { | ||
| 773 | # if no prev then look for one among other $data records, as for | ||
| 774 | # when the group is a few consecutive daily data | ||
| 775 | my $prev_date; | ||
| 776 | foreach my $data (@{$h->{'data'}}) { | ||
| 777 | if ($data->{'symbol'} eq $symbol | ||
| 778 | && exists $data->{'date'} | ||
| 779 | && $data->{'date'} lt $last_date | ||
| 780 | && (! $prev_date | ||
| 781 | || $data->{'date'} gt $prev_date)) { | ||
| 782 | $prev_date = $data->{'date'}; | ||
| 783 | $prev = $data->{'close'}; | ||
| 784 | } | ||
| 785 | } | ||
| 786 | } | ||
| 787 | if ($prev) { | ||
| 788 | $change = App::Chart::decimal_sub ($last, $prev); | ||
| 789 | } | ||
| 790 | } | ||
| 791 | |||
| 792 | $sth->execute ($symbol, | ||
| 793 | $h->{'names'}->{$symbol}, | ||
| 794 | $data->{'month'}, | ||
| 795 | $h->{'exchanges'}->{$symbol}, | ||
| 796 | $h->{'currencies'}->{$symbol}, | ||
| 797 | |||
| 798 | $quote_date, $quote_time, | ||
| 799 | $bid, $offer, | ||
| 800 | |||
| 801 | $last_date, $last_time, | ||
| 802 | $open, $high, $low, $last, $change, $volume, | ||
| 803 | |||
| 804 | $data->{'note'}, | ||
| 805 | $data->{'error'} || $error, | ||
| 806 | $data->{'dividend'}, | ||
| 807 | $h->{'copyright'}, | ||
| 808 | $source, | ||
| 809 | |||
| 810 | $fetch_timestamp, | ||
| 811 | $h->{'url'}, | ||
| 812 | $etag, | ||
| 813 | $last_modified); | ||
| 814 | $sth->finish; | ||
| 815 | } | ||
| 816 | }); | ||
| 817 | |||
| 818 | App::Chart::chart_dirbroadcast()->send ('latest-changed', \%latest); | ||
| 819 | |||
| 820 | require App::Chart::Annotation; | ||
| 821 | foreach my $symbol (keys %latest) { | ||
| 822 | App::Chart::Annotation::Alert::update_alert ($symbol); | ||
| 823 | } | ||
| 824 | } | ||
| 825 | |||
| 826 | sub iso_to_MMM_YY { | ||
| 827 | my ($iso) = @_; | ||
| 828 | my ($year, $month, $day) = App::Chart::iso_to_ymd ($iso); | ||
| 829 | return sprintf ("%.3s %02d", | ||
| 830 | uc(Date::Calc::Month_to_Text ($month)), | ||
| 831 | $year % 100); | ||
| 832 | } | ||
| 833 | |||
| 834 | #------------------------------------------------------------------------------ | ||
| 835 | |||
| 836 | my $iso_date_re = qr/^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]$/; | ||
| 837 | |||
| 838 | my %date_format_to_func | ||
| 839 | = ('ymd' => \&App::Chart::Download::Decode_Date_YMD_to_iso, | ||
| 840 | 'dmy' => \&App::Chart::Download::Decode_Date_EU_to_iso, | ||
| 841 | 'mdy' => \&App::Chart::Download::Decode_Date_US_to_iso); | ||
| 842 | sub noop { | ||
| 843 | return $_[0]; | ||
| 844 | } | ||
| 845 | |||
| 846 | my %date_format_to_month_func | ||
| 847 | = ('ymd' => \&crunch_month_ymd); | ||
| 848 | |||
| 849 | sub crunch_month_ymd { | ||
| 850 | my ($str) = @_; | ||
| 851 | my ($year, $month, $day) = Date::Calc::Decode_Date_EU ($str); | ||
| 852 | if (! defined $year) { | ||
| 853 | croak "unrecognised month string: $str"; | ||
| 854 | } | ||
| 855 | if (defined $day) { | ||
| 856 | if ($day != 1) { croak "month with day!=1: '$str' got $day"; } | ||
| 857 | } else { | ||
| 858 | $day = 1; | ||
| 859 | } | ||
| 860 | return App::Chart::ymd_to_iso ($year, $month, $day); | ||
| 861 | } | ||
| 862 | |||
| 863 | sub crunch_h { | ||
| 864 | my ($h) = @_; | ||
| 865 | my $database_symbols_hash = App::Chart::Database::database_symbols_hash(); | ||
| 866 | |||
| 867 | # ignore undef where expecting hashrefs in 'data' and 'dividends' arefs | ||
| 868 | foreach my $elem ('data', | ||
| 869 | 'dividends') { | ||
| 870 | my $aref = $h->{$elem} // next; | ||
| 871 | @$aref = grep { | ||
| 872 | defined $_ | ||
| 873 | # && App::Chart::Database->symbol_exists($_->{'symbol'}) | ||
| 874 | } @$aref; | ||
| 875 | } | ||
| 876 | |||
| 877 | my $suffix = $h->{'suffix'}; | ||
| 878 | my $month_format = $h->{'month_format'}; | ||
| 879 | my $prefer_decimals = $h->{'prefer_decimals'}; | ||
| 880 | |||
| 881 | my $date_format = delete $h->{'date_format'}; | ||
| 882 | my $date_func = ($date_format ? $date_format_to_func{$date_format} : \&noop) | ||
| 883 | || croak "Unrecognised date_format '$date_format'"; | ||
| 884 | # my $month_func = $date_format ? $date_format_to_month_func{$date_format} : \&noop; | ||
| 885 | |||
| 886 | my %currencies; $h->{'currencies'} = \%currencies; | ||
| 887 | my %isins; $h->{'isins'} = \%isins; | ||
| 888 | my %names; $h->{'names'} = \%names; | ||
| 889 | my %exchanges; $h->{'exchanges'} = \%exchanges; | ||
| 890 | |||
| 891 | foreach my $info (@{$h->{'info'}}) { | ||
| 892 | my $symbol = $info->{'symbol'} | ||
| 893 | or croak "write_daily_group: missing symbol in info record"; | ||
| 894 | if (! exists $database_symbols_hash->{$symbol}) { next; } | ||
| 895 | |||
| 896 | $names{$symbol} ||= delete $info->{'name'}; | ||
| 897 | $currencies{$symbol} ||= delete $info->{'currency'}; | ||
| 898 | $exchanges{$symbol} ||= delete $info->{'exchange'}; | ||
| 899 | $isins{$symbol} ||= delete $info->{'isin'}; | ||
| 900 | } | ||
| 901 | |||
| 902 | foreach my $data (@{$h->{'data'}}) { | ||
| 903 | my $month = $data->{'month'}; | ||
| 904 | if (defined $month) { | ||
| 905 | # $month = $data->{'month'} = $month_func->($month); | ||
| 906 | $month =~ $iso_date_re or croak "Bad month value '$month'"; | ||
| 907 | } | ||
| 908 | |||
| 909 | my $symbol = $data->{'symbol'}; | ||
| 910 | if (! defined $symbol) { | ||
| 911 | # symbols built from commodity + month | ||
| 912 | my $commodity = $data->{'commodity'} | ||
| 913 | // croak "neither symbol nor commodity in 'data' element"; | ||
| 914 | $month // croak "Group data: no 'month' to go with 'commodity' in data element"; | ||
| 915 | $symbol = $data->{'symbol'} | ||
| 916 | = $commodity . ' ' . iso_to_MMM_YY($month) . $suffix; | ||
| 917 | } | ||
| 918 | # if ($month_format eq 'MMM_YY') { | ||
| 919 | # $month = iso_to_MMM_YY ($month); | ||
| 920 | # } else { | ||
| 921 | # croak "unrecognised month format: $month_format"; | ||
| 922 | # } | ||
| 923 | } | ||
| 924 | |||
| 925 | if ($h->{'front_month'}) { | ||
| 926 | my %front; | ||
| 927 | foreach my $data (@{$h->{'data'}}) { | ||
| 928 | my $symbol = $data->{'symbol'}; | ||
| 929 | my $front_symbol | ||
| 930 | = App::Chart::symbol_commodity($symbol) | ||
| 931 | . App::Chart::symbol_suffix($symbol); | ||
| 932 | if (! $front{$front_symbol} | ||
| 933 | || $data->{'month'} gt $front{$front_symbol}->{'month'}) { | ||
| 934 | $front{$front_symbol} = $data; | ||
| 935 | } | ||
| 936 | } | ||
| 937 | |||
| 938 | while (my ($symbol, $data) = each %front) { | ||
| 939 | $data = { %$data }; | ||
| 940 | $data->{'symbol'} = $symbol; | ||
| 941 | push @{$h->{'data'}}, $data; | ||
| 942 | } | ||
| 943 | } | ||
| 944 | |||
| 945 | foreach my $data (@{$h->{'data'}}) { | ||
| 946 | |||
| 947 | foreach my $field ('date', 'quote_date', 'last_date') { | ||
| 948 | if (defined $data->{$field}) { | ||
| 949 | $data->{$field} = crunch_date ($data->{$field}, $date_format); | ||
| 950 | } | ||
| 951 | } | ||
| 952 | |||
| 953 | # empty volume or openint taken to be no data (as opposed to '0' for | ||
| 954 | # zero volume or int) | ||
| 955 | foreach my $field ('volume', 'openint') { | ||
| 956 | if (defined $data->{$field} && $data->{$field} eq '') { | ||
| 957 | $data->{$field} = undef; | ||
| 958 | } | ||
| 959 | } | ||
| 960 | |||
| 961 | if (my $sessions = delete $data->{'sessions'}) { | ||
| 962 | my @sessions = grep {defined} map {crunch_price($_)} @$sessions; | ||
| 963 | $data->{'open'} = $sessions[0]; | ||
| 964 | ($data->{'low'}, $data->{'high'}) = List::MoreUtils::minmax (@sessions); | ||
| 965 | $data->{'close'} = $sessions[-1]; | ||
| 966 | } | ||
| 967 | |||
| 968 | if (exists $data->{'change'}) { | ||
| 969 | $data->{'change'} = crunch_change ($data->{'change'}, $prefer_decimals); | ||
| 970 | } | ||
| 971 | foreach my $field (qw(bid offer open high low close last)) { | ||
| 972 | if (exists $data->{$field}) { | ||
| 973 | $data->{$field} = crunch_price ($data->{$field}, $prefer_decimals); | ||
| 974 | } | ||
| 975 | } | ||
| 976 | foreach my $field (qw(volume openint)) { | ||
| 977 | if (exists $data->{$field}) { | ||
| 978 | $data->{$field} = crunch_number ($data->{$field}); | ||
| 979 | } | ||
| 980 | } | ||
| 981 | |||
| 982 | my $symbol = $data->{'symbol'}; | ||
| 983 | $currencies{$symbol} ||= delete $data->{'currency'} || $h->{'currency'}; | ||
| 984 | $names{$symbol} ||= delete $data->{'name'} || $h->{'name'}; | ||
| 985 | $exchanges{$symbol} ||= delete $data->{'exchange'}; | ||
| 986 | $isins{$symbol} ||= delete $data->{'isin'}; | ||
| 987 | } | ||
| 988 | |||
| 989 | foreach my $dividend (@{$h->{'dividends'}}) { | ||
| 990 | foreach my $field ('ex_date', 'record_date', 'pay_date') { | ||
| 991 | if (defined $dividend->{$field}) { | ||
| 992 | $dividend->{$field} = crunch_date ($dividend->{$field}, $date_format); | ||
| 993 | } | ||
| 994 | } | ||
| 995 | $dividend->{'ex_date'} | ||
| 996 | or croak 'Group data: missing ex_date in dividend record'; | ||
| 997 | |||
| 998 | foreach my $field (qw(amount imputation)) { | ||
| 999 | if (exists $dividend->{$field}) { | ||
| 1000 | $dividend->{$field} = crunch_price ($dividend->{$field}, $prefer_decimals); | ||
| 1001 | } | ||
| 1002 | } | ||
| 1003 | } | ||
| 1004 | |||
| 1005 | foreach my $split (@{$h->{'splits'}}) { | ||
| 1006 | if (defined $split->{'date'}) { | ||
| 1007 | $split->{'date'} = crunch_date ($split->{'date'}, $date_format); | ||
| 1008 | } | ||
| 1009 | $split->{'date'} | ||
| 1010 | or croak "Group data: missing 'date' in split record"; | ||
| 1011 | } | ||
| 1012 | |||
| 1013 | # whitespace in names, and possible leading/trailing in isins | ||
| 1014 | foreach my $href (\%names, \%currencies, \%exchanges, \%isins) { | ||
| 1015 | hash_delete_undefs ($href); | ||
| 1016 | } | ||
| 1017 | foreach (values %names, values %isins) { | ||
| 1018 | $_ = App::Chart::collapse_whitespace ($_); | ||
| 1019 | } | ||
| 1020 | |||
| 1021 | if (eval { require Business::ISIN; 1 }) { | ||
| 1022 | my $bi = Business::ISIN->new; | ||
| 1023 | while (my ($symbol, $isin) = each %isins) { | ||
| 1024 | $bi->set ($isin); | ||
| 1025 | if ($bi->is_valid) { next; } | ||
| 1026 | warn "$symbol ISIN '$isin' is invalid, ignoring: ", $bi->error; | ||
| 1027 | delete $isins{$symbol}; | ||
| 1028 | } | ||
| 1029 | } | ||
| 1030 | } | ||
| 1031 | |||
| 1032 | sub hash_delete_undefs { | ||
| 1033 | my ($href) = @_; | ||
| 1034 | while (my ($key, $value) = each %$href) { | ||
| 1035 | if (! defined $value) { | ||
| 1036 | delete $href->{$key}; | ||
| 1037 | } | ||
| 1038 | } | ||
| 1039 | } | ||
| 1040 | |||
| 1041 | sub crunch_date { | ||
| 1042 | my ($str, $format) = @_; | ||
| 1043 | $str = crunch_empty_undef ($str); | ||
| 1044 | if (! defined $str) { return $str; } | ||
| 1045 | |||
| 1046 | if (defined $format) { | ||
| 1047 | my $func = $date_format_to_func{$format}; | ||
| 1048 | defined $func or croak "Unrecognised date_format spec: $format"; | ||
| 1049 | $str = &$func ($str); | ||
| 1050 | } | ||
| 1051 | $str =~ $iso_date_re or croak "Bad date '$str'"; | ||
| 1052 | return $str; | ||
| 1053 | } | ||
| 1054 | |||
| 1055 | sub crunch_price { | ||
| 1056 | my ($str, $prefer_decimals) = @_; | ||
| 1057 | $str = crunch_change ($str, $prefer_decimals) // return undef; | ||
| 1058 | |||
| 1059 | if (str_is_zero ($str)) { return undef; } | ||
| 1060 | return $str; | ||
| 1061 | } | ||
| 1062 | sub crunch_change { | ||
| 1063 | my ($str, $prefer_decimals) = @_; | ||
| 1064 | $str = crunch_number ($str) // return undef; | ||
| 1065 | |||
| 1066 | if ($str eq '') { return undef; } # empty | ||
| 1067 | if (uc($str) eq 'CLOSED') { return undef; } # RBA 2003-2006.xls | ||
| 1068 | if ($str eq 'unch') { return '0'; } # unchanged | ||
| 1069 | if (defined $prefer_decimals) { | ||
| 1070 | return App::Chart::Download::trim_decimals ($str, $prefer_decimals); | ||
| 1071 | } else { | ||
| 1072 | return $str; | ||
| 1073 | } | ||
| 1074 | } | ||
| 1075 | |||
| 1076 | sub crunch_number { | ||
| 1077 | my ($str) = @_; | ||
| 1078 | if (! defined $str) { return undef; } | ||
| 1079 | |||
| 1080 | $str =~ s/$RE{ws}{crop}//go; # leading and trailing whitespace | ||
| 1081 | $str =~ s/^\+//; # leading + sign | ||
| 1082 | $str =~ s/^0+([1-9]|0\.|0$)/$1/; # leading extra zeros | ||
| 1083 | |||
| 1084 | if ($str eq '' | ||
| 1085 | || $str eq '-' | ||
| 1086 | || $str eq 'N/A' | ||
| 1087 | || $str eq 'n/a') { | ||
| 1088 | return undef; | ||
| 1089 | } | ||
| 1090 | $str =~ s/,//g; # commas for thousands | ||
| 1091 | return $str; | ||
| 1092 | } | ||
| 1093 | |||
| 1094 | sub crunch_time { | ||
| 1095 | my ($str) = @_; | ||
| 1096 | $str = crunch_empty_undef ($str); | ||
| 1097 | if (! defined $str) { return undef; } | ||
| 1098 | |||
| 1099 | $str =~ /^([0-9]?[0-9])(:[0-9][0-9])(:[0-9][0-9])?([ap]m)?$/i | ||
| 1100 | or croak "bad time '$str'"; | ||
| 1101 | my $hour = $1; | ||
| 1102 | my $minute = $2; | ||
| 1103 | my $second = $3; | ||
| 1104 | my $am_pm = $4; | ||
| 1105 | $second //= ':00'; | ||
| 1106 | if (defined $am_pm && lc $am_pm eq 'pm') { $hour += 12; } | ||
| 1107 | $hour = sprintf '%02d', $hour; | ||
| 1108 | return "$hour$minute$second"; | ||
| 1109 | } | ||
| 1110 | |||
| 1111 | sub crunch_empty_undef { | ||
| 1112 | my ($str) = @_; | ||
| 1113 | if (! defined $str) { return $str; } | ||
| 1114 | $str =~ s/$RE{ws}{crop}//go; # leading and trailing whitespace | ||
| 1115 | |||
| 1116 | # eg. string "N/A" in dates and times from Yahoo | ||
| 1117 | if ($str eq '' || $str eq 'N/A') { return undef; } | ||
| 1118 | return $str; | ||
| 1119 | } | ||
| 1120 | |||
| 1121 | |||
| 1122 | #------------------------------------------------------------------------------ | ||
| 1123 | |||
| 1124 | sub set_symbol_name { | ||
| 1125 | my ($symbol, $name) = @_; | ||
| 1126 | my $dbh = App::Chart::DBI->instance; | ||
| 1127 | my $sth = $dbh->prepare_cached('UPDATE info SET name=? WHERE symbol=?'); | ||
| 1128 | my $changed = $sth->execute($name, $symbol); | ||
| 1129 | $sth->finish(); | ||
| 1130 | return $changed; | ||
| 1131 | } | ||
| 1132 | sub set_currency { | ||
| 1133 | my ($symbol, $currency) = @_; | ||
| 1134 | my $dbh = App::Chart::DBI->instance; | ||
| 1135 | my $sth = $dbh->prepare_cached('UPDATE info SET currency=? WHERE symbol=?'); | ||
| 1136 | my $changed = $sth->execute($currency, $symbol); | ||
| 1137 | $sth->finish(); | ||
| 1138 | return $changed; | ||
| 1139 | } | ||
| 1140 | sub set_exchange { | ||
| 1141 | my ($symbol, $exchange) = @_; | ||
| 1142 | my $dbh = App::Chart::DBI->instance; | ||
| 1143 | my $sth = $dbh->prepare_cached('UPDATE info SET exchange=? WHERE symbol=?'); | ||
| 1144 | my $changed = $sth->execute($exchange, $symbol); | ||
| 1145 | $sth->finish(); | ||
| 1146 | return $changed; | ||
| 1147 | } | ||
| 1148 | sub set_decimals { | ||
| 1149 | my ($symbol, $decimals) = @_; | ||
| 1150 | my $dbh = App::Chart::DBI->instance; | ||
| 1151 | my $sth = $dbh->prepare_cached('UPDATE info SET decimals=? WHERE symbol=?'); | ||
| 1152 | my $changed = $sth->execute($decimals, $symbol); | ||
| 1153 | $sth->finish(); | ||
| 1154 | return $changed; | ||
| 1155 | } | ||
| 1156 | sub set_isin { | ||
| 1157 | my ($symbol, $isin) = @_; | ||
| 1158 | my $dbh = App::Chart::DBI->instance; | ||
| 1159 | my $sth = $dbh->prepare_cached('UPDATE info SET isin=? WHERE symbol=?'); | ||
| 1160 | my $changed = $sth->execute($isin, $symbol); | ||
| 1161 | $sth->finish(); | ||
| 1162 | return $changed; | ||
| 1163 | } | ||
| 1164 | |||
| 1165 | |||
| 1166 | #------------------------------------------------------------------------------ | ||
| 1167 | |||
| 1168 | # return true if $timestamp string is within the past $seconds from now | ||
| 1169 | # also return true if $timestamp is some strange future value | ||
| 1170 | sub timestamp_within { | ||
| 1171 | my ($timestamp, $seconds) = @_; | ||
| 1172 | if (! defined $timestamp) { return 0; } # undef stamp always out of range | ||
| 1173 | my ($lo, $hi) = timestamp_range ($seconds); | ||
| 1174 | return (($timestamp ge $lo) && ($timestamp le $hi)); | ||
| 1175 | } | ||
| 1176 | sub timestamp_range { | ||
| 1177 | my ($seconds) = @_; | ||
| 1178 | my $t = time(); | ||
| 1179 | my $lo = $t - $seconds; | ||
| 1180 | my $hi = $t + 6*3600; # 2 hours future | ||
| 1181 | return (timet_to_timestamp($lo), | ||
| 1182 | timet_to_timestamp($hi)); | ||
| 1183 | } | ||
| 1184 | sub timestamp_now { | ||
| 1185 | return timet_to_timestamp(time()); | ||
| 1186 | } | ||
| 1187 | sub timet_to_timestamp { | ||
| 1188 | my ($t) = @_; | ||
| 1189 | return POSIX::strftime ('%Y-%m-%d %H:%M:%S+00:00', gmtime($t)); | ||
| 1190 | } | ||
| 1191 | sub timestamp_to_ymdhms { | ||
| 1192 | my ($timestamp) = @_; | ||
| 1193 | return split /[- :+]/, $timestamp; | ||
| 1194 | } | ||
| 1195 | sub timestamp_to_timet { | ||
| 1196 | my ($timestamp) = @_; | ||
| 1197 | my ($year, $month, $day, $hour, $minute, $second) | ||
| 1198 | = timestamp_to_ymdhms($timestamp); | ||
| 1199 | require Time::Local; | ||
| 1200 | return Time::Local::timegm_modern | ||
| 1201 | ($second, $minute, $hour, $day, $month-1, $year); | ||
| 1202 | } | ||
| 1203 | |||
| 1204 | |||
| 1205 | #------------------------------------------------------------------------------ | ||
| 1206 | |||
| 1207 | sub tdate_strftime { | ||
| 1208 | my ($format, $tdate) = @_; | ||
| 1209 | my ($year, $month, $day) = App::Chart::tdate_to_ymd ($tdate); | ||
| 1210 | require App::Chart::Timebase; | ||
| 1211 | return App::Chart::Timebase::strftime_ymd ($format, $year, $month, $day); | ||
| 1212 | } | ||
| 1213 | |||
| 1214 | sub tdate_range_string { | ||
| 1215 | my ($lo, $hi) = @_; | ||
| 1216 | if (@_ < 2) { $hi = $lo; } | ||
| 1217 | my $d_fmt = $App::Chart::option{'d_fmt'}; | ||
| 1218 | if ($lo == $hi) { | ||
| 1219 | return tdate_strftime ($d_fmt, $lo); | ||
| 1220 | } else { | ||
| 1221 | return __x('{lodate} to {hidate}', | ||
| 1222 | lodate => tdate_strftime ($d_fmt, $lo), | ||
| 1223 | hidate => tdate_strftime ($d_fmt, $hi)); | ||
| 1224 | } | ||
| 1225 | } | ||
| 1226 | |||
| 1227 | sub symbol_range_string { | ||
| 1228 | my ($symbol_list) = @_; | ||
| 1229 | if (@$symbol_list == 0) { | ||
| 1230 | return ''; | ||
| 1231 | } elsif (@$symbol_list == 1) { | ||
| 1232 | return $symbol_list->[0]; | ||
| 1233 | } else { | ||
| 1234 | return __x('{start_symbol} to {end_symbol}', | ||
| 1235 | start_symbol => $symbol_list->[0], | ||
| 1236 | end_symbol => $symbol_list->[-1]); | ||
| 1237 | } | ||
| 1238 | } | ||
| 1239 | |||
| 1240 | #------------------------------------------------------------------------------ | ||
| 1241 | |||
| 1242 | sub weekday_date_after_time { | ||
| 1243 | return App::Chart::tdate_to_iso (weekday_tdate_after_time (@_)); | ||
| 1244 | } | ||
| 1245 | sub weekday_tdate_after_time { | ||
| 1246 | my ($after_hour,$after_min, $timezone, $offset) = @_; | ||
| 1247 | |||
| 1248 | local $Tie::TZ::TZ = $timezone->tz; | ||
| 1249 | my ($year,$month,$day, $hour,$min,$sec, $doy,$dow,$dst) | ||
| 1250 | = Date::Calc::Localtime(); | ||
| 1251 | |||
| 1252 | my $tdate = App::Chart::ymd_to_tdate_floor ($year,$month,$day) | ||
| 1253 | + ($offset // 0); | ||
| 1254 | |||
| 1255 | if ($dow >= 6 # Saturday or Sunday | ||
| 1256 | || ($hour*60+$min < $after_hour*60+$after_min)) { | ||
| 1257 | $tdate--; | ||
| 1258 | } | ||
| 1259 | return $tdate; | ||
| 1260 | } | ||
| 1261 | |||
| 1262 | #------------------------------------------------------------------------------ | ||
| 1263 | |||
| 1264 | sub download { | ||
| 1265 | my (%options) = @_; | ||
| 1266 | |||
| 1267 | my @symbol_list = (); | ||
| 1268 | { | ||
| 1269 | my $symbol_list = $options{'symbol_list'} | ||
| 1270 | || croak "download() missing symbol_list\n"; | ||
| 1271 | @symbol_list = @$symbol_list; | ||
| 1272 | } | ||
| 1273 | |||
| 1274 | @symbol_list = List::MoreUtils::uniq (@symbol_list); | ||
| 1275 | verbose_message (__('Download:'), @symbol_list); | ||
| 1276 | |||
| 1277 | foreach my $symbol (@symbol_list) { | ||
| 1278 | App::Chart::symbol_setups ($symbol); | ||
| 1279 | } | ||
| 1280 | App::Chart::Database->add_symbol (@symbol_list); | ||
| 1281 | |||
| 1282 | require App::Chart::DownloadHandler; | ||
| 1283 | my $all_ok = 1; | ||
| 1284 | foreach my $handler (@App::Chart::DownloadHandler::handler_list) { | ||
| 1285 | my @this_list = grep {$handler->match($_)} @symbol_list; | ||
| 1286 | my $ok = $handler->download (\@this_list); | ||
| 1287 | if (! $ok) { $all_ok = 0; } | ||
| 1288 | } | ||
| 1289 | |||
| 1290 | # my %handler_result = (); | ||
| 1291 | # foreach my $symbol (@symbol_list) { | ||
| 1292 | # | ||
| 1293 | # my @handlers = App::Chart::DownloadHandler->handlers_for_symbol ($symbol); | ||
| 1294 | # foreach my $handler (@handlers) { | ||
| 1295 | # if (exists $handler_result{$handler}) { | ||
| 1296 | # if (! $handler_result{$handler}) { $all_ok = 0; } | ||
| 1297 | # next; | ||
| 1298 | # } | ||
| 1299 | # | ||
| 1300 | # my @this_list = grep { my $symbol = $_; | ||
| 1301 | # List::MoreUtils::all | ||
| 1302 | # {$_->match($symbol)} @handlers | ||
| 1303 | # } @symbol_list; | ||
| 1304 | # my $ok = $handler->download (\@this_list); | ||
| 1305 | # $handler_result{$handler} = $ok; | ||
| 1306 | # } | ||
| 1307 | # | ||
| 1308 | # } | ||
| 1309 | status (__('Checking historical')); | ||
| 1310 | if ($all_ok) { | ||
| 1311 | consider_historical (\@symbol_list); | ||
| 1312 | } | ||
| 1313 | } | ||
| 1314 | |||
| 1315 | # return a list of symbols, either just ($symbol), or if $symbol has | ||
| 1316 | # wildcards then the result of matching that in the "all" list | ||
| 1317 | sub symbol_glob { | ||
| 1318 | my ($symbol) = @_; | ||
| 1319 | |||
| 1320 | if ($symbol =~ /[*?]/) { | ||
| 1321 | require Text::Glob; | ||
| 1322 | require App::Chart::Gtk2::Symlist::All; | ||
| 1323 | my $symlist = App::Chart::Gtk2::Symlist::All->instance; | ||
| 1324 | my $regexp = Text::Glob::glob_to_regex ($symbol); | ||
| 1325 | my @list = grep {$_ =~ $regexp} $symlist->symbols; | ||
| 1326 | if (! @list) { | ||
| 1327 | print __x("Warning, pattern \"{pattern}\" doesn't match anything in the database, ignoring\n", | ||
| 1328 | pattern => $symbol); | ||
| 1329 | } | ||
| 1330 | return @list; | ||
| 1331 | } else { | ||
| 1332 | return ($symbol); | ||
| 1333 | } | ||
| 1334 | } | ||
| 1335 | |||
| 1336 | sub command_line_download { | ||
| 1337 | my ($class, $output, $args) = @_; | ||
| 1338 | my $hash; | ||
| 1339 | |||
| 1340 | if ($output eq 'tty') { | ||
| 1341 | if (-t STDOUT) { | ||
| 1342 | binmode (STDOUT, ':via(EscStatus)') | ||
| 1343 | or die 'Cannot push EscStatus'; | ||
| 1344 | } else { | ||
| 1345 | require PerlIO::via::EscStatus::ShowNone; | ||
| 1346 | binmode (STDOUT, ':via(EscStatus::ShowNone)') | ||
| 1347 | or die 'Cannot push EscStatus::ShowNone'; | ||
| 1348 | } | ||
| 1349 | } elsif ($output eq 'all-status') { | ||
| 1350 | require PerlIO::via::EscStatus::ShowAll; | ||
| 1351 | binmode (STDOUT, ':via(EscStatus::ShowAll)') | ||
| 1352 | or die 'Cannot push EscStatus::ShowAll'; | ||
| 1353 | } | ||
| 1354 | |||
| 1355 | if (! @$args) { | ||
| 1356 | print __"No symbols specified to download\n"; | ||
| 1357 | return; | ||
| 1358 | } | ||
| 1359 | |||
| 1360 | my @symbol_list = (); | ||
| 1361 | foreach my $arg (@$args) { | ||
| 1362 | if (ref $arg) { | ||
| 1363 | # only what's already in the database | ||
| 1364 | $hash ||= App::Chart::Database::database_symbols_hash(); | ||
| 1365 | my @list = grep {exists $hash->{$_}} $arg->symbols; | ||
| 1366 | push @symbol_list, @list; | ||
| 1367 | if (! @list) { | ||
| 1368 | print __x("Warning, no symbols in \"{symlist}\" are currently in the database\n(only symbols already in the database are downloaded from lists)\n", | ||
| 1369 | symlist => $arg->name); | ||
| 1370 | } | ||
| 1371 | } else { | ||
| 1372 | push @symbol_list, symbol_glob ($arg); | ||
| 1373 | } | ||
| 1374 | } | ||
| 1375 | |||
| 1376 | App::Chart::Download::download (symbol_list => \@symbol_list); | ||
| 1377 | |||
| 1378 | require App::Chart::LatestHandler; | ||
| 1379 | App::Chart::LatestHandler->download (\@symbol_list); | ||
| 1380 | |||
| 1381 | status (''); | ||
| 1382 | } | ||
| 1383 | |||
| 1384 | #------------------------------------------------------------------------------ | ||
| 1385 | |||
| 1386 | sub iso_to_tdate_floor { | ||
| 1387 | my ($str) = @_; | ||
| 1388 | my ($year, $month, $day) = App::Chart::iso_to_ymd ($str); | ||
| 1389 | return App::Chart::ymd_to_tdate_floor ($year, $month, $day); | ||
| 1390 | } | ||
| 1391 | |||
| 1392 | sub iso_to_tdate_ceil { | ||
| 1393 | my ($str) = @_; | ||
| 1394 | my ($year, $month, $day) = App::Chart::iso_to_ymd ($str); | ||
| 1395 | return App::Chart::ymd_to_tdate_ceil ($year, $month, $day); | ||
| 1396 | } | ||
| 1397 | |||
| 1398 | sub tdate_today { | ||
| 1399 | my ($timezone) = @_; | ||
| 1400 | $timezone //= App::Chart::TZ->loco; | ||
| 1401 | my ($year, $month, $day) = $timezone->ymd; | ||
| 1402 | return App::Chart::ymd_to_tdate_floor ($year, $month, $day); | ||
| 1403 | } | ||
| 1404 | |||
| 1405 | my $default_download_tdates = 5 * 265; # 5 years | ||
| 1406 | |||
| 1407 | sub start_tdate_for_update { | ||
| 1408 | my (@symbol_list) = @_; | ||
| 1409 | if (! @symbol_list) { croak "start_tdate_for_update(): no symbols"; } | ||
| 1410 | my $ret; | ||
| 1411 | foreach my $symbol (@symbol_list) { | ||
| 1412 | my $iso = App::Chart::DBI->read_single | ||
| 1413 | ('SELECT date FROM daily WHERE symbol=? ORDER BY date DESC LIMIT 1', | ||
| 1414 | $symbol); | ||
| 1415 | if (! defined $iso) { | ||
| 1416 | return (tdate_today() - $default_download_tdates); | ||
| 1417 | } | ||
| 1418 | my $tdate = iso_to_tdate_floor ($iso) + 1; | ||
| 1419 | $ret = App::Chart::min_maybe ($ret, $tdate); | ||
| 1420 | } | ||
| 1421 | return $ret; | ||
| 1422 | } | ||
| 1423 | |||
| 1424 | sub tdate_today_after { | ||
| 1425 | my ($after_hour, $after_minute, $timezone) = @_; | ||
| 1426 | |||
| 1427 | { local $Tie::TZ::TZ = $timezone->tz; | ||
| 1428 | my ($year,$month,$day, $hour,$min,$sec, $doy,$dow,$dst) = | ||
| 1429 | Date::Calc::System_Clock(); | ||
| 1430 | |||
| 1431 | my $tdate = App::Chart::ymd_to_tdate_floor ($year, $month, $day); | ||
| 1432 | if ($dow <= 5 # is a weekday | ||
| 1433 | && (App::Chart::hms_to_seconds ($hour, $min, 0) | ||
| 1434 | < App::Chart::hms_to_seconds ($after_hour, $after_minute, 0))) { | ||
| 1435 | $tdate--; | ||
| 1436 | } | ||
| 1437 | return $tdate; | ||
| 1438 | } | ||
| 1439 | } | ||
| 1440 | |||
| 1441 | |||
| 1442 | #----------------------------------------------------------------------------- | ||
| 1443 | # selecting among possibly overlapping files | ||
| 1444 | |||
| 1445 | # $files is an arrayref containing hash records with keys | ||
| 1446 | # | ||
| 1447 | # lo_tdate,hi_tdate inclusive coverage of the record | ||
| 1448 | # lo_year,hi_year alterative form for date range | ||
| 1449 | # cost size of the file in bytes | ||
| 1450 | # | ||
| 1451 | sub choose_files { | ||
| 1452 | my ($files, $lo_tdate, $hi_tdate) = @_; | ||
| 1453 | if ($lo_tdate > $hi_tdate) { return []; } | ||
| 1454 | |||
| 1455 | if (DEBUG) { print "choose_files $lo_tdate to $hi_tdate\n"; } | ||
| 1456 | |||
| 1457 | foreach my $f (@$files) { | ||
| 1458 | if (! defined $f->{'lo_tdate'}) { | ||
| 1459 | if (my $m = $f->{'month_iso'}) { | ||
| 1460 | $f->{'lo_tdate'} = App::Chart::Download::iso_to_tdate_ceil ($m); | ||
| 1461 | } elsif ($f->{'lo_year'}) { | ||
| 1462 | $f->{'lo_tdate'} | ||
| 1463 | = App::Chart::ymd_to_tdate_ceil ($f->{'lo_year'}, 1, 1); | ||
| 1464 | } else { | ||
| 1465 | croak 'choose_files: missing lo date'; | ||
| 1466 | } | ||
| 1467 | } | ||
| 1468 | |||
| 1469 | if (! defined $f->{'hi_tdate'}) { | ||
| 1470 | if (my $m = $f->{'month_iso'}) { | ||
| 1471 | $f->{'hi_tdate'} | ||
| 1472 | = tdate_end_of_month (App::Chart::Download::iso_to_tdate_ceil ($m)); | ||
| 1473 | } elsif ($f->{'hi_year'}) { | ||
| 1474 | $f->{'hi_tdate'} | ||
| 1475 | = App::Chart::ymd_to_tdate_floor($f->{'hi_year'}, 12, 31); | ||
| 1476 | } else { | ||
| 1477 | croak 'choose_files: missing hi date'; | ||
| 1478 | } | ||
| 1479 | } | ||
| 1480 | } | ||
| 1481 | if (DEBUG >= 2) { require Data::Dumper; | ||
| 1482 | print Data::Dumper::Dumper($files); } | ||
| 1483 | |||
| 1484 | # restrict wanted range to what's available | ||
| 1485 | my $lo_available = min (map {$_->{'lo_tdate'}} @$files); | ||
| 1486 | my $hi_available = max (map {$_->{'hi_tdate'}} @$files); | ||
| 1487 | $lo_tdate = max ($lo_tdate, $lo_available); | ||
| 1488 | $hi_tdate = min ($hi_tdate, $hi_available); | ||
| 1489 | if (DEBUG) { print " available $lo_available to $hi_available\n"; | ||
| 1490 | print " restricted range $lo_tdate to $hi_tdate\n"; } | ||
| 1491 | if ($lo_tdate > $hi_tdate) { return []; } | ||
| 1492 | |||
| 1493 | # ignore file elements not covering any of the desired range | ||
| 1494 | $files = [ grep {App::Chart::overlap_inclusive_p ($lo_tdate, $hi_tdate, | ||
| 1495 | $_->{'lo_tdate'}, | ||
| 1496 | $_->{'hi_tdate'})} | ||
| 1497 | @$files ]; | ||
| 1498 | |||
| 1499 | # Algorithm::ChooseSubsets would be another way to iterate, or | ||
| 1500 | # Math::Subset::List to get all combinations | ||
| 1501 | my $best_cost; | ||
| 1502 | my $best_files; | ||
| 1503 | foreach my $this_files (all_combinations ($files)) { | ||
| 1504 | if (! cover_p ($this_files, $lo_tdate, $hi_tdate)) { next; } | ||
| 1505 | my $cost = List::Util::sum (map {$_->{'cost'}||0} @$this_files); | ||
| 1506 | $cost += $App::Chart::option{'http_get_cost'} * scalar(@$this_files); | ||
| 1507 | if (! defined $best_cost || $cost < $best_cost) { | ||
| 1508 | $best_cost = $cost; | ||
| 1509 | $best_files = $this_files; | ||
| 1510 | } | ||
| 1511 | } | ||
| 1512 | return $best_files; | ||
| 1513 | } | ||
| 1514 | |||
| 1515 | # return true if the set of file records in arrayref $files covers all of | ||
| 1516 | # $lo_tdate through $hi_tdate inclusive | ||
| 1517 | # | ||
| 1518 | sub cover_p { | ||
| 1519 | my ($files, $lo_tdate, $hi_tdate) = @_; | ||
| 1520 | require Set::IntSpan::Fast; | ||
| 1521 | my $set = Set::IntSpan::Fast->new; | ||
| 1522 | foreach my $f (@$files) { | ||
| 1523 | $set->add_range ($f->{'lo_tdate'}, $f->{'hi_tdate'}); | ||
| 1524 | } | ||
| 1525 | $set->contains_all_range ($lo_tdate, $hi_tdate); | ||
| 1526 | } | ||
| 1527 | |||
| 1528 | # return a list which is all the combinations of elements of @$aref | ||
| 1529 | # for example $aref == [ 10, 20 ] would return ([], [10], [20], [10,20]) | ||
| 1530 | # there's 2**N combinations for aref length N | ||
| 1531 | # | ||
| 1532 | sub all_combinations { | ||
| 1533 | my ($aref) = @_; | ||
| 1534 | my @ret = ([]); | ||
| 1535 | foreach my $i (0 .. $#$aref) { | ||
| 1536 | push @ret, map {[ @$_, $aref->[$i] ]} @ret; | ||
| 1537 | } | ||
| 1538 | return @ret; | ||
| 1539 | } | ||
| 1540 | |||
| 1541 | # return the last tdate in the month containing the given $tdate | ||
| 1542 | sub tdate_end_of_month { | ||
| 1543 | my ($tdate) = @_; | ||
| 1544 | my ($year, $month, $day) = App::Chart::tdate_to_ymd ($tdate); | ||
| 1545 | ($year, $month, $day) = Date::Calc::Add_Delta_YM ($year, $month, $day, 0,1); | ||
| 1546 | $day = 1; | ||
| 1547 | ($year, $month, $day) = Date::Calc::Add_Delta_Days ($year, $month, $day, -1); | ||
| 1548 | return App::Chart::ymd_to_tdate_floor ($year, $month, $day); | ||
| 1549 | } | ||
| 1550 | |||
| 1551 | 1; | ||
| 1552 | __END__ | ||