App/Chart/Download.pm

filter: Statementstmt 94.2% 33 / 35 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 12 / 12 Podpod n/a totaltotal 95.7% 45 / 47 CC 1 SCAR 0.2
CC
1
Cov
94%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« App/Chart/Barchart.pm ↑ Summary App/Chart/DownloadHandler.pm »
1 # Download functions.
2  
3 # Copyright 2007, 2008, 2009, 2010, 2011, 2013, 2015, 2016, 2017, 2018, 2020, 2023, 2024 Kevin Ryde
4  
5 # This file is part of Chart.
6 #
7 # Chart is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 3, or (at your option) any later version.
10 #
11 # Chart is distributed in the hope that it will be useful, but WITHOUT ANY
12 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
14 # details.
15 #
16 # You should have received a copy of the GNU General Public License along
17 # with Chart. If not, see <http://www.gnu.org/licenses/>.
18  
19 package App::Chart::Download;
20 2 ▶use 5.010;
Subroutine BEGIN: called CC=1 SCAR=0.0
21 2 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
22 2 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
23 2 ▶use Carp 'carp','croak';
Subroutine BEGIN: called CC=1 SCAR=0.0
24 2 ▶use Date::Calc;
Subroutine BEGIN: called CC=1 SCAR=0.0
25 2 ▶use List::Util qw(min max);
Subroutine BEGIN: called CC=1 SCAR=0.0
26 2 ▶use List::MoreUtils;
Subroutine BEGIN: called CC=1 SCAR=0.0
27 2 ▶use Regexp::Common 'whitespace';
Subroutine BEGIN: called CC=1 SCAR=0.0
28 2 ▶use Locale::TextDomain ('App-Chart');
Subroutine BEGIN: called CC=1 SCAR=0.0
29  
30 2 ▶use PerlIO::via::EscStatus;
Subroutine BEGIN: called CC=1 SCAR=0.0
31 2 ▶use Tie::TZ;
Subroutine BEGIN: called CC=1 SCAR=0.0
32  
33 0 ▶use App::Chart;
Subroutine BEGIN: called CC=1 SCAR=2.6
34 use App::Chart::Database;
35 use App::Chart::DBI;
36 use App::Chart::TZ;
37  
38 # uncomment this to run the ### lines
39 # use Smart::Comments;
40  
41 use constant DEBUG => 0;
42  
43 #------------------------------------------------------------------------------
44  
45 sub http_code_is_allowed {
46 my ($aref, $code) = @_;
47 return (grep {$code eq $_} @{$aref // []}) > 0;
48 }
49  
50 sub get {
51 my ($class, $url, %options) = @_;
52 ### Download get(): $url
53  
54 # URI object becomes string
55 $url = "$url";
56  
57 my $ua = $options{'ua'} || do { require App::Chart::UserAgent;
58 App::Chart::UserAgent->instance };
59 $ua->cookie_jar ($options{'cookie_jar'}); # undef for none
60 ### cookie_jar object: ($options{'cookie_jar'} // "").""
61  
62 require HTTP::Request;
63 my $method = $options{'method'} || 'GET';
64 my @headers = (Referer => $options{'referer'});
65 my $data = $options{'data'};
66 if (defined $data) {
67 push @headers, 'Content-Type' => 'application/x-www-form-urlencoded';
68 }
69 my $req = HTTP::Request->new ($method, $url, \@headers, $data);
70  
71 # possible override
72 if (my $user_agent = $options{'user_agent'}) {
73 $req->user_agent($user_agent);
74 }
75  
76 my $etag = $options{'etag'};
77 my $lastmod = $options{'last_modified'};
78  
79 if (my $key = $options{'url_tags_key'}) {
80 my $symbol = $options{'symbol'};
81 my $prev_url = App::Chart::Database->read_extra($symbol,"$key-URL");
82 if (defined $prev_url && $url eq $prev_url) {
83 $etag = App::Chart::Database->read_extra($symbol,"$key-ETag");
84 $lastmod = App::Chart::Database->read_extra($symbol,"$key-Last-Modified");
85 }
86 }
87  
88 if ($etag) { $req->header ('If-None-Match' => $etag); }
89 if ($lastmod) { $req->header ('If-Modified-Since' => $lastmod); }
90  
91 my $resp = $ua->request ($req);
92  
93 # internal message from LWP when a keep-alive has missed the boat
94 if ($resp->status_line =~ /500 Server closed connection/i) {
95 substatus (__('retry'));
96 $resp = $ua->request ($req);
97 }
98  
99 my $code = $resp->code;
100 if ($resp->is_success
101 || ($options{'allow_401'} && $code == 401)
102 || ($options{'allow_404'} && $code == 404)
103 || (($etag || $lastmod) && $code == 304)
104 || http_code_is_allowed($options{'allow_http_codes'}, $code)) {
105 substatus (__('processing'));
106 return $resp;
107 } else {
108 my $error = $resp->status_line . "\n";
109 if ($code == 500) {
110 $error .= $resp->decoded_content;
111 unless ($error =~ /\n$/s) { $error .= "\n"; }
112 }
113 croak "Cannot download $url\n" . $error;
114 }
115 }
116  
117 #------------------------------------------------------------------------------
118  
119 my $last_status = ''; # without substatus addition
120  
121 sub download_message {
122 print join (' ',@_),"\n";
123 }
124 sub verbose_message {
125 if ($App::Chart::option{'verbose'}) {
126 print join (' ',@_),"\n";
127 }
128 }
129  
130 sub status {
131 my $str = join (' ', @_);
132 $last_status = $str;
133 PerlIO::via::EscStatus::print_status ($str);
134 }
135 sub substatus {
136 my ($str) = @_;
137 if ($str) {
138 PerlIO::via::EscStatus::print_status ($last_status, ' [', $str, ']');
139 }
140 }
141  
142 #------------------------------------------------------------------------------
143  
144 sub split_lines {
145 my ($str) = @_;
146 my @lines = split (/[\r\n]+/, $str); # LF or CRLF
147 foreach (@lines) { $_ =~ s/[ \t]+$// } # trailing whitespace
148 return grep {$_ ne ''} @lines; # no blanks
149 }
150  
151 #------------------------------------------------------------------------------
152  
153 # Return $str with trailing "0"s removed.
154 # Don't go below $want_deicmals digits after the decimal point.
155 sub trim_decimals {
156 my ($str, $want_decimals) = @_;
157 if ($str && $str =~ /(.*\.[0-9]{$want_decimals}[0-9]*?)0+$/) {
158 return $1;
159 } else {
160 return $str;
161 }
162 }
163  
164 #------------------------------------------------------------------------------
165  
166 sub str_is_zero {
167 my ($str) = @_;
168 return ($str =~ /^0+(\.0*)?$|^0*(\.0+)$/ ? 1 : 0);
169 }
170  
171  
172 #------------------------------------------------------------------------------
173  
174 sub cents_to_dollars {
175 my ($str) = @_;
176 $str =~ /^([^.]*)(\.(.*))?$/
177 or croak "cents_to_dollars(): bad string: \"$str\"";
178 my $int = $1;
179 my $frac = (defined $3 ? $3 : '');
180 if (length ($int) < 3) {
181 $int = sprintf ('%03s', $int);
182 }
183 return substr ($int, 0, length($int)-2) . '.' .
184 substr ($int, length($int)-2) . $frac;
185 }
186  
187  
188 #------------------------------------------------------------------------------
189  
190 sub month_to_nearest_year {
191 my ($target_month) = @_;
192 my ($year, $month, $day) = Date::Calc::Today();
193 $month --; # 0=January
194 $target_month --;
195  
196 my $diff = $target_month - $month;
197 $diff += 5;
198 $diff %= 12;
199 $diff -= 5; # now range -5 to +6
200  
201 # applying $diff makes $month == $target_month modulo 12,
202 # but $month<0 is last year, 0 to 11 this year, >= 12 next year
203 $month += $diff;
204 return $year + ($month < 0 ? -1 : $month < 12 ? 0 : 1);
205 }
206  
207  
208 #------------------------------------------------------------------------------
209  
210 sub Decode_Date_EU_to_iso {
211 my ($str, $noerror) = @_;
212 my ($year, $month, $day) = Date::Calc::Decode_Date_EU ($str);
213 unless (defined $year && defined $month && defined $day) {
214 if ($noerror) {
215 return undef;
216 } else {
217 croak "Decode_Date_EU_to_iso: unrecognised date \"$str\"\n";
218 }
219 }
220 return App::Chart::ymd_to_iso ($year, $month, $day);
221 }
222  
223 sub Decode_Date_US_to_iso {
224 my ($str) = @_;
225 my ($year, $month, $day) = Date::Calc::Decode_Date_US ($str);
226 unless (defined $year && defined $month && defined $day) {
227 croak "Decode_Date_US_to_iso: unrecognised date \"$str\"\n";
228 }
229 return App::Chart::ymd_to_iso ($year, $month, $day);
230 }
231  
232 #------------------------------------------------------------------------------
233  
234 sub Decode_Date_YMD {
235 my ($str) = @_;
236 ($str =~ m{^ # 6 or 8 digits are yyyymmdd or yymmdd
237 ["'[:space:]]*
238 (\d{2,4}) # $1 year
239 ((\d{2})|([A-Za-z]+)) # $3 numeric month, $4 alpha month
240 (\d{2}) # $5 day
241 ["'[:space:]]*
242 $}x)
243 or
244 ($str =~ m{^
245 ["'[:space:]]*
246 (\d{2,4}) # $1 year
247 [-_/:.[:space:]]*
248 ((\d{1,2})|([A-Za-z]+)) # $3 numeric month, $4 alpha month
249 [-_/:.[:space:]]*
250 (\d{1,2}) # $5 day
251 ["'[:space:]]*
252 $}x)
253 or return;
254  
255 my $year = $1;
256 my $num_month = $3;
257 my $alpha_month = $4,
258 my $day = $5;
259 my $month = $num_month || Date::Calc::Decode_Month ($alpha_month);
260 $year = Date::Calc::Moving_Window ($year);
261 return ($year, $month, $day);
262 }
263  
264 sub Decode_Date_YMD_to_iso {
265 my ($str) = @_;
266 my ($year, $month, $day) = Decode_Date_YMD ($str);
267 if (! defined $year || ! defined $month || ! defined $day
268 || ! Date::Calc::check_date ($year, $month, $day)) {
269 croak "Decode_Date_YMD_to_iso: invalid date \"$str\"\n";
270 }
271 return App::Chart::ymd_to_iso ($year, $month, $day);
272 }
273  
274  
275 #------------------------------------------------------------------------------
276  
277 sub date_parse_to_iso {
278 my ($str) = @_;
279 require Date::Parse;
280 my ($ss,$mm,$hh,$day,$month,$year,$zone) = Date::Parse::strptime ($str);
281 if (! defined ($day) || ! defined ($month) || ! defined ($year)) {
282 croak "date_parse_to_iso: unrecognised date \"$str\"\n";
283 }
284 return App::Chart::ymd_to_iso ($year + 1900, $month + 1, $day);
285 }
286  
287 #------------------------------------------------------------------------------
288  
289 # $h is a hash of share prices
290 # data => [ { symbol=>$str,
291 # date => $str,
292 # open => $price,
293 # high => $price,
294 # low => $price,
295 # close => $price,
296 # volume => $number,
297 # openint => $number,
298 # },
299 # ...
300 # ]
301 # dividends => [ { symbol => $str,
302 # ex_date => $date,
303 # record_date => $date,
304 # pay_date => $date,
305 # type => $str,
306 # amount => $price,
307 # imputation => $price,
308 # qualifier => $str,
309 # note => $str,
310 # },
311 # ...
312 # ],
313 # write it as daily data in the database
314 sub write_daily_group {
315 my ($h) = @_;
316  
317 crunch_h ($h);
318  
319 my $prefer_decimals = $h->{'prefer_decimals'};
320 my $database_symbols_hash = App::Chart::Database::database_symbols_hash();
321  
322 my %decimals;
323 my %data_changed;
324  
325 substatus (__('writing database'));
326 my $dbh = App::Chart::DBI->instance;
327 App::Chart::Database::call_with_transaction
328 ($dbh, sub {
329 if ($h->{'cost_key'}) {
330 require App::Chart::DownloadCost;
331 App::Chart::DownloadCost::cost_store_h ($h);
332 }
333  
334 my %symbol_hash;
335 {
336 my $sth = $dbh->prepare_cached
337 ('INSERT OR REPLACE INTO daily
338 (symbol, date, open, high, low, close, volume, openint)
339 VALUES (?,?,?,?,?,?,?,?)');
340  
341 foreach my $data (@{$h->{'data'}}) {
342 my $symbol = $data->{'symbol'};
343 next unless exists $database_symbols_hash->{$symbol};
344  
345 if (defined $prefer_decimals) {
346 $decimals{$symbol} = $prefer_decimals;
347 }
348 $data_changed{$symbol} = 1;
349  
350 $sth->execute($symbol,
351 $data->{'date'},
352 $data->{'open'},
353 $data->{'high'},
354 $data->{'low'},
355 $data->{'close'},
356 $data->{'volume'},
357 $data->{'openint'});
358 $sth->finish;
359 }
360 }
361  
362 foreach my $dividend (@{$h->{'dividends'}}) {
363 my $symbol = $dividend->{'symbol'}
364 or croak "write_daily_group: missing symbol in dividend record";
365 next unless exists $database_symbols_hash->{$symbol};
366  
367 my $ex_date = $dividend->{'ex_date'};
368 my $record_date = $dividend->{'record_date'};
369 my $pay_date = $dividend->{'pay_date'};
370  
371 my $type = $dividend->{'type'} || '';
372 my $amount = $dividend->{'amount'};
373 my $imputation = $dividend->{'imputation'};
374 my $qualifier = crunch_empty_undef ($dividend->{'qualifier'});
375 my $note = crunch_empty_undef ($dividend->{'note'});
376  
377 my $old_qualifier = App::Chart::DBI->read_single
378 ('SELECT qualifier FROM dividend WHERE symbol=? AND ex_date=? AND type=?', $symbol, $ex_date, $type);
379  
380 my $sth = $dbh->prepare_cached
381 ('INSERT OR REPLACE INTO dividend
382 (symbol, ex_date, record_date, pay_date,
383 type, amount, imputation, qualifier, note)
384 VALUES (?,?,?,?, ?,?,?,?,?)');
385  
386 $sth->execute ($symbol, $ex_date, $record_date, $pay_date,
387 $type, $amount, $imputation, $qualifier, $note);
388 $sth->finish;
389 $data_changed{$symbol} = 1;
390 }
391  
392 foreach my $split (@{$h->{'splits'}}) {
393 my $symbol = $split->{'symbol'}
394 or croak "write_daily_group: missing symbol in split record";
395 next unless exists $database_symbols_hash->{$symbol};
396  
397 my $date = $split->{'date'};
398 my $new = crunch_number ($split->{'new'});
399 my $old = crunch_number ($split->{'old'});
400 my $note = crunch_empty_undef ($split->{'note'});
401  
402 my $sth = $dbh->prepare_cached
403 ('INSERT OR REPLACE INTO split (symbol, date, new, old, note)
404 VALUES (?,?,?,?,?)');
405 $sth->execute ($symbol, $date, $new, $old, $note);
406 $sth->finish;
407 $data_changed{$symbol} = 1;
408 }
409  
410 if (my $names = $h->{'names'}) {
411 while (my ($symbol, $name) = each %$names) {
412 if (defined $name) {
413 $data_changed{$symbol} |= set_symbol_name ($symbol, $name);
414 }
415 }
416 }
417 if (my $currencies = $h->{'currencies'}) {
418 while (my ($symbol, $currency) = each %$currencies) {
419 if (defined $currency) {
420 $data_changed{$symbol} |= set_currency ($symbol, $currency);
421 }
422 }
423 }
424 if (my $isins = $h->{'isins'}) {
425 while (my ($symbol, $isin) = each %$isins) {
426 if (defined $isin) {
427 $data_changed{$symbol} |= set_isin ($symbol, $isin);
428 }
429 }
430 }
431 if (my $exchanges = $h->{'exchanges'}) {
432 while (my ($symbol, $exchange) = each %$exchanges) {
433 if (defined $exchange) {
434 $data_changed{$symbol} |= set_exchange ($symbol, $exchange);
435 }
436 }
437 }
438 while (my ($symbol, $decimals) = each %decimals) {
439 if (defined $decimals) {
440 $data_changed{$symbol} |= set_decimals ($symbol, $decimals);
441 }
442 }
443  
444 my $symbol_list = [ keys %data_changed ];
445  
446 if (my $key = $h->{'url_tags_key'}) {
447 my $resp = $h->{'resp'};
448 foreach my $symbol (@$symbol_list) {
449 my $url = (defined $resp ? $resp->request->uri : undef);
450 my $etag = (defined $resp ? scalar $resp->header('ETag') : undef);
451 my $last_modified =(defined $resp ? $resp->last_modified : undef);
452  
453 App::Chart::Database->write_extra
454 ($symbol, "$key-URL", "$url"); # stringize URI object
455 App::Chart::Database->write_extra
456 ($symbol, "$key-ETag", $etag);
457 App::Chart::Database->write_extra
458 ($symbol, "$key-Last-Modified", $last_modified);
459 }
460 }
461  
462 if (my $key = $h->{'copyright_key'}) {
463 foreach my $symbol (@$symbol_list) {
464 App::Chart::Database->write_extra
465 ($symbol, $key, $h->{'copyright'});
466 }
467 }
468  
469 my $timestamp = timestamp_now();
470  
471 if (my $key = $h->{'recheck_key'}) {
472 my @recheck_list;
473 if (my $l = $h->{'recheck_list'}) {
474 @recheck_list = @$l;
475 }
476 if (my $pred = $h->{'recheck_pred'}) {
477 push @recheck_list,
478 grep { $pred->match($_) } keys %$database_symbols_hash;
479 }
480 if (DEBUG) { print "Recheck write ",join(' ',@recheck_list),"\n"; }
481 foreach my $symbol (@recheck_list) {
482 App::Chart::Database->write_extra ($symbol, $key, $timestamp);
483 }
484 }
485  
486 if ($h->{'last_download'}) {
487 foreach my $symbol (keys %data_changed) {
488 App::Chart::Database->write_extra ($symbol, 'last-download',
489 $timestamp);
490 }
491 consider_historical ($symbol_list);
492 }
493 consider_latest_from_daily ($h, $database_symbols_hash);
494 });
495  
496 ### data_changed: %data_changed
497 App::Chart::chart_dirbroadcast()->send ('data-changed', \%data_changed);
498 }
499  
500 sub consider_historical {
501 my ($symbol_list) = @_;
502  
503 my $all_list;
504 my $historical_list;
505  
506 foreach my $symbol (@$symbol_list) {
507 if (App::Chart::Database->symbol_is_historical ($symbol)) {
508 next; # already marked
509 }
510 my $reason = want_historical($symbol) // next;
511 download_message ($reason);
512  
513 my $dbh = App::Chart::DBI->instance;
514 $dbh->do ('UPDATE info SET historical=1 WHERE symbol=?',
515 {}, $symbol);
516  
517 require App::Chart::Gtk2::Symlist;
518 $all_list ||= App::Chart::Gtk2::Symlist::All->instance;
519 $historical_list ||= App::Chart::Gtk2::Symlist::Historical->instance;
520  
521 $all_list->delete_symbol ($symbol);
522 $historical_list->insert_symbol ($symbol);
523 }
524 }
525  
526 # return true if $symbol should be marked historical, meaning it has had no
527 # new daily data for a long time
528 #
529 sub want_historical {
530 my ($symbol) = @_;
531 my $last_download_timestamp
532 = App::Chart::Database->read_extra ($symbol, 'last-download')
533 // return undef; # no download attempted, not historical
534  
535 my $date = App::Chart::DBI->read_single
536 ('SELECT date FROM daily WHERE (symbol=?) AND (close NOTNULL)
537 ORDER BY date DESC LIMIT 1',
538 $symbol);
539 if (! defined $date) {
540 return __x('{symbol} no data at all, marked historical',
541 symbol => $symbol);
542 }
543 my $days = iso_timestamp_days_ago ($date, $last_download_timestamp);
544 if ($days > 21) {
545 return __x('{symbol} no data for {days} days, marked historical',
546 days => $days,
547 symbol => $symbol);
548 }
549 return undef;
550 }
551  
552 sub iso_timestamp_days_ago {
553 my ($iso, $prev_timestamp) = @_;
554 my ($prev_year,$prev_month,$prev_day) = timestamp_to_ymdhms($prev_timestamp);
555 return Date::Calc::Delta_Days (App::Chart::iso_to_ymd($iso),
556 $prev_year, $prev_month, $prev_day);
557 }
558  
559 sub consider_latest_from_daily {
560 my ($h) = @_;
561 my $dbh = App::Chart::DBI->instance;
562 my %latest_changed;
563 my $timestamp;
564  
565 my $database_symbols_hash = App::Chart::Database::database_symbols_hash();
566  
567 # find the newest and second newest data record of each symbol
568 my %data_newest;
569 my %data_second;
570 foreach my $data (@{$h->{'data'}}) {
571 my $symbol = $data->{'symbol'};
572 if ($data->{'date'} ge ($data_newest{$symbol}->{'date'} // '')) {
573 $data_second{$symbol} = $data_newest{$symbol};
574 $data_newest{$symbol} = $data;
575 } elsif ($data->{'date'} ge ($data_second{$symbol}->{'date'} // '')) {
576 $data_second{$symbol} = $data;
577 }
578 }
579  
580 foreach my $symbol (keys %data_newest) {
581 my $newest = $data_newest{$symbol};
582 my $date = $newest->{'date'};
583  
584 # For symbols in the database, if newest daily is >= latest quote then
585 # delete that quote in order to prefer the daily data in the database.
586 if (exists $database_symbols_hash->{$symbol}) {
587 my $latest_delete_sth = $dbh->prepare_cached
588 ('DELETE FROM latest WHERE symbol=? AND quote_date < ?');
589 if ($latest_delete_sth->execute ($symbol, $date)) {
590 $latest_changed{$symbol} = 1;
591 }
592 next;
593 }
594  
595 # For symbols not in the database, if the newest daily is >= latest quote
596 # then replace that quote with the daily.
597 #
598 # Times in the latest record are not considered, so it's possible a
599 # quote taken after close of trading will be deleted or overwritten.
600 # Would want something in the latest to say it's after the close ...
601  
602 my $latest_get_sth = $dbh->prepare_cached
603 ('SELECT last_date, name FROM latest WHERE symbol=?');
604 my ($last_date, $name, $dividend)
605 = $dbh->selectrow_array ($latest_get_sth, undef, $symbol);
606 if (defined $last_date && $last_date gt $date) { next; }
607  
608 # "name" from the daily, or retain name from existing latest record.
609 $name = $h->{'names'}->{$symbol} // $name;
610  
611 # "dividend" from existing latest record retained, but only if same date.
612 unless (defined $last_date && $last_date eq $date) {
613 undef $dividend;
614 }
615  
616 # change by difference from second newest daily, if have one
617 my $change = undef;
618 if (defined(my $second_close = $data_second{$symbol}->{'close'})) {
619 $change = App::Chart::decimal_sub ($newest->{'close'}, $second_close);
620 }
621  
622 $timestamp ||= timestamp_now();
623  
624 my $latest_set_sth = $dbh->prepare_cached
625 ('INSERT OR REPLACE INTO latest
626 (symbol, name, currency, exchange, dividend,
627 last_date, open, high, low, last, change, volume,
628 source, fetch_timestamp)
629 VALUES (?,?,?,?,?, ?,?,?,?,?,?,?, ?,?)');
630 $latest_set_sth->execute
631 ($symbol,
632 $name,
633 $h->{'currencies'}->{$symbol},
634 $h->{'exchanges'}->{$symbol},
635 $dividend,
636 #
637 $newest->{'date'},
638 $newest->{'open'},
639 $newest->{'high'},
640 $newest->{'low'},
641 $newest->{'close'},
642 $change,
643 $newest->{'volume'},
644 #
645 $h->{'source'},
646 $timestamp);
647 $latest_changed{$symbol} = 1;
648 }
649  
650 foreach my $dividend (@{$h->{'dividends'}}) {
651 my $latest_dividend_sth = $dbh->prepare_cached
652 ('UPDATE latest SET dividend=? WHERE symbol=? AND last_date=?');
653 my $symbol = $dividend->{'symbol'};
654 if ($latest_dividend_sth->execute ($dividend->{'amount'},
655 $symbol,
656 $dividend->{'ex_date'})) {
657 $latest_changed{$symbol} = 1;
658 }
659 }
660  
661 App::Chart::chart_dirbroadcast()->send ('latest-changed', \%latest_changed);
662  
663 require App::Chart::Annotation;
664 foreach my $symbol (keys %latest_changed) {
665 App::Chart::Annotation::Alert::update_alert ($symbol);
666 }
667 }
668  
669  
670 #------------------------------------------------------------------------------
671  
672 sub write_latest_group {
673 my ($h) = @_;
674 ### write_latest_group(): $h
675  
676 crunch_h ($h);
677 ### crunched: $h
678  
679 my $fetch_timestamp = timestamp_now();
680 my $prefer_decimals = $h->{'prefer_decimals'};
681 my $source = $h->{'source'}
682 or croak 'missing "source" for latest records';
683 my %latest;
684  
685 my $dbh = App::Chart::DBI->instance;
686 App::Chart::Database::call_with_transaction
687 ($dbh, sub {
688  
689 my $sth = $dbh->prepare_cached
690 ('INSERT OR REPLACE INTO latest
691 (symbol, name, month, exchange, currency,
692 quote_date, quote_time, bid, offer,
693 last_date, last_time, open, high, low, last, change, volume,
694 note, error, dividend, copyright, source,
695 fetch_timestamp, url, etag, last_modified)
696 VALUES (?,?,?,?,?, ?,?,?,?, ?,?,?,?,?,?,?,?, ?,?,?,?,?, ?,?,?,?)');
697  
698 my $resp = $h->{'resp'};
699 my $etag = (defined $resp ? scalar $resp->header('ETag') : undef);
700 my $last_modified = (defined $resp ? $resp->last_modified : undef);
701  
702 foreach my $data (@{$h->{'data'}}) {
703 my $symbol = $data->{'symbol'};
704 my $this_date = $data->{'date'};
705 if ($latest{$symbol}) {
706 my $got_date = $latest{$symbol}->{'date'};
707 if (! defined $got_date || ! defined $this_date) {
708 carp "write_latest_group: $source: two records for '$symbol', but no 'date' field";
709 if (DEBUG || 1) {
710 require Data::Dumper;
711 print Data::Dumper->Dump([$data,$latest{$symbol}],
712 ['data','latest-so-far']);
713 }
714 next;
715 }
716 if ($got_date ge $this_date) { next; }
717 }
718 $latest{$symbol} = $data;
719 }
720  
721 my $error = $h->{'error'};
722 if (! defined $error && defined $resp && ! $resp->is_success) {
723 $error = $resp->status_line;
724 }
725  
726 foreach my $data (values %latest) {
727 my $symbol = $data->{'symbol'};
728  
729 my $bid = $data->{'bid'};
730 my $offer = $data->{'offer'};
731  
732 # disallow 0 for prices
733 if ($bid && $bid == 0) { $bid = undef; }
734 if ($offer && $offer == 0) { $offer = undef; }
735  
736 my $quote_date = crunch_date ($data->{'quote_date'});
737 my $quote_time = crunch_time ($data->{'quote_time'});
738 if ($quote_time && ! $quote_date) {
739 croak "quote_time without quote_date for $symbol";
740 }
741 # default quote date/time to now
742 if (($bid || $offer) && ! $quote_date) {
743 my $symbol_timezone = App::Chart::TZ->for_symbol ($symbol);
744 ($quote_date, $quote_time)
745 = $symbol_timezone->iso_date_time
746 (time() - 60 * ($data->{'quote_delay_minutes'} || 0));
747 }
748  
749 my $last_date = crunch_date ($data->{'last_date'} || $data->{'date'});
750 my $last_time = crunch_time ($data->{'last_time'});
751  
752 my $open = $data->{'open'};
753 my $high = $data->{'high'};
754 my $low = $data->{'low'};
755 my $last = $data->{'last'} || $data->{'close'};
756 my $change = $data->{'change'};
757 my $prev = crunch_price ($data->{'prev'}, $prefer_decimals);
758 my $volume = $data->{'volume'};
759  
760 if (! defined $last) {
761 # if there's no last price then try to use the prev
762 $open = $high = $low = undef;
763 $last = $prev;
764 $prev = undef;
765 $change = undef;
766 $last_date = undef;
767 $last_time = undef;
768  
769 } elsif (! defined $change) {
770 # if no change given then try to calculate it from last and prev
771  
772 if (! defined $prev) {
773 # if no prev then look for one among other $data records, as for
774 # when the group is a few consecutive daily data
775 my $prev_date;
776 foreach my $data (@{$h->{'data'}}) {
777 if ($data->{'symbol'} eq $symbol
778 && exists $data->{'date'}
779 && $data->{'date'} lt $last_date
780 && (! $prev_date
781 || $data->{'date'} gt $prev_date)) {
782 $prev_date = $data->{'date'};
783 $prev = $data->{'close'};
784 }
785 }
786 }
787 if ($prev) {
788 $change = App::Chart::decimal_sub ($last, $prev);
789 }
790 }
791  
792 $sth->execute ($symbol,
793 $h->{'names'}->{$symbol},
794 $data->{'month'},
795 $h->{'exchanges'}->{$symbol},
796 $h->{'currencies'}->{$symbol},
797  
798 $quote_date, $quote_time,
799 $bid, $offer,
800  
801 $last_date, $last_time,
802 $open, $high, $low, $last, $change, $volume,
803  
804 $data->{'note'},
805 $data->{'error'} || $error,
806 $data->{'dividend'},
807 $h->{'copyright'},
808 $source,
809  
810 $fetch_timestamp,
811 $h->{'url'},
812 $etag,
813 $last_modified);
814 $sth->finish;
815 }
816 });
817  
818 App::Chart::chart_dirbroadcast()->send ('latest-changed', \%latest);
819  
820 require App::Chart::Annotation;
821 foreach my $symbol (keys %latest) {
822 App::Chart::Annotation::Alert::update_alert ($symbol);
823 }
824 }
825  
826 sub iso_to_MMM_YY {
827 my ($iso) = @_;
828 my ($year, $month, $day) = App::Chart::iso_to_ymd ($iso);
829 return sprintf ("%.3s %02d",
830 uc(Date::Calc::Month_to_Text ($month)),
831 $year % 100);
832 }
833  
834 #------------------------------------------------------------------------------
835  
836 my $iso_date_re = qr/^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]$/;
837  
838 my %date_format_to_func
839 = ('ymd' => \&App::Chart::Download::Decode_Date_YMD_to_iso,
840 'dmy' => \&App::Chart::Download::Decode_Date_EU_to_iso,
841 'mdy' => \&App::Chart::Download::Decode_Date_US_to_iso);
842 sub noop {
843 return $_[0];
844 }
845  
846 my %date_format_to_month_func
847 = ('ymd' => \&crunch_month_ymd);
848  
849 sub crunch_month_ymd {
850 my ($str) = @_;
851 my ($year, $month, $day) = Date::Calc::Decode_Date_EU ($str);
852 if (! defined $year) {
853 croak "unrecognised month string: $str";
854 }
855 if (defined $day) {
856 if ($day != 1) { croak "month with day!=1: '$str' got $day"; }
857 } else {
858 $day = 1;
859 }
860 return App::Chart::ymd_to_iso ($year, $month, $day);
861 }
862  
863 sub crunch_h {
864 my ($h) = @_;
865 my $database_symbols_hash = App::Chart::Database::database_symbols_hash();
866  
867 # ignore undef where expecting hashrefs in 'data' and 'dividends' arefs
868 foreach my $elem ('data',
869 'dividends') {
870 my $aref = $h->{$elem} // next;
871 @$aref = grep {
872 defined $_
873 # && App::Chart::Database->symbol_exists($_->{'symbol'})
874 } @$aref;
875 }
876  
877 my $suffix = $h->{'suffix'};
878 my $month_format = $h->{'month_format'};
879 my $prefer_decimals = $h->{'prefer_decimals'};
880  
881 my $date_format = delete $h->{'date_format'};
882 my $date_func = ($date_format ? $date_format_to_func{$date_format} : \&noop)
883 || croak "Unrecognised date_format '$date_format'";
884 # my $month_func = $date_format ? $date_format_to_month_func{$date_format} : \&noop;
885  
886 my %currencies; $h->{'currencies'} = \%currencies;
887 my %isins; $h->{'isins'} = \%isins;
888 my %names; $h->{'names'} = \%names;
889 my %exchanges; $h->{'exchanges'} = \%exchanges;
890  
891 foreach my $info (@{$h->{'info'}}) {
892 my $symbol = $info->{'symbol'}
893 or croak "write_daily_group: missing symbol in info record";
894 if (! exists $database_symbols_hash->{$symbol}) { next; }
895  
896 $names{$symbol} ||= delete $info->{'name'};
897 $currencies{$symbol} ||= delete $info->{'currency'};
898 $exchanges{$symbol} ||= delete $info->{'exchange'};
899 $isins{$symbol} ||= delete $info->{'isin'};
900 }
901  
902 foreach my $data (@{$h->{'data'}}) {
903 my $month = $data->{'month'};
904 if (defined $month) {
905 # $month = $data->{'month'} = $month_func->($month);
906 $month =~ $iso_date_re or croak "Bad month value '$month'";
907 }
908  
909 my $symbol = $data->{'symbol'};
910 if (! defined $symbol) {
911 # symbols built from commodity + month
912 my $commodity = $data->{'commodity'}
913 // croak "neither symbol nor commodity in 'data' element";
914 $month // croak "Group data: no 'month' to go with 'commodity' in data element";
915 $symbol = $data->{'symbol'}
916 = $commodity . ' ' . iso_to_MMM_YY($month) . $suffix;
917 }
918 # if ($month_format eq 'MMM_YY') {
919 # $month = iso_to_MMM_YY ($month);
920 # } else {
921 # croak "unrecognised month format: $month_format";
922 # }
923 }
924  
925 if ($h->{'front_month'}) {
926 my %front;
927 foreach my $data (@{$h->{'data'}}) {
928 my $symbol = $data->{'symbol'};
929 my $front_symbol
930 = App::Chart::symbol_commodity($symbol)
931 . App::Chart::symbol_suffix($symbol);
932 if (! $front{$front_symbol}
933 || $data->{'month'} gt $front{$front_symbol}->{'month'}) {
934 $front{$front_symbol} = $data;
935 }
936 }
937  
938 while (my ($symbol, $data) = each %front) {
939 $data = { %$data };
940 $data->{'symbol'} = $symbol;
941 push @{$h->{'data'}}, $data;
942 }
943 }
944  
945 foreach my $data (@{$h->{'data'}}) {
946  
947 foreach my $field ('date', 'quote_date', 'last_date') {
948 if (defined $data->{$field}) {
949 $data->{$field} = crunch_date ($data->{$field}, $date_format);
950 }
951 }
952  
953 # empty volume or openint taken to be no data (as opposed to '0' for
954 # zero volume or int)
955 foreach my $field ('volume', 'openint') {
956 if (defined $data->{$field} && $data->{$field} eq '') {
957 $data->{$field} = undef;
958 }
959 }
960  
961 if (my $sessions = delete $data->{'sessions'}) {
962 my @sessions = grep {defined} map {crunch_price($_)} @$sessions;
963 $data->{'open'} = $sessions[0];
964 ($data->{'low'}, $data->{'high'}) = List::MoreUtils::minmax (@sessions);
965 $data->{'close'} = $sessions[-1];
966 }
967  
968 if (exists $data->{'change'}) {
969 $data->{'change'} = crunch_change ($data->{'change'}, $prefer_decimals);
970 }
971 foreach my $field (qw(bid offer open high low close last)) {
972 if (exists $data->{$field}) {
973 $data->{$field} = crunch_price ($data->{$field}, $prefer_decimals);
974 }
975 }
976 foreach my $field (qw(volume openint)) {
977 if (exists $data->{$field}) {
978 $data->{$field} = crunch_number ($data->{$field});
979 }
980 }
981  
982 my $symbol = $data->{'symbol'};
983 $currencies{$symbol} ||= delete $data->{'currency'} || $h->{'currency'};
984 $names{$symbol} ||= delete $data->{'name'} || $h->{'name'};
985 $exchanges{$symbol} ||= delete $data->{'exchange'};
986 $isins{$symbol} ||= delete $data->{'isin'};
987 }
988  
989 foreach my $dividend (@{$h->{'dividends'}}) {
990 foreach my $field ('ex_date', 'record_date', 'pay_date') {
991 if (defined $dividend->{$field}) {
992 $dividend->{$field} = crunch_date ($dividend->{$field}, $date_format);
993 }
994 }
995 $dividend->{'ex_date'}
996 or croak 'Group data: missing ex_date in dividend record';
997  
998 foreach my $field (qw(amount imputation)) {
999 if (exists $dividend->{$field}) {
1000 $dividend->{$field} = crunch_price ($dividend->{$field}, $prefer_decimals);
1001 }
1002 }
1003 }
1004  
1005 foreach my $split (@{$h->{'splits'}}) {
1006 if (defined $split->{'date'}) {
1007 $split->{'date'} = crunch_date ($split->{'date'}, $date_format);
1008 }
1009 $split->{'date'}
1010 or croak "Group data: missing 'date' in split record";
1011 }
1012  
1013 # whitespace in names, and possible leading/trailing in isins
1014 foreach my $href (\%names, \%currencies, \%exchanges, \%isins) {
1015 hash_delete_undefs ($href);
1016 }
1017 foreach (values %names, values %isins) {
1018 $_ = App::Chart::collapse_whitespace ($_);
1019 }
1020  
1021 if (eval { require Business::ISIN; 1 }) {
1022 my $bi = Business::ISIN->new;
1023 while (my ($symbol, $isin) = each %isins) {
1024 $bi->set ($isin);
1025 if ($bi->is_valid) { next; }
1026 warn "$symbol ISIN '$isin' is invalid, ignoring: ", $bi->error;
1027 delete $isins{$symbol};
1028 }
1029 }
1030 }
1031  
1032 sub hash_delete_undefs {
1033 my ($href) = @_;
1034 while (my ($key, $value) = each %$href) {
1035 if (! defined $value) {
1036 delete $href->{$key};
1037 }
1038 }
1039 }
1040  
1041 sub crunch_date {
1042 my ($str, $format) = @_;
1043 $str = crunch_empty_undef ($str);
1044 if (! defined $str) { return $str; }
1045  
1046 if (defined $format) {
1047 my $func = $date_format_to_func{$format};
1048 defined $func or croak "Unrecognised date_format spec: $format";
1049 $str = &$func ($str);
1050 }
1051 $str =~ $iso_date_re or croak "Bad date '$str'";
1052 return $str;
1053 }
1054  
1055 sub crunch_price {
1056 my ($str, $prefer_decimals) = @_;
1057 $str = crunch_change ($str, $prefer_decimals) // return undef;
1058  
1059 if (str_is_zero ($str)) { return undef; }
1060 return $str;
1061 }
1062 sub crunch_change {
1063 my ($str, $prefer_decimals) = @_;
1064 $str = crunch_number ($str) // return undef;
1065  
1066 if ($str eq '') { return undef; } # empty
1067 if (uc($str) eq 'CLOSED') { return undef; } # RBA 2003-2006.xls
1068 if ($str eq 'unch') { return '0'; } # unchanged
1069 if (defined $prefer_decimals) {
1070 return App::Chart::Download::trim_decimals ($str, $prefer_decimals);
1071 } else {
1072 return $str;
1073 }
1074 }
1075  
1076 sub crunch_number {
1077 my ($str) = @_;
1078 if (! defined $str) { return undef; }
1079  
1080 $str =~ s/$RE{ws}{crop}//go; # leading and trailing whitespace
1081 $str =~ s/^\+//; # leading + sign
1082 $str =~ s/^0+([1-9]|0\.|0$)/$1/; # leading extra zeros
1083  
1084 if ($str eq ''
1085 || $str eq '-'
1086 || $str eq 'N/A'
1087 || $str eq 'n/a') {
1088 return undef;
1089 }
1090 $str =~ s/,//g; # commas for thousands
1091 return $str;
1092 }
1093  
1094 sub crunch_time {
1095 my ($str) = @_;
1096 $str = crunch_empty_undef ($str);
1097 if (! defined $str) { return undef; }
1098  
1099 $str =~ /^([0-9]?[0-9])(:[0-9][0-9])(:[0-9][0-9])?([ap]m)?$/i
1100 or croak "bad time '$str'";
1101 my $hour = $1;
1102 my $minute = $2;
1103 my $second = $3;
1104 my $am_pm = $4;
1105 $second //= ':00';
1106 if (defined $am_pm && lc $am_pm eq 'pm') { $hour += 12; }
1107 $hour = sprintf '%02d', $hour;
1108 return "$hour$minute$second";
1109 }
1110  
1111 sub crunch_empty_undef {
1112 my ($str) = @_;
1113 if (! defined $str) { return $str; }
1114 $str =~ s/$RE{ws}{crop}//go; # leading and trailing whitespace
1115  
1116 # eg. string "N/A" in dates and times from Yahoo
1117 if ($str eq '' || $str eq 'N/A') { return undef; }
1118 return $str;
1119 }
1120  
1121  
1122 #------------------------------------------------------------------------------
1123  
1124 sub set_symbol_name {
1125 my ($symbol, $name) = @_;
1126 my $dbh = App::Chart::DBI->instance;
1127 my $sth = $dbh->prepare_cached('UPDATE info SET name=? WHERE symbol=?');
1128 my $changed = $sth->execute($name, $symbol);
1129 $sth->finish();
1130 return $changed;
1131 }
1132 sub set_currency {
1133 my ($symbol, $currency) = @_;
1134 my $dbh = App::Chart::DBI->instance;
1135 my $sth = $dbh->prepare_cached('UPDATE info SET currency=? WHERE symbol=?');
1136 my $changed = $sth->execute($currency, $symbol);
1137 $sth->finish();
1138 return $changed;
1139 }
1140 sub set_exchange {
1141 my ($symbol, $exchange) = @_;
1142 my $dbh = App::Chart::DBI->instance;
1143 my $sth = $dbh->prepare_cached('UPDATE info SET exchange=? WHERE symbol=?');
1144 my $changed = $sth->execute($exchange, $symbol);
1145 $sth->finish();
1146 return $changed;
1147 }
1148 sub set_decimals {
1149 my ($symbol, $decimals) = @_;
1150 my $dbh = App::Chart::DBI->instance;
1151 my $sth = $dbh->prepare_cached('UPDATE info SET decimals=? WHERE symbol=?');
1152 my $changed = $sth->execute($decimals, $symbol);
1153 $sth->finish();
1154 return $changed;
1155 }
1156 sub set_isin {
1157 my ($symbol, $isin) = @_;
1158 my $dbh = App::Chart::DBI->instance;
1159 my $sth = $dbh->prepare_cached('UPDATE info SET isin=? WHERE symbol=?');
1160 my $changed = $sth->execute($isin, $symbol);
1161 $sth->finish();
1162 return $changed;
1163 }
1164  
1165  
1166 #------------------------------------------------------------------------------
1167  
1168 # return true if $timestamp string is within the past $seconds from now
1169 # also return true if $timestamp is some strange future value
1170 sub timestamp_within {
1171 my ($timestamp, $seconds) = @_;
1172 if (! defined $timestamp) { return 0; } # undef stamp always out of range
1173 my ($lo, $hi) = timestamp_range ($seconds);
1174 return (($timestamp ge $lo) && ($timestamp le $hi));
1175 }
1176 sub timestamp_range {
1177 my ($seconds) = @_;
1178 my $t = time();
1179 my $lo = $t - $seconds;
1180 my $hi = $t + 6*3600; # 2 hours future
1181 return (timet_to_timestamp($lo),
1182 timet_to_timestamp($hi));
1183 }
1184 sub timestamp_now {
1185 return timet_to_timestamp(time());
1186 }
1187 sub timet_to_timestamp {
1188 my ($t) = @_;
1189 return POSIX::strftime ('%Y-%m-%d %H:%M:%S+00:00', gmtime($t));
1190 }
1191 sub timestamp_to_ymdhms {
1192 my ($timestamp) = @_;
1193 return split /[- :+]/, $timestamp;
1194 }
1195 sub timestamp_to_timet {
1196 my ($timestamp) = @_;
1197 my ($year, $month, $day, $hour, $minute, $second)
1198 = timestamp_to_ymdhms($timestamp);
1199 require Time::Local;
1200 return Time::Local::timegm_modern
1201 ($second, $minute, $hour, $day, $month-1, $year);
1202 }
1203  
1204  
1205 #------------------------------------------------------------------------------
1206  
1207 sub tdate_strftime {
1208 my ($format, $tdate) = @_;
1209 my ($year, $month, $day) = App::Chart::tdate_to_ymd ($tdate);
1210 require App::Chart::Timebase;
1211 return App::Chart::Timebase::strftime_ymd ($format, $year, $month, $day);
1212 }
1213  
1214 sub tdate_range_string {
1215 my ($lo, $hi) = @_;
1216 if (@_ < 2) { $hi = $lo; }
1217 my $d_fmt = $App::Chart::option{'d_fmt'};
1218 if ($lo == $hi) {
1219 return tdate_strftime ($d_fmt, $lo);
1220 } else {
1221 return __x('{lodate} to {hidate}',
1222 lodate => tdate_strftime ($d_fmt, $lo),
1223 hidate => tdate_strftime ($d_fmt, $hi));
1224 }
1225 }
1226  
1227 sub symbol_range_string {
1228 my ($symbol_list) = @_;
1229 if (@$symbol_list == 0) {
1230 return '';
1231 } elsif (@$symbol_list == 1) {
1232 return $symbol_list->[0];
1233 } else {
1234 return __x('{start_symbol} to {end_symbol}',
1235 start_symbol => $symbol_list->[0],
1236 end_symbol => $symbol_list->[-1]);
1237 }
1238 }
1239  
1240 #------------------------------------------------------------------------------
1241  
1242 sub weekday_date_after_time {
1243 return App::Chart::tdate_to_iso (weekday_tdate_after_time (@_));
1244 }
1245 sub weekday_tdate_after_time {
1246 my ($after_hour,$after_min, $timezone, $offset) = @_;
1247  
1248 local $Tie::TZ::TZ = $timezone->tz;
1249 my ($year,$month,$day, $hour,$min,$sec, $doy,$dow,$dst)
1250 = Date::Calc::Localtime();
1251  
1252 my $tdate = App::Chart::ymd_to_tdate_floor ($year,$month,$day)
1253 + ($offset // 0);
1254  
1255 if ($dow >= 6 # Saturday or Sunday
1256 || ($hour*60+$min < $after_hour*60+$after_min)) {
1257 $tdate--;
1258 }
1259 return $tdate;
1260 }
1261  
1262 #------------------------------------------------------------------------------
1263  
1264 sub download {
1265 my (%options) = @_;
1266  
1267 my @symbol_list = ();
1268 {
1269 my $symbol_list = $options{'symbol_list'}
1270 || croak "download() missing symbol_list\n";
1271 @symbol_list = @$symbol_list;
1272 }
1273  
1274 @symbol_list = List::MoreUtils::uniq (@symbol_list);
1275 verbose_message (__('Download:'), @symbol_list);
1276  
1277 foreach my $symbol (@symbol_list) {
1278 App::Chart::symbol_setups ($symbol);
1279 }
1280 App::Chart::Database->add_symbol (@symbol_list);
1281  
1282 require App::Chart::DownloadHandler;
1283 my $all_ok = 1;
1284 foreach my $handler (@App::Chart::DownloadHandler::handler_list) {
1285 my @this_list = grep {$handler->match($_)} @symbol_list;
1286 my $ok = $handler->download (\@this_list);
1287 if (! $ok) { $all_ok = 0; }
1288 }
1289  
1290 # my %handler_result = ();
1291 # foreach my $symbol (@symbol_list) {
1292 #
1293 # my @handlers = App::Chart::DownloadHandler->handlers_for_symbol ($symbol);
1294 # foreach my $handler (@handlers) {
1295 # if (exists $handler_result{$handler}) {
1296 # if (! $handler_result{$handler}) { $all_ok = 0; }
1297 # next;
1298 # }
1299 #
1300 # my @this_list = grep { my $symbol = $_;
1301 # List::MoreUtils::all
1302 # {$_->match($symbol)} @handlers
1303 # } @symbol_list;
1304 # my $ok = $handler->download (\@this_list);
1305 # $handler_result{$handler} = $ok;
1306 # }
1307 #
1308 # }
1309 status (__('Checking historical'));
1310 if ($all_ok) {
1311 consider_historical (\@symbol_list);
1312 }
1313 }
1314  
1315 # return a list of symbols, either just ($symbol), or if $symbol has
1316 # wildcards then the result of matching that in the "all" list
1317 sub symbol_glob {
1318 my ($symbol) = @_;
1319  
1320 if ($symbol =~ /[*?]/) {
1321 require Text::Glob;
1322 require App::Chart::Gtk2::Symlist::All;
1323 my $symlist = App::Chart::Gtk2::Symlist::All->instance;
1324 my $regexp = Text::Glob::glob_to_regex ($symbol);
1325 my @list = grep {$_ =~ $regexp} $symlist->symbols;
1326 if (! @list) {
1327 print __x("Warning, pattern \"{pattern}\" doesn't match anything in the database, ignoring\n",
1328 pattern => $symbol);
1329 }
1330 return @list;
1331 } else {
1332 return ($symbol);
1333 }
1334 }
1335  
1336 sub command_line_download {
1337 my ($class, $output, $args) = @_;
1338 my $hash;
1339  
1340 if ($output eq 'tty') {
1341 if (-t STDOUT) {
1342 binmode (STDOUT, ':via(EscStatus)')
1343 or die 'Cannot push EscStatus';
1344 } else {
1345 require PerlIO::via::EscStatus::ShowNone;
1346 binmode (STDOUT, ':via(EscStatus::ShowNone)')
1347 or die 'Cannot push EscStatus::ShowNone';
1348 }
1349 } elsif ($output eq 'all-status') {
1350 require PerlIO::via::EscStatus::ShowAll;
1351 binmode (STDOUT, ':via(EscStatus::ShowAll)')
1352 or die 'Cannot push EscStatus::ShowAll';
1353 }
1354  
1355 if (! @$args) {
1356 print __"No symbols specified to download\n";
1357 return;
1358 }
1359  
1360 my @symbol_list = ();
1361 foreach my $arg (@$args) {
1362 if (ref $arg) {
1363 # only what's already in the database
1364 $hash ||= App::Chart::Database::database_symbols_hash();
1365 my @list = grep {exists $hash->{$_}} $arg->symbols;
1366 push @symbol_list, @list;
1367 if (! @list) {
1368 print __x("Warning, no symbols in \"{symlist}\" are currently in the database\n(only symbols already in the database are downloaded from lists)\n",
1369 symlist => $arg->name);
1370 }
1371 } else {
1372 push @symbol_list, symbol_glob ($arg);
1373 }
1374 }
1375  
1376 App::Chart::Download::download (symbol_list => \@symbol_list);
1377  
1378 require App::Chart::LatestHandler;
1379 App::Chart::LatestHandler->download (\@symbol_list);
1380  
1381 status ('');
1382 }
1383  
1384 #------------------------------------------------------------------------------
1385  
1386 sub iso_to_tdate_floor {
1387 my ($str) = @_;
1388 my ($year, $month, $day) = App::Chart::iso_to_ymd ($str);
1389 return App::Chart::ymd_to_tdate_floor ($year, $month, $day);
1390 }
1391  
1392 sub iso_to_tdate_ceil {
1393 my ($str) = @_;
1394 my ($year, $month, $day) = App::Chart::iso_to_ymd ($str);
1395 return App::Chart::ymd_to_tdate_ceil ($year, $month, $day);
1396 }
1397  
1398 sub tdate_today {
1399 my ($timezone) = @_;
1400 $timezone //= App::Chart::TZ->loco;
1401 my ($year, $month, $day) = $timezone->ymd;
1402 return App::Chart::ymd_to_tdate_floor ($year, $month, $day);
1403 }
1404  
1405 my $default_download_tdates = 5 * 265; # 5 years
1406  
1407 sub start_tdate_for_update {
1408 my (@symbol_list) = @_;
1409 if (! @symbol_list) { croak "start_tdate_for_update(): no symbols"; }
1410 my $ret;
1411 foreach my $symbol (@symbol_list) {
1412 my $iso = App::Chart::DBI->read_single
1413 ('SELECT date FROM daily WHERE symbol=? ORDER BY date DESC LIMIT 1',
1414 $symbol);
1415 if (! defined $iso) {
1416 return (tdate_today() - $default_download_tdates);
1417 }
1418 my $tdate = iso_to_tdate_floor ($iso) + 1;
1419 $ret = App::Chart::min_maybe ($ret, $tdate);
1420 }
1421 return $ret;
1422 }
1423  
1424 sub tdate_today_after {
1425 my ($after_hour, $after_minute, $timezone) = @_;
1426  
1427 { local $Tie::TZ::TZ = $timezone->tz;
1428 my ($year,$month,$day, $hour,$min,$sec, $doy,$dow,$dst) =
1429 Date::Calc::System_Clock();
1430  
1431 my $tdate = App::Chart::ymd_to_tdate_floor ($year, $month, $day);
1432 if ($dow <= 5 # is a weekday
1433 && (App::Chart::hms_to_seconds ($hour, $min, 0)
1434 < App::Chart::hms_to_seconds ($after_hour, $after_minute, 0))) {
1435 $tdate--;
1436 }
1437 return $tdate;
1438 }
1439 }
1440  
1441  
1442 #-----------------------------------------------------------------------------
1443 # selecting among possibly overlapping files
1444  
1445 # $files is an arrayref containing hash records with keys
1446 #
1447 # lo_tdate,hi_tdate inclusive coverage of the record
1448 # lo_year,hi_year alterative form for date range
1449 # cost size of the file in bytes
1450 #
1451 sub choose_files {
1452 my ($files, $lo_tdate, $hi_tdate) = @_;
1453 if ($lo_tdate > $hi_tdate) { return []; }
1454  
1455 if (DEBUG) { print "choose_files $lo_tdate to $hi_tdate\n"; }
1456  
1457 foreach my $f (@$files) {
1458 if (! defined $f->{'lo_tdate'}) {
1459 if (my $m = $f->{'month_iso'}) {
1460 $f->{'lo_tdate'} = App::Chart::Download::iso_to_tdate_ceil ($m);
1461 } elsif ($f->{'lo_year'}) {
1462 $f->{'lo_tdate'}
1463 = App::Chart::ymd_to_tdate_ceil ($f->{'lo_year'}, 1, 1);
1464 } else {
1465 croak 'choose_files: missing lo date';
1466 }
1467 }
1468  
1469 if (! defined $f->{'hi_tdate'}) {
1470 if (my $m = $f->{'month_iso'}) {
1471 $f->{'hi_tdate'}
1472 = tdate_end_of_month (App::Chart::Download::iso_to_tdate_ceil ($m));
1473 } elsif ($f->{'hi_year'}) {
1474 $f->{'hi_tdate'}
1475 = App::Chart::ymd_to_tdate_floor($f->{'hi_year'}, 12, 31);
1476 } else {
1477 croak 'choose_files: missing hi date';
1478 }
1479 }
1480 }
1481 if (DEBUG >= 2) { require Data::Dumper;
1482 print Data::Dumper::Dumper($files); }
1483  
1484 # restrict wanted range to what's available
1485 my $lo_available = min (map {$_->{'lo_tdate'}} @$files);
1486 my $hi_available = max (map {$_->{'hi_tdate'}} @$files);
1487 $lo_tdate = max ($lo_tdate, $lo_available);
1488 $hi_tdate = min ($hi_tdate, $hi_available);
1489 if (DEBUG) { print " available $lo_available to $hi_available\n";
1490 print " restricted range $lo_tdate to $hi_tdate\n"; }
1491 if ($lo_tdate > $hi_tdate) { return []; }
1492  
1493 # ignore file elements not covering any of the desired range
1494 $files = [ grep {App::Chart::overlap_inclusive_p ($lo_tdate, $hi_tdate,
1495 $_->{'lo_tdate'},
1496 $_->{'hi_tdate'})}
1497 @$files ];
1498  
1499 # Algorithm::ChooseSubsets would be another way to iterate, or
1500 # Math::Subset::List to get all combinations
1501 my $best_cost;
1502 my $best_files;
1503 foreach my $this_files (all_combinations ($files)) {
1504 if (! cover_p ($this_files, $lo_tdate, $hi_tdate)) { next; }
1505 my $cost = List::Util::sum (map {$_->{'cost'}||0} @$this_files);
1506 $cost += $App::Chart::option{'http_get_cost'} * scalar(@$this_files);
1507 if (! defined $best_cost || $cost < $best_cost) {
1508 $best_cost = $cost;
1509 $best_files = $this_files;
1510 }
1511 }
1512 return $best_files;
1513 }
1514  
1515 # return true if the set of file records in arrayref $files covers all of
1516 # $lo_tdate through $hi_tdate inclusive
1517 #
1518 sub cover_p {
1519 my ($files, $lo_tdate, $hi_tdate) = @_;
1520 require Set::IntSpan::Fast;
1521 my $set = Set::IntSpan::Fast->new;
1522 foreach my $f (@$files) {
1523 $set->add_range ($f->{'lo_tdate'}, $f->{'hi_tdate'});
1524 }
1525 $set->contains_all_range ($lo_tdate, $hi_tdate);
1526 }
1527  
1528 # return a list which is all the combinations of elements of @$aref
1529 # for example $aref == [ 10, 20 ] would return ([], [10], [20], [10,20])
1530 # there's 2**N combinations for aref length N
1531 #
1532 sub all_combinations {
1533 my ($aref) = @_;
1534 my @ret = ([]);
1535 foreach my $i (0 .. $#$aref) {
1536 push @ret, map {[ @$_, $aref->[$i] ]} @ret;
1537 }
1538 return @ret;
1539 }
1540  
1541 # return the last tdate in the month containing the given $tdate
1542 sub tdate_end_of_month {
1543 my ($tdate) = @_;
1544 my ($year, $month, $day) = App::Chart::tdate_to_ymd ($tdate);
1545 ($year, $month, $day) = Date::Calc::Add_Delta_YM ($year, $month, $day, 0,1);
1546 $day = 1;
1547 ($year, $month, $day) = Date::Calc::Add_Delta_Days ($year, $month, $day, -1);
1548 return App::Chart::ymd_to_tdate_floor ($year, $month, $day);
1549 }
1550  
1551 1;
1552 __END__
« App/Chart/Barchart.pm ↑ Summary App/Chart/DownloadHandler.pm »