| 1 | # Copyright 2007, 2008, 2009, 2010, 2011, 2015, 2016, 2017, 2019, 2020, 2023, 2024 Kevin Ryde | ||
| 2 | |||
| 3 | # This file is part of Chart. | ||
| 4 | # | ||
| 5 | # Chart is free software; you can redistribute it and/or modify it under the | ||
| 6 | # terms of the GNU General Public License as published by the Free Software | ||
| 7 | # Foundation; either version 3, or (at your option) any later version. | ||
| 8 | # | ||
| 9 | # Chart is distributed in the hope that it will be useful, but WITHOUT ANY | ||
| 10 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
| 11 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | ||
| 12 | # details. | ||
| 13 | # | ||
| 14 | # You should have received a copy of the GNU General Public License along | ||
| 15 | # with Chart. If not, see <http://www.gnu.org/licenses/>. | ||
| 16 | |||
| 17 | |||
| 18 | # Protocols: | ||
| 19 | # | ||
| 20 | # Don't know the full state of what Yahoo intends to offer. | ||
| 21 | # There's a range of "v1", "v7", "v8", "v11" URL forms. | ||
| 22 | # These might be meant as data sources for Yahoo client-side viewing. | ||
| 23 | # Don't think they're documented as such, but a few people have | ||
| 24 | # explored or put traces under the web site operation. | ||
| 25 | # | ||
| 26 | # The download forms might be only for logged-in Yahoo users. | ||
| 27 | # Would prefer not to require a Yahoo account for every user, | ||
| 28 | # esp since it's not as easy to do now as in the past. | ||
| 29 | # (But if that's a condition of use then would do so.) | ||
| 30 | # | ||
| 31 | # Here currently using | ||
| 32 | # v1 JSON for info (company name, exchange, decimals) | ||
| 33 | # v8 JSON for historical daily data | ||
| 34 | # v7 JSON for latest quote | ||
| 35 | # | ||
| 36 | # Each seem to be without protocol level hoops to jump through. | ||
| 37 | # Presumably all fall under the general Yahoo terms of service | ||
| 38 | # which think are personal use and no re-publication. | ||
| 39 | # | ||
| 40 | # | ||
| 41 | # Data Format: | ||
| 42 | # | ||
| 43 | # Each of these methods returns results in JSON format. | ||
| 44 | # The "download" URL form(s), as distinct from "chart", were/are CSV | ||
| 45 | # but might be restricted access now. | ||
| 46 | # | ||
| 47 | # JSON has prices put through single-precision floats, | ||
| 48 | # ie. 24 bit mantissa, which causes some price strings like | ||
| 49 | # 123.44999999 instead of 123.45. | ||
| 50 | # | ||
| 51 | # The parse here rounds that back to an apparent intended number | ||
| 52 | # of decimals (eg. 2 decimals for dollars and cents), but still | ||
| 53 | # allowing for trading in fractions of a cent as 3 or more decimals. | ||
| 54 | # | ||
| 55 | # In daily data, the current day's trading in progress shows at | ||
| 56 | # today's date and it changes though the course of trading. | ||
| 57 | # Don't know how or whether pre-market or post-market trading is | ||
| 58 | # applied, or what happens to a few futures or similar which might | ||
| 59 | # even be 24 hour trading. | ||
| 60 | # | ||
| 61 | # Dividends and splits seem to only appear on the ex date. | ||
| 62 | # That's disappointing since upcoming dividends often influence price | ||
| 63 | # action. (Which they shouldn't since everyone knows in advance.) | ||
| 64 | # | ||
| 65 | # | ||
| 66 | # Cookies and Crumb: | ||
| 67 | # | ||
| 68 | # In recent times there's been some protocol hoops to jump through. | ||
| 69 | # It seemed to be sometimes on the v7 download, maybe always on v11. | ||
| 70 | # The v7 JSON has been fine asking for the latest few days daily data, | ||
| 71 | # but not sure about bigger historical data. | ||
| 72 | # | ||
| 73 | # The hoops consisted of | ||
| 74 | # | ||
| 75 | # - Fetch one of the finance.yahoo.com web pages to get a | ||
| 76 | # HTTP Set-Cookie header. | ||
| 77 | # As of September 2024, there seems no such cookie any more | ||
| 78 | # (only a general user-tracking one for www.yahoo.com). | ||
| 79 | # - Maybe answer the ridiculous EU cookie consent on the page. | ||
| 80 | # Maybe that depends on where your IP seems to be from. | ||
| 81 | # (Not that an IP reliably indicates legal jurisdiction.) | ||
| 82 | # - Look deep within script in that page for a "crumb" string. | ||
| 83 | # Or maybe a further "getcrumb" web fetch, but seems result | ||
| 84 | # crumb is embedded in the page. | ||
| 85 | # - On each data download, HTTP Cookie header, and URL crumb | ||
| 86 | # field. | ||
| 87 | # | ||
| 88 | # Presumably this is designed either to track user activity, or | ||
| 89 | # as a level of difficulty to stop what Yahoo had said was | ||
| 90 | # widespread mis-use (contrary to personal use terms). | ||
| 91 | # | ||
| 92 | |||
| 93 | |||
| 94 | package App::Chart::Yahoo; | ||
| 95 | 1 | ▶ | use 5.010; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 96 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 97 | 1 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 98 | 1 | ▶ | use Carp; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 99 | 1 | ▶ | use Date::Calc; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 100 | 1 | ▶ | use Date::Parse; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 101 | 1 | ▶ | use JSON; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 102 | 1 | ▶ | use List::Util qw (min max); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 103 | 1 | ▶ | use POSIX (); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 104 | 1 | ▶ | use Time::Local; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 105 | 1 | ▶ | use URI::Escape; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 106 | 1 | ▶ | use Locale::TextDomain ('App-Chart'); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 107 | |||
| 108 | 1 | ▶ | use Tie::TZ; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 109 | 0 | ▶ | use App::Chart; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 110 | use App::Chart::Database; | ||
| 111 | use App::Chart::Download; | ||
| 112 | use App::Chart::DownloadHandler; | ||
| 113 | use App::Chart::DownloadHandler::IndivChunks; | ||
| 114 | use App::Chart::DownloadHandler::IndivInfo; | ||
| 115 | use App::Chart::IntradayHandler; | ||
| 116 | use App::Chart::Latest; | ||
| 117 | use App::Chart::Sympred; | ||
| 118 | use App::Chart::TZ; | ||
| 119 | use App::Chart::Weblink; | ||
| 120 | |||
| 121 | # uncomment this to run the ### lines | ||
| 122 | # use Smart::Comments; | ||
| 123 | |||
| 124 | |||
| 125 | # .X or .XY or no suffix | ||
| 126 | our $yahoo_pred = App::Chart::Sympred::Proc->new | ||
| 127 | (sub { | ||
| 128 | my ($symbol) = @_; | ||
| 129 | return ($symbol !~ /\.(FQ|LJ)$/ | ||
| 130 | && $symbol =~ /[.=]..?$|^[^.]+$/); | ||
| 131 | }); | ||
| 132 | |||
| 133 | my $download_pred = App::Chart::Sympred::Any->new ($yahoo_pred); | ||
| 134 | our $latest_pred = App::Chart::Sympred::Any->new ($yahoo_pred); | ||
| 135 | our $index_pred = App::Chart::Sympred::Regexp->new (qr/^\^|^0.*\.SS$/); | ||
| 136 | my $futures_pred = App::Chart::Sympred::Any->new; | ||
| 137 | |||
| 138 | # overridden by specific nodes | ||
| 139 | App::Chart::setup_source_help | ||
| 140 | ($yahoo_pred, __p('manual-node','Yahoo Finance')); | ||
| 141 | |||
| 142 | |||
| 143 | #----------------------------------------------------------------------------- | ||
| 144 | # Weblink - basic quote page | ||
| 145 | # | ||
| 146 | # This is for instance | ||
| 147 | # https://finance.yahoo.com/quote/CSCO/ | ||
| 148 | # https://finance.yahoo.com/quote/BHP.AX/ | ||
| 149 | # | ||
| 150 | # There was a short time in July 2024 when seemed only the national | ||
| 151 | # page like au.finaince.yahoo.com worked for BHP.AX, and similar. | ||
| 152 | # Think that's no longer so. But the page is badly bloated by | ||
| 153 | # script and junk, so might not want to use. | ||
| 154 | # | ||
| 155 | # The accelerator key is "_Y" so as not to clash with "_S" for shares | ||
| 156 | # on various stock exchange links like "ASX IRM _Stock Information" | ||
| 157 | |||
| 158 | App::Chart::Weblink->new | ||
| 159 | (pred => $yahoo_pred, | ||
| 160 | name => __('_Yahoo Stock Page'), | ||
| 161 | desc => __('Open web browser at the Yahoo quote page for this stock'), | ||
| 162 | proc => sub { | ||
| 163 | my ($symbol) = @_; | ||
| 164 | return "https://" | ||
| 165 | . App::Chart::Database->preference_get ('yahoo-quote-host', | ||
| 166 | 'finance.yahoo.com') | ||
| 167 | . "/quote/" | ||
| 168 | . URI::Escape::uri_escape($symbol); | ||
| 169 | }); | ||
| 170 | |||
| 171 | |||
| 172 | #----------------------------------------------------------------------------- | ||
| 173 | # Quote Delays - Exchanges Page | ||
| 174 | # | ||
| 175 | # This uses the help page | ||
| 176 | # | ||
| 177 | use constant EXCHANGES_URL => | ||
| 178 | 'https://help.yahoo.com/kb/finance-app-for-android/exchanges-data-providers-yahoo-finance-sln2310.html'; | ||
| 179 | # | ||
| 180 | # which has a table of quote delays for exchanges, by suffix. | ||
| 181 | # | ||
| 182 | # Past URL was https://help.yahoo.com/kb/SLN2310.html and the | ||
| 183 | # same SLN part would suggest it's about everything, meaning both | ||
| 184 | # web site data and any Yahoo mobile phone applications. | ||
| 185 | # Distant past URL was http://finance.yahoo.com/exchanges | ||
| 186 | |||
| 187 | # Refetch the exchanges page after EXCHANGES_UPDATE_DAYS. | ||
| 188 | # The page is bloated by a script greatly exceeding the actual info, | ||
| 189 | # and doesn't offer a Last-Modified. Expect it changes infrequently. | ||
| 190 | # | ||
| 191 | use constant EXCHANGES_UPDATE_DAYS => 14; | ||
| 192 | |||
| 193 | # containing arefs [$pred,'.XX'] | ||
| 194 | my @quote_delay_aliases; | ||
| 195 | |||
| 196 | sub setup_quote_delay_alias { | ||
| 197 | my ($pred, $suffix) = @_; | ||
| 198 | push @quote_delay_aliases, [ $pred, $suffix ]; | ||
| 199 | } | ||
| 200 | |||
| 201 | sub symbol_quote_delay { | ||
| 202 | my ($symbol) = @_; | ||
| 203 | |||
| 204 | # indices all in real time | ||
| 205 | if ($index_pred->match($symbol)) { return 0; } | ||
| 206 | |||
| 207 | my $suffix = App::Chart::symbol_suffix ($symbol); | ||
| 208 | my $h = exchanges_data(); | ||
| 209 | my $delay = $h->{$suffix}; | ||
| 210 | |||
| 211 | if (! defined $delay) { | ||
| 212 | if (my $elem = List::Util::first { $_->[0]->match ($symbol) } | ||
| 213 | @quote_delay_aliases) { | ||
| 214 | $suffix = $elem->[1]; | ||
| 215 | $delay = $h->{$suffix}; | ||
| 216 | } | ||
| 217 | } | ||
| 218 | if (! defined $delay) { | ||
| 219 | # guess default 20 minutes | ||
| 220 | $delay = 20; | ||
| 221 | } | ||
| 222 | return $delay; | ||
| 223 | } | ||
| 224 | |||
| 225 | # exchanges_data() return a hashref of exchange delay data like | ||
| 226 | # { '.AX' => 20, '.BI' => 15 } | ||
| 227 | # which means .AX quotes are delayed by 20 minutes. | ||
| 228 | # | ||
| 229 | sub exchanges_data { | ||
| 230 | require App::Chart::Pagebits; | ||
| 231 | return App::Chart::Pagebits::get | ||
| 232 | (name => __('Yahoo exchanges page'), | ||
| 233 | url => EXCHANGES_URL, | ||
| 234 | key => 'yahoo-quote-delays', | ||
| 235 | freq_days => EXCHANGES_UPDATE_DAYS, | ||
| 236 | parse => \&exchanges_parse); | ||
| 237 | } | ||
| 238 | sub exchanges_parse { | ||
| 239 | my ($content) = @_; | ||
| 240 | my $h = {}; | ||
| 241 | |||
| 242 | require HTML::TableExtract; | ||
| 243 | my $te = HTML::TableExtract->new (headers => ['Suffix', 'Delay']); | ||
| 244 | $te->parse($content); | ||
| 245 | if (! $te->tables) { | ||
| 246 | warn "Yahoo exchanges page unrecognised, assuming 15 min quote delay"; | ||
| 247 | return $h; | ||
| 248 | } | ||
| 249 | |||
| 250 | foreach my $row ($te->rows) { | ||
| 251 | my $suffix = $row->[0]; | ||
| 252 | my $delay = $row->[1]; | ||
| 253 | next if ($suffix eq 'N/A'); | ||
| 254 | |||
| 255 | # eg "15 min" | ||
| 256 | # or "15 min**" with footnote | ||
| 257 | # | ||
| 258 | if ($delay =~ /^(\d+) min/) { | ||
| 259 | $delay = $1; | ||
| 260 | } elsif ($delay =~ /real/i) { | ||
| 261 | $delay = 0; | ||
| 262 | } else { | ||
| 263 | warn "Yahoo exchanges page unrecognised delay: \"$delay\"\n"; | ||
| 264 | next; | ||
| 265 | } | ||
| 266 | |||
| 267 | $h->{$suffix} = $delay + 0; | ||
| 268 | } | ||
| 269 | return $h; | ||
| 270 | } | ||
| 271 | |||
| 272 | |||
| 273 | #----------------------------------------------------------------------------- | ||
| 274 | # Info - Share Names | ||
| 275 | # | ||
| 276 | # This uses the info pages like | ||
| 277 | # | ||
| 278 | # https://query2.finance.yahoo.com/v1/finance/search?q=CSCO&enableFuzzyQuery=false | ||
| 279 | # | ||
| 280 | # which is a JSON format of company information like | ||
| 281 | # | ||
| 282 | # {"explains":[], | ||
| 283 | # "count":7, | ||
| 284 | # "quotes":[{"exchange":"NMS", | ||
| 285 | # "shortname":"Cisco Systems, Inc.", | ||
| 286 | # "quoteType":"EQUITY", | ||
| 287 | # "symbol":"CSCO", | ||
| 288 | # ... | ||
| 289 | # | ||
| 290 | # There can be multiple exchanges in the quotes list. Use the first. | ||
| 291 | # Other fields include | ||
| 292 | # | ||
| 293 | # longname occasionally shorter than shortname actually | ||
| 294 | # but use shortname | ||
| 295 | # | ||
| 296 | # exchDisp longer display name for the exchange, | ||
| 297 | # eg. exchange=NMS and exchDisp=NASDAQ. | ||
| 298 | # or exchange=ASX and exchDisp=Australian. | ||
| 299 | # Think the exchange code more useful. | ||
| 300 | # | ||
| 301 | # The URL has an optional &newsCount=0 for no news events. | ||
| 302 | # Think that's the default anyway. | ||
| 303 | |||
| 304 | |||
| 305 | # {"chart":{"result":null,"error":{"code":"Bad Request","description":"Data doesn't exist for startDate = 1565740800, endDate = 1596067200"}}} | ||
| 306 | |||
| 307 | |||
| 308 | App::Chart::DownloadHandler::IndivInfo->new | ||
| 309 | (name => __('Yahoo info'), | ||
| 310 | key => 'Yahoo-info', | ||
| 311 | pred => $download_pred, | ||
| 312 | url_func => \&info_url_func, | ||
| 313 | parse => \&info_parse, | ||
| 314 | recheck_days => 14); | ||
| 315 | |||
| 316 | sub info_url_func { | ||
| 317 | my ($symbol) = @_; | ||
| 318 | return 'https://query2.finance.yahoo.com/v1/finance/search?q=' | ||
| 319 | . URI::Escape::uri_escape ($symbol) | ||
| 320 | . '&enableFuzzyQuery=false'; | ||
| 321 | } | ||
| 322 | |||
| 323 | sub info_parse { | ||
| 324 | my ($symbol, $resp) = @_; | ||
| 325 | my @info; | ||
| 326 | my $h = { source => __PACKAGE__, | ||
| 327 | info => \@info }; | ||
| 328 | |||
| 329 | my $content = $resp->decoded_content (raise_error => 1); | ||
| 330 | my $json = JSON::decode_json($content) // {}; | ||
| 331 | my $quotes = $json->{'quotes'} // []; | ||
| 332 | my $e = $quotes->[0] // {}; | ||
| 333 | |||
| 334 | # Should have symbol in the data equal to $symbol requested. | ||
| 335 | # May want to be relaxed about that, as the Yahoo server allows | ||
| 336 | # different upper/lower case. | ||
| 337 | # | ||
| 338 | # my $quotes_symbol = $e->{'symbol'} // ''; | ||
| 339 | # if ($quotes_symbol ne $symbol) { | ||
| 340 | # die "Yahoo info: oops, wanted symbol $symbol got \"$quotes_symbol\""; | ||
| 341 | # } | ||
| 342 | |||
| 343 | my $name = $e->{'shortname'}; | ||
| 344 | if (defined $name) { | ||
| 345 | # ASX shares have the symbol repeated at the end of shortname, | ||
| 346 | # like "BHP FPO [BHP]". Seems unnecessary, so strip that. | ||
| 347 | my $end = '[' . App::Chart::symbol_sans_suffix ($symbol) .']'; | ||
| 348 | $name =~ s/\s*\Q$end\E$//; | ||
| 349 | } | ||
| 350 | |||
| 351 | return { symbol => $symbol, | ||
| 352 | name => $name, | ||
| 353 | exchange => $e->{'exchange'}, | ||
| 354 | }; | ||
| 355 | } | ||
| 356 | |||
| 357 | |||
| 358 | #------------------------------------------------------------------------------ | ||
| 359 | # Latest | ||
| 360 | # | ||
| 361 | # This uses for example | ||
| 362 | # | ||
| 363 | # https://query1.finance.yahoo.com/v7/finance/chart/BHP.AX?period1=1718841600&period2=1719532800&interval=1d&events=history&close=unadjusted | ||
| 364 | # | ||
| 365 | # periodi1 and period2 are Unix style seconds since 1 Jan 1970 GMT. | ||
| 366 | # | ||
| 367 | # https://stackoverflow.com/questions/47076404/currency-helper-of-yahoo-sorry-unable-to-process-request-at-this-time-erro | ||
| 368 | # -> | ||
| 369 | # https://stackoverflow.com/questions/47064776/has-yahoo-suddenly-today-terminated-its-finance-download-api | ||
| 370 | # | ||
| 371 | # FUTURE: Intending to switch this over to v8 the same as the daily | ||
| 372 | # data, and possibly a single common parse. But this latest quote | ||
| 373 | # way has survived without problem during cooking and crumb troubles, | ||
| 374 | # not don't need to rush to change what's working. | ||
| 375 | |||
| 376 | App::Chart::LatestHandler->new | ||
| 377 | (pred => $latest_pred, | ||
| 378 | proc => \&latest_download, | ||
| 379 | max_symbols => 1); # downloads go 1 at a time | ||
| 380 | |||
| 381 | sub latest_download { | ||
| 382 | my ($symbol_list) = @_; | ||
| 383 | |||
| 384 | foreach my $symbol (@$symbol_list) { | ||
| 385 | my $tdate = daily_available_tdate ($symbol); | ||
| 386 | App::Chart::Download::status(__('Yahoo quote'), $symbol); | ||
| 387 | |||
| 388 | my $lo_timet = tdate_to_unix($tdate - 4); | ||
| 389 | my $hi_timet = tdate_to_unix($tdate + 2); | ||
| 390 | |||
| 391 | my $events = 'history'; | ||
| 392 | my $url = "https://query1.finance.yahoo.com/v7/finance/chart/" | ||
| 393 | . URI::Escape::uri_escape($symbol) | ||
| 394 | ."?period1=$lo_timet" | ||
| 395 | ."&period2=$hi_timet" | ||
| 396 | ."&interval=1d" | ||
| 397 | ."&events=$events" | ||
| 398 | ."&close=unadjusted"; | ||
| 399 | |||
| 400 | # unknown symbol is 404 with JSON error details | ||
| 401 | # | ||
| 402 | my $resp = App::Chart::Download->get ($url, allow_404 => 1,); | ||
| 403 | App::Chart::Download::write_latest_group | ||
| 404 | (latest_parse($symbol,$resp,$tdate)); | ||
| 405 | } | ||
| 406 | } | ||
| 407 | |||
| 408 | sub latest_parse { | ||
| 409 | my ($symbol, $resp, $tdate) = @_; | ||
| 410 | my $content = $resp->decoded_content (raise_error => 1); | ||
| 411 | my $json = JSON::from_json($content); | ||
| 412 | |||
| 413 | my %record = (symbol => $symbol, | ||
| 414 | ); | ||
| 415 | my $h = { source => __PACKAGE__, | ||
| 416 | resp => $resp, | ||
| 417 | prefer_decimals => 2, | ||
| 418 | date_format => 'ymd', | ||
| 419 | data => [ \%record ], | ||
| 420 | }; | ||
| 421 | if (defined (my $error = $json->{'chart'}->{'error'}->{'code'})) { | ||
| 422 | $record{'error'} = $error; | ||
| 423 | } | ||
| 424 | |||
| 425 | if (my $result = $json->{'chart'}->{'result'}->[0]) { | ||
| 426 | my $meta = $result->{'meta'} | ||
| 427 | // die "Yahoo JSON oops, no meta"; | ||
| 428 | $record{'currency'} = $meta->{'currency'}; | ||
| 429 | $record{'exchange'} = $meta->{'exchangeName'}; | ||
| 430 | my $symbol_timezone = App::Chart::TZ->for_symbol ($symbol); | ||
| 431 | |||
| 432 | # Delisted shares are known symbols and currency, | ||
| 433 | # possibly junk name and exchange, | ||
| 434 | # and no data which means no timestamp field exists. | ||
| 435 | my $timestamps = $result->{'timestamp'} // []; | ||
| 436 | if (@$timestamps) { | ||
| 437 | |||
| 438 | # timestamps are time of last trade, as can be seen by looking at | ||
| 439 | # something with low enough volume, eg. RMX.AX | ||
| 440 | # | ||
| 441 | if (defined (my $timet = $timestamps->[-1])) { | ||
| 442 | ($record{'last_date'}, $record{'last_time'}) | ||
| 443 | = $symbol_timezone->iso_date_time($timet); | ||
| 444 | } | ||
| 445 | |||
| 446 | if (my $indicators = $result->{'indicators'}->{'quote'}->[0]) { | ||
| 447 | foreach my $key ('open','high','low') { | ||
| 448 | if (my $aref = $indicators->{$key}) { | ||
| 449 | $record{$key} = crunch_trailing_nines($aref->[$#$timestamps]); | ||
| 450 | } | ||
| 451 | } | ||
| 452 | if (my $aref = $indicators->{'volume'}) { | ||
| 453 | $record{'volume'} = $aref->[$#$timestamps]; | ||
| 454 | } | ||
| 455 | if (my $aref = $indicators->{'close'}) { | ||
| 456 | my $last = $record{'last'} | ||
| 457 | = crunch_trailing_nines($aref->[$#$timestamps]); | ||
| 458 | |||
| 459 | # "change" from second last timestamp, if there is one. | ||
| 460 | # As of Nov 2017, XAUUSD=X only ever gives a single latest | ||
| 461 | # quote from v7, no previous day to compare. | ||
| 462 | # | ||
| 463 | if (defined $last | ||
| 464 | && scalar(@$timestamps) >= 2 | ||
| 465 | && defined(my $prev = $aref->[$#$timestamps - 1])) { | ||
| 466 | $record{'change'} | ||
| 467 | = App::Chart::decimal_sub($last, crunch_trailing_nines($prev)); | ||
| 468 | } | ||
| 469 | } | ||
| 470 | } | ||
| 471 | } | ||
| 472 | |||
| 473 | if (defined $record{'last_date'} | ||
| 474 | && (my $splits = $result->{'events'}->{'splits'})) { | ||
| 475 | while (my ($timet, $href) = each %$splits) { | ||
| 476 | my $split_date = $symbol_timezone->iso_date($timet); | ||
| 477 | if ($split_date eq $record{'last_date'}) { | ||
| 478 | __x('Split {ratio}', ratio => $href->{'splitRatio'}) | ||
| 479 | } | ||
| 480 | } | ||
| 481 | } | ||
| 482 | } | ||
| 483 | return $h; | ||
| 484 | } | ||
| 485 | |||
| 486 | |||
| 487 | #----------------------------------------------------------------------------- | ||
| 488 | # Download Data, including dividends and splits | ||
| 489 | # | ||
| 490 | # This uses the "v8" historical prices downloads in JSON format like | ||
| 491 | # | ||
| 492 | # https://query2.finance.yahoo.com/v8/finance/chart/IBM?period1=1504028419&period2=1504428419&interval=1d&events=div%7Csplit&close=unadjusted | ||
| 493 | # | ||
| 494 | # period1 is the start time, period2 the end time, both as Unix | ||
| 495 | # seconds since 1 Jan 1970 in GMT. | ||
| 496 | # | ||
| 497 | # close=unadjusted means prices are without any adjustment for | ||
| 498 | # splits, so prices as traded at the time. | ||
| 499 | # | ||
| 500 | # If no trading in the date range (eg. before first listing) then | ||
| 501 | # | ||
| 502 | # 400 Bad Request | ||
| 503 | # {"chart":{"result":null,"error":{"code":"Bad Request","description":"Data doesn't exist for startDate = 1565827200, endDate = 1596153600"}}} | ||
| 504 | # | ||
| 505 | |||
| 506 | # One download for each symbol. | ||
| 507 | # Date ranges in limited size chunks. | ||
| 508 | # (Don't know whether Yahoo has a limit on size of download, | ||
| 509 | # but let's try not to discover one.) | ||
| 510 | # | ||
| 511 | App::Chart::DownloadHandler::IndivChunks->new | ||
| 512 | (name => __('Yahoo'), | ||
| 513 | pred => $download_pred, | ||
| 514 | url_func => \&daily_url_func, | ||
| 515 | parse => \&daily_parse, | ||
| 516 | allow_http_codes => [400,404], | ||
| 517 | chunk_size => 250, # about 1 year at a time | ||
| 518 | available_tdate_by_symbol => \&daily_available_tdate, | ||
| 519 | available_tdate_extra => 2, | ||
| 520 | ); | ||
| 521 | |||
| 522 | sub daily_available_tdate { | ||
| 523 | my ($symbol) = @_; | ||
| 524 | |||
| 525 | # As of September 2017, daily data is present for the current | ||
| 526 | # day's trade, during the trading session. | ||
| 527 | # Try reckoning it complete at 6pm. | ||
| 528 | return App::Chart::Download::tdate_today_after | ||
| 529 | (18,0, App::Chart::TZ->for_symbol ($symbol)); | ||
| 530 | } | ||
| 531 | |||
| 532 | sub daily_url_func { | ||
| 533 | my ($symbol, $lo_tdate, $hi_tdate) = @_; | ||
| 534 | my $lo_timet = tdate_to_unix($lo_tdate - 2); | ||
| 535 | my $hi_timet = tdate_to_unix($hi_tdate); | ||
| 536 | |||
| 537 | # As of September 2024, dividends only appear on (or after?) | ||
| 538 | # the ex date. But try hi_timet well ahead hoping for | ||
| 539 | # upcoming dividends (ex date announced). | ||
| 540 | if ($hi_tdate >= daily_available_tdate($symbol)) { | ||
| 541 | $hi_timet += 60 * 86400; | ||
| 542 | } | ||
| 543 | |||
| 544 | return "https://query1.finance.yahoo.com/v8/finance/chart/" | ||
| 545 | . URI::Escape::uri_escape($symbol) | ||
| 546 | ."?formatted=true&lang=en-US®ion=US" | ||
| 547 | ."&period1=$lo_timet" | ||
| 548 | ."&period2=$hi_timet" | ||
| 549 | ."&interval=1d" | ||
| 550 | ."&events=". URI::Escape::uri_escape('div|split') | ||
| 551 | ."&close=unadjusted"; | ||
| 552 | } | ||
| 553 | |||
| 554 | # $resp is a HTTP::Response object with is Yahoo v8 JSON data for $symbol.. | ||
| 555 | # Return $h which is a write_daily_group() style hashref of the data. | ||
| 556 | # | ||
| 557 | sub daily_parse { | ||
| 558 | my ($symbol, $resp) = @_; | ||
| 559 | my $hi_tdate = daily_available_tdate ($symbol); | ||
| 560 | my @data = (); | ||
| 561 | my $h = { source => __PACKAGE__, | ||
| 562 | prefer_decimals => 2, # default | ||
| 563 | date_format => 'ymd', | ||
| 564 | data => \@data, | ||
| 565 | }; | ||
| 566 | |||
| 567 | my $content = $resp->decoded_content (raise_error => 1); | ||
| 568 | my $json = JSON::decode_json($content); | ||
| 569 | my $result = $json->{'chart'}->{'result'}->[0]; | ||
| 570 | |||
| 571 | my $meta = $result->{'meta'} // {}; | ||
| 572 | |||
| 573 | # Should have symbol in the data equal to $symbol requested. | ||
| 574 | # May want to be relaxed about that, as the Yahoo server allows | ||
| 575 | # different upper/lower case. The intention is to have all | ||
| 576 | # symbols in the database as a canonical form. | ||
| 577 | # | ||
| 578 | # my $meta_symbol = $meta->{'symbol'} // ''; | ||
| 579 | # if ($meta_symbol ne $symbol) { | ||
| 580 | # die "Yahoo JSON oops, symbol wanted $symbol got $meta_symbol"; | ||
| 581 | # } | ||
| 582 | |||
| 583 | # Trading in pence Sterling is "GBp", such as TSCO.L | ||
| 584 | $h->{'currencies'}->{$symbol} = $meta->{'currency'}; | ||
| 585 | $h->{'exchanges'}->{$symbol} = $meta->{'exchangeName'}; | ||
| 586 | my $decimals = $meta->{'priceHint'}; | ||
| 587 | if (defined $decimals) { | ||
| 588 | $h->{'prefer_decimals'} = $decimals; | ||
| 589 | } | ||
| 590 | $decimals //= 2; | ||
| 591 | |||
| 592 | my $gmtoffset = $meta->{'gmtoffset'}; | ||
| 593 | my $timet_to_iso = sub { | ||
| 594 | my ($t) = @_; | ||
| 595 | return (defined $t ? POSIX::strftime('%Y-%m-%d', gmtime($t + $gmtoffset)) : undef); | ||
| 596 | }; | ||
| 597 | |||
| 598 | my $timestamps = $result->{'timestamp'}; | ||
| 599 | my $quote= $result->{'indicators'}->{'quote'}->[0]; | ||
| 600 | my $opens = $quote->{'open'} // []; | ||
| 601 | my $highs = $quote->{'high'} // []; | ||
| 602 | my $lows = $quote->{'low'} // []; | ||
| 603 | my $closes = $quote->{'close'} // []; | ||
| 604 | my $volumes = $quote->{'volume'} // []; | ||
| 605 | foreach my $i (0 .. $#$timestamps) { | ||
| 606 | my $date = $timet_to_iso->($timestamps->[$i]); | ||
| 607 | if (App::Chart::Download::iso_to_tdate_floor($date) > $hi_tdate) { | ||
| 608 | # Current day's trading shows in the data. | ||
| 609 | # Don't enter it in the database until close of trade. | ||
| 610 | next; | ||
| 611 | } | ||
| 612 | push @data, { symbol => $symbol, | ||
| 613 | date => $date, | ||
| 614 | open => crunch_trailing_nines($opens->[$i]), | ||
| 615 | high => crunch_trailing_nines($highs->[$i]), | ||
| 616 | low => crunch_trailing_nines($lows->[$i]), | ||
| 617 | close => crunch_trailing_nines($closes->[$i]), | ||
| 618 | volume => $volumes->[$i] }; | ||
| 619 | } | ||
| 620 | |||
| 621 | my $events = $result->{'events'} // {}; | ||
| 622 | ### $events | ||
| 623 | |||
| 624 | # Eg. BHP.AX | ||
| 625 | # date 1709766000 | ||
| 626 | # amount 1.096196 | ||
| 627 | # | ||
| 628 | foreach my $dividend (values %{$events->{'dividends'} // {}}) { | ||
| 629 | push @{$h->{'dividends'}}, | ||
| 630 | { symbol => $symbol, | ||
| 631 | ex_date => $timet_to_iso->($dividend->{'date'}), | ||
| 632 | amount => $dividend->{'amount'}, | ||
| 633 | }; | ||
| 634 | } | ||
| 635 | |||
| 636 | # eg. 1:30 split from NKLA.MX on 21 Jun 2024 is like | ||
| 637 | # date 1719322200 | ||
| 638 | # numerator 1 | ||
| 639 | # denominator 30 | ||
| 640 | # splitRatio 1:30 | ||
| 641 | # closing price (unadjusted) drops from 193.80 to 6.47 | ||
| 642 | # new 30 shares = denominator | ||
| 643 | # older prices when displayed adjusted are factor | ||
| 644 | # old/new = numerator/denominator | ||
| 645 | # | ||
| 646 | foreach my $split (values %{$events->{'splits'} // {}}) { | ||
| 647 | push @{$h->{'splits'}}, | ||
| 648 | { symbol => $symbol, | ||
| 649 | date => $timet_to_iso->($split->{'date'}), | ||
| 650 | new => $split->{'denominator'}, | ||
| 651 | old => $split->{'numerator'}, | ||
| 652 | }; | ||
| 653 | } | ||
| 654 | |||
| 655 | return $h; | ||
| 656 | } | ||
| 657 | |||
| 658 | # $str is a string like "30.299999" | ||
| 659 | # Return it with trailing 9s turned into trailing 0s. | ||
| 660 | sub crunch_trailing_nines { | ||
| 661 | my ($str) = @_; | ||
| 662 | |||
| 663 | # ENHANCE-ME: The digits are 23-bit float formatted badly, or so it seems. | ||
| 664 | # That makes it 23 / (log(10)/log(2)) = 6.92 many high digits good. | ||
| 665 | # Maybe round-to-nearest of high 7 digits, or maybe only high 6 depending | ||
| 666 | # where the bit range falls (how many bits used by the high digit). | ||
| 667 | |||
| 668 | if (defined $str) { | ||
| 669 | $str =~ s/(\....(99|00)).*/$1/; # trailing garbage | ||
| 670 | |||
| 671 | if ($str =~ /(.*)\.(....9+)$/) { | ||
| 672 | $str = decimal_add_low($str,1); | ||
| 673 | } elsif ($str =~ /(.*)\.(....*01)$/) { | ||
| 674 | $str = decimal_add_low($str,-1); | ||
| 675 | } | ||
| 676 | |||
| 677 | if ($str =~ /(.*)\./) { | ||
| 678 | my $ilen = length($1); | ||
| 679 | my $decimals = ($ilen >= 4 ? 2 | ||
| 680 | : $ilen == 3 ? 3 | ||
| 681 | : 4); | ||
| 682 | $str = round_decimals($str,$decimals); | ||
| 683 | } | ||
| 684 | $str = pad_decimals($str, 2); | ||
| 685 | } | ||
| 686 | return $str; | ||
| 687 | } | ||
| 688 | sub decimal_add_low { | ||
| 689 | my ($str, $add) = @_; | ||
| 690 | ### decimal_add_low(): "$str add $add" | ||
| 691 | $str =~ /(.*)\.(.+)$/ or return $str+$add; | ||
| 692 | my $pre = $1; | ||
| 693 | my $post = $2; | ||
| 694 | ### $pre | ||
| 695 | ### $post | ||
| 696 | $str = $pre * 10**length($post) + $post + $add; | ||
| 697 | while (length($post) >= length($str)) { $str = '0'.$str; } | ||
| 698 | substr($str, -length($post),0, '.'); | ||
| 699 | return $str; | ||
| 700 | } | ||
| 701 | sub round_decimals { | ||
| 702 | my ($str, $decimals) = @_; | ||
| 703 | if (defined $str && $str =~ /(.*\.[0-9]{$decimals})([0-9])/) { | ||
| 704 | $str = $1; | ||
| 705 | if ($2 >= 5) { $str = decimal_add_low($str, 1); } | ||
| 706 | } | ||
| 707 | return $str; | ||
| 708 | } | ||
| 709 | sub pad_decimals { | ||
| 710 | my ($str, $decimals) = @_; | ||
| 711 | ### pad_decimals(): "$str $decimals" | ||
| 712 | my $got; | ||
| 713 | if ($str =~ /\.(.*)/) { | ||
| 714 | $got = length($1); | ||
| 715 | } else { | ||
| 716 | $got = 0; | ||
| 717 | $str .= '.'; | ||
| 718 | } | ||
| 719 | if ((my $add = $decimals - $got) > 0) { | ||
| 720 | $str .= '0' x $add; | ||
| 721 | } | ||
| 722 | return $str; | ||
| 723 | } | ||
| 724 | |||
| 725 | # Return seconds since 00:00:00, 1 Jan 1970 GMT. | ||
| 726 | sub tdate_to_unix { | ||
| 727 | my ($tdate) = @_; | ||
| 728 | my $adate = App::Chart::tdate_to_adate ($tdate); | ||
| 729 | return ($adate + 4)*86400; | ||
| 730 | } | ||
| 731 | |||
| 732 | #------------------------------------------------------------------------------ | ||
| 733 | 1; | ||
| 734 | __END__ | ||