| 1 | # Reserve Bank of Australia setups. | ||
| 2 | |||
| 3 | # Copyright 2007, 2008, 2009, 2010, 2011, 2012, 2016, 2019 Kevin Ryde | ||
| 4 | |||
| 5 | # This file is part of Chart. | ||
| 6 | # | ||
| 7 | # Chart is free software; you can redistribute it and/or modify it under the | ||
| 8 | # terms of the GNU General Public License as published by the Free Software | ||
| 9 | # Foundation; either version 3, or (at your option) any later version. | ||
| 10 | # | ||
| 11 | # Chart is distributed in the hope that it will be useful, but WITHOUT ANY | ||
| 12 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
| 13 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | ||
| 14 | # details. | ||
| 15 | # | ||
| 16 | # You should have received a copy of the GNU General Public License along | ||
| 17 | # with Chart. If not, see <http://www.gnu.org/licenses/>. | ||
| 18 | |||
| 19 | package App::Chart::Suffix::RBA; | ||
| 20 | 1 | ▶ | use 5.010; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 21 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 22 | 1 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 23 | 1 | ▶ | use Carp; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 24 | 1 | ▶ | use Date::Calc; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 25 | 1 | ▶ | use List::Util qw(min max); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 26 | 1 | ▶ | use Set::IntSpan::Fast; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 27 | 1 | ▶ | use Locale::TextDomain ('App-Chart'); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 28 | |||
| 29 | 0 | ▶ | use App::Chart; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 30 | use App::Chart::Database; | ||
| 31 | use App::Chart::Download; | ||
| 32 | use App::Chart::DownloadHandler; | ||
| 33 | use App::Chart::Sympred; | ||
| 34 | use App::Chart::Latest; | ||
| 35 | use App::Chart::TZ; | ||
| 36 | use App::Chart::Weblink; | ||
| 37 | |||
| 38 | # uncomment this to run the ### lines | ||
| 39 | # use Smart::Comments; | ||
| 40 | |||
| 41 | # Not yet using Finance::Quote::RBA | ||
| 42 | |||
| 43 | my $pred = App::Chart::Sympred::Suffix->new ('.RBA'); | ||
| 44 | App::Chart::TZ->sydney->setup_for_symbol ($pred); | ||
| 45 | App::Chart::setup_source_help | ||
| 46 | ($pred, __p('manual-node','Reserve Bank of Australia')); | ||
| 47 | |||
| 48 | use constant RBA_COPYRIGHT_URL => | ||
| 49 | 'https://www.rba.gov.au/copyright/'; | ||
| 50 | |||
| 51 | |||
| 52 | #------------------------------------------------------------------------------ | ||
| 53 | # weblink - home page | ||
| 54 | |||
| 55 | App::Chart::Weblink->new | ||
| 56 | (pred => $pred, | ||
| 57 | name => __('_RBA Home Page'), | ||
| 58 | desc => __('Open web browser at the Reserve Bank of Australia home page'), | ||
| 59 | url => 'https://www.rba.gov.au'); | ||
| 60 | |||
| 61 | |||
| 62 | #------------------------------------------------------------------------------ | ||
| 63 | # three day page | ||
| 64 | # | ||
| 65 | # This uses the rates at: | ||
| 66 | # | ||
| 67 | use constant RBA_EXCHANGE_URL => | ||
| 68 | 'https://www.rba.gov.au/statistics/frequency/exchange-rates.html'; | ||
| 69 | use constant RBA_EXCHANGE_URL_DAYS => 3; | ||
| 70 | |||
| 71 | # would it be a few minutes after 4pm ? | ||
| 72 | sub threeday_available_date_time { | ||
| 73 | return (App::Chart::Download::weekday_date_after_time | ||
| 74 | (16,0, App::Chart::TZ->sydney), | ||
| 75 | '16:01:00'); | ||
| 76 | } | ||
| 77 | |||
| 78 | sub threeday_available_tdate { | ||
| 79 | my ($iso, $time) = threeday_available_date_time(); | ||
| 80 | return App::Chart::Download::iso_to_tdate_floor ($iso); | ||
| 81 | } | ||
| 82 | |||
| 83 | sub threeday_parse { | ||
| 84 | my ($resp) = @_; | ||
| 85 | my @data = (); | ||
| 86 | my $h = { url => RBA_EXCHANGE_URL, | ||
| 87 | copyright => RBA_COPYRIGHT_URL, | ||
| 88 | source => __PACKAGE__, | ||
| 89 | resp => $resp, | ||
| 90 | cover_pred => $pred, | ||
| 91 | data => \@data }; | ||
| 92 | |||
| 93 | my $content = $resp->decoded_content(raise_error=>1); | ||
| 94 | |||
| 95 | # mung <tr id="USD"> to add <td>USD</td> so it appears in the TableExtract | ||
| 96 | $content =~ s{<tr>}{<tr><td></td>}ig; | ||
| 97 | $content =~ s{(<tr +id="([^"]*)">)}{$1<td>$2</td>}ig; | ||
| 98 | |||
| 99 | require HTML::TableExtract; | ||
| 100 | my $te = HTML::TableExtract->new | ||
| 101 | ( | ||
| 102 | # is now a <caption> | ||
| 103 | # headers => ['Units of foreign currency per'], | ||
| 104 | slice_columns => 0); | ||
| 105 | $te->parse($content); | ||
| 106 | my $ts = $te->first_table_found(); | ||
| 107 | if (! $ts) { die "RBA: html table not found\n"; } | ||
| 108 | |||
| 109 | my $rows = $ts->rows(); | ||
| 110 | my $lastrow = $#$rows; | ||
| 111 | my $lastcol = $#{$rows->[0]}; | ||
| 112 | |||
| 113 | # date like "03 Sep 2007" | ||
| 114 | my @dates; | ||
| 115 | foreach my $c (2 .. $lastcol) { | ||
| 116 | $dates[$c] = App::Chart::Download::Decode_Date_EU_to_iso($rows->[0]->[$c]); | ||
| 117 | } | ||
| 118 | $h->{'lo_date'} = List::Util::minstr (grep {defined} @dates); | ||
| 119 | |||
| 120 | foreach my $r (1 .. $lastrow) { | ||
| 121 | my $row = $rows->[$r]; | ||
| 122 | |||
| 123 | my $symbol = $row->[0] // next; | ||
| 124 | $symbol =~ s/_.*//; # _4pm on TWI | ||
| 125 | $symbol = "AUD$symbol.RBA"; | ||
| 126 | |||
| 127 | my $name = $row->[1]; | ||
| 128 | $name =~ s/ \(4pm\)$//; # 4pm on TWI | ||
| 129 | |||
| 130 | foreach my $c (2 .. $lastcol) { | ||
| 131 | my $rate = $row->[$c]; | ||
| 132 | # bank holiday columns have "BANK HOLIDAY" with one letter per row or | ||
| 133 | # blank which comes through as undef, skip those | ||
| 134 | next if ! Scalar::Util::looks_like_number($rate); | ||
| 135 | |||
| 136 | push @data, { symbol => $symbol, | ||
| 137 | name => $name, | ||
| 138 | date => $dates[$c], | ||
| 139 | last_time => '16:00:00', | ||
| 140 | close => $rate, | ||
| 141 | currency => substr($symbol,3,3), | ||
| 142 | }; | ||
| 143 | } | ||
| 144 | } | ||
| 145 | |||
| 146 | return $h; | ||
| 147 | } | ||
| 148 | |||
| 149 | #------------------------------------------------------------------------------ | ||
| 150 | # latest quotes | ||
| 151 | |||
| 152 | App::Chart::LatestHandler->new | ||
| 153 | (pred => $pred, | ||
| 154 | url_tags_key => 'RBA-latest', | ||
| 155 | proc => \&latest_download, | ||
| 156 | available_date_time => \&threeday_available_date_time); | ||
| 157 | |||
| 158 | sub latest_download { | ||
| 159 | my ($symbol_list) = @_; | ||
| 160 | |||
| 161 | App::Chart::Download::status (__('RBA past three days')); | ||
| 162 | my $resp = App::Chart::Download->get (RBA_EXCHANGE_URL, | ||
| 163 | url_tags_key => 'RBA-latest'); | ||
| 164 | App::Chart::Download::write_latest_group (threeday_parse ($resp)); | ||
| 165 | } | ||
| 166 | |||
| 167 | |||
| 168 | #------------------------------------------------------------------------------ | ||
| 169 | # historical xls page | ||
| 170 | # | ||
| 171 | # This downloads and parses up the page: | ||
| 172 | # https://www.rba.gov.au/statistics/historical-data.html | ||
| 173 | # | ||
| 174 | # 2014 to present | ||
| 175 | # https://www.rba.gov.au/statistics/tables/csv/f11.1-data.csv | ||
| 176 | # 213k or 47k compressed, also byte ranges | ||
| 177 | # https://www.rba.gov.au/statistics/tables/xls-hist/2014-current.xls | ||
| 178 | # 438k | ||
| 179 | # | ||
| 180 | use constant RBA_HISTORICAL_PAGE_URL => | ||
| 181 | 'https://www.rba.gov.au/statistics/hist-exchange-rates/index.html'; | ||
| 182 | # | ||
| 183 | # which offers various xls files for past rates. | ||
| 184 | |||
| 185 | sub historical_info { | ||
| 186 | require App::Chart::Pagebits; | ||
| 187 | return App::Chart::Pagebits::get | ||
| 188 | (name => __('RBA historical page'), | ||
| 189 | url => RBA_HISTORICAL_PAGE_URL, | ||
| 190 | key => 'rba-historical', | ||
| 191 | freq_days => 1, | ||
| 192 | parse => \&historical_parse); | ||
| 193 | } | ||
| 194 | |||
| 195 | sub historical_parse { | ||
| 196 | my ($content) = @_; | ||
| 197 | |||
| 198 | # Eg. | ||
| 199 | # <a href="/statistics/hist-exchange-rates/2007-2009.xls" | ||
| 200 | # target="_blank" title="Link, opening in a new window, to XLS file."> | ||
| 201 | # 2007 to 2009</a> <span class="nonHtml">[XLS 227K]</span> | ||
| 202 | # | ||
| 203 | # because the size is outside the link it doesn't really suit | ||
| 204 | # HTML::LinkExtor / HTML::Parser etc | ||
| 205 | # | ||
| 206 | my @files; | ||
| 207 | while ($content =~ m%href=\"([^\"]*[0-9][0-9][0-9][0-9]\.xls)\" | ||
| 208 | (.*\n){0,3} | ||
| 209 | .*\[XLS\s*([0-9.]+)([MK]) | ||
| 210 | %igmx) { | ||
| 211 | my $link = $1; | ||
| 212 | my $size = $3; | ||
| 213 | my $size_units = uc($4); | ||
| 214 | |||
| 215 | if ($size_units eq 'K') { $size *= 1_000 } | ||
| 216 | if ($size_units eq 'M') { $size *= 1_000_000 } | ||
| 217 | |||
| 218 | my $uri = URI->new_abs($link, RBA_HISTORICAL_PAGE_URL); | ||
| 219 | |||
| 220 | # eg per above: 2003-2007.xls | ||
| 221 | # or just: 2007.xls | ||
| 222 | $link =~ /([0-9][0-9][0-9][0-9])((to|-)([0-9][0-9][0-9][0-9]))?\.xls$/ | ||
| 223 | or die "RBA: oops, unrecognised link: $link"; | ||
| 224 | my $lo_year = $1; | ||
| 225 | my $hi_year = $4 || $1; | ||
| 226 | |||
| 227 | push @files, { url => $uri->as_string, | ||
| 228 | cost => $size, | ||
| 229 | lo_year => $lo_year, | ||
| 230 | hi_year => $hi_year, | ||
| 231 | }; | ||
| 232 | } | ||
| 233 | return { files => \@files }; | ||
| 234 | } | ||
| 235 | |||
| 236 | |||
| 237 | #----------------------------------------------------------------------------- | ||
| 238 | # download - historical monthly prices | ||
| 239 | # | ||
| 240 | # This parses the monthly rates spreadsheet file from the | ||
| 241 | # RBA_HISTORICAL_PAGE_URL page above, | ||
| 242 | # | ||
| 243 | # https://www.rba.gov.au/statistics/tables/xls/f11hist-1969-2009.xls | ||
| 244 | # | ||
| 245 | # but only the part from 1983 back is wanted since there's daily data for | ||
| 246 | # 1983 onwards. | ||
| 247 | |||
| 248 | my %monthly_fx_to_currency | ||
| 249 | = ( | ||
| 250 | # 'TWI' # trade weighted index | ||
| 251 | 'CR' => 'CNY', # chinese renminbi | ||
| 252 | 'JY' => 'JPY', # japanese yen | ||
| 253 | # 'EUR' | ||
| 254 | # 'USD' | ||
| 255 | 'SKW' => 'KRW', # South Korean won | ||
| 256 | 'UKPS' => 'GBP', # british pound sterling | ||
| 257 | 'SD' => 'SGD', # singapore dollar | ||
| 258 | 'IRE' => 'INR', # Indian rupee | ||
| 259 | 'TB' => 'THB', # Thai baht | ||
| 260 | # 'NZD' | ||
| 261 | 'NTD' => 'TWD', # taiwan dollar | ||
| 262 | 'MR' => 'MYR', # malaysian ringgit | ||
| 263 | 'IR' => 'IDR', # indonesian rupiah | ||
| 264 | 'VD' => 'VND', # Vietnamese dong | ||
| 265 | 'UAED' => 'AED', # UAE dirham | ||
| 266 | 'PNGK' => 'PGK', # PNG kina | ||
| 267 | # 'HKD' # Hong Kong dollar | ||
| 268 | 'CD' => 'CAD', # Canadian dollar | ||
| 269 | 'SARD' => 'ZAR', # South African rand | ||
| 270 | 'SARY' => 'SAR', # Saudi riyal | ||
| 271 | 'SF' => 'CHF', # Swiss franc | ||
| 272 | 'SK' => 'SEK', # Swedish krona | ||
| 273 | # 'SDR' # special drawing right | ||
| 274 | ); | ||
| 275 | |||
| 276 | sub monthly_parse { | ||
| 277 | my ($resp, $stop_iso) = @_; | ||
| 278 | ### RBA monthly_parse() ... | ||
| 279 | my $content = $resp->decoded_content(raise_error=>1); | ||
| 280 | |||
| 281 | my @data = (); | ||
| 282 | my $h = { source => __PACKAGE__, | ||
| 283 | copyright => RBA_COPYRIGHT_URL, | ||
| 284 | data => \@data }; | ||
| 285 | |||
| 286 | require Spreadsheet::ParseExcel; | ||
| 287 | require Spreadsheet::ParseExcel::Utility; | ||
| 288 | |||
| 289 | my $excel = Spreadsheet::ParseExcel::Workbook->Parse (\$content); | ||
| 290 | my $sheet = $excel->Worksheet (0); | ||
| 291 | ### SheetCount: $excel->{'SheetCount'} | ||
| 292 | ### Name: $sheet->{'Name'} | ||
| 293 | |||
| 294 | my ($minrow, $maxrow) = $sheet->RowRange; | ||
| 295 | my ($mincol, $maxcol) = $sheet->ColRange; | ||
| 296 | ### rows: $minrow, $maxrow | ||
| 297 | ### cols: $mincol, $maxcol | ||
| 298 | |||
| 299 | # heading row repeats the filename "F11HIST.XLS" and then the currencies | ||
| 300 | # in columns as say "FXRJY" | ||
| 301 | my $heading_row = List::Util::first { | ||
| 302 | my $cell = $sheet->Cell($_,$mincol); | ||
| 303 | $cell && $cell->Value eq 'F11HIST.XLS' } | ||
| 304 | ($minrow .. $maxrow) | ||
| 305 | or die "RBA monthly: headings not found"; | ||
| 306 | ### $heading_row | ||
| 307 | |||
| 308 | my @currencies = map { | ||
| 309 | my $cell = $sheet->Cell($heading_row,$_); | ||
| 310 | my $currency = $cell ? $cell->Value : ''; | ||
| 311 | $currency =~ s/^FXR//; | ||
| 312 | ($monthly_fx_to_currency{$currency} || $currency) | ||
| 313 | } ($mincol .. $maxcol); | ||
| 314 | ### @currencies | ||
| 315 | ### count: scalar(@currencies) | ||
| 316 | |||
| 317 | my %currency_started; | ||
| 318 | |||
| 319 | ROW: foreach my $row ($heading_row+1 .. $maxrow) { | ||
| 320 | my $datecell = $sheet->Cell($row,0) or next; | ||
| 321 | # seen 'Numeric', but presumably 'Date' is ok | ||
| 322 | if ($datecell->{'Type'} ne 'Numeric' | ||
| 323 | && $datecell->{'Type'} ne 'Date') { | ||
| 324 | next; # skip blanks at end | ||
| 325 | } | ||
| 326 | my $month = Spreadsheet::ParseExcel::Utility::ExcelFmt | ||
| 327 | ('yyyy-mm-dd', $datecell->{'Val'}, $excel->{'Flg1904'}); | ||
| 328 | |||
| 329 | foreach my $col ($mincol+1 .. $maxcol) { | ||
| 330 | my $currency = $currencies[$col-$mincol] or next; | ||
| 331 | my $ratecell = $sheet->Cell($row,$col) or next; | ||
| 332 | my $rate = $ratecell->Value; | ||
| 333 | |||
| 334 | # avoid empty records until the start of data for a given currency is | ||
| 335 | # reached | ||
| 336 | if (! $rate && ! $currency_started{$currency}) { next; } | ||
| 337 | |||
| 338 | my $symbol = "AUD$currency.RBA"; | ||
| 339 | $currency_started{$currency} = 1; | ||
| 340 | foreach my $date (iso_weekdays_in_month ($month)) { | ||
| 341 | if ($date gt $stop_iso) { last ROW; } | ||
| 342 | push @data, { symbol => $symbol, | ||
| 343 | currency => $currency, | ||
| 344 | date => $date, | ||
| 345 | close => $rate, | ||
| 346 | }; | ||
| 347 | } | ||
| 348 | } | ||
| 349 | } | ||
| 350 | |||
| 351 | return $h; | ||
| 352 | } | ||
| 353 | |||
| 354 | # return a list of ISO date strings like '2008-09-08' which is all the | ||
| 355 | # weekdays in the month of ISO date $str | ||
| 356 | sub iso_weekdays_in_month { | ||
| 357 | my ($str) = @_; | ||
| 358 | my ($lo_year, $lo_month, undef) = App::Chart::iso_to_ymd ($str); | ||
| 359 | my ($hi_year, $hi_month, undef) = Date::Calc::Add_Delta_YM | ||
| 360 | ($lo_year,$lo_month,1, 0,1); | ||
| 361 | my $lo = App::Chart::ymd_to_tdate_ceil ($lo_year, $lo_month, 1); | ||
| 362 | my $hi = App::Chart::ymd_to_tdate_ceil ($hi_year, $hi_month, 1) - 1; | ||
| 363 | return map { App::Chart::tdate_to_iso($_) } ($lo .. $hi); | ||
| 364 | } | ||
| 365 | |||
| 366 | |||
| 367 | #------------------------------------------------------------------------------ | ||
| 368 | # xls parse | ||
| 369 | # | ||
| 370 | # This parses xls spreadsheet files like | ||
| 371 | # | ||
| 372 | # https://www.rba.gov.au/Statistics/HistoricalExchangeRates/2003to2007.xls | ||
| 373 | # | ||
| 374 | # The files aren't huge (500k upwards) but there's a lot of cells, which | ||
| 375 | # makes Spreadsheet::ParseExcel fairly slow and eat up about 50Mb of core | ||
| 376 | # (as of version 0.32), even before getting to the $h data build. | ||
| 377 | |||
| 378 | sub xls_parse { | ||
| 379 | my ($resp) = @_; | ||
| 380 | ### RBA xls_parse() ... | ||
| 381 | my $content = $resp->decoded_content(raise_error=>1); | ||
| 382 | |||
| 383 | my @data = (); | ||
| 384 | my $h = { source => __PACKAGE__, | ||
| 385 | copyright => RBA_COPYRIGHT_URL, | ||
| 386 | data => \@data }; | ||
| 387 | |||
| 388 | require Spreadsheet::ParseExcel; | ||
| 389 | require Spreadsheet::ParseExcel::Utility; | ||
| 390 | |||
| 391 | my $excel = Spreadsheet::ParseExcel::Workbook->Parse (\$content); | ||
| 392 | my $sheet = $excel->Worksheet (0); | ||
| 393 | ### sheet: $sheet->{'Name'} | ||
| 394 | |||
| 395 | my ($minrow, $maxrow) = $sheet->RowRange; | ||
| 396 | my ($mincol, $maxcol) = $sheet->ColRange; | ||
| 397 | |||
| 398 | # heading row "DAILY 4PM", or "Mnemonic" and the currencies in columns | ||
| 399 | my $heading_row = List::Util::first { | ||
| 400 | my $cell = $sheet->Cell($_,$mincol); | ||
| 401 | $cell && ($cell->Value eq 'DAILY 4PM' | ||
| 402 | || $cell->Value eq 'Mnemonic') } | ||
| 403 | ($minrow .. $maxrow) | ||
| 404 | or die "RBA historical: currency code headings row not found"; | ||
| 405 | ### heading row: $heading_row | ||
| 406 | |||
| 407 | foreach my $row ($heading_row+1 .. $maxrow) { | ||
| 408 | my $datecell = $sheet->Cell($row,$mincol) or next; | ||
| 409 | $datecell->{'Type'} eq 'Date' or next; # skip blanks | ||
| 410 | my $date = Spreadsheet::ParseExcel::Utility::ExcelFmt | ||
| 411 | ('yyyy-mm-dd', $datecell->{'Val'}, $excel->{'Flg1904'}); | ||
| 412 | |||
| 413 | foreach my $col ($mincol+1 .. $maxcol) { | ||
| 414 | my $cell = $sheet->Cell($row,$col) | ||
| 415 | or next; # skip lots of blanks | ||
| 416 | my $rate = $cell->Value | ||
| 417 | or next; # skip lots of blanks | ||
| 418 | my $currency = $sheet->Cell($heading_row,$col)->Value; | ||
| 419 | $currency =~ s/^FXR//; # leading "FXR" circa 2012 | ||
| 420 | $currency = ($monthly_fx_to_currency{$currency} || $currency); | ||
| 421 | |||
| 422 | push @data, { symbol => "AUD$currency.RBA", | ||
| 423 | currency => $currency, | ||
| 424 | date => $date, | ||
| 425 | close => $rate, | ||
| 426 | }; | ||
| 427 | } | ||
| 428 | } | ||
| 429 | |||
| 430 | return $h; | ||
| 431 | } | ||
| 432 | |||
| 433 | |||
| 434 | |||
| 435 | #------------------------------------------------------------------------------ | ||
| 436 | # csv parse | ||
| 437 | # | ||
| 438 | # This parses csv download files like | ||
| 439 | # | ||
| 440 | # https://www.rba.gov.au/statistics/tables/csv/f11.1-data.csv | ||
| 441 | # | ||
| 442 | # row Title, A$1=USD, Trade-weighted Index May 1970 = 100, A$1=CNY, A$1=JPY, | ||
| 443 | # | ||
| 444 | # row Description,AUD/USD Exchange Rate; see notes for further detail., | ||
| 445 | # Australian Dollar Trade-weighted Index, | ||
| 446 | # AUD/CNY Exchange Rate,AUD/JPY Exchange Rate, | ||
| 447 | # | ||
| 448 | # row Frequency,Daily,Daily,Daily | ||
| 449 | # row Type,Indicative,Indicative, | ||
| 450 | # row Units,USD,Index,CNY,JPY, | ||
| 451 | # row Source,WM/Reuters,RBA,RBA, | ||
| 452 | # row Publication date,17-Feb-2016,17-Feb-2016 | ||
| 453 | # row Series ID,FXRUSD,FXRTWI,FXRCR,FXRJY,FXREUR, | ||
| 454 | # | ||
| 455 | # row 17-Feb-2016,0.7090,60.80,4.6265,80.61, | ||
| 456 | # | ||
| 457 | # csv | ||
| 458 | # AED CAD CHF CNY EUR GBP HKD IDR INR JPY KRW MYR NZD PGK PHP SDR SGD THB TWD TWI USD VND ZAR | ||
| 459 | # | ||
| 460 | # exchange page | ||
| 461 | # AED CAD CHF CNY EUR GBP HKD IDR INR JPY KRW MYR NZD PGK PHP SDR SGD THB TWD TWI USD VND | ||
| 462 | |||
| 463 | use constant RBA_CURRENT_CSV_URL => | ||
| 464 | 'https://www.rba.gov.au/statistics/tables/csv/f11.1-data.csv'; | ||
| 465 | |||
| 466 | sub csv_parse { | ||
| 467 | my ($resp) = @_; | ||
| 468 | ### RBA csv_parse() ... | ||
| 469 | my $content = $resp->decoded_content(raise_error=>1); | ||
| 470 | |||
| 471 | my @data = (); | ||
| 472 | my $h = { source => __PACKAGE__, | ||
| 473 | copyright => RBA_COPYRIGHT_URL, | ||
| 474 | date_format => 'dmy', | ||
| 475 | data => \@data }; | ||
| 476 | |||
| 477 | my @currencies; | ||
| 478 | foreach my $line (split /\n/, $content) { | ||
| 479 | my @fields = split /,\s*/, $line; | ||
| 480 | my $key = shift @fields // next; | ||
| 481 | ### $key | ||
| 482 | |||
| 483 | if ($key eq 'Title') { | ||
| 484 | @currencies = map { | ||
| 485 | my $field = $_; | ||
| 486 | if ($field eq '') { # empty fields at end of line | ||
| 487 | undef; | ||
| 488 | } elsif ($field =~ /Trade.Weighted.Index/i) { | ||
| 489 | 'TWI'; | ||
| 490 | } elsif ($field =~ /A\$1=(.*)/i) { | ||
| 491 | $1; | ||
| 492 | } else { | ||
| 493 | warn "RBA: unrecognised Title field: $field"; | ||
| 494 | undef; | ||
| 495 | } | ||
| 496 | } @fields; | ||
| 497 | |||
| 498 | } elsif ($key =~ /\d+-[a-z]+-\d+/i) { | ||
| 499 | # row like 17-Feb-2016,0.7090,60.80,4.6265,80.61, | ||
| 500 | foreach my $i (0 .. $#fields) { | ||
| 501 | my $currency = $currencies[$i] // next; | ||
| 502 | push @data, { symbol => "AUD$currency.RBA", | ||
| 503 | currency => $currency, | ||
| 504 | date => $key, | ||
| 505 | close => $fields[$i], | ||
| 506 | }; | ||
| 507 | } | ||
| 508 | } | ||
| 509 | } | ||
| 510 | return $h; | ||
| 511 | } | ||
| 512 | |||
| 513 | |||
| 514 | |||
| 515 | #------------------------------------------------------------------------------ | ||
| 516 | # data downloading | ||
| 517 | |||
| 518 | App::Chart::DownloadHandler->new | ||
| 519 | (name => __('RBA'), | ||
| 520 | pred => $pred, | ||
| 521 | proc => \&download, | ||
| 522 | # backto => \&backto, | ||
| 523 | available_date_time => \&threeday_available_date_time); | ||
| 524 | |||
| 525 | sub download { | ||
| 526 | my ($symbol_list) = @_; | ||
| 527 | |||
| 528 | my $lo_tdate = App::Chart::Download::start_tdate_for_update (@$symbol_list); | ||
| 529 | my $hi_tdate = threeday_available_tdate(); | ||
| 530 | ### RBA wanting ... | ||
| 531 | ### $lo_tdate | ||
| 532 | ### $hi_tdate | ||
| 533 | |||
| 534 | # desired range is <= 3 days, so try the threeday page | ||
| 535 | if ($hi_tdate - $lo_tdate + 1 <= RBA_EXCHANGE_URL_DAYS) { | ||
| 536 | App::Chart::Download::status (__('RBA past three days')); | ||
| 537 | my $resp = App::Chart::Download->get (RBA_EXCHANGE_URL); | ||
| 538 | my $h = threeday_parse ($resp); | ||
| 539 | App::Chart::Download::write_latest_group($h); | ||
| 540 | |||
| 541 | # if $lo_tdate is within the threeday data then write that and done | ||
| 542 | my $threeday_lo_tdate | ||
| 543 | = App::Chart::Download::iso_to_tdate_ceil ($h->{'lo_date'}); | ||
| 544 | if ($threeday_lo_tdate <= $lo_tdate) { | ||
| 545 | App::Chart::Download::write_daily_group ($h); | ||
| 546 | return; | ||
| 547 | } | ||
| 548 | } | ||
| 549 | |||
| 550 | # desired range > 3 days, or the threeday found was in fact not enough, | ||
| 551 | # get the csv | ||
| 552 | |||
| 553 | my $url = RBA_CURRENT_CSV_URL; | ||
| 554 | require File::Basename; | ||
| 555 | my $filename = File::Basename::basename($url); | ||
| 556 | App::Chart::Download::status (__x('RBA data {filename}', | ||
| 557 | filename => $filename)); | ||
| 558 | my $resp = App::Chart::Download->get($url); | ||
| 559 | my $h = csv_parse($resp); | ||
| 560 | App::Chart::Download::write_daily_group ($h); | ||
| 561 | |||
| 562 | # This code looked at the historical downloads page and chose among the | ||
| 563 | # various XLS files. | ||
| 564 | # | ||
| 565 | # my $info = historical_info(); | ||
| 566 | # my $files = $info->{'files'}; | ||
| 567 | # $files = App::Chart::Download::choose_files ($files, $lo_tdate, $hi_tdate); | ||
| 568 | # $files = [ sort {$a->{'lo_tdate'} <=> $b->{'lo_tdate'}} @$files ]; | ||
| 569 | # foreach my $f (@$files) { | ||
| 570 | # my $url = $f->{'url'}; | ||
| 571 | # require File::Basename; | ||
| 572 | # my $filename = File::Basename::basename($url); | ||
| 573 | # App::Chart::Download::status (__x('RBA data {filename}', | ||
| 574 | # filename => $filename)); | ||
| 575 | # my $resp = App::Chart::Download->get ($url); | ||
| 576 | # my $h = xls_parse ($resp); | ||
| 577 | # App::Chart::Download::write_daily_group ($h); | ||
| 578 | # } | ||
| 579 | } | ||
| 580 | |||
| 581 | sub backto { | ||
| 582 | my ($symbol_list, $backto_tdate) = @_; | ||
| 583 | die "Not implemented"; | ||
| 584 | } | ||
| 585 | |||
| 586 | 1; | ||
| 587 | __END__ | ||