App/Chart/Suffix/RBA.pm

filter: Statementstmt 92.3% 24 / 26 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 9 / 9 Podpod n/a totaltotal 94.2% 33 / 35 CC 1 SCAR 0.3
CC
1
Cov
92%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« App/Chart/Suffix/NoSuffix.pm ↑ Summary App/Chart/TZ.pm »
1 # Reserve Bank of Australia setups.
2  
3 # Copyright 2007, 2008, 2009, 2010, 2011, 2012, 2016, 2019 Kevin Ryde
4  
5 # This file is part of Chart.
6 #
7 # Chart is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 3, or (at your option) any later version.
10 #
11 # Chart is distributed in the hope that it will be useful, but WITHOUT ANY
12 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
14 # details.
15 #
16 # You should have received a copy of the GNU General Public License along
17 # with Chart. If not, see <http://www.gnu.org/licenses/>.
18  
19 package App::Chart::Suffix::RBA;
20 1 ▶use 5.010;
Subroutine BEGIN: called CC=1 SCAR=0.0
21 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
22 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
23 1 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
24 1 ▶use Date::Calc;
Subroutine BEGIN: called CC=1 SCAR=0.0
25 1 ▶use List::Util qw(min max);
Subroutine BEGIN: called CC=1 SCAR=0.0
26 1 ▶use Set::IntSpan::Fast;
Subroutine BEGIN: called CC=1 SCAR=0.0
27 1 ▶use Locale::TextDomain ('App-Chart');
Subroutine BEGIN: called CC=1 SCAR=0.0
28  
29 0 ▶use App::Chart;
Subroutine BEGIN: called CC=1 SCAR=2.6
30 use App::Chart::Database;
31 use App::Chart::Download;
32 use App::Chart::DownloadHandler;
33 use App::Chart::Sympred;
34 use App::Chart::Latest;
35 use App::Chart::TZ;
36 use App::Chart::Weblink;
37  
38 # uncomment this to run the ### lines
39 # use Smart::Comments;
40  
41 # Not yet using Finance::Quote::RBA
42  
43 my $pred = App::Chart::Sympred::Suffix->new ('.RBA');
44 App::Chart::TZ->sydney->setup_for_symbol ($pred);
45 App::Chart::setup_source_help
46 ($pred, __p('manual-node','Reserve Bank of Australia'));
47  
48 use constant RBA_COPYRIGHT_URL =>
49 'https://www.rba.gov.au/copyright/';
50  
51  
52 #------------------------------------------------------------------------------
53 # weblink - home page
54  
55 App::Chart::Weblink->new
56 (pred => $pred,
57 name => __('_RBA Home Page'),
58 desc => __('Open web browser at the Reserve Bank of Australia home page'),
59 url => 'https://www.rba.gov.au');
60  
61  
62 #------------------------------------------------------------------------------
63 # three day page
64 #
65 # This uses the rates at:
66 #
67 use constant RBA_EXCHANGE_URL =>
68 'https://www.rba.gov.au/statistics/frequency/exchange-rates.html';
69 use constant RBA_EXCHANGE_URL_DAYS => 3;
70  
71 # would it be a few minutes after 4pm ?
72 sub threeday_available_date_time {
73 return (App::Chart::Download::weekday_date_after_time
74 (16,0, App::Chart::TZ->sydney),
75 '16:01:00');
76 }
77  
78 sub threeday_available_tdate {
79 my ($iso, $time) = threeday_available_date_time();
80 return App::Chart::Download::iso_to_tdate_floor ($iso);
81 }
82  
83 sub threeday_parse {
84 my ($resp) = @_;
85 my @data = ();
86 my $h = { url => RBA_EXCHANGE_URL,
87 copyright => RBA_COPYRIGHT_URL,
88 source => __PACKAGE__,
89 resp => $resp,
90 cover_pred => $pred,
91 data => \@data };
92  
93 my $content = $resp->decoded_content(raise_error=>1);
94  
95 # mung <tr id="USD"> to add <td>USD</td> so it appears in the TableExtract
96 $content =~ s{<tr>}{<tr><td></td>}ig;
97 $content =~ s{(<tr +id="([^"]*)">)}{$1<td>$2</td>}ig;
98  
99 require HTML::TableExtract;
100 my $te = HTML::TableExtract->new
101 (
102 # is now a <caption>
103 # headers => ['Units of foreign currency per'],
104 slice_columns => 0);
105 $te->parse($content);
106 my $ts = $te->first_table_found();
107 if (! $ts) { die "RBA: html table not found\n"; }
108  
109 my $rows = $ts->rows();
110 my $lastrow = $#$rows;
111 my $lastcol = $#{$rows->[0]};
112  
113 # date like "03 Sep 2007"
114 my @dates;
115 foreach my $c (2 .. $lastcol) {
116 $dates[$c] = App::Chart::Download::Decode_Date_EU_to_iso($rows->[0]->[$c]);
117 }
118 $h->{'lo_date'} = List::Util::minstr (grep {defined} @dates);
119  
120 foreach my $r (1 .. $lastrow) {
121 my $row = $rows->[$r];
122  
123 my $symbol = $row->[0] // next;
124 $symbol =~ s/_.*//; # _4pm on TWI
125 $symbol = "AUD$symbol.RBA";
126  
127 my $name = $row->[1];
128 $name =~ s/ \(4pm\)$//; # 4pm on TWI
129  
130 foreach my $c (2 .. $lastcol) {
131 my $rate = $row->[$c];
132 # bank holiday columns have "BANK HOLIDAY" with one letter per row or
133 # blank which comes through as undef, skip those
134 next if ! Scalar::Util::looks_like_number($rate);
135  
136 push @data, { symbol => $symbol,
137 name => $name,
138 date => $dates[$c],
139 last_time => '16:00:00',
140 close => $rate,
141 currency => substr($symbol,3,3),
142 };
143 }
144 }
145  
146 return $h;
147 }
148  
149 #------------------------------------------------------------------------------
150 # latest quotes
151  
152 App::Chart::LatestHandler->new
153 (pred => $pred,
154 url_tags_key => 'RBA-latest',
155 proc => \&latest_download,
156 available_date_time => \&threeday_available_date_time);
157  
158 sub latest_download {
159 my ($symbol_list) = @_;
160  
161 App::Chart::Download::status (__('RBA past three days'));
162 my $resp = App::Chart::Download->get (RBA_EXCHANGE_URL,
163 url_tags_key => 'RBA-latest');
164 App::Chart::Download::write_latest_group (threeday_parse ($resp));
165 }
166  
167  
168 #------------------------------------------------------------------------------
169 # historical xls page
170 #
171 # This downloads and parses up the page:
172 # https://www.rba.gov.au/statistics/historical-data.html
173 #
174 # 2014 to present
175 # https://www.rba.gov.au/statistics/tables/csv/f11.1-data.csv
176 # 213k or 47k compressed, also byte ranges
177 # https://www.rba.gov.au/statistics/tables/xls-hist/2014-current.xls
178 # 438k
179 #
180 use constant RBA_HISTORICAL_PAGE_URL =>
181 'https://www.rba.gov.au/statistics/hist-exchange-rates/index.html';
182 #
183 # which offers various xls files for past rates.
184  
185 sub historical_info {
186 require App::Chart::Pagebits;
187 return App::Chart::Pagebits::get
188 (name => __('RBA historical page'),
189 url => RBA_HISTORICAL_PAGE_URL,
190 key => 'rba-historical',
191 freq_days => 1,
192 parse => \&historical_parse);
193 }
194  
195 sub historical_parse {
196 my ($content) = @_;
197  
198 # Eg.
199 # <a href="/statistics/hist-exchange-rates/2007-2009.xls"
200 # target="_blank" title="Link, opening in a new window, to XLS file.">
201 # 2007 to 2009</a> <span class="nonHtml">[XLS 227K]</span>
202 #
203 # because the size is outside the link it doesn't really suit
204 # HTML::LinkExtor / HTML::Parser etc
205 #
206 my @files;
207 while ($content =~ m%href=\"([^\"]*[0-9][0-9][0-9][0-9]\.xls)\"
208 (.*\n){0,3}
209 .*\[XLS\s*([0-9.]+)([MK])
210 %igmx) {
211 my $link = $1;
212 my $size = $3;
213 my $size_units = uc($4);
214  
215 if ($size_units eq 'K') { $size *= 1_000 }
216 if ($size_units eq 'M') { $size *= 1_000_000 }
217  
218 my $uri = URI->new_abs($link, RBA_HISTORICAL_PAGE_URL);
219  
220 # eg per above: 2003-2007.xls
221 # or just: 2007.xls
222 $link =~ /([0-9][0-9][0-9][0-9])((to|-)([0-9][0-9][0-9][0-9]))?\.xls$/
223 or die "RBA: oops, unrecognised link: $link";
224 my $lo_year = $1;
225 my $hi_year = $4 || $1;
226  
227 push @files, { url => $uri->as_string,
228 cost => $size,
229 lo_year => $lo_year,
230 hi_year => $hi_year,
231 };
232 }
233 return { files => \@files };
234 }
235  
236  
237 #-----------------------------------------------------------------------------
238 # download - historical monthly prices
239 #
240 # This parses the monthly rates spreadsheet file from the
241 # RBA_HISTORICAL_PAGE_URL page above,
242 #
243 # https://www.rba.gov.au/statistics/tables/xls/f11hist-1969-2009.xls
244 #
245 # but only the part from 1983 back is wanted since there's daily data for
246 # 1983 onwards.
247  
248 my %monthly_fx_to_currency
249 = (
250 # 'TWI' # trade weighted index
251 'CR' => 'CNY', # chinese renminbi
252 'JY' => 'JPY', # japanese yen
253 # 'EUR'
254 # 'USD'
255 'SKW' => 'KRW', # South Korean won
256 'UKPS' => 'GBP', # british pound sterling
257 'SD' => 'SGD', # singapore dollar
258 'IRE' => 'INR', # Indian rupee
259 'TB' => 'THB', # Thai baht
260 # 'NZD'
261 'NTD' => 'TWD', # taiwan dollar
262 'MR' => 'MYR', # malaysian ringgit
263 'IR' => 'IDR', # indonesian rupiah
264 'VD' => 'VND', # Vietnamese dong
265 'UAED' => 'AED', # UAE dirham
266 'PNGK' => 'PGK', # PNG kina
267 # 'HKD' # Hong Kong dollar
268 'CD' => 'CAD', # Canadian dollar
269 'SARD' => 'ZAR', # South African rand
270 'SARY' => 'SAR', # Saudi riyal
271 'SF' => 'CHF', # Swiss franc
272 'SK' => 'SEK', # Swedish krona
273 # 'SDR' # special drawing right
274 );
275  
276 sub monthly_parse {
277 my ($resp, $stop_iso) = @_;
278 ### RBA monthly_parse() ...
279 my $content = $resp->decoded_content(raise_error=>1);
280  
281 my @data = ();
282 my $h = { source => __PACKAGE__,
283 copyright => RBA_COPYRIGHT_URL,
284 data => \@data };
285  
286 require Spreadsheet::ParseExcel;
287 require Spreadsheet::ParseExcel::Utility;
288  
289 my $excel = Spreadsheet::ParseExcel::Workbook->Parse (\$content);
290 my $sheet = $excel->Worksheet (0);
291 ### SheetCount: $excel->{'SheetCount'}
292 ### Name: $sheet->{'Name'}
293  
294 my ($minrow, $maxrow) = $sheet->RowRange;
295 my ($mincol, $maxcol) = $sheet->ColRange;
296 ### rows: $minrow, $maxrow
297 ### cols: $mincol, $maxcol
298  
299 # heading row repeats the filename "F11HIST.XLS" and then the currencies
300 # in columns as say "FXRJY"
301 my $heading_row = List::Util::first {
302 my $cell = $sheet->Cell($_,$mincol);
303 $cell && $cell->Value eq 'F11HIST.XLS' }
304 ($minrow .. $maxrow)
305 or die "RBA monthly: headings not found";
306 ### $heading_row
307  
308 my @currencies = map {
309 my $cell = $sheet->Cell($heading_row,$_);
310 my $currency = $cell ? $cell->Value : '';
311 $currency =~ s/^FXR//;
312 ($monthly_fx_to_currency{$currency} || $currency)
313 } ($mincol .. $maxcol);
314 ### @currencies
315 ### count: scalar(@currencies)
316  
317 my %currency_started;
318  
319 ROW: foreach my $row ($heading_row+1 .. $maxrow) {
320 my $datecell = $sheet->Cell($row,0) or next;
321 # seen 'Numeric', but presumably 'Date' is ok
322 if ($datecell->{'Type'} ne 'Numeric'
323 && $datecell->{'Type'} ne 'Date') {
324 next; # skip blanks at end
325 }
326 my $month = Spreadsheet::ParseExcel::Utility::ExcelFmt
327 ('yyyy-mm-dd', $datecell->{'Val'}, $excel->{'Flg1904'});
328  
329 foreach my $col ($mincol+1 .. $maxcol) {
330 my $currency = $currencies[$col-$mincol] or next;
331 my $ratecell = $sheet->Cell($row,$col) or next;
332 my $rate = $ratecell->Value;
333  
334 # avoid empty records until the start of data for a given currency is
335 # reached
336 if (! $rate && ! $currency_started{$currency}) { next; }
337  
338 my $symbol = "AUD$currency.RBA";
339 $currency_started{$currency} = 1;
340 foreach my $date (iso_weekdays_in_month ($month)) {
341 if ($date gt $stop_iso) { last ROW; }
342 push @data, { symbol => $symbol,
343 currency => $currency,
344 date => $date,
345 close => $rate,
346 };
347 }
348 }
349 }
350  
351 return $h;
352 }
353  
354 # return a list of ISO date strings like '2008-09-08' which is all the
355 # weekdays in the month of ISO date $str
356 sub iso_weekdays_in_month {
357 my ($str) = @_;
358 my ($lo_year, $lo_month, undef) = App::Chart::iso_to_ymd ($str);
359 my ($hi_year, $hi_month, undef) = Date::Calc::Add_Delta_YM
360 ($lo_year,$lo_month,1, 0,1);
361 my $lo = App::Chart::ymd_to_tdate_ceil ($lo_year, $lo_month, 1);
362 my $hi = App::Chart::ymd_to_tdate_ceil ($hi_year, $hi_month, 1) - 1;
363 return map { App::Chart::tdate_to_iso($_) } ($lo .. $hi);
364 }
365  
366  
367 #------------------------------------------------------------------------------
368 # xls parse
369 #
370 # This parses xls spreadsheet files like
371 #
372 # https://www.rba.gov.au/Statistics/HistoricalExchangeRates/2003to2007.xls
373 #
374 # The files aren't huge (500k upwards) but there's a lot of cells, which
375 # makes Spreadsheet::ParseExcel fairly slow and eat up about 50Mb of core
376 # (as of version 0.32), even before getting to the $h data build.
377  
378 sub xls_parse {
379 my ($resp) = @_;
380 ### RBA xls_parse() ...
381 my $content = $resp->decoded_content(raise_error=>1);
382  
383 my @data = ();
384 my $h = { source => __PACKAGE__,
385 copyright => RBA_COPYRIGHT_URL,
386 data => \@data };
387  
388 require Spreadsheet::ParseExcel;
389 require Spreadsheet::ParseExcel::Utility;
390  
391 my $excel = Spreadsheet::ParseExcel::Workbook->Parse (\$content);
392 my $sheet = $excel->Worksheet (0);
393 ### sheet: $sheet->{'Name'}
394  
395 my ($minrow, $maxrow) = $sheet->RowRange;
396 my ($mincol, $maxcol) = $sheet->ColRange;
397  
398 # heading row "DAILY 4PM", or "Mnemonic" and the currencies in columns
399 my $heading_row = List::Util::first {
400 my $cell = $sheet->Cell($_,$mincol);
401 $cell && ($cell->Value eq 'DAILY 4PM'
402 || $cell->Value eq 'Mnemonic') }
403 ($minrow .. $maxrow)
404 or die "RBA historical: currency code headings row not found";
405 ### heading row: $heading_row
406  
407 foreach my $row ($heading_row+1 .. $maxrow) {
408 my $datecell = $sheet->Cell($row,$mincol) or next;
409 $datecell->{'Type'} eq 'Date' or next; # skip blanks
410 my $date = Spreadsheet::ParseExcel::Utility::ExcelFmt
411 ('yyyy-mm-dd', $datecell->{'Val'}, $excel->{'Flg1904'});
412  
413 foreach my $col ($mincol+1 .. $maxcol) {
414 my $cell = $sheet->Cell($row,$col)
415 or next; # skip lots of blanks
416 my $rate = $cell->Value
417 or next; # skip lots of blanks
418 my $currency = $sheet->Cell($heading_row,$col)->Value;
419 $currency =~ s/^FXR//; # leading "FXR" circa 2012
420 $currency = ($monthly_fx_to_currency{$currency} || $currency);
421  
422 push @data, { symbol => "AUD$currency.RBA",
423 currency => $currency,
424 date => $date,
425 close => $rate,
426 };
427 }
428 }
429  
430 return $h;
431 }
432  
433  
434  
435 #------------------------------------------------------------------------------
436 # csv parse
437 #
438 # This parses csv download files like
439 #
440 # https://www.rba.gov.au/statistics/tables/csv/f11.1-data.csv
441 #
442 # row Title, A$1=USD, Trade-weighted Index May 1970 = 100, A$1=CNY, A$1=JPY,
443 #
444 # row Description,AUD/USD Exchange Rate; see notes for further detail.,
445 # Australian Dollar Trade-weighted Index,
446 # AUD/CNY Exchange Rate,AUD/JPY Exchange Rate,
447 #
448 # row Frequency,Daily,Daily,Daily
449 # row Type,Indicative,Indicative,
450 # row Units,USD,Index,CNY,JPY,
451 # row Source,WM/Reuters,RBA,RBA,
452 # row Publication date,17-Feb-2016,17-Feb-2016
453 # row Series ID,FXRUSD,FXRTWI,FXRCR,FXRJY,FXREUR,
454 #
455 # row 17-Feb-2016,0.7090,60.80,4.6265,80.61,
456 #
457 # csv
458 # AED CAD CHF CNY EUR GBP HKD IDR INR JPY KRW MYR NZD PGK PHP SDR SGD THB TWD TWI USD VND ZAR
459 #
460 # exchange page
461 # AED CAD CHF CNY EUR GBP HKD IDR INR JPY KRW MYR NZD PGK PHP SDR SGD THB TWD TWI USD VND
462  
463 use constant RBA_CURRENT_CSV_URL =>
464 'https://www.rba.gov.au/statistics/tables/csv/f11.1-data.csv';
465  
466 sub csv_parse {
467 my ($resp) = @_;
468 ### RBA csv_parse() ...
469 my $content = $resp->decoded_content(raise_error=>1);
470  
471 my @data = ();
472 my $h = { source => __PACKAGE__,
473 copyright => RBA_COPYRIGHT_URL,
474 date_format => 'dmy',
475 data => \@data };
476  
477 my @currencies;
478 foreach my $line (split /\n/, $content) {
479 my @fields = split /,\s*/, $line;
480 my $key = shift @fields // next;
481 ### $key
482  
483 if ($key eq 'Title') {
484 @currencies = map {
485 my $field = $_;
486 if ($field eq '') { # empty fields at end of line
487 undef;
488 } elsif ($field =~ /Trade.Weighted.Index/i) {
489 'TWI';
490 } elsif ($field =~ /A\$1=(.*)/i) {
491 $1;
492 } else {
493 warn "RBA: unrecognised Title field: $field";
494 undef;
495 }
496 } @fields;
497  
498 } elsif ($key =~ /\d+-[a-z]+-\d+/i) {
499 # row like 17-Feb-2016,0.7090,60.80,4.6265,80.61,
500 foreach my $i (0 .. $#fields) {
501 my $currency = $currencies[$i] // next;
502 push @data, { symbol => "AUD$currency.RBA",
503 currency => $currency,
504 date => $key,
505 close => $fields[$i],
506 };
507 }
508 }
509 }
510 return $h;
511 }
512  
513  
514  
515 #------------------------------------------------------------------------------
516 # data downloading
517  
518 App::Chart::DownloadHandler->new
519 (name => __('RBA'),
520 pred => $pred,
521 proc => \&download,
522 # backto => \&backto,
523 available_date_time => \&threeday_available_date_time);
524  
525 sub download {
526 my ($symbol_list) = @_;
527  
528 my $lo_tdate = App::Chart::Download::start_tdate_for_update (@$symbol_list);
529 my $hi_tdate = threeday_available_tdate();
530 ### RBA wanting ...
531 ### $lo_tdate
532 ### $hi_tdate
533  
534 # desired range is <= 3 days, so try the threeday page
535 if ($hi_tdate - $lo_tdate + 1 <= RBA_EXCHANGE_URL_DAYS) {
536 App::Chart::Download::status (__('RBA past three days'));
537 my $resp = App::Chart::Download->get (RBA_EXCHANGE_URL);
538 my $h = threeday_parse ($resp);
539 App::Chart::Download::write_latest_group($h);
540  
541 # if $lo_tdate is within the threeday data then write that and done
542 my $threeday_lo_tdate
543 = App::Chart::Download::iso_to_tdate_ceil ($h->{'lo_date'});
544 if ($threeday_lo_tdate <= $lo_tdate) {
545 App::Chart::Download::write_daily_group ($h);
546 return;
547 }
548 }
549  
550 # desired range > 3 days, or the threeday found was in fact not enough,
551 # get the csv
552  
553 my $url = RBA_CURRENT_CSV_URL;
554 require File::Basename;
555 my $filename = File::Basename::basename($url);
556 App::Chart::Download::status (__x('RBA data {filename}',
557 filename => $filename));
558 my $resp = App::Chart::Download->get($url);
559 my $h = csv_parse($resp);
560 App::Chart::Download::write_daily_group ($h);
561  
562 # This code looked at the historical downloads page and chose among the
563 # various XLS files.
564 #
565 # my $info = historical_info();
566 # my $files = $info->{'files'};
567 # $files = App::Chart::Download::choose_files ($files, $lo_tdate, $hi_tdate);
568 # $files = [ sort {$a->{'lo_tdate'} <=> $b->{'lo_tdate'}} @$files ];
569 # foreach my $f (@$files) {
570 # my $url = $f->{'url'};
571 # require File::Basename;
572 # my $filename = File::Basename::basename($url);
573 # App::Chart::Download::status (__x('RBA data {filename}',
574 # filename => $filename));
575 # my $resp = App::Chart::Download->get ($url);
576 # my $h = xls_parse ($resp);
577 # App::Chart::Download::write_daily_group ($h);
578 # }
579 }
580  
581 sub backto {
582 my ($symbol_list, $backto_tdate) = @_;
583 die "Not implemented";
584 }
585  
586 1;
587 __END__
« App/Chart/Suffix/NoSuffix.pm ↑ Summary App/Chart/TZ.pm »