| 1 | # MLC data downloading. | ||
| 2 | |||
| 3 | # Copyright 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2017 Kevin Ryde | ||
| 4 | |||
| 5 | # This file is part of Chart. | ||
| 6 | # | ||
| 7 | # Chart is free software; you can redistribute it and/or modify it under the | ||
| 8 | # terms of the GNU General Public License as published by the Free Software | ||
| 9 | # Foundation; either version 3, or (at your option) any later version. | ||
| 10 | # | ||
| 11 | # Chart is distributed in the hope that it will be useful, but WITHOUT ANY | ||
| 12 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
| 13 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | ||
| 14 | # details. | ||
| 15 | # | ||
| 16 | # You should have received a copy of the GNU General Public License along | ||
| 17 | # with Chart. If not, see <http://www.gnu.org/licenses/>. | ||
| 18 | |||
| 19 | package App::Chart::Suffix::MLC; | ||
| 20 | 1 | ▶ | use 5.010; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 21 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 22 | 1 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 23 | 1 | ▶ | use Carp; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 24 | 1 | ▶ | use Date::Calc; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 25 | 1 | ▶ | use URI::Escape; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 26 | 1 | ▶ | use Locale::TextDomain ('App-Chart'); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 27 | |||
| 28 | 0 | ▶ | use App::Chart; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 29 | use App::Chart::Database; | ||
| 30 | use App::Chart::Download; | ||
| 31 | use App::Chart::DownloadHandler; | ||
| 32 | use App::Chart::DownloadHandler::IndivChunks; | ||
| 33 | use App::Chart::Latest; | ||
| 34 | use App::Chart::Sympred; | ||
| 35 | use App::Chart::TZ; | ||
| 36 | |||
| 37 | # uncomment this to run the ### lines | ||
| 38 | # use Smart::Comments; | ||
| 39 | |||
| 40 | my $pred = App::Chart::Sympred::Suffix->new ('.MLC'); | ||
| 41 | |||
| 42 | # not sure exactly where MLC operates out of, but Sydney is close enough | ||
| 43 | App::Chart::TZ->sydney->setup_for_symbol ($pred); | ||
| 44 | |||
| 45 | App::Chart::setup_source_help | ||
| 46 | ($pred, __p('manual-node','MLC Funds')); | ||
| 47 | |||
| 48 | # The home page has a link to fund descriptions, unfortunately it's in | ||
| 49 | # flash format. | ||
| 50 | |||
| 51 | |||
| 52 | #----------------------------------------------------------------------------- | ||
| 53 | # download | ||
| 54 | # | ||
| 55 | # This uses the unit prices under | ||
| 56 | # | ||
| 57 | # https://www.mlc.com.au/masterkeyWeb/execute/FramesetUnitPrices | ||
| 58 | # | ||
| 59 | # Filling in the boxes ends up with a url like the following, with full | ||
| 60 | # fund and product name, and a requested date range (dd/mm/yyyy). | ||
| 61 | # | ||
| 62 | # https://www.mlc.com.au/masterkeyWeb/execute/UnitPricesWQO?openAgent&reporttype=HistoricalDateRange&product=MasterKey%20Superannuation%20%28Gold%20Star%29&fund=Property%20Securities%20Fund&begindate=19/07/2003&enddate=19/07/2004& | ||
| 63 | # | ||
| 64 | # Old data has prices for weekends too, but the same as Friday. | ||
| 65 | # | ||
| 66 | |||
| 67 | App::Chart::DownloadHandler::IndivChunks->new | ||
| 68 | (name => __('MLC'), | ||
| 69 | pred => $pred, | ||
| 70 | available_tdate => \&available_tdate, | ||
| 71 | url_func => \&url_func, | ||
| 72 | parse => \&parse, | ||
| 73 | |||
| 74 | # If you fill in the web page boxes asking for more than 1 year it | ||
| 75 | # explains you can only get 1 year at a time (you get 1 year from the | ||
| 76 | # given start date). | ||
| 77 | chunk_size => 250); | ||
| 78 | |||
| 79 | # Return the expected available tdate for data. | ||
| 80 | # | ||
| 81 | # This is only based on observation, in the morning it seems to be not the | ||
| 82 | # previous weekday but the one before that, and Thursday on a weekend. | ||
| 83 | # Don't know what time of day it ticks over, assume midnight for now. | ||
| 84 | # | ||
| 85 | sub available_tdate { | ||
| 86 | return App::Chart::Download::tdate_today_after | ||
| 87 | (23,59, App::Chart::TZ->sydney) | ||
| 88 | - 1; | ||
| 89 | } | ||
| 90 | |||
| 91 | # Sample url: | ||
| 92 | # https://www.mlc.com.au/masterkeyWeb/execute/UnitPricesWQO?openAgent&reporttype=HistoricalDateRange&product=MasterKey%20Allocated%20Pension%20%28Five%20Star%29&fund=MLC%20MasterKey%20Horizon%201%20-%20Bond%20Portfolio&begindate=07/01/2007&enddate=07/01/2008& | ||
| 93 | # | ||
| 94 | # In the past it was plain http: | ||
| 95 | # http://www.mlc.com.au/masterkeyWeb/execute/UnitPricesWQO?openAgent&reporttype=HistoricalDateRange&product=MasterKey%20Superannuation%20%28Gold%20Star%29&fund=Property%20Securities%20Fund&begindate=19/07/2003&enddate=19/07/2004& | ||
| 96 | # | ||
| 97 | sub url_func { | ||
| 98 | my ($symbol, $lo, $hi) = @_; | ||
| 99 | my ($lo_year, $lo_month, $lo_day) = App::Chart::tdate_to_ymd ($lo); | ||
| 100 | my ($hi_year, $hi_month, $hi_day) = App::Chart::tdate_to_ymd ($hi); | ||
| 101 | my ($fund, $product) = split /,/, App::Chart::symbol_sans_suffix ($symbol); | ||
| 102 | return sprintf ('https://www.mlc.com.au/masterkeyWeb/execute/UnitPricesWQO?openAgent&reporttype=HistoricalDateRange&product=%s&fund=%s&begindate=%02d/%02d/%04d&enddate=%02d/%02d/%04d&', | ||
| 103 | URI::Escape::uri_escape ($product), | ||
| 104 | URI::Escape::uri_escape ($fund), | ||
| 105 | $lo_day, $lo_month, $lo_year, | ||
| 106 | $hi_day, $hi_month, $hi_year); | ||
| 107 | } | ||
| 108 | |||
| 109 | # Lines like: | ||
| 110 | # historicalProduct1funds[1]="MLC Property Securities Fund,MasterKey Superannuation (Gold Star),29 March 2007,64.71567,0.00000"; | ||
| 111 | # | ||
| 112 | sub parse { | ||
| 113 | my ($symbol, $resp) = @_; | ||
| 114 | my $content = $resp->decoded_content(raise_error=>1); | ||
| 115 | |||
| 116 | my @data = (); | ||
| 117 | my $h = { source => __PACKAGE__, | ||
| 118 | resp => $resp, | ||
| 119 | currency => 'AUD', | ||
| 120 | data => \@data }; | ||
| 121 | |||
| 122 | while ($content =~ /^historicalProduct1funds.*=\"(.*)\"/mg) { | ||
| 123 | ### $& | ||
| 124 | my ($fund, $product, $date, $price) = split /,/, $1; | ||
| 125 | |||
| 126 | # skip historicalProduct1funds[0]="All Funds" bit | ||
| 127 | if (! $product) { next; } | ||
| 128 | |||
| 129 | my ($year, $month, $day) = Date::Calc::Decode_Date_EU ($date); | ||
| 130 | # skip weekends in some old data | ||
| 131 | next if (ymd_is_weekend ($year, $month, $day)); | ||
| 132 | $date = App::Chart::ymd_to_iso ($year, $month, $day); | ||
| 133 | |||
| 134 | push @data, { symbol => $fund . ',' . $product . '.MLC', | ||
| 135 | date => $date, | ||
| 136 | close => $price }; | ||
| 137 | } | ||
| 138 | return $h; | ||
| 139 | } | ||
| 140 | |||
| 141 | sub validate_symbol { | ||
| 142 | my ($symbol) = @_; | ||
| 143 | if ($symbol !~ /,/) { | ||
| 144 | print __x("MLC: invalid symbol, should be \"Fund,Product.MLC\": {symbol}\n", | ||
| 145 | symbol => $symbol); | ||
| 146 | return 0; | ||
| 147 | } | ||
| 148 | return 1; | ||
| 149 | } | ||
| 150 | |||
| 151 | |||
| 152 | #------------------------------------------------------------------------------ | ||
| 153 | # latest | ||
| 154 | # | ||
| 155 | # The approach here is to download the latest price the same as for the | ||
| 156 | # database above, getting the latest and second latest, so as to calculate a | ||
| 157 | # "change", and back a few extra days to allow for public holidays. | ||
| 158 | # | ||
| 159 | # There's a single download of latest prices for all funds and products. If | ||
| 160 | # $symbol_list was big then it might be worth doing that instead of | ||
| 161 | # individual downloads, but not sure if a set of immediately preceding | ||
| 162 | # prices would be available to make the "change" amount. In any case for | ||
| 163 | # now it's probably unlikely there'll be many funds in the watchlist that | ||
| 164 | # are not in the database. | ||
| 165 | # | ||
| 166 | |||
| 167 | App::Chart::LatestHandler->new | ||
| 168 | (pred => $pred, | ||
| 169 | proc => \&latest, | ||
| 170 | max_symbols => 1, | ||
| 171 | available_tdate => \&available_tdate); | ||
| 172 | |||
| 173 | sub latest { | ||
| 174 | my ($symbol_list) = @_; | ||
| 175 | my $symbol = $symbol_list->[0]; | ||
| 176 | if (! validate_symbol ($symbol)) { return; } | ||
| 177 | my $avail_tdate = available_tdate(); | ||
| 178 | my $url = url_func ($symbol, $avail_tdate-3, $avail_tdate+1); | ||
| 179 | my $resp = App::Chart::Download->get ($url); | ||
| 180 | App::Chart::Download::write_latest_group (parse ($symbol, $resp)); | ||
| 181 | } | ||
| 182 | |||
| 183 | |||
| 184 | #----------------------------------------------------------------------------- | ||
| 185 | # generic helpers | ||
| 186 | |||
| 187 | sub ymd_is_weekend { | ||
| 188 | my ($year, $month, $day) = @_; | ||
| 189 | return (Date::Calc::Day_of_Week ($year, $month, $day) >= 6); # 6 or 7 | ||
| 190 | } | ||
| 191 | |||
| 192 | |||
| 193 | 1; | ||
| 194 | __END__ | ||