App/Chart/Suffix/TGE.pm untested

filter: Statementstmt 0.0% 0 / 122 Branchbran 0.0% 0 / 12 Conditioncond 0.0% 0 / 18 MC/DCmcdc 0.0% 0 / 9 Subroutinesub 0.0% 0 / 22 Podpod 0.0% 0 / 10 totaltotal 0.0% 0 / 193 CC 7 SCAR 20.2
CC
7
Cov
0%
download_day
30.0
download_year_attempt
6.0
zip_parse
6.0
CRAP 7.6
1 # Tokyo Grain Exchange (TGE) setups. -*- coding: euc-jp -*-
2  
3 # Copyright 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2013, 2016 Kevin Ryde
4  
5 # This file is part of Chart.
6 #
7 # Chart is free software; you can redistribute it and/or modify it under
8 # the terms of the GNU General Public License as published by the Free
9 # Software Foundation; either version 3, or (at your option) any later
10 # version.
11 #
12 # Chart is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
15 # details.
16 #
17 # You should have received a copy of the GNU General Public License
18 # along with Chart. If not, see <http://www.gnu.org/licenses/>.
19  
20  
21 package App::Chart::Suffix::TGE;
22 use 5.010;
23 use strict;
24 use warnings;
25 use Locale::TextDomain 'App-Chart';
26  
27 use App::Chart::Glib::Ex::MoreUtils;
28 use App::Chart;
29 use App::Chart::DownloadHandler;
30 use App::Chart::Sympred;
31 use App::Chart::TZ;
32 use App::Chart::Weblink;
33  
34  
35 my $pred = App::Chart::Sympred::Suffix->new ('.TGE');
36 App::Chart::TZ->tokyo->setup_for_symbol ($pred);
37 # (source-help! tge-symbol?
38 # __p('manual-node','Tokyo Grain Exchange'))
39  
40  
41  
42 #-----------------------------------------------------------------------------
43 # weblinks - contract specs
44 #
45 # eg. http://www.tge.or.jp/japanese/contract/cont_s_corn.shtml
46 # http://www.tge.or.jp/english/contract/cont_s_soym.shtml
47 #
48  
49 my %commodity_to_weblink = ('CO' => 'corn', # Corn
50 'SM' => 'soym', # Soybean Meal
51 'SB' => 'soy', # Soybean
52 'NG' => 'gmo', # Non-GMO Soybean
53 'RB' => 'azuki', # Azuki
54 'AC' => 'ara', # Arabica Coffee
55 'RC' => 'rob', # Robusta Coffee
56 'SG' => 'raw', # Raw Sugar
57 'SL' => 'silk', # Raw Silk
58 'VG' => 'vege'); # Vegetables
59  
60 App::Chart::Weblink->new
61 (pred => $pred,
62 name => __('TGE _Contract Specifications'),
63 desc => __('Open web browser at the Tokyo Grain Exchange contract specifications for this commodity'),
64 proc => sub {
65 my ($symbol) = @_;
66 my $commodity = App::Chart::symbol_commodity ($symbol);
67 my $code = $commodity_to_weblink{$commodity} || return undef;
68 my $lang = App::Chart::Glib::Ex::MoreUtils::lang_select (ja => 'japanese',
69 en => 'english');
70 return "http://www.tge.or.jp/$lang/contract/cont_s_$code.shtml";
71 });
72  
73 #-----------------------------------------------------------------------------
74 # misc
75  
76 # return ISO date string of last trading for SYMBOL, or possibly a day or
77 # two after (because public holidays are not accounted for)
78 #
79 sub symbol_expiry {
80 my ($symbol) = @_;
81 my $mdate = symbol_mdate($symbol) // return undef;
82  
83 my $commodity = App::Chart::symbol_commodity ($symbol);
84 # if ($commodity eq 'CO') {
85 # # corn
86 # # 15th calendar day of the month preceding the delivery month
87 # # (receive-list (year month day)
88 # # (mdate->ymd mdate)
89 # # (ymd->tdate year (1- month) 15)))
90 # }
91 # when (['NG','RB','SB']) {
92 # # non-gm soybeans, azuki
93 # # 2 business days prior to delivery day; delivery day
94 # # is the business day prior to the last business day of
95 # # the delivery month or 24th for December
96 # # (receive-list (year month day)
97 # # (mdate->ymd mdate)
98 # # (- (if (= 12 month)
99 # # (ymd->tdate year month 24)
100 # # (- (mdate->tdate (1+ mdate)) 2))
101 # # 2))
102 # }
103 # when (['NG','RB','SB']) {
104 # # arabica, robusta, soybean meal
105 # # 10 business days prior to the last business day of
106 # # the delivery month
107 # # (- (mdate->tdate (1+ mdate)) 11)
108 # }
109 # when ('SG') {
110 # # raw sugar
111 # # last business day two months prior to delivery month
112 # # (1- (mdate->tdate (1- mdate))))
113 # }
114 # default {
115 # # unknown, lets guess the last day of the contract month,
116 # # this won't be right but at least won't be early
117 # # (1- (mdate->tdate (1+ mdate))))))
118 # }
119 # }
120  
121 # FIXME:
122 return undef;
123 }
124  
125 # see table in http://www.tge.or.jp/japanese/price/prisel.js.php
126 # ENHANCE-ME: maybe download and parse that page
127 my %commodity_to_nav
128 = ('CO' => 'tou', # corn
129 'SM' => 'dai', # soybean meal
130 'SB' => 'ipd', # soybeans
131 'NG' => 'ngd', # non-gmo soybeans
132 'RB' => 'azu', # azuki
133 'AC' => 'ara', # arabica coffee
134 'RC' => 'rob', # robusta coffee
135 'SG' => 'sot', # raw sugar
136 'SL' => 'rsl', # raw silk
137 'VG' => 'veg'); # vegetables
138  
139 #-----------------------------------------------------------------------------
140 # download
141 #
142 # This uses the "download" under the price data page
143 #
144 # http://www.tge.or.jp/english/price/pri_sel_01.shtml
145 #
146 # There's two files for each commodity,
147 #
148 # http://www.tge.or.jp/data/down_load/co01.csv
149 # http://www.tge.or.jp/data/down_load/co01040610050609.zip
150 #
151 # The csv is only about 600 bytes, it has just the most recent day and is
152 # used when that's all that's needed to update (which will be the case if
153 # you're updating every day).
154 #
155 # The zip is about 35kbytes and has the same most recent day as the csv,
156 # plus the past 12 months. The filename is a date range
157 # (XX01yymmddyymmdd.zip), but there's only one file, you can't get an
158 # arbitrary range.
159 #
160 # The zip filename is formed from today's date, or a day or two earlier,
161 # but if we don't hit it that way there's a fallback to fetch and parse the
162 # actual download page. The pattern in the filename is pretty clear, so
163 # perhaps that fallback is unnecessary.
164 #
165 # There's an ETag / Last-Modified on the year file, but it's hardly needed
166 # since the dates are in the name.
167  
168 App::Chart::DownloadHandler->new
169 (name => __('TGE'),
170 pred => $pred,
171 proc => \&download,
172 backto => undef,
173 available_tdate => \&available_tdate,
174 by_commodity => 1);
175  
176 sub download {
177 my ($symbol_list) = @_;
178 download_day ($symbol_list)
179 || download_year ($symbol_list);
180  
181 # (download-name-from-latest (_ "TGE") symbol-list))
182 }
183  
184 # return tdate for available download data
185 # the .csv downloads have been seen with a Last-Modified headers
186 # Wed, 24 Aug 2005 01:00:20 GMT == 10am tokyo
187 # Thu, 01 Sep 2005 22:33:05 GMT == 7am tokyo
188 # so try at 10:05am tokyo
189 #
190 sub available_tdate {
191 App::Chart::Download::weekday_tdate_after_time
192 (10,5, App::Chart::TZ->tokyo, -1);
193 }
194  
195 # do a download of the last day .csv file for $symbol_list, if that would
196 # update all those
197 # return true if updated successfully, or false if not (should use zip instead)
198 #
199 sub download_day {
200 my ($symbol_list) = @_;
201  
202 my $commodity = App::Chart::symbol_commodity ($symbol_list->[0]);
203 my $avail_tdate = available_tdate();
204 my $start_tdate = App::Chart::Download::start_tdate_for_update(@$symbol_list);
205  
206 # use csv if all of $symbol_list just wanting $avail_tdate
207 if ($start_tdate < $avail_tdate) {
208 return 0;
209 }
210 my $filename = "\L$commodity\E01.csv";
211 my $url = 'http://www.tge.or.jp/data/down_load/'
212 . URI::Escape::uri_escape ($filename);
213  
214 App::Chart::Download::status (__x('TGE data {filename}',
215 filename => $filename));
216 my $resp = App::Chart::Download->get ($url,
217 url_tags_key => 'TGE-day');
218 if (! $resp->is_success) {
219 # not modified, no new data
220 return 1;
221 }
222 my $got_tdate = csv_tdate ($resp);
223 if ($got_tdate == $start_tdate) {
224 # got the expected data, process it
225 my $content = $resp->decoded_content (charset => 'none');
226 my $h = csv_parse ($content);
227 $h->{'url_tags_key'} = 'TGE-day';
228 return 1;
229 } elsif ($got_tdate < $avail_tdate) {
230 # got something older, there's no new data
231 return 1;
232 } else {
233 return 0;
234 }
235 }
236  
237 sub download_year {
238 my ($symbol_list) = @_;
239 my $commodity = App::Chart::symbol_commodity ($symbol_list->[0]);
240 my $avail_tdate = available_tdate();
241  
242  
243 # try 0, -1, -2 direct filenames, then finally download page
244 # (allowing for new COMM_NUM values too)
245 #
246 download_year_attempt (download_tdate_url ($commodity, $avail_tdate))
247 || download_year_attempt (download_tdate_url ($commodity, $avail_tdate - 1))
248 || download_year_attempt (download_tdate_url ($commodity, $avail_tdate - 2))
249 || download_year_attempt (download_page ($commodity));
250 }
251  
252 # return true if successful
253 sub download_year_attempt {
254 my ($commodity, $url) = @_;
255  
256 $url =~ m{/([^/]+)$/};
257 my $filename = $1;
258 App::Chart::Download::status (__x('TGE data {filename}',
259 filename => $filename));
260  
261 my $resp = App::Chart::Download->get ($url, allow_404 => 1);
262 if (! $resp->is_success) { return 0; }
263  
264 # got the expected data, process it
265 my $h = zip_parse ($resp);
266 $h->{'url_tags_key'} = 'TGE-day';
267 return 1;
268 }
269  
270 # eg. http://www.tge.or.jp/data/down_load/co01040610050609.zip
271 # for Jun/10/2004 - Jun/09/2005
272 sub download_tdate_url {
273 my ($commodity, $tdate) = @_;
274 my ($end_year, $end_month, $end_day) = App::Chart::tdate_to_ymd ($tdate);
275  
276 # a year ago, plus one day
277 my ($start_year, $start_month, $start_day) = Date::Calc::Add_Delta_YMD
278 ($end_year, $end_month, $end_day, -1, 0, 1);
279  
280 return sprintf 'http://www.tge.or.jp/data/down_load/%s01%02d%02d%02d%02d%02d%02d.zip',
281 lc($commodity),
282 $start_year % 100, $start_month, $start_day,
283 $end_year % 100, $end_month, $end_day;
284 }
285  
286 sub zip_parse {
287 my ($resp) = @_;
288 my $zipstr = $resp->decoded_content (charset => 'none', raise_error => 1);
289 my $h;
290  
291 require Archive::Zip;
292 require IO::String;
293 my $zip = Archive::Zip->new;
294 my $io = IO::String->new ($zipstr);
295 $zip->readFromFileHandle ($io);
296  
297 foreach my $member ($zip->members) {
298 my $csv = $member->contents;
299 my $hh = csv_parse ($csv);
300 if ($h) {
301 push @{$h->{'data'}}, @{$hh->{'data'}};
302 } else {
303 $h = $hh;
304 }
305 }
306 return $h;
307 }
308  
309 sub csv_parse {
310 my ($content) = @_;
311 $content =~ s/\r//g;
312 my @lines = split /\n/, $content;
313  
314 my $heading = shift @lines;
315 $heading eq 'yr_mo_dy,contract,contract_month,contract_price_m1,contract_price_m2,contract_price_m3,contract_price_a1,contract_price_a2,contract_price_a3,sett_price,volume,open_int,net_position'
316 or die "TGE: unrecognised CSV headings: $heading";
317  
318 my @data;
319 my $h = { source => __PACKAGE__,
320 currency => 'JPY',
321 month_format => 'MMM_YY',
322 expiry_proc => \&symbol_expiry_date,
323 date_format => 'ymd',
324 suffix => '.TGE',
325 last_download => 1,
326 data => \@data };
327  
328 foreach my $line (@lines) {
329 my ($date, $commodity, $month, $m1, $m2, $m3, $a1, $a2, $a3, $settle,
330 $volume, $openint, $net)
331 = split /,/, $line;
332  
333 push @data, { date => $date, # eg. '20040603'
334 commodity => $commodity,
335 month => YYYYMM_to_iso($month), # eg. '200407'
336 sessions => [$m1, $m2, $m3, $a1, $a2, $a3, $settle],
337 volume => $volume,
338 openint => $openint };
339 }
340 return $h;
341 }
342  
343 sub YYYYMM_to_iso {
344 my ($str) = @_;
345 return substr($str,0,4) . '-' . substr($str,4) . '-01';
346 }
347  
348 1;
349 __END__