App/Chart/Barchart.pm

filter: Statementstmt 93.7% 30 / 32 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 11 / 11 Podpod n/a totaltotal 95.3% 41 / 43 CC 1 SCAR 0.3
CC
1
Cov
94%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« App/Chart/TZ.pm ↑ Summary App/Chart/Download.pm »
1 # Copyright 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015, 2017 Kevin Ryde
2  
3 # This file is part of Chart.
4 #
5 # Chart is free software; you can redistribute it and/or modify it under the
6 # terms of the GNU General Public License as published by the Free Software
7 # Foundation; either version 3, or (at your option) any later version.
8 #
9 # Chart is distributed in the hope that it will be useful, but WITHOUT ANY
10 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
11 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
12 # details.
13 #
14 # You should have received a copy of the GNU General Public License along
15 # with Chart. If not, see <http://www.gnu.org/licenses/>.
16  
17  
18 # Broken by hideous recent barchart.com site.
19  
20  
21 package App::Chart::Barchart;
22 1 ▶use 5.006;
Subroutine BEGIN: called CC=1 SCAR=0.0
23 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
24 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
25 1 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
26 1 ▶use Date::Calc;
Subroutine BEGIN: called CC=1 SCAR=0.0
27 1 ▶use Date::Parse;
Subroutine BEGIN: called CC=1 SCAR=0.0
28 1 ▶use List::Util qw (min max);
Subroutine BEGIN: called CC=1 SCAR=0.0
29 1 ▶use POSIX ();
Subroutine BEGIN: called CC=1 SCAR=0.0
30 1 ▶use URI::Escape;
Subroutine BEGIN: called CC=1 SCAR=0.0
31 1 ▶use Locale::TextDomain ('App-Chart');
Subroutine BEGIN: called CC=1 SCAR=0.0
32  
33 0 ▶use App::Chart;
Subroutine BEGIN: called CC=1 SCAR=2.6
34 use App::Chart::Database;
35 use App::Chart::Download;
36 use App::Chart::DownloadHandler;
37 use App::Chart::IntradayHandler;
38 use App::Chart::Latest;
39 use App::Chart::Pagebits;
40 use App::Chart::Sympred;
41 use App::Chart::TZ;
42 use App::Chart::Weblink;
43  
44 # uncomment this to run the ### lines
45 #use Smart::Comments;
46  
47  
48 our $latest_pred = App::Chart::Sympred::Any->new;
49 our $intraday_pred = App::Chart::Sympred::Any->new;
50 our $fiveday_pred = App::Chart::Sympred::Any->new;
51  
52 # overridden by specific nodes
53 App::Chart::setup_source_help
54 ($fiveday_pred, __p('manual-node','Barchart'));
55  
56 sub setup {
57 my ($setup_pred) = @_;
58 $latest_pred->add ($setup_pred);
59 $intraday_pred->add ($setup_pred);
60 $fiveday_pred->add ($setup_pred);
61 }
62  
63 #-----------------------------------------------------------------------------
64 # various
65  
66 sub cookie_jar {
67 require HTTP::Cookies;
68 my $jar = HTTP::Cookies->new;
69  
70 # Set-Cookie: bcad_int=1; path=/; domain=barchart.com;
71 $jar->set_cookie(undef, # version
72 'bcad_int', # key
73 '1', # value
74 '/', # path
75 '.barchart.com'); # domain
76 return $jar;
77 }
78  
79  
80 #-----------------------------------------------------------------------------
81 # symbol munging
82  
83 my @commodity_mung;
84  
85 sub commodity_mung {
86 my ($pred, %table) = @_;
87 push @commodity_mung, [ $pred, \%table ];
88 }
89  
90 sub symbol_to_barchart {
91 my ($symbol) = @_;
92 foreach my $elem (@commodity_mung) {
93 if ($elem->[0]->match ($symbol)) {
94 my $commodity = App::Chart::symbol_commodity ($symbol);
95 my $barchart = $elem->[1]->{$commodity};
96 if (defined $barchart) {
97 $symbol =~ s/^\Q$commodity/$barchart/;
98 }
99 last;
100 }
101 }
102 $symbol =~ s/.[^.]*$//;
103 return $symbol;
104 }
105  
106  
107 #------------------------------------------------------------------------------
108 # latest
109 #
110 # The intraday commodity quotes pages are used, like oats
111 #
112 # http://www2.barchart.com/ifutpage.asp?code=BSTK&sym=O
113 #
114 # which is about 35 kbytes each. An alternative would be the combined
115 # pages like all grains
116 #
117 # http://www2.barchart.com/mktcom.asp?code=BSTK&section=grains
118 #
119 # which has the front month or two of various at about 50kbytes the lot.
120  
121 App::Chart::LatestHandler->new
122 (pred => $latest_pred,
123 proc => \&latest_download,
124 max_symbols => 1);
125  
126 sub latest_download {
127 my ($symbol_list) = @_;
128  
129 my $symbol = $symbol_list->[0];
130 my $commodity = App::Chart::symbol_commodity ($symbol);
131 my $suffix = App::Chart::symbol_suffix ($symbol);
132 my $barchart_commodity = symbol_to_barchart ("$commodity$suffix");
133  
134 App::Chart::Download::status
135 (__x('Barchart quote {symbol} ({barchart_commodity})',
136 symbol => "$commodity$suffix",
137 barchart_commodity => $barchart_commodity));
138  
139 my $url = 'http://www2.barchart.com/ifutpage.asp?code=BSTK&sym='
140 . URI::Escape::uri_escape ($barchart_commodity);
141  
142 my $resp = App::Chart::Download->get ($url,
143 cookie_jar => cookie_jar(),
144 referer => $url);
145 my $h = ifutpage_parse ($commodity, $suffix, $resp);
146 App::Chart::Download::write_latest_group ($h);
147 }
148  
149 sub ifutpage_parse {
150 my ($commodity, $suffix, $resp) = @_;
151 my $content = $resp->decoded_content (raise_error => 1);
152  
153 my @data = ();
154 my $h = { source => __PACKAGE__,
155 resp => $resp,
156 front_month => 1,
157 data => \@data };
158  
159 # eg. " <B>CRUDE OIL</B> Delayed Futures -20:10 - Sunday, 19 June"
160 # " <B>SIMEX NIKKEI 225</B> Delayed Futures -18:20 - Tuesday, 12 December</td>"
161 # " <B>OATS </B> Daily Futures - Friday, 20 April </td>
162 $content =~ m{([^<>\r\n]+) *</B> Delayed Futures *- *([0-9]+:[0-9]+) *- *[A-Za-z]+, ([0-9]+ [A-Za-z]+)}is
163 or die 'Barchart: ifutpage name/date/time not matched';
164 my $name = $1;
165 my $head_time = $2;
166 my $head_date = $3;
167 ### ifutpage head name: $name
168 ### $head_time
169 ### $head_date
170 require App::Chart::Suffix::NZ;
171 $head_date = App::Chart::Suffix::NZ::dm_str_to_nearest_iso ($head_date);
172  
173 require HTML::TableExtract;
174  
175 my $te = HTML::TableExtract->new
176 (headers => ['Contract', 'Last', 'Change', 'Open', 'High', 'Low', 'Time']);
177 $te->parse($content);
178 if (! $te->tables) { die 'Barchart: ifutpage price columns not matched'; }
179  
180 foreach my $row ($te->rows) {
181 ### ifutpage row: $row
182  
183 my ($month, $last, $change, $open, $high, $low, $time) = @$row;
184  
185 # eg. "August '05 ( CLQ05 )"
186 $month =~ /\( *([^ )]+) *\)/p
187 or die 'Barchart: ifutpage month form not recognised';
188 my $month_name = ${^PREMATCH}; # "August '05"
189 my $bar_symbol = $1; # "CLQ05"
190 my $MYY = substr ($bar_symbol, -3);
191 my $symbol = $commodity . $MYY . $suffix;
192 $month_name = App::Chart::collapse_whitespace ($month_name);
193 my $month_iso =App::Chart::Download::Decode_Date_EU_to_iso("1 $month_name");
194  
195 # # subtract normal exchange delay to get quote time
196 # (set! quote-time (- quote-time (barchart-symbol-delay symbol)))
197 # (if (negative? quote-time)
198 # (begin
199 # (set! quote-time (+ quote-time 86400))
200 # (set! quote-adate (1- quote-adate))))
201 #
202  
203 # trailing "s" on last for settlement price
204 # have also seen "c", maybe for close
205 $last =~ s/[cs]$//i;
206  
207 my ($last_date, $last_time);
208 if ($time =~ /:/) {
209 # time is HH:MM on same day as the quote
210 $last_date = $head_date;
211 $last_time = $time;
212 ($last_date, $last_time)
213 = datetime_chicago_to_symbol ($symbol, $last_date, $last_time);
214  
215 } else {
216 # time is a date MM/DD/YY later (on the weekend)
217 $last_date = App::Chart::Download::Decode_Date_US_to_iso ($time);
218  
219 # FIXME: convert date from Chicago ?
220 }
221  
222 # dash is frac in various CBOT
223 if ($last =~ /-/) {
224 $open = dash_frac_to_decimals ($open);
225 $high = dash_frac_to_decimals ($high);
226 $low = dash_frac_to_decimals ($low);
227 $last = dash_frac_to_decimals ($last);
228 $change = dash_frac_to_decimals ($change);
229 }
230  
231 push @data, { symbol => $symbol,
232 name => "$name $month_name",
233 month => $month_iso,
234  
235 # quote_date => $quote_date,
236 # quote_time => $quote_time,
237  
238 last_date => $last_date,
239 last_time => $last_time,
240 open => $open,
241 high => $high,
242 low => $low,
243 last => $last,
244 change => $change,
245 };
246 }
247 return $h;
248 }
249  
250 sub datetime_chicago_to_symbol {
251 my ($symbol, $date, $time) = @_;
252  
253 my $chicago = App::Chart::TZ->chicago;
254 my $stimezone = App::Chart::TZ->for_symbol ($symbol);
255 if ($stimezone == $chicago) { return ($date, $time); }
256  
257 my ($sec,$min,$hour,$mday,$mon,$year)
258 = Date::Parse::strptime($date . ' ' . $time);
259 require App::Chart::Yahoo;
260 my $timet = App::Chart::Yahoo::mktime_in_zone
261 ($sec, $min, $hour, $mday, $mon, $year, $chicago);
262 return $stimezone->iso_date_time ($timet);
263 }
264  
265 # FIXME: Share with Finance::Quote::Barchart
266 #
267 # convert number like "99-1" with dash fraction to decimals like "99.125"
268 # single dash digit is 1/8s
269 # three dash digits -xxy is xx 1/32s and y is 0,2,5,7 for further 1/4, 2/4,
270 # or 3/4 of 1/32
271 #
272 my %qu_to_quarter = (''=>0, 0=>0, 2=>1, 5=>2, 7=>3);
273 sub dash_frac_to_decimals {
274 my ($str) = @_;
275  
276 $str =~ /^\+?(.+)-(.*)/p or return $str;
277 my $int = $1;
278 my $frac = $2;
279  
280 if (length ($frac) == 1) {
281 # 99-1
282 # only 2 decimals for 1/4s, since for various commodities that's the
283 # minimum tick
284 return $int + ($frac / 8);
285  
286 } elsif (length ($frac) == 2 || length ($frac) == 3) {
287 # 109-30, in 1/32nds
288 # 99-130, in 1/32s then last dig 0,2,5,7 further 1/4s of that
289 my $th = substr $frac, 0, 2;
290 if ($th > 31) {
291 die "Barchart: dash thirtyseconds out of range: $str";
292 }
293 my $qu = substr($frac, 2, 1);
294 if (! exists $qu_to_quarter{$qu}) {
295 die "Barchart: dash thirtyseconds further quarters unrecognised: $str";
296 }
297 $qu = $qu_to_quarter{$qu};
298 return $int + (($th + $qu / 4) / 32);
299  
300 } else {
301 die "Barchart: unrecognised dash number: $str";
302 }
303 }
304  
305 sub dm_str_to_nearest_iso {
306 my ($str) = @_;
307 require Date::Parse;
308 my ($sec, $min, $hour, $mday, $mon, $year) = Date::Parse::strptime($str);
309 if ($year) {
310 $year += 1900;
311 } else {
312 $year = App::Chart::Download::month_to_nearest_year ($mon+1);
313 }
314 return App::Chart::ymd_to_iso ($year, $mon+1, $mday);
315 }
316  
317  
318 #-----------------------------------------------------------------------------
319 # 5-day download
320 #
321 # This uses the rolling 5-day quote pages like
322 #
323 # http://www.barchart.com/detailedquote/futures/CLZ16
324 #
325 use constant FIVEDAY_URL_BASE =>
326 'http://www.barchart.com/detailedquote/futures/';
327 #
328 # which has daily open, high, low and close, volume.
329 #
330 # Going back only five days isn't good for much, but it's better than
331 # nothing and regular updates accumulate enough for a short-term picture.
332  
333 App::Chart::DownloadHandler->new
334 (name => __('Barchart'),
335 pred => $fiveday_pred,
336 available_tdate => \&fiveday_available_tdate,
337 proc => \&fiveday_download,
338 max_symbols => 1);
339  
340 # latest data available from barchart 5-day quote page
341 # the support page says the daily pages update at 5pm US central, try 5pm
342 # local for the 5-day
343 sub fiveday_available_tdate {
344 return App::Chart::Download::weekday_tdate_after_time
345 (17,0, App::Chart::TZ->chicago, -1);
346 }
347  
348 sub fiveday_download {
349 my ($symbol_list) = @_;
350  
351 my $symbol = $symbol_list->[0];
352 if (App::Chart::symbol_is_front ($symbol)) { return; }
353  
354 # my $commodity = App::Chart::symbol_commodity ($symbol);
355 my $suffix = App::Chart::symbol_suffix ($symbol);
356 my $barchart_symbol = symbol_to_barchart ($symbol);
357  
358 if ($barchart_symbol.$suffix eq $symbol) {
359 App::Chart::Download::status
360 (__x('Barchart five day {symbol}',
361 symbol => $symbol));
362 } else {
363 App::Chart::Download::status
364 (__x('Barchart five day {symbol} ({barchart_symbol})',
365 symbol => $symbol,
366 barchart_symbol => $barchart_symbol));
367 }
368  
369 my $url = FIVEDAY_URL_BASE . URI::Escape::uri_escape ($barchart_symbol);
370  
371 my $resp = App::Chart::Download->get ($url,
372 cookie_jar => cookie_jar(),
373 referer => $url);
374 my $h = fiveday_parse ($symbol, $resp);
375 App::Chart::Download::write_daily_group ($h);
376 }
377  
378 sub fiveday_parse {
379 my ($symbol, $resp) = @_;
380  
381 my @data;
382 my $h = { source => __PACKAGE__,
383 prefer_decimals => 2,
384 date_format => 'mdy',
385 data => \@data };
386  
387 my $content = $resp->decoded_content (raise_error => 1);
388  
389 # message in table when no info for given symbol (eg. an old month/year)
390 if ($content =~ /In order to form an extended quote/) {
391 return $h;
392 }
393  
394 # eg. <h1 class="fl" id="symname">Crude Oil WTI December 2016 (CLZ16)</h1>
395 #
396 $content =~ m{<h1[^>]* id="symname">(([^<\r\n]*?) [a-z]+ [0-9]+)}si
397 or die "Barchart fiveday heading not matched";
398 my $name = $1;
399 ### $name
400  
401 require HTML::TableExtract;
402 my $te = HTML::TableExtract->new
403 (headers => ['Date', 'Open', 'High', 'Low', 'Last', 'Volume']);
404 $te->parse($content);
405 my ($ts) = $te->tables
406 or die 'Barchart: fiveday price columns not matched';
407  
408 foreach my $row ($ts->rows) {
409 ### fiveday row: $row
410 my ($date, $open, $high, $low, $close, $volume) = @$row;
411  
412 $open = dash_frac_to_decimals ($open);
413 $high = dash_frac_to_decimals ($high);
414 $low = dash_frac_to_decimals ($low);
415 $close = dash_frac_to_decimals ($close);
416  
417 push @data, { symbol => $symbol,
418 name => $name,
419 date => $date,
420 open => $open,
421 high => $high,
422 low => $low,
423 close => $close,
424 volume => $volume,
425 };
426 }
427 return $h;
428 }
429  
430  
431 #-----------------------------------------------------------------------------
432 # intraday
433 #
434 # This uses the charting pages like
435 #
436 # http://www.barchart.com/charts/futures/CLZ10
437 #
438 # which for a 2-day intraday is
439 #
440 # http://www.barchart.com/chart.php?sym=CLZ10&style=technical&p=I&d=O&im=&sd=&ed=&size=M&log=0&t=BAR&v=2&g=1&evnt=1&late=1&o1=&o2=&o3=&x=41&y=11&indicators=&addindicator=&submitted=1&fpage=&txtDate=#jump
441 #
442 # must be fetched (about 50kbytes unfortunately), and it has a gif url.
443 # That url is some generated 4-digit number, apparently different each
444 # time, even outside trading hours. The server doesn't give an etag or
445 # last-modified to avoid re-downloading.
446 #
447 # The form fields are
448 #
449 # sym=
450 # date=
451 # size=A 504 by 288
452 # E 576 by 360
453 # B 612 by 360
454 # C 720 by 432
455 # D 864 by 504
456 # data=Z05 minutes
457 # Z10 "
458 # Z15 "
459 # Z30 "
460 # Z45 "
461 # Z60 "
462 # Z90 "
463 # A daily
464 # D weekly
465 # G monthly
466 # den=HIGH density
467 # MEDHI
468 # MED [default]
469 # MEDLO
470 # LOW
471 # this combines with data period for how much is shown
472 # 5min 10min 15min
473 # HIGH 3.5day 7day 10day ... etc
474 # MEDHI 2day 4.5day 7day
475 # MED 1.5day 3day 4day
476 # MEDLO 1day 2day 3day
477 # LOW 0.5day 1.5day 2day
478 # evnt=ADV events
479 # off
480 # grid=Y/N background green grid
481 # sky=Y/N fine grid lines
482 # jav=ADV prices on, gif file
483 # [other options for subscribers]
484 # size=A 504 by 288
485 # E 576 by 360
486 # B 612 by 360
487 # C 720 by 432
488 # D 864 by 504
489 # sly=N linear
490 # L log
491 # Y fit available space
492 # late=
493 # ch1=011 OHLC [default]
494 # 012 close-only
495 # 013 candlestick
496 # ...
497 # ov1=
498 # ch2=
499 # ov2=
500 # code=BSTKIC IC for interactive chart
501 # vol=y/n volume (not avail for 5min)
502  
503 foreach my $n (1, 2, 5) {
504 App::Chart::IntradayHandler->new
505 (pred => $intraday_pred,
506 proc => \&intraday_url,
507 mode => "${n}d",
508 name => __nx('_{n} Day',
509 '_{n} Days',
510 $n,
511 n => $n));
512 }
513 App::Chart::IntradayHandler->new
514 (pred => $intraday_pred,
515 proc => \&intraday_url,
516 mode => 'daily',
517 name => __('_Daily 2 Months'));
518 App::Chart::IntradayHandler->new
519 (pred => $intraday_pred,
520 proc => \&intraday_url,
521 mode => 'daily',
522 name => __('_Daily 1 Year'));
523  
524 # 5 minute, linear scale
525  
526 my %intraday_mode_to_data = ('1d' => '&p=I&d=L',
527 '2d' => '&p=I&d=O',
528 '3d' => '&p=I&d=M',
529 '4d' => '&p=I&d=H',
530 '5d' => '&p=I&d=X',
531 'daily2m' => '&p=DO&d=L',
532 'daily1y' => '&p=DO&d=X');
533  
534 # '7d' => '&data=Z60&den=MEDHIGH',
535 # 'daily' => '&data=A');
536  
537 sub intraday_url {
538 my ($self, $symbol, $mode) = @_;
539  
540 App::Chart::Download::status
541 (__x('Barchart intraday page {symbol} {mode}',
542 symbol => $symbol,
543 mode => $mode));
544 my $url = 'http://www.barchart.com/chart.php?sym='
545 . URI::Escape::uri_escape (symbol_to_barchart ($symbol))
546 . $intraday_mode_to_data{$mode};
547 App::Chart::Download::verbose_message ("Intraday page", $url);
548  
549 my $jar = cookie_jar();
550 my $resp = App::Chart::Download->get ($url,
551 cookie_jar => $jar,
552 referer => $url);
553 return (intraday_resp_to_url ($resp, $symbol),
554 cookie_jar => $jar,
555 referer => $url);
556 }
557 # separate func for offline testing ...
558 sub intraday_resp_to_url {
559 my ($resp, $symbol) = @_;
560 my $content = $resp->decoded_content (raise_error => 1);
561  
562 require HTML::LinkExtor;
563 my $parser = HTML::LinkExtor->new(undef, $resp->base);
564 $parser->parse($content);
565 $parser->eof;
566  
567 # eg.
568  
569 # </map><img src="/cache/bde71ebe23ddac66f2d25081b1b5f953.png"
570 # must match some of the link target name since there's other images in
571 # the page
572 foreach my $link ($parser->links) {
573 my ($tag, %attr) = @$link;
574 $tag eq 'img' or next;
575 my $url = $attr{'src'};
576 index ($url, '/cache/') >= 0 or next;
577 ### $url
578 return URI->new_abs($url,$resp->base)->as_string;
579 }
580  
581 if ($content =~ /Could not find any symbols/i) {
582 die __x("No such symbol {symbol}\n",
583 symbol => $symbol);
584 } else {
585 die 'Barchart Customer: Intraday page not matched';
586 }
587 }
588  
589 1;
590 __END__
« App/Chart/TZ.pm ↑ Summary App/Chart/Download.pm »