App/Chart/Suffix/TSP.pm untested

filter: Statementstmt 0.0% 0 / 118 Branchbran 0.0% 0 / 6 Conditioncond 0.0% 0 / 4 MC/DCmcdc 0.0% 0 / 2 Subroutinesub 0.0% 0 / 24 Podpod 0.0% 0 / 5 totaltotal 0.0% 0 / 159 CC 4 SCAR 12.3
CC
4
Cov
0%
parse
12.0
download
6.0
available_tdate
2.0
CRAP 3.4
1 # Copyright 2008, 2009, 2010, 2011, 2015, 2016, 2023 Kevin Ryde
2  
3 # This file is part of Chart.
4 #
5 # Chart is free software; you can redistribute it and/or modify it under the
6 # terms of the GNU General Public License as published by the Free Software
7 # Foundation; either version 3, or (at your option) any later version.
8 #
9 # Chart is distributed in the hope that it will be useful, but WITHOUT ANY
10 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
11 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
12 # details.
13 #
14 # You should have received a copy of the GNU General Public License along
15 # with Chart. If not, see <http://www.gnu.org/licenses/>.
16  
17 package App::Chart::Suffix::TSP;
18 use 5.006;
19 use strict;
20 use warnings;
21 no warnings 'once';
22 use List::Util qw(min max);
23 use Locale::TextDomain 'App-Chart';
24  
25 use App::Chart;
26 use App::Chart::Download;
27 use App::Chart::DownloadHandler;
28 use App::Chart::FinanceQuote;
29 use App::Chart::IntradayHandler;
30 use App::Chart::Sympred;
31 use App::Chart::TZ;
32 use App::Chart::Weblink;
33  
34 # uncomment this to run the ### lines
35 # use Smart::Comments;
36  
37 use constant DEBUG => 0;
38  
39  
40 my $pred = App::Chart::Sympred::Suffix->new ('.TSP');
41  
42 # FIXME: is east coast right?
43 App::Chart::TZ->newyork->setup_for_symbol ($pred);
44  
45 App::Chart::setup_source_help
46 ($pred, __p('manual-node','Thrift Savings Plan'));
47  
48 App::Chart::FinanceQuote->setup (pred => $pred,
49 suffix => '.TSP',
50 modules => ['TSP'],
51 method => 'tsp');
52  
53  
54 #------------------------------------------------------------------------------
55 # weblink
56  
57 # only home page, per "Linkage to the TSP Web Site" requirement
58 App::Chart::Weblink->new
59 (pred => $pred,
60 name => __('_TSP Home Page'),
61 desc => __('Open web browser at the US Government Thrift Savings Plan'),
62 proc => sub {
63 require Finance::Quote::TSP;
64 return $Finance::Quote::TSP::TSP_MAIN_URL;
65 });
66  
67  
68 #-----------------------------------------------------------------------------
69 # download
70 #
71 # This uses the historical prices page
72 # https://www.tsp.gov/share-price-history/
73 #
74 # The site won't even speak to you without a right User-Agent header.
75 # This is recent site breakage, as it was happy with anything in the past.
76 # Here follow Finance::Quote::TSP which uses the following,
77 #
78 use constant TSP_USER_AGENT =>
79 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.61 Safari/537.36';
80 #
81 # The download button is URL parameters after a base:
82 #
83 use constant TSP_SHARE_PRICES_URL =>
84 'https://www.tsp.gov/data/fund-price-history.csv';
85 #
86 # The parameters on the page are buried in some unhelpful script, eg.
87 #
88 # https://www.tsp.gov/data/fund-price-history.csv?startdate=$2023-01-01&enddate=2023-02-01&Lfunds=1&InvFunds=1&download=1
89 #
90 # The check boxes select individual funds for the display but seem to have
91 # effect on the download, so the download gets all funds at once, in the
92 # date range.
93 #
94 # Seems no limit to the size of the date range. In the past there it was 30
95 # days per request (and had made it 20 business days here). Data used to go
96 # back to 2003-06-02.
97 #
98 # --------
99 # Old POST:
100 # https://www.tsp.gov/InvestmentFunds/FundPerformance/index.html
101 # reloaded=1&startdate=04%2F27%2F2015&enddate=06%2F02%2F2015&fundgroup=L2020&fundgroup=G&fundgroup=C&whichButton=CSV
102 #
103 # Old POST:
104 # startdate=20030801&enddate=20031128&submit=Retrieve+Share+Prices&prev=0&next=30&whichButton=CSV
105 #
106 # Old page (now 403 redirects):
107 # https://www.tsp.gov/investmentfunds/shareprice/sharePriceHistory.shtml
108  
109 App::Chart::DownloadHandler->new
110 (name => __('TSP'),
111 pred => $pred,
112 available_tdate => \&available_tdate,
113 proc => \&download);
114  
115 # 18:55 still showing prev
116 # 20:22 showing today
117 sub available_tdate {
118 App::Chart::Download::tdate_today_after
119 (20,0, App::Chart::TZ->newyork);
120 }
121  
122 sub download {
123 my ($symbol_list) = @_;
124  
125 my $avail_tdate = available_tdate();
126 my $lo_tdate = App::Chart::Download::start_tdate_for_update (@$symbol_list);
127  
128 if ($lo_tdate > $avail_tdate) {
129 App::Chart::Download::verbose_message
130 (__('TSP nothing further expected yet'));
131 return;
132 }
133  
134 # ask for extra in case available_tdate() is a bit short
135 $avail_tdate += 2;
136  
137 my $resp = get_chunk ($symbol_list, $lo_tdate, $avail_tdate);
138 my $h = parse ($resp);
139 $h->{'last_download'} = 1;
140 App::Chart::Download::write_daily_group ($h);
141 }
142  
143 sub backto {
144 my ($symbol_list, $backto_tdate) = @_;
145 my $hi_tdate = App::Chart::Download::start_tdate_for_backto (@$symbol_list);
146 my $resp = get_chunk ($backto_tdate, $hi_tdate);
147 my $h = parse ($resp);
148 App::Chart::Download::write_daily_group ($h);
149 }
150  
151 # return a HTTP::Response for data $lo_tdate to $hi_tdate, inclusive
152 sub get_chunk {
153 my ($symbol_list, $lo_tdate, $hi_tdate) = @_;
154  
155 App::Chart::Download::status
156 (__x('TSP data {date_range}',
157 date_range =>
158 App::Chart::Download::tdate_range_string ($lo_tdate, $hi_tdate)));
159  
160 my ($lo_year, $lo_month, $lo_day) = App::Chart::tdate_to_ymd ($lo_tdate);
161 my ($hi_year, $hi_month, $hi_day) = App::Chart::tdate_to_ymd ($hi_tdate);
162 my $startdate = sprintf "%04d-%02d-%02d", $lo_year, $lo_month, $lo_day;
163 my $enddate = sprintf "%04d-%02d-%02d", $hi_year, $hi_month, $hi_day;
164  
165 return App::Chart::Download->get
166 (TSP_SHARE_PRICES_URL
167 . "?startdate=$startdate&enddate=$enddate&Lfunds=1&InvFunds=1&download=1",
168 user_agent => TSP_USER_AGENT);
169 }
170  
171 # $resp is a HTTP::Response object, return a hashref of data
172 sub parse {
173 my ($resp, $symbol_list) = @_;
174  
175 # From 1 Jul 2008 prices are 4 decimal places, and old data is padded with
176 # zeros. Could prefer_decimals to strip them, but may as well leave at
177 # current precision.
178 #
179 my @data = ();
180 my $h = { source => __PACKAGE__,
181 currency => 'USD',
182 date_format => 'ymd',
183 suffix => '.TSP',
184 data => \@data };
185  
186 my $content = $resp->decoded_content (raise_error=>1);
187 ### $content
188  
189 my @lines = App::Chart::Download::split_lines ($content);
190 my ($date, @symbols) = split /,/, $lines[0];
191 lc($date) eq 'date'
192 or die 'TSP csv doesn\'t start with "date": ', $lines[0];
193 shift @lines;
194 ### @symbols
195  
196 # "L Income" becomes uppercase "LINCOME.TSP" (documented in chart.texi
197 # that way)
198 @symbols = map { my $symbol = $_;
199 $symbol =~ s/ Fund//i;
200 $symbol =~ s/ //g;
201 "\U$symbol.TSP" }
202 @symbols;
203 ### @symbols
204 $symbol_list //= \@symbols;
205 my %symbol_list = map {$_ => 1} @$symbol_list;
206  
207 foreach my $line (@lines) {
208 my ($date, @prices) = split /,/, $line;
209 foreach my $c (0 .. $#prices) {
210 my $symbol = $symbols[$c];
211 next unless exists $symbol_list{$symbol};
212 push @data, { symbol => $symbol,
213 date => $date,
214 close => $prices[$c],
215 };
216 }
217 }
218 if (DEBUG) {
219 require List::Util;
220 print "min date ",List::Util::minstr(map {$_->{'date'}} @data),"\n";
221 print "max date ",List::Util::maxstr(map {$_->{'date'}} @data),"\n";
222 }
223 return $h;
224 }
225  
226 1;
227 __END__