| 1 | # Copyright 2008, 2009, 2010, 2011, 2016, 2018, 2024 Kevin Ryde | ||
| 2 | |||
| 3 | # This file is part of Chart. | ||
| 4 | # | ||
| 5 | # Chart is free software; you can redistribute it and/or modify it under the | ||
| 6 | # terms of the GNU General Public License as published by the Free Software | ||
| 7 | # Foundation; either version 3, or (at your option) any later version. | ||
| 8 | # | ||
| 9 | # Chart is distributed in the hope that it will be useful, but WITHOUT ANY | ||
| 10 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
| 11 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | ||
| 12 | # details. | ||
| 13 | # | ||
| 14 | # You should have received a copy of the GNU General Public License along | ||
| 15 | # with Chart. If not, see <http://www.gnu.org/licenses/>. | ||
| 16 | |||
| 17 | package App::Chart::UserAgent; | ||
| 18 | use 5.006; | ||
| 19 | use strict; | ||
| 20 | use warnings; | ||
| 21 | use App::Chart; | ||
| 22 | use Locale::TextDomain ('App-Chart'); | ||
| 23 | use HTTP::Message 5.814; # for decodable() | ||
| 24 | use base 'LWP::UserAgent'; | ||
| 25 | |||
| 26 | use Class::Singleton 1.03; # 1.03 for _new_instance() | ||
| 27 | use base 'Class::Singleton'; | ||
| 28 | *_new_instance = \&new; | ||
| 29 | |||
| 30 | # Crib notes: | ||
| 31 | # | ||
| 32 | # URIs: | ||
| 33 | # Current RFC3986 | ||
| 34 | # | ||
| 35 | # Past RFC2396 -- previous URI spec | ||
| 36 | # RFC1808 -- relative URI spec | ||
| 37 | # RFC1738 -- previous again | ||
| 38 | # RFC1736, RFC1737 -- functional specs | ||
| 39 | # | ||
| 40 | # Proxies: | ||
| 41 | # /usr/share/doc/libwww-doc/html/Library/User/Using/Proxy.html | ||
| 42 | # http_proxy, no_proxy | ||
| 43 | # Eg. http_proxy=http://proxy.zipworld.com.au:8080/ | ||
| 44 | # | ||
| 45 | # /usr/share/doc/lynx/lynx_help/keystrokes/environments.html | ||
| 46 | # shows spaces in no_proxy | ||
| 47 | # | ||
| 48 | # wget.info Proxies | ||
| 49 | # | ||
| 50 | # | ||
| 51 | # something ... | ||
| 52 | # @LWP::Protocol::http::EXTRA_SOCK_OPTS = (PeerAddr => "foo.com"); | ||
| 53 | # | ||
| 54 | |||
| 55 | # LWP::Protocol::http::EXTRA_SOCK_OPTS is used by LWP::Protocol::http | ||
| 56 | # when creating its I/O socket class. That class is | ||
| 57 | # LWP::Protocol::http::Socket, which is a subclass Net::HTTP. | ||
| 58 | # | ||
| 59 | # Net::HTTP made an incompatible change, circa its version 6.23, | ||
| 60 | # to its default MaxLineLength, dropping from 32 kbytes to 8 kbytes. | ||
| 61 | # This breaks on some long header lines from finance.yahoo.com in | ||
| 62 | # App::Chart::Yahoo data downloads. Those lines are somewhere up | ||
| 63 | # 6 kbytes, maybe more depending on the share symbol or something. | ||
| 64 | # That's a foolish amount to have in a header line, but want it to | ||
| 65 | # work here. | ||
| 66 | # | ||
| 67 | # MaxLineLength => 0 here means no length limit. | ||
| 68 | # Would have preferred to get this in via the UserAgent instance | ||
| 69 | # or class here, but looks like no way to get through to that | ||
| 70 | # protocol bits. | ||
| 71 | # | ||
| 72 | use LWP::Protocol::http; | ||
| 73 | push @LWP::Protocol::http::EXTRA_SOCK_OPTS, MaxLineLength => 0; | ||
| 74 | |||
| 75 | |||
| 76 | sub new { | ||
| 77 | my ($class, %options) = @_; | ||
| 78 | if (! exists $options{'keep_alive'} | ||
| 79 | && ! exists $options{'conn_cache'}) { | ||
| 80 | $options{'keep_alive'} = 1; # connection cache 1 sock | ||
| 81 | } | ||
| 82 | if (! exists $options{'agent'}) { | ||
| 83 | # not given, use our default | ||
| 84 | $options{'agent'} = $class->_agent; | ||
| 85 | } elsif (defined $options{'agent'} && $options{'agent'} =~ / $/) { | ||
| 86 | # ends in space, append our default | ||
| 87 | $options{'agent'} .= $class->_agent; | ||
| 88 | } | ||
| 89 | my $self = $class->SUPER::new (timeout => 60, # seconds, default | ||
| 90 | %options); | ||
| 91 | |||
| 92 | # with lwp bit appended | ||
| 93 | # $self->agent (); | ||
| 94 | |||
| 95 | # ask for everything decoded_content() accepts | ||
| 96 | $self->default_header ('Accept-Encoding' => scalar HTTP::Message::decodable()); | ||
| 97 | |||
| 98 | # trace on redirect ... | ||
| 99 | |||
| 100 | return $self; | ||
| 101 | } | ||
| 102 | |||
| 103 | sub _agent { | ||
| 104 | my ($class) = @_; | ||
| 105 | return "Chart/$App::Chart::VERSION " . $class->SUPER::_agent; | ||
| 106 | } | ||
| 107 | |||
| 108 | sub redirect_ok { | ||
| 109 | my ($self, $prospective_request, $response) = @_; | ||
| 110 | my $ret = $self->SUPER::redirect_ok ($prospective_request, $response); | ||
| 111 | if ($ret) { | ||
| 112 | print "Redirect to ",$prospective_request->uri,"\n"; | ||
| 113 | } | ||
| 114 | return $ret; | ||
| 115 | } | ||
| 116 | |||
| 117 | sub request { | ||
| 118 | my ($self, @args) = @_; | ||
| 119 | $self->{__PACKAGE__.'.last_time'} = time() - 1; # provoke initial display | ||
| 120 | return $self->SUPER::request (@args); | ||
| 121 | } | ||
| 122 | |||
| 123 | # method call up from LWP::UserAgent | ||
| 124 | sub progress { | ||
| 125 | my ($self, $status, $response) = @_; | ||
| 126 | |||
| 127 | my $str = $status; | ||
| 128 | if ($status eq 'begin') { | ||
| 129 | $str = __('connect'); | ||
| 130 | |||
| 131 | } elsif ($response && $status ne 'end') { | ||
| 132 | my $time = time(); | ||
| 133 | if ($time != $self->{__PACKAGE__.'.last_time'}) { | ||
| 134 | $self->{__PACKAGE__.'.last_time'} = $time; | ||
| 135 | |||
| 136 | my $got = length ($response->content); | ||
| 137 | my $total = $response->content_length; | ||
| 138 | if (defined $total) { | ||
| 139 | $str = __x('{got} of {total} bytes', | ||
| 140 | got => $got, total => $total); | ||
| 141 | } else { | ||
| 142 | $str = __x('{got} bytes', got => $got); | ||
| 143 | } | ||
| 144 | } else { | ||
| 145 | $str = undef; | ||
| 146 | } | ||
| 147 | } | ||
| 148 | if (defined $str) { | ||
| 149 | require App::Chart::Download; | ||
| 150 | App::Chart::Download::substatus ($str); | ||
| 151 | } | ||
| 152 | |||
| 153 | return $self->SUPER::progress ($status, $response); | ||
| 154 | } | ||
| 155 | |||
| 156 | sub run_handlers { | ||
| 157 | my ($self, $phase, $o) = @_; | ||
| 158 | if ($phase eq 'request_send') { | ||
| 159 | if ($App::Chart::option{'verbose'} >= 2) { | ||
| 160 | print $o->as_string,"\n"; | ||
| 161 | } elsif ($App::Chart::option{'verbose'} >= 1) { | ||
| 162 | print $o->method," ",$o->url,"\n"; | ||
| 163 | } | ||
| 164 | } | ||
| 165 | if ($phase eq 'response_done') { | ||
| 166 | if ( $App::Chart::option{'verbose'} >= 1) { | ||
| 167 | print $o->status_line,"\n"; | ||
| 168 | } | ||
| 169 | if ( $App::Chart::option{'verbose'} >= 2) { | ||
| 170 | print $o->headers->as_string,"\n"; | ||
| 171 | my $content = $o->decoded_content; | ||
| 172 | if (length $content <= 1000) { | ||
| 173 | print "content:\n", $content, "\n\n"; | ||
| 174 | } | ||
| 175 | } | ||
| 176 | } | ||
| 177 | return $self->SUPER::run_handlers ($phase, $o); | ||
| 178 | } | ||
| 179 | |||
| 180 | # use Data::Dumper; | ||
| 181 | # *LWP::Protocol::http::SocketMethods::configure = sub { | ||
| 182 | # my $self = shift; | ||
| 183 | # print Dumper (\@_); | ||
| 184 | # $self->SUPER::configure (@_); | ||
| 185 | # }; | ||
| 186 | |||
| 187 | |||
| 188 | 1; | ||
| 189 | __END__ |