line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package Feed::Data::Stream; |
2
|
|
|
|
|
|
|
|
3
|
12
|
|
|
12
|
|
92
|
use Moo; |
|
12
|
|
|
|
|
32
|
|
|
12
|
|
|
|
|
76
|
|
4
|
12
|
|
|
12
|
|
4438
|
use Carp qw/croak/; |
|
12
|
|
|
|
|
29
|
|
|
12
|
|
|
|
|
744
|
|
5
|
12
|
|
|
12
|
|
8576
|
use LWP::UserAgent; |
|
12
|
|
|
|
|
565344
|
|
|
12
|
|
|
|
|
484
|
|
6
|
12
|
|
|
12
|
|
113
|
use HTTP::Request; |
|
12
|
|
|
|
|
31
|
|
|
12
|
|
|
|
|
355
|
|
7
|
12
|
|
|
12
|
|
5903
|
use Compiled::Params::OO qw/cpo/; |
|
12
|
|
|
|
|
354559
|
|
|
12
|
|
|
|
|
174
|
|
8
|
|
|
|
|
|
|
|
9
|
12
|
|
|
12
|
|
1310
|
use Types::Standard qw/Str Any Object/; |
|
12
|
|
|
|
|
29
|
|
|
12
|
|
|
|
|
83
|
|
10
|
|
|
|
|
|
|
our $validate; |
11
|
|
|
|
|
|
|
BEGIN { |
12
|
12
|
|
|
12
|
|
10132
|
$validate = cpo( |
13
|
|
|
|
|
|
|
open_stream => [Object], |
14
|
|
|
|
|
|
|
open_url => [Object], |
15
|
|
|
|
|
|
|
open_file => [Object], |
16
|
|
|
|
|
|
|
open_string => [Object], |
17
|
|
|
|
|
|
|
write_file => [Object, Str] |
18
|
|
|
|
|
|
|
); |
19
|
|
|
|
|
|
|
} |
20
|
|
|
|
|
|
|
|
21
|
|
|
|
|
|
|
our $VERSION = '0.01'; |
22
|
|
|
|
|
|
|
|
23
|
|
|
|
|
|
|
has 'stream' => ( |
24
|
|
|
|
|
|
|
is => 'rw', |
25
|
|
|
|
|
|
|
isa => Str, |
26
|
|
|
|
|
|
|
lazy => 1, |
27
|
|
|
|
|
|
|
default => q{} |
28
|
|
|
|
|
|
|
); |
29
|
|
|
|
|
|
|
|
30
|
|
|
|
|
|
|
has 'stream_type' => ( |
31
|
|
|
|
|
|
|
is => 'ro', |
32
|
|
|
|
|
|
|
isa => Str, |
33
|
|
|
|
|
|
|
default => sub { |
34
|
|
|
|
|
|
|
my $self = shift; |
35
|
|
|
|
|
|
|
return 'url' if $self->stream =~ m{^http}xms; |
36
|
|
|
|
|
|
|
return 'string' if $self->stream =~ m{\<\?xml}xms; |
37
|
|
|
|
|
|
|
return 'file' if $self->stream =~ m{(\.xml|\.html|\.txt|\.json|\.csv)}xms; |
38
|
|
|
|
|
|
|
} |
39
|
|
|
|
|
|
|
); |
40
|
|
|
|
|
|
|
|
41
|
|
|
|
|
|
|
sub open_stream { |
42
|
12
|
|
|
12
|
0
|
712
|
my ($self) = $validate->open_stream->(@_); |
43
|
12
|
|
|
|
|
186
|
my $type = 'open_' . $self->stream_type; |
44
|
12
|
|
|
|
|
69
|
return $self->$type; |
45
|
|
|
|
|
|
|
} |
46
|
|
|
|
|
|
|
|
47
|
|
|
|
|
|
|
sub open_url { |
48
|
0
|
|
|
0
|
0
|
0
|
my ($self) = $validate->open_url->(@_); |
49
|
0
|
|
|
|
|
0
|
my $stream = $self->stream; |
50
|
0
|
|
|
|
|
0
|
my $ua = LWP::UserAgent->new(ssl_opts => { verify_hostname => 1 }); |
51
|
0
|
|
|
|
|
0
|
$ua->env_proxy; |
52
|
0
|
|
|
|
|
0
|
$ua->agent("Mozilla/8.0"); |
53
|
0
|
|
|
|
|
0
|
my $req = HTTP::Request->new( GET => $stream ); |
54
|
0
|
|
|
|
|
0
|
$req->header( 'Accept-Encoding', 'gzip' ); |
55
|
0
|
0
|
|
|
|
0
|
my $res = $ua->request($req) or croak "Failed to fetch URI: $stream"; |
56
|
0
|
0
|
|
|
|
0
|
if ( $res->code == 410 ) { |
57
|
0
|
|
|
|
|
0
|
croak "This feed has been permantly removed"; |
58
|
|
|
|
|
|
|
} |
59
|
0
|
|
|
|
|
0
|
my $content = $res->decoded_content(charset => 'utf8'); |
60
|
0
|
|
|
|
|
0
|
return \$content; |
61
|
|
|
|
|
|
|
} |
62
|
|
|
|
|
|
|
|
63
|
|
|
|
|
|
|
sub open_file { |
64
|
12
|
|
|
12
|
0
|
80
|
my ($self) = $validate->open_file->(@_); |
65
|
|
|
|
|
|
|
|
66
|
12
|
|
|
|
|
372
|
my $stream = $self->stream; |
67
|
|
|
|
|
|
|
|
68
|
12
|
50
|
|
|
|
739
|
open ( my $fh, '<', $stream ) or croak "could not open file: $stream"; |
69
|
|
|
|
|
|
|
|
70
|
12
|
|
|
|
|
44
|
my $content = do { local $/; <$fh> }; |
|
12
|
|
|
|
|
64
|
|
|
12
|
|
|
|
|
4100
|
|
71
|
12
|
|
|
|
|
171
|
close $fh; |
72
|
|
|
|
|
|
|
|
73
|
12
|
|
|
|
|
230
|
return \$content; |
74
|
|
|
|
|
|
|
} |
75
|
|
|
|
|
|
|
|
76
|
|
|
|
|
|
|
sub open_string { |
77
|
0
|
|
|
0
|
0
|
0
|
my ($self) = $validate->open_string(@_); |
78
|
0
|
|
|
|
|
0
|
return shift->stream; |
79
|
|
|
|
|
|
|
} |
80
|
|
|
|
|
|
|
|
81
|
|
|
|
|
|
|
sub write_file { |
82
|
5
|
|
|
5
|
0
|
59344
|
my ($self, $feed) = $validate->write_file->(@_); |
83
|
5
|
|
|
|
|
195
|
my $stream = $self->stream; |
84
|
5
|
50
|
|
|
|
822
|
open my $FILE, ">", $stream or croak "could not open file: $stream"; |
85
|
5
|
|
|
|
|
136
|
print $FILE $feed; |
86
|
5
|
|
|
|
|
3681
|
close $FILE; |
87
|
|
|
|
|
|
|
} |
88
|
|
|
|
|
|
|
|
89
|
|
|
|
|
|
|
1; # End of Feed::Data |