line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package HTML::ExtractText; |
2
|
|
|
|
|
|
|
|
3
|
4
|
|
|
4
|
|
131234
|
use strict; |
|
4
|
|
|
|
|
6
|
|
|
4
|
|
|
|
|
108
|
|
4
|
4
|
|
|
4
|
|
13
|
use warnings; |
|
4
|
|
|
|
|
5
|
|
|
4
|
|
|
|
|
118
|
|
5
|
|
|
|
|
|
|
|
6
|
|
|
|
|
|
|
our $VERSION = '1.002003'; # VERSION |
7
|
|
|
|
|
|
|
|
8
|
4
|
|
|
4
|
|
1157
|
use Try::Tiny; |
|
4
|
|
|
|
|
2653
|
|
|
4
|
|
|
|
|
176
|
|
9
|
4
|
|
|
4
|
|
14
|
use Scalar::Util qw/blessed/; |
|
4
|
|
|
|
|
6
|
|
|
4
|
|
|
|
|
125
|
|
10
|
4
|
|
|
4
|
|
13
|
use Carp qw/croak/; |
|
4
|
|
|
|
|
5
|
|
|
4
|
|
|
|
|
110
|
|
11
|
4
|
|
|
4
|
|
1155
|
use Devel::TakeHashArgs; |
|
4
|
|
|
|
|
834
|
|
|
4
|
|
|
|
|
150
|
|
12
|
4
|
|
|
4
|
|
1195
|
use Mojo::DOM; |
|
4
|
|
|
|
|
154361
|
|
|
4
|
|
|
|
|
194
|
|
13
|
0
|
|
|
0
|
|
0
|
use overload q|""| => sub { shift->error }, |
14
|
4
|
|
|
4
|
|
32
|
q|%{}| => sub { shift->last_results }; |
|
4
|
|
|
7
|
|
4
|
|
|
4
|
|
|
|
|
40
|
|
|
7
|
|
|
|
|
29885
|
|
15
|
|
|
|
|
|
|
|
16
|
|
|
|
|
|
|
sub new { |
17
|
13
|
|
|
13
|
1
|
20200
|
my $self = bless [], shift; |
18
|
13
|
50
|
|
|
|
88
|
get_args_as_hash( \@_, \ my %args, |
19
|
|
|
|
|
|
|
{ # these are optional with defaults |
20
|
|
|
|
|
|
|
separator => "\n", |
21
|
|
|
|
|
|
|
ignore_not_found => 1, |
22
|
|
|
|
|
|
|
}, |
23
|
|
|
|
|
|
|
[ ], # these are mandatory (none, ATM) |
24
|
|
|
|
|
|
|
[ qw/separator ignore_not_found/ ], # only these args are valid ones |
25
|
|
|
|
|
|
|
) or croak $@; |
26
|
|
|
|
|
|
|
|
27
|
13
|
|
|
|
|
343
|
$self->$_( $args{$_} ) for keys %args; |
28
|
|
|
|
|
|
|
|
29
|
13
|
|
|
|
|
37
|
return $self; |
30
|
|
|
|
|
|
|
} |
31
|
|
|
|
|
|
|
|
32
|
|
|
|
|
|
|
sub extract { |
33
|
11
|
|
|
11
|
1
|
2406
|
my ( $self, $what, $html, $obj ) = @_; |
34
|
|
|
|
|
|
|
|
35
|
11
|
|
|
|
|
27
|
$self->error( undef ); |
36
|
11
|
|
|
|
|
19
|
$self->last_results( undef ); |
37
|
|
|
|
|
|
|
|
38
|
11
|
50
|
|
|
|
27
|
ref $what eq 'HASH' |
39
|
|
|
|
|
|
|
or return $self->_set_error('First argument to ' |
40
|
|
|
|
|
|
|
. 'extract_text() must be a hashref'); |
41
|
|
|
|
|
|
|
|
42
|
11
|
50
|
|
|
|
23
|
defined $html |
43
|
|
|
|
|
|
|
or return $self->_set_error('Second argument to extract_text() is ' |
44
|
|
|
|
|
|
|
. 'an undef, expected HTML'); |
45
|
|
|
|
|
|
|
|
46
|
11
|
100
|
|
|
|
25
|
if ( defined $obj ) { |
47
|
3
|
100
|
|
|
|
17
|
blessed $obj |
48
|
|
|
|
|
|
|
or return $self->_set_error('Third argument must be an object'); |
49
|
|
|
|
|
|
|
|
50
|
2
|
|
|
|
|
5
|
for ( keys %$what ) { |
51
|
2
|
100
|
|
|
|
19
|
$obj->can($_) |
52
|
|
|
|
|
|
|
or return $self->_set_error( |
53
|
|
|
|
|
|
|
'The object your provided does not implement the ->' |
54
|
|
|
|
|
|
|
. $_ . '() method that you requested in the first' |
55
|
|
|
|
|
|
|
. ' argument', |
56
|
|
|
|
|
|
|
); |
57
|
|
|
|
|
|
|
} |
58
|
|
|
|
|
|
|
} |
59
|
|
|
|
|
|
|
|
60
|
|
|
|
|
|
|
|
61
|
9
|
|
|
|
|
53
|
my $dom = Mojo::DOM->new( $html ); |
62
|
|
|
|
|
|
|
|
63
|
9
|
|
|
|
|
3081
|
my $did_have_error = 0; |
64
|
9
|
|
|
|
|
43
|
for my $selector ( sort keys %$what ) { |
65
|
23
|
|
|
|
|
21
|
my $result; |
66
|
|
|
|
|
|
|
|
67
|
|
|
|
|
|
|
try { |
68
|
23
|
|
|
23
|
|
528
|
my @results = $self->_extract( $dom, $selector, $what, ); |
69
|
23
|
50
|
66
|
|
|
1391
|
die "NOT FOUND\n" |
70
|
|
|
|
|
|
|
if not @results and not $self->ignore_not_found; |
71
|
|
|
|
|
|
|
|
72
|
22
|
100
|
|
|
|
34
|
if ( defined (my $sep = $self->separator) ) { |
73
|
10
|
|
|
|
|
30
|
$result = join $sep, @results; |
74
|
|
|
|
|
|
|
} |
75
|
|
|
|
|
|
|
else { |
76
|
12
|
|
|
|
|
27
|
$result = [ @results ]; |
77
|
|
|
|
|
|
|
} |
78
|
|
|
|
|
|
|
|
79
|
|
|
|
|
|
|
} catch { |
80
|
1
|
|
|
1
|
|
9
|
chomp($_); |
81
|
1
|
|
|
|
|
4
|
$self->error("ERROR: [$selector]: $_"); |
82
|
1
|
|
|
|
|
1
|
$result = "ERROR: $_"; |
83
|
1
|
|
|
|
|
3
|
$did_have_error = 1; |
84
|
23
|
|
|
|
|
131
|
}; |
85
|
|
|
|
|
|
|
|
86
|
23
|
|
|
|
|
281
|
$what->{ $selector } = $result; |
87
|
|
|
|
|
|
|
} |
88
|
|
|
|
|
|
|
|
89
|
9
|
100
|
|
|
|
23
|
if ( defined $obj ) { |
90
|
1
|
|
|
|
|
2
|
for ( keys %$what ) { |
91
|
1
|
|
|
|
|
4
|
$obj->$_( $what->{ $_ } ); |
92
|
|
|
|
|
|
|
} |
93
|
|
|
|
|
|
|
} |
94
|
|
|
|
|
|
|
|
95
|
9
|
|
|
|
|
23
|
$self->last_results( $what ); |
96
|
|
|
|
|
|
|
|
97
|
9
|
100
|
|
|
|
18
|
if ( $did_have_error ) { |
98
|
1
|
|
|
|
|
6
|
return; |
99
|
|
|
|
|
|
|
} |
100
|
|
|
|
|
|
|
else { |
101
|
8
|
|
|
|
|
47
|
return $what; |
102
|
|
|
|
|
|
|
} |
103
|
|
|
|
|
|
|
} |
104
|
|
|
|
|
|
|
|
105
|
|
|
|
|
|
|
sub separator { |
106
|
39
|
|
|
39
|
1
|
44
|
my $self = shift; |
107
|
39
|
100
|
|
|
|
71
|
if ( @_ ) { $self->[0]->{SEPARATOR} = shift; } |
|
15
|
|
|
|
|
33
|
|
108
|
39
|
|
|
|
|
107
|
return $self->[0]->{SEPARATOR}; |
109
|
|
|
|
|
|
|
} |
110
|
|
|
|
|
|
|
|
111
|
|
|
|
|
|
|
sub ignore_not_found { |
112
|
16
|
|
|
16
|
1
|
17
|
my $self = shift; |
113
|
16
|
100
|
|
|
|
34
|
if ( @_ ) { $self->[0]->{IGNORE_NOT_FOUND} = shift; } |
|
14
|
|
|
|
|
153
|
|
114
|
16
|
|
|
|
|
52
|
return $self->[0]->{IGNORE_NOT_FOUND}; |
115
|
|
|
|
|
|
|
} |
116
|
|
|
|
|
|
|
|
117
|
|
|
|
|
|
|
sub last_results { |
118
|
33
|
|
|
33
|
1
|
37
|
my $self = shift; |
119
|
33
|
100
|
|
|
|
65
|
if ( @_ ) { $self->[0]->{LAST_RESULTS} = shift; } |
|
20
|
|
|
|
|
29
|
|
120
|
33
|
|
|
|
|
89
|
return $self->[0]->{LAST_RESULTS}; |
121
|
|
|
|
|
|
|
} |
122
|
|
|
|
|
|
|
|
123
|
|
|
|
|
|
|
sub error { |
124
|
17
|
|
|
17
|
1
|
301
|
my $self = shift; |
125
|
17
|
100
|
|
|
|
37
|
if ( @_ ) { $self->[0]->{ERROR} = shift; } |
|
14
|
|
|
|
|
26
|
|
126
|
17
|
|
|
|
|
31
|
return $self->[0]->{ERROR}; |
127
|
|
|
|
|
|
|
} |
128
|
|
|
|
|
|
|
|
129
|
|
|
|
|
|
|
sub _set_error { |
130
|
2
|
|
|
2
|
|
2
|
my ( $self, $error ) = @_; |
131
|
2
|
|
|
|
|
3
|
$self->error( $error ); |
132
|
2
|
|
|
|
|
7
|
return; |
133
|
|
|
|
|
|
|
} |
134
|
|
|
|
|
|
|
|
135
|
|
|
|
|
|
|
sub _process { |
136
|
27
|
|
|
27
|
|
50
|
my $tag = $_->tag; |
137
|
27
|
100
|
|
|
|
349
|
return $_->all_text unless $tag =~ /input|img/; |
138
|
|
|
|
|
|
|
|
139
|
7
|
100
|
100
|
|
|
32
|
return $_->attr('alt')//'' |
|
|
|
100
|
|
|
|
|
|
|
|
66
|
|
|
|
|
|
|
|
66
|
|
|
|
|
140
|
|
|
|
|
|
|
if $tag eq 'img' or |
141
|
|
|
|
|
|
|
($tag eq 'input' and ($_->attr('type')//'') eq 'image'); |
142
|
|
|
|
|
|
|
|
143
|
4
|
|
100
|
|
|
55
|
return $_->attr('value')//''; |
144
|
|
|
|
|
|
|
} |
145
|
|
|
|
|
|
|
|
146
|
|
|
|
|
|
|
sub _extract { |
147
|
22
|
|
|
22
|
|
37
|
my ( $self, $dom, $selector, $what ) = @_; |
148
|
|
|
|
|
|
|
return $dom->find( $what->{ $selector } ) |
149
|
22
|
|
|
28
|
|
57
|
->map( sub { $self->_process( @_ ) } )->each; |
|
28
|
|
|
|
|
5940
|
|
150
|
|
|
|
|
|
|
} |
151
|
|
|
|
|
|
|
|
152
|
|
|
|
|
|
|
|
153
|
|
|
|
|
|
|
|
154
|
|
|
|
|
|
|
q| |
155
|
|
|
|
|
|
|
Programming is 10% science, 20% ingenuity, |
156
|
|
|
|
|
|
|
and 70% getting the ingenuity to work with the science. |
157
|
|
|
|
|
|
|
|; |
158
|
|
|
|
|
|
|
|
159
|
|
|
|
|
|
|
__END__ |