blib/lib/HTML/ExtractText.pm

filter: Statementstmt 97.4% 116 / 119 Branchbran 80.3% 45 / 56 Conditioncond 68.5% 24 / 35 MC/DCmcdc 44.4% 8 / 18 Subroutinesub 96.1% 25 / 26 Podpod 100% 6 / 6 totaltotal 86.1% 224 / 260 CC 46 SCAR 20.9
CC
46
Cov
88%
_text
19.6
extract
11.0
_process
8.0
CRAP 8.1
↑ Summary
1 package HTML::ExtractText;
2  
3 5 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 5 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5  
6 5 our $VERSION = '1.002005'; # VERSION
7  
8 5 ▶use Try::Tiny;
Subroutine BEGIN: called CC=1 SCAR=0.0
9 5 ▶use Scalar::Util qw/blessed/;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 5 ▶use Carp qw/croak/;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 5 ▶use Devel::TakeHashArgs;
Subroutine BEGIN: called CC=1 SCAR=0.0
12 5 ▶use Mojo::DOM;
Subroutine BEGIN: called CC=1 SCAR=0.0
13 0 ▶use overload q|""| => sub { shift->error },
Subroutine BEGIN: called CC=1 SCAR=0.1
Subroutine __ANON__: not called CC=1 SCAR=6.9
14 5 ▶ q|%{}| => sub { shift->last_results };
Subroutine __ANON__: called CC=1 SCAR=0.0
15  
16 13 ▶sub new {
Subroutine new: called CC=3 SCAR=11.1
Pod documented
17 13 my $self = bless [], shift;
18 13 ▶ get_args_as_hash( \@_, \ my %args,
Branch: 1 branch
BranchTrueFalseTotal
get_args_as_hash(\@_, \my %args, {"separator", "\n", "ignore_not_found", 1}, scalar [], ["separator", "ignore_not_found"]) or croak($@) 13 0 13
19 { # these are optional with defaults
20 separator => "\n",
21 ignore_not_found => 1,
22 },
23 [ ], # these are mandatory (none, ATM)
24 [ qw/separator ignore_not_found/ ], # only these args are valid ones
25 ) or croak $@;
26  
27 13 $self->$_( $args{$_} ) for keys %args;
28  
29 13 return $self;
30 }
31  
32 11 ▶sub extract {
Subroutine extract: called CC=11 SCAR=24.0
Pod documented
33 11 my ( $self, $what, $html, $obj ) = @_;
34  
35 11 $self->error( undef );
36 11 $self->last_results( undef );
37  
38 11 ▶ ref $what eq 'HASH'
Branch: 1 branch
BranchTrueFalseTotal
ref $what eq "HASH" or return $self->_set_error("First argument to extract_text() must be a hashref") 11 0 11
39 or return $self->_set_error('First argument to '
40 . 'extract_text() must be a hashref');
41  
42 11 ▶ defined $html
Branch: 1 branch
BranchTrueFalseTotal
defined $html or return $self->_set_error("Second argument to extract_text() is an undef, expected HTML") 11 0 11
43 or return $self->_set_error('Second argument to extract_text() is '
44 . 'an undef, expected HTML');
45  
46 11 ▶ if ( defined $obj ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $obj) 3 8 11
47 3 ▶ blessed $obj
Branch: 1 branch
BranchTrueFalseTotal
builtin::blessed($obj) or return $self->_set_error("Third argument must be an object") 2 1 3
48 or return $self->_set_error('Third argument must be an object');
49  
50 2 for ( keys %$what ) {
51 2 ▶ $obj->can($_)
Branch: 1 branch
BranchTrueFalseTotal
$obj->can($_) or return $self->_set_error('The object your provided does not implement the ->' . $_ . '() method that you requested in the first' . " argument") 1 1 2
52 or return $self->_set_error(
53 'The object your provided does not implement the ->'
54 . $_ . '() method that you requested in the first'
55 . ' argument',
56 );
57 }
58 }
59  
60  
61 9 my $dom = Mojo::DOM->new( $html );
62  
63 9 my $did_have_error = 0;
64 9 for my $selector ( sort keys %$what ) {
65 23 my $result;
66  
67 23 ▶ try {
Subroutine __ANON__: called CC=4 SCAR=13.9
68 23 my @results = $self->_extract( $dom, $selector, $what, );
69 23 ▶ die "NOT FOUND\n"
Branch: 1 branch
BranchTrueFalseTotal
unless @results or $self->ignore_not_found 1 22 23
Condition66%2/3
@results or $self->ignore_not_found
l!l&&r!l&&!r
2201
MC/DC50%1/2
@results or $self->ignore_not_found
@results $self->ignore_not_found
Truth table66%2/3
A or B
A= @resultsB= $self->ignore_not_found
ABresult
1X1
011
000
70 if not @results and not $self->ignore_not_found;
71  
72 22 ▶ if ( defined (my $sep = $self->separator) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $sep = $self->separator)) { } 10 12 22
73 10 $result = join $sep, @results;
74 }
75 else {
76 12 $result = [ @results ];
77 }
78  
79 1 ▶ } catch {
Subroutine __ANON__: called CC=1 SCAR=0.0
80 1 chomp($_);
81 1 $self->error("ERROR: [$selector]: $_");
82 1 $result = "ERROR: $_";
83 1 $did_have_error = 1;
84 23 };
85  
86 23 $what->{ $selector } = $result;
87 }
88  
89 9 ▶ if ( defined $obj ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $obj) 1 8 9
90 1 for ( keys %$what ) {
91 1 $obj->$_( $what->{ $_ } );
92 }
93 }
94  
95 9 $self->last_results( $what );
96  
97 9 ▶ if ( $did_have_error ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($did_have_error) { } 1 8 9
98 1 return;
99 }
100 else {
101 8 return $what;
102 }
103 }
104  
105 39 ▶sub separator {
Subroutine separator: called CC=2 SCAR=6.9
Pod documented
106 39 my $self = shift;
107 39 ▶ if ( @_ ) { $self->[0]->{SEPARATOR} = shift; }
Branch: 1 branch
BranchTrueFalseTotal
if (@_) 15 24 39
108 39 return $self->[0]->{SEPARATOR};
109 }
110  
111 16 ▶sub ignore_not_found {
Subroutine ignore_not_found: called CC=2 SCAR=6.9
Pod documented
112 16 my $self = shift;
113 16 ▶ if ( @_ ) { $self->[0]->{IGNORE_NOT_FOUND} = shift; }
Branch: 1 branch
BranchTrueFalseTotal
if (@_) 14 2 16
114 16 return $self->[0]->{IGNORE_NOT_FOUND};
115 }
116  
117 33 ▶sub last_results {
Subroutine last_results: called CC=2 SCAR=6.9
Pod documented
118 33 my $self = shift;
119 33 ▶ if ( @_ ) { $self->[0]->{LAST_RESULTS} = shift; }
Branch: 1 branch
BranchTrueFalseTotal
if (@_) 20 13 33
120 33 return $self->[0]->{LAST_RESULTS};
121 }
122  
123 17 ▶sub error {
Subroutine error: called CC=2 SCAR=6.9
Pod documented
124 17 my $self = shift;
125 17 ▶ if ( @_ ) { $self->[0]->{ERROR} = shift; }
Branch: 1 branch
BranchTrueFalseTotal
if (@_) 14 3 17
126 17 return $self->[0]->{ERROR};
127 }
128  
129 2 ▶sub _set_error {
Subroutine _set_error: called CC=1 SCAR=0.0
130 2 my ( $self, $error ) = @_;
131 2 $self->error( $error );
132 2 return;
133 }
134  
135 27 ▶sub _process {
Subroutine _process: called CC=8 SCAR=20.8
136 27 my $tag = $_->tag;
137 27 ▶ return _all_text($_) unless $tag =~ /input|img/;
Branch: 1 branch
BranchTrueFalseTotal
unless $tag =~ /input|img/ 20 7 27
138  
139 7 ▶ return $_->attr('alt')//''
Branch: 1 branch
BranchTrueFalseTotal
if $tag eq "img" or $tag eq "input" and ($_->attr("type") // "") eq "image" 3 4 7
Condition90%9/10
$tag eq "img" or $tag eq "input" and ($_->attr("type") // "") eq "image"
l!l&&r!l&&!r
214
$tag eq "input" and ($_->attr("type") // "") eq "image"
!ll&&!rl&&r
041
$_->attr("type") // ""
l!l
41
$_->attr("alt") // ""
l!l
21
MC/DC66%2/3
$tag eq "img" or $tag eq "input" and ($_->attr("type") // "") eq "image"
$tag eq "img" $tag eq "input" $_->attr("type")
MC/DC100%1/1
$_->attr("alt") // ""
$_->attr("alt")
Truth table75%3/4
A or B and C
A= $tag eq "img"B= $tag eq "input"C= $_->attr("type")
ABCresult
1XX1
0111
00X0
0100
Truth table100%2/2
A
A= $_->attr("alt")
Aresult
11
00
140 if $tag eq 'img' or
141 ($tag eq 'input' and ($_->attr('type')//'') eq 'image');
142  
143 4 ▶ return $_->attr('value')//'';
Condition100%2/2
$_->attr("value") // ""
l!l
31
MC/DC100%1/1
$_->attr("value") // ""
$_->attr("value")
Truth table100%2/2
A
A= $_->attr("value")
Aresult
11
00
144 }
145  
146 22 ▶sub _extract {
Subroutine _extract: called CC=1 SCAR=0.0
147 22 my ( $self, $dom, $selector, $what ) = @_;
148 return $dom->find( $what->{ $selector } )
149 22 ▶ ->map( sub { $self->_process( @_ ) } )->each;
Subroutine __ANON__: called CC=1 SCAR=0.0
150 }
151  
152 # The _all_text & _text functions copied from Mojo::DOM 6.66, adapted to
153 # avoid Mojo::DOM's private _nodes()/_ancestors() internals, whose return
154 # conventions changed in later Mojo::DOM releases (which silently broke text
155 # extraction). _child_nodes() reads the stable ->tree node structure directly.
156 22 ▶sub _child_nodes {
Subroutine _child_nodes: called CC=3 SCAR=11.7
157 22 my $tree = shift;
158 22 ▶ return unless $tree;
Branch: 1 branch
BranchTrueFalseTotal
unless $tree 0 22 22
159 22 ▶ return @$tree[ ( $tree->[0] eq 'tag' ? 4 : 1 ) .. $#$tree ];
Branch: 1 branch
BranchTrueFalseTotal
$tree->[0] eq 'tag' ? : 22 0 22
160 }
161  
162 20 ▶sub _all_text {
Subroutine _all_text: called CC=3 SCAR=12.0
163 20 my ( $dom ) = @_;
164 20 my $trim = 1;
165  
166 # Detect "pre" tag on the element itself or any of its ancestors
167 20 ▶ $trim = 0 if $dom->tag eq 'pre' or $dom->ancestors('pre')->size;
Branch: 1 branch
BranchTrueFalseTotal
if $dom->tag eq "pre" or $dom->ancestors("pre")->size 0 20 20
Condition33%1/3
$dom->tag eq "pre" or $dom->ancestors("pre")->size
l!l&&r!l&&!r
0020
MC/DC0%0/2
$dom->tag eq "pre" or $dom->ancestors("pre")->size
$dom->tag eq "pre" $dom->ancestors("pre")->size
Truth table33%1/3
A or B
A= $dom->tag eq "pre"B= $dom->ancestors("pre")->size
ABresult
1X1
011
000
168  
169 20 return _text( [ _child_nodes( $dom->tree ) ], $trim );
170 }
171  
172 22 ▶sub _text {
Subroutine _text: called CC=16 SCAR=29.8
173 22 my ( $nodes, $trim ) = @_;
174  
175 # Merge successive text nodes.
176 22 my $i = 0;
177 22 ▶ while ( my $next = $nodes->[$i + 1] ) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $next = $$nodes[$i + 1]) 4 22 26
178 4 ▶ ++$i and next unless $nodes->[$i][0] eq 'text' && $next->[0] eq 'text';
Branch: 1 branch
BranchTrueFalseTotal
unless $nodes->[$i][0] eq "text" and $next->[0] eq "text" 4 0 4
Condition50%4/8
$nodes->[$i][0] eq 'text' && $next->[0] eq 'text' || ++$i && (next)
l!l&&r!l&&!r
040
$nodes->[$i][0] eq "text" and $next->[0] eq "text"
!ll&&!rl&&r
220
++$i and next
!ll
04
MC/DC0%0/3
$nodes->[$i][0] eq 'text' && $next->[0] eq 'text' || ++$i && (next)
$nodes->[$i][0] eq "text" $next->[0] eq "text" ++$i
Truth table0%0/5
A and B || C
A= $nodes->[$i][0] eq "text"B= $next->[0] eq "text"C= ++$i
ABCresult
11X1
0X11
1011
0X00
1000
179 0 splice @$nodes, $i, 2, ['text', $nodes->[$i][1] . $next->[1]];
180 }
181  
182 22 my $text = '';
183 22 for my $node ( @$nodes ) {
184 26 my $type = $node->[0];
185 26 my $chunk = '';
186  
187 # Text.
188 26 ▶ if ( $type eq 'text' ) {
Branch: 3 branches
BranchTrueFalseTotal
if ($type eq 'text') { } 24 2 26
elsif ($type eq 'cdata' or $type eq 'raw') { } 0 2 2
elsif ($type eq 'tag') { } 2 0 2
189 24 $chunk = $node->[1];
190 24 ▶ if ( $trim ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($trim) 24 0 24
191 24 $chunk =~ s/^\s+//;
192 24 $chunk =~ s/\s+$//;
193 24 $chunk =~ s/\s+/ /g;
194 }
195 }
196 # CDATA or raw text.
197 ▶ elsif ( $type eq 'cdata' || $type eq 'raw' ) {
Condition33%1/3
$type eq "cdata" or $type eq "raw"
l!l&&r!l&&!r
002
MC/DC0%0/2
$type eq "cdata" or $type eq "raw"
$type eq "cdata" $type eq "raw"
Truth table33%1/3
A or B
A= $type eq "cdata"B= $type eq "raw"
ABresult
1X1
011
000
198 0 $chunk = $node->[1];
199 }
200 # Nested tag.
201 elsif ( $type eq 'tag' ) {
202 5 ▶ no warnings 'recursion';
Subroutine BEGIN: called CC=1 SCAR=0.0
203 2 ▶ $chunk = _text( [ _child_nodes($node) ], 1, $node->[1] eq 'pre' ? 0 : $trim );
Branch: 1 branch
BranchTrueFalseTotal
$node->[1] eq 'pre' ? : 0 2 2
204 }
205  
206 # Add leading whitespace if punctuation allows it.
207 26 ▶ $chunk = " $chunk" if $text =~ /\S\z/ && $chunk =~ /^[^.!?,;:\s]+/;
Branch: 1 branch
BranchTrueFalseTotal
if $text =~ /\S\z/ and $chunk =~ /^[^.!?,;:\s]+/ 1 25 26
Condition100%3/3
$text =~ /\S\z/ and $chunk =~ /^[^.!?,;:\s]+/
!ll&&!rl&&r
2321
MC/DC100%2/2
$text =~ /\S\z/ and $chunk =~ /^[^.!?,;:\s]+/
$text =~ /\S\z/ $chunk =~ /^[^.!?,;:\s]+/
Truth table100%3/3
A and B
A= $text =~ /\S\z/B= $chunk =~ /^[^.!?,;:\s]+/
ABresult
0X0
100
111
208  
209 # Trim whitespace blocks.
210 26 ▶ $text .= $chunk if $chunk =~ /\S+/ || !$trim;
Branch: 1 branch
BranchTrueFalseTotal
if $chunk =~ /\S+/ or not $trim 23 3 26
Condition66%2/3
$chunk =~ /\S+/ or not $trim
l!l&&r!l&&!r
2303
MC/DC50%1/2
$chunk =~ /\S+/ or not $trim
$chunk =~ /\S+/ not $trim
Truth table66%2/3
A or not(B)
A= $chunk =~ /\S+/B= not $trim
ABresult
1X1
011
000
211 }
212  
213 22 return $text;
214 }
215  
216  
217 5 q|
218 Programming is 10% science, 20% ingenuity,
219 and 70% getting the ingenuity to work with the science.
220 |;
221  
222 __END__
↑ Summary