blib/lib/WWW/RobotRules/Parser/MultiValue.pm

filter: Statementstmt 100% 107 / 107 Branchbran 81.4% 44 / 54 Conditioncond 53.8% 7 / 13 MC/DCmcdc 12.5% 1 / 8 Subroutinesub 100% 22 / 22 Podpod 100% 5 / 5 totaltotal 89.0% 186 / 209 CC 37 SCAR 17.4
CC
37
Cov
91%
parse
13.1
_uri
7.0
_translate_path_pattern
6.0
CRAP 5.7
↑ Summary
1 package WWW::RobotRules::Parser::MultiValue;
2 4 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 4 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 4 ▶use 5.014;
Subroutine BEGIN: called CC=1 SCAR=0.0
5  
6 4 our $VERSION = '0.02';
7  
8 # core
9 4 ▶use Scalar::Util qw(blessed);
Subroutine BEGIN: called CC=1 SCAR=0.0
10  
11 # cpan
12 4 ▶use URI;
Subroutine BEGIN: called CC=1 SCAR=0.0
13 4 ▶use Text::Glob qw(match_glob);
Subroutine BEGIN: called CC=1 SCAR=0.0
14 4 ▶use Hash::MultiValue;
Subroutine BEGIN: called CC=1 SCAR=0.0
15 4 ▶use Class::Accessor::Lite (
Subroutine BEGIN: called CC=1 SCAR=0.0
16 4 new => 1,
17 ro => [qw(agent ignore_default)],
18 4 );
19  
20 4 ▶use constant {
Subroutine BEGIN: called CC=1 SCAR=0.0
21 4 WILDCARD => 'wc',
22 ME => 'me',
23 TRANSLATOR => {
24 allow => '_translate_path_pattern',
25 disallow => '_translate_path_pattern',
26 'crawl-delay' => '_translate_delay',
27 'request-rate' => '_translate_rate',
28 },
29 4 };
30  
31 115 ▶sub _uri {
Subroutine _uri: called CC=5 SCAR=19.4
32 115 my ($uri) = @_;
33 115 ▶ $uri = URI->new($uri.q()) unless blessed($uri) && $uri->isa('URI');
Branch: 1 branch
BranchTrueFalseTotal
unless builtin::blessed($uri) and $uri->isa("URI") 115 0 115
Condition33%1/3
builtin::blessed($uri) and $uri->isa("URI")
!ll&&!rl&&r
11500
MC/DC0%0/2
builtin::blessed($uri) and $uri->isa("URI")
builtin::blessed($uri) $uri->isa("URI")
Truth table33%1/3
A and B
A= builtin::blessed($uri)B= $uri->isa("URI")
ABresult
0X0
100
111
34 115 ▶ return unless $uri->can('host') && $uri->can('port');
Branch: 1 branch
BranchTrueFalseTotal
unless $uri->can("host") and $uri->can("port") 0 115 115
Condition33%1/3
$uri->can("host") and $uri->can("port")
!ll&&!rl&&r
00115
MC/DC0%0/2
$uri->can("host") and $uri->can("port")
$uri->can("host") $uri->can("port")
Truth table33%1/3
A and B
A= $uri->can("host")B= $uri->can("port")
ABresult
0X0
100
111
35 115 return $uri;
36 }
37  
38 115 ▶sub _domain {
Subroutine _domain: called CC=1 SCAR=0.0
39 115 my ($uri) = @_;
40 115 return sprintf '%s:%d', $uri->host, $uri->port;
41 }
42  
43 199 ▶sub _rules {
Subroutine _rules: called CC=2 SCAR=7.1
44 199 my ($self, $domain) = @_;
45 199 ▶ return $self->{rules}->{$domain} //= Hash::MultiValue->new;
Condition66%2/3
$self->{"rules"}{$domain} //= "Hash::MultiValue"->new
l!l&&r!l&&!r
157420
MC/DC0%0/2
$self->{"rules"}{$domain} //= "Hash::MultiValue"->new
$self->{"rules"}{$domain} "Hash::MultiValue"->new
Truth table66%2/3
A //= B
A= $self->{"rules"}{$domain}B= "Hash::MultiValue"->new
ABresult
1X1
011
000
46 }
47  
48 22 ▶sub rules_for {
Subroutine rules_for: called CC=2 SCAR=7.0
Pod documented
49 22 my ($self, $uri) = @_;
50 22 ▶ $uri = _uri($uri)
Branch: 1 branch
BranchTrueFalseTotal
$uri = _uri($uri) or return "Hash::MultiValue"->new 22 0 22
51 or return Hash::MultiValue->new;
52 22 my $path_query = $uri->path_query;
53 22 my $domain = _domain($uri);
54 22 return $self->_rules($domain);
55 }
56  
57 28 ▶sub parse {
Subroutine parse: called CC=13 SCAR=25.8
Pod documented
58 28 my ($self, $robots_txt_uri, $txt) = @_;
59 28 ▶ $robots_txt_uri = _uri($robots_txt_uri)
Branch: 1 branch
BranchTrueFalseTotal
$robots_txt_uri = _uri($robots_txt_uri) or return 28 0 28
60 or return;
61 28 my $domain = _domain($robots_txt_uri);
62  
63 28 my $ua = WILDCARD;
64 28 my $anon_rules = Hash::MultiValue->new;
65  
66 28 ▶ $txt = ($txt//'') =~ s|\r\n|\n|gr;
Condition50%1/2
$txt // ""
l!l
280
MC/DC0%0/1
$txt // ""
$txt
Truth table50%1/2
A
A= $txt
Aresult
11
00
67 28 for my $line (split /[\r\n]/, $txt) {
68 202 $line =~ s/(?:^\s*|\s*$|\s*[#].*$)//g;
69 202 ▶ next if $line =~ /^\s*$/; # skip empty line
Branch: 1 branch
BranchTrueFalseTotal
if $line =~ /^\s*$/u 43 159 202
70  
71 159 ▶ if ($line =~ /^User-Agent\s*:\s*(.*)$/i) {
Branch: 1 branch
BranchTrueFalseTotal
if ($line =~ /^User-Agent\s*:\s*(.*)$/iu) { } 41 118 159
72 41 $ua = $self->match_ua($1);
73 } else {
74 118 ▶ next unless $ua; # skip directives for other UA
Branch: 1 branch
BranchTrueFalseTotal
unless $ua 32 86 118
75  
76 86 ▶ if ($line =~ /^([^:]+?)\s*:\s*(.*)$/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($line =~ /^([^:]+?)\s*:\s*(.*)$/u) 86 0 86
77 86 my ($rule, $value) = (lc $1, $2);
78 86 ▶ if (my $method = TRANSLATOR->{$rule}) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $method = {"allow" => "_translate_path_pattern","crawl-delay" => "_translate_delay","disallow" => "_translate_path_pattern","request-rate" => "_translate_rate"}->{$rule}) 86 0 86
79 86 ($rule, $value) = $self->$method(
80 $rule, $value, $robots_txt_uri,
81 );
82 }
83 86 ▶ next unless $rule;
Branch: 1 branch
BranchTrueFalseTotal
unless $rule 11 75 86
84  
85 75 ▶ if ($ua eq ME) {
Branch: 1 branch
BranchTrueFalseTotal
if ($ua eq 'me') { } 36 39 75
86 36 $self->_rules($domain)->add($rule => $value);
87 } else {
88 39 $anon_rules->add($rule => $value);
89 }
90 }
91 }
92 }
93  
94 28 ▶ unless ($self->ignore_default) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($self->ignore_default) 20 8 28
95 # Add rules for default UA as a lower precedence
96 $self->_rules($domain)->add($_ => $anon_rules->get_all($_))
97 20 for $anon_rules->keys;
98 }
99  
100 28 return $self;
101 }
102  
103 53 ▶sub match_ua {
Subroutine match_ua: called CC=3 SCAR=11.0
Pod documented
104 53 my ($self, $pattern) = @_;
105 53 ▶ return WILDCARD if $pattern eq '*';
Branch: 1 branch
BranchTrueFalseTotal
if $pattern eq "*" 16 37 53
106 37 ▶ return ME if index(lc $self->_short_agent, lc($pattern)) >= 0;
Branch: 1 branch
BranchTrueFalseTotal
if index(lc $self->_short_agent, lc $pattern) != -1 15 22 37
107 22 return undef;
108 }
109  
110 205 ▶sub _match_path ($$) {
Subroutine _match_path: called CC=1 SCAR=0.0
111 205 my ($str, $pattern) = @_;
112 205 local $Text::Glob::strict_leading_dot = 0;
113 205 local $Text::Glob::strict_wildcard_slash = 0;
114 205 return match_glob($pattern.'*', $str);
115 }
116  
117 65 ▶sub allows {
Subroutine allows: called CC=6 SCAR=17.9
Pod documented
118 65 my ($self, $uri) = @_;
119 65 ▶ $uri = _uri($uri)
Branch: 1 branch
BranchTrueFalseTotal
$uri = _uri($uri) or return 65 0 65
120 or return;
121 65 my $path_query = $uri->path_query;
122 65 my $domain = _domain($uri);
123 65 for my $pattern ($self->_rules($domain)->get_all('allow')) {
124 139 ▶ return 1 if _match_path $path_query, $pattern;
Branch: 1 branch
BranchTrueFalseTotal
if &_match_path($path_query, $pattern) 17 122 139
125 }
126 48 for my $pattern ($self->_rules($domain)->get_all('disallow')) {
127 66 ▶ return 0 if _match_path $path_query, $pattern;
Branch: 1 branch
BranchTrueFalseTotal
if &_match_path($path_query, $pattern) 23 43 66
128 }
129 25 return 1;
130 }
131  
132 22 ▶sub delay_for {
Subroutine delay_for: called CC=3 SCAR=11.0
Pod documented
133 22 my ($self, $uri, $base) = @_;
134 22 my ($delay) = $self->rules_for($uri)->get_all('crawl-delay');
135 22 ▶ $delay *= ( $base || 1 ) if defined $delay;
Branch: 1 branch
BranchTrueFalseTotal
if defined $delay 16 6 22
Condition100%2/2
$base || 1
l!l
115
MC/DC100%1/1
$base || 1
$base
Truth table100%2/2
A
A= $base
Aresult
11
00
136 22 return $delay;
137 }
138  
139 37 ▶sub _short_agent {
Subroutine _short_agent: called CC=2 SCAR=7.0
140 37 my ($self) = @_;
141 37 my $name = $self->agent;
142 37 ▶ $name = $1 if $name =~ m!^(\S+)!; # first word
Branch: 1 branch
BranchTrueFalseTotal
if $name =~ /^(\S+)/u 37 0 37
143 37 $name =~ s!/.*$!!; # no version
144 37 return $name;
145 }
146  
147 59 ▶sub _translate_path_pattern {
Subroutine _translate_path_pattern: called CC=6 SCAR=18.0
148 59 my ($self, $key, $value, $base_uri) = @_;
149  
150 59 my $ignore;
151 59 eval {
152 59 my $uri = URI->new_abs($value, $base_uri);
153 59 ▶ $ignore++ unless $uri->scheme eq $base_uri->scheme;
Branch: 1 branch
BranchTrueFalseTotal
unless $uri->scheme eq $base_uri->scheme 7 52 59
154 59 ▶ $ignore++ unless lc($uri->host) eq lc($base_uri->host);
Branch: 1 branch
BranchTrueFalseTotal
unless lc $uri->host eq lc $base_uri->host 0 59 59
155 59 ▶ $ignore++ unless $uri->port eq $base_uri->port;
Branch: 1 branch
BranchTrueFalseTotal
unless $uri->port eq $base_uri->port 7 52 59
156 };
157 59 ▶ return () if $@;
Branch: 1 branch
BranchTrueFalseTotal
if $@ 0 59 59
158 59 ▶ return () if $ignore;
Branch: 1 branch
BranchTrueFalseTotal
if $ignore 7 52 59
159  
160 52 return ($key, $value);
161 }
162  
163 11 ▶sub _translate_delay { # into delay in milliseconds
Subroutine _translate_delay: called CC=2 SCAR=6.9
164 11 my ($self, $key, $value) = @_;
165 11 ▶ return () unless $value =~ qr!\A[0-9.]+\z!;
Branch: 1 branch
BranchTrueFalseTotal
unless $value =~ qr/\A[0-9.]+\z/u 1 10 11
166 10 return ('crawl-delay', $value);
167 }
168  
169 16 ▶sub _translate_rate { # into delay in milliseconds
Subroutine _translate_rate: called CC=3 SCAR=11.0
170 16 my ($self, $key, $value) = @_;
171 16 ▶ return () unless $value =~ qr!\A([0-9.]+)\s*/\s*([0-9.]+)\z!;
Branch: 1 branch
BranchTrueFalseTotal
unless $value =~ qr"\A([0-9.]+)\s*/\s*([0-9.]+)\z"u 1 15 16
172 15 ▶ return () unless $1+0;
Branch: 1 branch
BranchTrueFalseTotal
unless $1 + 0 2 13 15
173 13 return ('crawl-delay', $2 / $1);
174 }
175  
176 4 1;
177 __END__
↑ Summary