blib/lib/URI/ParseSearchString/More.pm

filter: Statementstmt 81.1% 116 / 143 Branchbran 52.9% 18 / 34 Conditioncond 0.0% 0 / 6 MC/DCmcdc 0.0% 0 / 4 Subroutinesub 86.9% 20 / 23 Podpod 100% 8 / 8 totaltotal 74.3% 162 / 218 CC 25 SCAR 24.5
CC
25
Cov
73%
parse_search_string
33.0
guess
30.0
parse_more
7.1
CRAP 11.6
↑ Summary
1 4 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
2 4 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
3  
4 package URI::ParseSearchString::More;
5 4 our $VERSION = '0.19';
6 4 ▶use base qw( URI::ParseSearchString );
Subroutine BEGIN: called CC=1 SCAR=0.0
7  
8 4 ▶use List::Compare ();
Subroutine BEGIN: called CC=1 SCAR=0.0
9 4 ▶use LWP::Protocol::https (); ## no perlimports
Subroutine BEGIN: called CC=1 SCAR=0.0
10 4 ▶use Params::Validate qw( SCALAR validate );
Subroutine BEGIN: called CC=1 SCAR=0.0
11 4 ▶use Try::Tiny qw( catch try );
Subroutine BEGIN: called CC=1 SCAR=0.0
12 4 ▶use URI ();
Subroutine BEGIN: called CC=1 SCAR=0.0
13 4 ▶use URI::Heuristic qw( uf_uristr );
Subroutine BEGIN: called CC=1 SCAR=0.0
14 4 ▶use URI::QueryParam ();
Subroutine BEGIN: called CC=1 SCAR=0.0
15 4 ▶use WWW::Mechanize::Cached ();
Subroutine BEGIN: called CC=1 SCAR=0.0
16  
17 4 my %search_regex = (
18 answers => [qr{(.*)}],
19 aol => [qr{(.*) - AOL Search Results}],
20 as => [qr{Starware (.*) Search Results}],
21 dogpile => [qr{(.*) - Dogpile Web Search}],
22 );
23  
24 4 my %url_regex = (
25 answers => qr{answers.yahoo.com},
26 aol => qr{aol.com/(?:aol|aolcom)/search\?encquery=},
27 as => qr{as.\w+.com/dp/search\?x=},
28 dogpile => qr{http://www.dogpile},
29 );
30  
31 # local.yahoo.com should come before Yahoo as it has different
32 # params
33 # ie. this list is for ordering some of the engines in the regex. if an
34 # engine has no reason to be at a certain point in the list, it should
35 # just be listed in the %query_lookup table
36  
37 4 my @engines = (
38 'local.google', 'maps.google',
39 'googlesyndication', 'google',
40 'local.yahoo.com', 'search.yahoo.com',
41 'shopping.yahoo.com', 'yahoo',
42 'alltheweb.com', 'errors.aol.com',
43 'sucheaol.aol.de', 'aol',
44 'ask.*', 'fastbrowsersearch.com',
45 'as.*.com', 'att.net',
46 'trustedsearch.com',
47 );
48  
49 4 my %query_lookup = (
50  
51 'abcsok.no' => ['q'],
52 'about.com' => ['terms'],
53 'alltheweb.com' => ['q'],
54 'answers.com' => ['s'],
55 'aol' => [ 'query', 'q' ],
56 'as.*.com' => ['qry'],
57 'ask.*' => ['q'],
58 'att.net' => [ 'qry', 'q' ],
59 'baidu.com' => ['bs'],
60 'bing.com' => ['q'],
61 'blingo.com' => ['q'],
62 'citysearch.com' => ['query'],
63 'clicknow.org.uk' => ['q'],
64 'clusty.com' => ['query'],
65 'comcast.net' => [ 'query', 'q' ],
66 'cuil.com' => ['q'],
67 'danielsearch.info' => ['q'],
68 'devilfinder.com' => ['q'],
69 'ebay' => ['satitle'],
70 'education.yahoo.com' => ['p'],
71 'errors.aol.com' => ['host'],
72 'excite' => ['search'],
73 'ez4search.com' => ['searchname'],
74 'fastbrowsersearch.com' => ['q'],
75 'fedstats.com' => ['s'],
76 'find.copernic.com' => ['query'],
77 'finna.is' => ['query'],
78 'googlesyndication' => [ 'q', 'ref', 'loc' ],
79 'google' => [ 'q', 'as_q' ],
80 'googel' => ['q'],
81 'hotbot.lycos.com' => ['query'],
82 'isearch.com' => ['Terms'],
83 'local.google' => [ 'q', 'near' ],
84 'local.yahoo.com' => [ 'stx', 'csz' ],
85 'looksmart.com' => ['key'],
86 'lycos' => ['query'],
87 'maps.google' => [ 'q', 'near' ],
88 'msntv.msn.com' => ['q'],
89 'munky.com' => ['term'],
90 'mysearch.com' => ['searchfor'],
91 'mywebsearch.com' => ['searchfor'],
92 'mytelus.com' => ['q'],
93 'netscape.com' => ['query'],
94 'nextag.com' => ['search'],
95 'overture.com' => ['Keywords'],
96 'pricescan.com' => ['SearchString'],
97 'reviews.search.com' => ['q'],
98 'search.com' => ['q'],
99 'searchalot.com' => ['q'],
100 'searchfusion.com' => ['t'],
101 'searchon.ca' => ['Terms'],
102 'search.cnn.com' => ['query'],
103 'search.bearshare.com' => ['q'],
104 'search.comcast.net' => ['q'],
105 'search.dmoz.org' => ['search'],
106 'search.earthlink.net' => ['q'],
107 'search.findsall.info' => ['s'],
108 'search.freeserve.com' => ['q'],
109 'search.freeze.com' => ['Keywords'],
110 'search.go.com' => ['search'],
111 qr/search\d?.incredimail.com/ => ['q'],
112 'search.juno.com' => ['query'],
113 'search.iol.ie' => ['q'],
114 'search.live.com' => ['q'],
115 'search.netzero.net' => ['query'],
116 'search.*.msn.' => ['q'],
117 'search.myway.com' => ['searchfor'],
118 'search.opera.com' => ['search'],
119 'search.rogers.com' => [ 'qf', 'qo' ],
120 'search.rr.com' => ['qs'],
121 'search.start.co.il' => ['q'],
122 'search.starware.com' => ['qry'],
123 'search.sympatico.msn.ca' => ['q'],
124 'search.sweetim.com' => ['q'],
125 'search.usatoday.com' => ['kw'],
126 'search.virgilio.it' => ['qs'],
127 'search.wanadoo.co.uk' => ['q'],
128 'search.yahoo.com' => [ 'q', 'va', 'p' ],
129 'searchservice.myspace.com' => ['qry'],
130 'shopping.yahoo.com' => ['p'],
131 'start.shaw.ca' => ['q'],
132 'startgoogle.startpagina.nl' => ['q'],
133 'stumbleupon.com' => ['url'],
134 'sucheaol.aol.de' => ['q'],
135 'teoma.com' => ['q'],
136 'toronto.com' => ['query'],
137 'trustedsearch.net' => ['w'],
138 'trustedsearch.com' => ['w'],
139 'yahoo' => ['p'],
140 'yandex.ru' => ['text'],
141 'youtube.com' => ['search_query'],
142 'websearch.cbc.ca' => ['query'],
143 'websearch.cs.com' => ['query'],
144 'webtv.net' => ['q'],
145 'www.bestsearchonearth.info' => ['Keywords'],
146 'www.boat.com' => ['HotKeysTopCategory'],
147 'www.factbites.com' => ['kp'],
148 'www.mweb.co.za' => ['q'],
149 'www.rr.com/html/search.cfm' => ['query'],
150 'www.wotbox.com' => ['q'],
151 );
152  
153 136 ▶sub parse_search_string {
Subroutine parse_search_string: called CC=9 SCAR=35.0
Pod documented
154 136 my $self = shift;
155 136 my $url = shift;
156  
157 136 ▶ die 'missing url.' unless $url;
Branch: 1 branch
BranchTrueFalseTotal
unless $url 1 135 136
158  
159 135 foreach my $engine ( keys %url_regex ) {
160  
161 540 ▶ if ( $url =~ $url_regex{$engine} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($url =~ /$url_regex{$engine}/) 0 540 540
162  
163 # fix funky URLs
164 0 $url = uf_uristr($url);
165  
166 0 my $mech = $self->get_mech();
167  
168 0 ▶ try { $mech->get($url); }
Subroutine __ANON__: not called CC=1 SCAR=6.9
169 0 ▶ catch {
Subroutine __ANON__: not called CC=1 SCAR=6.9
170 0 warn "Issue with url: $url";
171 0 warn $_;
172 0 };
173  
174 0 ▶ if ( $mech->status && $mech->status == 403 ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($mech->status and $mech->status == 403) 0 0 0
Condition0%0/3
$mech->status and $mech->status == 403
!ll&&!rl&&r
000
MC/DC0%0/2
$mech->status and $mech->status == 403
$mech->status $mech->status == 403
Truth table0%0/3
A and B
A= $mech->statusB= $mech->status == 403
ABresult
0X0
100
111
175 0 warn "403 returned for $url Are you being blocked?";
176 }
177  
178 0 ▶ if ( $mech->title() ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($mech->title) 0 0 0
179 0 my $search_term = $self->_apply_regex(
180 string => $mech->title(),
181 engine => $engine,
182 );
183  
184 0 ▶ if ($search_term) {
Branch: 1 branch
BranchTrueFalseTotal
if ($search_term) 0 0 0
185 0 $self->{'more'}->{'blame'} = __PACKAGE__;
186 0 return $search_term;
187 }
188 }
189 }
190 }
191  
192 135 my $terms = $self->parse_more($url);
193  
194 135 ▶ if ($terms) {
Branch: 1 branch
BranchTrueFalseTotal
if ($terms) 135 0 135
195 135 $self->{'more'}->{'blame'} = __PACKAGE__;
196 135 return $terms;
197 }
198  
199 # We've come up empty. Let's see what the superclass can do
200 0 $self->{'more'}->{'blame'} = 'URI::ParseSearchString';
201 0 return $self->SUPER::parse_search_string( $url, @_ );
202 }
203  
204 1 ▶sub se_term {
Subroutine se_term: called CC=1 SCAR=0.0
Pod documented
205 1 my $self = shift;
206 1 return $self->parse_search_string(@_);
207 }
208  
209 135 ▶sub parse_more {
Subroutine parse_more: called CC=7 SCAR=19.6
Pod documented
210 135 my $self = shift;
211 135 my $url = shift;
212  
213 135 ▶ die 'you need to supply at least one argument' unless $url;
Branch: 1 branch
BranchTrueFalseTotal
unless $url 0 135 135
214  
215 135 $self->{'more'} = undef;
216 135 $self->{'more'}->{'string'} = $url;
217  
218 135 my $regex = join ' | ', $self->_get_engines;
219 135 $self->{'more'}->{'regex'} = $regex;
220 135 $self->{'more'}->{'url'} = $url;
221  
222 135 ▶ if ( $url =~ m{ ( (?: $regex ) .* ?/ ) .* ?\? (.*)\z }xms ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($url =~ m[ ( (?: $regex ) .* ?/ ) .* ?\? (.*)\z ]msx) 135 0 135
223  
224 135 my $uri = URI->new( uf_uristr($url) );
225 135 my $domain = $uri->host;
226  
227 # remove trailing slash
228 135 $domain =~ s{/\z}{};
229  
230 135 my @param_parts = ();
231 135 my %params = ();
232  
233 ENGINE:
234 135 foreach my $engine ( $self->_get_engines ) {
235  
236 5531 ▶ if ( $domain =~ /$engine/i ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($domain =~ /$engine/i) 135 5396 5531
237  
238 135 my @names = @{ $query_lookup{$engine} };
239  
240 135 $self->{'more'}->{'domain'} = $domain;
241 135 $self->{'more'}->{'names'} = \@names;
242  
243 135 foreach my $name (@names) {
244 170 push @param_parts, $uri->query_param($name);
245 170 $params{$name} = $uri->query_param($name);
246 }
247  
248 135 last ENGINE;
249 }
250 }
251  
252 135 my $params = join( q{ }, @param_parts );
253 135 my $orig_domain = $domain;
254 135 $domain =~ s/\/.*//g;
255 135 ▶ unless ( $domain =~ /\w/ ) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($domain =~ /\w/) 0 135 135
256 0 $domain = $orig_domain;
257 }
258  
259 135 $self->{'more'}->{'terms'} = \@param_parts;
260 135 $self->{'more'}->{'params'} = \%params;
261  
262 135 return $params;
263 }
264  
265 0 return;
266 }
267  
268 133 ▶sub blame {
Subroutine blame: called CC=1 SCAR=0.0
Pod documented
269 133 my $self = shift;
270 133 return $self->{more}->{blame};
271 }
272  
273 0 ▶sub guess {
Subroutine guess: not called CC=5 SCAR=34.0
Pod documented
274 0 my $self = shift;
275 0 ▶ my $url = shift || $self->{'more'}->{'string'};
Condition0%0/3
shift() || $self->{'more'}{'string'}
l!l&&r!l&&!r
000
MC/DC0%0/2
shift() || $self->{'more'}{'string'}
shift() $self->{'more'}{'string'}
Truth table0%0/3
A || B
A= shift()B= $self->{'more'}{'string'}
ABresult
1X1
011
000
276  
277 0 my @guesses = ( 'q', 'query', 'searchfor' );
278  
279 0 my $uri = URI->new($url);
280 0 ▶ if ( $uri->query_params ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($uri->query_params) 0 0 0
281  
282 0 foreach my $guess (@guesses) {
283 0 ▶ if ( $uri->query_param($guess) ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($uri->query_param($guess)) 0 0 0
284 0 return $uri->query_param($guess);
285 }
286 }
287 }
288  
289 0 return;
290 }
291  
292 3 ▶sub set_cached {
Subroutine set_cached: called CC=2 SCAR=6.9
Pod documented
293 3 my $self = shift;
294 3 my $switch = shift;
295  
296 3 ▶ if ($switch) {
Branch: 1 branch
BranchTrueFalseTotal
if ($switch) { } 1 2 3
297 1 $self->{'__more_cached'} = 1;
298 }
299 else {
300 2 $self->{'__more_cached'} = 0;
301 }
302  
303 3 return $self->{'__more_cached'};
304 }
305  
306 271 ▶sub get_cached {
Subroutine get_cached: called CC=1 SCAR=0.0
Pod documented
307 271 my $self = shift;
308  
309 271 return $self->{'__more_cached'};
310 }
311  
312 269 ▶sub get_mech {
Subroutine get_mech: called CC=4 SCAR=13.9
Pod documented
313 269 my $self = shift;
314 269 my $cache = $self->get_cached;
315  
316 269 ▶ if ($cache) {
Branch: 1 branch
BranchTrueFalseTotal
if ($cache) 2 267 269
317  
318 2 ▶ if ( !exists $self->{'__more_mech_cached'} ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (exists $self->{"__more_mech_cached"}) 1 1 2
319  
320 1 my $mech = WWW::Mechanize::Cached->new();
321 1 $mech->agent('URI::ParseSearchString::More');
322 1 $self->{'__more_mech_cached'} = $mech;
323  
324 }
325  
326 2 return $self->{'__more_mech_cached'};
327  
328 }
329  
330 # return a non-caching object
331 267 ▶ if ( !exists $self->{'__more_mech'} ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (exists $self->{"__more_mech"}) 2 265 267
332  
333 2 my $mech = WWW::Mechanize->new();
334 2 $mech->agent('URI::ParseSearchString::More');
335 2 $self->{'__more_mech'} = $mech;
336  
337 }
338  
339 267 return $self->{'__more_mech'};
340 }
341  
342 1 ▶sub _apply_regex {
Subroutine _apply_regex: called CC=3 SCAR=11.2
343 1 my $self = shift;
344 1 my %rules = (
345 string => { type => SCALAR },
346 engine => { type => SCALAR },
347 );
348  
349 1 my %args = validate( @_, \%rules );
350  
351 1 foreach my $regex ( @{ $search_regex{ $args{'engine'} } } ) {
352 1 ▶ if ( $args{'string'} =~ $regex ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($args{"string"} =~ /$regex/) 0 1 1
353 0 return $1;
354 }
355 }
356 1 return;
357 }
358  
359 271 ▶sub _get_engines {
Subroutine _get_engines: called CC=1 SCAR=0.0
360 271 my $lc = List::Compare->new( \@engines, [ keys %query_lookup ] );
361 271 my @remaining_engines = $lc->get_complement;
362  
363 271 my @all_engines = @engines;
364 271 push @all_engines, @remaining_engines;
365  
366 271 return @all_engines;
367 }
368  
369 4 1;
370  
371 # ABSTRACT: Extract search strings from more referrers.
372  
373 __END__
↑ Summary