AlltheWeb.pm untested

filter: Statementstmt 0.0% 0 / 85 Branchbran 0.0% 0 / 50 Conditioncond 0.0% 0 / 9 MC/DCmcdc 0.0% 0 / 6 Subroutinesub 0.0% 0 / 6 Podpod 0.0% 0 / 2 totaltotal 0.0% 0 / 158 CC 26 SCAR 57.7
CC
26
Cov
0%
native_retrieve_some
462.0
native_setup_search
42.0
BEGIN
2.0
CRAP 321.4
1 # AlltheWeb.pm
2 # by Jim Smyser
3 # Copyright (c) 2000 by Jim Smyser
4 # $Id: AlltheWeb.pm,v 1.5 2000/07/17 07:10:59 jims Exp $
5  
6 package WWW::Search::AlltheWeb;
7  
8 =head1 NAME
9  
10 WWW::Search::AlltheWeb - class for searching AlltheWeb
11  
12 =head1 SYNOPSIS
13  
14 use WWW::Search;
15 $query = "sprinkler system installation how to";
16 $search = new WWW::Search('AlltheWeb');
17 $search->native_query(WWW::Search::escape_query($query));
18 $search->maximum_to_retrieve(100);
19 while (my $result = $search->next_result()) {
20  
21 $url = $result->url;
22 $title = $result->title;
23 $desc = $result->description;
24  
25 print "<a href=$url>$title</a> $source<br>$date<br>$desc<p>\n";
26 }
27  
28 =head1 DESCRIPTION
29  
30 AlltheWeb is a class specialization of WWW::Search.
31 It handles making and interpreting AlltheWeb searches.
32 This is one of the fastest and largest search engines around.
33 F<http://www.alltheweb.com>.
34  
35 This class exports no public interface; all interaction should
36 be done through L<WWW::Search> objects. See SYNOPSIS.
37  
38 =head1 AUTHOR
39  
40 C<WWW::Search::AlltheWeb> is written by Jim Smyser
41 Author e-mail <jsmyser@bigfoot.com>
42  
43 =head1 COPYRIGHT
44  
45 Copyright (c) 1996-1999 University of Southern California.
46 All rights reserved.
47  
48 THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
49 WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
50 MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
51  
52  
53 =cut
54 #'
55  
56 #####################################################################
57 require Exporter;
58 @EXPORT = qw();
59 @EXPORT_OK = qw();
60 @ISA = qw(WWW::Search Exporter);
61 $VERSION = '1.5';
62  
63 $MAINTAINER = 'Jim Smyser <jsmyser@bigfoot.com>';
64 $TEST_CASES = <<"ENDTESTCASES";
65 &test('AlltheWeb', '$MAINTAINER', 'zero', \$bogus_query, \$TEST_EXACTLY);
66 &test('AlltheWeb', '$MAINTAINER', 'one', 'vbt'.'hread', \$TEST_RANGE, 2,50);
67 &test('AlltheWeb', '$MAINTAINER', 'two_page', '+LS'.'AM +IS'.'I', \$TEST_GREATER_THAN, 26);
68 ENDTESTCASES
69  
70 use Carp ();
71 use WWW::Search(generic_option);
72 require WWW::SearchResult;
73  
74 sub native_setup_search {
75 my($self, $native_query, $native_options_ref) = @_;
76 $self->{_debug} = $native_options_ref->{'search_debug'};
77 $self->{_debug} = 2 if ($native_options_ref->{'search_parse_debug'});
78 $self->{_debug} = 0 if (!defined($self->{_debug}));
79 $self->{agent_e_mail} = 'jsmyser@bigfoot.com';
80 $self->user_agent('user');
81 $self->{_next_to_retrieve} = 1;
82 $self->{'_num_hits'} = 0;
83 $self->{'_hits_per_page'} = 100;
84
85 if (!defined($self->{_options})) {
86 $self->{'search_base_url'} = 'http://www.ussc.alltheweb.com';
87 $self->{_options} = {
88 'search_url' => 'http://www.ussc.alltheweb.com/cgi-bin/search',
89 'query' => $native_query,
90 'type' => 'all',
91 'hits' => $self->{'_hits_per_page'},
92 };
93 }
94 my $options_ref = $self->{_options};
95 if (defined($native_options_ref))
96 {
97 foreach (keys %$native_options_ref)
98 {
99 $options_ref->{$_} = $native_options_ref->{$_};
100 }
101 }
102 my($options) = '';
103 foreach (sort keys %$options_ref)
104 {
105 next if (generic_option($_));
106 $options .= $_ . '=' . $options_ref->{$_} . '&';
107 }
108 chop $options;
109 $self->{_next_url} = $self->{_options}{'search_url'} .'?'. $options;
110 }
111
112 sub native_retrieve_some {
113 my ($self) = @_;
114 print STDERR "**Getting Some**\n" if $self->{_debug};
115 return undef if (!defined($self->{_next_url}));
116 $self->user_agent_delay if 1 < $self->{'_next_to_retrieve'};
117 print STDERR "*Sending request (",$self->{_next_url},")\n" if $self->{_debug};
118 my($response) = $self->http_request('GET', $self->{_next_url});
119 $self->{response} = $response;
120 if (!$response->is_success)
121 {
122 return undef;
123 }
124 $self->{'_next_url'} = undef;
125 print STDERR "**Response\n" if $self->{_debug};
126
127 # parse the output
128 my ($HEADER, $HITS, $TITLE, $DESC) = qw(HE HI TI DE);
129 my $hits_found = 0;
130 my $state = $HEADER;
131 my $hit = ();
132 foreach ($self->split_lines($response->content()))
133 {
134 next if m@^$@; # short circuit for blank lines
135 print STDERR " $state ===$_=== " if 2 <= $self->{'_debug'};
136
137 if (m|(\d+)\sdocuments\sfound|i)
138 {
139 print STDERR "**Found Count**\n" if ($self->{_debug});
140 $self->approximate_result_count($1);
141 $state = $HITS;
142 }
143 elsif ($state eq $HITS && m@<dt>.*?<a href="([^"]+)">(.+)$@i)
144 {
145 print "**Found Hit URL/Title**\n" if 2 <= $self->{_debug};
146 if (defined($hit))
147 {
148 push(@{$self->{cache}}, $hit);
149 }
150 $hit = new WWW::SearchResult;
151 $hits_found++;
152 $hit->add_url($1);
153 $hit->title($2);
154 $state = $DESC;
155 }
156 elsif ($state eq $DESC && m|<dd><span.*?>(.*)$|i)
157 {
158 print STDERR "**Found Description**\n" if 2 <= $self->{_debug};
159 $hit->description($1);
160 $state = $HITS;
161 }
162 elsif ($state eq $HITS && m|.*?<A HREF="([^"]+)"><span class="resultbar">&gt;&gt;</span></a>|i)
163 {
164 print STDERR "**Going to Next Page**\n" if 2 <= $self->{_debug};
165 my $URL = $1;
166 $self->{'_next_url'} = $self->{'search_base_url'} . $URL;
167 print STDERR "Next URL is ", $self->{'_next_url'}, "\n" if 2 <= $self->{_debug};
168 $state = $HITS;
169 }
170 else
171 {
172 print STDERR "**Nothing matched.**\n" if 2 <= $self->{_debug};
173 }
174 }
175 if (defined($hit))
176 {
177 push(@{$self->{cache}}, $hit);
178 }
179 return $hits_found;
180 }
181 1;
182