TextSearch.pm untested

filter: Statementstmt 0.0% 0 / 291 Branchbran 0.0% 0 / 58 Conditioncond 0.0% 0 / 21 MC/DCmcdc 0.0% 0 / 13 Subroutinesub 0.0% 0 / 34 Podpod 0.0% 0 / 15 totaltotal 0.0% 0 / 432 CC 30 SCAR 33.2
CC
30
Cov
0%
index_document
110.0
_rem_newer
42.0
open
30.0
CRAP 27.8
1 ######################################################################
2 ## ##
3 ## TextSearch - generic module to index and search ASCII files ##
4 ## This will hook up to a set of database specific routines ##
5 ## ##
6 ######################################################################
7  
8  
9 package DBIx::TextSearch;
10  
11 use DBI;
12 use Carp qw(croak cluck carp); # croak is die, cluck is warn;
13 use Env qw(TEMP TMP);
14 use English;
15 use Net::FTP;
16 use LWP::Simple;
17 use LWP::UserAgent;
18 use HTTP::Response;
19 use SGML::StripParser;
20 use Text::Query;
21 use URI;
22 use File::Basename;
23 use HTML::TokeParser;
24 use strict;
25 no strict qw/refs/;
26 use warnings; # - incompatible due to SGML::StripParser
27 use Socket;
28 use Sys::Hostname;
29 use Digest::MD5 qw/md5_hex/;
30 $DBIx::TextSearch::VERSION = '0.3';
31  
32 ######################################################################
33 sub say {
34 my $self = shift();
35 if ($self->{debug}) {
36 print @_;
37 }
38 }
39 ######################################################################
40 #sub AUTOLOAD {
41 # # allow variables to be accessed directly via methods
42 # # i.e. print $index->match() or $index->match = 5;
43 # my $self = shift;
44 # my $type = ref $self or Carp::croak "$self is not an object";
45 # my $field = $AUTOLOAD;
46 # $field =~ s/.*://s;
47 # unless (exists $self->{$field}) {
48 # Carp::croak "$field does not exist in object/class $type";
49 # }
50 # if (@_) {
51 # return $self->($name) = shift;
52 # } else {
53 # return $self->($name);
54 # }
55 #}
56 ######################################################################
57 sub new {
58 # create an index
59 # takes as input: database handle, index name
60 my $type = shift;
61 my $self = {};
62 my $dbh = shift;
63 my $name = shift;
64 my %par = @_;
65  
66 # set database handle
67 $self->{dbh} = $dbh;
68  
69 # set index name
70 $self->{name} = $name;
71  
72 # debug
73 $self->{debug} = $par{debug};
74  
75 # test for sufficient parameters
76 unless ($self->{dbh}) {
77 croak "A database handle is required.\n";
78 }
79 unless ($self->{name}) {
80 croak "An index name is required.\n";
81 }
82  
83 bless $self, $type;
84 # create appropriate tables
85 $self->CreateIndex; # CreateIndex will return its own errors
86  
87 # end.
88 return $self, $type;
89 }
90 ######################################################################
91 sub open {
92 # connect to an existing index (or barf if there isn't one)
93 # no strict qw/vars/;
94 my $type = shift;
95 my $self = {};
96 my $dbh = shift;
97 my $name = shift;
98 my %par = @_;
99  
100 # set database handle
101 $self->{dbh} = $dbh;
102  
103 # set index name
104 $self->{name} = $name;
105  
106 # debug
107 $self->{debug} = $par{debug};
108  
109 # test for sufficient parameters
110 unless ($self->{dbh}) {
111 Carp::croak "A database handle is required.\n";
112 }
113 unless ($self->{name}) {
114 Carp::croak "An index name is required.\n";
115 }
116  
117 # check if this index exists
118 my @tables = $self->{dbh}->tables;
119 # my @tables = $dbh->tables;
120 my $exists = 0;
121 foreach my $table (@tables) {
122 $table =~ s/\`//g;
123 #print $table, "\n";
124 if ($table =~ m/$self->{name}_(docid|words|description)/) {
125 # index tables exist
126 $exists = 'True';
127 }
128 }
129  
130 # die with message if index tables don't exist
131 if ($exists ne "True") {
132 croak "Index $name is not available";
133 }
134  
135 # else return an index object
136 return bless $self, $type;
137 }
138 ######################################################################
139 sub _get_unique_filename {
140 # get a unique local filename to store a remote file as
141 my ($i, $temp_dir);
142 $i = 0;
143  
144 if ($ENV{TEMP}) {
145 $temp_dir = $ENV{TEMP};
146 } elsif ($ENV{TMP}) {
147 $temp_dir = $ENV{TMP};
148 } else {
149 $temp_dir = '/tmp';
150 }
151 while (-e "$temp_dir/$PID.$i") {
152 ++$i;
153 }
154 return "$temp_dir/$PID.$i";
155 }
156 ######################################################################
157 sub _ftp {
158 my $self = shift();
159 $self->say("fetching via ftp\n");
160 # fetch a file via ftp and store locally
161 my $url = shift();
162 # parse an url like ftp://user:password@foo.bar.com/wibble/barf.txt into
163 # something usable
164 my $uri = URI->new($url);
165 my $host = $uri->host();
166 my $path = $uri->path();
167 my $auth = $uri->authority(); # user:password@host
168  
169 my ($username, $passwd);
170 if ($auth =~ /:/) {
171 # get username and password from $auth
172 $username = $auth;
173 $username =~ s/:.*//;
174 $passwd = $auth;
175 $passwd =~ s/$username://;
176 $passwd =~ s/@.*//;
177 $self->say("auth'd ftp\nusername: $username\nPassword $passwd\n");
178 } else {
179 # set username to anonymous, password to local (linux) email address
180 $username = 'anonymous';
181 my $hostname = `hostname`; # need to get domain name as well.
182 my $me = $ENV{USER};
183 $passwd = $me . '@' . $hostname;
184 $self->say("anon ftp\nuser : $username\npass : $passwd\n");
185 }
186  
187 # remove remote file name from $path into a separate variable
188 my $dir = dirname($path);
189 my $remote_file = basename($path);
190  
191  
192 # get unique name for local file
193 my $local_file = _get_unique_filename();
194  
195 # fetch the file
196 $self->say("logging into $host as $username with password $passwd\n");
197 my $ftp = Net::FTP->new($host,
198 Debug => 1,
199 Passive => 1);
200 $ftp->login($username, $passwd);
201 $ftp->cwd("$dir");
202 $ftp->ascii();
203 $ftp->get($remote_file, $local_file);
204 $ftp->quit();
205  
206 # file transferred, return its location
207 $self->say("Local file is: $local_file\n");
208 return $local_file;
209 }
210 ######################################################################
211 sub _http {
212 # fetch a file via http and store locally
213 my ($self, $url) = @_;
214 print "URL to fetch: $url\n";
215  
216 # get unique name for local file
217 my $local_file = _get_unique_filename();
218  
219 # fetch the file
220 my $ua = LWP::UserAgent->new;
221 my $request = HTTP::Request->new('GET', $url);
222 my $response = $ua->request($request);
223  
224 if ($response->is_success) {
225 # sucessful fetch
226 # write to disk
227 my $html = $response->content;
228 CORE::open(HTML, ">$local_file") or
229 croak "Can't save HTML file $url to $local_file: $!";
230 print HTML $html;
231 close HTML;
232 # file transferred, return its location
233 return $local_file;
234 } else {
235 # error message
236 my $error = $response->status_line;
237 cluck $error;
238 }
239  
240 }
241 ######################################################################
242 sub _rem_newer {
243 # check the md5 sum of a URI against db.
244 # return md5. If not in index, md5 from MD5i eq 'none'
245 # par 1 = http|ftp|file. par2 = uri
246 my $self = shift();
247 my ($ftype, $loc) = @_;
248 my $md5_file; # file checksum
249  
250 my $md5_db = $self->MD5($loc); # mtime of indexed file
251 # $md5_db = 'none' if not in index
252  
253 $self->say("is file newer than already indexed version?\n");
254 if ($ftype eq 'http') {
255 $self->say("checking md5 sum with http\n");
256 my $ua = LWP::UserAgent->new(env_proxy => 1,
257 keep_alive => 1,
258 timeout => 30);
259 my $response = $ua->get($loc);
260 cluck "Error while getting ", $response->request->uri,
261 " -- ", $response->status_line, "\nAborting"
262 unless $response->is_success;
263 my $doc = $response->content();
264 $md5_file = md5_hex($doc);
265 undef $ua;
266 } elsif ($ftype eq 'ftp') {
267 my $file = $self->_ftp($loc);
268 $md5_file = md5_hex($file);
269 unlink($file);
270 } elsif ($ftype eq 'file') {
271 $md5_file = md5_hex($loc);
272 }
273  
274 $self->say("file checksum : $md5_file\nindex checksum: $md5_db\n");
275  
276 if ($md5_file ne $md5_db) {
277 # remote file is different from indexed version
278 $self->say("uri is different from indexed version\n");
279 return ($md5_file, 1);
280 } else {
281 $self->say("uri is is the same as the indexed version\n");
282 return ($md5_file, 0);
283 }
284  
285 }
286 ######################################################################
287 sub index_document {
288 # given a document URI, add it to the index.
289 # each word is to be indexed once.
290 # also, only index if file is newer than database copy.
291 my ($file, $http_content_type, @head, $toIndex, $md5, $changed, $toRemove);
292 my $self = shift();
293 my %params = @_;
294 $http_content_type = 0;
295  
296 my $uri = $params{uri};
297  
298 $self->say("about to index $uri\n");
299  
300 # get file contents
301  
302 # if an ftp or http uri, call a sub to fetch the remote file, save
303 # it somewhere useful (/tmp) and return the name that the file has
304 # been saved under ($file)
305 my $url = URI->new($uri) or $self->say("couldn't create URI object to check url options\n");
306 $self->say("url is $uri\n");
307 $self->say("URI object is $url\n");
308  
309 if ($url->scheme() eq 'ftp') {
310 # an FTP address
311 # fetch and index only if remote file is newer than db
312 $self->say("fetching $uri via ftp\n");
313 ($md5, $changed) = $self->_rem_newer('ftp', $uri);
314 if ($changed == 1) {
315 $file = $self->_ftp($uri);
316 }
317 } elsif ($url->scheme() eq 'http') {
318 # an HTTP address
319 # fetch and index only if remote file is newer than db
320 $self->say("fetching $uri via http\n");
321 ($md5, $changed) = $self->_rem_newer('http', $uri);
322 $self->say("uri is $uri\n");
323 if ($changed == 1) {
324 $file = $self->_http($uri);
325 @head = head($uri);
326 $http_content_type = shift(@head);
327 }
328 } elsif ($url->scheme() eq 'file') {
329 # a local file
330 my $orig_file = $url->path();
331 ($md5, $changed) = $self->_rem_newer('file', $orig_file);
332 if ($changed == 1) {
333 $file = _get_unique_filename;
334 system('cp', $orig_file, $file) or croak "Can't create temp file $file for indexing: $!";
335 }
336 } else {
337 Carp::cluck "Unrecognised URI type, assuming $uri is a local file\n";
338 ($md5, $changed) = $self->_rem_newer($self, 'file', $uri);
339 if ($changed == 1) {
340 $file = _get_unique_filename;
341 system('cp', $uri, $file) or croak "Can't create temp file $file for indexing: $!";
342 }
343 }
344  
345 # OK, now know where the file is, can open and index it
346  
347 # index the file differently depending on whether HTML or not. Use
348 # either file extension or HTTP content type
349 # HTML files get meta
350 # description tag as description, all others get 1st paragraph of
351 # file
352 if ($changed == 1) {
353 if ( ($uri =~ /html$|htm$/i) or ($http_content_type =~ /html/i) ) {
354 $self->say("processing $uri as html\n");
355 &_store_html($self,
356 file => $file,
357 uri => $uri,
358 md5 => $md5);
359 } else {
360 &_store_plain($self,
361 file => $file,
362 uri => $uri,
363 md5 => $md5);
364 }
365 }
366 return('ok');
367 }
368 ######################################################################
369 sub _uniqify {
370 # uniqify a string
371 my $str = shift;
372 my $prev = 0;
373 $str = lc($str); # for case-insensitive matching
374 my @words = split(/ /, $str);
375 @words = sort @words;
376 my @out = grep($_ ne $prev && ($prev = $_, 1), @words);
377 my $uniq = join(' ', @out);
378 return $uniq;
379 }
380 ######################################################################
381 sub _store_plain {
382 # index the content of a local copy of an ASCII file, giving it the
383 # uri of wherever it came from initially
384  
385 # get parameters
386 my $self = shift();
387 my %params = @_;
388 # open the file
389 CORE::open (INFILE, "<$params{file}");
390  
391 # use the first non-blank line line as the title
392 # and the next paragraph as the description
393 my $RS = "\n\n"; # read data terminated by 2 newlines
394 my @parags = <INFILE>;
395 my $title = $parags[0];
396 my $description = $parags[1];
397  
398 # reform @parags array into a string of words
399 my $doc = join(" ", @parags);
400 # tidy up doc so it contains only alnums and single spaces
401 $doc =~ s/[^A-Za-z]+/ /gm;
402 $doc = _uniqify($doc);
403 $title =~ s/[^A-Za-z]+/ /gm;
404 $description =~ s/[^A-Za-z]+/ /gm;
405  
406 # store this data
407 $self->IndexFile($params{uri}, $title, $description, $params{md5}, $doc);
408 $self->say("URI : $params{uri}\n",
409 "Title: $title\n",
410 "Desc : $description\n",
411 "Doc: $doc\n");
412 # remove local copy of file
413 unlink($params{file});
414 }
415 ######################################################################
416 sub _store_html {
417 # index the content of a local copy of an HTML file, giving it the
418 # uri of wherever it came from initially.
419  
420 # get parameters
421 my $self = shift();
422 my %params = @_;
423 my $title = 'Untitled document';
424 my $description = 'No description meta tag';
425 my $keywords;
426  
427 # title
428 my $HTML_Parser = new HTML::TokeParser($params{file});
429 if ($HTML_Parser->get_tag("title")) {
430 $title = $HTML_Parser->get_trimmed_text;
431 }
432 # description and keywords
433 while (my $token = $HTML_Parser->get_tag('meta')) {
434 if ($token->[1]{name} eq 'description') {
435 # description
436 $description = $token->[1]{content};
437 $description =~ s/\n/ /g;
438 } elsif ($token->[1]{name} eq 'keywords') {
439 # keywords
440 $keywords = $token->[1]{content};
441 $keywords =~ s/\n/ /g;
442 }
443 }
444  
445 # now have the title, description and keywords seperated out, can
446 # dump the rest of the HTML code
447 my $plain = _get_unique_filename;
448 CORE::open PLAIN, ">$plain";
449 CORE::open INFILE, $params{file};
450 my $sgmlp = new SGML::StripParser;
451 $sgmlp->set_outhandle(\*PLAIN); # plain text output file
452 $sgmlp->parse_data(\*INFILE); # html input file
453 close INFILE;
454 close PLAIN;
455  
456 # read the non-html file in.
457 undef $RS;
458 CORE::open PLAIN, $plain;
459 my $doc = <PLAIN>;
460 # tidy up doc so it contains only alnums and single spaces
461 $doc =~ s/[^A-Za-z]+/ /gm;
462 close PLAIN;
463 # uniqify $doc
464 $doc = _uniqify($doc);
465 $doc .= $keywords; # add keywords to doc content
466  
467 # store this data
468 $self->IndexFile($params{uri}, $title, $description, $params{md5}, $doc);
469  
470 # clean up temp files
471 unlink($plain);
472 unlink($params{file});
473 }
474 ######################################################################
475 sub find_document {
476 # read an altavista style advanced query in, pass through to an appropriate
477 # Text::Query::BuildSQL interface and return the URI, title and
478 # description of each matching document
479 my $self = shift;
480 my %params = @_;
481  
482 # get a fully parsed query to run
483 $params{query} = $self->GetQuery(query => $params{query},
484 parser => $params{parser});
485 $self->say("Got SQL query\n");
486  
487 $self->say("Query: $params{query}\n");
488 # the column names which will be returned when this query is run
489 # are uri, title, description (in that order)
490  
491 $self->say("preparing query\n");
492 my $sth = $self->{dbh}->prepare($params{query}) or
493 cluck "Can't prepare query: $params{query}\n$self->{dbh}->errstr";
494 $self->say("executing query (", time(), ")\n");
495 $sth->execute or
496 cluck "Can't execute query $params{query}: $self->{dbh}->errstr";
497  
498 # build an array of hashrefs - each hashref refers to a single row
499 # by column name
500 $self->say("Building result array of hashrefs\n");
501 my (@documents, $hash_row);
502 while ($hash_row = $sth->fetchrow_hashref) {
503 die $self->{dbh}->errstr() unless $hash_row;
504 push @documents, $hash_row;
505 $self->{matches} = scalar(@documents);
506 }
507 # die unless @documents;
508 return \@documents;
509 }
510 #####################################################################
511 sub DESTROY {
512 my $self = shift();
513 $self->{dbh}->disconnect;
514 }
515 ######################################################################
516 sub flush_index {
517 my $self = shift();
518 $self->FlushIndex();
519 }
520 ######################################################################
521 sub delete_document {
522 my ($self, $uri) = @_;
523 $self->RemoveDocumnt($uri);
524 }
525 ######################################################################
526 __END__