| 1 | ###################################################################### | ||
| 2 | ## ## | ||
| 3 | ## TextSearch - generic module to index and search ASCII files ## | ||
| 4 | ## This will hook up to a set of database specific routines ## | ||
| 5 | ## ## | ||
| 6 | ###################################################################### | ||
| 7 | |||
| 8 | |||
| 9 | package DBIx::TextSearch; | ||
| 10 | |||
| 11 | use DBI; | ||
| 12 | use Carp qw(croak cluck carp); # croak is die, cluck is warn; | ||
| 13 | use Env qw(TEMP TMP); | ||
| 14 | use English; | ||
| 15 | use Net::FTP; | ||
| 16 | use LWP::Simple; | ||
| 17 | use LWP::UserAgent; | ||
| 18 | use HTTP::Response; | ||
| 19 | use SGML::StripParser; | ||
| 20 | use Text::Query; | ||
| 21 | use URI; | ||
| 22 | use File::Basename; | ||
| 23 | use HTML::TokeParser; | ||
| 24 | use strict; | ||
| 25 | no strict qw/refs/; | ||
| 26 | use warnings; # - incompatible due to SGML::StripParser | ||
| 27 | use Socket; | ||
| 28 | use Sys::Hostname; | ||
| 29 | use Digest::MD5 qw/md5_hex/; | ||
| 30 | $DBIx::TextSearch::VERSION = '0.3'; | ||
| 31 | |||
| 32 | ###################################################################### | ||
| 33 | sub say { | ||
| 34 | my $self = shift(); | ||
| 35 | if ($self->{debug}) { | ||
| 36 | print @_; | ||
| 37 | } | ||
| 38 | } | ||
| 39 | ###################################################################### | ||
| 40 | #sub AUTOLOAD { | ||
| 41 | # # allow variables to be accessed directly via methods | ||
| 42 | # # i.e. print $index->match() or $index->match = 5; | ||
| 43 | # my $self = shift; | ||
| 44 | # my $type = ref $self or Carp::croak "$self is not an object"; | ||
| 45 | # my $field = $AUTOLOAD; | ||
| 46 | # $field =~ s/.*://s; | ||
| 47 | # unless (exists $self->{$field}) { | ||
| 48 | # Carp::croak "$field does not exist in object/class $type"; | ||
| 49 | # } | ||
| 50 | # if (@_) { | ||
| 51 | # return $self->($name) = shift; | ||
| 52 | # } else { | ||
| 53 | # return $self->($name); | ||
| 54 | # } | ||
| 55 | #} | ||
| 56 | ###################################################################### | ||
| 57 | sub new { | ||
| 58 | # create an index | ||
| 59 | # takes as input: database handle, index name | ||
| 60 | my $type = shift; | ||
| 61 | my $self = {}; | ||
| 62 | my $dbh = shift; | ||
| 63 | my $name = shift; | ||
| 64 | my %par = @_; | ||
| 65 | |||
| 66 | # set database handle | ||
| 67 | $self->{dbh} = $dbh; | ||
| 68 | |||
| 69 | # set index name | ||
| 70 | $self->{name} = $name; | ||
| 71 | |||
| 72 | # debug | ||
| 73 | $self->{debug} = $par{debug}; | ||
| 74 | |||
| 75 | # test for sufficient parameters | ||
| 76 | unless ($self->{dbh}) { | ||
| 77 | croak "A database handle is required.\n"; | ||
| 78 | } | ||
| 79 | unless ($self->{name}) { | ||
| 80 | croak "An index name is required.\n"; | ||
| 81 | } | ||
| 82 | |||
| 83 | bless $self, $type; | ||
| 84 | # create appropriate tables | ||
| 85 | $self->CreateIndex; # CreateIndex will return its own errors | ||
| 86 | |||
| 87 | # end. | ||
| 88 | return $self, $type; | ||
| 89 | } | ||
| 90 | ###################################################################### | ||
| 91 | sub open { | ||
| 92 | # connect to an existing index (or barf if there isn't one) | ||
| 93 | # no strict qw/vars/; | ||
| 94 | my $type = shift; | ||
| 95 | my $self = {}; | ||
| 96 | my $dbh = shift; | ||
| 97 | my $name = shift; | ||
| 98 | my %par = @_; | ||
| 99 | |||
| 100 | # set database handle | ||
| 101 | $self->{dbh} = $dbh; | ||
| 102 | |||
| 103 | # set index name | ||
| 104 | $self->{name} = $name; | ||
| 105 | |||
| 106 | # debug | ||
| 107 | $self->{debug} = $par{debug}; | ||
| 108 | |||
| 109 | # test for sufficient parameters | ||
| 110 | unless ($self->{dbh}) { | ||
| 111 | Carp::croak "A database handle is required.\n"; | ||
| 112 | } | ||
| 113 | unless ($self->{name}) { | ||
| 114 | Carp::croak "An index name is required.\n"; | ||
| 115 | } | ||
| 116 | |||
| 117 | # check if this index exists | ||
| 118 | my @tables = $self->{dbh}->tables; | ||
| 119 | # my @tables = $dbh->tables; | ||
| 120 | my $exists = 0; | ||
| 121 | foreach my $table (@tables) { | ||
| 122 | $table =~ s/\`//g; | ||
| 123 | #print $table, "\n"; | ||
| 124 | if ($table =~ m/$self->{name}_(docid|words|description)/) { | ||
| 125 | # index tables exist | ||
| 126 | $exists = 'True'; | ||
| 127 | } | ||
| 128 | } | ||
| 129 | |||
| 130 | # die with message if index tables don't exist | ||
| 131 | if ($exists ne "True") { | ||
| 132 | croak "Index $name is not available"; | ||
| 133 | } | ||
| 134 | |||
| 135 | # else return an index object | ||
| 136 | return bless $self, $type; | ||
| 137 | } | ||
| 138 | ###################################################################### | ||
| 139 | sub _get_unique_filename { | ||
| 140 | # get a unique local filename to store a remote file as | ||
| 141 | my ($i, $temp_dir); | ||
| 142 | $i = 0; | ||
| 143 | |||
| 144 | if ($ENV{TEMP}) { | ||
| 145 | $temp_dir = $ENV{TEMP}; | ||
| 146 | } elsif ($ENV{TMP}) { | ||
| 147 | $temp_dir = $ENV{TMP}; | ||
| 148 | } else { | ||
| 149 | $temp_dir = '/tmp'; | ||
| 150 | } | ||
| 151 | while (-e "$temp_dir/$PID.$i") { | ||
| 152 | ++$i; | ||
| 153 | } | ||
| 154 | return "$temp_dir/$PID.$i"; | ||
| 155 | } | ||
| 156 | ###################################################################### | ||
| 157 | sub _ftp { | ||
| 158 | my $self = shift(); | ||
| 159 | $self->say("fetching via ftp\n"); | ||
| 160 | # fetch a file via ftp and store locally | ||
| 161 | my $url = shift(); | ||
| 162 | # parse an url like ftp://user:password@foo.bar.com/wibble/barf.txt into | ||
| 163 | # something usable | ||
| 164 | my $uri = URI->new($url); | ||
| 165 | my $host = $uri->host(); | ||
| 166 | my $path = $uri->path(); | ||
| 167 | my $auth = $uri->authority(); # user:password@host | ||
| 168 | |||
| 169 | my ($username, $passwd); | ||
| 170 | if ($auth =~ /:/) { | ||
| 171 | # get username and password from $auth | ||
| 172 | $username = $auth; | ||
| 173 | $username =~ s/:.*//; | ||
| 174 | $passwd = $auth; | ||
| 175 | $passwd =~ s/$username://; | ||
| 176 | $passwd =~ s/@.*//; | ||
| 177 | $self->say("auth'd ftp\nusername: $username\nPassword $passwd\n"); | ||
| 178 | } else { | ||
| 179 | # set username to anonymous, password to local (linux) email address | ||
| 180 | $username = 'anonymous'; | ||
| 181 | my $hostname = `hostname`; # need to get domain name as well. | ||
| 182 | my $me = $ENV{USER}; | ||
| 183 | $passwd = $me . '@' . $hostname; | ||
| 184 | $self->say("anon ftp\nuser : $username\npass : $passwd\n"); | ||
| 185 | } | ||
| 186 | |||
| 187 | # remove remote file name from $path into a separate variable | ||
| 188 | my $dir = dirname($path); | ||
| 189 | my $remote_file = basename($path); | ||
| 190 | |||
| 191 | |||
| 192 | # get unique name for local file | ||
| 193 | my $local_file = _get_unique_filename(); | ||
| 194 | |||
| 195 | # fetch the file | ||
| 196 | $self->say("logging into $host as $username with password $passwd\n"); | ||
| 197 | my $ftp = Net::FTP->new($host, | ||
| 198 | Debug => 1, | ||
| 199 | Passive => 1); | ||
| 200 | $ftp->login($username, $passwd); | ||
| 201 | $ftp->cwd("$dir"); | ||
| 202 | $ftp->ascii(); | ||
| 203 | $ftp->get($remote_file, $local_file); | ||
| 204 | $ftp->quit(); | ||
| 205 | |||
| 206 | # file transferred, return its location | ||
| 207 | $self->say("Local file is: $local_file\n"); | ||
| 208 | return $local_file; | ||
| 209 | } | ||
| 210 | ###################################################################### | ||
| 211 | sub _http { | ||
| 212 | # fetch a file via http and store locally | ||
| 213 | my ($self, $url) = @_; | ||
| 214 | print "URL to fetch: $url\n"; | ||
| 215 | |||
| 216 | # get unique name for local file | ||
| 217 | my $local_file = _get_unique_filename(); | ||
| 218 | |||
| 219 | # fetch the file | ||
| 220 | my $ua = LWP::UserAgent->new; | ||
| 221 | my $request = HTTP::Request->new('GET', $url); | ||
| 222 | my $response = $ua->request($request); | ||
| 223 | |||
| 224 | if ($response->is_success) { | ||
| 225 | # sucessful fetch | ||
| 226 | # write to disk | ||
| 227 | my $html = $response->content; | ||
| 228 | CORE::open(HTML, ">$local_file") or | ||
| 229 | croak "Can't save HTML file $url to $local_file: $!"; | ||
| 230 | print HTML $html; | ||
| 231 | close HTML; | ||
| 232 | # file transferred, return its location | ||
| 233 | return $local_file; | ||
| 234 | } else { | ||
| 235 | # error message | ||
| 236 | my $error = $response->status_line; | ||
| 237 | cluck $error; | ||
| 238 | } | ||
| 239 | |||
| 240 | } | ||
| 241 | ###################################################################### | ||
| 242 | sub _rem_newer { | ||
| 243 | # check the md5 sum of a URI against db. | ||
| 244 | # return md5. If not in index, md5 from MD5i eq 'none' | ||
| 245 | # par 1 = http|ftp|file. par2 = uri | ||
| 246 | my $self = shift(); | ||
| 247 | my ($ftype, $loc) = @_; | ||
| 248 | my $md5_file; # file checksum | ||
| 249 | |||
| 250 | my $md5_db = $self->MD5($loc); # mtime of indexed file | ||
| 251 | # $md5_db = 'none' if not in index | ||
| 252 | |||
| 253 | $self->say("is file newer than already indexed version?\n"); | ||
| 254 | if ($ftype eq 'http') { | ||
| 255 | $self->say("checking md5 sum with http\n"); | ||
| 256 | my $ua = LWP::UserAgent->new(env_proxy => 1, | ||
| 257 | keep_alive => 1, | ||
| 258 | timeout => 30); | ||
| 259 | my $response = $ua->get($loc); | ||
| 260 | cluck "Error while getting ", $response->request->uri, | ||
| 261 | " -- ", $response->status_line, "\nAborting" | ||
| 262 | unless $response->is_success; | ||
| 263 | my $doc = $response->content(); | ||
| 264 | $md5_file = md5_hex($doc); | ||
| 265 | undef $ua; | ||
| 266 | } elsif ($ftype eq 'ftp') { | ||
| 267 | my $file = $self->_ftp($loc); | ||
| 268 | $md5_file = md5_hex($file); | ||
| 269 | unlink($file); | ||
| 270 | } elsif ($ftype eq 'file') { | ||
| 271 | $md5_file = md5_hex($loc); | ||
| 272 | } | ||
| 273 | |||
| 274 | $self->say("file checksum : $md5_file\nindex checksum: $md5_db\n"); | ||
| 275 | |||
| 276 | if ($md5_file ne $md5_db) { | ||
| 277 | # remote file is different from indexed version | ||
| 278 | $self->say("uri is different from indexed version\n"); | ||
| 279 | return ($md5_file, 1); | ||
| 280 | } else { | ||
| 281 | $self->say("uri is is the same as the indexed version\n"); | ||
| 282 | return ($md5_file, 0); | ||
| 283 | } | ||
| 284 | |||
| 285 | } | ||
| 286 | ###################################################################### | ||
| 287 | sub index_document { | ||
| 288 | # given a document URI, add it to the index. | ||
| 289 | # each word is to be indexed once. | ||
| 290 | # also, only index if file is newer than database copy. | ||
| 291 | my ($file, $http_content_type, @head, $toIndex, $md5, $changed, $toRemove); | ||
| 292 | my $self = shift(); | ||
| 293 | my %params = @_; | ||
| 294 | $http_content_type = 0; | ||
| 295 | |||
| 296 | my $uri = $params{uri}; | ||
| 297 | |||
| 298 | $self->say("about to index $uri\n"); | ||
| 299 | |||
| 300 | # get file contents | ||
| 301 | |||
| 302 | # if an ftp or http uri, call a sub to fetch the remote file, save | ||
| 303 | # it somewhere useful (/tmp) and return the name that the file has | ||
| 304 | # been saved under ($file) | ||
| 305 | my $url = URI->new($uri) or $self->say("couldn't create URI object to check url options\n"); | ||
| 306 | $self->say("url is $uri\n"); | ||
| 307 | $self->say("URI object is $url\n"); | ||
| 308 | |||
| 309 | if ($url->scheme() eq 'ftp') { | ||
| 310 | # an FTP address | ||
| 311 | # fetch and index only if remote file is newer than db | ||
| 312 | $self->say("fetching $uri via ftp\n"); | ||
| 313 | ($md5, $changed) = $self->_rem_newer('ftp', $uri); | ||
| 314 | if ($changed == 1) { | ||
| 315 | $file = $self->_ftp($uri); | ||
| 316 | } | ||
| 317 | } elsif ($url->scheme() eq 'http') { | ||
| 318 | # an HTTP address | ||
| 319 | # fetch and index only if remote file is newer than db | ||
| 320 | $self->say("fetching $uri via http\n"); | ||
| 321 | ($md5, $changed) = $self->_rem_newer('http', $uri); | ||
| 322 | $self->say("uri is $uri\n"); | ||
| 323 | if ($changed == 1) { | ||
| 324 | $file = $self->_http($uri); | ||
| 325 | @head = head($uri); | ||
| 326 | $http_content_type = shift(@head); | ||
| 327 | } | ||
| 328 | } elsif ($url->scheme() eq 'file') { | ||
| 329 | # a local file | ||
| 330 | my $orig_file = $url->path(); | ||
| 331 | ($md5, $changed) = $self->_rem_newer('file', $orig_file); | ||
| 332 | if ($changed == 1) { | ||
| 333 | $file = _get_unique_filename; | ||
| 334 | system('cp', $orig_file, $file) or croak "Can't create temp file $file for indexing: $!"; | ||
| 335 | } | ||
| 336 | } else { | ||
| 337 | Carp::cluck "Unrecognised URI type, assuming $uri is a local file\n"; | ||
| 338 | ($md5, $changed) = $self->_rem_newer($self, 'file', $uri); | ||
| 339 | if ($changed == 1) { | ||
| 340 | $file = _get_unique_filename; | ||
| 341 | system('cp', $uri, $file) or croak "Can't create temp file $file for indexing: $!"; | ||
| 342 | } | ||
| 343 | } | ||
| 344 | |||
| 345 | # OK, now know where the file is, can open and index it | ||
| 346 | |||
| 347 | # index the file differently depending on whether HTML or not. Use | ||
| 348 | # either file extension or HTTP content type | ||
| 349 | # HTML files get meta | ||
| 350 | # description tag as description, all others get 1st paragraph of | ||
| 351 | # file | ||
| 352 | if ($changed == 1) { | ||
| 353 | if ( ($uri =~ /html$|htm$/i) or ($http_content_type =~ /html/i) ) { | ||
| 354 | $self->say("processing $uri as html\n"); | ||
| 355 | &_store_html($self, | ||
| 356 | file => $file, | ||
| 357 | uri => $uri, | ||
| 358 | md5 => $md5); | ||
| 359 | } else { | ||
| 360 | &_store_plain($self, | ||
| 361 | file => $file, | ||
| 362 | uri => $uri, | ||
| 363 | md5 => $md5); | ||
| 364 | } | ||
| 365 | } | ||
| 366 | return('ok'); | ||
| 367 | } | ||
| 368 | ###################################################################### | ||
| 369 | sub _uniqify { | ||
| 370 | # uniqify a string | ||
| 371 | my $str = shift; | ||
| 372 | my $prev = 0; | ||
| 373 | $str = lc($str); # for case-insensitive matching | ||
| 374 | my @words = split(/ /, $str); | ||
| 375 | @words = sort @words; | ||
| 376 | my @out = grep($_ ne $prev && ($prev = $_, 1), @words); | ||
| 377 | my $uniq = join(' ', @out); | ||
| 378 | return $uniq; | ||
| 379 | } | ||
| 380 | ###################################################################### | ||
| 381 | sub _store_plain { | ||
| 382 | # index the content of a local copy of an ASCII file, giving it the | ||
| 383 | # uri of wherever it came from initially | ||
| 384 | |||
| 385 | # get parameters | ||
| 386 | my $self = shift(); | ||
| 387 | my %params = @_; | ||
| 388 | # open the file | ||
| 389 | CORE::open (INFILE, "<$params{file}"); | ||
| 390 | |||
| 391 | # use the first non-blank line line as the title | ||
| 392 | # and the next paragraph as the description | ||
| 393 | my $RS = "\n\n"; # read data terminated by 2 newlines | ||
| 394 | my @parags = <INFILE>; | ||
| 395 | my $title = $parags[0]; | ||
| 396 | my $description = $parags[1]; | ||
| 397 | |||
| 398 | # reform @parags array into a string of words | ||
| 399 | my $doc = join(" ", @parags); | ||
| 400 | # tidy up doc so it contains only alnums and single spaces | ||
| 401 | $doc =~ s/[^A-Za-z]+/ /gm; | ||
| 402 | $doc = _uniqify($doc); | ||
| 403 | $title =~ s/[^A-Za-z]+/ /gm; | ||
| 404 | $description =~ s/[^A-Za-z]+/ /gm; | ||
| 405 | |||
| 406 | # store this data | ||
| 407 | $self->IndexFile($params{uri}, $title, $description, $params{md5}, $doc); | ||
| 408 | $self->say("URI : $params{uri}\n", | ||
| 409 | "Title: $title\n", | ||
| 410 | "Desc : $description\n", | ||
| 411 | "Doc: $doc\n"); | ||
| 412 | # remove local copy of file | ||
| 413 | unlink($params{file}); | ||
| 414 | } | ||
| 415 | ###################################################################### | ||
| 416 | sub _store_html { | ||
| 417 | # index the content of a local copy of an HTML file, giving it the | ||
| 418 | # uri of wherever it came from initially. | ||
| 419 | |||
| 420 | # get parameters | ||
| 421 | my $self = shift(); | ||
| 422 | my %params = @_; | ||
| 423 | my $title = 'Untitled document'; | ||
| 424 | my $description = 'No description meta tag'; | ||
| 425 | my $keywords; | ||
| 426 | |||
| 427 | # title | ||
| 428 | my $HTML_Parser = new HTML::TokeParser($params{file}); | ||
| 429 | if ($HTML_Parser->get_tag("title")) { | ||
| 430 | $title = $HTML_Parser->get_trimmed_text; | ||
| 431 | } | ||
| 432 | # description and keywords | ||
| 433 | while (my $token = $HTML_Parser->get_tag('meta')) { | ||
| 434 | if ($token->[1]{name} eq 'description') { | ||
| 435 | # description | ||
| 436 | $description = $token->[1]{content}; | ||
| 437 | $description =~ s/\n/ /g; | ||
| 438 | } elsif ($token->[1]{name} eq 'keywords') { | ||
| 439 | # keywords | ||
| 440 | $keywords = $token->[1]{content}; | ||
| 441 | $keywords =~ s/\n/ /g; | ||
| 442 | } | ||
| 443 | } | ||
| 444 | |||
| 445 | # now have the title, description and keywords seperated out, can | ||
| 446 | # dump the rest of the HTML code | ||
| 447 | my $plain = _get_unique_filename; | ||
| 448 | CORE::open PLAIN, ">$plain"; | ||
| 449 | CORE::open INFILE, $params{file}; | ||
| 450 | my $sgmlp = new SGML::StripParser; | ||
| 451 | $sgmlp->set_outhandle(\*PLAIN); # plain text output file | ||
| 452 | $sgmlp->parse_data(\*INFILE); # html input file | ||
| 453 | close INFILE; | ||
| 454 | close PLAIN; | ||
| 455 | |||
| 456 | # read the non-html file in. | ||
| 457 | undef $RS; | ||
| 458 | CORE::open PLAIN, $plain; | ||
| 459 | my $doc = <PLAIN>; | ||
| 460 | # tidy up doc so it contains only alnums and single spaces | ||
| 461 | $doc =~ s/[^A-Za-z]+/ /gm; | ||
| 462 | close PLAIN; | ||
| 463 | # uniqify $doc | ||
| 464 | $doc = _uniqify($doc); | ||
| 465 | $doc .= $keywords; # add keywords to doc content | ||
| 466 | |||
| 467 | # store this data | ||
| 468 | $self->IndexFile($params{uri}, $title, $description, $params{md5}, $doc); | ||
| 469 | |||
| 470 | # clean up temp files | ||
| 471 | unlink($plain); | ||
| 472 | unlink($params{file}); | ||
| 473 | } | ||
| 474 | ###################################################################### | ||
| 475 | sub find_document { | ||
| 476 | # read an altavista style advanced query in, pass through to an appropriate | ||
| 477 | # Text::Query::BuildSQL interface and return the URI, title and | ||
| 478 | # description of each matching document | ||
| 479 | my $self = shift; | ||
| 480 | my %params = @_; | ||
| 481 | |||
| 482 | # get a fully parsed query to run | ||
| 483 | $params{query} = $self->GetQuery(query => $params{query}, | ||
| 484 | parser => $params{parser}); | ||
| 485 | $self->say("Got SQL query\n"); | ||
| 486 | |||
| 487 | $self->say("Query: $params{query}\n"); | ||
| 488 | # the column names which will be returned when this query is run | ||
| 489 | # are uri, title, description (in that order) | ||
| 490 | |||
| 491 | $self->say("preparing query\n"); | ||
| 492 | my $sth = $self->{dbh}->prepare($params{query}) or | ||
| 493 | cluck "Can't prepare query: $params{query}\n$self->{dbh}->errstr"; | ||
| 494 | $self->say("executing query (", time(), ")\n"); | ||
| 495 | $sth->execute or | ||
| 496 | cluck "Can't execute query $params{query}: $self->{dbh}->errstr"; | ||
| 497 | |||
| 498 | # build an array of hashrefs - each hashref refers to a single row | ||
| 499 | # by column name | ||
| 500 | $self->say("Building result array of hashrefs\n"); | ||
| 501 | my (@documents, $hash_row); | ||
| 502 | while ($hash_row = $sth->fetchrow_hashref) { | ||
| 503 | die $self->{dbh}->errstr() unless $hash_row; | ||
| 504 | push @documents, $hash_row; | ||
| 505 | $self->{matches} = scalar(@documents); | ||
| 506 | } | ||
| 507 | # die unless @documents; | ||
| 508 | return \@documents; | ||
| 509 | } | ||
| 510 | ##################################################################### | ||
| 511 | sub DESTROY { | ||
| 512 | my $self = shift(); | ||
| 513 | $self->{dbh}->disconnect; | ||
| 514 | } | ||
| 515 | ###################################################################### | ||
| 516 | sub flush_index { | ||
| 517 | my $self = shift(); | ||
| 518 | $self->FlushIndex(); | ||
| 519 | } | ||
| 520 | ###################################################################### | ||
| 521 | sub delete_document { | ||
| 522 | my ($self, $uri) = @_; | ||
| 523 | $self->RemoveDocumnt($uri); | ||
| 524 | } | ||
| 525 | ###################################################################### | ||
| 526 | __END__ |