Site.pm untested

filter: Statementstmt 0.0% 0 / 306 Branchbran 0.0% 0 / 112 Conditioncond 0.0% 0 / 23 MC/DCmcdc 0.0% 0 / 14 Subroutinesub 0.0% 0 / 27 Podpod 0.0% 0 / 17 totaltotal 0.0% 0 / 499 CC 57 SCAR 39.3
CC
57
Cov
0%
_validate_setting
110.0
save
90.0
merge
90.0
CRAP 51.2
1 package HtDig::Site;
2  
3 use strict;
4 use vars qw($VERSION @ISA @EXPORT @EXPORT_OK @accepted_fuzzy_types);
5 use File::stat;
6 use File::Copy;
7 use File::Spec;
8 use Date::Manip;
9 use Net::SMTP;
10  
11 require Exporter;
12 require AutoLoader;
13  
14 @ISA = qw(Exporter AutoLoader);
15 # Items to export into callers namespace by default. Note: do not export
16 # names by default without a very good reason. Use EXPORT_OK instead.
17 # Do not simply export all your public functions/methods/constants.
18 @EXPORT = qw(time_dig);
19  
20 $VERSION = '1.02';
21  
22  
23  
24 ###
25 #This array defines the known fuzzy index types that can be
26 # created by htfuzzy
27 ###
28 @accepted_fuzzy_types = qw/soundex metaphones endings synonyms/;
29 sub fuzzy_types {@accepted_fuzzy_types}
30  
31 ##
32 #Conf metadata is used to validate input and pre-existing values
33 # when importing conf files. It is based on the htdig documentation
34 # and its interpretation of what each config setting represents.
35 #For more information on htdig config file data types, see
36 # http://www.htdig.org/cf_types.html
37 #
38 #P.S.: It can also be used to get a list of stock settings
39 ##
40 my $conf_metadata = {
41 database_dir => 'STRING',
42 word_db => 'STRING',
43 start_url => 'STRING LIST',
44 limit_urls_to => 'STRING LIST',
45 exclude_urls => 'STRING LIST',
46 maintainer => 'STRING',
47 max_head_length => 'NUMBER',
48 max_doc_size => 'NUMBER',
49 no_excerpt_show_top => 'BOOLEAN',
50 search_algorithm => 'STRING LIST',
51 template_map => 'STRING LIST',
52 template_name => 'STRING',
53 next_page_text => 'STRING',
54 no_next_page_text => 'STRING',
55 prev_page_text => 'STRING',
56 no_prev_page_text => 'STRING',
57 page_number_text => 'STRING',
58 no_page_number_text => 'STRING',
59 configdig_notify => 'STRING LIST',
60 bad_extensions => 'STRING LIST',
61 };
62  
63 ##
64 #Constructor for Site object
65 ##
66 sub new {
67 my ($class, %attribs) = @_;
68 my $conf_path = $attribs{conf_path};
69  
70 #Autocreate allows us to create a conf that doesn't
71 # already exist.
72 if ($attribs{auto_create}) {
73 unless (-f $conf_path) {
74 #I avoid using "touch" on the file to ensure cross-platform
75 # compatibility, opting instead to just open it for writing.
76 open(TMP,">$conf_path");
77 close(TMP);
78 }
79 }
80  
81 my $st = stat($conf_path);
82 if (!$st) {
83 die "Couldn't access file $conf_path: $!\n(try using parameter value auto_create=>1 if you want this file to be created)\n";
84 }
85 my $mtime = $st->mtime;
86  
87 #If we weren't provided with a Config object ref, we create one
88 if($attribs{config} && ref($attribs{config}) ne "HtDig::Config") {
89 # print STDERR $attribs{config} . "\n";
90 $attribs{config} = new HtDig::Config($attribs{config});
91 }
92  
93 my $self = {
94 site_name => $attribs{site_name},
95 trace_lvl => $attribs{trace_lvl},
96 conf_path => $conf_path,
97 mtime => $mtime, #For avoiding concurrency problems
98 settings => {},
99 auto_create => $attribs{auto_create},
100 config => $attribs{config},
101 };
102 bless $self, $class;
103  
104  
105 $self->_get_settings($conf_path) or die "Couldn't interpret conf file $conf_path";
106 return $self;
107 }
108  
109 ##
110 #Retrieve or modify a configuration setting
111 ##
112 sub setting {
113 my ($self, $setting, $value) = @_;
114 #Check to see if they provided a new value for this setting
115 if (defined($value) and $value eq "") {
116 $self->_trace("Deleting value for $setting",1);
117 return delete($self->{settings}->{$setting});
118 }
119 elsif ($value) {
120 #Get the cleaned up value from validation routine
121 my $validated = $self->_validate_setting(type=>$conf_metadata->{$setting}, value=>$value);
122 #Validation returns undef on failure
123 if ($validated) {
124 #Set value and return it
125 $self->{settings}->{$setting} = $validated;
126 return $validated;
127 }
128 else {
129 #Return undef to indicate failure
130 $self->_trace("Value provided for $setting failed validation",1);
131 return undef;
132 }
133 }
134 #They didn't assign a value so just return the current value
135 else {
136 return $self->_validate_setting(type=>$conf_metadata->{$setting}, value=>$self->{settings}->{$setting});
137 }
138 }
139  
140 sub _get_settings {
141 my ($self, $conf_path) = @_;
142 my $settings = {};
143 open(CONF, $conf_path) or die "Couldn't open configuration file at $conf_path for reading: $!";
144 my $more = 1;
145 while($more) {
146 my ($pretext, $key, $value);
147 #We throw away the pretext when reading the conf file, because
148 # it's preserved when writing the file later
149 ($pretext, $key, $value, $more) = $self->_get_next_setting(*CONF);
150 $settings->{$key} = $self->_validate_setting(value=>$value, type=>$conf_metadata->{$key});
151 }
152 $self->{settings} = $settings;
153 return 1;
154 }
155  
156 sub _get_next_setting {
157 my ($self, $fh) = @_;
158 my ($line, $setting, $pretext, $key, $value);
159 my $setting_found = 0;
160 until ($setting_found) {
161 $line = <$fh>;
162 return($pretext, undef, undef, 0) if !$line;
163 if ($line =~ /^#/) {
164 $pretext .= $line;
165 # print "Pretext: $pretext\n";
166 next;
167 };
168 ($key, $value) = $line =~ /^\s*(\w+)\s*:\s*(.+)/;
169 if (!$value) {
170 $pretext .= $line;
171 next;
172 }
173 while ($value =~ /\\$/) {
174 defined(my $next_line = <$fh>) or last;
175 $value .= $next_line;
176 }
177 $setting_found = 1;
178 }
179 #print "Pretext: $pretext\n";
180
181 $value =~ s/\s+$//;
182 if ($key) {
183 $self->_trace("Value for $key extracted:\n$value\n",3);
184 #Use the internal validation routine to get the right data format
185 $value = $self->_validate_setting(value=>$value, type=>$conf_metadata->{$key});
186 if (!$value) {
187 $self->_trace("Using null value for setting $key. This could be the result of a badly formatted conf file being imported for the first time, or it might just be an empty setting.",1);
188 }
189 }
190 #print "returning $pretext\n";
191 return ($pretext, $key, $value, 1);
192 }
193  
194 ##
195 #Reload the configuration file
196 # Useful if someone's modified it since first load
197 ##
198 sub refresh {
199 my ($self) = @_;
200 $self->{settings} = _get_settings($self->{conf_path});
201 my $st = stat($self->{conf_path});
202 $self->{mtime} = $st->mtime;
203 }
204  
205 sub save {
206 my ($self, %attribs) = @_;
207 my ($save_path, $tmp_path);
208 #Allow saving to a different file name
209 $save_path = $attribs{save_to} ? $attribs{save_to} : $self->{conf_path};
210 $tmp_path = $save_path . ".bak";
211  
212 my $st = stat($self->{conf_path});
213 if ($st->mtime != $self->{mtime}) {
214 $self->_trace("Configuration was modified by another process, cannot update without refresh.");
215 $self->{errstr} = "Configuration was modified by another process, cannot update without refresh.";
216 return 0;
217 }
218
219 #Create a backup/temp copy of the file
220 if (!copy($self->{conf_path}, "$tmp_path")) {
221 $self->{errstr} = "Couldn't backup original configuration file to $tmp_path: $!";
222 return 0;
223 }
224 #Open the tmp_path file for reading
225 if (!open(ORIG,$tmp_path)) {
226 $self->{errstr} = "Couldn't open temp file at " . $self->{conf_path} . " for reading: $!";
227 return 0;
228 }
229 #Open the save_path file for saving
230 if (!open(NEW, ">$save_path")) {
231 $self->{errstr} = "Couldn't open configuration file at " . $save_path . " for writing: $!";
232 return 0;
233 }
234  
235 my (@saved);
236 my $more = 1;
237 # print NEW "#This file was generated by ConfigDig\n\n";
238 while ($more) {
239 my ($pretext, $key, $value);
240 ($pretext, $key, $value, $more) = $self->_get_next_setting(*ORIG);
241 print NEW $pretext unless $attribs{no_preserve};
242 if ($self->setting($key)) {
243 print NEW "$key: " . $self->_setting2string($self->setting($key)) . "\n";
244 push(@saved, $key);
245 }
246 else {
247 # print "$key: " . $self->setting2string($value;
248 }
249 #$self->_trace("Setting value for $key to:\n$string_setting",3);
250 }
251  
252 #Finish up with settings that didn't already exist in the file
253 for my $key (keys %{$self->{settings}}) {
254 if ($key and !grep(/^$key$/, @saved)) {
255 print NEW "$key: " . $self->_setting2string($self->setting($key)) . "\n";
256 }
257 }
258  
259 #Close the filehandles
260 close(NEW);
261 close(ORIG);
262  
263 #After the update, we renew the timestamp
264 $st = stat($self->{conf_path});
265 $self->{mtime} = $st->mtime;
266 return 1;
267 }
268  
269 ##
270 #Recursive convenience function for timed "at" jobs
271 ##
272 sub time_dig {
273 use HtDig::Config;
274 my ($htdig_conf, $site_name, $notify) = @_;
275 my $htdig = new HtDig::Config(conf_path=>$htdig_conf);
276 my $site = $htdig->get_site($site_name);
277 if(my $PID = $site->dig(notify=>$notify)) {
278 print "Site dig successfully started. PID: $PID\n";
279 }
280 else {
281 print "Couldn't initiate dig: " . $site->errstr;
282 }
283 }
284  
285 ##
286 #Crawl the site(s)
287 ##
288 sub dig {
289 #Watch out:
290 # this func returns the PID of the rundig process on success
291 # unless you've requested a timed dig, where you'll get the "at"
292 # queue job id.
293 # Any case returns undef on internal error
294  
295 my ($self, %attribs) = @_;
296  
297 my $options .= " -c " . $self->{conf_path};
298 #Currently the only options supported are "verbosity" and "notify"
299 #Others should be added soon
300 $options .= $attribs{verbose} ? " -" . ("v" x $attribs{verbose}) : "";
301  
302  
303 #Figure out all the necessary details for running the dig
304 my $site_name = $self->{site_name};
305 my $conf_path = $self->{conf_path};
306 my $index_type = $attribs{type};
307 my $notify = $attribs{notify};
308 my $log_file = $site_name . "_dig_" . ParseDate("now");
309 $log_file =~ s/\:/\_/g;
310 my $htdig_path = $self->{config}->{htdig_base_path} . "/bin/rundig";
311 my $log_file = File::Spec->catfile($self->{config}->{configdig_log_path},$log_file);
312  
313 my $system_cmd = "$htdig_path -c $conf_path $options >> $log_file";
314  
315  
316 #If immediate dig was requested, we fork, if not, we schedule for
317 # later
318  
319 if ($attribs{at}) {
320 my ($job_id, $result);
321  
322 my $trace_log = File::Spec->catfile($self->{config}->{configdig_log_path},"configdig_at_log");
323 open(LOG, ">>$trace_log");
324 my $at_time = $attribs{"at"};
325  
326 #We must schedule an "at" job
327 #For simplicity, I assume that if you aren't mswin32
328 # then you have Unix "at". I know this will cause problems,
329 # but this seems the best way to start off, since I'm not
330 # familiar with scheduling programs on systems other than Linux and NT
331 if ($^O =~ /mswin32/i) {
332 #Use NT's "at" command, win95/98 can't do this
333 #but, then again, I don't think they can run ht://Dig, anyway
334 $self->{errstr} = "Scheduled digs are not yet implemented on $^O operating system\n";
335 return undef;
336 }
337 else {
338 #I'm not sure how better to pick up custom library directories, so
339 # I'm only handling one at this time. If you haven't specified a custom
340 # library, then this should be harmless...
341 #This seems easier than getting the entire @INC array and rebuilding it
342 # explicitly with -I parameters!
343 my $custom_lib = $INC[0];
344 my $at_cmd = qq|perl -I$custom_lib -MHtDig::Site -e "time_dig('| . $self->{config}->{path} . qq|','| . $self->{site_name} . qq|','$notify')"|;
345 $result = `$at_cmd | at $at_time 2>&1`;
346 $self->_trace("Piping command to at:\n$at_cmd | at $at_time",2);
347 ($job_id) = $result =~ /job (\d+)/;
348 $self->_trace("Successfully piped system command to 'at':\nResult: $result");
349 }
350  
351 if ($job_id) {
352 #Success! Return job id we parsed from return value
353 return $job_id;
354 }
355 else {
356 #Prettify the error returned from "at" and indicate error condition
357 $result =~ s/\n/. /g;
358 $self->{errstr} = "Error scheduling \"at\" job: $result";
359 return undef;
360 }
361 }
362 else {
363 #Fork before we start the process so we are "non-blocking"
364 if (my $kid = fork) {
365 #I'm the parent
366 return $kid;
367 }
368 elsif (!defined $kid) {
369 $self->{errstr} = "Couldn't spawn dig process: $!";
370 return undef;
371 }
372  
373 #From this point onward, we are the forked "child" and are not
374 # part of the CGI process that spawned us.
375  
376 #These are filehandles for the CGI. We don't want to tamper with
377 # them accidentally, so we close them.
378 #(Comment these out if you're doing debugging, and your error log should
379 # display the trace messages)
380 close STDERR;
381 close STDIN;
382 close STDOUT;
383  
384 my $retval = system("$system_cmd");
385  
386 #If notify flag is set, we attempt to notify the admins
387 if ($attribs{notify}) {
388 $self->_notify_admins("Site indexing run for ht://Dig configuration named " . $self->{site_name} . " has completed.\nRundig exit code: $retval\nLog saved at $log_file\n\nThis message was autogenerated by the ConfigDig configuration system.");
389 }
390  
391  
392 $self->_trace("htdig returned exit code $retval when running dig\n",1);
393 #We exit because we are the child process
394 exit;
395 }
396 }
397  
398 ##
399 #Generate fuzzy indexes
400 ##
401 sub generate_fuzzy_index {
402 #Watch out:
403 # this func returns the PID of the htfuzzy process on success
404 # or undef on internal error
405 #
406  
407 my ($self, %attribs) = @_;
408 my $idx_type = $attribs{type};
409  
410 #Check to make sure we have a valid Config reference
411 if (!$self->{config}) {
412 $self->{errstr} = "To generate fuzzy indexes, you must access the Site object via the Config object's get_site() method.";
413 return undef;
414 }
415  
416 #Generate the verbosity flag, if requested
417 my $options .= $attribs{verbose} ? " -" . ("v" x $attribs{verbose}) : "";
418  
419 #Check the accepted types to make sure the requested type is valid
420 if (!grep(/^$idx_type$/, @accepted_fuzzy_types)) {
421 $self->_trace("Unsupported fuzzy index of type $idx_type was requested. Only " . join(", ", @accepted_fuzzy_types) . " are supported.",1);
422 $self->{errstr} = "Unsupported fuzzy index of type $idx_type was requested. Only " . join(", ", @accepted_fuzzy_types) . " are supported.";
423 return undef;
424 }
425
426 #Fork before we start the process so we are "non-blocking"
427 if (my $kid = fork) {
428 #I'm the parent
429 return $kid;
430 }
431 elsif (!defined $kid) {
432 $self->{errstr} = "Couldn't spawn index creation process: $!";
433 return undef;
434 }
435  
436 #Generate the index using htfuzzy at the appropriate location
437 close STDERR;
438 close STDIN;
439 close STDOUT;
440 my $site_name = $self->{site_name};
441 my $conf_path = $self->{conf_path};
442 my $index_type = $attribs{type};
443 my $log_file = $site_name . "_" . $index_type . "_" . ParseDate("now");
444 $log_file =~ s/\:/\_/g;
445 my $htfuzzy_path = $self->{config}->{htdig_base_path} . "/bin/htfuzzy";
446 my $log_file = File::Spec->catfile($self->{config}->{configdig_log_path},$log_file);
447 my $system_cmd = "$htfuzzy_path -c $conf_path $options $index_type >> $log_file";
448  
449 # $self->_trace($system_cmd,1);
450 my $retval = system("$system_cmd");
451 $self->_trace("htfuzzy returned exit code $retval when generating $idx_type\n",1);
452 #We exit because we are the child process
453 exit;
454 }
455  
456 ##
457 #Merge database functionality
458 ##
459 sub merge {
460 #Watch out:
461 # this func returns the PID of the htmerge process on success
462 # or undef on internal error
463 #
464  
465 my ($self, %attribs) = @_;
466  
467 #Check to make sure we have a valid Config reference
468 if (!$self->{config}) {
469 $self->{errstr} = "To generate fuzzy indexes, you must access the Site object via the Config object's get_site() method.";
470 return undef;
471 }
472  
473 #Generate the verbosity flag, if requested
474 my $options .= $attribs{verbose} ? " -" . ("v" x $attribs{verbose}) : "";
475  
476 #Check to see if we're merging another database into this one
477 if ($attribs{merge_site}) {
478 my $msite = $self->{config}->get_site($attribs{merge_site});
479 $options .= " -m " . $msite->{conf_path};
480 }
481
482 #Check to see if they requested alternate work files, no words db merge, or
483 # no document db merge
484 $options .= " -a" if $attribs{work_files};
485 $options .= " -w" if $attribs{not_words};
486 $options .= " -d" if $attribs{not_documents};
487
488 #Fork before we start the process so we are "non-blocking"
489 if (my $kid = fork) {
490 #I'm the parent
491 return $kid;
492 }
493 elsif (!defined $kid) {
494 $self->{errstr} = "Couldn't spawn merge process: $!";
495 return undef;
496 }
497  
498 #Merge using htmerge at the appropriate location
499 #Close the filehandles the parent might be using
500 close STDERR;
501 close STDIN;
502 close STDOUT;
503  
504 #Set up the system call
505 my $site_name = $self->{site_name};
506 my $conf_path = $self->{conf_path};
507 my $index_type = $attribs{type};
508 my $log_file = $site_name . "_merge_" . ParseDate("now");
509 $log_file =~ s/\:/\_/g;
510 my $htmerge_path = $self->{config}->{htdig_base_path} . "/bin/htmerge";
511 my $log_file = File::Spec->catfile($self->{config}->{configdig_log_path},$log_file);
512 my $system_cmd = "$htmerge_path -c $conf_path $options >> $log_file";
513  
514 $self->_trace($system_cmd,1);
515 my $retval = system("$system_cmd");
516 $self->_trace("htmerge returned exit code $retval when performing merge\n",1);
517  
518 #Exit because we are the child process
519 exit;
520 }
521  
522 ##
523 #Error handling, debugging, tracing
524 ##
525 sub errstr {
526 my ($self) = @_;
527 return $self->{errstr};
528 }
529  
530 sub _notify_admins {
531 my ($self, $msg) = @_;
532  
533 #Check to see if extra notification recipients have been defined
534 my @admins = @{$self->setting("configdig_notify")};
535 push(@admins, $self->setting("maintainer"));
536 if (!@admins) {
537 $self->{errstr} = "Notification requested, but no administrators or maintainer configured.";
538 return 0;
539 }
540  
541 my $mailhost = $self->{config}->{mailhost} || 'localhost';
542  
543 print "Mailhost is $mailhost\n";
544 print "Admins are " . join(", ", @admins) . "\n";
545  
546  
547 my $smtp = new Net::SMTP($mailhost, Debug=>0);
548 $smtp->mail("ConfigDig\@localhost");
549 $smtp->to(@admins);
550 $smtp->data();
551 $smtp->datasend($msg);
552 $smtp->datasend("\n");
553 $smtp->dataend();
554 $smtp->quit;
555  
556 return 1;
557 }
558  
559 sub _trace {
560 my($self, $msg, $trace_lvl) = @_;
561 return if !$self->{trace_lvl};
562 if (!$trace_lvl or ($self->{trace_lvl} ge $trace_lvl)) {
563 if (ref($self->{trace_cb}) eq "CODE") {
564 &{$self->{trace_ob}}($msg);
565 }
566 elsif ($self->{trace_cb}) {
567 print {$self->{trace_cb}} "TRACE: " . $msg . "\n";
568 }
569 else {
570 print STDERR "TRACE: " . $msg . "\n";
571 }
572 }
573 }
574  
575 sub datatypes {return $conf_metadata;}
576  
577 sub _setting2string {
578 my ($self, $setting) = @_;
579 my $reftype = ref($setting);
580 if ($reftype eq "ARRAY") {
581 return join(" ", @{$setting});
582 }
583 else {
584 return $setting;
585 }
586 }
587  
588 sub _validate_setting {
589 #Returns cleaned-up setting value on success, and undef on failure
590 my ($self, %attribs) = @_;
591 if ($attribs{type} eq 'STRING' or !$attribs{type}) {
592 $attribs{value} =~ s/[^\\]\n/\\\n/g; #end lines with \ to indicate continuation
593 return $attribs{value};
594 }
595 elsif ($attribs{type} eq 'STRING LIST') {
596 if (ref($attribs{value}) eq 'ARRAY') {
597 return $attribs{value};
598 }
599 elsif (my @vals = split(/\s/, $attribs{value})) {
600 return \@vals;
601 }
602 else {
603 return undef;
604 }
605 }
606 elsif ($attribs{type} eq 'NUMBER') {
607 if ($attribs{value} =~ /^\d*$/) {
608 return $attribs{value};
609 }
610 else {
611 return undef;
612 }
613 }
614 elsif ($attribs{type} eq 'BOOLEAN') {
615 if ($attribs{value} =~ /0|false|f|no/i) {
616 return 0;
617 }
618 elsif ($attribs{value} =~ /1|true|t|yes/i) {
619 return 1;
620 }
621 else {
622 return undef;
623 }
624 }
625 else {
626 $self->{errstr} = "Datatype $attribs{type} cannot be validated.";
627 $self->_trace("Datatype $attribs{type} cannot be validated.");
628 return undef;
629 }
630 }
631  
632  
633 # Autoload methods go after =cut, and are processed by the autosplit program.
634  
635 1;
636 __END__