| 1 | #!perl | ||
| 2 | |||
| 3 | =pod | ||
| 4 | |||
| 5 | =head1 NAME | ||
| 6 | |||
| 7 | WWW::ImageSpool - Cache images of interest from the web. | ||
| 8 | |||
| 9 | =head1 SYNOPSIS | ||
| 10 | |||
| 11 | use WWW::ImageSpool; | ||
| 12 | |||
| 13 | mkdir("/var/tmp/imagespool", 0700); | ||
| 14 | |||
| 15 | my $spool = WWW::ImageSpool->new | ||
| 16 | ( | ||
| 17 | limit => 3, | ||
| 18 | searchlimit => 10, | ||
| 19 | max => 5 * 1048576, | ||
| 20 | dictionary => "sushi.txt", | ||
| 21 | verbose => 1, | ||
| 22 | dir => "/var/tmp/imagespool" | ||
| 23 | ); | ||
| 24 | |||
| 25 | $spool->run(); | ||
| 26 | while($spool->uptime < 86400); | ||
| 27 | |||
| 28 | =head1 DESCRIPTION | ||
| 29 | |||
| 30 | When A WWW::ImageSpool object's run() method is called, it randomly picks keywords | ||
| 31 | out of a chosen dictionary file and attempts to download images off of the | ||
| 32 | internet by doing searches on these keywords. (Currently only a Google Image | ||
| 33 | Search is done, via Guillaume Rousse's WWW::Google::Images module, but the | ||
| 34 | internals have been set up to make it easy to hook into other engines in the future.) | ||
| 35 | Images are stored in the specified directory. If the directory grows beyond | ||
| 36 | the maximum size, the oldest files in the directory are deleted. | ||
| 37 | |||
| 38 | The intended purpose behind this module is to supply images on demand for any | ||
| 39 | piece of software that wants abstract images, such as screensavers or | ||
| 40 | webpage generators or voice synthesizers (wouldn't it be cool if a voice | ||
| 41 | synthesizer extracted all the popular nouns out of a book and scrolled by | ||
| 42 | pertanent images as it read to you?) | ||
| 43 | |||
| 44 | =head1 Constructor | ||
| 45 | |||
| 46 | =head2 new(I<%args>) | ||
| 47 | |||
| 48 | Creates and returns a new C<WWW::ImageSpool> object. | ||
| 49 | |||
| 50 | Required parameters: | ||
| 51 | |||
| 52 | =over | ||
| 53 | |||
| 54 | =item dir => I<$dir> | ||
| 55 | |||
| 56 | Directory to hold the image files in. C<WWW::ImageSpool> will | ||
| 57 | delete files out of this directory when it reaches the | ||
| 58 | maximum size, so there shouldn't be anything in there that you | ||
| 59 | want to keep. | ||
| 60 | |||
| 61 | =back | ||
| 62 | |||
| 63 | Optional parameters: | ||
| 64 | |||
| 65 | =over | ||
| 66 | |||
| 67 | =item limit => I<$limit> | ||
| 68 | |||
| 69 | Maximum number of images to fetch from any one keyword search. Defaults to 3. | ||
| 70 | |||
| 71 | =item searchlimit => I<$searchlimit> | ||
| 72 | |||
| 73 | Maximum number of search results to ask the search engine for. | ||
| 74 | I<limit> results will be randomly picked out of the list that the search engine | ||
| 75 | returns. Default is search-engine specific (50 for Google). Most search engines will | ||
| 76 | return the results in the same order each time they are called with | ||
| 77 | the same keywords, so if you are using a small dictionary file it is generally | ||
| 78 | a good idea to make this a lot higher than I<limit>. | ||
| 79 | |||
| 80 | =item consume => 0 | 1 | ||
| 81 | |||
| 82 | WWW::ImageSpool re-loads the dictionary file whenever it is modified, or | ||
| 83 | whenever it runs out of words. With I<consume> set to 0, WWW::ImageSpool | ||
| 84 | will never run out of words because it can re-use them as much as they | ||
| 85 | want. With I<consume> set to 1, WWW::ImageSpool deletes each word | ||
| 86 | from it's internal list as it uses it, ensuring that every single word | ||
| 87 | in the dictionary must be used once before any word may be used twice. | ||
| 88 | |||
| 89 | I<consume> is set to 1 by default. | ||
| 90 | |||
| 91 | =item retry => I<$retry> | ||
| 92 | |||
| 93 | How many times to retry image-searching or fetching operations if they | ||
| 94 | fail. | ||
| 95 | |||
| 96 | The actual maximum number of retries is (I<$retry> * I<$retry>); | ||
| 97 | WWW::ImageSpool will try up to I<$retry> times to find a word with | ||
| 98 | good search results, then with that word, will try up to I<$retry> | ||
| 99 | times to get images from it, stopping after at least one image | ||
| 100 | is successfully downloaded (or the retry is exhausted.) | ||
| 101 | |||
| 102 | I<retry> is set to 5 by default. | ||
| 103 | |||
| 104 | =item minx=> I<$minx>, miny => I<$miny> | ||
| 105 | |||
| 106 | Minimum X / Y resolution of images to return. Smaller images are discarded. | ||
| 107 | |||
| 108 | By default, I<minx> is set to 160, and I<miny> is set to 120. | ||
| 109 | |||
| 110 | =item max => I<$bytes> | ||
| 111 | |||
| 112 | Maximum size of the spool directory, in bytes. If the total size of all files in | ||
| 113 | that directory ever goes over this size, the oldest file in the directory is deleted | ||
| 114 | to make more room. | ||
| 115 | |||
| 116 | =item dictionary => I<$file> | ||
| 117 | |||
| 118 | Path to the dictionary file to use. Defaults to "/usr/share/dict/words". | ||
| 119 | |||
| 120 | =item verbose => I<0 - 4> | ||
| 121 | |||
| 122 | Level of verbosity. Defaults to 0, which prints nothing. | ||
| 123 | 1 prints a logfile-like status line for each iteration of run(). | ||
| 124 | 2 prints each word that is picked, and advises if C<WWW::ImageSpool> picked a file that already exists in the spool. | ||
| 125 | 3-4 print more verbose debugging information. | ||
| 126 | |||
| 127 | =back | ||
| 128 | |||
| 129 | Paramaters for making C<WWW::ImageSpool> re-entrant: | ||
| 130 | |||
| 131 | These parameters are only really useful if you are creating and destroying | ||
| 132 | C<WWW::ImageSpool> objects throughout the lifespan of an application, but | ||
| 133 | want your statistics to remain constant throughout: | ||
| 134 | |||
| 135 | =over | ||
| 136 | |||
| 137 | =item n => I<$n> | ||
| 138 | |||
| 139 | How many iterations of C<run()> the application has done so far. | ||
| 140 | |||
| 141 | =item s => I<$s> | ||
| 142 | |||
| 143 | UNIX timestamp of when the application did it's first call to C<run()> | ||
| 144 | on a C<WWW::ImageSpool> object. | ||
| 145 | |||
| 146 | =item l => I<$l> | ||
| 147 | |||
| 148 | UNIX timestamp of when the application last did a call to C<run()> | ||
| 149 | on a C<WWW::ImageSpool> object. | ||
| 150 | |||
| 151 | =item got => I<$got> | ||
| 152 | |||
| 153 | How many images have been downloaded and stored over the life of the | ||
| 154 | application (including ones that have been deleted). | ||
| 155 | |||
| 156 | =back | ||
| 157 | |||
| 158 | =head1 Methods | ||
| 159 | |||
| 160 | =head2 run() | ||
| 161 | |||
| 162 | Pick a new keyword and attemt to download up to I<limit> images | ||
| 163 | from an image search. | ||
| 164 | |||
| 165 | Returns the actual number of images downloaded and stored. | ||
| 166 | |||
| 167 | =head2 s() | ||
| 168 | |||
| 169 | Returns the UNIX timestamp of the object's first operation. | ||
| 170 | |||
| 171 | =head2 l() | ||
| 172 | |||
| 173 | Returns the UNIX timestamp of the object's last operation. | ||
| 174 | |||
| 175 | =head2 n() | ||
| 176 | |||
| 177 | Returns how many times C<run()> has been called on this object. | ||
| 178 | |||
| 179 | =head2 uptime() | ||
| 180 | |||
| 181 | Returns the number of seconds between the object's first operation and it's last operation. | ||
| 182 | |||
| 183 | =head2 lag() | ||
| 184 | |||
| 185 | Returns the number of seconds between the object's last operation and the current time. | ||
| 186 | |||
| 187 | =head2 got() | ||
| 188 | |||
| 189 | Returns the total number of images that have been downloaded and stored by this object, | ||
| 190 | including images that have been deleted. | ||
| 191 | |||
| 192 | =head1 BUGS | ||
| 193 | |||
| 194 | If the dictionary file suddenly disappears, C<WWW::ImageSpool> does not act very | ||
| 195 | graceful. | ||
| 196 | |||
| 197 | =head1 TODO | ||
| 198 | |||
| 199 | There should be size limitations on individual files with a HEAD check before | ||
| 200 | they are actually downloaded. | ||
| 201 | |||
| 202 | Underlying modules (C<WWW::ImageSpool::Source::Google>, C<WWW::ImageSpool::Dictionary>, | ||
| 203 | etc need to be documented. | ||
| 204 | |||
| 205 | Support for multiple "Source" and "Dictionary" objects in one "ImageSpool" object. | ||
| 206 | |||
| 207 | Per-C<run()> control over the search configuration. | ||
| 208 | |||
| 209 | =head1 NOTE | ||
| 210 | |||
| 211 | This module may violate the terms of service of some search engines or content | ||
| 212 | providers. Use at your own risk. | ||
| 213 | |||
| 214 | =head1 VERSION | ||
| 215 | |||
| 216 | 0.01 | ||
| 217 | |||
| 218 | =head1 LICENSE | ||
| 219 | |||
| 220 | Copyright 2004, Tyler "Crackerjack" MacDonald <tyler@yi.org> | ||
| 221 | This is free software; you may redistribute it under the same terms as perl itself. | ||
| 222 | |||
| 223 | =cut | ||
| 224 | |||
| 225 | package WWW::ImageSpool; | ||
| 226 | use strict; | ||
| 227 | use warnings; | ||
| 228 | |||
| 229 | use POSIX qw(strftime); | ||
| 230 | use WWW::ImageSpool::Source::Google; | ||
| 231 | use WWW::ImageSpool::Directory; | ||
| 232 | use WWW::ImageSpool::Dictionary; | ||
| 233 | |||
| 234 | our $VERSION = "0.01"; | ||
| 235 | |||
| 236 | return 1; | ||
| 237 | |||
| 238 | sub new | ||
| 239 | { | ||
| 240 | my $class = shift; | ||
| 241 | my (%args) = (@_); | ||
| 242 | my $self = bless { args => \%args }, $class; | ||
| 243 | my (%dict_args, %dir_args, %search_args) = (); | ||
| 244 | |||
| 245 | if(!$args{dir}) | ||
| 246 | { | ||
| 247 | warn "WWW::ImageSpool->new(): \"dir\" is required.\n"; | ||
| 248 | return; | ||
| 249 | } | ||
| 250 | |||
| 251 | $args{limit} = 3 | ||
| 252 | if(!defined($args{limit})); | ||
| 253 | |||
| 254 | $args{consume} = 1 | ||
| 255 | if(!defined($args{consume})); | ||
| 256 | |||
| 257 | $args{retry} = 5 | ||
| 258 | if(!defined($args{retry})); | ||
| 259 | |||
| 260 | $dir_args{dir} = $args{dir}; | ||
| 261 | |||
| 262 | $dir_args{minx} = $args{minx} | ||
| 263 | if($args{minx}); | ||
| 264 | |||
| 265 | $dir_args{miny} = $args{miny} | ||
| 266 | if($args{miny}); | ||
| 267 | |||
| 268 | $search_args{limit} = $args{limit} | ||
| 269 | if($args{limit}); | ||
| 270 | |||
| 271 | $search_args{searchlimit} = $args{searchlimit} | ||
| 272 | if($args{searchlimit}); | ||
| 273 | |||
| 274 | $dict_args{file} = $args{dictionary} | ||
| 275 | if($args{dictionary}); | ||
| 276 | |||
| 277 | $dir_args{max} = $args{max} | ||
| 278 | if($args{max}); | ||
| 279 | |||
| 280 | $args{verbose} = 0 | ||
| 281 | if(!defined($args{verbose})); | ||
| 282 | |||
| 283 | $dir_args{verbose} = $dict_args{verbose} = $search_args{verbose} = $args{verbose}; | ||
| 284 | |||
| 285 | return | ||
| 286 | if(!($self->{dict} = WWW::ImageSpool::Dictionary->new(%dict_args))); | ||
| 287 | |||
| 288 | return | ||
| 289 | if(!($self->{dir} = WWW::ImageSpool::Directory->new(%dir_args))); | ||
| 290 | |||
| 291 | return | ||
| 292 | if(!($self->{search} = WWW::ImageSpool::Source::Google->new(%search_args))); | ||
| 293 | |||
| 294 | return $self; | ||
| 295 | } | ||
| 296 | |||
| 297 | sub word | ||
| 298 | { | ||
| 299 | shift->{dict}->word(@_); | ||
| 300 | } | ||
| 301 | |||
| 302 | sub search | ||
| 303 | { | ||
| 304 | shift->{search}->search(@_); | ||
| 305 | } | ||
| 306 | |||
| 307 | sub fetch | ||
| 308 | { | ||
| 309 | shift->{dir}->fetch(@_); | ||
| 310 | } | ||
| 311 | |||
| 312 | sub run | ||
| 313 | { | ||
| 314 | my $self = shift; | ||
| 315 | my $iretry = 0; | ||
| 316 | my $oretry = 0; | ||
| 317 | my @images; | ||
| 318 | my $rv; | ||
| 319 | |||
| 320 | if(!$self->{n}) | ||
| 321 | { | ||
| 322 | $self->{s} = time(); | ||
| 323 | |||
| 324 | if($self->{args}->{verbose}) | ||
| 325 | { | ||
| 326 | printf | ||
| 327 | ( | ||
| 328 | "[%s] PID %d % 3dstor % 3.2fMb\n", | ||
| 329 | strftime("%y-%m-%d %H:%M:%S", localtime()), $$, scalar(keys(%{$self->{dir}->{files}})), $self->{dir}->{total_size} / 1048576 | ||
| 330 | ); | ||
| 331 | } | ||
| 332 | } | ||
| 333 | |||
| 334 | while(($oretry < $self->{args}->{retry}) && (!$rv)) | ||
| 335 | { | ||
| 336 | $self->{n}++; | ||
| 337 | my $word = $self->word($self->{args}->{consume}); | ||
| 338 | while(($oretry < $self->{args}->{retry}) && (!$rv)) | ||
| 339 | { | ||
| 340 | @images = ($self->search($word)); | ||
| 341 | $rv = $self->fetch(@images); | ||
| 342 | $iretry ++; | ||
| 343 | } | ||
| 344 | $oretry++; | ||
| 345 | } | ||
| 346 | |||
| 347 | $self->{l} = time(); | ||
| 348 | |||
| 349 | if($rv) | ||
| 350 | { | ||
| 351 | $self->{got}+=$rv; | ||
| 352 | if($self->{args}->{verbose}) | ||
| 353 | { | ||
| 354 | printf | ||
| 355 | ( | ||
| 356 | "[%s] % 4d: % 3dnow/% 3dstor/% 3dt % 3.2fMb % 2.2ffpm %dw %d*%dtry\n", | ||
| 357 | strftime("%y-%m-%d %H:%M:%S", localtime()), $self->{n}, $rv, scalar(keys(%{$self->{dir}->{files}})), $self->{got}, | ||
| 358 | $self->{dir}->{total_size} / 1048576, $self->{got} / (($self->{l} - $self->{s}) / 60), scalar(@{$self->{dict}->{words}}), $iretry, $oretry | ||
| 359 | ); | ||
| 360 | } | ||
| 361 | return $rv; | ||
| 362 | } | ||
| 363 | else | ||
| 364 | { | ||
| 365 | return; | ||
| 366 | } | ||
| 367 | } | ||
| 368 | |||
| 369 | sub s | ||
| 370 | { | ||
| 371 | return shift->{s}; | ||
| 372 | } | ||
| 373 | |||
| 374 | sub l | ||
| 375 | { | ||
| 376 | return shift->{l}; | ||
| 377 | } | ||
| 378 | |||
| 379 | sub n | ||
| 380 | { | ||
| 381 | return shift->{n}; | ||
| 382 | } | ||
| 383 | |||
| 384 | sub got | ||
| 385 | { | ||
| 386 | return shift->{got}; | ||
| 387 | } | ||
| 388 | |||
| 389 | sub uptime | ||
| 390 | { | ||
| 391 | return $_[0]->{l} - $_[0]->{s}; | ||
| 392 | } | ||
| 393 | |||
| 394 | sub lag | ||
| 395 | { | ||
| 396 | return time() - $_[0]->{l}; | ||
| 397 | } |