| 1 | # ===================================================================== | ||
| 2 | # Copyright (c) 2002,2003 IBM Corporation | ||
| 3 | # All rights reserved. This program and the accompanying materials | ||
| 4 | # are made available under the terms of the Common Public License v1.0 | ||
| 5 | # which accompanies this distribution, and is available at | ||
| 6 | # http://www.opensource.org/licenses/cpl.php | ||
| 7 | # | ||
| 8 | # ===================================================================== | ||
| 9 | |||
| 10 | package LS::Cache::Manager; | ||
| 11 | |||
| 12 | use strict; | ||
| 13 | use warnings; | ||
| 14 | |||
| 15 | use vars qw( $WSDL_EXT $DATA_EXT $SERVICE_FILENAME $EXPIRATION_FILENAME $METADATA_EXT ); | ||
| 16 | |||
| 17 | |||
| 18 | use IO::File; | ||
| 19 | use File::Copy; | ||
| 20 | use Fcntl ':flock'; | ||
| 21 | use Time::HiRes qw( gettimeofday ); | ||
| 22 | |||
| 23 | use Carp qw(:DEFAULT); | ||
| 24 | |||
| 25 | use LS::Cache::Config; | ||
| 26 | use LS::Cache::Logger; | ||
| 27 | |||
| 28 | |||
| 29 | # | ||
| 30 | # Cache constants | ||
| 31 | # | ||
| 32 | $WSDL_EXT = 'wsdl'; | ||
| 33 | $DATA_EXT = 'tmp'; | ||
| 34 | $METADATA_EXT = 'rdf'; | ||
| 35 | $SERVICE_FILENAME = 'authority.wsdl'; | ||
| 36 | |||
| 37 | |||
| 38 | # | ||
| 39 | # | ||
| 40 | # | ||
| 41 | $EXPIRATION_FILENAME = 'EXPIRES'; | ||
| 42 | |||
| 43 | |||
| 44 | |||
| 45 | |||
| 46 | # | ||
| 47 | # new( %options ) - | ||
| 48 | # | ||
| 49 | sub new { | ||
| 50 | |||
| 51 | my ($self, %options) = @_; | ||
| 52 | |||
| 53 | unless (ref $self) { | ||
| 54 | |||
| 55 | $self = bless { | ||
| 56 | |||
| 57 | _config=> LS::Cache::Config->new(), | ||
| 58 | _logger=> undef, | ||
| 59 | }, $self; | ||
| 60 | } | ||
| 61 | |||
| 62 | $self->{'_logger'} = LS::Cache::Logger->new('log-file' => $self->{'_config'}->get_config_entry('log-file')); | ||
| 63 | |||
| 64 | $self->enable_log() if($self->{'_config'}->get_config_entry('logging-on') eq 'true'); | ||
| 65 | |||
| 66 | return $self; | ||
| 67 | } | ||
| 68 | |||
| 69 | |||
| 70 | # | ||
| 71 | # Accessor functions | ||
| 72 | # | ||
| 73 | |||
| 74 | |||
| 75 | # | ||
| 76 | # logger( ) - | ||
| 77 | # | ||
| 78 | sub logger { | ||
| 79 | |||
| 80 | my $self = shift; | ||
| 81 | return $self->{'_logger'}; | ||
| 82 | } | ||
| 83 | |||
| 84 | |||
| 85 | # | ||
| 86 | # config( ) - | ||
| 87 | # | ||
| 88 | sub config { | ||
| 89 | |||
| 90 | my $self = shift; | ||
| 91 | return $self->{'_config'}; | ||
| 92 | } | ||
| 93 | |||
| 94 | |||
| 95 | # | ||
| 96 | # enable_log( ) - | ||
| 97 | # | ||
| 98 | sub enable_log { | ||
| 99 | |||
| 100 | my $self = shift; | ||
| 101 | $self->{_config}->{'logging-on'} = 'true'; | ||
| 102 | } | ||
| 103 | |||
| 104 | |||
| 105 | # | ||
| 106 | # is_logging( ) - | ||
| 107 | # | ||
| 108 | sub is_logging { | ||
| 109 | |||
| 110 | my $self = shift; | ||
| 111 | return $self->{_config}->get_config_entry('logging-on') eq 'true'; | ||
| 112 | } | ||
| 113 | |||
| 114 | |||
| 115 | # | ||
| 116 | # get_time( ) - | ||
| 117 | # | ||
| 118 | sub get_time { | ||
| 119 | |||
| 120 | my $self = shift; | ||
| 121 | |||
| 122 | my ($seconds, $microsecs) = Time::HiRes::gettimeofday(); | ||
| 123 | return $seconds * 1_000 + $microsecs / 1000 ; | ||
| 124 | } | ||
| 125 | |||
| 126 | |||
| 127 | |||
| 128 | # | ||
| 129 | # Cache functions | ||
| 130 | # | ||
| 131 | |||
| 132 | |||
| 133 | # | ||
| 134 | # maintain_cache( timestampt, $currentDirectory, $cacheConfig ) - | ||
| 135 | # | ||
| 136 | sub maintain_cache { | ||
| 137 | |||
| 138 | my ($self, $timestamp, $current_dir, $cache_config) = @_; | ||
| 139 | |||
| 140 | # Useful for when we are called at the top level | ||
| 141 | $current_dir = $self->{_config}->get_config_entry('cache-root-dir') if(!$current_dir); | ||
| 142 | |||
| 143 | # Build a configuration reference if one was not specified | ||
| 144 | # (so that we don't modify the root configuration while traversing the cache) | ||
| 145 | if(!$cache_config) { | ||
| 146 | |||
| 147 | my %hash; | ||
| 148 | |||
| 149 | foreach(keys( %{$self->{_config}}) ) { | ||
| 150 | |||
| 151 | $hash{$_} = $self->{_config}->get_config_entry($_); | ||
| 152 | } | ||
| 153 | |||
| 154 | $cache_config = \%hash; | ||
| 155 | } | ||
| 156 | |||
| 157 | $self->{_logger}->log_message("Begin cache maintenance on: $current_dir") if($self->is_logging); | ||
| 158 | |||
| 159 | if(!opendir(CACHE_DIR, $current_dir)) { | ||
| 160 | |||
| 161 | $self->{_logger}->log_message(" Unable to open directory: $current_dir") if($self->is_logging); | ||
| 162 | |||
| 163 | return undef; | ||
| 164 | } | ||
| 165 | |||
| 166 | my @files = readdir(CACHE_DIR); | ||
| 167 | |||
| 168 | closedir(CACHE_DIR); | ||
| 169 | |||
| 170 | |||
| 171 | # Load the more specific configuration if it exists | ||
| 172 | if ( -e "$current_dir/$LS::Cache::Config::CACHE_FILENAME" ) { | ||
| 173 | |||
| 174 | $self->{_logger}->log_message(" Reading local cache settings from:") if($self->is_logging); | ||
| 175 | $self->{_logger}->log_message(" $current_dir/$LS::Cache::Config::CACHE_FILENAME") if($self->is_logging); | ||
| 176 | |||
| 177 | $cache_config = $self->{_config}->read_cache_settings($current_dir, $cache_config); | ||
| 178 | } | ||
| 179 | |||
| 180 | $self->{_logger}->log_message(" Found " . ( $#files + 1 ) . " files.") if($self->is_logging); | ||
| 181 | |||
| 182 | # Remove cache entries that have expired based on their timestamp | ||
| 183 | my @remaining_files = (); | ||
| 184 | my $total_size = 0; | ||
| 185 | |||
| 186 | foreach my $f (@files) { | ||
| 187 | |||
| 188 | next if($f eq '.' || $f eq '..'); | ||
| 189 | next if($f eq $LS::Cache::Config::CACHE_FILENAME || | ||
| 190 | $f eq $EXPIRATION_FILENAME || | ||
| 191 | $f eq $LS::Cache::Config::LOG_FILENAME); | ||
| 192 | |||
| 193 | if( -d "$current_dir/$f") { | ||
| 194 | |||
| 195 | $self->{_logger}->log_message(" New directory found, recursing: $current_dir/$f") if($self->is_logging); | ||
| 196 | |||
| 197 | push(@remaining_files, $self->maintain_cache($timestamp, "$current_dir/$f", $cache_config)); | ||
| 198 | } | ||
| 199 | else { | ||
| 200 | |||
| 201 | $self->{_logger}->log_message(" Checking timestamp for: $current_dir/$f") if($self->is_logging); | ||
| 202 | |||
| 203 | if(!$self->is_expired(config => $cache_config, directory => $current_dir, | ||
| 204 | filename => $f, timestamp => $timestamp)) { | ||
| 205 | |||
| 206 | # Save some time and calculate the total size of the cache | ||
| 207 | # in order to weed out the largets files to be compliant | ||
| 208 | $total_size += -s "$current_dir/$f"; | ||
| 209 | |||
| 210 | push(@remaining_files, "$current_dir/$f"); | ||
| 211 | |||
| 212 | $self->{_logger}->log_message(" Keeping file based on date: $current_dir/$f") if($self->is_logging); | ||
| 213 | } | ||
| 214 | else { | ||
| 215 | |||
| 216 | $self->expire_file($current_dir, $f); | ||
| 217 | |||
| 218 | $self->{_logger}->log_message(" Removing file based on date: $current_dir/$f") if($self->is_logging); | ||
| 219 | } | ||
| 220 | } | ||
| 221 | } | ||
| 222 | |||
| 223 | |||
| 224 | $self->{_logger}->log_message(" Processing remaining files by size.") if($self->is_logging); | ||
| 225 | |||
| 226 | # filter by size | ||
| 227 | |||
| 228 | # Largest files are the first to go | ||
| 229 | my @sorted_files = sort { (-s $b) <=> (-s $a) } @remaining_files; | ||
| 230 | |||
| 231 | @remaining_files = (); | ||
| 232 | |||
| 233 | # Remove files until the size of the directory is less than the max | ||
| 234 | # If the max size is zero, make sure to remove zero byte files | ||
| 235 | while($cache_config->{'max-cache-size'} != -1 && | ||
| 236 | $total_size > $cache_config->{'max-cache-size'} || | ||
| 237 | ($cache_config->{'max-cache-size'} == 0 && $#sorted_files > -1) ) { | ||
| 238 | |||
| 239 | my $f = pop(@sorted_files); | ||
| 240 | |||
| 241 | $total_size -= ( -s $f ); | ||
| 242 | |||
| 243 | $self->{_logger}->log_message(" Max size/current size: $cache_config->{'max-cache-size'} / $total_size") if($self->is_logging); | ||
| 244 | |||
| 245 | $self->expire_file($current_dir, $f); | ||
| 246 | |||
| 247 | $self->{_logger}->log_message(" Removing file based on size: $f") if($self->is_logging); | ||
| 248 | } | ||
| 249 | |||
| 250 | # Remove the directory if necessary | ||
| 251 | if( $#sorted_files == -1) { | ||
| 252 | |||
| 253 | # We really should check to see if the directory doesn't contain the log and Cache configuration | ||
| 254 | #(!grep($LS::Cache::Config::CACHE_FILENAME|$LS::Cache::Config::LOG_FILENAME, @files))) { | ||
| 255 | |||
| 256 | $self->{_logger}->log_message(" Removing directory: $current_dir") if($self->is_logging); | ||
| 257 | rmdir($current_dir); | ||
| 258 | } | ||
| 259 | |||
| 260 | return @sorted_files; | ||
| 261 | } | ||
| 262 | |||
| 263 | |||
| 264 | |||
| 265 | |||
| 266 | |||
| 267 | # | ||
| 268 | # Retrieve cached entries | ||
| 269 | # | ||
| 270 | |||
| 271 | |||
| 272 | |||
| 273 | # | ||
| 274 | # lookupFile( %options ) - | ||
| 275 | # | ||
| 276 | sub lookupFile { | ||
| 277 | |||
| 278 | my ($self, %options) = @_; | ||
| 279 | |||
| 280 | my $filename = $options{'filename'}; | ||
| 281 | my $directory = $options{'directory'}; | ||
| 282 | |||
| 283 | unless( -e "$directory/$filename" ) { | ||
| 284 | |||
| 285 | $self->{_logger}->log_message("File not cached: $directory/$filename") if($self->is_logging); | ||
| 286 | |||
| 287 | return undef; | ||
| 288 | } | ||
| 289 | |||
| 290 | my $fh = new IO::File("$directory/$filename", "r"); | ||
| 291 | |||
| 292 | unless($fh) { | ||
| 293 | |||
| 294 | $self->{_logger}->log_message("Unable to open cache file: $directory/$filename") if($self->is_logging); | ||
| 295 | |||
| 296 | return undef; | ||
| 297 | } | ||
| 298 | |||
| 299 | my $expiration = $self->getExpiration(filename=> $filename, | ||
| 300 | directory=> $directory); | ||
| 301 | |||
| 302 | return LS::Service::Response->new(response=> $fh, | ||
| 303 | expiration=> $expiration); | ||
| 304 | } | ||
| 305 | |||
| 306 | |||
| 307 | # | ||
| 308 | # lookupWSDL( %options ) - | ||
| 309 | # | ||
| 310 | sub lookupWSDL { | ||
| 311 | |||
| 312 | my ($self, %options) = @_; | ||
| 313 | |||
| 314 | my $lsid = $options{'lsid'}; | ||
| 315 | |||
| 316 | |||
| 317 | my $authority = $lsid->authority; | ||
| 318 | my $directory = $self->{_config}->get_config_entry('cache-root-dir') . '/wsdl/' . $authority; | ||
| 319 | |||
| 320 | my $filename = $self->canonicalized_disk_form($lsid) . ".$WSDL_EXT"; | ||
| 321 | |||
| 322 | |||
| 323 | |||
| 324 | $self->{_logger}->log_message("** Attempting to read WSDL for $authority from cache") if($self->is_logging); | ||
| 325 | |||
| 326 | # Make sure the directory exists | ||
| 327 | $self->{_config}->create_dir($directory); | ||
| 328 | |||
| 329 | # Load the expiration data and expire the file if necessary | ||
| 330 | if($self->is_expired(directory=> $directory, | ||
| 331 | filename=> $filename)) { | ||
| 332 | |||
| 333 | $self->{_logger}->log_message("WSDL file has expired: $directory/$filename") if($self->is_logging); | ||
| 334 | |||
| 335 | $self->expire_file($directory, $filename); | ||
| 336 | |||
| 337 | return undef; | ||
| 338 | } | ||
| 339 | |||
| 340 | my $response; | ||
| 341 | |||
| 342 | if( ($response = $self->lookupFile(filename=> $filename, directory=> $directory)) ) { | ||
| 343 | |||
| 344 | $self->{_logger}->log_message("WSDL is cached: $directory/$filename") if($self->is_logging); | ||
| 345 | } | ||
| 346 | |||
| 347 | return $response; | ||
| 348 | } | ||
| 349 | |||
| 350 | |||
| 351 | # | ||
| 352 | # lookupData( %options ) - | ||
| 353 | # | ||
| 354 | sub lookupData { | ||
| 355 | |||
| 356 | my ($self, %options) = @_; | ||
| 357 | |||
| 358 | my $lsid = $options{'lsid'}; | ||
| 359 | |||
| 360 | my $authority = $lsid->authority; | ||
| 361 | my $directory = $self->{_config}->get_config_entry('cache-root-dir') . '/data/' . $authority; | ||
| 362 | |||
| 363 | |||
| 364 | $self->{_logger}->log_message("** Attempting to read DATA for $authority from cache") if($self->is_logging); | ||
| 365 | |||
| 366 | # Make sure the directory exists | ||
| 367 | $self->{_config}->create_dir($directory); | ||
| 368 | |||
| 369 | my $filename = 'data'; | ||
| 370 | |||
| 371 | # Find a file in the directory and return the filename | ||
| 372 | |||
| 373 | my $response; | ||
| 374 | |||
| 375 | if( ($response = $self->lookupFile(filename=> $filename, directory=> $directory)) ) { | ||
| 376 | |||
| 377 | $self->{_logger}->log_message("DATA is cached: $directory/$filename") if($self->is_logging); | ||
| 378 | } | ||
| 379 | |||
| 380 | return $response; | ||
| 381 | } | ||
| 382 | |||
| 383 | |||
| 384 | # | ||
| 385 | # lookupMetadata( %options ) - | ||
| 386 | # | ||
| 387 | sub lookupMetadata { | ||
| 388 | |||
| 389 | my ($self, %options) = @_; | ||
| 390 | |||
| 391 | my $lsid = $options{'lsid'}; | ||
| 392 | |||
| 393 | my $portName = $options{'portName'}; | ||
| 394 | my $serviceName = $options{'serviceName'}; | ||
| 395 | |||
| 396 | my $authority = $lsid->authority; | ||
| 397 | my $directory = $self->{_config}->get_config_entry('cache-root-dir') . '/metadata'; | ||
| 398 | my $canonical = $self->canonicalized_disk_form($lsid); | ||
| 399 | |||
| 400 | my $filename = "$serviceName;$portName.$METADATA_EXT"; | ||
| 401 | |||
| 402 | $directory .= "/$authority/$canonical"; | ||
| 403 | |||
| 404 | $self->{_logger}->log_message("** Attempting to read METADATA for $authority from cache") if($self->is_logging); | ||
| 405 | |||
| 406 | # Make sure the directory exists | ||
| 407 | $self->{_config}->create_dir($directory); | ||
| 408 | |||
| 409 | # Load the expiration data and expire the file if necessary | ||
| 410 | if($self->is_expired(directory => $directory, filename => $filename)) { | ||
| 411 | |||
| 412 | #$self->{_logger}->log_message("Expiring METADATA: $directory/$filename") if($self->is_logging); | ||
| 413 | |||
| 414 | $self->expire_file($directory, $filename); | ||
| 415 | |||
| 416 | return undef; | ||
| 417 | } | ||
| 418 | |||
| 419 | my $response; | ||
| 420 | |||
| 421 | if( ($response = $self->lookupFile(filename=> $filename, directory=> $directory)) ) { | ||
| 422 | |||
| 423 | $self->{_logger}->log_message("METADATA is cached: $directory/$filename") if($self->is_logging); | ||
| 424 | } | ||
| 425 | |||
| 426 | return $response; | ||
| 427 | } | ||
| 428 | |||
| 429 | |||
| 430 | # | ||
| 431 | # cacheFile( %options ) - | ||
| 432 | # | ||
| 433 | sub cacheFile { | ||
| 434 | |||
| 435 | my ($self, %options) = @_; | ||
| 436 | |||
| 437 | |||
| 438 | my $filename = $options{'filename'}; | ||
| 439 | my $directory = $options{'directory'}; | ||
| 440 | |||
| 441 | my $cacheData = $options{'cacheData'}; | ||
| 442 | my $expiration = $options{'expiration'}; | ||
| 443 | |||
| 444 | # 1. Can the file/data fit in the directory? | ||
| 445 | my $size; | ||
| 446 | |||
| 447 | if(UNIVERSAL::isa($cacheData, 'File::Temp') || | ||
| 448 | UNIVERSAL::isa($cacheData, 'IO::File') ) { | ||
| 449 | |||
| 450 | $size = -s $cacheData; | ||
| 451 | } | ||
| 452 | else { | ||
| 453 | |||
| 454 | $size = length($cacheData); | ||
| 455 | } | ||
| 456 | |||
| 457 | return undef unless($self->cacheable(directory=> $directory, | ||
| 458 | size=> $size, | ||
| 459 | timestamp=> $expiration)); | ||
| 460 | |||
| 461 | # 2a. If our cachedData is a temporary file, move it in to a permanent location | ||
| 462 | my $fh; | ||
| 463 | |||
| 464 | if(UNIVERSAL::isa($cacheData, 'File::Temp')) { | ||
| 465 | |||
| 466 | unless(File::Copy::move($cacheData->filename, "$directory/$filename")) { | ||
| 467 | |||
| 468 | $self->{_logger}->log_message("Unable to cache file: $directory/$filename. " . | ||
| 469 | "Could not save temporary file $cacheData to a permanent location.") | ||
| 470 | if($self->is_logging); | ||
| 471 | |||
| 472 | return undef; | ||
| 473 | } | ||
| 474 | |||
| 475 | # 3. Update the file's expiration | ||
| 476 | $self->update_expiration($directory, $filename, $expiration) if($expiration); | ||
| 477 | |||
| 478 | # 4. Return a response in the form of a file handle | ||
| 479 | $fh = new IO::File("$directory/$filename", "r"); | ||
| 480 | |||
| 481 | return LS::Service::Response->new(response=> $fh, | ||
| 482 | expiration=> $expiration); | ||
| 483 | } | ||
| 484 | |||
| 485 | # 2b. If we were passed raw data, write that to the store | ||
| 486 | $fh = new IO::File("$directory/$filename", "w"); | ||
| 487 | |||
| 488 | if($fh->opened) { | ||
| 489 | |||
| 490 | $self->{_logger}->log_message("Caching file: $directory/$filename") if($self->is_logging); | ||
| 491 | |||
| 492 | binmode $fh; | ||
| 493 | |||
| 494 | flock($fh, LOCK_EX); | ||
| 495 | |||
| 496 | if(UNIVERSAL::isa($cacheData, 'IO::File')) { | ||
| 497 | |||
| 498 | while(<$cacheData>) { | ||
| 499 | |||
| 500 | $fh->print($_); | ||
| 501 | } | ||
| 502 | } | ||
| 503 | else { | ||
| 504 | |||
| 505 | $fh->print($cacheData); | ||
| 506 | } | ||
| 507 | |||
| 508 | flock($fh, LOCK_UN); | ||
| 509 | |||
| 510 | $fh->close; | ||
| 511 | } | ||
| 512 | else { | ||
| 513 | |||
| 514 | $self->{_logger}->log_message("Unable to cache file: $directory/$filename. " . | ||
| 515 | "Could not open location for writing.") | ||
| 516 | if($self->is_logging); | ||
| 517 | |||
| 518 | return undef; | ||
| 519 | } | ||
| 520 | |||
| 521 | # 3. Update the file's expiration | ||
| 522 | $self->update_expiration($directory, $filename, $expiration) if($expiration); | ||
| 523 | |||
| 524 | # 4. Return a response in the form of a filehandle | ||
| 525 | $fh = new IO::File("$directory/$filename", 'r'); | ||
| 526 | |||
| 527 | unless($fh->opened) { | ||
| 528 | |||
| 529 | $self->{_logger}->log_message("Unable to open file: $directory/$filename. " . | ||
| 530 | "Could not obtain file handle.") | ||
| 531 | if($self->is_logging); | ||
| 532 | |||
| 533 | return undef; | ||
| 534 | } | ||
| 535 | |||
| 536 | return LS::Service::Response->new(response=> $fh, | ||
| 537 | expiration=> $expiration); | ||
| 538 | } | ||
| 539 | |||
| 540 | |||
| 541 | # | ||
| 542 | # cacheWSDL( %options ) - | ||
| 543 | # | ||
| 544 | sub cacheWSDL { | ||
| 545 | |||
| 546 | my ($self, %options) = @_; | ||
| 547 | |||
| 548 | my $lsid = $options{'lsid'}; | ||
| 549 | my $response = $options{'response'}; | ||
| 550 | |||
| 551 | my $authority = $lsid->authority; | ||
| 552 | |||
| 553 | my $directory = $self->{_config}->get_config_entry('cache-root-dir') . '/wsdl'; | ||
| 554 | |||
| 555 | if( -d "$directory/$authority" || mkdir("$directory/$authority")) { | ||
| 556 | |||
| 557 | $directory .= "/$authority"; | ||
| 558 | } | ||
| 559 | else { | ||
| 560 | |||
| 561 | die("Unable to create cache direcory: $directory/$authority\n"); | ||
| 562 | } | ||
| 563 | |||
| 564 | my $filename = $self->canonicalized_disk_form($lsid) . ".$WSDL_EXT"; | ||
| 565 | |||
| 566 | return $self->cacheFile(filename=> $filename, | ||
| 567 | directory=> $directory, | ||
| 568 | expiration=> $response->expiration, | ||
| 569 | cacheData=> $response->response); | ||
| 570 | } | ||
| 571 | |||
| 572 | |||
| 573 | # | ||
| 574 | # cacheData( %options ) - | ||
| 575 | # | ||
| 576 | sub cacheData { | ||
| 577 | |||
| 578 | my ($self, %options) = @_; | ||
| 579 | |||
| 580 | my $lsid = $options{'lsid'}; | ||
| 581 | |||
| 582 | my $response = $options{'response'}; | ||
| 583 | |||
| 584 | my $authority = $lsid->authority; | ||
| 585 | my $directory = $self->{_config}->get_config_entry('cache-root-dir') . '/data'; | ||
| 586 | |||
| 587 | if(-d "$directory/$authority" || mkdir("$directory/$authority")) { | ||
| 588 | |||
| 589 | $directory .= "/$authority"; | ||
| 590 | } | ||
| 591 | else { | ||
| 592 | |||
| 593 | die("Unable to create cache direcory: $directory/$authority\n"); | ||
| 594 | } | ||
| 595 | |||
| 596 | my $filename = $self->canonicalized_disk_form($lsid) . ".$DATA_EXT"; | ||
| 597 | |||
| 598 | return $self->cacheFile(filename=> $filename, | ||
| 599 | directory=> $directory, | ||
| 600 | cacheData=> $response->response); | ||
| 601 | } | ||
| 602 | |||
| 603 | |||
| 604 | # | ||
| 605 | # cacheMetadata( %options ) - | ||
| 606 | # | ||
| 607 | sub cacheMetadata { | ||
| 608 | |||
| 609 | my ($self, %options) = @_; | ||
| 610 | |||
| 611 | #my ($self, $lsid, $metadata, $expiration, $service_name, $port) = @_; | ||
| 612 | |||
| 613 | my $lsid = $options{'lsid'}; | ||
| 614 | |||
| 615 | my $portName = $options{'portName'}; | ||
| 616 | my $serviceName = $options{'serviceName'}; | ||
| 617 | |||
| 618 | my $response = $options{'response'}; | ||
| 619 | |||
| 620 | my $authority = $lsid->authority; | ||
| 621 | my $directory = $self->{_config}->get_config_entry('cache-root-dir') . '/metadata'; | ||
| 622 | my $canonical = $self->canonicalized_disk_form($lsid); | ||
| 623 | |||
| 624 | if(-d "$directory/$authority" || mkdir("$directory/$authority")) { | ||
| 625 | |||
| 626 | $directory .= "/$authority"; | ||
| 627 | } | ||
| 628 | else { | ||
| 629 | |||
| 630 | $self->{_logger}->log_message("Unable to create METADATA cache directory: $directory/$authority") if($self->is_logging); | ||
| 631 | } | ||
| 632 | |||
| 633 | if(-d "$directory/$canonical" || mkdir("$directory/$canonical")) { | ||
| 634 | |||
| 635 | $directory .= "/$canonical"; | ||
| 636 | } | ||
| 637 | else { | ||
| 638 | |||
| 639 | $self->{_logger}->log_message("Unable to create METADATA cache directory: $directory/$canonical") if($self->is_logging); | ||
| 640 | } | ||
| 641 | |||
| 642 | |||
| 643 | my $filename = "$serviceName;$portName.$METADATA_EXT"; | ||
| 644 | |||
| 645 | return $self->cacheFile(filename=> $filename, | ||
| 646 | directory=> $directory, | ||
| 647 | expiration=> $response->expiration, | ||
| 648 | cacheData=> $response->response); | ||
| 649 | } | ||
| 650 | |||
| 651 | |||
| 652 | # | ||
| 653 | # Utility functions | ||
| 654 | # | ||
| 655 | |||
| 656 | |||
| 657 | # | ||
| 658 | # canonicalizedDiskForm( $lsid ) - | ||
| 659 | # | ||
| 660 | sub canonicalizedDiskForm { | ||
| 661 | |||
| 662 | my ($self, $lsid) = @_; | ||
| 663 | |||
| 664 | $lsid = $lsid->canonical; | ||
| 665 | |||
| 666 | my $authority = $lsid->authority; | ||
| 667 | my $namespace = $lsid->namespace; | ||
| 668 | my $object = $lsid->object; | ||
| 669 | my $revision = $lsid->revision; | ||
| 670 | |||
| 671 | return "urn;lsid;$authority;$namespace;$object;$revision" if($revision); | ||
| 672 | |||
| 673 | return "urn;lsid;$authority;$namespace;$object"; | ||
| 674 | } | ||
| 675 | |||
| 676 | |||
| 677 | # | ||
| 678 | # canonicalized_disk_form( $lsid ) - Synonym for canonicalizedDiskForm. | ||
| 679 | # | ||
| 680 | sub canonicalized_disk_form { | ||
| 681 | |||
| 682 | my $self = shift; | ||
| 683 | return $self->canonicalizedDiskForm(@_); | ||
| 684 | } | ||
| 685 | |||
| 686 | |||
| 687 | |||
| 688 | # | ||
| 689 | # cacheable( %options ) - | ||
| 690 | # | ||
| 691 | # | ||
| 692 | # Parameters: %options, Required. Keys used: | ||
| 693 | # size: | ||
| 694 | # directory: | ||
| 695 | # timestamp: | ||
| 696 | # | ||
| 697 | # | ||
| 698 | # Returns: True if the file can be cached in the specifed directory | ||
| 699 | # This loads the most specific configuration found and uses it to | ||
| 700 | # determine cacheability. | ||
| 701 | # | ||
| 702 | sub cacheable { | ||
| 703 | |||
| 704 | my ($self, %options) = @_; | ||
| 705 | |||
| 706 | return undef if(!$options{'size'} || !$options{'directory'}); | ||
| 707 | |||
| 708 | my $config_ref = $self->{_config}->import_cache_settings($options{'directory'}); | ||
| 709 | |||
| 710 | my $dir_size = $self->dir_size($options{'directory'}); | ||
| 711 | |||
| 712 | if($config_ref->{'max-cache-size'} != -1 && | ||
| 713 | $options{'size'} + $dir_size > $config_ref->{'max-cache-size'}) { | ||
| 714 | |||
| 715 | $self->{_logger}->log_message("File can not be cached - Directory ($options{'directory'}) has reached its maximum capacity: $config_ref->{'max-cache-size'}"); | ||
| 716 | |||
| 717 | return undef; | ||
| 718 | } | ||
| 719 | |||
| 720 | if($config_ref->{'max-cache-lifetime'} != -1 && $options{'timestamp'}) { | ||
| 721 | |||
| 722 | my $cur_time = $self->get_time; | ||
| 723 | my $time_lived = $cur_time - $options{'timestamp'}; | ||
| 724 | |||
| 725 | |||
| 726 | if($time_lived > $config_ref->{'max-cache-lifetime'}) { | ||
| 727 | |||
| 728 | $self->{_logger}->log_message("File can not be cached - Directory ($options{'directory'}) has low TTL: $time_lived > $config_ref->{'max-cache-lifetime'}"); | ||
| 729 | |||
| 730 | return undef; | ||
| 731 | } | ||
| 732 | } | ||
| 733 | |||
| 734 | $self->{_logger}->log_message("File can be cached in: $options{'directory'}"); | ||
| 735 | } | ||
| 736 | |||
| 737 | # | ||
| 738 | # Returns true if the file has expired | ||
| 739 | # | ||
| 740 | # %options keys - config, timestamp, filename, directory | ||
| 741 | # | ||
| 742 | sub is_expired { | ||
| 743 | |||
| 744 | my ($self, %options) = @_; | ||
| 745 | |||
| 746 | die("Invalid paramaters: is_expired\n") if(!$options{'filename'} || !$options{'directory'}); | ||
| 747 | |||
| 748 | my $config_ref = $self->{_config}; | ||
| 749 | my $curr_time = $self->get_time; | ||
| 750 | |||
| 751 | $config_ref = $options{'config'} if($options{'config'}); | ||
| 752 | $curr_time = $options{'timestamp'} if($options{'timestamp'}); | ||
| 753 | |||
| 754 | my $max = $config_ref->{'max-cache-lifetime'}; | ||
| 755 | |||
| 756 | return undef if($max == -1); | ||
| 757 | |||
| 758 | my $expiration = $self->getExpiration($options{'directory'}, $options{'filename'}); | ||
| 759 | |||
| 760 | if(!$expiration) { | ||
| 761 | |||
| 762 | # Grab the last modified time and use that | ||
| 763 | # (maybe use the creation time instead?) | ||
| 764 | $expiration = ( stat ($options{filename}) )[9]; | ||
| 765 | $expiration *= 1000; | ||
| 766 | } | ||
| 767 | |||
| 768 | return undef if($expiration == -1); | ||
| 769 | |||
| 770 | if($self->is_logging) { | ||
| 771 | |||
| 772 | my $time_lived = $curr_time - $expiration; | ||
| 773 | |||
| 774 | #$self->{_logger}->log_message("Is expired: $time_lived > $max"); | ||
| 775 | } | ||
| 776 | |||
| 777 | return ($curr_time - $expiration) > $max; | ||
| 778 | } | ||
| 779 | |||
| 780 | |||
| 781 | |||
| 782 | # | ||
| 783 | # Expiration options for individual files | ||
| 784 | # | ||
| 785 | |||
| 786 | # | ||
| 787 | # getExpiration( $directory, $caononicalFilename ) - | ||
| 788 | # | ||
| 789 | # Parameters: | ||
| 790 | # $directory, Required. | ||
| 791 | # $canonicalFilename, Required. | ||
| 792 | # | ||
| 793 | # | ||
| 794 | # Returns: | ||
| 795 | # | ||
| 796 | sub getExpiration { | ||
| 797 | |||
| 798 | my ($self, $directory, $canonical_filename) = @_; | ||
| 799 | |||
| 800 | my $exp_hash_ref = $self->read_expiration_file($directory); | ||
| 801 | |||
| 802 | if(!defined($exp_hash_ref->{$canonical_filename}) || | ||
| 803 | $exp_hash_ref->{$canonical_filename} == -1) { | ||
| 804 | |||
| 805 | return undef; | ||
| 806 | } | ||
| 807 | |||
| 808 | $self->{_logger}->log_message("Retrieving expiration for: $directory/$canonical_filename") if($self->is_logging); | ||
| 809 | $self->{_logger}->log_message("Expiration Time: " . $exp_hash_ref->{$canonical_filename}) if($self->is_logging); | ||
| 810 | |||
| 811 | return $exp_hash_ref->{$canonical_filename}; | ||
| 812 | } | ||
| 813 | |||
| 814 | sub expire_file { | ||
| 815 | |||
| 816 | my ($self, $directory, $canonical_filename) = @_; | ||
| 817 | |||
| 818 | if ( -f "$directory/$canonical_filename" ) { | ||
| 819 | |||
| 820 | $self->{_logger}->log_message("Removing file: $directory/$canonical_filename") if($self->is_logging); | ||
| 821 | |||
| 822 | unlink("$directory/$canonical_filename"); | ||
| 823 | } | ||
| 824 | |||
| 825 | my $exp_hash_ref = $self->read_expiration_file($directory); | ||
| 826 | |||
| 827 | delete($exp_hash_ref->{$canonical_filename}); | ||
| 828 | |||
| 829 | $self->write_expiration_file($directory, $exp_hash_ref); | ||
| 830 | } | ||
| 831 | |||
| 832 | sub update_expiration { | ||
| 833 | |||
| 834 | my ($self, $directory, $canonical_filename, $expiration_time) = @_; | ||
| 835 | |||
| 836 | $self->{_logger}->log_message("Updating expiration for: $directory/$canonical_filename") if($self->is_logging); | ||
| 837 | #$self->{_logger}->log_message(" Expiration time: $expiration_time") if($self->is_logging); | ||
| 838 | |||
| 839 | my %exp; | ||
| 840 | my $exp_hash_ref = $self->read_expiration_file($directory); | ||
| 841 | |||
| 842 | if($exp_hash_ref) { | ||
| 843 | |||
| 844 | # Update the expiration time for this entry. This has the side effect | ||
| 845 | # of creating the entry if it doesn't exist | ||
| 846 | $exp_hash_ref->{$canonical_filename} = $expiration_time; | ||
| 847 | } | ||
| 848 | else { | ||
| 849 | |||
| 850 | $exp{$canonical_filename} = $expiration_time; | ||
| 851 | $exp_hash_ref = \%exp; | ||
| 852 | } | ||
| 853 | |||
| 854 | $self->write_expiration_file($directory, $exp_hash_ref); | ||
| 855 | } | ||
| 856 | |||
| 857 | # | ||
| 858 | # Read and write to expiration files | ||
| 859 | # | ||
| 860 | # It's important to know that these deal with complete files, | ||
| 861 | # in otherwords write_expiration_file overwrites the entire file, | ||
| 862 | # read_expiration_file read the entire file in to a hash. | ||
| 863 | # | ||
| 864 | sub read_expiration_file { | ||
| 865 | |||
| 866 | my ($self, $directory) = @_; | ||
| 867 | |||
| 868 | my %exp_hash; | ||
| 869 | |||
| 870 | return undef if(! -e "$directory/$EXPIRATION_FILENAME"); | ||
| 871 | #$self->{_logger}->log_message("Expiration file does not exist: $directory/$EXPIRATION_FILENAME") if($self->is_logging); | ||
| 872 | |||
| 873 | if(!open(EXPIRE_FILE, "$directory/$EXPIRATION_FILENAME")) { | ||
| 874 | |||
| 875 | $self->{_logger}->log_message("Unable to open expiration file: $directory/$EXPIRATION_FILENAME") if($self->is_logging); | ||
| 876 | |||
| 877 | return undef; | ||
| 878 | } | ||
| 879 | |||
| 880 | flock(EXPIRE_FILE, LOCK_EX); | ||
| 881 | |||
| 882 | #$self->{_logger}->log_message("Reading expiration file: $directory/$EXPIRATION_FILENAME") if($self->is_logging); | ||
| 883 | |||
| 884 | # The file is of the form - canonical_filename::expiration | ||
| 885 | # All other lines are ignored | ||
| 886 | while(<EXPIRE_FILE>) { | ||
| 887 | |||
| 888 | if(/(.*)::(.*)/) { | ||
| 889 | |||
| 890 | $exp_hash{$1} = $2; | ||
| 891 | } | ||
| 892 | } | ||
| 893 | |||
| 894 | flock(EXPIRE_FILE, LOCK_UN); | ||
| 895 | |||
| 896 | close(EXPIRE_FILE); | ||
| 897 | |||
| 898 | return \%exp_hash; | ||
| 899 | } | ||
| 900 | |||
| 901 | sub write_expiration_file { | ||
| 902 | |||
| 903 | my ($self, $directory, $exp_hash_ref) = @_; | ||
| 904 | |||
| 905 | return undef if(!$exp_hash_ref); | ||
| 906 | |||
| 907 | if(!open(EXPIRE_FILE, ">$directory/$EXPIRATION_FILENAME")) { | ||
| 908 | |||
| 909 | $self->{_logger}->log_message("Unable to open expiration file: $directory/$EXPIRATION_FILENAME") if($self->is_logging); | ||
| 910 | |||
| 911 | return undef; | ||
| 912 | } | ||
| 913 | |||
| 914 | flock(EXPIRE_FILE, LOCK_EX); | ||
| 915 | |||
| 916 | #$self->{_logger}->log_message("Writing expiration file: $directory/$EXPIRATION_FILENAME") if($self->is_logging); | ||
| 917 | |||
| 918 | foreach my $cf (keys(%{$exp_hash_ref})) { | ||
| 919 | |||
| 920 | # The file is of the form - canonical_filename::expiration | ||
| 921 | print EXPIRE_FILE "$cf\:\:" . $exp_hash_ref->{$cf} . "\n"; | ||
| 922 | } | ||
| 923 | |||
| 924 | flock(EXPIRE_FILE, LOCK_UN); | ||
| 925 | |||
| 926 | close(EXPIRE_FILE); | ||
| 927 | |||
| 928 | # Remove the empty file | ||
| 929 | if( -s "$directory/$EXPIRATION_FILENAME" == 0) { | ||
| 930 | |||
| 931 | #$self->{_logger}->log_message("Empty expiration file, removing: $directory/$EXPIRATION_FILENAME") if($self->is_logging); | ||
| 932 | |||
| 933 | unlink("$directory/$EXPIRATION_FILENAME"); | ||
| 934 | } | ||
| 935 | } | ||
| 936 | |||
| 937 | # | ||
| 938 | # Calculates the size of the directory | ||
| 939 | # excluding system files | ||
| 940 | # | ||
| 941 | sub dir_size { | ||
| 942 | |||
| 943 | my ($self, $dir) = @_; | ||
| 944 | |||
| 945 | opendir(DIR, $dir); | ||
| 946 | my @files = readdir(DIR); | ||
| 947 | closedir(DIR); | ||
| 948 | |||
| 949 | my $sz = 0; | ||
| 950 | foreach(@files) { | ||
| 951 | |||
| 952 | next if($_ eq '.' || $_ eq '..'); | ||
| 953 | next if($_ eq $LS::Cache::Config::CACHE_FILENAME || | ||
| 954 | $_ eq $EXPIRATION_FILENAME || | ||
| 955 | $_ eq $LS::Cache::Config::LOG_FILENAME); | ||
| 956 | |||
| 957 | $sz += (-s "$dir/$_"); | ||
| 958 | } | ||
| 959 | |||
| 960 | return $sz; | ||
| 961 | } | ||
| 962 | |||
| 963 | 1; | ||
| 964 | |||
| 965 | __END__ |