| 1 | package HTTP::OAIPMH::Validator; | ||
| 2 | |||
| 3 | =head1 NAME | ||
| 4 | |||
| 5 | HTTP::OAIPMH::Validator - OAI-PMH validator class | ||
| 6 | |||
| 7 | =head1 SYNOPSIS | ||
| 8 | |||
| 9 | Validation suite for OAI-PMH data providers that checks for responses | ||
| 10 | in accord with OAI-PMH v2 | ||
| 11 | L<http://www.openarchives.org/OAI/2.0/openarchivesprotocol.htm>. | ||
| 12 | |||
| 13 | Typical use: | ||
| 14 | |||
| 15 | use HTTP::OAIPMH::Validator; | ||
| 16 | use Try::Tiny; | ||
| 17 | my $val = HTTP::OAIPMH::Validator->new( base_url=>'http://example.com/oai' ); | ||
| 18 | try { | ||
| 19 | $val->run_complete_validation; | ||
| 20 | } catch { | ||
| 21 | warn "oops, validation didn't run to completion: $!\n"; | ||
| 22 | }; | ||
| 23 | print "Validation status of data provider ".$val->base_url." is ".$val->status."\n"; | ||
| 24 | |||
| 25 | =cut | ||
| 26 | |||
| 27 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 28 | |||
| 29 | our $VERSION = '1.06'; | ||
| 30 | |||
| 31 | 1 | ▶ | use base qw(Class::Accessor::Fast); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 32 | 1 | ▶ | use Data::UUID; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 33 | 1 | ▶ | use Date::Manip; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 34 | 1 | ▶ | use HTTP::Request; # for rendering http queries |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 35 | 1 | ▶ | use HTTP::Headers; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 36 | 1 | ▶ | use HTTP::Request::Common; # makes POST easier |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 37 | 1 | ▶ | use HTTP::Status; # for checking error codes |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 38 | 1 | ▶ | use LWP::UserAgent; # send http requests |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 39 | 1 | ▶ | use LWP::Protocol::https; # explicit include so we fail without https support |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 40 | 1 | ▶ | use URI::Escape; # excape special characters |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 41 | 0 | ▶ | use XML::DOM; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 42 | use HTTP::OAIPMH::Log; | ||
| 43 | |||
| 44 | =head2 METHODS | ||
| 45 | |||
| 46 | =head3 new(%args) | ||
| 47 | |||
| 48 | Create new HTTP::OAIPMH::Validator object and initialize counters. | ||
| 49 | |||
| 50 | The following instance variables may be set via %args and have read-write | ||
| 51 | accessors (via L<Class::Accessor::Fast>): | ||
| 52 | |||
| 53 | base_url - base URL of the data provdier being validated | ||
| 54 | run_id - UUID identifying the run (will be generated if none supplied) | ||
| 55 | protocol_version - protocol version supported | ||
| 56 | admin_email - admin email extracted from Identify response | ||
| 57 | granularity - datestamp granularity (defaults to 'days', else 'seconds') | ||
| 58 | uses_https - set true if the validator sees an https URL at any stage | ||
| 59 | |||
| 60 | debug - set true to add extra debugging output | ||
| 61 | log - logging object (usually L<HTTP::OAIPMH::Log>) | ||
| 62 | parser - XML DOM parser instance | ||
| 63 | |||
| 64 | identify_response - string of identify response (used for registration record) | ||
| 65 | earliest_datestamp - value extracted from earliestDatestamp in Identify response | ||
| 66 | namespace_id - if the oai-identifier is used then this records the namespace identifier extracted | ||
| 67 | set_names - array of all the set names reported in listSets | ||
| 68 | |||
| 69 | example_record_id - example id used for tests that require a specific identifier | ||
| 70 | example_set_spec - example setSpec ("&set=name") used for tests that require a set | ||
| 71 | example_metadata_prefix - example metadataPrefix which defaults to 'oai_dc' | ||
| 72 | |||
| 73 | =cut | ||
| 74 | |||
| 75 | HTTP::OAIPMH::Validator->mk_accessors( qw( base_url protocol_version | ||
| 76 | admin_email granularity uses_503 uses_https | ||
| 77 | debug parser run_id ua allow_https doc save_all_responses | ||
| 78 | response_number http_timeout max_retries max_size | ||
| 79 | protocol guidelines | ||
| 80 | identify_response earliest_datestamp namespace_id set_names | ||
| 81 | example_record_id example_set_spec example_metadata_prefix | ||
| 82 | log status | ||
| 83 | ) ); | ||
| 84 | |||
| 85 | sub new { | ||
| 86 | my $this=shift; | ||
| 87 | my $class=ref($this) || $this; | ||
| 88 | my $self={ | ||
| 89 | 'base_url' => undef, | ||
| 90 | 'protocol_version' => undef, | ||
| 91 | # Repository features extracted | ||
| 92 | 'granularity' => 'days', # can also be "seconds" | ||
| 93 | 'uses_503' => 0, # set true if 503 responses ever used | ||
| 94 | 'uses_https' => 0, # set to true if https is ever used | ||
| 95 | # Control | ||
| 96 | 'debug' => 0, | ||
| 97 | 'parser' => XML::DOM::Parser->new(), | ||
| 98 | 'run_id' => undef, | ||
| 99 | 'ua' => undef, | ||
| 100 | 'allow_https' => 0, # allow https URIs | ||
| 101 | 'doc' => undef, # current parsed xml document | ||
| 102 | 'save_all_responses' => 0, # set True to save all HTTP responses | ||
| 103 | 'response_number' => 1, # initial response number | ||
| 104 | 'http_timeout' => 600, | ||
| 105 | 'max_retries' => 5, # number of 503's in a row that we will accept | ||
| 106 | 'max_size' => 100000000, # max response size in bytes (100MB) | ||
| 107 | 'protocol' => 'http://www.openarchives.org/OAI/2.0/openarchivesprotocol.htm', #URL of protocol spec | ||
| 108 | 'guidelines' => 'http://www.openarchives.org/OAI/2.0/guidelines-repository.htm', #URL of repository guidelines doc | ||
| 109 | # Results | ||
| 110 | 'namespace_id' => undef, | ||
| 111 | 'set_names' => [], | ||
| 112 | 'example_record_id' => undef, | ||
| 113 | 'example_set_spec' => undef, | ||
| 114 | 'example_metadata_prefix' => 'oai_dc', | ||
| 115 | 'log' => HTTP::OAIPMH::Log->new(), | ||
| 116 | 'status' => 'unknown', | ||
| 117 | @_}; | ||
| 118 | bless($self, $class); | ||
| 119 | $self->setup_run_id if (not $self->run_id); | ||
| 120 | $self->setup_user_agent if (not $self->ua); | ||
| 121 | return($self); | ||
| 122 | } | ||
| 123 | |||
| 124 | =head3 setup_run_id() | ||
| 125 | |||
| 126 | Set a UUID for the run_id. | ||
| 127 | |||
| 128 | =cut | ||
| 129 | |||
| 130 | sub setup_run_id { | ||
| 131 | my $self=shift; | ||
| 132 | my $ug=Data::UUID->new; | ||
| 133 | $self->run_id(lc($ug->to_string($ug->create))); | ||
| 134 | } | ||
| 135 | |||
| 136 | =head3 setup_user_agent() | ||
| 137 | |||
| 138 | Setup L<LWP::UserAgent> for the validator. | ||
| 139 | |||
| 140 | =cut | ||
| 141 | |||
| 142 | sub setup_user_agent { | ||
| 143 | my $self=shift; | ||
| 144 | my $ua = LWP::UserAgent->new(); # User agent, to render http requests | ||
| 145 | $ua->timeout($self->http_timeout); # give responses 10 minutes | ||
| 146 | $ua->max_size($self->max_size); # size limit ##seems to break http://eprints.soton.ac.uk/perl/oai2 [Simeon/2005-06-06] | ||
| 147 | $ua->requests_redirectable([]); # we will do redirects manually | ||
| 148 | $ua->agent('OAIPMH_Validator'); # set user agent | ||
| 149 | $ua->from('https://groups.google.com/d/forum/oai-pmh'); # set a default From: address -> direct to google group for dicussion | ||
| 150 | $self->ua($ua); | ||
| 151 | } | ||
| 152 | |||
| 153 | |||
| 154 | =head3 abort($msg) | ||
| 155 | |||
| 156 | Special purpose "die" routine because tests cannot continue. Logs | ||
| 157 | failure and then dies. | ||
| 158 | |||
| 159 | =cut | ||
| 160 | |||
| 161 | sub abort { | ||
| 162 | my $self=shift; | ||
| 163 | my ($msg)=@_; | ||
| 164 | $self->log->fail('ABORT: '.$msg); | ||
| 165 | $self->status('FAILED'); | ||
| 166 | die('ABORT: '.$msg."\n"); | ||
| 167 | } | ||
| 168 | |||
| 169 | |||
| 170 | =head3 run_complete_validation($skip_test_identify) | ||
| 171 | |||
| 172 | Run all tests for a complete validation and return true is the data provider passes, | ||
| 173 | false otherwise. All actions are logged and may be accessed to provide a report | ||
| 174 | (including warnings that do not indicate failure) after the run. | ||
| 175 | |||
| 176 | Arguments: | ||
| 177 | $skip_identify - set true to skip the text_identify() step | ||
| 178 | |||
| 179 | =cut | ||
| 180 | |||
| 181 | sub run_complete_validation { | ||
| 182 | my $self=shift; | ||
| 183 | my ($skip_identify)=@_; | ||
| 184 | |||
| 185 | $self->response_number(1); | ||
| 186 | $self->test_identify unless ($skip_identify); | ||
| 187 | $self->test_list_sets; | ||
| 188 | $self->test_list_identifiers; | ||
| 189 | |||
| 190 | my $baseURL = $self->base_url; | ||
| 191 | my ($formats, $gotDC) = $self->test_list_metadata_formats; | ||
| 192 | |||
| 193 | # If the repository doesn't support oai_dc then this is a failure (because | ||
| 194 | # the standard demands it) but see whether we can find another metadataPrefix | ||
| 195 | # in order to continue the tests | ||
| 196 | if ( $gotDC ) { | ||
| 197 | $self->log->pass("Data provider supports oai_dc metadataPrefix"); | ||
| 198 | } else { | ||
| 199 | if ($formats and $formats->getLength()>0) { | ||
| 200 | $self->example_metadata_prefix( $formats->item(0)->getFirstChild->getData ); | ||
| 201 | $self->log->fail("Data provider does not support the simple Dublin Core metadata ". | ||
| 202 | "format with metadataPrefix oai_dc. Tests that require a ". | ||
| 203 | "metadataPrefix to be specified will use '". | ||
| 204 | $self->example_metadata_prefix."'"); | ||
| 205 | } else { | ||
| 206 | $self->log->fail("There are no metadata formats available to use with the GetRecord ". | ||
| 207 | "request. The metadataPrefix ". | ||
| 208 | $self->example_metadata_prefix. | ||
| 209 | " will be used for later tests even though it seems unsupported."); | ||
| 210 | } | ||
| 211 | } | ||
| 212 | |||
| 213 | my ($dateStamp)=$self->test_get_record($self->example_record_id,$self->example_metadata_prefix); | ||
| 214 | $self->test_list_records($dateStamp,$self->example_metadata_prefix); | ||
| 215 | |||
| 216 | # Check responses to erroneous queries | ||
| 217 | $self->test_expected_errors($self->example_record_id); | ||
| 218 | |||
| 219 | if ($self->protocol_version eq '2.0') { | ||
| 220 | $self->test_expected_v2_errors($self->earliest_datestamp,$self->example_metadata_prefix); | ||
| 221 | # As of version 2.0, data providers must support HTTP POST requests | ||
| 222 | $self->test_post_requests($self->example_metadata_prefix); | ||
| 223 | } | ||
| 224 | $self->test_resumption_tokens; | ||
| 225 | |||
| 226 | # Getting here with no failures means that the data provider is compliant | ||
| 227 | # (there may be warnings which are not counted in num_fail) | ||
| 228 | $self->status( $self->log->num_fail==0 ? 'COMPLIANT' : 'FAILED' ); | ||
| 229 | return($self->log->num_fail==0); | ||
| 230 | } | ||
| 231 | |||
| 232 | |||
| 233 | =head3 failures() | ||
| 234 | |||
| 235 | Return Markdown summary of failure log entries, along with the appropriate | ||
| 236 | titles and request details. Will return empty string if there are no | ||
| 237 | failures in the log. | ||
| 238 | |||
| 239 | =cut | ||
| 240 | |||
| 241 | sub failures { | ||
| 242 | my $self=shift; | ||
| 243 | return($self->log->failures()); | ||
| 244 | } | ||
| 245 | |||
| 246 | |||
| 247 | =head3 summary() | ||
| 248 | |||
| 249 | Return summary statistics for the validation in Markdown (designed to agree | ||
| 250 | with conversion to HTML by L<Text::Markdown>). | ||
| 251 | |||
| 252 | =cut | ||
| 253 | |||
| 254 | sub summary { | ||
| 255 | my $self=shift; | ||
| 256 | |||
| 257 | my $sf=($self->log->num_fail>0?'failure':'success'); | ||
| 258 | |||
| 259 | my $str="\n## Summary - *$sf*\n\n"; | ||
| 260 | my $namespace_id = $self->namespace_id; | ||
| 261 | if ($namespace_id) { | ||
| 262 | if ($namespace_id=~/\./) { #v2.0 | ||
| 263 | $str.=" * Namespace declared for v2.0 oai-identifiers is $namespace_id\n"; | ||
| 264 | } else { #v1.1 | ||
| 265 | $str.=" * Namespace declared for v1.1 oai-identifiers (the repositoryIdentifier) is $namespace_id\n"; | ||
| 266 | } | ||
| 267 | } | ||
| 268 | $str.=" * Uses 503 for flow control\n" if ($self->uses_503); | ||
| 269 | $str.=" * Uses https URIs (not specified in protocol)\n" if ($self->uses_https); | ||
| 270 | $str.=" * Total tests passed: ".$self->log->num_pass."\n"; | ||
| 271 | $str.=" * Total warnings: ".$self->log->num_warn."\n"; | ||
| 272 | $str.=" * Total error count: ".$self->log->num_fail."\n"; | ||
| 273 | $str.=" * Validation status: ".($self->status || 'unknown')."\n"; | ||
| 274 | return($str); | ||
| 275 | } | ||
| 276 | |||
| 277 | |||
| 278 | =head2 METHODS TESTING SPECIFIC OAI-PMH VERBS | ||
| 279 | |||
| 280 | =head3 test_identify() | ||
| 281 | |||
| 282 | Check response to an Identify request. Returns false if tests cannot | ||
| 283 | continue, true otherwise. | ||
| 284 | |||
| 285 | Side effects based on values extracted: | ||
| 286 | |||
| 287 | - $self->admin_email set to email extracted from adminEmail element | ||
| 288 | - $self->granularity set to 'days' or 'seconds' | ||
| 289 | |||
| 290 | =cut | ||
| 291 | |||
| 292 | sub test_identify { | ||
| 293 | my $self=shift; | ||
| 294 | |||
| 295 | my $cantContinue=0; | ||
| 296 | $self->log->start("Checking Identify response"); | ||
| 297 | |||
| 298 | # Send the verb request to the base URL - vet extracts the email address | ||
| 299 | my $burl=$self->base_url; | ||
| 300 | my $req = $burl."?verb=Identify"; | ||
| 301 | |||
| 302 | my $response = $self->make_request($req); #don't use make_request_and_validate() just do simplest thing here | ||
| 303 | unless ($response->is_success) { | ||
| 304 | my $r="Server at base URL '$burl' failed to respond to Identify. The HTTP GET request with URL $req received response code '".$response->code()."'."; | ||
| 305 | if ($response->code() == 301) { | ||
| 306 | $self->log->fail("$r HTTP code 301 'Moved Permanently' is not widely supported by ". | ||
| 307 | "harvesters and is anyway inappropriate for registration of a ". | ||
| 308 | "service. If requests must be redirected then an HTTP response 302 ". | ||
| 309 | "may be used as outlined in the guidelines [". | ||
| 310 | $self->guidelines."#LoadBalancing]."); | ||
| 311 | } else { | ||
| 312 | $self->log->fail($r); | ||
| 313 | } | ||
| 314 | $self->abort("Failed to get Identify response from server at base URL '$burl'.\n"); | ||
| 315 | return; | ||
| 316 | } | ||
| 317 | |||
| 318 | # Parse the XML response | ||
| 319 | unless ($self->parse_response($req,$response)) { | ||
| 320 | $self->log->fail("Failed to parse Identify response"); | ||
| 321 | $self->abort("Failed to parse Identify response from server at base URL '$burl'.\n"); | ||
| 322 | } | ||
| 323 | |||
| 324 | # Check that this really is a Identify response | ||
| 325 | my $oaipmhNode=$self->doc->getFirstChild(); | ||
| 326 | # skip over and processing instructions such as XML stylesheets | ||
| 327 | while ($oaipmhNode->getNodeType==PROCESSING_INSTRUCTION_NODE or | ||
| 328 | $oaipmhNode->getNodeType==COMMENT_NODE) { | ||
| 329 | $oaipmhNode=$oaipmhNode->getNextSibling(); | ||
| 330 | } | ||
| 331 | unless (defined $oaipmhNode and $oaipmhNode->getNodeName eq 'OAI-PMH') { | ||
| 332 | $self->log->fail("Identify response does not have OAI-PMH as root element! ". | ||
| 333 | "Found node named '".$oaipmhNode->getNodeName."' instead"); | ||
| 334 | $self->abort("Identify response from server at base URL '$burl' does not have ". | ||
| 335 | "OAI-PMH as root element!\n"); | ||
| 336 | } | ||
| 337 | my $identifyNode=$oaipmhNode->getElementsByTagName('Identify',0); | ||
| 338 | unless ($identifyNode->getLength()>0) { | ||
| 339 | my $errorNode=$oaipmhNode->getElementsByTagName('error',0); | ||
| 340 | if ($errorNode->getLength()>0) { | ||
| 341 | # give specific message if response is and error | ||
| 342 | $self->log->fail("Error response to Identify request!\n"); | ||
| 343 | $self->abort("Error response to Identify request from server at base URL '$burl'.\n"); | ||
| 344 | return; | ||
| 345 | } else { | ||
| 346 | $self->log->fail("Identify response does not contain <Identify> block.\n"); | ||
| 347 | $self->abort("Identify response does not contain Identify block from server at base URL '$burl'.\n"); | ||
| 348 | return; | ||
| 349 | } | ||
| 350 | } | ||
| 351 | |||
| 352 | # Extract admin email and protocol version numbers, check | ||
| 353 | my ($admin_email,$email_error)=$self->get_admin_email; | ||
| 354 | if (not $admin_email or $email_error) { | ||
| 355 | $self->abort(($email_error || "Failed to extract adminEmail").", aborting.\n"); | ||
| 356 | return; | ||
| 357 | } | ||
| 358 | $self->admin_email($admin_email); | ||
| 359 | $self->check_protocol_version; # bails if not Version 2.0 | ||
| 360 | |||
| 361 | # URL is valid, Identify response was provided, extract content as string | ||
| 362 | $self->identify_response( $response->content ); | ||
| 363 | |||
| 364 | my $baseURL = $self->doc->getElementsByTagName('baseURL'); | ||
| 365 | |||
| 366 | # BUG FOUND ON AUGUST 26, 2002: empty baseURL still returns length > 0 | ||
| 367 | # So it is necessary to explicity check for an empty element. | ||
| 368 | if ( $baseURL->getLength() > 0 ) { | ||
| 369 | $baseURL = $baseURL->item(0)->getFirstChild; | ||
| 370 | if ( $baseURL ) { $baseURL = $baseURL->getData; } | ||
| 371 | |||
| 372 | # $burl is the one given on the form; $baseURL is the one in the XML doc. | ||
| 373 | if ($burl eq $baseURL) { | ||
| 374 | $self->log->pass("baseURL supplied matches the Identify response"); | ||
| 375 | } else { | ||
| 376 | # report the error, but keep the form URL | ||
| 377 | # (at least it answered Identify!) | ||
| 378 | $self->log->fail("baseURL supplied '$burl' does not match the baseURL in the ". | ||
| 379 | "Identify response '$baseURL'. The baseURL you enter must EXACTLY ". | ||
| 380 | "match the baseURL returned in the Identify response. It must ". | ||
| 381 | "match in case (http://Wibble.org/ does not match http://wibble.org/) ". | ||
| 382 | "and include any trailing slashes etc."); | ||
| 383 | $cantContinue++; | ||
| 384 | } | ||
| 385 | } | ||
| 386 | |||
| 387 | # For Version 2.0, Check for seconds granularity | ||
| 388 | if ($self->protocol_version eq '2.0') { | ||
| 389 | my $gran_el = $self->doc->getElementsByTagName('granularity'); | ||
| 390 | if ($self->parse_granularity($gran_el)) { | ||
| 391 | $self->log->pass("Datestamp granularity is '".$self->granularity."'"); | ||
| 392 | } else { | ||
| 393 | $cantContinue++; | ||
| 394 | } | ||
| 395 | } | ||
| 396 | |||
| 397 | # For an exception check new to Version 2.0, extract the earliest date | ||
| 398 | # and also check that its granularity is right | ||
| 399 | if (my $err=$self->get_earliest_datestamp) { | ||
| 400 | $self->log->fail("Bad earliestDatestamp: $err"); | ||
| 401 | $cantContinue++; | ||
| 402 | } else { | ||
| 403 | $self->log->pass("Extracted earliestDatestamp ".$self->earliest_datestamp); | ||
| 404 | } | ||
| 405 | |||
| 406 | # Check for OAI-identifier. If already in use by another base URL, bump | ||
| 407 | # the error count to avoid having this URL register. | ||
| 408 | # | ||
| 409 | my $oaiIds = $self->doc->getElementsByTagName('oai-identifier'); | ||
| 410 | if ($oaiIds and $oaiIds->getLength()>0) { | ||
| 411 | if ($oaiIds->getLength()>1) { | ||
| 412 | $self->log->fail("Found more than one oai-identifier element. The intention ". | ||
| 413 | "is that this declaration only be used by repositories ". | ||
| 414 | "declaring the use of a single identifier namespace."); | ||
| 415 | $cantContinue++; | ||
| 416 | } else { | ||
| 417 | $oaiIds=$oaiIds->item(0); | ||
| 418 | |||
| 419 | # Now find out if this is v1.1 or v2.0 | ||
| 420 | my $oai_id_version='2.0'; | ||
| 421 | if (my $xmlns=$oaiIds->getAttribute('xmlns')) { #FIXME this requires default namespace to be set to oai-id | ||
| 422 | if ($xmlns eq 'http://www.openarchives.org/OAI/2.0/oai-identifier') { | ||
| 423 | $oai_id_version='2.0'; | ||
| 424 | $self->log->pass("oai-identifier description for version $oai_id_version is being used"); | ||
| 425 | } elsif ($xmlns eq 'http://www.openarchives.org/OAI/1.1/oai-identifier') { | ||
| 426 | $oai_id_version='1.1'; | ||
| 427 | $self->log->pass("oai-identifier description for version $oai_id_version is being used"); | ||
| 428 | } elsif ($xmlns) { | ||
| 429 | $self->log->fail("Unrecognized namespace declaration '$xmlns' for ". | ||
| 430 | "oai-identifier, expected ". | ||
| 431 | "http://www.openarchives.org/OAI/2.0/oai-identifier ". | ||
| 432 | "(for v2.0) or ". | ||
| 433 | "http://www.openarchives.org/OAI/1.1/oai-identifier ". | ||
| 434 | "(for v1.1). Assuming version $oai_id_version."); | ||
| 435 | } else { | ||
| 436 | $self->log->fail("No namespace declaration found for oai-identifier, expected ". | ||
| 437 | "http://www.openarchives.org/OAI/2.0/oai-identifier ". | ||
| 438 | "(for v2.0) or ". | ||
| 439 | "http://www.openarchives.org/OAI/1.1/oai-identifier ". | ||
| 440 | "(for v1.1). Assuming version $oai_id_version/"); | ||
| 441 | } | ||
| 442 | } else { | ||
| 443 | $self->log->fail("Can't find namespace declaration for the oai-identifier description. ". | ||
| 444 | "This must be added as <oai-identifier xmlns=\"http://www.openarchives.org/OAI/2.0/oai-identifier\" ...> ". | ||
| 445 | "(or 1.1), there will likely also be schema validation weeors. Will ". | ||
| 446 | "assume that the oai-identifier is version $oai_id_version for ". | ||
| 447 | "later tests"); | ||
| 448 | } | ||
| 449 | my $repoIds = $oaiIds->getElementsByTagName('repositoryIdentifier'); | ||
| 450 | if ($repoIds) { | ||
| 451 | my $temp = $repoIds->item(0); | ||
| 452 | if (!defined($temp)) { | ||
| 453 | $self->log->fail("No namespace-identifier (repositoryIdentifier element) in ". | ||
| 454 | "the oai-identifier block of the Identify description"); | ||
| 455 | return; | ||
| 456 | } | ||
| 457 | my $nsel = $temp->getFirstChild; | ||
| 458 | unless ( $nsel ) { | ||
| 459 | # Empty repositoryIdentifier element, squawk loudly | ||
| 460 | $self->log->fail("Empty namespace-identifier (repositoryIdentifier element) in ". | ||
| 461 | "the oai-identifier block of the Identify description"); | ||
| 462 | return; | ||
| 463 | } | ||
| 464 | my $namespace_id = $nsel->getData; | ||
| 465 | # Having validated the value of namespace-identifier, we can now tell if it is v1.1 or v2.0 based | ||
| 466 | # on whether is has a . in it (i.e. if /\./) | ||
| 467 | if ($oai_id_version eq '2.0') { | ||
| 468 | #schema: <pattern value="[a-zA-Z][a-zA-Z0-9\-]*(\.[a-zA-Z][a-zA-Z0-9\-]+)+"/> | ||
| 469 | unless ($namespace_id=~/^[a-z][a-z0-9\-]*(\.[a-z][a-z0-9\-]+)+$/i) { | ||
| 470 | $self->log->fail("Bad namespace-identifier (repositoryIdentifier element) ". | ||
| 471 | "'$namespace_id' in oai-identifier declaration. See section ". | ||
| 472 | "2.1 of the OAI Identifier specification for details ". | ||
| 473 | "(http://www.openarchives.org/OAI/2.0/guidelines-oai-identifier.htm)."); | ||
| 474 | $cantContinue++; | ||
| 475 | } else { | ||
| 476 | $self->log->pass("namespace-identifier (repositoryIdentifier element) in oai-identifier ". | ||
| 477 | "declaration is $namespace_id"); | ||
| 478 | $self->namespace_id( $namespace_id ); | ||
| 479 | } | ||
| 480 | } else { #v1.1 schema: <pattern value="[a-zA-Z0-9]+"/> | ||
| 481 | unless ($namespace_id=~/^[a-z0-9]+$/i) { | ||
| 482 | $self->log->fail("Bad namespace-identifier (repositoryIdentifier element) ". | ||
| 483 | "'$namespace_id' in oai-identifier declaration. See section ". | ||
| 484 | "2.1 of the OAI Identifier specification for details ". | ||
| 485 | "(http://www.openarchives.org/OAI/1.1/guidelines-oai-identifier.htm)."); | ||
| 486 | $cantContinue++; | ||
| 487 | } else { | ||
| 488 | $self->log->pass("namespace-identifier (repositoryIdentifier element) in oai-identifier ". | ||
| 489 | "declaration is $namespace_id"); | ||
| 490 | $self->namespace_id( $namespace_id ); | ||
| 491 | } | ||
| 492 | } | ||
| 493 | } | ||
| 494 | } | ||
| 495 | } | ||
| 496 | return(not $cantContinue); | ||
| 497 | } | ||
| 498 | |||
| 499 | |||
| 500 | =head3 test_list_sets() | ||
| 501 | |||
| 502 | Check response to the ListSets verb. | ||
| 503 | |||
| 504 | Save the setSpecs for later use. | ||
| 505 | |||
| 506 | Note that the any set might be empty. So if test_list_identifiers doesn't | ||
| 507 | get a match, we need to try the second set identifier, and so on. | ||
| 508 | So keep a list of the setSpec elements. | ||
| 509 | |||
| 510 | =cut | ||
| 511 | |||
| 512 | sub test_list_sets { | ||
| 513 | my $self=shift; | ||
| 514 | |||
| 515 | $self->log->start("Checking ListSets response"); | ||
| 516 | my $req=$self->base_url."?verb=ListSets"; | ||
| 517 | my $response = $self->make_request_and_validate("ListSets", $req); | ||
| 518 | unless ($response) { | ||
| 519 | $self->log->fail("Can't check set names"); | ||
| 520 | return; | ||
| 521 | } | ||
| 522 | |||
| 523 | unless ($self->parse_response($req,$response)) { | ||
| 524 | $self->log->fail("Can't parse response"); | ||
| 525 | $self->abort("failed to parse response to ListSets"); | ||
| 526 | } | ||
| 527 | |||
| 528 | $self->set_names( [] ); | ||
| 529 | $self->example_set_spec( '' ); | ||
| 530 | my $set_elements=$self->doc->getElementsByTagName('setSpec'); | ||
| 531 | if (not defined($set_elements) or ($set_elements->getLength<1)) { | ||
| 532 | # No setSpec elements, so there should be an <error code="noSetHierarchy"> element | ||
| 533 | my $details={}; | ||
| 534 | if ($self->is_error_response($details)) { | ||
| 535 | if ($details->{'noSetHierarchy'}) { | ||
| 536 | $self->log->pass("Repository does not support sets and the is correctly reported with a ". | ||
| 537 | "noSetHierarchy exception in the ListSets response"); | ||
| 538 | } else { | ||
| 539 | $self->log->fail("Failed to extract any setSpec elements from ListSets ". | ||
| 540 | "but did not find a noSetHierarchy exception. Found instead a '". | ||
| 541 | join(', ',keys %{$details})."' exception(s). See <". | ||
| 542 | $self->protocol."#ListSets>."); | ||
| 543 | } | ||
| 544 | } else { | ||
| 545 | $self->log->fail("Failed to extract any setSpec elements from ListSets but did not ". | ||
| 546 | "find an exception message. If sets are not supported by the ". | ||
| 547 | "repository then the ListSets response must be the noSetHierarchy ". | ||
| 548 | "error. See <".$self->protocol."#ListSets>."); | ||
| 549 | } | ||
| 550 | } else { | ||
| 551 | # Have setSpec elements, record all set names and pick an example set spec | ||
| 552 | for (my $j=0; $j<$set_elements->getLength; $j++) { | ||
| 553 | my $set_name=$set_elements->item($j)->getFirstChild->getData; | ||
| 554 | ##FIXME - should validate each set name | ||
| 555 | push(@{$self->set_names},$set_name); | ||
| 556 | } | ||
| 557 | # Sanity check, did we get the number we expected? | ||
| 558 | my $num_sets=scalar(@{$self->set_names}); | ||
| 559 | if ($num_sets!=$set_elements->getLength) { | ||
| 560 | $self->log->fail("Failed to extract the expected number of set names (got ". | ||
| 561 | "$num_sets, expected ".$set_elements->getLength.")"); | ||
| 562 | } | ||
| 563 | if ($num_sets>0) { | ||
| 564 | $self->example_set_spec( "&set=".$self->set_names->[0] ); | ||
| 565 | } | ||
| 566 | my $msg=''; | ||
| 567 | for (my $j=0; $j<$num_sets and $j<3; $j++) { $msg.=" ".$self->set_names->[$j]; } | ||
| 568 | $msg.=" ..." if ($num_sets>3); | ||
| 569 | $self->log->pass("Extracted $num_sets set names: {$msg }, will use setSpec ". | ||
| 570 | $self->example_set_spec." in tests"); | ||
| 571 | } | ||
| 572 | } | ||
| 573 | |||
| 574 | |||
| 575 | =head3 test_list_identifiers() | ||
| 576 | |||
| 577 | Check response to ListIdentifiers and record an example record id in | ||
| 578 | $self->example_record_id to be used in other tests. | ||
| 579 | |||
| 580 | If there are no identifiers, but the response is legal, stop the test with | ||
| 581 | errors=0, number of verbs checked is three. | ||
| 582 | |||
| 583 | As of version 2.0, a metadataPrefix argument is required. Unfortunately | ||
| 584 | we need to call test_list_identifiers first in order to get an id for | ||
| 585 | GetRecord, so we simply use oai_dc. | ||
| 586 | |||
| 587 | =cut | ||
| 588 | |||
| 589 | sub test_list_identifiers { | ||
| 590 | my $self=shift; | ||
| 591 | |||
| 592 | $self->log->start("Checking ListIdentifiers response"); | ||
| 593 | |||
| 594 | ### FIXME -- skip the set= restriction because this code doesn't | ||
| 595 | ### FIXME work right for set hierarchies - 2002-10-17 | ||
| 596 | ### FIXME 2015-01-02 - put/left in, is it OK? | ||
| 597 | my $set_spec = $self->example_set_spec; | ||
| 598 | my $req = $self->base_url."?verb=ListIdentifiers&metadataPrefix=oai_dc".$set_spec; | ||
| 599 | my $response = $self->make_request_and_validate("ListIdentifiers", $req); | ||
| 600 | |||
| 601 | # Note: $response will come back null if an error code was returned | ||
| 602 | # An error code of "noRecordsMatch" comes back if specified set is | ||
| 603 | # empty. In that case we should drop the set and try again. | ||
| 604 | if ($set_spec and (!$response or $self->is_no_records_match)) { | ||
| 605 | $self->log->note("Tried empty set, will look for set with items..."); | ||
| 606 | foreach my $set_name (@{$self->set_names}) { | ||
| 607 | $set_spec = "&set=".$set_name; | ||
| 608 | if ($set_spec eq $self->example_set_spec) { | ||
| 609 | # This was the first one we tried, don't try again | ||
| 610 | next; | ||
| 611 | } | ||
| 612 | $self->log->note("Trying set ".$set_spec); | ||
| 613 | $req = $self->base_url."?verb=ListIdentifiers&metadataPrefix=oai_dc".$set_spec; | ||
| 614 | $response = $self->make_request_and_validate("ListIdentifiers", $req); | ||
| 615 | if ($response and not $self->is_no_records_match) { | ||
| 616 | last; | ||
| 617 | } | ||
| 618 | } | ||
| 619 | # If we were successful then set the example_set_spec for any future tests | ||
| 620 | if ($response and not $self->is_no_records_match) { | ||
| 621 | $self->example_set_spec( $set_spec ); | ||
| 622 | } else { | ||
| 623 | $self->log->note("Failed to find non-empty set, expect failure"); | ||
| 624 | } | ||
| 625 | } | ||
| 626 | |||
| 627 | # None of the sets had any identifiers in them. Try the whole entire | ||
| 628 | # list of identifiers. | ||
| 629 | if ($set_spec and !$response) { | ||
| 630 | $self->log->note("Last attempt is without any sets..."); | ||
| 631 | $req = $self->base_url."?verb=ListIdentifiers&metadataPrefix=oai_dc"; | ||
| 632 | $response = $self->make_request_and_validate("ListIdentifiers",$req); | ||
| 633 | } | ||
| 634 | |||
| 635 | # Now we are for real in trouble if $response is null | ||
| 636 | unless ($response) { | ||
| 637 | $self->log->fail("No ListIdentifiers response with content"); | ||
| 638 | $self->log->note("The base URL did not respond to the ListIdentifiers verb.". | ||
| 639 | "Without that, we cannot proceed with the validation test. Exiting."); | ||
| 640 | $self->abort("The base URL did not respond to the ListIdentifiers verb. Without that, we cannot proceed with the validation test. Exiting."); | ||
| 641 | } | ||
| 642 | |||
| 643 | # Grab the first identifier for later use | ||
| 644 | unless ($self->parse_response($req,$response)) { | ||
| 645 | $self->log->fail("Can't parse ListIdentifiers response"); | ||
| 646 | $self->abort("unable to parse response"); | ||
| 647 | } | ||
| 648 | # | ||
| 649 | # Now look for the identifier of a non-deleted record | ||
| 650 | # If there are no identifiers to be harvested, we cannot complete the | ||
| 651 | # validation test. | ||
| 652 | # | ||
| 653 | # FIXME - this still doesn't solve the problem that there may be no | ||
| 654 | # non-deleted items listed in the particular response or partial | ||
| 655 | # response that we are looking at [Simeon/2005-07-20] | ||
| 656 | # | ||
| 657 | my $headers = $self->doc->getElementsByTagName('header'); | ||
| 658 | my $h; | ||
| 659 | my $record_id; | ||
| 660 | for ($h=0; $h<$headers->getLength(); $h++) { | ||
| 661 | my $hdnode=$headers->item($h); | ||
| 662 | my $idnode=$hdnode->getElementsByTagName('identifier',0); | ||
| 663 | next unless ($idnode and $idnode->getLength()==1); | ||
| 664 | $record_id=$idnode->item(0)->getFirstChild->getData; | ||
| 665 | last unless ($hdnode->getAttribute('status') and $hdnode->getAttribute('status') eq 'deleted'); | ||
| 666 | $self->log->warn("Identifier ".($h+1).", '$record_id', is for a deleted record, skipping"); | ||
| 667 | } | ||
| 668 | if ($h==$headers->getLength()) { | ||
| 669 | # No identifiers were in the ListIdentifiers response. Further testing | ||
| 670 | # is not possible. | ||
| 671 | $self->log->fail("The response to the ListIdentifiers verb with metadataPrefix oai_dc ". | ||
| 672 | "contained no identifiers. Without at least one identifier, we cannot ". | ||
| 673 | "proceed with the validation tests."); | ||
| 674 | $self->abort("No identifiers in response to ListIdentifiers. Without an identifier ". | ||
| 675 | "we cannot proceed with validation tests."); | ||
| 676 | } | ||
| 677 | $self->log->pass("Good ListIdentifiers response, extracted id '$record_id' for use in future tests."); | ||
| 678 | $self->example_record_id( $record_id ); | ||
| 679 | } | ||
| 680 | |||
| 681 | |||
| 682 | =head3 test_list_metadata_formats() | ||
| 683 | |||
| 684 | Vet the verb as usual, and then make sure that Dublin Core in included | ||
| 685 | In particular, we will check the metadata formats available for "record_id", | ||
| 686 | obtained from checking the ListIdentifier verb. | ||
| 687 | Side effect: save available formats for later use (global "formats"). | ||
| 688 | NOTE:if there are no formats, error will be picked up by getRecord | ||
| 689 | |||
| 690 | =cut | ||
| 691 | |||
| 692 | sub test_list_metadata_formats { | ||
| 693 | my $self=shift; | ||
| 694 | |||
| 695 | $self->log->start("Checking ListMetadataFormats response"); | ||
| 696 | |||
| 697 | # Do we have an example record id to check with? | ||
| 698 | my $record_id = $self->example_record_id; | ||
| 699 | unless ($record_id) { | ||
| 700 | $self->log->fail("Cannot check ListMetadataFormats as we do not have an example id"); | ||
| 701 | return; | ||
| 702 | } | ||
| 703 | |||
| 704 | my $req = $self->base_url."?verb=ListMetadataFormats&identifier=".url_encode($record_id); | ||
| 705 | my $response = $self->make_request_and_validate("ListMetadataFormats",$req); | ||
| 706 | unless ($response) { | ||
| 707 | $self->log->fail("Can't check metadataFormats available for item $record_id, no ". | ||
| 708 | "response to ListMetadataFormats request."); | ||
| 709 | return; | ||
| 710 | } | ||
| 711 | |||
| 712 | # Check for Dublin Core | ||
| 713 | unless ($self->parse_response($req,$response)) { | ||
| 714 | $self->log->fail("Can't parse response to ListMetadataFormats request for item $record_id."); | ||
| 715 | return; | ||
| 716 | } | ||
| 717 | |||
| 718 | my $formats = $self->doc->getElementsByTagName('metadataPrefix'); | ||
| 719 | unless ($formats->getLength() > 0) { | ||
| 720 | $self->log->fail("No metadata formats are listed in the response to a ListMetadataFormats ". | ||
| 721 | "request for item $record_id."); | ||
| 722 | return; | ||
| 723 | } | ||
| 724 | |||
| 725 | if ($self->debug) { | ||
| 726 | $self->log->note("debug: ".$formats->getLength()." formats supported for identifier '$record_id'"); | ||
| 727 | } | ||
| 728 | my $gotDC=0; | ||
| 729 | for my $i (0..$formats->getLength()-1) { | ||
| 730 | my $format = $formats->item($i); | ||
| 731 | #assume this node has only one child, and its data for a format | ||
| 732 | if ( $format->getFirstChild->getData =~ /^\s*oai_dc\s*$/ ) { | ||
| 733 | $gotDC = 1; | ||
| 734 | last; | ||
| 735 | } | ||
| 736 | } | ||
| 737 | if ($gotDC) { | ||
| 738 | $self->log->pass("Good ListMetadataFormats response, includes oai_dc"); | ||
| 739 | } else { | ||
| 740 | $self->log->pass("Good ListMetadataFormats response, BUT DID NOT FIND oai_dc"); | ||
| 741 | } | ||
| 742 | return($formats, $gotDC); | ||
| 743 | } | ||
| 744 | |||
| 745 | |||
| 746 | =head3 test_get_record($record_id, $format) | ||
| 747 | |||
| 748 | Try to get record $record_id in $format. | ||
| 749 | |||
| 750 | If either $record_id or $format are undef then we have an error | ||
| 751 | right off the bat. Else make the request and return the | ||
| 752 | datestamp of the record. | ||
| 753 | |||
| 754 | =cut | ||
| 755 | |||
| 756 | sub test_get_record { | ||
| 757 | my $self=shift; | ||
| 758 | my ($record_id, $format)=@_; | ||
| 759 | |||
| 760 | $self->log->start("Checking GetRecord response"); | ||
| 761 | |||
| 762 | unless (defined $format) { | ||
| 763 | $self->log->fail("Skipping GetRecord test as no metadata format is listed as being available."); | ||
| 764 | return; | ||
| 765 | } | ||
| 766 | unless (defined $record_id) { | ||
| 767 | $self->log->fail("Skipping GetRecord test as no items are listed as having metadata available."); | ||
| 768 | return; | ||
| 769 | } | ||
| 770 | |||
| 771 | my $numerr=0; #count up non-fatal errors | ||
| 772 | |||
| 773 | my $req = $self->base_url."?verb=GetRecord&identifier=".url_encode($record_id)."&metadataPrefix=".url_encode($format); | ||
| 774 | my $response = $self->make_request_and_validate("GetRecord", $req); | ||
| 775 | unless ($response) { | ||
| 776 | $self->log->fail("Can't complete datestamp check for GetRecord"); | ||
| 777 | $self->abort("Can't complete datestamp check for GetRecord"); | ||
| 778 | } | ||
| 779 | |||
| 780 | # Save the datestamp for later use by ListRecords | ||
| 781 | # As of version 2.0, Identify response can have a granularity and the | ||
| 782 | # datestamp MUST be in the finest granularity supported by the repository | ||
| 783 | unless ($self->parse_response($req,$response)) { | ||
| 784 | $self->log->fail("Can't parse response"); | ||
| 785 | $self->abort("Unable to parse response from GetRecord"); | ||
| 786 | } | ||
| 787 | |||
| 788 | if (my $msg=$self->is_error_response) { | ||
| 789 | $self->log->fail("The response to the GetRecord verb was the OAI exception $msg. ". | ||
| 790 | "It is this not possible to extract a valid datestamp for remaining tests"); | ||
| 791 | $self->abort("Unexpected OAI exception response"); | ||
| 792 | } | ||
| 793 | |||
| 794 | my $datestamps = $self->doc->getElementsByTagName('datestamp'); | ||
| 795 | # If there is no <record> there is no datestamp ... but there should be a record | ||
| 796 | unless ( $datestamps->getLength() > 0 ) { | ||
| 797 | $self->log->fail("The response to the GetRecord verb did not have a datestamp, which is ". | ||
| 798 | "needed to continue checking verbs."); | ||
| 799 | $self->abort("No datestamp in the response for GetRecord"); | ||
| 800 | } | ||
| 801 | |||
| 802 | my $datestamp=undef; | ||
| 803 | eval { | ||
| 804 | $datestamp = $datestamps->item(0)->getFirstChild->getData; | ||
| 805 | }; | ||
| 806 | if (not defined($datestamp)) { | ||
| 807 | $self->log->fail("Failed to extract datestamp from the GetRecord response. See <". | ||
| 808 | $self->protocol."#Dates>."); | ||
| 809 | $numerr++; | ||
| 810 | } elsif ( my $granularity=$self->get_datestamp_granularity($datestamp) ) { | ||
| 811 | $self->log->pass("Datestamp in GetRecord response ($datestamp) has the correct form for ". | ||
| 812 | "$granularity granularity."); | ||
| 813 | if ( $granularity eq $self->granularity ) { | ||
| 814 | # The granularity in v2.0 must match the finest granularity supported (see sec3.3.2) | ||
| 815 | $self->log->pass("Datestamp in GetRecord response ($datestamp) matched the ". | ||
| 816 | $self->granularity." granularity specified in the Identify response. "); | ||
| 817 | } else { | ||
| 818 | $self->log->fail("Datestamp in GetRecord response ($datestamp) is not consistent ". | ||
| 819 | "with the ".$self->granularity." granularity specified in the ". | ||
| 820 | "Identify response"); | ||
| 821 | $numerr++; | ||
| 822 | } | ||
| 823 | } else { | ||
| 824 | $self->log->fail("Datestamp in GetRecord response ($datestamp) is not valid. See <". | ||
| 825 | $self->protocol."#Dates>."); | ||
| 826 | $numerr++; | ||
| 827 | } | ||
| 828 | |||
| 829 | # As of OAI-PMH Version 2.0, GetRecord must return a set spec if the | ||
| 830 | # repository supports sets and the item is in a set | ||
| 831 | if (not $self->example_set_spec) { | ||
| 832 | $self->log->pass("Valid GetRecord response") unless ($numerr>0); | ||
| 833 | return($datestamp); | ||
| 834 | } | ||
| 835 | |||
| 836 | my $set_list = $self->doc->getElementsByTagName('setSpec'); | ||
| 837 | my $set_value = $self->example_set_spec; | ||
| 838 | $set_value =~ s/&set=//; | ||
| 839 | $self->log->note("Looking for set '".$set_value."' or a descendant set.") if $self->debug; | ||
| 840 | my $i; | ||
| 841 | my $subset_str = ''; | ||
| 842 | for ($i=0; $i<$set_list->getLength; $i++) { | ||
| 843 | my $s = $set_list->item($i)->getFirstChild->getData; | ||
| 844 | last if ($s eq $set_value); | ||
| 845 | if ($s =~ m/^${set_value}:/) { | ||
| 846 | $subset_str = " (implied by a descendant setSpec)"; | ||
| 847 | last; | ||
| 848 | } | ||
| 849 | } | ||
| 850 | if ($i==$set_list->getLength) { # error | ||
| 851 | $self->log->fail("Expected setSpec was missing from the response. The GetRecord ". | ||
| 852 | "response for identifier $record_id did not contain a set ". | ||
| 853 | "specification for $set_value"); | ||
| 854 | } else { | ||
| 855 | $self->log->pass("Expected setSpec was returned in the response".$subset_str); | ||
| 856 | } | ||
| 857 | return($datestamp); | ||
| 858 | } | ||
| 859 | |||
| 860 | |||
| 861 | =head3 test_list_records($datestamp,$metadata_prefix) | ||
| 862 | |||
| 863 | Test the response for the ListRecords verb. In addition, if there is | ||
| 864 | no Dublin Core available for this repository, this is an error. | ||
| 865 | (And the error has already been counted in test_get_record) | ||
| 866 | We can still test the verb, however, with one of the available | ||
| 867 | formats found by testGetMetadataFormats. Since the output of | ||
| 868 | ListRecords is likely to be large, use the datestamp of the one | ||
| 869 | record we did retrieve to limit the output. | ||
| 870 | |||
| 871 | =cut | ||
| 872 | |||
| 873 | sub test_list_records { | ||
| 874 | my $self=shift; | ||
| 875 | my ($datestamp,$metadata_prefix)=@_; | ||
| 876 | |||
| 877 | $self->log->start("Checking ListRecords response"); | ||
| 878 | |||
| 879 | my $req = $self->base_url."?verb=ListRecords"; | ||
| 880 | if ($datestamp) { | ||
| 881 | $req.="&from=".$datestamp."&until=".$datestamp; | ||
| 882 | } else { | ||
| 883 | $self->log->warn("Omitting datestamp parameter as none was obtained from earlier test"); | ||
| 884 | } | ||
| 885 | $req.="&metadataPrefix=".$metadata_prefix; | ||
| 886 | my $list_not_complete=1; | ||
| 887 | |||
| 888 | while ($list_not_complete) { | ||
| 889 | $list_not_complete=0; | ||
| 890 | my $response = $self->make_request_and_validate("ListRecords", $req); | ||
| 891 | unless ($response) { | ||
| 892 | #Nothing else to say since we don't do other tests | ||
| 893 | return; | ||
| 894 | } | ||
| 895 | |||
| 896 | if ($self->parse_response($req,$response)) { | ||
| 897 | $self->log->pass("Response is well formed"); | ||
| 898 | } else { | ||
| 899 | $self->log->fail("The ListRecords response was not well formed XML"); | ||
| 900 | } | ||
| 901 | |||
| 902 | # Now check to make sure that we got back the record for the identifier | ||
| 903 | # $self->example_record_id if there is one specified, else fail that | ||
| 904 | # test. | ||
| 905 | my $record_id=$self->example_record_id; | ||
| 906 | unless ($record_id) { | ||
| 907 | $self->log->fail("Cannot check for correct record inclusion without an example record id"); | ||
| 908 | return; | ||
| 909 | } | ||
| 910 | my $details={}; | ||
| 911 | if ($self->is_error_response($details)) { | ||
| 912 | if ($details->{'noRecordsMatch'}) { | ||
| 913 | $self->log->fail("ListRecords response gave a noRecordsMatch error when it should ". | ||
| 914 | "have included at least the record with identifier $record_id. ". | ||
| 915 | "The from and until parameters of the request were set to the ". | ||
| 916 | "datestamp of this record ($datestamp). The from and until parameters ". | ||
| 917 | "are inclusive, see protocol spec section 2.7.1. The message ". | ||
| 918 | "included in the error response was: '". | ||
| 919 | $details->{'noRecordsMatch'}."'"); | ||
| 920 | } else { | ||
| 921 | my @txt=(); | ||
| 922 | foreach my $k (keys %$details) { | ||
| 923 | push(@txt,"$k (".$details->{$k}.")"); | ||
| 924 | } | ||
| 925 | $self->log->fail("ListRecords gave an unexpected error response to a request using ". | ||
| 926 | "from and until datestamps taken from the previous GetRecord response: ". | ||
| 927 | join(', ',@txt)); | ||
| 928 | } | ||
| 929 | } else { | ||
| 930 | my $id_list = $self->doc->getElementsByTagName('identifier'); | ||
| 931 | my $i; | ||
| 932 | my $badly_formed=0; | ||
| 933 | for ($i=0; $i<$id_list->getLength; $i++) { | ||
| 934 | if (my $child=$id_list->item($i)->getFirstChild()) { | ||
| 935 | last if ($id_list->item($i)->getFirstChild->getData eq $record_id); | ||
| 936 | } else { | ||
| 937 | $badly_formed++; | ||
| 938 | last; | ||
| 939 | } | ||
| 940 | } | ||
| 941 | if ($badly_formed) { | ||
| 942 | $self->log->fail("ListRecords response badly formed, identifier element for record ". | ||
| 943 | ($i+1)." is empty"); | ||
| 944 | } elsif ($i<$id_list->getLength) { | ||
| 945 | $self->log->pass("ListRecords response correctly included record with identifier $record_id"); | ||
| 946 | } elsif (my $token=$self->get_resumption_token) { | ||
| 947 | # More responses to come, may just not have got to the | ||
| 948 | # record yet... roll around for more: | ||
| 949 | $self->log->pass("ListRecords response includes resumptionToken. Haven't found ". | ||
| 950 | "record with identifier $record_id yet, will continue with resumptionToken..."); | ||
| 951 | $list_not_complete=1; | ||
| 952 | $req = $self->base_url."?verb=ListRecords&resumptionToken=".url_encode($token); | ||
| 953 | } else { | ||
| 954 | $self->log->fail("ListRecords response did not include the identifier $record_id ". | ||
| 955 | "which should have been included because both the from and until ". | ||
| 956 | "parameters were set to the datestamp of this record ($datestamp). ". | ||
| 957 | "The from and until parameters are inclusive, see protocol spec ". | ||
| 958 | "section 2.7.1"); | ||
| 959 | } | ||
| 960 | } | ||
| 961 | } | ||
| 962 | } | ||
| 963 | |||
| 964 | |||
| 965 | =head3 test_resumption_tokens() | ||
| 966 | |||
| 967 | Request an unlimited ListRecords. If there is a resumption token, see | ||
| 968 | if it works. It is an error if resumption is there but doesn't work. | ||
| 969 | Empty resumption tokens are OK -- this ends the list. | ||
| 970 | |||
| 971 | CGI takes care of URL-encoding the resumption token. | ||
| 972 | |||
| 973 | =cut | ||
| 974 | |||
| 975 | sub test_resumption_tokens { | ||
| 976 | my $self=shift; | ||
| 977 | |||
| 978 | $self->log->start("Checking for correct use of resumptionToken (if used)"); | ||
| 979 | |||
| 980 | my $req = $self->base_url."?verb=ListRecords&metadataPrefix=oai_dc"; | ||
| 981 | my $response = $self->make_request($req); | ||
| 982 | |||
| 983 | # was there a resumption token? | ||
| 984 | unless ($self->parse_response($req,$response)) { | ||
| 985 | $self->log->fail("Can't parse malformed XML in response to ListRecords request. ". | ||
| 986 | "Cannot complete test for correct use of resumptionToken (if used)"); | ||
| 987 | return; | ||
| 988 | } | ||
| 989 | |||
| 990 | my $tokenList = $self->doc->getElementsByTagName('resumptionToken'); | ||
| 991 | if ( !$tokenList or $tokenList->getLength()==0 ) { | ||
| 992 | $self->log->pass("resumptionToken not used"); | ||
| 993 | return; | ||
| 994 | } | ||
| 995 | if ( $tokenList->getLength()>1 ) { | ||
| 996 | $self->log->fail("More than one resumptionToken in response!"); | ||
| 997 | return; | ||
| 998 | } | ||
| 999 | |||
| 1000 | # Dig out the resumption token from the document | ||
| 1001 | my $tokenElement = $tokenList->item(0); | ||
| 1002 | |||
| 1003 | # Try getting the resumption token, $token will be will be undefined | ||
| 1004 | # unless the element has content | ||
| 1005 | my $token = $tokenElement->getFirstChild; | ||
| 1006 | my $tokenString; | ||
| 1007 | if ($token) { | ||
| 1008 | $tokenString = $token->getData; | ||
| 1009 | } | ||
| 1010 | unless ($tokenString) { | ||
| 1011 | $self->log->fail("Empty resumption token in response to $req There should never ". | ||
| 1012 | "be an empty resumptionToken in response to a request without a ". | ||
| 1013 | "resumptionToken argument"); | ||
| 1014 | return; | ||
| 1015 | } | ||
| 1016 | |||
| 1017 | # If there us a 'cursor' value given then check that it is | ||
| 1018 | # correct. It must have the value 0 in the first response | ||
| 1019 | my $usingCursor=0; | ||
| 1020 | if (my $cursor=$tokenElement->getAttribute('cursor')) { | ||
| 1021 | $usingCursor=1; | ||
| 1022 | if ($cursor==0) { | ||
| 1023 | $self->log->pass("A cursor value was supplied with the resumptionToken and it ". | ||
| 1024 | "correctly had the value zero in the first response"); | ||
| 1025 | } else { | ||
| 1026 | $self->log->fail("A cursor value was supplied with the resumptionToken but it ". | ||
| 1027 | "did not have the correct value zero for the first response. ". | ||
| 1028 | "The value was '$cursor'."); | ||
| 1029 | } | ||
| 1030 | } | ||
| 1031 | |||
| 1032 | $self->log->note("Got resumptionToken ".$tokenString); | ||
| 1033 | |||
| 1034 | # Try using the resumption token. Before including a resumptionToken in | ||
| 1035 | # the URL of a subsequent request, we must encode all special characters | ||
| 1036 | # getData in this version of XML::DOM expands entitities | ||
| 1037 | $req = $self->base_url."?verb=ListRecords&resumptionToken=".url_encode($tokenString); | ||
| 1038 | $response = $self->make_request($req); | ||
| 1039 | unless ( $response ) { | ||
| 1040 | $self->log->fail("Site failed to respond to request using resumptionToken: $req"); | ||
| 1041 | return; | ||
| 1042 | } | ||
| 1043 | unless ( $self->parse_response($req,$response)) { | ||
| 1044 | $self->log->fail("Response to request is using resumptionToken not valid XML: $req"); | ||
| 1045 | return; | ||
| 1046 | } | ||
| 1047 | |||
| 1048 | my $errorList = $self->doc->getElementsByTagName('error'); | ||
| 1049 | if ( $errorList and $errorList->getLength() > 0 ) { | ||
| 1050 | $self->log->fail("Response to request using resumptionToken was an error code: $req"); | ||
| 1051 | return; | ||
| 1052 | } | ||
| 1053 | |||
| 1054 | ###FIXME: put in test for cursor again, should be number of items returned in the | ||
| 1055 | ###FIXME: first response [Simeon/2005-10-11] | ||
| 1056 | |||
| 1057 | $self->log->pass("Resumption tokens appear to work"); | ||
| 1058 | } | ||
| 1059 | |||
| 1060 | |||
| 1061 | =head2 METHODS CHECKING ERRORS AND EXCEPTIONS | ||
| 1062 | |||
| 1063 | =head3 test_expected_errors($record_id) | ||
| 1064 | |||
| 1065 | Each one of these requests should get a 400 response in OAI-PHM v1.1, | ||
| 1066 | or a 200 response in 2.0, along with a Reason_Phrase. Bump error_count | ||
| 1067 | if this does not hold. Return the number of errorneous responses. | ||
| 1068 | |||
| 1069 | $record_id is a valid record identifier to be used in tests that require | ||
| 1070 | one. | ||
| 1071 | |||
| 1072 | =cut | ||
| 1073 | |||
| 1074 | sub test_expected_errors { | ||
| 1075 | my $self=shift; | ||
| 1076 | my ($record_id)=@_; | ||
| 1077 | |||
| 1078 | $self->log->start("Checking exception handling (errors)"); | ||
| 1079 | |||
| 1080 | my @request_list = ( | ||
| 1081 | [ 'junk', [ 'badVerb' ], '', '' ], | ||
| 1082 | [ 'verb=junk', [ 'badVerb' ], '', '' ], | ||
| 1083 | [ 'verb=GetRecord&metadataPrefix=oai_dc', [ 'badArgument' ], '', '' ], | ||
| 1084 | [ 'verb=GetRecord&identifier='.$record_id, [ 'badArgument' ], '', '' ], | ||
| 1085 | [ 'verb=GetRecord&identifier=invalid"id&metadataPrefix=oai_dc', [ 'badArgument','idDoesNotExist' ], 'An XML parsing error may be due to incorrectly including the invalid identifier in the <request> element of your XML error response; only valid arguments should be included. A response that includes <request verb="GetRecord" identifier="invalid"id" metadataPrefix="oai_dc">..baseURL..</request> is not well-formed XML because of the quotation mark (") in the identifier.', 'Either the badArgument or idDoesNotExist error codes would be appropriate to report this case.' ], | ||
| 1086 | [ 'verb=ListIdentifiers&until=junk', [ 'badArgument' ], '', '' ], | ||
| 1087 | [ 'verb=ListIdentifiers&from=junk', [ 'badArgument' ], '', '' ], | ||
| 1088 | [ 'verb=ListIdentifiers&resumptionToken=junk&until=2000-02-05', [ 'badArgument','badResumptionToken' ], '', 'Either the badArgument and/or badResumptionToken error codes may be reported in this case. If only one is reported then the badArgument error is to be preferred because the resumptionToken and until parameters are exclusive.' ], | ||
| 1089 | [ 'verb=ListRecords&metadataPrefix=oai_dc&from=junk', [ 'badArgument' ], '', '' ], | ||
| 1090 | [ 'verb=ListRecords&resumptionToken=junk', [ 'badResumptionToken' ], '', '' ], | ||
| 1091 | [ 'verb=ListRecords&metadataPrefix=oai_dc&resumptionToken=junk&until=1990-01-10', [ 'badArgument','badResumptionToken' ] , '', 'Either the badArgument and/or badResumptionToken error codes may be reported in this case. If only one is reported then the badArgument error is to be preferred because the resumptionToken and until parameters are exclusive.' ], | ||
| 1092 | [ 'verb=ListRecords&metadataPrefix=oai_dc&until=junk', [ 'badArgument' ], '', '' ], | ||
| 1093 | [ 'verb=ListRecords', [ 'badArgument' ], '', '' ] | ||
| 1094 | ); | ||
| 1095 | |||
| 1096 | my $n=0; | ||
| 1097 | foreach my $rrr ( @request_list ) { | ||
| 1098 | my ($request_string, $error_codes, $xml_reason, $reason)=@$rrr; | ||
| 1099 | my $req = $self->base_url.'?'.$request_string; | ||
| 1100 | my $ok_errors=join(' or ',@$error_codes); | ||
| 1101 | |||
| 1102 | my $response=$self->make_request($req); | ||
| 1103 | |||
| 1104 | # TBD: $response->status_line should also be checked? see output from | ||
| 1105 | # physnet.uni-oldenburg.de/oai/oai.php | ||
| 1106 | if ($self->protocol_version eq "1.1") { | ||
| 1107 | if ($response->code ne "400") { | ||
| 1108 | $self->log->note("Invalid requests which failed to return 400:") if $n == 0; | ||
| 1109 | $n++; | ||
| 1110 | $self->log->fail("Expected 400 from: $request_string"); | ||
| 1111 | } | ||
| 1112 | } elsif ($self->protocol_version eq "2.0") { | ||
| 1113 | # The document must contain the proper error code | ||
| 1114 | unless ($self->parse_response($req,$response,$xml_reason)) { | ||
| 1115 | $self->log->fail("Can't parse malformed response. ".html_escape($xml_reason)); | ||
| 1116 | $n++; | ||
| 1117 | next; | ||
| 1118 | } | ||
| 1119 | # check that the error code is in the error_list | ||
| 1120 | my $error_elements = $self->doc->getElementsByTagName('error'); | ||
| 1121 | if (my $matching_code=$self->error_elements_include($error_elements, $error_codes)) { | ||
| 1122 | $self->log->pass("Error response correctly includes error code '$matching_code'"); | ||
| 1123 | } else { | ||
| 1124 | $self->log->fail("Exception/error response did not contain error code ". | ||
| 1125 | "'$ok_errors' ".html_escape($reason)); | ||
| 1126 | $n++; | ||
| 1127 | next; | ||
| 1128 | } | ||
| 1129 | } else { | ||
| 1130 | $self->log->fail("Invalid protocol version returned"); | ||
| 1131 | $self->abort("test_expected_errors - invalid protocol version"); | ||
| 1132 | } | ||
| 1133 | } | ||
| 1134 | my $total = scalar @request_list; | ||
| 1135 | if ($n==0) { | ||
| 1136 | $self->log->pass("All $total error requests properly handled"); | ||
| 1137 | } else { | ||
| 1138 | $self->log->warn("Only ".($total-$n)." out of $total error requests properly handled"); | ||
| 1139 | } | ||
| 1140 | return($n); | ||
| 1141 | } | ||
| 1142 | |||
| 1143 | |||
| 1144 | =head3 test_expected_v2_errors($earliest_datestamp,$metadata_prefix) | ||
| 1145 | |||
| 1146 | There are some additional exception tests for OAI-PMH version 2.0. | ||
| 1147 | |||
| 1148 | =cut | ||
| 1149 | |||
| 1150 | sub test_expected_v2_errors { | ||
| 1151 | my $self=shift; | ||
| 1152 | my ($earliest_datestamp,$metadata_prefix)=@_; | ||
| 1153 | |||
| 1154 | $self->log->start("Checking for version 2.0 specific exceptions"); | ||
| 1155 | |||
| 1156 | my $too_early_date=one_year_before($earliest_datestamp); | ||
| 1157 | |||
| 1158 | # format of entries: [ request_string, [error_codes_accepable], resaon ] | ||
| 1159 | my @request_list = ( | ||
| 1160 | [ "verb=ListRecords&metadataPrefix=".url_encode($metadata_prefix)."&from=2002-02-05&until=2002-02-06T05:35:00Z", ['badArgument'], | ||
| 1161 | 'The request has different granularities for the from and until parameters.' ], | ||
| 1162 | [ "verb=ListRecords&metadataPrefix=".url_encode($metadata_prefix)."&until=$too_early_date" , ['noRecordsMatch'], | ||
| 1163 | 'The request specified a date one year before the earliestDatestamp given in the Identify response. '. | ||
| 1164 | 'There should therefore not be any records with datestamps on or before this date and a noRecordsMatch '. | ||
| 1165 | 'error code should be returned.' ] | ||
| 1166 | ); | ||
| 1167 | |||
| 1168 | foreach my $rrr ( @request_list ) { | ||
| 1169 | my ($request_string,$error_codes,$reason)=@$rrr; | ||
| 1170 | |||
| 1171 | my $req=$self->base_url."?$request_string"; | ||
| 1172 | my $response = $self->make_request($req); | ||
| 1173 | # parse the response content for the desired error code | ||
| 1174 | unless ( $self->parse_response($req,$response) ) { | ||
| 1175 | $self->log->fail("Error in parsing XML response to exception request: $request_string"); | ||
| 1176 | next; | ||
| 1177 | } | ||
| 1178 | # check that there is at least the desired error code | ||
| 1179 | my $ok_errors=join(' or ',@$error_codes); | ||
| 1180 | my $error_elements = $self->doc->getElementsByTagName('error'); | ||
| 1181 | if ( !$error_elements or $error_elements->getLength==0 ) { | ||
| 1182 | $self->log->fail("Failed to extract error code from the response to request: ". | ||
| 1183 | "$request_string $reason"); | ||
| 1184 | } elsif (my $matching_code=$self->error_elements_include($error_elements, $error_codes) ) { | ||
| 1185 | $self->log->pass("Error response correctly includes error code '$matching_code'"); | ||
| 1186 | } else { | ||
| 1187 | $self->log->fail("Error code $ok_errors not found in response but should be given ". | ||
| 1188 | "to the request: $request_string $reason"); | ||
| 1189 | } | ||
| 1190 | } | ||
| 1191 | return; | ||
| 1192 | } | ||
| 1193 | |||
| 1194 | |||
| 1195 | =head2 METHODS TO TEST USE OF HTTP POST | ||
| 1196 | |||
| 1197 | =head3 test_post_requests() | ||
| 1198 | |||
| 1199 | Test responses to POST requests. Do both the simplest possible -- the Identify | ||
| 1200 | verb -- and a GetRecord request which uses two additional parameters. | ||
| 1201 | |||
| 1202 | =cut | ||
| 1203 | |||
| 1204 | sub test_post_requests { | ||
| 1205 | my $self=shift; | ||
| 1206 | my ($metadata_prefix)=@_; | ||
| 1207 | |||
| 1208 | $self->log->start("Checking that HTTP POST requests are handled correctly"); | ||
| 1209 | |||
| 1210 | $self->test_post_request(1,{verb => "Identify"}); | ||
| 1211 | |||
| 1212 | my $record_id=$self->example_record_id; | ||
| 1213 | if ($record_id) { | ||
| 1214 | $self->test_post_request(2,{verb => "GetRecord", 'identifier' => $record_id, 'metadataPrefix' => $metadata_prefix}); | ||
| 1215 | } else { | ||
| 1216 | $self->log->fail("Cannot test GetRecord via POST without and example record identifier"); | ||
| 1217 | } | ||
| 1218 | } | ||
| 1219 | |||
| 1220 | |||
| 1221 | # Called just by test_post_requests to actually run the test | ||
| 1222 | # | ||
| 1223 | sub test_post_request { | ||
| 1224 | my $self=shift; | ||
| 1225 | my ($num, $post_data) = @_; | ||
| 1226 | my $response = $self->make_request($self->base_url, $post_data); | ||
| 1227 | if ($response->is_success) { | ||
| 1228 | my $verb = $post_data->{verb}; | ||
| 1229 | if ( $self->is_verb_response($response,$verb) ) { | ||
| 1230 | $self->log->pass("POST test $num for $verb was successful"); | ||
| 1231 | } elsif ( $self->check_error_response($response) ) { | ||
| 1232 | $self->log->fail("POST test $num for $verb was unsuccessful, an OAI error ". | ||
| 1233 | "response was received"); | ||
| 1234 | } else { | ||
| 1235 | $self->log->fail("POST test $num for $verb was unsuccessful, got neither a ". | ||
| 1236 | "valid response nor an error"); | ||
| 1237 | } | ||
| 1238 | } else { | ||
| 1239 | $self->log->fail("POST test $num was unsuccessful. Server returned HTTP Status: '". | ||
| 1240 | $response->status_line."'"); | ||
| 1241 | } | ||
| 1242 | } | ||
| 1243 | |||
| 1244 | |||
| 1245 | =head2 METHODS CHECKING ELEMENTS WITHIN VERB AND ERROR RESPONSES | ||
| 1246 | |||
| 1247 | =head3 check_response_date($req, $doc) | ||
| 1248 | |||
| 1249 | Check responseDate for being in UTC format | ||
| 1250 | (should perhaps also check that it is at least the current day?) | ||
| 1251 | |||
| 1252 | =cut | ||
| 1253 | |||
| 1254 | sub check_response_date { | ||
| 1255 | my $self=shift; | ||
| 1256 | my ($req, $doc) = @_; | ||
| 1257 | |||
| 1258 | my $elements = $self->doc->getElementsByTagName('responseDate'); | ||
| 1259 | # assume rest of validity already checked, just take first | ||
| 1260 | my $item; | ||
| 1261 | my $child; | ||
| 1262 | if ($elements and $item=$elements->item(0) and $child=$item->getFirstChild()) { | ||
| 1263 | my $date = $child->getData(); | ||
| 1264 | if ($date=~/\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ/) { | ||
| 1265 | $self->log->pass("responseDate has correct format: $date"); | ||
| 1266 | } else { | ||
| 1267 | $self->log->fail("Bad responseDate of $date, this is not in UTC DateTime ". | ||
| 1268 | "(YYYY-MM-DDThh:mm:ssZ) format"); | ||
| 1269 | } | ||
| 1270 | } else { | ||
| 1271 | $self->log->fail("Failed to extract responseDate"); | ||
| 1272 | } | ||
| 1273 | } | ||
| 1274 | |||
| 1275 | |||
| 1276 | =head3 check_schema_name($req, $doc) | ||
| 1277 | |||
| 1278 | Given the response to one of the OAI verbs, make sure that it it | ||
| 1279 | going to be validated against the "official" OAI schema, and not | ||
| 1280 | one that the repository made up for itself. If the response can't | ||
| 1281 | be parsed, or if there is no OAI-PMH element, or if the schema is | ||
| 1282 | incorrect, print an error message and bump the error_count. | ||
| 1283 | |||
| 1284 | Return true if the schema name and date check out, else return undef | ||
| 1285 | |||
| 1286 | =cut | ||
| 1287 | |||
| 1288 | sub check_schema_name { | ||
| 1289 | my $self=shift; | ||
| 1290 | my ($req, $doc) = @_; | ||
| 1291 | |||
| 1292 | my $namespace = 'http://www.openarchives.org/OAI/2.0/'; | ||
| 1293 | my $location = 'http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd'; | ||
| 1294 | |||
| 1295 | my $elements = $self->doc->getElementsByTagName('OAI-PMH'); #NodeList | ||
| 1296 | unless ( $elements->getLength() > 0 ) { | ||
| 1297 | $self->log->fail("Response to $req did not contain a OAI-PMH element"); | ||
| 1298 | return(0); | ||
| 1299 | } | ||
| 1300 | my $attributes = $elements->item(0)->getAttributes; #Node->NamedNodeMap | ||
| 1301 | my $attr = $attributes->getNamedItem('xsi:schemaLocation'); #Node | ||
| 1302 | unless ( $attr ) { | ||
| 1303 | $self->log->fail("No xsi:schemaLocation attribute for the OAI-PMH element was ". | ||
| 1304 | "found, expected xsi:schemaLocation=\"$namespace $location\""); | ||
| 1305 | return(0); | ||
| 1306 | } | ||
| 1307 | $attr = $attributes->getNamedItem('xsi:schemaLocation'); #Node | ||
| 1308 | my $pair = $attr->getNodeValue(); # must pair OAI namespace with schema | ||
| 1309 | unless ( $pair =~ /^\s?$namespace\s*$location/ ) { | ||
| 1310 | $self->log->fail("Error in pairing OAI namespace with schema location, expected: ". | ||
| 1311 | "xsi:schemaLocation=\"$namespace $location\" but got $pair"); | ||
| 1312 | return(0); | ||
| 1313 | } | ||
| 1314 | return(1); | ||
| 1315 | } | ||
| 1316 | |||
| 1317 | |||
| 1318 | =head3 check_protocol_version | ||
| 1319 | |||
| 1320 | Extract the protocol version being used from the Identify response, check that it is | ||
| 1321 | valid and then abort unless 2.0. | ||
| 1322 | |||
| 1323 | =cut | ||
| 1324 | |||
| 1325 | sub check_protocol_version { | ||
| 1326 | my $self=shift; | ||
| 1327 | my $doc; | ||
| 1328 | # Extract the version number of the validator to run | ||
| 1329 | my $x = $self->doc->getElementsByTagName('protocolVersion'); | ||
| 1330 | if (not $x) { | ||
| 1331 | $self->abort("Unknown protocol version, failed to extract protocolVersion element from Identify response"); | ||
| 1332 | } | ||
| 1333 | my $protocol_version = $x->item(0)->getFirstChild->getData; | ||
| 1334 | if ($protocol_version ne '2.0' and | ||
| 1335 | $protocol_version ne '1.1' and | ||
| 1336 | $protocol_version ne '1.0') { | ||
| 1337 | $self->abort("Invalid protocol version ($protocol_version)"); | ||
| 1338 | } | ||
| 1339 | $self->protocol_version( $protocol_version ); | ||
| 1340 | if ($protocol_version ne '2.0') { | ||
| 1341 | $self->abort("Repository reports OAI-PMH protocol version $protocol_version and will not be validated. Guidelines for upgrading to 2.0 can be found at http://www.openarchives.org/OAI/2.0/migration.htm\n\n"); | ||
| 1342 | } | ||
| 1343 | $self->log->pass("Correctly reports OAI-PMH protocol version 2.0"); | ||
| 1344 | } | ||
| 1345 | |||
| 1346 | |||
| 1347 | =head2 is_verb_response($reponse,$verb) | ||
| 1348 | |||
| 1349 | Return true if $response is a response for the specified $verb. | ||
| 1350 | |||
| 1351 | FIXME -- need better checks! | ||
| 1352 | |||
| 1353 | =cut | ||
| 1354 | |||
| 1355 | sub is_verb_response { | ||
| 1356 | my $self=shift; | ||
| 1357 | my ($response,$verb) = @_; | ||
| 1358 | my $doc; | ||
| 1359 | eval { $doc=$self->parser->parse($response->content); }; | ||
| 1360 | return unless $doc; # We can't parse it so it isn't a valid doc | ||
| 1361 | my $verb_elements = $doc->getElementsByTagName($verb); | ||
| 1362 | return(1) if ( $verb_elements and $verb_elements->getLength==1 ); | ||
| 1363 | return; # not the one element we expected | ||
| 1364 | } | ||
| 1365 | |||
| 1366 | |||
| 1367 | =head3 error_elements_include($error_elements,$error_codes) | ||
| 1368 | |||
| 1369 | Determine whether the list of error elements ($error_elements) includes at least | ||
| 1370 | one of the desired codes. Return string with first matching error code, else | ||
| 1371 | return false/nothing. | ||
| 1372 | |||
| 1373 | Does a sanity check on $error_list to check that it is set and has length>0 | ||
| 1374 | before trying to match, so cose calling it can simply do a | ||
| 1375 | getElementsByTagName or similar before caling. | ||
| 1376 | |||
| 1377 | =cut | ||
| 1378 | |||
| 1379 | sub error_elements_include { | ||
| 1380 | my $self=shift; | ||
| 1381 | my ($error_elements, $error_codes) = @_; | ||
| 1382 | # sanity check | ||
| 1383 | return if (!$error_elements or $error_elements->getLength==0); | ||
| 1384 | for (my $i=0; $i<$error_elements->getLength; $i++) { | ||
| 1385 | foreach my $ec (@$error_codes) { | ||
| 1386 | my $code = $error_elements->item($i)->getAttribute('code') || 'no-code-attribute'; | ||
| 1387 | $self->log->note("$code =? $ec") if ($self->debug); | ||
| 1388 | return($ec) if ($code eq $ec); | ||
| 1389 | } | ||
| 1390 | } | ||
| 1391 | return; | ||
| 1392 | } | ||
| 1393 | |||
| 1394 | |||
| 1395 | |||
| 1396 | =head3 check_error_response($response) | ||
| 1397 | |||
| 1398 | Given the response to an HTTP request, make sure it is not an | ||
| 1399 | OAI-PMH error message. The $response is a success. If it is an | ||
| 1400 | OAI error message, return 2; if the response cannot be parsed, return | ||
| 1401 | -1; otherwise return undef (it must be a real Identify response). | ||
| 1402 | |||
| 1403 | FIXME -- need better checks! | ||
| 1404 | |||
| 1405 | FIXME -- need to merge thic functionality in with is_error_response | ||
| 1406 | |||
| 1407 | =cut | ||
| 1408 | |||
| 1409 | sub check_error_response { | ||
| 1410 | my $self=shift; | ||
| 1411 | my ($response) = @_; | ||
| 1412 | my $doc; | ||
| 1413 | eval { $doc=$self->parser->parse($response->content); }; | ||
| 1414 | return unless $doc; # We can't parse it so it isn't a valid error | ||
| 1415 | my $error_elements = $doc->getElementsByTagName('error'); | ||
| 1416 | return(1) if ( $error_elements and $error_elements->getLength() > 0 ); | ||
| 1417 | return; # no error codes so not an error response | ||
| 1418 | } | ||
| 1419 | |||
| 1420 | |||
| 1421 | =head3 get_earliest_datestamp() | ||
| 1422 | |||
| 1423 | A new exception check for Version 2.0 raises noRecordsMatch errorcode | ||
| 1424 | if the set of records returned by ListRecords is empty. This requires | ||
| 1425 | that we know the earliest date in the repository. Also check that the | ||
| 1426 | earliest date matches the specified granularity. | ||
| 1427 | |||
| 1428 | Called only for version 2.0 or greater. | ||
| 1429 | |||
| 1430 | Since the Identify response has already been validated, we know | ||
| 1431 | there is exactly one earliestDatestamp element in the current document. | ||
| 1432 | Extract this value, check it, and if it looks good then set | ||
| 1433 | $self->earliest_datestamp and return false. | ||
| 1434 | |||
| 1435 | If there is an error then return string explaining that. | ||
| 1436 | |||
| 1437 | =cut | ||
| 1438 | |||
| 1439 | sub get_earliest_datestamp { | ||
| 1440 | my $self=shift; | ||
| 1441 | |||
| 1442 | my $earliest = $self->doc->getElementsByTagName('earliestDatestamp'); | ||
| 1443 | my $el = $earliest->item(0); | ||
| 1444 | return("Can't get earliestDatestamp element from Identify response.") unless ($el); | ||
| 1445 | return("earliestDatestamp element is empty in Identify response.") unless ($el->getFirstChild); | ||
| 1446 | |||
| 1447 | my $error=''; | ||
| 1448 | my $earliest_datestamp = $el->getFirstChild->getData; | ||
| 1449 | $self->log->note("Earliest datestamp in repository is $earliest_datestamp") if $self->debug; | ||
| 1450 | |||
| 1451 | $earliest_datestamp =~ /^([0-9]{4})-([0-9][0-9])-([0-9][0-9])(.*)$/; | ||
| 1452 | if ($1 eq '' || $2 eq '' || $3 eq '') { | ||
| 1453 | $error="is not in ISO8601 format"; | ||
| 1454 | } elsif ( $4 eq '' and $self->granularity eq 'seconds') { | ||
| 1455 | $error="must have seconds granularity (format YYYY-MM-DDThh:mm:ssZ) to match ". | ||
| 1456 | "the granularity for the repository. The granularity has been set to seconds ". | ||
| 1457 | "by the granularity element of the Identify response.\n"; | ||
| 1458 | } elsif ( $4 ne '' and $self->granularity eq 'days') { | ||
| 1459 | $error="must have days granularity (format YYYY-MM-DD) to match the granularity for ". | ||
| 1460 | "the repository. The granularity has been set to days by the granularity ". | ||
| 1461 | "element of the Identify response (or that element is bad/missing).\n"; | ||
| 1462 | } elsif ( $self->granularity eq 'seconds' and $4 !~ /^T\d\d:\d\d:\d\d(\.\d+)?Z$/ ) { | ||
| 1463 | $error="does not have the correct format for the time part of the UTCdatetime. The ". | ||
| 1464 | "overall format must be YYYY-MM-DDThh:mm:ssZ.\n"; | ||
| 1465 | } | ||
| 1466 | if ($error) { | ||
| 1467 | # Sanitize for error message | ||
| 1468 | return("The earliestDatestamp in the identify response (". | ||
| 1469 | sanitize($earliest_datestamp).") $error"); | ||
| 1470 | } else { | ||
| 1471 | $self->earliest_datestamp($earliest_datestamp); | ||
| 1472 | return; | ||
| 1473 | } | ||
| 1474 | } | ||
| 1475 | |||
| 1476 | |||
| 1477 | =head3 parse_granularity($granularity_element) | ||
| 1478 | |||
| 1479 | Parse contents of the granularity element of the Identify response. Returns either | ||
| 1480 | 'days', 'seconds' or nothing on failure. Sets $self->granularity if valid, otherwise | ||
| 1481 | does not change setting. | ||
| 1482 | |||
| 1483 | As of v2.0 the granularity element is mandatory, see: | ||
| 1484 | http://www.openarchives.org/OAI/openarchivesprotocol.html#Identify | ||
| 1485 | |||
| 1486 | =cut | ||
| 1487 | |||
| 1488 | sub parse_granularity { | ||
| 1489 | my $self=shift; | ||
| 1490 | my ($gran) = @_; | ||
| 1491 | if (!$gran or $gran->getLength==0) { | ||
| 1492 | $self->log->fail("Missing granularity element"); | ||
| 1493 | return; | ||
| 1494 | } elsif ($gran->getLength>1) { | ||
| 1495 | $self->log->fail("Multiple granularity elements"); | ||
| 1496 | return; | ||
| 1497 | } | ||
| 1498 | #schema validation guarantees that there is a spec here | ||
| 1499 | my $el=$gran->item(0)->getFirstChild->getData; | ||
| 1500 | if ($el eq 'YYYY-MM-DD') { | ||
| 1501 | $self->granularity('days'); | ||
| 1502 | return($self->granularity); | ||
| 1503 | } elsif ($el eq 'YYYY-MM-DDThh:mm:ssZ') { | ||
| 1504 | $self->granularity('seconds'); | ||
| 1505 | return($self->granularity); | ||
| 1506 | } else { | ||
| 1507 | $self->log->fail("Bad value for the granularity element '$el', must be either ". | ||
| 1508 | "YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ"); | ||
| 1509 | return; | ||
| 1510 | } | ||
| 1511 | } | ||
| 1512 | |||
| 1513 | |||
| 1514 | =head3 get_datestamp_granularity($datestamp) | ||
| 1515 | |||
| 1516 | Parse the datestamp supplied and return 'days' if it is valid with granularity | ||
| 1517 | of days, 'seconds' if it is valid for seconds granularity, and nothing if it is not | ||
| 1518 | valid. | ||
| 1519 | |||
| 1520 | # FIXME - should add more validation | ||
| 1521 | |||
| 1522 | =cut | ||
| 1523 | |||
| 1524 | sub get_datestamp_granularity { | ||
| 1525 | my $self=shift; | ||
| 1526 | my ($datestamp)=@_; | ||
| 1527 | if ($datestamp=~/^(\d\d\d\d)-(\d\d)-(\d\d)$/) { | ||
| 1528 | return 'days' if ($2>=1 and $2<=12 and $3>=1 and $3<=31); | ||
| 1529 | } elsif ($datestamp=~/^(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)(\.\d+)?Z$/) { | ||
| 1530 | return 'seconds' if ($2>=1 and $2<=12 and $3>=1 and $3<=31 and $4<24 and $5<60); | ||
| 1531 | } | ||
| 1532 | return; | ||
| 1533 | } | ||
| 1534 | |||
| 1535 | |||
| 1536 | =head3 is_no_records_match | ||
| 1537 | |||
| 1538 | Returns true if the current document contains and error code element with the code "noRecordsMatch" | ||
| 1539 | |||
| 1540 | ### FIXME - should be merged into an extended is_error_response | ||
| 1541 | |||
| 1542 | =cut | ||
| 1543 | |||
| 1544 | sub is_no_records_match { | ||
| 1545 | my $self=shift; | ||
| 1546 | my $error_elements = $self->doc->getElementsByTagName('error'); | ||
| 1547 | return( $self->error_elements_include($error_elements, ['noRecordsMatch']) ); | ||
| 1548 | } | ||
| 1549 | |||
| 1550 | |||
| 1551 | =head3 get_resumption_token() | ||
| 1552 | |||
| 1553 | See if there is a resumptionToken with this response, return | ||
| 1554 | value if present, empty if not or if there is some other error. | ||
| 1555 | |||
| 1556 | =cut | ||
| 1557 | |||
| 1558 | sub get_resumption_token { | ||
| 1559 | my $self=shift; | ||
| 1560 | |||
| 1561 | my $tokenList = $self->doc->getElementsByTagName('resumptionToken'); | ||
| 1562 | if ( !$tokenList or $tokenList->getLength()==0 ) { | ||
| 1563 | return; #no resumptionToken | ||
| 1564 | } | ||
| 1565 | |||
| 1566 | # Dig out the resumption token from the document | ||
| 1567 | my $token = $tokenList->item(0)->getFirstChild(); | ||
| 1568 | |||
| 1569 | # Try getting the resumption token, $token will be will be undefined | ||
| 1570 | # unless the element has content | ||
| 1571 | if ($token) { | ||
| 1572 | return($token->getData()); | ||
| 1573 | } | ||
| 1574 | return; | ||
| 1575 | } | ||
| 1576 | |||
| 1577 | |||
| 1578 | =head3 is_error_response($details) | ||
| 1579 | |||
| 1580 | Look at the parsed response in $self->doc to see if it is an error response, | ||
| 1581 | parse data and return true if it is. | ||
| 1582 | |||
| 1583 | Returns true (a printable string containing the error messages) if response was a valid | ||
| 1584 | OAI_PMH error response, codes in %$details if a hash reference is passed in. | ||
| 1585 | |||
| 1586 | =cut | ||
| 1587 | |||
| 1588 | sub is_error_response { | ||
| 1589 | my $self=shift; | ||
| 1590 | my ($details)=@_; | ||
| 1591 | $details={} unless (ref($details) eq 'HASH'); #dummy hash unless one supplied | ||
| 1592 | # | ||
| 1593 | my $error_elements = $self->doc->getElementsByTagName('error'); | ||
| 1594 | if ($error_elements and $error_elements->getLength()>=1) { | ||
| 1595 | my $msg=''; | ||
| 1596 | for (my $i=0; $i<$error_elements->getLength; $i++) { | ||
| 1597 | my $code=$error_elements->item($i)->getAttribute("code"); | ||
| 1598 | my $child=$error_elements->item($i)->getFirstChild(); | ||
| 1599 | unless ($child) { | ||
| 1600 | # Warn about no content unless it is the special case of noSetHierarchy | ||
| 1601 | # where the error code really is sufficient | ||
| 1602 | unless ($code eq 'noSetHierarchy') { | ||
| 1603 | $self->log->warn("No human readable message included in error element for ". | ||
| 1604 | "$code error, this is discouraged"); | ||
| 1605 | } | ||
| 1606 | $details->{$code}='[NO MESSAGE RETURNED]'; | ||
| 1607 | $msg.="[$code] "; | ||
| 1608 | } else { | ||
| 1609 | $details->{$code}=$child->getData(); | ||
| 1610 | $msg.="[$code: $details->{$code}] "; | ||
| 1611 | } | ||
| 1612 | } | ||
| 1613 | return($msg); | ||
| 1614 | } else { | ||
| 1615 | return; | ||
| 1616 | } | ||
| 1617 | } | ||
| 1618 | |||
| 1619 | |||
| 1620 | =head3 get_admin_email() | ||
| 1621 | |||
| 1622 | Extract admin email from a parsed Identify response in $self->doc). | ||
| 1623 | Also note that the email target may have been set via form option | ||
| 1624 | |||
| 1625 | Returns the pair of ($email,$error) where $email is the combined | ||
| 1626 | set of email addresses (comma separated). $error will be undef | ||
| 1627 | or a string with error message to users. | ||
| 1628 | |||
| 1629 | =cut | ||
| 1630 | |||
| 1631 | sub get_admin_email { | ||
| 1632 | my $self=shift; | ||
| 1633 | |||
| 1634 | my $adminEmailElements = $self->doc->getElementsByTagName('adminEmail'); | ||
| 1635 | my @emails=(); | ||
| 1636 | my $n = $adminEmailElements->getLength; | ||
| 1637 | if ($n > 0) { | ||
| 1638 | my $name_node = $adminEmailElements->item(0)->getFirstChild; | ||
| 1639 | if ($name_node) { | ||
| 1640 | for (my $i=0; $i<$n; $i++) { | ||
| 1641 | my $e=$adminEmailElements->item($i)->getFirstChild->getData; | ||
| 1642 | if ($e=~s/mailto://g) { | ||
| 1643 | $self->log->warn("Stripped mailto: prefix from adminEmail address, this ". | ||
| 1644 | "should not be included."); | ||
| 1645 | } | ||
| 1646 | if (my $msg=$self->bad_admin_email($e)) { | ||
| 1647 | return(undef,$msg); | ||
| 1648 | } | ||
| 1649 | push(@emails,$e); | ||
| 1650 | } | ||
| 1651 | } else { | ||
| 1652 | $self->log->fail("adminEmail element is empty!"); | ||
| 1653 | return(undef); | ||
| 1654 | } | ||
| 1655 | } else { | ||
| 1656 | $self->log->fail("No adminEmail element!"); | ||
| 1657 | return(undef); | ||
| 1658 | } | ||
| 1659 | my $email=join(',',@emails); | ||
| 1660 | $self->log->pass("Administrator email address is '$email'"); | ||
| 1661 | return($email); | ||
| 1662 | } | ||
| 1663 | |||
| 1664 | |||
| 1665 | =head3 bad_admin_email($admin_email) | ||
| 1666 | |||
| 1667 | Check for some stupid email addresses to avoid so much bounced email. | ||
| 1668 | Returns a string (True) if bad, else nothing. | ||
| 1669 | |||
| 1670 | =cut | ||
| 1671 | |||
| 1672 | sub bad_admin_email { | ||
| 1673 | my $self=shift; | ||
| 1674 | my ($admin_email)=@_; | ||
| 1675 | if ($admin_email=~/\@localhost$/) { | ||
| 1676 | $self->log->fail("adminEmail '$admin_email' is local. This must be corrected to a ". | ||
| 1677 | "valid globally resolvable email address before tests can continue"); | ||
| 1678 | return("local adminEmail"); | ||
| 1679 | } elsif ($admin_email!~/^\w[\w\-\.]+\@[a-zA-Z0-9\-\.]+\.[a-z]{2,}$/) { | ||
| 1680 | $self->log->fail("adminEmail '$admin_email' looks bogus. This must be corrected to ". | ||
| 1681 | "a valid email address before tests can continue"); | ||
| 1682 | return("looks like bogus adminEmail"); | ||
| 1683 | } | ||
| 1684 | return; | ||
| 1685 | } | ||
| 1686 | |||
| 1687 | |||
| 1688 | =head2 METHODS FOR MAKING REQUESTS AND PARSING RESPONSES | ||
| 1689 | |||
| 1690 | =head3 make_request_and_validate($verb, $req) | ||
| 1691 | |||
| 1692 | Given the base URL that we are validating, the Verb that we are checking | ||
| 1693 | and the complete query to be sent to the OAI server, get the response to | ||
| 1694 | the verb. Validation has already been done, so we need only do some | ||
| 1695 | special checks here. Return the response to the OAI verb, | ||
| 1696 | or undef if the OAI server failed to respond to that verb. | ||
| 1697 | |||
| 1698 | Side effects: errors may be printed and error_count bumped. | ||
| 1699 | If the verb involved is "Identify" then set the version number and the | ||
| 1700 | email address, assuming that some response has been obtained. | ||
| 1701 | |||
| 1702 | Simple well-formedness is checked by this routine. An undef exit means | ||
| 1703 | that any calling code should fail the test but need not report 'no response'. | ||
| 1704 | |||
| 1705 | If the response is true then $self->doc contains a parsed XML | ||
| 1706 | document. | ||
| 1707 | |||
| 1708 | This is the usual way we make requests with integrated parsing and error | ||
| 1709 | checking. This method is built around calls to L<make_request> and | ||
| 1710 | L<parse_response>. | ||
| 1711 | |||
| 1712 | =cut | ||
| 1713 | |||
| 1714 | sub make_request_and_validate { | ||
| 1715 | my $self=shift; | ||
| 1716 | my ($verb, $req) = @_; | ||
| 1717 | |||
| 1718 | my $response = $self->make_request($req); | ||
| 1719 | |||
| 1720 | unless ( $response->is_success ) { | ||
| 1721 | my $status = $response->status_line; | ||
| 1722 | my $age = $response->current_age; | ||
| 1723 | my $lifetime = $response->freshness_lifetime; | ||
| 1724 | my $is_fresh = $response->is_fresh; | ||
| 1725 | $self->log->fail("Server failed to respond to the $verb request (HTTP header ". | ||
| 1726 | "values: status=$status, age=$age, lifetime=$lifetime, ". | ||
| 1727 | "is fresh:=$is_fresh)"); | ||
| 1728 | return; | ||
| 1729 | } | ||
| 1730 | |||
| 1731 | unless ($self->parse_response($req, $response)) { | ||
| 1732 | $self->log->fail("Failed to parse response"); | ||
| 1733 | return; | ||
| 1734 | } | ||
| 1735 | |||
| 1736 | # Check that the responseDate is in UTC format | ||
| 1737 | $self->check_response_date($req,$self->doc); | ||
| 1738 | # Check that the response refers to the "official" OAI schema | ||
| 1739 | $self->check_schema_name($req,$self->doc); | ||
| 1740 | |||
| 1741 | return($response); | ||
| 1742 | } | ||
| 1743 | |||
| 1744 | |||
| 1745 | =head3 make_request($url,$post_data) | ||
| 1746 | |||
| 1747 | Routine to GET or POST a request, handle 503's, and return the response | ||
| 1748 | |||
| 1749 | Second parameter, $post_data, must be hasfref to POST data to indicate that | ||
| 1750 | the request should be an HTTP POST request instead of a GET. | ||
| 1751 | |||
| 1752 | =cut | ||
| 1753 | |||
| 1754 | sub make_request { | ||
| 1755 | my $self=shift; | ||
| 1756 | my ($url,$post_data) = @_; | ||
| 1757 | |||
| 1758 | # Is this https and do we allow that? | ||
| 1759 | if (is_https_uri($url)) { | ||
| 1760 | $self->uses_https(1); | ||
| 1761 | if (not $self->allow_https) { | ||
| 1762 | $self->abort("URI $url is https. Use of https URIs is not allowed ". | ||
| 1763 | "by the OAI-PMH v2.0 specification"); | ||
| 1764 | } | ||
| 1765 | } | ||
| 1766 | |||
| 1767 | my $request; | ||
| 1768 | if ($post_data) { | ||
| 1769 | my $content_msg=''; #nice string to report | ||
| 1770 | # Sort keys in alpha order for consistent behavior | ||
| 1771 | foreach my $k (sort keys(%$post_data)) { | ||
| 1772 | my $v=$post_data->{$k}; | ||
| 1773 | $content_msg.="$k:$v "; | ||
| 1774 | } | ||
| 1775 | $self->log->request($url,'POST',$content_msg); | ||
| 1776 | $request = POST($url,'Content'=>$post_data); | ||
| 1777 | } else { | ||
| 1778 | $self->log->request($url,'GET'); | ||
| 1779 | $request = GET($url); | ||
| 1780 | } | ||
| 1781 | my $response; | ||
| 1782 | my $tries=0; | ||
| 1783 | my $try_again = 1; | ||
| 1784 | while ( $try_again ) { | ||
| 1785 | #$ua->max_redirect(0); | ||
| 1786 | $response = $self->ua->request($request); | ||
| 1787 | # | ||
| 1788 | # Write response if requested | ||
| 1789 | if ($self->save_all_responses) { | ||
| 1790 | my $response_file="/tmp/".$self->run_id.".".$self->response_number; | ||
| 1791 | open(my $fh,'>',$response_file) || $self->abort("Can't write response $response_file: $!"); | ||
| 1792 | print {$fh} $response->content(); | ||
| 1793 | $self->log->note("Response saved as $response_file") if ($self->debug); | ||
| 1794 | close($fh); | ||
| 1795 | $self->{response_number}++; | ||
| 1796 | } | ||
| 1797 | $tries++; | ||
| 1798 | if ($tries > $self->max_retries) { | ||
| 1799 | $self->abort("Too many 503 Retry-After or 302 Redirect responses received in a row"); | ||
| 1800 | } | ||
| 1801 | # | ||
| 1802 | # Check response for 503 and 302 | ||
| 1803 | if ($response->code eq '503') { | ||
| 1804 | # 503 (Retry-After), expect to get a time too | ||
| 1805 | $self->uses_503(1); | ||
| 1806 | if (defined $response->header("Retry-After")) { | ||
| 1807 | my $retryAfter=$response->header("Retry-After"); | ||
| 1808 | if ($retryAfter=~/^\d+$/) { | ||
| 1809 | if ($retryAfter<=3600) { | ||
| 1810 | ###FIXME: Should check the Retry-After value carefully and barf if bad | ||
| 1811 | my $sleep_time = 1 + $response->header("Retry-After"); | ||
| 1812 | $self->log->note("Status: ".$response->code(). | ||
| 1813 | " -- going to sleep for $sleep_time seconds."); | ||
| 1814 | sleep $sleep_time; | ||
| 1815 | } else { | ||
| 1816 | $self->abort("503 response with Retry-After > 1hour (3600s), aborting"); | ||
| 1817 | } | ||
| 1818 | } else { | ||
| 1819 | $self->log->fail("503 response with bad (non-numeric) Retry-After time, ". | ||
| 1820 | "will wait 10s"); | ||
| 1821 | sleep 10; | ||
| 1822 | } | ||
| 1823 | } else { | ||
| 1824 | $self->log->warn("503 response without Retry-After time, will wait 10s"); | ||
| 1825 | sleep 10; | ||
| 1826 | } | ||
| 1827 | } elsif ($response->code eq '302') { | ||
| 1828 | # 302 (Found) redirect | ||
| 1829 | my $loc=$response->header('Location'); | ||
| 1830 | if ($loc!~m%^http://([^\?&]+)%) { | ||
| 1831 | if (is_https_uri($loc)) { | ||
| 1832 | $self->uses_https(1); | ||
| 1833 | if (not $self->allow_https) { | ||
| 1834 | $self->abort("Redirect URI specified in 302 response is https. Use of ". | ||
| 1835 | "https URIs is not allowed by the OAI-PMH v2.0 specification"); | ||
| 1836 | } | ||
| 1837 | } else { | ||
| 1838 | $self->abort("Bad redirect URI specified in 302 response"); | ||
| 1839 | } | ||
| 1840 | } | ||
| 1841 | # Make new request | ||
| 1842 | if ($post_data and $loc!~/\?/) { #don't do POST if new Location includes ? | ||
| 1843 | $request = POST($loc,'Content'=>$post_data); | ||
| 1844 | } else { | ||
| 1845 | $request = GET($loc); | ||
| 1846 | } | ||
| 1847 | } elsif ($response->code eq '501') { | ||
| 1848 | $self->abort("Got 501 Not Implemented response which may either have come from ". | ||
| 1849 | "the server or have been generated within the validator because the ". | ||
| 1850 | "request type (perhaps https) is not supported."); | ||
| 1851 | } else { | ||
| 1852 | $try_again=0; | ||
| 1853 | } | ||
| 1854 | } | ||
| 1855 | # Check for oversize limit (indicated by X-Content-Range header) | ||
| 1856 | if (defined $response->header('X-Content-Range')) { | ||
| 1857 | $self->log->fail("Response to <$url> exceeds maximum size limit (".$self->max_size." bytes), discarded. ". | ||
| 1858 | "While this limit is set only in this validation program you should not use excessively ". | ||
| 1859 | "large responses as service providers will likely have problems parsing the XML. You ". | ||
| 1860 | "should split the responses using the resumptionToken mechanism. (X-Content-Range: '". | ||
| 1861 | $response->header('X-Content-Range')."' Content-Length: '".$response->content_length."')\n"); | ||
| 1862 | $response->content(''); | ||
| 1863 | } | ||
| 1864 | return $response; | ||
| 1865 | } | ||
| 1866 | |||
| 1867 | |||
| 1868 | =head3 parse_response($request_url,$response,$xml_reason) | ||
| 1869 | |||
| 1870 | Attempt to parse the HTTP response $response, examining both the response code | ||
| 1871 | and then attempting to parse the content as XML. | ||
| 1872 | |||
| 1873 | If $xml_reason is specified then this is added to the failure message, if | ||
| 1874 | nothing is specified then a standard message about UTF-8 issues is | ||
| 1875 | added. | ||
| 1876 | |||
| 1877 | Returns true on success and sets $self->doc with the parsed XML document. | ||
| 1878 | If unsuccessful, log an error message, bump the error count, and | ||
| 1879 | return false. | ||
| 1880 | |||
| 1881 | =cut | ||
| 1882 | |||
| 1883 | sub parse_response { | ||
| 1884 | my $self=shift; | ||
| 1885 | my ($request_url,$response,$xml_reason) = @_; | ||
| 1886 | $xml_reason='' unless (defined $xml_reason); | ||
| 1887 | # | ||
| 1888 | # Fail if reponse=undef, else check to see if response is ref to | ||
| 1889 | # response object or is string | ||
| 1890 | if (!defined($response) or not ref($response)) { | ||
| 1891 | $self->log->warn("Bad response from server"); | ||
| 1892 | return; | ||
| 1893 | } | ||
| 1894 | # Unpack the bits we want from response object | ||
| 1895 | my $code=$response->code; | ||
| 1896 | my $content=$response->content; | ||
| 1897 | # Check return code (if given) | ||
| 1898 | if ($code and $code=~/^[45]/) { | ||
| 1899 | $self->log->warn("Bad HTTP status code from server: $code"); | ||
| 1900 | return; | ||
| 1901 | } | ||
| 1902 | # | ||
| 1903 | # Check content | ||
| 1904 | my $doc; | ||
| 1905 | eval { $doc=$self->parser->parse($content); }; | ||
| 1906 | unless ( $doc ) { | ||
| 1907 | my $err=$@; | ||
| 1908 | $err=~s/^\s+//; | ||
| 1909 | $err=~s%at\s+/usr/lib/perl.*%%i; #trim stuff about our perl installation | ||
| 1910 | if ($request_url) { | ||
| 1911 | unless ($xml_reason) { | ||
| 1912 | $xml_reason="The most common reason for malformed responses is illegal bytes in ". | ||
| 1913 | "UTF-8 streams (e.g. the inclusion of Latin1 characters with codes>127 ". | ||
| 1914 | "without creating proper UTF-8 mutli-byte sequences). You might find ". | ||
| 1915 | "the utf8conditioner, found on the OAI tools page helpful for debugging."; | ||
| 1916 | } | ||
| 1917 | $self->log->warn("Malformed response: $err. $xml_reason"); | ||
| 1918 | } | ||
| 1919 | return; | ||
| 1920 | } | ||
| 1921 | # Set parsed document | ||
| 1922 | $self->doc( $doc ); | ||
| 1923 | return(1); | ||
| 1924 | } | ||
| 1925 | |||
| 1926 | |||
| 1927 | =head2 UTILITY FUNCTIONS | ||
| 1928 | |||
| 1929 | =head3 html_escape($str) | ||
| 1930 | |||
| 1931 | Escapes characters which have special meanings in HTML | ||
| 1932 | |||
| 1933 | =cut | ||
| 1934 | |||
| 1935 | sub html_escape { | ||
| 1936 | my $string = shift; | ||
| 1937 | $string =~ s/&/&/g; #must be first! | ||
| 1938 | $string =~ s/</</g; | ||
| 1939 | $string =~ s/>/>/g; | ||
| 1940 | $string =~ s/"/"/g; | ||
| 1941 | $string =~ s/'/'/g; | ||
| 1942 | return $string; | ||
| 1943 | } | ||
| 1944 | |||
| 1945 | =head3 one_year_before($date) | ||
| 1946 | |||
| 1947 | Assumes properly formatted date, decrements year by one | ||
| 1948 | via string manipulation and returns date. | ||
| 1949 | |||
| 1950 | =cut | ||
| 1951 | |||
| 1952 | sub one_year_before { | ||
| 1953 | my ($date)=@_; | ||
| 1954 | my ($year) = $date =~ /^([0-9]{4})/; | ||
| 1955 | my $year_minus_one = sprintf('%04d',($year - 1)); #make sure we get leading zeros | ||
| 1956 | $date =~ s/^$year/$year_minus_one/; | ||
| 1957 | return($date); | ||
| 1958 | } | ||
| 1959 | |||
| 1960 | =head3 url_encode($str) | ||
| 1961 | |||
| 1962 | Escape/encode any characters that aren't in the small safe set for URLs | ||
| 1963 | |||
| 1964 | =cut | ||
| 1965 | |||
| 1966 | sub url_encode { | ||
| 1967 | my $str=shift; | ||
| 1968 | $str =~ s/([^\w\/\,\- ])/sprintf("%%%02X",ord($1))/eg; | ||
| 1969 | $str =~ tr/ /+/; | ||
| 1970 | return($str); | ||
| 1971 | } | ||
| 1972 | |||
| 1973 | |||
| 1974 | =head3 is_https_uri($uri) | ||
| 1975 | |||
| 1976 | Return true if the URI is an https URI, false otherwise. | ||
| 1977 | |||
| 1978 | =cut | ||
| 1979 | |||
| 1980 | sub is_https_uri { | ||
| 1981 | my $uri=shift; | ||
| 1982 | return($uri=~m%^https:%); | ||
| 1983 | } | ||
| 1984 | |||
| 1985 | |||
| 1986 | =head3 sanitize($str) | ||
| 1987 | |||
| 1988 | Return a sanitized version of $str that doesn't contain odd | ||
| 1989 | characters and it not over 80 chars long. Will have the | ||
| 1990 | string '(sanitized)' appended if changed. | ||
| 1991 | |||
| 1992 | =cut | ||
| 1993 | |||
| 1994 | sub sanitize { | ||
| 1995 | my ($str)=@_; | ||
| 1996 | my $out=$str; | ||
| 1997 | $out=~s/[^\w\-:;.!@#%^*\(\) ]/_/g; | ||
| 1998 | $out=substr($out,0,80); | ||
| 1999 | if ($out ne $str) { | ||
| 2000 | $out.='(sanitized)'; | ||
| 2001 | } | ||
| 2002 | return($out); | ||
| 2003 | } | ||
| 2004 | |||
| 2005 | |||
| 2006 | =head1 SUPPORT | ||
| 2007 | |||
| 2008 | Please report any bugs of questions about validation via the | ||
| 2009 | OAI-PMH discussion list at L<https://groups.google.com/d/forum/oai-pmh>. | ||
| 2010 | Be sure to make it clear that you are talking about the | ||
| 2011 | HTTP::OAIPMH::Validator module. | ||
| 2012 | |||
| 2013 | =head1 AUTHORS | ||
| 2014 | |||
| 2015 | Simeon Warner, Donna Bergmark | ||
| 2016 | |||
| 2017 | =head1 HISTORY | ||
| 2018 | |||
| 2019 | This module is based on an OAI-PMH validator first written by Donna Bergmark | ||
| 2020 | (Cornell University) in 2001-01 for the OAI-PMH validation and registration | ||
| 2021 | service (L<http://www.openarchives.org/data/registerasprovider.html>). | ||
| 2022 | Simeon Warner (Cornell University) took over the validator and operation of | ||
| 2023 | the registration service in 2004-01, and then did a significant tidy/rework | ||
| 2024 | of the code. That code ran the validation and registration service with | ||
| 2025 | few changes through 2015-01. Some of the early work on the OAI-PMH validation | ||
| 2026 | service was supported through NSF award number 0127308. | ||
| 2027 | |||
| 2028 | Code was abstracted into this module 2015-01 by Simeon Warner and is | ||
| 2029 | used for the OAI-PMH validation and registration service on | ||
| 2030 | L<http://www.openarchives.org/pmh/>. | ||
| 2031 | |||
| 2032 | =head1 COPYRIGHT | ||
| 2033 | |||
| 2034 | Copyright 2001..2017 by Simeon Warner, Donna Bergmark. | ||
| 2035 | |||
| 2036 | This library is free software; you can redistribute it and/or modify it under | ||
| 2037 | the same terms as Perl itself. | ||
| 2038 | |||
| 2039 | =cut | ||
| 2040 | |||
| 2041 | 1; | ||