Validator.pm

filter: Statementstmt 94.4% 34 / 36 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 12 / 12 Podpod n/a totaltotal 95.8% 46 / 48 CC 1 SCAR 0.2
CC
1
Cov
94%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« Log.pm ↑ Summary
1 package HTTP::OAIPMH::Validator;
2  
3 =head1 NAME
4  
5 HTTP::OAIPMH::Validator - OAI-PMH validator class
6  
7 =head1 SYNOPSIS
8  
9 Validation suite for OAI-PMH data providers that checks for responses
10 in accord with OAI-PMH v2
11 L<http://www.openarchives.org/OAI/2.0/openarchivesprotocol.htm>.
12  
13 Typical use:
14  
15 use HTTP::OAIPMH::Validator;
16 use Try::Tiny;
17 my $val = HTTP::OAIPMH::Validator->new( base_url=>'http://example.com/oai' );
18 try {
19 $val->run_complete_validation;
20 } catch {
21 warn "oops, validation didn't run to completion: $!\n";
22 };
23 print "Validation status of data provider ".$val->base_url." is ".$val->status."\n";
24  
25 =cut
26  
27 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
28  
29 our $VERSION = '1.06';
30  
31 1 ▶use base qw(Class::Accessor::Fast);
Subroutine BEGIN: called CC=1 SCAR=0.0
32 1 ▶use Data::UUID;
Subroutine BEGIN: called CC=1 SCAR=0.0
33 1 ▶use Date::Manip;
Subroutine BEGIN: called CC=1 SCAR=0.0
34 1 ▶use HTTP::Request; # for rendering http queries
Subroutine BEGIN: called CC=1 SCAR=0.0
35 1 ▶use HTTP::Headers;
Subroutine BEGIN: called CC=1 SCAR=0.0
36 1 ▶use HTTP::Request::Common; # makes POST easier
Subroutine BEGIN: called CC=1 SCAR=0.0
37 1 ▶use HTTP::Status; # for checking error codes
Subroutine BEGIN: called CC=1 SCAR=0.0
38 1 ▶use LWP::UserAgent; # send http requests
Subroutine BEGIN: called CC=1 SCAR=0.0
39 1 ▶use LWP::Protocol::https; # explicit include so we fail without https support
Subroutine BEGIN: called CC=1 SCAR=0.0
40 1 ▶use URI::Escape; # excape special characters
Subroutine BEGIN: called CC=1 SCAR=0.0
41 0 ▶use XML::DOM;
Subroutine BEGIN: called CC=1 SCAR=2.6
42 use HTTP::OAIPMH::Log;
43  
44 =head2 METHODS
45  
46 =head3 new(%args)
47  
48 Create new HTTP::OAIPMH::Validator object and initialize counters.
49  
50 The following instance variables may be set via %args and have read-write
51 accessors (via L<Class::Accessor::Fast>):
52  
53 base_url - base URL of the data provdier being validated
54 run_id - UUID identifying the run (will be generated if none supplied)
55 protocol_version - protocol version supported
56 admin_email - admin email extracted from Identify response
57 granularity - datestamp granularity (defaults to 'days', else 'seconds')
58 uses_https - set true if the validator sees an https URL at any stage
59  
60 debug - set true to add extra debugging output
61 log - logging object (usually L<HTTP::OAIPMH::Log>)
62 parser - XML DOM parser instance
63  
64 identify_response - string of identify response (used for registration record)
65 earliest_datestamp - value extracted from earliestDatestamp in Identify response
66 namespace_id - if the oai-identifier is used then this records the namespace identifier extracted
67 set_names - array of all the set names reported in listSets
68  
69 example_record_id - example id used for tests that require a specific identifier
70 example_set_spec - example setSpec ("&set=name") used for tests that require a set
71 example_metadata_prefix - example metadataPrefix which defaults to 'oai_dc'
72  
73 =cut
74  
75 HTTP::OAIPMH::Validator->mk_accessors( qw( base_url protocol_version
76 admin_email granularity uses_503 uses_https
77 debug parser run_id ua allow_https doc save_all_responses
78 response_number http_timeout max_retries max_size
79 protocol guidelines
80 identify_response earliest_datestamp namespace_id set_names
81 example_record_id example_set_spec example_metadata_prefix
82 log status
83 ) );
84  
85 sub new {
86 my $this=shift;
87 my $class=ref($this) || $this;
88 my $self={
89 'base_url' => undef,
90 'protocol_version' => undef,
91 # Repository features extracted
92 'granularity' => 'days', # can also be "seconds"
93 'uses_503' => 0, # set true if 503 responses ever used
94 'uses_https' => 0, # set to true if https is ever used
95 # Control
96 'debug' => 0,
97 'parser' => XML::DOM::Parser->new(),
98 'run_id' => undef,
99 'ua' => undef,
100 'allow_https' => 0, # allow https URIs
101 'doc' => undef, # current parsed xml document
102 'save_all_responses' => 0, # set True to save all HTTP responses
103 'response_number' => 1, # initial response number
104 'http_timeout' => 600,
105 'max_retries' => 5, # number of 503's in a row that we will accept
106 'max_size' => 100000000, # max response size in bytes (100MB)
107 'protocol' => 'http://www.openarchives.org/OAI/2.0/openarchivesprotocol.htm', #URL of protocol spec
108 'guidelines' => 'http://www.openarchives.org/OAI/2.0/guidelines-repository.htm', #URL of repository guidelines doc
109 # Results
110 'namespace_id' => undef,
111 'set_names' => [],
112 'example_record_id' => undef,
113 'example_set_spec' => undef,
114 'example_metadata_prefix' => 'oai_dc',
115 'log' => HTTP::OAIPMH::Log->new(),
116 'status' => 'unknown',
117 @_};
118 bless($self, $class);
119 $self->setup_run_id if (not $self->run_id);
120 $self->setup_user_agent if (not $self->ua);
121 return($self);
122 }
123  
124 =head3 setup_run_id()
125  
126 Set a UUID for the run_id.
127  
128 =cut
129  
130 sub setup_run_id {
131 my $self=shift;
132 my $ug=Data::UUID->new;
133 $self->run_id(lc($ug->to_string($ug->create)));
134 }
135  
136 =head3 setup_user_agent()
137  
138 Setup L<LWP::UserAgent> for the validator.
139  
140 =cut
141  
142 sub setup_user_agent {
143 my $self=shift;
144 my $ua = LWP::UserAgent->new(); # User agent, to render http requests
145 $ua->timeout($self->http_timeout); # give responses 10 minutes
146 $ua->max_size($self->max_size); # size limit ##seems to break http://eprints.soton.ac.uk/perl/oai2 [Simeon/2005-06-06]
147 $ua->requests_redirectable([]); # we will do redirects manually
148 $ua->agent('OAIPMH_Validator'); # set user agent
149 $ua->from('https://groups.google.com/d/forum/oai-pmh'); # set a default From: address -> direct to google group for dicussion
150 $self->ua($ua);
151 }
152  
153  
154 =head3 abort($msg)
155  
156 Special purpose "die" routine because tests cannot continue. Logs
157 failure and then dies.
158  
159 =cut
160  
161 sub abort {
162 my $self=shift;
163 my ($msg)=@_;
164 $self->log->fail('ABORT: '.$msg);
165 $self->status('FAILED');
166 die('ABORT: '.$msg."\n");
167 }
168  
169  
170 =head3 run_complete_validation($skip_test_identify)
171  
172 Run all tests for a complete validation and return true is the data provider passes,
173 false otherwise. All actions are logged and may be accessed to provide a report
174 (including warnings that do not indicate failure) after the run.
175  
176 Arguments:
177 $skip_identify - set true to skip the text_identify() step
178  
179 =cut
180  
181 sub run_complete_validation {
182 my $self=shift;
183 my ($skip_identify)=@_;
184  
185 $self->response_number(1);
186 $self->test_identify unless ($skip_identify);
187 $self->test_list_sets;
188 $self->test_list_identifiers;
189  
190 my $baseURL = $self->base_url;
191 my ($formats, $gotDC) = $self->test_list_metadata_formats;
192  
193 # If the repository doesn't support oai_dc then this is a failure (because
194 # the standard demands it) but see whether we can find another metadataPrefix
195 # in order to continue the tests
196 if ( $gotDC ) {
197 $self->log->pass("Data provider supports oai_dc metadataPrefix");
198 } else {
199 if ($formats and $formats->getLength()>0) {
200 $self->example_metadata_prefix( $formats->item(0)->getFirstChild->getData );
201 $self->log->fail("Data provider does not support the simple Dublin Core metadata ".
202 "format with metadataPrefix oai_dc. Tests that require a ".
203 "metadataPrefix to be specified will use '".
204 $self->example_metadata_prefix."'");
205 } else {
206 $self->log->fail("There are no metadata formats available to use with the GetRecord ".
207 "request. The metadataPrefix ".
208 $self->example_metadata_prefix.
209 " will be used for later tests even though it seems unsupported.");
210 }
211 }
212  
213 my ($dateStamp)=$self->test_get_record($self->example_record_id,$self->example_metadata_prefix);
214 $self->test_list_records($dateStamp,$self->example_metadata_prefix);
215  
216 # Check responses to erroneous queries
217 $self->test_expected_errors($self->example_record_id);
218  
219 if ($self->protocol_version eq '2.0') {
220 $self->test_expected_v2_errors($self->earliest_datestamp,$self->example_metadata_prefix);
221 # As of version 2.0, data providers must support HTTP POST requests
222 $self->test_post_requests($self->example_metadata_prefix);
223 }
224 $self->test_resumption_tokens;
225  
226 # Getting here with no failures means that the data provider is compliant
227 # (there may be warnings which are not counted in num_fail)
228 $self->status( $self->log->num_fail==0 ? 'COMPLIANT' : 'FAILED' );
229 return($self->log->num_fail==0);
230 }
231  
232  
233 =head3 failures()
234  
235 Return Markdown summary of failure log entries, along with the appropriate
236 titles and request details. Will return empty string if there are no
237 failures in the log.
238  
239 =cut
240  
241 sub failures {
242 my $self=shift;
243 return($self->log->failures());
244 }
245  
246  
247 =head3 summary()
248  
249 Return summary statistics for the validation in Markdown (designed to agree
250 with conversion to HTML by L<Text::Markdown>).
251  
252 =cut
253  
254 sub summary {
255 my $self=shift;
256  
257 my $sf=($self->log->num_fail>0?'failure':'success');
258  
259 my $str="\n## Summary - *$sf*\n\n";
260 my $namespace_id = $self->namespace_id;
261 if ($namespace_id) {
262 if ($namespace_id=~/\./) { #v2.0
263 $str.=" * Namespace declared for v2.0 oai-identifiers is $namespace_id\n";
264 } else { #v1.1
265 $str.=" * Namespace declared for v1.1 oai-identifiers (the repositoryIdentifier) is $namespace_id\n";
266 }
267 }
268 $str.=" * Uses 503 for flow control\n" if ($self->uses_503);
269 $str.=" * Uses https URIs (not specified in protocol)\n" if ($self->uses_https);
270 $str.=" * Total tests passed: ".$self->log->num_pass."\n";
271 $str.=" * Total warnings: ".$self->log->num_warn."\n";
272 $str.=" * Total error count: ".$self->log->num_fail."\n";
273 $str.=" * Validation status: ".($self->status || 'unknown')."\n";
274 return($str);
275 }
276  
277  
278 =head2 METHODS TESTING SPECIFIC OAI-PMH VERBS
279  
280 =head3 test_identify()
281  
282 Check response to an Identify request. Returns false if tests cannot
283 continue, true otherwise.
284  
285 Side effects based on values extracted:
286  
287 - $self->admin_email set to email extracted from adminEmail element
288 - $self->granularity set to 'days' or 'seconds'
289  
290 =cut
291  
292 sub test_identify {
293 my $self=shift;
294  
295 my $cantContinue=0;
296 $self->log->start("Checking Identify response");
297  
298 # Send the verb request to the base URL - vet extracts the email address
299 my $burl=$self->base_url;
300 my $req = $burl."?verb=Identify";
301  
302 my $response = $self->make_request($req); #don't use make_request_and_validate() just do simplest thing here
303 unless ($response->is_success) {
304 my $r="Server at base URL '$burl' failed to respond to Identify. The HTTP GET request with URL $req received response code '".$response->code()."'.";
305 if ($response->code() == 301) {
306 $self->log->fail("$r HTTP code 301 'Moved Permanently' is not widely supported by ".
307 "harvesters and is anyway inappropriate for registration of a ".
308 "service. If requests must be redirected then an HTTP response 302 ".
309 "may be used as outlined in the guidelines [".
310 $self->guidelines."#LoadBalancing].");
311 } else {
312 $self->log->fail($r);
313 }
314 $self->abort("Failed to get Identify response from server at base URL '$burl'.\n");
315 return;
316 }
317  
318 # Parse the XML response
319 unless ($self->parse_response($req,$response)) {
320 $self->log->fail("Failed to parse Identify response");
321 $self->abort("Failed to parse Identify response from server at base URL '$burl'.\n");
322 }
323  
324 # Check that this really is a Identify response
325 my $oaipmhNode=$self->doc->getFirstChild();
326 # skip over and processing instructions such as XML stylesheets
327 while ($oaipmhNode->getNodeType==PROCESSING_INSTRUCTION_NODE or
328 $oaipmhNode->getNodeType==COMMENT_NODE) {
329 $oaipmhNode=$oaipmhNode->getNextSibling();
330 }
331 unless (defined $oaipmhNode and $oaipmhNode->getNodeName eq 'OAI-PMH') {
332 $self->log->fail("Identify response does not have OAI-PMH as root element! ".
333 "Found node named '".$oaipmhNode->getNodeName."' instead");
334 $self->abort("Identify response from server at base URL '$burl' does not have ".
335 "OAI-PMH as root element!\n");
336 }
337 my $identifyNode=$oaipmhNode->getElementsByTagName('Identify',0);
338 unless ($identifyNode->getLength()>0) {
339 my $errorNode=$oaipmhNode->getElementsByTagName('error',0);
340 if ($errorNode->getLength()>0) {
341 # give specific message if response is and error
342 $self->log->fail("Error response to Identify request!\n");
343 $self->abort("Error response to Identify request from server at base URL '$burl'.\n");
344 return;
345 } else {
346 $self->log->fail("Identify response does not contain &lt;Identify&gt; block.\n");
347 $self->abort("Identify response does not contain Identify block from server at base URL '$burl'.\n");
348 return;
349 }
350 }
351  
352 # Extract admin email and protocol version numbers, check
353 my ($admin_email,$email_error)=$self->get_admin_email;
354 if (not $admin_email or $email_error) {
355 $self->abort(($email_error || "Failed to extract adminEmail").", aborting.\n");
356 return;
357 }
358 $self->admin_email($admin_email);
359 $self->check_protocol_version; # bails if not Version 2.0
360  
361 # URL is valid, Identify response was provided, extract content as string
362 $self->identify_response( $response->content );
363  
364 my $baseURL = $self->doc->getElementsByTagName('baseURL');
365  
366 # BUG FOUND ON AUGUST 26, 2002: empty baseURL still returns length > 0
367 # So it is necessary to explicity check for an empty element.
368 if ( $baseURL->getLength() > 0 ) {
369 $baseURL = $baseURL->item(0)->getFirstChild;
370 if ( $baseURL ) { $baseURL = $baseURL->getData; }
371  
372 # $burl is the one given on the form; $baseURL is the one in the XML doc.
373 if ($burl eq $baseURL) {
374 $self->log->pass("baseURL supplied matches the Identify response");
375 } else {
376 # report the error, but keep the form URL
377 # (at least it answered Identify!)
378 $self->log->fail("baseURL supplied '$burl' does not match the baseURL in the ".
379 "Identify response '$baseURL'. The baseURL you enter must EXACTLY ".
380 "match the baseURL returned in the Identify response. It must ".
381 "match in case (http://Wibble.org/ does not match http://wibble.org/) ".
382 "and include any trailing slashes etc.");
383 $cantContinue++;
384 }
385 }
386  
387 # For Version 2.0, Check for seconds granularity
388 if ($self->protocol_version eq '2.0') {
389 my $gran_el = $self->doc->getElementsByTagName('granularity');
390 if ($self->parse_granularity($gran_el)) {
391 $self->log->pass("Datestamp granularity is '".$self->granularity."'");
392 } else {
393 $cantContinue++;
394 }
395 }
396  
397 # For an exception check new to Version 2.0, extract the earliest date
398 # and also check that its granularity is right
399 if (my $err=$self->get_earliest_datestamp) {
400 $self->log->fail("Bad earliestDatestamp: $err");
401 $cantContinue++;
402 } else {
403 $self->log->pass("Extracted earliestDatestamp ".$self->earliest_datestamp);
404 }
405  
406 # Check for OAI-identifier. If already in use by another base URL, bump
407 # the error count to avoid having this URL register.
408 #
409 my $oaiIds = $self->doc->getElementsByTagName('oai-identifier');
410 if ($oaiIds and $oaiIds->getLength()>0) {
411 if ($oaiIds->getLength()>1) {
412 $self->log->fail("Found more than one oai-identifier element. The intention ".
413 "is that this declaration only be used by repositories ".
414 "declaring the use of a single identifier namespace.");
415 $cantContinue++;
416 } else {
417 $oaiIds=$oaiIds->item(0);
418  
419 # Now find out if this is v1.1 or v2.0
420 my $oai_id_version='2.0';
421 if (my $xmlns=$oaiIds->getAttribute('xmlns')) { #FIXME this requires default namespace to be set to oai-id
422 if ($xmlns eq 'http://www.openarchives.org/OAI/2.0/oai-identifier') {
423 $oai_id_version='2.0';
424 $self->log->pass("oai-identifier description for version $oai_id_version is being used");
425 } elsif ($xmlns eq 'http://www.openarchives.org/OAI/1.1/oai-identifier') {
426 $oai_id_version='1.1';
427 $self->log->pass("oai-identifier description for version $oai_id_version is being used");
428 } elsif ($xmlns) {
429 $self->log->fail("Unrecognized namespace declaration '$xmlns' for ".
430 "oai-identifier, expected ".
431 "http://www.openarchives.org/OAI/2.0/oai-identifier ".
432 "(for v2.0) or ".
433 "http://www.openarchives.org/OAI/1.1/oai-identifier ".
434 "(for v1.1). Assuming version $oai_id_version.");
435 } else {
436 $self->log->fail("No namespace declaration found for oai-identifier, expected ".
437 "http://www.openarchives.org/OAI/2.0/oai-identifier ".
438 "(for v2.0) or ".
439 "http://www.openarchives.org/OAI/1.1/oai-identifier ".
440 "(for v1.1). Assuming version $oai_id_version/");
441 }
442 } else {
443 $self->log->fail("Can't find namespace declaration for the oai-identifier description. ".
444 "This must be added as <oai-identifier xmlns=\"http://www.openarchives.org/OAI/2.0/oai-identifier\" ...> ".
445 "(or 1.1), there will likely also be schema validation weeors. Will ".
446 "assume that the oai-identifier is version $oai_id_version for ".
447 "later tests");
448 }
449 my $repoIds = $oaiIds->getElementsByTagName('repositoryIdentifier');
450 if ($repoIds) {
451 my $temp = $repoIds->item(0);
452 if (!defined($temp)) {
453 $self->log->fail("No namespace-identifier (repositoryIdentifier element) in ".
454 "the oai-identifier block of the Identify description");
455 return;
456 }
457 my $nsel = $temp->getFirstChild;
458 unless ( $nsel ) {
459 # Empty repositoryIdentifier element, squawk loudly
460 $self->log->fail("Empty namespace-identifier (repositoryIdentifier element) in ".
461 "the oai-identifier block of the Identify description");
462 return;
463 }
464 my $namespace_id = $nsel->getData;
465 # Having validated the value of namespace-identifier, we can now tell if it is v1.1 or v2.0 based
466 # on whether is has a . in it (i.e. if /\./)
467 if ($oai_id_version eq '2.0') {
468 #schema: <pattern value="[a-zA-Z][a-zA-Z0-9\-]*(\.[a-zA-Z][a-zA-Z0-9\-]+)+"/>
469 unless ($namespace_id=~/^[a-z][a-z0-9\-]*(\.[a-z][a-z0-9\-]+)+$/i) {
470 $self->log->fail("Bad namespace-identifier (repositoryIdentifier element) ".
471 "'$namespace_id' in oai-identifier declaration. See section ".
472 "2.1 of the OAI Identifier specification for details ".
473 "(http://www.openarchives.org/OAI/2.0/guidelines-oai-identifier.htm).");
474 $cantContinue++;
475 } else {
476 $self->log->pass("namespace-identifier (repositoryIdentifier element) in oai-identifier ".
477 "declaration is $namespace_id");
478 $self->namespace_id( $namespace_id );
479 }
480 } else { #v1.1 schema: <pattern value="[a-zA-Z0-9]+"/>
481 unless ($namespace_id=~/^[a-z0-9]+$/i) {
482 $self->log->fail("Bad namespace-identifier (repositoryIdentifier element) ".
483 "'$namespace_id' in oai-identifier declaration. See section ".
484 "2.1 of the OAI Identifier specification for details ".
485 "(http://www.openarchives.org/OAI/1.1/guidelines-oai-identifier.htm).");
486 $cantContinue++;
487 } else {
488 $self->log->pass("namespace-identifier (repositoryIdentifier element) in oai-identifier ".
489 "declaration is $namespace_id");
490 $self->namespace_id( $namespace_id );
491 }
492 }
493 }
494 }
495 }
496 return(not $cantContinue);
497 }
498  
499  
500 =head3 test_list_sets()
501  
502 Check response to the ListSets verb.
503  
504 Save the setSpecs for later use.
505  
506 Note that the any set might be empty. So if test_list_identifiers doesn't
507 get a match, we need to try the second set identifier, and so on.
508 So keep a list of the setSpec elements.
509  
510 =cut
511  
512 sub test_list_sets {
513 my $self=shift;
514  
515 $self->log->start("Checking ListSets response");
516 my $req=$self->base_url."?verb=ListSets";
517 my $response = $self->make_request_and_validate("ListSets", $req);
518 unless ($response) {
519 $self->log->fail("Can't check set names");
520 return;
521 }
522  
523 unless ($self->parse_response($req,$response)) {
524 $self->log->fail("Can't parse response");
525 $self->abort("failed to parse response to ListSets");
526 }
527  
528 $self->set_names( [] );
529 $self->example_set_spec( '' );
530 my $set_elements=$self->doc->getElementsByTagName('setSpec');
531 if (not defined($set_elements) or ($set_elements->getLength<1)) {
532 # No setSpec elements, so there should be an <error code="noSetHierarchy"> element
533 my $details={};
534 if ($self->is_error_response($details)) {
535 if ($details->{'noSetHierarchy'}) {
536 $self->log->pass("Repository does not support sets and the is correctly reported with a ".
537 "noSetHierarchy exception in the ListSets response");
538 } else {
539 $self->log->fail("Failed to extract any setSpec elements from ListSets ".
540 "but did not find a noSetHierarchy exception. Found instead a '".
541 join(', ',keys %{$details})."' exception(s). See <".
542 $self->protocol."#ListSets>.");
543 }
544 } else {
545 $self->log->fail("Failed to extract any setSpec elements from ListSets but did not ".
546 "find an exception message. If sets are not supported by the ".
547 "repository then the ListSets response must be the noSetHierarchy ".
548 "error. See <".$self->protocol."#ListSets>.");
549 }
550 } else {
551 # Have setSpec elements, record all set names and pick an example set spec
552 for (my $j=0; $j<$set_elements->getLength; $j++) {
553 my $set_name=$set_elements->item($j)->getFirstChild->getData;
554 ##FIXME - should validate each set name
555 push(@{$self->set_names},$set_name);
556 }
557 # Sanity check, did we get the number we expected?
558 my $num_sets=scalar(@{$self->set_names});
559 if ($num_sets!=$set_elements->getLength) {
560 $self->log->fail("Failed to extract the expected number of set names (got ".
561 "$num_sets, expected ".$set_elements->getLength.")");
562 }
563 if ($num_sets>0) {
564 $self->example_set_spec( "&set=".$self->set_names->[0] );
565 }
566 my $msg='';
567 for (my $j=0; $j<$num_sets and $j<3; $j++) { $msg.=" ".$self->set_names->[$j]; }
568 $msg.=" ..." if ($num_sets>3);
569 $self->log->pass("Extracted $num_sets set names: {$msg }, will use setSpec ".
570 $self->example_set_spec." in tests");
571 }
572 }
573  
574  
575 =head3 test_list_identifiers()
576  
577 Check response to ListIdentifiers and record an example record id in
578 $self->example_record_id to be used in other tests.
579  
580 If there are no identifiers, but the response is legal, stop the test with
581 errors=0, number of verbs checked is three.
582  
583 As of version 2.0, a metadataPrefix argument is required. Unfortunately
584 we need to call test_list_identifiers first in order to get an id for
585 GetRecord, so we simply use oai_dc.
586  
587 =cut
588  
589 sub test_list_identifiers {
590 my $self=shift;
591  
592 $self->log->start("Checking ListIdentifiers response");
593  
594 ### FIXME -- skip the set= restriction because this code doesn't
595 ### FIXME work right for set hierarchies - 2002-10-17
596 ### FIXME 2015-01-02 - put/left in, is it OK?
597 my $set_spec = $self->example_set_spec;
598 my $req = $self->base_url."?verb=ListIdentifiers&metadataPrefix=oai_dc".$set_spec;
599 my $response = $self->make_request_and_validate("ListIdentifiers", $req);
600  
601 # Note: $response will come back null if an error code was returned
602 # An error code of "noRecordsMatch" comes back if specified set is
603 # empty. In that case we should drop the set and try again.
604 if ($set_spec and (!$response or $self->is_no_records_match)) {
605 $self->log->note("Tried empty set, will look for set with items...");
606 foreach my $set_name (@{$self->set_names}) {
607 $set_spec = "&set=".$set_name;
608 if ($set_spec eq $self->example_set_spec) {
609 # This was the first one we tried, don't try again
610 next;
611 }
612 $self->log->note("Trying set ".$set_spec);
613 $req = $self->base_url."?verb=ListIdentifiers&metadataPrefix=oai_dc".$set_spec;
614 $response = $self->make_request_and_validate("ListIdentifiers", $req);
615 if ($response and not $self->is_no_records_match) {
616 last;
617 }
618 }
619 # If we were successful then set the example_set_spec for any future tests
620 if ($response and not $self->is_no_records_match) {
621 $self->example_set_spec( $set_spec );
622 } else {
623 $self->log->note("Failed to find non-empty set, expect failure");
624 }
625 }
626  
627 # None of the sets had any identifiers in them. Try the whole entire
628 # list of identifiers.
629 if ($set_spec and !$response) {
630 $self->log->note("Last attempt is without any sets...");
631 $req = $self->base_url."?verb=ListIdentifiers&metadataPrefix=oai_dc";
632 $response = $self->make_request_and_validate("ListIdentifiers",$req);
633 }
634  
635 # Now we are for real in trouble if $response is null
636 unless ($response) {
637 $self->log->fail("No ListIdentifiers response with content");
638 $self->log->note("The base URL did not respond to the ListIdentifiers verb.".
639 "Without that, we cannot proceed with the validation test. Exiting.");
640 $self->abort("The base URL did not respond to the ListIdentifiers verb. Without that, we cannot proceed with the validation test. Exiting.");
641 }
642  
643 # Grab the first identifier for later use
644 unless ($self->parse_response($req,$response)) {
645 $self->log->fail("Can't parse ListIdentifiers response");
646 $self->abort("unable to parse response");
647 }
648 #
649 # Now look for the identifier of a non-deleted record
650 # If there are no identifiers to be harvested, we cannot complete the
651 # validation test.
652 #
653 # FIXME - this still doesn't solve the problem that there may be no
654 # non-deleted items listed in the particular response or partial
655 # response that we are looking at [Simeon/2005-07-20]
656 #
657 my $headers = $self->doc->getElementsByTagName('header');
658 my $h;
659 my $record_id;
660 for ($h=0; $h<$headers->getLength(); $h++) {
661 my $hdnode=$headers->item($h);
662 my $idnode=$hdnode->getElementsByTagName('identifier',0);
663 next unless ($idnode and $idnode->getLength()==1);
664 $record_id=$idnode->item(0)->getFirstChild->getData;
665 last unless ($hdnode->getAttribute('status') and $hdnode->getAttribute('status') eq 'deleted');
666 $self->log->warn("Identifier ".($h+1).", '$record_id', is for a deleted record, skipping");
667 }
668 if ($h==$headers->getLength()) {
669 # No identifiers were in the ListIdentifiers response. Further testing
670 # is not possible.
671 $self->log->fail("The response to the ListIdentifiers verb with metadataPrefix oai_dc ".
672 "contained no identifiers. Without at least one identifier, we cannot ".
673 "proceed with the validation tests.");
674 $self->abort("No identifiers in response to ListIdentifiers. Without an identifier ".
675 "we cannot proceed with validation tests.");
676 }
677 $self->log->pass("Good ListIdentifiers response, extracted id '$record_id' for use in future tests.");
678 $self->example_record_id( $record_id );
679 }
680  
681  
682 =head3 test_list_metadata_formats()
683  
684 Vet the verb as usual, and then make sure that Dublin Core in included
685 In particular, we will check the metadata formats available for "record_id",
686 obtained from checking the ListIdentifier verb.
687 Side effect: save available formats for later use (global "formats").
688 NOTE:if there are no formats, error will be picked up by getRecord
689  
690 =cut
691  
692 sub test_list_metadata_formats {
693 my $self=shift;
694  
695 $self->log->start("Checking ListMetadataFormats response");
696  
697 # Do we have an example record id to check with?
698 my $record_id = $self->example_record_id;
699 unless ($record_id) {
700 $self->log->fail("Cannot check ListMetadataFormats as we do not have an example id");
701 return;
702 }
703  
704 my $req = $self->base_url."?verb=ListMetadataFormats&identifier=".url_encode($record_id);
705 my $response = $self->make_request_and_validate("ListMetadataFormats",$req);
706 unless ($response) {
707 $self->log->fail("Can't check metadataFormats available for item $record_id, no ".
708 "response to ListMetadataFormats request.");
709 return;
710 }
711  
712 # Check for Dublin Core
713 unless ($self->parse_response($req,$response)) {
714 $self->log->fail("Can't parse response to ListMetadataFormats request for item $record_id.");
715 return;
716 }
717  
718 my $formats = $self->doc->getElementsByTagName('metadataPrefix');
719 unless ($formats->getLength() > 0) {
720 $self->log->fail("No metadata formats are listed in the response to a ListMetadataFormats ".
721 "request for item $record_id.");
722 return;
723 }
724  
725 if ($self->debug) {
726 $self->log->note("debug: ".$formats->getLength()." formats supported for identifier '$record_id'");
727 }
728 my $gotDC=0;
729 for my $i (0..$formats->getLength()-1) {
730 my $format = $formats->item($i);
731 #assume this node has only one child, and its data for a format
732 if ( $format->getFirstChild->getData =~ /^\s*oai_dc\s*$/ ) {
733 $gotDC = 1;
734 last;
735 }
736 }
737 if ($gotDC) {
738 $self->log->pass("Good ListMetadataFormats response, includes oai_dc");
739 } else {
740 $self->log->pass("Good ListMetadataFormats response, BUT DID NOT FIND oai_dc");
741 }
742 return($formats, $gotDC);
743 }
744  
745  
746 =head3 test_get_record($record_id, $format)
747  
748 Try to get record $record_id in $format.
749  
750 If either $record_id or $format are undef then we have an error
751 right off the bat. Else make the request and return the
752 datestamp of the record.
753  
754 =cut
755  
756 sub test_get_record {
757 my $self=shift;
758 my ($record_id, $format)=@_;
759  
760 $self->log->start("Checking GetRecord response");
761  
762 unless (defined $format) {
763 $self->log->fail("Skipping GetRecord test as no metadata format is listed as being available.");
764 return;
765 }
766 unless (defined $record_id) {
767 $self->log->fail("Skipping GetRecord test as no items are listed as having metadata available.");
768 return;
769 }
770  
771 my $numerr=0; #count up non-fatal errors
772  
773 my $req = $self->base_url."?verb=GetRecord&identifier=".url_encode($record_id)."&metadataPrefix=".url_encode($format);
774 my $response = $self->make_request_and_validate("GetRecord", $req);
775 unless ($response) {
776 $self->log->fail("Can't complete datestamp check for GetRecord");
777 $self->abort("Can't complete datestamp check for GetRecord");
778 }
779  
780 # Save the datestamp for later use by ListRecords
781 # As of version 2.0, Identify response can have a granularity and the
782 # datestamp MUST be in the finest granularity supported by the repository
783 unless ($self->parse_response($req,$response)) {
784 $self->log->fail("Can't parse response");
785 $self->abort("Unable to parse response from GetRecord");
786 }
787  
788 if (my $msg=$self->is_error_response) {
789 $self->log->fail("The response to the GetRecord verb was the OAI exception $msg. ".
790 "It is this not possible to extract a valid datestamp for remaining tests");
791 $self->abort("Unexpected OAI exception response");
792 }
793  
794 my $datestamps = $self->doc->getElementsByTagName('datestamp');
795 # If there is no <record> there is no datestamp ... but there should be a record
796 unless ( $datestamps->getLength() > 0 ) {
797 $self->log->fail("The response to the GetRecord verb did not have a datestamp, which is ".
798 "needed to continue checking verbs.");
799 $self->abort("No datestamp in the response for GetRecord");
800 }
801  
802 my $datestamp=undef;
803 eval {
804 $datestamp = $datestamps->item(0)->getFirstChild->getData;
805 };
806 if (not defined($datestamp)) {
807 $self->log->fail("Failed to extract datestamp from the GetRecord response. See <".
808 $self->protocol."#Dates>.");
809 $numerr++;
810 } elsif ( my $granularity=$self->get_datestamp_granularity($datestamp) ) {
811 $self->log->pass("Datestamp in GetRecord response ($datestamp) has the correct form for ".
812 "$granularity granularity.");
813 if ( $granularity eq $self->granularity ) {
814 # The granularity in v2.0 must match the finest granularity supported (see sec3.3.2)
815 $self->log->pass("Datestamp in GetRecord response ($datestamp) matched the ".
816 $self->granularity." granularity specified in the Identify response. ");
817 } else {
818 $self->log->fail("Datestamp in GetRecord response ($datestamp) is not consistent ".
819 "with the ".$self->granularity." granularity specified in the ".
820 "Identify response");
821 $numerr++;
822 }
823 } else {
824 $self->log->fail("Datestamp in GetRecord response ($datestamp) is not valid. See <".
825 $self->protocol."#Dates>.");
826 $numerr++;
827 }
828  
829 # As of OAI-PMH Version 2.0, GetRecord must return a set spec if the
830 # repository supports sets and the item is in a set
831 if (not $self->example_set_spec) {
832 $self->log->pass("Valid GetRecord response") unless ($numerr>0);
833 return($datestamp);
834 }
835  
836 my $set_list = $self->doc->getElementsByTagName('setSpec');
837 my $set_value = $self->example_set_spec;
838 $set_value =~ s/&set=//;
839 $self->log->note("Looking for set '".$set_value."' or a descendant set.") if $self->debug;
840 my $i;
841 my $subset_str = '';
842 for ($i=0; $i<$set_list->getLength; $i++) {
843 my $s = $set_list->item($i)->getFirstChild->getData;
844 last if ($s eq $set_value);
845 if ($s =~ m/^${set_value}:/) {
846 $subset_str = " (implied by a descendant setSpec)";
847 last;
848 }
849 }
850 if ($i==$set_list->getLength) { # error
851 $self->log->fail("Expected setSpec was missing from the response. The GetRecord ".
852 "response for identifier $record_id did not contain a set ".
853 "specification for $set_value");
854 } else {
855 $self->log->pass("Expected setSpec was returned in the response".$subset_str);
856 }
857 return($datestamp);
858 }
859  
860  
861 =head3 test_list_records($datestamp,$metadata_prefix)
862  
863 Test the response for the ListRecords verb. In addition, if there is
864 no Dublin Core available for this repository, this is an error.
865 (And the error has already been counted in test_get_record)
866 We can still test the verb, however, with one of the available
867 formats found by testGetMetadataFormats. Since the output of
868 ListRecords is likely to be large, use the datestamp of the one
869 record we did retrieve to limit the output.
870  
871 =cut
872  
873 sub test_list_records {
874 my $self=shift;
875 my ($datestamp,$metadata_prefix)=@_;
876  
877 $self->log->start("Checking ListRecords response");
878  
879 my $req = $self->base_url."?verb=ListRecords";
880 if ($datestamp) {
881 $req.="&from=".$datestamp."&until=".$datestamp;
882 } else {
883 $self->log->warn("Omitting datestamp parameter as none was obtained from earlier test");
884 }
885 $req.="&metadataPrefix=".$metadata_prefix;
886 my $list_not_complete=1;
887  
888 while ($list_not_complete) {
889 $list_not_complete=0;
890 my $response = $self->make_request_and_validate("ListRecords", $req);
891 unless ($response) {
892 #Nothing else to say since we don't do other tests
893 return;
894 }
895  
896 if ($self->parse_response($req,$response)) {
897 $self->log->pass("Response is well formed");
898 } else {
899 $self->log->fail("The ListRecords response was not well formed XML");
900 }
901  
902 # Now check to make sure that we got back the record for the identifier
903 # $self->example_record_id if there is one specified, else fail that
904 # test.
905 my $record_id=$self->example_record_id;
906 unless ($record_id) {
907 $self->log->fail("Cannot check for correct record inclusion without an example record id");
908 return;
909 }
910 my $details={};
911 if ($self->is_error_response($details)) {
912 if ($details->{'noRecordsMatch'}) {
913 $self->log->fail("ListRecords response gave a noRecordsMatch error when it should ".
914 "have included at least the record with identifier $record_id. ".
915 "The from and until parameters of the request were set to the ".
916 "datestamp of this record ($datestamp). The from and until parameters ".
917 "are inclusive, see protocol spec section 2.7.1. The message ".
918 "included in the error response was: '".
919 $details->{'noRecordsMatch'}."'");
920 } else {
921 my @txt=();
922 foreach my $k (keys %$details) {
923 push(@txt,"$k (".$details->{$k}.")");
924 }
925 $self->log->fail("ListRecords gave an unexpected error response to a request using ".
926 "from and until datestamps taken from the previous GetRecord response: ".
927 join(', ',@txt));
928 }
929 } else {
930 my $id_list = $self->doc->getElementsByTagName('identifier');
931 my $i;
932 my $badly_formed=0;
933 for ($i=0; $i<$id_list->getLength; $i++) {
934 if (my $child=$id_list->item($i)->getFirstChild()) {
935 last if ($id_list->item($i)->getFirstChild->getData eq $record_id);
936 } else {
937 $badly_formed++;
938 last;
939 }
940 }
941 if ($badly_formed) {
942 $self->log->fail("ListRecords response badly formed, identifier element for record ".
943 ($i+1)." is empty");
944 } elsif ($i<$id_list->getLength) {
945 $self->log->pass("ListRecords response correctly included record with identifier $record_id");
946 } elsif (my $token=$self->get_resumption_token) {
947 # More responses to come, may just not have got to the
948 # record yet... roll around for more:
949 $self->log->pass("ListRecords response includes resumptionToken. Haven't found ".
950 "record with identifier $record_id yet, will continue with resumptionToken...");
951 $list_not_complete=1;
952 $req = $self->base_url."?verb=ListRecords&resumptionToken=".url_encode($token);
953 } else {
954 $self->log->fail("ListRecords response did not include the identifier $record_id ".
955 "which should have been included because both the from and until ".
956 "parameters were set to the datestamp of this record ($datestamp). ".
957 "The from and until parameters are inclusive, see protocol spec ".
958 "section 2.7.1");
959 }
960 }
961 }
962 }
963  
964  
965 =head3 test_resumption_tokens()
966  
967 Request an unlimited ListRecords. If there is a resumption token, see
968 if it works. It is an error if resumption is there but doesn't work.
969 Empty resumption tokens are OK -- this ends the list.
970  
971 CGI takes care of URL-encoding the resumption token.
972  
973 =cut
974  
975 sub test_resumption_tokens {
976 my $self=shift;
977  
978 $self->log->start("Checking for correct use of resumptionToken (if used)");
979  
980 my $req = $self->base_url."?verb=ListRecords&metadataPrefix=oai_dc";
981 my $response = $self->make_request($req);
982  
983 # was there a resumption token?
984 unless ($self->parse_response($req,$response)) {
985 $self->log->fail("Can't parse malformed XML in response to ListRecords request. ".
986 "Cannot complete test for correct use of resumptionToken (if used)");
987 return;
988 }
989  
990 my $tokenList = $self->doc->getElementsByTagName('resumptionToken');
991 if ( !$tokenList or $tokenList->getLength()==0 ) {
992 $self->log->pass("resumptionToken not used");
993 return;
994 }
995 if ( $tokenList->getLength()>1 ) {
996 $self->log->fail("More than one resumptionToken in response!");
997 return;
998 }
999  
1000 # Dig out the resumption token from the document
1001 my $tokenElement = $tokenList->item(0);
1002  
1003 # Try getting the resumption token, $token will be will be undefined
1004 # unless the element has content
1005 my $token = $tokenElement->getFirstChild;
1006 my $tokenString;
1007 if ($token) {
1008 $tokenString = $token->getData;
1009 }
1010 unless ($tokenString) {
1011 $self->log->fail("Empty resumption token in response to $req There should never ".
1012 "be an empty resumptionToken in response to a request without a ".
1013 "resumptionToken argument");
1014 return;
1015 }
1016  
1017 # If there us a 'cursor' value given then check that it is
1018 # correct. It must have the value 0 in the first response
1019 my $usingCursor=0;
1020 if (my $cursor=$tokenElement->getAttribute('cursor')) {
1021 $usingCursor=1;
1022 if ($cursor==0) {
1023 $self->log->pass("A cursor value was supplied with the resumptionToken and it ".
1024 "correctly had the value zero in the first response");
1025 } else {
1026 $self->log->fail("A cursor value was supplied with the resumptionToken but it ".
1027 "did not have the correct value zero for the first response. ".
1028 "The value was '$cursor'.");
1029 }
1030 }
1031  
1032 $self->log->note("Got resumptionToken ".$tokenString);
1033  
1034 # Try using the resumption token. Before including a resumptionToken in
1035 # the URL of a subsequent request, we must encode all special characters
1036 # getData in this version of XML::DOM expands entitities
1037 $req = $self->base_url."?verb=ListRecords&resumptionToken=".url_encode($tokenString);
1038 $response = $self->make_request($req);
1039 unless ( $response ) {
1040 $self->log->fail("Site failed to respond to request using resumptionToken: $req");
1041 return;
1042 }
1043 unless ( $self->parse_response($req,$response)) {
1044 $self->log->fail("Response to request is using resumptionToken not valid XML: $req");
1045 return;
1046 }
1047  
1048 my $errorList = $self->doc->getElementsByTagName('error');
1049 if ( $errorList and $errorList->getLength() > 0 ) {
1050 $self->log->fail("Response to request using resumptionToken was an error code: $req");
1051 return;
1052 }
1053  
1054 ###FIXME: put in test for cursor again, should be number of items returned in the
1055 ###FIXME: first response [Simeon/2005-10-11]
1056  
1057 $self->log->pass("Resumption tokens appear to work");
1058 }
1059  
1060  
1061 =head2 METHODS CHECKING ERRORS AND EXCEPTIONS
1062  
1063 =head3 test_expected_errors($record_id)
1064  
1065 Each one of these requests should get a 400 response in OAI-PHM v1.1,
1066 or a 200 response in 2.0, along with a Reason_Phrase. Bump error_count
1067 if this does not hold. Return the number of errorneous responses.
1068  
1069 $record_id is a valid record identifier to be used in tests that require
1070 one.
1071  
1072 =cut
1073  
1074 sub test_expected_errors {
1075 my $self=shift;
1076 my ($record_id)=@_;
1077  
1078 $self->log->start("Checking exception handling (errors)");
1079  
1080 my @request_list = (
1081 [ 'junk', [ 'badVerb' ], '', '' ],
1082 [ 'verb=junk', [ 'badVerb' ], '', '' ],
1083 [ 'verb=GetRecord&metadataPrefix=oai_dc', [ 'badArgument' ], '', '' ],
1084 [ 'verb=GetRecord&identifier='.$record_id, [ 'badArgument' ], '', '' ],
1085 [ 'verb=GetRecord&identifier=invalid"id&metadataPrefix=oai_dc', [ 'badArgument','idDoesNotExist' ], 'An XML parsing error may be due to incorrectly including the invalid identifier in the <request> element of your XML error response; only valid arguments should be included. A response that includes <request verb="GetRecord" identifier="invalid"id" metadataPrefix="oai_dc">..baseURL..</request> is not well-formed XML because of the quotation mark (") in the identifier.', 'Either the badArgument or idDoesNotExist error codes would be appropriate to report this case.' ],
1086 [ 'verb=ListIdentifiers&until=junk', [ 'badArgument' ], '', '' ],
1087 [ 'verb=ListIdentifiers&from=junk', [ 'badArgument' ], '', '' ],
1088 [ 'verb=ListIdentifiers&resumptionToken=junk&until=2000-02-05', [ 'badArgument','badResumptionToken' ], '', 'Either the badArgument and/or badResumptionToken error codes may be reported in this case. If only one is reported then the badArgument error is to be preferred because the resumptionToken and until parameters are exclusive.' ],
1089 [ 'verb=ListRecords&metadataPrefix=oai_dc&from=junk', [ 'badArgument' ], '', '' ],
1090 [ 'verb=ListRecords&resumptionToken=junk', [ 'badResumptionToken' ], '', '' ],
1091 [ 'verb=ListRecords&metadataPrefix=oai_dc&resumptionToken=junk&until=1990-01-10', [ 'badArgument','badResumptionToken' ] , '', 'Either the badArgument and/or badResumptionToken error codes may be reported in this case. If only one is reported then the badArgument error is to be preferred because the resumptionToken and until parameters are exclusive.' ],
1092 [ 'verb=ListRecords&metadataPrefix=oai_dc&until=junk', [ 'badArgument' ], '', '' ],
1093 [ 'verb=ListRecords', [ 'badArgument' ], '', '' ]
1094 );
1095  
1096 my $n=0;
1097 foreach my $rrr ( @request_list ) {
1098 my ($request_string, $error_codes, $xml_reason, $reason)=@$rrr;
1099 my $req = $self->base_url.'?'.$request_string;
1100 my $ok_errors=join(' or ',@$error_codes);
1101  
1102 my $response=$self->make_request($req);
1103  
1104 # TBD: $response->status_line should also be checked? see output from
1105 # physnet.uni-oldenburg.de/oai/oai.php
1106 if ($self->protocol_version eq "1.1") {
1107 if ($response->code ne "400") {
1108 $self->log->note("Invalid requests which failed to return 400:") if $n == 0;
1109 $n++;
1110 $self->log->fail("Expected 400 from: $request_string");
1111 }
1112 } elsif ($self->protocol_version eq "2.0") {
1113 # The document must contain the proper error code
1114 unless ($self->parse_response($req,$response,$xml_reason)) {
1115 $self->log->fail("Can't parse malformed response. ".html_escape($xml_reason));
1116 $n++;
1117 next;
1118 }
1119 # check that the error code is in the error_list
1120 my $error_elements = $self->doc->getElementsByTagName('error');
1121 if (my $matching_code=$self->error_elements_include($error_elements, $error_codes)) {
1122 $self->log->pass("Error response correctly includes error code '$matching_code'");
1123 } else {
1124 $self->log->fail("Exception/error response did not contain error code ".
1125 "'$ok_errors' ".html_escape($reason));
1126 $n++;
1127 next;
1128 }
1129 } else {
1130 $self->log->fail("Invalid protocol version returned");
1131 $self->abort("test_expected_errors - invalid protocol version");
1132 }
1133 }
1134 my $total = scalar @request_list;
1135 if ($n==0) {
1136 $self->log->pass("All $total error requests properly handled");
1137 } else {
1138 $self->log->warn("Only ".($total-$n)." out of $total error requests properly handled");
1139 }
1140 return($n);
1141 }
1142  
1143  
1144 =head3 test_expected_v2_errors($earliest_datestamp,$metadata_prefix)
1145  
1146 There are some additional exception tests for OAI-PMH version 2.0.
1147  
1148 =cut
1149  
1150 sub test_expected_v2_errors {
1151 my $self=shift;
1152 my ($earliest_datestamp,$metadata_prefix)=@_;
1153  
1154 $self->log->start("Checking for version 2.0 specific exceptions");
1155  
1156 my $too_early_date=one_year_before($earliest_datestamp);
1157  
1158 # format of entries: [ request_string, [error_codes_accepable], resaon ]
1159 my @request_list = (
1160 [ "verb=ListRecords&metadataPrefix=".url_encode($metadata_prefix)."&from=2002-02-05&until=2002-02-06T05:35:00Z", ['badArgument'],
1161 'The request has different granularities for the from and until parameters.' ],
1162 [ "verb=ListRecords&metadataPrefix=".url_encode($metadata_prefix)."&until=$too_early_date" , ['noRecordsMatch'],
1163 'The request specified a date one year before the earliestDatestamp given in the Identify response. '.
1164 'There should therefore not be any records with datestamps on or before this date and a noRecordsMatch '.
1165 'error code should be returned.' ]
1166 );
1167  
1168 foreach my $rrr ( @request_list ) {
1169 my ($request_string,$error_codes,$reason)=@$rrr;
1170  
1171 my $req=$self->base_url."?$request_string";
1172 my $response = $self->make_request($req);
1173 # parse the response content for the desired error code
1174 unless ( $self->parse_response($req,$response) ) {
1175 $self->log->fail("Error in parsing XML response to exception request: $request_string");
1176 next;
1177 }
1178 # check that there is at least the desired error code
1179 my $ok_errors=join(' or ',@$error_codes);
1180 my $error_elements = $self->doc->getElementsByTagName('error');
1181 if ( !$error_elements or $error_elements->getLength==0 ) {
1182 $self->log->fail("Failed to extract error code from the response to request: ".
1183 "$request_string $reason");
1184 } elsif (my $matching_code=$self->error_elements_include($error_elements, $error_codes) ) {
1185 $self->log->pass("Error response correctly includes error code '$matching_code'");
1186 } else {
1187 $self->log->fail("Error code $ok_errors not found in response but should be given ".
1188 "to the request: $request_string $reason");
1189 }
1190 }
1191 return;
1192 }
1193  
1194  
1195 =head2 METHODS TO TEST USE OF HTTP POST
1196  
1197 =head3 test_post_requests()
1198  
1199 Test responses to POST requests. Do both the simplest possible -- the Identify
1200 verb -- and a GetRecord request which uses two additional parameters.
1201  
1202 =cut
1203  
1204 sub test_post_requests {
1205 my $self=shift;
1206 my ($metadata_prefix)=@_;
1207  
1208 $self->log->start("Checking that HTTP POST requests are handled correctly");
1209  
1210 $self->test_post_request(1,{verb => "Identify"});
1211  
1212 my $record_id=$self->example_record_id;
1213 if ($record_id) {
1214 $self->test_post_request(2,{verb => "GetRecord", 'identifier' => $record_id, 'metadataPrefix' => $metadata_prefix});
1215 } else {
1216 $self->log->fail("Cannot test GetRecord via POST without and example record identifier");
1217 }
1218 }
1219  
1220  
1221 # Called just by test_post_requests to actually run the test
1222 #
1223 sub test_post_request {
1224 my $self=shift;
1225 my ($num, $post_data) = @_;
1226 my $response = $self->make_request($self->base_url, $post_data);
1227 if ($response->is_success) {
1228 my $verb = $post_data->{verb};
1229 if ( $self->is_verb_response($response,$verb) ) {
1230 $self->log->pass("POST test $num for $verb was successful");
1231 } elsif ( $self->check_error_response($response) ) {
1232 $self->log->fail("POST test $num for $verb was unsuccessful, an OAI error ".
1233 "response was received");
1234 } else {
1235 $self->log->fail("POST test $num for $verb was unsuccessful, got neither a ".
1236 "valid response nor an error");
1237 }
1238 } else {
1239 $self->log->fail("POST test $num was unsuccessful. Server returned HTTP Status: '".
1240 $response->status_line."'");
1241 }
1242 }
1243  
1244  
1245 =head2 METHODS CHECKING ELEMENTS WITHIN VERB AND ERROR RESPONSES
1246  
1247 =head3 check_response_date($req, $doc)
1248  
1249 Check responseDate for being in UTC format
1250 (should perhaps also check that it is at least the current day?)
1251  
1252 =cut
1253  
1254 sub check_response_date {
1255 my $self=shift;
1256 my ($req, $doc) = @_;
1257  
1258 my $elements = $self->doc->getElementsByTagName('responseDate');
1259 # assume rest of validity already checked, just take first
1260 my $item;
1261 my $child;
1262 if ($elements and $item=$elements->item(0) and $child=$item->getFirstChild()) {
1263 my $date = $child->getData();
1264 if ($date=~/\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ/) {
1265 $self->log->pass("responseDate has correct format: $date");
1266 } else {
1267 $self->log->fail("Bad responseDate of $date, this is not in UTC DateTime ".
1268 "(YYYY-MM-DDThh:mm:ssZ) format");
1269 }
1270 } else {
1271 $self->log->fail("Failed to extract responseDate");
1272 }
1273 }
1274  
1275  
1276 =head3 check_schema_name($req, $doc)
1277  
1278 Given the response to one of the OAI verbs, make sure that it it
1279 going to be validated against the "official" OAI schema, and not
1280 one that the repository made up for itself. If the response can't
1281 be parsed, or if there is no OAI-PMH element, or if the schema is
1282 incorrect, print an error message and bump the error_count.
1283  
1284 Return true if the schema name and date check out, else return undef
1285  
1286 =cut
1287  
1288 sub check_schema_name {
1289 my $self=shift;
1290 my ($req, $doc) = @_;
1291  
1292 my $namespace = 'http://www.openarchives.org/OAI/2.0/';
1293 my $location = 'http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd';
1294  
1295 my $elements = $self->doc->getElementsByTagName('OAI-PMH'); #NodeList
1296 unless ( $elements->getLength() > 0 ) {
1297 $self->log->fail("Response to $req did not contain a OAI-PMH element");
1298 return(0);
1299 }
1300 my $attributes = $elements->item(0)->getAttributes; #Node->NamedNodeMap
1301 my $attr = $attributes->getNamedItem('xsi:schemaLocation'); #Node
1302 unless ( $attr ) {
1303 $self->log->fail("No xsi:schemaLocation attribute for the OAI-PMH element was ".
1304 "found, expected xsi:schemaLocation=\"$namespace $location\"");
1305 return(0);
1306 }
1307 $attr = $attributes->getNamedItem('xsi:schemaLocation'); #Node
1308 my $pair = $attr->getNodeValue(); # must pair OAI namespace with schema
1309 unless ( $pair =~ /^\s?$namespace\s*$location/ ) {
1310 $self->log->fail("Error in pairing OAI namespace with schema location, expected: ".
1311 "xsi:schemaLocation=\"$namespace $location\" but got $pair");
1312 return(0);
1313 }
1314 return(1);
1315 }
1316  
1317  
1318 =head3 check_protocol_version
1319  
1320 Extract the protocol version being used from the Identify response, check that it is
1321 valid and then abort unless 2.0.
1322  
1323 =cut
1324  
1325 sub check_protocol_version {
1326 my $self=shift;
1327 my $doc;
1328 # Extract the version number of the validator to run
1329 my $x = $self->doc->getElementsByTagName('protocolVersion');
1330 if (not $x) {
1331 $self->abort("Unknown protocol version, failed to extract protocolVersion element from Identify response");
1332 }
1333 my $protocol_version = $x->item(0)->getFirstChild->getData;
1334 if ($protocol_version ne '2.0' and
1335 $protocol_version ne '1.1' and
1336 $protocol_version ne '1.0') {
1337 $self->abort("Invalid protocol version ($protocol_version)");
1338 }
1339 $self->protocol_version( $protocol_version );
1340 if ($protocol_version ne '2.0') {
1341 $self->abort("Repository reports OAI-PMH protocol version $protocol_version and will not be validated. Guidelines for upgrading to 2.0 can be found at http://www.openarchives.org/OAI/2.0/migration.htm\n\n");
1342 }
1343 $self->log->pass("Correctly reports OAI-PMH protocol version 2.0");
1344 }
1345  
1346  
1347 =head2 is_verb_response($reponse,$verb)
1348  
1349 Return true if $response is a response for the specified $verb.
1350  
1351 FIXME -- need better checks!
1352  
1353 =cut
1354  
1355 sub is_verb_response {
1356 my $self=shift;
1357 my ($response,$verb) = @_;
1358 my $doc;
1359 eval { $doc=$self->parser->parse($response->content); };
1360 return unless $doc; # We can't parse it so it isn't a valid doc
1361 my $verb_elements = $doc->getElementsByTagName($verb);
1362 return(1) if ( $verb_elements and $verb_elements->getLength==1 );
1363 return; # not the one element we expected
1364 }
1365  
1366  
1367 =head3 error_elements_include($error_elements,$error_codes)
1368  
1369 Determine whether the list of error elements ($error_elements) includes at least
1370 one of the desired codes. Return string with first matching error code, else
1371 return false/nothing.
1372  
1373 Does a sanity check on $error_list to check that it is set and has length>0
1374 before trying to match, so cose calling it can simply do a
1375 getElementsByTagName or similar before caling.
1376  
1377 =cut
1378  
1379 sub error_elements_include {
1380 my $self=shift;
1381 my ($error_elements, $error_codes) = @_;
1382 # sanity check
1383 return if (!$error_elements or $error_elements->getLength==0);
1384 for (my $i=0; $i<$error_elements->getLength; $i++) {
1385 foreach my $ec (@$error_codes) {
1386 my $code = $error_elements->item($i)->getAttribute('code') || 'no-code-attribute';
1387 $self->log->note("$code =? $ec") if ($self->debug);
1388 return($ec) if ($code eq $ec);
1389 }
1390 }
1391 return;
1392 }
1393  
1394  
1395  
1396 =head3 check_error_response($response)
1397  
1398 Given the response to an HTTP request, make sure it is not an
1399 OAI-PMH error message. The $response is a success. If it is an
1400 OAI error message, return 2; if the response cannot be parsed, return
1401 -1; otherwise return undef (it must be a real Identify response).
1402  
1403 FIXME -- need better checks!
1404  
1405 FIXME -- need to merge thic functionality in with is_error_response
1406  
1407 =cut
1408  
1409 sub check_error_response {
1410 my $self=shift;
1411 my ($response) = @_;
1412 my $doc;
1413 eval { $doc=$self->parser->parse($response->content); };
1414 return unless $doc; # We can't parse it so it isn't a valid error
1415 my $error_elements = $doc->getElementsByTagName('error');
1416 return(1) if ( $error_elements and $error_elements->getLength() > 0 );
1417 return; # no error codes so not an error response
1418 }
1419  
1420  
1421 =head3 get_earliest_datestamp()
1422  
1423 A new exception check for Version 2.0 raises noRecordsMatch errorcode
1424 if the set of records returned by ListRecords is empty. This requires
1425 that we know the earliest date in the repository. Also check that the
1426 earliest date matches the specified granularity.
1427  
1428 Called only for version 2.0 or greater.
1429  
1430 Since the Identify response has already been validated, we know
1431 there is exactly one earliestDatestamp element in the current document.
1432 Extract this value, check it, and if it looks good then set
1433 $self->earliest_datestamp and return false.
1434  
1435 If there is an error then return string explaining that.
1436  
1437 =cut
1438  
1439 sub get_earliest_datestamp {
1440 my $self=shift;
1441  
1442 my $earliest = $self->doc->getElementsByTagName('earliestDatestamp');
1443 my $el = $earliest->item(0);
1444 return("Can't get earliestDatestamp element from Identify response.") unless ($el);
1445 return("earliestDatestamp element is empty in Identify response.") unless ($el->getFirstChild);
1446  
1447 my $error='';
1448 my $earliest_datestamp = $el->getFirstChild->getData;
1449 $self->log->note("Earliest datestamp in repository is $earliest_datestamp") if $self->debug;
1450  
1451 $earliest_datestamp =~ /^([0-9]{4})-([0-9][0-9])-([0-9][0-9])(.*)$/;
1452 if ($1 eq '' || $2 eq '' || $3 eq '') {
1453 $error="is not in ISO8601 format";
1454 } elsif ( $4 eq '' and $self->granularity eq 'seconds') {
1455 $error="must have seconds granularity (format YYYY-MM-DDThh:mm:ssZ) to match ".
1456 "the granularity for the repository. The granularity has been set to seconds ".
1457 "by the granularity element of the Identify response.\n";
1458 } elsif ( $4 ne '' and $self->granularity eq 'days') {
1459 $error="must have days granularity (format YYYY-MM-DD) to match the granularity for ".
1460 "the repository. The granularity has been set to days by the granularity ".
1461 "element of the Identify response (or that element is bad/missing).\n";
1462 } elsif ( $self->granularity eq 'seconds' and $4 !~ /^T\d\d:\d\d:\d\d(\.\d+)?Z$/ ) {
1463 $error="does not have the correct format for the time part of the UTCdatetime. The ".
1464 "overall format must be YYYY-MM-DDThh:mm:ssZ.\n";
1465 }
1466 if ($error) {
1467 # Sanitize for error message
1468 return("The earliestDatestamp in the identify response (".
1469 sanitize($earliest_datestamp).") $error");
1470 } else {
1471 $self->earliest_datestamp($earliest_datestamp);
1472 return;
1473 }
1474 }
1475  
1476  
1477 =head3 parse_granularity($granularity_element)
1478  
1479 Parse contents of the granularity element of the Identify response. Returns either
1480 'days', 'seconds' or nothing on failure. Sets $self->granularity if valid, otherwise
1481 does not change setting.
1482  
1483 As of v2.0 the granularity element is mandatory, see:
1484 http://www.openarchives.org/OAI/openarchivesprotocol.html#Identify
1485  
1486 =cut
1487  
1488 sub parse_granularity {
1489 my $self=shift;
1490 my ($gran) = @_;
1491 if (!$gran or $gran->getLength==0) {
1492 $self->log->fail("Missing granularity element");
1493 return;
1494 } elsif ($gran->getLength>1) {
1495 $self->log->fail("Multiple granularity elements");
1496 return;
1497 }
1498 #schema validation guarantees that there is a spec here
1499 my $el=$gran->item(0)->getFirstChild->getData;
1500 if ($el eq 'YYYY-MM-DD') {
1501 $self->granularity('days');
1502 return($self->granularity);
1503 } elsif ($el eq 'YYYY-MM-DDThh:mm:ssZ') {
1504 $self->granularity('seconds');
1505 return($self->granularity);
1506 } else {
1507 $self->log->fail("Bad value for the granularity element '$el', must be either ".
1508 "YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ");
1509 return;
1510 }
1511 }
1512  
1513  
1514 =head3 get_datestamp_granularity($datestamp)
1515  
1516 Parse the datestamp supplied and return 'days' if it is valid with granularity
1517 of days, 'seconds' if it is valid for seconds granularity, and nothing if it is not
1518 valid.
1519  
1520 # FIXME - should add more validation
1521  
1522 =cut
1523  
1524 sub get_datestamp_granularity {
1525 my $self=shift;
1526 my ($datestamp)=@_;
1527 if ($datestamp=~/^(\d\d\d\d)-(\d\d)-(\d\d)$/) {
1528 return 'days' if ($2>=1 and $2<=12 and $3>=1 and $3<=31);
1529 } elsif ($datestamp=~/^(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)(\.\d+)?Z$/) {
1530 return 'seconds' if ($2>=1 and $2<=12 and $3>=1 and $3<=31 and $4<24 and $5<60);
1531 }
1532 return;
1533 }
1534  
1535  
1536 =head3 is_no_records_match
1537  
1538 Returns true if the current document contains and error code element with the code "noRecordsMatch"
1539  
1540 ### FIXME - should be merged into an extended is_error_response
1541  
1542 =cut
1543  
1544 sub is_no_records_match {
1545 my $self=shift;
1546 my $error_elements = $self->doc->getElementsByTagName('error');
1547 return( $self->error_elements_include($error_elements, ['noRecordsMatch']) );
1548 }
1549  
1550  
1551 =head3 get_resumption_token()
1552  
1553 See if there is a resumptionToken with this response, return
1554 value if present, empty if not or if there is some other error.
1555  
1556 =cut
1557  
1558 sub get_resumption_token {
1559 my $self=shift;
1560  
1561 my $tokenList = $self->doc->getElementsByTagName('resumptionToken');
1562 if ( !$tokenList or $tokenList->getLength()==0 ) {
1563 return; #no resumptionToken
1564 }
1565  
1566 # Dig out the resumption token from the document
1567 my $token = $tokenList->item(0)->getFirstChild();
1568  
1569 # Try getting the resumption token, $token will be will be undefined
1570 # unless the element has content
1571 if ($token) {
1572 return($token->getData());
1573 }
1574 return;
1575 }
1576  
1577  
1578 =head3 is_error_response($details)
1579  
1580 Look at the parsed response in $self->doc to see if it is an error response,
1581 parse data and return true if it is.
1582  
1583 Returns true (a printable string containing the error messages) if response was a valid
1584 OAI_PMH error response, codes in %$details if a hash reference is passed in.
1585  
1586 =cut
1587  
1588 sub is_error_response {
1589 my $self=shift;
1590 my ($details)=@_;
1591 $details={} unless (ref($details) eq 'HASH'); #dummy hash unless one supplied
1592 #
1593 my $error_elements = $self->doc->getElementsByTagName('error');
1594 if ($error_elements and $error_elements->getLength()>=1) {
1595 my $msg='';
1596 for (my $i=0; $i<$error_elements->getLength; $i++) {
1597 my $code=$error_elements->item($i)->getAttribute("code");
1598 my $child=$error_elements->item($i)->getFirstChild();
1599 unless ($child) {
1600 # Warn about no content unless it is the special case of noSetHierarchy
1601 # where the error code really is sufficient
1602 unless ($code eq 'noSetHierarchy') {
1603 $self->log->warn("No human readable message included in error element for ".
1604 "$code error, this is discouraged");
1605 }
1606 $details->{$code}='[NO MESSAGE RETURNED]';
1607 $msg.="[$code] ";
1608 } else {
1609 $details->{$code}=$child->getData();
1610 $msg.="[$code: $details->{$code}] ";
1611 }
1612 }
1613 return($msg);
1614 } else {
1615 return;
1616 }
1617 }
1618  
1619  
1620 =head3 get_admin_email()
1621  
1622 Extract admin email from a parsed Identify response in $self->doc).
1623 Also note that the email target may have been set via form option
1624  
1625 Returns the pair of ($email,$error) where $email is the combined
1626 set of email addresses (comma separated). $error will be undef
1627 or a string with error message to users.
1628  
1629 =cut
1630  
1631 sub get_admin_email {
1632 my $self=shift;
1633  
1634 my $adminEmailElements = $self->doc->getElementsByTagName('adminEmail');
1635 my @emails=();
1636 my $n = $adminEmailElements->getLength;
1637 if ($n > 0) {
1638 my $name_node = $adminEmailElements->item(0)->getFirstChild;
1639 if ($name_node) {
1640 for (my $i=0; $i<$n; $i++) {
1641 my $e=$adminEmailElements->item($i)->getFirstChild->getData;
1642 if ($e=~s/mailto://g) {
1643 $self->log->warn("Stripped mailto: prefix from adminEmail address, this ".
1644 "should not be included.");
1645 }
1646 if (my $msg=$self->bad_admin_email($e)) {
1647 return(undef,$msg);
1648 }
1649 push(@emails,$e);
1650 }
1651 } else {
1652 $self->log->fail("adminEmail element is empty!");
1653 return(undef);
1654 }
1655 } else {
1656 $self->log->fail("No adminEmail element!");
1657 return(undef);
1658 }
1659 my $email=join(',',@emails);
1660 $self->log->pass("Administrator email address is '$email'");
1661 return($email);
1662 }
1663  
1664  
1665 =head3 bad_admin_email($admin_email)
1666  
1667 Check for some stupid email addresses to avoid so much bounced email.
1668 Returns a string (True) if bad, else nothing.
1669  
1670 =cut
1671  
1672 sub bad_admin_email {
1673 my $self=shift;
1674 my ($admin_email)=@_;
1675 if ($admin_email=~/\@localhost$/) {
1676 $self->log->fail("adminEmail '$admin_email' is local. This must be corrected to a ".
1677 "valid globally resolvable email address before tests can continue");
1678 return("local adminEmail");
1679 } elsif ($admin_email!~/^\w[\w\-\.]+\@[a-zA-Z0-9\-\.]+\.[a-z]{2,}$/) {
1680 $self->log->fail("adminEmail '$admin_email' looks bogus. This must be corrected to ".
1681 "a valid email address before tests can continue");
1682 return("looks like bogus adminEmail");
1683 }
1684 return;
1685 }
1686  
1687  
1688 =head2 METHODS FOR MAKING REQUESTS AND PARSING RESPONSES
1689  
1690 =head3 make_request_and_validate($verb, $req)
1691  
1692 Given the base URL that we are validating, the Verb that we are checking
1693 and the complete query to be sent to the OAI server, get the response to
1694 the verb. Validation has already been done, so we need only do some
1695 special checks here. Return the response to the OAI verb,
1696 or undef if the OAI server failed to respond to that verb.
1697  
1698 Side effects: errors may be printed and error_count bumped.
1699 If the verb involved is "Identify" then set the version number and the
1700 email address, assuming that some response has been obtained.
1701  
1702 Simple well-formedness is checked by this routine. An undef exit means
1703 that any calling code should fail the test but need not report 'no response'.
1704  
1705 If the response is true then $self->doc contains a parsed XML
1706 document.
1707  
1708 This is the usual way we make requests with integrated parsing and error
1709 checking. This method is built around calls to L<make_request> and
1710 L<parse_response>.
1711  
1712 =cut
1713  
1714 sub make_request_and_validate {
1715 my $self=shift;
1716 my ($verb, $req) = @_;
1717  
1718 my $response = $self->make_request($req);
1719  
1720 unless ( $response->is_success ) {
1721 my $status = $response->status_line;
1722 my $age = $response->current_age;
1723 my $lifetime = $response->freshness_lifetime;
1724 my $is_fresh = $response->is_fresh;
1725 $self->log->fail("Server failed to respond to the $verb request (HTTP header ".
1726 "values: status=$status, age=$age, lifetime=$lifetime, ".
1727 "is fresh:=$is_fresh)");
1728 return;
1729 }
1730  
1731 unless ($self->parse_response($req, $response)) {
1732 $self->log->fail("Failed to parse response");
1733 return;
1734 }
1735  
1736 # Check that the responseDate is in UTC format
1737 $self->check_response_date($req,$self->doc);
1738 # Check that the response refers to the "official" OAI schema
1739 $self->check_schema_name($req,$self->doc);
1740  
1741 return($response);
1742 }
1743  
1744  
1745 =head3 make_request($url,$post_data)
1746  
1747 Routine to GET or POST a request, handle 503's, and return the response
1748  
1749 Second parameter, $post_data, must be hasfref to POST data to indicate that
1750 the request should be an HTTP POST request instead of a GET.
1751  
1752 =cut
1753  
1754 sub make_request {
1755 my $self=shift;
1756 my ($url,$post_data) = @_;
1757  
1758 # Is this https and do we allow that?
1759 if (is_https_uri($url)) {
1760 $self->uses_https(1);
1761 if (not $self->allow_https) {
1762 $self->abort("URI $url is https. Use of https URIs is not allowed ".
1763 "by the OAI-PMH v2.0 specification");
1764 }
1765 }
1766  
1767 my $request;
1768 if ($post_data) {
1769 my $content_msg=''; #nice string to report
1770 # Sort keys in alpha order for consistent behavior
1771 foreach my $k (sort keys(%$post_data)) {
1772 my $v=$post_data->{$k};
1773 $content_msg.="$k:$v ";
1774 }
1775 $self->log->request($url,'POST',$content_msg);
1776 $request = POST($url,'Content'=>$post_data);
1777 } else {
1778 $self->log->request($url,'GET');
1779 $request = GET($url);
1780 }
1781 my $response;
1782 my $tries=0;
1783 my $try_again = 1;
1784 while ( $try_again ) {
1785 #$ua->max_redirect(0);
1786 $response = $self->ua->request($request);
1787 #
1788 # Write response if requested
1789 if ($self->save_all_responses) {
1790 my $response_file="/tmp/".$self->run_id.".".$self->response_number;
1791 open(my $fh,'>',$response_file) || $self->abort("Can't write response $response_file: $!");
1792 print {$fh} $response->content();
1793 $self->log->note("Response saved as $response_file") if ($self->debug);
1794 close($fh);
1795 $self->{response_number}++;
1796 }
1797 $tries++;
1798 if ($tries > $self->max_retries) {
1799 $self->abort("Too many 503 Retry-After or 302 Redirect responses received in a row");
1800 }
1801 #
1802 # Check response for 503 and 302
1803 if ($response->code eq '503') {
1804 # 503 (Retry-After), expect to get a time too
1805 $self->uses_503(1);
1806 if (defined $response->header("Retry-After")) {
1807 my $retryAfter=$response->header("Retry-After");
1808 if ($retryAfter=~/^\d+$/) {
1809 if ($retryAfter<=3600) {
1810 ###FIXME: Should check the Retry-After value carefully and barf if bad
1811 my $sleep_time = 1 + $response->header("Retry-After");
1812 $self->log->note("Status: ".$response->code().
1813 " -- going to sleep for $sleep_time seconds.");
1814 sleep $sleep_time;
1815 } else {
1816 $self->abort("503 response with Retry-After > 1hour (3600s), aborting");
1817 }
1818 } else {
1819 $self->log->fail("503 response with bad (non-numeric) Retry-After time, ".
1820 "will wait 10s");
1821 sleep 10;
1822 }
1823 } else {
1824 $self->log->warn("503 response without Retry-After time, will wait 10s");
1825 sleep 10;
1826 }
1827 } elsif ($response->code eq '302') {
1828 # 302 (Found) redirect
1829 my $loc=$response->header('Location');
1830 if ($loc!~m%^http://([^\?&]+)%) {
1831 if (is_https_uri($loc)) {
1832 $self->uses_https(1);
1833 if (not $self->allow_https) {
1834 $self->abort("Redirect URI specified in 302 response is https. Use of ".
1835 "https URIs is not allowed by the OAI-PMH v2.0 specification");
1836 }
1837 } else {
1838 $self->abort("Bad redirect URI specified in 302 response");
1839 }
1840 }
1841 # Make new request
1842 if ($post_data and $loc!~/\?/) { #don't do POST if new Location includes ?
1843 $request = POST($loc,'Content'=>$post_data);
1844 } else {
1845 $request = GET($loc);
1846 }
1847 } elsif ($response->code eq '501') {
1848 $self->abort("Got 501 Not Implemented response which may either have come from ".
1849 "the server or have been generated within the validator because the ".
1850 "request type (perhaps https) is not supported.");
1851 } else {
1852 $try_again=0;
1853 }
1854 }
1855 # Check for oversize limit (indicated by X-Content-Range header)
1856 if (defined $response->header('X-Content-Range')) {
1857 $self->log->fail("Response to <$url> exceeds maximum size limit (".$self->max_size." bytes), discarded. ".
1858 "While this limit is set only in this validation program you should not use excessively ".
1859 "large responses as service providers will likely have problems parsing the XML. You ".
1860 "should split the responses using the resumptionToken mechanism. (X-Content-Range: '".
1861 $response->header('X-Content-Range')."' Content-Length: '".$response->content_length."')\n");
1862 $response->content('');
1863 }
1864 return $response;
1865 }
1866  
1867  
1868 =head3 parse_response($request_url,$response,$xml_reason)
1869  
1870 Attempt to parse the HTTP response $response, examining both the response code
1871 and then attempting to parse the content as XML.
1872  
1873 If $xml_reason is specified then this is added to the failure message, if
1874 nothing is specified then a standard message about UTF-8 issues is
1875 added.
1876  
1877 Returns true on success and sets $self->doc with the parsed XML document.
1878 If unsuccessful, log an error message, bump the error count, and
1879 return false.
1880  
1881 =cut
1882  
1883 sub parse_response {
1884 my $self=shift;
1885 my ($request_url,$response,$xml_reason) = @_;
1886 $xml_reason='' unless (defined $xml_reason);
1887 #
1888 # Fail if reponse=undef, else check to see if response is ref to
1889 # response object or is string
1890 if (!defined($response) or not ref($response)) {
1891 $self->log->warn("Bad response from server");
1892 return;
1893 }
1894 # Unpack the bits we want from response object
1895 my $code=$response->code;
1896 my $content=$response->content;
1897 # Check return code (if given)
1898 if ($code and $code=~/^[45]/) {
1899 $self->log->warn("Bad HTTP status code from server: $code");
1900 return;
1901 }
1902 #
1903 # Check content
1904 my $doc;
1905 eval { $doc=$self->parser->parse($content); };
1906 unless ( $doc ) {
1907 my $err=$@;
1908 $err=~s/^\s+//;
1909 $err=~s%at\s+/usr/lib/perl.*%%i; #trim stuff about our perl installation
1910 if ($request_url) {
1911 unless ($xml_reason) {
1912 $xml_reason="The most common reason for malformed responses is illegal bytes in ".
1913 "UTF-8 streams (e.g. the inclusion of Latin1 characters with codes>127 ".
1914 "without creating proper UTF-8 mutli-byte sequences). You might find ".
1915 "the utf8conditioner, found on the OAI tools page helpful for debugging.";
1916 }
1917 $self->log->warn("Malformed response: $err. $xml_reason");
1918 }
1919 return;
1920 }
1921 # Set parsed document
1922 $self->doc( $doc );
1923 return(1);
1924 }
1925  
1926  
1927 =head2 UTILITY FUNCTIONS
1928  
1929 =head3 html_escape($str)
1930  
1931 Escapes characters which have special meanings in HTML
1932  
1933 =cut
1934  
1935 sub html_escape {
1936 my $string = shift;
1937 $string =~ s/&/&amp;/g; #must be first!
1938 $string =~ s/</&lt;/g;
1939 $string =~ s/>/&gt;/g;
1940 $string =~ s/"/&quot;/g;
1941 $string =~ s/'/&apos;/g;
1942 return $string;
1943 }
1944  
1945 =head3 one_year_before($date)
1946  
1947 Assumes properly formatted date, decrements year by one
1948 via string manipulation and returns date.
1949  
1950 =cut
1951  
1952 sub one_year_before {
1953 my ($date)=@_;
1954 my ($year) = $date =~ /^([0-9]{4})/;
1955 my $year_minus_one = sprintf('%04d',($year - 1)); #make sure we get leading zeros
1956 $date =~ s/^$year/$year_minus_one/;
1957 return($date);
1958 }
1959  
1960 =head3 url_encode($str)
1961  
1962 Escape/encode any characters that aren't in the small safe set for URLs
1963  
1964 =cut
1965  
1966 sub url_encode {
1967 my $str=shift;
1968 $str =~ s/([^\w\/\,\- ])/sprintf("%%%02X",ord($1))/eg;
1969 $str =~ tr/ /+/;
1970 return($str);
1971 }
1972  
1973  
1974 =head3 is_https_uri($uri)
1975  
1976 Return true if the URI is an https URI, false otherwise.
1977  
1978 =cut
1979  
1980 sub is_https_uri {
1981 my $uri=shift;
1982 return($uri=~m%^https:%);
1983 }
1984  
1985  
1986 =head3 sanitize($str)
1987  
1988 Return a sanitized version of $str that doesn't contain odd
1989 characters and it not over 80 chars long. Will have the
1990 string '(sanitized)' appended if changed.
1991  
1992 =cut
1993  
1994 sub sanitize {
1995 my ($str)=@_;
1996 my $out=$str;
1997 $out=~s/[^\w\-:;.!@#%^*\(\) ]/_/g;
1998 $out=substr($out,0,80);
1999 if ($out ne $str) {
2000 $out.='(sanitized)';
2001 }
2002 return($out);
2003 }
2004  
2005  
2006 =head1 SUPPORT
2007  
2008 Please report any bugs of questions about validation via the
2009 OAI-PMH discussion list at L<https://groups.google.com/d/forum/oai-pmh>.
2010 Be sure to make it clear that you are talking about the
2011 HTTP::OAIPMH::Validator module.
2012  
2013 =head1 AUTHORS
2014  
2015 Simeon Warner, Donna Bergmark
2016  
2017 =head1 HISTORY
2018  
2019 This module is based on an OAI-PMH validator first written by Donna Bergmark
2020 (Cornell University) in 2001-01 for the OAI-PMH validation and registration
2021 service (L<http://www.openarchives.org/data/registerasprovider.html>).
2022 Simeon Warner (Cornell University) took over the validator and operation of
2023 the registration service in 2004-01, and then did a significant tidy/rework
2024 of the code. That code ran the validation and registration service with
2025 few changes through 2015-01. Some of the early work on the OAI-PMH validation
2026 service was supported through NSF award number 0127308.
2027  
2028 Code was abstracted into this module 2015-01 by Simeon Warner and is
2029 used for the OAI-PMH validation and registration service on
2030 L<http://www.openarchives.org/pmh/>.
2031  
2032 =head1 COPYRIGHT
2033  
2034 Copyright 2001..2017 by Simeon Warner, Donna Bergmark.
2035  
2036 This library is free software; you can redistribute it and/or modify it under
2037 the same terms as Perl itself.
2038  
2039 =cut
2040  
2041 1;
« Log.pm ↑ Summary