Test/Attean/W3CManifestTestSuite.pm untested

filter: Statementstmt 0.0% 0 / 1038 Branchbran 0.0% 0 / 348 Conditioncond 0.0% 0 / 191 MC/DCmcdc 0.0% 0 / 102 Subroutinesub 0.0% 0 / 46 Podpod 0.0% 0 / 18 totaltotal 0.0% 0 / 1743 CC 175 SCAR 63.6
CC
175
Cov
0%
data_syntax_eval_test
1406.0
update_eval_test
1260.0
query_eval_test
552.0
CRAP 579.2
1 package Test::Attean::W3CManifestTestSuite;
2  
3 use v5.14;
4 use warnings;
5  
6 use Attean;
7 use Attean::RDF;
8 use AtteanX::Parser::SPARQL;
9 use Attean::SimpleQueryEvaluator;
10 use Test::Attean::TestSimpleQueryEvaluator;
11 use Test::Attean::TestIDPQueryPlanner;
12 use Module::Load::Conditional qw[can_load];
13  
14 use Carp;
15 use HTTP::Request;
16 use HTTP::Response;
17 use HTTP::Message::PSGI;
18 use Data::Dumper;
19 use Encode qw(encode encode_utf8 decode_utf8);
20 use Getopt::Long;
21 use Regexp::Common qw /URI/;
22 use Scalar::Util qw(blessed reftype);
23 use List::Util qw(all);
24 use Test::Modern;
25 use Test::Roo::Role;
26 use Text::CSV;
27 use Text::Table;
28 use Try::Tiny;
29 use URI::file;
30 use File::Spec;
31 use Types::Standard qw(Str Bool ArrayRef HashRef InstanceOf ConsumerOf);
32 require XML::Simple;
33  
34 my $XSD = 'http://www.w3.org/2001/XMLSchema#';
35 my $RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
36 my $RDFS = 'http://www.w3.org/2000/01/rdf-schema#';
37 my $RDFT = 'http://www.w3.org/ns/rdftest#';
38 my $RS = 'http://www.w3.org/2001/sw/DataAccess/tests/result-set#';
39 my $MF = 'http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#';
40 my $UT = 'http://www.w3.org/2009/sparql/tests/test-update#';
41 my $RQ = 'http://www.w3.org/2001/sw/DataAccess/tests/test-query#';
42 my $DAWGT = 'http://www.w3.org/2001/sw/DataAccess/tests/test-dawg#';
43  
44 requires 'test_model';
45 requires 'manifest_paths';
46  
47 has run_update_tests => (is => 'rw', isa => Bool, default => 1);
48 has run_query_tests => (is => 'rw', isa => Bool, default => 1);
49 has run_syntax_tests => (is => 'rw', isa => Bool, default => 1);
50  
51 has debug => (is => 'rw', isa => Bool, default => 0);
52 has results => (is => 'rw', isa => Bool, default => 0);
53 has strict_approval => (is => 'rw', isa => Bool, default => 0);
54 has use_idp_planner => (is => 'rw', isa => Bool, default => 1);
55 has pattern => (is => 'rw', isa => Str, default => '');
56  
57 has tests_dir => (is => 'rw', required => 1, default => sub { $ENV{ATTEAN_SPARQL_TESTS_DIR} });
58 has model => (is => 'rw', isa => ConsumerOf['Attean::API::Model'], init_arg => undef);
59 has manifests => (is => 'rw', isa => ArrayRef, init_arg => undef);
60 has default_graph => (is => 'rw');
61 has failures => (is => 'rw', isa => HashRef, default => sub { +{} });
62 has supported_extensions => (is => 'rw', isa => HashRef, default => sub { +{} });
63  
64 has earl => (is => 'rw', isa => InstanceOf['RDF::EARL']);
65  
66 sub BUILD {
67 my $self = shift;
68 if ($self->pattern) {
69 $self->results(1);
70 }
71 }
72  
73 sub memory_model {
74 my $self = shift;
75 my $store = Attean->get_store('Memory')->new();
76 my $model = Attean::MutableQuadModel->new( store => $store );
77 return $model;
78 }
79  
80 sub setup {
81 my $self = shift;
82 $self->default_graph(iri('http://graph/'));
83  
84 warn 'setting up sparql test harness' if ($self->debug);
85 my @manifests_iris = $self->manifest_paths();
86 unless (scalar(@manifests_iris)) {
87 my $dir = $self->tests_dir;
88 plan skip_all => "No manifest files found in $dir";
89 exit(0);
90 }
91  
92 my $model = $self->memory_model();
93 $self->model($model);
94 my $class = Attean->get_parser("turtle") || die "Failed to load parser for 'turtle'";
95  
96 my %loaded;
97
98 my %manifests;
99 my @load = map { iri("file://" . File::Spec->rel2abs($_)) } @manifests_iris;
100 while (scalar(@load)) {
101 foreach my $iri (@load) {
102 warn "Loading " . $iri->value . "\n" if ($self->debug);
103 $loaded{ $iri->value }++;
104 }
105 $model->load_urls_into_graph($self->default_graph, @load);
106 @load = ();
107  
108 my $subjects = $model->subjects( iri("${RDF}type"), iri("${MF}Manifest") );
109 my @manifest_matches = $subjects->elements;
110 foreach my $m (@manifest_matches) {
111 if (not exists $manifests{ $m->value }) {
112 $manifests{ $m->value } = $m;
113 my @list_heads = $model->objects( $m, iri("${MF}include") )->elements;
114 my @elements = map { $model->get_list(undef, $_)->elements() } @list_heads;
115 push(@load, grep { not exists($loaded{$_->value}) } @elements);
116 }
117 }
118 }
119
120 my @manifests = values %manifests;
121
122 if ($self->debug) {
123 warn "done parsing manifests" if $self->debug;
124 warn "setting manifests:\n";
125 foreach my $m (@manifests) {
126 warn "- " . $m->as_string . "\n";
127 }
128 }
129 $self->manifests(\@manifests);
130 }
131  
132 sub sparql_syntax_test {
133 my $self = shift;
134 my $test_type = shift;
135 my $model = shift;
136 my $test = shift;
137 my %args = @_;
138 my $specVersion = $args{'specVersion'} // '1.1';
139  
140 my $handled = 0;
141
142 my $type = iri( "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" );
143 my $mfname = iri( "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#name" );
144 my ($queryd) = $model->objects( $test, iri("${MF}action") )->elements;
145 my ($approved) = $model->objects( $test, iri("${DAWGT}approval") )->elements;
146 my ($name) = $model->objects( $test, $mfname )->elements;
147 my $testname = $test->value;
148 $testname =~ s{^.*[/#]}{};
149
150 if ($self->strict_approval) {
151 unless ($approved) {
152 warn "- skipping test because it isn't approved\n" if ($self->debug);
153 return;
154 }
155 if ($approved->equal("${DAWGT}NotClassified")) {
156 warn "- skipping test because its approval is dawgt:NotClassified\n" if ($self->debug);
157 return;
158 }
159 }
160  
161 my $is_pos_query = $model->count_quads($test, $type, iri("${MF}PositiveSyntaxTest")) + $model->count_quads($test, $type, iri("${MF}PositiveSyntaxTest11"));
162 my $is_pos_update = $model->count_quads($test, $type, iri("${MF}PositiveUpdateSyntaxTest")) + $model->count_quads($test, $type, iri("${MF}PositiveUpdateSyntaxTest11"));
163 my $is_neg_query = $model->count_quads($test, $type, iri("${MF}NegativeSyntaxTest")) + $model->count_quads($test, $type, iri("${MF}NegativeSyntaxTest11"));
164 my $is_neg_update = $model->count_quads($test, $type, iri("${MF}NegativeUpdateSyntaxTest")) + $model->count_quads($test, $type, iri("${MF}NegativeUpdateSyntaxTest11"));
165  
166 my $uri = URI->new( $queryd->value );
167 my $filename = $uri->file;
168 my (undef,$base,undef) = File::Spec->splitpath( $filename );
169 $base = "file://${base}";
170 # warn "Loading SPARQL query from file $filename" if ($self->debug);
171 my $sparql = do { local($/) = undef; open(my $fh, '<:utf8', $filename) or do { warn("$!: $filename; " . $test->as_string); return }; <$fh> };
172 my $bytes = encode_utf8($sparql);
173  
174 if ($self->debug) {
175 my $q = $sparql;
176 $q =~ s/\s+/ /g;
177 warn "### test : " . $test->as_string . "\n";
178 warn "# sparql : $q\n";
179 warn "# file : $filename\n";
180 }
181
182 my $pclass = Attean->get_parser('SPARQL');
183 my $parser = $pclass->new(version => $specVersion);
184 if ($test_type eq 'update') {
185 $parser->update(1);
186 }
187 if ($is_pos_query or $is_pos_update) {
188 my ($query) = eval { $parser->parse_list_from_bytes($bytes) };
189 my $ok = blessed($query);
190 $self->record_result('syntax', $ok, $test->as_string, $TODO);
191 $handled++;
192 if ($ok) {
193 pass("$testname - $filename");
194 } else {
195 fail("$testname - $filename: $@");
196 }
197 } elsif ($is_neg_query or $is_neg_update) {
198 my ($query) = eval { $parser->parse_list_from_bytes($bytes) };
199 my $ok = $@ ? 1 : 0;
200 $self->record_result('syntax', $ok, $test->as_string, $TODO);
201 $handled++;
202 if ($ok) {
203 pass("$testname - $filename");
204 } else {
205 if ($self->debug) {
206 warn $query->as_string;
207 }
208 fail("$testname - $filename (unexpected successful parse)");
209 }
210 }
211
212 unless ($handled) {
213 fail("Unhandled test: $testname");
214 }
215
216 }
217  
218 sub data_syntax_eval_test {
219 my $self = shift;
220 my $model = shift;
221 my $test = shift;
222 my $count = shift // 1;
223
224 my $type = iri( "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" );
225 my $mfname = iri( "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#name" );
226 my ($queryd) = $model->objects( $test, iri("${MF}action") )->elements;
227 my ($resultd) = $model->objects( $test, iri("${MF}result") )->elements;
228 my ($approved) = $model->objects( $test, iri("${DAWGT}approval") )->elements;
229 my ($name) = $model->objects( $test, $mfname )->elements;
230 my $testname = $test->value;
231 $testname =~ s{^.*[/#]([^/#]+[/#])}{$1};
232
233 if ($self->strict_approval) {
234 unless ($approved) {
235 warn "- skipping test because it isn't approved\n" if ($self->debug);
236 return;
237 }
238 if ($approved->equal("${DAWGT}NotClassified")) {
239 warn "- skipping test because its approval is dawgt:NotClassified\n" if ($self->debug);
240 return;
241 }
242 }
243  
244 my $is_eval_ttl = $model->count_quads($test, $type, iri("${RDFT}TestTurtleEval"));
245 my $is_pos_ttl = $model->count_quads($test, $type, iri("${RDFT}TestTurtlePositiveSyntax"));
246 my $is_neg_ttl = ($model->count_quads($test, $type, iri("${RDFT}TestTurtleNegativeSyntax")) + $model->count_quads($test, $type, iri("${RDFT}TestTurtleNegativeEval")));
247 my $is_pos_c14n_ttl = $model->count_quads($test, $type, iri("${RDFT}TestTurtlePositiveC14N"));
248  
249 my $is_eval_trig = $model->count_quads($test, $type, iri("${RDFT}TestTrigEval"));
250 my $is_pos_trig = $model->count_quads($test, $type, iri("${RDFT}TestTrigPositiveSyntax"));
251 my $is_neg_trig = ($model->count_quads($test, $type, iri("${RDFT}TestTrigNegativeSyntax")) + $model->count_quads($test, $type, iri("${RDFT}TestTrigNegativeEval")));
252 my $is_pos_c14n_trig = $model->count_quads($test, $type, iri("${RDFT}TestTrigPositiveC14N"));
253  
254 my $is_eval_xml = $model->count_quads($test, $type, iri("${RDFT}TestXMLEval"));
255 my $is_pos_xml = $model->count_quads($test, $type, iri("${RDFT}TestXMLPositiveSyntax"));
256 my $is_neg_xml = ($model->count_quads($test, $type, iri("${RDFT}TestXMLNegativeSyntax")) + $model->count_quads($test, $type, iri("${RDFT}TestXMLNegativeEval")));
257 my $is_pos_c14n_xml = $model->count_quads($test, $type, iri("${RDFT}TestXMLPositiveC14N"));
258  
259 my $is_eval_nt = $model->count_quads($test, $type, iri("${RDFT}TestNTriplesEval"));
260 my $is_pos_nt = $model->count_quads($test, $type, iri("${RDFT}TestNTriplesPositiveSyntax"));
261 my $is_neg_nt = $model->count_quads($test, $type, iri("${RDFT}TestNTriplesNegativeSyntax"));
262 my $is_pos_c14n_nt = $model->count_quads($test, $type, iri("${RDFT}TestNTriplesPositiveC14N"));
263  
264 my $is_eval_nq = $model->count_quads($test, $type, iri("${RDFT}TestNQuadsEval"));
265 my $is_pos_nq = $model->count_quads($test, $type, iri("${RDFT}TestNQuadsPositiveSyntax"));
266 my $is_neg_nq = $model->count_quads($test, $type, iri("${RDFT}TestNQuadsNegativeSyntax"));
267 my $is_pos_c14n_nq = $model->count_quads($test, $type, iri("${RDFT}TestNQuadsPositiveC14N"));
268  
269 my $is_eval = ($is_eval_ttl or $is_eval_nt or $is_eval_xml or $is_eval_trig or $is_eval_nq);
270 my $is_eval_triples = ($is_eval_ttl or $is_eval_nt or $is_eval_xml);
271 my $is_pos_triples = ($is_pos_ttl or $is_pos_nt or $is_pos_xml);
272 my $is_eval_quads = ($is_eval_trig or $is_eval_nq);
273 my $is_pos_quads = ($is_pos_trig or $is_pos_nq);
274 my $is_pos = ($is_pos_ttl or $is_pos_trig or $is_pos_xml or $is_pos_nt or $is_pos_nq);
275 my $is_neg = ($is_neg_ttl or $is_neg_trig or $is_neg_xml or $is_neg_nt or $is_neg_nq);
276 my $is_pos_c14n = ($is_pos_c14n_ttl or $is_pos_c14n_nt or $is_pos_c14n_nq);
277  
278 my $is_ttl = ($is_eval_ttl or $is_pos_ttl or $is_neg_ttl or $is_pos_c14n_ttl);
279 my $is_trig = ($is_eval_trig or $is_pos_trig or $is_neg_trig or $is_pos_c14n_trig);
280 my $is_xml = ($is_eval_xml or $is_pos_xml or $is_neg_xml or $is_pos_c14n_xml);
281 my $is_nt = ($is_eval_nt or $is_pos_nt or $is_neg_nt or $is_pos_c14n_nt);
282 my $is_nq = ($is_eval_nq or $is_pos_nq or $is_neg_nq or $is_pos_c14n_nq);
283
284 my $parser_name;
285 if ($is_ttl) {
286 $parser_name = 'Turtle';
287 } elsif ($is_trig) {
288 $parser_name = 'Trig';
289 } elsif ($is_xml) {
290 $parser_name = 'RDFXML';
291 } elsif ($is_nq) {
292 $parser_name = 'NQuads';
293 } elsif ($is_nt) {
294 $parser_name = 'NTriples';
295 } else {
296 die "Unexpected format";
297 }
298  
299 # my @types = $model->objects( $test, $type )->elements();
300 # warn Dumper([map {$_->value} @types]);
301
302
303 # warn "is_eval_ttl = $is_eval_ttl\n";
304 # warn "is_pos_ttl = $is_pos_ttl\n";
305 # warn "is_neg_ttl = $is_neg_ttl\n";
306 # warn "\n";
307 # warn "is_eval_nt = $is_eval_nt\n";
308 # warn "is_pos_nt = $is_pos_nt\n";
309 # warn "is_neg_nt = $is_neg_nt\n";
310 # warn "\n";
311 # warn "is_eval = $is_eval\n";
312 # warn "is_ttl = $is_ttl\n";
313 # warn "is_nt = $is_nt\n";
314
315
316
317
318 my $base;
319
320 my ($result_filename, $expected_bytes, $nt_content);
321 if ($is_eval) {
322 my $result_uri = URI->new( $resultd->value );
323 $result_filename = $result_uri->file;
324 (undef,$base,undef) = File::Spec->splitpath( $result_filename );
325 $base = "file://${base}";
326 warn "Loading expected RDF data from file $result_filename\n" if ($self->debug);
327 $nt_content = do { local($/) = undef; open(my $fh, '<:utf8', $result_filename) or do { warn("$!: $result_filename; " . $test->as_string); return }; <$fh> };
328 unless (defined($nt_content)) {
329 warn "*** Undefined expected data from $result_filename...\n";
330 }
331 $expected_bytes = encode_utf8($nt_content);
332 }
333
334 my $uri = URI->new( $queryd->value );
335 my $filename = $uri->file;
336 (undef,$base,undef) = File::Spec->splitpath( $filename );
337 $base = "file://${base}";
338 warn "Loading test RDF data from file $filename\n" if ($self->debug);
339 my $content = do { local($/) = undef; open(my $fh, '<:utf8', $filename) or do { warn("$!: $filename; " . $test->as_string); return }; <$fh> };
340 my $bytes = encode_utf8($content);
341  
342 if ($self->debug) {
343 my $q = $content;
344 $q =~ s/\s+/ /g;
345
346 warn "### test : " . $test->as_string . "\n";
347 warn "# file : $filename\n";
348 warn "# content : $q\n";
349 if ($is_eval) {
350 my $expected = $nt_content;
351 $expected =~ s/\s+/ /g;
352 warn "# expected : $result_filename\n";
353 warn "# expected : $expected\n";
354 }
355 }
356  
357 my $test_uri = $uri->as_string;
358 my $test_base = $uri->as_string;
359 $test_base = $self->fix_local_to_w3c($test_base);
360 warn "Test base: " . $test_base . "\n" if ($self->debug);
361
362 my $test_parser = Attean->get_parser($parser_name)->new(base => $test_base);
363 my $nt_parser = Attean->get_parser('NTriples')->new();
364 my $nq_parser = Attean->get_parser('NQuads')->new();
365 SKIP: {
366 if ($is_pos_triples or $is_eval_triples) {
367 my (@triples) = eval { $test_parser->parse_iter_from_bytes($bytes)->uniq()->elements() };
368 my $parse_error = $@;
369 if ($is_eval) {
370 my $ser = Attean->get_serializer('NTriples')->new();
371 my @nt_triples = eval { $nt_parser->parse_list_from_bytes($expected_bytes) };
372 if ($@) {
373 fail("Failed to parse expected data from $result_filename: $@");
374 return;
375 }
376 my $expected_iter = Attean::ListIterator->new( values => \@nt_triples, item_type => 'Attean::API::Triple' );
377  
378 my $eq_test = Attean::BindingEqualityTest->new();
379 my $actual_iter = Attean::ListIterator->new( values => \@triples, item_type => 'Attean::API::Triple' );
380 my $ok = $eq_test->equals($actual_iter, $expected_iter);
381 ok($ok, "$testname - $filename");
382 $self->record_result('evaluation', $ok, $test->as_string, $TODO);
383 if (not $ok) {
384 warn "Expecting " . scalar(@nt_triples) . " triples:\n";
385 if (scalar(@nt_triples)) {
386 warn $ser->serialize_list_to_bytes(@nt_triples);
387 }
388 warn "But found " . scalar(@triples) . " triples:\n";
389 my $bytes = $ser->serialize_list_to_bytes(@triples);
390 warn((length($bytes)) ? decode_utf8($bytes) : '(empty content)');
391 }
392 } else {
393 my $ok = scalar(not $parse_error);
394 $self->record_result('syntax', $ok, $test->as_string, $TODO);
395 if ($ok) {
396 pass("$testname - $filename");
397 } else {
398 fail("$testname - $filename: $@");
399 }
400 }
401 } elsif ($is_pos_quads or $is_eval_quads) {
402 my (@triples) = eval { $test_parser->parse_iter_from_bytes($bytes)->uniq()->elements() };
403 my $err = $@;
404 if ($is_eval) {
405 my $ser = Attean->get_serializer('NQuads')->new();
406 my $nq_parser = Attean->get_parser('NQuads')->new();
407 my @nq_quads = eval { $nq_parser->parse_iter_from_bytes($expected_bytes)->uniq()->elements() };
408 $err ||= $@;
409 if ($err) {
410 fail("$testname - Failed to parse expected data from $result_filename: $err");
411 return;
412 }
413 my $nq_iter = Attean::ListIterator->new( values => \@nq_quads, item_type => 'Attean::API::TripleOrQuad' );
414  
415 my $eq_test = Attean::BindingEqualityTest->new();
416 my $trig_iter = Attean::ListIterator->new( values => \@triples, item_type => 'Attean::API::TripleOrQuad' );
417 my $ok = $eq_test->equals($trig_iter, $nq_iter);
418 $self->record_result('syntax', $ok, $test->as_string, $TODO);
419 ok($ok, "$testname - $filename");
420 if (not $ok) {
421 warn "BindingEqualityTest failures: " . $eq_test->error();
422 warn "Expecting " . scalar(@nq_quads) . " triples:\n";
423 if (scalar(@nq_quads)) {
424 warn $ser->serialize_list_to_bytes(@nq_quads);
425 }
426 warn "But found " . scalar(@triples) . " triples:\n";
427 my $bytes = $ser->serialize_list_to_bytes(@triples);
428 warn((length($bytes)) ? decode_utf8($bytes) : '(empty content)');
429 }
430 } else {
431 my $ok = not($err);
432 $self->record_result('syntax', $ok, $test->as_string, $TODO);
433 if ($ok) {
434 pass("$testname - $filename");
435 } else {
436 fail("$testname - $filename: $err");
437 }
438 }
439 } elsif ($is_neg) {
440 my (@triples) = eval { $test_parser->parse_list_from_bytes($bytes) };
441 my $ok = $@ ? 1 : 0;
442 $self->record_result('syntax', $ok, $test->as_string, $TODO);
443 if ($ok) {
444 pass("$testname - $filename");
445 } else {
446 if ($self->debug) {
447 warn "Unexpected successful parse of:\n" . $content;
448 }
449 fail("$testname - $filename (unexpected successful parse)");
450 }
451 } elsif ($is_pos_c14n) {
452 skip("$testname - C14N tests not supported");
453 } else {
454 my @types = $model->objects( $test, $type )->elements();
455 my $types = join(',', map {$_->value} @types);
456 fail("$testname - unrecognized test type(s): $types");
457 }
458 }
459 }
460  
461 # sub data_syntax_test {
462 # my $self = shift;
463 # my $model = shift;
464 # my $test = shift;
465 # my $count = shift // 1;
466 #
467 # my $type = iri( "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" );
468 # my $mfname = iri( "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#name" );
469 # my ($queryd) = $model->objects( $test, iri("${MF}action") )->elements;
470 # my ($resultd) = $model->objects( $test, iri("${MF}result") )->elements;
471 # my ($approved) = $model->objects( $test, iri("${DAWGT}approval") )->elements;
472 # my ($name) = $model->objects( $test, $mfname )->elements;
473 #
474 # if ($self->strict_approval) {
475 # unless ($approved) {
476 # warn "- skipping test because it isn't approved\n" if ($self->debug);
477 # return;
478 # }
479 # if ($approved->equal("${DAWGT}NotClassified")) {
480 # warn "- skipping test because its approval is dawgt:NotClassified\n" if ($self->debug);
481 # return;
482 # }
483 # }
484 #
485 # my $is_eval_ttl = $model->count_quads($test, $type, iri("${RDFT}TestTurtleEval"));
486 # my $is_pos_ttl = $model->count_quads($test, $type, iri("${RDFT}TestTurtlePositiveSyntax"));
487 # my $is_neg_ttl = $model->count_quads($test, $type, iri("${RDFT}TestTurtleNegativeSyntax"));
488 #
489 # my $is_eval_nt = $model->count_quads($test, $type, iri("${RDFT}TestNTriplesEval"));
490 # my $is_pos_nt = $model->count_quads($test, $type, iri("${RDFT}TestNTriplesPositiveSyntax"));
491 # my $is_neg_nt = $model->count_quads($test, $type, iri("${RDFT}TestNTriplesNegativeSyntax"));
492 #
493 #
494 # # my @types = $model->objects( $test, $type )->elements();
495 # # warn Dumper([map {$_->value} @types]);
496 #
497 # my $base;
498 #
499 # my $uri = URI->new( $queryd->value );
500 # my $filename = $uri->file;
501 # (undef,$base,undef) = File::Spec->splitpath( $filename );
502 # $base = "file://${base}";
503 # warn "Loading test RDF data from file $filename\n" if ($self->debug);
504 # my $content = do { local($/) = undef; open(my $fh, '<:utf8', $filename) or do { warn("$!: $filename; " . $test->as_string); return }; <$fh> };
505 # my $bytes = encode_utf8($content);
506 #
507 # # if ($self->debug) {
508 # # my $q = $content;
509 # # $q =~ s/\s+/ /g;
510 # # warn "### test : " . $test->as_string . "\n";
511 # # warn "# file : $filename\n";
512 # # warn "# content : $q\n";
513 # # }
514 #
515 # if ($is_pos_ttl or $is_neg_ttl or $is_eval_ttl) {
516 # my $test_base = $uri->as_string;
517 # $test_base =~ s{^.*/rdf-tests/}{https://w3c.github.io/rdf-tests/}; # XXX hack
518 #
519 # my $ttl_parser = Attean->get_parser('Turtle')->new(base => $test_base);
520 # my $nt_parser = Attean->get_parser('NTriples')->new();
521 # if ($is_pos_ttl or $is_eval_ttl) {
522 # my (@triples) = eval { $ttl_parser->parse_list_from_bytes($bytes) };
523 # my $ok = scalar(@triples);
524 # $self->record_result('syntax', $ok, $test->as_string);
525 # if ($ok) {
526 # pass("$testname - $filename");
527 # } else {
528 # fail("$testname - $filename: $@");
529 # }
530 # } elsif ($is_pos_nt or $is_eval_nt) {
531 # my (@triples) = eval { $nt_parser->parse_list_from_bytes($bytes) };
532 # my $ok = scalar(@triples);
533 # $self->record_result('syntax', $ok, $test->as_string);
534 # if ($ok) {
535 # pass("$testname - $filename");
536 # } else {
537 # fail("$testname - $filename: $@");
538 # }
539 # } elsif ($is_neg_ttl) {
540 # my (@triples) = eval { $ttl_parser->parse_list_from_bytes($bytes) };
541 # my $ok = $@ ? 1 : 0;
542 # $self->record_result('syntax', $ok, $test->as_string);
543 # if ($ok) {
544 # pass("$testname - $filename");
545 # } else {
546 # # if ($self->debug) {
547 # # warn $query->as_string;
548 # # }
549 # fail("$testname - $filename (unexpected successful parse)");
550 # }
551 # } elsif ($is_neg_nt) {
552 # my (@triples) = eval { $nt_parser->parse_list_from_bytes($bytes) };
553 # my $ok = $@ ? 1 : 0;
554 # $self->record_result('syntax', $ok, $test->as_string);
555 # if ($ok) {
556 # pass("$testname - $filename");
557 # } else {
558 # # if ($self->debug) {
559 # # warn $query->as_string;
560 # # }
561 # fail("$testname - $filename (unexpected successful parse)");
562 # }
563 # }
564 # }
565 # }
566  
567 sub update_eval_test {
568 my $self = shift;
569 my $model = shift;
570 my $test = shift;
571 my %args = @_;
572 my $specVersion = $args{'specVersion'} // '1.1';
573  
574 my ($action) = $model->objects( $test, iri("${MF}action") )->elements;
575 my ($result) = $model->objects( $test, iri("${MF}result") )->elements;
576 my @reqs = $model->objects( $test, iri("${MF}requires") )->elements;
577 my ($approved) = $model->objects( $test, iri("${DAWGT}approval") )->elements;
578 my ($queryd) = $model->objects( $action, iri("${UT}request") )->elements;
579 my @data = $model->objects( $action, iri("${UT}data") )->elements;
580 my @gdata = $model->objects( $action, iri("${UT}graphData") )->elements;
581
582 if ($self->strict_approval) {
583 unless ($approved) {
584 warn "- skipping test because it isn't approved\n" if ($self->debug);
585 return;
586 }
587 if ($approved->equal(iri("${DAWGT}NotClassified"))) {
588 warn "- skipping test because its approval is dawgt:NotClassified\n" if ($self->debug);
589 return;
590 }
591 }
592
593 my $uri = URI->new( $queryd->value );
594 my $filename = $uri->file;
595 my (undef,$base,undef) = File::Spec->splitpath( $filename );
596 $base = "file://${base}";
597 warn "Loading SPARQL query from file $filename" if ($self->debug);
598 my $sparql = do { local($/) = undef; open(my $fh, '<', $filename) or do { fail("$!: $filename; " . $test->as_string); return }; binmode($fh, ':utf8'); <$fh> };
599  
600 my $q = $sparql;
601 $q =~ s/\s+/ /g;
602 if ($self->debug) {
603 warn "### test : " . $test->value . "\n";
604 warn "# sparql : $q\n";
605 warn "# file : $filename\n";
606 foreach my $data (@data) {
607 warn "# data : " . $data->value . "\n" if (blessed($data));
608 }
609 warn "# graph data : " . $_->value . "\n" for (@gdata);
610 warn "# result : " . $result->value . "\n";
611 foreach my $req (@reqs) {
612 warn "# requires : " . $req->value . "\n" if (blessed($req));
613 }
614 }
615
616 my $todo_because_of_test_req = 0;
617 my @unsupported_reqs;
618 foreach my $req (@reqs) {
619 unless (exists ${ $self->supported_extensions }{ $req->value }) {
620 $todo_because_of_test_req++;
621 push(@unsupported_reqs, $req->value);
622 }
623 }
624
625
626
627 # TODO: set up remote endpoint mock
628 TODO: {
629 local($TODO) = $todo_because_of_test_req ? "requires " . join(', ', @unsupported_reqs) : '';
630  
631 warn "constructing model...\n" if ($self->debug);
632 my $test_model = $self->test_model();
633 foreach my $data (@data) {
634 eval {
635 if (blessed($data)) {
636 $test_model->load_urls_into_graph($self->default_graph, $data);
637 }
638 };
639
640 if ($self->debug) {
641 warn "TEST MODEL AFTER DEFAULT GRAPH LOAD:\n";
642 $self->debug_model($test_model);
643 }
644
645 if ($@) {
646 fail($test->value);
647 $self->record_result('update', 0, $test->value, $TODO);
648 print "# died: " . $test->value . ": $@\n";
649 return;
650 }
651 }
652 foreach my $gdata (@gdata) {
653 my ($data) = ($model->objects( $gdata, iri("${UT}data") )->elements)[0] || ($model->objects( $gdata, iri("${UT}graph") )->elements)[0];
654 my ($graph) = $model->objects( $gdata, iri("${RDFS}label") )->elements;
655 my $uri = $graph->value;
656 eval {
657 $test_model->load_urls_into_graph(iri($uri), $data);
658 };
659 if ($@) {
660 fail($test->as_string);
661 $self->record_result('update', 0, $test->value, $TODO);
662 print "# died: " . $test->value . ": $@\n";
663 return;
664 };
665 }
666
667 my ($result_status) = $model->objects( $result, iri("${UT}result") )->elements;
668 my @resgdata = $model->objects( $result, iri("${UT}graphData") )->elements;
669 my ($resdata) = $model->objects( $result, iri("${UT}data") )->elements;
670 my $expected_model = memory_model;
671 eval {
672 if (blessed($resdata)) {
673 $expected_model->load_urls_into_graph($self->default_graph, $resdata);
674 }
675 };
676 if ($@) {
677 fail($test->as_string);
678 $self->record_result('update', 0, $test->value, $TODO);
679 print "# died: " . $test->value . ": $@\n";
680 return;
681 };
682 foreach my $gdata (@resgdata) {
683 my ($data) = ($model->objects( $gdata, iri("${UT}data") )->elements)[0] || ($model->objects( $gdata, iri("${UT}graph") )->elements)[0];
684 my ($graph) = $model->objects( $gdata, iri("${RDFS}label") )->elements;
685 my $uri = $graph->value;
686 my $return = 0;
687 if ($data) {
688 eval {
689 $expected_model->load_urls_into_graph(iri($uri), $data);
690 };
691 if ($@) {
692 fail($test->as_string);
693 $self->record_result('update', 0, $test->value, $TODO);
694 print "# died: " . $test->value . ": $@\n";
695 $return = 1;
696 };
697 return if ($return);
698 }
699 }
700
701 if ($self->debug) {
702 warn "Dataset before update operation:\n";
703 warn $self->model_as_string($test_model);
704 }
705
706 my $ok = 0;
707 eval {
708 my $algebra = eval { Attean->get_parser('SPARQL')->parse_update($sparql) };
709 if ($@) {
710 warn "Failed to parse query $filename: $@";
711 die $@;
712 }
713 unless ($algebra) {
714 warn "No algebra generated for update\n";
715 fail($test->value);
716 return;
717 }
718 if ($self->debug) {
719 warn "# Algebra:\n" . $algebra->as_string . "\n";
720 }
721
722 my $default_graphs = [$self->default_graph];
723 my $planner = Attean::IDPQueryPlanner->new();
724 my $plan = $planner->plan_for_algebra($algebra, $test_model, $default_graphs);
725 if ($self->debug) {
726 warn "# Plan:\n" . $plan->as_string . "\n";
727 }
728
729 if ($self->debug) {
730 warn "Running update...\n";
731 }
732 my $iter = $plan->evaluate($test_model);
733 $iter->elements;
734 if ($self->debug) {
735 warn "done.\n";
736 }
737
738 if ($self->debug) {
739 warn "Comparing results...\n";
740 }
741 my $eqtest = Attean::BindingEqualityTest->new();
742 my $eq = $eqtest->equals($test_model, $expected_model);
743 if ($self->debug) {
744 warn "done.\n";
745 }
746 $ok = is( $eq, 1, $test->value );
747 unless ($ok) {
748 warn $eqtest->error;
749 warn "Got model:\n" . $self->model_as_string($test_model);
750 warn "Expected model:\n" . $self->model_as_string($expected_model);
751 }
752 };
753 if ($@) {
754 warn "Failed to execute update: $@";
755 fail($test->value);
756 }
757 if (not($ok)) {
758 print "# failed: " . $test->value . "\n";
759 }
760
761 $self->record_result('update', $ok, $test->value, $TODO);
762 warn "ok\n" if ($self->debug);
763 }
764 }
765  
766 sub fix_local_to_w3c {
767 my $self = shift;
768 my $file_url = shift;
769 $file_url =~ s{^.*/rdf-tests/}{https://w3c.github.io/rdf-tests/}; # XXX hack
770 return $file_url;
771 }
772  
773 sub construct_data {
774 my $self = shift;
775 my $model = shift;
776 my $action = shift;
777 my %args = @_;
778 my $specVersion = $args{'specVersion'} // '1.1';
779  
780 my ($queryd) = $model->objects( $action, iri("${RQ}query") )->elements;
781 my @data = $model->objects( $action, iri("${RQ}data") )->elements;
782 my @gdata = $model->objects( $action, iri("${RQ}graphData") )->elements;
783 my @sdata = $model->objects( $action, iri("${RQ}serviceData") )->elements;
784 my @cdata = $model->objects( $action, iri("${RQ}constructDataFile") )->elements;
785 my ($fnode) = $model->objects( $action, iri("${RQ}format") )->elements;
786 my $format = blessed($fnode) ? $fnode->value : 'text/turtle';
787  
788 my $uri = URI->new( $queryd->value );
789 my $filename = $uri->file;
790 my (undef,$base,undef) = File::Spec->splitpath( $filename );
791 $base = "file://${base}";
792 warn "Loading SPARQL query from file $filename" if ($self->debug);
793 my $sparql = do { local($/) = undef; open(my $fh, '<', $filename) or do { warn("$!: $filename"); return }; binmode($fh, ':utf8'); <$fh> };
794
795 my $test_model = $self->test_model();
796 foreach my $data (@data) {
797 if (blessed($data)) {
798 $test_model->load_urls_into_graph($self->default_graph, $data);
799 }
800 }
801 foreach my $g (@gdata) {
802 my $start = $test_model->size;
803 my $file_url = $g->value;
804 $file_url = $self->fix_local_to_w3c($file_url);
805 $test_model->load_urls_into_graph(iri($file_url), $g);
806 my $end = $test_model->size;
807 unless ($start < $end) {
808 warn "*** Loading file did not result in any new quads: " . $g;
809 }
810 }
811 foreach my $n (@cdata) {
812 my ($bytes, $format) = $self->construct_data($model, $n, %args);
813 my $p = Attean->get_parser(media_type => $format)->new();
814 my $iter = $p->parse_iter_from_bytes($bytes);
815 $test_model->add_iter($iter->as_quads($self->default_graph));
816 }
817
818 print STDERR "ok\n" if ($self->debug);
819  
820 if ($self->debug) {
821 my $q = $sparql;
822 $q =~ s/([\x{256}-\x{1000}])/'\x{' . sprintf('%x', ord($1)) . '}'/eg;
823 warn $q;
824 }
825
826 my $test_base = $base;
827 $test_base = $self->fix_local_to_w3c($test_base);
828  
829 my ($iter, $type) = $self->get_actual_results( $filename, $test_model, $sparql, $test_base, $model, \@sdata, %args );
830 my $sclass = Attean->get_serializer(media_type => $format);
831 my $s = $sclass->new();
832 my $bytes = $s->serialize_iter_to_bytes($iter);
833 return ($bytes, $format);
834 }
835  
836 sub query_eval_test {
837 my $self = shift;
838 my $model = shift;
839 my $test = shift;
840 my %args = @_;
841 my $specVersion = $args{'specVersion'} // '1.1';
842
843 my ($action) = $model->objects( $test, iri("${MF}action") )->elements;
844 my ($result) = $model->objects( $test, iri("${MF}result") )->elements;
845 my ($req) = $model->objects( $test, iri("${MF}requires") )->elements;
846 my ($approved) = $model->objects( $test, iri("${DAWGT}approval") )->elements;
847 my ($res_card) = $model->objects( $test, iri("${MF}resultCardinality") )->elements;
848 my ($queryd) = $model->objects( $action, iri("${RQ}query") )->elements;
849 my @data = $model->objects( $action, iri("${RQ}data") )->elements;
850 my @gdata = $model->objects( $action, iri("${RQ}graphData") )->elements;
851 my @sdata = $model->objects( $action, iri("${RQ}serviceData") )->elements;
852 my @cdata = $model->objects( $action, iri("${RQ}constructDataFile") )->elements;
853 my $reduced_mode = (defined($res_card) and $res_card->value eq "${MF}LaxCardinality");
854
855 if ($self->strict_approval) {
856 unless ($approved) {
857 warn "- skipping test because it isn't approved\n" if ($self->debug);
858 return;
859 }
860 if ($approved->equal("${DAWGT}NotClassified")) {
861 warn "- skipping test because its approval is dawgt:NotClassified\n" if ($self->debug);
862 return;
863 }
864 }
865
866 my $uri = URI->new( $queryd->value );
867 my $filename = $uri->file;
868 my (undef,$base,undef) = File::Spec->splitpath( $filename );
869 $base = "file://${base}";
870 warn "Loading SPARQL query from file $filename\n" if ($self->debug);
871  
872 my $test_base = $base;
873 $test_base = $self->fix_local_to_w3c($test_base);
874  
875 my $sparql = do { local($/) = undef; open(my $fh, '<', $filename) or do { warn("$!: $filename; " . $test->value); return }; binmode($fh, ':utf8'); <$fh> };
876
877 my $q = $sparql;
878 $q =~ s/\s+/ /g;
879 if ($self->debug) {
880 warn "### test : " . $test->value . "\n";
881 warn "# sparql : $q\n";
882 warn "# file : $filename\n";
883 foreach my $data (@data) {
884 warn "# data : " . ($data->value =~ s#file://##r) . "\n" if (blessed($data));
885 }
886 warn "# graph data : " . ($_->value =~ s#file://##r) . "\n" for (@gdata);
887 warn "# constructed data : " . ($_->value =~ s#file://##r) . "\n" for (@cdata);
888 warn "# result : " . ($result->value =~ s#file://##r) . "\n";
889 warn "# requires : " . ($req->value =~ s#file://##r) . "\n" if (blessed($req));
890 }
891
892 print STDERR "constructing model...\n" if ($self->debug);
893 my $test_model = $self->test_model();
894 TODO: {
895 local($TODO) = (blessed($req)) ? "requires " . $req->value : '';
896 try {
897 foreach my $data (@data) {
898 my $file_url = $data->value;
899 $file_url = $self->fix_local_to_w3c($file_url);
900 if (blessed($data)) {
901 $test_model->load_urls_into_graph_with_base($self->default_graph, [$data], $file_url);
902 }
903 }
904
905 if ($self->debug) {
906 warn "TEST MODEL AFTER DEFAULT GRAPH LOAD:\n";
907 $self->debug_model($test_model);
908 }
909  
910 foreach my $g (@gdata) {
911 my $start = $test_model->size;
912 my $file_url = $g->value;
913 $file_url = $self->fix_local_to_w3c($file_url);
914 $test_model->load_urls_into_graph_with_base(iri($file_url), [$g], $file_url);
915 my $end = $test_model->size;
916 unless ($start < $end) {
917 warn "*** Loading file did not result in any new quads: " . $g;
918 }
919 }
920 foreach my $n (@cdata) {
921 my ($bytes, $format) = $self->construct_data($model, $n, %args);
922 my $p = Attean->get_parser(media_type => $format)->new();
923 my $iter = $p->parse_iter_from_bytes($bytes);
924 $test_model->add_iter($iter->as_quads($self->default_graph));
925 }
926 } catch {
927 fail($test->value);
928 $self->record_result('evaluation', 0, $test->value, $TODO);
929 print "# died: " . $test->value . ": $_\n";
930 return;
931 };
932 print STDERR "ok\n" if ($self->debug);
933
934 my $resuri = URI->new( $result->value );
935 my $resfilename = $resuri->file;
936
937 my $comment;
938 eval {
939 if ($self->debug) {
940 my $q = $sparql;
941 $q =~ s/([\x{256}-\x{1000}])/'\x{' . sprintf('%x', ord($1)) . '}'/eg;
942 warn $q;
943 }
944
945 my ($actual, $type);
946 {
947 local($::DEBUG) = 1;
948 print STDERR "getting actual results...\n" if ($self->debug);
949 ($actual, $type) = $self->get_actual_results( $filename, $test_model, $sparql, $test_base, $model, \@sdata, %args );
950 print STDERR "ok\n" if ($self->debug);
951 }
952
953 print STDERR "getting expected results...\n" if ($self->debug);
954 my $expected = $self->get_expected_results( $resfilename, $type );
955 print STDERR "ok\n" if ($self->debug);
956
957 # warn "comparing results...";
958 $self->compare_results( $expected, $actual, $test->value, \$comment, $reduced_mode, $TODO );
959 };
960
961 my $ok = not($@);
962 unless ($ok) {
963 warn $@;
964 fail($test->value);
965 $self->record_result('evaluation', 0, $test->value, $TODO);
966 };
967 if ($ok) {
968 } else {
969 print "# failed: " . $test->value . "\n";
970 }
971 }
972 }
973  
974 sub mock_endpoints {
975 my $self = shift;
976 my $mock = shift;
977 my $model = shift;
978 my $sdata = shift;
979
980 foreach my $sd (@$sdata) {
981 my ($e) = $model->objects( $sd, iri("${RQ}endpoint") )->elements;
982 my @data = $model->objects( $sd, iri("${RQ}data") )->elements;
983 my @gdata = $model->objects( $sd, iri("${RQ}graphData") )->elements;
984  
985 my $endpoint = $e->value;
986 my $test_model = $self->test_model();
987 foreach my $data (@data) {
988 if (blessed($data)) {
989 $test_model->load_urls_into_graph($self->default_graph, $data);
990 }
991 }
992 foreach my $g (@gdata) {
993 my $start = $test_model->size;
994 $test_model->load_urls_into_graph($g, $g);
995 my $end = $test_model->size;
996 unless ($start < $end) {
997 warn "*** Loading file did not result in any new quads: " . $g;
998 }
999 }
1000 $mock->register_test_endpoint($endpoint, [$test_model, $self->default_graph]);
1001 }
1002 }
1003  
1004 sub debug_model {
1005 my $self = shift;
1006 my $model = shift;
1007 if ($self->debug) {
1008 my $iter = $model->get_quads;
1009 warn "Dataset:\n-------------\n";
1010 my $s = Attean->get_serializer('TextTable')->new();
1011 my $table = decode_utf8($s->serialize_iter_to_bytes($iter));
1012 warn $table;
1013 warn "-------------\n";
1014 }
1015 }
1016  
1017 sub get_actual_results {
1018 my $self = shift;
1019 my $filename = shift;
1020 my $model = shift;
1021 my $sparql = shift;
1022 my $base = shift;
1023 my $manifest_model = shift;
1024 my $sdata = shift;
1025 my %args = @_;
1026 my $specVersion = $args{'specVersion'} // '1.1';
1027  
1028 my $bytes = encode_utf8($sparql);
1029  
1030 my $s = AtteanX::Parser::SPARQL->new(base => $base, version => $specVersion);
1031 my $algebra;
1032 eval {
1033 ($algebra) = $s->parse_list_from_bytes($bytes);
1034 };
1035 if ($@) {
1036 warn "Failed to parse query $filename: $@";
1037 warn "--------- [contents of $filename]\n";
1038 warn $bytes . "\n";
1039 warn "---------\n";
1040 die $@;
1041 }
1042
1043 if ($self->debug) {
1044 warn "Walking algebra:\n";
1045 warn $algebra->as_string;
1046 }
1047
1048 $self->debug_model($model);
1049
1050 my $testns = 'http://example.com/test-results#';
1051 my $rmodel = memory_model();
1052  
1053 my $results;
1054 if ($self->use_idp_planner) {
1055 my $default_graphs = [$self->default_graph];
1056 my $planner = Test::Attean::TestIDPQueryPlanner->new();
1057 $self->mock_endpoints($planner, $manifest_model, $sdata);
1058 my $plan = $planner->plan_for_algebra($algebra, $model, $default_graphs);
1059 if ($self->debug) {
1060 warn "Walking plan:\n";
1061 warn $plan->as_string;
1062 }
1063 $results = eval { $plan->evaluate($model) };
1064 warn $@ if $@;
1065 } else {
1066 my $e = Test::Attean::TestSimpleQueryEvaluator->new( model => $model, default_graph => $self->default_graph );
1067 $self->mock_endpoints($e, $manifest_model, $sdata);
1068 $results = eval { $e->evaluate($algebra, $self->default_graph) };
1069 warn $@ if $@;
1070 }
1071 my $count = 1;
1072
1073 unless (blessed($results)) {
1074 die 'Failed to produce actual results';
1075 }
1076
1077 $results = $results->materialize;
1078 my $item = $results->peek;
1079
1080 my $type = 'bindings';
1081 if ($item) {
1082 if ($item->does('Attean::API::Triple')) {
1083 $type = 'graph';
1084 } elsif ($item->does('Attean::API::Term')) {
1085 $type = 'boolean';
1086 }
1087 }
1088  
1089 $self->print_results("Actual results", \$results) if ($self->results);
1090 return ($results, $type);
1091
1092 if ($results->is_bindings) {
1093 return ($results, 'bindings');
1094 } elsif ($results->is_boolean) {
1095 $rmodel->add_statement( triple( iri("${testns}result"), iri("${testns}boolean"), literal(($results->get_boolean ? 'true' : 'false'), undef, "${XSD}boolean") ) );
1096 return ($rmodel->get_statements, 'boolean');
1097 } elsif ($results->is_graph) {
1098 return ($results, 'graph');
1099 } else {
1100 warn "unknown result type: " . Dumper($results);
1101 }
1102 }
1103  
1104 sub print_results {
1105 my $self = shift;
1106 my $name = shift;
1107 my $results = shift;
1108 $$results = $$results->materialize;
1109 print "$name:\n";
1110 my $count = 1;
1111 while (my $r = $$results->next) {
1112 printf("%3d %s\n", $count++, $r->as_string);
1113 }
1114 $$results->reset;
1115 }
1116  
1117 sub get_expected_results {
1118 my $self = shift;
1119 my $file = shift;
1120 my $type = shift;
1121
1122 if ($type eq 'graph') {
1123 my $model = memory_model();
1124 $model->load_urls_into_graph($self->default_graph, iri("file://$file"));
1125 my $results = $model->get_quads->map(sub { shift->as_triple }, 'Attean::API::Triple');
1126 $self->print_results("Expected results", \$results) if ($self->results);
1127 return $results;
1128 } elsif ($file =~ /[.](srj|json)/) {
1129 my $model = memory_model();
1130 open(my $fh, '<', $file) or die $!;
1131 my $parser = Attean->get_parser('SPARQLJSON')->new();
1132 my $results = $parser->parse_iter_from_io($fh)->materialize;
1133 my $item = $results->peek;
1134 if (blessed($item) and $item->does('Attean::API::Term')) {
1135 if ($self->results) {
1136 warn "Expected result: " . $item->as_string . "\n";
1137 }
1138 return $results;
1139 } else {
1140 $self->print_results("Expected results", \$results) if ($self->results);
1141 return $results;
1142 }
1143 } elsif ($file =~ /[.]srx/) {
1144 my $model = memory_model();
1145 my $parser = Attean->get_parser('sparqlxml')->new();
1146 open(my $fh, '<', $file);
1147 my $results = $parser->parse_iter_from_io($fh);
1148
1149 $self->print_results("Expected results", \$results) if ($self->results);
1150 return $results;
1151 } elsif ($file =~ /[.]csv/) {
1152 my $csv = Text::CSV->new({binary => 1});
1153 open( my $fh, "<:encoding(utf8)", $file ) or die $!;
1154 my $header = $csv->getline($fh);
1155 my @vars = @$header;
1156 my @data;
1157 while (my $row = $csv->getline($fh)) {
1158 my %result;
1159 foreach my $i (0 .. $#vars) {
1160 my $var = $vars[$i];
1161 my $value = $row->[ $i ];
1162 # XXX @@ heuristics that won't always work.
1163 # XXX @@ expected to work on the test suite, though
1164 if ($value =~ /^_:(\w+)$/) {
1165 $value = blank($1);
1166 } elsif ($value =~ /$RE{URI}/) {
1167 $value = iri($value);
1168 } elsif (defined($value) and length($value)) {
1169 $value = literal($value);
1170 }
1171 if (ref($value)) {
1172 $result{ $var } = $value;
1173 }
1174 }
1175 push(@data, Attean::Result->new( bindings => \%result ));
1176 }
1177 my $results = Attean::ListIterator->new(values => \@data, item_type => 'Attean::API::Result', variables => \@vars);
1178 $self->print_results("Expected results", \$results) if ($self->results);
1179 return $results;
1180 } elsif ($file =~ /[.]tsv/) {
1181 my $parser = Attean->get_parser('SPARQLTSV')->new();
1182 open( my $fh, "<:encoding(utf8)", $file ) or die $!;
1183 my $iter = $parser->parse_iter_from_io($fh);
1184 return $iter;
1185 } elsif ($file =~ /[.](ttl|rdf|nt)/) {
1186 my $model = memory_model();
1187 my $file_url = "file://$file";
1188 my $base = $file_url;
1189 $base = $self->fix_local_to_w3c($base);
1190 $model->load_urls_into_graph_with_base($self->default_graph, [iri($file_url)], iri($base));
1191 my ($res) = $model->subjects( iri("${RDF}type"), iri("${RS}ResultSet") )->elements;
1192 if (my($b) = $model->objects( $res, iri("${RS}boolean") )->elements) {
1193 my $bool = $b->value;
1194 my $term = literal(value => $bool, datatype => "${XSD}boolean");
1195 if ($self->results) {
1196 warn "Expected result: " . $term->as_string . "\n";
1197 }
1198 return Attean::ListIterator->new(values => [$term], item_type => 'Attean::API::Term');
1199 } else {
1200 my @vars = $model->objects( $res, iri("${RS}resultVariable") )->elements;
1201 my @sols = $model->objects( $res, iri("${RS}solution") )->elements;
1202 my @names = map { $_->value } @vars;
1203 my @bindings;
1204 my %vars;
1205 foreach my $r (@sols) {
1206 my %data;
1207 my @b = $model->objects( $r, iri("${RS}binding") )->elements;
1208 foreach my $b (@b) {
1209 my ($value) = $model->objects( $b, iri("${RS}value") )->elements;
1210 my ($var) = $model->objects( $b, iri("${RS}variable") )->elements;
1211 $data{ $var->value } = $value;
1212 $vars{ $var->value }++;
1213 }
1214 push(@bindings, Attean::Result->new( bindings => \%data ));
1215 }
1216 my $results = Attean::ListIterator->new(values => \@bindings, item_type => 'Attean::API::Result', variables => [keys %vars]);
1217 $self->print_results("Expected results", \$results) if ($self->results);
1218 return $results;
1219 }
1220 } else {
1221 die "Unrecognized type of expected results: $file";
1222 }
1223 }
1224  
1225 sub compare_results {
1226 my $self = shift;
1227 my $expected = shift->canonicalize->materialize;
1228 my $actual = shift->canonicalize->materialize;
1229 my $test = shift;
1230 my $comment = shift || do { my $foo; \$foo };
1231 my $reduced_mode = shift;
1232 my $todo = shift;
1233 if ($reduced_mode) {
1234 for ($actual, $expected) {
1235 $_ = $_->uniq();
1236 }
1237 }
1238
1239 if ($reduced_mode and $self->debug) {
1240 warn "Comparing data using REDUCED cardinality semantics\n";
1241 }
1242
1243 if ($actual->does('Attean::API::ResultIterator') or $actual->does('Attean::API::TripleIterator')) {
1244 my $eqtest = Attean::BindingEqualityTest->new();
1245 if ($test =~ /csv0/) {
1246 # CSV is a lossy format, so strip the languages and datatypes off of literals in the actual results (so that they'll match up with the (lossy) expected results
1247 my $mapper = Attean::TermMap->new(mapper => sub {
1248 my $term = shift;
1249 if ($term->does('Attean::API::Literal')) {
1250 return Attean::Literal->new(value => $term->value);
1251 }
1252 return $term;
1253 });
1254 $actual = $actual->map($mapper->binding_mapper);
1255 }
1256  
1257 my $ok = eval { ok( $eqtest->equals( $actual, $expected ), $test ) or diag($eqtest->error) };
1258 if ($@) {
1259 diag($@);
1260 }
1261 $self->record_result('evaluation', $ok, $test, $todo);
1262 return $ok;
1263 } elsif ($actual->does('Attean::API::TermIterator')) {
1264 my $a = $actual->next;
1265 my $e = $expected->next;
1266 my $name = $self->debug ? sprintf("$test: %s == %s", $a->as_string, $e->as_string) : $test;
1267 my $eq = $a->equals($e);
1268 my $ok = ok( $eq, $name );
1269 $self->record_result('evaluation', $ok, $test, $todo);
1270 return $ok;
1271 } else {
1272 die "Unexpected result type $actual";
1273 }
1274 }
1275  
1276 sub record_result {
1277 my $self = shift;
1278 my $type = shift;
1279 my $ok = shift;
1280 my $name = shift;
1281 my $todo = shift;
1282 unless ($ok) {
1283 if ($todo) {
1284 push(@{ $self->failures->{'TODO'}{$type} }, $name);
1285 } else {
1286 push(@{ $self->failures->{$type} }, $name);
1287 }
1288 }
1289
1290 if (my $earl = $self->earl) {
1291 if ($ok) {
1292 $earl->pass($name);
1293 } else {
1294 $earl->fail($name);
1295 }
1296 }
1297 }
1298  
1299 sub model_as_string {
1300 my $self = shift;
1301 my $model = shift;
1302 my $ser = Attean->get_serializer('nquads');
1303 my $sep = ('####' x 25) . "\n";
1304 my $s = sprintf("Model with %d quads:\n", $model->size);
1305 $s .= $ser->serialize_iter_to_bytes($model->get_quads);
1306 return $sep . $s . $sep;
1307 }
1308  
1309 sub DESTROY {
1310 my $self = shift;
1311 my $count = 0;
1312 my $todo = 0;
1313 while (my ($type, $failures) = each(%{ $self->failures })) {
1314 if ($type eq 'TODO') {
1315 while (my ($type, $failures) = each(%$failures)) {
1316 $todo += scalar(@$failures);
1317 }
1318 next;
1319 }
1320 $count += scalar(@$failures);
1321 }
1322 my $timestamp = scalar(time);
1323 if ($self->run_query_tests and ($count or $todo)) {
1324 my $d = Data::Dumper->new([$self->failures], [qw(tests)]);
1325 $d->Sortkeys(1)->Indent(2);
1326 my $msg = "Failing and/or TODO tests: " . $d->Dump;
1327 warn $msg;
1328 unless ($self->pattern) {
1329 open(my $fh, '>', sprintf('.sparql-test-suite-%d', $timestamp));
1330 while (my ($type, $failures) = each(%{ $self->failures })) {
1331 if ($type eq 'TODO') {
1332 while (my ($type, $failures) = each(%$failures)) {
1333 say $fh "$type - TODO";
1334 say $fh join("\n", sort @$failures);
1335 }
1336 next;
1337 }
1338 say $fh $type;
1339 say $fh join("\n", sort @$failures);
1340 }
1341 }
1342 }
1343 if (my $earl = $self->earl) {
1344 my $filename = sprintf('.sparql-test-suite-%d.ttl', $timestamp);
1345 open(my $fh, '>', $filename);
1346 my $model = $earl->model;
1347 my $iter = $model->get_quads()->as_triples();
1348 my $rdf = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
1349 my $EARL = 'http://www.w3.org/ns/earl#';
1350 my $map = URI::NamespaceMap->new( { earl => iri($EARL), rdf => iri($rdf) } );
1351 my $ser = Attean->get_serializer('Turtle')->new(namespaces => $map);
1352 $ser->serialize_iter_to_io($fh, $iter);
1353 warn "EARL results: $filename\n" if ($self->debug);
1354 }
1355 }
1356  
1357 ###############################################################################
1358  
1359 1;