Harness/Compatible.pm

filter: Statementstmt 36.2% 41 / 113 Branchbran 0.0% 0 / 24 Conditioncond 10.4% 5 / 48 MC/DCmcdc 0.0% 0 / 24 Subroutinesub 56.2% 9 / 16 Podpod 100% 2 / 2 totaltotal 25.1% 57 / 227 CC 28 SCAR 51.6
CC
28
Cov
25%
execute_tests
380.0
_canon
30.0
_check_sequence
20.0
CRAP 174.1
↑ Summary Harness/Compatible/Straps.pm »
1 package TAPx::Harness::Compatible;
2  
3 1 require 5.00405;
4  
5 1 ▶use TAPx::Harness;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 1 ▶use TAPx::Parser::Aggregator;
Subroutine BEGIN: called CC=1 SCAR=0.0
7  
8 #use TAPx::Harness::Compatible::Straps;
9 1 ▶use Exporter;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 1 ▶use Benchmark;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 1 ▶use Config;
Subroutine BEGIN: called CC=1 SCAR=0.0
12 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
13  
14 # TODO: Emulate at least some of these
15 1 ▶use vars qw(
Subroutine BEGIN: called CC=1 SCAR=0.0
16 $VERSION
17 @ISA @EXPORT @EXPORT_OK
18 $Verbose $Switches $Debug
19 $verbose $switches $debug
20 $Columns
21 $Directives
22 $Timer
23 $ML $Last_ML_Print
24 $Strap
25 $has_time_hires
26 1 );
27  
28 1 ▶BEGIN {
Subroutine BEGIN: called CC=1 SCAR=0.0
29 1 ▶ eval q{use Time::HiRes 'time'};
Subroutine BEGIN: called CC=1 SCAR=0.0
30 1 $has_time_hires = !$@;
31 }
32  
33 =head1 NAME
34  
35 TAPx::Harness::Compatible - Run Perl standard test scripts with statistics
36  
37 =head1 VERSION
38  
39 Version 0.50_07
40  
41 =cut
42  
43 1 $VERSION = '0.50_07';
44  
45 # Backwards compatibility for exportable variable names.
46 1 *verbose = *Verbose;
47 1 *switches = *Switches;
48 1 *debug = *Debug;
49  
50 #
51 # $ENV{HARNESS_ACTIVE} = 1;
52 # $ENV{HARNESS_VERSION} = $VERSION;
53 #
54 # END {
55 # # For VMS.
56 # delete $ENV{HARNESS_ACTIVE};
57 # delete $ENV{HARNESS_VERSION};
58 # }
59 #
60 # my $Files_In_Dir = $ENV{HARNESS_FILELEAK_IN_DIR};
61 #
62 # # Stolen from Params::Util
63 # sub _CLASS {
64 # (defined $_[0] and ! ref $_[0] and $_[0] =~ m/^[^\W\d]\w*(?:::\w+)*$/s) ? $_[0] : undef;
65 # }
66 #
67 # # Strap Overloading
68 # if ( $ENV{HARNESS_STRAPS_CLASS} ) {
69 # die 'Set HARNESS_STRAP_CLASS, singular, not HARNESS_STRAPS_CLASS';
70 # }
71 # my $HARNESS_STRAP_CLASS = $ENV{HARNESS_STRAP_CLASS} || 'TAPx::Harness::Compatible::Straps';
72 # if ( $HARNESS_STRAP_CLASS =~ /\.pm$/ ) {
73 # # "Class" is actually a filename, that should return the
74 # # class name as its true return value.
75 # $HARNESS_STRAP_CLASS = require $HARNESS_STRAP_CLASS;
76 # if ( !_CLASS($HARNESS_STRAP_CLASS) ) {
77 # die "HARNESS_STRAP_CLASS '$HARNESS_STRAP_CLASS' is not a valid class name";
78 # }
79 # }
80 # else {
81 # # It is a class name within the current @INC
82 # if ( !_CLASS($HARNESS_STRAP_CLASS) ) {
83 # die "HARNESS_STRAP_CLASS '$HARNESS_STRAP_CLASS' is not a valid class name";
84 # }
85 # eval "require $HARNESS_STRAP_CLASS";
86 # die $@ if $@;
87 # }
88 # if ( !$HARNESS_STRAP_CLASS->isa('TAPx::Harness::Compatible::Straps') ) {
89 # die "HARNESS_STRAP_CLASS '$HARNESS_STRAP_CLASS' must be a TAPx::Harness::Compatible::Straps subclass";
90 # }
91 #
92 # $Strap = $HARNESS_STRAP_CLASS->new;
93 #
94 # sub strap { return $Strap };
95 #
96  
97 1 @ISA = ('Exporter');
98 1 @EXPORT = qw(&runtests);
99 1 @EXPORT_OK = qw(&execute_tests $verbose $switches);
100  
101 1 ▶$Verbose = $ENV{HARNESS_VERBOSE} || 0;
Condition50%1/2
$ENV{'HARNESS_VERBOSE'} || 0
l!l
01
MC/DC0%0/1
$ENV{'HARNESS_VERBOSE'} || 0
$ENV{'HARNESS_VERBOSE'}
Truth table50%1/2
A
A= $ENV{'HARNESS_VERBOSE'}
Aresult
11
00
102 1 ▶$Debug = $ENV{HARNESS_DEBUG} || 0;
Condition50%1/2
$ENV{'HARNESS_DEBUG'} || 0
l!l
01
MC/DC0%0/1
$ENV{'HARNESS_DEBUG'} || 0
$ENV{'HARNESS_DEBUG'}
Truth table50%1/2
A
A= $ENV{'HARNESS_DEBUG'}
Aresult
11
00
103 1 $Switches = '-w';
104 1 ▶$Columns = $ENV{HARNESS_COLUMNS} || $ENV{COLUMNS} || 80;
Condition40%2/5
$ENV{'HARNESS_COLUMNS'} || $ENV{'COLUMNS'} || 80
l!l
01
$ENV{"HARNESS_COLUMNS"} or $ENV{"COLUMNS"}
l!l&&r!l&&!r
001
MC/DC0%0/2
$ENV{'HARNESS_COLUMNS'} || $ENV{'COLUMNS'} || 80
$ENV{"HARNESS_COLUMNS"} $ENV{"COLUMNS"}
Truth table33%1/3
A or B
A= $ENV{"HARNESS_COLUMNS"}B= $ENV{"COLUMNS"}
ABresult
1X1
011
000
105 1 $Columns--; # Some shells have trouble with a full line of text.
106 1 ▶$Timer = $ENV{HARNESS_TIMER} || 0;
Condition50%1/2
$ENV{'HARNESS_TIMER'} || 0
l!l
01
MC/DC0%0/1
$ENV{'HARNESS_TIMER'} || 0
$ENV{'HARNESS_TIMER'}
Truth table50%1/2
A
A= $ENV{'HARNESS_TIMER'}
Aresult
11
00
107  
108 =head1 SYNOPSIS
109  
110 use TAPx::Harness::Compatible;
111  
112 runtests(@test_files);
113  
114 =head1 DESCRIPTION
115  
116 B<STOP!> If all you want to do is write a test script, consider
117 using Test::Simple. TAPx::Harness::Compatible is the module that reads the
118 output from Test::Simple, Test::More and other modules based on
119 Test::Builder. You don't need to know about TAPx::Harness::Compatible to use
120 those modules.
121  
122 TAPx::Harness::Compatible runs tests and expects output from the test in a
123 certain format. That format is called TAP, the Test Anything
124 Protocol. It is defined in L<TAPx::Harness::Compatible::TAP>.
125  
126 C<TAPx::Harness::Compatible::runtests(@tests)> runs all the testscripts named
127 as arguments and checks standard output for the expected strings
128 in TAP format.
129  
130 The F<prove> utility is a thin wrapper around TAPx::Harness::Compatible.
131  
132 =head2 Taint mode
133  
134 TAPx::Harness::Compatible will honor the C<-T> or C<-t> in the #! line on your
135 test files. So if you begin a test with:
136  
137 #!perl -T
138  
139 the test will be run with taint mode on.
140  
141 =head2 Configuration variables.
142  
143 These variables can be used to configure the behavior of
144 TAPx::Harness::Compatible. They are exported on request.
145  
146 =over 4
147  
148 =item C<$TAPx::Harness::Compatible::Verbose>
149  
150 The package variable C<$TAPx::Harness::Compatible::Verbose> is exportable and can be
151 used to let C<runtests()> display the standard output of the script
152 without altering the behavior otherwise. The F<prove> utility's C<-v>
153 flag will set this.
154  
155 =item C<$TAPx::Harness::Compatible::switches>
156  
157 The package variable C<$TAPx::Harness::Compatible::switches> is exportable and can be
158 used to set perl command line options used for running the test
159 script(s). The default value is C<-w>. It overrides C<HARNESS_PERL_SWITCHES>.
160  
161 =item C<$TAPx::Harness::Compatible::Timer>
162  
163 If set to true, and C<Time::HiRes> is available, print elapsed seconds
164 after each test file.
165  
166 =back
167  
168  
169 =head2 Failure
170  
171 When tests fail, analyze the summary report:
172  
173 t/base..............ok
174 t/nonumbers.........ok
175 t/ok................ok
176 t/test-harness......ok
177 t/waterloo..........dubious
178 Test returned status 3 (wstat 768, 0x300)
179 DIED. FAILED tests 1, 3, 5, 7, 9, 11, 13, 15, 17, 19
180 Failed 10/20 tests, 50.00% okay
181 Failed Test Stat Wstat Total Fail List of Failed
182 ---------------------------------------------------------------
183 t/waterloo.t 3 768 20 10 1 3 5 7 9 11 13 15 17 19
184 Failed 1/5 test scripts, 80.00% okay. 10/44 subtests failed, 77.27% okay.
185  
186 Everything passed but F<t/waterloo.t>. It failed 10 of 20 tests and
187 exited with non-zero status indicating something dubious happened.
188  
189 The columns in the summary report mean:
190  
191 =over 4
192  
193 =item B<Failed Test>
194  
195 The test file which failed.
196  
197 =item B<Stat>
198  
199 If the test exited with non-zero, this is its exit status.
200  
201 =item B<Wstat>
202  
203 The wait status of the test.
204  
205 =item B<Total>
206  
207 Total number of tests expected to run.
208  
209 =item B<Fail>
210  
211 Number which failed, either from "not ok" or because they never ran.
212  
213 =item B<List of Failed>
214  
215 A list of the tests which failed. Successive failures may be
216 abbreviated (ie. 15-20 to indicate that tests 15, 16, 17, 18, 19 and
217 20 failed).
218  
219 =back
220  
221  
222 =head1 FUNCTIONS
223  
224 The following functions are available.
225  
226 =head2 runtests( @test_files )
227  
228 This runs all the given I<@test_files> and divines whether they passed
229 or failed based on their output to STDOUT (details above). It prints
230 out each individual test which failed along with a summary report and
231 a how long it all took.
232  
233 It returns true if everything was ok. Otherwise it will C<die()> with
234 one of the messages in the DIAGNOSTICS section.
235  
236 =cut
237  
238 0 ▶sub runtests {
Subroutine runtests: not called CC=2 SCAR=17.9
Pod documented
239 0 my @tests = @_;
240  
241 0 my $harness = _new_harness();
242 0 my $aggregate = TAPx::Parser::Aggregator->new();
243  
244 0 my $results = $harness->aggregate_tests( $aggregate, @tests );
245  
246 0 $harness->summary($results);
247  
248 0 my $total = $aggregate->total;
249 0 my $passed = $aggregate->passed;
250  
251 0 ▶ return $total && $total == $passed;
Condition0%0/3
$total && $total == $passed
!ll&&!rl&&r
000
MC/DC0%0/2
$total && $total == $passed
$total $total == $passed
Truth table0%0/3
A && B
A= $totalB= $total == $passed
ABresult
0X0
100
111
252 }
253  
254 0 ▶sub _canon {
Subroutine _canon: not called CC=5 SCAR=34.0
255 0 my @list = sort { $a <=> $b } @_;
256 0 my @ranges = ();
257 0 my $count = scalar @list;
258 0 my $pos = 0;
259  
260 0 ▶ while ( $pos < $count ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($pos < $count) 0 0 0
261 0 my $end = $pos + 1;
262 0 ▶ $end++ while $end < $count && $list[$end] <= $list[ $end - 1 ] + 1;
Branch: 1 branch
BranchTrueFalseTotal
$end < $count and $list[$end] <= $list[$end - 1] + 1 and do { ++$end } 0 0 0
Condition0%0/3
$end < $count and $list[$end] <= $list[$end - 1] + 1
!ll&&!rl&&r
000
MC/DC0%0/2
$end < $count and $list[$end] <= $list[$end - 1] + 1
$end < $count $list[$end] <= $list[$end - 1] + 1
Truth table0%0/3
A and B
A= $end < $countB= $list[$end] <= $list[$end - 1] + 1
ABresult
0X0
100
111
263 0 ▶ push @ranges, ( $end == $pos + 1 )
Branch: 1 branch
BranchTrueFalseTotal
$end == $pos + 1 ? : 0 0 0
264 ? $list[$pos]
265 : join( '-', $list[$pos], $list[ $end - 1 ] );
266 0 $pos = $end;
267 }
268  
269 0 return join( ' ', @ranges );
270 }
271  
272 0 ▶sub _new_harness {
Subroutine _new_harness: not called CC=1 SCAR=6.9
273  
274 # TODO: lib? switches?
275 0 my $args = {
276 verbose => $Verbose,
277 timer => $Timer,
278 directives => $Directives,
279 };
280  
281 0 return TAPx::Harness->new($args);
282 }
283  
284 0 ▶sub _check_sequence {
Subroutine _check_sequence: not called CC=4 SCAR=30.0
285 0 my @list = @_;
286 0 my $prev;
287 0 ▶ while ( my $next = shift @list ) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $next = shift @list) 0 0 0
288 0 ▶ return if defined $prev && $next <= $prev;
Branch: 1 branch
BranchTrueFalseTotal
if defined $prev and $next <= $prev 0 0 0
Condition0%0/3
defined $prev and $next <= $prev
!ll&&!rl&&r
000
MC/DC0%0/2
defined $prev and $next <= $prev
defined $prev $next <= $prev
Truth table0%0/3
A and B
A= defined $prevB= $next <= $prev
ABresult
0X0
100
111
289 0 $prev = $next;
290 }
291  
292 0 return 1;
293 }
294  
295 0 ▶sub execute_tests {
Subroutine execute_tests: not called CC=19 SCAR=59.4
Pod documented
296 0 my %args = @_;
297  
298 # TODO: Handle out option
299  
300 0 my $harness = _new_harness();
301 0 my $aggregate = TAPx::Parser::Aggregator->new();
302  
303 0 my %tot = (
304 bonus => 0,
305 max => 0,
306 ok => 0,
307 bad => 0,
308 good => 0,
309 files => 0,
310 tests => 0,
311 sub_skipped => 0,
312 todo => 0,
313 skipped => 0,
314 bench => undef,
315 );
316  
317 # Install a callback so we get to see any plans the
318 # harness executes.
319 $harness->callback(
320 0 ▶ made_parser => sub {
Subroutine __ANON__: not called CC=1 SCAR=6.9
321 0 my $parser = shift;
322 $parser->callback(
323 0 ▶ plan => sub {
Subroutine __ANON__: not called CC=2 SCAR=17.9
324 0 my $plan = shift;
325 0 ▶ if ( $plan->directive eq 'SKIP' ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($plan->directive eq "SKIP") 0 0 0
326 0 $tot{skipped}++;
327 }
328 }
329 0 );
330 }
331 0 );
332  
333 0 my $results = $harness->aggregate_tests( $aggregate, @{ $args{tests} } );
334  
335 0 $tot{bench} = timediff( $results->{end}, $results->{start} );
336  
337 # TODO: Work out the circumstances under which the files
338 # and tests totals can differ.
339 0 $tot{files} = $tot{tests} = @{ $results->{tests} };
340  
341 0 my %failedtests = ();
342 0 my %todo_passed = ();
343  
344 0 for my $test ( @{ $results->{tests} } ) {
345 0 my ($parser) = $aggregate->parsers($test);
346  
347 0 my @failed = $parser->failed;
348  
349 0 my $wstat = $parser->wait;
350 0 my $estat = $parser->exit;
351 0 my $planned = $parser->tests_planned;
352 0 my @errors = $parser->parse_errors;
353 0 my $passed = $parser->passed;
354 0 my $actual_passed = $parser->actual_passed;
355  
356 0 my $ok_seq = _check_sequence( $parser->actual_passed );
357  
358 # Duplicate exit, wait status semantics of old version
359 0 ▶ $estat ||= '' unless $wstat;
Branch: 1 branch
BranchTrueFalseTotal
unless $wstat 0 0 0
Condition0%0/2
$estat ||= ""
l!l
00
MC/DC0%0/1
$estat ||= ""
$estat
Truth table0%0/2
A
A= $estat
Aresult
11
00
360 0 ▶ $wstat ||= '';
Condition0%0/2
$wstat ||= ""
l!l
00
MC/DC0%0/1
$wstat ||= ""
$wstat
Truth table0%0/2
A
A= $wstat
Aresult
11
00
361  
362 0 ▶ $tot{max} += ( $planned || 0 );
Condition0%0/2
$planned || 0
l!l
00
MC/DC0%0/1
$planned || 0
$planned
Truth table0%0/2
A
A= $planned
Aresult
11
00
363 0 $tot{bonus} += $parser->todo_passed;
364 0 ▶ $tot{ok} += $passed > $actual_passed ? $passed : $actual_passed;
Branch: 1 branch
BranchTrueFalseTotal
$passed > $actual_passed ? : 0 0 0
365 0 $tot{sub_skipped} += $parser->skipped;
366 0 $tot{todo} += $parser->todo;
367  
368 0 ▶ if ( @failed || $estat || @errors ) {
Branch: 1 branch
BranchTrueFalseTotal
if (@failed or $estat or @errors) { } 0 0 0
Condition0%0/6
@failed or $estat or @errors
l!l&&r!l&&!r
000
@failed or $estat
l!l&&r!l&&!r
000
MC/DC0%0/3
@failed or $estat or @errors
@failed $estat @errors
Truth table0%0/4
A or B or C
A= @failedB= $estatC= @errors
ABCresult
1XX1
01X1
0011
0000
369 0 $tot{bad}++;
370  
371 0 ▶ my $huh_planned = $planned ? undef: '??';
Branch: 1 branch
BranchTrueFalseTotal
$planned ? : 0 0 0
372 0 ▶ my $huh_errors = $ok_seq ? undef: '??';
Branch: 1 branch
BranchTrueFalseTotal
$ok_seq ? : 0 0 0
373  
374 0 ▶ $failedtests{$test} = {
Condition0%0/16
$huh_planned || $huh_errors || _canon(@failed) || '??'
l!l
00
$huh_planned or $huh_errors or _canon(@failed)
l!l&&r!l&&!r
000
$huh_planned or $huh_errors
l!l&&r!l&&!r
000
$huh_planned || $huh_errors || scalar @failed
l!l
00
$huh_planned or $huh_errors
l!l&&r!l&&!r
000
$huh_planned || $planned
l!l&&r!l&&!r
000
MC/DC0%0/3
$huh_planned || $huh_errors || _canon(@failed) || '??'
$huh_planned $huh_errors _canon(@failed)
MC/DC0%0/2
$huh_planned || $huh_errors || scalar @failed
$huh_planned $huh_errors
MC/DC0%0/2
$huh_planned || $planned
$huh_planned $planned
Truth table0%0/4
A or B or C
A= $huh_plannedB= $huh_errorsC= _canon(@failed)
ABCresult
1XX1
01X1
0011
0000
Truth table0%0/3
A or B
A= $huh_plannedB= $huh_errors
ABresult
1X1
011
000
Truth table0%0/3
A || B
A= $huh_plannedB= $planned
ABresult
1X1
011
000
375 'canon' => $huh_planned
376 || $huh_errors
377 || _canon(@failed)
378 || '??',
379 'estat' => $estat,
380 'failed' => $huh_planned || $huh_errors || scalar @failed,
381 'max' => $huh_planned || $planned,
382 'name' => $test,
383 'wstat' => $wstat
384 };
385 }
386 else {
387 0 $tot{good}++;
388 }
389  
390 0 my @todo = $parser->todo_passed;
391 0 ▶ if (@todo) {
Branch: 1 branch
BranchTrueFalseTotal
if (@todo) 0 0 0
392 0 $todo_passed{$test} = {
393 'canon' => _canon(@todo),
394 'estat' => $estat,
395 'failed' => scalar @todo,
396 'max' => scalar $parser->todo,
397 'name' => $test,
398 'wstat' => $wstat
399 };
400 }
401 }
402  
403 0 return ( \%tot, \%failedtests, \%todo_passed );
404 }
405  
406 =head2 execute_tests( tests => \@test_files, out => \*FH )
407  
408 Runs all the given C<@test_files> (just like C<runtests()>) but
409 doesn't generate the final report. During testing, progress
410 information will be written to the currently selected output
411 filehandle (usually C<STDOUT>), or to the filehandle given by the
412 C<out> parameter. The I<out> is optional.
413  
414 Returns a list of two values, C<$total> and C<$failed>, describing the
415 results. C<$total> is a hash ref summary of all the tests run. Its
416 keys and values are this:
417  
418 bonus Number of individual todo tests unexpectedly passed
419 max Number of individual tests ran
420 ok Number of individual tests passed
421 sub_skipped Number of individual tests skipped
422 todo Number of individual todo tests
423  
424 files Number of test files ran
425 good Number of test files passed
426 bad Number of test files failed
427 tests Number of test files originally given
428 skipped Number of test files skipped
429  
430 If C<< $total->{bad} == 0 >> and C<< $total->{max} > 0 >>, you've
431 got a successful test.
432  
433 C<$failed> is a hash ref of all the test scripts that failed. Each key
434 is the name of a test script, each value is another hash representing
435 how that script failed. Its keys are these:
436  
437 name Name of the test which failed
438 estat Script's exit value
439 wstat Script's wait status
440 max Number of individual tests
441 failed Number which failed
442 canon List of tests which failed (as string).
443  
444 C<$failed> should be empty if everything passed.
445  
446 =cut
447  
448 1 1;
449 __END__
↑ Summary Harness/Compatible/Straps.pm »