bin/test-generator-mutate

filter: Statementstmt 88.2% 187 / 212 Branchbran 61.2% 49 / 80 Conditioncond 44.4% 16 / 36 MC/DCmcdc 15.0% 3 / 20 Subroutinesub 88.2% 15 / 17 Podpod n/a totaltotal 73.9% 270 / 365 CC 12 SCAR 15.1
CC
12
Cov
77%
__ANON__
12.0
group_mutants
12.0
mutation_difficulty
5.1
CRAP 4.5
« lib/App/Test/Generator/Sample/Module.pm ↑ Summary lib/App/Test/Generator/Planner.pm »
1 #!/usr/bin/env perl
2  
3 7 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 7 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5  
6 7 ▶use Carp qw(croak);
Subroutine BEGIN: called CC=1 SCAR=0.0
7 7 ▶use Config;
Subroutine BEGIN: called CC=1 SCAR=0.0
8 7 ▶use Cwd qw(abs_path);
Subroutine BEGIN: called CC=1 SCAR=0.0
9 7 ▶use File::Find qw(find);
Subroutine BEGIN: called CC=1 SCAR=0.0
10 7 ▶use File::Spec;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 7 ▶use Getopt::Long qw(GetOptions);
Subroutine BEGIN: called CC=1 SCAR=0.0
12 7 ▶use JSON::MaybeXS;
Subroutine BEGIN: called CC=1 SCAR=0.0
13 7 ▶use Pod::Usage;
Subroutine BEGIN: called CC=1 SCAR=0.0
14 7 ▶use Readonly;
Subroutine BEGIN: called CC=1 SCAR=0.0
15 7 ▶use Time::HiRes qw(time);
Subroutine BEGIN: called CC=1 SCAR=0.0
16  
17 7 ▶use App::Test::Generator::Mutator;
Subroutine BEGIN: called CC=1 SCAR=0.0
18  
19 =pod
20  
21 =head1 NAME
22  
23 test-generator-mutate - Run mutation testing against a Perl test suite
24  
25 =head1 SYNOPSIS
26  
27 test-generator-mutate [options]
28  
29 test-generator-mutate --lib lib --tests t
30 test-generator-mutate --file lib/My/Module.pm
31 test-generator-mutate --json mutation.json
32 test-generator-mutate --min-score 75
33  
34 =head1 QUICK START
35  
36 test-generator-mutate --lib lib --min-score 85 --json mutation.json
37  
38 =head2 Skipping Lines from Mutation
39  
40 Some routines involve logic that must not be mutated because doing so
41 would cause the test suite to hang rather than produce a clean
42 pass/fail result. Two common cases:
43  
44 =over 4
45  
46 =item Recursion base cases
47  
48 The C<ConditionalInversion> and C<BooleanNegation> operators will negate
49 any C<if> condition, including the guard that terminates a recursive
50 function. A mutant that removes the base case causes infinite recursion
51 and the per-mutant C<prove> run hangs indefinitely (there is currently
52 no per-mutant timeout). Protect the base-case condition:
53  
54 sub factorial {
55 my ($n) = @_;
56 ## MUTANT_SKIP_BEGIN
57 return 1 if $n <= 1;
58 ## MUTANT_SKIP_END
59 return $n * factorial($n - 1);
60 }
61  
62 =item Process-management side-effects
63  
64 Forking, signalling, and C<waitpid> are inherently difficult to assert
65 against in a unit test. A mutant that corrupts a PID or signal value
66 may cause the test suite to hang or emit warnings from an C<END> block:
67  
68 sub disable_hot_reload {
69 ## MUTANT_SKIP_BEGIN
70 kill 'HUP', $pid if $pid;
71 waitpid $pid, 0;
72 ## MUTANT_SKIP_END
73 }
74  
75 =back
76  
77 To exclude a block of lines from mutation testing entirely, wrap it with
78 C<## MUTANT_SKIP_BEGIN> and C<## MUTANT_SKIP_END> annotations as shown
79 above.
80  
81 Skipped lines are silently excluded from the mutation candidate list and
82 do not appear in the killed or survived counts. The number of skipped
83 lines is shown in the per-file mutation report.
84  
85 A mutant that does not compile (C<perl -Ilib -c> fails) is not run. It is
86 not in the killed or survived counts either, but the summary counts it as
87 C<Skipped (compile)>, and the JSON report gives the number per file under
88 C<compile_skipped>. Each mutant is compiled against the project's own
89 C<--lib> directory, so a large count usually means a broken mutation or a
90 dependency that is not installed, not a problem with the tests.
91  
92 Mismatched markers (a C<## MUTANT_SKIP_BEGIN> with no matching
93 C<## MUTANT_SKIP_END>, or vice versa) are fatal errors.
94  
95 =head2 Numeric Boundary Mutants
96  
97 Kill Numeric Boundary Mutants first,
98 these are the easiest wins.
99 For example,
100 C<NUM_BOUNDARY_1295>
101 means something like C<if ($x > 10)> became C<if ($x >= 10)> or C<if ($x > 9)>.
102 If that survived, it means, there is a missing edge value.
103 Numeric mutations are important because they reveal missing edge coverage.
104 This example means line 1295.
105  
106 So if that line contains something like this:
107  
108 if(((scalar keys %input) == 1) && exists($input{'type'}) && !ref($input{'type'})) {
109  
110 You need to add a test where
111  
112 =over 4
113  
114 =item * %input contains more than one key
115  
116 =item * One of them is type
117  
118 =item * And behavior must be different
119  
120 =back
121  
122 For example, if you have a test with
123  
124 %input = ( type => 'string' )
125  
126 add a test which sets
127  
128 %input = (
129 type => 'string',
130 something_else => 'value'
131 )
132  
133 =head2 Conditional Inversions
134  
135 Then kill Conditional Inversions, for example, C<COND_INV_1186>,
136 where C<unless (-f $file)> became C<if (-f $file)>.
137 If that survives,
138 test did not assert the negative case.
139  
140 Focus by file,
141 if one file contributes 200 survivors, that's the weakest module.
142  
143 Frequently re-run.
144 The loop should be: add 5-10 targeted tests, re-run mutation tool, watch score climb, repeat.
145  
146 =head1 DESCRIPTION
147  
148 This command-line tool performs mutation testing on a Perl codebase.
149  
150 It scans one or more C<.pm> files, generates code mutations using
151 L<App::Test::Generator::Mutator>, and runs the project's test suite
152 against each mutated version inside an isolated workspace.
153  
154 For each generated mutant:
155  
156 =over 4
157  
158 =item *
159  
160 The mutant is applied in a temporary workspace.
161  
162 =item *
163  
164 The mutated file is syntax-checked.
165  
166 =item *
167  
168 The test suite is executed using C<prove>.
169  
170 =item *
171  
172 If the tests fail, the mutant is considered I<killed>.
173  
174 =item *
175  
176 If the tests pass, the mutant is considered I<survived>.
177  
178 =back
179  
180 A mutation score is then calculated:
181  
182 (killed / total) * 100
183  
184 Mutation testing measures the effectiveness of a test suite. A higher
185 mutation score indicates that the tests are better at detecting behavioral
186 changes in the code.
187  
188 =head1 OPTIONS
189  
190 =head2 --lib <dir>
191  
192 Directory containing Perl modules to mutate.
193  
194 Defaults to C<lib>.
195  
196 =head2 --file <file>
197  
198 Mutate a single file instead of scanning the entire C<--lib> directory.
199  
200 =head2 --tests <dir>
201  
202 Directory containing test files.
203  
204 Defaults to C<t>.
205  
206 =head2 --changed_only
207  
208 Only mutate files that were changed in the most recent commit, as
209 determined by C<git diff --name-only HEAD~1 HEAD>. Files not changed
210 in the current commit retain their mutation results from the previous
211 dashboard run. This significantly reduces CI runtime while preserving
212 accuracy for the files that actually changed.
213  
214 =head2 --exclude <path>
215  
216 Exclude files matching the given path fragment from mutation testing.
217 May be specified multiple times. For example:
218  
219 --exclude lib/Devel --exclude lib/App/Test/Generator/Sample
220  
221 =head2 --base_sha <sha>
222  
223 The git commit SHA to use as the base when computing which files have
224 changed under C<--changed_only>. Defaults to C<HEAD~1>.
225  
226 Use this when automated commits (such as coverage snapshots or generated
227 test stubs) have landed on top of your last code commit, causing
228 C<HEAD~1> to point to an automated commit rather than a real change.
229 Typically set by the CI workflow using the output of C<git log> filtered
230 to exclude automated commits.
231  
232 =head2 --min-score <int>
233  
234 Minimum acceptable mutation score (percentage).
235  
236 If the final score is below this value, the program exits with a
237 non-zero status.
238  
239 =head2 --json <file>
240  
241 Write mutation results to the specified JSON file.
242  
243 The output structure:
244  
245 {
246 score => "85.32",
247 total => 120,
248 killed => 102,
249 survived => [ ... mutant IDs ... ],
250 compile_skipped => { 'lib/My/Module.pm' => 2 },
251 }
252  
253 =head2 --cover_json <file>
254  
255 The location of the file generated by C<cover -report json>.
256 That file is used to generate an approximation for an LCSAJ table.
257  
258 =head2 --fail-fast
259  
260 (Reserved for future use.)
261  
262 =head2 --mutation_level <full|fast>
263  
264 Setting to C<fast> removes redundant mutations and dedups mutations before running.
265 The default is C<full>.
266  
267 =head2 --timeout <seconds>
268  
269 (Reserved for future use.)
270  
271 =head2 --verbose
272  
273 Print progress information.
274  
275 =head2 --quiet
276  
277 Suppress final summary output.
278  
279 =head1 EXIT CODES
280  
281 =over 4
282  
283 =item = 0
284  
285 Success and mutation score meets minimum threshold.
286  
287 =item = 1
288  
289 Mutation score below C<--min-score>.
290  
291 =item = 2
292  
293 Baseline test suite failed before mutation testing began.
294  
295 =item = 3
296  
297 Invalid command-line options.
298  
299 =back
300  
301 =head1 WORKFLOW
302  
303 The tool performs the following steps:
304  
305 =over 4
306  
307 =item 1.
308  
309 Collect target files (either a single file or all C<.pm> files under C<--lib>).
310  
311 =item 2.
312  
313 Run baseline tests to ensure the suite passes before mutation.
314  
315 =item 3.
316  
317 Generate mutants for each file.
318  
319 =item 4.
320  
321 Apply each mutant in isolation and re-run the test suite.
322  
323 =item 5.
324  
325 Calculate and report mutation statistics.
326  
327 =back
328  
329 =encoding utf-8
330  
331 =head1 WORKFLOW DIAGRAM
332  
333 The mutation testing process follows this execution flow:
334  
335 ┌───────────────────────────────┐
336 │ Start │
337 └───────────────┬───────────────┘
338 │
339 ▼
340 ┌───────────────────────────────┐
341 │ Collect Target Files │
342 │ --file OR scan --lib/*.pm │
343 └───────────────┬───────────────┘
344 │
345 ▼
346 ┌───────────────────────────────┐
347 │ Run Baseline Tests │
348 │ prove -l t │
349 └───────────────┬───────────────┘
350 │
351 Baseline OK? ── No ──► Exit (code 2)
352 │
353 Yes
354 │
355 ▼
356 ┌───────────────────────────────┐
357 │ For Each File │
358 └───────────────┬───────────────┘
359 │
360 ▼
361 ┌───────────────────────────────┐
362 │ Generate Mutants │
363 │ (conditional flips, etc.) │
364 └───────────────┬───────────────┘
365 │
366 ▼
367 ┌───────────────────────────────┐
368 │ For Each Mutant │
369 └───────────────┬───────────────┘
370 │
371 ▼
372 ┌───────────────────────────────┐
373 │ Prepare Workspace │
374 │ (isolated temp directory) │
375 └───────────────┬───────────────┘
376 │
377 ▼
378 ┌───────────────────────────────┐
379 │ Apply Mutant │
380 └───────────────┬───────────────┘
381 │
382 ▼
383 ┌───────────────────────────────┐
384 │ Syntax Check │
385 │ perl -Ilib -c mutated_file.pm│
386 └───────────────┬───────────────┘
387 │
388 Compiles? ── No ──► Skip Mutant (Skipped (compile))
389 │
390 Yes
391 │
392 ▼
393 ┌───────────────────────────────┐
394 │ Run Test Suite │
395 │ prove t │
396 └───────────────┬───────────────┘
397 │
398 Tests Fail? ── Yes ──► Killed++
399 │
400 No
401 │
402 ▼
403 Survived++
404 │
405 ▼
406 ┌───────────────────────────────┐
407 │ Repeat for Next Mutant │
408 └───────────────┬───────────────┘
409 │
410 ▼
411 ┌───────────────────────────────┐
412 │ Calculate Mutation Score │
413 │ (killed / total) * 100 │
414 └───────────────┬───────────────┘
415 │
416 ▼
417 ┌───────────────────────────────┐
418 │ Print Report / Write JSON │
419 └───────────────┬───────────────┘
420 │
421 ▼
422 Finish
423  
424 =cut
425  
426 # --------------------------------------------------
427 # Difficulty label strings and priority scores.
428 # TODO: move to App::Test::Generator::Constants once
429 # that module is created, so they can be shared with
430 # SchemaExtractor and other consumers.
431 # --------------------------------------------------
432 7 Readonly my $DIFFICULTY_HIGH_LABEL => 'HIGH';
433 7 Readonly my $DIFFICULTY_MEDIUM_LABEL => 'MEDIUM';
434 7 Readonly my $DIFFICULTY_LOW_LABEL => 'LOW';
435  
436 7 Readonly my $DIFFICULTY_HIGH_SCORE => 3;
437 7 Readonly my $DIFFICULTY_MEDIUM_SCORE => 2;
438 7 Readonly my $DIFFICULTY_LOW_SCORE => 1;
439  
440 # --------------------------------------------------
441 # Status strings written into the JSON output and
442 # used by test-generator-index to colour-code the
443 # mutation heatmap.
444 # TODO: move to App::Test::Generator::Constants.
445 # --------------------------------------------------
446 7 Readonly my $STATUS_SURVIVED => 'Survived';
447 7 Readonly my $STATUS_KILLED => 'Killed';
448  
449 # --------------------------------------------------
450 # Mutation level strings accepted by --mutation_level.
451 # 'full' runs every mutant independently.
452 # 'fast' deduplicates mutants by site before running.
453 # --------------------------------------------------
454 7 Readonly my $MUTATION_LEVEL_FULL => 'full';
455 7 Readonly my $MUTATION_LEVEL_FAST => 'fast';
456  
457 7 my @excludes;
458  
459 7 my %opt = (
460 changed_only => 0,
461 lib => 'lib',
462 tests => 't',
463 min_score => 0,
464 fail_fast => 0,
465 verbose => 0,
466 quiet => 0,
467 man => 0,
468 help => 0,
469 mutation_level => $MUTATION_LEVEL_FULL,
470 lcsaj_root => 'lcsaj',
471 lcsaj_hits => 'cover_html/lcsaj_hits.json',
472 );
473  
474 # Exit code 3 is "invalid command-line options" per the EXIT CODES POD section
475 GetOptions(
476 'base_sha=s' => \$opt{base_sha},
477 'changed_only' => \$opt{changed_only},
478 'lib=s' => \$opt{lib},
479 'file=s' => \$opt{file},
480 'tests=s' => \$opt{tests},
481 'min-score=i' => \$opt{min_score},
482 'cover_json=s' => \$opt{cover_json},
483 'exclude=s' => \@excludes,
484 'json=s' => \$opt{json},
485 'fail-fast' => \$opt{fail_fast},
486 'timeout=i' => \$opt{timeout},
487 'verbose' => \$opt{verbose},
488 'quiet' => \$opt{quiet},
489 'help|h' => \$opt{help},
490 'mutation_level=s' => \$opt{mutation_level},
491 'man|m' => \$opt{man},
492 'lcsaj_root=s' => \$opt{lcsaj_root},
493 'lcsaj_hits=s' => \$opt{lcsaj_hits},
494 7 ▶) or pod2usage(3);
Branch: 1 branch
BranchTrueFalseTotal
&GetOptions("base_sha=s", \$opt{"base_sha"}, "changed_only", \$opt{"changed_only"}, "lib=s", \$opt{"lib"}, "file=s", \$opt{"file"}, "tests=s", \$opt{"tests"}, "min-score=i", \$opt{"min_score"}, "cover_json=s", \$opt{"cover_json"}, "exclude=s", \@excludes, "json=s", \$opt{"json"}, "fail-fast", \$opt{"fail_fast"}, "timeout=i", \$opt{"timeout"}, "verbose", \$opt{"verbose"}, "quiet", \$opt{"quiet"}, "help|h", \$opt{"help"}, "mutation_level=s", \$opt{"mutation_level"}, "man|m", \$opt{"man"}, "lcsaj_root=s", \$opt{"lcsaj_root"}, "lcsaj_hits=s", \$opt{"lcsaj_hits"}) or pod2usage(3) 7 0 7
495  
496 7 ▶pod2usage(-exitval => 0, -verbose => 1) if $opt{help};
Branch: 1 branch
BranchTrueFalseTotal
if $opt{"help"} 0 7 7
497 7 ▶pod2usage(-exitval => 0, -verbose => 2) if $opt{man};
Branch: 1 branch
BranchTrueFalseTotal
if $opt{"man"} 0 7 7
498  
499 # warn "DEBUG options: changed_only=$opt{changed_only} base_sha=", $opt{base_sha} // 'undef';
500  
501 # Warn when reserved-for-future-use options are supplied so the
502 # caller knows they have no effect yet rather than silently ignoring
503 7 ▶warn "--fail-fast is reserved for future use and has no effect" if $opt{fail_fast};
Branch: 1 branch
BranchTrueFalseTotal
if $opt{"fail_fast"} 0 7 7
504 7 ▶warn "--timeout is reserved for future use and has no effect" if $opt{timeout};
Branch: 1 branch
BranchTrueFalseTotal
if $opt{"timeout"} 0 7 7
505  
506 # -------------------------
507 # Collect Files
508 # -------------------------
509 7 my @files;
510  
511 7 ▶if($opt{file}) {
Branch: 2 branches
BranchTrueFalseTotal
if ($opt{'file'}) { } 4 3 7
elsif ($opt{'changed_only'}) { } 3 0 3
512 4 push @files, $opt{file};
513 } elsif($opt{changed_only}) {
514 # Only mutate files changed since the base commit
515 croak("Invalid base_sha: $opt{base_sha}")
516 3 ▶ if defined($opt{base_sha}) && $opt{base_sha} !~ /^[0-9A-Za-z._\/-]+\z/;
Branch: 1 branch
BranchTrueFalseTotal
if defined $opt{"base_sha"} and not $opt{"base_sha"} =~ m[^[0-9A-Za-z._/-]+\z] 1 2 3
Condition66%2/3
defined $opt{"base_sha"} and not $opt{"base_sha"} =~ m[^[0-9A-Za-z._/-]+\z]
!ll&&!rl&&r
021
MC/DC50%1/2
defined $opt{"base_sha"} and not $opt{"base_sha"} =~ m[^[0-9A-Za-z._/-]+\z]
defined $opt{"base_sha"} not $opt{"base_sha"} =~ m[^[0-9A-Za-z._/-]+\z]
Truth table66%2/3
A and not(B)
A= defined $opt{"base_sha"}B= not $opt{"base_sha"} =~ m[^[0-9A-Za-z._/-]+\z]
ABresult
0X0
100
111
517 2 ▶ my $base = $opt{base_sha} ? "$opt{base_sha}^" : 'HEAD~1';
Branch: 1 branch
BranchTrueFalseTotal
$opt{'base_sha'} ? : 2 0 2
518 2 my $diff = '';
519 {
520 2 ▶ open(my $git_fh, '-|', 'git', 'diff', '--name-only', $base, 'HEAD')
Branch: 1 branch
BranchTrueFalseTotal
open my $git_fh, "-|", "git", "diff", "--name-only", $base, "HEAD" or croak("Could not run git diff: $!") 2 0 2
521 or croak("Could not run git diff: $!");
522 2 local $/;
523 2 ▶ $diff = <$git_fh> // '';
Condition50%1/2
readline $git_fh // ""
l!l
20
MC/DC0%0/1
readline $git_fh // ""
readline $git_fh
Truth table50%1/2
A
A= readline $git_fh
Aresult
11
00
524 2 close($git_fh);
525 }
526 my @changed = grep {
527 # Must be a lib .pm file, must exist, and must not match any exclusion
528 2 my $f = $_;
529 0 ▶ /^lib\/.*\.pm$/ && -f $f && !grep { $f =~ m{\Q$_\E} } @excludes
Condition33%2/6
m[^lib/.*\.pm$] && -f $f && !grep({$f =~ /\Q$_\E/;} @excludes)
!ll&&!rl&&r
002
m[^lib/.*\.pm$] and -f $f
!ll&&!rl&&r
002
MC/DC0%0/3
m[^lib/.*\.pm$] && -f $f && !grep({$f =~ /\Q$_\E/;} @excludes)
m[^lib/.*\.pm$] -f $f !grep({$f =~ /\Q$_\E/;} @excludes)
Truth table25%1/4
A and B && not(C)
A= m[^lib/.*\.pm$]B= -f $fC= !grep({$f =~ /\Q$_\E/;} @excludes)
ABCresult
0XX0
10X0
1100
1111
530 } split /\n/, $diff;
531  
532 2 ▶ if(@changed) {
Branch: 1 branch
BranchTrueFalseTotal
if (@changed) { } 1 1 2
533 1 ▶ print 'Changed-only mode: mutating ', scalar(@changed), " file(s)\n" if $opt{verbose};
Branch: 1 branch
BranchTrueFalseTotal
if $opt{"verbose"} 0 1 1
534 1 @files = @changed;
535 } else {
536 # No lib/*.pm files changed -- fall through to merge and write
537 # existing results back to JSON without running new mutations
538 1 print "No lib/*.pm files changed -- retaining previous mutation results\n";
539 }
540 } else {
541 find(
542 0 ▶ sub {
Subroutine __ANON__: not called CC=3 SCAR=24.8
543 # Skip files matching any exclusion pattern
544 0 ▶ return if grep { $File::Find::name =~ m{\Q$_\E} } @excludes;
Branch: 1 branch
BranchTrueFalseTotal
if grep {$File::Find::name =~ /\Q$_\E/;} @excludes 0 0 0
545 0 ▶ push @files, $File::Find::name if /\.pm$/;
Branch: 1 branch
BranchTrueFalseTotal
if /\.pm$/ 0 0 0
546 },
547 $opt{lib}
548 0 );
549 }
550  
551 # -------------------------
552 # Verify baseline tests
553 # -------------------------
554  
555 6 ▶print "Running baseline tests...\n" if $opt{verbose};
Branch: 1 branch
BranchTrueFalseTotal
if $opt{"verbose"} 0 6 6
556  
557 # Run with whatever LCSAJ_TARGETS (if any) is already in the parent
558 # environment, not the mutation-scoped value set below -- this baseline
559 # pass verifies the unmodified suite, and t/LCSAJ-Runtime.t exercises
560 # DB::DB() directly against its own file, which is never one of @files,
561 # so scoping LCSAJ_TARGETS to @files here makes that test spuriously fail.
562 #
563 # Use list form of system() to avoid shell interpolation of $opt{tests},
564 # which could be a security risk if the path contains special characters
565 6 ▶if(system('prove', '-Mblib', '-l', '-r', $opt{tests}) != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (system("prove", "-Mblib", "-l", "-r", $opt{"tests"}) != 0) 1 5 6
566 1 print STDERR "Baseline tests failed.\n";
567 1 exit 2;
568 }
569  
570 # Pass absolute target file paths to the LCSAJ runtime debugger
571 # via environment variable so it knows which files to instrument
572 # during the mutation runs below.
573 5 ▶if(@files) {
Branch: 1 branch
BranchTrueFalseTotal
if (@files) { } 5 0 5
574 5 $ENV{LCSAJ_TARGETS} = join($Config{path_sep}, map { abs_path($_) } @files);
575 } else {
576 # No files to mutate -- preserve existing LCSAJ hit data
577 0 delete $ENV{LCSAJ_TARGETS};
578 }
579  
580 # -------------------------
581 # Run Mutation Testing
582 # -------------------------
583  
584 5 my $total = 0;
585 5 my $killed = 0;
586 5 my @survivors;
587 my @killed_mutants;
588 5 my %skipped_by_file; # lines excluded by MUTANT_SKIP annotations, keyed by filename
589 5 my %compile_skipped_by_file; # mutants dropped because they do not compile, keyed by filename
590  
591 5 for my $file (@files) {
592 my $mutator = App::Test::Generator::Mutator->new(
593 file => $file,
594 lib_dir => $opt{lib},
595 mutation_level => $opt{mutation_level},
596 6 );
597  
598 6 my @mutants = $mutator->generate_mutants();
599  
600 # Collect skip count from mutator
601 6 ▶ my $skipped = scalar keys %{$mutator->{skip_lines} // {}};
Condition50%1/2
$mutator->{"skip_lines"} // {}
l!l
60
MC/DC0%0/1
$mutator->{"skip_lines"} // {}
$mutator->{"skip_lines"}
Truth table50%1/2
A
A= $mutator->{"skip_lines"}
Aresult
11
00
602  
603 # Add to per-file tracking (needs a new %skipped_by_file hash)
604 6 ▶ $skipped_by_file{$file} = $skipped if $skipped;
Branch: 1 branch
BranchTrueFalseTotal
if $skipped 0 6 6
605  
606 6 my $groups;
607  
608 6 ▶ if($opt{mutation_level} eq $MUTATION_LEVEL_FAST) {
Branch: 1 branch
BranchTrueFalseTotal
if ($opt{'mutation_level'} eq $MUTATION_LEVEL_FAST) { } 0 6 6
609 # Fast mode: deduplicate mutants by mutation site before
610 # running to avoid redundant test runs
611 0 $groups = group_mutants($file, \@mutants);
612 } else {
613 # Full mode: run every mutant independently
614 6 $groups = { map { $_->id => [$_] } @mutants };
615 }
616  
617 # warn "DEBUG file=$file groups=", scalar(keys %{$groups});
618  
619 6 for my $group (values %{$groups}) {
620 30 my $mutant = representative_mutant($group);
621  
622 print 'Testing representative mutant ',
623 $mutant->id,
624 " ($file line ",
625 $mutant->line,
626 ') representing ',
627 0 scalar(@{$group}),
628 30 ▶ " mutants\n" if $opt{verbose};
Branch: 1 branch
BranchTrueFalseTotal
if $opt{"verbose"} 0 30 30
629  
630 30 my $workspace = $mutator->prepare_workspace();
631  
632 30 $mutator->apply_mutant($mutant);
633  
634 30 my $workspace_file = File::Spec->catfile($workspace, $file);
635  
636 # Without -I the mutant's own "use" lines resolve to installed copies of
637 # the project's modules, which may be older or missing; the mutant then
638 # "fails to compile" and is skipped rather than run.
639 30 my $workspace_lib = File::Spec->catdir($workspace, (File::Spec->splitdir($opt{lib}))[-1]);
640 30 my $compile = system($^X, '-I', $workspace_lib, '-c', $workspace_file);
641 30 ▶ if($compile != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if ($compile != 0) 10 20 30
642 10 $compile_skipped_by_file{$file} += scalar(@{$group});
643 10 next;
644 }
645  
646 # Apply mutation in-place so test files that use 'use lib "$Bin/../lib"'
647 # (a common CPAN idiom) load the mutated code. Those files do
648 # 'unshift @INC' at compile time, which prepends the project's own lib/
649 # BEFORE any PERL5LIB-injected workspace path, making the workspace
650 # approach invisible to them and causing all mutants to survive.
651 # In-place mutation writes to the actual project file, so every @INC
652 # strategy (PERL5LIB, -I, 'use lib', and blib) sees the mutant.
653 20 ▶ my $orig_content = do { open(my $fh, '<', $file) or croak "Cannot read $file: $!"; local $/; <$fh> };
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, "<", $file or croak("Cannot read ${file}: $!") 20 0 20
654 20 ▶ my $mut_content = do { open(my $fh, '<', $workspace_file) or croak "Cannot read $workspace_file: $!"; local $/; <$fh> };
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, "<", $workspace_file or croak("Cannot read ${workspace_file}: $!") 20 0 20
655  
656 20 ▶ { open(my $fh, '>', $file) or croak "Cannot write $file: $!"; print $fh $mut_content; }
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, ">", $file or croak("Cannot write ${file}: $!") 20 0 20
657  
658 # Clear PERL5OPT so no stray -Ilib or -M flags interfere with prove.
659 20 local $ENV{PERL5OPT} = '';
660  
661 # -l adds the project's lib/ to @INC for tests that don't use 'use lib'.
662 20 my $survived = (system('prove', '-l', '-r', $opt{tests}) == 0);
663  
664 # Always restore the original, even if prove crashes.
665 20 ▶ { open(my $fh, '>', $file) or croak "Cannot restore $file: $!"; print $fh $orig_content; }
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, ">", $file or croak("Cannot restore ${file}: $!") 20 0 20
666  
667 20 ▶ if($survived) {
Branch: 1 branch
BranchTrueFalseTotal
if ($survived) { } 10 10 20
668 # Representative survived — mark the entire group as survived
669 10 for my $m (@{$group}) {
670 10 my $difficulty = mutation_difficulty($m);
671  
672 push @survivors, {
673 id => $m->id(),
674 line => $m->line(),
675 file => $file,
676 description => $m->description(),
677 status => $STATUS_SURVIVED,
678 difficulty => $difficulty->{label},
679 priority => $difficulty->{score},
680 hint => $difficulty->{hint},
681 10 };
682  
683 10 $total++;
684 }
685 } else {
686 # Representative killed — assume the whole group is killed
687 10 for my $m (@{$group}) {
688 10 push @killed_mutants, {
689 id => $m->id(),
690 line => $m->line(),
691 file => $file,
692 description => $m->description(),
693 status => $STATUS_KILLED,
694 };
695  
696 10 $total++;
697 10 $killed++;
698 }
699 }
700  
701 # workspace auto-destroyed when $workspace goes out of scope
702 }
703 }
704  
705 # -------------------------
706 # Report
707 # -------------------------
708  
709 5 ▶my $score = $total ? sprintf('%.2f', ($killed / $total) * 100) : 100;
Branch: 1 branch
BranchTrueFalseTotal
$total ? : 4 1 5
710 5 my $compile_skipped = 0;
711 5 $compile_skipped += $_ for values %compile_skipped_by_file;
712  
713 5 ▶unless($opt{quiet}) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($opt{"quiet"}) 5 0 5
714 5 print "\nMutation Score: $score%\n",
715 "Total: $total\n",
716 "Killed: $killed\n",
717 'Survived: ', scalar(@survivors), "\n",
718 'Skipped (compile): ', $compile_skipped, "\n";
719 }
720  
721 # printf "DEBUG argv: json=%s changed_only=%s\n",
722 # $opt{json} // 'undef',
723 # $opt{changed_only} // 'undef';
724  
725 5 ▶if($opt{json}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($opt{"json"}) 2 3 5
726 2 ▶ if($opt{changed_only} && -f $opt{json}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($opt{"changed_only"} and -f $opt{"json"}) 1 1 2
Condition66%2/3
$opt{"changed_only"} and -f $opt{"json"}
!ll&&!rl&&r
101
MC/DC50%1/2
$opt{"changed_only"} and -f $opt{"json"}
$opt{"changed_only"} -f $opt{"json"}
Truth table66%2/3
A and B
A= $opt{"changed_only"}B= -f $opt{"json"}
ABresult
0X0
100
111
727 # printf "DEBUG merge: file=%s size=%d\n", $opt{json}, -s $opt{json};
728 1 my $raw = do {
729 1 local $/;
730 1 ▶ open(my $fh, '<', $opt{json}) or die "Cannot open $opt{json}: $!";
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, "<", $opt{"json"} or die qq[Cannot open $opt{"json"}: $!] 1 0 1
731 <$fh>
732 1 };
733 # printf "DEBUG merge: raw_length=%d first_char=%s\n", length($raw // ''), substr($raw // '', 0, 1);
734 1 my $prev = eval { decode_json($raw) };
735 1 ▶ if($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) { } 0 1 1
736 # printf "DEBUG merge: PARSE ERROR: %s\n", substr($@, 0, 200);
737 0 $prev = {};
738 } else {
739 1 ▶ $prev //= {};
Condition50%1/2
$prev //= {}
l!l
10
MC/DC0%0/1
$prev //= {}
$prev
Truth table50%1/2
A
A= $prev
Aresult
11
00
740 }
741  
742 # printf "DEBUG merge: keys=%s survived=%d killed=%d\n",
743 # join(',', sort keys %{$prev // {}}),
744 # scalar(@{ $prev->{survived} || [] }),
745 # scalar(@{ $prev->{killed} || [] });
746  
747 # Keep survived/killed entries for files NOT in this run
748 1 my %mutated_files = map { $_ => 1 } @files;
749 0 ▶ my @prev_survived = grep { !$mutated_files{ $_->{file} // '' } } @{ $prev->{survived} || [] };
Condition25%1/4
$_->{"file"} // ""
l!l
00
$prev->{'survived'} || []
l!l
10
MC/DC0%0/1
$_->{"file"} // ""
$_->{"file"}
MC/DC0%0/1
$prev->{'survived'} || []
$prev->{'survived'}
Truth table0%0/2
A
A= $_->{"file"}
Aresult
11
00
Truth table50%1/2
A
A= $prev->{'survived'}
Aresult
11
00
750 0 ▶ my @prev_killed = grep { !$mutated_files{ $_->{file} // '' } } @{ $prev->{killed} || [] };
Condition25%1/4
$_->{"file"} // ""
l!l
00
$prev->{'killed'} || []
l!l
10
MC/DC0%0/1
$_->{"file"} // ""
$_->{"file"}
MC/DC0%0/1
$prev->{'killed'} || []
$prev->{'killed'}
Truth table0%0/2
A
A= $_->{"file"}
Aresult
11
00
Truth table50%1/2
A
A= $prev->{'killed'}
Aresult
11
00
751  
752 # Merge previous data with current run
753 1 unshift @survivors, @prev_survived;
754 1 unshift @killed_mutants, @prev_killed;
755 1 ▶ my %prev_compile_skipped = %{ $prev->{compile_skipped} || {} };
Condition50%1/2
$prev->{'compile_skipped'} || {}
l!l
10
MC/DC0%0/1
$prev->{'compile_skipped'} || {}
$prev->{'compile_skipped'}
Truth table50%1/2
A
A= $prev->{'compile_skipped'}
Aresult
11
00
756 1 ▶ $compile_skipped_by_file{$_} //= $prev_compile_skipped{$_} for grep { !$mutated_files{$_} } keys %prev_compile_skipped;
Condition33%1/3
$compile_skipped_by_file{$_} //= $prev_compile_skipped{$_}
l!l&&r!l&&!r
010
MC/DC0%0/2
$compile_skipped_by_file{$_} //= $prev_compile_skipped{$_}
$compile_skipped_by_file{$_} $prev_compile_skipped{$_}
Truth table33%1/3
A //= B
A= $compile_skipped_by_file{$_}B= $prev_compile_skipped{$_}
ABresult
1X1
011
000
757  
758 # Recalculate totals to include previous data
759 1 $total += scalar(@prev_survived) + scalar(@prev_killed);
760 1 $killed += scalar(@prev_killed);
761 1 ▶ $score = $total ? sprintf('%.2f', ($killed / $total) * 100) : 100;
Branch: 1 branch
BranchTrueFalseTotal
$total ? : 1 0 1
762 }
763  
764 # Sort survivors worst-first so the most impactful gaps
765 # appear at the top of the generated test stub file
766 2 @survivors = sort { $b->{priority} <=> $a->{priority} } @survivors;
767 2 ▶ open(my $fh, '>', $opt{json}) or die "Cannot write $opt{json}: $!";
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, ">", $opt{"json"} or die qq[Cannot write $opt{"json"}: $!] 2 0 2
768 2 print $fh encode_json({
769 score => $score,
770 total => $total,
771 killed_count => $killed,
772 survived_count => scalar(@survivors),
773 killed => \@killed_mutants,
774 survived => \@survivors,
775 skipped => \%skipped_by_file,
776 compile_skipped => \%compile_skipped_by_file,
777 });
778 2 close $fh;
779 }
780  
781 5 ▶if($score < $opt{min_score}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($score < $opt{"min_score"}) 0 5 5
782 0 print STDERR "score $score is less than min_score $opt{min_score}\n";
783 0 exit 1;
784 }
785  
786 5 exit 0;
787  
788 # --------------------------------------------------
789 # group_mutants
790 #
791 # Purpose: Group mutants that affect the same
792 # mutation site (file, line, and original
793 # operator) so that only one representative
794 # mutant needs to be tested per site in
795 # fast mode.
796 #
797 # Entry: $file - path to the source file being
798 # mutated.
799 # $mutants - arrayref of mutant objects as
800 # returned by generate_mutants().
801 #
802 # Exit: Returns a hashref keyed by site key
803 # (file:line:operator), where each value
804 # is an arrayref of mutant objects at that
805 # site.
806 #
807 # Side effects: None.
808 #
809 # Notes: The site key uses the original token
810 # (operator or keyword being mutated) with
811 # whitespace stripped, so that formatting
812 # differences do not create spurious groups.
813 # Used only when --mutation_level=fast.
814 # --------------------------------------------------
815 0 ▶sub group_mutants {
Subroutine group_mutants: not called CC=3 SCAR=24.8
816 0 my ($file, $mutants) = @_;
817  
818 0 my %groups;
819  
820 0 for my $m (@{$mutants}) {
821 0 my $line = $m->line;
822  
823 # Original token being mutated (e.g. >, ==, !)
824 0 ▶ my $orig = defined($m->original) ? $m->original : '';
Branch: 1 branch
BranchTrueFalseTotal
defined $m->original ? : 0 0 0
825  
826 # Strip whitespace so formatting differences don't
827 # create spurious distinct groups for the same site
828 0 $orig =~ s/\s+//g;
829  
830 # Group key identifies a unique mutation site
831 0 my $key = join(':', $file, $line, $orig);
832  
833 0 push @{$groups{$key}}, $m;
834 }
835  
836 0 return \%groups;
837 }
838  
839 # --------------------------------------------------
840 # representative_mutant
841 #
842 # Purpose: Select a single mutant from a group to
843 # stand in for all mutants at that site
844 # when running in fast mode. If the
845 # representative is killed, the whole
846 # group is assumed killed; if it survives,
847 # the whole group is assumed to survive.
848 #
849 # Entry: $group - arrayref of mutant objects at
850 # a single mutation site.
851 #
852 # Exit: Returns a single mutant object. Never
853 # returns undef (falls back to the first
854 # element of the group).
855 #
856 # Side effects: None.
857 #
858 # Notes: Prefers boundary-type mutations when
859 # available since they probe the most
860 # meaningful behavioural edge. Falls back
861 # to the first mutant in the group if no
862 # boundary mutant is present.
863 # --------------------------------------------------
864 30 ▶sub representative_mutant {
Subroutine representative_mutant: called CC=4 SCAR=13.9
865 30 my $group = $_[0];
866  
867 30 for my $m (@{$group}) {
868 # Prefer boundary-type mutations — they are the most
869 # informative representatives since they probe numeric
870 # edge cases that other mutation types may not exercise
871 30 ▶ if($m->id() && $m->id() =~ /BOUNDARY|NUM/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($m->id and $m->id =~ /BOUNDARY|NUM/) 18 12 30
Condition66%2/3
$m->id and $m->id =~ /BOUNDARY|NUM/
!ll&&!rl&&r
01218
MC/DC50%1/2
$m->id and $m->id =~ /BOUNDARY|NUM/
$m->id $m->id =~ /BOUNDARY|NUM/
Truth table66%2/3
A and B
A= $m->idB= $m->id =~ /BOUNDARY|NUM/
ABresult
0X0
100
111
872 18 return $m;
873 }
874 }
875  
876 # Fallback: use the first mutant in the group
877 12 return $group->[0];
878 }
879  
880 # --------------------------------------------------
881 # mutation_difficulty
882 #
883 # Purpose: Assign a difficulty label, priority
884 # score, and hint string to a surviving
885 # mutant, reflecting how meaningful it
886 # would be to kill it and what kind of
887 # test is likely to do so.
888 #
889 # Entry: $mutant - a mutant object with an id()
890 # method returning the mutant ID string.
891 #
892 # Exit: Returns a hashref with three keys:
893 # score - numeric priority (higher is
894 # more important to kill)
895 # label - one of HIGH, MEDIUM, LOW
896 # hint - human-readable advice on what
897 # test to write
898 #
899 # Side effects: None.
900 #
901 # Notes: Classification is based on the mutant
902 # ID prefix, which encodes the mutation
903 # type. This is a heuristic — the same
904 # type of mutation may be more or less
905 # impactful depending on context.
906 #
907 # The label and score constants are
908 # candidates for extraction to
909 # App::Test::Generator::Constants once
910 # that module is created.
911 # --------------------------------------------------
912 10 ▶sub mutation_difficulty {
Subroutine mutation_difficulty: called CC=5 SCAR=16.4
913 10 my $mutant = $_[0];
914  
915 10 ▶ my $id = $mutant->id // '';
Condition50%1/2
$mutant->id // ""
l!l
100
MC/DC0%0/1
$mutant->id // ""
$mutant->id
Truth table50%1/2
A
A= $mutant->id
Aresult
11
00
916  
917 # Numeric boundary mutants expose missing edge-case tests —
918 # the highest priority since they reveal concrete value gaps
919 10 ▶ if($id =~ /BOUNDARY|NUM/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($id =~ /BOUNDARY|NUM/) 6 4 10
920 return {
921 6 score => $DIFFICULTY_HIGH_SCORE,
922 label => $DIFFICULTY_HIGH_LABEL,
923 hint => 'Likely missing edge-case test (boundary value)',
924 };
925 }
926  
927 # Conditional inversions reveal missing negative assertions —
928 # medium priority since one branch was left untested
929 4 ▶ if($id =~ /COND|INV|NEG/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($id =~ /COND|INV|NEG/) 2 2 4
930 return {
931 2 score => $DIFFICULTY_MEDIUM_SCORE,
932 label => $DIFFICULTY_MEDIUM_LABEL,
933 hint => 'Add tests asserting both true and false outcomes',
934 };
935 }
936  
937 # Logical operator changes reveal missing combination tests
938 2 ▶ if($id =~ /LOGIC|AND|OR/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($id =~ /LOGIC|AND|OR/) 0 2 2
939 return {
940 0 score => $DIFFICULTY_MEDIUM_SCORE,
941 label => $DIFFICULTY_MEDIUM_LABEL,
942 hint => 'Test combinations where only one logical operand is true',
943 };
944 }
945  
946 # Default: minor mutation with limited behavioural impact
947 return {
948 2 score => $DIFFICULTY_LOW_SCORE,
949 label => $DIFFICULTY_LOW_LABEL,
950 hint => 'Mutation survived, but impact may be minor',
951 };
952 }
953  
954 =head1 AUTHOR
955  
956 Nigel Horne
957  
958 =cut
« lib/App/Test/Generator/Sample/Module.pm ↑ Summary lib/App/Test/Generator/Planner.pm »