File Coverage

blib/lib/SimpleFlow.pm

Criterion	Covered	Total	%
statement	202	219	92.2
branch	52	70	74.2
condition	25	34	73.5
subroutine	20	20	100.0
pod	2	2	100.0
total	301	345	87.2

# ABSTRACT: SimpleFlow - easy, simple workflow manager (and logger); for keeping track of and debugging large and complex shell command workflows
use strict;
use warnings FATAL => 'all';
require 5.010;
use feature 'say';
use DDP {output => 'STDOUT', array_max => 10, show_memsize => 1};
use Devel::Confess 'color';
use Cwd 'getcwd';
package SimpleFlow;
our $VERSION = 0.13;
use Time::HiRes;
use Term::ANSIColor;
# Windows portability: the legacy Windows console (cmd.exe) prints raw ANSI
# escape sequences as garbage. Disable colouring there unless a terminal that
# understands ANSI is in use (Windows Terminal, ConEmu, ANSICON). Unix and
# modern Windows terminals are left untouched.
BEGIN {
	$ENV{ANSI_COLORS_DISABLED} = 1
		if $^O eq 'MSWin32'
		&& !$ENV{WT_SESSION} # Windows Terminal
		&& !$ENV{ConEmuANSI} # ConEmu
		&& !$ENV{ANSICON};   # ANSICON
}
use Scalar::Util 'openhandle';
use DDP {output => 'STDOUT', array_max => 10, show_memsize => 1};
use Devel::Confess 'color';
use Cwd 'getcwd';
use warnings FATAL => 'all';
use Capture::Tiny 'capture';
use List::Util 'max';
use Exporter 'import';
our @EXPORT = qw(say2 task);
our @EXPORT_OK = @EXPORT;

sub say2 { # say to both command line and log file
	my ($msg, $fh) = @_;
	my $current_sub = (split(/::/,(caller(0))[3]))[-1]; # https://stackoverflow.com/questions/2559792/how-can-i-get-the-name-of-the-current-subroutine-in-perl
	my @c = caller;
	if (not openhandle($fh)) {
		die "the filehandle given to $current_sub with \"$msg\" from $c[1] line $c[2] isn't actually a filehandle";
	}
	$msg = "\@ $c[1] line $c[2] $msg";
	say $msg;
	say $fh $msg;
	return $msg;
}

sub task {
	my ($args) = @_;
	my $current_sub = (split(/::/,(caller(0))[3]))[-1];
	unless (ref $args eq 'HASH') {
		die "args must be given as a hash ref, e.g. \"$current_sub({ data => \@blah })\"";
	}
	my @c = caller;
	my @reqd_args = (
		'cmd', # the shell command
	);
	my @undef_args = grep { !defined $args->{$_}} @reqd_args;
	if (scalar @undef_args > 0) {
		p @undef_args;
		die 'the above args are necessary, but were not defined.';
	}
	my @defined_args = ( @reqd_args,
		'die',			  # die if not successful; 0 or 1
		'dry.run',       # dry run or not
		'input.files',   # check for input files; SCALAR or ARRAY
		'log.fh',
		'note',          # a note for the log
		'overwrite',     #
		'output.files'	  # product files that need to be checked; can be scalar or array
	);
	my @bad_args = grep { my $key = $_; not grep {$_ eq $key} @defined_args} keys %{ $args };
	if (scalar @bad_args > 0) {
		p @bad_args, array_max => scalar @bad_args;
		say "the above arguments are not recognized by $current_sub";
		p @defined_args, array_max => scalar @defined_args;
		die "The above args are accepted by $current_sub";
	}
	if (
			(defined $args->{'log.fh'}) &&
			(not openhandle($args->{'log.fh'}))
		) {
		p $args;
		die "the filehandle given to $current_sub isn't actually a filehandle";
	}
	my (%input_file_size, @existing_files, @output_files, @empty_filenames);
	if (defined $args->{'input.files'}) {
		my $ref = ref $args->{'input.files'};
		my @missing_files;
		if ($ref eq 'ARRAY') {
			@missing_files = grep {not -f -r $_ } @{ $args->{'input.files'} };
			%input_file_size = map { $_ => -s $_ } @{ $args->{'input.files'} };
			@empty_filenames = grep {length $_ == 0} @{ $args->{'input.files'} };
		} elsif ($ref eq '') { # scalar
			@missing_files = grep {not -f -r $_ } ($args->{'input.files'});
			%input_file_size = map { $_ => -s $_ } ($args->{'input.files'} );
			@empty_filenames = grep {(defined $_) && (length $_ == 0)} ($args->{'input.files'});
		} else {
			p $args;
			die 'ref type "' . $ref . '" is not allowed for "input.files"';
		}
		if (scalar @missing_files > 0) {
			say STDERR 'this list of arguments:';
			p $args;
			say STDERR 'Cannot run because these files are either missing or unreadable in: ' . getcwd();
			p @missing_files;
			die 'the above files are missing or are not readable';
		}
	}
	if (scalar @empty_filenames > 0) {
		p $args;
		die '0-length filenames are not allowed (found in "input.files")';
	}
	my $msg = "\@ $c[1] line $c[2] The command is:\n" . colored(['blue on_bright_red'], $args->{cmd});
	say $msg;
	say {$args->{'log.fh'}} "\@ $c[1] line $c[2] The command is:\n $args->{cmd})" if defined $args->{'log.fh'};
	if (defined $args->{'output.files'}) { # avoid "uninitialized value" warning
		my $ref = ref $args->{'output.files'};
		if ($ref eq 'ARRAY') {
			@output_files = @{ $args->{'output.files'} };
		} elsif ($ref eq '') { # a scalar
			@output_files = $args->{'output.files'};
		} else {
			p $args;
			die "$ref isn't allowed for \"output.files\"";
		}
	}
	@empty_filenames = grep {length $_ == 0} @output_files; # 0-length filenames aren't allowed
	if (scalar @empty_filenames > 0) {
		p $args;
		die '0-length filenames are not allowed (found in "output.files"';
	}
	if (scalar @output_files > 0) {
		@existing_files = grep {-f $_} @output_files;
	}
	my %r = (
		cmd            => $args->{cmd},
		dir				=> getcwd(),
		'source.file'  => $c[1],
		'source.line'  => $c[2],
		'output.files' => [@output_files],
	);
	$r{'die'}     = $args->{'die'}     // 1; # by default, true
	$r{'dry.run'} = $args->{'dry.run'} // 0; # by default, false
	$r{note}      = $args->{note}      // '';# by default, false
	$r{overwrite} = $args->{overwrite} // 0; # by default, false
	$r{'will.do'} = 'yes';
	$r{'will.do'} = 'no: dry run'  if $args->{'dry.run'};
	my $string_max = 0;
	if (defined $args->{'input.files'}) {
		$r{'input.files'} = $args->{'input.files'};
		$r{'input.file.size'} = \%input_file_size;
	}
	my %output_file_size = map {$_ => -s $_} @output_files;
	foreach my $val (grep {ref $r{$_} eq ''} keys %r) {
		$string_max = max($string_max, length $r{$val});
	}
	if ((!$args->{overwrite}) && (scalar @output_files > 0) && (scalar @existing_files == scalar @output_files)) { # this has been done before
		$r{done} = 'before';
		$r{'will.do'} = 'no';
		say colored(['black on_green'], "\"$args->{cmd}\"\n") . ' has been done before';
		$r{'output.file.size'} = \%output_file_size;
		$r{duration} = 0;
		p(%r, output => $args->{'log.fh'}, string_max => $string_max) if defined $args->{'log.fh'};
		p %r, string_max => $string_max;
		return \%r;
	} else {
		$r{done} = 'not yet';
	}
	if ($r{'dry.run'}) {
		say "\@ $c[1] line $c[2] in $r{dir} the command was going to be:";
		say colored(['red on_black'], "\"$args->{cmd}\"");
		say 'But this is a dry run';
		say '-------------';
		$r{duration} = 0;
		return \%r;
	}
	my $t0 = Time::HiRes::time();
	my $status;
	($r{stdout}, $r{stderr}, $status) = capture {
		system( $args->{cmd} );
	};
	my $t1 = Time::HiRes::time();
	$r{duration} = $t1-$t0;
	# Decode the raw wait status. On Unix the low 7 bits hold the death
	# signal and the high byte holds the exit code. The signal MUST be read
	# from the raw status *before* shifting -- the old code shifted first and
	# then did ($exit & 127), so $r{signal} was always 0 and could never
	# detect a kill by signal 9/15. Windows has no POSIX signals, and a -1
	# return from system() means the command never launched.
	if (!defined $status || $status == -1) {
		$r{'exit'}   = -1;
		$r{signal}   = 0;
	} elsif ($^O eq 'MSWin32') {
		$r{signal}   = 0;
		$r{'exit'}   = $status >> 8;
	} else {
		$r{signal}   = $status & 127; # FIX: taken from raw status, not from $exit
		$r{'exit'}   = $status >> 8;
	}
	foreach my $std ('stderr', 'stdout') {
		$r{$std} =~ s/\s+$//; # remove trailing whitespace/newline
		$string_max = max($string_max, length $r{$std});
	}
	$r{done} = 'now';
	$r{'will.do'} = 'done';
	my @missing_output_files = grep {not -f -r $_} @output_files;
	if (scalar @missing_output_files > 0) {
		$r{'will.do'} = 'FAILED';
		say STDERR "this input to $current_sub:";
		p $args;
		say {$args->{'log.fh'}} "this input to $current_sub:" if defined $args->{'log.fh'};
		p($args, output => $args->{'log.fh'}, string_max => $string_max) if defined $args->{'log.fh'};
		say STDERR 'has these output files missing:';
		say {$args->{'log.fh'}} 'has these output files missing:' if defined $args->{'log.fh'};
		p @missing_output_files;
		p(@missing_output_files, output => $args->{'log.fh'}, string_max => $string_max) if defined $args->{'log.fh'};
		p %r, string_max => $string_max;
		p(%r, output => $args->{'log.fh'}, string_max => $string_max) if defined $args->{'log.fh'};
		if ($args->{'die'}) {
			die 'those above files should have been made but are missing';
		} else {
			say STDERR 'those above files should have been made but are missing';
		}
	}
	%output_file_size = map {$_ => -s $_} @output_files;
	$r{'output.file.size'} = \%output_file_size;
#	p %output_file_size;
	my @files_with_zero_size = grep { ($output_file_size{$_} // 0) == 0 } @output_files;
	if (scalar @files_with_zero_size > 0) {
		p @files_with_zero_size;
		warn 'the above output files have 0 size.';
	}
	p(%r, output => $args->{'log.fh'}) if defined $args->{'log.fh'};
	if (($r{'die'}) && ($r{'exit'} != 0)) {
		$r{'will.do'} = 'FAILED';
		p %r, string_max => $string_max;
		die "\"$args->{cmd}\" failed from $c[1] line $c[2]"
	}
	p %r, string_max => $string_max;
	return \%r;
}
1;

=encoding utf8

A tiny workflow manager and logger for Perl, like SnakeMake or NextFlow, but in pure Perl and aimed at making long, error-prone shell pipelines easy to B and B.

Every step is a single C call. SimpleFlow checks the inputs before a
command runs and the outputs after, times the command, captures its C,
C, exit code and signal, optionally logs a full structured record, and
skips work that has already been done.

Two subroutines are exported by default: L and L.

=head1 Install

With a CPAN client:

 cpanm SimpleFlow

Or from a checkout:

 perl Makefile.PL
 make
 make test
 make install

=head1 Synopsis

The simplest useful case: run a command and confirm it produced its output:

 use SimpleFlow qw(task say2);

 my $t = task({
     cmd            => 'which ls',
     'output.files' => '/tmp/AFK3mnEK8L.log',
 });

C returns a hash reference describing exactly what happened:

 {
     cmd            "which ls",
     die            1,
     dir            "/home/con/Scripts/SimpleFlow",
     done           "now",
     dry.run        0,
     duration       0.00191903114318848,
     exit           0,
     note           "",
     output.files   [
         [0] "/tmp/AFK3mnEK8L.log"
     ],
     overwrite      1,
     signal         0,
     source.file    "t/01.t",
     source.line    29,
     stderr         "",
     stdout         "/usr/bin/ls",
     will.do        "done"
 }

 > B SimpleFlow runs whatever shell command you give it via
 > C, so the I are your responsibility to keep
 > cross-platform (e.g. C is Unix-only). SimpleFlow's own behaviour
 > exit/signal decoding and coloured output is cross-platform; see the
 > L.

=head1 C

 my $result = task(\%args);

Runs one shell command with checking, timing, capture and logging. Takes a
B; the only required key is C.

=head2 Arguments



=begin html

 




323




 





 





 





 





 





 



  




324




 





 





 





 





 





 



  




325




 





 





 





 





 





 



   Key 




326




 





 





 





 





 





 



   Type 




327




 





 





 





 





 





 



   Default 




328




 





 





 





 





 





 



   Description 




329




 





 





 





 





 





 



  




330




 





 





 





 





 





 



  




331




 





 





 





 





 





 



  




332




 





 





 





 





 





 



  




333




 





 





 





 





 





 



   cmd 




334




 





 





 





 





 





 



   scalar 




335




 





 





 





 





 





 



   undef 




336




 





 





 





 





 





 



   Required. The shell command to run. 




337




 





 





 





 





 





 



  




338




 





 





 





 





 





 



  




339




 





 





 





 





 





 



   die 




340




 





 





 





 





 





 



   bool (0/1) 




341




 





 





 





 





 





 



   1 




342




 





 





 





 





 





 



   Die if the command fails (non-zero exit) or an output file is missing. Set to 0 to warn and continue instead. 




343




 





 





 





 





 





 



  




344




 





 





 





 





 





 



  




345




 





 





 





 





 





 



   dry.run 




346




 





 





 





 





 





 



   bool 




347




 





 





 





 





 





 



   0 




348




 





 





 





 





 





 



   Print the command (and log it) but do not execute it. 




349




 





 





 





 





 





 



  




350




 





 





 





 





 





 



  




351




 





 





 





 





 





 



   input.files 




352




 





 





 





 





 





 



   scalar or array 




353




 





 





 





 





 





 



   undef 




354




 





 





 





 





 





 



   File(s) that must exist and be readable before running; otherwise task dies. 




355




 





 





 





 





 





 



  




356




 





 





 





 





 





 



  




357




 





 





 





 





 





 



   output.files 




358




 





 





 





 





 





 



   scalar or array 




359




 





 





 





 





 





 



   undef 




360




 





 





 





 





 





 



   File(s) expected to exist after running; used both for the missing-output check and for skip detection. 




361




 





 





 





 





 





 



  




362




 





 





 





 





 





 



  




363




 





 





 





 





 





 



   log.fh 




364




 





 





 





 





 





 



   open filehandle 




365




 





 





 





 





 





 



   undef 




366




 





 





 





 





 





 



   If given, the full result record is also written here. Must be a real, open filehandle. 




367




 





 





 





 





 





 



  




368




 





 





 





 





 





 



  




369




 





 





 





 





 





 



   note 




370




 





 





 





 





 





 



   scalar 




371




 





 





 





 





 





 



   '' 




372




 





 





 





 





 





 



   Free-text note copied into the result and the log. 




373




 





 





 





 





 





 



  




374




 





 





 





 





 





 



  




375




 





 





 





 





 





 



   overwrite 




376




 





 





 





 





 





 



   bool 




377




 





 





 





 





 





 



   0 




378




 





 





 





 





 





 



   If false and all output.files already exist, the command is skipped. Set true to always run. 




379




 





 





 





 





 





 



  




380




 





 





 





 





 





 



  




381




























































=end html



Passing an unrecognised key, an empty filename, or a non-filehandle C
causes C to die: these are usually mistakes worth catching early.

=head2 Return value

C always returns a hash reference. The fields below are present after a
normal run; the L and L paths
omit the execution-only fields (C, C, C, C).



=begin html

 




401




 





 





 





 





 





 



  




402




 





 





 





 





 





 



  




403




 





 





 





 





 





 



   Field 




404




 





 





 





 





 





 



   Meaning 




405




 





 





 





 





 





 



  




406




 





 





 





 





 





 



  




407




 





 





 





 





 





 



  




408




 





 





 





 





 





 



  




409




 





 





 





 





 





 



   cmd 




410




 





 





 





 





 





 



   The command that was run. 




411




 





 





 





 





 





 



  




412




 





 





 





 





 





 



  




413




 





 





 





 





 





 



   dir 




414




 





 





 





 





 





 



   Working directory at execution time. 




415




 





 





 





 





 





 



  




416




 





 





 





 





 





 



  




417




 





 





 





 





 





 



   done 




418




 





 





 





 





 





 



   "now" (just ran), "before" (skipped, outputs already existed), or "not yet" (dry run). 




419




 





 





 





 





 





 



  




420




 





 





 





 





 





 



  




421




 





 





 





 





 





 



   will.do 




422




 





 





 





 





 





 



   "done", "no" (skipped), "no: dry run", or "FAILED". 




423




 





 





 





 





 





 



  




424




 





 





 





 





 





 



  




425




 





 





 





 





 





 



   duration 




426




 





 





 





 





 





 



   Wall-clock seconds the command took (0 for skips/dry runs). 




427




 





 





 





 





 





 



  




428




 





 





 





 





 





 



  




429




 





 





 





 





 





 



   exit 




430




 





 





 





 





 





 



   Exit code of the command (-1 if it could not be launched). 




431




 





 





 





 





 





 



  




432




 





 





 





 





 





 



  




433




 





 





 





 





 





 



   signal 




434




 





 





 





 





 





 



   Signal number if the command process was killed by a signal, else 0. Always 0 on Windows (no POSIX signals). 




435




 





 





 





 





 





 



  




436




 





 





 





 





 





 



  




437




 





 





 





 





 





 



   stdout, stderr 




438




 





 





 





 





 





 



   Captured output, with trailing whitespace stripped. 




439




 





 





 





 





 





 



  




440




 





 





 





 





 





 



  




441




 





 





 





 





 





 



   die, dry.run, overwrite, note 




442




 





 





 





 





 





 



   The (defaulted) argument values used. 




443




 





 





 





 





 





 



  




444




 





 





 





 





 





 



  




445




 





 





 





 





 





 



   output.files 




446




 





 





 





 





 





 



   Array ref of the output files (a scalar argument is normalised to a one-element array). 




447




 





 





 





 





 





 



  




448




 





 





 





 





 





 



  




449




 





 





 





 





 





 



   output.file.size 




450




 





 





 





 





 





 



   Hash of filename => size in bytes for the outputs. 




451




 





 





 





 





 





 



  




452




 





 





 





 





 





 



  




453




 





 





 





 





 





 



   input.files 




454




 





 





 





 





 





 



   The input argument, as given (present only if you passed input.files). 




455




 





 





 





 





 





 



  




456




 





 





 





 





 





 



  




457




 





 





 





 





 





 



   input.file.size 




458




 





 





 





 





 





 



   Hash of filename => size in bytes for the inputs (present only if you passed input.files). 




459




 





 





 





 





 





 



  




460




 





 





 





 





 





 



  




461




 





 





 





 





 





 



   source.file, source.line 




462




 





 





 





 





 





 



   Where in your code the task was called: handy when debugging a long pipeline. 




463




 





 





 





 





 





 



  




464




 





 





 





 





 





 



  




465


































































=end html



=head2 Skipping completed work

If C is false (the default) and every file in C already
exists, C does B re-run the command. This makes pipelines
restartable: re-running the script picks up where it left off.

 open my $log, '>', 'logfile.txt';
 my $t = task({
     cmd            => 'gmx grompp -f em.mdp -c box.gro -p topol.top -o em.tpr',
     'input.files'  => ['em.mdp', 'box.gro', 'topol.top'],
     'output.files' => 'em.tpr',
     'log.fh'       => $log,
 });
 close $log;

On the first run C is C<"now">; on a re-run (with C present) C
is C<"before"> and C is C<"no">. Pass C<< overwrite =E 1 >> to force it.

=head2 Dry runs

Useful for inspecting a pipeline without executing anything expensive:

 my $t = task({
     cmd       => 'a long-running, time-consuming command',
     'dry.run' => 1,
     'log.fh'  => $fh,
 });

The command is printed (and logged) but not run; C is C<"no: dry run">.

=head2 Failure behaviour

By default (C<< die =E 1 >>) C dies if the command exits non-zero or if any
declared C are missing afterwards, so a broken step stops the
pipeline immediately. With C<< die =E 0 >>, C instead warns and returns its
result hash (with C<< will.do =E "FAILED" >>), letting you decide what to do.

=head2 C

 say2($message, $filehandle);

"Say to two places": prints C<$message> to standard output B to the given
log filehandle, prefixed with the calling file and line number so log entries
are traceable. The filehandle must be open, or C dies.

 open my $log, '>', 'run.log';
 say2('starting equilibration', $log);   # -> STDOUT and run.log
 close $log;

=head1 Dependencies

Core/runtime modules used by SimpleFlow:

=over

=item * L captures C/C

=item * L (C) pretty result/record printing

=item * L better backtraces on death

=item * L coloured terminal output

=item * C, C, C, C core utilities

=back

The test suite additionally uses C and
L.

=head1 Change log

=head2 0.13 (2026-06-11)

=head3 Fixed (Claude Opus 4.8 helped)

=over

=item * B C previously
computed the exit code (C<< $status EE 8 >>) and I derived the signal as
C<$exit & 127>. Because the signal lives in the low byte of the raw wait
status, which C<< EE 8 >> discards the C field was always wrong: a clean
C was reported as C, and a process actually killed by a
signal reported C. The signal is now read from the raw status before
shifting, so C and C are independent and accurate.

=item * B<< No longer dies on a missing output file when C<< die =E 0 >>. >> The zero-size
check did C<(-s $file) == 0>, which is C when a declared output file
is absent. Under C<< use warnings FATAL =E 'all' >> that "uninitialized value"
warning was fatal, so a task that was meant to I about missing output
(with C<< die =E 0 >>) crashed instead. Missing sizes are now treated as C<0>, so
the task warns and returns its result hash as intended.

=item * B<< The "already done" result is now logged with its C. >> In the
short-circuit path (output files already exist), C was set I
the record was written to the log, so the logged hash was missing it; the
duplicate C<< done =E 'before' >> assignment was also removed.

=back

=head3 Changed / Windows support

=over

=item * B Decoding now branches on C<$^O>: Windows has
no POSIX signals (C is reported as C<0> there), and a C that
fails to launch the command (C<-1>) yields C<< exit =E -1 >> instead of a garbage
value from shifting C<-1>.

=item * B C
output is suppressed on C unless an ANSI-capable terminal is detected
(Windows Terminal, ConEmu, or ANSICON), so C no longer prints raw
escape sequences and redirected logs stay clean. Unix and modern Windows
terminals are unaffected.

=back

=head3 Tests

=over

=item * Rewrote C to be cross-platform: shell commands now invoke the running
Perl interpreter (C<"$^X" -e ...>) instead of Unix-only tools (C, C,
C, C), and temp files use the system temp directory instead of a
hard-coded C.

=item * Added regression tests for both fixed bugs (exit/signal decoding; surviving a
missing output file with C<< die =E 0 >>).

=item * Added coverage for the C field, the C / C
hashes, scalar-vs-array normalisation of C / C, the
C / C / C metadata, captured C / C
(including trailing-whitespace stripping), and argument validation (missing
C, unknown keys, bad C, missing input files).

=back

=head2 0.12

exit code now matches what shell would show it as; signal now appears

=head2 0.11

max string length now corresponds to max of output strings, no more truncated output
added List::Util dependency for string length maxes
memory size now shows when output
directory is now output during dry runs