line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package HPC::Runner::Command::submit_jobs::Plugin::Slurm; |
2
|
|
|
|
|
|
|
|
3
|
1
|
|
|
1
|
|
738
|
use Moose::Role; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
5
|
|
4
|
1
|
|
|
1
|
|
4081
|
use namespace::autoclean; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
7
|
|
5
|
|
|
|
|
|
|
|
6
|
1
|
|
|
1
|
|
71
|
use File::Temp qw/ tempfile /; |
|
1
|
|
|
|
|
3
|
|
|
1
|
|
|
|
|
65
|
|
7
|
1
|
|
|
1
|
|
8
|
use File::Slurp; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
58
|
|
8
|
1
|
|
|
1
|
|
6
|
use File::Spec; |
|
1
|
|
|
|
|
3
|
|
|
1
|
|
|
|
|
373
|
|
9
|
|
|
|
|
|
|
|
10
|
|
|
|
|
|
|
with 'HPC::Runner::Command::submit_jobs::Plugin::Role::Log'; |
11
|
|
|
|
|
|
|
|
12
|
|
|
|
|
|
|
=head1 HPC::Runner::Command::Plugin::Scheduler::Slurm; |
13
|
|
|
|
|
|
|
|
14
|
|
|
|
|
|
|
Use the SLURM scheduler |
15
|
|
|
|
|
|
|
|
16
|
|
|
|
|
|
|
=cut |
17
|
|
|
|
|
|
|
|
18
|
|
|
|
|
|
|
has 'submit_command' => ( |
19
|
|
|
|
|
|
|
is => 'rw', |
20
|
|
|
|
|
|
|
isa => 'Str', |
21
|
|
|
|
|
|
|
default => 'sbatch', |
22
|
|
|
|
|
|
|
); |
23
|
|
|
|
|
|
|
|
24
|
|
|
|
|
|
|
|
25
|
|
|
|
|
|
|
=head2 Subroutines |
26
|
|
|
|
|
|
|
|
27
|
|
|
|
|
|
|
=cut |
28
|
|
|
|
|
|
|
|
29
|
|
|
|
|
|
|
=head3 submit_jobs |
30
|
|
|
|
|
|
|
|
31
|
|
|
|
|
|
|
Submit jobs to slurm queue using sbatch. |
32
|
|
|
|
|
|
|
|
33
|
|
|
|
|
|
|
Format is |
34
|
|
|
|
|
|
|
|
35
|
|
|
|
|
|
|
Submitted batch job <job_id> |
36
|
|
|
|
|
|
|
|
37
|
|
|
|
|
|
|
Where <job_id> is just only numeric |
38
|
|
|
|
|
|
|
=cut |
39
|
|
|
|
|
|
|
|
40
|
|
|
|
|
|
|
sub submit_jobs { |
41
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
42
|
|
|
|
|
|
|
|
43
|
0
|
|
|
|
|
|
my ( $exitcode, $stdout, $stderr ) = |
44
|
|
|
|
|
|
|
$self->submit_to_scheduler( |
45
|
|
|
|
|
|
|
$self->submit_command . " " . $self->slurmfile ); |
46
|
|
|
|
|
|
|
|
47
|
0
|
|
|
|
|
|
sleep(3); |
48
|
|
|
|
|
|
|
|
49
|
0
|
0
|
0
|
|
|
|
if ( ! defined $exitcode || $exitcode != 0 ) { |
50
|
0
|
|
|
|
|
|
$self->log->warn("Job was not submitted successfully"); |
51
|
0
|
0
|
|
|
|
|
$self->log->warn( "STDERR: " . $stderr ) if $stderr; |
52
|
0
|
0
|
|
|
|
|
$self->log->warn( "STDOUT: " . $stdout ) if $stdout; |
53
|
|
|
|
|
|
|
} |
54
|
|
|
|
|
|
|
|
55
|
0
|
0
|
|
|
|
|
my ($jobid) = $stdout =~ m/(\d.*)$/ if $stdout; |
56
|
|
|
|
|
|
|
|
57
|
0
|
0
|
|
|
|
|
if ( !$jobid ) { |
58
|
0
|
|
|
|
|
|
$self->job_failure; |
59
|
|
|
|
|
|
|
} |
60
|
|
|
|
|
|
|
else { |
61
|
0
|
|
|
|
|
|
$self->app_log->info( "Submitted job " |
62
|
|
|
|
|
|
|
. $self->slurmfile |
63
|
|
|
|
|
|
|
. "\n\tWith Slurm jobid $jobid" ); |
64
|
|
|
|
|
|
|
} |
65
|
|
|
|
|
|
|
|
66
|
0
|
|
|
|
|
|
return $jobid; |
67
|
|
|
|
|
|
|
} |
68
|
|
|
|
|
|
|
|
69
|
|
|
|
|
|
|
=head3 update_job_deps |
70
|
|
|
|
|
|
|
|
71
|
|
|
|
|
|
|
Update the job dependencies if using job_array (not batches) |
72
|
|
|
|
|
|
|
|
73
|
|
|
|
|
|
|
=cut |
74
|
|
|
|
|
|
|
|
75
|
|
|
|
|
|
|
sub update_job_deps { |
76
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
77
|
|
|
|
|
|
|
|
78
|
0
|
0
|
|
|
|
|
return unless $self->has_array_deps; |
79
|
|
|
|
|
|
|
|
80
|
0
|
|
|
|
|
|
my $array_deps_file = File::Spec->catdir( $self->logdir, 'array_deps.tsv' ); |
81
|
0
|
|
|
|
|
|
my $array_log_file = File::Spec->catdir( $self->logdir, 'array_deps.log' ); |
82
|
|
|
|
|
|
|
|
83
|
0
|
|
|
|
|
|
while ( my ( $current_task, $v ) = each %{ $self->array_deps } ) { |
|
0
|
|
|
|
|
|
|
84
|
0
|
|
|
|
|
|
my $dep_tasks = join( ':', @{$v} ); |
|
0
|
|
|
|
|
|
|
85
|
0
|
|
|
|
|
|
my $cmd = |
86
|
|
|
|
|
|
|
"scontrol update job=$current_task depend=afterok:$dep_tasks"; |
87
|
|
|
|
|
|
|
|
88
|
0
|
|
|
|
|
|
my ( $exitcode, $stdout, $stderr ) = $self->submit_to_scheduler($cmd); |
89
|
0
|
|
|
|
|
|
write_file( |
90
|
|
|
|
|
|
|
$array_deps_file, |
91
|
|
|
|
|
|
|
{ append => 1 }, |
92
|
|
|
|
|
|
|
$current_task . "\t" . $dep_tasks . "\n" |
93
|
|
|
|
|
|
|
); |
94
|
|
|
|
|
|
|
|
95
|
0
|
|
|
|
|
|
my $info = |
96
|
|
|
|
|
|
|
"Task Deps:\t" |
97
|
|
|
|
|
|
|
. $current_task . "\t" |
98
|
|
|
|
|
|
|
. $dep_tasks . "\n" |
99
|
|
|
|
|
|
|
. "ExitCode: $exitcode\n"; |
100
|
0
|
0
|
|
|
|
|
$info .= "Stderr: $stderr\n" if $stderr; |
101
|
0
|
0
|
|
|
|
|
$info .= "Stdout: $stdout\n" if $stdout; |
102
|
|
|
|
|
|
|
|
103
|
0
|
|
|
|
|
|
write_file( $array_log_file, {append => 1}, $info ); |
104
|
|
|
|
|
|
|
} |
105
|
|
|
|
|
|
|
} |
106
|
|
|
|
|
|
|
|
107
|
|
|
|
|
|
|
1; |