SLURM/Run.pm untested

filter: Statementstmt 0.0% 0 / 115 Branchbran 0.0% 0 / 32 Conditioncond 0.0% 0 / 11 MC/DCmcdc 0.0% 0 / 7 Subroutinesub 0.0% 0 / 14 Podpod 0.0% 0 / 9 totaltotal 0.0% 0 / 188 CC 14 SCAR 25.8
CC
14
Cov
0%
_to_MB
30.0
_reformat_time
20.0
_collect_sacct_info
12.0
CRAP 13.2
1 package HPCD::SLURM::Run;
2  
3 =head1 NAME
4  
5 HPCD::SLURM::Run
6  
7 =head1 SYNOPSIS
8  
9 use HPCD::SLURM::Run;
10  
11 =head1 DESCRIPTION
12  
13 This module helps execute srun, scancel and sacct.
14 srun: This module puts together user input for stage attributes and submits the srun command for
15 the system to run.
16 sacct: This module contains a method to parse account information given
17 back by the sacct command.
18 scancel: This module executes the scancel command when the job needs to be killed.
19  
20 =cut
21  
22 ### INCLUDES ######################################################################################
23  
24 # safe Perl
25 use warnings;
26 use strict;
27 use Carp;
28  
29 use Moose;
30 use Config;
31  
32 my %sig_num;
33 my @sig_name;
34  
35 @sig_num{split ' ', $Config{sig_name}} = split ' ', $Config{sig_num};
36 while (my ($k, $v) = each %sig_num) {
37 $sig_name[$v] ||= $k;
38 }
39  
40 sub _register_status {
41 }
42  
43 sub _collect_job_stats {
44 }
45  
46 with 'HPCI::Run' => { theDriver => 'HPCD::SLURM' };
47  
48 # %job_info_mappings is a hash table that maps account information keys used by SLURM to more
49 # human readable names
50 my %job_info_mappings = (
51 MaxRSS => 'peak_real_memory',
52 MaxVMSize => 'peak_virtual_memory'
53 );
54  
55 # @job_info_fields contains a list of account information keys that can be obtained by the
56 # "sacct -e" command
57 my @job_info_fields = map { m/\S+/g } `sacct -e`;
58  
59 =head1 ATTRIBUTE
60  
61 =head2 stats_keys
62  
63 stats_keys record the names of the account information keys. Default to @job_info_fields.
64  
65 =cut
66  
67 has 'stats_keys' => (
68 is => 'ro',
69 isa => 'ArrayRef[Str]',
70 init_arg => undef,
71 default => sub { [ values @job_info_fields ] }
72 );
73  
74 =head1 METHODS
75  
76 =head2 after '_collect_job_stats'
77  
78 Calls the subroutine _collect_sacct_info, updates the hash in the stats attribute
79 with the exit status and accounting info.
80  
81 =cut
82  
83 after '_collect_job_stats' => sub {
84 my $self = shift;
85 my $status = shift;
86  
87 my $stath = $self->stats;
88  
89 my $stage = $self->stage;
90 my $id = $self->unique_id;
91  
92 $stath->{exit_status} = $self->status;
93 $stath->{aborted} = $self->abort if $self->abort;
94 $stath->{internal_killed} = $self->killsignal if $self->kill;
95  
96 $stage->info("collecting job stats ($id)\n");
97 my $sacct_info = $self->_collect_sacct_info;
98 while (my ( $k, $v ) = each %$sacct_info) {
99 $k = $job_info_mappings{$k} // $k;
100 $stath->{$k} = $v;
101 }
102 $self->_base_register_status( $stath->{ExitCode} );
103 $stath->{killed} = $self->killsignal if $self->killsignal;
104 };
105  
106 =head2 _collect_sacct_info
107  
108 Executes the command 'sacct --name $id --format=$key' n times, where $key is a singular key
109 returned by sacct -e and n is the number of keys returned by sacct -e. Maps $key and the
110 returned account information value in hash %info and returns %info as the result.
111  
112 The reason why sacct is called multiple times instead of once (by sacct -name $id --format=
113 $key1,$key2,$key3...) is that sometimes the value field might be blank, e.g. the result of
114 the command sacct -name $id --format=Account,User,Comment,ReqMem might be
115  
116 Account User Comment ReqMem
117 ----------------------------------
118 jdoe 2Gn
119  
120 It is thus difficult to parse the values and map them with corresponding keys. By calling
121 --format=$key separately for each $key value each time, we can catch all the blank values and
122 ensure that the key-value pairs are matched correctly.
123  
124 =cut
125  
126 sub _collect_sacct_info {
127 my $self = shift;
128 my $id = $self->unique_id;
129 my %info = ();
130 my $stage = $self->stage;
131 foreach my $key (@job_info_fields) {
132 open my $fh, '-|', "sacct --name $id -n --format=$key 2>&1"
133 or $self->croak("cannot open sacct command: $!");
134 while (defined(my $val = <$fh>)) {
135 $val =~ s/^\s*//;
136 $val =~ s/\s*$//;
137 # The number1:number2 formatted string returned by sacct for ExitCode has number1
138 # as the exit status and number2 as the signal value.
139 if ($key eq "ExitCode") {
140 my( $exit, $kill ) = $val =~ m/(.*):(.*)/;
141 $val = $exit;
142 if ($kill) {
143 $val += 128;
144 $self->kill( 1 );
145 $self->killsignal( $sig_name[$kill] // "Unknown signal #".($kill) );
146 }
147 }
148 $info{$key} = $val;
149 }
150 }
151 return \%info;
152 }
153  
154 =head2 around 'soft_timeout'
155  
156 Replaces the original 'soft_timeout' method in HPCI, and cancels the job directly.
157  
158 =cut
159  
160 around 'soft_timeout' => sub {
161 my $orig = shift;
162 my $self = shift;
163 $self->_delete_job;
164 };
165  
166 =head2 around 'hard_timeout'
167  
168 Cancels the job before calling the original HPCI method 'hard_timeout'.
169  
170 The original hard_timeout sends a kill signal to the child process. In this case,
171 that is the "srun" program, not the actual child job (which is on some other
172 computer so kill cannot be used). The sleep and continue with sending the kill
173 signal at least cleans up the local process if the cancellation does not work
174 properly. Usually it will, and the kill will be sent to a process that has terminated
175 already.
176  
177 =cut
178  
179 around 'hard_timeout' => sub {
180 my $orig = shift;
181 my $self = shift;
182 $self->_delete_job;
183 sleep(2);
184 $self->$orig(@_);
185 };
186  
187 =head2 _delete_job
188  
189 Terminates the job by calling scancel -n $id.
190  
191 =cut
192  
193 sub _delete_job {
194 my $self = shift;
195 my $id = $self->unique_id;
196 $self->info( "Running: (scancel -n $id) to terminate stage job" );
197 system( "scancel -n $id" );
198 }
199  
200 =head2 _to_MB
201  
202 A subroutine which converts any memory value in unit KMGT to a number in MB,
203 since the srun --mem= option takes only a number which by default is in MB.
204  
205 Example:
206 $self->_to_MB('2G') would return 2048
207 $self->_to_MB('100M') would return 100
208  
209 =cut
210  
211 sub _to_MB {
212 my $self = shift;
213 my $val = shift;
214 return ($1 * 1024 * 1024)
215 if $val =~ /^(\d+)T$/;
216 return ($1 * 1024)
217 if $val =~ /^(\d+)G$/;
218 return ($1)
219 if $val =~ /^(\d+)M$/;
220 return ($1 / 1024)
221 if $val =~ /^(\d+)K$/;
222 }
223  
224 =head2 _reformat_time
225  
226 A subroutine which reformats the input $sec (a number in seconds) to either
227 minute:second, hour:minute:second, or day-hour:minute:second, which are the
228 formats acceptable by the srun --time= option.
229  
230 Example:
231 $self->_reformat_time(1) would give '0:1'
232 $self->_reformat_time(70) would give '1:10'
233 $self->_reformat_time(3601) would give '1:0:1'
234 $self->_reformat_time(86400) would give '1-0:0:0'
235  
236 =cut
237  
238 sub _reformat_time {
239 my $self = shift;
240 my $sec = shift;
241 return "0:$sec" if $sec < 60;
242 my $min = int $sec / 60;
243 $sec %= 60;
244 return "$min:$sec" if $min < 60;
245 my $hour = int $min / 60;
246 $min %= 60;
247 return "$hour:$min:$sec" if $hour < 24;
248 my $day = int $hour / 24;
249 $hour %= 24;
250 return "$day-$hour:$min:$sec";
251 }
252  
253 =head2 _res_value_map
254  
255 A subroutine which reformats key and value in stage attribute resources_required
256 to the option format acceptable by srun.
257  
258 Example:
259 If the key and value in resources_required is 'mem' and '3G', then
260 _res_value_map would give '--mem=3072' as the output.
261  
262 =cut
263  
264 sub _res_value_map {
265 my $self = shift;
266 my $key = shift;
267 my $val = shift;
268 $val = $self->_reformat_time($self->stage->_time_to_secs( $val ))
269 if $key eq 'time';
270 $val = $self->_to_MB( $val )
271 if $key eq 'mem';
272 return "--$key=$val";
273 }
274  
275 =head2 _get_mapped_resources_string
276  
277 A subroutine which maps parameters in resources_required to a string of srun
278 options.
279  
280 Example:
281 Say resources_required is {"mem" => "100M", "h_time" => 1000}, then the output
282 will be '--mem=100 --time=16:40'.
283  
284 =cut
285  
286 sub _get_mapped_resource_string {
287 my $self = shift;
288 my $res = shift;
289 join( " ",
290 map { $self->_res_value_map( $_, $res->{$_} ) }
291 sort keys %$res
292 );
293 }
294  
295 =head2 _get_submit_command
296  
297 A subroutine which incorporates attributes of one certain stage (i.e. shell_script,
298 unique name, stdout, stderr, native_args_string, resrouces_required) into one single
299 srun command for the system to execute.
300  
301 Example:
302 If the stage has its script_file named script.sh, unique_id being NAME12345,
303 native_args_string being "-N 2 -n 4 --mail-type=ALL --mail-user=jdoe@xyz.com",
304 resources_required being {"mem" => "5G", "h_time" => 200}, then the output of this
305 subroutine would be 'srun -N 2 -n 4 --mail-type=ALL --mail-user=jdoe@xyz.com
306 --mem=5120 --time=3:20 -J NAME12345 -o someoutputpath -e someerrorpath script.sh'.
307  
308 =cut
309  
310 sub _get_submit_command {
311 my $self = shift;
312 my $stage = $self->stage;
313 my $shell_script = $stage->script_file;
314 my $name = $self->unique_id;
315 my $extras = $stage->native_args_string;
316 my $res = $stage->_use_resources_required;
317 my $resources = $self->_get_mapped_resource_string( $res );
318  
319 # Note how extras are positioned before resources: so that if a conflict arises
320 # (in the case when the user accidentally define time/memory limit in native_args_string
321 # as well), SLURM will adopt the ones defined in resources because resources appear later
322 my $srun_command = join( ' ',
323 "srun",
324 ( length($extras) ? $extras : () ),
325 ( length($resources) ? $resources : () ),
326 "-J '$name'",
327 "-o", $self->_stdout,
328 "-e", $self->_stderr,
329 "'$shell_script'" );
330  
331 return $srun_command;
332 }
333  
334 =head1 AUTHOR
335  
336 John Macdonald - Boutros Lab
337  
338 Anqi (Joyce) Yang - Boutros Lab
339  
340 =head1 ACKNOWLEDGEMENTS
341  
342 Paul Boutros, Phd, PI - Boutros Lab
343  
344 The Ontario Institute for Cancer Research
345  
346 =cut
347  
348 1;