| 1 | package HPCD::SLURM::Run; | ||
| 2 | |||
| 3 | =head1 NAME | ||
| 4 | |||
| 5 | HPCD::SLURM::Run | ||
| 6 | |||
| 7 | =head1 SYNOPSIS | ||
| 8 | |||
| 9 | use HPCD::SLURM::Run; | ||
| 10 | |||
| 11 | =head1 DESCRIPTION | ||
| 12 | |||
| 13 | This module helps execute srun, scancel and sacct. | ||
| 14 | srun: This module puts together user input for stage attributes and submits the srun command for | ||
| 15 | the system to run. | ||
| 16 | sacct: This module contains a method to parse account information given | ||
| 17 | back by the sacct command. | ||
| 18 | scancel: This module executes the scancel command when the job needs to be killed. | ||
| 19 | |||
| 20 | =cut | ||
| 21 | |||
| 22 | ### INCLUDES ###################################################################################### | ||
| 23 | |||
| 24 | # safe Perl | ||
| 25 | use warnings; | ||
| 26 | use strict; | ||
| 27 | use Carp; | ||
| 28 | |||
| 29 | use Moose; | ||
| 30 | use Config; | ||
| 31 | |||
| 32 | my %sig_num; | ||
| 33 | my @sig_name; | ||
| 34 | |||
| 35 | @sig_num{split ' ', $Config{sig_name}} = split ' ', $Config{sig_num}; | ||
| 36 | while (my ($k, $v) = each %sig_num) { | ||
| 37 | $sig_name[$v] ||= $k; | ||
| 38 | } | ||
| 39 | |||
| 40 | sub _register_status { | ||
| 41 | } | ||
| 42 | |||
| 43 | sub _collect_job_stats { | ||
| 44 | } | ||
| 45 | |||
| 46 | with 'HPCI::Run' => { theDriver => 'HPCD::SLURM' }; | ||
| 47 | |||
| 48 | # %job_info_mappings is a hash table that maps account information keys used by SLURM to more | ||
| 49 | # human readable names | ||
| 50 | my %job_info_mappings = ( | ||
| 51 | MaxRSS => 'peak_real_memory', | ||
| 52 | MaxVMSize => 'peak_virtual_memory' | ||
| 53 | ); | ||
| 54 | |||
| 55 | # @job_info_fields contains a list of account information keys that can be obtained by the | ||
| 56 | # "sacct -e" command | ||
| 57 | my @job_info_fields = map { m/\S+/g } `sacct -e`; | ||
| 58 | |||
| 59 | =head1 ATTRIBUTE | ||
| 60 | |||
| 61 | =head2 stats_keys | ||
| 62 | |||
| 63 | stats_keys record the names of the account information keys. Default to @job_info_fields. | ||
| 64 | |||
| 65 | =cut | ||
| 66 | |||
| 67 | has 'stats_keys' => ( | ||
| 68 | is => 'ro', | ||
| 69 | isa => 'ArrayRef[Str]', | ||
| 70 | init_arg => undef, | ||
| 71 | default => sub { [ values @job_info_fields ] } | ||
| 72 | ); | ||
| 73 | |||
| 74 | =head1 METHODS | ||
| 75 | |||
| 76 | =head2 after '_collect_job_stats' | ||
| 77 | |||
| 78 | Calls the subroutine _collect_sacct_info, updates the hash in the stats attribute | ||
| 79 | with the exit status and accounting info. | ||
| 80 | |||
| 81 | =cut | ||
| 82 | |||
| 83 | after '_collect_job_stats' => sub { | ||
| 84 | my $self = shift; | ||
| 85 | my $status = shift; | ||
| 86 | |||
| 87 | my $stath = $self->stats; | ||
| 88 | |||
| 89 | my $stage = $self->stage; | ||
| 90 | my $id = $self->unique_id; | ||
| 91 | |||
| 92 | $stath->{exit_status} = $self->status; | ||
| 93 | $stath->{aborted} = $self->abort if $self->abort; | ||
| 94 | $stath->{internal_killed} = $self->killsignal if $self->kill; | ||
| 95 | |||
| 96 | $stage->info("collecting job stats ($id)\n"); | ||
| 97 | my $sacct_info = $self->_collect_sacct_info; | ||
| 98 | while (my ( $k, $v ) = each %$sacct_info) { | ||
| 99 | $k = $job_info_mappings{$k} // $k; | ||
| 100 | $stath->{$k} = $v; | ||
| 101 | } | ||
| 102 | $self->_base_register_status( $stath->{ExitCode} ); | ||
| 103 | $stath->{killed} = $self->killsignal if $self->killsignal; | ||
| 104 | }; | ||
| 105 | |||
| 106 | =head2 _collect_sacct_info | ||
| 107 | |||
| 108 | Executes the command 'sacct --name $id --format=$key' n times, where $key is a singular key | ||
| 109 | returned by sacct -e and n is the number of keys returned by sacct -e. Maps $key and the | ||
| 110 | returned account information value in hash %info and returns %info as the result. | ||
| 111 | |||
| 112 | The reason why sacct is called multiple times instead of once (by sacct -name $id --format= | ||
| 113 | $key1,$key2,$key3...) is that sometimes the value field might be blank, e.g. the result of | ||
| 114 | the command sacct -name $id --format=Account,User,Comment,ReqMem might be | ||
| 115 | |||
| 116 | Account User Comment ReqMem | ||
| 117 | ---------------------------------- | ||
| 118 | jdoe 2Gn | ||
| 119 | |||
| 120 | It is thus difficult to parse the values and map them with corresponding keys. By calling | ||
| 121 | --format=$key separately for each $key value each time, we can catch all the blank values and | ||
| 122 | ensure that the key-value pairs are matched correctly. | ||
| 123 | |||
| 124 | =cut | ||
| 125 | |||
| 126 | sub _collect_sacct_info { | ||
| 127 | my $self = shift; | ||
| 128 | my $id = $self->unique_id; | ||
| 129 | my %info = (); | ||
| 130 | my $stage = $self->stage; | ||
| 131 | foreach my $key (@job_info_fields) { | ||
| 132 | open my $fh, '-|', "sacct --name $id -n --format=$key 2>&1" | ||
| 133 | or $self->croak("cannot open sacct command: $!"); | ||
| 134 | while (defined(my $val = <$fh>)) { | ||
| 135 | $val =~ s/^\s*//; | ||
| 136 | $val =~ s/\s*$//; | ||
| 137 | # The number1:number2 formatted string returned by sacct for ExitCode has number1 | ||
| 138 | # as the exit status and number2 as the signal value. | ||
| 139 | if ($key eq "ExitCode") { | ||
| 140 | my( $exit, $kill ) = $val =~ m/(.*):(.*)/; | ||
| 141 | $val = $exit; | ||
| 142 | if ($kill) { | ||
| 143 | $val += 128; | ||
| 144 | $self->kill( 1 ); | ||
| 145 | $self->killsignal( $sig_name[$kill] // "Unknown signal #".($kill) ); | ||
| 146 | } | ||
| 147 | } | ||
| 148 | $info{$key} = $val; | ||
| 149 | } | ||
| 150 | } | ||
| 151 | return \%info; | ||
| 152 | } | ||
| 153 | |||
| 154 | =head2 around 'soft_timeout' | ||
| 155 | |||
| 156 | Replaces the original 'soft_timeout' method in HPCI, and cancels the job directly. | ||
| 157 | |||
| 158 | =cut | ||
| 159 | |||
| 160 | around 'soft_timeout' => sub { | ||
| 161 | my $orig = shift; | ||
| 162 | my $self = shift; | ||
| 163 | $self->_delete_job; | ||
| 164 | }; | ||
| 165 | |||
| 166 | =head2 around 'hard_timeout' | ||
| 167 | |||
| 168 | Cancels the job before calling the original HPCI method 'hard_timeout'. | ||
| 169 | |||
| 170 | The original hard_timeout sends a kill signal to the child process. In this case, | ||
| 171 | that is the "srun" program, not the actual child job (which is on some other | ||
| 172 | computer so kill cannot be used). The sleep and continue with sending the kill | ||
| 173 | signal at least cleans up the local process if the cancellation does not work | ||
| 174 | properly. Usually it will, and the kill will be sent to a process that has terminated | ||
| 175 | already. | ||
| 176 | |||
| 177 | =cut | ||
| 178 | |||
| 179 | around 'hard_timeout' => sub { | ||
| 180 | my $orig = shift; | ||
| 181 | my $self = shift; | ||
| 182 | $self->_delete_job; | ||
| 183 | sleep(2); | ||
| 184 | $self->$orig(@_); | ||
| 185 | }; | ||
| 186 | |||
| 187 | =head2 _delete_job | ||
| 188 | |||
| 189 | Terminates the job by calling scancel -n $id. | ||
| 190 | |||
| 191 | =cut | ||
| 192 | |||
| 193 | sub _delete_job { | ||
| 194 | my $self = shift; | ||
| 195 | my $id = $self->unique_id; | ||
| 196 | $self->info( "Running: (scancel -n $id) to terminate stage job" ); | ||
| 197 | system( "scancel -n $id" ); | ||
| 198 | } | ||
| 199 | |||
| 200 | =head2 _to_MB | ||
| 201 | |||
| 202 | A subroutine which converts any memory value in unit KMGT to a number in MB, | ||
| 203 | since the srun --mem= option takes only a number which by default is in MB. | ||
| 204 | |||
| 205 | Example: | ||
| 206 | $self->_to_MB('2G') would return 2048 | ||
| 207 | $self->_to_MB('100M') would return 100 | ||
| 208 | |||
| 209 | =cut | ||
| 210 | |||
| 211 | sub _to_MB { | ||
| 212 | my $self = shift; | ||
| 213 | my $val = shift; | ||
| 214 | return ($1 * 1024 * 1024) | ||
| 215 | if $val =~ /^(\d+)T$/; | ||
| 216 | return ($1 * 1024) | ||
| 217 | if $val =~ /^(\d+)G$/; | ||
| 218 | return ($1) | ||
| 219 | if $val =~ /^(\d+)M$/; | ||
| 220 | return ($1 / 1024) | ||
| 221 | if $val =~ /^(\d+)K$/; | ||
| 222 | } | ||
| 223 | |||
| 224 | =head2 _reformat_time | ||
| 225 | |||
| 226 | A subroutine which reformats the input $sec (a number in seconds) to either | ||
| 227 | minute:second, hour:minute:second, or day-hour:minute:second, which are the | ||
| 228 | formats acceptable by the srun --time= option. | ||
| 229 | |||
| 230 | Example: | ||
| 231 | $self->_reformat_time(1) would give '0:1' | ||
| 232 | $self->_reformat_time(70) would give '1:10' | ||
| 233 | $self->_reformat_time(3601) would give '1:0:1' | ||
| 234 | $self->_reformat_time(86400) would give '1-0:0:0' | ||
| 235 | |||
| 236 | =cut | ||
| 237 | |||
| 238 | sub _reformat_time { | ||
| 239 | my $self = shift; | ||
| 240 | my $sec = shift; | ||
| 241 | return "0:$sec" if $sec < 60; | ||
| 242 | my $min = int $sec / 60; | ||
| 243 | $sec %= 60; | ||
| 244 | return "$min:$sec" if $min < 60; | ||
| 245 | my $hour = int $min / 60; | ||
| 246 | $min %= 60; | ||
| 247 | return "$hour:$min:$sec" if $hour < 24; | ||
| 248 | my $day = int $hour / 24; | ||
| 249 | $hour %= 24; | ||
| 250 | return "$day-$hour:$min:$sec"; | ||
| 251 | } | ||
| 252 | |||
| 253 | =head2 _res_value_map | ||
| 254 | |||
| 255 | A subroutine which reformats key and value in stage attribute resources_required | ||
| 256 | to the option format acceptable by srun. | ||
| 257 | |||
| 258 | Example: | ||
| 259 | If the key and value in resources_required is 'mem' and '3G', then | ||
| 260 | _res_value_map would give '--mem=3072' as the output. | ||
| 261 | |||
| 262 | =cut | ||
| 263 | |||
| 264 | sub _res_value_map { | ||
| 265 | my $self = shift; | ||
| 266 | my $key = shift; | ||
| 267 | my $val = shift; | ||
| 268 | $val = $self->_reformat_time($self->stage->_time_to_secs( $val )) | ||
| 269 | if $key eq 'time'; | ||
| 270 | $val = $self->_to_MB( $val ) | ||
| 271 | if $key eq 'mem'; | ||
| 272 | return "--$key=$val"; | ||
| 273 | } | ||
| 274 | |||
| 275 | =head2 _get_mapped_resources_string | ||
| 276 | |||
| 277 | A subroutine which maps parameters in resources_required to a string of srun | ||
| 278 | options. | ||
| 279 | |||
| 280 | Example: | ||
| 281 | Say resources_required is {"mem" => "100M", "h_time" => 1000}, then the output | ||
| 282 | will be '--mem=100 --time=16:40'. | ||
| 283 | |||
| 284 | =cut | ||
| 285 | |||
| 286 | sub _get_mapped_resource_string { | ||
| 287 | my $self = shift; | ||
| 288 | my $res = shift; | ||
| 289 | join( " ", | ||
| 290 | map { $self->_res_value_map( $_, $res->{$_} ) } | ||
| 291 | sort keys %$res | ||
| 292 | ); | ||
| 293 | } | ||
| 294 | |||
| 295 | =head2 _get_submit_command | ||
| 296 | |||
| 297 | A subroutine which incorporates attributes of one certain stage (i.e. shell_script, | ||
| 298 | unique name, stdout, stderr, native_args_string, resrouces_required) into one single | ||
| 299 | srun command for the system to execute. | ||
| 300 | |||
| 301 | Example: | ||
| 302 | If the stage has its script_file named script.sh, unique_id being NAME12345, | ||
| 303 | native_args_string being "-N 2 -n 4 --mail-type=ALL --mail-user=jdoe@xyz.com", | ||
| 304 | resources_required being {"mem" => "5G", "h_time" => 200}, then the output of this | ||
| 305 | subroutine would be 'srun -N 2 -n 4 --mail-type=ALL --mail-user=jdoe@xyz.com | ||
| 306 | --mem=5120 --time=3:20 -J NAME12345 -o someoutputpath -e someerrorpath script.sh'. | ||
| 307 | |||
| 308 | =cut | ||
| 309 | |||
| 310 | sub _get_submit_command { | ||
| 311 | my $self = shift; | ||
| 312 | my $stage = $self->stage; | ||
| 313 | my $shell_script = $stage->script_file; | ||
| 314 | my $name = $self->unique_id; | ||
| 315 | my $extras = $stage->native_args_string; | ||
| 316 | my $res = $stage->_use_resources_required; | ||
| 317 | my $resources = $self->_get_mapped_resource_string( $res ); | ||
| 318 | |||
| 319 | # Note how extras are positioned before resources: so that if a conflict arises | ||
| 320 | # (in the case when the user accidentally define time/memory limit in native_args_string | ||
| 321 | # as well), SLURM will adopt the ones defined in resources because resources appear later | ||
| 322 | my $srun_command = join( ' ', | ||
| 323 | "srun", | ||
| 324 | ( length($extras) ? $extras : () ), | ||
| 325 | ( length($resources) ? $resources : () ), | ||
| 326 | "-J '$name'", | ||
| 327 | "-o", $self->_stdout, | ||
| 328 | "-e", $self->_stderr, | ||
| 329 | "'$shell_script'" ); | ||
| 330 | |||
| 331 | return $srun_command; | ||
| 332 | } | ||
| 333 | |||
| 334 | =head1 AUTHOR | ||
| 335 | |||
| 336 | John Macdonald - Boutros Lab | ||
| 337 | |||
| 338 | Anqi (Joyce) Yang - Boutros Lab | ||
| 339 | |||
| 340 | =head1 ACKNOWLEDGEMENTS | ||
| 341 | |||
| 342 | Paul Boutros, Phd, PI - Boutros Lab | ||
| 343 | |||
| 344 | The Ontario Institute for Cancer Research | ||
| 345 | |||
| 346 | =cut | ||
| 347 | |||
| 348 | 1; |