| 1 | package HPCD::SLURM::Stage; | ||
| 2 | |||
| 3 | =head1 NAME | ||
| 4 | |||
| 5 | HPCD::SLURM::Stage | ||
| 6 | |||
| 7 | =head1 SYNOPSIS | ||
| 8 | |||
| 9 | use HPCD::SLURM::Stage; | ||
| 10 | |||
| 11 | =head1 DESCRIPTION | ||
| 12 | |||
| 13 | This module defines SLURM-specific stage attributes and contains the method | ||
| 14 | that decides if the program has failed due to memory shortage, and if so if | ||
| 15 | the program can be retried with a larger memory limit. | ||
| 16 | |||
| 17 | =cut | ||
| 18 | |||
| 19 | ### INCLUDES ################################################################## | ||
| 20 | |||
| 21 | # safe Perl | ||
| 22 | use warnings; | ||
| 23 | use strict; | ||
| 24 | use Carp; | ||
| 25 | use Data::Dumper; | ||
| 26 | use DateTime; | ||
| 27 | |||
| 28 | use Moose; | ||
| 29 | use Moose::Util::TypeConstraints; | ||
| 30 | use MooseX::ClassAttribute; | ||
| 31 | |||
| 32 | use HPCD::SLURM::Run; | ||
| 33 | |||
| 34 | with 'HPCI::Stage', | ||
| 35 | HPCI::get_extra_roles('SLURM', 'stage'); | ||
| 36 | |||
| 37 | =head1 BUILDERS | ||
| 38 | |||
| 39 | =head2 _build_cluster_specific_valid_resources | ||
| 40 | |||
| 41 | The only two resources user can define via user_input are (hard) time and | ||
| 42 | (hard) memory. | ||
| 43 | |||
| 44 | =cut | ||
| 45 | |||
| 46 | sub _build_cluster_specific_valid_resources { | ||
| 47 | return ( | ||
| 48 | mem => 'mem', | ||
| 49 | time => 'h_time' | ||
| 50 | ) | ||
| 51 | } | ||
| 52 | |||
| 53 | =head2 _build_cluster_specific_default_resources | ||
| 54 | |||
| 55 | The default memory limit is 2G, and default time limit is there is no limit. | ||
| 56 | |||
| 57 | =cut | ||
| 58 | |||
| 59 | sub _build_cluster_specific_default_resources { | ||
| 60 | return ( mem => '2G' ) | ||
| 61 | }; | ||
| 62 | |||
| 63 | =head2 _build_cluster_specific_default_retry_resources | ||
| 64 | |||
| 65 | A list of possible retry memory limits (when a program fails because of lack | ||
| 66 | of memory it will look through this list to find and retry with the next biggest | ||
| 67 | memory available). Should be in ascending order. Default is qw(2G 4G 16G 32G). | ||
| 68 | |||
| 69 | =cut | ||
| 70 | |||
| 71 | sub _build_cluster_specific_default_retry_resources { | ||
| 72 | mem => [qw(2G 4G 8G 16G 32G)] | ||
| 73 | }; | ||
| 74 | |||
| 75 | =head1 ATTRIBUTES | ||
| 76 | |||
| 77 | =cut | ||
| 78 | |||
| 79 | has '+runs' => ( | ||
| 80 | isa => 'ArrayRef[HPCD::SLURM::Run]' | ||
| 81 | ); | ||
| 82 | |||
| 83 | has '_run_class' => ( | ||
| 84 | is => 'ro', | ||
| 85 | isa => 'Str', | ||
| 86 | init_arg => undef, | ||
| 87 | default => "HPCD::SLURM::Run" | ||
| 88 | ); | ||
| 89 | |||
| 90 | around '_run_args' => sub { | ||
| 91 | my $orig = shift; | ||
| 92 | my $self = shift; | ||
| 93 | return ( | ||
| 94 | ( | ||
| 95 | $self->$orig(), resources_required => $self->_use_resources_required | ||
| 96 | ) | ||
| 97 | ); | ||
| 98 | }; | ||
| 99 | |||
| 100 | has 'native_args_string' => ( | ||
| 101 | is => 'ro', | ||
| 102 | isa => 'Str', | ||
| 103 | default => '' | ||
| 104 | ); | ||
| 105 | |||
| 106 | has 'memory_too_small' => ( | ||
| 107 | is => 'ro', | ||
| 108 | isa => 'CodeRef', | ||
| 109 | default => sub { return sub { return 0 } } | ||
| 110 | ); | ||
| 111 | |||
| 112 | =head1 METHODS | ||
| 113 | |||
| 114 | =head2 _analyse_completion_state | ||
| 115 | |||
| 116 | Analyzes and returns the completion state. | ||
| 117 | Exit status 0 => pass, otherwise, if the program fails due to lack of memory and | ||
| 118 | the program can be retried with the next biggest memory, then returns retry, else | ||
| 119 | returns fail. | ||
| 120 | |||
| 121 | =cut | ||
| 122 | |||
| 123 | sub _analyse_completion_state { | ||
| 124 | my $self = shift; | ||
| 125 | my $run = shift; | ||
| 126 | my $stats = $run->stats; | ||
| 127 | $self->debug( "Stats from finished stage(" . $self->name . "): " . Dumper($stats) ); | ||
| 128 | my $vmem_retry = $self->_vmem_usage( $run ); | ||
| 129 | my $new_state = | ||
| 130 | 0 == $stats->{ExitCode} ? 'pass' | ||
| 131 | : $vmem_retry ? 'retry' | ||
| 132 | : 'fail'; | ||
| 133 | $self->_set_state($new_state); | ||
| 134 | } | ||
| 135 | |||
| 136 | =head2 _can_retry_from_vmem | ||
| 137 | |||
| 138 | Searches for the next biggest memory in _use_retry_resources_required for retry. | ||
| 139 | |||
| 140 | =cut | ||
| 141 | |||
| 142 | sub _can_retry_from_vmem { | ||
| 143 | my ($self, $res, $dres) = @_; | ||
| 144 | return 0 | ||
| 145 | unless $self->_use_retry_resources_required | ||
| 146 | and my $res_avail = $self->_use_retry_resources_required->{mem}; | ||
| 147 | for my $nres (@$res_avail) { | ||
| 148 | $self->info( "Considering ($nres) as resource replacement for ($dres)" ); | ||
| 149 | next unless $res < _res_to_num($nres); | ||
| 150 | $self->info( "Accepted ($nres) as resource replacement for ($dres)" ); | ||
| 151 | $self->_use_resources_required->{mem} = $nres; | ||
| 152 | return 1; | ||
| 153 | } | ||
| 154 | $self->info( "No resource replacement found for ($dres)" ); | ||
| 155 | return 0; | ||
| 156 | } | ||
| 157 | |||
| 158 | =head2 _res_to_num | ||
| 159 | |||
| 160 | Takes in a memory size (units: KMG or none(no unit means the unit is byte already)) | ||
| 161 | and converts it to a number representing the original size in bytes. | ||
| 162 | |||
| 163 | Example: | ||
| 164 | &_res_to_num('1024') returns 1024 (because 1024 bytes = 1024 bytes) | ||
| 165 | &_res_to_num('2K') returns 2048 (because 2KB = 2048 bytes) | ||
| 166 | &_res_to_num('1M') returns 1048576 (because 1MB = 1048576 bytes) | ||
| 167 | |||
| 168 | =cut | ||
| 169 | |||
| 170 | sub _res_to_num { | ||
| 171 | my $str = shift; | ||
| 172 | return 0 unless my ($val, $unit) = ($str =~ /^(\d+(?:\.\d+)?)([KMG])?B?$/); | ||
| 173 | if ($unit) { | ||
| 174 | $unit = uc $unit; | ||
| 175 | if ($unit eq 'K') { | ||
| 176 | return $val * 1024; | ||
| 177 | } | ||
| 178 | elsif ($unit eq 'M') { | ||
| 179 | return $val * 1024 * 1024; | ||
| 180 | } | ||
| 181 | elsif ($unit eq 'G') { | ||
| 182 | return $val * 1024 * 1024 * 1024; | ||
| 183 | } | ||
| 184 | else { | ||
| 185 | return 0; | ||
| 186 | } | ||
| 187 | } | ||
| 188 | return $val; | ||
| 189 | } | ||
| 190 | |||
| 191 | =head2 _num_to_res | ||
| 192 | |||
| 193 | Takes in a number in bytes and converts it into designated units (K/M/G). | ||
| 194 | |||
| 195 | Example: | ||
| 196 | &_num_to_res(2048) returns '2K' | ||
| 197 | &_num_to_res(1048576) returns '1M' | ||
| 198 | |||
| 199 | =cut | ||
| 200 | |||
| 201 | sub _num_to_res { | ||
| 202 | my $num = shift; | ||
| 203 | my $codes = ' KMG'; | ||
| 204 | my $unit = 0; | ||
| 205 | while ($unit < length($codes)-1 && $num >= 1024) { | ||
| 206 | $num /= 1024; | ||
| 207 | ++$unit; | ||
| 208 | } | ||
| 209 | $num = int($num) == $num ? int($num) : sprintf( "%.2f", $num ); | ||
| 210 | $num .= substr( $codes, $unit, 1 ); | ||
| 211 | return $num; | ||
| 212 | } | ||
| 213 | |||
| 214 | =head2 _vmem_usage | ||
| 215 | |||
| 216 | Gives out warning based on memory usage account information and decides if the program | ||
| 217 | fails because of lack of memory. If so, invoke _can_retry_from_vmem and return the memory | ||
| 218 | limit for retry, otherwise return 0. | ||
| 219 | |||
| 220 | =cut | ||
| 221 | |||
| 222 | sub _vmem_usage { | ||
| 223 | my $self = shift; | ||
| 224 | my $run = shift; | ||
| 225 | my $stats = $run->stats; | ||
| 226 | my ( $req, $used ) = map { _res_to_num($_) } | ||
| 227 | $self->_use_resources_required->{mem}, | ||
| 228 | $stats->{peak_virtual_memory}; | ||
| 229 | my $dreq = _num_to_res($req); | ||
| 230 | my $dused = _num_to_res($used); | ||
| 231 | my $pct = ( $used / $req ) * 100; # percentage of actual memory that got used | ||
| 232 | my $warn = ($pct < 50 && $req > _res_to_num('2G')) | ||
| 233 | ? " ** Requested resource too large" | ||
| 234 | : ""; | ||
| 235 | my $loglevel = $warn ? 'warn' : 'info'; | ||
| 236 | $self->$loglevel( sprintf "Resource(%s): Requested(%s) Usage(%s) %.2f%%%s", | ||
| 237 | 'mem', $dreq, $dused, $pct, $warn ); | ||
| 238 | my $stderr = $run->_stderr; | ||
| 239 | my $cmd = "tail -1 $stderr"; | ||
| 240 | my $last = `$cmd`; | ||
| 241 | # Decides if the program fails due to lack of memory | ||
| 242 | # Sometimes SLURM will throw an error message 'exceeded memory limit', but sometimes | ||
| 243 | # there will be no such error message. In the second case, we need to check the memory | ||
| 244 | # usage percentage. But even that may fail, if account gathering frequency is set too big | ||
| 245 | # (if the interval for checking memory usage is 30 sec then the memory usage info we get | ||
| 246 | # back will likely to be a lot less that what we expect), so in that case we will check | ||
| 247 | # the signal - if the program gets KILLED and the completion state is not TIMEOUT, then | ||
| 248 | # it is most likely that the job has failed due to shortage of memory. | ||
| 249 | if ($last =~ /exceeded memory limit/ | ||
| 250 | || $pct >= 99 | ||
| 251 | || $self->memory_too_small->($stats, $stderr) | ||
| 252 | || ((defined $stats->{killed}) && ($stats->{killed} eq 'KILL') && ($stats->{State} ne 'TIMEOUT'))) { | ||
| 253 | return $self->_can_retry_from_vmem( | ||
| 254 | $pct > 100 | ||
| 255 | ? ( $used, $dused ) | ||
| 256 | : ( $req, $dreq ) | ||
| 257 | ); | ||
| 258 | } | ||
| 259 | return 0; | ||
| 260 | } | ||
| 261 | |||
| 262 | =head1 AUTHOR | ||
| 263 | |||
| 264 | John Macdonald - Boutros Lab | ||
| 265 | |||
| 266 | Anqi (Joyce) Yang - Boutros Lab | ||
| 267 | |||
| 268 | =head1 ACKNOWLEDGEMENTS | ||
| 269 | |||
| 270 | Paul Boutros, Phd, PI - Boutros Lab | ||
| 271 | |||
| 272 | The Ontario Institute for Cancer Research | ||
| 273 | |||
| 274 | =cut | ||
| 275 | |||
| 276 | 1; |