SLURM/Stage.pm untested

filter: Statementstmt 0.0% 0 / 97 Branchbran 0.0% 0 / 28 Conditioncond 0.0% 0 / 24 MC/DCmcdc 0.0% 0 / 13 Subroutinesub 0.0% 0 / 17 Podpod 0.0% 0 / 8 totaltotal 0.0% 0 / 187 CC 15 SCAR 28.0
CC
15
Cov
0%
_res_to_num
42.0
_vmem_usage
30.0
_analyse_completion_state
12.0
CRAP 16.5
1 package HPCD::SLURM::Stage;
2  
3 =head1 NAME
4  
5 HPCD::SLURM::Stage
6  
7 =head1 SYNOPSIS
8  
9 use HPCD::SLURM::Stage;
10  
11 =head1 DESCRIPTION
12  
13 This module defines SLURM-specific stage attributes and contains the method
14 that decides if the program has failed due to memory shortage, and if so if
15 the program can be retried with a larger memory limit.
16  
17 =cut
18  
19 ### INCLUDES ##################################################################
20  
21 # safe Perl
22 use warnings;
23 use strict;
24 use Carp;
25 use Data::Dumper;
26 use DateTime;
27  
28 use Moose;
29 use Moose::Util::TypeConstraints;
30 use MooseX::ClassAttribute;
31  
32 use HPCD::SLURM::Run;
33  
34 with 'HPCI::Stage',
35 HPCI::get_extra_roles('SLURM', 'stage');
36  
37 =head1 BUILDERS
38  
39 =head2 _build_cluster_specific_valid_resources
40  
41 The only two resources user can define via user_input are (hard) time and
42 (hard) memory.
43  
44 =cut
45  
46 sub _build_cluster_specific_valid_resources {
47 return (
48 mem => 'mem',
49 time => 'h_time'
50 )
51 }
52  
53 =head2 _build_cluster_specific_default_resources
54  
55 The default memory limit is 2G, and default time limit is there is no limit.
56  
57 =cut
58  
59 sub _build_cluster_specific_default_resources {
60 return ( mem => '2G' )
61 };
62  
63 =head2 _build_cluster_specific_default_retry_resources
64  
65 A list of possible retry memory limits (when a program fails because of lack
66 of memory it will look through this list to find and retry with the next biggest
67 memory available). Should be in ascending order. Default is qw(2G 4G 16G 32G).
68  
69 =cut
70  
71 sub _build_cluster_specific_default_retry_resources {
72 mem => [qw(2G 4G 8G 16G 32G)]
73 };
74  
75 =head1 ATTRIBUTES
76  
77 =cut
78  
79 has '+runs' => (
80 isa => 'ArrayRef[HPCD::SLURM::Run]'
81 );
82  
83 has '_run_class' => (
84 is => 'ro',
85 isa => 'Str',
86 init_arg => undef,
87 default => "HPCD::SLURM::Run"
88 );
89  
90 around '_run_args' => sub {
91 my $orig = shift;
92 my $self = shift;
93 return (
94 (
95 $self->$orig(), resources_required => $self->_use_resources_required
96 )
97 );
98 };
99  
100 has 'native_args_string' => (
101 is => 'ro',
102 isa => 'Str',
103 default => ''
104 );
105  
106 has 'memory_too_small' => (
107 is => 'ro',
108 isa => 'CodeRef',
109 default => sub { return sub { return 0 } }
110 );
111  
112 =head1 METHODS
113  
114 =head2 _analyse_completion_state
115  
116 Analyzes and returns the completion state.
117 Exit status 0 => pass, otherwise, if the program fails due to lack of memory and
118 the program can be retried with the next biggest memory, then returns retry, else
119 returns fail.
120  
121 =cut
122  
123 sub _analyse_completion_state {
124 my $self = shift;
125 my $run = shift;
126 my $stats = $run->stats;
127 $self->debug( "Stats from finished stage(" . $self->name . "): " . Dumper($stats) );
128 my $vmem_retry = $self->_vmem_usage( $run );
129 my $new_state =
130 0 == $stats->{ExitCode} ? 'pass'
131 : $vmem_retry ? 'retry'
132 : 'fail';
133 $self->_set_state($new_state);
134 }
135  
136 =head2 _can_retry_from_vmem
137  
138 Searches for the next biggest memory in _use_retry_resources_required for retry.
139  
140 =cut
141  
142 sub _can_retry_from_vmem {
143 my ($self, $res, $dres) = @_;
144 return 0
145 unless $self->_use_retry_resources_required
146 and my $res_avail = $self->_use_retry_resources_required->{mem};
147 for my $nres (@$res_avail) {
148 $self->info( "Considering ($nres) as resource replacement for ($dres)" );
149 next unless $res < _res_to_num($nres);
150 $self->info( "Accepted ($nres) as resource replacement for ($dres)" );
151 $self->_use_resources_required->{mem} = $nres;
152 return 1;
153 }
154 $self->info( "No resource replacement found for ($dres)" );
155 return 0;
156 }
157  
158 =head2 _res_to_num
159  
160 Takes in a memory size (units: KMG or none(no unit means the unit is byte already))
161 and converts it to a number representing the original size in bytes.
162  
163 Example:
164 &_res_to_num('1024') returns 1024 (because 1024 bytes = 1024 bytes)
165 &_res_to_num('2K') returns 2048 (because 2KB = 2048 bytes)
166 &_res_to_num('1M') returns 1048576 (because 1MB = 1048576 bytes)
167  
168 =cut
169  
170 sub _res_to_num {
171 my $str = shift;
172 return 0 unless my ($val, $unit) = ($str =~ /^(\d+(?:\.\d+)?)([KMG])?B?$/);
173 if ($unit) {
174 $unit = uc $unit;
175 if ($unit eq 'K') {
176 return $val * 1024;
177 }
178 elsif ($unit eq 'M') {
179 return $val * 1024 * 1024;
180 }
181 elsif ($unit eq 'G') {
182 return $val * 1024 * 1024 * 1024;
183 }
184 else {
185 return 0;
186 }
187 }
188 return $val;
189 }
190  
191 =head2 _num_to_res
192  
193 Takes in a number in bytes and converts it into designated units (K/M/G).
194  
195 Example:
196 &_num_to_res(2048) returns '2K'
197 &_num_to_res(1048576) returns '1M'
198  
199 =cut
200  
201 sub _num_to_res {
202 my $num = shift;
203 my $codes = ' KMG';
204 my $unit = 0;
205 while ($unit < length($codes)-1 && $num >= 1024) {
206 $num /= 1024;
207 ++$unit;
208 }
209 $num = int($num) == $num ? int($num) : sprintf( "%.2f", $num );
210 $num .= substr( $codes, $unit, 1 );
211 return $num;
212 }
213  
214 =head2 _vmem_usage
215  
216 Gives out warning based on memory usage account information and decides if the program
217 fails because of lack of memory. If so, invoke _can_retry_from_vmem and return the memory
218 limit for retry, otherwise return 0.
219  
220 =cut
221  
222 sub _vmem_usage {
223 my $self = shift;
224 my $run = shift;
225 my $stats = $run->stats;
226 my ( $req, $used ) = map { _res_to_num($_) }
227 $self->_use_resources_required->{mem},
228 $stats->{peak_virtual_memory};
229 my $dreq = _num_to_res($req);
230 my $dused = _num_to_res($used);
231 my $pct = ( $used / $req ) * 100; # percentage of actual memory that got used
232 my $warn = ($pct < 50 && $req > _res_to_num('2G'))
233 ? " ** Requested resource too large"
234 : "";
235 my $loglevel = $warn ? 'warn' : 'info';
236 $self->$loglevel( sprintf "Resource(%s): Requested(%s) Usage(%s) %.2f%%%s",
237 'mem', $dreq, $dused, $pct, $warn );
238 my $stderr = $run->_stderr;
239 my $cmd = "tail -1 $stderr";
240 my $last = `$cmd`;
241 # Decides if the program fails due to lack of memory
242 # Sometimes SLURM will throw an error message 'exceeded memory limit', but sometimes
243 # there will be no such error message. In the second case, we need to check the memory
244 # usage percentage. But even that may fail, if account gathering frequency is set too big
245 # (if the interval for checking memory usage is 30 sec then the memory usage info we get
246 # back will likely to be a lot less that what we expect), so in that case we will check
247 # the signal - if the program gets KILLED and the completion state is not TIMEOUT, then
248 # it is most likely that the job has failed due to shortage of memory.
249 if ($last =~ /exceeded memory limit/
250 || $pct >= 99
251 || $self->memory_too_small->($stats, $stderr)
252 || ((defined $stats->{killed}) && ($stats->{killed} eq 'KILL') && ($stats->{State} ne 'TIMEOUT'))) {
253 return $self->_can_retry_from_vmem(
254 $pct > 100
255 ? ( $used, $dused )
256 : ( $req, $dreq )
257 );
258 }
259 return 0;
260 }
261  
262 =head1 AUTHOR
263  
264 John Macdonald - Boutros Lab
265  
266 Anqi (Joyce) Yang - Boutros Lab
267  
268 =head1 ACKNOWLEDGEMENTS
269  
270 Paul Boutros, Phd, PI - Boutros Lab
271  
272 The Ontario Institute for Cancer Research
273  
274 =cut
275  
276 1;