line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package HPC::Runner::Command::execute_job::Logger::JSON; |
2
|
|
|
|
|
|
|
|
3
|
1
|
|
|
1
|
|
438
|
use Moose::Role; |
|
1
|
|
|
|
|
3
|
|
|
1
|
|
|
|
|
6
|
|
4
|
1
|
|
|
1
|
|
4429
|
use namespace::autoclean; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
7
|
|
5
|
|
|
|
|
|
|
|
6
|
|
|
|
|
|
|
with 'HPC::Runner::Command::execute_job::Logger::Lock'; |
7
|
|
|
|
|
|
|
|
8
|
1
|
|
|
1
|
|
70
|
use JSON; |
|
1
|
|
|
|
|
3
|
|
|
1
|
|
|
|
|
8
|
|
9
|
1
|
|
|
1
|
|
116
|
use File::Spec; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
20
|
|
10
|
1
|
|
|
1
|
|
4
|
use DateTime; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
24
|
|
11
|
1
|
|
|
1
|
|
4
|
use Try::Tiny; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
50
|
|
12
|
1
|
|
|
1
|
|
5
|
use File::Path qw(make_path remove_tree); |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
44
|
|
13
|
1
|
|
|
1
|
|
5
|
use File::Slurp; |
|
1
|
|
|
|
|
3
|
|
|
1
|
|
|
|
|
52
|
|
14
|
1
|
|
|
1
|
|
6
|
use Cwd; |
|
1
|
|
|
|
|
1
|
|
|
1
|
|
|
|
|
40
|
|
15
|
1
|
|
|
1
|
|
5
|
use Time::HiRes; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
9
|
|
16
|
|
|
|
|
|
|
|
17
|
|
|
|
|
|
|
has 'task_json' => ( |
18
|
|
|
|
|
|
|
is => 'rw', |
19
|
|
|
|
|
|
|
isa => 'Str', |
20
|
|
|
|
|
|
|
default => '', |
21
|
|
|
|
|
|
|
required => 0, |
22
|
|
|
|
|
|
|
); |
23
|
|
|
|
|
|
|
|
24
|
|
|
|
|
|
|
has 'task_jobname' => ( |
25
|
|
|
|
|
|
|
is => 'rw', |
26
|
|
|
|
|
|
|
isa => 'Str', |
27
|
|
|
|
|
|
|
required => 0, |
28
|
|
|
|
|
|
|
); |
29
|
|
|
|
|
|
|
|
30
|
|
|
|
|
|
|
sub parse_meta_str { |
31
|
0
|
|
|
0
|
0
|
|
my $self = shift; |
32
|
|
|
|
|
|
|
|
33
|
0
|
|
|
|
|
|
my $job_meta = {}; |
34
|
0
|
0
|
|
|
|
|
if ( $self->metastr ) { |
35
|
0
|
|
|
|
|
|
$job_meta = decode_json( $self->metastr ); |
36
|
|
|
|
|
|
|
} |
37
|
|
|
|
|
|
|
|
38
|
0
|
0
|
0
|
|
|
|
if ( !$job_meta || !exists $job_meta->{jobname} ) { |
39
|
|
|
|
|
|
|
##TO account for single_node mode |
40
|
0
|
|
|
|
|
|
$job_meta->{jobname} = $self->jobname; |
41
|
|
|
|
|
|
|
} |
42
|
|
|
|
|
|
|
|
43
|
0
|
|
|
|
|
|
$self->task_jobname( $job_meta->{jobname} ); |
44
|
0
|
|
|
|
|
|
return $job_meta; |
45
|
|
|
|
|
|
|
} |
46
|
|
|
|
|
|
|
|
47
|
|
|
|
|
|
|
sub create_json_task { |
48
|
0
|
|
|
0
|
0
|
|
my $self = shift; |
49
|
0
|
|
|
|
|
|
my $cmdpid = shift; |
50
|
|
|
|
|
|
|
|
51
|
0
|
|
|
|
|
|
my $job_meta = $self->parse_meta_str; |
52
|
|
|
|
|
|
|
|
53
|
|
|
|
|
|
|
my $task_obj = { |
54
|
|
|
|
|
|
|
pid => $cmdpid, |
55
|
|
|
|
|
|
|
start_time => $self->table_data->{start_time}, |
56
|
|
|
|
|
|
|
jobname => $job_meta->{jobname}, |
57
|
0
|
|
|
|
|
|
task_id => $self->counter, |
58
|
|
|
|
|
|
|
}; |
59
|
|
|
|
|
|
|
|
60
|
0
|
0
|
|
|
|
|
$task_obj->{scheduler_id} = $self->job_scheduler_id |
61
|
|
|
|
|
|
|
if $self->can('scheduler_id'); |
62
|
|
|
|
|
|
|
|
63
|
0
|
|
|
|
|
|
my $data_dir = File::Spec->catdir( $self->data_dir, $self->task_jobname ); |
64
|
0
|
|
|
|
|
|
make_path($data_dir); |
65
|
|
|
|
|
|
|
|
66
|
0
|
0
|
|
|
|
|
if ( !$self->no_log_json ) { |
67
|
0
|
|
|
|
|
|
$self->check_lock; |
68
|
0
|
|
|
|
|
|
$self->write_lock; |
69
|
0
|
|
|
|
|
|
$self->add_to_running( $data_dir, $task_obj ); |
70
|
|
|
|
|
|
|
try { |
71
|
0
|
|
|
0
|
|
|
$self->lock_file->remove; |
72
|
0
|
|
|
|
|
|
}; |
73
|
|
|
|
|
|
|
} |
74
|
|
|
|
|
|
|
|
75
|
0
|
|
|
|
|
|
return $task_obj; |
76
|
|
|
|
|
|
|
} |
77
|
|
|
|
|
|
|
|
78
|
|
|
|
|
|
|
##TODO Once we add to the complete |
79
|
|
|
|
|
|
|
## Get all the for the tasks |
80
|
|
|
|
|
|
|
## Compute mean, min, max of all tasks |
81
|
|
|
|
|
|
|
sub update_json_task { |
82
|
0
|
|
|
0
|
0
|
|
my $self = shift; |
83
|
|
|
|
|
|
|
|
84
|
|
|
|
|
|
|
# my @stats = ( 'vmpeak', 'vmrss', 'vmsize', 'vmhwm' ); |
85
|
|
|
|
|
|
|
# my $job_meta = $self->parse_meta_str; |
86
|
|
|
|
|
|
|
# my $basename = $self->data_tar->basename('.tar.gz'); |
87
|
|
|
|
|
|
|
# my $data_dir = File::Spec->catdir( $basename, $job_meta->{jobname} ); |
88
|
|
|
|
|
|
|
|
89
|
0
|
|
|
|
|
|
my $data_dir = File::Spec->catdir( $self->data_dir, $self->task_jobname ); |
90
|
0
|
|
|
|
|
|
make_path($data_dir); |
91
|
|
|
|
|
|
|
|
92
|
0
|
|
|
|
|
|
my $tags = ""; |
93
|
0
|
0
|
|
|
|
|
if ( exists $self->table_data->{task_tags} ) { |
94
|
0
|
|
|
|
|
|
my $task_tags = $self->table_data->{task_tags}; |
95
|
0
|
0
|
|
|
|
|
if ($task_tags) { |
96
|
0
|
|
|
|
|
|
$tags = $task_tags; |
97
|
|
|
|
|
|
|
} |
98
|
|
|
|
|
|
|
} |
99
|
|
|
|
|
|
|
|
100
|
|
|
|
|
|
|
# my $task_obj = $self->get_from_running($data_dir); |
101
|
0
|
|
|
|
|
|
my $task_obj = {}; |
102
|
|
|
|
|
|
|
|
103
|
0
|
|
|
|
|
|
$task_obj->{exit_time} = $self->table_data->{exit_time}; |
104
|
0
|
|
|
|
|
|
$task_obj->{duration} = $self->table_data->{duration}; |
105
|
0
|
|
|
|
|
|
$task_obj->{exit_code} = $self->table_data->{exitcode}; |
106
|
0
|
|
|
|
|
|
$task_obj->{task_tags} = $tags; |
107
|
0
|
|
|
|
|
|
$task_obj->{cmdpid} = $self->table_data->{cmdpid}; |
108
|
0
|
|
|
|
|
|
$task_obj->{start_time} = $self->table_data->{start_time}; |
109
|
0
|
|
|
|
|
|
$task_obj->{task_id} = $self->table_data->{task_id}; |
110
|
|
|
|
|
|
|
|
111
|
|
|
|
|
|
|
# $task_obj->{memory_profile} = {}; |
112
|
|
|
|
|
|
|
# |
113
|
|
|
|
|
|
|
# foreach my $stat (@stats) { |
114
|
|
|
|
|
|
|
# $task_obj->{memory_profile}->{$stat}->{low} = |
115
|
|
|
|
|
|
|
# $self->task_mem_data->{low}->{$stat}; |
116
|
|
|
|
|
|
|
# $task_obj->{memory_profile}->{$stat}->{high} = |
117
|
|
|
|
|
|
|
# $self->task_mem_data->{high}->{$stat}; |
118
|
|
|
|
|
|
|
# $task_obj->{memory_profile}->{$stat}->{mean} = |
119
|
|
|
|
|
|
|
# $self->task_mem_data->{mean}->{$stat}; |
120
|
|
|
|
|
|
|
# $task_obj->{memory_profile}->{$stat}->{count} = |
121
|
|
|
|
|
|
|
# $self->task_mem_data->{count}->{$stat}; |
122
|
|
|
|
|
|
|
# } |
123
|
|
|
|
|
|
|
|
124
|
0
|
0
|
|
|
|
|
if ( !$self->no_log_json ) { |
125
|
0
|
|
|
|
|
|
$self->check_lock; |
126
|
0
|
|
|
|
|
|
$self->write_lock; |
127
|
|
|
|
|
|
|
|
128
|
0
|
|
|
|
|
|
$self->remove_from_running($data_dir); |
129
|
|
|
|
|
|
|
##TODO Add in mem for job |
130
|
0
|
|
|
|
|
|
$self->add_to_complete( $data_dir, $task_obj ); |
131
|
|
|
|
|
|
|
try { |
132
|
0
|
|
|
0
|
|
|
$self->lock_file->remove; |
133
|
0
|
|
|
|
|
|
}; |
134
|
|
|
|
|
|
|
} |
135
|
|
|
|
|
|
|
|
136
|
0
|
|
|
|
|
|
$task_obj->{pid} = $self->table_data->{cmdpid}; |
137
|
0
|
|
|
|
|
|
$task_obj->{start_time_dt} = $self->table_data->{start_time_dt}; |
138
|
0
|
|
|
|
|
|
return $task_obj; |
139
|
|
|
|
|
|
|
} |
140
|
|
|
|
|
|
|
|
141
|
|
|
|
|
|
|
sub add_to_complete { |
142
|
0
|
|
|
0
|
0
|
|
my $self = shift; |
143
|
0
|
|
|
|
|
|
my $data_dir = shift; |
144
|
0
|
|
|
|
|
|
my $task_data = shift; |
145
|
|
|
|
|
|
|
|
146
|
0
|
|
|
|
|
|
my $c_file = File::Spec->catfile( $data_dir, 'complete.json' ); |
147
|
|
|
|
|
|
|
|
148
|
0
|
|
|
|
|
|
my $json_obj = $self->read_json($c_file); |
149
|
|
|
|
|
|
|
|
150
|
0
|
|
|
|
|
|
$json_obj->{ $self->counter } = $task_data; |
151
|
0
|
|
|
|
|
|
$self->write_json( $c_file, $json_obj ); |
152
|
|
|
|
|
|
|
|
153
|
0
|
|
|
|
|
|
return $json_obj; |
154
|
|
|
|
|
|
|
} |
155
|
|
|
|
|
|
|
|
156
|
|
|
|
|
|
|
## keep this or no? |
157
|
|
|
|
|
|
|
##TODO Create Mem profile file |
158
|
|
|
|
|
|
|
sub create_task_file { |
159
|
0
|
|
|
0
|
0
|
|
my $self = shift; |
160
|
0
|
|
|
|
|
|
my $data_dir = shift; |
161
|
0
|
|
|
|
|
|
my $json_obj = shift; |
162
|
|
|
|
|
|
|
|
163
|
0
|
|
|
|
|
|
my $t_file = File::Spec->catfile( $data_dir, $self->counter . '.json' ); |
164
|
0
|
|
|
|
|
|
$self->write_json( $t_file, $json_obj, ); |
165
|
|
|
|
|
|
|
} |
166
|
|
|
|
|
|
|
|
167
|
|
|
|
|
|
|
sub add_to_running { |
168
|
0
|
|
|
0
|
0
|
|
my $self = shift; |
169
|
0
|
|
|
|
|
|
my $data_dir = shift; |
170
|
0
|
|
|
|
|
|
my $task_data = shift; |
171
|
|
|
|
|
|
|
|
172
|
0
|
|
|
|
|
|
my $r_file = File::Spec->catfile( $data_dir, 'running.json' ); |
173
|
|
|
|
|
|
|
|
174
|
0
|
|
|
|
|
|
my $json_obj = $self->read_json( $r_file, ); |
175
|
0
|
|
|
|
|
|
$json_obj->{ $self->counter } = $task_data; |
176
|
|
|
|
|
|
|
|
177
|
0
|
|
|
|
|
|
$self->write_json( $r_file, $json_obj, ); |
178
|
|
|
|
|
|
|
} |
179
|
|
|
|
|
|
|
|
180
|
|
|
|
|
|
|
sub remove_from_running { |
181
|
0
|
|
|
0
|
0
|
|
my $self = shift; |
182
|
0
|
|
|
|
|
|
my $data_dir = shift; |
183
|
|
|
|
|
|
|
|
184
|
0
|
|
|
|
|
|
my $r_file = File::Spec->catfile( $data_dir, 'running.json' ); |
185
|
|
|
|
|
|
|
|
186
|
0
|
|
|
|
|
|
my $json_obj = $self->read_json( $r_file, ); |
187
|
|
|
|
|
|
|
|
188
|
0
|
|
|
|
|
|
delete $json_obj->{ $self->table_data->{task_id} }; |
189
|
0
|
|
|
|
|
|
$self->write_json( $r_file, $json_obj, ); |
190
|
|
|
|
|
|
|
} |
191
|
|
|
|
|
|
|
|
192
|
|
|
|
|
|
|
sub get_from_running { |
193
|
0
|
|
|
0
|
0
|
|
my $self = shift; |
194
|
0
|
|
|
|
|
|
my $data_dir = shift; |
195
|
|
|
|
|
|
|
|
196
|
0
|
|
|
|
|
|
my $r_file = File::Spec->catfile( $data_dir, 'running.json' ); |
197
|
0
|
|
|
|
|
|
my $json_obj = $self->read_json( $r_file, ); |
198
|
|
|
|
|
|
|
|
199
|
0
|
|
|
|
|
|
return $json_obj->{ $self->table_data->{task_id} }; |
200
|
|
|
|
|
|
|
} |
201
|
|
|
|
|
|
|
|
202
|
|
|
|
|
|
|
sub read_json { |
203
|
0
|
|
|
0
|
0
|
|
my $self = shift; |
204
|
0
|
|
|
|
|
|
my $file = shift; |
205
|
|
|
|
|
|
|
|
206
|
0
|
|
|
|
|
|
my $json_obj = {}; |
207
|
0
|
|
|
|
|
|
my $text; |
208
|
|
|
|
|
|
|
|
209
|
0
|
0
|
|
|
|
|
if ( -e $file ) { |
210
|
0
|
|
|
|
|
|
$text = read_file($file); |
211
|
|
|
|
|
|
|
try { |
212
|
0
|
0
|
|
0
|
|
|
$json_obj = decode_json($text) if $text; |
213
|
0
|
|
|
|
|
|
}; |
214
|
|
|
|
|
|
|
} |
215
|
|
|
|
|
|
|
|
216
|
0
|
|
|
|
|
|
return $json_obj; |
217
|
|
|
|
|
|
|
} |
218
|
|
|
|
|
|
|
|
219
|
|
|
|
|
|
|
sub write_json { |
220
|
0
|
|
|
0
|
0
|
|
my $self = shift; |
221
|
0
|
|
|
|
|
|
my $file = shift; |
222
|
0
|
|
|
|
|
|
my $json_obj = shift; |
223
|
|
|
|
|
|
|
|
224
|
0
|
0
|
|
|
|
|
return unless $json_obj; |
225
|
0
|
|
|
|
|
|
my $json_text = ''; |
226
|
|
|
|
|
|
|
|
227
|
|
|
|
|
|
|
try { |
228
|
0
|
|
|
0
|
|
|
$json_text = encode_json($json_obj); |
229
|
|
|
|
|
|
|
} |
230
|
|
|
|
|
|
|
catch { |
231
|
0
|
|
|
0
|
|
|
$json_text = ''; |
232
|
0
|
|
|
|
|
|
}; |
233
|
|
|
|
|
|
|
|
234
|
0
|
|
|
|
|
|
write_file($file, $json_text); |
235
|
|
|
|
|
|
|
} |
236
|
|
|
|
|
|
|
|
237
|
|
|
|
|
|
|
1; |