| line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
|
1
|
|
|
|
|
|
|
#!/usr/bin/env perl |
|
2
|
|
|
|
|
|
|
#PODNAME:printslurm |
|
3
|
|
|
|
|
|
|
#ABSTRACT:Utility to print info from Slurm |
|
4
|
1
|
|
|
1
|
|
4336
|
use v5.12; |
|
|
1
|
|
|
|
|
2
|
|
|
5
|
1
|
|
|
1
|
|
3
|
use warnings; |
|
|
1
|
|
|
|
|
1
|
|
|
|
1
|
|
|
|
|
42
|
|
|
6
|
1
|
|
|
1
|
|
618
|
use Getopt::Long; |
|
|
1
|
|
|
|
|
14268
|
|
|
|
1
|
|
|
|
|
5
|
|
|
7
|
1
|
|
|
1
|
|
748
|
use Term::ANSIColor; |
|
|
1
|
|
|
|
|
9636
|
|
|
|
1
|
|
|
|
|
105
|
|
|
8
|
1
|
|
|
1
|
|
446
|
use FindBin qw($RealBin); |
|
|
1
|
|
|
|
|
1140
|
|
|
|
1
|
|
|
|
|
155
|
|
|
9
|
1
|
50
|
|
|
|
87994
|
if (-e "$RealBin/../dist.ini") { |
|
|
0
|
|
|
|
|
0
|
|
|
10
|
1
|
50
|
|
|
|
5
|
say STDERR "[dev mode] Using local lib" if ($ENV{"DEBUG"}); |
|
11
|
1
|
|
|
1
|
|
480
|
use lib "$RealBin/../lib"; |
|
|
1
|
|
|
|
|
725
|
|
|
|
1
|
|
|
|
|
7
|
|
|
12
|
|
|
|
|
|
|
} |
|
13
|
1
|
|
|
1
|
|
524
|
use NBI::Slurm; |
|
|
1
|
|
|
|
|
2
|
|
|
|
1
|
|
|
|
|
65230
|
|
|
14
|
|
|
|
|
|
|
# Define command line options |
|
15
|
1
|
|
|
|
|
2
|
my $help = 0; |
|
16
|
1
|
|
|
|
|
31
|
my $node_info = 0; |
|
17
|
|
|
|
|
|
|
|
|
18
|
|
|
|
|
|
|
GetOptions( |
|
19
|
|
|
|
|
|
|
"help|h" => \$help, |
|
20
|
|
|
|
|
|
|
"node|n" => \$node_info, |
|
21
|
0
|
|
|
0
|
|
0
|
"version" => sub { say "printslurm v", $NBI::Slurm::VERSION; exit }, |
|
|
0
|
|
|
|
|
0
|
|
|
22
|
1
|
50
|
|
|
|
13
|
) or show_help(); |
|
23
|
|
|
|
|
|
|
|
|
24
|
|
|
|
|
|
|
# Show help if requested |
|
25
|
1
|
50
|
|
|
|
638
|
show_help() if $help; |
|
26
|
|
|
|
|
|
|
|
|
27
|
|
|
|
|
|
|
sub show_help { |
|
28
|
1
|
|
|
1
|
|
5
|
print color('bold white'); |
|
29
|
1
|
|
|
|
|
41
|
print "Usage: $0 [options]\n\n"; |
|
30
|
1
|
|
|
|
|
3
|
print color('reset'); |
|
31
|
|
|
|
|
|
|
|
|
32
|
1
|
|
|
|
|
16
|
print "Options:\n"; |
|
33
|
1
|
|
|
|
|
2
|
print " -h, --help Show this help message and exit\n"; |
|
34
|
1
|
|
|
|
|
1
|
print " -n, --node Display node-specific job information (when inside a Slurm job)\n\n"; |
|
35
|
|
|
|
|
|
|
|
|
36
|
1
|
|
|
|
|
2
|
print "Description:\n"; |
|
37
|
1
|
|
|
|
|
2
|
print " This script displays information about the Slurm environment,\n"; |
|
38
|
1
|
|
|
|
|
2
|
print " including queues (partitions) and Slurm environment variables.\n"; |
|
39
|
1
|
|
|
|
|
2
|
print " When run with the --node option, it displays detailed information\n"; |
|
40
|
1
|
|
|
|
|
1
|
print " about the current job allocation.\n\n"; |
|
41
|
|
|
|
|
|
|
|
|
42
|
1
|
|
|
|
|
72
|
exit(0); |
|
43
|
|
|
|
|
|
|
} |
|
44
|
|
|
|
|
|
|
|
|
45
|
|
|
|
|
|
|
# Function to run shell commands |
|
46
|
|
|
|
|
|
|
sub run_command { |
|
47
|
0
|
|
|
0
|
|
|
my $cmd = shift; |
|
48
|
0
|
|
|
|
|
|
my @output = `$cmd 2>/dev/null`; |
|
49
|
0
|
|
|
|
|
|
chomp(@output); |
|
50
|
0
|
|
|
|
|
|
return @output; |
|
51
|
|
|
|
|
|
|
} |
|
52
|
|
|
|
|
|
|
|
|
53
|
|
|
|
|
|
|
# Print a section header |
|
54
|
|
|
|
|
|
|
sub print_header { |
|
55
|
0
|
|
|
0
|
|
|
my $text = shift; |
|
56
|
0
|
|
|
|
|
|
print "\n", color('bold cyan'), "=== $text ===", color('reset'), "\n"; |
|
57
|
|
|
|
|
|
|
} |
|
58
|
|
|
|
|
|
|
|
|
59
|
|
|
|
|
|
|
# Print a key-value pair |
|
60
|
|
|
|
|
|
|
sub print_key_value { |
|
61
|
0
|
|
|
0
|
|
|
my ($key, $value, $color_name) = @_; |
|
62
|
0
|
|
0
|
|
|
|
$color_name //= 'green'; # Default color |
|
63
|
|
|
|
|
|
|
|
|
64
|
0
|
|
|
|
|
|
print color('yellow'), "$key: ", color($color_name), "$value", color('reset'), "\n"; |
|
65
|
|
|
|
|
|
|
} |
|
66
|
|
|
|
|
|
|
|
|
67
|
|
|
|
|
|
|
# Check if sinfo command exists (are we on a Slurm system?) |
|
68
|
0
|
|
|
|
|
0
|
my @check_slurm = run_command("command -v sinfo"); |
|
69
|
0
|
0
|
|
|
|
0
|
if (!@check_slurm) { |
|
70
|
0
|
|
|
|
|
0
|
print color('bold red'), "Error: Slurm commands not found. Are you on a Slurm cluster?", color('reset'), "\n"; |
|
71
|
0
|
|
|
|
|
0
|
exit(1); |
|
72
|
|
|
|
|
|
|
} |
|
73
|
|
|
|
|
|
|
|
|
74
|
|
|
|
|
|
|
# Display node-specific information if requested |
|
75
|
0
|
0
|
|
|
|
0
|
if ($node_info) { |
|
76
|
0
|
|
|
|
|
0
|
print_header("Current Job Information"); |
|
77
|
|
|
|
|
|
|
|
|
78
|
|
|
|
|
|
|
# Check if we're in a Slurm job |
|
79
|
0
|
0
|
|
|
|
0
|
if (!defined $ENV{SLURM_JOB_ID}) { |
|
80
|
0
|
|
|
|
|
0
|
print color('bold red'), "Error: Not running inside a Slurm job allocation.", color('reset'), "\n"; |
|
81
|
0
|
|
|
|
|
0
|
print "Run this script with the --node option only within a Slurm job.\n"; |
|
82
|
0
|
|
|
|
|
0
|
exit(1); |
|
83
|
|
|
|
|
|
|
} |
|
84
|
|
|
|
|
|
|
|
|
85
|
|
|
|
|
|
|
# Display job information |
|
86
|
0
|
|
|
|
|
0
|
print_key_value("Job ID", $ENV{SLURM_JOB_ID}); |
|
87
|
0
|
0
|
|
|
|
0
|
print_key_value("Job Name", $ENV{SLURM_JOB_NAME}) if defined $ENV{SLURM_JOB_NAME}; |
|
88
|
0
|
0
|
|
|
|
0
|
print_key_value("Queue/Partition", $ENV{SLURM_JOB_PARTITION}) if defined $ENV{SLURM_JOB_PARTITION}; |
|
89
|
|
|
|
|
|
|
|
|
90
|
|
|
|
|
|
|
# Get more detailed job info using scontrol |
|
91
|
0
|
|
|
|
|
0
|
my @job_info = run_command("scontrol show job $ENV{SLURM_JOB_ID}"); |
|
92
|
|
|
|
|
|
|
|
|
93
|
|
|
|
|
|
|
# Extract and display specific job details |
|
94
|
0
|
|
|
|
|
0
|
my ($time_limit) = grep { /TimeLimit=/ } @job_info; |
|
|
0
|
|
|
|
|
0
|
|
|
95
|
0
|
|
|
|
|
0
|
my ($mem_per_node) = grep { /MinMemoryNode=/ } @job_info; |
|
|
0
|
|
|
|
|
0
|
|
|
96
|
0
|
|
|
|
|
0
|
my ($num_nodes) = grep { /NumNodes=/ } @job_info; |
|
|
0
|
|
|
|
|
0
|
|
|
97
|
0
|
|
|
|
|
0
|
my ($num_cpus) = grep { /NumCPUs=/ } @job_info; |
|
|
0
|
|
|
|
|
0
|
|
|
98
|
|
|
|
|
|
|
|
|
99
|
0
|
0
|
|
|
|
0
|
if ($time_limit) { |
|
100
|
0
|
|
|
|
|
0
|
$time_limit =~ s/.*TimeLimit=(\S+).*/$1/; |
|
101
|
0
|
|
|
|
|
0
|
print_key_value("Time Limit", $time_limit); |
|
102
|
|
|
|
|
|
|
} |
|
103
|
|
|
|
|
|
|
|
|
104
|
0
|
0
|
|
|
|
0
|
if ($mem_per_node) { |
|
105
|
0
|
|
|
|
|
0
|
$mem_per_node =~ s/.*MinMemoryNode=(\S+).*/$1/; |
|
106
|
0
|
|
|
|
|
0
|
print_key_value("Memory Per Node", $mem_per_node); |
|
107
|
|
|
|
|
|
|
} |
|
108
|
|
|
|
|
|
|
|
|
109
|
0
|
0
|
|
|
|
0
|
if ($num_nodes) { |
|
110
|
0
|
|
|
|
|
0
|
$num_nodes =~ s/.*NumNodes=(\S+).*/$1/; |
|
111
|
0
|
|
|
|
|
0
|
print_key_value("Number of Nodes", $num_nodes); |
|
112
|
|
|
|
|
|
|
} |
|
113
|
|
|
|
|
|
|
|
|
114
|
0
|
0
|
|
|
|
0
|
if ($num_cpus) { |
|
115
|
0
|
|
|
|
|
0
|
$num_cpus =~ s/.*NumCPUs=(\S+).*/$1/; |
|
116
|
0
|
|
|
|
|
0
|
print_key_value("Number of CPUs", $num_cpus); |
|
117
|
|
|
|
|
|
|
} |
|
118
|
|
|
|
|
|
|
|
|
119
|
|
|
|
|
|
|
# Node list |
|
120
|
0
|
0
|
|
|
|
0
|
if (defined $ENV{SLURM_JOB_NODELIST}) { |
|
121
|
0
|
|
|
|
|
0
|
print_key_value("Node List", $ENV{SLURM_JOB_NODELIST}); |
|
122
|
|
|
|
|
|
|
|
|
123
|
|
|
|
|
|
|
# Expand the node list if it's compressed |
|
124
|
0
|
0
|
|
|
|
0
|
if ($ENV{SLURM_JOB_NODELIST} =~ /\[/) { |
|
125
|
0
|
|
|
|
|
0
|
my @expanded_nodes = run_command("scontrol show hostnames $ENV{SLURM_JOB_NODELIST}"); |
|
126
|
0
|
0
|
|
|
|
0
|
print " ", color('magenta'), join(", ", @expanded_nodes), color('reset'), "\n" if @expanded_nodes; |
|
127
|
|
|
|
|
|
|
} |
|
128
|
|
|
|
|
|
|
} |
|
129
|
|
|
|
|
|
|
|
|
130
|
|
|
|
|
|
|
# Print current working directory |
|
131
|
0
|
|
0
|
|
|
0
|
print_key_value("Working Directory", $ENV{PWD} || `pwd`); |
|
132
|
|
|
|
|
|
|
} |
|
133
|
|
|
|
|
|
|
# Default: display general Slurm information |
|
134
|
|
|
|
|
|
|
else { |
|
135
|
|
|
|
|
|
|
# Display Slurm version |
|
136
|
0
|
|
|
|
|
0
|
print_header("Slurm Version"); |
|
137
|
0
|
|
|
|
|
0
|
my @version = run_command("sinfo --version"); |
|
138
|
0
|
0
|
|
|
|
0
|
print color('green'), $version[0], color('reset'), "\n" if @version; |
|
139
|
|
|
|
|
|
|
|
|
140
|
|
|
|
|
|
|
# Display available partitions (queues) |
|
141
|
0
|
|
|
|
|
0
|
print_header("Available Partitions (Queues)"); |
|
142
|
0
|
|
|
|
|
0
|
my @partitions = run_command("sinfo --format=\"%20P %10l %5D %8m %11T %N\""); |
|
143
|
|
|
|
|
|
|
|
|
144
|
0
|
0
|
|
|
|
0
|
if (@partitions) { |
|
145
|
|
|
|
|
|
|
# Print header with bold |
|
146
|
0
|
|
|
|
|
0
|
print color('bold'), $partitions[0], color('reset'), "\n"; |
|
147
|
|
|
|
|
|
|
|
|
148
|
|
|
|
|
|
|
# Print remaining lines |
|
149
|
0
|
|
|
|
|
0
|
foreach my $i (1..$#partitions) { |
|
150
|
0
|
|
|
|
|
0
|
my @parts = split(/\s+/, $partitions[$i]); |
|
151
|
|
|
|
|
|
|
# Color the partition name |
|
152
|
0
|
|
|
|
|
0
|
print color('yellow'), sprintf("%-20s", $parts[0]), color('reset'); |
|
153
|
|
|
|
|
|
|
# Print the rest |
|
154
|
0
|
|
|
|
|
0
|
print join(" ", @parts[1..$#parts]), "\n"; |
|
155
|
|
|
|
|
|
|
} |
|
156
|
|
|
|
|
|
|
} |
|
157
|
|
|
|
|
|
|
|
|
158
|
|
|
|
|
|
|
# Display user's current jobs |
|
159
|
0
|
|
|
|
|
0
|
print_header("Your Current Jobs"); |
|
160
|
0
|
|
|
|
|
0
|
my @jobs = run_command("squeue --user=\$USER --format=\"%12i %20j %10P %10T %10M %10l %6D %R\""); |
|
161
|
|
|
|
|
|
|
|
|
162
|
0
|
0
|
|
|
|
0
|
if (@jobs > 1) { # More than just the header |
|
163
|
|
|
|
|
|
|
# Print header with bold |
|
164
|
0
|
|
|
|
|
0
|
print color('bold'), $jobs[0], color('reset'), "\n"; |
|
165
|
|
|
|
|
|
|
|
|
166
|
|
|
|
|
|
|
# Print jobs with colors based on state |
|
167
|
0
|
|
|
|
|
0
|
foreach my $i (1..$#jobs) { |
|
168
|
0
|
|
|
|
|
0
|
my @parts = split(/\s+/, $jobs[$i], 8); # Limit to 8 parts to keep the reason as one field |
|
169
|
|
|
|
|
|
|
|
|
170
|
|
|
|
|
|
|
# Color job ID |
|
171
|
0
|
|
|
|
|
0
|
print color('cyan'), sprintf("%-12s", $parts[0]), color('reset'); |
|
172
|
|
|
|
|
|
|
|
|
173
|
|
|
|
|
|
|
# Color job name |
|
174
|
0
|
|
|
|
|
0
|
print color('green'), sprintf("%-20s", $parts[1]), color('reset'); |
|
175
|
|
|
|
|
|
|
|
|
176
|
|
|
|
|
|
|
# Color partition |
|
177
|
0
|
|
|
|
|
0
|
print color('yellow'), sprintf("%-10s", $parts[2]), color('reset'); |
|
178
|
|
|
|
|
|
|
|
|
179
|
|
|
|
|
|
|
# Color state based on value |
|
180
|
0
|
|
|
|
|
0
|
my $state_color = 'white'; |
|
181
|
0
|
0
|
|
|
|
0
|
if ($parts[3] eq 'RUNNING') { |
|
|
|
0
|
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
|
182
|
0
|
|
|
|
|
0
|
$state_color = 'green'; |
|
183
|
|
|
|
|
|
|
} elsif ($parts[3] eq 'PENDING') { |
|
184
|
0
|
|
|
|
|
0
|
$state_color = 'yellow'; |
|
185
|
|
|
|
|
|
|
} elsif ($parts[3] =~ /^(FAILED|CANCELLED|TIMEOUT)$/) { |
|
186
|
0
|
|
|
|
|
0
|
$state_color = 'red'; |
|
187
|
|
|
|
|
|
|
} |
|
188
|
0
|
|
|
|
|
0
|
print color($state_color), sprintf("%-10s", $parts[3]), color('reset'); |
|
189
|
|
|
|
|
|
|
|
|
190
|
|
|
|
|
|
|
# Print the rest |
|
191
|
0
|
|
|
|
|
0
|
print join(" ", @parts[4..$#parts]), "\n"; |
|
192
|
|
|
|
|
|
|
} |
|
193
|
|
|
|
|
|
|
} else { |
|
194
|
0
|
|
|
|
|
0
|
print color('yellow'), "No active jobs found for your user.\n", color('reset'); |
|
195
|
|
|
|
|
|
|
} |
|
196
|
|
|
|
|
|
|
|
|
197
|
|
|
|
|
|
|
# Display Slurm environment variables |
|
198
|
0
|
|
|
|
|
0
|
print_header("Slurm Environment Variables"); |
|
199
|
0
|
|
|
|
|
0
|
my $found_vars = 0; |
|
200
|
|
|
|
|
|
|
|
|
201
|
0
|
|
|
|
|
0
|
foreach my $key (sort keys %ENV) { |
|
202
|
0
|
0
|
|
|
|
0
|
if ($key =~ /^SLURM_/) { |
|
203
|
0
|
|
|
|
|
0
|
print_key_value($key, $ENV{$key}); |
|
204
|
0
|
|
|
|
|
0
|
$found_vars = 1; |
|
205
|
|
|
|
|
|
|
} |
|
206
|
|
|
|
|
|
|
} |
|
207
|
|
|
|
|
|
|
|
|
208
|
0
|
0
|
|
|
|
0
|
if (!$found_vars) { |
|
209
|
0
|
|
|
|
|
0
|
print color('yellow'), "No SLURM_* environment variables found.\n", color('reset'); |
|
210
|
0
|
|
|
|
|
0
|
print "This is normal when not inside a Slurm job allocation.\n"; |
|
211
|
|
|
|
|
|
|
} |
|
212
|
|
|
|
|
|
|
|
|
213
|
|
|
|
|
|
|
# Display system-wide Slurm configuration |
|
214
|
0
|
|
|
|
|
0
|
print_header("Slurm Configuration"); |
|
215
|
0
|
|
|
|
|
0
|
my @config = run_command("scontrol show config | grep -v '^ '"); |
|
216
|
|
|
|
|
|
|
|
|
217
|
0
|
0
|
|
|
|
0
|
if (@config) { |
|
218
|
|
|
|
|
|
|
# Print only key configuration items |
|
219
|
0
|
|
|
|
|
0
|
my @important_configs = grep { /^(ClusterName|DefMemPerNode|MaxJobCount|SchedulerType|SelectType)/ } @config; |
|
|
0
|
|
|
|
|
0
|
|
|
220
|
|
|
|
|
|
|
|
|
221
|
0
|
|
|
|
|
0
|
foreach my $line (@important_configs) { |
|
222
|
0
|
|
|
|
|
0
|
my ($key, $value) = split(/\s*=\s*/, $line, 2); |
|
223
|
0
|
|
|
|
|
0
|
print_key_value($key, $value, 'magenta'); |
|
224
|
|
|
|
|
|
|
} |
|
225
|
|
|
|
|
|
|
|
|
226
|
0
|
|
|
|
|
0
|
print color('yellow'), "...\n", color('reset'); |
|
227
|
|
|
|
|
|
|
} |
|
228
|
|
|
|
|
|
|
} |
|
229
|
|
|
|
|
|
|
|
|
230
|
0
|
|
|
|
|
0
|
exit(0); |
|
231
|
|
|
|
|
|
|
|
|
232
|
|
|
|
|
|
|
__END__ |