| 1 | #!/usr/local/bin/perl -w | ||
| 2 | # | ||
| 3 | # Example Utility using Lustre::Info | ||
| 4 | # | ||
| 5 | # (C) 2010 Adrian Ulrich - <adrian.ulrich@id.ethz.ch> | ||
| 6 | # | ||
| 7 | # | ||
| 8 | # This program is free software; you can redistribute it and/or | ||
| 9 | # modify it under the same terms as Perl itself. | ||
| 10 | # | ||
| 11 | use strict; | ||
| 12 | use Lustre::Info; | ||
| 13 | use Getopt::Long; | ||
| 14 | use Data::Dumper; | ||
| 15 | |||
| 16 | use constant ANSI_ESC => "\x1b["; | ||
| 17 | use constant ANSI_RSET => '0m'; | ||
| 18 | |||
| 19 | $| = 1; | ||
| 20 | |||
| 21 | my $opts = {}; | ||
| 22 | my $l = Lustre::Info->new; | ||
| 23 | |||
| 24 | GetOptions($opts, "summary|s", "ost-stats", "filter=s", "monitor=s", "help", "version", "as-list", "delay=i"); | ||
| 25 | |||
| 26 | # fixup default delay value | ||
| 27 | $opts->{delay} = 3 if !exists($opts->{delay}) or $opts->{delay} < 1; | ||
| 28 | |||
| 29 | |||
| 30 | if($opts->{help}) { | ||
| 31 | usage(); | ||
| 32 | } | ||
| 33 | elsif($opts->{version}) { | ||
| 34 | version(); | ||
| 35 | } | ||
| 36 | elsif($opts->{summary}) { | ||
| 37 | show_summary(); | ||
| 38 | } | ||
| 39 | elsif(exists($opts->{'ost-stats'})) { | ||
| 40 | loop_ost_stats(); | ||
| 41 | } | ||
| 42 | elsif($opts->{monitor}) { | ||
| 43 | $opts->{filter} = "." if !exists($opts->{filter}); | ||
| 44 | |||
| 45 | if($l->is_ost) { | ||
| 46 | loop_client_stats($opts->{filter},'ost','traffic') if $opts->{monitor} eq 'ost'; | ||
| 47 | loop_client_stats($opts->{filter},'ost','pattern') if $opts->{monitor} eq 'ost-pattern'; | ||
| 48 | loop_client_stats($opts->{filter},'export','traffic') if $opts->{monitor} eq 'nid'; | ||
| 49 | loop_client_stats($opts->{filter},'export','pattern') if $opts->{monitor} eq 'nid-pattern'; | ||
| 50 | loop_brw_stats($opts->{filter}, 'disk_i_o_size') if $opts->{monitor} eq 'io-size'; | ||
| 51 | loop_brw_stats($opts->{filter}, 'disk_fragmented_i_os') if $opts->{monitor} eq 'io-frag'; | ||
| 52 | loop_brw_stats($opts->{filter}, 'i_o_time__1_1000s_') if $opts->{monitor} eq 'io-time'; | ||
| 53 | loop_brw_stats($opts->{filter}, 'disk_i_os_in_flight') if $opts->{monitor} eq 'in-flight'; | ||
| 54 | } | ||
| 55 | die "$0: unknown value for --monitor: `$opts->{monitor}'\n"; | ||
| 56 | } | ||
| 57 | else { | ||
| 58 | usage(); | ||
| 59 | } | ||
| 60 | |||
| 61 | |||
| 62 | |||
| 63 | ####################################################################### | ||
| 64 | # Display usage information / help | ||
| 65 | sub usage { | ||
| 66 | die join("\n", | ||
| 67 | "Usage: $0 OPTION", | ||
| 68 | "", | ||
| 69 | "Valid OPTIONs are:", | ||
| 70 | "--help Display this information and exit", | ||
| 71 | "--version Display version information", | ||
| 72 | "--summary Display some information about this host", | ||
| 73 | "--ost-stats Display per-OST statistics", | ||
| 74 | "--monitor=ACTION Display various statistics, valid values for `ACTION' are:", | ||
| 75 | " --- IF RUNNING ON AN OST: ---", | ||
| 76 | " ost : show ost->client stats (filter affects ost)", | ||
| 77 | " ost-pattern : show `metadata' rpcs of clients per ost", | ||
| 78 | " nid : show client->ost stats (filter affects nid)", | ||
| 79 | " nid-pattern : same as ost-pattern, but filter affects nid", | ||
| 80 | " io-size : track io-size per client", | ||
| 81 | " io-frag : track io-fragmentation per client", | ||
| 82 | " io-time : track service time per client", | ||
| 83 | " in-flight : track in-flight queue per client", | ||
| 84 | "--filter=REGEXP Filter --monitor output (use `.' to show everything)", | ||
| 85 | "--delay=SECONDS Refresh interval (defaults to 3 sec.)", | ||
| 86 | "--as-list Do not clear the screen", | ||
| 87 | "", | ||
| 88 | "", | ||
| 89 | "Report bugs to <adrian.ulrich\@id.ethz.ch>.", | ||
| 90 | ""); | ||
| 91 | } | ||
| 92 | |||
| 93 | ####################################################################### | ||
| 94 | # Display version and exit | ||
| 95 | sub version { | ||
| 96 | die join("\n", | ||
| 97 | "lustre-info $Lustre::Info::VERSION, running with Perl ".sprintf("%vd", $^V)." ($^X) on `$^O'", | ||
| 98 | "Report bugs to <adrian.ulrich\@id.ethz.ch>.", | ||
| 99 | ""); | ||
| 100 | } | ||
| 101 | |||
| 102 | ####################################################################### | ||
| 103 | # Return 'clear-screen' ascii sequence | ||
| 104 | sub _cls { | ||
| 105 | return "# ".localtime()."\n" if $opts->{'as-list'}; | ||
| 106 | return ANSI_ESC.'H'.ANSI_ESC.'2J'; | ||
| 107 | } | ||
| 108 | |||
| 109 | |||
| 110 | |||
| 111 | ####################################################################### | ||
| 112 | # Display a short summary about this host | ||
| 113 | sub show_summary { | ||
| 114 | my @summary_objs = (); | ||
| 115 | |||
| 116 | print "Lustre module version : ".($l->get_lustre_version)."\n"; | ||
| 117 | print "Host is acting as OST : ".($l->is_ost ? 'yes' : 'no')."\n"; | ||
| 118 | print "Host is acting as MDS : ".($l->is_mds ? 'yes' : 'no')."\n"; | ||
| 119 | print "Host is acting as MDT : ".($l->is_mdt ? 'yes' : 'no')."\n"; | ||
| 120 | |||
| 121 | if($l->is_ost) { | ||
| 122 | print "\nOST list:\n"; | ||
| 123 | @summary_objs = map( { $l->get_ost($_) } @{$l->get_ost_list} ); | ||
| 124 | } | ||
| 125 | elsif($l->is_mdt) { | ||
| 126 | print "\nMDT list:\n"; | ||
| 127 | @summary_objs = map( { $l->get_mdt($_) } @{$l->get_mdt_list} ); | ||
| 128 | } | ||
| 129 | |||
| 130 | foreach my $so (@summary_objs) { | ||
| 131 | my $size = sprintf("%.2f",$so->get_kbytes_total/1024/1024); | ||
| 132 | my $free = sprintf("%.2f",$so->get_kbytes_free/1024/1024); | ||
| 133 | |||
| 134 | my $f_free = $so->get_files_free; | ||
| 135 | my $f_total= $so->get_files_total; | ||
| 136 | my $f_used = $f_total - $f_free; | ||
| 137 | my $f_pct = ( $f_total ? sprintf("%.2f",( $f_free / $f_total * 100)) : 0 ); | ||
| 138 | |||
| 139 | my $rcinfo = $so->get_recovery_info; | ||
| 140 | my $blkdev = $so->get_blockdevice; | ||
| 141 | |||
| 142 | print join("\n",( "\t".$so->get_name, | ||
| 143 | "\t\tblock_device : $blkdev" , "\t\tlast_recovery : ".gmtime($rcinfo->{recovery_start})." (UTC)", | ||
| 144 | "\t\ttotal_size : $size GB", "\t\tfree_space : $free GB", | ||
| 145 | "\t\tfiles : $f_used in use (~ $f_pct\% free)", | ||
| 146 | "", | ||
| 147 | )); | ||
| 148 | } | ||
| 149 | |||
| 150 | } | ||
| 151 | |||
| 152 | ####################################################################### | ||
| 153 | # Display per-OST statistics | ||
| 154 | sub loop_ost_stats { | ||
| 155 | |||
| 156 | die "This host is not an OST\n" unless $l->is_ost; | ||
| 157 | |||
| 158 | my $orx = {}; # reference with all ost data | ||
| 159 | foreach my $this_ost (@{$l->get_ost_list}) { | ||
| 160 | my $obj = $l->get_ost($this_ost) or next; | ||
| 161 | my $blkdev = $obj->get_blockdevice; | ||
| 162 | $orx->{$this_ost} = { name=>$this_ost, blkdev=>$blkdev, obj=>$obj }; | ||
| 163 | } | ||
| 164 | |||
| 165 | # loop forever: | ||
| 166 | for(;;) { | ||
| 167 | print _cls(); | ||
| 168 | foreach my $this_ost (sort(keys(%$orx))) { | ||
| 169 | my $oref = $orx->{$this_ost}; | ||
| 170 | $oref->{obj}->collect_ost_stats; | ||
| 171 | |||
| 172 | my $stats = $oref->{obj}->get_ost_stats or next; # no data (yet); | ||
| 173 | my $slice = $stats->{_slice} or next; # ??? | ||
| 174 | my $wps = $stats->{write_bytes}/$slice/1024/1024; | ||
| 175 | my $rps = $stats->{read_bytes}/$slice/1024/1024; | ||
| 176 | printf("%16s (\@ %8s) : write=%8.3f MB/s, read=%8.3f MB/s", $this_ost, $oref->{blkdev}, $wps, $rps); | ||
| 177 | |||
| 178 | # Add some 'metadata' info | ||
| 179 | foreach my $type (qw(create destroy setattr preprw)) { | ||
| 180 | printf(", %s=%5.1f R/s",$type,$stats->{$type}/$slice); | ||
| 181 | } | ||
| 182 | print "\n"; | ||
| 183 | } | ||
| 184 | sleep($opts->{delay}); | ||
| 185 | } | ||
| 186 | } | ||
| 187 | |||
| 188 | ####################################################################### | ||
| 189 | # Display per-client statistics for OSTs matching regexp | ||
| 190 | sub loop_client_stats { | ||
| 191 | my($regexp, $provider, $xmode) = @_; | ||
| 192 | |||
| 193 | my($gx, $gx_list) = ("get_${provider}", "get_${provider}_list"); | ||
| 194 | my @ost_ref = map { $l->$gx($_) } grep(/$regexp/, @{$l->$gx_list}); | ||
| 195 | my($kludge, $what, $div_by, $fields) = ( $xmode eq 'traffic' ? ('_bytes', 'MB/s', 1024*1024, ['read','write']) : ('', 'op/s', 1, ['setattr','preprw','create', 'destroy']) ); | ||
| 196 | |||
| 197 | for(;;) { | ||
| 198 | my $memhog = {}; | ||
| 199 | foreach my $ost_obj (sort @ost_ref) { | ||
| 200 | $ost_obj->collect_client_stats; # Trigger new data collection | ||
| 201 | _update_memhog( Memhog=>$memhog, Data=>$ost_obj->get_client_stats, Name=>$ost_obj->get_name, PostfixKludge=>$kludge, | ||
| 202 | Fields=>$fields, Reverse=>($provider eq 'export' ? 1 : 0)); | ||
| 203 | } | ||
| 204 | |||
| 205 | my $lvl2_seen = delete($memhog->{_LVL2_SEEN_}); | ||
| 206 | my @ost_seen = sort(keys(%$lvl2_seen)); | ||
| 207 | _dump_matrix(Data=>$memhog, Items=>\@ost_seen, What=>$what, Divide=>$div_by, Fields=>$fields, ShowAll=>($provider =~ /^ost/ ? 0 : 1) ); | ||
| 208 | sleep($opts->{delay}); | ||
| 209 | } | ||
| 210 | # NOT REACHED | ||
| 211 | } | ||
| 212 | |||
| 213 | |||
| 214 | ####################################################################### | ||
| 215 | # | ||
| 216 | sub loop_brw_stats { | ||
| 217 | my($regexp, $kind) = @_; | ||
| 218 | |||
| 219 | my @exp_ref = map { $l->get_export($_) } grep(/$regexp/, @{$l->get_export_list}); | ||
| 220 | |||
| 221 | for(;;) { | ||
| 222 | my $memhog = {}; | ||
| 223 | my $seen_lbl = {}; | ||
| 224 | foreach my $exp_obj (@exp_ref) { | ||
| 225 | $exp_obj->collect_brw_stats; | ||
| 226 | my $r = $exp_obj->get_brw_stats or next; # -> first run | ||
| 227 | my $data = $r->{data}->{$kind}; | ||
| 228 | my $slice = $r->{_slice} or next; | ||
| 229 | foreach my $label (keys(%$data)) { | ||
| 230 | my $wps = $data->{$label}->{'write'}/$slice; | ||
| 231 | my $rps = $data->{$label}->{'read'}/$slice; | ||
| 232 | $memhog->{$exp_obj->get_name}->{$label} = { write=>$wps, read=>$rps }; | ||
| 233 | $seen_lbl->{$label}=1; | ||
| 234 | } | ||
| 235 | } | ||
| 236 | |||
| 237 | my @xitems = sort(keys(%$seen_lbl)); | ||
| 238 | _dump_matrix(Data=>$memhog, Items=>\@xitems, What=>"OPs/s of `$kind'", ShowAll=>1, Divide=>1); | ||
| 239 | sleep($opts->{delay}); | ||
| 240 | } | ||
| 241 | } | ||
| 242 | |||
| 243 | |||
| 244 | ####################################################################### | ||
| 245 | # Extract data from memhog and create per-second stats | ||
| 246 | sub _update_memhog { | ||
| 247 | my(%args) = @_; | ||
| 248 | my $memhog = $args{Memhog}; | ||
| 249 | my $dataset = $args{Data}; | ||
| 250 | my $name = $args{Name}; | ||
| 251 | my $pfk = ($args{PostfixKludge} || ''); # for ugli postfix'es :-( | ||
| 252 | my $fields = $args{Fields} || ['read', 'write']; | ||
| 253 | |||
| 254 | my $reverse = ($args{Reverse} ? 1 : 0 ); | ||
| 255 | foreach my $key (keys(%$dataset)) { | ||
| 256 | my $slice = $dataset->{$key}->{_slice} or next; | ||
| 257 | my($lx1, $lx2) = ( $reverse ? ($name,$key) : ($key,$name) ); | ||
| 258 | foreach my $mh_key (@$fields) { | ||
| 259 | $memhog->{$lx1}->{$lx2}->{$mh_key} = $dataset->{$key}->{$mh_key.$pfk}/$slice; | ||
| 260 | } | ||
| 261 | $memhog->{_LVL2_SEEN_}->{$lx2} = 1; | ||
| 262 | } | ||
| 263 | } | ||
| 264 | |||
| 265 | |||
| 266 | sub _dump_matrix { | ||
| 267 | my(%args) = @_; | ||
| 268 | |||
| 269 | my $memhog = $args{Data}; # Current dataset | ||
| 270 | my $items = $args{Items}; # Items to display in header | ||
| 271 | my $what = $args{What}; # Type of data (MB/s) | ||
| 272 | my $divide = $args{Divide}; # Divide values by .... | ||
| 273 | my $hide = ($args{ShowAll} ? 0 : 1 ); # Hide/Show clients with total_wps+total_rps < 1 | ||
| 274 | my $fields = $args{Fields} || ['read', 'write']; | ||
| 275 | my $n_items = int(@$items); | ||
| 276 | my $n_fields = int(@$fields); | ||
| 277 | my $seen_c = 0; | ||
| 278 | my $shown_c = 0; | ||
| 279 | |||
| 280 | print _cls().ANSI_ESC."7m"; # Switch into 'invert' mode | ||
| 281 | if($n_items > 0) { | ||
| 282 | printf("%-19s|", "> client nid"); | ||
| 283 | print join("|", map({sprintf("%-${n_fields}0s", " $_")} @$items))."| "; | ||
| 284 | print "+++ TOTALS +++ " if $n_items > 1; | ||
| 285 | print "($what)"; | ||
| 286 | } | ||
| 287 | else { | ||
| 288 | print "collecting data, please wait..."; | ||
| 289 | } | ||
| 290 | print ANSI_ESC.ANSI_RSET."\n"; # switch back into normal mode | ||
| 291 | |||
| 292 | foreach my $nid (sort(keys(%$memhog))) { | ||
| 293 | my $total = {}; | ||
| 294 | my $tsum = 0; | ||
| 295 | my $str = sprintf("%-19s|",$nid); # Add client ID to list | ||
| 296 | $seen_c++; | ||
| 297 | |||
| 298 | foreach my $this_item (@$items) { | ||
| 299 | my $qref = ( $memhog->{$nid}->{$this_item} || { map( { ($_,0) } @$fields ) } ); # <-- note the ultra smart use of map ;-) | ||
| 300 | my @data = (); | ||
| 301 | foreach my $this_field (@$fields) { | ||
| 302 | push(@data, sprintf("%s=%6.1f", substr($this_field,0,1), $qref->{$this_field}/$divide)); | ||
| 303 | $total->{$this_field} += $qref->{$this_field}; | ||
| 304 | $tsum += $qref->{$this_field}; | ||
| 305 | } | ||
| 306 | $str .= sprintf("%-${n_fields}0s|", " ".join(", ",@data)); # each field has 10 chars | ||
| 307 | } | ||
| 308 | next if $tsum/$divide < 1 && $hide; | ||
| 309 | print "$str"; | ||
| 310 | if($n_items > 1) { | ||
| 311 | # add totals: | ||
| 312 | print " ".join(", ",map( { sprintf("%s=%6.1f", $_,$total->{$_}/$divide) } @$fields) ); | ||
| 313 | } | ||
| 314 | print "\n"; | ||
| 315 | $shown_c++; | ||
| 316 | } | ||
| 317 | |||
| 318 | printf("# %d client(s) hidden\n", $seen_c-$shown_c) if $hide; | ||
| 319 | |||
| 320 | } | ||
| 321 | __END__ |