« lib/App/ElasticSearch/Utilities/VersionHacks.pm
↑ Summary
lib/App/ElasticSearch/Utilities/QueryString/Nested.pm »
| 1 | #!/usr/local/bin/perl | ||
| 2 | # PODNAME: es-storage-overview.pl | ||
| 3 | # ABSTRACT: Index Storage Overview by Index Name without Dates | ||
| 4 | use strict; | ||
| 5 | use warnings; | ||
| 6 | use feature qw(state); | ||
| 7 | |||
| 8 | use App::ElasticSearch::Utilities qw(es_request es_index_strip_date es_index_bases); | ||
| 9 | use CHI; | ||
| 10 | use CLI::Helpers qw(:output); | ||
| 11 | use Getopt::Long::Descriptive; | ||
| 12 | use Pod::Usage; | ||
| 13 | |||
| 14 | #------------------------------------------------------------------------# | ||
| 15 | # Argument Collection | ||
| 16 | my ($opt,$usage) = describe_options('%c %o', | ||
| 17 | ['sort=s', "sort by name or size, default: name", | ||
| 18 | { default => 'name', callbacks => { 'must be name or size' => sub { $_[0] =~ /^name|size$/ } } } | ||
| 19 | ], | ||
| 20 | ['asc', "Sort ascending (default by name)"], | ||
| 21 | ['desc', "Sort descending (default by size)"], | ||
| 22 | ['limit=i', "Limit to showing only this many, ie top N", { default => 0 }], | ||
| 23 | ['raw', "Display numeric data without rollups"], | ||
| 24 | ['hidden', "Include system indices that start with a dot"], | ||
| 25 | [], | ||
| 26 | ['clear-cache', "Clear the _cat/indices cache"], | ||
| 27 | [], | ||
| 28 | ['help|h', "Display this help", { shortcircuit => 1 }], | ||
| 29 | ['manual|m',"Display the full manual", { shortcircuit => 1 }], | ||
| 30 | ); | ||
| 31 | |||
| 32 | #------------------------------------------------------------------------# | ||
| 33 | # Documentations! | ||
| 34 | if( $opt->help ) { | ||
| 35 | print $usage->text; | ||
| 36 | exit; | ||
| 37 | } | ||
| 38 | pod2usage(-exitstatus => 0, -verbose => 2) if $opt->manual; | ||
| 39 | |||
| 40 | my $cache = CHI->new( | ||
| 41 | driver => 'File', | ||
| 42 | root_dir => "$ENV{HOME}/.es-utils/cache", | ||
| 43 | namespace => 'storage', | ||
| 44 | expires_in => '10min', | ||
| 45 | ); | ||
| 46 | |||
| 47 | $cache->clear if $opt->clear_cache; | ||
| 48 | |||
| 49 | my %indices = (); | ||
| 50 | my %bases = (); | ||
| 51 | my %overview = ( | ||
| 52 | shards => 0, | ||
| 53 | indices => 0, | ||
| 54 | docs => 0, | ||
| 55 | size => 0, | ||
| 56 | memory => 0, | ||
| 57 | ); | ||
| 58 | |||
| 59 | my %fields = qw( | ||
| 60 | index index | ||
| 61 | pri primary | ||
| 62 | rep replica | ||
| 63 | docs.count docs | ||
| 64 | store.size size | ||
| 65 | memory.total memory | ||
| 66 | ); | ||
| 67 | my $result = $cache->get('_cat/indices'); | ||
| 68 | if( !defined $result ) { | ||
| 69 | output({color=>'cyan'}, "Fetching Index Meta-Data"); | ||
| 70 | $result ||= es_request('_cat/indices', | ||
| 71 | { | ||
| 72 | uri_param => { | ||
| 73 | h => join(',', sort keys %fields), | ||
| 74 | bytes => 'b', | ||
| 75 | format => 'json' | ||
| 76 | } | ||
| 77 | } | ||
| 78 | ); | ||
| 79 | $cache->set('_cat/indices', $result); | ||
| 80 | } | ||
| 81 | |||
| 82 | foreach my $row (@{ $result }) { | ||
| 83 | # Index Name | ||
| 84 | my $index = delete $row->{index}; | ||
| 85 | |||
| 86 | next if $index =~ /^\./ && !$opt->hidden; | ||
| 87 | |||
| 88 | $overview{indices}++; | ||
| 89 | verbose({color=>'green'}, "$index - Gathering statistics"); | ||
| 90 | |||
| 91 | my @bases = es_index_bases($index); | ||
| 92 | foreach my $base ( @bases ) { | ||
| 93 | # Count Indexes | ||
| 94 | $bases{$base}->{indices} ||= 0; | ||
| 95 | $bases{$base}->{indices}++; | ||
| 96 | # Handle keys | ||
| 97 | foreach my $k (keys %{ $row }) { | ||
| 98 | next unless exists $fields{$k}; | ||
| 99 | my $dk = $fields{$k}; | ||
| 100 | # Grab Overview Data | ||
| 101 | $overview{$dk} += $row->{$k}; | ||
| 102 | # Counts against bases | ||
| 103 | $bases{$base} ||= {}; | ||
| 104 | $bases{$base}->{$dk} ||= 0; | ||
| 105 | $bases{$base}->{$dk} += $row->{$k}; | ||
| 106 | } | ||
| 107 | } | ||
| 108 | } | ||
| 109 | |||
| 110 | output({color=>'white'}, "Storage Overview"); | ||
| 111 | my $displayed = 0; | ||
| 112 | foreach my $index (sort indices_by keys %bases) { | ||
| 113 | output({color=>"magenta",indent=>1}, $index); | ||
| 114 | output({color=>"cyan",kv=>1,indent=>2}, 'size', pretty_size( $bases{$index}->{size})); | ||
| 115 | output({color=>"cyan",kv=>1,indent=>2}, 'indices', $bases{$index}->{indices}); | ||
| 116 | output({color=>"cyan",kv=>1,indent=>2}, 'avgsize', pretty_size( $bases{$index}->{size} / $bases{$index}->{indices} )); | ||
| 117 | output({color=>"cyan",kv=>1,indent=>2}, 'primary', $bases{$index}->{primary}); | ||
| 118 | output({color=>"cyan",kv=>1,indent=>2}, 'replica', $bases{$index}->{replica}); | ||
| 119 | output({color=>"cyan",kv=>1,indent=>2}, 'docs', $bases{$index}->{docs}); | ||
| 120 | output({color=>"cyan",kv=>1,indent=>2}, 'memory', pretty_size( $bases{$index}->{memory})); | ||
| 121 | $displayed++; | ||
| 122 | last if $opt->limit > 0 && $displayed >= $opt->limit; | ||
| 123 | } | ||
| 124 | output({color=>'white',clear=>1},"Total for scanned data"); | ||
| 125 | output({color=>"cyan",kv=>1,indent=>1}, 'size', pretty_size( $overview{size})); | ||
| 126 | output({color=>"cyan",kv=>1,indent=>1}, 'indices', $overview{indices}); | ||
| 127 | output({color=>"cyan",kv=>1,indent=>1}, 'shards', $overview{primary} + $overview{replica}); | ||
| 128 | output({color=>"cyan",kv=>1,indent=>1}, 'docs', $overview{docs}); | ||
| 129 | output({color=>"cyan",kv=>1,indent=>1}, 'memory', pretty_size( $overview{memory})); | ||
| 130 | |||
| 131 | |||
| 132 | exit (0); | ||
| 133 | |||
| 134 | sub pretty_size { | ||
| 135 | my ($size)=@_; | ||
| 136 | state $warned = 0; | ||
| 137 | |||
| 138 | my $value = $size; | ||
| 139 | if( !$opt->raw ) { | ||
| 140 | my @indicators = qw(kb mb gb tb); | ||
| 141 | my $indicator = ''; | ||
| 142 | |||
| 143 | while( $size > 1024 && @indicators ) { | ||
| 144 | $indicator = shift @indicators; | ||
| 145 | $size /= 1024; | ||
| 146 | } | ||
| 147 | $value = sprintf('%0.2f %s', $size, $indicator); | ||
| 148 | } | ||
| 149 | |||
| 150 | return $value; | ||
| 151 | } | ||
| 152 | |||
| 153 | sub indices_by { | ||
| 154 | if( $opt->sort eq 'size' ) { | ||
| 155 | return $opt->asc ? | ||
| 156 | $bases{$a}->{size} <=> $bases{$b}->{size} : | ||
| 157 | $bases{$b}->{size} <=> $bases{$a}->{size} ; | ||
| 158 | } | ||
| 159 | return $opt->desc ? $b cmp $a : $a cmp $b; | ||
| 160 | } | ||
| 161 | |||
| 162 | __END__ |