script/es-daily-index-maintenance.pl untested

filter: Statementstmt 0.0% 0 / 136 Branchbran 0.0% 0 / 66 Conditioncond 0.0% 0 / 39 MC/DCmcdc 0.0% 0 / 24 Subroutinesub 0.0% 0 / 7 Podpod n/a totaltotal 0.0% 0 / 272 CC 1 SCAR 6.9
CC
1
Cov
0%
BEGIN
2.0
BEGIN
2.0
BEGIN
2.0
CRAP 2.0
1 #!/usr/local/bin/perl
2 # PODNAME: es-daily-index-maintenance.pl
3 # ABSTRACT: Run to prune old indexes and optimize existing
4 use strict;
5 use warnings;
6  
7 use App::ElasticSearch::Utilities qw(:all);
8 use CLI::Helpers qw(:all);
9 use Getopt::Long qw(:config no_ignore_case no_ignore_case_always);
10 use Pod::Usage;
11 use POSIX qw(ceil);
12  
13 #------------------------------------------------------------------------#
14 # Argument Collection
15 my %opt;
16 GetOptions(\%opt,
17 'all',
18 'delete',
19 'delete-days=i',
20 'close',
21 'close-days=i',
22 'bloom',
23 'replicas',
24 'replicas',
25 'replicas-min|replica-min=i',
26 'replicas-max|replica-max=i',
27 'replicas-age|replica-age=i',
28 'optimize',
29 'optimize-days=i',
30 'index-basename=s',
31 'timezone=s',
32 'skip-alias=s@',
33 # Basic options
34 'help|h',
35 'manual|m',
36 );
37  
38 #------------------------------------------------------------------------#
39 # Documentations!
40 pod2usage(-exitval => 0) if $opt{help};
41 pod2usage(-exitval => 0, -verbose => 2) if $opt{manual};
42  
43 my %CFG = (
44 'optimize-days' => 1,
45 'delete-days' => 90,
46 'close-days' => 60,
47 'replicas-min' => 0,
48 'replicas-max' => 1,
49 'replicas-age' => 60,
50 timezone => 'UTC',
51 delete => 0,
52 close => 0,
53 bloom => 0,
54 optimize => 0,
55 replicas => 0,
56 'skip-alias' => [],
57 );
58 # Extract from our options if we've overridden defaults
59 foreach my $setting (keys %CFG) {
60 $CFG{$setting} = $opt{$setting} if exists $opt{$setting} and defined $opt{$setting};
61 }
62  
63 # Figure out what to run
64 my @MODES = qw(close delete optimize replicas);
65 if ( exists $opt{all} && $opt{all} ) {
66 map {
67 $CFG{$_} = 1 unless $_ eq 'replicas';
68 } @MODES;
69 }
70 else {
71 my $operate = 0;
72 foreach my $mode (@MODES) {
73 $operate++ if $CFG{$mode};
74 last if $operate;
75 }
76 pod2usage(-message => "No operation selected, use --close, --delete, --optimize, or --replicas.", -exitval => 1) unless $operate;
77 }
78 # Set skip alias hash
79 push @{ $CFG{'skip-alias'} }, qw( .hold .do_not_erase );
80 my %SKIP = map { $_ => 1 } @{ $CFG{'skip-alias'} };
81  
82 # Can't have replicas-min below 0
83 $CFG{'replicas-min'} = 0 if $CFG{'replicas-min'} < 0;
84  
85 # Create the target uri for the ES Cluster
86 my $es = es_connect();
87  
88 # This setting is no longer supported
89 output({color=>'red',sticky=>1,stderr=>1}, "WARNING: The index.codec.bloom.load is now disabled as of v1.4")
90 if $CFG{bloom};
91  
92 # Ages for replica management
93 my $AGE = $CFG{'replicas-age'};
94  
95 # Retrieve a list of indexes
96 my @indices = es_indices(
97 check_state => 0,
98 check_dates => 0,
99 );
100  
101 # Loop through the indices and take appropriate actions;
102 my @alias_changes=();
103 foreach my $index (sort @indices) {
104 my $days_old = es_index_days_old( $index );
105 verbose({color=>"cyan"}, sprintf "%s: index is %s days old",
106 $index, defined $days_old ? $days_old : '(unknown)'
107 );
108  
109 # If we can't calculate how old it is, skip it
110 if( !$days_old || $days_old < 1 ) {
111 verbose({color=>'magenta'},"$index for today, skipping.");
112 next;
113 }
114  
115 # Get aliases from index root
116 my %aliases = ();
117 my $idx_meta = es_request($index);
118 if( exists $idx_meta->{$index}{aliases} ) {
119 my $skipped;
120 foreach my $alias ( keys %{ $idx_meta->{$index}{aliases} } ) {
121 $skipped = $alias if exists $SKIP{$alias};
122 last if $skipped;
123 }
124 if( $skipped ) {
125 output("$index contains a skipped alias: $skipped");
126 next;
127 }
128 }
129  
130 # Delete the Index if it's too old
131 if( $CFG{delete} && $CFG{'delete-days'} <= $days_old ) {
132 output({color=>"red"}, "$index will be deleted.");
133 my $rc = es_delete_index($index);
134 next;
135 }
136  
137 # Manage replicas
138 if( $CFG{replicas} ) {
139 my $replicas;
140 if( $days_old > $AGE ) {
141 $replicas = $CFG{'replicas-min'};
142 }
143 else {
144 my $daily = $CFG{'replicas-max'} / $AGE;
145 $replicas = ceil( $CFG{'replicas-max'} - ($daily * $days_old) );
146 $replicas = $replicas < $CFG{'replicas-min'} ? $CFG{'replicas-min'} : $replicas;
147 $replicas = $replicas > $CFG{'replicas-max'} ? $CFG{'replicas-max'} : $replicas;
148 }
149 my %shards = es_index_shards($index);
150 debug({indent=>1}, "+ replica aging (P:$shards{primaries} R:$shards{replicas}->$replicas)");
151 if ( $shards{primaries} > 0 && $shards{replicas} != $replicas ) {
152 verbose({color=>'yellow'}, "$index: should have $replicas replicas, has $shards{replicas}");
153 my $result = es_request('_settings',
154 { index => $index, method => 'PUT' },
155 { index => { number_of_replicas => $replicas} },
156 );
157 if($result) {
158 output({color=>"green"}, "$index: Successfully set replicas to $replicas");
159 }
160 else {
161 output({color=>'red'}, "$index: Unable to set replicas");
162 }
163 }
164 }
165  
166 # Run optimize?
167 if( $CFG{optimize} ) {
168 my $segdata = es_segment_stats( $index );
169  
170 my $segment_ratio = undef;
171 if( defined $segdata && $segdata->{shards} > 0 ) {
172 $segment_ratio = sprintf( "%0.2f", $segdata->{segments} / $segdata->{shards} );
173 }
174  
175 if( $days_old >= $CFG{'optimize-days'} && defined($segment_ratio) && $segment_ratio > 1 ) {
176 verbose({color=>"yellow"}, "$index: required (segment_ratio: $segment_ratio).");
177  
178 my $o_res = es_optimize_index($index);
179 if( !defined $o_res ) {
180 output({color=>"red"}, "$index: Encountered error during optimize");
181 }
182 else {
183 output({color=>"green"}, "$index: $o_res->{_shards}{successful} of $o_res->{_shards}{total} shards optimized.");
184 }
185 }
186 else {
187 if( defined($segment_ratio) && $segment_ratio > 1 ) {
188 verbose("$index is active not optimizing (segment_ratio:$segment_ratio)");
189 }
190 else {
191 verbose("$index already optimized");
192 }
193 }
194 }
195  
196 # Close the index?
197 if( $CFG{close} && $CFG{'close-days'} <= $days_old ) {
198 my $status = es_request('_stats/store',{index=>$index});
199 if( defined $status ) {
200 if( $status->{_shards} && $status->{_shards}{total} && $status->{_shards}{total} > 0 ) {
201 # retrieve aliases
202 my $ars = es_request('_alias', {index=>$index});
203 foreach my $alias ( keys %{ $ars->{$index}{aliases} } ) {
204 debug({indent=>1}, "- Will remove alias $alias from $index");
205 push @alias_changes, { remove => { index => $index, alias => $alias }};
206 }
207 verbose({indent=>1}," - closing index.");
208 my $result = es_request('_close' => {method=>'POST',index=>$index});
209 if( defined $result && $result->{acknowledged}) {
210 output({color=>'magenta'},"+ Closed $index.");
211 }
212 else {
213 output({color=>'red'},"! Attempted to close $index, but did not succeed.");
214 }
215 }
216 else {
217 debug({indent=>1},"- $index already closed.");
218 }
219 }
220 else {
221 output({color=>'red'},"! Error establishing status of $index");
222 }
223 }
224 }
225 # If we closed indexes with aliases, we need to remove those aliases so searches to those aliases don't fail.
226 if(@alias_changes) {
227 verbose("+ Indexes closed had aliases, removing those aliases to prevent searches against closed indices.");
228 my $result = es_request('_aliases', {method=>'POST'}, { actions => \@alias_changes });
229 debug_var($result);
230 }
231  
232 __END__