| 1 | #!/usr/local/bin/perl | ||
| 2 | # PODNAME: es-alias-manager.pl | ||
| 3 | # ABSTRACT: Allow easy alias management for daily indexes | ||
| 4 | use strict; | ||
| 5 | use warnings; | ||
| 6 | |||
| 7 | use App::ElasticSearch::Utilities qw(:default); | ||
| 8 | use CLI::Helpers qw(:all); | ||
| 9 | use DateTime; | ||
| 10 | use Getopt::Long qw(:config no_ignore_case no_ignore_case_always); | ||
| 11 | use Pod::Usage; | ||
| 12 | use Ref::Util qw( is_ref ); | ||
| 13 | use YAML::XS (); | ||
| 14 | |||
| 15 | #------------------------------------------------------------------------# | ||
| 16 | # Argument Collection | ||
| 17 | my %opt; | ||
| 18 | GetOptions(\%opt, | ||
| 19 | 'all', | ||
| 20 | 'config=s', | ||
| 21 | 'skip=s', | ||
| 22 | # Basic options | ||
| 23 | 'help|h', | ||
| 24 | 'manual|m', | ||
| 25 | ); | ||
| 26 | |||
| 27 | #------------------------------------------------------------------------# | ||
| 28 | # Documentations! | ||
| 29 | pod2usage(1) if $opt{help}; | ||
| 30 | pod2usage(-exitstatus => 0, -verbose => 2) if $opt{manual}; | ||
| 31 | |||
| 32 | my %actions = ( | ||
| 33 | add => 'Create any missing aliases', | ||
| 34 | remove => 'Remove any aliases not in the desired set', | ||
| 35 | ); | ||
| 36 | |||
| 37 | # We might skip one thing or another | ||
| 38 | if( exists $opt{skip} && !exists $actions{$opt{skip}} ) { | ||
| 39 | output({color=>'red',sticky=>1}, "Invalid action to skip: $opt{skip}"); | ||
| 40 | output({clear=>1},"Valid actions to skip are:"); | ||
| 41 | output({indent=>1}, sprintf "%s - %s", $_, $actions{$_}) for sort keys %actions; | ||
| 42 | pod2usage(-exitstatus => 1); | ||
| 43 | } | ||
| 44 | |||
| 45 | my %CFG = ( | ||
| 46 | config => '/etc/elasticsearch/aliases.yml', | ||
| 47 | ); | ||
| 48 | # Extract from our options if we've overridden defaults | ||
| 49 | foreach my $setting (keys %CFG) { | ||
| 50 | $CFG{$setting} = $opt{$setting} if exists $opt{$setting} and defined $opt{$setting}; | ||
| 51 | } | ||
| 52 | if ( !exists $CFG{config} and ! -f $CFG{config} ) { | ||
| 53 | pod2usage(1); | ||
| 54 | } | ||
| 55 | my $ALIAS = YAML::XS::LoadFile( $CFG{config} ) or die "unable to read $CFG{config}"; | ||
| 56 | |||
| 57 | # Create the target uri for the ES Cluster | ||
| 58 | my $TARGET = exists $opt{host} && defined $opt{host} ? $opt{host} : 'localhost'; | ||
| 59 | |||
| 60 | # Grab a connection to ElasticSearch | ||
| 61 | my $es = es_connect(); | ||
| 62 | |||
| 63 | # Delete Indexes older than a certain point | ||
| 64 | my $TODAY = DateTime->now()->truncate( to => 'day' ); | ||
| 65 | my $indices = es_request('_aliases'); | ||
| 66 | |||
| 67 | if ( !defined $indices ) { | ||
| 68 | output({color=>"red"}, "Unable to locate indices by get_aliases()!"); | ||
| 69 | exit 1; | ||
| 70 | } | ||
| 71 | debug_var($indices); | ||
| 72 | |||
| 73 | my %PARTS = ( | ||
| 74 | DATE => { | ||
| 75 | RE => '(?<year>\d{4})[.\-](?<month>\d{2})[.\-](?<day>\d{2})', | ||
| 76 | FMT => join('.', '%Y', '%m', '%d'), | ||
| 77 | }, | ||
| 78 | PERIOD => { | ||
| 79 | FMT => '%s', | ||
| 80 | } | ||
| 81 | ); | ||
| 82 | |||
| 83 | my %IGNORE = (); | ||
| 84 | |||
| 85 | foreach my $base (keys %{ $ALIAS }) { | ||
| 86 | # Index Aliases Managed Manually | ||
| 87 | if( ! is_ref($ALIAS->{$base}) ) { | ||
| 88 | $IGNORE{$base} = 1; | ||
| 89 | delete $ALIAS->{$base}; | ||
| 90 | next; | ||
| 91 | } | ||
| 92 | |||
| 93 | my $re = $ALIAS->{$base}{pattern}; | ||
| 94 | $re =~ s/[^\w{}*?-]+//g; | ||
| 95 | |||
| 96 | # Wildcards | ||
| 97 | $re =~ s/\*+/.\+/g; | ||
| 98 | $re =~ s/\?/./g; | ||
| 99 | |||
| 100 | # Varaibles | ||
| 101 | $re =~ s/\{\{([^\}]+)\}\}/$PARTS{$1}->{RE}/g; | ||
| 102 | $ALIAS->{$base}{re} = qr/$re/; | ||
| 103 | |||
| 104 | # Setup Formatting | ||
| 105 | if (exists $ALIAS->{$base}{daily}) { | ||
| 106 | $ALIAS->{$base}{daily} =~ s/\{\{([^\}]+)\}\}/$PARTS{$1}->{FMT}/g; | ||
| 107 | } | ||
| 108 | if (exists $ALIAS->{$base}{relative}) { | ||
| 109 | $ALIAS->{$base}{relative}{alias} =~ s/\{\{([^\}]+)\}\}/$PARTS{$1}->{FMT}/g; | ||
| 110 | while( my ($period,$def) = each %{ $ALIAS->{$base}{relative}{periods} }) { | ||
| 111 | my %dt = ( | ||
| 112 | to => $TODAY->clone(), | ||
| 113 | from => exists $def->{from} ? $TODAY->clone() : DateTime->from_epoch(epoch => 0)->truncate( to => 'day'), | ||
| 114 | ); | ||
| 115 | debug("Period[$period] subtracting: "); | ||
| 116 | debug_var($def); | ||
| 117 | foreach my $d (keys %dt) { | ||
| 118 | if ( exists $def->{$d} ) { | ||
| 119 | $dt{$d}->subtract( %{ $def->{$d}} ); | ||
| 120 | debug("$period $d " . $dt{$d}->ymd); | ||
| 121 | } | ||
| 122 | } | ||
| 123 | $ALIAS->{$base}{relative}{periods}{$period} = \%dt; | ||
| 124 | } | ||
| 125 | } | ||
| 126 | } | ||
| 127 | debug("Aliases being applied:"); | ||
| 128 | debug_var($ALIAS); | ||
| 129 | |||
| 130 | # Loop through the indices and take appropriate actions; | ||
| 131 | foreach my $index (sort keys %{ $indices }) { | ||
| 132 | next if $index =~ /^[._]/; | ||
| 133 | |||
| 134 | debug("$index being evaluated"); | ||
| 135 | my %current = map { $_ => 1 } | ||
| 136 | grep { !/^\./ } | ||
| 137 | grep { not exists $IGNORE{$_} } | ||
| 138 | keys %{ $indices->{$index}{aliases} }; | ||
| 139 | my $managed = 0; | ||
| 140 | |||
| 141 | my %desired = (); | ||
| 142 | while( my($name,$map) = each %{ $ALIAS }) { | ||
| 143 | if ($index =~ /$map->{re}/) { | ||
| 144 | $managed++; | ||
| 145 | my $idx_dt = DateTime->new( map { $_ => $+{$_} } qw(year month day) ); | ||
| 146 | verbose("$index is a $name index."); | ||
| 147 | |||
| 148 | if ( exists $map->{daily} ) { | ||
| 149 | my $daily = $idx_dt->strftime($map->{daily}); | ||
| 150 | $desired{$daily} = 1; | ||
| 151 | } | ||
| 152 | if ( exists $map->{relative} ) { | ||
| 153 | while (my ($period,$def) = each %{ $map->{relative}{periods} }) { | ||
| 154 | debug(sprintf("Checking index date (%s) is between %s and %s", | ||
| 155 | $idx_dt->ymd, | ||
| 156 | $def->{from}->ymd, | ||
| 157 | $def->{to}->ymd, | ||
| 158 | )); | ||
| 159 | if( $idx_dt <= $def->{to} && $idx_dt >= $def->{from} ) { | ||
| 160 | my $alias = sprintf( $map->{relative}{alias}, $period ); | ||
| 161 | $desired{$alias} = 1; | ||
| 162 | } | ||
| 163 | } | ||
| 164 | } | ||
| 165 | } | ||
| 166 | } | ||
| 167 | my @updates = (); | ||
| 168 | my %checks = map { $_ => 1 } keys(%desired), keys(%current); | ||
| 169 | if( $managed ) { | ||
| 170 | foreach my $alias (keys %checks) { | ||
| 171 | if( exists $desired{$alias} && exists $current{$alias} ) { | ||
| 172 | next; | ||
| 173 | } | ||
| 174 | my $action = exists $desired{$alias} ? 'add' : 'remove'; | ||
| 175 | push @updates, { $action => { index => $index, alias => $alias} }; | ||
| 176 | |||
| 177 | # Do we skip an add or remove? | ||
| 178 | next if exists $opt{skip} && $action eq $opt{skip}; | ||
| 179 | verbose({color=>'cyan'}, "$index: $action alias '$alias'"); | ||
| 180 | } | ||
| 181 | } | ||
| 182 | debug({color=>'magenta'}, "Aliases for $index : " . join(',', keys %desired) ); | ||
| 183 | if( @updates ) { | ||
| 184 | eval { | ||
| 185 | es_request('_aliases', { method => 'POST' }, { actions => \@updates }); | ||
| 186 | output({color=>'green'}, "Updates applied for $index : " . join(',', keys %desired) ); | ||
| 187 | }; | ||
| 188 | if( my $err = $@ ){ | ||
| 189 | output({color=>'red'}, " + Failed to set aliases for $index\n", $err); | ||
| 190 | } | ||
| 191 | } | ||
| 192 | } | ||
| 193 | |||
| 194 | __END__ |