| 1 | #!/usr/local/bin/perl | ||
| 2 | # PODNAME: es-copy-index.pl | ||
| 3 | # ABSTRACT: Copy an index from one cluster to another | ||
| 4 | |||
| 5 | use strict; | ||
| 6 | use warnings; | ||
| 7 | |||
| 8 | use App::ElasticSearch::Utilities qw(:default :index); | ||
| 9 | use App::ElasticSearch::Utilities::Query; | ||
| 10 | use App::ElasticSearch::Utilities::QueryString; | ||
| 11 | use CLI::Helpers qw(:all); | ||
| 12 | use File::Basename; | ||
| 13 | use File::Slurp::Tiny qw(read_lines); | ||
| 14 | use Getopt::Long qw(:config posix_default no_ignore_case no_ignore_case_always); | ||
| 15 | use Hash::Merge::Simple qw(clone_merge); | ||
| 16 | use JSON::MaybeXS; | ||
| 17 | use Pod::Usage; | ||
| 18 | use Ref::Util qw(is_hashref); | ||
| 19 | use Time::HiRes qw(time); | ||
| 20 | |||
| 21 | #------------------------------------------------------------------------# | ||
| 22 | # Argument Parsing | ||
| 23 | my %OPT; | ||
| 24 | GetOptions(\%OPT, qw( | ||
| 25 | from=s | ||
| 26 | to=s | ||
| 27 | source=s | ||
| 28 | destination=s | ||
| 29 | append|A | ||
| 30 | block=i | ||
| 31 | mapping=s | ||
| 32 | settings=s | ||
| 33 | help|h | ||
| 34 | manual|m | ||
| 35 | )); | ||
| 36 | |||
| 37 | #------------------------------------------------------------------------# | ||
| 38 | # Documentation | ||
| 39 | pod2usage(1) if $OPT{help}; | ||
| 40 | pod2usage(-exitstatus => 0, -verbose => 2) if $OPT{manual}; | ||
| 41 | debug_var(\%OPT); | ||
| 42 | |||
| 43 | #------------------------------------------------------------------------# | ||
| 44 | # Copy To/From | ||
| 45 | my %INDEX = ( | ||
| 46 | from => $OPT{source}, | ||
| 47 | to => exists $OPT{destination} ? $OPT{destination} : $OPT{source}, | ||
| 48 | block => exists $OPT{block} ? $OPT{block} : 1000, | ||
| 49 | ); | ||
| 50 | my %HOST = ( | ||
| 51 | from => $OPT{from}, | ||
| 52 | to => exists $OPT{to} ? $OPT{to} : $OPT{from}, | ||
| 53 | ); | ||
| 54 | if( $HOST{to} eq $HOST{from} && $INDEX{to} eq $INDEX{from} ) { | ||
| 55 | output({color=>'red',IMPORTANT=>1}, | ||
| 56 | "FATAL: Cannot copy from the same host to the same index name!" | ||
| 57 | ); | ||
| 58 | exit 1; | ||
| 59 | } | ||
| 60 | |||
| 61 | #------------------------------------------------------------------------# | ||
| 62 | # Build the Query | ||
| 63 | my $JSON = JSON->new->pretty->canonical; | ||
| 64 | my $qs = App::ElasticSearch::Utilities::QueryString->new(); | ||
| 65 | my $q = @ARGV ? $qs->expand_query_string(@ARGV) | ||
| 66 | : App::ElasticSearch::Utilities::Query->new(must => {match_all=>{}}); | ||
| 67 | |||
| 68 | $q->set_scan_scroll('1m'); | ||
| 69 | $q->set_size( $INDEX{block} ); | ||
| 70 | |||
| 71 | # Connect to ElasticSearch | ||
| 72 | my %ES = (); | ||
| 73 | foreach my $dir (qw(from to)) { | ||
| 74 | $ES{$dir} = es_connect( $HOST{$dir} ); | ||
| 75 | } | ||
| 76 | |||
| 77 | die "Invalid index: $INDEX{from}" unless $ES{from}->exists( index => $INDEX{from} ); | ||
| 78 | my $TO_EXISTS = $ES{to}->exists( index => $INDEX{to} ); | ||
| 79 | |||
| 80 | my $RECORDS = 0; | ||
| 81 | my $TOTAL=0; | ||
| 82 | my $LAST = time; | ||
| 83 | my ($status, $res); | ||
| 84 | |||
| 85 | # Mappings/Settings for Non-existant index. | ||
| 86 | unless( exists $OPT{append} ) { | ||
| 87 | die "Index $INDEX{to} already exists in $HOST{to}" if $TO_EXISTS; | ||
| 88 | $res = es_request($ES{from}, '_settings', {index => $INDEX{from}} ); | ||
| 89 | debug_var($res); | ||
| 90 | my $from_settings = $res->{$INDEX{from}}{settings}; | ||
| 91 | my @settings = ({ | ||
| 92 | index => { | ||
| 93 | number_of_shards => $from_settings->{index}{number_of_shards}, | ||
| 94 | number_of_replicas => $from_settings->{index}{number_of_replicas}, | ||
| 95 | } | ||
| 96 | }); | ||
| 97 | if( exists $OPT{settings} && -f $OPT{settings} ) { | ||
| 98 | my $content = join '', read_lines($OPT{settings}); | ||
| 99 | eval { | ||
| 100 | push @settings, $JSON->decode($content); | ||
| 101 | 1; | ||
| 102 | } or do { | ||
| 103 | debug($content); | ||
| 104 | die "Parsing JSON from $OPT{settings} failed: $@"; | ||
| 105 | }; | ||
| 106 | } | ||
| 107 | my $to_settings = clone_merge(@settings); | ||
| 108 | |||
| 109 | # Determine if we get mappings from a file or from the index. | ||
| 110 | my $mappings; | ||
| 111 | if( exists $OPT{mapping} && -f $OPT{mapping} ) { | ||
| 112 | my $content = join '', read_lines($OPT{mapping}); | ||
| 113 | eval { | ||
| 114 | $mappings = $JSON->decode($content); | ||
| 115 | 1; | ||
| 116 | } or do { | ||
| 117 | debug($content); | ||
| 118 | die "Parsing JSON from $OPT{mapping} failed: $@"; | ||
| 119 | }; | ||
| 120 | } | ||
| 121 | else { | ||
| 122 | $mappings = $res->{$INDEX{from}}{mappings}; | ||
| 123 | } | ||
| 124 | |||
| 125 | $res = es_request($ES{to}, '/', | ||
| 126 | { | ||
| 127 | method => 'PUT', | ||
| 128 | index => $INDEX{to}, | ||
| 129 | }, | ||
| 130 | { | ||
| 131 | settings => $to_settings, | ||
| 132 | $mappings ? ( mappings => $mappings ) : (), | ||
| 133 | } | ||
| 134 | ); | ||
| 135 | |||
| 136 | if (!defined $res || !is_hashref($res) || !$res->{ok}) { | ||
| 137 | die "Failed to create index in $HOST{to} : " . $JSON->encode($res); | ||
| 138 | } | ||
| 139 | } | ||
| 140 | else { | ||
| 141 | my @ignored=(); | ||
| 142 | foreach my $k (qw(settings mapping)) { | ||
| 143 | push @ignored, $k if exists $OPT{$k} && -f $OPT{$k}; | ||
| 144 | } | ||
| 145 | output({color=>'yellow',sticky=>1}, | ||
| 146 | sprintf "%s - warning ignoring %s as they are invalid in this context.", basename($0), join(', ', map { "--$_" } @ignored) | ||
| 147 | ) if @ignored; | ||
| 148 | } # End Mappings/Settings for Non-existant index. | ||
| 149 | |||
| 150 | debug_var($q->request_body); | ||
| 151 | $res = es_request($ES{from}, '_search', | ||
| 152 | # Search Parameters | ||
| 153 | { | ||
| 154 | index => $INDEX{from}, | ||
| 155 | uri_param => $q->uri_params, | ||
| 156 | method => 'GET', | ||
| 157 | }, | ||
| 158 | # Search Body | ||
| 159 | $q->request_body, | ||
| 160 | ); | ||
| 161 | debug_var($res); | ||
| 162 | |||
| 163 | while( $res && @{ $res->{hits}{hits} }) { | ||
| 164 | $TOTAL ||= $res->{hits}{total}; | ||
| 165 | my $start=time; | ||
| 166 | my $batch=0; | ||
| 167 | my $body = [ | ||
| 168 | map { | ||
| 169 | $batch++; | ||
| 170 | ( | ||
| 171 | { create => { _type => $_->{_type}, _id => $_->{_id}, } }, | ||
| 172 | $_->{_source} | ||
| 173 | ) | ||
| 174 | } @{ $res->{hits}{hits} } | ||
| 175 | ]; | ||
| 176 | my $max_retries = 3; | ||
| 177 | my $success = 0; | ||
| 178 | while ($max_retries--) { | ||
| 179 | debug("Attempting bulk load of $batch documents"); | ||
| 180 | my ($s2, $r2) = $ES{to}->bulk( | ||
| 181 | index => $INDEX{to}, | ||
| 182 | body => $body | ||
| 183 | ); | ||
| 184 | if ($s2 ne "200") { | ||
| 185 | output({stderr=>1,color=>'red'},"Failed to put documents to $HOST{to} (http status = $s2): " . $JSON->encode([ $s2, $r2 ])); | ||
| 186 | next; | ||
| 187 | } | ||
| 188 | $success=1; | ||
| 189 | last; | ||
| 190 | } | ||
| 191 | die "Failed to write data to $HOST{to}:9200/$INDEX{to} after $RECORDS docs indexed." | ||
| 192 | unless $success; | ||
| 193 | my $took = time - $start; | ||
| 194 | show_counts( scalar @{$res->{hits}{hits}} ); | ||
| 195 | $res = es_request($ES{from}, '_search/scroll', { | ||
| 196 | uri_param => { | ||
| 197 | scroll_id => $res->{_scroll_id}, | ||
| 198 | scroll => '1m', | ||
| 199 | } | ||
| 200 | }); | ||
| 201 | |||
| 202 | verbose(sprintf "Batch of %d done in %00.2fs.", $batch, $took); | ||
| 203 | } | ||
| 204 | |||
| 205 | sub show_counts { | ||
| 206 | my $inc_records = shift; | ||
| 207 | |||
| 208 | output({color=>'green'}, "Starting copy of $INDEX{from} to $HOST{to}:$INDEX{to}.") if $RECORDS == 0; | ||
| 209 | |||
| 210 | $RECORDS += $inc_records; | ||
| 211 | if( $RECORDS % ($INDEX{block} * 10) == 0 ) { | ||
| 212 | my $now = time; | ||
| 213 | my $diff = $now - $LAST; | ||
| 214 | my @time=localtime; | ||
| 215 | my $msg = sprintf "%00.2f%% %02d:%02d:%02d Records: %d of %d in %0.2fs", ($RECORDS/$TOTAL)*100, @time[2,1,0], $RECORDS, $TOTAL, $diff; | ||
| 216 | output({color=>'yellow'}, $msg); | ||
| 217 | $LAST=$now; | ||
| 218 | } | ||
| 219 | } | ||
| 220 | |||
| 221 | __END__ |