script/es-copy-index.pl untested

filter: Statementstmt 0.0% 0 / 143 Branchbran 0.0% 0 / 40 Conditioncond 0.0% 0 / 30 MC/DCmcdc 0.0% 0 / 19 Subroutinesub 0.0% 0 / 15 Podpod 0.0% 0 / 1 totaltotal 0.0% 0 / 248 CC 3 SCAR 13.3
CC
3
Cov
0%
show_counts
12.0
BEGIN
2.0
BEGIN
2.0
CRAP 3.8
1 #!/usr/local/bin/perl
2 # PODNAME: es-copy-index.pl
3 # ABSTRACT: Copy an index from one cluster to another
4  
5 use strict;
6 use warnings;
7  
8 use App::ElasticSearch::Utilities qw(:default :index);
9 use App::ElasticSearch::Utilities::Query;
10 use App::ElasticSearch::Utilities::QueryString;
11 use CLI::Helpers qw(:all);
12 use File::Basename;
13 use File::Slurp::Tiny qw(read_lines);
14 use Getopt::Long qw(:config posix_default no_ignore_case no_ignore_case_always);
15 use Hash::Merge::Simple qw(clone_merge);
16 use JSON::MaybeXS;
17 use Pod::Usage;
18 use Ref::Util qw(is_hashref);
19 use Time::HiRes qw(time);
20  
21 #------------------------------------------------------------------------#
22 # Argument Parsing
23 my %OPT;
24 GetOptions(\%OPT, qw(
25 from=s
26 to=s
27 source=s
28 destination=s
29 append|A
30 block=i
31 mapping=s
32 settings=s
33 help|h
34 manual|m
35 ));
36  
37 #------------------------------------------------------------------------#
38 # Documentation
39 pod2usage(1) if $OPT{help};
40 pod2usage(-exitstatus => 0, -verbose => 2) if $OPT{manual};
41 debug_var(\%OPT);
42  
43 #------------------------------------------------------------------------#
44 # Copy To/From
45 my %INDEX = (
46 from => $OPT{source},
47 to => exists $OPT{destination} ? $OPT{destination} : $OPT{source},
48 block => exists $OPT{block} ? $OPT{block} : 1000,
49 );
50 my %HOST = (
51 from => $OPT{from},
52 to => exists $OPT{to} ? $OPT{to} : $OPT{from},
53 );
54 if( $HOST{to} eq $HOST{from} && $INDEX{to} eq $INDEX{from} ) {
55 output({color=>'red',IMPORTANT=>1},
56 "FATAL: Cannot copy from the same host to the same index name!"
57 );
58 exit 1;
59 }
60  
61 #------------------------------------------------------------------------#
62 # Build the Query
63 my $JSON = JSON->new->pretty->canonical;
64 my $qs = App::ElasticSearch::Utilities::QueryString->new();
65 my $q = @ARGV ? $qs->expand_query_string(@ARGV)
66 : App::ElasticSearch::Utilities::Query->new(must => {match_all=>{}});
67  
68 $q->set_scan_scroll('1m');
69 $q->set_size( $INDEX{block} );
70  
71 # Connect to ElasticSearch
72 my %ES = ();
73 foreach my $dir (qw(from to)) {
74 $ES{$dir} = es_connect( $HOST{$dir} );
75 }
76  
77 die "Invalid index: $INDEX{from}" unless $ES{from}->exists( index => $INDEX{from} );
78 my $TO_EXISTS = $ES{to}->exists( index => $INDEX{to} );
79  
80 my $RECORDS = 0;
81 my $TOTAL=0;
82 my $LAST = time;
83 my ($status, $res);
84  
85 # Mappings/Settings for Non-existant index.
86 unless( exists $OPT{append} ) {
87 die "Index $INDEX{to} already exists in $HOST{to}" if $TO_EXISTS;
88 $res = es_request($ES{from}, '_settings', {index => $INDEX{from}} );
89 debug_var($res);
90 my $from_settings = $res->{$INDEX{from}}{settings};
91 my @settings = ({
92 index => {
93 number_of_shards => $from_settings->{index}{number_of_shards},
94 number_of_replicas => $from_settings->{index}{number_of_replicas},
95 }
96 });
97 if( exists $OPT{settings} && -f $OPT{settings} ) {
98 my $content = join '', read_lines($OPT{settings});
99 eval {
100 push @settings, $JSON->decode($content);
101 1;
102 } or do {
103 debug($content);
104 die "Parsing JSON from $OPT{settings} failed: $@";
105 };
106 }
107 my $to_settings = clone_merge(@settings);
108  
109 # Determine if we get mappings from a file or from the index.
110 my $mappings;
111 if( exists $OPT{mapping} && -f $OPT{mapping} ) {
112 my $content = join '', read_lines($OPT{mapping});
113 eval {
114 $mappings = $JSON->decode($content);
115 1;
116 } or do {
117 debug($content);
118 die "Parsing JSON from $OPT{mapping} failed: $@";
119 };
120 }
121 else {
122 $mappings = $res->{$INDEX{from}}{mappings};
123 }
124  
125 $res = es_request($ES{to}, '/',
126 {
127 method => 'PUT',
128 index => $INDEX{to},
129 },
130 {
131 settings => $to_settings,
132 $mappings ? ( mappings => $mappings ) : (),
133 }
134 );
135  
136 if (!defined $res || !is_hashref($res) || !$res->{ok}) {
137 die "Failed to create index in $HOST{to} : " . $JSON->encode($res);
138 }
139 }
140 else {
141 my @ignored=();
142 foreach my $k (qw(settings mapping)) {
143 push @ignored, $k if exists $OPT{$k} && -f $OPT{$k};
144 }
145 output({color=>'yellow',sticky=>1},
146 sprintf "%s - warning ignoring %s as they are invalid in this context.", basename($0), join(', ', map { "--$_" } @ignored)
147 ) if @ignored;
148 } # End Mappings/Settings for Non-existant index.
149  
150 debug_var($q->request_body);
151 $res = es_request($ES{from}, '_search',
152 # Search Parameters
153 {
154 index => $INDEX{from},
155 uri_param => $q->uri_params,
156 method => 'GET',
157 },
158 # Search Body
159 $q->request_body,
160 );
161 debug_var($res);
162  
163 while( $res && @{ $res->{hits}{hits} }) {
164 $TOTAL ||= $res->{hits}{total};
165 my $start=time;
166 my $batch=0;
167 my $body = [
168 map {
169 $batch++;
170 (
171 { create => { _type => $_->{_type}, _id => $_->{_id}, } },
172 $_->{_source}
173 )
174 } @{ $res->{hits}{hits} }
175 ];
176 my $max_retries = 3;
177 my $success = 0;
178 while ($max_retries--) {
179 debug("Attempting bulk load of $batch documents");
180 my ($s2, $r2) = $ES{to}->bulk(
181 index => $INDEX{to},
182 body => $body
183 );
184 if ($s2 ne "200") {
185 output({stderr=>1,color=>'red'},"Failed to put documents to $HOST{to} (http status = $s2): " . $JSON->encode([ $s2, $r2 ]));
186 next;
187 }
188 $success=1;
189 last;
190 }
191 die "Failed to write data to $HOST{to}:9200/$INDEX{to} after $RECORDS docs indexed."
192 unless $success;
193 my $took = time - $start;
194 show_counts( scalar @{$res->{hits}{hits}} );
195 $res = es_request($ES{from}, '_search/scroll', {
196 uri_param => {
197 scroll_id => $res->{_scroll_id},
198 scroll => '1m',
199 }
200 });
201  
202 verbose(sprintf "Batch of %d done in %00.2fs.", $batch, $took);
203 }
204  
205 sub show_counts {
206 my $inc_records = shift;
207  
208 output({color=>'green'}, "Starting copy of $INDEX{from} to $HOST{to}:$INDEX{to}.") if $RECORDS == 0;
209  
210 $RECORDS += $inc_records;
211 if( $RECORDS % ($INDEX{block} * 10) == 0 ) {
212 my $now = time;
213 my $diff = $now - $LAST;
214 my @time=localtime;
215 my $msg = sprintf "%00.2f%% %02d:%02d:%02d Records: %d of %d in %0.2fs", ($RECORDS/$TOTAL)*100, @time[2,1,0], $RECORDS, $TOTAL, $diff;
216 output({color=>'yellow'}, $msg);
217 $LAST=$now;
218 }
219 }
220  
221 __END__