0_90/Bulk.pm untested

filter: Statementstmt 0.0% 0 / 66 Branchbran 0.0% 0 / 14 Conditioncond 0.0% 0 / 21 MC/DCmcdc 0.0% 0 / 13 Subroutinesub 0.0% 0 / 7 Podpod 0.0% 0 / 3 totaltotal 0.0% 0 / 124 CC 8 SCAR 26.9
CC
8
Cov
0%
flush
30.0
reindex
12.0
add_action
6.0
CRAP 14.7
1 package Search::Elasticsearch::Client::0_90::Bulk;
2 $Search::Elasticsearch::Client::0_90::Bulk::VERSION = '6.81';
3 use Moo;
4 with 'Search::Elasticsearch::Client::0_90::Role::Bulk',
5 'Search::Elasticsearch::Role::Is_Sync';
6 use Search::Elasticsearch::Util qw(parse_params throw);
7 use Try::Tiny;
8 use namespace::clean;
9  
10 #===================================
11 sub add_action {
12 #===================================
13 my $self = shift;
14 my $buffer = $self->_buffer;
15 my $max_size = $self->max_size;
16 my $max_count = $self->max_count;
17 my $max_time = $self->max_time;
18  
19 while (@_) {
20 my @json = $self->_encode_action( splice( @_, 0, 2 ) );
21  
22 push @$buffer, @json;
23  
24 my $size = $self->_buffer_size;
25 $size += length($_) + 1 for @json;
26 $self->_buffer_size($size);
27  
28 my $count = $self->_buffer_count( $self->_buffer_count + 1 );
29  
30 $self->flush
31 if ( $max_size and $size >= $max_size )
32 || ( $max_count and $count >= $max_count )
33 || ( $max_time and time >= $self->_last_flush + $max_time );
34 }
35 return 1;
36 }
37  
38 #===================================
39 sub flush {
40 #===================================
41 my $self = shift;
42 $self->_last_flush(time);
43  
44 return { items => [] }
45 unless $self->_buffer_size;
46  
47 if ( $self->verbose ) {
48 local $| = 1;
49 print ".";
50 }
51 my $buffer = $self->_buffer;
52 my $results = try {
53 my $res = $self->es->bulk( %{ $self->_bulk_args }, body => $buffer );
54 $self->clear_buffer;
55 return $res;
56 }
57 catch {
58 my $error = $_;
59 $self->clear_buffer
60 unless $error->is( 'Cxn', 'NoNodes' );
61  
62 die $error;
63 };
64 $self->_report( $buffer, $results );
65 return defined wantarray ? $results : undef;
66 }
67  
68 #===================================
69 sub reindex {
70 #===================================
71 my ( $self, $params ) = parse_params(@_);
72 my $src = $params->{source}
73 or throw( 'Param', "Missing required param <source>" );
74  
75 my $transform = $self->_doc_transformer($params);
76  
77 if ( ref $src eq 'HASH' ) {
78 $src = {%$src};
79 my $es = delete $src->{es} || $self->es;
80 my $scroll = $es->scroll_helper(
81 search_type => 'scan',
82 size => 500,
83 %$src
84 );
85  
86 $src = sub {
87 $scroll->refill_buffer;
88 $scroll->drain_buffer;
89 };
90  
91 print "Reindexing " . $scroll->total . " docs\n"
92 if $self->verbose;
93 }
94  
95 while ( my @docs = grep {defined} $src->() ) {
96 $self->index( grep {$_} map { $transform->($_) } @docs );
97 }
98 $self->flush;
99 return 1;
100 }
101  
102 1;
103  
104 =pod
105  
106 =encoding UTF-8
107  
108 =head1 NAME
109  
110 Search::Elasticsearch::Client::0_90::Bulk - A helper module for the Bulk API and for reindexing
111  
112 =head1 VERSION
113  
114 version 6.81
115  
116 =head1 SYNOPSIS
117  
118 use Search::Elasticsearch;
119  
120 my $es = Search::Elasticsearch->new;
121 my $bulk = $es->bulk_helper(
122 index => 'my_index',
123 type => 'my_type'
124 );
125  
126 # Index docs:
127 $bulk->index({ id => 1, source => { foo => 'bar' }});
128 $bulk->add_action( index => { id => 1, source => { foo=> 'bar' }});
129  
130 # Create docs:
131 $bulk->create({ id => 1, source => { foo => 'bar' }});
132 $bulk->add_action( create => { id => 1, source => { foo=> 'bar' }});
133 $bulk->create_docs({ foo => 'bar' })
134  
135 # Delete docs:
136 $bulk->delete({ id => 1});
137 $bulk->add_action( delete => { id => 1 });
138 $bulk->delete_ids(1,2,3)
139  
140 # Update docs:
141 $bulk->update({ id => 1, script => '...' });
142 $bulk->add_action( update => { id => 1, script => '...' });
143  
144 # Manual flush
145 $bulk->flush;
146  
147 # Reindex docs:
148 my $bulk = $es->bulk_helper(
149 index => 'new_index',
150 verbose => 1
151 );
152  
153 $bulk->reindex( source => { index => 'old_index' });
154  
155 =head1 DESCRIPTION
156  
157 This module provides a wrapper for the L<Search::Elasticsearch::Client::0_90::Direct/bulk()>
158 method which makes it easier to run multiple create, index, update or delete
159 actions in a single request. It also provides a simple interface
160 for L<reindexing documents|/REINDEXING DOCUMENTS>.
161  
162 The L<Search::Elasticsearch::Client::0_90::Bulk> module acts as a queue, buffering up actions
163 until it reaches a maximum count of actions, or a maximum size of JSON request
164 body, at which point it issues a C<bulk()> request.
165  
166 Once you have finished adding actions, call L</flush()> to force the final
167 C<bulk()> request on the items left in the queue.
168  
169 This class does L<Search::Elasticsearch::Client::0_90::Role::Bulk> and
170 L<Search::Elasticsearch::Role::Is_Sync>.
171  
172 =head1 CREATING A NEW INSTANCE
173  
174 =head2 C<new()>
175  
176 my $bulk = $es->bulk_helper(
177  
178 index => 'default_index', # optional
179 type => 'default_type', # optional
180 %other_bulk_params # optional
181  
182 max_count => 1_000, # optional
183 max_size => 1_000_000, # optional
184 max_time => 5, # optional
185  
186 verbose => 0 | 1, # optional
187  
188 on_success => sub {...}, # optional
189 on_error => sub {...}, # optional
190 on_conflict => sub {...}, # optional
191  
192  
193 );
194  
195 The C<new()> method returns a new C<$bulk> object. You must pass your
196 Search::Elasticsearch client as the C<es> argument.
197  
198 The C<index> and C<type> parameters provide default values for
199 C<index> and C<type>, which can be overridden in each action.
200 You can also pass any other values which are accepted
201 by the L<bulk()|Search::Elasticsearch::Client::0_90::Direct/bulk()> method.
202  
203 See L</flush()> for more information about the other parameters.
204  
205 =head1 FLUSHING THE BUFFER
206  
207 =head2 C<flush()>
208  
209 $result = $bulk->flush;
210  
211 The C<flush()> method sends all buffered actions to Elasticsearch using
212 a L<bulk()|Search::Elasticsearch::Client::0_90::Direct/bulk()> request.
213  
214 =head2 Auto-flushing
215  
216 An automatic L</flush()> is triggered whenever the C<max_count>, C<max_size>,
217 or C<max_time> threshold is breached. This causes all actions in the buffer to be
218 sent to Elasticsearch.
219  
220 =over
221  
222 =item * C<max_count>
223  
224 The maximum number of actions to allow before triggering a L</flush()>.
225 This can be disabled by setting C<max_count> to C<0>. Defaults to
226 C<1,000>.
227  
228 =item * C<max_size>
229  
230 The maximum size of JSON request body to allow before triggering a
231 L</flush()>. This can be disabled by setting C<max_size> to C<0>. Defaults
232 to C<1_000,000> bytes.
233  
234 =item * C<max_time>
235  
236 The maximum number of seconds to wait before triggering a flush. Defaults
237 to C<0> seconds, which means that it is disabled. B<Note:> This timeout
238 is only triggered when new items are added to the queue, not in the background.
239  
240 =back
241  
242 =head2 Errors when flushing
243  
244 There are two types of error which can be thrown when L</flush()>
245 is called, either manually or automatically.
246  
247 =over
248  
249 =item * Temporary Elasticsearch errors
250  
251 A C<Cxn> error like a C<NoNodes> error which indicates that your cluster is down.
252 These errors do not clear the buffer, as they can be retried later on.
253  
254 =item * Action errors
255  
256 Individual actions may fail. For instance, a C<create> action will fail
257 if a document with the same C<index>, C<type> and C<id> already exists.
258 These action errors are reported via L<callbacks|/Using callbacks>.
259  
260 =back
261  
262 =head2 Using callbacks
263  
264 By default, any I<Action errors> (see above) cause warnings to be
265 written to C<STDERR>. However, you can use the C<on_error>, C<on_conflict>
266 and C<on_success> callbacks for more fine-grained control.
267  
268 All callbacks receive the following arguments:
269  
270 =over
271  
272 =item C<$action>
273  
274 The name of the action, ie C<index>, C<create>, C<update> or C<delete>.
275  
276 =item C<$response>
277  
278 The response that Elasticsearch returned for this action.
279  
280 =item C<$i>
281  
282 The index of the action, ie the first action in the flush request
283 will have C<$i> set to C<0>, the second will have C<$i> set to C<1> etc.
284  
285 =back
286  
287 =head3 C<on_success>
288  
289 my $bulk = $es->bulk_helper(
290 on_success => sub {
291 my ($action,$response,$i) = @_;
292 # do something
293 },
294 );
295  
296 The C<on_success> callback is called for every action that has a successful
297 response.
298  
299 =head3 C<on_conflict>
300  
301 my $bulk = $es->bulk_helper(
302 on_conflict => sub {
303 my ($action,$response,$i,$version) = @_;
304 # do something
305 },
306 );
307  
308 The C<on_conflict> callback is called for actions that have triggered
309 a C<Conflict> error, eg trying to C<create> a document which already
310 exists. The C<$version> argument will contain the version number
311 of the document currently stored in Elasticsearch (if found).
312  
313 =head3 C<on_error>
314  
315 my $bulk = $es->bulk_helper(
316 on_error => sub {
317 my ($action,$response,$i) = @_;
318 # do something
319 },
320 );
321  
322 The C<on_error> callback is called for any error (unless the C<on_conflict>)
323 callback has already been called).
324  
325 =head2 Disabling callbacks and autoflush
326  
327 If you want to be in control of flushing, and you just want to receive
328 the raw response that Elasticsearch sends instead of using callbacks,
329 then you can do so as follows:
330  
331 my $bulk = $es->bulk_helper(
332 max_count => 0,
333 max_size => 0,
334 on_error => undef
335 );
336  
337 $bulk->add_actions(....);
338 $response = $bulk->flush;
339  
340 =head1 CREATE, INDEX, UPDATE, DELETE
341  
342 =head2 C<add_action()>
343  
344 $bulk->add_action(
345 create => { ...params... },
346 index => { ...params... },
347 update => { ...params... },
348 delete => { ...params... }
349 );
350  
351 The C<add_action()> method allows you to add multiple C<create>, C<index>,
352 C<update> and C<delete> actions to the queue. The first value is the action
353 type, and the second value is the parameters that describe that action.
354 See the individual helper methods below for details.
355  
356 B<Note:> Parameters like C<index> or C<type> can be specified as C<index> or as
357 C<_index>, so the following two lines are equivalent:
358  
359 index => { index => 'index', type => 'type', id => 1, source => {...}},
360 index => { _index => 'index', _type => 'type', _id => 1, _source => {...}},
361  
362 B<Note:> The C<index> and C<type> parameters can be specified in the
363 params for any action, but if not specified, will default to the C<index>
364 and C<type> values specified in L</new()>. These are required parameters:
365 they must be specified either in L</new()> or in every action.
366  
367 =head2 C<create()>
368  
369 $bulk->create(
370 { index => 'custom_index', source => { doc body }},
371 { type => 'custom_type', id => 1, source => { doc body }},
372 ...
373 );
374  
375 The C<create()> helper method allows you to add multiple C<create> actions.
376 It accepts the same parameters as L<Search::Elasticsearch::Client::0_90::Direct/create()>
377 except that the document body should be passed as the C<source> or C<_source>
378 parameter, instead of as C<body>.
379  
380 =head2 C<create_docs()>
381  
382 $bulk->create_docs(
383 { doc body },
384 { doc body },
385 ...
386 );
387  
388 The C<create_docs()> helper is a shorter form of L</create()> which can be used
389 when you are using the default C<index> and C<type> as set in L</new()>
390 and you are not specifying a custom C<id> per document. In this case,
391 you can just pass the individual document bodies.
392  
393 =head2 C<index()>
394  
395 $bulk->index(
396 { index => 'custom_index', source => { doc body }},
397 { type => 'custom_type', id => 1, source => { doc body }},
398 ...
399 );
400  
401 The C<index()> helper method allows you to add multiple C<index> actions.
402 It accepts the same parameters as L<Search::Elasticsearch::Client::0_90::Direct/index()>
403 except that the document body should be passed as the C<source> or C<_source>
404 parameter, instead of as C<body>.
405  
406 =head2 C<delete()>
407  
408 $bulk->delete(
409 { index => 'custom_index', id => 1},
410 { type => 'custom_type', id => 2},
411 ...
412 );
413  
414 The C<delete()> helper method allows you to add multiple C<delete> actions.
415 It accepts the same parameters as L<Search::Elasticsearch::Client::0_90::Direct/delete()>.
416  
417 =head2 C<delete_ids()>
418  
419 $bulk->delete_ids(1,2,3...)
420  
421 The C<delete_ids()> helper method can be used when all of the documents you
422 want to delete have the default C<index> and C<type> as set in L</new()>.
423 In this case, all you have to do is to pass in a list of IDs.
424  
425 =head2 C<update()>
426  
427 $bulk->update(
428 { id => 1,
429 doc => { partial doc },
430 doc_as_upsert => 1
431 },
432 { id => 2,
433 lang => 'mvel',
434 script => { script }
435 upsert => { upsert doc }
436 },
437 ...
438 );
439  
440 The C<update()> helper method allows you to add multiple C<update> actions.
441 It accepts the same parameters as L<Search::Elasticsearch::Client::0_90::Direct/update()>.
442 An update can either use a I<partial doc> which gets merged with an existing
443 doc (example 1 above), or can use a C<script> to update an existing doc
444 (example 2 above). More information on C<script> can be found here:
445 L<Search::Elasticsearch::Client::0_90::Direct/update()>.
446  
447 =head1 REINDEXING DOCUMENTS
448  
449 A common use case for bulk indexing is to reindex a whole index when
450 changing the type mappings or analysis chain. This typically
451 combines bulk indexing with L<scrolled searches|Search::Elasticsearch::Client::0_90::Scroll>:
452 the scrolled search pulls all of the data from the source index, and
453 the bulk indexer indexes the data into the new index.
454  
455 =head2 C<reindex()>
456  
457 $bulk->reindex(
458 source => $source, # required
459 transform => \&transform, # optional
460 version_type => 'external|internal', # optional
461 );
462  
463 The C<reindex()> method requires a C<$source> parameter, which provides
464 the source for the documents which are to be reindexed.
465  
466 =head2 Reindexing from another index
467  
468 If the C<source> argument is a HASH ref, then the hash is passed to
469 L<Search::Elasticsearch::Client::0_90::Scroll/new()> to create a new scrolled search.
470  
471 my $bulk = $es->bulk_helper(
472 index => 'new_index',
473 verbose => 1
474 );
475  
476 $bulk->reindex(
477 source => {
478 index => 'old_index',
479 size => 500, # default
480 search_type => 'scan' # default
481 }
482 );
483  
484 If a default C<index> or C<type> has been specified in the call to
485 L</new()>, then it will replace the C<index> and C<type> values for
486 the docs returned from the scrolled search. In the example above,
487 all docs will be retrieved from C<"old_index"> and will be bulk indexed
488 into C<"new_index">.
489  
490 =head2 Reindexing from a generic source
491  
492 The C<source> parameter also accepts a coderef or an anonymous sub,
493 which should return one or more new documents every time it is executed.
494 This allows you to pass any iterator, wrapped in an anonymous sub:
495  
496 my $iter = get_iterator_from_somewhere();
497  
498 $bulk->reindex(
499 source => sub { $iter->next }
500 );
501  
502 =head2 Transforming docs on the fly
503  
504 The C<transform> parameter allows you to change documents on the fly,
505 using a callback. The callback receives the document as the only argument,
506 and should return the updated document, or C<undef> if the document should
507 not be indexed:
508  
509 $bulk->reindex(
510 source => { index => 'old_index' },
511 transform => sub {
512 my $doc = shift;
513  
514 # don't index doc marked as valid:false
515 return undef unless $doc->{_source}{valid};
516  
517 # convert $tag to @tags
518 $doc->{_source}{tags} = [ delete $doc->{_source}{tag}];
519 return $doc
520 }
521 );
522  
523 =head2 Reindexing from another cluster
524  
525 By default, L</reindex()> expects the source and destination indices
526 to be in the same cluster. To pull data from one cluster and index it into
527 another, you can use two separate C<$es> objects:
528  
529 $es_target = Search::Elasticsearch->new( nodes => 'localhost:9200' );
530 $es_source = Search::Elasticsearch->new( nodes => 'search1:9200' );
531  
532 my $bulk = $es_targert->bulk_helper(
533 verbose => 1
534 )
535 -> reindex(
536 source => {
537 es => $es_source,
538 index => 'my_index'
539 }
540 );
541  
542 =head2 Parents and routing
543  
544 If you are using parent-child relationships or custom C<routing> values,
545 and you want to preserve these when you reindex your documents, then
546 you will need to request these values specifically, as follows:
547  
548 $bulk->reindex(
549 source => {
550 index => 'old_index',
551 fields => ['_source','_parent','_routing']
552 }
553 );
554  
555 =head2 Working with version numbers
556  
557 Every document in Elasticsearch has a current C<version> number, which
558 is used for L<optimistic concurrency control|http://en.wikipedia.org/wiki/Optimistic_concurrency_control>,
559 that is, to ensure that you don't overwrite changes that have been made
560 by another process.
561  
562 All CRUD operations accept a C<version> parameter and a C<version_type>
563 parameter which tells Elasticsearch that the change should only be made
564 if the current document corresponds to these parameters. The
565 C<version_type> parameter can have the following values:
566  
567 =over
568  
569 =item * C<internal>
570  
571 Use Elasticsearch version numbers. Documents are only changed if the
572 document in Elasticsearch has the B<same> C<version> number that is
573 specified in the CRUD operation. After the change, the new
574 version number is C<version+1>.
575  
576 =item * C<external>
577  
578 Use an external versioning system, such as timestamps or version numbers
579 from an external database. Documents are only changed if the document
580 in Elasticsearch has a B<lower> C<version> number than the one
581 specified in the CRUD operation. After the change, the new version
582 number is C<version>.
583  
584 =back
585  
586 If you would like to reindex documents from one index to another, preserving
587 the C<version> numbers from the original index, then you need the following:
588  
589 $bulk->reindex(
590 source => {
591 index => 'old_index',
592 version => 1, # retrieve version numbers in search
593 },
594 version_type => 'external' # use these "external" version numbers
595 );
596  
597 =head1 AUTHOR
598  
599 Enrico Zimuel <enrico.zimuel@elastic.co>
600  
601 =head1 COPYRIGHT AND LICENSE
602  
603 This software is Copyright (c) 2020 by Elasticsearch BV.
604  
605 This is free software, licensed under:
606  
607 The Apache License, Version 2.0, January 2004
608  
609 =cut
610  
611 __END__