0_90/Scroll.pm untested

filter: Statementstmt 0.0% 0 / 58 Branchbran 0.0% 0 / 18 Conditioncond 0.0% 0 / 14 MC/DCmcdc 0.0% 0 / 8 Subroutinesub 0.0% 0 / 9 Podpod 0.0% 0 / 6 totaltotal 0.0% 0 / 113 CC 10 SCAR 24.0
CC
10
Cov
0%
refill_buffer
20.0
BUILDARGS
12.0
next
12.0
CRAP 11.0
1 package Search::Elasticsearch::Client::0_90::Scroll;
2 $Search::Elasticsearch::Client::0_90::Scroll::VERSION = '6.81';
3 use Moo;
4 use Search::Elasticsearch::Util qw(parse_params throw);
5 use namespace::clean;
6  
7 has '_buffer' => ( is => 'ro' );
8  
9 with 'Search::Elasticsearch::Role::Is_Sync',
10 'Search::Elasticsearch::Client::0_90::Role::Scroll';
11  
12 #===================================
13 sub BUILDARGS {
14 #===================================
15 my ( $class, $params ) = parse_params(@_);
16 my $es = delete $params->{es};
17 my $scroll = $params->{scroll} ||= '1m';
18  
19 throw( 'Param',
20 'The (scroll_in_body) parameter has been replaced by (scroll_in_qs)' )
21 if exists $params->{scroll_in_body};
22  
23 my $scroll_in_qs = delete $params->{scroll_in_qs};
24 my $results = $es->search($params);
25  
26 my $total = $results->{hits}{total};
27  
28 return {
29 es => $es,
30 scroll => $scroll,
31 scroll_in_qs => $scroll_in_qs,
32 aggregations => $results->{aggregations},
33 facets => $results->{facets},
34 suggest => $results->{suggest},
35 took => $results->{took},
36 total_took => $results->{took},
37 total => $total,
38 max_score => $results->{hits}{max_score},
39 _buffer => $results->{hits}{hits},
40 $total
41 ? ( _scroll_id => $results->{_scroll_id} )
42 : ( is_finished => 1 )
43 };
44 }
45  
46 #===================================
47 sub next {
48 #===================================
49 my ( $self, $n ) = @_;
50 $n ||= 1;
51 while ( $self->_has_scroll_id and $self->buffer_size < $n ) {
52 $self->refill_buffer;
53 }
54 my @return = splice( @{ $self->_buffer }, 0, $n );
55 $self->finish if @return < $n;
56 return wantarray ? @return : $return[-1];
57 }
58  
59 #===================================
60 sub drain_buffer {
61 #===================================
62 my $self = shift;
63 return splice( @{ $self->_buffer } );
64 }
65  
66 #===================================
67 sub buffer_size { 0 + @{ shift->_buffer } }
68 #===================================
69  
70 #===================================
71 sub refill_buffer {
72 #===================================
73 my $self = shift;
74  
75 return 0 if $self->is_finished;
76  
77 my $buffer = $self->_buffer;
78 my $scroll_id = $self->_scroll_id
79 || return 0 + @$buffer;
80  
81 my $results = $self->scroll_request;
82  
83 my $hits = $results->{hits}{hits};
84 $self->_set_total_took( $self->total_took + $results->{took} );
85  
86 if ( @$hits == 0 ) {
87 $self->_clear_scroll_id;
88 }
89 else {
90 $self->_set__scroll_id( $results->{_scroll_id} );
91 push @$buffer, @$hits;
92 }
93 $self->finish if @$buffer == 0;
94 return 0 + @$buffer;
95 }
96  
97 #===================================
98 sub finish {
99 #===================================
100 my $self = shift;
101 return if $self->is_finished || $self->_pid != $$;
102  
103 $self->_set_is_finished(1);
104 @{ $self->_buffer } = ();
105  
106 my $scroll_id = $self->_scroll_id or return;
107 $self->_clear_scroll_id;
108  
109 my %args
110 = $self->scroll_in_qs
111 ? ( scroll_id => $scroll_id )
112 : ( body => $scroll_id );
113 eval { $self->es->clear_scroll(%args) };
114 return 1;
115 }
116  
117 1;
118  
119 =pod
120  
121 =encoding UTF-8
122  
123 =head1 NAME
124  
125 Search::Elasticsearch::Client::0_90::Scroll - A helper module for scrolled searches
126  
127 =head1 VERSION
128  
129 version 6.81
130  
131 =head1 SYNOPSIS
132  
133 use Search::Elasticsearch;
134  
135 my $es = Search::Elasticsearch->new;
136  
137 my $scroll = $es->scroll_helper(
138 index => 'my_index',
139 search_type => 'scan',
140 size => 500
141 );
142  
143 say "Total hits: ". $scroll->total;
144  
145 while (my $doc = $scroll->next) {
146 # do something
147 }
148  
149 =head1 DESCRIPTION
150  
151 A I<scrolled search> is a search that allows you to keep pulling results
152 until there are no more matching results, much like a cursor in an SQL
153 database.
154  
155 Unlike paginating through results (with the C<from> parameter in
156 L<search()|Search::Elasticsearch::Client::0_90::Direct/search()>),
157 scrolled searches take a snapshot of the current state of the index. Even
158 if you keep adding new documents to the index or updating existing documents,
159 a scrolled search will only see the index as it was when the search began.
160  
161 This module is a helper utility that wraps the functionality of the
162 L<search()|Search::Elasticsearch::Client::0_90::Direct/search()> and
163 L<scroll()|Search::Elasticsearch::Client::0_90::Direct/scroll()> methods to make
164 them easier to use.
165  
166 B<IMPORTANT>: Deep scrolling can be expensive. See L</DEEP SCROLLING>
167 for more.
168  
169 This class does L<Search::Elasticsearch::Client::0_90::Role::Scroll> and
170 L<Search::Elasticsearch::Role::Is_Sync>.
171  
172 =head1 USE CASES
173  
174 There are two primary use cases:
175  
176 =head2 Pulling enough results
177  
178 Perhaps you want to group your results by some field, and you don't know
179 exactly how many results you will need in order to return 10 grouped
180 results. With a scrolled search you can keep pulling more results
181 until you have enough. For instance, you can search emails in a mailing
182 list, and return results grouped by C<thread_id>:
183  
184 my (%groups,@results);
185  
186 my $scroll = $es->scroll_helper(
187 index => 'my_emails',
188 type => 'email',
189 body => { query => {... some query ... }}
190 );
191  
192 my $doc;
193 while (@results < 10 and $doc = $scroll->next) {
194  
195 my $thread = $doc->{_source}{thread_id};
196  
197 unless ($groups{$thread}) {
198 $groups{$thread} = [];
199 push @results, $groups{$thread};
200 }
201 push @{$groups{$thread}},$doc;
202  
203 }
204  
205 =head2 Extracting all documents
206  
207 Often you will want to extract all (or a subset of) documents in an index.
208 If you want to change your type mappings, you will need to reindex all of your
209 data. Or perhaps you want to move a subset of the data in one index into
210 a new dedicated index. In these cases, you don't care about sort
211 order, you just want to retrieve all documents which match a query, and do
212 something with them. For instance, to retrieve all the docs for a particular
213 C<client_id>:
214  
215 my $scroll = $es->scroll_helper(
216 index => 'my_index',
217 search_type => 'scan', # important!
218 size => 500,
219 body => {
220 query => {
221 match => {
222 client_id => 123
223 }
224 }
225 }
226 );
227  
228 while (my $doc = $scroll->next) {
229 # do something
230 }
231  
232 Very often the I<something> that you will want to do with these results
233 involves bulk-indexing them into a new index. The easiest way to
234 marry a scrolled search with bulk indexing is to use the
235 L<Search::Elasticsearch::Client::0_90::Bulk/reindex()> method.
236  
237 =head1 DEEP SCROLLING
238  
239 Deep scrolling (and deep pagination) are very expensive in a distributed
240 environment, and the reason they are expensive is that results need to
241 be sorted in a global order.
242  
243 For example, if we have an index with 5 shards, and we request the first
244 10 results, each shard has to return its top 10, and then the I<requesting
245 node> (the node that is handling the search request) has to resort these
246 50 results to return a global top 10. Now, if we request page 1,000
247 (ie results 10,001 .. 10,010), then each shard has to return 10,010 results,
248 and the requesting node has to sort through 50,050 results just to return
249 10 of them!
250  
251 You can see how this can get very heavy very quickly. This is the reason that
252 web search engines never return more than 1,000 results.
253  
254 =head2 Disable sorting for efficient scrolling
255  
256 The problem with deep scrolling is the sorting phase. If we disable sorting,
257 then we can happily scroll through millions of documents efficiently. The
258 way to do this is to set C<search_type> to C<scan>:
259  
260 my $scroll = $es->scroll_helper(
261 search_type => 'scan',
262 size => 500,
263 );
264  
265 Scanning disables sorting and will just return C<size> results from each
266 shard until there are no more results to return. B<Note>: this means
267 that, when querying an index with 5 shards, the scrolled search
268 will pull C<size * 5> results at a time. If you have large documents or
269 are memory constrained, you will need to take this into account.
270  
271 =head1 METHODS
272  
273 =head2 C<new()>
274  
275 use Search::Elasticsearch;
276  
277 my $es = Search::Elasticsearch->new(...);
278 my $scroll = $es->scroll_helper(
279 scroll => '1m', # optional
280 scroll_in_qs => 0|1, # optional
281 %search_params
282 );
283  
284 The L<Search::Elasticsearch::Client::0_90::Direct/scroll_helper()> method loads
285 L<Search::Elasticsearch::Client::0_90::Scroll> class and calls L</new()>,
286 passing in any arguments.
287  
288 You can specify a C<scroll> duration (which defaults to C<"1m">) and
289 C<scroll_in_qs> (which defaults to C<false>). Any other parameters are
290 passed directly to L<Search::Elasticsearch::Client::0_90::Direct/search()>.
291  
292 The C<scroll> duration tells Elasticearch how long it should keep the scroll
293 alive. B<Note>: this duration doesn't need to be long enough to process
294 all results, just long enough to process a single B<batch> of results.
295 The expiry gets renewed for another C<scroll> period every time new
296 a new batch of results is retrieved from the cluster.
297  
298 By default, the C<scroll_id> is passed as the C<body> to the
299 L<scroll|Search::Elasticsearch::Client::0_90::Direct/scroll()> request.
300 To send it in the query string instead, set C<scroll_in_qs> to a true value,
301 but be aware: when querying very many indices, the scroll ID can become
302 too long for intervening proxies.
303  
304 The C<scroll> request uses C<GET> by default. To use C<POST> instead,
305 set L<send_get_body_as|Search::Elasticsearch::Transport/send_get_body_as> to
306 C<POST>.
307  
308 =head2 C<next()>
309  
310 $doc = $scroll->next;
311 @docs = $scroll->next($num);
312  
313 The C<next()> method returns the next result, or the next C<$num> results
314 (pulling more results if required). If all results have been exhausted,
315 it returns an empty list.
316  
317 =head2 C<drain_buffer()>
318  
319 @docs = $scroll->drain_buffer;
320  
321 The C<drain_buffer()> method returns all of the documents currently in the
322 buffer, without fetching any more from the cluster.
323  
324 =head2 C<refill_buffer()>
325  
326 $total = $scroll->refill_buffer;
327  
328 The C<refill_buffer()> method fetches the next batch of results from the
329 cluster, stores them in the buffer, and returns the total number of docs
330 currently in the buffer.
331  
332 =head2 C<buffer_size()>
333  
334 $total = $scroll->buffer_size;
335  
336 The C<buffer_size()> method returns the total number of docs currently in
337 the buffer.
338  
339 =head2 C<finish()>
340  
341 $scroll->finish;
342  
343 The C<finish()> method clears out the buffer, sets L</is_finished()> to C<true>
344 and tries to clear the C<scroll_id> on Elasticsearch. This API is only
345 supported since v0.90.5, but the call to C<clear_scroll> is wrapped in an
346 C<eval> so the C<finish()> method can be safely called with any version
347 of Elasticsearch.
348  
349 When the C<$scroll> instance goes out of scope, L</finish()> is called
350 automatically if required.
351  
352 =head2 C<is_finished()>
353  
354 $bool = $scroll->is_finished;
355  
356 A flag which returns C<true> if all results have been processed or
357 L</finish()> has been called.
358  
359 =head1 INFO ACCESSORS
360  
361 The information from the original search is returned via the following
362 accessors:
363  
364 =head2 C<total>
365  
366 The total number of documents that matched your query.
367  
368 =head2 C<max_score>
369  
370 The maximum score of any documents in your query.
371  
372 =head2 C<aggregations>
373  
374 Any aggregations that were specified, or C<undef>
375  
376 =head2 C<facets>
377  
378 Any facets that were specified, or C<undef>
379  
380 =head2 C<suggest>
381  
382 Any suggestions that were specified, or C<undef>
383  
384 =head2 C<took>
385  
386 How long the original search took, in milliseconds
387  
388 =head2 C<took_total>
389  
390 How long the original search plus all subsequent batches took, in milliseconds.
391  
392 =head1 SEE ALSO
393  
394 =over
395  
396 =item * L<Search::Elasticsearch::Client::0_90::Bulk/reindex()>
397  
398 =item * L<Search::Elasticsearch::Client::0_90::Direct/search()>
399  
400 =item * L<Search::Elasticsearch::Client::0_90::Direct/scroll()>
401  
402 =back
403  
404 =head1 AUTHOR
405  
406 Enrico Zimuel <enrico.zimuel@elastic.co>
407  
408 =head1 COPYRIGHT AND LICENSE
409  
410 This software is Copyright (c) 2020 by Elasticsearch BV.
411  
412 This is free software, licensed under:
413  
414 The Apache License, Version 2.0, January 2004
415  
416 =cut
417  
418 __END__