| 1 | package Search::Elasticsearch::Client::0_90::Scroll; | ||
| 2 | $Search::Elasticsearch::Client::0_90::Scroll::VERSION = '6.81'; | ||
| 3 | use Moo; | ||
| 4 | use Search::Elasticsearch::Util qw(parse_params throw); | ||
| 5 | use namespace::clean; | ||
| 6 | |||
| 7 | has '_buffer' => ( is => 'ro' ); | ||
| 8 | |||
| 9 | with 'Search::Elasticsearch::Role::Is_Sync', | ||
| 10 | 'Search::Elasticsearch::Client::0_90::Role::Scroll'; | ||
| 11 | |||
| 12 | #=================================== | ||
| 13 | sub BUILDARGS { | ||
| 14 | #=================================== | ||
| 15 | my ( $class, $params ) = parse_params(@_); | ||
| 16 | my $es = delete $params->{es}; | ||
| 17 | my $scroll = $params->{scroll} ||= '1m'; | ||
| 18 | |||
| 19 | throw( 'Param', | ||
| 20 | 'The (scroll_in_body) parameter has been replaced by (scroll_in_qs)' ) | ||
| 21 | if exists $params->{scroll_in_body}; | ||
| 22 | |||
| 23 | my $scroll_in_qs = delete $params->{scroll_in_qs}; | ||
| 24 | my $results = $es->search($params); | ||
| 25 | |||
| 26 | my $total = $results->{hits}{total}; | ||
| 27 | |||
| 28 | return { | ||
| 29 | es => $es, | ||
| 30 | scroll => $scroll, | ||
| 31 | scroll_in_qs => $scroll_in_qs, | ||
| 32 | aggregations => $results->{aggregations}, | ||
| 33 | facets => $results->{facets}, | ||
| 34 | suggest => $results->{suggest}, | ||
| 35 | took => $results->{took}, | ||
| 36 | total_took => $results->{took}, | ||
| 37 | total => $total, | ||
| 38 | max_score => $results->{hits}{max_score}, | ||
| 39 | _buffer => $results->{hits}{hits}, | ||
| 40 | $total | ||
| 41 | ? ( _scroll_id => $results->{_scroll_id} ) | ||
| 42 | : ( is_finished => 1 ) | ||
| 43 | }; | ||
| 44 | } | ||
| 45 | |||
| 46 | #=================================== | ||
| 47 | sub next { | ||
| 48 | #=================================== | ||
| 49 | my ( $self, $n ) = @_; | ||
| 50 | $n ||= 1; | ||
| 51 | while ( $self->_has_scroll_id and $self->buffer_size < $n ) { | ||
| 52 | $self->refill_buffer; | ||
| 53 | } | ||
| 54 | my @return = splice( @{ $self->_buffer }, 0, $n ); | ||
| 55 | $self->finish if @return < $n; | ||
| 56 | return wantarray ? @return : $return[-1]; | ||
| 57 | } | ||
| 58 | |||
| 59 | #=================================== | ||
| 60 | sub drain_buffer { | ||
| 61 | #=================================== | ||
| 62 | my $self = shift; | ||
| 63 | return splice( @{ $self->_buffer } ); | ||
| 64 | } | ||
| 65 | |||
| 66 | #=================================== | ||
| 67 | sub buffer_size { 0 + @{ shift->_buffer } } | ||
| 68 | #=================================== | ||
| 69 | |||
| 70 | #=================================== | ||
| 71 | sub refill_buffer { | ||
| 72 | #=================================== | ||
| 73 | my $self = shift; | ||
| 74 | |||
| 75 | return 0 if $self->is_finished; | ||
| 76 | |||
| 77 | my $buffer = $self->_buffer; | ||
| 78 | my $scroll_id = $self->_scroll_id | ||
| 79 | || return 0 + @$buffer; | ||
| 80 | |||
| 81 | my $results = $self->scroll_request; | ||
| 82 | |||
| 83 | my $hits = $results->{hits}{hits}; | ||
| 84 | $self->_set_total_took( $self->total_took + $results->{took} ); | ||
| 85 | |||
| 86 | if ( @$hits == 0 ) { | ||
| 87 | $self->_clear_scroll_id; | ||
| 88 | } | ||
| 89 | else { | ||
| 90 | $self->_set__scroll_id( $results->{_scroll_id} ); | ||
| 91 | push @$buffer, @$hits; | ||
| 92 | } | ||
| 93 | $self->finish if @$buffer == 0; | ||
| 94 | return 0 + @$buffer; | ||
| 95 | } | ||
| 96 | |||
| 97 | #=================================== | ||
| 98 | sub finish { | ||
| 99 | #=================================== | ||
| 100 | my $self = shift; | ||
| 101 | return if $self->is_finished || $self->_pid != $$; | ||
| 102 | |||
| 103 | $self->_set_is_finished(1); | ||
| 104 | @{ $self->_buffer } = (); | ||
| 105 | |||
| 106 | my $scroll_id = $self->_scroll_id or return; | ||
| 107 | $self->_clear_scroll_id; | ||
| 108 | |||
| 109 | my %args | ||
| 110 | = $self->scroll_in_qs | ||
| 111 | ? ( scroll_id => $scroll_id ) | ||
| 112 | : ( body => $scroll_id ); | ||
| 113 | eval { $self->es->clear_scroll(%args) }; | ||
| 114 | return 1; | ||
| 115 | } | ||
| 116 | |||
| 117 | 1; | ||
| 118 | |||
| 119 | =pod | ||
| 120 | |||
| 121 | =encoding UTF-8 | ||
| 122 | |||
| 123 | =head1 NAME | ||
| 124 | |||
| 125 | Search::Elasticsearch::Client::0_90::Scroll - A helper module for scrolled searches | ||
| 126 | |||
| 127 | =head1 VERSION | ||
| 128 | |||
| 129 | version 6.81 | ||
| 130 | |||
| 131 | =head1 SYNOPSIS | ||
| 132 | |||
| 133 | use Search::Elasticsearch; | ||
| 134 | |||
| 135 | my $es = Search::Elasticsearch->new; | ||
| 136 | |||
| 137 | my $scroll = $es->scroll_helper( | ||
| 138 | index => 'my_index', | ||
| 139 | search_type => 'scan', | ||
| 140 | size => 500 | ||
| 141 | ); | ||
| 142 | |||
| 143 | say "Total hits: ". $scroll->total; | ||
| 144 | |||
| 145 | while (my $doc = $scroll->next) { | ||
| 146 | # do something | ||
| 147 | } | ||
| 148 | |||
| 149 | =head1 DESCRIPTION | ||
| 150 | |||
| 151 | A I<scrolled search> is a search that allows you to keep pulling results | ||
| 152 | until there are no more matching results, much like a cursor in an SQL | ||
| 153 | database. | ||
| 154 | |||
| 155 | Unlike paginating through results (with the C<from> parameter in | ||
| 156 | L<search()|Search::Elasticsearch::Client::0_90::Direct/search()>), | ||
| 157 | scrolled searches take a snapshot of the current state of the index. Even | ||
| 158 | if you keep adding new documents to the index or updating existing documents, | ||
| 159 | a scrolled search will only see the index as it was when the search began. | ||
| 160 | |||
| 161 | This module is a helper utility that wraps the functionality of the | ||
| 162 | L<search()|Search::Elasticsearch::Client::0_90::Direct/search()> and | ||
| 163 | L<scroll()|Search::Elasticsearch::Client::0_90::Direct/scroll()> methods to make | ||
| 164 | them easier to use. | ||
| 165 | |||
| 166 | B<IMPORTANT>: Deep scrolling can be expensive. See L</DEEP SCROLLING> | ||
| 167 | for more. | ||
| 168 | |||
| 169 | This class does L<Search::Elasticsearch::Client::0_90::Role::Scroll> and | ||
| 170 | L<Search::Elasticsearch::Role::Is_Sync>. | ||
| 171 | |||
| 172 | =head1 USE CASES | ||
| 173 | |||
| 174 | There are two primary use cases: | ||
| 175 | |||
| 176 | =head2 Pulling enough results | ||
| 177 | |||
| 178 | Perhaps you want to group your results by some field, and you don't know | ||
| 179 | exactly how many results you will need in order to return 10 grouped | ||
| 180 | results. With a scrolled search you can keep pulling more results | ||
| 181 | until you have enough. For instance, you can search emails in a mailing | ||
| 182 | list, and return results grouped by C<thread_id>: | ||
| 183 | |||
| 184 | my (%groups,@results); | ||
| 185 | |||
| 186 | my $scroll = $es->scroll_helper( | ||
| 187 | index => 'my_emails', | ||
| 188 | type => 'email', | ||
| 189 | body => { query => {... some query ... }} | ||
| 190 | ); | ||
| 191 | |||
| 192 | my $doc; | ||
| 193 | while (@results < 10 and $doc = $scroll->next) { | ||
| 194 | |||
| 195 | my $thread = $doc->{_source}{thread_id}; | ||
| 196 | |||
| 197 | unless ($groups{$thread}) { | ||
| 198 | $groups{$thread} = []; | ||
| 199 | push @results, $groups{$thread}; | ||
| 200 | } | ||
| 201 | push @{$groups{$thread}},$doc; | ||
| 202 | |||
| 203 | } | ||
| 204 | |||
| 205 | =head2 Extracting all documents | ||
| 206 | |||
| 207 | Often you will want to extract all (or a subset of) documents in an index. | ||
| 208 | If you want to change your type mappings, you will need to reindex all of your | ||
| 209 | data. Or perhaps you want to move a subset of the data in one index into | ||
| 210 | a new dedicated index. In these cases, you don't care about sort | ||
| 211 | order, you just want to retrieve all documents which match a query, and do | ||
| 212 | something with them. For instance, to retrieve all the docs for a particular | ||
| 213 | C<client_id>: | ||
| 214 | |||
| 215 | my $scroll = $es->scroll_helper( | ||
| 216 | index => 'my_index', | ||
| 217 | search_type => 'scan', # important! | ||
| 218 | size => 500, | ||
| 219 | body => { | ||
| 220 | query => { | ||
| 221 | match => { | ||
| 222 | client_id => 123 | ||
| 223 | } | ||
| 224 | } | ||
| 225 | } | ||
| 226 | ); | ||
| 227 | |||
| 228 | while (my $doc = $scroll->next) { | ||
| 229 | # do something | ||
| 230 | } | ||
| 231 | |||
| 232 | Very often the I<something> that you will want to do with these results | ||
| 233 | involves bulk-indexing them into a new index. The easiest way to | ||
| 234 | marry a scrolled search with bulk indexing is to use the | ||
| 235 | L<Search::Elasticsearch::Client::0_90::Bulk/reindex()> method. | ||
| 236 | |||
| 237 | =head1 DEEP SCROLLING | ||
| 238 | |||
| 239 | Deep scrolling (and deep pagination) are very expensive in a distributed | ||
| 240 | environment, and the reason they are expensive is that results need to | ||
| 241 | be sorted in a global order. | ||
| 242 | |||
| 243 | For example, if we have an index with 5 shards, and we request the first | ||
| 244 | 10 results, each shard has to return its top 10, and then the I<requesting | ||
| 245 | node> (the node that is handling the search request) has to resort these | ||
| 246 | 50 results to return a global top 10. Now, if we request page 1,000 | ||
| 247 | (ie results 10,001 .. 10,010), then each shard has to return 10,010 results, | ||
| 248 | and the requesting node has to sort through 50,050 results just to return | ||
| 249 | 10 of them! | ||
| 250 | |||
| 251 | You can see how this can get very heavy very quickly. This is the reason that | ||
| 252 | web search engines never return more than 1,000 results. | ||
| 253 | |||
| 254 | =head2 Disable sorting for efficient scrolling | ||
| 255 | |||
| 256 | The problem with deep scrolling is the sorting phase. If we disable sorting, | ||
| 257 | then we can happily scroll through millions of documents efficiently. The | ||
| 258 | way to do this is to set C<search_type> to C<scan>: | ||
| 259 | |||
| 260 | my $scroll = $es->scroll_helper( | ||
| 261 | search_type => 'scan', | ||
| 262 | size => 500, | ||
| 263 | ); | ||
| 264 | |||
| 265 | Scanning disables sorting and will just return C<size> results from each | ||
| 266 | shard until there are no more results to return. B<Note>: this means | ||
| 267 | that, when querying an index with 5 shards, the scrolled search | ||
| 268 | will pull C<size * 5> results at a time. If you have large documents or | ||
| 269 | are memory constrained, you will need to take this into account. | ||
| 270 | |||
| 271 | =head1 METHODS | ||
| 272 | |||
| 273 | =head2 C<new()> | ||
| 274 | |||
| 275 | use Search::Elasticsearch; | ||
| 276 | |||
| 277 | my $es = Search::Elasticsearch->new(...); | ||
| 278 | my $scroll = $es->scroll_helper( | ||
| 279 | scroll => '1m', # optional | ||
| 280 | scroll_in_qs => 0|1, # optional | ||
| 281 | %search_params | ||
| 282 | ); | ||
| 283 | |||
| 284 | The L<Search::Elasticsearch::Client::0_90::Direct/scroll_helper()> method loads | ||
| 285 | L<Search::Elasticsearch::Client::0_90::Scroll> class and calls L</new()>, | ||
| 286 | passing in any arguments. | ||
| 287 | |||
| 288 | You can specify a C<scroll> duration (which defaults to C<"1m">) and | ||
| 289 | C<scroll_in_qs> (which defaults to C<false>). Any other parameters are | ||
| 290 | passed directly to L<Search::Elasticsearch::Client::0_90::Direct/search()>. | ||
| 291 | |||
| 292 | The C<scroll> duration tells Elasticearch how long it should keep the scroll | ||
| 293 | alive. B<Note>: this duration doesn't need to be long enough to process | ||
| 294 | all results, just long enough to process a single B<batch> of results. | ||
| 295 | The expiry gets renewed for another C<scroll> period every time new | ||
| 296 | a new batch of results is retrieved from the cluster. | ||
| 297 | |||
| 298 | By default, the C<scroll_id> is passed as the C<body> to the | ||
| 299 | L<scroll|Search::Elasticsearch::Client::0_90::Direct/scroll()> request. | ||
| 300 | To send it in the query string instead, set C<scroll_in_qs> to a true value, | ||
| 301 | but be aware: when querying very many indices, the scroll ID can become | ||
| 302 | too long for intervening proxies. | ||
| 303 | |||
| 304 | The C<scroll> request uses C<GET> by default. To use C<POST> instead, | ||
| 305 | set L<send_get_body_as|Search::Elasticsearch::Transport/send_get_body_as> to | ||
| 306 | C<POST>. | ||
| 307 | |||
| 308 | =head2 C<next()> | ||
| 309 | |||
| 310 | $doc = $scroll->next; | ||
| 311 | @docs = $scroll->next($num); | ||
| 312 | |||
| 313 | The C<next()> method returns the next result, or the next C<$num> results | ||
| 314 | (pulling more results if required). If all results have been exhausted, | ||
| 315 | it returns an empty list. | ||
| 316 | |||
| 317 | =head2 C<drain_buffer()> | ||
| 318 | |||
| 319 | @docs = $scroll->drain_buffer; | ||
| 320 | |||
| 321 | The C<drain_buffer()> method returns all of the documents currently in the | ||
| 322 | buffer, without fetching any more from the cluster. | ||
| 323 | |||
| 324 | =head2 C<refill_buffer()> | ||
| 325 | |||
| 326 | $total = $scroll->refill_buffer; | ||
| 327 | |||
| 328 | The C<refill_buffer()> method fetches the next batch of results from the | ||
| 329 | cluster, stores them in the buffer, and returns the total number of docs | ||
| 330 | currently in the buffer. | ||
| 331 | |||
| 332 | =head2 C<buffer_size()> | ||
| 333 | |||
| 334 | $total = $scroll->buffer_size; | ||
| 335 | |||
| 336 | The C<buffer_size()> method returns the total number of docs currently in | ||
| 337 | the buffer. | ||
| 338 | |||
| 339 | =head2 C<finish()> | ||
| 340 | |||
| 341 | $scroll->finish; | ||
| 342 | |||
| 343 | The C<finish()> method clears out the buffer, sets L</is_finished()> to C<true> | ||
| 344 | and tries to clear the C<scroll_id> on Elasticsearch. This API is only | ||
| 345 | supported since v0.90.5, but the call to C<clear_scroll> is wrapped in an | ||
| 346 | C<eval> so the C<finish()> method can be safely called with any version | ||
| 347 | of Elasticsearch. | ||
| 348 | |||
| 349 | When the C<$scroll> instance goes out of scope, L</finish()> is called | ||
| 350 | automatically if required. | ||
| 351 | |||
| 352 | =head2 C<is_finished()> | ||
| 353 | |||
| 354 | $bool = $scroll->is_finished; | ||
| 355 | |||
| 356 | A flag which returns C<true> if all results have been processed or | ||
| 357 | L</finish()> has been called. | ||
| 358 | |||
| 359 | =head1 INFO ACCESSORS | ||
| 360 | |||
| 361 | The information from the original search is returned via the following | ||
| 362 | accessors: | ||
| 363 | |||
| 364 | =head2 C<total> | ||
| 365 | |||
| 366 | The total number of documents that matched your query. | ||
| 367 | |||
| 368 | =head2 C<max_score> | ||
| 369 | |||
| 370 | The maximum score of any documents in your query. | ||
| 371 | |||
| 372 | =head2 C<aggregations> | ||
| 373 | |||
| 374 | Any aggregations that were specified, or C<undef> | ||
| 375 | |||
| 376 | =head2 C<facets> | ||
| 377 | |||
| 378 | Any facets that were specified, or C<undef> | ||
| 379 | |||
| 380 | =head2 C<suggest> | ||
| 381 | |||
| 382 | Any suggestions that were specified, or C<undef> | ||
| 383 | |||
| 384 | =head2 C<took> | ||
| 385 | |||
| 386 | How long the original search took, in milliseconds | ||
| 387 | |||
| 388 | =head2 C<took_total> | ||
| 389 | |||
| 390 | How long the original search plus all subsequent batches took, in milliseconds. | ||
| 391 | |||
| 392 | =head1 SEE ALSO | ||
| 393 | |||
| 394 | =over | ||
| 395 | |||
| 396 | =item * L<Search::Elasticsearch::Client::0_90::Bulk/reindex()> | ||
| 397 | |||
| 398 | =item * L<Search::Elasticsearch::Client::0_90::Direct/search()> | ||
| 399 | |||
| 400 | =item * L<Search::Elasticsearch::Client::0_90::Direct/scroll()> | ||
| 401 | |||
| 402 | =back | ||
| 403 | |||
| 404 | =head1 AUTHOR | ||
| 405 | |||
| 406 | Enrico Zimuel <enrico.zimuel@elastic.co> | ||
| 407 | |||
| 408 | =head1 COPYRIGHT AND LICENSE | ||
| 409 | |||
| 410 | This software is Copyright (c) 2020 by Elasticsearch BV. | ||
| 411 | |||
| 412 | This is free software, licensed under: | ||
| 413 | |||
| 414 | The Apache License, Version 2.0, January 2004 | ||
| 415 | |||
| 416 | =cut | ||
| 417 | |||
| 418 | __END__ |