script/es-search.pl untested

filter: Statementstmt 0.0% 0 / 501 Branchbran 0.0% 0 / 308 Conditioncond 0.0% 0 / 142 MC/DCmcdc 0.0% 0 / 85 Subroutinesub 0.0% 0 / 20 Podpod 0.0% 0 / 6 totaltotal 0.0% 0 / 1062 CC 50 SCAR 61.3
CC
50
Cov
0%
display_aggregations
992.0
show_fields
56.0
show_bases
56.0
CRAP 460.3
1 #!/usr/local/bin/perl
2 # PODNAME: es-search.pl
3 # ABSTRACT: Provides a CLI for quick searches of data in ElasticSearch daily indexes
4 use strict;
5 use warnings;
6  
7 use App::ElasticSearch::Utilities qw(:all);
8 use App::ElasticSearch::Utilities::Query;
9 use App::ElasticSearch::Utilities::QueryString;
10 use Carp;
11 use CLI::Helpers qw(:all);
12 use Getopt::Long qw(:config no_ignore_case no_ignore_case_always);
13 use JSON::MaybeXS qw(:legacy);
14 use Pod::Usage;
15 use POSIX qw(strftime);
16 use Ref::Util qw(is_ref is_arrayref is_hashref);
17 use Time::HiRes qw(sleep time);
18 use YAML::XS;
19 local $YAML::XS::Boolean = "JSON::PP";
20  
21 #------------------------------------------------------------------------#
22 # Argument Parsing
23 my %OPT;
24 GetOptions(\%OPT, qw(
25 all
26 asc
27 bases
28 bg-filter=s
29 by=s
30 desc
31 exists=s@
32 fields
33 filter
34 format=s
35 help|h
36 json|jq
37 manual|m
38 match-all
39 max-batch-size=i
40 missing=s@
41 no-decorators|no-header
42 no-implications|no-imply
43 precision=i
44 prefix=s@
45 pretty
46 show=s@
47 size|n|limit=i
48 sort=s
49 tail
50 timestamp=s
51 top=s
52 interval=s
53 with=s@
54 with-missing
55 or
56 ));
57  
58 # Search string is the rest of the argument string
59 my $context = $OPT{filter} ? 'filter' : 'must';
60 my $qs = App::ElasticSearch::Utilities::QueryString->new(
61 $OPT{filter} ? (context => 'filter') : (),
62 default_join => $OPT{or} ? 'OR' : 'AND',
63 );
64 my $q = exists $OPT{'match-all'} && $OPT{'match-all'}
65 ? App::ElasticSearch::Utilities::Query->new($context => { match_all => {} })
66 : $qs->expand_query_string(@ARGV);
67  
68 $q->set_timeout('10s');
69 $q->set_scroll('30s');
70  
71 if( exists $OPT{prefix} ){
72 foreach my $prefix (@{ $OPT{prefix} }) {
73 my ($f,$v) = split /:/, $prefix, 2;
74 next unless $f && $v;
75 $q->add_bool( $context => { prefix => { $f => $v } } );
76 }
77 }
78  
79 #------------------------------------------------------------------------#
80 # Documentation
81 pod2usage({-sections => 'SYNOPSIS'}) if $OPT{help};
82 pod2usage(-exitval => 0, -verbose => 2) if $OPT{manual};
83 my $unknown_options = join ', ', grep /^--/, @ARGV;
84 pod2usage({-exitval => 1, -sections => 'SYNOPSIS', -msg =>"Unknown option(s): $unknown_options"}) if $unknown_options;
85  
86 #--------------------------------------------------------------------------#
87 # Information Gathering Routines
88 if( $OPT{bases} ) {
89 show_bases();
90 exit 0;
91 }
92 #--------------------------------------------------------------------------#
93 # App Config
94 my %CONFIG = (
95 size => ($OPT{size} && $OPT{size} > 0) ? int($OPT{size}) : 20,
96 format => $OPT{json} ? 'json'
97 : $OPT{format} ? lc $OPT{format}
98 : 'yaml',
99 'max-batch-size' => $OPT{'max-batch-size'} || 50,
100 precision => $OPT{precision} || 3,
101 $OPT{timestamp} ? ( timestamp => $OPT{timestamp} ) : (),
102 );
103 $OPT{'no-decorators'} = 1 if $CONFIG{format} eq 'json';
104 $CONFIG{pretty} = $OPT{pretty} ? 1
105 : $CONFIG{format} =~ /pretty/ ? 1
106 : 0;
107 my $JSON = JSON->new->utf8->canonical;
108 #------------------------------------------------------------------------#
109 # Handle Indices
110 my $ORDER = exists $OPT{asc} && $OPT{asc} ? 'asc' : 'desc';
111 $ORDER = 'asc' if exists $OPT{tail};
112 my %by_age = ();
113 my %indices = map { $_ => (es_index_days_old($_) || 0) } es_indices();
114 die "# Failed to retrieve any indices using your paramaters." unless keys %indices;
115 my %FIELDS = ();
116 my $TimeStampCheck=0;
117 foreach my $index (sort by_index_age keys %indices) {
118 my $age = $indices{$index};
119 $by_age{$age} ||= [];
120 push @{ $by_age{$age} }, $index;
121 my $fields = es_index_fields($index);
122 foreach my $k ( keys %{ $fields } ) {
123 $FIELDS{$k} = $fields->{$k}
124 unless $FIELDS{$k};
125 }
126 # Lookup the Index in our local YAML
127 if( !$TimeStampCheck ) {
128 $TimeStampCheck++;
129 $CONFIG{timestamp} ||= es_local_index_meta(timestamp => $index);
130 }
131 }
132  
133 # Set fields so we know how to construct complex aggs
134 $q->fields_meta( \%FIELDS );
135  
136 #------------------------------------------------------------------------#
137 # Figure out the timestamp
138 $CONFIG{timestamp} ||= es_globals('timestamp') || '@timestamp';
139 debug_var(\%by_age);
140 my @AGES = sort { $ORDER eq 'asc' ? $b <=> $a : $a <=> $b } keys %by_age;
141 # Figure out if we summarize
142 $CONFIG{summary} = @AGES > 1 && $OPT{top} && ( !$OPT{by} && !$OPT{with} && !$OPT{interval} );
143 debug({color=>"cyan"}, "Fields discovered.");
144  
145 if( $OPT{fields} ) {
146 show_fields();
147 exit 0;
148 }
149 # Attempt date autodiscovery
150 if( !exists $FIELDS{$CONFIG{timestamp}} ) {
151 my @dates = grep { $FIELDS{$_}->{type} eq 'date' } keys %FIELDS;
152 if( @dates == 0 ) {
153 output({color=>'red',stderr=>1},"FATAL: No date fields found in the indices specified" );
154 exit 1;
155 }
156 elsif( @dates == 1 ) {
157 output({color=>'yellow',stderr=>1}, "WARNING: Timestamp field '$CONFIG{timestamp}' not found, using '$dates[0]' instead");
158 $CONFIG{timestamp} = $dates[0];
159 }
160 else {
161 output({color=>'red',stderr=>1},
162 sprintf "FATAL: Timestamp field '%s' not found and discovered multiple date fields: %s",
163 $CONFIG{timestamp},
164 join(', ', sort @dates)
165 );
166 output({color=>'yellow',indent=>1}, "Try again with '--timestamp $dates[0]' for example.");
167 exit 1;
168 }
169 }
170  
171 # Which fields to show
172 my @SHOW = ();
173 if ( exists $OPT{show} && scalar @{ $OPT{show} } ) {
174 foreach my $args (@{ $OPT{show} }) {
175 push @SHOW, grep { defined && length } split /,/, $args;
176 }
177 $q->set_fields([$CONFIG{timestamp},@SHOW]);
178 }
179 # How to sort
180 my $SORT = [ { $CONFIG{timestamp} => $ORDER } ];
181 if( exists $OPT{sort} && length $OPT{sort} ) {
182 $SORT = [
183 map { /:/ ? +{ split /:/ } : $_ }
184 split /,/,
185 $OPT{sort}
186 ]
187 }
188 $q->set_sort($SORT);
189  
190 # Improper Usage
191 pod2usage({-exitval=>1, -verbose=>0, -sections=>'SYNOPSIS', -msg=>'No search string specified'})
192 unless keys %{ $q->query };
193 pod2usage({-exitval=>1, -verbose=>0, -sections=>'SYNOPSIS', -msg=>'Cannot use --tail and --top together'})
194 if exists $OPT{tail} && $OPT{top};
195 pod2usage({-exitval=>1, -verbose=>0, -sections=>'SYNOPSIS', -msg=>'Cannot use --tail and --sort together'})
196 if exists $OPT{tail} && $OPT{sort};
197 pod2usage({-exitval=>1, -verbose=>0, -sections=>'SYNOPSIS', -msg=>'Cannot use --sort along with --asc or --desc'})
198 if $OPT{sort} && ($OPT{asc} || $OPT{desc});
199 pod2usage({-exitval=>1, -verbose=>0, -sections=>'SYNOPSIS', -msg=>'Please specify --show or --jq with --tail'})
200 if exists $OPT{tail} && !( @SHOW || $OPT{json});
201  
202 # Process extra parameters
203 foreach my $presence ( qw( exists missing ) ) {
204 if( exists $OPT{$presence} ) {
205 my @fields = map { split /[,:]/ } (is_arrayref($OPT{$presence}) ? @{ $OPT{$presence} } : ($OPT{$presence}));
206 my $context = $presence eq 'exists' ? 'must' : 'must_not';
207 foreach my $field (@fields) {
208 $q->add_bool( $context => { exists => { field => $field } } );
209 }
210 }
211 }
212  
213 my %SUPPORTED_AGGREGATIONS = map {$_=>'simple_value'} qw(cardinality sum min max avg value_count);
214 my $agg_header = '';
215 if( exists $OPT{top} ) {
216 my @top = split /:/, $OPT{top};
217 my $top_field = pop @top;
218 my $top_agg = @top ? shift @top : 'terms';
219  
220 my @agg_fields = grep { exists $FIELDS{$_} } split /\s*,\s*/, $top_field;
221 croak(sprintf("Option --top takes a field, found %d fields: %s\n", scalar(@agg_fields),join(',',@agg_fields)))
222 unless @agg_fields == 1;
223  
224 my %agg = ();
225 my %sub_agg = ();
226 if( $OPT{by}) {
227 my ($type,$field) = split /\:/, $OPT{by};
228 if( exists $SUPPORTED_AGGREGATIONS{$type} ) {
229 $sub_agg{by} = { $type => {field => $field} };
230 }
231 else {
232 output({color=>'red'}, "Aggregation '$type' is not currently supported, ignoring.");
233 }
234 }
235 if( $OPT{with} ) {
236 my @with = is_arrayref($OPT{with}) ? @{ $OPT{with} } : ( $OPT{with} );
237 foreach my $with ( @with ) {
238 my @attrs = split /:/, $with, 3;
239 # Process Args from Right to Left
240 my $arg = @attrs == 3 ? pop @attrs
241 : $attrs[-1] =~ /^\d/ ? pop @attrs
242 : '';
243 my $pcts = $arg =~ /^\d{1,2}(?:\.\d+)?(?:,\d{1,2}(?:\.\d+)?)*$/ ? $arg : '25,50,75,90,95,99';
244 my $size = $arg =~ /^\d+$/ ? $arg : 3;
245 my $hi = $arg || 0.1;
246 my $field = exists $FIELDS{$attrs[-1]} ? pop @attrs : undef;
247 my $type = @attrs ? pop @attrs : 'terms';
248 # Skip invalid elements
249 next unless defined $field and defined $size and $size > 0;
250  
251 my %params = ();
252 my $id = "$type-$field";
253 # If a term agg and we haven't used this field name, simplify it
254 if( $type =~ /terms$/ && !$sub_agg{$field} ) {
255 $id = $field;
256 $params{size} = $size;
257 $params{missing} = 'MISSING' if $OPT{'with-missing'};
258 }
259  
260 if( $type =~ /histogram|stats|percentiles/ && !$OPT{'no-implications'} ) {
261 output({color=>'magenta',sticky=>1}, "* Using a statistical aggregation implies an exists filter on $field, use --no-implications to disable this");
262 $q->add_bool( must => { exists => { field => $field } } );
263 }
264  
265 $sub_agg{$id} = {
266 $type => {
267 field => $field,
268 $type eq 'percentiles' ? ( percents => [split /,/, $pcts] ) : (),
269 $type eq 'histogram' ? ( interval => $hi ) : (),
270 %params,
271 }
272 };
273 }
274 }
275  
276 my %params = ();
277 $params{missing} = 'MISSING' if $OPT{'with-missing'} and $top_agg eq 'terms';
278  
279 my $field = shift @agg_fields;
280 $agg_header = "count\tpct\t" . $field;
281 $agg{$top_agg} = { field => $field, %params };
282  
283 if( $OPT{'bg-filter'} && $top_agg eq 'significant_terms' ) {
284 my $bgf = App::ElasticSearch::Utilities::QueryString->new();
285 my $bgq = $bgf->expand_query_string($OPT{'bg-filter'});
286 $agg{$top_agg}->{background_filter} = $bgq->query;
287  
288 }
289  
290 if( exists $sub_agg{by} ) {
291 $agg_header = "$OPT{by}\t" . $agg_header;
292 $agg{$top_agg}->{order} = [ { by => $ORDER }, { "_count" => "desc" } ];
293 }
294 $agg{aggregations} = \%sub_agg if keys %sub_agg;
295  
296 if( exists $OPT{all} ) {
297 verbose({color=>'cyan'}, "# Aggregations with --all are limited to returning 1,000,000 results.");
298 $agg{$top_agg}->{size} = 1_000_000;
299 }
300 else {
301 $agg{$top_agg}->{size} = $CONFIG{size};
302 }
303 $q->add_aggregations( top => \%agg );
304 $q->add_aggregations( out_of => { cardinality => { field => $field } } );
305  
306 if( $OPT{interval} ) {
307 $q->wrap_aggregations( step => {
308 date_histogram => {
309 field => $CONFIG{timestamp},
310 interval => $OPT{interval},
311 }
312 });
313 }
314 }
315 elsif(exists $OPT{tail}) {
316 $q->set_size($CONFIG{'max-batch-size'});
317 @AGES = ($AGES[-1]);
318 }
319 elsif( $OPT{all} ) {
320 $q->set_size( $CONFIG{'max-batch-size'} );
321 }
322 else {
323 $q->set_size( $CONFIG{size} < $CONFIG{'max-batch-size'} ? $CONFIG{size} : $CONFIG{'max-batch-size'} );
324 }
325  
326 my %displayed_indices = ();
327 my $TOTAL_HITS = 0;
328 my $OUT_OF = 0;
329 my $last_hit_ts = undef;
330 my $duration = 0;
331 my $displayed = 0;
332 my $header = 0;
333 my $age = undef;
334 my %last_batch_id = ();
335 my %AGGS_TOTALS = ();
336 my %AGES_SEEN = ();
337 # Handle CTRL+C During the Loop
338 my $DONE = 0;
339 local $SIG{INT} = sub { $DONE=1 };
340  
341 verbose({color=>'green',level=>1}, "= Query setup complete, beginning request.");
342 AGES: while( !$DONE && @AGES ) {
343 # With --tail, we don't want to deplete @AGES
344 $age = $OPT{tail} ? $AGES[0] : shift @AGES;
345  
346 # Pause for 200ms if we're tailing
347 sleep(0.2) if exists $OPT{tail} && $last_hit_ts;
348  
349 my $start=time();
350 $last_hit_ts ||= strftime('%Y-%m-%dT%H:%M:%S%z',localtime($start-30));
351  
352 # If we're tailing, bump the @query with a timestamp range
353 $q->stash( filter => {range => { $CONFIG{timestamp} => {gte => $last_hit_ts}}} ) if $OPT{tail};
354  
355 # Header
356 if( !exists $AGES_SEEN{$age} ) {
357 output({color=>'yellow'}, "= Querying Indexes: " . join(',', @{ $by_age{$age} })) unless $OPT{'no-decorators'};
358 $AGES_SEEN{$age}=1;
359 $header=0;
360 }
361  
362 debug("== Request Parameters");
363 debug_var($q->uri_params);
364 debug("== Query");
365 debug(to_json $q->request_body,{allow_nonref=>1,canonical=>1,pretty=>1});
366  
367 # Execute the query
368 my $result = $q->execute( $by_age{$age} );
369  
370 debug({clear=>1},"== Results");
371 debug_var($result);
372 $duration += time() - $start;
373  
374 # Advance if we don't have a result
375 next unless defined $result;
376  
377 if ( $result->{error} ) {
378 my $simple_error;
379 eval {
380 $simple_error = $result->{error}{caused_by}{caused_by}{reason};
381 } or do {
382 ($simple_error) = $result->{error} =~ m/(QueryParsingException\[\[[^\]]+\][^\]]+\]\]);/;
383 };
384 $simple_error ||= '';
385 output({stderr=>1,color=>'red'},
386 "# Received an error from the cluster. $simple_error"
387 );
388 last;
389 }
390 $displayed_indices{$_} = 1 for @{ $by_age{$age} };
391 $TOTAL_HITS += $result->{hits}{total} if $result->{hits}{total};
392  
393 my @always = ();
394 push @always, $CONFIG{timestamp} unless $OPT{'no-decorators'};
395 if(!$header && @SHOW) {
396 output({color=>'cyan'}, join("\t", @always, @SHOW));
397 $header++;
398 }
399  
400 while( $result && !$DONE ) {
401 my $hits = is_arrayref($result->{hits}{hits}) ? $result->{hits}{hits} : [];
402  
403 # Handle Aggregations
404 if( my $aggregations = $result->{aggregations} ) {
405 display_aggregations($aggregations, $result->{hits}{total});
406 next AGES;
407 }
408  
409 # Reset the last batch ID if we have new data
410 %last_batch_id = () if @{$hits} > 0 && $last_hit_ts ne $hits->[-1]->{_source}{$CONFIG{timestamp}};
411 debug({color=>'magenta'}, "+ ID cache is now empty.") unless keys %last_batch_id;
412  
413 foreach my $hit (@{ $hits }) {
414 # Skip if we've seen this record
415 next if exists $last_batch_id{$hit->{_id}};
416  
417 $last_hit_ts = $hit->{_source}{$CONFIG{timestamp}};
418 $last_batch_id{$hit->{_id}}=1;
419 my $record = {};
420  
421 # Add the _id field to the source so that it is listed
422 # when showing full records and can be and can be
423 # used in @SHOW
424 $hit->{_source}->{_id} = $hit->{_id} unless defined($hit->{_source}->{_id});
425  
426 if( @SHOW ) {
427 my $flat = es_flatten_hash( $hit->{_source} );
428 debug_var($flat);
429 foreach my $f (@always) {
430 $record->{$f} = $flat->{$f};
431 }
432 foreach my $f (@SHOW) {
433 my $value = undef;
434 if( exists $flat->{$f} ) {
435 $value = $flat->{$f};
436 }
437 elsif( my $v = document_lookdown($hit->{_source},$f) ) {
438 $value = $v;
439 }
440 elsif(index($f, '.') > 0) {
441 # Try path matching the key
442 my @values = ();
443 foreach my $k (keys %{ $flat }) {
444 if( index($k,$f) == 0 ) {
445 push @values, $flat->{$k};
446 }
447 elsif( $k =~ /\.\d+\./ ) {
448 my $flatter = join '.', grep { !/^\d+$/ } split /\./, $k;
449 if ( $flatter eq $f ) {
450 push @values, $flat->{$k};
451 }
452 }
453 }
454 $value = @values ? @values == 1 ? $values[0] : \@values : undef;
455 }
456 $record->{$f} = $value;
457 }
458 }
459 else {
460 $record = $hit->{_source};
461 }
462 # Determine how this record is output
463 my $output = undef;
464 if( @SHOW ) {
465 my @cols=();
466 foreach my $f (@always,@SHOW) {
467 my $v = '-';
468 if( exists $record->{$f} && defined $record->{$f} ) {
469 $v = is_arrayref($record->{$f}) && @{ $record->{$f} } == 1 ? $record->{$f}[0]
470 : is_ref($record->{$f}) ? $JSON->encode($record->{$f})
471 : $record->{$f};
472 }
473 push @cols,$v;
474 }
475 $output = join("\t",@cols);
476 }
477 else {
478 $output = $CONFIG{format} =~ /^json/? to_json($record,{allow_nonref=>1,canonical=>1,pretty=>$CONFIG{pretty}})
479 : Dump $record;
480 }
481  
482 output({data=>1}, $output);
483 $displayed++;
484 last if all_records_displayed();
485 }
486 last if all_records_displayed();
487  
488 # Scroll forward
489 $start = time;
490 $result = $q->scroll_results();
491 $duration += time - $start;
492 # Check if we need to keep going
493 last unless defined $result;
494 last unless $result->{hits} && $result->{hits}{hits} && @{ $result->{hits}{hits} } > 0
495 }
496 last if all_records_displayed();
497 }
498  
499 output({stderr=>1,color=>'yellow'},
500 "# Search Parameters:",
501 (map { "# $_" } split /\r?\n/, $JSON->encode($q->query)),
502 sprintf("# Displaying %d of %d results%s took %0.2f seconds.",
503 $displayed,
504 $OUT_OF || $TOTAL_HITS,
505 $OUT_OF ? " in $TOTAL_HITS documents" : '',
506 $duration,
507 ),
508 sprintf("# Indexes (%d of %d) searched: %s\n",
509 scalar(keys %displayed_indices),
510 scalar(keys %indices),
511 join(',', sort keys %displayed_indices)
512 ),
513 ) unless $OPT{'no-decorators'};
514  
515 if($CONFIG{summary} && keys %AGGS_TOTALS) {
516 unless ( $OPT{'no-decorators'} ) {
517 output({color=>'yellow'}, '#', '# Totals across batch', '#');
518 output({color=>'cyan'},$agg_header);
519 }
520 foreach my $k (sort { $AGGS_TOTALS{$b} <=> $AGGS_TOTALS{$a} } keys %AGGS_TOTALS) {
521 output({data=>1,color=>'green'}, join "\t",
522 $AGGS_TOTALS{$k}, es_format_numeric($AGGS_TOTALS{$k} / $TOTAL_HITS, $CONFIG{precision}), $k
523 );
524 }
525 }
526  
527 sub all_records_displayed {
528 return 1 if $DONE;
529 return 0 if exists $OPT{tail};
530 return 0 if exists $OPT{all};
531 return 1 if $displayed >= $CONFIG{size};
532 return 0;
533 }
534  
535 sub document_lookdown {
536 my ($href,$field) = @_;
537  
538 return $href->{$field} if exists $href->{$field};
539  
540 foreach my $k (keys %{ $href }) {
541 if( is_hashref($href->{$k}) ) {
542 return document_lookdown($href->{$k},$field);
543 }
544 }
545 return;
546 }
547  
548 sub show_fields {
549 output({color=>'cyan'}, 'Fields available for search:' );
550 my $total = 0;
551 my %types = ();
552 foreach my $field (sort keys %FIELDS) {
553 $total++;
554 my $type = $FIELDS{$field}->{type};
555 $types{$type} ||= 0;
556 $types{$type}++;
557 my $color = $type eq 'ip' ? 'magenta'
558 : $type eq 'text' ? 'red'
559 : $type =~ /float|integer|short|byte|double/ ? 'cyan'
560 : $type =~ /^geo/ ? 'green'
561 : $type =~ /^date/ ? 'yellow'
562 : 'white';
563 output({indent=>1,kv=>1,color=>$color}, $field => $type);
564 output({indent=>2}, sprintf "nested: %s - %s",
565 @{ $FIELDS{$field} }{qw(nested_path nested_key)}
566 ) if exists $FIELDS{$field}->{nested_path};
567 }
568 output({color=>"yellow"},
569 sprintf("# Fields: %d from a combined %d indices.\n",
570 $total,
571 scalar(keys %indices),
572 )
573 );
574 # Type Meta Roll Up
575 output({indent=>1}, join(', ',
576 map { "$types{$_} $_ fields" }
577 sort { $types{$b} <=> $types{$a} }
578 keys %types
579 )
580 );
581 }
582  
583 sub show_bases {
584 output({color=>'cyan'}, 'Bases available for search:' );
585 my @all = es_indices(_all => 1);
586 my %bases = ();
587  
588 foreach my $index (@all) {
589 next if $index =~ /^\./;
590 my $days_old = es_index_days_old( $index ) || 0;
591 next unless defined $days_old;
592 $days_old = 0 if $days_old < 0;
593 foreach my $base (es_index_bases($index)) {
594 if( exists $bases{$base} ) {
595 $bases{$base}->{oldest} = $days_old if $days_old > $bases{$base}->{oldest};
596 $bases{$base}->{youngest} = $days_old if $days_old < $bases{$base}->{youngest};
597 }
598 else {
599 $bases{$base} = { oldest => $days_old, youngest => $days_old };
600 }
601 }
602 }
603 foreach my $base (sort keys %bases) {
604 output({indent=>1,color=>'green'},$base);
605 verbose({indent=>2,kv=>1},
606 map {
607 $_ => sprintf "%d days old", $bases{$base}->{$_}
608 } qw( youngest oldest )
609 );
610 }
611  
612 output({color=>"yellow"},
613 sprintf("# Bases: %d from a combined %d indices.\n",
614 scalar(keys %bases),
615 scalar(@all),
616 )
617 );
618 }
619  
620 sub display_aggregations {
621 my ($aggregations,$total_docs) = (@_);
622  
623 my $out_of = $aggregations->{out_of}{value} || 0;
624 $OUT_OF = $out_of if $out_of > $OUT_OF;
625 my $steps = exists $aggregations->{step} ? $aggregations->{step}{buckets}
626 : [ $aggregations ];
627 my $indent = exists $aggregations->{step} ? 1 : 0;
628  
629 foreach my $step ( @$steps ) {
630 my $aggs = exists $step->{top} ? $step->{top}{buckets} : [];
631 if( exists $step->{key_as_string} ) {
632 output({color=>'cyan',clear=>1}, sprintf "%d\t%s", @{$step}{qw(doc_count key_as_string)});
633 }
634 if( @$aggs ) {
635 # For top the N of T needs to represent maximums
636 $displayed = scalar(@$aggs) if scalar(@$aggs) > $displayed;
637 output({color=>'cyan',indent=>$indent},$agg_header) unless $OPT{'no-decorators'};
638 foreach my $agg ( @$aggs ) {
639 $AGGS_TOTALS{$agg->{key}} ||= 0;
640 $AGGS_TOTALS{$agg->{key}} += $agg->{doc_count};
641 my @out = ();
642 my $top_docs;
643 foreach my $k (qw(score doc_count bg_count key)) {
644 next unless exists $agg->{$k};
645 my $value = delete $agg->{$k};
646 push @out, defined $value ? es_format_numeric($value, $CONFIG{precision}) : '-';
647 if( $k eq 'doc_count' ) {
648 push @out, es_format_numeric($value ? $value / $total_docs : 0, $CONFIG{precision});
649 $top_docs = $value;
650 }
651 }
652 if(exists $agg->{by} ) {
653 my $by = delete $agg->{by};
654 if( exists $by->{value_as_string} ) {
655 unshift @out, $by->{value_as_string};
656 }
657 elsif( exists $by->{value} ) {
658 unshift @out, es_format_numeric($by->{value});
659 }
660 }
661 # Handle the --with elements
662 my %subaggs = ();
663 if( keys %{ $agg } ) {
664 foreach my $k (sort keys %{ $agg }) {
665 next unless is_hashref($agg->{$k});
666 if( exists $agg->{$k}{buckets} ) {
667 my @sub;
668 foreach my $subagg (@{ $agg->{$k}{buckets} }) {
669 my @elms = ();
670 next unless exists $subagg->{key};
671 push @elms, $subagg->{key};
672 foreach my $dk (qw(score doc_count bg_count)) {
673 next unless exists $subagg->{$dk};
674 my $v = delete $subagg->{$dk};
675 push @elms, defined $v ? es_format_numeric($v, $CONFIG{precision}) : '-';
676 push @elms, es_format_numeric($v / $top_docs,, $CONFIG{precision})
677 if $dk eq 'doc_count' and $top_docs;
678 }
679 push @sub, \@elms;
680 }
681 $subaggs{$k} = \@sub if @sub;
682 }
683 # Simple Numeric Aggs
684 elsif( exists $agg->{$k}{value_as_string} ) {
685 $subaggs{$k} = [ [ $agg->{$k}{value_as_string} ] ];
686 }
687 elsif( exists $agg->{$k}{value} ) {
688 my $v = es_format_numeric($agg->{$k}{value},$CONFIG{precision});
689 $subaggs{$k} = [ [ $v ] ];
690 }
691 # Percentiles
692 elsif( exists $agg->{$k}{values} ) {
693 my @pcts;
694 foreach my $pctl (sort { $a <=> $b } keys %{ $agg->{$k}{values} }) {
695 push @pcts, "p$pctl", es_format_numeric($agg->{$k}{values}{$pctl}, $CONFIG{precision});
696 }
697 $subaggs{$k} = [ \@pcts ];
698 }
699 else {
700 # Statistics
701 my @stats;
702 my %alias = qw( variance var std_deviation stdev );
703 foreach my $stat (qw(count min avg max sum variance std_deviation)) {
704 next unless exists $agg->{$k}{$stat};
705 my $v = es_format_numeric($agg->{$k}{$stat}, $CONFIG{precision});
706 push @stats, $alias{$stat} || $stat => $v;
707 }
708 $subaggs{$k} = [ \@stats ] if @stats;
709 }
710 }
711 }
712 if( keys %subaggs ) {
713 foreach my $subagg (sort keys %subaggs) {
714 foreach my $extra ( @{ $subaggs{$subagg} } ) {
715 output({indent=>$indent,data=>1},
716 join "\t", @out, $subagg, @{ $extra }
717 );
718 }
719 }
720 }
721 else {
722 # Simple output
723 output({indent=>$indent,data=>!$CONFIG{summary}}, join("\t",@out));
724 }
725 }
726 }
727 elsif(exists $aggregations->{top}) {
728 output({indent=>1,color=>'red'}, "= No results.");
729 }
730 }
731 }
732  
733 sub by_index_age {
734 return $ORDER eq 'asc'
735 ? $indices{$b} <=> $indices{$a}
736 : $indices{$a} <=> $indices{$b};
737 }
738  
739 __END__