| 1 | #!/usr/local/bin/perl | ||
| 2 | # PODNAME: es-search.pl | ||
| 3 | # ABSTRACT: Provides a CLI for quick searches of data in ElasticSearch daily indexes | ||
| 4 | use strict; | ||
| 5 | use warnings; | ||
| 6 | |||
| 7 | use App::ElasticSearch::Utilities qw(:all); | ||
| 8 | use App::ElasticSearch::Utilities::Query; | ||
| 9 | use App::ElasticSearch::Utilities::QueryString; | ||
| 10 | use Carp; | ||
| 11 | use CLI::Helpers qw(:all); | ||
| 12 | use Getopt::Long qw(:config no_ignore_case no_ignore_case_always); | ||
| 13 | use JSON::MaybeXS qw(:legacy); | ||
| 14 | use Pod::Usage; | ||
| 15 | use POSIX qw(strftime); | ||
| 16 | use Ref::Util qw(is_ref is_arrayref is_hashref); | ||
| 17 | use Time::HiRes qw(sleep time); | ||
| 18 | use YAML::XS; | ||
| 19 | local $YAML::XS::Boolean = "JSON::PP"; | ||
| 20 | |||
| 21 | #------------------------------------------------------------------------# | ||
| 22 | # Argument Parsing | ||
| 23 | my %OPT; | ||
| 24 | GetOptions(\%OPT, qw( | ||
| 25 | all | ||
| 26 | asc | ||
| 27 | bases | ||
| 28 | bg-filter=s | ||
| 29 | by=s | ||
| 30 | desc | ||
| 31 | exists=s@ | ||
| 32 | fields | ||
| 33 | filter | ||
| 34 | format=s | ||
| 35 | help|h | ||
| 36 | json|jq | ||
| 37 | manual|m | ||
| 38 | match-all | ||
| 39 | max-batch-size=i | ||
| 40 | missing=s@ | ||
| 41 | no-decorators|no-header | ||
| 42 | no-implications|no-imply | ||
| 43 | precision=i | ||
| 44 | prefix=s@ | ||
| 45 | pretty | ||
| 46 | show=s@ | ||
| 47 | size|n|limit=i | ||
| 48 | sort=s | ||
| 49 | tail | ||
| 50 | timestamp=s | ||
| 51 | top=s | ||
| 52 | interval=s | ||
| 53 | with=s@ | ||
| 54 | with-missing | ||
| 55 | or | ||
| 56 | )); | ||
| 57 | |||
| 58 | # Search string is the rest of the argument string | ||
| 59 | my $context = $OPT{filter} ? 'filter' : 'must'; | ||
| 60 | my $qs = App::ElasticSearch::Utilities::QueryString->new( | ||
| 61 | $OPT{filter} ? (context => 'filter') : (), | ||
| 62 | default_join => $OPT{or} ? 'OR' : 'AND', | ||
| 63 | ); | ||
| 64 | my $q = exists $OPT{'match-all'} && $OPT{'match-all'} | ||
| 65 | ? App::ElasticSearch::Utilities::Query->new($context => { match_all => {} }) | ||
| 66 | : $qs->expand_query_string(@ARGV); | ||
| 67 | |||
| 68 | $q->set_timeout('10s'); | ||
| 69 | $q->set_scroll('30s'); | ||
| 70 | |||
| 71 | if( exists $OPT{prefix} ){ | ||
| 72 | foreach my $prefix (@{ $OPT{prefix} }) { | ||
| 73 | my ($f,$v) = split /:/, $prefix, 2; | ||
| 74 | next unless $f && $v; | ||
| 75 | $q->add_bool( $context => { prefix => { $f => $v } } ); | ||
| 76 | } | ||
| 77 | } | ||
| 78 | |||
| 79 | #------------------------------------------------------------------------# | ||
| 80 | # Documentation | ||
| 81 | pod2usage({-sections => 'SYNOPSIS'}) if $OPT{help}; | ||
| 82 | pod2usage(-exitval => 0, -verbose => 2) if $OPT{manual}; | ||
| 83 | my $unknown_options = join ', ', grep /^--/, @ARGV; | ||
| 84 | pod2usage({-exitval => 1, -sections => 'SYNOPSIS', -msg =>"Unknown option(s): $unknown_options"}) if $unknown_options; | ||
| 85 | |||
| 86 | #--------------------------------------------------------------------------# | ||
| 87 | # Information Gathering Routines | ||
| 88 | if( $OPT{bases} ) { | ||
| 89 | show_bases(); | ||
| 90 | exit 0; | ||
| 91 | } | ||
| 92 | #--------------------------------------------------------------------------# | ||
| 93 | # App Config | ||
| 94 | my %CONFIG = ( | ||
| 95 | size => ($OPT{size} && $OPT{size} > 0) ? int($OPT{size}) : 20, | ||
| 96 | format => $OPT{json} ? 'json' | ||
| 97 | : $OPT{format} ? lc $OPT{format} | ||
| 98 | : 'yaml', | ||
| 99 | 'max-batch-size' => $OPT{'max-batch-size'} || 50, | ||
| 100 | precision => $OPT{precision} || 3, | ||
| 101 | $OPT{timestamp} ? ( timestamp => $OPT{timestamp} ) : (), | ||
| 102 | ); | ||
| 103 | $OPT{'no-decorators'} = 1 if $CONFIG{format} eq 'json'; | ||
| 104 | $CONFIG{pretty} = $OPT{pretty} ? 1 | ||
| 105 | : $CONFIG{format} =~ /pretty/ ? 1 | ||
| 106 | : 0; | ||
| 107 | my $JSON = JSON->new->utf8->canonical; | ||
| 108 | #------------------------------------------------------------------------# | ||
| 109 | # Handle Indices | ||
| 110 | my $ORDER = exists $OPT{asc} && $OPT{asc} ? 'asc' : 'desc'; | ||
| 111 | $ORDER = 'asc' if exists $OPT{tail}; | ||
| 112 | my %by_age = (); | ||
| 113 | my %indices = map { $_ => (es_index_days_old($_) || 0) } es_indices(); | ||
| 114 | die "# Failed to retrieve any indices using your paramaters." unless keys %indices; | ||
| 115 | my %FIELDS = (); | ||
| 116 | my $TimeStampCheck=0; | ||
| 117 | foreach my $index (sort by_index_age keys %indices) { | ||
| 118 | my $age = $indices{$index}; | ||
| 119 | $by_age{$age} ||= []; | ||
| 120 | push @{ $by_age{$age} }, $index; | ||
| 121 | my $fields = es_index_fields($index); | ||
| 122 | foreach my $k ( keys %{ $fields } ) { | ||
| 123 | $FIELDS{$k} = $fields->{$k} | ||
| 124 | unless $FIELDS{$k}; | ||
| 125 | } | ||
| 126 | # Lookup the Index in our local YAML | ||
| 127 | if( !$TimeStampCheck ) { | ||
| 128 | $TimeStampCheck++; | ||
| 129 | $CONFIG{timestamp} ||= es_local_index_meta(timestamp => $index); | ||
| 130 | } | ||
| 131 | } | ||
| 132 | |||
| 133 | # Set fields so we know how to construct complex aggs | ||
| 134 | $q->fields_meta( \%FIELDS ); | ||
| 135 | |||
| 136 | #------------------------------------------------------------------------# | ||
| 137 | # Figure out the timestamp | ||
| 138 | $CONFIG{timestamp} ||= es_globals('timestamp') || '@timestamp'; | ||
| 139 | debug_var(\%by_age); | ||
| 140 | my @AGES = sort { $ORDER eq 'asc' ? $b <=> $a : $a <=> $b } keys %by_age; | ||
| 141 | # Figure out if we summarize | ||
| 142 | $CONFIG{summary} = @AGES > 1 && $OPT{top} && ( !$OPT{by} && !$OPT{with} && !$OPT{interval} ); | ||
| 143 | debug({color=>"cyan"}, "Fields discovered."); | ||
| 144 | |||
| 145 | if( $OPT{fields} ) { | ||
| 146 | show_fields(); | ||
| 147 | exit 0; | ||
| 148 | } | ||
| 149 | # Attempt date autodiscovery | ||
| 150 | if( !exists $FIELDS{$CONFIG{timestamp}} ) { | ||
| 151 | my @dates = grep { $FIELDS{$_}->{type} eq 'date' } keys %FIELDS; | ||
| 152 | if( @dates == 0 ) { | ||
| 153 | output({color=>'red',stderr=>1},"FATAL: No date fields found in the indices specified" ); | ||
| 154 | exit 1; | ||
| 155 | } | ||
| 156 | elsif( @dates == 1 ) { | ||
| 157 | output({color=>'yellow',stderr=>1}, "WARNING: Timestamp field '$CONFIG{timestamp}' not found, using '$dates[0]' instead"); | ||
| 158 | $CONFIG{timestamp} = $dates[0]; | ||
| 159 | } | ||
| 160 | else { | ||
| 161 | output({color=>'red',stderr=>1}, | ||
| 162 | sprintf "FATAL: Timestamp field '%s' not found and discovered multiple date fields: %s", | ||
| 163 | $CONFIG{timestamp}, | ||
| 164 | join(', ', sort @dates) | ||
| 165 | ); | ||
| 166 | output({color=>'yellow',indent=>1}, "Try again with '--timestamp $dates[0]' for example."); | ||
| 167 | exit 1; | ||
| 168 | } | ||
| 169 | } | ||
| 170 | |||
| 171 | # Which fields to show | ||
| 172 | my @SHOW = (); | ||
| 173 | if ( exists $OPT{show} && scalar @{ $OPT{show} } ) { | ||
| 174 | foreach my $args (@{ $OPT{show} }) { | ||
| 175 | push @SHOW, grep { defined && length } split /,/, $args; | ||
| 176 | } | ||
| 177 | $q->set_fields([$CONFIG{timestamp},@SHOW]); | ||
| 178 | } | ||
| 179 | # How to sort | ||
| 180 | my $SORT = [ { $CONFIG{timestamp} => $ORDER } ]; | ||
| 181 | if( exists $OPT{sort} && length $OPT{sort} ) { | ||
| 182 | $SORT = [ | ||
| 183 | map { /:/ ? +{ split /:/ } : $_ } | ||
| 184 | split /,/, | ||
| 185 | $OPT{sort} | ||
| 186 | ] | ||
| 187 | } | ||
| 188 | $q->set_sort($SORT); | ||
| 189 | |||
| 190 | # Improper Usage | ||
| 191 | pod2usage({-exitval=>1, -verbose=>0, -sections=>'SYNOPSIS', -msg=>'No search string specified'}) | ||
| 192 | unless keys %{ $q->query }; | ||
| 193 | pod2usage({-exitval=>1, -verbose=>0, -sections=>'SYNOPSIS', -msg=>'Cannot use --tail and --top together'}) | ||
| 194 | if exists $OPT{tail} && $OPT{top}; | ||
| 195 | pod2usage({-exitval=>1, -verbose=>0, -sections=>'SYNOPSIS', -msg=>'Cannot use --tail and --sort together'}) | ||
| 196 | if exists $OPT{tail} && $OPT{sort}; | ||
| 197 | pod2usage({-exitval=>1, -verbose=>0, -sections=>'SYNOPSIS', -msg=>'Cannot use --sort along with --asc or --desc'}) | ||
| 198 | if $OPT{sort} && ($OPT{asc} || $OPT{desc}); | ||
| 199 | pod2usage({-exitval=>1, -verbose=>0, -sections=>'SYNOPSIS', -msg=>'Please specify --show or --jq with --tail'}) | ||
| 200 | if exists $OPT{tail} && !( @SHOW || $OPT{json}); | ||
| 201 | |||
| 202 | # Process extra parameters | ||
| 203 | foreach my $presence ( qw( exists missing ) ) { | ||
| 204 | if( exists $OPT{$presence} ) { | ||
| 205 | my @fields = map { split /[,:]/ } (is_arrayref($OPT{$presence}) ? @{ $OPT{$presence} } : ($OPT{$presence})); | ||
| 206 | my $context = $presence eq 'exists' ? 'must' : 'must_not'; | ||
| 207 | foreach my $field (@fields) { | ||
| 208 | $q->add_bool( $context => { exists => { field => $field } } ); | ||
| 209 | } | ||
| 210 | } | ||
| 211 | } | ||
| 212 | |||
| 213 | my %SUPPORTED_AGGREGATIONS = map {$_=>'simple_value'} qw(cardinality sum min max avg value_count); | ||
| 214 | my $agg_header = ''; | ||
| 215 | if( exists $OPT{top} ) { | ||
| 216 | my @top = split /:/, $OPT{top}; | ||
| 217 | my $top_field = pop @top; | ||
| 218 | my $top_agg = @top ? shift @top : 'terms'; | ||
| 219 | |||
| 220 | my @agg_fields = grep { exists $FIELDS{$_} } split /\s*,\s*/, $top_field; | ||
| 221 | croak(sprintf("Option --top takes a field, found %d fields: %s\n", scalar(@agg_fields),join(',',@agg_fields))) | ||
| 222 | unless @agg_fields == 1; | ||
| 223 | |||
| 224 | my %agg = (); | ||
| 225 | my %sub_agg = (); | ||
| 226 | if( $OPT{by}) { | ||
| 227 | my ($type,$field) = split /\:/, $OPT{by}; | ||
| 228 | if( exists $SUPPORTED_AGGREGATIONS{$type} ) { | ||
| 229 | $sub_agg{by} = { $type => {field => $field} }; | ||
| 230 | } | ||
| 231 | else { | ||
| 232 | output({color=>'red'}, "Aggregation '$type' is not currently supported, ignoring."); | ||
| 233 | } | ||
| 234 | } | ||
| 235 | if( $OPT{with} ) { | ||
| 236 | my @with = is_arrayref($OPT{with}) ? @{ $OPT{with} } : ( $OPT{with} ); | ||
| 237 | foreach my $with ( @with ) { | ||
| 238 | my @attrs = split /:/, $with, 3; | ||
| 239 | # Process Args from Right to Left | ||
| 240 | my $arg = @attrs == 3 ? pop @attrs | ||
| 241 | : $attrs[-1] =~ /^\d/ ? pop @attrs | ||
| 242 | : ''; | ||
| 243 | my $pcts = $arg =~ /^\d{1,2}(?:\.\d+)?(?:,\d{1,2}(?:\.\d+)?)*$/ ? $arg : '25,50,75,90,95,99'; | ||
| 244 | my $size = $arg =~ /^\d+$/ ? $arg : 3; | ||
| 245 | my $hi = $arg || 0.1; | ||
| 246 | my $field = exists $FIELDS{$attrs[-1]} ? pop @attrs : undef; | ||
| 247 | my $type = @attrs ? pop @attrs : 'terms'; | ||
| 248 | # Skip invalid elements | ||
| 249 | next unless defined $field and defined $size and $size > 0; | ||
| 250 | |||
| 251 | my %params = (); | ||
| 252 | my $id = "$type-$field"; | ||
| 253 | # If a term agg and we haven't used this field name, simplify it | ||
| 254 | if( $type =~ /terms$/ && !$sub_agg{$field} ) { | ||
| 255 | $id = $field; | ||
| 256 | $params{size} = $size; | ||
| 257 | $params{missing} = 'MISSING' if $OPT{'with-missing'}; | ||
| 258 | } | ||
| 259 | |||
| 260 | if( $type =~ /histogram|stats|percentiles/ && !$OPT{'no-implications'} ) { | ||
| 261 | output({color=>'magenta',sticky=>1}, "* Using a statistical aggregation implies an exists filter on $field, use --no-implications to disable this"); | ||
| 262 | $q->add_bool( must => { exists => { field => $field } } ); | ||
| 263 | } | ||
| 264 | |||
| 265 | $sub_agg{$id} = { | ||
| 266 | $type => { | ||
| 267 | field => $field, | ||
| 268 | $type eq 'percentiles' ? ( percents => [split /,/, $pcts] ) : (), | ||
| 269 | $type eq 'histogram' ? ( interval => $hi ) : (), | ||
| 270 | %params, | ||
| 271 | } | ||
| 272 | }; | ||
| 273 | } | ||
| 274 | } | ||
| 275 | |||
| 276 | my %params = (); | ||
| 277 | $params{missing} = 'MISSING' if $OPT{'with-missing'} and $top_agg eq 'terms'; | ||
| 278 | |||
| 279 | my $field = shift @agg_fields; | ||
| 280 | $agg_header = "count\tpct\t" . $field; | ||
| 281 | $agg{$top_agg} = { field => $field, %params }; | ||
| 282 | |||
| 283 | if( $OPT{'bg-filter'} && $top_agg eq 'significant_terms' ) { | ||
| 284 | my $bgf = App::ElasticSearch::Utilities::QueryString->new(); | ||
| 285 | my $bgq = $bgf->expand_query_string($OPT{'bg-filter'}); | ||
| 286 | $agg{$top_agg}->{background_filter} = $bgq->query; | ||
| 287 | |||
| 288 | } | ||
| 289 | |||
| 290 | if( exists $sub_agg{by} ) { | ||
| 291 | $agg_header = "$OPT{by}\t" . $agg_header; | ||
| 292 | $agg{$top_agg}->{order} = [ { by => $ORDER }, { "_count" => "desc" } ]; | ||
| 293 | } | ||
| 294 | $agg{aggregations} = \%sub_agg if keys %sub_agg; | ||
| 295 | |||
| 296 | if( exists $OPT{all} ) { | ||
| 297 | verbose({color=>'cyan'}, "# Aggregations with --all are limited to returning 1,000,000 results."); | ||
| 298 | $agg{$top_agg}->{size} = 1_000_000; | ||
| 299 | } | ||
| 300 | else { | ||
| 301 | $agg{$top_agg}->{size} = $CONFIG{size}; | ||
| 302 | } | ||
| 303 | $q->add_aggregations( top => \%agg ); | ||
| 304 | $q->add_aggregations( out_of => { cardinality => { field => $field } } ); | ||
| 305 | |||
| 306 | if( $OPT{interval} ) { | ||
| 307 | $q->wrap_aggregations( step => { | ||
| 308 | date_histogram => { | ||
| 309 | field => $CONFIG{timestamp}, | ||
| 310 | interval => $OPT{interval}, | ||
| 311 | } | ||
| 312 | }); | ||
| 313 | } | ||
| 314 | } | ||
| 315 | elsif(exists $OPT{tail}) { | ||
| 316 | $q->set_size($CONFIG{'max-batch-size'}); | ||
| 317 | @AGES = ($AGES[-1]); | ||
| 318 | } | ||
| 319 | elsif( $OPT{all} ) { | ||
| 320 | $q->set_size( $CONFIG{'max-batch-size'} ); | ||
| 321 | } | ||
| 322 | else { | ||
| 323 | $q->set_size( $CONFIG{size} < $CONFIG{'max-batch-size'} ? $CONFIG{size} : $CONFIG{'max-batch-size'} ); | ||
| 324 | } | ||
| 325 | |||
| 326 | my %displayed_indices = (); | ||
| 327 | my $TOTAL_HITS = 0; | ||
| 328 | my $OUT_OF = 0; | ||
| 329 | my $last_hit_ts = undef; | ||
| 330 | my $duration = 0; | ||
| 331 | my $displayed = 0; | ||
| 332 | my $header = 0; | ||
| 333 | my $age = undef; | ||
| 334 | my %last_batch_id = (); | ||
| 335 | my %AGGS_TOTALS = (); | ||
| 336 | my %AGES_SEEN = (); | ||
| 337 | # Handle CTRL+C During the Loop | ||
| 338 | my $DONE = 0; | ||
| 339 | local $SIG{INT} = sub { $DONE=1 }; | ||
| 340 | |||
| 341 | verbose({color=>'green',level=>1}, "= Query setup complete, beginning request."); | ||
| 342 | AGES: while( !$DONE && @AGES ) { | ||
| 343 | # With --tail, we don't want to deplete @AGES | ||
| 344 | $age = $OPT{tail} ? $AGES[0] : shift @AGES; | ||
| 345 | |||
| 346 | # Pause for 200ms if we're tailing | ||
| 347 | sleep(0.2) if exists $OPT{tail} && $last_hit_ts; | ||
| 348 | |||
| 349 | my $start=time(); | ||
| 350 | $last_hit_ts ||= strftime('%Y-%m-%dT%H:%M:%S%z',localtime($start-30)); | ||
| 351 | |||
| 352 | # If we're tailing, bump the @query with a timestamp range | ||
| 353 | $q->stash( filter => {range => { $CONFIG{timestamp} => {gte => $last_hit_ts}}} ) if $OPT{tail}; | ||
| 354 | |||
| 355 | # Header | ||
| 356 | if( !exists $AGES_SEEN{$age} ) { | ||
| 357 | output({color=>'yellow'}, "= Querying Indexes: " . join(',', @{ $by_age{$age} })) unless $OPT{'no-decorators'}; | ||
| 358 | $AGES_SEEN{$age}=1; | ||
| 359 | $header=0; | ||
| 360 | } | ||
| 361 | |||
| 362 | debug("== Request Parameters"); | ||
| 363 | debug_var($q->uri_params); | ||
| 364 | debug("== Query"); | ||
| 365 | debug(to_json $q->request_body,{allow_nonref=>1,canonical=>1,pretty=>1}); | ||
| 366 | |||
| 367 | # Execute the query | ||
| 368 | my $result = $q->execute( $by_age{$age} ); | ||
| 369 | |||
| 370 | debug({clear=>1},"== Results"); | ||
| 371 | debug_var($result); | ||
| 372 | $duration += time() - $start; | ||
| 373 | |||
| 374 | # Advance if we don't have a result | ||
| 375 | next unless defined $result; | ||
| 376 | |||
| 377 | if ( $result->{error} ) { | ||
| 378 | my $simple_error; | ||
| 379 | eval { | ||
| 380 | $simple_error = $result->{error}{caused_by}{caused_by}{reason}; | ||
| 381 | } or do { | ||
| 382 | ($simple_error) = $result->{error} =~ m/(QueryParsingException\[\[[^\]]+\][^\]]+\]\]);/; | ||
| 383 | }; | ||
| 384 | $simple_error ||= ''; | ||
| 385 | output({stderr=>1,color=>'red'}, | ||
| 386 | "# Received an error from the cluster. $simple_error" | ||
| 387 | ); | ||
| 388 | last; | ||
| 389 | } | ||
| 390 | $displayed_indices{$_} = 1 for @{ $by_age{$age} }; | ||
| 391 | $TOTAL_HITS += $result->{hits}{total} if $result->{hits}{total}; | ||
| 392 | |||
| 393 | my @always = (); | ||
| 394 | push @always, $CONFIG{timestamp} unless $OPT{'no-decorators'}; | ||
| 395 | if(!$header && @SHOW) { | ||
| 396 | output({color=>'cyan'}, join("\t", @always, @SHOW)); | ||
| 397 | $header++; | ||
| 398 | } | ||
| 399 | |||
| 400 | while( $result && !$DONE ) { | ||
| 401 | my $hits = is_arrayref($result->{hits}{hits}) ? $result->{hits}{hits} : []; | ||
| 402 | |||
| 403 | # Handle Aggregations | ||
| 404 | if( my $aggregations = $result->{aggregations} ) { | ||
| 405 | display_aggregations($aggregations, $result->{hits}{total}); | ||
| 406 | next AGES; | ||
| 407 | } | ||
| 408 | |||
| 409 | # Reset the last batch ID if we have new data | ||
| 410 | %last_batch_id = () if @{$hits} > 0 && $last_hit_ts ne $hits->[-1]->{_source}{$CONFIG{timestamp}}; | ||
| 411 | debug({color=>'magenta'}, "+ ID cache is now empty.") unless keys %last_batch_id; | ||
| 412 | |||
| 413 | foreach my $hit (@{ $hits }) { | ||
| 414 | # Skip if we've seen this record | ||
| 415 | next if exists $last_batch_id{$hit->{_id}}; | ||
| 416 | |||
| 417 | $last_hit_ts = $hit->{_source}{$CONFIG{timestamp}}; | ||
| 418 | $last_batch_id{$hit->{_id}}=1; | ||
| 419 | my $record = {}; | ||
| 420 | |||
| 421 | # Add the _id field to the source so that it is listed | ||
| 422 | # when showing full records and can be and can be | ||
| 423 | # used in @SHOW | ||
| 424 | $hit->{_source}->{_id} = $hit->{_id} unless defined($hit->{_source}->{_id}); | ||
| 425 | |||
| 426 | if( @SHOW ) { | ||
| 427 | my $flat = es_flatten_hash( $hit->{_source} ); | ||
| 428 | debug_var($flat); | ||
| 429 | foreach my $f (@always) { | ||
| 430 | $record->{$f} = $flat->{$f}; | ||
| 431 | } | ||
| 432 | foreach my $f (@SHOW) { | ||
| 433 | my $value = undef; | ||
| 434 | if( exists $flat->{$f} ) { | ||
| 435 | $value = $flat->{$f}; | ||
| 436 | } | ||
| 437 | elsif( my $v = document_lookdown($hit->{_source},$f) ) { | ||
| 438 | $value = $v; | ||
| 439 | } | ||
| 440 | elsif(index($f, '.') > 0) { | ||
| 441 | # Try path matching the key | ||
| 442 | my @values = (); | ||
| 443 | foreach my $k (keys %{ $flat }) { | ||
| 444 | if( index($k,$f) == 0 ) { | ||
| 445 | push @values, $flat->{$k}; | ||
| 446 | } | ||
| 447 | elsif( $k =~ /\.\d+\./ ) { | ||
| 448 | my $flatter = join '.', grep { !/^\d+$/ } split /\./, $k; | ||
| 449 | if ( $flatter eq $f ) { | ||
| 450 | push @values, $flat->{$k}; | ||
| 451 | } | ||
| 452 | } | ||
| 453 | } | ||
| 454 | $value = @values ? @values == 1 ? $values[0] : \@values : undef; | ||
| 455 | } | ||
| 456 | $record->{$f} = $value; | ||
| 457 | } | ||
| 458 | } | ||
| 459 | else { | ||
| 460 | $record = $hit->{_source}; | ||
| 461 | } | ||
| 462 | # Determine how this record is output | ||
| 463 | my $output = undef; | ||
| 464 | if( @SHOW ) { | ||
| 465 | my @cols=(); | ||
| 466 | foreach my $f (@always,@SHOW) { | ||
| 467 | my $v = '-'; | ||
| 468 | if( exists $record->{$f} && defined $record->{$f} ) { | ||
| 469 | $v = is_arrayref($record->{$f}) && @{ $record->{$f} } == 1 ? $record->{$f}[0] | ||
| 470 | : is_ref($record->{$f}) ? $JSON->encode($record->{$f}) | ||
| 471 | : $record->{$f}; | ||
| 472 | } | ||
| 473 | push @cols,$v; | ||
| 474 | } | ||
| 475 | $output = join("\t",@cols); | ||
| 476 | } | ||
| 477 | else { | ||
| 478 | $output = $CONFIG{format} =~ /^json/? to_json($record,{allow_nonref=>1,canonical=>1,pretty=>$CONFIG{pretty}}) | ||
| 479 | : Dump $record; | ||
| 480 | } | ||
| 481 | |||
| 482 | output({data=>1}, $output); | ||
| 483 | $displayed++; | ||
| 484 | last if all_records_displayed(); | ||
| 485 | } | ||
| 486 | last if all_records_displayed(); | ||
| 487 | |||
| 488 | # Scroll forward | ||
| 489 | $start = time; | ||
| 490 | $result = $q->scroll_results(); | ||
| 491 | $duration += time - $start; | ||
| 492 | # Check if we need to keep going | ||
| 493 | last unless defined $result; | ||
| 494 | last unless $result->{hits} && $result->{hits}{hits} && @{ $result->{hits}{hits} } > 0 | ||
| 495 | } | ||
| 496 | last if all_records_displayed(); | ||
| 497 | } | ||
| 498 | |||
| 499 | output({stderr=>1,color=>'yellow'}, | ||
| 500 | "# Search Parameters:", | ||
| 501 | (map { "# $_" } split /\r?\n/, $JSON->encode($q->query)), | ||
| 502 | sprintf("# Displaying %d of %d results%s took %0.2f seconds.", | ||
| 503 | $displayed, | ||
| 504 | $OUT_OF || $TOTAL_HITS, | ||
| 505 | $OUT_OF ? " in $TOTAL_HITS documents" : '', | ||
| 506 | $duration, | ||
| 507 | ), | ||
| 508 | sprintf("# Indexes (%d of %d) searched: %s\n", | ||
| 509 | scalar(keys %displayed_indices), | ||
| 510 | scalar(keys %indices), | ||
| 511 | join(',', sort keys %displayed_indices) | ||
| 512 | ), | ||
| 513 | ) unless $OPT{'no-decorators'}; | ||
| 514 | |||
| 515 | if($CONFIG{summary} && keys %AGGS_TOTALS) { | ||
| 516 | unless ( $OPT{'no-decorators'} ) { | ||
| 517 | output({color=>'yellow'}, '#', '# Totals across batch', '#'); | ||
| 518 | output({color=>'cyan'},$agg_header); | ||
| 519 | } | ||
| 520 | foreach my $k (sort { $AGGS_TOTALS{$b} <=> $AGGS_TOTALS{$a} } keys %AGGS_TOTALS) { | ||
| 521 | output({data=>1,color=>'green'}, join "\t", | ||
| 522 | $AGGS_TOTALS{$k}, es_format_numeric($AGGS_TOTALS{$k} / $TOTAL_HITS, $CONFIG{precision}), $k | ||
| 523 | ); | ||
| 524 | } | ||
| 525 | } | ||
| 526 | |||
| 527 | sub all_records_displayed { | ||
| 528 | return 1 if $DONE; | ||
| 529 | return 0 if exists $OPT{tail}; | ||
| 530 | return 0 if exists $OPT{all}; | ||
| 531 | return 1 if $displayed >= $CONFIG{size}; | ||
| 532 | return 0; | ||
| 533 | } | ||
| 534 | |||
| 535 | sub document_lookdown { | ||
| 536 | my ($href,$field) = @_; | ||
| 537 | |||
| 538 | return $href->{$field} if exists $href->{$field}; | ||
| 539 | |||
| 540 | foreach my $k (keys %{ $href }) { | ||
| 541 | if( is_hashref($href->{$k}) ) { | ||
| 542 | return document_lookdown($href->{$k},$field); | ||
| 543 | } | ||
| 544 | } | ||
| 545 | return; | ||
| 546 | } | ||
| 547 | |||
| 548 | sub show_fields { | ||
| 549 | output({color=>'cyan'}, 'Fields available for search:' ); | ||
| 550 | my $total = 0; | ||
| 551 | my %types = (); | ||
| 552 | foreach my $field (sort keys %FIELDS) { | ||
| 553 | $total++; | ||
| 554 | my $type = $FIELDS{$field}->{type}; | ||
| 555 | $types{$type} ||= 0; | ||
| 556 | $types{$type}++; | ||
| 557 | my $color = $type eq 'ip' ? 'magenta' | ||
| 558 | : $type eq 'text' ? 'red' | ||
| 559 | : $type =~ /float|integer|short|byte|double/ ? 'cyan' | ||
| 560 | : $type =~ /^geo/ ? 'green' | ||
| 561 | : $type =~ /^date/ ? 'yellow' | ||
| 562 | : 'white'; | ||
| 563 | output({indent=>1,kv=>1,color=>$color}, $field => $type); | ||
| 564 | output({indent=>2}, sprintf "nested: %s - %s", | ||
| 565 | @{ $FIELDS{$field} }{qw(nested_path nested_key)} | ||
| 566 | ) if exists $FIELDS{$field}->{nested_path}; | ||
| 567 | } | ||
| 568 | output({color=>"yellow"}, | ||
| 569 | sprintf("# Fields: %d from a combined %d indices.\n", | ||
| 570 | $total, | ||
| 571 | scalar(keys %indices), | ||
| 572 | ) | ||
| 573 | ); | ||
| 574 | # Type Meta Roll Up | ||
| 575 | output({indent=>1}, join(', ', | ||
| 576 | map { "$types{$_} $_ fields" } | ||
| 577 | sort { $types{$b} <=> $types{$a} } | ||
| 578 | keys %types | ||
| 579 | ) | ||
| 580 | ); | ||
| 581 | } | ||
| 582 | |||
| 583 | sub show_bases { | ||
| 584 | output({color=>'cyan'}, 'Bases available for search:' ); | ||
| 585 | my @all = es_indices(_all => 1); | ||
| 586 | my %bases = (); | ||
| 587 | |||
| 588 | foreach my $index (@all) { | ||
| 589 | next if $index =~ /^\./; | ||
| 590 | my $days_old = es_index_days_old( $index ) || 0; | ||
| 591 | next unless defined $days_old; | ||
| 592 | $days_old = 0 if $days_old < 0; | ||
| 593 | foreach my $base (es_index_bases($index)) { | ||
| 594 | if( exists $bases{$base} ) { | ||
| 595 | $bases{$base}->{oldest} = $days_old if $days_old > $bases{$base}->{oldest}; | ||
| 596 | $bases{$base}->{youngest} = $days_old if $days_old < $bases{$base}->{youngest}; | ||
| 597 | } | ||
| 598 | else { | ||
| 599 | $bases{$base} = { oldest => $days_old, youngest => $days_old }; | ||
| 600 | } | ||
| 601 | } | ||
| 602 | } | ||
| 603 | foreach my $base (sort keys %bases) { | ||
| 604 | output({indent=>1,color=>'green'},$base); | ||
| 605 | verbose({indent=>2,kv=>1}, | ||
| 606 | map { | ||
| 607 | $_ => sprintf "%d days old", $bases{$base}->{$_} | ||
| 608 | } qw( youngest oldest ) | ||
| 609 | ); | ||
| 610 | } | ||
| 611 | |||
| 612 | output({color=>"yellow"}, | ||
| 613 | sprintf("# Bases: %d from a combined %d indices.\n", | ||
| 614 | scalar(keys %bases), | ||
| 615 | scalar(@all), | ||
| 616 | ) | ||
| 617 | ); | ||
| 618 | } | ||
| 619 | |||
| 620 | sub display_aggregations { | ||
| 621 | my ($aggregations,$total_docs) = (@_); | ||
| 622 | |||
| 623 | my $out_of = $aggregations->{out_of}{value} || 0; | ||
| 624 | $OUT_OF = $out_of if $out_of > $OUT_OF; | ||
| 625 | my $steps = exists $aggregations->{step} ? $aggregations->{step}{buckets} | ||
| 626 | : [ $aggregations ]; | ||
| 627 | my $indent = exists $aggregations->{step} ? 1 : 0; | ||
| 628 | |||
| 629 | foreach my $step ( @$steps ) { | ||
| 630 | my $aggs = exists $step->{top} ? $step->{top}{buckets} : []; | ||
| 631 | if( exists $step->{key_as_string} ) { | ||
| 632 | output({color=>'cyan',clear=>1}, sprintf "%d\t%s", @{$step}{qw(doc_count key_as_string)}); | ||
| 633 | } | ||
| 634 | if( @$aggs ) { | ||
| 635 | # For top the N of T needs to represent maximums | ||
| 636 | $displayed = scalar(@$aggs) if scalar(@$aggs) > $displayed; | ||
| 637 | output({color=>'cyan',indent=>$indent},$agg_header) unless $OPT{'no-decorators'}; | ||
| 638 | foreach my $agg ( @$aggs ) { | ||
| 639 | $AGGS_TOTALS{$agg->{key}} ||= 0; | ||
| 640 | $AGGS_TOTALS{$agg->{key}} += $agg->{doc_count}; | ||
| 641 | my @out = (); | ||
| 642 | my $top_docs; | ||
| 643 | foreach my $k (qw(score doc_count bg_count key)) { | ||
| 644 | next unless exists $agg->{$k}; | ||
| 645 | my $value = delete $agg->{$k}; | ||
| 646 | push @out, defined $value ? es_format_numeric($value, $CONFIG{precision}) : '-'; | ||
| 647 | if( $k eq 'doc_count' ) { | ||
| 648 | push @out, es_format_numeric($value ? $value / $total_docs : 0, $CONFIG{precision}); | ||
| 649 | $top_docs = $value; | ||
| 650 | } | ||
| 651 | } | ||
| 652 | if(exists $agg->{by} ) { | ||
| 653 | my $by = delete $agg->{by}; | ||
| 654 | if( exists $by->{value_as_string} ) { | ||
| 655 | unshift @out, $by->{value_as_string}; | ||
| 656 | } | ||
| 657 | elsif( exists $by->{value} ) { | ||
| 658 | unshift @out, es_format_numeric($by->{value}); | ||
| 659 | } | ||
| 660 | } | ||
| 661 | # Handle the --with elements | ||
| 662 | my %subaggs = (); | ||
| 663 | if( keys %{ $agg } ) { | ||
| 664 | foreach my $k (sort keys %{ $agg }) { | ||
| 665 | next unless is_hashref($agg->{$k}); | ||
| 666 | if( exists $agg->{$k}{buckets} ) { | ||
| 667 | my @sub; | ||
| 668 | foreach my $subagg (@{ $agg->{$k}{buckets} }) { | ||
| 669 | my @elms = (); | ||
| 670 | next unless exists $subagg->{key}; | ||
| 671 | push @elms, $subagg->{key}; | ||
| 672 | foreach my $dk (qw(score doc_count bg_count)) { | ||
| 673 | next unless exists $subagg->{$dk}; | ||
| 674 | my $v = delete $subagg->{$dk}; | ||
| 675 | push @elms, defined $v ? es_format_numeric($v, $CONFIG{precision}) : '-'; | ||
| 676 | push @elms, es_format_numeric($v / $top_docs,, $CONFIG{precision}) | ||
| 677 | if $dk eq 'doc_count' and $top_docs; | ||
| 678 | } | ||
| 679 | push @sub, \@elms; | ||
| 680 | } | ||
| 681 | $subaggs{$k} = \@sub if @sub; | ||
| 682 | } | ||
| 683 | # Simple Numeric Aggs | ||
| 684 | elsif( exists $agg->{$k}{value_as_string} ) { | ||
| 685 | $subaggs{$k} = [ [ $agg->{$k}{value_as_string} ] ]; | ||
| 686 | } | ||
| 687 | elsif( exists $agg->{$k}{value} ) { | ||
| 688 | my $v = es_format_numeric($agg->{$k}{value},$CONFIG{precision}); | ||
| 689 | $subaggs{$k} = [ [ $v ] ]; | ||
| 690 | } | ||
| 691 | # Percentiles | ||
| 692 | elsif( exists $agg->{$k}{values} ) { | ||
| 693 | my @pcts; | ||
| 694 | foreach my $pctl (sort { $a <=> $b } keys %{ $agg->{$k}{values} }) { | ||
| 695 | push @pcts, "p$pctl", es_format_numeric($agg->{$k}{values}{$pctl}, $CONFIG{precision}); | ||
| 696 | } | ||
| 697 | $subaggs{$k} = [ \@pcts ]; | ||
| 698 | } | ||
| 699 | else { | ||
| 700 | # Statistics | ||
| 701 | my @stats; | ||
| 702 | my %alias = qw( variance var std_deviation stdev ); | ||
| 703 | foreach my $stat (qw(count min avg max sum variance std_deviation)) { | ||
| 704 | next unless exists $agg->{$k}{$stat}; | ||
| 705 | my $v = es_format_numeric($agg->{$k}{$stat}, $CONFIG{precision}); | ||
| 706 | push @stats, $alias{$stat} || $stat => $v; | ||
| 707 | } | ||
| 708 | $subaggs{$k} = [ \@stats ] if @stats; | ||
| 709 | } | ||
| 710 | } | ||
| 711 | } | ||
| 712 | if( keys %subaggs ) { | ||
| 713 | foreach my $subagg (sort keys %subaggs) { | ||
| 714 | foreach my $extra ( @{ $subaggs{$subagg} } ) { | ||
| 715 | output({indent=>$indent,data=>1}, | ||
| 716 | join "\t", @out, $subagg, @{ $extra } | ||
| 717 | ); | ||
| 718 | } | ||
| 719 | } | ||
| 720 | } | ||
| 721 | else { | ||
| 722 | # Simple output | ||
| 723 | output({indent=>$indent,data=>!$CONFIG{summary}}, join("\t",@out)); | ||
| 724 | } | ||
| 725 | } | ||
| 726 | } | ||
| 727 | elsif(exists $aggregations->{top}) { | ||
| 728 | output({indent=>1,color=>'red'}, "= No results."); | ||
| 729 | } | ||
| 730 | } | ||
| 731 | } | ||
| 732 | |||
| 733 | sub by_index_age { | ||
| 734 | return $ORDER eq 'asc' | ||
| 735 | ? $indices{$b} <=> $indices{$a} | ||
| 736 | : $indices{$a} <=> $indices{$b}; | ||
| 737 | } | ||
| 738 | |||
| 739 | __END__ |