lib/Log/Munger/LogProcessor.pm

filter: Statementstmt 89.0% 357 / 401 Branchbran 79.5% 210 / 264 Conditioncond 58.7% 37 / 63 MC/DCmcdc 36.5% 15 / 41 Subroutinesub 100% 22 / 22 Podpod 100% 3 / 3 totaltotal 81.1% 644 / 794 CC 172 SCAR 33.6
CC
172
Cov
83%
_compile_rule
63.5
_compile_decompose
50.7
_decompose
25.2
CRAP 28.9
« lib/Log/Munger/RulesTest.pm ↑ Summary lib/Log/Munger/App/Command/degrok.pm »
1 package Log::Munger::LogProcessor;
2  
3 27 ▶use 5.006;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 27 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 27 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 27 ▶use Scalar::Util qw(looks_like_number);
Subroutine BEGIN: called CC=1 SCAR=0.0
7 27 ▶use JSON ();
Subroutine BEGIN: called CC=1 SCAR=0.0
8 27 ▶use Log::Munger::RuleFileParser ();
Subroutine BEGIN: called CC=1 SCAR=0.0
9 27 ▶use Log::Munger::RulesUsable ();
Subroutine BEGIN: called CC=1 SCAR=0.0
10  
11 # the "json" decompose type decodes with JSON::decode_json (utf8-on, and backed
12 # by JSON::XS when installed via the JSON front end) so it handles a raw-bytes
13 # MESSAGE without a separate decode step.
14  
15 =head1 NAME
16  
17 Log::Munger::LogProcessor - Compiles rule files and matches log records against them.
18  
19 =head1 VERSION
20  
21 Version 0.0.1
22  
23 =cut
24  
25 27 our $VERSION = '0.0.1';
26  
27 =head1 SYNOPSIS
28  
29 use Log::Munger::LogProcessor;
30  
31 my $processor = Log::Munger::LogProcessor->new( 'rules' => [ 'base', 'postfix' ] );
32  
33 my $fields = $processor->process_item( 'item' => $decoded_log_hashref );
34 if ( defined($fields) ) {
35 # $fields is a hashref of the named captures from the first matching rule
36 }
37  
38 Each named rule file is loaded and its C<rules:> section compiled into an ordered,
39 merged dispatch list. A rule file with no C<rules:> section (a pure primitive
40 library such as C<base>) is accepted and skipped for dispatch.
41  
42 A rule dispatches when B<all> of its gates pass B<and> one of its patterns matches
43 the target field. A gate passes when its field matches any of the gate's values,
44 patterns are first-match-wins, and rules are tried in load order (files in the
45 order given, rules in file order).
46  
47 =head1 METHODS
48  
49 =head2 new
50  
51 Loads and compiles the named rule files.
52  
53 - rules :: Rule files to load. The taken value is an array ref.
54 Default :: undef (required)
55  
56 - geoip :: Path to a MaxMind .mmdb database. When given, rules may flag
57 captured fields (via a rule-level or file-level C<geoip:> list) whose
58 values are looked up and stored under C<< $result->{geoip}{$field} >>.
59 Requires IP::Geolocation::MMDB (a soft dependency, only loaded when this
60 option is used).
61 Default :: undef (geoip disabled)
62  
63 =cut
64  
65 32 ▶sub new {
Subroutine new: called CC=15 SCAR=28.5
Pod documented
66 32 my ( $blank, %opts ) = @_;
67  
68 32 ▶ if ( !defined( $opts{'rules'} ) ) {
Branch: 3 branches
BranchTrueFalseTotal
if (not defined $opts{'rules'}) { } 0 32 32
elsif (ref $opts{'rules'} ne 'ARRAY') { } 0 32 32
elsif (not defined $opts{'rules'}[0]) { } 0 32 32
69 0 die('$opts{rules} is undef');
70 } elsif ( ref( $opts{'rules'} ) ne 'ARRAY' ) {
71 0 die( '$opts{rules} has a ref of "' . ref( $opts{'rules'} ) . '" and not "ARRAY"' );
72 } elsif ( !defined( $opts{'rules'}[0] ) ) {
73 0 die('$opts{rules}[0] is undef... no rules have been specified to use');
74 }
75  
76 32 my $self = {
77 'rules' => [], # compiled rules, ordered and merged across files
78 'rule_files' => [], # names of the files loaded, in order
79 'geoip_reader' => undef, # IP::Geolocation::MMDB, if geoip is enabled
80 };
81 32 bless $self;
82  
83 # geoip is an opt-in feature with a soft dependency: only when a database
84 # path is given do we require IP::Geolocation::MMDB and open the database
85 32 ▶ if ( defined( $opts{'geoip'} ) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $opts{"geoip"}) 11 21 32
86 11 ▶ if ( ref( $opts{'geoip'} ) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $opts{"geoip"} ne "") 0 11 11
87 0 die( '$opts{geoip} has a ref of "' . ref( $opts{'geoip'} ) . '" and not "" (a database path)' );
88 }
89 11 my $have_reader = eval { require IP::Geolocation::MMDB; 1; };
90 11 ▶ if ( !$have_reader ) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($have_reader) 0 11 11
91 0 die( 'geoip was requested but IP::Geolocation::MMDB is not installed... ' . $@ );
92 }
93 11 eval { $self->{'geoip_reader'} = IP::Geolocation::MMDB->new( 'file' => $opts{'geoip'} ); };
94 11 ▶ if ($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 0 11 11
95 0 die( 'Failed to open the geoip database "' . $opts{'geoip'} . '"... ' . $@ );
96 }
97 } ## end if ( defined( $opts{'geoip'} ) )
98  
99 32 my $parser = Log::Munger::RuleFileParser->new;
100  
101 32 my $rules_int = 0;
102 32 ▶ while ( defined( $opts{'rules'}[$rules_int] ) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $opts{"rules"}[$rules_int]) 130 32 162
103 130 my $name = $opts{'rules'}[$rules_int];
104  
105 # RuleFileParser->load resolves the name (or path) itself via
106 # WhichRuleFile, so hand it over untouched; resolving here too would
107 # repeat the same work and turn error messages into resolved paths
108 # rather than the name the caller actually passed
109 130 my $rules;
110 130 eval { $rules = $parser->load( 'file' => $name ); };
111 130 ▶ if ($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 0 130 130
112 0 die( 'Failed to load $opts{rules}[' . $rules_int . '], "' . $name . '"... ' . $@ );
113 }
114  
115 # a file used purely as a primitive library (e.g. base) has no rules:
116 # section; it is legal to list it, it just contributes no dispatch rules
117 130 ▶ if ( !defined( $rules->{'rules'} ) ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $rules->{"rules"}) 7 123 130
118 7 push( @{ $self->{'rule_files'} }, $name );
119 7 $rules_int++;
120 7 next;
121 }
122  
123 123 eval { Log::Munger::RulesUsable->usable( 'rules' => $rules ); };
124 123 ▶ if ($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 0 123 123
125 0 die( '.rules in "' . $name . '" is not usable... ' . $@ );
126 }
127  
128 123 my $vars = $rules->{'vars'};
129 123 ▶ if ( ref($vars) ne 'HASH' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $vars ne "HASH") 1 122 123
130 1 $vars = {};
131 }
132  
133 # file-level geoip / decompose / convert defaults for every rule in the
134 # file that does not carry its own
135 123 my $file_geoip = $rules->{'geoip'};
136 123 my $file_decompose = $rules->{'decompose'};
137 123 my $file_convert = $rules->{'convert'};
138  
139 123 my $rule_int = 0;
140 123 foreach my $rule ( @{ $rules->{'rules'} } ) {
141 299 my $compiled;
142 299 eval {
143 299 $compiled = $self->_compile_rule(
144 'rule' => $rule,
145 'vars' => $vars,
146 'default_geoip' => $file_geoip,
147 'default_decompose' => $file_decompose,
148 'default_convert' => $file_convert,
149 );
150 };
151 299 ▶ if ($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 0 299 299
152 0 die( 'Failed to compile .rules[' . $rule_int . '] in "' . $name . '"... ' . $@ );
153 }
154 299 push( @{ $self->{'rules'} }, $compiled );
155 299 $rule_int++;
156 } ## end foreach my $rule ( @{ $rules->{'rules'} } )
157  
158 123 push( @{ $self->{'rule_files'} }, $name );
159 123 $rules_int++;
160 } ## end while ( defined( $opts{'rules'}[$rules_int] ))
161  
162 32 return $self;
163 } ## end sub new
164  
165 =head2 process_item
166  
167 Runs an item through the compiled rules and returns the named captures of the
168 first matching rule.
169  
170 The item may be given directly with C<item>, or assembled from the individual
171 syslog fields with C<message> (plus the optional C<program> / C<priority> /
172 C<facility>). This lets a caller that already has the fields split out (a syslog
173 reader, C<baphomet>, etc.) hand them over without building the record hash
174 itself.
175  
176 - item :: The decoded log record (a hash ref), or a bare string. A bare
177 string is treated as a raw log line and matched as the C<MESSAGE> field,
178 so callers feeding whole log lines (e.g. apache/nginx access logs) need
179 not wrap them. If given, C<item> takes precedence and the C<message> /
180 C<program> / C<priority> / C<facility> args below are ignored.
181 Default :: undef
182  
183 - message :: The raw log message; assembled into C<< { MESSAGE => ... } >>.
184 Default :: undef
185  
186 - program :: Adds a C<PROGRAM> field (the usual gate for daemon rule files).
187 Default :: undef
188  
189 - priority :: Adds a C<PRIORITY> field.
190 Default :: undef
191  
192 - facility :: Adds a C<FACILITY> field.
193 Default :: undef
194  
195 my $fields = $processor->process_item( 'item' => $decoded_hashref );
196 my $fields = $processor->process_item( 'message' => $line, 'program' => 'sshd' );
197  
198 Returns a hash ref of the winning pattern's named captures on a match (which may
199 be an empty hash ref if the pattern had no named captures), or C<undef> if no rule
200 matched. This method never dies: an exception during matching is caught and comes
201 back as C<undef>. That bounds failures, not runtime -- a pattern prone to
202 catastrophic backtracking can still burn CPU on a hostile line, so keep patterns
203 anchored and tested.
204  
205 =cut
206  
207 132 ▶sub process_item {
Subroutine process_item: called CC=2 SCAR=6.9
Pod documented
208 132 my ( $self, %opts ) = @_;
209  
210 132 my $match = $self->_run( $self->_item_from_opts(%opts) );
211 132 ▶ return $match->{'matched'} ? $match->{'fields'} : undef;
Branch: 1 branch
BranchTrueFalseTotal
$match->{'matched'} ? : 119 13 132
212 }
213  
214 =head2 explain_item
215  
216 Like L</process_item> but returns match metadata instead of just the fields,
217 for tooling that needs to know I<which> rule fired. Accepts the same arguments as
218 L</process_item> (C<item>, or C<message> plus optional C<program> / C<priority> /
219 C<facility>). Returns a hash ref:
220  
221 { matched => 0 } # nothing matched
222 { matched => 1, rule => <name>, pattern => <index>,
223 field => <target field>, fields => { ...captures... } } # a rule fired
224  
225 Never dies.
226  
227 =cut
228  
229 4 ▶sub explain_item {
Subroutine explain_item: called CC=1 SCAR=0.0
Pod documented
230 4 my ( $self, %opts ) = @_;
231  
232 4 return $self->_run( $self->_item_from_opts(%opts) );
233 }
234  
235 # Resolves the item to run from the caller's arguments. An explicit item (hash
236 # ref or bare string) is returned as-is for backward compatibility. Otherwise,
237 # when message is given, a record hash is assembled from it plus any of the
238 # optional program / priority / facility fields, mapped to the syslog-ng-style
239 # upper-case keys PROGRAM / PRIORITY / FACILITY that rule gates match on.
240 #
241 # Takes the same %opts as process_item and explain_item.
242 #
243 # Returns the item to hand to _run: the item as given, an assembled hash ref,
244 # or undef when neither item nor message was given (which _run treats as a
245 # non-match).
246 #
247 # my $item = $self->_item_from_opts(%opts);
248 136 ▶sub _item_from_opts {
Subroutine _item_from_opts: called CC=6 SCAR=17.9
249 136 my ( $self, %opts ) = @_;
250  
251 # an explicit item wins, preserving the original hashref/bare-string behavior
252 136 ▶ return $opts{'item'} if ( exists( $opts{'item'} ) );
Branch: 1 branch
BranchTrueFalseTotal
if exists $opts{"item"} 130 6 136
253  
254 # nothing to assemble
255 6 ▶ return undef if ( !defined( $opts{'message'} ) );
Branch: 1 branch
BranchTrueFalseTotal
unless defined $opts{"message"} 1 5 6
256  
257 5 my $item = { 'MESSAGE' => $opts{'message'} };
258 5 ▶ $item->{'PROGRAM'} = $opts{'program'} if ( defined( $opts{'program'} ) );
Branch: 1 branch
BranchTrueFalseTotal
if defined $opts{"program"} 4 1 5
259 5 ▶ $item->{'PRIORITY'} = $opts{'priority'} if ( defined( $opts{'priority'} ) );
Branch: 1 branch
BranchTrueFalseTotal
if defined $opts{"priority"} 1 4 5
260 5 ▶ $item->{'FACILITY'} = $opts{'facility'} if ( defined( $opts{'facility'} ) );
Branch: 1 branch
BranchTrueFalseTotal
if defined $opts{"facility"} 1 4 5
261  
262 5 return $item;
263 } ## end sub _item_from_opts
264  
265 # The shared matcher for process_item and explain_item. Coerces a bare-string
266 # item to { MESSAGE => $string }, walks the rules, and on the first
267 # gate-passing pattern match runs decompose -> geoip -> convert.
268 #
269 # Args:
270 #
271 # - $item :: The record to match. A hash ref, a bare string, or undef.
272 #
273 # Returns the match-metadata hash explain_item documents: { matched => 0 } on
274 # no match, else { matched => 1, rule, pattern, field, fields }. Never dies.
275 #
276 # my $match = $self->_run($item);
277 136 ▶sub _run {
Subroutine _run: called CC=17 SCAR=28.4
278 136 my ( $self, $item ) = @_;
279  
280 # a bare string is treated as a raw log line and matched as MESSAGE, so
281 # callers feeding whole log lines (apache/nginx access logs, etc.) do not
282 # have to wrap them in a hash themselves
283 136 ▶ if ( defined($item) && ref($item) eq '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $item and ref $item eq "") 24 112 136
Condition100%3/3
defined $item and ref $item eq ""
!ll&&!rl&&r
211024
MC/DC100%2/2
defined $item and ref $item eq ""
defined $item ref $item eq ""
Truth table100%3/3
A and B
A= defined $itemB= ref $item eq ""
ABresult
0X0
100
111
284 24 $item = { 'MESSAGE' => $item };
285 }
286 136 ▶ if ( !defined($item) || ref($item) ne 'HASH' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $item or ref $item ne "HASH") 2 134 136
Condition66%2/3
not defined $item or ref $item ne "HASH"
l!l&&r!l&&!r
20134
MC/DC50%1/2
not defined $item or ref $item ne "HASH"
not defined $item ref $item ne "HASH"
Truth table66%2/3
not(A) or B
A= not defined $itemB= ref $item ne "HASH"
ABresult
1X1
011
000
287 2 return { 'matched' => 0 };
288 }
289  
290 134 my $result = { 'matched' => 0 };
291 134 eval {
292 134 RULE: foreach my $rule ( @{ $self->{'rules'} } ) {
293  
294 # gates: every gate must pass (ANDed)
295 4187 foreach my $gate ( @{ $rule->{'gate'} } ) {
296 3769 ▶ if ( !$self->_gate_passes( $gate, $item->{ $gate->{'field'} } ) ) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($self->_gate_passes($gate, $$item{$gate->{"field"}})) 3666 103 3769
297 3666 next RULE;
298 }
299 }
300  
301 # target field to munge
302 521 my $target = $item->{ $rule->{'field'} };
303 521 ▶ if ( !defined($target) || ref($target) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $target or ref $target ne "") 1 520 521
Condition66%2/3
not defined $target or ref $target ne ""
l!l&&r!l&&!r
10520
MC/DC50%1/2
not defined $target or ref $target ne ""
not defined $target ref $target ne ""
Truth table66%2/3
not(A) or B
A= not defined $targetB= ref $target ne ""
ABresult
1X1
011
000
304 1 next RULE;
305 }
306  
307 # patterns: first match wins
308 520 my $pattern_int = 0;
309 520 foreach my $pattern ( @{ $rule->{'patterns'} } ) {
310 2044 ▶ if ( $target =~ $pattern ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($target =~ /$pattern/) 122 1922 2044
311 # copy %+ immediately, before any later match can clobber it
312 122 my %captures = %+;
313 # decompose first (it can produce fields geoip then looks up)
314 122 ▶ if ( @{ $rule->{'decompose'} } ) {
Branch: 1 branch
BranchTrueFalseTotal
if (@{$rule->{"decompose"};}) 50 72 122
315 50 $self->_decompose( $rule, \%captures );
316 }
317 122 ▶ if ( $self->{'geoip_reader'} && @{ $rule->{'geoip'} } ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->{"geoip_reader"} and @{$rule->{"geoip"};}) 12 110 122
Condition66%2/3
$self->{"geoip_reader"} and @{$rule->{"geoip"};}
!ll&&!rl&&r
110012
MC/DC50%1/2
$self->{"geoip_reader"} and @{$rule->{"geoip"};}
$self->{"geoip_reader"} @{$rule->{"geoip"};}
Truth table66%2/3
A and B
A= $self->{"geoip_reader"}B= @{$rule->{"geoip"};}
ABresult
0X0
100
111
318 12 $self->_geoip_enrich( $rule, \%captures );
319 }
320 # coerce numeric fields last, after geoip has seen the
321 # string addresses
322 122 ▶ if ( keys( %{ $rule->{'convert'} } ) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (keys %{$$rule{"convert"};}) 88 34 122
323 88 $self->_convert( $rule, \%captures );
324 }
325 $result = {
326 'matched' => 1,
327 'rule' => $rule->{'name'},
328 'pattern' => $pattern_int,
329 122 'field' => $rule->{'field'},
330 'fields' => \%captures,
331 };
332 122 last RULE;
333 } ## end if ( $target =~ $pattern )
334 1922 $pattern_int++;
335 } ## end foreach my $pattern ( @{ $rule->{'patterns'} } )
336 } ## end RULE: foreach my $rule ( @{ $self->{'rules'} } )
337 };
338 134 ▶ if ($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 0 134 134
339 0 return { 'matched' => 0 };
340 }
341  
342 134 return $result;
343 } ## end sub _run
344  
345 # Checks a single compiled gate against the value of its field.
346 #
347 # A gate passes when the value is a defined plain scalar and either sits in the
348 # gate's literals set or matches one of its //regexp// values. An absent,
349 # undef, or non-scalar value fails the gate. This is the one place gate
350 # semantics live; Log::Munger::RulesTest calls it too, so a test's gate check
351 # is the engine's rather than a parallel copy.
352 #
353 # Args:
354 #
355 # - $gate :: A compiled gate entry, as _compile_rule builds them:
356 # { field, literals => {}, regexps => [ qr//, ... ] }.
357 #
358 # - $value :: The value of the gate's field out of the record being
359 # matched. May be undef.
360 #
361 # Returns 1 when the gate passes, otherwise 0.
362 #
363 # if ( !$self->_gate_passes( $gate, $item->{ $gate->{'field'} } ) ) {
364 # next RULE;
365 # }
366 4478 ▶sub _gate_passes {
Subroutine _gate_passes: called CC=6 SCAR=17.9
367 4478 my ( $self, $gate, $value ) = @_;
368  
369 4478 ▶ if ( !defined($value) || ref($value) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $value or ref $value ne "") 165 4313 4478
Condition66%2/3
not defined $value or ref $value ne ""
l!l&&r!l&&!r
16504313
MC/DC50%1/2
not defined $value or ref $value ne ""
not defined $value ref $value ne ""
Truth table66%2/3
not(A) or B
A= not defined $valueB= ref $value ne ""
ABresult
1X1
011
000
370 165 return 0;
371 }
372 4313 ▶ if ( $gate->{'literals'}{$value} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($gate->{"literals"}{$value}) 30 4283 4313
373 30 return 1;
374 }
375 4283 foreach my $re ( @{ $gate->{'regexps'} } ) {
376 4111 ▶ if ( $value =~ $re ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($value =~ /$re/) 535 3576 4111
377 535 return 1;
378 }
379 }
380  
381 3748 return 0;
382 } ## end sub _gate_passes
383  
384 # For each of the rule's flagged geoip fields that was captured, looks the
385 # value up in the geoip database and stores the record under
386 # $captures->{geoip}{$field}.
387 #
388 # Args:
389 #
390 # - $rule :: The compiled rule, read for its geoip field list.
391 #
392 # - $captures :: Hash ref of the captured fields. Modified in place.
393 #
394 # Returns nothing. A field that is undef, empty, not an address, or absent from
395 # the database is simply skipped; a lookup never dies.
396 #
397 # $self->_geoip_enrich( $rule, \%captures );
398 12 ▶sub _geoip_enrich {
Subroutine _geoip_enrich: called CC=8 SCAR=20.9
399 12 my ( $self, $rule, $captures ) = @_;
400  
401 12 my %geo;
402 12 foreach my $field ( @{ $rule->{'geoip'} } ) {
403 16 my $address = $captures->{$field};
404 16 ▶ next if ( !defined($address) || $address eq '' );
Branch: 1 branch
BranchTrueFalseTotal
if not defined $address or $address eq "" 4 12 16
Condition66%2/3
not defined $address or $address eq ""
l!l&&r!l&&!r
4012
MC/DC50%1/2
not defined $address or $address eq ""
not defined $address $address eq ""
Truth table66%2/3
not(A) or B
A= not defined $addressB= $address eq ""
ABresult
1X1
011
000
405  
406 12 my $record;
407 # record_for_address dies on a non-address; keep that local so a bad
408 # value never costs the rest of the enrichment
409 12 eval { $record = $self->{'geoip_reader'}->record_for_address($address); };
410 12 ▶ next if ( $@ || !defined($record) );
Branch: 1 branch
BranchTrueFalseTotal
if $@ or not defined $record 1 11 12
Condition66%2/3
$@ or not defined $record
l!l&&r!l&&!r
0111
MC/DC50%1/2
$@ or not defined $record
$@ not defined $record
Truth table66%2/3
A or not(B)
A= $@B= not defined $record
ABresult
1X1
011
000
411  
412 11 $geo{$field} = $record;
413 } ## end foreach my $field ( @{ $rule->{'geoip'} } )
414  
415 # never clobber a capture that happens to be named geoip; no enrichment
416 # step overwrites an existing capture
417 12 ▶ if ( %geo && !exists( $captures->{'geoip'} ) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (%geo and not exists $captures->{"geoip"}) 11 1 12
Condition66%2/3
%geo and not exists $captures->{"geoip"}
!ll&&!rl&&r
1011
MC/DC50%1/2
%geo and not exists $captures->{"geoip"}
%geo not exists $captures->{"geoip"}
Truth table66%2/3
A and not(B)
A= %geoB= not exists $captures->{"geoip"}
ABresult
0X0
100
111
418 11 $captures->{'geoip'} = \%geo;
419 }
420  
421 12 return;
422 } ## end sub _geoip_enrich
423  
424 # Compiles a decompose: list (an array of {field, type, ...} entries) into
425 # runtime structures. Three types are supported:
426 #
427 # - kv :: split a "k=v k=v" blob into fields. Options: field_split
428 # (default " "), value_split (default "="), prefix (default ""),
429 # trim (chars stripped from each end of a value), remove. With
430 # quoted => true it is quote-aware instead: a value may be
431 # "double" or 'single' quoted (quotes stripped, the separator
432 # allowed inside them), and pairs are found by scanning for
433 # key=value shapes rather than by splitting on field_split.
434 # - pattern :: re-match the field against a named var (or inline regexp),
435 # anchored, and merge its named captures. Options: pattern,
436 # remove.
437 # - json :: JSON-decode the field (for logs that embed a JSON document in
438 # a sub-field, e.g. MongoDB). By default the decoded structure
439 # is flattened into keys of prefix + path joined by separator
440 # (default "_"); MongoDB extended-JSON wrappers ({"$date":...},
441 # {"$oid":...}, {"$numberLong":...}) collapse to their scalar,
442 # booleans normalize to 1/0, and JSON null is skipped. Arrays
443 # are keyed by index. Options: prefix (default ""), separator
444 # (default "_"), remove, and nested => true (store the decoded
445 # structure whole under a single key -- the prefix minus a
446 # trailing separator, or with no prefix the source field itself,
447 # replacing the raw string -- instead of flattening). A field
448 # whose value is not valid JSON, or is JSON for a bare scalar,
449 # is left untouched.
450 #
451 # Every entry may set remove: true to drop the source field afterwards. An
452 # entry may also carry a tests: [ { input, result }, ... ] list, which
453 # Log::Munger::RulesTest runs in isolation; the tests key is ignored here.
454 #
455 # Args:
456 #
457 # - $decompose :: The decompose list as written in the rule file. An array
458 # ref of { field, type, ... } hash refs.
459 #
460 # - $vars :: The compiled vars hash ref a "type: pattern" entry resolves
461 # its pattern name against.
462 #
463 # Returns an array ref of compiled entries. Dies on a malformed entry, an
464 # unknown type, un-degrokked grok, or a pattern that will not compile.
465 #
466 # my $compiled = $self->_compile_decompose( $rules->{'decompose'}, $vars );
467 502 ▶sub _compile_decompose {
Subroutine _compile_decompose: called CC=28 SCAR=39.3
468 502 my ( $self, $decompose, $vars ) = @_;
469  
470 502 ▶ if ( ref($decompose) ne 'ARRAY' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $decompose ne "ARRAY") 0 502 502
471 0 die( '.decompose has a ref of "' . ref($decompose) . '" and not "ARRAY"' );
472 }
473  
474 502 my @compiled;
475 502 my $int = 0;
476 502 foreach my $d ( @{$decompose} ) {
477 1866 ▶ if ( ref($d) ne 'HASH' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $d ne "HASH") 0 1866 1866
478 0 die( '.decompose[' . $int . '] has a ref of "' . ref($d) . '" and not "HASH"' );
479 }
480 1866 ▶ if ( !defined( $d->{'field'} ) || ref( $d->{'field'} ) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $d->{"field"} or ref $d->{"field"} ne "") 0 1866 1866
Condition33%1/3
not defined $d->{"field"} or ref $d->{"field"} ne ""
l!l&&r!l&&!r
001866
MC/DC0%0/2
not defined $d->{"field"} or ref $d->{"field"} ne ""
not defined $d->{"field"} ref $d->{"field"} ne ""
Truth table33%1/3
not(A) or B
A= not defined $d->{"field"}B= ref $d->{"field"} ne ""
ABresult
1X1
011
000
481 0 die( '.decompose[' . $int . '].field is undef or not a string' );
482 }
483 1866 ▶ my $type = defined( $d->{'type'} ) ? $d->{'type'} : 'kv';
Branch: 1 branch
BranchTrueFalseTotal
defined $d->{'type'} ? : 1866 0 1866
484 my $c = {
485 'field' => $d->{'field'},
486 'type' => $type,
487 1866 ▶ 'remove' => ( $d->{'remove'} ? 1 : 0 ),
Branch: 1 branch
BranchTrueFalseTotal
$d->{'remove'} ? : 1688 178 1866
488 };
489  
490 1866 ▶ if ( $type eq 'kv' ) {
Branch: 3 branches
BranchTrueFalseTotal
if ($type eq 'kv') { } 419 1447 1866
elsif ($type eq 'pattern') { } 1433 14 1447
elsif ($type eq 'json') { } 14 0 14
491 419 ▶ $c->{'prefix'} = defined( $d->{'prefix'} ) ? $d->{'prefix'} : '';
Branch: 1 branch
BranchTrueFalseTotal
defined $d->{'prefix'} ? : 414 5 419
492 419 $c->{'trim'} = $d->{'trim'}; # may be undef
493 419 ▶ $c->{'field_split'} = defined( $d->{'field_split'} ) ? $d->{'field_split'} : ' ';
Branch: 1 branch
BranchTrueFalseTotal
defined $d->{'field_split'} ? : 53 366 419
494 419 ▶ $c->{'value_split'} = defined( $d->{'value_split'} ) ? $d->{'value_split'} : '=';
Branch: 1 branch
BranchTrueFalseTotal
defined $d->{'value_split'} ? : 27 392 419
495 419 ▶ $c->{'quoted'} = ( $d->{'quoted'} ? 1 : 0 );
Branch: 1 branch
BranchTrueFalseTotal
$d->{'quoted'} ? : 66 353 419
496 419 ▶ if ( $c->{'field_split'} eq '' ) { die( '.decompose[' . $int . '].field_split is empty' ); }
Branch: 1 branch
BranchTrueFalseTotal
if ($c->{"field_split"} eq "") 0 419 419
497 419 ▶ if ( $c->{'value_split'} eq '' ) { die( '.decompose[' . $int . '].value_split is empty' ); }
Branch: 1 branch
BranchTrueFalseTotal
if ($c->{"value_split"} eq "") 0 419 419
498 } elsif ( $type eq 'pattern' ) {
499 1433 my $pname = $d->{'pattern'};
500 1433 ▶ if ( !defined($pname) || ref($pname) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $pname or ref $pname ne "") 0 1433 1433
Condition33%1/3
not defined $pname or ref $pname ne ""
l!l&&r!l&&!r
001433
MC/DC0%0/2
not defined $pname or ref $pname ne ""
not defined $pname ref $pname ne ""
Truth table33%1/3
not(A) or B
A= not defined $pnameB= ref $pname ne ""
ABresult
1X1
011
000
501 0 die( '.decompose[' . $int . '].pattern is undef or not a string' );
502 }
503 1433 ▶ my $rx_string = exists( $vars->{$pname} ) ? $vars->{$pname} : $pname;
Branch: 1 branch
BranchTrueFalseTotal
exists $vars->{$pname} ? : 1433 0 1433
504 1433 ▶ if ( $rx_string =~ /%\{[^}]*\}/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($rx_string =~ /%\{[^}]*\}/) 0 1433 1433
505 0 die( '.decompose[' . $int . '].pattern ("' . $pname . '") contains un-degrokked grok "%{...}"' );
506 }
507 1433 my $rx;
508 1433 eval { $rx = qr/\A(?:$rx_string)\z/; };
509 1433 ▶ if ($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 0 1433 1433
510 0 die( '.decompose[' . $int . '].pattern ("' . $pname . '") does not compile... ' . $@ );
511 }
512 1433 $c->{'regexp'} = $rx;
513 } elsif ( $type eq 'json' ) {
514 # JSON-decode the field and either flatten the structure into
515 # prefixed keys (default) or store the decoded structure whole
516 # (nested => true).
517 14 ▶ $c->{'prefix'} = defined( $d->{'prefix'} ) ? $d->{'prefix'} : '';
Branch: 1 branch
BranchTrueFalseTotal
defined $d->{'prefix'} ? : 13 1 14
518 14 ▶ $c->{'separator'} = defined( $d->{'separator'} ) ? $d->{'separator'} : '_';
Branch: 1 branch
BranchTrueFalseTotal
defined $d->{'separator'} ? : 0 14 14
519 14 ▶ $c->{'nested'} = ( $d->{'nested'} ? 1 : 0 );
Branch: 1 branch
BranchTrueFalseTotal
$d->{'nested'} ? : 2 12 14
520 14 ▶ if ( ref( $c->{'prefix'} ) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $c->{"prefix"} ne "") 0 14 14
521 0 die( '.decompose[' . $int . '].prefix is not a string' );
522 }
523 14 ▶ if ( ref( $c->{'separator'} ) ne '' || $c->{'separator'} eq '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $c->{"separator"} ne "" or $c->{"separator"} eq "") 0 14 14
Condition33%1/3
ref $c->{"separator"} ne "" or $c->{"separator"} eq ""
l!l&&r!l&&!r
0014
MC/DC0%0/2
ref $c->{"separator"} ne "" or $c->{"separator"} eq ""
ref $c->{"separator"} ne "" $c->{"separator"} eq ""
Truth table33%1/3
A or B
A= ref $c->{"separator"} ne ""B= $c->{"separator"} eq ""
ABresult
1X1
011
000
524 0 die( '.decompose[' . $int . '].separator is undef, empty, or not a string' );
525 }
526 } else {
527 0 die( '.decompose[' . $int . '].type "' . $type . '" is unknown (expected "kv", "pattern", or "json")' );
528 }
529  
530 1866 push( @compiled, $c );
531 1866 $int++;
532 } ## end foreach my $d ( @{$decompose} )
533  
534 502 return \@compiled;
535 } ## end sub _compile_decompose
536  
537 # Applies a rule's compiled decompose entries to the captures hash, in order,
538 # so a later entry can break down a field produced by an earlier one.
539 #
540 # Args:
541 #
542 # - $rule :: The compiled rule, read for its decompose list.
543 #
544 # - $captures :: Hash ref of the captured fields. Modified in place. New
545 # fields never clobber an existing capture; the one overwrite is a
546 # nested json decompose with no prefix, which replaces its own source
547 # field with the decoded structure.
548 #
549 # Returns nothing. Never dies.
550 #
551 # $self->_decompose( $rule, \%captures );
552 1794 ▶sub _decompose {
Subroutine _decompose: called CC=25 SCAR=32.3
553 1794 my ( $self, $rule, $captures ) = @_;
554  
555 1794 foreach my $d ( @{ $rule->{'decompose'} } ) {
556 4593 my $value = $captures->{ $d->{'field'} };
557 4593 ▶ next if ( !defined($value) || ref($value) ne '' );
Branch: 1 branch
BranchTrueFalseTotal
if not defined $value or ref $value ne "" 3324 1269 4593
Condition66%2/3
not defined $value or ref $value ne ""
l!l&&r!l&&!r
332401269
MC/DC50%1/2
not defined $value or ref $value ne ""
not defined $value ref $value ne ""
Truth table66%2/3
not(A) or B
A= not defined $valueB= ref $value ne ""
ABresult
1X1
011
000
558  
559 1269 ▶ if ( $d->{'type'} eq 'kv' ) {
Branch: 3 branches
BranchTrueFalseTotal
if ($d->{'type'} eq 'kv') { } 721 548 1269
elsif ($d->{'type'} eq 'pattern') { } 512 36 548
elsif ($d->{'type'} eq 'json') { } 36 0 36
560 721 ▶ if ( $d->{'quoted'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($d->{'quoted'}) { } 262 459 721
561 # quote-aware: a value may be "double" or 'single' quoted (quotes
562 # stripped, the field separator allowed inside), else a bareword.
563 # pairs are found by scanning for key=value shapes (value_split
564 # joining key and value) rather than by splitting on field_split.
565 262 my $vs = quotemeta( $d->{'value_split'} );
566 262 ▶ while ( $value =~ /([\w.\-]+)$vs(?:"([^"]*)"|'([^']*)'|(\S*))/g ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($value =~ /([\w.\-]+)$vs(?:"([^"]*)"|'([^']*)'|(\S*))/g) 1604 262 1866
567 1604 my $key = $1;
568 1604 ▶ my $val = defined($2) ? $2 : ( defined($3) ? $3 : $4 );
Branch: 2 branches
BranchTrueFalseTotal
defined $2 ? : 686 918 1604
defined $3 ? : 18 900 918
569 1604 $self->_store_kv_pair( $d, $captures, $key, $val );
570 }
571 } else {
572 459 my $fs = quotemeta( $d->{'field_split'} );
573 459 foreach my $token ( split( /$fs/, $value, -1 ) ) {
574 2452 ▶ next if ( $token eq '' );
Branch: 1 branch
BranchTrueFalseTotal
if $token eq "" 8 2444 2452
575 2444 my $idx = index( $token, $d->{'value_split'} );
576 2444 ▶ next if ( $idx < 1 ); # need at least a one-char key before the split
Branch: 1 branch
BranchTrueFalseTotal
if $idx < 1 76 2368 2444
577 2368 my $key = substr( $token, 0, $idx );
578 2368 my $val = substr( $token, $idx + length( $d->{'value_split'} ) );
579 2368 $self->_store_kv_pair( $d, $captures, $key, $val );
580 }
581 } ## end else [ if ( $d->{'quoted'} ) ]
582 } elsif ( $d->{'type'} eq 'pattern' ) {
583 512 ▶ if ( $value =~ $d->{'regexp'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($value =~ /$d->{"regexp"}/) 370 142 512
584 370 my %sub = %+;
585 370 foreach my $key ( keys(%sub) ) {
586 769 ▶ next if ( exists( $captures->{$key} ) );
Branch: 1 branch
BranchTrueFalseTotal
if exists $captures->{$key} 0 769 769
587 769 $captures->{$key} = $sub{$key};
588 }
589 }
590 } elsif ( $d->{'type'} eq 'json' ) {
591 36 my $decoded;
592 36 eval { $decoded = JSON::decode_json($value); };
593 # a value that is not valid JSON (or decodes to a bare scalar) is left
594 # untouched -- skip this entry entirely so the raw field survives
595 36 ▶ next if ( $@ || !defined($decoded) || ref($decoded) eq '' );
Branch: 1 branch
BranchTrueFalseTotal
if $@ or not defined $decoded or ref $decoded eq "" 1 35 36
Condition66%4/6
$@ or not defined $decoded or ref $decoded eq ""
l!l&&r!l&&!r
1035
$@ or not defined $decoded
l!l&&r!l&&!r
1035
MC/DC33%1/3
$@ or not defined $decoded or ref $decoded eq ""
$@ not defined $decoded ref $decoded eq ""
Truth table50%2/4
A or not(B) or C
A= $@B= not defined $decodedC= ref $decoded eq ""
ABCresult
1XX1
01X1
0011
0000
596 35 ▶ if ( $d->{'nested'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($d->{'nested'}) { } 2 33 35
597 2 my $key = $d->{'prefix'};
598 2 ▶ if ( $key ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($key ne '') { } 1 1 2
599 1 $key =~ s/\Q$d->{'separator'}\E\z//; # trim a trailing separator
600 } else {
601 1 $key = $d->{'field'};
602 }
603 2 ▶ if ( $key eq $d->{'field'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($key eq $d->{"field"}) 1 1 2
604 # landing on the source field itself replaces the raw JSON
605 # string; remove: would delete what was just stored, so skip it
606 1 $captures->{$key} = $decoded;
607 1 next;
608 }
609 1 ▶ $captures->{$key} = $decoded if ( !exists( $captures->{$key} ) );
Branch: 1 branch
BranchTrueFalseTotal
unless exists $captures->{$key} 1 0 1
610 } else {
611 33 $self->_json_flatten( $decoded, '', $d->{'separator'}, $d->{'prefix'}, $captures );
612 }
613 } ## end elsif ( $d->{'type'} eq 'json' )
614  
615 1267 ▶ if ( $d->{'remove'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($d->{"remove"}) 872 395 1267
616 872 delete $captures->{ $d->{'field'} };
617 }
618 } ## end foreach my $d ( @{ $rule->{'decompose'} } )
619  
620 1794 return;
621 } ## end sub _decompose
622  
623 # Stores one key/value pair a kv decompose produced: trims the entry's trim
624 # characters off each end of the value, prepends the entry's prefix to the key,
625 # and stores the pair -- unless the key already exists, since no decompose ever
626 # clobbers a real capture.
627 #
628 # Args:
629 #
630 # - $d :: The compiled kv decompose entry, read for its trim and prefix.
631 #
632 # - $captures :: Hash ref of the captured fields. Modified in place.
633 #
634 # - $key :: The key as parsed out of the blob, before the prefix.
635 #
636 # - $value :: The value as parsed out of the blob, before the trim.
637 #
638 # Returns nothing.
639 #
640 # $self->_store_kv_pair( $d, $captures, $key, $val );
641 3972 ▶sub _store_kv_pair {
Subroutine _store_kv_pair: called CC=4 SCAR=13.9
642 3972 my ( $self, $d, $captures, $key, $value ) = @_;
643  
644 3972 ▶ if ( defined( $d->{'trim'} ) && length( $d->{'trim'} ) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $d->{"trim"} and length $d->{"trim"}) 642 3330 3972
Condition66%2/3
defined $d->{"trim"} and length $d->{"trim"}
!ll&&!rl&&r
33300642
MC/DC50%1/2
defined $d->{"trim"} and length $d->{"trim"}
defined $d->{"trim"} length $d->{"trim"}
Truth table66%2/3
A and B
A= defined $d->{"trim"}B= length $d->{"trim"}
ABresult
0X0
100
111
645 642 my $tc = $d->{'trim'};
646 642 $value =~ s/\A[\Q$tc\E]+//;
647 642 $value =~ s/[\Q$tc\E]+\z//;
648 }
649  
650 3972 $key = $d->{'prefix'} . $key;
651 3972 ▶ if ( !exists( $captures->{$key} ) ) { # do not clobber a real capture
Branch: 1 branch
BranchTrueFalseTotal
unless (exists $captures->{$key}) 3936 36 3972
652 3936 $captures->{$key} = $value;
653 }
654  
655 3972 return;
656 } ## end sub _store_kv_pair
657  
658 # Recursively flattens a decoded-JSON structure into the captures hash. Each
659 # leaf becomes prefix + path, where path is the sequence of object keys and
660 # array indices joined by the separator. MongoDB extended-JSON wrappers (a hash
661 # with a single $-prefixed key such as $date / $oid / $numberLong) collapse
662 # transparently to their inner value; booleans normalize to 1/0; JSON null is
663 # skipped.
664 #
665 # Args:
666 #
667 # - $data :: The decoded structure (or, on recursion, a piece of it).
668 #
669 # - $path :: The path down to $data so far, "" at the top.
670 #
671 # - $sep :: The separator joining path segments.
672 #
673 # - $prefix :: Prepended to every produced key.
674 #
675 # - $captures :: Hash ref the produced fields land in. Modified in place;
676 # an already-present capture is never clobbered.
677 #
678 # Returns nothing. Never dies.
679 #
680 # $self->_json_flatten( $decoded, '', '_', 'mongo_', \%captures );
681 255 ▶sub _json_flatten {
Subroutine _json_flatten: called CC=15 SCAR=28.0
682 255 my ( $self, $data, $path, $sep, $prefix, $captures ) = @_;
683  
684 255 ▶ if ( ref($data) eq 'HASH' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $data eq "HASH") 64 191 255
685 # collapse a MongoDB extended-JSON wrapper: { "$date" => ... } etc.
686 64 my @keys = keys( %{$data} );
687 64 ▶ if ( scalar(@keys) == 1 && $keys[0] =~ /\A\$/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if (scalar @keys == 1 and $keys[0] =~ /\A\$/) 14 50 64
Condition100%3/3
scalar @keys == 1 and $keys[0] =~ /\A\$/
!ll&&!rl&&r
47314
MC/DC100%2/2
scalar @keys == 1 and $keys[0] =~ /\A\$/
scalar @keys == 1 $keys[0] =~ /\A\$/
Truth table100%3/3
A and B
A= scalar @keys == 1B= $keys[0] =~ /\A\$/
ABresult
0X0
100
111
688 14 return $self->_json_flatten( $data->{ $keys[0] }, $path, $sep, $prefix, $captures );
689 }
690 50 foreach my $key ( keys( %{$data} ) ) {
691 208 ▶ my $child = ( $path eq '' ) ? $key : $path . $sep . $key;
Branch: 1 branch
BranchTrueFalseTotal
$path eq '' ? : 171 37 208
692 208 $self->_json_flatten( $data->{$key}, $child, $sep, $prefix, $captures );
693 }
694 50 return;
695 } ## end if ( ref($data) eq 'HASH' )
696 191 ▶ if ( ref($data) eq 'ARRAY' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $data eq "ARRAY") 2 189 191
697 2 my $index = 0;
698 2 foreach my $element ( @{$data} ) {
699 0 ▶ my $child = ( $path eq '' ) ? $index : $path . $sep . $index;
Branch: 1 branch
BranchTrueFalseTotal
$path eq '' ? : 0 0 0
700 0 $self->_json_flatten( $element, $child, $sep, $prefix, $captures );
701 0 $index++;
702 }
703 2 return;
704 }
705  
706 # leaf
707 189 ▶ return if ( !defined($data) ); # skip JSON null
Branch: 1 branch
BranchTrueFalseTotal
unless defined $data 3 186 189
708 186 ▶ if ( JSON::is_bool($data) ) { # backend-agnostic (JSON::XS or JSON::PP boolean)
Branch: 2 branches
BranchTrueFalseTotal
if (&JSON::is_bool($data)) { } 3 183 186
elsif (ref $data) { } 0 183 183
709 3 ▶ $data = $data ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
$data ? : 3 0 3
710 } elsif ( ref($data) ) {
711 0 $data = "$data"; # any other blessed leaf: stringify defensively
712 }
713 186 ▶ return if ( $path eq '' ); # a top-level bare scalar has no key to store under
Branch: 1 branch
BranchTrueFalseTotal
if $path eq "" 0 186 186
714  
715 186 my $key = $prefix . $path;
716 186 ▶ return if ( exists( $captures->{$key} ) ); # do not clobber an existing capture
Branch: 1 branch
BranchTrueFalseTotal
if exists $captures->{$key} 0 186 186
717 186 $captures->{$key} = $data;
718  
719 186 return;
720 } ## end sub _json_flatten
721  
722 # Validates a convert: map (field => type) and returns a normalized copy.
723 #
724 # lc/uc exist so a rule file can normalize a captured token whose case varies
725 # between the log sources that produce it; mac normalizes the several spellings
726 # a MAC address arrives in (see _normalize_mac). The full rationale lives in
727 # docs/rule-files.md.
728 #
729 # Args:
730 #
731 # - $convert :: The convert map as written in the rule file. A hash ref of
732 # field => type, where type is one of int, float, lc, uc, or mac.
733 # Aliases: integer for int; num/number for float; lower/lowercase and
734 # upper/uppercase for the case folds; macaddr/mac_address for mac.
735 #
736 # Returns a hash ref of field => canonical type. Dies on an unknown type.
737 #
738 # my $normalized = $self->_compile_convert( $rules->{'convert'} );
739 417 ▶sub _compile_convert {
Subroutine _compile_convert: called CC=10 SCAR=23.6
740 417 my ( $self, $convert ) = @_;
741  
742 417 ▶ if ( ref($convert) ne 'HASH' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $convert ne "HASH") 0 417 417
743 0 die( '.convert has a ref of "' . ref($convert) . '" and not "HASH"' );
744 }
745  
746 417 my %normalized;
747 417 foreach my $field ( keys( %{$convert} ) ) {
748 1848 my $type = $convert->{$field};
749 1848 ▶ if ( !defined($type) || ref($type) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $type or ref $type ne "") 0 1848 1848
Condition33%1/3
not defined $type or ref $type ne ""
l!l&&r!l&&!r
001848
MC/DC0%0/2
not defined $type or ref $type ne ""
not defined $type ref $type ne ""
Truth table33%1/3
not(A) or B
A= not defined $typeB= ref $type ne ""
ABresult
1X1
011
000
750 0 die( '.convert.' . $field . ' type is undef or not a string' );
751 }
752 1848 ▶ if ( $type =~ /\A(?:int|integer)\z/i ) {
Branch: 5 branches
BranchTrueFalseTotal
if ($type =~ /\A(?:int|integer)\z/i) { } 1697 151 1848
elsif ($type =~ /\A(?:float|num|number)\z/i) { } 87 64 151
elsif ($type =~ /\A(?:lc|lower|lowercase)\z/i) { } 41 23 64
elsif ($type =~ /\A(?:uc|upper|uppercase)\z/i) { } 5 18 23
elsif ($type =~ /\A(?:mac|macaddr|mac_address)\z/i) { } 16 2 18
753 1697 $normalized{$field} = 'int';
754 } elsif ( $type =~ /\A(?:float|num|number)\z/i ) {
755 87 $normalized{$field} = 'float';
756 } elsif ( $type =~ /\A(?:lc|lower|lowercase)\z/i ) {
757 41 $normalized{$field} = 'lc';
758 } elsif ( $type =~ /\A(?:uc|upper|uppercase)\z/i ) {
759 5 $normalized{$field} = 'uc';
760 } elsif ( $type =~ /\A(?:mac|macaddr|mac_address)\z/i ) {
761 16 $normalized{$field} = 'mac';
762 } else {
763 2 die( '.convert.' . $field . ' type "' . $type . '" is unknown (expected int, float, lc, uc, or mac)' );
764 }
765 } ## end foreach my $field ( keys( %{$convert} ) )
766  
767 415 return \%normalized;
768 } ## end sub _compile_convert
769  
770 # Coerces the rule's convert fields in the captures hash: int/float to numbers
771 # so they serialize as JSON numbers rather than strings, lc/uc to a case-folded
772 # string, mac to lowercase colon-separated hex.
773 #
774 # Args:
775 #
776 # - $rule :: The compiled rule, read for its convert map.
777 #
778 # - $captures :: Hash ref of the captured fields. Modified in place. A
779 # field that is not present is left untouched, as is a numeric
780 # conversion of something that does not look like a number and a mac
781 # conversion of something that is not twelve hex digits.
782 #
783 # Returns nothing. Never dies.
784 #
785 # $self->_convert( $rule, \%captures );
786 526 ▶sub _convert {
Subroutine _convert: called CC=10 SCAR=23.1
787 526 my ( $self, $rule, $captures ) = @_;
788  
789 526 foreach my $field ( keys( %{ $rule->{'convert'} } ) ) {
790 4969 ▶ next if ( !exists( $captures->{$field} ) );
Branch: 1 branch
BranchTrueFalseTotal
unless exists $captures->{$field} 3907 1062 4969
791 1062 my $value = $captures->{$field};
792 1062 ▶ next if ( !defined($value) || ref($value) ne '' );
Branch: 1 branch
BranchTrueFalseTotal
if not defined $value or ref $value ne "" 0 1062 1062
Condition33%1/3
not defined $value or ref $value ne ""
l!l&&r!l&&!r
001062
MC/DC0%0/2
not defined $value or ref $value ne ""
not defined $value ref $value ne ""
Truth table33%1/3
not(A) or B
A= not defined $valueB= ref $value ne ""
ABresult
1X1
011
000
793  
794 1062 my $type = $rule->{'convert'}{$field};
795  
796 # the case folds apply to any string, so they sit ahead of the
797 # looks-like-a-number guard the numeric conversions need
798 1062 ▶ if ( $type eq 'lc' ) {
Branch: 3 branches
BranchTrueFalseTotal
if ($type eq 'lc') { } 42 1020 1062
elsif ($type eq 'uc') { } 1 1019 1020
elsif ($type eq 'mac') { } 10 1009 1019
799 42 $captures->{$field} = lc($value);
800 42 next;
801 } elsif ( $type eq 'uc' ) {
802 1 $captures->{$field} = uc($value);
803 1 next;
804 } elsif ( $type eq 'mac' ) {
805 10 $captures->{$field} = $self->_normalize_mac($value);
806 10 next;
807 }
808  
809 1009 ▶ next if ( !looks_like_number($value) );
Branch: 1 branch
BranchTrueFalseTotal
unless &looks_like_number($value) 7 1002 1009
810  
811 1002 ▶ if ( $type eq 'int' ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($type eq 'int') { } 927 75 1002
812 927 $captures->{$field} = int( $value + 0 );
813 } else {
814 75 $captures->{$field} = $value + 0;
815 }
816 } ## end foreach my $field ( keys( %{ $rule->{'convert'}...}))
817  
818 526 return;
819 } ## end sub _convert
820  
821 # Rewrites a MAC address into the one spelling everything else in the
822 # distribution uses: twelve lowercase hex digits in six colon-separated pairs.
823 #
824 # Args:
825 #
826 # - $value :: The captured string to rewrite. Any of the spellings a log
827 # line might carry are understood -- colon-separated
828 # ("C4:D8:D5:3B:8C:4B"), hyphen-separated as Windows writes it
829 # ("C4-D8-D5-3B-8C-4B"), dotted quads as Cisco writes them
830 # ("c4d8.d53b.8c4b"), space-separated bytes as the kernel's link layer
831 # header dump writes them ("c4 d8 d5 3b 8c 4b"), and bare
832 # ("c4d8d53b8c4b").
833 #
834 # Returns the normalized address as a string. If the value does not reduce to
835 # exactly twelve hex digits once the separators are removed it is returned
836 # unchanged, so a field that turns out not to hold a MAC after all is passed
837 # through rather than mangled -- this runs against whatever the pattern
838 # captured, and a pattern can be looser than its author intended.
839 #
840 # # all five of these return 'c4:d8:d5:3b:8c:4b'
841 # $processor->_normalize_mac('C4:D8:D5:3B:8C:4B');
842 # $processor->_normalize_mac('C4-D8-D5-3B-8C-4B');
843 # $processor->_normalize_mac('c4d8.d53b.8c4b');
844 # $processor->_normalize_mac('c4 d8 d5 3b 8c 4b');
845 # $processor->_normalize_mac('c4d8d53b8c4b');
846 #
847 # # not twelve hex digits, so returned as-is
848 # $processor->_normalize_mac('unknown');
849 10 ▶sub _normalize_mac {
Subroutine _normalize_mac: called CC=2 SCAR=6.9
850 10 my ( $self, $value ) = @_;
851  
852 10 my $digits = $value;
853 10 $digits =~ s/[:.\-\s]//g;
854  
855 10 ▶ return $value if ( $digits !~ /\A[0-9A-Fa-f]{12}\z/ );
Branch: 1 branch
BranchTrueFalseTotal
unless $digits =~ /\A[0-9A-Fa-f]{12}\z/ 1 9 10
856  
857 9 return join( ':', lc($digits) =~ /([0-9a-f]{2})/g );
858 } ## end sub _normalize_mac
859  
860 # Compiles a single rules: entry into the runtime structure:
861 #
862 # {
863 # name => ..., # optional, diagnostics only
864 # field => 'MESSAGE', # target field, defaults to MESSAGE
865 # gate => [ { field => ..., literals => {...}, regexps => [ qr//, ... ] }, ... ],
866 # patterns => [ qr//, ... ],
867 # geoip => [ 'field_name', ... ],
868 # decompose => [ ... ], # see _compile_decompose
869 # convert => { field_name => 'int'|'float'|'lc'|'uc'|'mac', ... },
870 # }
871 #
872 # A rule may set anchored: true, in which case each pattern is wrapped as
873 # \A(?:...)\z so it must match the whole target field (the equivalent of a
874 # logstash "^...$" grok) rather than any substring.
875 #
876 # geoip:, decompose:, and convert: may each be given per-rule or once at the
877 # top of the file as a default for every rule. A rule-level one replaces the
878 # file-level default rather than merging with it. They run in the order
879 # decompose -> geoip -> convert, so geoip can look up a field a decompose step
880 # produced, and convert only coerces once geoip has seen the string addresses.
881 #
882 # Args:
883 #
884 # - rule :: The raw rule hash ref.
885 #
886 # - vars :: The compiled vars hash ref (pattern names resolve against this).
887 #
888 # - default_geoip :: The file-level geoip list, used when the rule has none.
889 #
890 # - default_decompose :: The file-level decompose list, used when the rule
891 # has none.
892 #
893 # - default_convert :: The file-level convert map, used when the rule has
894 # none.
895 #
896 # Returns the compiled rule hash ref. Dies on a malformed rule, an un-degrokked
897 # %{...} remnant, or a pattern that will not compile (which is where an illegal
898 # named capture such as a "-" in the name is caught, at load time rather than
899 # at match time).
900 #
901 # my $compiled = $self->_compile_rule( 'rule' => $rule, 'vars' => $vars );
902 575 ▶sub _compile_rule {
Subroutine _compile_rule: called CC=37 SCAR=41.5
903 575 my ( $self, %opts ) = @_;
904  
905 575 my $rule = $opts{'rule'};
906 575 my $vars = $opts{'vars'};
907 575 ▶ if ( ref($rule) ne 'HASH' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $rule ne "HASH") 0 575 575
908 0 die( 'rule has a ref of "' . ref($rule) . '" and not "HASH"' );
909 }
910 575 ▶ if ( ref($vars) ne 'HASH' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $vars ne "HASH") 0 575 575
911 0 $vars = {};
912 }
913  
914 my $compiled = {
915 'name' => $rule->{'name'},
916 575 ▶ 'field' => ( defined( $rule->{'field'} ) ? $rule->{'field'} : 'MESSAGE' ),
Branch: 1 branch
BranchTrueFalseTotal
defined $rule->{'field'} ? : 574 1 575
917 'gate' => [],
918 'patterns' => [],
919 'geoip' => [],
920 'decompose' => [],
921 'convert' => {},
922 };
923 575 ▶ if ( ref( $compiled->{'field'} ) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $compiled->{"field"} ne "") 0 575 575
924 0 die( '.field has a ref of "' . ref( $compiled->{'field'} ) . '" and not ""' );
925 }
926  
927 # numeric coercion of captured fields (rule-level, else the file default)
928 575 ▶ my $convert = defined( $rule->{'convert'} ) ? $rule->{'convert'} : $opts{'default_convert'};
Branch: 1 branch
BranchTrueFalseTotal
defined $rule->{'convert'} ? : 0 575 575
929 575 ▶ if ( defined($convert) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $convert) 256 319 575
930 256 $compiled->{'convert'} = $self->_compile_convert($convert);
931 }
932  
933 # captured fields to enrich with geoip (rule-level, else the file default)
934 575 ▶ my $geoip = defined( $rule->{'geoip'} ) ? $rule->{'geoip'} : $opts{'default_geoip'};
Branch: 1 branch
BranchTrueFalseTotal
defined $rule->{'geoip'} ? : 17 558 575
935 575 ▶ if ( defined($geoip) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $geoip) 432 143 575
936 432 ▶ if ( ref($geoip) ne 'ARRAY' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $geoip ne "ARRAY") 0 432 432
937 0 die( '.geoip has a ref of "' . ref($geoip) . '" and not "ARRAY"' );
938 }
939 432 foreach my $field ( @{$geoip} ) {
940 1015 ▶ if ( ref($field) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $field ne "") 0 1015 1015
941 0 die( '.geoip entry has a ref of "' . ref($field) . '" and not ""' );
942 }
943 1015 push( @{ $compiled->{'geoip'} }, $field );
944 }
945 } ## end if ( defined($geoip) )
946  
947 # captured fields to break down further (rule-level, else the file default)
948 575 ▶ my $decompose = defined( $rule->{'decompose'} ) ? $rule->{'decompose'} : $opts{'default_decompose'};
Branch: 1 branch
BranchTrueFalseTotal
defined $rule->{'decompose'} ? : 4 571 575
949 575 ▶ if ( defined($decompose) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $decompose) 350 225 575
950 350 $compiled->{'decompose'} = $self->_compile_decompose( $decompose, $vars );
951 }
952  
953 # when anchored, each pattern must match the whole target field (like a
954 # logstash "^...$" grok), not just a substring
955 575 ▶ my $anchored = ( defined( $rule->{'anchored'} ) && $rule->{'anchored'} ) ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
defined $rule->{'anchored'} && $rule->{'anchored'} ? : 551 24 575
Condition66%2/3
defined $rule->{"anchored"} and $rule->{"anchored"}
!ll&&!rl&&r
240551
MC/DC50%1/2
defined $rule->{"anchored"} and $rule->{"anchored"}
defined $rule->{"anchored"} $rule->{"anchored"}
Truth table66%2/3
A and B
A= defined $rule->{"anchored"}B= $rule->{"anchored"}
ABresult
0X0
100
111
956  
957 # gates
958 575 ▶ if ( defined( $rule->{'gate'} ) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $rule->{"gate"}) 515 60 575
959 515 ▶ if ( ref( $rule->{'gate'} ) ne 'ARRAY' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $rule->{"gate"} ne "ARRAY") 0 515 515
960 0 die( '.gate has a ref of "' . ref( $rule->{'gate'} ) . '" and not "ARRAY"' );
961 }
962 515 my $gate_int = 0;
963 515 foreach my $gate ( @{ $rule->{'gate'} } ) {
964 515 ▶ if ( ref($gate) ne 'HASH' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $gate ne "HASH") 0 515 515
965 0 die( '.gate[' . $gate_int . '] has a ref of "' . ref($gate) . '" and not "HASH"' );
966 }
967 515 ▶ if ( !defined( $gate->{'field'} ) || ref( $gate->{'field'} ) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $gate->{"field"} or ref $gate->{"field"} ne "") 0 515 515
Condition33%1/3
not defined $gate->{"field"} or ref $gate->{"field"} ne ""
l!l&&r!l&&!r
00515
MC/DC0%0/2
not defined $gate->{"field"} or ref $gate->{"field"} ne ""
not defined $gate->{"field"} ref $gate->{"field"} ne ""
Truth table33%1/3
not(A) or B
A= not defined $gate->{"field"}B= ref $gate->{"field"} ne ""
ABresult
1X1
011
000
968 0 die( '.gate[' . $gate_int . '].field is undef or not a string' );
969 }
970 515 ▶ if ( !defined( $gate->{'values'} ) || ref( $gate->{'values'} ) ne 'ARRAY' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $gate->{"values"} or ref $gate->{"values"} ne "ARRAY") 0 515 515
Condition33%1/3
not defined $gate->{"values"} or ref $gate->{"values"} ne "ARRAY"
l!l&&r!l&&!r
00515
MC/DC0%0/2
not defined $gate->{"values"} or ref $gate->{"values"} ne "ARRAY"
not defined $gate->{"values"} ref $gate->{"values"} ne "ARRAY"
Truth table33%1/3
not(A) or B
A= not defined $gate->{"values"}B= ref $gate->{"values"} ne "ARRAY"
ABresult
1X1
011
000
971 0 die( '.gate[' . $gate_int . '].values is undef or not an "ARRAY"' );
972 }
973  
974 my $compiled_gate = {
975 515 'field' => $gate->{'field'},
976 'literals' => {},
977 'regexps' => [],
978 };
979 515 my $value_int = 0;
980 515 foreach my $value ( @{ $gate->{'values'} } ) {
981 537 ▶ if ( ref($value) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $value ne "") 0 537 537
982 0 die( '.gate['
983 . $gate_int
984 . '].values['
985 . $value_int
986 . '] has a ref of "'
987 . ref($value)
988 . '" and not ""' );
989 }
990 # //regexp// is the gate-value regexp marker; the outer slashes
991 # are stripped strictly so an interior // (e.g. http://) survives
992 537 ▶ if ( $value =~ /^\/\/(.*)\/\/\z/s ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($value =~ m[^//(.*)//\z]s) { } 503 34 537
993 503 my $re = $1;
994 503 my $compiled_re;
995 503 eval { $compiled_re = qr/$re/; };
996 503 ▶ if ($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 0 503 503
997 0 die( '.gate['
998 . $gate_int
999 . '].values['
1000 . $value_int
1001 . '] //regexp// "'
1002 . $value
1003 . '" does not compile... '
1004 . $@ );
1005 } ## end if ($@)
1006 503 push( @{ $compiled_gate->{'regexps'} }, $compiled_re );
1007 } else {
1008 34 $compiled_gate->{'literals'}{$value} = 1;
1009 }
1010 537 $value_int++;
1011 } ## end foreach my $value ( @{ $gate->{'values'} } )
1012  
1013 515 push( @{ $compiled->{'gate'} }, $compiled_gate );
1014 515 $gate_int++;
1015 } ## end foreach my $gate ( @{ $rule->{'gate'} } )
1016 } ## end if ( defined( $rule->{'gate'} ) )
1017  
1018 # patterns
1019 575 ▶ if ( !defined( $rule->{'patterns'} ) ) {
Branch: 3 branches
BranchTrueFalseTotal
if (not defined $rule->{'patterns'}) { } 0 575 575
elsif (ref $rule->{'patterns'} ne 'ARRAY') { } 0 575 575
elsif (not defined $rule->{'patterns'}[0]) { } 0 575 575
1020 0 die('.patterns is undef; a rule needs at least one pattern');
1021 } elsif ( ref( $rule->{'patterns'} ) ne 'ARRAY' ) {
1022 0 die( '.patterns has a ref of "' . ref( $rule->{'patterns'} ) . '" and not "ARRAY"' );
1023 } elsif ( !defined( $rule->{'patterns'}[0] ) ) {
1024 0 die('.patterns[0] is undef; a rule needs at least one pattern');
1025 }
1026 575 my $pattern_int = 0;
1027 575 foreach my $pattern ( @{ $rule->{'patterns'} } ) {
1028 2659 ▶ if ( ref($pattern) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $pattern ne "") 0 2659 2659
1029 0 die( '.patterns[' . $pattern_int . '] has a ref of "' . ref($pattern) . '" and not ""' );
1030 }
1031 # a bare name that resolves to a compiled var uses that var's regexp
1032 # string; anything else is treated as an inline regexp
1033 2659 ▶ my $regexp_string = ( exists( $vars->{$pattern} ) ) ? $vars->{$pattern} : $pattern;
Branch: 1 branch
BranchTrueFalseTotal
exists $vars->{$pattern} ? : 2654 5 2659
1034  
1035 2659 ▶ if ( $regexp_string =~ /%\{[^}]*\}/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($regexp_string =~ /%\{[^}]*\}/) 0 2659 2659
1036 0 die( '.patterns[' . $pattern_int . '] ("' . $pattern . '") contains un-degrokked grok "%{...}"' );
1037 }
1038  
1039 2659 ▶ if ($anchored) {
Branch: 1 branch
BranchTrueFalseTotal
if ($anchored) 2635 24 2659
1040 2635 $regexp_string = '\A(?:' . $regexp_string . ')\z';
1041 }
1042  
1043 2659 my $compiled_re;
1044 2659 eval { $compiled_re = qr/$regexp_string/; };
1045 2659 ▶ if ($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 0 2659 2659
1046 0 die( '.patterns[' . $pattern_int . '] ("' . $pattern . '") does not compile as a regexp... ' . $@ );
1047 }
1048 2659 push( @{ $compiled->{'patterns'} }, $compiled_re );
1049 2659 $pattern_int++;
1050 } ## end foreach my $pattern ( @{ $rule->{'patterns'} } )
1051  
1052 575 return $compiled;
1053 } ## end sub _compile_rule
1054  
1055 =head1 AUTHOR
1056  
1057 Zane C. Bowers-Hadley, C<< <vvelox at vvelox.net> >>
1058  
1059 =head1 LICENSE AND COPYRIGHT
1060  
1061 This software is Copyright (c) 2026 by Zane C. Bowers-Hadley.
1062  
1063 This is free software, licensed under:
1064  
1065 The GNU Lesser General Public License, Version 3, June 2007
1066  
1067 =cut
1068  
1069 27 1; # End of Log::Munger::LogProcessor
« lib/Log/Munger/RulesTest.pm ↑ Summary lib/Log/Munger/App/Command/degrok.pm »