lib/Log/Munger.pm

filter: Statementstmt 75.5% 34 / 45 Branchbran 42.8% 6 / 14 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 88.8% 8 / 9 Podpod 100% 4 / 4 totaltotal 72.2% 52 / 72 CC 8 SCAR 13.5
CC
8
Cov
68%
load
12.0
new
3.1
process_item
2.1
CRAP 3.9
« lib/Log/Munger/WhichRuleFile.pm ↑ Summary lib/Log/Munger/App/Command/dump_rule_file.pm »
1 package Log::Munger;
2  
3 25 ▶use 5.006;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 25 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 25 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 25 ▶use Log::Munger::LogProcessor ();
Subroutine BEGIN: called CC=1 SCAR=0.0
7  
8 =head1 NAME
9  
10 Log::Munger - Extracts structured fields from log records using YAML rule files.
11  
12 =head1 VERSION
13  
14 Version 0.0.1
15  
16 =cut
17  
18 25 our $VERSION = '0.0.1';
19  
20 =head1 SYNOPSIS
21  
22 use Log::Munger;
23  
24 my $munger = Log::Munger->new( 'rules' => [ 'sshd', 'postfix' ] );
25  
26 # a decoded record, such as what journald or syslog-ng JSON output gives you
27 my $fields = $munger->process_item(
28 'item' => {
29 'PROGRAM' => 'sshd',
30 'MESSAGE' => 'Failed password for root from 203.0.113.7 port 44444 ssh2',
31 }
32 );
33 # $fields = { ssh_method => 'password', ssh_user => 'root', ssh_src_ip => '203.0.113.7', ... }
34  
35 # a bare string is matched as the MESSAGE field, so whole log lines work as-is
36 my $access = $munger->process_item( 'item' => $raw_apache_line );
37  
38 Each rule file is a YAML document holding a library of named regexps and an ordered
39 list of rules built from them. A record is walked against those rules in load order,
40 and the first one whose gates pass and whose pattern matches returns its named
41 captures. Captured fields can then be broken down further, looked up in a GeoIP
42 database, and coerced to numbers.
43  
44 This is the same idea as grok for Logstash, minus the Logstash. See
45 L<Log::Munger::LogProcessor> for the matching engine and the C<log_munger> command
46 for the CLI.
47  
48 =head1 METHODS
49  
50 =head2 new
51  
52 Creates a new munger. Rule files may be supplied up front, added later via
53 L</load>, or both.
54  
55 Each name is resolved through L<Log::Munger::WhichRuleFile>, so it may be a bare
56 name such as C<sshd> or a path. A rule file that fails to load or compile is
57 fatal, which means a broken file is caught here rather than silently producing no
58 matches later on.
59  
60 my $munger = Log::Munger->new( 'rules' => [ 'base', 'postfix' ] );
61 my $munger = Log::Munger->new( 'rules' => ['postfix'], 'geoip' => '/path/to/GeoLite2-City.mmdb' );
62  
63 - rules :: Rule files to load. The taken value is an array ref.
64 Default :: undef
65  
66 - geoip :: Path to a MaxMind .mmdb database. When set, rules that flag
67 captured fields with a C<geoip:> list have those looked up, with the
68 result stored under C<< $result->{geoip}{$field} >>. Needs
69 L<IP::Geolocation::MMDB>, which is only loaded when this is used.
70 Default :: undef (geoip disabled)
71  
72 =cut
73  
74 32 ▶sub new {
Subroutine new: called CC=3 SCAR=11.2
Pod documented
75 32 my ( $blank, %opts ) = @_;
76  
77 my $self = {
78 'rule_files' => [],
79 'processor' => undef,
80 32 'geoip' => $opts{'geoip'},
81 };
82 32 bless $self;
83  
84 32 ▶ if ( defined( $opts{'rules'} ) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $opts{"rules"}) 32 0 32
85 32 ▶ if ( ref( $opts{'rules'} ) ne 'ARRAY' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $opts{"rules"} ne "ARRAY") 0 32 32
86 0 die( '$opts{rules} has a ref of "' . ref( $opts{'rules'} ) . '" and not "ARRAY"' );
87 }
88 32 push( @{ $self->{'rule_files'} }, @{ $opts{'rules'} } );
89 32 $self->_build_processor;
90 }
91  
92 32 return $self;
93 } ## end sub new
94  
95 =head2 load
96  
97 Loads an additional rule file and rebuilds the processor. Rules from files loaded
98 later are tried after rules from files loaded earlier, so load order is match
99 priority.
100  
101 Returns 1. Dies if the file cannot be found, loaded, or compiled.
102  
103 - file :: The file to load. Required.
104 Default :: undef
105  
106 $munger->load( 'file' => 'sshd' );
107  
108 =cut
109  
110 0 ▶sub load {
Subroutine load: not called CC=3 SCAR=24.8
Pod documented
111 0 my ( $self, %opts ) = @_;
112  
113 0 ▶ if ( !defined( $opts{'file'} ) ) {
Branch: 2 branches
BranchTrueFalseTotal
if (not defined $opts{'file'}) { } 0 0 0
elsif (ref $opts{'file'} ne '') { } 0 0 0
114 0 die('$opts{file} is undef');
115 } elsif ( ref( $opts{'file'} ) ne '' ) {
116 0 die( '$opts{file} ref is "' . ref( $opts{'file'} ) . '" and not ""' );
117 }
118  
119 0 push( @{ $self->{'rule_files'} }, $opts{'file'} );
120 0 $self->_build_processor;
121  
122 0 return 1;
123 } ## end sub load
124  
125 =head2 process_item
126  
127 Runs a decoded log record through the loaded rules and returns the named
128 captures of the first matching rule, or undef if nothing matched (or no rules
129 have been loaded).
130  
131 - item :: The decoded log record (a hash ref), or a bare string. A bare
132 string is treated as a raw log line and matched as the C<MESSAGE> field.
133 If given, C<item> takes precedence over the field args below.
134 Default :: undef
135  
136 - message :: The raw log message, assembled into C<< { MESSAGE => ... } >>.
137 Default :: undef
138  
139 - program :: Optional C<PROGRAM> field (the usual daemon-rule gate).
140 Default :: undef
141  
142 - priority :: Optional C<PRIORITY> field.
143 Default :: undef
144  
145 - facility :: Optional C<FACILITY> field.
146 Default :: undef
147  
148 my $fields = $munger->process_item( 'item' => $json );
149 my $fields = $munger->process_item( 'item' => $raw_access_log_line );
150  
151 # from a syslog reader that already has the fields split out (e.g. baphomet):
152 my $fields = $munger->process_item(
153 'message' => $message,
154 'program' => $program,
155 'priority' => $priority,
156 'facility' => $facility,
157 );
158  
159 Returns a hash ref of the winning rule's named captures, or undef if nothing
160 matched. Never dies: an exception during matching comes back as undef rather
161 than taking down the stream. That bounds failures, not runtime -- a pattern
162 prone to catastrophic backtracking can still burn CPU on a hostile line.
163  
164 =cut
165  
166 132 ▶sub process_item {
Subroutine process_item: called CC=2 SCAR=7.6
Pod documented
167 132 my ( $self, %opts ) = @_;
168  
169 132 ▶ if ( !defined( $self->{'processor'} ) ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $self->{"processor"}) 0 132 132
170 0 return undef;
171 }
172  
173 132 return $self->{'processor'}->process_item(%opts);
174 }
175  
176 =head2 explain_item
177  
178 Like L</process_item>, but reports which rule and pattern fired alongside the
179 fields. Handy when a rule file is not matching what you expected it to. Takes the
180 same args and never dies. See L<Log::Munger::LogProcessor/explain_item>.
181  
182 my $why = $munger->explain_item( 'item' => $json );
183  
184 # { matched => 0 }
185 # { matched => 1, rule => 'sshd', pattern => 1, field => 'MESSAGE', fields => { ... } }
186  
187 =cut
188  
189 4 ▶sub explain_item {
Subroutine explain_item: called CC=2 SCAR=7.6
Pod documented
190 4 my ( $self, %opts ) = @_;
191  
192 4 ▶ if ( !defined( $self->{'processor'} ) ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $self->{"processor"}) 0 4 4
193 0 return { 'matched' => 0 };
194 }
195  
196 4 return $self->{'processor'}->explain_item(%opts);
197 }
198  
199 # Rebuilds the LogProcessor from the rule file names collected so far, passing
200 # along the geoip database path if one was given to new.
201 #
202 # Log::Munger::LogProcessor has no way to append a rule file to an existing
203 # instance, so every call to new or load throws the old processor away and
204 # compiles the whole set again. That keeps the rules in load order and keeps the
205 # compile errors happening at load time.
206 #
207 # Takes no args beyond $self.
208 #
209 # Returns the new Log::Munger::LogProcessor object, which is also stashed in
210 # $self->{processor}. Dies if any of the rule files fails to load or compile.
211 #
212 # $self->_build_processor;
213 32 ▶sub _build_processor {
Subroutine _build_processor: called CC=2 SCAR=6.9
214 32 my ($self) = @_;
215  
216 32 my %args = ( 'rules' => $self->{'rule_files'} );
217 32 ▶ $args{'geoip'} = $self->{'geoip'} if ( defined( $self->{'geoip'} ) );
Branch: 1 branch
BranchTrueFalseTotal
if defined $self->{"geoip"} 11 21 32
218  
219 32 $self->{'processor'} = Log::Munger::LogProcessor->new(%args);
220  
221 32 return $self->{'processor'};
222 } ## end sub _build_processor
223  
224 =head1 AUTHOR
225  
226 Zane C. Bowers-Hadley, C<< <vvelox at vvelox.net> >>
227  
228 =head1 SEE ALSO
229  
230 =over 4
231  
232 =item * L<Log::Munger::LogProcessor> - the matching engine.
233  
234 =item * L<Log::Munger::RulesTest> - the test harness behind C<log_munger test_all>.
235  
236 =item * L<Log::Munger::Degrok> - converting existing grok patterns.
237  
238 =item * The C<docs/> directory in the distribution, which covers the rule file
239 format, the primitive library, and writing your own rules.
240  
241 =back
242  
243 =head1 BUGS
244  
245 Report bugs and feature requests through GitHub at
246 L<https://github.com/LilithSec/Log-Munger>, or to C<bug-log-munger at rt.cpan.org>.
247  
248 =head1 SUPPORT
249  
250 perldoc Log::Munger
251  
252 You can also find this distribution at:
253  
254 =over 4
255  
256 =item * GitHub
257  
258 L<https://github.com/LilithSec/Log-Munger>
259  
260 =item * MetaCPAN
261  
262 L<https://metacpan.org/release/Log-Munger>
263  
264 =item * RT, CPAN's request tracker
265  
266 L<https://rt.cpan.org/NoAuth/Bugs.html?Dist=Log-Munger>
267  
268 =back
269  
270 =head1 LICENSE AND COPYRIGHT
271  
272 This software is Copyright (c) 2026 by Zane C. Bowers-Hadley.
273  
274 This is free software, licensed under:
275  
276 The GNU Lesser General Public License, Version 3, June 2007
277  
278  
279 =cut
280  
281 25 1; # End of Log::Munger
« lib/Log/Munger/WhichRuleFile.pm ↑ Summary lib/Log/Munger/App/Command/dump_rule_file.pm »