lib/Log/Munger/Degrok.pm

filter: Statementstmt 89.2% 91 / 102 Branchbran 73.9% 34 / 46 Conditioncond 55.5% 5 / 9 MC/DCmcdc 16.6% 1 / 6 Subroutinesub 100% 11 / 11 Podpod 100% 3 / 3 totaltotal 81.9% 145 / 177 CC 29 SCAR 29.6
CC
29
Cov
83%
grok2rules
30.1
file
4.2
string
3.0
CRAP 19.4
« lib/Log/Munger/App/Command/enrich.pm ↑ Summary lib/Log/Munger/RulesTemplateOrder.pm »
1 package Log::Munger::Degrok;
2  
3 17 ▶use 5.006;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 17 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 17 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 17 ▶use Scalar::Util qw(looks_like_number);
Subroutine BEGIN: called CC=1 SCAR=0.0
7 17 ▶use File::Slurp qw(read_file);
Subroutine BEGIN: called CC=1 SCAR=0.0
8 17 ▶use Hash::Merge ();
Subroutine BEGIN: called CC=1 SCAR=0.0
9 17 ▶use YAML::XS qw(Dump);
Subroutine BEGIN: called CC=1 SCAR=0.0
10 17 ▶use Log::Munger::RuleFileParser ();
Subroutine BEGIN: called CC=1 SCAR=0.0
11  
12 =head1 NAME
13  
14 Log::Munger::Degrok - Rewrites grok templating into the form Log::Munger uses.
15  
16 =head1 VERSION
17  
18 Version 0.0.1
19  
20 =cut
21  
22 17 our $VERSION = '0.0.1';
23  
24 =head1 SYNOPSIS
25  
26 use Log::Munger::Degrok;
27  
28 print Log::Munger::Degrok->string( 'string' => 'client=%{IP:client_ip}' ) . "\n";
29 # client=(?<client_ip>[% IP %])
30  
31 print Log::Munger::Degrok->file( 'file' => '/path/to/patterns.grok' );
32  
33 print Log::Munger::Degrok->grok2rules( 'file' => '/path/to/patterns.grok', 'includes' => ['base'] );
34  
35 Grok references a named pattern as C<%{TEMPLATE}> and captures one as
36 C<%{TEMPLATE:VAR}>. Log::Munger uses L<Template> for the first and Perl's own
37 named capture syntax for the second, so those become C<[% TEMPLATE %]> and
38 C<< (?<VAR>[% TEMPLATE %]) >>. This module does that rewrite.
39  
40 Only the delimiters change. The primitive names in F<base.yaml> deliberately track
41 grok's, so a pattern that referenced C<%{IP}> before goes on referencing the same
42 thing afterwards, and most patterns work straight after conversion. What the
43 rewrite cannot fix is a capture name Perl will not accept, since grok is happy
44 with names like C<src-ip> and Perl is not. Those come across unchanged and are
45 caught later, either at load time or by L<Log::Munger::RulesTest>.
46  
47 =head1 METHODS
48  
49 =head2 string
50  
51 Rewrites the grok templating in a string and returns the result.
52  
53 Every C<%{...}> reference is rewritten, however many there are, and anything that
54 is not one is left exactly as it was.
55  
56 - string :: The string to convert. Required.
57 Default :: undef
58  
59 Returns the rewritten string. Dies only if C<string> is undef.
60  
61 my $results;
62 my $some_string = 'client=%{IP:client_ip} user=%{USERNAME:user}';
63 eval { $results = Log::Munger::Degrok->string( 'string' => $some_string ); };
64 if ($@) {
65 die( 'Failed to process the string "' . $some_string . '"... ' . $@ );
66 }
67 print $results . "\n";
68 # client=(?<client_ip>[% IP %]) user=(?<user>[% USERNAME %])
69  
70 =cut
71  
72 13 ▶sub string {
Subroutine string: called CC=3 SCAR=11.0
Pod documented
73 13 my ( $blank, %opts ) = @_;
74  
75 13 ▶ if ( !defined( $opts{'string'} ) ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $opts{"string"}) 1 12 13
76 1 die('$opts{string} is undef');
77 }
78 12 my $string = $opts{'string'};
79  
80 # one pass: %{NAME} becomes [% NAME %] and %{NAME:capture} becomes
81 # (?<capture>[% NAME %]). s///g resumes after each replacement, so the
82 # replacement text itself is never re-matched
83 12 $string =~ s/\%\{([A-Za-z0-9_]+)(?:\:([A-Za-z0-9_]+))?\}/
84 17 ▶ defined($2) ? "(?<$2>[% $1 %])" : "[% $1 %]"/ge;
Branch: 1 branch
BranchTrueFalseTotal
defined $2 ? : 5 12 17
85  
86 12 return $string;
87 } ## end sub string
88  
89 =head2 file
90  
91 L</string> applied to a whole file, a line at a time.
92  
93 The file's structure is untouched. Only the C<%{...}> references change, so a grok
94 patterns file comes back as the same file with Log::Munger templating in it. To
95 turn one into an actual rule file, use L</grok2rules> instead.
96  
97 - file :: The file to convert. Required.
98 Default :: undef
99  
100 Returns the whole rewritten file as a single string, line endings and all. Dies if
101 C<file> is undef or the file cannot be read.
102  
103 my $results;
104 my $file = '/tmp/patterns.grok';
105 eval { $results = Log::Munger::Degrok->file( 'file' => $file ); };
106 if ($@) {
107 die( 'Failed to process "' . $file . '"... ' . $@ );
108 }
109 print $results;
110  
111 =cut
112  
113 1 ▶sub file {
Subroutine file: called CC=4 SCAR=14.4
Pod documented
114 1 my ( $blank, %opts ) = @_;
115  
116 1 ▶ if ( !defined( $opts{'file'} ) ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $opts{"file"}) 0 1 1
117 0 die('$opts{file} is undef');
118 }
119  
120 1 my @lines;
121 1 eval { @lines = read_file( $opts{'file'} ); };
122 1 ▶ if ($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 0 1 1
123 0 die( 'Failed to read "' . $opts{'file'} . '"... ' . $@ );
124 }
125  
126 1 my @processed_lines;
127 1 foreach my $line (@lines) {
128 2 $line = Log::Munger::Degrok->string( 'string' => $line );
129 2 push( @processed_lines, $line );
130 }
131  
132 1 return join( '', @processed_lines );
133 } ## end sub file
134  
135 =head2 grok2rules
136  
137 Turns a grok patterns file into the skeleton of a Log::Munger rule file.
138  
139 A grok patterns file is a list of C<NAME regexp> lines. Each one becomes a var:
140 plain lines go under C<vars>, and lines that reference other patterns go under
141 C<vars_templated> once they have been through L</string>. Comments and blank lines
142 are dropped.
143  
144 Naming what you already have as C<includes> is worth doing. Most grok patterns
145 files start with their own copies of C<IP>, C<WORD>, C<HOSTNAME> and the rest, and
146 without the includes those all come across and shadow the ones in F<base.yaml>.
147 With them, the duplicates are recognised and handled per C<overwrite> instead.
148  
149 What comes back is a skeleton, not a finished rule file. There is no C<rules>
150 section, no gates, no tests and no enrichment, because none of that exists in a
151 grok patterns file to convert. Writing those is the part still left to do.
152  
153 - file :: The grok patterns file to convert. Required.
154 Default :: undef
155  
156 - includes :: Rule files to treat as includes. Their vars are loaded so
157 names already provided by one can be recognised, and they are written
158 into the skeleton's own includes list. The taken value is an array ref.
159 Default :: []
160  
161 - overwrite :: What to do about a name an include already defines.
162 Accepted values are:
163 - yes :: Take the grok file's version.
164 - no_silent :: Keep the include's version, saying nothing.
165 - no_warn :: Keep the include's version and warn about it.
166 - no_die :: Die on the first one.
167 Default :: no_warn
168  
169 Returns the skeleton as a YAML string, ready to write to a file. Dies if C<file>
170 is undef or unreadable, if C<overwrite> is not one of the four values above, if an
171 include cannot be loaded, or if a line in the patterns file has a name but no
172 regexp after it.
173  
174 my $results;
175 my $file = '/tmp/patterns.grok';
176 eval { $results = Log::Munger::Degrok->grok2rules( 'file' => $file, 'includes' => ['base'] ); };
177 if ($@) {
178 die( 'Failed to process "' . $file . '"... ' . $@ );
179 }
180 print $results;
181  
182 =cut
183  
184 7 ▶sub grok2rules {
Subroutine grok2rules: called CC=24 SCAR=34.0
Pod documented
185 7 my ( $blank, %opts ) = @_;
186  
187 7 ▶ if ( !defined( $opts{'file'} ) ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $opts{"file"}) 0 7 7
188 0 die('$opts{file} is undef');
189 }
190  
191 7 my $overwrite_types = {
192 'yes' => 1,
193 'no_die' => 1,
194 'no_warn' => 1,
195 'no_silent' => 1,
196 };
197  
198 7 ▶ if ( !defined( $opts{'overwrite'} ) ) {
Branch: 3 branches
BranchTrueFalseTotal
if (not defined $opts{'overwrite'}) { } 2 5 7
elsif (ref $opts{'overwrite'} ne '') { } 0 5 5
elsif (not $$overwrite_types{$opts{'overwrite'}}) { } 1 4 5
199 2 $opts{'overwrite'} = 'no_warn';
200 } elsif ( ref( $opts{'overwrite'} ) ne '' ) {
201 0 die( '$opts{overwrite} is specified but has a ref of "' . ref( $opts{'overwrite'} ) . '" instead of ""' );
202 } elsif ( !$overwrite_types->{ $opts{'overwrite'} } ) {
203 die( '$opts{overwrite} has a value of "'
204 . $opts{'overwrite'}
205 . '"... expected values: '
206 1 . join( ', ', sort( keys( %{$overwrite_types} ) ) ) );
207 }
208  
209 # the rules file being generated
210 6 my $rules = {};
211  
212 6 my $includes = { 'vars' => {}, 'vars_templated' => {} };
213 6 ▶ if ( defined( $opts{'includes'} )
Branch: 1 branch
BranchTrueFalseTotal
if (defined $opts{"includes"} and ref $opts{"includes"} ne "ARRAY") 0 6 6
Condition66%2/3
defined $opts{"includes"} and ref $opts{"includes"} ne "ARRAY"
!ll&&!rl&&r
510
MC/DC0%0/2
defined $opts{"includes"} and ref $opts{"includes"} ne "ARRAY"
defined $opts{"includes"} ref $opts{"includes"} ne "ARRAY"
Truth table66%2/3
A and B
A= defined $opts{"includes"}B= ref $opts{"includes"} ne "ARRAY"
ABresult
0X0
100
111
214 && ( ref( $opts{'includes'} ) ne 'ARRAY' ) )
215 {
216 0 die( '$opts{includes} is specified but has a ref of "' . ref( $opts{'includes'} ) . '" instead of "ARRAY"' );
217 }
218 6 ▶ if ( defined( $opts{'includes'} ) && defined( $opts{'includes'}[0] ) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $opts{"includes"} and defined $opts{"includes"}[0]) 1 5 6
Condition66%2/3
defined $opts{"includes"} and defined $opts{"includes"}[0]
!ll&&!rl&&r
501
MC/DC50%1/2
defined $opts{"includes"} and defined $opts{"includes"}[0]
defined $opts{"includes"} defined $opts{"includes"}[0]
Truth table66%2/3
A and B
A= defined $opts{"includes"}B= defined $opts{"includes"}[0]
ABresult
0X0
100
111
219 1 $rules->{'includes'} = [];
220  
221 # LEFT_PRECEDENT so an earlier include wins over a later one, the
222 # same way RuleFileParser merges. Only existence is checked below,
223 # so this is consistency rather than behavior
224 1 my $merger = Hash::Merge->new('LEFT_PRECEDENT');
225 1 my $parser = Log::Munger::RuleFileParser->new;
226  
227 # use a while loop instead of foreach for basically simplifying display of errors
228 1 my $include_int = 0;
229 1 ▶ while ( defined( $opts{'includes'}[$include_int] ) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $opts{"includes"}[$include_int]) 1 1 2
230 1 my $include = $opts{'includes'}[$include_int];
231 1 ▶ if ( ref($include) ne '' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $include ne "") 0 1 1
232 0 die( '$opts{includes}[' . $include_int . '] has a ref of "' . ref($include) . '" and not ""' );
233 }
234  
235 1 push( @{ $rules->{'includes'} }, $include );
236  
237 1 my $include_rules;
238 1 eval { $include_rules = $parser->load( 'file' => $include ); };
239 1 ▶ if ($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 0 1 1
240 0 die( '$opts{includes}[' . $include_int . '], "' . $include . '", could not be loaded... ' . $@ );
241 }
242  
243 1 $includes = $merger->merge( $includes, $include_rules );
244  
245 1 $include_int++;
246 } ## end while ( defined( $opts{'includes'}[$include_int...]))
247 } ## end if ( defined( $opts{'includes'} ) && defined...)
248  
249 6 my @lines;
250 6 eval { @lines = read_file( $opts{'file'} ); };
251 6 ▶ if ($@) {
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 1 5 6
252 1 die( 'Failed to read "' . $opts{'file'} . '"... ' . $@ );
253 }
254 # chomp before anything else so the regexp half of a line never carries its
255 # newline into the emitted YAML
256 5 chomp(@lines);
257 # drop comments and blank lines. A name with no regexp after it is kept so
258 # it hits the die below rather than vanishing silently
259 5 ▶ @lines = grep( !/^#/ && !/^\s*$/, @lines );
Condition33%1/3
/^#/ || /^\s*$/
l!l&&r!l&&!r
0014
MC/DC0%0/2
/^#/ || /^\s*$/
/^#/ /^\s*$/
Truth table33%1/3
A || B
A= /^#/B= /^\s*$/
ABresult
1X1
011
000
260  
261 5 foreach my $line (@lines) {
262 14 my ( $var, $regexp ) = split( /[\ \t]+/, $line, 2 );
263 # If we don't have $regexp, there is almost certainly something wrong with that line
264 # It is not useful and likely should be removed.
265 14 ▶ if ( !defined($regexp) ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $regexp) 0 14 14
266 0 die( 'The line "'
267 . $line
268 . '" could not be split into a name and a regexp on whitespace, meaning there is likely something wrong with this line as there is no regexp for the variable'
269 );
270 }
271  
272 # a name an include already defines is handled per the overwrite policy:
273 # yes takes the grok file's version, the no_* policies keep the
274 # include's
275 14 my $process_line = 1;
276 14 ▶ if ( $opts{'overwrite'} ne 'yes' ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($opts{"overwrite"} ne "yes") 14 0 14
277 14 my $already;
278 14 ▶ if ( defined( $includes->{'vars'}{$var} ) ) {
Branch: 2 branches
BranchTrueFalseTotal
if (defined $includes->{'vars'}{$var}) { } 2 12 14
elsif (defined $includes->{'vars_templated'}{$var}) { } 0 12 12
279 2 $already = '.vars.' . $var;
280 } elsif ( defined( $includes->{'vars_templated'}{$var} ) ) {
281 0 $already = '.vars_templated.' . $var;
282 }
283 14 ▶ if ( defined($already) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $already) 2 12 14
284 2 ▶ if ( $opts{'overwrite'} eq 'no_die' ) {
Branch: 2 branches
BranchTrueFalseTotal
if ($opts{'overwrite'} eq 'no_die') { } 0 2 2
elsif ($opts{'overwrite'} eq 'no_warn') { } 0 2 2
285 0 die( '"' . $already . '" is already defined in one of the includes' );
286 } elsif ( $opts{'overwrite'} eq 'no_warn' ) {
287 0 warn( '"' . $already . '" is already defined in one of the includes... skipping...' );
288 }
289 2 $process_line = 0;
290 }
291 } ## end if ( $opts{'overwrite'} ne 'yes' )
292  
293 14 ▶ if ($process_line) {
Branch: 1 branch
BranchTrueFalseTotal
if ($process_line) 12 2 14
294 # a line referencing another pattern becomes a templated var
295 12 ▶ if ( $regexp =~ /\%\{[A-Za-z0-9_]+(?:\:[A-Za-z0-9_]+)?\}/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($regexp =~ /\%\{[A-Za-z0-9_]+(?:\:[A-Za-z0-9_]+)?\}/) { } 5 7 12
296 5 $rules->{'vars_templated'}{$var} = Log::Munger::Degrok->string( 'string' => $regexp );
297 } else {
298 7 $rules->{'vars'}{$var} = $regexp;
299 }
300 }
301  
302 } ## end foreach my $line (@lines)
303  
304 5 return Dump($rules);
305 } ## end sub grok2rules
« lib/Log/Munger/App/Command/enrich.pm ↑ Summary lib/Log/Munger/RulesTemplateOrder.pm »