bin/loghack

filter: Statementstmt 5.9% 49 / 829 Branchbran 0.8% 3 / 372 Conditioncond 0.0% 0 / 39 MC/DCmcdc 0.0% 0 / 23 Subroutinesub 19.1% 14 / 73 Podpod n/a totaltotal 4.9% 66 / 1336 CC 238 SCAR 45.3
CC
238
Cov
4%
cluster
342.0
do_prep
272.0
repo_files
240.0
CRAP 93.0
« blib/lib/ApacheLog/Parser/Report.pm ↑ Summary blib/lib/ApacheLog/Parser.pm »
1 #!/usr/bin/perl
2  
3 # Copyright (C) 2007 Eric L. Wilhelm
4  
5 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
7  
8 =head1 NAME
9  
10 loghack - process and query apache logs
11  
12 =cut
13  
14 package bin::loghack;
15  
16 1 ▶use ApacheLog::Parser qw(parse_line :fields);
Subroutine BEGIN: called CC=1 SCAR=0.0
17 1 ▶use ApacheLog::Parser::SkipList;
Subroutine BEGIN: called CC=1 SCAR=0.0
18  
19 1 ▶use Getopt::Helpful;
Subroutine BEGIN: called CC=1 SCAR=0.0
20  
21 1 ▶use YAML ();
Subroutine BEGIN: called CC=1 SCAR=0.0
22 1 ▶use POSIX ();
Subroutine BEGIN: called CC=1 SCAR=0.0
23 1 ▶use File::Basename qw(basename dirname);
Subroutine BEGIN: called CC=1 SCAR=0.0
24 1 ▶use Digest::MD5 ();
Subroutine BEGIN: called CC=1 SCAR=0.0
25  
26 1 ▶use Date::Piece qw(date months weeks);
Subroutine BEGIN: called CC=1 SCAR=0.0
27  
28 1 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
29  
30 1 ▶BEGIN {
Subroutine BEGIN: called CC=3 SCAR=19.2
31 1 ▶ unless($ENV{HOSTNAME}) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($ENV{"HOSTNAME"}) 0 1 1
32 0 ▶ if(open(my $fh, '<', '/etc/hostname')) {
Branch: 1 branch
BranchTrueFalseTotal
if (open my $fh, "<", "/etc/hostname") 0 0 0
33 0 chomp(my $line = <$fh>);
34 0 $ENV{HOSTNAME} = $line;
35 }
36 }
37 }
38 1 ▶$SIG{CHLD} = sub {
Subroutine __ANON__: called CC=3 SCAR=18.1
39 1 my $child;
40 1 ▶ while(($child = waitpid(-1, POSIX::WNOHANG())) > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (($child = waitpid(-1, 1)) > 0) 0 1 1
41 # XXX this dies before we get a cluster child's output?
42 0 ▶ if($?) {
Branch: 1 branch
BranchTrueFalseTotal
if ($?) 0 0 0
43 0 my $code = $? >> 8;
44 0 my $sig = $? & 127;
45 0 die "$child status: $? ($code/$sig)";
46 }
47 }
48 1 };
49  
50 0 ▶sub open_file {
Subroutine open_file: not called CC=7 SCAR=40.3
51 0 my ($file) = @_;
52  
53 # better than IO::Zlib because it leverages multiple cores, duh
54 0 ▶ if($file =~ m/\.(gz|bz2)$/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($file =~ /\.(gz|bz2)$/) { } 0 0 0
55 0 my $ext = $1;
56 0 my %prog = (
57 gz => 'gunzip',
58 bz2 => 'bunzip2',
59 );
60 0 ▶ my $prog = $prog{$ext} or die "cannot read $ext files";
Branch: 1 branch
BranchTrueFalseTotal
my $prog = $prog{$ext} or die "cannot read $ext files" 0 0 0
61 0 my $pid = open(my $fh, '-|');
62 0 ▶ unless($pid) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($pid) 0 0 0
63 0 local $SIG{CHLD};
64 0 ▶ open(STDIN, '<', $file) or die "cannot open $file $!";
Branch: 1 branch
BranchTrueFalseTotal
open STDIN, "<", $file or die "cannot open $file $!" 0 0 0
65 0 ▶ exec($prog, '-c') or die "ack $!";
Branch: 1 branch
BranchTrueFalseTotal
exec $prog, "-c" or die "ack $!" 0 0 0
66 }
67 #warn "launch $prog < $file on $pid\n";
68 0 return($fh);
69 }
70 else {
71 0 ▶ open(my $fh, '<', $file) or die "cannot open '$file' $!";
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, "<", $file or die "cannot open '${file}' $!" 0 0 0
72 0 return($fh);
73 }
74 }
75  
76 0 ▶sub pipe_out {
Subroutine pipe_out: not called CC=6 SCAR=37.4
77 0 my ($file) = @_;
78 0 ▶ $file =~ m/\.(gz|bz2)$/ or die "unknown extension on $file";
Branch: 1 branch
BranchTrueFalseTotal
$file =~ /\.(gz|bz2)$/ or die "unknown extension on $file" 0 0 0
79 0 my $ext = $1;
80 0 my %prog = (
81 gz => 'gzip',
82 bz2 => 'bzip2',
83 );
84 0 ▶ my $prog = $prog{$ext} or die "cannot write $ext files";
Branch: 1 branch
BranchTrueFalseTotal
my $prog = $prog{$ext} or die "cannot write $ext files" 0 0 0
85  
86 0 my $pid = open(my $fh, '|-');
87 0 ▶ unless($pid) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($pid) 0 0 0
88 0 local $SIG{CHLD};
89 0 ▶ open(STDOUT, '>', $file) or die "cannot write '$file' $!";
Branch: 1 branch
BranchTrueFalseTotal
open STDOUT, ">", $file or die "cannot write '${file}' $!" 0 0 0
90 0 ▶ exec($prog, '-c') or die "ack $!";
Branch: 1 branch
BranchTrueFalseTotal
exec $prog, "-c" or die "ack $!" 0 0 0
91 }
92 #warn "launch $prog > $file on $pid\n";
93 0 return($fh);
94 }
95  
96 0 ▶sub main {
Subroutine main: not called CC=7 SCAR=40.3
97 0 my (@args) = @_;
98  
99 0 my %o = (
100 archive => '',
101 repository => '',
102 missok => 0,
103 daemon => '',
104 cluster => '',
105 skip => 1,
106 quiet => 0,
107 );
108 my $hopt = Getopt::Helpful->new(
109 usage => 'CALLER <mode> [options] <arguments>',
110 ['a|archive=s', \$o{archive}, '<dir>', 'archive dir'],
111 ['r|repository=s', \$o{repository}, '<dir>', 'repository dir'],
112 ['missok', \$o{missok}, '', 'skip missing files'],
113 ['d|daemon=s', \$o{daemon}, '<dir>', 'daemon mode - needs chdir'],
114 ['c|cluster=s', \$o{cluster}, '<hosts>', 'cluster mode'],
115 ['s|skip!', \$o{skip}, '', 'use skipper (default yes)'],
116 0 ['q|quiet', \$o{quiet}, '', 'suppress status'],
117 '+help',
118 );
119 0 $hopt->Get_from(\@args);
120  
121 0 ▶ if(not $o{repository}) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($o{"repository"}) 0 0 0
122 0 ▶ $o{repository} = '.' if(-e '.config');
Branch: 1 branch
BranchTrueFalseTotal
if -e ".config" 0 0 0
123 }
124  
125 0 ▶ if($o{daemon}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($o{"daemon"}) 0 0 0
126 0 daemon(\%o, @args);
127 0 exit;
128 }
129  
130 0 my %modes = map({$_ => 1} qw(
131 makelinks
132 import
133 prep check sweep verify confirm list
134 unique day_unique month_unique month_unique2
135 compile
136 aggregate report date dump tabulate count reskip
137 ));
138 0 my $mode = shift(@args);
139 0 ▶ $modes{$mode} or die "USAGE: mode must be one of ",
Branch: 1 branch
BranchTrueFalseTotal
$modes{$mode} or die "USAGE: mode must be one of ", join(", ", sort(keys %modes)), "\n" 0 0 0
140 join(", ", sort(keys(%modes))), "\n";
141  
142 # TODO deal with the do_ stuff
143 0 ▶ if($o{cluster}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($o{'cluster'}) { } 0 0 0
144 0 cluster(\%o, $mode, @args);
145 }
146 else {
147 0 ▶ my $run = __PACKAGE__->can('do_' . $mode) or
Branch: 1 branch
BranchTrueFalseTotal
my $run = __PACKAGE__->can('do_' . $mode) or die "cannot find method 'do_${mode}'" 0 0 0
148 die "cannot find method 'do_$mode'";
149 0 $run->(\%o, @args);
150 }
151 }
152  
153 0 ▶sub name_as_date {
Subroutine name_as_date: not called CC=2 SCAR=17.9
154 0 my ($n) = @_;
155 0 ▶ $n =~ s/(?:.*\.)?(\d{4}-\d{2}-\d{2})\..*/$1/ or
Branch: 1 branch
BranchTrueFalseTotal
$n =~ s/(?:.*\.)?(\d{4}-\d{2}-\d{2})\..*/$1/ or croak("weird name -- $n") 0 0 0
156 croak("weird name -- $n");
157 0 $n =~ s#.*/##;
158 0 return($n);
159 }
160 0 ▶sub nice_name {
Subroutine nice_name: not called CC=2 SCAR=17.9
161 0 my ($name) = @_;
162 0 my @d = split(/\/+/, $name);
163 0 my $n = '*.' . name_as_date(pop(@d)) . '.*';
164 0 ▶ @d or return($n);
Branch: 1 branch
BranchTrueFalseTotal
@d or return $n 0 0 0
165 0 return(join("/", $d[-1], $n));
166 }
167 0 ▶sub record_source {
Subroutine record_source: not called CC=4 SCAR=30.0
168 0 my ($opt, $file, $dir, $md5) = @_;
169  
170 0 my $writefile = $dir.$md5;
171  
172 0 want_dir($dir);
173  
174 0 ▶ if(-e $writefile) {
Branch: 1 branch
BranchTrueFalseTotal
if (-e $writefile) 0 0 0
175 0 warn "skipping $writefile ($file)\n";
176 0 return;
177 }
178 0 ▶ open(my $fh, '>', $writefile) or die "cannot write '$writefile' $!";
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, ">", $writefile or die "cannot write '${writefile}' $!" 0 0 0
179 0 print $fh File::Basename::basename($file), "\n";
180 0 ▶ close($fh) or die "cannot write '$writefile' $!";
Branch: 1 branch
BranchTrueFalseTotal
close $fh or die "cannot write '${writefile}' $!" 0 0 0
181 }
182 0 ▶sub want_dir {
Subroutine want_dir: not called CC=4 SCAR=30.0
183 0 my ($dir) = @_;
184  
185 0 ▶ return if(-d $dir);
Branch: 1 branch
BranchTrueFalseTotal
if -d $dir 0 0 0
186  
187 0 ▶ unless(mkdir($dir)) {
Branch: 1 branch
BranchTrueFalseTotal
unless (mkdir $dir) 0 0 0
188 0 ▶ die "cannot create $dir $!" unless(-d $dir);
Branch: 1 branch
BranchTrueFalseTotal
unless -d $dir 0 0 0
189 }
190 }
191  
192 0 ▶sub daemon {
Subroutine daemon: not called CC=3 SCAR=24.8
193 0 my ($opt, @args) = @_;
194  
195 0 my $dir = $opt->{daemon};
196 0 ▶ chdir($dir) or die "no such dir $dir\n";
Branch: 1 branch
BranchTrueFalseTotal
chdir $dir or die "no such dir $dir\n" 0 0 0
197 0 $| = 1;
198 0 ▶ while(my $line = <STDIN>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline STDIN)) 0 0 0
199 0 chomp($line);
200 0 main(split(/\t/, $line));
201 0 print ".done\n";
202 #warn "done\n";
203 #die ".done\n";
204 #sleep(1);
205 }
206 }
207  
208 0 ▶sub start_cluster {
Subroutine start_cluster: not called CC=4 SCAR=30.0
209 0 my ($dir, @hosts) = @_;
210 0 require IPC::Open3;
211 0 require IO::Select;
212 0 require IO::Handle;
213 0 my $sel = IO::Select->new();
214 0 my %track;
215 0 my $prog = basename($0);
216 0 foreach my $host (@hosts) {
217 0 (my $realhost = $host) =~ s/#\d+$//;
218 0 my $stdin;
219 0 my ($stdout, $stderr) = map({IO::Handle->new} 1..3);
220 0 ▶ my $pid = IPC::Open3::open3(
Branch: 1 branch
BranchTrueFalseTotal
$realhost eq 'localhost' ? : 0 0 0
221 $stdin, $stdout, $stderr,
222 ($realhost eq 'localhost' ? () : ('ssh', $realhost)),
223 $prog, '-d', $dir
224 );
225 #warn "started $pid to $host";
226 0 $stdout->autoflush;
227 0 $stderr->autoflush;
228 0 ▶ $pid or die "gah no pid\n";
Branch: 1 branch
BranchTrueFalseTotal
$pid or die "gah no pid\n" 0 0 0
229 #warn "$stdin, $stdout, $stderr";
230 0 $track{$pid} = my $obj = {
231 stdin => $stdin,
232 stdout => $stdout,
233 stderr => $stderr,
234 host => $host,
235 };
236 0 $sel->add($obj->{sel_o} = [$stdout, $pid, 'stdout']);
237 0 $sel->add($obj->{sel_e} = [$stderr, $pid, 'stderr']);
238 }
239 0 return($sel, %track);
240 }
241 0 ▶my $lglob = sub {
Subroutine __ANON__: not called CC=1 SCAR=6.9
242 0 my ($opt, @spec) = @_;
243 0 local $opt->{lazy_glob} = 1;
244 0 return(repo_files($opt, @spec));
245 1 };
246 0 ▶my $datethru = sub {shift(@_); _date_dwim(@_) };
Subroutine __ANON__: not called CC=1 SCAR=2.6
247 my %cluster_fspec = (
248 report => $lglob,
249 compile => $datethru,
250 unique => $lglob,
251 day_unique => $datethru,
252 0 ▶ month_unique => sub {shift(@_); @_},
Subroutine __ANON__: not called CC=1 SCAR=6.9
253 0 ▶ month_unique2 => sub {$_[1]},
Subroutine __ANON__: not called CC=1 SCAR=6.9
254 1 );
255 0 ▶sub cluster {
Subroutine cluster: not called CC=18 SCAR=58.3
256 0 my ($opt, $mode, @files) = @_;
257  
258 0 require Cwd;
259 0 my $dir = Cwd::abs_path($opt->{repository});
260  
261 0 ▶ if(my $code = $cluster_fspec{$mode}) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $code = $cluster_fspec{$mode}) { } 0 0 0
262 0 @files = $code->($opt, @files);
263 #die join("\n ", 'files', @files);
264 }
265 else {
266 0 @files = repo_files($opt, @files);
267 0 foreach my $file (@files) {
268 0 my $msg = "missing $file\n";
269 0 ▶ (-e $file) or $opt->{missok} ? warn $msg : die $msg;
Branch: 2 branches
BranchTrueFalseTotal
-e $file or $opt->{"missok"} ? warn($msg) : die($msg) 0 0 0
$opt->{'missok'} ? : 0 0 0
270 }
271 }
272  
273 0 my @hosts = map({my ($h, $n) = split(/:/, $_);
274 0 ▶ ($n ? map({$h.'#'.$_} 1..$n) : $h)
Branch: 1 branch
BranchTrueFalseTotal
$n ? : 0 0 0
275 0 } split(/, ?/, $opt->{cluster}));
276  
277 0 ▶ if(@hosts > @files) { # XXX weighting?
Branch: 1 branch
BranchTrueFalseTotal
if (@hosts > @files) 0 0 0
278 0 warn "that would get boring\n";
279 0 $#hosts = $#files;
280 }
281 0 my ($sel, %track) = start_cluster($dir, @hosts);
282  
283 0 my $hlen = 0;
284 0 foreach my $host (@hosts) {
285 0 my $l = length($host);
286 0 ▶ $hlen = $l if($l > $hlen);
Branch: 1 branch
BranchTrueFalseTotal
if $l > $hlen 0 0 0
287 }
288  
289 #die map({"$_ => " . join(", ", %{$track{$_}})} keys(%track));
290 0 my %hmap = map({$track{$_}{host} => $_} keys(%track));
291 0 my %sels = map({$track{$_->[1]}{host} => $_} $sel->handles);
292  
293 0 my %blacklist;
294 0 ▶ my $output = sub {
Subroutine __ANON__: not called CC=3 SCAR=24.8
295 0 my ($host, $which, @lines) = @_;
296 0 ▶ my $pref = ($which eq 'stderr' ? '!' : '#');
Branch: 1 branch
BranchTrueFalseTotal
$which eq 'stderr' ? : 0 0 0
297 0 printf("%-${hlen}s %s %s", $host, $pref, $_) for(@lines);
298 0 };
299 0 ▶ my $end_host = sub {
Subroutine __ANON__: not called CC=4 SCAR=30.0
300 0 my ($host) = @_;
301  
302 0 ▶ my $pid = delete($hmap{$host}) or die "no pid at $host";
Branch: 1 branch
BranchTrueFalseTotal
my $pid = delete $hmap{$host} or die "no pid at $host" 0 0 0
303 0 my $obj = delete($track{$pid});
304  
305 0 warn ' 'x($hlen+1), "closing $host\n";
306 0 close($obj->{stdin});
307 0 my $errh = $obj->{stdout};
308 0 ▶ local $SIG{ALRM} = sub { warn "no stderr on $host\n"};
Subroutine __ANON__: not called CC=1 SCAR=6.9
309 0 alarm(2);
310 0 $output->($host, 'stderr', <$errh>);
311 0 alarm(0);
312 0 ▶ $sel->remove(delete($obj->{sel_o})) or die;
Branch: 1 branch
BranchTrueFalseTotal
$sel->remove(delete $obj->{"sel_o"}) or die 0 0 0
313 #$errh->blocking(0);
314 0 ▶ $sel->remove(delete($obj->{sel_e})) or die;
Branch: 1 branch
BranchTrueFalseTotal
$sel->remove(delete $obj->{"sel_e"}) or die 0 0 0
315 0 };
316 0 ▶ my $fill_host = sub {
Subroutine __ANON__: not called CC=9 SCAR=45.0
317 0 my ($host) = @_;
318  
319 0 ▶ if($blacklist{$host}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($blacklist{$host}) 0 0 0
320 0 warn "$host is blacklisted\n";
321 0 eval { $end_host->($host) };
322 0 return;
323 }
324  
325 0 ▶ my $pid = $hmap{$host} or die "no pid at $host";
Branch: 1 branch
BranchTrueFalseTotal
my $pid = $hmap{$host} or die "no pid at $host" 0 0 0
326 0 my $obj = $track{$pid};
327 0 my $fh = $obj->{stdin};
328  
329 0 ▶ unless(@files) {
Branch: 1 branch
BranchTrueFalseTotal
unless (@files) 0 0 0
330 0 $end_host->($host);
331 0 return;
332 }
333  
334 0 my $file = shift(@files);
335 #warn "fill $host with $file\n";
336 0 ▶ if($opt->{missok} and not -e $file) {
Branch: 1 branch
BranchTrueFalseTotal
if ($opt->{"missok"} and not -e $file) 0 0 0
Condition0%0/3
$opt->{"missok"} and not -e $file
!ll&&!rl&&r
000
MC/DC0%0/2
$opt->{"missok"} and not -e $file
$opt->{"missok"} not -e $file
Truth table0%0/3
A and not(B)
A= $opt->{"missok"}B= not -e $file
ABresult
0X0
100
111
337 0 warn "still missing '$file'\n";
338 0 my @later = ($file);
339 0 ▶ while($file = shift(@files)) {
Branch: 1 branch
BranchTrueFalseTotal
if ($file = shift @files) 0 0 0
340 0 ▶ if(-e $file) {
Branch: 1 branch
BranchTrueFalseTotal
if (-e $file) { } 0 0 0
341 0 push(@files, @later);
342 0 last;
343 }
344 else {
345 0 warn "still missing '$file'\n";
346 0 push(@later, $file);
347 }
348 }
349 0 ▶ $file or die "out of files to use while waiting\n";
Branch: 1 branch
BranchTrueFalseTotal
$file or die "out of files to use while waiting\n" 0 0 0
350 # grr, needs a loop
351 }
352 #warn "send $host $mode\t$file\n";
353 0 print $fh "$mode\t$file\n";
354 0 };
355 0 ▶ local $SIG{CHLD} = sub {
Subroutine __ANON__: not called CC=3 SCAR=24.8
356 0 my $child;
357 0 ▶ while(($child = waitpid(-1, POSIX::WNOHANG())) > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (($child = waitpid(-1, 1)) > 0) 0 0 0
358 0 ▶ if($?) {
Branch: 1 branch
BranchTrueFalseTotal
if ($?) 0 0 0
359 0 my $code = $? >> 8;
360 0 my $sig = $? & 127;
361 0 my $host = $track{$child}{host};
362 0 warn " error $host ($child) status: $? ($code/$sig)\n";
363 0 $blacklist{$host} = 1;
364 0 $end_host->($host);
365 }
366 }
367 0 };
368  
369  
370 # go!
371 0 $fill_host->($_) for(@hosts);
372  
373 0 my %f = (stderr => 0, stdout => 1);
374 0 ▶ while($sel->count) {
Branch: 1 branch
BranchTrueFalseTotal
if ($sel->count) 0 0 0
375 0 ▶ READ: while(my @ready = $sel->can_read) {
Branch: 1 branch
BranchTrueFalseTotal
if (my(@ready) = $sel->can_read) 0 0 0
376 0 @ready = sort({$f{$a->[2]} <=> $f{$b->[2]}} @ready);
377  
378 0 foreach my $bit (@ready) {
379 0 my ($fh, $pid, $which) = @$bit;
380 0 my $obj = $track{$pid};
381 0 my $host = $obj->{host};
382 0 $fh->blocking(0);
383 0 ▶ until(eof($fh)) {
Branch: 1 branch
BranchTrueFalseTotal
unless (eof $fh) 0 0 0
384 0 my $line = <$fh>;
385  
386 # XXX probably never need this bit
387 0 ▶ unless(defined($line)) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $line) 0 0 0
388 0 warn "undef line from $host\n";
389 0 $sel->remove($bit);
390 0 last;
391 }
392 # TODO handle death
393 0 ▶ if(($which eq 'stdout') and ($line =~ m/^.done$/)) {
Branch: 1 branch
BranchTrueFalseTotal
if ($which eq "stdout" and $line =~ /^.done$/) 0 0 0
Condition0%0/3
$which eq "stdout" and $line =~ /^.done$/
!ll&&!rl&&r
000
MC/DC0%0/2
$which eq "stdout" and $line =~ /^.done$/
$which eq "stdout" $line =~ /^.done$/
Truth table0%0/3
A and B
A= $which eq "stdout"B= $line =~ /^.done$/
ABresult
0X0
100
111
394 #warn "$host said done\n";
395 0 $fill_host->($host);
396 0 last;
397 }
398 0 $output->($host, $which, $line);
399 }
400 0 $fh->blocking(1);
401 }
402 }
403 #warn "twiddling\n";
404 }
405 0 ▶ if(@files) {
Branch: 1 branch
BranchTrueFalseTotal
if (@files) 0 0 0
406 0 die "ACK all my hosts died! (",
407 scalar(@files), " files left to process.)\n";
408 }
409 }
410  
411 =head2 reskip
412  
413 Regenerate the skiplist for a given chunk.
414  
415 =cut
416  
417 0 ▶sub do_reskip {
Subroutine do_reskip: not called CC=6 SCAR=37.4
418 0 my ($opt, @files) = @_;
419  
420 0 @files = repo_files($opt, @files);
421  
422 0 my $skipper = get_skipper($opt);
423 0 my $doskip = $skipper->get_matcher;
424 0 foreach my $file (@files) {
425 0 ▶ unless(-e $file) {
Branch: 1 branch
BranchTrueFalseTotal
unless (-e $file) 0 0 0
426 0 die "no such file:\n $file\n";
427 }
428 0 my $fh = open_file($file);
429  
430 0 my $nicename = nice_name($file);
431 0 my $start = time;
432 0 print "$nicename -- ",
433 sprintf("%02d:%02d:%02d", (localtime($start))[2,1,0]), "\n";
434  
435 0 my $skipfile = skipfilename($opt, $file);
436 0 my $sw = $skipper->new_writer($skipfile);
437 0 my $lnum = 0;
438 0 ▶ while(my $line = <$fh>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $fh)) 0 0 0
439 0 $lnum++;
440 0 chomp($line);
441 0 my @v = split(/\t/, $line);
442 # create skiplist
443 0 ▶ if($doskip and $doskip->(\@v)) {$sw->skip($lnum);}
Branch: 1 branch
BranchTrueFalseTotal
if ($doskip and &$doskip(\@v)) 0 0 0
Condition0%0/3
$doskip and &$doskip(\@v)
!ll&&!rl&&r
000
MC/DC0%0/2
$doskip and &$doskip(\@v)
$doskip &$doskip(\@v)
Truth table0%0/3
A and B
A= $doskipB= &$doskip(\@v)
ABresult
0X0
100
111
444 }
445 }
446  
447 }
448  
449 =begin notes
450  
451 The files are split per-hour. Time zone adjustments are going to be an
452 issue. There's also a potential race condition between two nodes, so
453 the outputs will always have a ".$chunk" appended to them. The value of
454 $part is either 0 or 1 (and only switched to 1 at the start of the file.)
455  
456 And another issue: delay. The request init time is what's shown, but
457 it doesn't get logged until the request completes. So a 10min request
458 will not appear until 10min later. If there are any large downloads,
459 they could possibly even span a couple of logrotates.
460  
461 This also means that tomorrow or the next day could concievably hold a
462 bit of data from a big download that started 24+ hours ago. In
463 practice, logrotate is actually just disposing of this data when it runs
464 gzip. That is, a request always goes in the logfile that was open when
465 the apache process spawned?
466  
467 Still need to figure out the cleanup pass. Add the skiplists together
468 (and/or rename them), figure out where to tie-off the last item, etc.
469 Probably need some tracking of sources and/or chunks. Chunks can
470 probably be treated as closed until further notice as long as a
471 chunkcount file is maintained somewhere.
472  
473 =end notes
474  
475 =begin tznotes
476  
477 Probably going to just leave the date string unprocessed (but we will
478 definitely slot it into files according to the adjusted zone.) Of
479 course, the date+hour+tz is used to memoize the outgoing date, so taking
480 the localtime and chunking that back together with the minutes+seconds
481 wouldn't be a big deal. We will need to address the dst issue though.
482  
483 =end tznotes
484  
485 =cut
486  
487 =head2 prep
488  
489 Parse a raw logfile and split it into hourly chunks.
490  
491 loghack prep servername/logfile.gz
492  
493 =cut
494  
495 0 ▶sub do_prep {
Subroutine do_prep: not called CC=16 SCAR=56.1
496 0 my ($opt, @files) = @_;
497  
498 my $repo = $opt->{repository} or
499 0 ▶ die "must have repository setting for prep()\n";
Branch: 1 branch
BranchTrueFalseTotal
my $repo = $opt->{"repository"} or die "must have repository setting for prep()\n" 0 0 0
500  
501 0 my $doskip;
502 my $skipper;
503 0 ▶ if(-e (my $skipconf = "$repo/.config/skips.conf")) {
Branch: 1 branch
BranchTrueFalseTotal
if (-e (my $skipconf = "$repo/.config/skips.conf")) 0 0 0
504 0 my ($skip) = YAML::LoadFile($skipconf);
505 0 $skipper = ApacheLog::Parser::SkipList->new();
506 0 $skipper->set_config($skip);
507 0 $doskip = $skipper->get_matcher;
508 }
509  
510 0 my @loaded;
511 0 foreach my $file (@files) {
512 0 ▶ unless(-e $file) {
Branch: 1 branch
BranchTrueFalseTotal
unless (-e $file) 0 0 0
513 0 my $msg = "no such file:\n $file\n";
514 0 ▶ if($opt->{missok}) { warn $msg; next };
Branch: 1 branch
BranchTrueFalseTotal
if ($opt->{"missok"}) 0 0 0
515 0 die $msg;
516 }
517 0 my $outpath = repository_path($opt, $file);
518 0 my $fh = open_file($file);
519  
520 0 my $nicename = nice_name($file);
521  
522 0 my $checksum = checksum($fh, 50);
523 0 my $checkfile = "$outpath.loaded/$checksum";
524 0 my $linecount = 0;
525 0 my $ch;
526 0 ▶ if(-e "$outpath/.loaded/$checksum") {
Branch: 1 branch
BranchTrueFalseTotal
if (-e "$outpath/.loaded/$checksum") { } 0 0 0
527 0 warn "assume $nicename is done\n";
528 0 {local $SIG{CHLD}; close($fh);} # stupid macs
529 0 next;
530 # TODO fast-forward support
531 # $linecount = $old_linecount; and etc
532 }
533 else {
534 0 record_source($opt, $file, "$outpath.sources/", $checksum);
535  
536 # TODO this could stand to be more atomic
537 { # record results
538 0 want_dir("$outpath.loaded");
539 0 ▶ my $tag = ($ENV{HOSTNAME} || '') . '.' . $$;
Condition0%0/2
$ENV{'HOSTNAME'} || ''
l!l
00
MC/DC0%0/1
$ENV{'HOSTNAME'} || ''
$ENV{'HOSTNAME'}
Truth table0%0/2
A
A= $ENV{'HOSTNAME'}
Aresult
11
00
540 0 ▶ open($ch, '>', "$checkfile.$tag") or
Branch: 1 branch
BranchTrueFalseTotal
open $ch, ">", "$checkfile.$tag" or die "cannot write '$checkfile.${tag}' $!" 0 0 0
541 die "cannot write '$checkfile.$tag' $!";
542 # TODO chmod
543 0 ▶ rename("$checkfile.$tag", $checkfile) or
Branch: 1 branch
BranchTrueFalseTotal
rename "$checkfile.$tag", $checkfile or die "cannot make $checkfile $!" 0 0 0
544 die "cannot make $checkfile $!";
545 }
546  
547 # TODO a replayable pipe would be nice
548 0 {local $SIG{CHLD}; close($fh);} # stupid macs
549 0 $fh = open_file($file);
550 }
551  
552 0 ▶ $opt->{quiet} or print "$nicename -- ",
Branch: 1 branch
BranchTrueFalseTotal
$opt->{"quiet"} or print "$nicename -- ", sprintf("%02d:%02d:%02d", (localtime)[2, 1, 0]), "\n" 0 0 0
553 sprintf("%02d:%02d:%02d", (localtime)[2,1,0]), "\n";
554  
555 0 my %outhandles;
556 my $sw;
557 0 my $out;
558  
559 0 my $chunk = 2;
560 0 ▶ my $next_chunk = sub {
Subroutine __ANON__: not called CC=5 SCAR=34.0
561 0 my ($date, $hour, $tz) = @_;
562  
563 # might have already started that chunk
564 0 ▶ if(my $handles = $outhandles{"$date$hour$tz"}) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $handles = $outhandles{"$date$hour$tz"}) 0 0 0
565 #warn "back to $date:$hour$tz\n";
566 0 ($out, $sw) = @$handles;
567 0 return;
568 }
569  
570 # TODO include timezone in this calc
571 0 my $datestring = get_datestring($date);
572  
573 # make the tz three digits
574 0 (my $tzout = $tz) =~ s/00$//;
575 0 ▶ $tzout = '+' . $tzout if(length($tzout) == 2);
Branch: 1 branch
BranchTrueFalseTotal
if length $tzout == 2 0 0 0
576  
577 0 my $outfile = $outpath . $datestring . ".$hour$tzout.$chunk.tsv.gz";
578 0 push(@loaded, $outfile);
579 #warn "writing $outfile\n";
580 0 ▶ if(-e $outfile) {
Branch: 1 branch
BranchTrueFalseTotal
if (-e $outfile) 0 0 0
581 # XXX how to decide whether to skip completely?
582 0 die "already have $outfile\n";
583 }
584 0 $chunk = 1; # from now on
585 0 ▶ if($skipper) { # TODO how to reset skipcount?
Branch: 1 branch
BranchTrueFalseTotal
if ($skipper) 0 0 0
586 0 my $skipfile = skipfilename($opt, $outfile);
587 0 $sw = $skipper->new_writer($skipfile);
588 }
589 0 $out = pipe_out($outfile);
590 0 print $ch File::Basename::basename($outfile), "\n";
591 0 $outhandles{"$date$hour$tz"} = [$out, $sw];
592 0 };
593  
594 0 my $cdate = '';
595 0 my %lc;
596 0 ▶ while(my $line = <$fh>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $fh)) 0 0 0
597 0 $linecount++;
598 0 chomp($line);
599 0 my $v = parse_line($line);
600  
601 # check date/time
602 0 my ($d, $h, $rest) = split(/:/, $v->[dtime], 3);
603 0 my ($tz) = ($rest =~ m/ ([-+]?\d+)/);
604 0 ▶ if("$d$h$tz" ne $cdate) {
Branch: 1 branch
BranchTrueFalseTotal
if ("$d$h$tz" ne $cdate) 0 0 0
605 0 $next_chunk->($d, $h, $tz);
606 0 $cdate = "$d$h$tz";
607 0 ▶ $lc{$cdate} ||= 0;
Condition0%0/2
$lc{$cdate} ||= 0
l!l
00
MC/DC0%0/1
$lc{$cdate} ||= 0
$lc{$cdate}
Truth table0%0/2
A
A= $lc{$cdate}
Aresult
11
00
608 #warn "$d $h $tz\n";
609 }
610 0 my $lnum = ++$lc{$cdate};
611  
612 # create skiplist
613 0 ▶ if($doskip->($v)) {$sw->skip($lnum);}
Branch: 1 branch
BranchTrueFalseTotal
if (&$doskip($v)) 0 0 0
614  
615 0 print $out join("\t", @$v), "\n";
616 }
617  
618 0 print $ch "$linecount\n";
619 0 ▶ close($ch) or die "write '$checkfile' failed $!";
Branch: 1 branch
BranchTrueFalseTotal
close $ch or die "write '${checkfile}' failed $!" 0 0 0
620 # TODO race checks/chmod
621  
622 }
623 0 wait(); # XXX need this?
624 0 return(@loaded);
625 }
626  
627 =for doc ###############################################################
628 Examine the */.loaded files and verify that each one has a linecount
629 (finished loading.)
630 loghack check */.loaded/*
631  
632 =cut
633  
634 0 ▶sub do_check {
Subroutine do_check: not called CC=3 SCAR=24.8
635 0 my ($opt, @files) = @_;
636  
637 0 foreach my $file (@files) {
638 0 ▶ my $err = run_check($file) or next;
Branch: 1 branch
BranchTrueFalseTotal
my $err = run_check($file) or next 0 0 0
639 0 print "NC $file (", scalar(@$err), " parts)\n";
640 }
641 }
642 0 ▶sub do_sweep {
Subroutine do_sweep: not called CC=7 SCAR=40.3
643 0 my ($opt, @files) = @_;
644  
645 0 foreach my $file (@files) {
646 0 ▶ my $err = run_check($file) or next;
Branch: 1 branch
BranchTrueFalseTotal
my $err = run_check($file) or next 0 0 0
647 0 print "NC $file (", scalar(@$err), " parts)\n";
648 0 foreach my $part (@$err) {
649 0 print " $part\n";
650 0 ▶ if(-e $part) {
Branch: 1 branch
BranchTrueFalseTotal
if (-e $part) 0 0 0
651 0 ▶ unlink($part) or die "cannot unlink('$part') $!";
Branch: 1 branch
BranchTrueFalseTotal
unlink $part or die "cannot unlink('${part}') $!" 0 0 0
652 }
653 }
654 0 ▶ unlink($file) or die "cannot unlink('$file') $!";
Branch: 1 branch
BranchTrueFalseTotal
unlink $file or die "cannot unlink('${file}') $!" 0 0 0
655 }
656 }
657 0 ▶sub run_check {
Subroutine run_check: not called CC=6 SCAR=37.4
658 0 my ($checkfile) = @_;
659  
660 0 ▶ die "'$checkfile' is a directory" if(-d $checkfile);
Branch: 1 branch
BranchTrueFalseTotal
if -d $checkfile 0 0 0
661 0 ▶ open(my $fh, '<', $checkfile) or die "cannot read '$checkfile' $!";
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, "<", $checkfile or die "cannot read '${checkfile}' $!" 0 0 0
662 0 my @list = map({chomp; $_} <$fh>);
663  
664 0 ▶ return() if(@list and $list[-1] and $list[-1] =~ m/^\d+$/);
Branch: 1 branch
BranchTrueFalseTotal
if @list and $list[-1] and $list[-1] =~ /^\d+$/ 0 0 0
Condition0%0/6
@list and $list[-1] and $list[-1] =~ /^\d+$/
!ll&&!rl&&r
000
@list and $list[-1]
!ll&&!rl&&r
000
MC/DC0%0/3
@list and $list[-1] and $list[-1] =~ /^\d+$/
@list $list[-1] $list[-1] =~ /^\d+$/
Truth table0%0/4
A and B and C
A= @listB= $list[-1]C= $list[-1] =~ /^\d+$/
ABCresult
0XX0
10X0
1100
1111
665  
666 0 my $dir = File::Basename::dirname(File::Basename::dirname($checkfile));
667 0 return([map({"$dir/$_"} @list)]);
668 }
669  
670 0 ▶sub _date_dwim {
Subroutine _date_dwim: not called CC=3 SCAR=24.8
671 0 my (@in) = @_;
672  
673 0 my @dates;
674 0 ▶ while(@in) {
Branch: 1 branch
BranchTrueFalseTotal
if (@in) 0 0 0
675 0 my $date = shift(@in);
676  
677 0 ▶ if($date eq 'thru') {
Branch: 1 branch
BranchTrueFalseTotal
if ($date eq "thru") 0 0 0
678 0 push(@dates, date(pop(@dates))->thru(date(shift(@in))));
679 0 next;
680 }
681 0 push(@dates, $date);
682 }
683 0 return(@dates);
684 }
685  
686 =for doc ###############################################################
687 Given a date range, verify that all files + hours for that server are
688 done (with the exception of those listed in the .MIA file.)
689  
690 =cut
691  
692 0 ▶sub do_verify {
Subroutine do_verify: not called CC=5 SCAR=28.4
693 0 my ($opt, @in) = @_;
694  
695 0 ▶ my @dates = _date_dwim(@in) or die "you gave no dates";
Branch: 1 branch
BranchTrueFalseTotal
my(@dates) = _date_dwim(@in) or die "you gave no dates" 0 0 0
696 0 ▶ foreach my $dir (glob('*')) {
Subroutine BEGIN: called CC=1 SCAR=0.2
697 0 ▶ (-d $dir) or next;
Branch: 1 branch
BranchTrueFalseTotal
-d $dir or next 0 0 0
698 0 foreach my $date (@dates) {
699 0 my @got = glob("$dir/$date*");
700 0 print "$dir/$date ", scalar(@got), "\n";
701 }
702 }
703 }
704  
705 =for doc ###############################################################
706 Make sure that all files are claimed somewhere. This is useful when a
707 load-in crashed.
708  
709 loghack confirm *
710  
711 =cut
712  
713 0 ▶sub do_confirm {
Subroutine do_confirm: not called CC=5 SCAR=34.0
714 0 my ($opt, @dirs) = @_;
715  
716 0 foreach my $dir (@dirs) {
717 0 ▶ my %loaded = map({$_ => 1} sub {
Subroutine __ANON__: not called CC=6 SCAR=37.4
718 0 my ($s_dir) = @_;
719 0 $s_dir .= '/.loaded';
720 0 ▶ -d $s_dir or return();
Branch: 1 branch
BranchTrueFalseTotal
-d $s_dir or return () 0 0 0
721 0 ▶ opendir(my $dh, $s_dir) or die "cannot opendir '$s_dir' $!";
Branch: 1 branch
BranchTrueFalseTotal
opendir my $dh, $s_dir or die "cannot opendir '${s_dir}' $!" 0 0 0
722 0 my @ans;
723 0 foreach my $name (grep({$_ !~ m/^\./} readdir($dh))) {
724 0 my $file = "$s_dir/$name";
725 0 ▶ open(my $fh, '<', $file) or die "cannot read '$file' $!";
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, "<", $file or die "cannot read '${file}' $!" 0 0 0
726 0 my @list = map({chomp; $_} <$fh>);
727 0 ▶ pop(@list) if($list[-1] =~ m/^\d+$/);
Branch: 1 branch
BranchTrueFalseTotal
if $list[-1] =~ /^\d+$/ 0 0 0
728 0 push(@ans, @list);
729 }
730 0 return(@ans);
731 0 }->($dir)
732 );
733 0 $dir =~ s#/*$#/#;
734 0 ▶ opendir(my $dh, $dir) or die "cannot opendir '$dir' $!";
Branch: 1 branch
BranchTrueFalseTotal
opendir my $dh, $dir or die "cannot opendir '${dir}' $!" 0 0 0
735 0 foreach my $name (grep({$_ !~ m/^\./} readdir($dh))) {
736 0 ▶ unless($loaded{$name}) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($loaded{$name}) 0 0 0
737 0 print "$dir$name\n";
738 }
739 }
740 }
741 # TODO exit with error?
742 }
743  
744 =head2 list
745  
746 List files in the repository.
747  
748 loghack list 2008-01-01 thru 2008-01-31 in *
749  
750 =cut
751  
752 0 ▶sub do_list {
Subroutine do_list: not called CC=1 SCAR=6.9
753 0 my ($opt, @files) = @_;
754  
755 0 @files = repo_files($opt, @files);
756 0 print join("\n", @files), "\n";
757 }
758  
759 # TODO repo_hash as groups of 'in' bits
760  
761 # note: also parses stuff like '2007-10-12 in foo bar baz'
762  
763 0 ▶sub repo_files {
Subroutine repo_files: not called CC=15 SCAR=54.8
764 0 my ($opt, @files) = @_;
765  
766 0 ▶ my $repo = $opt->{repository} or die "must have repository";
Branch: 1 branch
BranchTrueFalseTotal
my $repo = $opt->{"repository"} or die "must have repository" 0 0 0
767  
768 0 my $spec_re = qr/(?:^\d{4}-\d{2}-\d{2}$)|\*/;
769 0 ▶ return(@files)
Branch: 1 branch
BranchTrueFalseTotal
unless $files[0] =~ /$spec_re/ and not -f $files[0] 0 0 0
Condition0%0/3
$files[0] =~ /$spec_re/ and not -f $files[0]
!ll&&!rl&&r
000
MC/DC0%0/2
$files[0] =~ /$spec_re/ and not -f $files[0]
$files[0] =~ /$spec_re/ not -f $files[0]
Truth table0%0/3
A and not(B)
A= $files[0] =~ /$spec_re/B= not -f $files[0]
ABresult
0X0
100
111
770 unless($files[0] =~ $spec_re and not -f $files[0]);
771  
772 0 my @dates;
773 0 ▶ while(@files) {
Branch: 1 branch
BranchTrueFalseTotal
if (@files) 0 0 0
774 0 my $date = shift(@files);
775 0 ▶ if($date eq 'in') {
Branch: 2 branches
BranchTrueFalseTotal
if ($date eq 'in') { } 0 0 0
elsif ($date eq 'thru') { } 0 0 0
776 0 last;
777 }
778 elsif($date eq 'thru') {
779 0 push(@dates, date(pop(@dates))->thru(date(shift(@files))));
780 0 next;
781 }
782 0 ▶ $date =~ $spec_re or die "$date doesn't look like a date";
Branch: 1 branch
BranchTrueFalseTotal
$date =~ /$spec_re/ or die "$date doesn't look like a date" 0 0 0
783 0 push(@dates, $date);
784 }
785  
786 # TODO need an iterator for this
787 0 my @globs;
788 0 ▶ if(my @dirs = @files) {
Branch: 1 branch
BranchTrueFalseTotal
if (my(@dirs) = @files) { } 0 0 0
789 0 ▶ (-d $_) or die "$_ not a dir" for(@dirs);
Branch: 1 branch
BranchTrueFalseTotal
-d $_ or die "$_ not a dir" 0 0 0
790 0 @globs = map({my $d = $_; map({"$repo/$_/$d.*"} @dirs)} @dates);
791 }
792 else {
793 0 @globs = map({"$repo/$_"} @dates);
794 }
795 0 ▶ $opt->{lazy_glob} and return(@globs);
Branch: 1 branch
BranchTrueFalseTotal
$opt->{"lazy_glob"} and return @globs 0 0 0
796 0 @files = ();
797 0 foreach my $glob (@globs) {
798 0 my @got = glob($glob);
799 0 ▶ (-f $_) or die "$_ is not a file" for(@got);
Branch: 1 branch
BranchTrueFalseTotal
-f $_ or die "$_ is not a file" 0 0 0
800 0 push(@files, @got);
801 }
802 0 return(@files);
803 }
804  
805 =head2 compile
806  
807 Assemble reports into daily chunks (in the .compiled/ directory.)
808  
809 loghack compile 2007-10-01
810  
811 =cut
812  
813 0 ▶sub do_compile {
Subroutine do_compile: not called CC=2 SCAR=17.9
814 0 my ($opt, @dates) = @_;
815 0 foreach my $date (@dates) {
816 0 do_aggregate($opt, 'compile', $date);
817 }
818 }
819  
820 =head2 aggregate
821  
822 Build aggregate reports.
823  
824 loghack aggregate month $start_date
825  
826 loghack aggregate week $start_date
827  
828 =cut
829  
830 0 ▶sub do_aggregate {
Subroutine do_aggregate: not called CC=10 SCAR=47.0
831 0 my ($opt, @spec) = @_;
832  
833 0 require ApacheLog::Parser::Report;
834  
835 0 my ($type, $date) = @spec;
836 0 my $name;
837 my @files;
838  
839 $opt->{quiet} or
840 0 ▶ printf("$date -- %02d:%02d:%02d\n", (localtime)[2,1,0]);
Branch: 1 branch
BranchTrueFalseTotal
$opt->{"quiet"} or printf "$date -- %02d:%02d:%02d\n", (localtime)[2, 1, 0] 0 0 0
841 0 ▶ if($type eq 'compile') {
Branch: 1 branch
BranchTrueFalseTotal
if ($type eq 'compile') { } 0 0 0
842 0 @files = repo_files($opt, $date, 'in', glob('*'));
843 0 want_dir($opt->{repository} . '/.compiled');
844 0 $name = '.compiled/' . $date;
845 }
846 else {
847 my %disp = (
848 0 ▶ month => sub {shift(@_)->end_of_month},
Subroutine __ANON__: not called CC=1 SCAR=6.9
849 0 ▶ week => sub {shift(@_)+6},
Subroutine __ANON__: not called CC=1 SCAR=6.9
850 0 );
851 0 $date = Date::Piece->new($date);
852 0 ▶ my $do = $disp{$type} or
Branch: 1 branch
BranchTrueFalseTotal
my $do = $disp{$type} or die "'${type}' must be one of ", join(", ", keys %disp) 0 0 0
853 die "'$type' must be one of ", join(', ', keys(%disp));
854 0 @files = map({$opt->{repository} . '/.compiled/' . $_ . '.yml'}
855 $date->thru($do->($date))
856 );
857 0 want_dir($opt->{repository} . '/.aggregate');
858 0 $name = ".aggregate/$type.$date";
859 }
860  
861 0 my $r_config = $opt->{repository} . '/.config/report.conf';
862 0 ▶ (-e $r_config) or die "must have a report config file $r_config";
Branch: 1 branch
BranchTrueFalseTotal
-e $r_config or die "must have a report config file $r_config" 0 0 0
863 0 my $rep = ApacheLog::Parser::Report->new(
864 conf => YAML::LoadFile($r_config)
865 );
866  
867 0 require YAML::Syck;
868  
869 0 foreach my $file (@files) {
870 0 ▶ my $report_file = ($file =~ m/\.yml$/) ?
Branch: 1 branch
BranchTrueFalseTotal
$file =~ /\.yml$/ ? : 0 0 0
871 $file : report_filename($opt, $file);
872 # warn "load $report_file\n";
873 0 my $data = YAML::Syck::LoadFile($report_file);
874 0 $rep->aggregate($data);
875 }
876  
877 # and save them
878 0 my $ag_name = $opt->{repository} . '/' . $name . '.yml';
879  
880 0 ▶ if($type eq 'compile') {
Branch: 1 branch
BranchTrueFalseTotal
if ($type eq "compile") 0 0 0
881 0 $rep->write_report($ag_name);
882 0 return;
883 }
884  
885 0 my ($text, $yaml) = $rep->print_report;
886  
887 {
888 0 ▶ open(my $ofh, '>', $ag_name) or
Branch: 1 branch
BranchTrueFalseTotal
open my $ofh, ">", $ag_name or die "cannot write $ag_name $!" 0 0 0
889 die "cannot write $ag_name $!";
890 0 print $ofh $yaml;
891 }
892 {
893 0 $ag_name =~ s/\.yml$/.txt/;
894 0 ▶ open(my $ofh, '>', $ag_name) or
Branch: 1 branch
BranchTrueFalseTotal
open my $ofh, ">", $ag_name or die "cannot write $ag_name $!" 0 0 0
895 die "cannot write $ag_name $!";
896 0 print $ofh $text;
897 }
898 }
899  
900 =head2 tabulate
901  
902 loghack tabulate daily 2007-10-01 thru 2007-10-31
903  
904 =cut
905  
906 0 ▶sub do_tabulate {
Subroutine do_tabulate: not called CC=15 SCAR=54.8
907 0 my ($opt, @list) = @_;
908  
909 0 ▶ my $daily = ($list[0] eq 'daily') ? shift(@list) : 0;
Branch: 1 branch
BranchTrueFalseTotal
$list[0] eq 'daily' ? : 0 0 0
910  
911 0 require ApacheLog::Parser::Report;
912  
913 0 my $outname;
914 my @files;
915 0 ▶ if($daily) {
Branch: 1 branch
BranchTrueFalseTotal
if ($daily) { } 0 0 0
916 0 ▶ if(@list == 3 and $list[1] eq 'thru') {
Branch: 1 branch
BranchTrueFalseTotal
if (@list == 3 and $list[1] eq "thru") 0 0 0
Condition0%0/3
@list == 3 and $list[1] eq "thru"
!ll&&!rl&&r
000
MC/DC0%0/2
@list == 3 and $list[1] eq "thru"
@list == 3 $list[1] eq "thru"
Truth table0%0/3
A and B
A= @list == 3B= $list[1] eq "thru"
ABresult
0X0
100
111
917 0 $outname = join('_', @list[0,2]);
918 0 ▶ $opt->{quiet} or print "$outname\n";
Branch: 1 branch
BranchTrueFalseTotal
$opt->{"quiet"} or print "$outname\n" 0 0 0
919 }
920 0 @list = sort(_date_dwim(@list));
921  
922 # TODO something to allow this-month-so-far
923 my %spec = map({
924 0 $_ => [
925 0 $opt->{repository} . '/.compiled/' . $_ . '.yml'
926 ]} @list);
927 0 @files = (\%spec, @list);
928 0 foreach my $file (map({@{$spec{$_}}} keys %spec)) {
929 0 ▶ (-e $file) or die "no such file: $file\n";
Branch: 1 branch
BranchTrueFalseTotal
-e $file or die "no such file: $file\n" 0 0 0
930 }
931 }
932 else {
933 0 @list = sort(@list);
934  
935 0 my $dir = $opt->{repository} . '/.aggregate/';
936 0 @files = map({$dir . $_ . '.yml'} @list);
937 0 foreach my $file (@files) {
938 0 ▶ (-e $file) or die "no such file: $file\n";
Branch: 1 branch
BranchTrueFalseTotal
-e $file or die "no such file: $file\n" 0 0 0
939 }
940 }
941  
942 0 my $r_config = $opt->{repository} . '/.config/report.conf';
943 0 ▶ (-e $r_config) or die "must have a report config file $r_config";
Branch: 1 branch
BranchTrueFalseTotal
-e $r_config or die "must have a report config file $r_config" 0 0 0
944 0 my $rep = ApacheLog::Parser::Report->new(
945 conf => YAML::LoadFile($r_config)
946 );
947  
948 0 my @table = $rep->table_report(@files);
949 0 unshift(@table, ['', @list]);
950 0 ▶ if($outname) {
Branch: 1 branch
BranchTrueFalseTotal
if ($outname) { } 0 0 0
951 0 my $t_dir = '.tables';
952 0 want_dir($t_dir);
953 0 my $file = "$t_dir/$outname.tsv";
954 0 ▶ open(my $fh, '>', $file) or die "cannot write '$file' $!";
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, ">", $file or die "cannot write '${file}' $!" 0 0 0
955 0 print $fh join("\n", map({join("\t", @$_)} @table)), "\n";
956 0 ▶ close($fh) or die "cannot write '$file' $!";
Branch: 1 branch
BranchTrueFalseTotal
close $fh or die "cannot write '${file}' $!" 0 0 0
957 0 ▶ $opt->{quiet} or print "wrote $file\n";
Branch: 1 branch
BranchTrueFalseTotal
$opt->{"quiet"} or print "wrote $file\n" 0 0 0
958 }
959 else {
960 0 print join("\n", map({join("\t", @$_)} @table)), "\n";
961 }
962 }
963  
964 =head2 report
965  
966 Crunch the prepared data and generate a report for the given chunk(s).
967  
968 loghack report $server/$chunk.tar.gz
969  
970 =cut
971  
972 0 ▶sub do_report {
Subroutine do_report: not called CC=11 SCAR=48.8
973 0 my ($opt, @files) = @_;
974  
975 0 require ApacheLog::Parser::Report;
976  
977 0 my $do_print = !$opt->{quiet};
978 0 ▶ my $show_status = sub {
Subroutine __ANON__: not called CC=2 SCAR=17.9
979 0 my ($status) = @_;
980 0 ▶ $do_print or return;
Branch: 1 branch
BranchTrueFalseTotal
$do_print or return 0 0 0
981 0 printf("$status -- %02d:%02d:%02d\n", (localtime)[2,1,0]);
982 0 };
983 0 ▶ if(@files == 1 and $files[0] =~ m/\*$/) {
Branch: 1 branch
BranchTrueFalseTotal
if (@files == 1 and $files[0] =~ /\*$/) 0 0 0
Condition0%0/3
@files == 1 and $files[0] =~ /\*$/
!ll&&!rl&&r
000
MC/DC0%0/2
@files == 1 and $files[0] =~ /\*$/
@files == 1 $files[0] =~ /\*$/
Truth table0%0/3
A and B
A= @files == 1B= $files[0] =~ /\*$/
ABresult
0X0
100
111
984 0 $show_status->($files[0]);
985 0 $do_print = 0;
986 }
987 0 @files = repo_files($opt, @files);
988 0 my $skipper = get_skipper($opt);
989  
990 0 my $r_config = $opt->{repository} . '/.config/report.conf';
991 0 ▶ (-e $r_config) or die "must have a report config file $r_config";
Branch: 1 branch
BranchTrueFalseTotal
-e $r_config or die "must have a report config file $r_config" 0 0 0
992  
993 0 foreach my $file (@files) {
994 0 my $fh = open_file($file);
995  
996 0 my $rep = ApacheLog::Parser::Report->new(
997 conf => YAML::LoadFile($r_config)
998 );
999 0 my $rfunc = $rep->get_func;
1000  
1001 0 my $report_file = report_filename($opt, $file);
1002 0 ▶ if(-e $report_file) {
Branch: 1 branch
BranchTrueFalseTotal
if (-e $report_file) 0 0 0
1003 # TODO unless force or check staleness or something
1004 0 warn "skip (got) $file\n";
1005 0 next;
1006 }
1007  
1008 0 $show_status->($file);
1009  
1010 0 my $sr;
1011 0 ▶ if($skipper) {
Branch: 1 branch
BranchTrueFalseTotal
if ($skipper) 0 0 0
1012 0 my $skipfile = skipfilename($opt, $file);
1013 0 ▶ (-e $skipfile) or die "missing $skipfile";
Branch: 1 branch
BranchTrueFalseTotal
-e $skipfile or die "missing $skipfile" 0 0 0
1014 0 $sr = $skipper->new_reader($skipfile);
1015 }
1016 0 ▶ my $skip = defined($sr) ? $sr->next_skip : 0;
Branch: 1 branch
BranchTrueFalseTotal
defined $sr ? : 0 0 0
1017  
1018 0 my $lnum = 0;
1019 0 ▶ while(my $line = <$fh>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $fh)) 0 0 0
1020 0 $lnum++;
1021 0 ▶ if($lnum == $skip) {
Branch: 1 branch
BranchTrueFalseTotal
if ($lnum == $skip) 0 0 0
1022 0 $lnum += $sr->skip_lines($fh);
1023 0 $skip = $sr->next_skip;
1024 0 next;
1025 }
1026  
1027 0 my @v = split(/\t/, $line);
1028 # now for some reporting
1029 0 $rfunc->(\@v);
1030 #if($lnum > 100000) { warn "exit hack"; last;}
1031 }
1032 0 $rep->write_report($report_file);
1033 }
1034  
1035 }
1036  
1037 =head2 unique
1038  
1039 Experimental: count/report unique visitors within a chunk.
1040  
1041 =cut
1042  
1043 0 ▶sub do_unique {
Subroutine do_unique: not called CC=6 SCAR=37.4
1044 0 my ($opt, @files) = @_;
1045  
1046 0 my $do_print = !$opt->{quiet};
1047 0 ▶ my $show_status = sub {
Subroutine __ANON__: not called CC=2 SCAR=17.9
1048 0 my ($status) = @_;
1049 0 ▶ $do_print or return;
Branch: 1 branch
BranchTrueFalseTotal
$do_print or return 0 0 0
1050 0 printf("$status -- %02d:%02d:%02d\n", (localtime)[2,1,0]);
1051 0 };
1052 0 ▶ if(@files == 1 and $files[0] =~ m/\*$/) {
Branch: 1 branch
BranchTrueFalseTotal
if (@files == 1 and $files[0] =~ /\*$/) 0 0 0
Condition0%0/3
@files == 1 and $files[0] =~ /\*$/
!ll&&!rl&&r
000
MC/DC0%0/2
@files == 1 and $files[0] =~ /\*$/
@files == 1 $files[0] =~ /\*$/
Truth table0%0/3
A and B
A= @files == 1B= $files[0] =~ /\*$/
ABresult
0X0
100
111
1053 0 $show_status->($files[0]);
1054 0 $do_print = 0;
1055 }
1056 0 @files = repo_files($opt, @files);
1057  
1058 0 foreach my $file (@files) {
1059 0 my $fh = open_file($file);
1060 0 $show_status->($file);
1061  
1062 0 my %unique;
1063 0 ▶ while(my $line = <$fh>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $fh)) 0 0 0
1064 0 my ($ip, $rest) = split(/\t/, $line);
1065 0 ▶ ($unique{$ip}||= 0)++;
Condition0%0/2
$unique{$ip} ||= 0
l!l
00
MC/DC0%0/1
$unique{$ip} ||= 0
$unique{$ip}
Truth table0%0/2
A
A= $unique{$ip}
Aresult
11
00
1066 }
1067  
1068 0 my $u_file = uniques_filename($opt, $file);
1069 0 want_dir(File::Basename::dirname($u_file));
1070 0 my $ofh = pipe_out($u_file);
1071 0 print $ofh map({"$_\t$unique{$_}\n"} sort keys %unique);
1072 }
1073 }
1074  
1075 =head2 day_unique
1076  
1077 Experimental: count/report unique visitors within a day.
1078  
1079 =cut
1080  
1081 0 ▶sub do_day_unique {
Subroutine do_day_unique: not called CC=6 SCAR=37.4
1082 0 my ($opt, @dates) = @_;
1083  
1084 0 want_dir($opt->{repository} . '/.day_uniques');
1085 0 foreach my $date (@dates) {
1086 $opt->{quiet} or
1087 0 ▶ printf("$date -- %02d:%02d:%02d\n", (localtime)[2,1,0]);
Branch: 1 branch
BranchTrueFalseTotal
$opt->{"quiet"} or printf "$date -- %02d:%02d:%02d\n", (localtime)[2, 1, 0] 0 0 0
1088 0 my @files = repo_files($opt, $date, 'in', glob('*'));
1089 0 my %unique;
1090 0 foreach my $file (@files) {
1091 0 my $u_file = uniques_filename($opt, $file);
1092 0 my $fh = open_file($u_file);
1093 0 ▶ while(my $line = <$fh>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $fh)) 0 0 0
1094 0 chomp($line);
1095 0 my ($ip, $count) = split(/\t/, $line, 2);
1096 0 ▶ ($unique{$ip}||=0)+= $count;
Condition0%0/2
$unique{$ip} ||= 0
l!l
00
MC/DC0%0/1
$unique{$ip} ||= 0
$unique{$ip}
Truth table0%0/2
A
A= $unique{$ip}
Aresult
11
00
1097 }
1098 }
1099 0 my $outfile = day_uniques_filename($opt, $date);
1100 0 my $ofh = pipe_out($outfile);
1101 0 print $ofh map({"$_\t$unique{$_}\n"} sort keys %unique);
1102 }
1103 }
1104  
1105 # here we run out of memory, so need a piecewise algorithm
1106  
1107 =head2 month_unique
1108  
1109 Experimental: count/report unique visitors within a month.
1110  
1111 =cut
1112  
1113 0 ▶sub do_month_unique {
Subroutine do_month_unique: not called CC=11 SCAR=48.8
1114 0 my ($opt, $month) = @_;
1115  
1116 0 my $out_dir = $opt->{repository} . '/.month_uniques';
1117 0 want_dir($out_dir);
1118 0 my $source_dir = $opt->{repository} . '/.day_uniques';
1119  
1120 $opt->{quiet} or
1121 0 ▶ printf("$month -- %02d:%02d:%02d\n", (localtime)[2,1,0]);
Branch: 1 branch
BranchTrueFalseTotal
$opt->{"quiet"} or printf "$month -- %02d:%02d:%02d\n", (localtime)[2, 1, 0] 0 0 0
1122 0 my @work;
1123 0 ▶ my $fill = sub {
Subroutine __ANON__: not called CC=2 SCAR=17.9
1124 0 my ($index) = @_;
1125 0 my $val;
1126 0 ▶ unless(defined($val = readline($work[$index][2]))) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined($val = readline $work[$index][2])) 0 0 0
1127 0 splice(@work, $index, 1);
1128 0 return;
1129 }
1130 0 chomp($val);
1131 0 @{$work[$index]}[0,1] = split(/\t/, $val, 2);
1132 0 };
1133 0 foreach my $file (glob("$source_dir/$month-*.gz")) {
1134 0 my $fh = open_file($file);
1135 0 push(@work, ['', 0, $fh]);
1136 0 $fill->($#work);
1137 }
1138  
1139 0 my $outfile = $out_dir . '/' . $month . '.gz';
1140 0 my $ofh = pipe_out($outfile);
1141  
1142 0 my $ucount = 0;
1143 0 my $outc = 0;
1144 0 ▶ while(@work) {
Branch: 1 branch
BranchTrueFalseTotal
if (@work) 0 0 0
1145 0 ▶ (++$ucount % 1_000_000) or do {
Branch: 1 branch
BranchTrueFalseTotal
++$ucount % 1000000 or do { $opt->{"quiet"} or printf "$ucount ($outc) -- %02d:%02d:%02d\n", (localtime)[2, 1, 0] } 0 0 0
1146 $opt->{quiet} or
1147 0 ▶ printf("$ucount ($outc) -- %02d:%02d:%02d\n", (localtime)[2,1,0])
Branch: 1 branch
BranchTrueFalseTotal
$opt->{"quiet"} or printf "$ucount ($outc) -- %02d:%02d:%02d\n", (localtime)[2, 1, 0] 0 0 0
1148 };
1149 0 my @l = sort({$work[$a][0] cmp $work[$b][0]} 0..$#work);
1150 0 my @o = (0);
1151 0 my $ip = $work[$l[0]][0];
1152 0 for(1..$#l) {
1153 0 ▶ ($ip eq $work[$l[$_]][0]) or last;
Branch: 1 branch
BranchTrueFalseTotal
$ip eq $work[$l[$_]][0] or last 0 0 0
1154 0 push(@o, $_);
1155 }
1156  
1157 0 my $count = 0;
1158 0 $count += $work[$l[$_]][1] for(@o);
1159  
1160 # have to be careful about stale indices in removal
1161 0 $fill->($_) for(sort({$b <=> $a} @l[@o]));
1162 0 $outc += scalar(@o);
1163 0 print $ofh "$ip\t$count\n";
1164 }
1165  
1166 # and write the count
1167 {
1168 0 my $cfile = $out_dir . '/' . $month . '.count';
1169 0 ▶ open(my $out, '>', $cfile) or die "cannot write '$cfile' $!";
Branch: 1 branch
BranchTrueFalseTotal
open my $out, ">", $cfile or die "cannot write '${cfile}' $!" 0 0 0
1170 0 print $out $ucount, "\n";
1171 0 print "$month -- $ucount\n";
1172 }
1173 }
1174  
1175 =head2 month_unique2
1176  
1177 Experimental: count/report unique visitors within a month (alternate,
1178 memory-hungry algorithm.)
1179  
1180 =cut
1181  
1182 0 ▶sub do_month_unique2 {
Subroutine do_month_unique2: not called CC=6 SCAR=37.4
1183 0 my ($opt, $month) = @_;
1184  
1185 0 my $out_dir = $opt->{repository} . '/.month_uniques';
1186 0 want_dir($out_dir);
1187 0 my $source_dir = $opt->{repository} . '/.day_uniques';
1188 0 my %unique;
1189 0 my $ucount = 0;
1190 0 foreach my $file (glob("$source_dir/$month-*.gz")) {
1191 0 my $fh = open_file($file);
1192 $opt->{quiet} or
1193 0 ▶ printf("$file -- %02d:%02d:%02d\n", (localtime)[2,1,0]);
Branch: 1 branch
BranchTrueFalseTotal
$opt->{"quiet"} or printf "$file -- %02d:%02d:%02d\n", (localtime)[2, 1, 0] 0 0 0
1194 0 ▶ while(my $line = <$fh>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $fh)) 0 0 0
1195 0 chomp($line);
1196 0 my ($ip, $count) = split(/\t/, $line, 2);
1197 0 ▶ unless($unique{$ip}) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($unique{$ip}) 0 0 0
1198 0 $unique{$ip} = 1;
1199 0 $ucount++;
1200 }
1201 }
1202 }
1203 0 my $outfile = $out_dir . '/' . $month . '.count';
1204 0 ▶ open(my $ofh, '>', $outfile) or
Branch: 1 branch
BranchTrueFalseTotal
open my $ofh, ">", $outfile or die "cannot write '${outfile}' $!" 0 0 0
1205 die "cannot write '$outfile' $!";
1206 0 print $ofh $ucount, "\n";
1207 0 print $ucount, "\n";
1208 }
1209  
1210 =head2 makelinks
1211  
1212 Create hardlinks with dated names.
1213  
1214 =cut
1215  
1216 0 ▶sub do_makelinks { # TODO --delete option?
Subroutine do_makelinks: not called CC=10 SCAR=47.0
1217 0 my ($opt, $ldir, @files) = @_;
1218  
1219 0 ▶ if(-e $ldir) {
Branch: 1 branch
BranchTrueFalseTotal
if (-e $ldir) 0 0 0
1220 0 ▶ (-d $ldir) or die "USAGE: makelinks <dir> <files>\n";
Branch: 1 branch
BranchTrueFalseTotal
-d $ldir or die "USAGE: makelinks <dir> <files>\n" 0 0 0
1221 }
1222 0 ▶ unless(-d $ldir) {
Branch: 1 branch
BranchTrueFalseTotal
unless (-d $ldir) 0 0 0
1223 0 ▶ mkdir($ldir) or die "cannot create $ldir $!";
Branch: 1 branch
BranchTrueFalseTotal
mkdir $ldir or die "cannot create $ldir $!" 0 0 0
1224 }
1225  
1226 0 foreach my $file (@files) {
1227 0 my $date = get_date($file);
1228  
1229 # XXX bah
1230 0 my $base = basename($file);
1231 0 my $dir = basename(dirname($file));
1232 0 ▶ my $dest = (-d "$ldir/$dir" ? "$ldir/$dir" : $ldir);
Branch: 1 branch
BranchTrueFalseTotal
-d "$ldir/$dir" ? : 0 0 0
1233 0 my $ext;
1234 0 ▶ if($base =~ s/(?:\.\d+)?(\.(?:gz|bz2))?$//) {
Branch: 1 branch
BranchTrueFalseTotal
if ($base =~ s/(?:\.\d+)?(\.(?:gz|bz2))?$//) 0 0 0
1235 0 ▶ $ext = $1 || '';
Condition0%0/2
$1 || ''
l!l
00
MC/DC0%0/1
$1 || ''
$1
Truth table0%0/2
A
A= $1
Aresult
11
00
1236 }
1237 0 $dest .= '/' . "$base.$date$ext";
1238  
1239 0 ▶ link($file, $dest) or die "cannot create link $!";
Branch: 1 branch
BranchTrueFalseTotal
link $file, $dest or die "cannot create link $!" 0 0 0
1240 }
1241 }
1242  
1243 =head2 import
1244  
1245 Run the prep, report, compile, and aggregate actions (nice for automatic
1246 daily imports.)
1247  
1248 loghack import $file1 $file2 ...
1249  
1250 =cut
1251  
1252 0 ▶sub do_import {
Subroutine do_import: not called CC=6 SCAR=37.4
1253 0 my ($opt, @files) = @_;
1254  
1255 0 my @loaded = do_prep($opt, @files);
1256  
1257 0 ▶ @loaded or die "imported nothing";
Branch: 1 branch
BranchTrueFalseTotal
@loaded or die "imported nothing" 0 0 0
1258 0 do_report($opt, @loaded);
1259  
1260 0 my @dates = do {
1261 0 my %dates = map({m#.*/(\d{4}-\d{2}-\d{2})\..*#; ($1 => 1)} @loaded);
1262 0 sort(keys(%dates));
1263 };
1264  
1265 0 do_compile($opt, @dates);
1266  
1267 0 my @actions;
1268 0 foreach my $date (@dates) {
1269 0 $date = date($date);
1270  
1271 # TODO tabulate daily $date, 'thru', 'latest'
1272  
1273 # note: these triggers might look strange (a day late), but we don't
1274 # know we have all of last week's/month's data until we see some
1275 # trickle of the new week/month in the input -- if logrotate happens
1276 # after midnight, this will be only the first few minutes of the
1277 # week/month (though weekly logrotate with no interim updates will
1278 # definitely delay that on a slow site.)
1279  
1280 # TODO may need to retabulate farther back in slow-site cases
1281 # (except that should trigger by any bit of the trigger date being
1282 # in the new file?)
1283  
1284 0 ▶ if($date->day == 1) {
Branch: 1 branch
BranchTrueFalseTotal
if ($date->day == 1) 0 0 0
1285 0 my $last_month = $date-1*months;
1286 0 push(@actions, ['aggregate', 'month', $last_month]);
1287 0 push(@actions, ['tabulate', 'daily',
1288 $last_month, 'thru', $last_month->end_of_month]
1289 );
1290 }
1291 0 ▶ if($date->iso_wday == 1) {
Branch: 1 branch
BranchTrueFalseTotal
if ($date->iso_wday == 1) 0 0 0
1292 0 push(@actions, ['aggregate', 'week', $date - 1*weeks]);
1293 }
1294 }
1295 0 foreach my $action (@actions) {
1296 0 my @do = @$action;
1297 #warn "run @do\n";
1298 0 my $method = shift(@do);
1299 0 my $ref = __PACKAGE__->can('do_'.$method); # XXX go oo already
1300 0 $ref->($opt, @do);
1301 }
1302 }
1303  
1304 0 ▶sub repository_path {
Subroutine repository_path: not called CC=3 SCAR=24.8
1305 0 my ($opt, $file) = @_;
1306  
1307 0 ▶ my $repo = $opt->{repository} or die "need repository name";
Branch: 1 branch
BranchTrueFalseTotal
my $repo = $opt->{"repository"} or die "need repository name" 0 0 0
1308 0 $repo =~ s#/*$##;
1309  
1310 # TODO we need to sort-out this server bit
1311 0 my $dir = basename(dirname($file));
1312 0 ▶ my $dest = (-d "$repo/$dir" ? "$repo/$dir" : $repo);
Branch: 1 branch
BranchTrueFalseTotal
-d "$repo/$dir" ? : 0 0 0
1313 0 return("$dest/");
1314 }
1315 0 ▶sub skipfilename {
Subroutine skipfilename: not called CC=5 SCAR=34.0
1316 0 my ($opt, $file) = @_;
1317  
1318 0 ▶ my $repo = $opt->{repository} or die "need repository name";
Branch: 1 branch
BranchTrueFalseTotal
my $repo = $opt->{"repository"} or die "need repository name" 0 0 0
1319 0 $repo =~ s#/*$##;
1320  
1321 0 my $base = basename($file);
1322 0 my $dest = dirname($file);
1323 0 $base =~ s/\.(tsv\.gz|gz|bz2)$//;
1324 0 $dest .= '/.skipdir/';
1325 0 ▶ unless(-d $dest) {
Branch: 1 branch
BranchTrueFalseTotal
unless (-d $dest) 0 0 0
1326 0 ▶ unless(mkdir($dest)) {
Branch: 1 branch
BranchTrueFalseTotal
unless (mkdir $dest) 0 0 0
1327 0 ▶ die "cannot create $dest dir $!" unless(-d $dest);
Branch: 1 branch
BranchTrueFalseTotal
unless -d $dest 0 0 0
1328 }
1329 }
1330 0 $dest .= $base . '.skip';
1331 0 return($dest);
1332 }
1333 0 ▶sub report_filename {
Subroutine report_filename: not called CC=5 SCAR=34.0
1334 0 my ($opt, $file) = @_;
1335 0 ▶ my $repo = $opt->{repository} or die "need repository name";
Branch: 1 branch
BranchTrueFalseTotal
my $repo = $opt->{"repository"} or die "need repository name" 0 0 0
1336 0 $repo =~ s#/*$##;
1337  
1338 0 my $base = basename($file);
1339 0 my $dir = basename(dirname($file));
1340 0 ▶ my $dest = (-d "$repo/$dir" ? "$repo/$dir" : $repo);
Branch: 1 branch
BranchTrueFalseTotal
-d "$repo/$dir" ? : 0 0 0
1341 0 $base =~ s/\.(tsv\.gz|gz|bz2)$//;
1342 0 $dest .= '/.reports/';
1343 0 ▶ unless(-d $dest) {
Branch: 1 branch
BranchTrueFalseTotal
unless (-d $dest) 0 0 0
1344 0 ▶ mkdir($dest) or die "cannot create $dest dir $!";
Branch: 1 branch
BranchTrueFalseTotal
mkdir $dest or die "cannot create $dest dir $!" 0 0 0
1345 }
1346 0 $dest .= $base . '.yml';
1347 0 return($dest);
1348 }
1349  
1350 0 ▶sub inner_filename {
Subroutine inner_filename: not called CC=5 SCAR=34.0
1351 0 my ($opt, $file, $indir, $ext) = @_;
1352 0 ▶ my $repo = $opt->{repository} or die "need repository name";
Branch: 1 branch
BranchTrueFalseTotal
my $repo = $opt->{"repository"} or die "need repository name" 0 0 0
1353 0 $repo =~ s#/*$##;
1354  
1355 0 my $base = basename($file);
1356 0 my $dir = basename(dirname($file));
1357 0 ▶ my $dest = (-d "$repo/$dir" ? "$repo/$dir" : $repo);
Branch: 1 branch
BranchTrueFalseTotal
-d "$repo/$dir" ? : 0 0 0
1358 0 $base =~ s/\.(tsv\.gz|gz|bz2)$//;
1359 0 $dest .= "/$indir/";
1360 0 ▶ unless(-d $dest) {
Branch: 1 branch
BranchTrueFalseTotal
unless (-d $dest) 0 0 0
1361 0 ▶ mkdir($dest) or die "cannot create $dest dir $!";
Branch: 1 branch
BranchTrueFalseTotal
mkdir $dest or die "cannot create $dest dir $!" 0 0 0
1362 }
1363 0 $dest .= $base . $ext;
1364 0 return($dest);
1365 }
1366 0 ▶sub uniques_filename {
Subroutine uniques_filename: not called CC=1 SCAR=6.9
1367 0 my ($opt, $file) = @_;
1368 0 inner_filename($opt, $file, '.uniques', '.gz');
1369 }
1370 0 ▶sub day_uniques_filename {
Subroutine day_uniques_filename: not called CC=1 SCAR=6.9
1371 0 my ($opt, $day) = @_;
1372 # TODO outer_filename ?
1373 0 $opt->{repository} . '/' . '.day_uniques/' . $day . '.gz';
1374 }
1375  
1376 0 ▶sub get_skipper {
Subroutine get_skipper: not called CC=4 SCAR=30.0
1377 0 my ($opt) = @_;
1378  
1379 0 ▶ my $repo = $opt->{repository} or die "need repository";
Branch: 1 branch
BranchTrueFalseTotal
my $repo = $opt->{"repository"} or die "need repository" 0 0 0
1380 0 ▶ $opt->{skip} or return();
Branch: 1 branch
BranchTrueFalseTotal
$opt->{"skip"} or return () 0 0 0
1381  
1382 0 my $skipper;
1383 0 ▶ if(-e (my $skipfile = "$repo/.config/skips.conf")) {
Branch: 1 branch
BranchTrueFalseTotal
if (-e (my $skipfile = "$repo/.config/skips.conf")) 0 0 0
1384 0 my ($skip) = YAML::LoadFile($skipfile);
1385 0 $skipper = ApacheLog::Parser::SkipList->new();
1386 0 $skipper->set_config($skip);
1387 }
1388 0 return($skipper);
1389 }
1390  
1391 =head2 count
1392  
1393 Count the records in a given chunk (accounting for skiplist.)
1394  
1395 =cut
1396  
1397 0 ▶sub do_count {
Subroutine do_count: not called CC=5 SCAR=34.0
1398 0 my ($opt, @files) = @_;
1399  
1400 0 @files = repo_files($opt, @files);
1401  
1402 0 my $skipper = get_skipper($opt);
1403 0 ▶ $skipper or die "you just want cat?";
Branch: 1 branch
BranchTrueFalseTotal
$skipper or die "you just want cat?" 0 0 0
1404 0 foreach my $file (@files) {
1405 0 my $fh = open_file($file);
1406  
1407 0 my $skipfile = skipfilename($opt, $file);
1408 0 my $sr = $skipper->new_reader($skipfile);
1409 0 my $skip = $sr->next_skip;
1410  
1411 0 my $lnum = 0;
1412 0 my $real = 0;
1413 0 ▶ while(my $line = <$fh>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $fh)) 0 0 0
1414 0 ▶ if(++$lnum == $skip) {
Branch: 1 branch
BranchTrueFalseTotal
if (++$lnum == $skip) 0 0 0
1415 0 $lnum += $sr->skip_lines($fh);
1416 0 $skip = $sr->next_skip;
1417 0 next;
1418 }
1419 0 $real++;
1420 }
1421 0 print join("\t", nice_name($file), $real, $lnum), "\n";
1422 }
1423 }
1424  
1425 =head2 dump
1426  
1427 Dump the records in a given chunk (accounting for skiplist.)
1428  
1429 =cut
1430  
1431 0 ▶sub do_dump {
Subroutine do_dump: not called CC=5 SCAR=34.0
1432 0 my ($opt, @files) = @_;
1433  
1434 0 @files = repo_files($opt, @files);
1435  
1436 0 my $skipper = get_skipper($opt);
1437 0 ▶ $skipper or die "you just want cat?";
Branch: 1 branch
BranchTrueFalseTotal
$skipper or die "you just want cat?" 0 0 0
1438 0 foreach my $file (@files) {
1439 0 my $fh = open_file($file);
1440  
1441 0 my $skipfile = skipfilename($opt, $file);
1442 0 my $sr = $skipper->new_reader($skipfile);
1443 0 my $skip = $sr->next_skip;
1444  
1445 0 my $lnum = 0;
1446 0 ▶ while(my $line = <$fh>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $fh)) 0 0 0
1447 0 ▶ if(++$lnum == $skip) {
Branch: 1 branch
BranchTrueFalseTotal
if (++$lnum == $skip) 0 0 0
1448 0 $lnum += $sr->skip_lines($fh);
1449 0 $skip = $sr->next_skip;
1450 0 next;
1451 }
1452 0 print $line;
1453 }
1454 }
1455 }
1456  
1457 =head2 date
1458  
1459 Print a date for the first line in a raw logfile.
1460  
1461 date=$(loghack date logfile.gz)
1462  
1463 =cut
1464  
1465 0 ▶sub do_date {
Subroutine do_date: not called CC=1 SCAR=6.9
1466 0 my ($opt, $file) = @_;
1467 0 my $date = get_date($file);
1468 0 print $date, "\n";
1469 }
1470  
1471 # TODO put this is the Parser module?
1472 {
1473 1 my @abbr = qw( Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec );
1474 1 my %months = map({$abbr[$_] => sprintf("%02d", $_ + 1)} 0..11);
1475 0 ▶sub get_datestring {
Subroutine get_datestring: not called CC=1 SCAR=6.9
1476 0 my ($date) = @_;
1477 0 $date =~ s#^(\d+)/(\w+)/(\d+)$#"$3-".$months{$2}."-$1"#e;
1478 0 return($date);
1479 }
1480 0 ▶sub get_date {
Subroutine get_date: not called CC=1 SCAR=6.9
1481 0 my ($file) = @_;
1482 0 my $fh = open_file($file);
1483  
1484 0 chomp(my $line = <$fh>);
1485 0 my $date = @{parse_line($line)}[dtime];
1486  
1487 0 $date =~ s#^(\d+)/(\w+)/(\d+):.*#"$3-".$months{$2}."-$1"#e;
1488 0 return($date);
1489 }
1490 }
1491  
1492 =begin doc
1493  
1494 =head2 checksum
1495  
1496 Returns an md5 hexdigest of the first $nlines lines of the file (or the
1497 whole thing if $nlines is omitted.)
1498  
1499 my $md5 = checksum($fh, $nlines);
1500  
1501 =end doc
1502  
1503 =cut
1504  
1505 1 ▶sub checksum {
Subroutine checksum: not called CC=5 SCAR=32.6
1506 0 my ($fh, $num) = @_;
1507 0 ▶ $num ||= 0;
Condition0%0/2
$num ||= 0
l!l
00
MC/DC0%0/1
$num ||= 0
$num
Truth table0%0/2
A
A= $num
Aresult
11
00
1508  
1509 0 my $data = '';
1510 0 my $count = 0;
1511 0 ▶ while(my $line = <$fh>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $fh)) 0 0 0
1512 0 $data .= $line;
1513 0 ▶ (++$count == $num) and last;
Branch: 1 branch
BranchTrueFalseTotal
++$count == $num and last 0 0 0
1514 }
1515 0 ▶ ($count >= $num) or croak("don't have $num lines to checksum");
Branch: 1 branch
BranchTrueFalseTotal
$count >= $num or croak("don't have $num lines to checksum") 0 0 0
1516  
1517 0 return(Digest::MD5::md5_hex($data));
1518 } # end subroutine checksum definition
1519 ########################################################################
1520  
1521 package main;
1522  
1523 1 ▶if($0 eq __FILE__) {
Branch: 1 branch
BranchTrueFalseTotal
if ($0 eq __FILE__) 0 1 1
1524 0 bin::loghack::main(@ARGV);
1525 }
1526  
1527 # vi:ts=2:sw=2:et:sta
1528 1 my $package = 'bin::loghack';
« blib/lib/ApacheLog/Parser/Report.pm ↑ Summary blib/lib/ApacheLog/Parser.pm »