blib/lib/Web/VWF/Display/meta_data.pm untested

filter: Statementstmt 0.0% 0 / 272 Branchbran 0.0% 0 / 74 Conditioncond 0.0% 0 / 39 MC/DCmcdc 0.0% 0 / 22 Subroutinesub 0.0% 0 / 26 Podpod 0.0% 0 / 10 totaltotal 0.0% 0 / 443 CC 38 SCAR 37.8
CC
38
Cov
0%
get_slow_endpoints_24h
110.0
html
72.0
get_latency_24h
72.0
CRAP 43.6
1 package Web::VWF::Display::meta_data;
2  
3 # Display the meta-data page - the internal status of the server and VWF system
4  
5 # TODO: Add breakdown by browser name; the chart counts the types of client
6 # (web, mobile, search engine, ...), which is not the same thing
7  
8 use strict;
9 use warnings;
10  
11 use version; our $VERSION = version->declare('0.003.0');
12  
13 use parent 'Web::VWF::Display';
14  
15 # 0.23 and earlier take positional arguments (render_pie_chart_snippet($data,
16 # $opts)) and die "Data must be an array of arrays" on the hashes passed here;
17 # 0.24 was the first to take them, and the charts were built and tested on 0.30.
18 use HTML::D3 0.30;
19  
20 # Filesys::Df refuses to install on Windows, so Makefile.PL does not require it
21 # there. A server without it just has no disk figure: df() then answers as if
22 # the call had failed, which get_server_metrics() already copes with.
23 BEGIN {
24 if(!eval { require Filesys::Df; Filesys::Df->import(); 1 }) {
25 no strict 'refs'; ## no critic (ProhibitNoStrict)
26 *{__PACKAGE__ . '::df'} = sub { return };
27 }
28 }
29  
30 use List::Util qw(max sum0);
31 use Scalar::Util qw(looks_like_number);
32 use POSIX qw(strftime);
33 use Readonly;
34 use System::Info;
35 use Sys::Uptime;
36 use Sys::MemInfo;
37 use Time::Piece;
38 use Time::Seconds;
39  
40 # The browser types counted in the breakdown chart.
41 # Keep this in step with VWF::Display::_types() and CGI::Info::browser_type().
42 Readonly my @BROWSER_TYPES => ('web', 'mobile', 'search', 'robot', 'ai');
43  
44 # A request cannot have taken longer than the window these charts cover, so a
45 # duration beyond one day is not a measurement but a stale row written by the
46 # old timer bug (about 1.8e12 ms). One such row would otherwise dominate the
47 # average, the 95th percentile and the slow-endpoint list for a whole day.
48 Readonly my $MAX_DURATION_MS => ONE_DAY * 1000;
49  
50 # What each panel says when it has nothing to draw. An empty frame looks like
51 # a broken chart, so every chart area explains itself instead - and the page
52 # has one place, not two, where that wording lives.
53 Readonly my $NO_REQUESTS => '<p>No requests logged yet.</p>';
54 Readonly my $NO_RECENT_REQUESTS => '<p>No requests logged in the last 24 hours.</p>';
55 Readonly my $NO_DURATIONS => '<p>No request durations logged in the last 24 hours.</p>';
56 Readonly my $NOT_ENOUGH_SAMPLES => '<p>Not enough data yet to rank endpoints.</p>';
57  
58 sub html {
59 my $self = shift;
60 my %args = (ref($_[0]) eq 'HASH') ? %{$_[0]} : @_;
61  
62 my $vwf_log = $args{'vwf_log'} or die "Missing 'vwf_log' handle";
63 my $domain_name = $self->{'info'}->domain_name();
64  
65 # --- Browser breakdown: one slice per browser type ---
66 my @browser;
67 foreach my $type (@BROWSER_TYPES) {
68 my @entries = $vwf_log->type({ domain_name => $domain_name, type => $type });
69 push @browser, [$type, scalar(@entries)];
70 if($self->{'logger'}) {
71 $self->{'logger'}->debug("$type = " . scalar(@entries));
72 }
73 }
74  
75 # --- Server metrics using CPAN modules ---
76 my $server_metrics = $self->get_server_metrics();
77  
78 # --- Traffic metrics from vwf_log ---
79 my $traffic_metrics = $self->get_traffic_metrics($vwf_log, $domain_name);
80  
81 # --- HTTP status breakdown: one slice per status code ---
82 my %status_count;
83 foreach my $http_code ($vwf_log->http_code({ domain_name => $domain_name })) {
84 $status_count{$http_code}++;
85 }
86 my @status = map { [$_, $status_count{$_}] } sort keys %status_count;
87  
88 my $rate_24h = $self->get_request_rate_24h($vwf_log, $domain_name);
89 my $latency_24h = $self->get_latency_24h($vwf_log, $domain_name);
90 my $slow = $self->get_slow_endpoints_24h($vwf_log, $domain_name);
91  
92 # The charts are built here and the template only places them. HTML::D3
93 # escapes the labels it writes into the page, so nothing from the access log
94 # is trusted as markup. A chart with no data is a placeholder, never an
95 # empty frame.
96 my $charts = HTML::D3->new(width => 800, height => 400);
97 my $browser_chart = (grep { $_->[1] } @browser)
98 ? _scoped_snippet($charts->render_pie_chart_snippet({ data => \@browser, id => 'browser_chart', donut => 1, sort_slices => 'value' }))
99 : $NO_REQUESTS;
100 my $status_chart = @status
101 ? _scoped_snippet($charts->render_pie_chart_snippet({ data => \@status, id => 'status_chart', donut => 1, sort_slices => 'label' }))
102 : $NO_REQUESTS;
103 # B::Lint reads an array dereference in a conditional as an implicit scalar
104 # context and fails CI over it; scalar() is the count that was wanted anyway.
105 my $rate_chart = scalar(@{$rate_24h->{'total'}})
106 ? _scoped_snippet($charts->render_multi_series_line_chart_snippet({
107 series_data => _series(['Requests', $rate_24h->{'total'}], ['Errors', $rate_24h->{'errors'}]),
108 id => 'rate_chart', x_label => 'Hour', y_label => 'Requests',
109 }))
110 : $NO_RECENT_REQUESTS;
111 my $latency_chart = scalar(@{$latency_24h->{'avg'}})
112 ? _scoped_snippet($charts->render_multi_series_line_chart_snippet({
113 series_data => _series(['Average', $latency_24h->{'avg'}], ['P95', $latency_24h->{'p95'}]),
114 id => 'latency_chart', x_label => 'Hour', y_label => 'Milliseconds',
115 }))
116 : $NO_DURATIONS;
117 my $slow_chart = scalar(@{$slow})
118 ? _scoped_snippet($charts->render_bar_chart_snippet({
119 data => $slow, id => 'slow_chart', orientation => 'horizontal',
120 sort_bars => 'value', show_values => 1, value_label => 'ms',
121 }))
122 # Too few requests to rank anything is normal, not an error
123 : $NOT_ENOUGH_SAMPLES;
124  
125 return $self->SUPER::html({
126 server => $server_metrics,
127 traffic => $traffic_metrics,
128 browser_chart => $browser_chart,
129 status_chart => $status_chart,
130 rate_chart => $rate_chart,
131 latency_chart => $latency_chart,
132 slow_chart => $slow_chart,
133 });
134 }
135  
136 # HTML::D3's multi-series charts take {name, data => [{label, value}, ...]};
137 # the counting methods above return [label, value] pairs.
138 sub _series
139 {
140 my @series = @_;
141  
142 return [ map {
143 my ($name, $points) = @{$_};
144 {
145 name => $name,
146 data => [ map { { label => $_->[0], value => $_->[1] } } @{$points} ],
147 }
148 } @series ];
149 }
150  
151 # The pie snippet writes its script at the top level of the page, while the
152 # line and bar snippets wrap theirs in a function. Two pies on one page would
153 # therefore both declare "const data": the second script dies with
154 # "Identifier 'data' has already been declared" and leaves an empty frame.
155 # Giving every snippet its own function makes them independent; the snippets
156 # that already wrap themselves are simply wrapped twice, which is harmless.
157 sub _scoped_snippet
158 {
159 my ($result) = @_;
160  
161 my $html = $result->{'html'};
162 $html =~ s{(<script\b[^>]*>)(.*?)(</script>)}{$1 . "(function () {\n" . $2 . "\n})();\n" . $3}gse;
163  
164 return $html;
165 }
166  
167 sub get_server_metrics {
168 my $metrics = {};
169  
170 # CPU info
171 my $si = System::Info->new;
172 $metrics->{cpu_count} = $si->ncpu // 0;
173 $metrics->{cpu_type} = $si->cpu_type // '';
174  
175 # Disk usage
176 if(my $df = df('/')) {
177 # prefer 'per' if present
178 # A filesystem reporting no blocks has no meaningful percentage
179 $metrics->{disk_used_pct} = $df->{per}
180 // ($df->{blocks} ? int(100 * ($df->{blocks} - $df->{bfree}) / $df->{blocks}) : undef);
181 } else {
182 $metrics->{disk_used_pct} = undef;
183 }
184  
185 # Memory usage (via Sys::MemInfo)
186 my $total_mem = Sys::MemInfo::totalmem();
187 my $free_mem = Sys::MemInfo::freemem();
188 if (defined $total_mem && defined $free_mem && $total_mem > 0) {
189 $metrics->{memory_used_pct} = int(100 * ($total_mem - $free_mem) / $total_mem);
190 } else {
191 $metrics->{memory_used_pct} = 0;
192 }
193  
194 return $metrics;
195 }
196  
197 # ------------------------------
198 # Traffic metrics from vwf_log
199 # ------------------------------
200  
201 sub get_traffic_metrics {
202 my (undef, $vwf_log, $domain_name) = @_;
203 my $metrics = {};
204  
205 # Current time and one hour ago (local time)
206 my $now = time();
207 my $hour_ago = $now - ONE_HOUR + _utc_offset();
208  
209 # Get all entries as array of hashrefs
210 my @entries = $vwf_log->selectall_array({ domain_name => $domain_name });
211  
212 # Filter entries from last hour
213 my @recent;
214 foreach my $entry (@entries) {
215 next unless ref $entry eq 'HASH' && $entry->{time};
216  
217 my $tp;
218 eval { $tp = Time::Piece->strptime($entry->{time}, '%Y-%m-%d %H:%M:%S') };
219 next unless $tp;
220  
221 my $epoch = $tp->epoch;
222 push @recent, $entry if $epoch > $hour_ago;
223 }
224  
225 # Requests per hour
226 $metrics->{requests_per_hour} = scalar @recent;
227  
228 # Active users (unique IPs)
229 my %ips;
230 foreach my $entry (@recent) {
231 my $ip = $entry->{ip} // 'unknown';
232 $ips{$ip} = 1;
233 }
234 $metrics->{active_users} = scalar keys %ips;
235  
236 # Top 5 endpoints
237 # my %urls;
238 # foreach my $entry (@recent) {
239 # my $url = $entry->{url} // '/';
240 # $urls{$url}++;
241 # }
242 # my @top_urls = sort { $urls{$b} <=> $urls{$a} } keys %urls;
243 # $metrics->{top_urls} = [ @top_urls[0..(4 > $#top_urls ? $#top_urls : 4)] ];
244  
245 # Error count (HTTP code >= 400)
246 my $errors = grep { ($_->{http_code} // 0) >= 400 } @recent;
247 $metrics->{errors_last_hour} = $errors;
248  
249 return $metrics;
250 }
251  
252 sub get_request_rate_24h {
253 my (undef, $vwf_log, $domain_name) = @_;
254  
255 my $now = time();
256 my $start = $now - ONE_DAY + _utc_offset();
257  
258 my %buckets;
259  
260 foreach my $entry ($vwf_log->selectall_array({ domain_name => $domain_name })) {
261 next unless ref $entry eq 'HASH' && $entry->{time};
262  
263 my $tp;
264 eval { $tp = Time::Piece->strptime($entry->{time}, '%Y-%m-%d %H:%M:%S') };
265 next unless $tp;
266  
267 my $epoch = $tp->epoch;
268 next if $epoch < $start;
269  
270 # Bucket by hour
271 my $bucket = $tp->strftime('%H:00');
272  
273 $buckets{$bucket}{total}++;
274 if(($entry->{http_code} // 0) >= 400) {
275 $buckets{$bucket}{errors}++;
276 }
277 }
278  
279 # One [label, value] pair per hour, ready for HTML::D3
280 my (@total, @errors, @error_pct);
281  
282 foreach my $hour (sort keys %buckets) {
283 my $total = $buckets{$hour}{total} // 0;
284 my $errors = $buckets{$hour}{errors} // 0;
285  
286 push @total, [$hour, $total];
287 push @errors, [$hour, $errors];
288 push @error_pct, [$hour, $total ? 0 + sprintf('%.2f', ($errors / $total) * 100) : 0];
289 }
290  
291 return {
292 total => \@total,
293 errors => \@errors,
294 error_pct => \@error_pct,
295 };
296 }
297  
298 sub get_latency_24h {
299 my (undef, $vwf_log, $domain_name) = @_;
300  
301 my $start = time() - ONE_DAY + _utc_offset();
302  
303 my %buckets;
304  
305 foreach my $entry ($vwf_log->selectall_array({ domain_name => $domain_name })) {
306 next unless ref $entry eq 'HASH';
307 next unless $entry->{time};
308 # The log is data, not code: only numbers are durations
309 next unless defined($entry->{duration_ms}) && looks_like_number($entry->{duration_ms});
310 next if $entry->{duration_ms} > $MAX_DURATION_MS;
311  
312 my $tp;
313 eval { $tp = Time::Piece->strptime($entry->{time}, '%Y-%m-%d %H:%M:%S') };
314 next unless $tp;
315  
316 my $epoch = $tp->epoch;
317 next if $epoch < $start;
318  
319 my $hour = $tp->strftime('%H:00');
320  
321 push @{ $buckets{$hour} }, $entry->{duration_ms};
322 }
323  
324 my (@avg, @p95);
325  
326 foreach my $hour (sort keys %buckets) {
327 my @vals = sort { $a <=> $b } @{ $buckets{$hour} };
328 next unless @vals;
329  
330 my $count = @vals;
331 # Never eval() log contents: a crafted duration_ms would run as Perl
332 my $avg = int(sum0(@vals) / $count);
333  
334 my $p95_index = int(0.95 * ($count - 1));
335 my $p95 = $vals[$p95_index];
336  
337 push @avg, [$hour, $avg];
338 push @p95, [$hour, $p95];
339 }
340  
341 return {
342 avg => \@avg,
343 p95 => \@p95,
344 };
345 }
346  
347 sub get_slow_endpoints_24h {
348 my (undef, $vwf_log, $domain_name) = @_;
349  
350 my $start = time() - ONE_DAY + _utc_offset();
351  
352 # Step 1: fetch rows filtered only by equality
353 my @rows = $vwf_log->selectall_array({ domain_name => $domain_name });
354  
355 # Step 2: collect durations per template
356 my %durations;
357  
358 foreach my $row (@rows) {
359 next unless ref($row) eq 'HASH';
360 next unless defined $row->{time};
361 next unless defined($row->{duration_ms}) && looks_like_number($row->{duration_ms});
362 next if $row->{duration_ms} <= 0;
363 next if $row->{duration_ms} > $MAX_DURATION_MS;
364  
365 my $tp;
366 eval { $tp = Time::Piece->strptime($row->{time}, '%Y-%m-%d %H:%M:%S') };
367 next unless $tp;
368 next if $tp->epoch < $start;
369  
370 my $tpl = $row->{template} // '';
371 push @{ $durations{$tpl} }, $row->{duration_ms};
372 }
373  
374 # Step 3: compute p95
375 my @ranked;
376  
377 foreach my $tpl (keys %durations) {
378 my @vals = sort { $a <=> $b } @{ $durations{$tpl} };
379 my $count = @vals;
380 next if $count < 5;
381  
382 my $p95_index = int(0.95 * ($count - 1));
383 my $p95 = $vals[$p95_index];
384  
385 push @ranked, {
386 template => $tpl,
387 p95_ms => $p95,
388 };
389 }
390  
391 # Step 4: sort + top 10
392 @ranked = sort { $b->{p95_ms} <=> $a->{p95_ms} } @ranked;
393 splice(@ranked, 10) if @ranked > 10;
394  
395 # Step 5: [label, value] pairs. The labels stay as the log wrote them;
396 # HTML::D3 escapes them when it writes the chart.
397 my @dp;
398 foreach my $row (@ranked) {
399 my $label = $row->{template};
400 $label =~ s/^.+\/templates\///;
401  
402 push @dp, [$label, $row->{p95_ms}];
403 }
404  
405 return \@dp;
406 }
407  
408 sub _utc_offset
409 {
410 my $local_time = time();
411 my @local = localtime($local_time);
412 my @gmt = gmtime($local_time);
413  
414 # Convert both to seconds since midnight
415 my $local_seconds = $local[2] * 3600 + $local[1] * 60 + $local[0];
416 my $gmt_seconds = $gmt[2] * 3600 + $gmt[1] * 60 + $gmt[0];
417  
418 # Account for day boundary crossing
419 my $day_diff = $local[3] - $gmt[3];
420 if ($day_diff > 1) { $day_diff = -1; } # wrapped backwards
421 if ($day_diff < -1) { $day_diff = 1; } # wrapped forwards
422  
423 return $local_seconds - $gmt_seconds + ($day_diff * 86400);
424 }
425  
426 1;
427  
428 =encoding utf8
429  
430 =head1 NAME
431  
432 Web::VWF::Display::meta_data - The status page: the server and its traffic
433  
434 =head1 VERSION
435  
436 Version 0.003.0
437  
438 =head1 DESCRIPTION
439  
440 A dashboard for the site's owner. It reads the access log through the
441 C<vwf_log> handle (see L<Web::VWF::Data::vwf_log>) and shows the server's load,
442 memory, disk and uptime, the counts of requests, and five L<HTML::D3> charts: the
443 browser types, the HTTP status codes, requests and errors per hour, latency per
444 hour, and the slowest endpoints. A chart with nothing to draw is replaced by a
445 sentence saying so. It needs L<HTML::D3> 0.30 or later.
446  
447 =head1 METHODS
448  
449 =head2 html
450  
451 Needs the C<vwf_log> handle and dies without it. Returns the HTML from
452 F<meta_data.tmpl>.
453  
454 =head3 API SPECIFICATION
455  
456 =head4 INPUT
457  
458 { vwf_log => { type => 'object' } }
459  
460 =head4 OUTPUT
461  
462 { type => 'string' }
463  
464 =head2 get_server_metrics
465  
466 The state of the server as a hash reference: C<cpu_count>, C<cpu_type>,
467 C<disk_used_pct> (undef when L<Filesys::Df> is not installed) and
468 C<memory_used_pct>.
469  
470 =head2 get_traffic_metrics
471  
472 Takes the log handle and the domain name. Returns a hash reference with
473 C<requests_per_hour>, C<active_users> (distinct addresses) and C<errors_last_hour>
474 for the last hour.
475  
476 =head2 get_request_rate_24h
477  
478 Takes the log handle and the domain name. Returns C<{ total, errors, error_pct }>,
479 each a list of C<[ hour, value ]> pairs for the last 24 hours.
480  
481 =head2 get_latency_24h
482  
483 Takes the log handle and the domain name. Returns C<{ avg, p95 }>, each a list of
484 C<[ hour, milliseconds ]> pairs for the last 24 hours. Durations over a day are
485 log rows written by an old bug and are left out.
486  
487 =head2 get_slow_endpoints_24h
488  
489 Takes the log handle and the domain name. Returns up to ten C<[ template,
490 milliseconds ]> pairs, the 95th percentile of the slowest templates in the last
491 24 hours, ignoring any that were asked for fewer than five times.
492  
493 =head1 AUTHOR
494  
495 Nigel Horne, C<< <njh at nigelhorne.com> >>
496  
497 =head1 LICENCE AND COPYRIGHT
498  
499 Copyright 2010-2026 Nigel Horne.
500  
501 Usage is subject to the GPL2 licence terms.
502 If you use it,
503 please let me know.
504  
505 =cut