lib/Test2/Harness/UI/RunProcessor.pm untested

filter: Statementstmt 0.0% 0 / 741 Branchbran 0.0% 0 / 324 Conditioncond 0.0% 0 / 263 MC/DCmcdc 0.0% 0 / 150 Subroutinesub 0.0% 0 / 60 Podpod 0.0% 0 / 42 totaltotal 0.0% 0 / 1580 CC 163 SCAR 49.7
CC
163
Cov
0%
_process_event
650.0
update_other
306.0
flush
240.0
CRAP 143.7
1 package Test2::Harness::UI::RunProcessor;
2 use strict;
3 use warnings;
4  
5 our $VERSION = '0.000150';
6  
7 use DateTime;
8 use Data::GUID;
9 use Time::HiRes qw/time/;
10 use List::Util qw/first min max/;
11 use MIME::Base64 qw/decode_base64/;
12  
13 use Clone qw/clone/;
14 use Carp qw/croak confess/;
15 use Scalar::Util qw/blessed/;
16 use bytes ();
17  
18 use Test2::Util::Facets2Legacy qw/causes_fail/;
19  
20 use Test2::Harness::UI::Util qw/format_duration is_invalid_subtest_name/;
21  
22 use Test2::Harness::UI::UUID qw/gen_uuid gen_deflated_uuid uuid_inflate uuid_deflate uuid_mass_deflate/;
23 use Test2::Harness::Util::JSON qw/encode_json decode_json/;
24 use JSON::PP();
25  
26 use Test2::Harness::UI::Util::ImportModes qw{
27 %MODES
28 record_all_events
29 event_in_mode
30 mode_check
31 record_subtest_events
32 };
33  
34 use Test2::Harness::UI::Util::HashBase qw{
35 <config
36  
37 <running <jobs <binaries
38  
39 signal
40  
41 <coverage <uncover <new_jobs <id_cache <file_cache
42  
43 <mode
44 <interval <last_flush
45 <run <run_id +run_id_deflated +bulk_byte_cap
46 +user +user_id
47 +project +project_id
48  
49 <first_stamp <last_stamp
50  
51 <passed <failed <retried
52 <job0_id <job_ord
53  
54 <disconnect_retry
55 };
56  
57 sub trim_error {
58 my ($msg, $err) = @_;
59  
60 my @lines;
61 if ($ENV{TEST2_HARNESS_IMPORT_VERBOSE}) {
62 @lines = ($err);
63 }
64 else {
65 @lines = split /\n/, $err;
66 @lines = (@lines[1 .. 5], "\n[... TRIMMED, set the TEST2_HARNESS_IMPORT_VERBOSE=1 env var to see the entire error ...]\n", @lines[-5 .. -1]) if @lines > 12;
67 }
68  
69 return join("\n" => $msg, @lines) . "\n";
70 }
71  
72 sub retry_on_disconnect {
73 my $self = shift;
74 my ($description, $callback) = @_;
75  
76 my ($attempt, $err);
77 for my $i (0 .. ($self->{+DISCONNECT_RETRY} - 1)) {
78 $attempt = $i;
79 return 1 if eval { $callback->(); 1 };
80 $err = $@;
81  
82 # Only the driver's message decides. DBI appends the statement and
83 # bind values, and Carp a stack trace, and either can contain
84 # "connect" (event facets, or this sub's own name).
85 (my $msg = $err) =~ s/\[for Statement.*//s;
86 ($msg) = split /\n/, $msg;
87 last unless $msg =~ m/(gone away|connect|timeout)/i;
88  
89 # Try to fix the connection
90 for (1 .. 10) {
91 $self->schema->storage->disconnect;
92 eval { $self->schema->storage->ensure_connected };
93 last if $self->schema->storage->connected;
94 sleep 1;
95 }
96 }
97  
98 die trim_error(qq{Failed "$description" (attempt $attempt)}, $err);
99 }
100  
101 sub populate {
102 my $self = shift;
103 my ($type, $data) = @_;
104  
105 return unless $data && @$data;
106  
107 $self->retry_on_disconnect(
108 "Populate '$type'",
109 sub {
110 my $rs = $self->schema->resultset($type);
111 my $ok = eval { $rs->populate($data); 1 };
112 my $err = $@;
113 return 1 if $ok;
114  
115 die $err unless $err =~ m/duplicate/i;
116  
117 warn "Duplicate found:\n====\n$err\n====\n\nPopulating '$type' 1 at a time.\n";
118 for my $item (@$data) {
119 uuid_mass_deflate($item);
120 next if eval { $rs->create($item); 1 };
121 my $err = $@;
122  
123 # I need to track down why we still get duplicates (Coverage mainly) for now skip them.
124 next if $err =~ m/duplicate/i;
125  
126 # Actual error
127 warn $err;
128 }
129  
130 return 1;
131 }
132 );
133 }
134  
135 # DBIx::Class populate() hands rows to DBI's execute_for_fetch(), and neither
136 # DBD::Pg nor DBD::mysql batch that, so every row is its own round trip to
137 # the database. Events and coverage arrive in the hundreds of thousands of
138 # rows for one run, so those go out as multi-row INSERT statements instead,
139 # all inside one transaction.
140 #
141 # Nothing here goes through the Result class, so rows must already hold what
142 # the database stores: JSON encoded, ids either deflated or still
143 # Test2::Harness::UI::UUID objects (those and DateTime objects are converted
144 # here, see _bulk_value). Rows need not all name the same columns, the
145 # column list is the union and missing columns are NULL.
146 #
147 # Duplicate recovery needs AutoCommit: on a duplicate the transaction rolls
148 # back and the chunks are replayed outside one, so do not call this inside
149 # txn_do().
150 sub populate_bulk {
151 my $self = shift;
152 my ($type, $data, %params) = @_;
153  
154 return unless $data && @$data;
155  
156 # Past this many rows the statement gets slower per row on PostgreSQL,
157 # and MySQL gains nothing from it. The byte cap keeps a statement under
158 # MySQL's max_allowed_packet when events carry large facets.
159 my $max_rows = $params{chunk} // 100;
160 my $max_bytes = $params{chunk_bytes} // $self->bulk_byte_cap;
161 croak "chunk must be a positive integer, got '$max_rows'" unless $max_rows =~ m/^\d+$/ && $max_rows > 0;
162  
163 my $schema = $self->schema;
164 my $storage = $schema->storage;
165 my $source = $schema->resultset($type)->result_source;
166  
167 my %seen;
168 my @cols = sort grep { !$seen{$_}++ } map { keys %$_ } @$data;
169  
170 # Binary columns (bytea on PostgreSQL) need their type bound, or the
171 # driver sends the bytes as text.
172 my $colinfo = $source->columns_info(\@cols);
173 my @attrs = map { $storage->bind_attribute_by_data_type($colinfo->{$_}->{data_type}) } @cols;
174 my $attrs = (grep { $_ } @attrs) ? \@attrs : undef;
175  
176 # Convert everything before touching the database so a bad value dies
177 # cleanly instead of inside a transaction. Every path below, including
178 # duplicate recovery, writes these values so no row can be converted two
179 # different ways.
180 my @chunks = ([]);
181 my $bytes = 0;
182 for my $item (@$data) {
183 my $values = [map { $self->_bulk_value($type, $_, $colinfo->{$_}->{data_type}, $item->{$_}) } @cols];
184  
185 my $size = 0;
186 $size += bytes::length($_) for grep { defined } @$values;
187  
188 if (@{$chunks[-1]} && (@{$chunks[-1]} >= $max_rows || $bytes + $size > $max_bytes)) {
189 push @chunks => [];
190 $bytes = 0;
191 }
192  
193 push @{$chunks[-1]} => $values;
194 $bytes += $size;
195 }
196  
197 my $dbh = $storage->dbh;
198 my $insert = "INSERT INTO " . $dbh->quote_identifier($source->name) . " (" . join(', ', map { $dbh->quote_identifier($_) } @cols) . ") VALUES ";
199 my $row = "(" . join(', ', ('?') x @cols) . ")";
200  
201 my $insert_chunk = sub {
202 my ($dbh, $chunk) = @_;
203  
204 my $sth = $dbh->prepare_cached($insert . join(', ', ($row) x @$chunk));
205  
206 if ($attrs) {
207 for my $r (0 .. $#$chunk) {
208 for my $c (0 .. $#cols) {
209 my $attr = $attrs->[$c] or next;
210 $sth->bind_param($r * @cols + $c + 1, undef, $attr);
211 }
212 }
213 }
214  
215 $sth->execute(map { @$_ } @$chunk);
216 };
217  
218 $self->retry_on_disconnect(
219 "Populate '$type'",
220 sub {
221 my $ok = eval {
222 $schema->txn_do(sub { $storage->dbh_do(sub { $insert_chunk->($_[1], $_) for @chunks }) });
223 1;
224 };
225 my $err = $@;
226 return 1 if $ok;
227  
228 die $err unless $err =~ m/duplicate/i;
229  
230 # The transaction rolled back, so nothing went in. Redo it with
231 # autocommit, and only the chunk that holds the duplicate goes
232 # in one row at a time. Chunks before a later failure stay
233 # committed. DBI appends the whole statement and every bind value
234 # to the error, drop those from the warning.
235 (my $short = $err) =~ s/\s*\[for Statement.*//s;
236 warn "Duplicate found:\n====\n$short\n====\n\nPopulating '$type' in chunks, falling back to 1 at a time on duplicates.\n";
237 for my $chunk (@chunks) {
238 next if eval { $storage->dbh_do(sub { $insert_chunk->($_[1], $chunk) }); 1 };
239 my $err = $@;
240 die $err unless $err =~ m/duplicate/i;
241  
242 for my $values (@$chunk) {
243 next if eval { $storage->dbh_do(sub { $insert_chunk->($_[1], [$values]) }); 1 };
244 my $err = $@;
245  
246 # Only duplicates may be skipped, anything else is a lost
247 # row and must fail the import.
248 next if $err =~ m/duplicate/i;
249 die $err;
250 }
251 }
252  
253 return 1;
254 }
255 );
256  
257 return;
258 }
259  
260 # MySQL rejects a statement larger than max_allowed_packet and drops the
261 # connection, and DBD::mysql interpolates the bind values into the statement
262 # text with escaping, so stay well under it. PostgreSQL has no such limit.
263 sub bulk_byte_cap {
264 my $self = shift;
265  
266 return $self->{+BULK_BYTE_CAP} //= do {
267 my $cap = 4 * 1024 * 1024;
268  
269 if ($Test2::Harness::UI::Schema::LOADED && $Test2::Harness::UI::Schema::LOADED =~ m/mysql/i) {
270 my ($packet) = $self->schema->storage->dbh->selectrow_array('SELECT @@max_allowed_packet');
271 $cap = min($cap, int($packet / 2)) if $packet;
272 }
273  
274 $cap;
275 };
276 }
277  
278 sub _bulk_value {
279 my $self = shift;
280 my ($type, $col, $data_type, $val) = @_;
281  
282 return $val unless ref $val;
283  
284 if (blessed($val)) {
285 # MySQL keeps most ids in BINARY(16) but some (trace_id) in CHAR(36),
286 # so the column decides the form, not the database.
287 return (($data_type // '') =~ m/binary/i ? $val->binary : $val->string) if $val->isa('Test2::Harness::UI::UUID');
288 return $self->schema->storage->datetime_parser->format_datetime($val) if $val->isa('DateTime');
289 return "$val" if overload::Method($val, '""');
290 }
291  
292 die "Cannot write a " . ref($val) . " to column '$col' of '$type', populate_bulk() rows must hold what the database stores\n";
293 }
294  
295 sub format_stamp {
296 my $self = shift;
297 my $stamp = shift;
298 return undef unless $stamp;
299  
300 unless (ref($stamp)) {
301 $self->{+FIRST_STAMP} = $self->{+FIRST_STAMP} ? min($self->{+FIRST_STAMP}, $stamp) : $stamp;
302 $self->{+LAST_STAMP} = $self->{+LAST_STAMP} ? max($self->{+LAST_STAMP}, $stamp) : $stamp;
303 }
304  
305 return DateTime->from_epoch(epoch => $stamp, time_zone => 'local');
306 }
307  
308 sub schema { $_[0]->{+CONFIG}->schema }
309  
310 sub init {
311 my $self = shift;
312  
313 croak "'config' is a required attribute"
314 unless $self->{+CONFIG};
315  
316 $self->{+DISCONNECT_RETRY} //= 15;
317  
318 my $run;
319 if ($run = $self->{+RUN}) {
320 $self->{+RUN_ID} = $run->run_id;
321 $self->{+MODE} = $MODES{$run->mode};
322  
323 $self->retry_on_disconnect("update status for run '$self->{+RUN_ID}'" => sub { $run->update({status => 'pending'}) });
324 }
325 else {
326 my $run_id = $self->{+RUN_ID} // croak "either 'run' or 'run_id' must be provided";
327 my $mode = $self->{+MODE} // croak "'mode' is a required attribute unless 'run' is specified";
328 $self->{+MODE} = $MODES{$mode} // croak "Invalid mode '$mode'";
329  
330 my $schema = $self->schema;
331 my $run = $schema->resultset('Run')->create({
332 run_id => $run_id,
333 user_id => $self->user_id,
334 project_id => $self->project_id,
335 mode => $mode,
336 status => 'pending',
337 });
338  
339 $self->{+RUN} = $run;
340 }
341  
342 $run->discard_changes;
343  
344 $self->{+PROJECT_ID} //= $run->project_id;
345  
346 $self->{+RUN_ID} = uuid_inflate($self->{+RUN_ID});
347 $self->{+USER_ID} = uuid_inflate($self->{+USER_ID});
348 $self->{+PROJECT_ID} = uuid_inflate($self->{+PROJECT_ID});
349  
350 $self->{+ID_CACHE} = {};
351 $self->{+COVERAGE} = [];
352  
353 $self->{+PASSED} = 0;
354 $self->{+FAILED} = 0;
355  
356 $self->{+JOB_ORD} = 1;
357 $self->{+JOB0_ID} = gen_uuid();
358 }
359  
360 sub flush_all {
361 my $self = shift;
362  
363 my $all = $self->{+JOBS};
364 for my $jobs (values %$all) {
365 for my $job (values %$jobs) {
366 $job->{done} = 'end';
367 $self->flush(job => $job);
368 }
369 }
370  
371 $self->flush_events();
372 $self->flush_reporting();
373 }
374  
375 sub flush {
376 my $self = shift;
377 my %params = @_;
378  
379 my $job = $params{job} or croak "job is required";
380 my $res = $job->{result};
381  
382 my $bmode = $self->run->buffer;
383 my $int = $self->{+INTERVAL};
384  
385 # Always update if needed
386 $self->retry_on_disconnect("update run" => sub { $self->run->insert_or_update() });
387  
388 my $flush = $params{force} ? 'force' : 0;
389 $flush ||= 'always' if $bmode eq 'none';
390 $flush ||= 'diag' if $bmode eq 'diag' && $res->fail && $params{is_diag};
391 $flush ||= 'job' if $job->{done};
392 $flush ||= 'status' if $res->is_column_changed('status');
393 $flush ||= 'fail' if $res->is_column_changed('fail');
394  
395 if ($int && !$flush) {
396 my $last = $self->{+LAST_FLUSH};
397 $flush = 'interval' if !$last || $int < time - $last;
398 }
399  
400 return "" unless $flush;
401 $self->{+LAST_FLUSH} = time;
402  
403 $self->retry_on_disconnect("update job result" => sub { $res->update() });
404  
405 $self->flush_events();
406 $self->flush_reporting();
407  
408 if (my $done = $job->{done}) {
409 # Last time we need to write this, so clear it.
410 delete $self->{+JOBS}->{$job->{job_id}}->{$job->{job_try}};
411  
412 unless ($res->status eq 'complete') {
413 my $status = $self->{+SIGNAL} ? 'canceled' : 'broken';
414 $status = 'canceled' if $done eq 'end';
415 $res->status($status);
416 }
417  
418 # Normalize the fail/pass
419 my $fail = $res->fail ? 1 : 0;
420 $res->fail($fail);
421  
422 $res->normalize_to_mode(mode => $self->{+MODE});
423 }
424  
425 $self->retry_on_disconnect("update job result" => sub { $res->update() });
426  
427 return $flush;
428 }
429  
430 sub flush_events {
431 my $self = shift;
432  
433 return if mode_check($self->{+MODE}, 'summary');
434  
435 my @write;
436  
437 my $jobs = $self->{+JOBS};
438 for my $tries (values %$jobs) {
439 for my $job (values %$tries) {
440 my $events = $job->{events};
441 my $deferred = $job->{deffered_events} //= [];
442  
443 if (record_all_events(mode => $self->{+MODE}, job => $job->{result})) {
444 push @write => (@$deferred, @$events);
445 @$deferred = ();
446 }
447 else {
448 for my $event (@$events) {
449 if (event_in_mode(event => $event, record_all_event => 0, mode => $self->{+MODE}, job => $job->{result})) {
450 push @write => $event;
451 }
452 else {
453 push @$deferred => $event;
454 }
455 }
456 }
457  
458 @$events = ();
459 }
460 }
461  
462 return unless @write;
463  
464 my @write_bin;
465 for my $e (@write) {
466 my $list = delete $e->{has_binary};
467  
468 $e->{has_binary} = $list && @$list ? 1 : 0;
469 next unless $e->{has_binary};
470  
471 $e->{has_binary} = 1;
472 for my $uuid (@$list) {
473 push @write_bin => delete $self->{+BINARIES}->{$uuid};
474 }
475 }
476  
477 local $ENV{DBIC_DT_SEARCH_OK} = 1;
478 $self->populate_bulk(Event => \@write);
479 $self->populate(Binary => \@write_bin);
480 }
481  
482 sub flush_reporting {
483 my $self = shift;
484  
485 my @write;
486  
487 my %mixin_run = (
488 user_id => $self->user_id,
489 run_id => $self->{+RUN_ID},
490 run_ord => $self->run->run_ord(),
491 project_id => $self->{+PROJECT_ID},
492 );
493  
494 my $jobs = $self->{+JOBS};
495 for my $tries (values %$jobs) {
496 for my $job (values %$tries) {
497 my $strip_event_id = 0;
498  
499 $strip_event_id = 1 unless record_subtest_events(
500 job => $job->{result},
501 fail => $job->{result}->fail,
502 mode => $self->{+MODE},
503  
504 is_harness_out => 0,
505 );
506  
507 my %mixin = (
508 %mixin_run,
509 job_try => $job->{job_try} // 0,
510 job_key => $job->{job_key},
511 test_file_id => $job->{result}->test_file_id,
512 );
513  
514 if (my $duration = $job->{duration}) {
515 my $raw_fail = $job->{result}->fail;
516 my $raw_retry = $job->{result}->retry;
517 my $abort = (defined($raw_fail) || defined($raw_retry)) ? 0 : 1;
518 my $fail = $raw_fail // 0;
519 my $pass = $fail ? 0 : 1;
520 my $retry = $raw_retry // 0;
521  
522 push @write => {
523 reporting_id => gen_uuid(),
524 duration => $duration,
525 pass => $pass,
526 fail => $fail,
527 abort => $abort,
528 retry => $retry,
529 %mixin,
530 };
531 }
532  
533 my $reporting = delete $job->{reporting};
534  
535 for my $rep (@$reporting) {
536 next unless defined $rep->{duration};
537 next unless defined $rep->{subtest};
538  
539 delete $rep->{event_id} if $strip_event_id;
540  
541 %$rep = (
542 reporting_id => gen_uuid(),
543 %mixin,
544 %$rep,
545 );
546  
547 push @write => $rep;
548 }
549 }
550 }
551  
552 return unless @write;
553  
554 local $ENV{DBIC_DT_SEARCH_OK} = 1;
555  
556 $self->populate_bulk(Reporting => \@write);
557 }
558  
559 sub user {
560 my $self = shift;
561  
562 return $self->{+RUN}->user if $self->{+RUN};
563 return $self->{+USER} if $self->{+USER};
564  
565 my $user_id = $self->{+USER_ID} // confess "No user or user_id specified";
566  
567 my $schema = $self->schema;
568 my $user = $schema->resultset('User')->search({user_id => $user_id})->first;
569 return $user if $user;
570 confess "Invalid user_id: $user_id";
571 }
572  
573 sub user_id {
574 my $self = shift;
575  
576 return $self->{+RUN}->user_id if $self->{+RUN};
577 return $self->{+USER}->user_id if $self->{+USER};
578 return $self->{+USER_ID} if $self->{+USER_ID};
579 }
580  
581 sub project {
582 my $self = shift;
583  
584 return $self->{+RUN}->project if $self->{+RUN};
585 return $self->{+PROJECT} if $self->{+PROJECT};
586  
587 my $project_id = $self->{+PROJECT_ID} // confess "No project or project_id specified";
588  
589 my $schema = $self->schema;
590 my $project = $schema->resultset('Project')->search({project_id => $project_id})->first;
591 return $project if $project;
592 confess "Invalid project_id: $project_id";
593 }
594  
595 sub project_id {
596 my $self = shift;
597  
598 return $self->{+RUN}->project_id if $self->{+RUN};
599 return $self->{+PROJECT}->project_id if $self->{+PROJECT};
600 return $self->{+PROJECT_ID} if $self->{+PROJECT_ID};
601 }
602  
603 sub start {
604 my $self = shift;
605 return if $self->{+RUNNING};
606  
607 $self->retry_on_disconnect("update status" => sub { $self->{+RUN}->update({status => 'running'}) });
608  
609 $self->{+RUNNING} = 1;
610 }
611  
612 sub get_job {
613 my $self = shift;
614 my (%params) = @_;
615  
616 my $is_harness_out = 0;
617 my $job_id = $params{job_id};
618  
619 if (!$job_id || $job_id eq '0') {
620 $job_id = $self->{+JOB0_ID};
621 $is_harness_out = 1;
622 }
623  
624 $job_id = uuid_inflate($job_id);
625 my $job_try = $params{job_try} // 0;
626  
627 my $job = $self->{+JOBS}->{$job_id}->{$job_try};
628 return $job if $job;
629  
630 my $key = gen_uuid();
631  
632 my $test_file_id = undef;
633 if (my $queue = $params{queue}) {
634 my $file = $queue->{rel_file} // $queue->{file};
635 $test_file_id = $self->get_test_file_id($file) if $file;
636 $self->{+FILE_CACHE}->{$job_id} //= $test_file_id if $test_file_id;
637 }
638  
639 $test_file_id //= $self->{+FILE_CACHE}->{$job_id};
640  
641 my $result;
642 $self->retry_on_disconnect(
643 "vivify job" => sub {
644 $result = $self->schema->resultset('Job')->update_or_create({
645 status => 'pending',
646 job_key => $key,
647 job_id => $job_id,
648 job_try => $job_try,
649 is_harness_out => $is_harness_out,
650 job_ord => $self->{+JOB_ORD}++,
651 run_id => $self->{+RUN}->run_id,
652 fail_count => 0,
653 pass_count => 0,
654 test_file_id => $test_file_id,
655  
656 $is_harness_out ? (name => "HARNESS INTERNAL LOG") : (),
657 });
658 }
659 );
660  
661 # In case we are resuming.
662 $self->retry_on_disconnect("delete old events" => sub { $result->events->delete_all() });
663  
664 # Prevent duplicate coverage when --retry is used
665 if ($job_try) {
666 if ($Test2::Harness::UI::Schema::LOADED =~ m/mysql/i) {
667 my $schema = $self->schema;
668 $schema->storage->connected; # Make sure we are connected
669 my $dbh = $schema->storage->dbh;
670  
671 my $query = <<" EOT";
672 DELETE coverage
673 FROM coverage
674 JOIN jobs USING(job_key)
675 WHERE job_id = ?
676 EOT
677  
678 my $sth = $dbh->prepare($query);
679 $sth->execute($job_id) or die $sth->errstr;
680 }
681 else {
682 $self->retry_on_disconnect(
683 "delete old coverage" => sub {
684 $self->schema->resultset('Coverage')->search({'job_key.job_id' => $job_id}, {join => 'job_key'})->delete;
685 }
686 );
687 }
688 }
689  
690 if (my $old = $self->{+JOBS}->{$job_id}->{$job_try - 1}) {
691 $self->{+UNCOVER}->{$old->{job_key}}++;
692 }
693  
694 $job = {
695 job_key => $key,
696 job_id => $job_id,
697 job_try => $job_try,
698  
699 events => [],
700 orphans => {},
701 reporting => [],
702  
703 event_ord => 1,
704 result => $result,
705 };
706  
707 return $self->{+JOBS}->{$job_id}->{$job_try} = $job;
708 }
709  
710 sub process_event {
711 my $self = shift;
712 my ($event, $f, %params) = @_;
713  
714 $f //= $event->{facet_data};
715 $f = $f ? clone($f) : {};
716  
717 $self->start unless $self->{+RUNNING};
718  
719 my $job = $params{job} // $self->get_job(%{$f->{harness} // {}}, queue => $f->{harness_job_queued});
720  
721 my $e = $self->_process_event($event, $f, %params, job => $job);
722 clean($e);
723  
724 if (my $od = $e->{orphan}) {
725 $job->{orphans}->{$e->{event_id}} = $e;
726 }
727 else {
728 if (my $o = delete $job->{orphans}->{$e->{event_id}}) {
729 $e->{orphan} = $o->{orphan};
730 $e->{orphan_line} = $o->{orphan_line} if defined $o->{orphan_line};
731 $e->{stamp} //= $o->{stamp};
732 }
733 push @{$job->{events}} => $e;
734 }
735  
736 $self->flush(job => $job, is_diag => $e->{is_diag});
737  
738 return;
739 }
740  
741 sub finish {
742 my $self = shift;
743 my (@errors) = @_;
744  
745 $self->flush_all();
746  
747 my $run = $self->run;
748  
749 my $status;
750 my $dur_stat;
751 my $aborted = 0;
752  
753 if (@errors) {
754 my $error = join "\n" => @errors;
755 $status = {status => 'broken', error => $error};
756 $dur_stat = 'abort';
757 }
758 else {
759 my $stat;
760 if ($self->{+SIGNAL}) {
761 $stat = 'canceled';
762 $dur_stat = 'abort';
763 $aborted = 1;
764 }
765 else {
766 $stat = 'complete';
767 $dur_stat = $self->{+FAILED} ? 'fail' : 'pass';
768 }
769  
770 $status = {status => $stat, passed => $self->{+PASSED}, failed => $self->{+FAILED}, retried => $self->{+RETRIED}};
771 }
772  
773 if ($self->{+FIRST_STAMP} && $self->{+LAST_STAMP}) {
774 my $duration = $self->{+LAST_STAMP} - $self->{+FIRST_STAMP};
775 $status->{duration} = format_duration($duration);
776  
777 $self->retry_on_disconnect("insert duration row" => sub {
778 my $fail = $aborted ? 0 : $self->{+FAILED} ? 1 : 0;
779 my $pass = ($fail || $aborted) ? 0 : 1;
780 $self->schema->resultset('Reporting')->create({
781 reporting_id => gen_uuid(),
782 user_id => $self->user_id,
783 run_id => $self->{+RUN_ID},
784 project_id => $self->{+PROJECT_ID},
785 run_ord => $self->run->run_ord(),
786 duration => $duration,
787 retry => 0,
788 pass => $pass,
789 fail => $fail,
790 abort => $aborted,
791 });
792 });
793 }
794  
795 $self->retry_on_disconnect("update run status" => sub { $run->update($status) });
796  
797 return $status;
798 }
799  
800 sub add_binary {
801 my $self = shift;
802 my $file = {@_};
803  
804 my $uuid = $file->{binary_id} //= gen_uuid();
805 $file->{is_image} //= $file->{filename} =~ m/\.(a?png|gif|jpe?g|svg|bmp|ico)$/ ? 1 : 0;
806 $file->{data} = decode_base64($file->{data});
807  
808 my $bins = $self->{+BINARIES} //= {};
809 $bins->{$uuid} = $file;
810  
811 return $uuid;
812 }
813  
814 sub _process_event {
815 my $self = shift;
816 my ($event, $f, %params) = @_;
817 my $job = $params{job};
818  
819 my $harness = $f->{harness} // {};
820 my $trace = $f->{trace} // {};
821  
822 my $e_id = uuid_inflate($harness->{event_id} // $event->{event_id} // die "No event id!");
823 my $nested = $f->{hubs}->[0]->{nested} || 0;
824  
825 my @has_binary;
826 if ($f->{binary} && @{$f->{binary}}) {
827 for my $file (@{$f->{binary}}) {
828 my $data = delete $file->{data};
829 $file->{data} = 'removed';
830 my $binary_id = $self->add_binary(event_id => $e_id, filename => $file->{filename}, description => $file->{details}, data => $data, is_image => $file->{is_image});
831 push @has_binary => $binary_id;
832 }
833 }
834  
835 my $fail = causes_fail($f) ? 1 : 0;
836  
837 my $is_diag = $fail;
838 $is_diag ||= 1 if $f->{errors} && @{$f->{errors}};
839 $is_diag ||= 1 if $f->{assert} && !($f->{assert}->{pass} || $f->{amnesty});
840 $is_diag ||= 1 if $f->{info} && first { $_->{debug} || $_->{important} } @{$f->{info}};
841 $is_diag //= 0;
842  
843 my $is_harness = (first { substr($_, 0, 8) eq 'harness_' } keys %$f) ? 1 : 0;
844  
845 my $is_time = $f->{harness_job_end} ? ($f->{harness_job_end}->{times} ? 1 : 0) : 0;
846  
847 my $is_subtest = $f->{parent} ? 1 : 0;
848  
849 my $e = {
850 event_id => $e_id,
851 nested => $nested,
852 is_subtest => $is_subtest,
853 is_diag => $is_diag,
854 is_harness => $is_harness,
855 is_time => $is_time,
856 trace_id => $trace->{uuid},
857 job_key => $job->{job_key},
858 event_ord => $job->{event_ord}++,
859 stamp => $self->format_stamp($harness->{stamp} || $event->{stamp} || $params{stamp}),
860 has_binary => \@has_binary,
861 };
862  
863 my $orphan = $nested ? 1 : 0;
864 if (my $p = $params{parent_id}) {
865 $e->{parent_id} ||= $p;
866 $orphan = 0;
867 }
868  
869 if ($orphan) {
870 clean($f);
871  
872 if ($f->{parent} && $f->{parent}->{children}) {
873 $f->{parent}->{children} = "Removed";
874 }
875  
876 $e->{orphan} = encode_json($f);
877 $e->{orphan_line} = $params{line} if $params{line};
878 }
879 else {
880 if (my $fields = $f->{run_fields}) {
881 $self->add_run_fields($fields);
882 }
883  
884 if (my $job_coverage = $f->{job_coverage}) {
885 $self->add_job_coverage($job, $job_coverage);
886 $f->{job_coverage} = "Removed, used to populate the job_coverage table";
887 }
888  
889 if (my $run_coverage = $f->{run_coverage}) {
890 $f->{run_coverage} = "Removed, used to populate the run_coverage table";
891 $self->add_run_coverage($run_coverage);
892 }
893  
894 if ($f->{parent} && $f->{parent}->{children}) {
895 $self->process_event({}, $_, job => $job, parent_id => $e_id, line => $params{line}) for @{$f->{parent}->{children}};
896 $f->{parent}->{children} = "Removed, used to populate events table";
897  
898 $self->add_subtest_duration($job, $e, $f) unless $nested;
899 }
900  
901 unless ($nested) {
902 my $res = $job->{result};
903 if ($fail) {
904 $res->fail_count($res->fail_count + 1);
905 $res->fail(1);
906 }
907 $res->pass_count($res->pass_count + 1) if $f->{assert} && !$fail;
908  
909 $self->update_other($job, $f) if $e->{is_harness};
910 }
911  
912 clean($f);
913 $e->{facets} = encode_json($f);
914 $e->{facets_line} = $params{line} if $params{line};
915 }
916  
917 return $e;
918 }
919  
920 sub add_subtest_duration {
921 my $self = shift;
922 my ($job, $e, $f) = @_;
923  
924 return if $f->{hubs}->[0]->{nested};
925  
926 my $parent = $f->{parent} // return;
927 my $assert = $f->{assert} // return;
928 my $st = $assert->{details} // return;
929 return if is_invalid_subtest_name($st);
930  
931 my $start = $parent->{start_stamp} // return;
932 my $stop = $parent->{stop_stamp} // return;
933 my $duration = $stop - $start;
934  
935 push @{$job->{reporting}} => {
936 duration => $duration,
937 subtest => $st,
938 event_id => $e->{event_id},
939 abort => 0,
940 retry => 0,
941 $assert->{pass} ? (pass => 1, fail => 0) : (fail => 1, pass => 0),
942 };
943 }
944  
945 sub add_job_coverage {
946 my $self = shift;
947 my ($job, $job_coverage) = @_;
948  
949 my $job_id = $job->{job_id};
950 my $job_try = $job->{job_try} // 0;
951  
952 # Do not add coverage if a retry has already started. Events could be out of order.
953 return if $self->{+JOBS}->{$job_id}->{$job_try + 1};
954 return if $self->{+UNCOVER} && $self->{+UNCOVER}->{$job->{job_key}};
955  
956 my $test = $job_coverage->{test} // $job->{result}->file;
957  
958 my $test_id = uuid_deflate($self->get_test_file_id($test)) or confess("Could not get test id (for '$test')");
959 my $manager_id = $self->get_coverage_manager_id($job_coverage->{manager});
960 my $job_key = uuid_deflate($job->{job_key});
961  
962 for my $source (keys %{$job_coverage->{files}}) {
963 my $subs = $job_coverage->{files}->{$source};
964 my $source_id = $self->get_source_file_id($source);
965  
966 for my $sub (keys %$subs) {
967 $self->_add_coverage(
968 job_key => $job_key,
969 test_file_id => $test_id,
970 source_file_id => $source_id,
971 source_sub_id => $self->get_source_sub_id($sub),
972 coverage_manager_id => $manager_id,
973 meta => $subs->{$sub},
974 );
975 }
976 }
977  
978 $self->flush_coverage;
979 }
980  
981 sub add_run_coverage {
982 my $self = shift;
983 my ($run_coverage) = @_;
984  
985 my $files = $run_coverage->{files};
986 my $meta = $run_coverage->{testmeta};
987  
988 my (%test_ids, %manager_ids);
989  
990 for my $source (keys %$files) {
991 my $subs = $files->{$source};
992 my $source_id = $self->get_source_file_id($source);
993  
994 for my $sub (keys %$subs) {
995 my $tests = $subs->{$sub};
996 my $sub_id = $self->get_source_sub_id($sub);
997  
998 for my $test (keys %$tests) {
999 my $test_id = $test_ids{$test} //= uuid_deflate($self->get_test_file_id($test)) or confess("Could not get test id (for '$test')");
1000  
1001 my $manager_id;
1002 if (defined(my $manager = $meta->{$test}->{manager})) {
1003 $manager_id = $manager_ids{$manager} //= $self->get_coverage_manager_id($manager);
1004 }
1005  
1006 $self->_add_coverage(
1007 test_file_id => $test_id,
1008 source_file_id => $source_id,
1009 source_sub_id => $sub_id,
1010 coverage_manager_id => $manager_id,
1011 meta => $tests->{$test},
1012 );
1013 }
1014 }
1015 }
1016  
1017 $self->flush_coverage;
1018 }
1019  
1020 # This runs once per coverage row, and there can be hundreds of thousands in
1021 # one job, so the ids arrive resolved and deflated: the row goes to the
1022 # database as it is, through populate_bulk().
1023 sub _add_coverage {
1024 my $self = shift;
1025 my %params = @_;
1026  
1027 my $manager_id = $params{coverage_manager_id};
1028 my $meta = $manager_id ? encode_json($params{meta}) : undef;
1029  
1030 my $coverage = $self->{+COVERAGE} //= [];
1031  
1032 push @$coverage => {
1033 coverage_id => gen_deflated_uuid(),
1034 run_id => $self->{+RUN_ID_DEFLATED} //= uuid_deflate($self->{+RUN_ID}),
1035 test_file_id => $params{test_file_id},
1036 source_file_id => $params{source_file_id},
1037 source_sub_id => $params{source_sub_id},
1038 coverage_manager_id => $manager_id,
1039 metadata => $meta,
1040 job_key => $params{job_key},
1041 };
1042 }
1043  
1044 sub flush_coverage {
1045 my $self = shift;
1046  
1047 my $coverage = $self->{+COVERAGE} or return;
1048 return unless @$coverage;
1049  
1050 $self->retry_on_disconnect("update has_coverage" => sub { $self->{+RUN}->update({has_coverage => 1}) })
1051 unless $self->{+RUN}->has_coverage;
1052  
1053 $self->populate_bulk(Coverage => $coverage);
1054  
1055 @$coverage = ();
1056  
1057 return;
1058 }
1059  
1060 sub _get__id {
1061 my $self = shift;
1062 my $id = $self->_get___id(@_);
1063 return $id unless defined $id;
1064 return uuid_inflate($id);
1065 }
1066  
1067 # The cache holds the deflated id, what the database stores, so a caller that
1068 # writes rows directly can use it as it is.
1069 sub _get___id {
1070 my $self = shift;
1071 my ($type, $id_field, $field, $id) = @_;
1072  
1073 return undef unless $id;
1074  
1075 return $self->{+ID_CACHE}->{$type}->{$id_field}->{$field}->{$id}
1076 if $self->{+ID_CACHE}->{$type}->{$id_field}->{$field}->{$id};
1077  
1078 my $spec = {$field => $id, $id_field => gen_uuid()};
1079 my $result = $self->schema->resultset($type)->find_or_create($spec);
1080  
1081 return $self->{+ID_CACHE}->{$type}->{$id_field}->{$field}->{$id} = uuid_deflate($result->$id_field);
1082 }
1083  
1084 sub get_test_file_id {
1085 my $self = shift;
1086 my ($file) = @_;
1087  
1088 return undef unless $file;
1089  
1090 return $self->_get__id('TestFile' => 'test_file_id', filename => $file);
1091 }
1092  
1093 # These return the deflated id, see _get___id().
1094 sub get_source_file_id {
1095 my $self = shift;
1096 my ($file) = @_;
1097  
1098 return $self->_get___id(SourceFile => 'source_file_id', filename => $file) // die "Could not get source id";
1099 }
1100  
1101 sub get_source_sub_id {
1102 my $self = shift;
1103 my ($sub) = @_;
1104  
1105 return $self->_get___id(SourceSub => 'source_sub_id', subname => $sub) // die "Could not get sub id";
1106 }
1107  
1108 sub get_coverage_manager_id {
1109 my $self = shift;
1110 my ($package) = @_;
1111  
1112 return $self->_get___id(CoverageManager => 'coverage_manager_id', package => $package);
1113 }
1114  
1115 sub add_run_fields {
1116 my $self = shift;
1117 my ($fields) = @_;
1118  
1119 my $run = $self->{+RUN};
1120 my $run_id = $run->run_id;
1121  
1122 return $self->_add_fields(
1123 fields => $fields,
1124 type => 'RunField',
1125 key_field => 'run_field_id',
1126 attrs => {run_id => $run_id},
1127 );
1128 }
1129  
1130 sub add_job_fields {
1131 my $self = shift;
1132 my ($job, $fields) = @_;
1133  
1134 my $job_key = $job->job_key;
1135  
1136 return $self->_add_fields(
1137 fields => $fields,
1138 type => 'JobField',
1139 key_field => 'job_field_id',
1140 attrs => {job_key => $job_key},
1141 );
1142 }
1143  
1144 sub _add_fields {
1145 my $self = shift;
1146 my %params = @_;
1147  
1148 my $fields = $params{fields};
1149 my $type = $params{type};
1150 my $key_field = $params{key_field};
1151 my $attrs = $params{attrs} // {};
1152  
1153 my @add;
1154 for my $field (@$fields) {
1155 my $id = gen_uuid;
1156 my $new = {%$attrs, $key_field => $id};
1157  
1158 $new->{name} = $field->{name} || 'unknown';
1159 $new->{details} = $field->{details} || $new->{name};
1160 $new->{raw} = $field->{raw} if $field->{raw};
1161 $new->{link} = $field->{link} if $field->{link};
1162 $new->{data} = encode_json($field->{data}) if $field->{data};
1163  
1164  
1165 push @add => $new;
1166  
1167 # Replace the item in the $fields array with the id
1168 $field = $id;
1169 }
1170  
1171 $self->populate($type => \@add);
1172 }
1173  
1174 sub clean_output {
1175 my $text = shift;
1176  
1177 return undef unless defined $text;
1178 $text =~ s/^T2-HARNESS-ESYNC: \d+\n//gm;
1179 chomp($text);
1180  
1181 return undef unless length($text);
1182 return $text;
1183 }
1184  
1185 sub clean {
1186 my ($s) = @_;
1187 return 0 unless defined $s;
1188 my $r = ref($_[0]) or return 1;
1189 if ($r eq 'HASH') { return clean_hash(@_) }
1190 elsif ($r eq 'ARRAY') { return clean_array(@_) }
1191 return 1;
1192 }
1193  
1194 sub clean_hash {
1195 my ($s) = @_;
1196 my $vals = 0;
1197  
1198 for my $key (keys %$s) {
1199 my $v = clean($s->{$key});
1200 if ($v) { $vals++ }
1201 else { delete $s->{$key} }
1202 }
1203  
1204 $_[0] = undef unless $vals;
1205  
1206 return $vals;
1207 }
1208  
1209 sub clean_array {
1210 my ($s) = @_;
1211  
1212 @$s = grep { clean($_) } @$s;
1213  
1214 return @$s if @$s;
1215  
1216 $_[0] = undef;
1217 return 0;
1218 }
1219  
1220 sub update_other {
1221 my $self = shift;
1222 my ($job, $f) = @_;
1223  
1224 my $run = $self->{+RUN};
1225  
1226 if (my $run_data = $f->{harness_run}) {
1227 my $settings = $run_data->{settings} //= $f->{harness_settings};
1228  
1229 if (my $j = $settings->{runner}->{job_count}) {
1230 $run->concurrency($j);
1231 }
1232  
1233 clean($run_data);
1234 $run->parameters($run_data);
1235  
1236 if (my $fields = $run_data->{harness_run_fields} // $run_data->{fields}) {
1237 $self->add_run_fields($fields);
1238 }
1239 }
1240  
1241 my $job_result = $job->{result};
1242 my %cols = $job_result->get_columns;
1243  
1244 # Handle job events
1245 if (my $job_data = $f->{harness_job}) {
1246 #$cols{test_file_id} ||= $self->get_test_file_id($job_data->{file});
1247 $cols{name} ||= $job_data->{job_name};
1248 clean($job_data);
1249 $cols{parameters} = encode_json($job_data);
1250 $f->{harness_job} = "Removed, see job with job_key $cols{job_key}";
1251 }
1252 if (my $job_exit = $f->{harness_job_exit}) {
1253 #$cols{test_file_id} ||= $self->get_test_file_id($job_exit->{file});
1254 $cols{exit_code} = $job_exit->{exit};
1255  
1256 if ($job_exit->{retry} && $job_exit->{retry} eq 'will-retry') {
1257 $cols{retry} = 1;
1258 $self->{+RETRIED}++;
1259 $self->{+FAILED}--;
1260 }
1261 else {
1262 $cols{retry} = 0;
1263 }
1264  
1265 $cols{stderr} = clean_output(delete $job_exit->{stderr});
1266 $cols{stdout} = clean_output(delete $job_exit->{stdout});
1267 }
1268 if (my $job_start = $f->{harness_job_start}) {
1269 $cols{test_file_id} ||= $self->get_test_file_id($job_start->{rel_file}) if $job_start->{rel_file};
1270 $cols{test_file_id} ||= $self->get_test_file_id($job_start->{file});
1271 $cols{start} = $self->format_stamp($job_start->{stamp});
1272 }
1273 if (my $job_launch = $f->{harness_job_launch}) {
1274 $cols{status} = 'running';
1275  
1276 $cols{test_file_id} ||= $self->get_test_file_id($job_launch->{file});
1277 $cols{launch} = $self->format_stamp($job_launch->{stamp});
1278 }
1279 if (my $job_end = $f->{harness_job_end}) {
1280 #$cols{test_file_id} ||= $self->get_test_file_id($job_end->{file});
1281 $cols{fail} ||= $job_end->{fail} ? 1 : 0;
1282 $cols{ended} = $self->format_stamp($job_end->{stamp});
1283  
1284 $cols{fail} ? $self->{+FAILED}++ : $self->{+PASSED}++;
1285  
1286 # All done
1287 $job->{done} = 1;
1288 $cols{status} = 'complete';
1289  
1290 if ($job_end->{rel_file} && $job_end->{times} && $job_end->{times}->{totals} && $job_end->{times}->{totals}->{total}) {
1291 my $tfile_id = $cols{test_file_id} ||= $self->get_test_file_id($job_end->{rel_file}) if $job_end->{rel_file};
1292  
1293 if (my $duration = $job_end->{times}->{totals}->{total}) {
1294 $job->{duration} = $duration;
1295 $cols{duration} = $duration;
1296 }
1297 }
1298 }
1299 if (my $job_fields = $f->{harness_job_fields}) {
1300 $self->add_job_fields($job_result, $job_fields);
1301 }
1302  
1303 $job_result->set_columns(\%cols);
1304  
1305 return;
1306 }
1307  
1308 1;
1309  
1310 __END__