lib/Test2/Harness/UI/Loader.pm untested

filter: Statementstmt 0.0% 0 / 97 Branchbran 0.0% 0 / 14 Conditioncond 0.0% 0 / 26 MC/DCmcdc 0.0% 0 / 15 Subroutinesub 0.0% 0 / 10 Podpod 0.0% 0 / 1 totaltotal 0.0% 0 / 163 CC 8 SCAR 35.5
CC
8
Cov
0%
load
72.0
BEGIN
2.0
BEGIN
2.0
CRAP 34.9
1 package Test2::Harness::UI::Loader;
2 use strict;
3 use warnings;
4  
5 our $VERSION = '0.000150';
6  
7 use Test2::Harness::UI::Util qw/format_duration/;
8 use Test2::Harness::Util::JSON qw/decode_json/;
9 use Test2::Harness::UI::UUID qw/uuid_deflate/;
10 use Time::HiRes qw/time/;
11 use Parallel::Runner;
12 use IO::Uncompress::Bunzip2;
13  
14 use Test2::Harness::UI::Util::HashBase qw{
15 <config
16 <procs
17 };
18  
19 my @LOAD_ORDER = (
20 [qw/User Email Host Session LogFile TestFile SourceFile SourceSub CoverageManager/],
21 [qw/PrimaryEmail ApiKey Project/],
22 [qw/EmailVerificationCode SessionHost Permission Run/],
23 [qw/Sweep RunField Job ResourceBatch/],
24 [qw/JobField Event Coverage Resource/],
25 [qw/Binary Reporting/],
26 );
27  
28 my %VALID = map { map {($_ => 1)} @{$_} } @LOAD_ORDER;
29  
30 sub load {
31 my $self = shift;
32  
33 my $start_all = time;
34  
35 my %todo;
36 my %seq;
37  
38 opendir(my $dh, './dump') or die "Could not open dump dir: $!";
39 for my $file (sort readdir($dh)) {
40 next unless $file =~ m/^(.+)-(\d+)\.jsonl\.bz2$/;
41 my ($type, $seq) = ($1, $2);
42 die "'$type' is not a valid type.\n" unless $VALID{$type};
43 $seq{$type}++;
44 die "Expected $seq{$type} got $seq.\n" unless $seq =~ m/^0+$/ or int($seq) eq $seq{$type};
45 push @{$todo{$type} //= []} => "./dump/$file";
46 }
47  
48 my $s = 0;
49 for my $set (@LOAD_ORDER) {
50 my $start_set = time;
51 $s++;
52 print "\n== START SET $s ==\n\n";
53 my $runner = Parallel::Runner->new($self->{+PROCS});
54  
55 for my $source (@$set) {
56 my $config = $self->config;
57 my $schema = $config->schema;
58 my $rs = $schema->resultset($source);
59 my $cols_info = $rs->result_source->columns_info;
60  
61 my $i = 0;
62 for my $file (@{$todo{$source} //= []}) {
63 $i++;
64 my $pageinfo = "$i/" . scalar(@{$todo{$source}});
65 $runner->run(sub {
66 my $start = time;
67 print "$$ $source STARTED ($pageinfo) <- $file\n";
68 my $fh = IO::Uncompress::Bunzip2->new($file) or die "Could not open log file: $IO::Uncompress::Bunzip2::Bunzip2Error";
69  
70 my $count = 0;
71 while (my $line = <$fh>) {
72 chomp($line);
73 $count++;
74  
75 my $row = decode_json($line);
76  
77 for my $col (keys %$cols_info) {
78 my $spec = $cols_info->{$col};
79 next if $col eq 'trace_id';
80 next
81 unless ($spec->{data_type} eq 'uuid')
82 || ($spec->{data_type} eq 'binary' && $spec->{size} == 16)
83 || ($spec->{data_type} eq 'char' && $spec->{size} == 36);
84  
85 $row->{$col} = uuid_deflate($row->{$col});
86 }
87  
88 my $ok = eval { $rs->create($row); 1 };
89 my $err = $@;
90 next if $ok;
91  
92 next if $err =~ m/Duplicate entry/ && $ENV{IGNORE_DUPLICATES};
93  
94 die $@;
95 }
96  
97 print "$$ $source [" . format_duration(time - $start) . " / " . format_duration(time - $start_all) . "] ($pageinfo) +$count\n";
98 });
99 }
100 }
101  
102 $runner->finish();
103 print "\n== END SET $s " . format_duration(time - $start_set) . " / " . format_duration(time - $start_all) . "==\n";
104 }
105  
106 print "\nCompleted in " . format_duration(time - $start_all) . "\n";
107 }
108  
109 1;