« lib/Test2/Harness/UI/Schema/Overlay/Run.pm
↑ Summary
lib/Test2/Harness/UI/Controller/Interactions.pm »
| 1 | package Test2::Harness::UI::Loader; | ||
| 2 | use strict; | ||
| 3 | use warnings; | ||
| 4 | |||
| 5 | our $VERSION = '0.000150'; | ||
| 6 | |||
| 7 | use Test2::Harness::UI::Util qw/format_duration/; | ||
| 8 | use Test2::Harness::Util::JSON qw/decode_json/; | ||
| 9 | use Test2::Harness::UI::UUID qw/uuid_deflate/; | ||
| 10 | use Time::HiRes qw/time/; | ||
| 11 | use Parallel::Runner; | ||
| 12 | use IO::Uncompress::Bunzip2; | ||
| 13 | |||
| 14 | use Test2::Harness::UI::Util::HashBase qw{ | ||
| 15 | <config | ||
| 16 | <procs | ||
| 17 | }; | ||
| 18 | |||
| 19 | my @LOAD_ORDER = ( | ||
| 20 | [qw/User Email Host Session LogFile TestFile SourceFile SourceSub CoverageManager/], | ||
| 21 | [qw/PrimaryEmail ApiKey Project/], | ||
| 22 | [qw/EmailVerificationCode SessionHost Permission Run/], | ||
| 23 | [qw/Sweep RunField Job ResourceBatch/], | ||
| 24 | [qw/JobField Event Coverage Resource/], | ||
| 25 | [qw/Binary Reporting/], | ||
| 26 | ); | ||
| 27 | |||
| 28 | my %VALID = map { map {($_ => 1)} @{$_} } @LOAD_ORDER; | ||
| 29 | |||
| 30 | sub load { | ||
| 31 | my $self = shift; | ||
| 32 | |||
| 33 | my $start_all = time; | ||
| 34 | |||
| 35 | my %todo; | ||
| 36 | my %seq; | ||
| 37 | |||
| 38 | opendir(my $dh, './dump') or die "Could not open dump dir: $!"; | ||
| 39 | for my $file (sort readdir($dh)) { | ||
| 40 | next unless $file =~ m/^(.+)-(\d+)\.jsonl\.bz2$/; | ||
| 41 | my ($type, $seq) = ($1, $2); | ||
| 42 | die "'$type' is not a valid type.\n" unless $VALID{$type}; | ||
| 43 | $seq{$type}++; | ||
| 44 | die "Expected $seq{$type} got $seq.\n" unless $seq =~ m/^0+$/ or int($seq) eq $seq{$type}; | ||
| 45 | push @{$todo{$type} //= []} => "./dump/$file"; | ||
| 46 | } | ||
| 47 | |||
| 48 | my $s = 0; | ||
| 49 | for my $set (@LOAD_ORDER) { | ||
| 50 | my $start_set = time; | ||
| 51 | $s++; | ||
| 52 | print "\n== START SET $s ==\n\n"; | ||
| 53 | my $runner = Parallel::Runner->new($self->{+PROCS}); | ||
| 54 | |||
| 55 | for my $source (@$set) { | ||
| 56 | my $config = $self->config; | ||
| 57 | my $schema = $config->schema; | ||
| 58 | my $rs = $schema->resultset($source); | ||
| 59 | my $cols_info = $rs->result_source->columns_info; | ||
| 60 | |||
| 61 | my $i = 0; | ||
| 62 | for my $file (@{$todo{$source} //= []}) { | ||
| 63 | $i++; | ||
| 64 | my $pageinfo = "$i/" . scalar(@{$todo{$source}}); | ||
| 65 | $runner->run(sub { | ||
| 66 | my $start = time; | ||
| 67 | print "$$ $source STARTED ($pageinfo) <- $file\n"; | ||
| 68 | my $fh = IO::Uncompress::Bunzip2->new($file) or die "Could not open log file: $IO::Uncompress::Bunzip2::Bunzip2Error"; | ||
| 69 | |||
| 70 | my $count = 0; | ||
| 71 | while (my $line = <$fh>) { | ||
| 72 | chomp($line); | ||
| 73 | $count++; | ||
| 74 | |||
| 75 | my $row = decode_json($line); | ||
| 76 | |||
| 77 | for my $col (keys %$cols_info) { | ||
| 78 | my $spec = $cols_info->{$col}; | ||
| 79 | next if $col eq 'trace_id'; | ||
| 80 | next | ||
| 81 | unless ($spec->{data_type} eq 'uuid') | ||
| 82 | || ($spec->{data_type} eq 'binary' && $spec->{size} == 16) | ||
| 83 | || ($spec->{data_type} eq 'char' && $spec->{size} == 36); | ||
| 84 | |||
| 85 | $row->{$col} = uuid_deflate($row->{$col}); | ||
| 86 | } | ||
| 87 | |||
| 88 | my $ok = eval { $rs->create($row); 1 }; | ||
| 89 | my $err = $@; | ||
| 90 | next if $ok; | ||
| 91 | |||
| 92 | next if $err =~ m/Duplicate entry/ && $ENV{IGNORE_DUPLICATES}; | ||
| 93 | |||
| 94 | die $@; | ||
| 95 | } | ||
| 96 | |||
| 97 | print "$$ $source [" . format_duration(time - $start) . " / " . format_duration(time - $start_all) . "] ($pageinfo) +$count\n"; | ||
| 98 | }); | ||
| 99 | } | ||
| 100 | } | ||
| 101 | |||
| 102 | $runner->finish(); | ||
| 103 | print "\n== END SET $s " . format_duration(time - $start_set) . " / " . format_duration(time - $start_all) . "==\n"; | ||
| 104 | } | ||
| 105 | |||
| 106 | print "\nCompleted in " . format_duration(time - $start_all) . "\n"; | ||
| 107 | } | ||
| 108 | |||
| 109 | 1; |