| 1 |
|
| package Test2::Harness::UI::Dumper; |
| 2 |
|
| use strict; |
| 3 |
|
| use warnings; |
| 4 |
|
| |
| 5 |
|
| our $VERSION = '0.000150'; |
| 6 |
|
| |
| 7 |
|
| use Test2::Harness::UI::Util qw/format_duration/; |
| 8 |
|
| use Test2::Harness::Util::JSON qw/encode_json/; |
| 9 |
|
| use Test2::Harness::UI::UUID qw/uuid_inflate/; |
| 10 |
|
| use Time::HiRes qw/time/; |
| 11 |
|
| use Parallel::Runner; |
| 12 |
|
| use IO::Compress::Bzip2; |
| 13 |
|
| |
| 14 |
|
| use Test2::Harness::UI::Util::HashBase qw{ |
| 15 |
|
| <config |
| 16 |
|
| <procs |
| 17 |
|
| }; |
| 18 |
|
| |
| 19 |
|
| my @DUMP_ORDER = qw{ |
| 20 |
|
| User |
| 21 |
|
| Email |
| 22 |
|
| PrimaryEmail |
| 23 |
|
| Host |
| 24 |
|
| EmailVerificationCode |
| 25 |
|
| Session |
| 26 |
|
| SessionHost |
| 27 |
|
| ApiKey |
| 28 |
|
| LogFile |
| 29 |
|
| Project |
| 30 |
|
| Permission |
| 31 |
|
| Run |
| 32 |
|
| Sweep |
| 33 |
|
| RunField |
| 34 |
|
| TestFile |
| 35 |
|
| Job |
| 36 |
|
| JobField |
| 37 |
|
| Event |
| 38 |
|
| Binary |
| 39 |
|
| SourceFile |
| 40 |
|
| SourceSub |
| 41 |
|
| CoverageManager |
| 42 |
|
| Coverage |
| 43 |
|
| Reporting |
| 44 |
|
| ResourceBatch |
| 45 |
|
| Resource |
| 46 |
|
| }; |
| 47 |
|
| |
| 48 |
|
| sub dump { |
| 49 |
|
| my $self = shift; |
| 50 |
|
| |
| 51 |
|
| my $start_all = time; |
| 52 |
|
| my $runner = Parallel::Runner->new($self->{+PROCS}); |
| 53 |
|
| |
| 54 |
|
| my $config = $self->config; |
| 55 |
|
| my $schema = $config->schema; |
| 56 |
|
| mkdir("./dump") unless -d "./dump"; |
| 57 |
|
| |
| 58 |
|
| my %seen; |
| 59 |
|
| for my $source (@DUMP_ORDER, $schema->sources) { |
| 60 |
|
| next if $seen{$source}++; |
| 61 |
|
| |
| 62 |
|
| my $rs = $schema->resultset($source); |
| 63 |
|
| my $res = $rs->search(undef, {page => 1, rows => $ENV{PAGE_SIZE} // 2000}); |
| 64 |
|
| my $pager = $res->pager; |
| 65 |
|
| my $cols_info = $rs->result_source->columns_info; |
| 66 |
|
| |
| 67 |
|
| my $pages = $pager->last_page; |
| 68 |
|
| my $page = $pager->first_page; |
| 69 |
|
| my $len = length($pages); |
| 70 |
|
| |
| 71 |
|
| while ($page <= $pages) { |
| 72 |
|
| my $file = "./dump/${source}-" . sprintf("%0${len}d", $page) . ".jsonl.bz2"; |
| 73 |
|
| die "Dump file '$file' already exists!\n" if -e $file; |
| 74 |
|
| |
| 75 |
|
| $runner->run(sub { |
| 76 |
|
| print "$$ $source STARTED ($page/$pages) -> $file\n"; |
| 77 |
|
| my $fh = IO::Compress::Bzip2->new("./dump/${source}-" . sprintf("%0${len}d", $page) . ".jsonl.bz2") or die "Could not open log file: $IO::Compress::Bzip2::Bzip2Error"; |
| 78 |
|
| |
| 79 |
|
| my $start_page = time; |
| 80 |
|
| my $page_rs = $res->page($page); |
| 81 |
|
| my $count = 0; |
| 82 |
|
| while (my $it = $page_rs->next) { |
| 83 |
|
| $count++; |
| 84 |
|
| my %data = (); |
| 85 |
|
| for my $col (keys %$cols_info) { |
| 86 |
|
| $data{$col} = $it->get_column($col); |
| 87 |
|
| |
| 88 |
|
| my $spec = $cols_info->{$col}; |
| 89 |
|
| next if $col eq 'trace_id'; |
| 90 |
|
| next |
| 91 |
|
| unless ($spec->{data_type} eq 'uuid') |
| 92 |
|
| || ($spec->{data_type} eq 'binary' && $spec->{size} == 16) |
| 93 |
|
| || ($spec->{data_type} eq 'char' && $spec->{size} == 36); |
| 94 |
|
| |
| 95 |
|
| $data{$col} = defined($data{$col}) ? uuid_inflate($data{$col})->string : undef; |
| 96 |
|
| } |
| 97 |
|
| |
| 98 |
|
| print $fh encode_json(\%data), "\n"; |
| 99 |
|
| } |
| 100 |
|
| |
| 101 |
|
| print "$$ $source [" . format_duration(time - $start_page) . " / " . format_duration(time - $start_all) . "] ($page/$pages) +$count\n"; |
| 102 |
|
| }); |
| 103 |
|
| |
| 104 |
|
| $page++; |
| 105 |
|
| } |
| 106 |
|
| } |
| 107 |
|
| |
| 108 |
|
| $runner->finish(); |
| 109 |
|
| |
| 110 |
|
| print "\nCompleted in " . format_duration(time - $start_all) . "\n"; |
| 111 |
|
| } |
| 112 |
|
| |
| 113 |
|
| 1; |