| 1 | package Test2::Harness::UI::Sweeper; | ||
| 2 | use strict; | ||
| 3 | use warnings; | ||
| 4 | use Time::HiRes qw/time/; | ||
| 5 | use Test2::Harness::UI::UUID qw/gen_uuid/; | ||
| 6 | |||
| 7 | our $VERSION = '0.000150'; | ||
| 8 | |||
| 9 | use Test2::Harness::UI::Util::HashBase qw{ | ||
| 10 | <config | ||
| 11 | <interval | ||
| 12 | }; | ||
| 13 | |||
| 14 | my @TABLES = qw{ | ||
| 15 | api_keys coverage coverage_manager email email_verification_codes events | ||
| 16 | job_fields jobs log_files permissions primary_email projects run_fields | ||
| 17 | runs session_hosts sessions source_files source_subs test_files users | ||
| 18 | }; | ||
| 19 | |||
| 20 | sub sweep { | ||
| 21 | my $self = shift; | ||
| 22 | my %params = @_; | ||
| 23 | |||
| 24 | $params{runs} //= 1; | ||
| 25 | $params{jobs} //= 1; | ||
| 26 | $params{coverage} //= 1; | ||
| 27 | $params{events} //= 1; | ||
| 28 | $params{resources} //= 1; | ||
| 29 | $params{subtests} //= 1; | ||
| 30 | $params{run_fields} //= 1; | ||
| 31 | $params{job_fields} //= 1; | ||
| 32 | $params{run_concurrency} //= $ENV{YATHUI_SWEEPER_RUN_CONCURRENCY} // 1; | ||
| 33 | $params{job_concurrency} //= $ENV{YATHUI_SWEEPER_JOB_CONCURRENCY} // 1; | ||
| 34 | $params{sweep_name} //= $ENV{YATHUI_SWEEPER_NAME}; | ||
| 35 | |||
| 36 | # Cannot remove jobs if we keep events | ||
| 37 | $params{jobs} = 0 unless $params{events} && $params{subtests} && $params{job_fields}; | ||
| 38 | |||
| 39 | # Cannot delete runs if we save jobs or coverage | ||
| 40 | $params{runs} = 0 unless $params{jobs} && $params{coverage} && $params{run_fields} && $params{resources}; | ||
| 41 | |||
| 42 | my $db_type = $self->config->guess_db_driver; | ||
| 43 | |||
| 44 | my $interval = $params{interval} || $self->{+INTERVAL}; | ||
| 45 | if ($db_type eq 'PostgreSQL') { | ||
| 46 | $interval = "< NOW() - INTERVAL '$interval'"; | ||
| 47 | } | ||
| 48 | elsif ($db_type =~ m/mysql/i) { | ||
| 49 | $interval = "< NOW() - INTERVAL $interval"; | ||
| 50 | } | ||
| 51 | else { | ||
| 52 | die "Not sure how to format interval for '$db_type'"; | ||
| 53 | } | ||
| 54 | |||
| 55 | my @sweep; | ||
| 56 | if (my $sweep_name = $params{sweep_name}) { | ||
| 57 | my $subquery = $self->config->schema->resultset('Sweep')->search({ | ||
| 58 | name => $sweep_name, | ||
| 59 | }); | ||
| 60 | |||
| 61 | push @sweep => ( | ||
| 62 | run_id => { '-not_in' => $subquery->get_column('run_id')->as_query } | ||
| 63 | ); | ||
| 64 | } | ||
| 65 | |||
| 66 | print "Finding runs ...\n"; | ||
| 67 | my $runs = $self->config->schema->resultset('Run')->search( | ||
| 68 | { | ||
| 69 | pinned => 'false', | ||
| 70 | added => \$interval, | ||
| 71 | @sweep, | ||
| 72 | }, | ||
| 73 | ); | ||
| 74 | |||
| 75 | my $counter = 0; | ||
| 76 | if ($params{run_concurrency} <= 1) { | ||
| 77 | while (my $run = $runs->next()) { | ||
| 78 | $self->sweep_run($run, %params, id => $counter++); | ||
| 79 | } | ||
| 80 | } | ||
| 81 | else { | ||
| 82 | require Parallel::Runner; | ||
| 83 | my $runner = Parallel::Runner->new($params{run_concurrency}); | ||
| 84 | |||
| 85 | while (my $run = $runs->next()) { | ||
| 86 | my $id = $counter++; | ||
| 87 | $runner->run(sub { | ||
| 88 | $self->config->schema->storage->disconnect; | ||
| 89 | $self->sweep_run($run, %params, id => $id); | ||
| 90 | }); | ||
| 91 | } | ||
| 92 | |||
| 93 | $runner->finish; | ||
| 94 | } | ||
| 95 | |||
| 96 | if ($db_type =~ m/mysql/i) { | ||
| 97 | my $dbh = $self->config->schema->storage->dbh; | ||
| 98 | $dbh->do('ANALYZE TABLE ' . join ', ' => @TABLES); | ||
| 99 | } | ||
| 100 | } | ||
| 101 | |||
| 102 | sub sweep_run { | ||
| 103 | my $self = shift; | ||
| 104 | my ($run, %params) = @_; | ||
| 105 | |||
| 106 | local $0 = "$0 - $params{id}: " . $run->run_id; | ||
| 107 | |||
| 108 | my $start = time; | ||
| 109 | |||
| 110 | my $jobs = $run->jobs; | ||
| 111 | |||
| 112 | my $counter = 0; | ||
| 113 | if ($params{job_concurrency} <= 1) { | ||
| 114 | while (my $job = $jobs->next()) { | ||
| 115 | $counter++; | ||
| 116 | $self->sweep_job($run, $job, %params); | ||
| 117 | } | ||
| 118 | } | ||
| 119 | else { | ||
| 120 | require Parallel::Runner; | ||
| 121 | my $runner = Parallel::Runner->new($params{job_concurrency}); | ||
| 122 | |||
| 123 | while (my $job = $jobs->next()) { | ||
| 124 | $counter++; | ||
| 125 | $runner->run(sub { | ||
| 126 | $self->config->schema->storage->disconnect; | ||
| 127 | $self->sweep_job($run, $job, %params); | ||
| 128 | }); | ||
| 129 | } | ||
| 130 | |||
| 131 | $runner->finish; | ||
| 132 | } | ||
| 133 | |||
| 134 | if ($params{run_fields}) { | ||
| 135 | $run->run_fields->delete; | ||
| 136 | } | ||
| 137 | |||
| 138 | if ($params{coverage}) { | ||
| 139 | $run->coverages->delete; | ||
| 140 | $run->update({has_coverage => 0}) unless $params{runs}; | ||
| 141 | } | ||
| 142 | |||
| 143 | if ($params{resources}) { | ||
| 144 | my $batches = $run->resource_batches; | ||
| 145 | while (my $batch = $batches->next) { | ||
| 146 | $batch->resources->delete; | ||
| 147 | $batch->delete; | ||
| 148 | } | ||
| 149 | } | ||
| 150 | |||
| 151 | if ($params{runs}) { | ||
| 152 | $run->reportings->delete; | ||
| 153 | $run->sweeps->delete; | ||
| 154 | $run->delete; | ||
| 155 | } | ||
| 156 | elsif (my $sweep_name = $params{sweep_name}) { | ||
| 157 | $self->config->schema->resultset('Sweep')->find_or_create({name => $sweep_name, run_id => $run->run_id, sweep_id => gen_uuid}); | ||
| 158 | } | ||
| 159 | |||
| 160 | print "[$$] ($params{id}) Run: " . $run->run_id . " took " . sprintf("%0.4f", time - $start) . " seconds to sweep $counter job(s).\n"; | ||
| 161 | } | ||
| 162 | |||
| 163 | sub sweep_job { | ||
| 164 | my $self = shift; | ||
| 165 | my ($run, $job, %params) = @_; | ||
| 166 | |||
| 167 | local $0 = "$0 > " . $job->job_key; | ||
| 168 | |||
| 169 | my $start = time; | ||
| 170 | |||
| 171 | if ($params{events}) { | ||
| 172 | if ($params{subtests}) { | ||
| 173 | $job->reportings->update({event_id => undef}); | ||
| 174 | |||
| 175 | my $has_binary = $job->events->search({has_binary => 1}); | ||
| 176 | while (my $e = $has_binary->next()) { | ||
| 177 | $e->binaries->delete; | ||
| 178 | $e->delete; | ||
| 179 | } | ||
| 180 | |||
| 181 | $job->events->delete; | ||
| 182 | } | ||
| 183 | else { | ||
| 184 | my $events = $job->events->search({'-not' => {is_subtest => 1, nested => 0}}); | ||
| 185 | while (my $e = $events->next()) { | ||
| 186 | $e->binaries->delete; | ||
| 187 | $e->delete; | ||
| 188 | } | ||
| 189 | } | ||
| 190 | } | ||
| 191 | |||
| 192 | if ($params{job_fields}) { | ||
| 193 | $job->job_fields->delete; | ||
| 194 | } | ||
| 195 | |||
| 196 | if ($params{jobs}) { | ||
| 197 | $job->reportings->delete; | ||
| 198 | $job->delete; | ||
| 199 | } | ||
| 200 | |||
| 201 | return unless (time - $start) > 1; | ||
| 202 | print "[$$] ($params{id}) Run: " . $run->run_id . " job " . $job->job_key . " took " . sprintf("%0.4f", time - $start) . " seconds\n"; | ||
| 203 | } | ||
| 204 | |||
| 205 | 1; | ||
| 206 |