| 1 | package Test2::Harness::UI::Controller::Stream; | ||
| 2 | use strict; | ||
| 3 | use warnings; | ||
| 4 | |||
| 5 | our $VERSION = '0.000150'; | ||
| 6 | |||
| 7 | use Data::GUID; | ||
| 8 | use List::Util qw/max/; | ||
| 9 | use Scalar::Util qw/blessed/; | ||
| 10 | use Test2::Harness::UI::Util qw/find_job/; | ||
| 11 | use Test2::Harness::UI::UUID qw/uuid_inflate/; | ||
| 12 | use Test2::Harness::UI::Response qw/resp error/; | ||
| 13 | use Test2::Harness::Util::JSON qw/encode_json/; | ||
| 14 | use JSON::PP(); | ||
| 15 | use Carp qw/croak/; | ||
| 16 | |||
| 17 | use parent 'Test2::Harness::UI::Controller'; | ||
| 18 | use Test2::Harness::UI::Util::HashBase qw{ | ||
| 19 | <run | ||
| 20 | <job | ||
| 21 | }; | ||
| 22 | |||
| 23 | use constant RUN_LIMIT => 100; | ||
| 24 | |||
| 25 | sub title { 'Stream' } | ||
| 26 | |||
| 27 | sub handle { | ||
| 28 | my $self = shift; | ||
| 29 | my ($route) = @_; | ||
| 30 | |||
| 31 | my $req = $self->{+REQUEST}; | ||
| 32 | my $res = resp(200); | ||
| 33 | |||
| 34 | my @sets = ( | ||
| 35 | $self->stream_runs($req, $route), | ||
| 36 | $self->stream_jobs($req, $route), | ||
| 37 | $self->stream_events($req, $route), | ||
| 38 | ); | ||
| 39 | |||
| 40 | my $cache = 1; | ||
| 41 | for my $it ($self->{+RUN}, $self->{+JOB}) { | ||
| 42 | next unless $it; | ||
| 43 | next if $it->complete; | ||
| 44 | $cache = 0; | ||
| 45 | last; | ||
| 46 | } | ||
| 47 | |||
| 48 | $res->stream( | ||
| 49 | env => $req->env, | ||
| 50 | content_type => 'application/x-jsonl; charset=utf-8', | ||
| 51 | cache => $cache, | ||
| 52 | |||
| 53 | done => sub { | ||
| 54 | my @keep; | ||
| 55 | while (my $set = shift @sets) { | ||
| 56 | my ($check) = @$set; | ||
| 57 | next if $check->(); # Next if done | ||
| 58 | |||
| 59 | # Not done, keep it | ||
| 60 | push @keep => $set; | ||
| 61 | } | ||
| 62 | |||
| 63 | @sets = @keep; | ||
| 64 | |||
| 65 | return @sets ? 0 : 1; | ||
| 66 | }, | ||
| 67 | |||
| 68 | fetch => sub { map { $_->[1]->() } @sets }, | ||
| 69 | ); | ||
| 70 | |||
| 71 | return $res; | ||
| 72 | } | ||
| 73 | |||
| 74 | sub stream_runs { | ||
| 75 | my $self = shift; | ||
| 76 | my ($req, $route) = @_; | ||
| 77 | |||
| 78 | my $schema = $self->{+CONFIG}->schema; | ||
| 79 | |||
| 80 | my $opts = { | ||
| 81 | remove_columns => [qw/log_data run_fields.data parameters/], | ||
| 82 | |||
| 83 | join => [qw/user_join project run_fields/], | ||
| 84 | '+columns' => { | ||
| 85 | 'prefetched_fields' => \'1', | ||
| 86 | 'run_fields.run_field_id' => 'run_fields.run_field_id', | ||
| 87 | 'run_fields.name' => 'run_fields.name', | ||
| 88 | 'run_fields.details' => 'run_fields.details', | ||
| 89 | 'run_fields.raw' => 'run_fields.raw', | ||
| 90 | 'run_fields.link' => 'run_fields.link', | ||
| 91 | 'run_fields.data', => \"run_fields.data IS NOT NULL", | ||
| 92 | 'user' => \'user_join.username', | ||
| 93 | 'project' => \'project.name', | ||
| 94 | }, | ||
| 95 | }; | ||
| 96 | |||
| 97 | my %params = ( | ||
| 98 | type => 'run', | ||
| 99 | |||
| 100 | req => $req, | ||
| 101 | |||
| 102 | track_status => 1, | ||
| 103 | id_field => 'run_id', | ||
| 104 | ord_field => 'added', | ||
| 105 | sort_field => 'added', | ||
| 106 | search_base => $schema->resultset('Run'), | ||
| 107 | initial_limit => RUN_LIMIT, | ||
| 108 | |||
| 109 | custom_opts => $opts, | ||
| 110 | |||
| 111 | timeout => 60 * 30, # 30 min. | ||
| 112 | ); | ||
| 113 | |||
| 114 | my $id = $route->{id}; | ||
| 115 | my $uuid = uuid_inflate($id); | ||
| 116 | my $run_id = $route->{run_id}; | ||
| 117 | my ($project, $user); | ||
| 118 | |||
| 119 | if ($id) { | ||
| 120 | my $p_rs = $schema->resultset('Project'); | ||
| 121 | $project //= eval { $p_rs->search({name => $id})->first }; | ||
| 122 | $project //= eval { $p_rs->search({project_id => $uuid})->first }; | ||
| 123 | |||
| 124 | if ($project) { | ||
| 125 | $uuid = uuid_inflate($project->project_id); | ||
| 126 | $params{search_base} = $params{search_base}->search_rs({'me.project_id' => $project->project_id}); | ||
| 127 | } | ||
| 128 | else { | ||
| 129 | my $u_rs = $schema->resultset('User'); | ||
| 130 | $user //= eval { $u_rs->search({username => $id})->first }; | ||
| 131 | $user //= eval { $u_rs->search({user_id => $uuid})->first }; | ||
| 132 | |||
| 133 | if ($user) { | ||
| 134 | $uuid = uuid_inflate($user->user_id); | ||
| 135 | $params{search_base} = $params{search_base}->search_rs({'me.user_id' => $user->user_id}); | ||
| 136 | } | ||
| 137 | else { | ||
| 138 | $run_id //= $uuid; | ||
| 139 | } | ||
| 140 | } | ||
| 141 | } | ||
| 142 | |||
| 143 | if($run_id) { | ||
| 144 | $run_id = uuid_inflate($run_id) or die error(404 => "Invalid run id"); | ||
| 145 | return $self->stream_single(%params, id => $run_id); | ||
| 146 | } | ||
| 147 | |||
| 148 | return $self->stream_set(%params); | ||
| 149 | } | ||
| 150 | |||
| 151 | sub stream_jobs { | ||
| 152 | my $self = shift; | ||
| 153 | my ($req, $route) = @_; | ||
| 154 | |||
| 155 | my $run = $self->{+RUN} // return; | ||
| 156 | |||
| 157 | my $opts = { | ||
| 158 | join => 'test_file', | ||
| 159 | remove_columns => [qw/stdout stderr parameters/], | ||
| 160 | '+select' => [ | ||
| 161 | 'test_file.filename AS file', | ||
| 162 | ], | ||
| 163 | '+as' => [ | ||
| 164 | 'file', | ||
| 165 | ], | ||
| 166 | }; | ||
| 167 | |||
| 168 | my %params = ( | ||
| 169 | type => 'job', | ||
| 170 | parent => $run, | ||
| 171 | |||
| 172 | req => $req, | ||
| 173 | |||
| 174 | track_status => 1, | ||
| 175 | id_field => 'job_key', | ||
| 176 | ord_field => 'job_ord', | ||
| 177 | method => 'glance_data', | ||
| 178 | search_base => scalar($run->jobs), | ||
| 179 | custom_opts => $opts, | ||
| 180 | |||
| 181 | order_by => [{'-desc' => 'status'}, {'-desc' => [qw/job_try job_ord name/]}], | ||
| 182 | ); | ||
| 183 | |||
| 184 | if (my $job_uuid = $route->{job}) { | ||
| 185 | $job_uuid = uuid_inflate($job_uuid) or die error(404 => "Invalid job id"); | ||
| 186 | |||
| 187 | my $schema = $self->{+CONFIG}->schema; | ||
| 188 | return $self->stream_single(%params, item => find_job($schema, $job_uuid, $route->{try})); | ||
| 189 | } | ||
| 190 | |||
| 191 | return $self->stream_set(%params); | ||
| 192 | } | ||
| 193 | |||
| 194 | sub stream_events { | ||
| 195 | my $self = shift; | ||
| 196 | my ($req, $route) = @_; | ||
| 197 | |||
| 198 | my $job = $self->{+JOB} // return; | ||
| 199 | |||
| 200 | # we only stream nested events when the job is still running | ||
| 201 | my $query = $job->complete ? {nested => 0} : undef; | ||
| 202 | |||
| 203 | my $opts = { | ||
| 204 | remove_columns => ['orphan'], | ||
| 205 | '+select' => [ | ||
| 206 | 'facets IS NOT NULL AS has_facets', | ||
| 207 | 'orphan IS NOT NULL AS has_orphan', | ||
| 208 | ], | ||
| 209 | '+as' => [ | ||
| 210 | 'has_facets', | ||
| 211 | 'has_orphan', | ||
| 212 | ], | ||
| 213 | }; | ||
| 214 | |||
| 215 | return $self->stream_set( | ||
| 216 | type => 'event', | ||
| 217 | parent => $job, | ||
| 218 | |||
| 219 | req => $req, | ||
| 220 | |||
| 221 | track_status => 0, | ||
| 222 | id_field => 'event_id', | ||
| 223 | ord_field => 'insert_ord', | ||
| 224 | sort_field => 'event_ord', | ||
| 225 | sort_dir => '-asc', | ||
| 226 | method => 'line_data', | ||
| 227 | custom_query => $query, | ||
| 228 | custom_opts => $opts, | ||
| 229 | search_base => scalar($job->events), | ||
| 230 | ); | ||
| 231 | } | ||
| 232 | |||
| 233 | sub stream_single { | ||
| 234 | my $self = shift; | ||
| 235 | my %params = @_; | ||
| 236 | |||
| 237 | my $id_field = $params{id_field}; | ||
| 238 | my $method = $params{method}; | ||
| 239 | my $search_base = $params{search_base}; | ||
| 240 | my $type = $params{type}; | ||
| 241 | my $id = $params{id}; | ||
| 242 | my $custom_opts = $params{custom_opts} // {}; | ||
| 243 | my $custom_query = $params{custom_query} // {}; | ||
| 244 | |||
| 245 | my $it; | ||
| 246 | if (exists $params{item}) { | ||
| 247 | $it = $params{item} or die error(404 => "Invalid Item"); | ||
| 248 | } | ||
| 249 | else { | ||
| 250 | $it = $search_base->search({%$custom_query, "me.$id_field" => $id}, $custom_opts)->first or die error(404 => "Invalid $type"); | ||
| 251 | } | ||
| 252 | $self->{$type} = $it; | ||
| 253 | |||
| 254 | my $sig; | ||
| 255 | return [ | ||
| 256 | sub { $sig && $it->complete ? 1 : 0 }, | ||
| 257 | sub { | ||
| 258 | my $update = JSON::PP::false; | ||
| 259 | if ($sig) { | ||
| 260 | $it->discard_changes; | ||
| 261 | $update = JSON::PP::true; | ||
| 262 | } | ||
| 263 | |||
| 264 | my $new_sig = $it->sig; | ||
| 265 | my $unchanged = $sig && $sig eq $new_sig; | ||
| 266 | $sig = $new_sig; | ||
| 267 | |||
| 268 | return if $unchanged; | ||
| 269 | |||
| 270 | my $data = $method ? $it->$method : $it->TO_JSON; | ||
| 271 | my $json; | ||
| 272 | local $@; | ||
| 273 | unless (eval { $json = encode_json({type => $type, update => $update, data => $data}); 1 }) { | ||
| 274 | warn "Failed to encode $type: $@"; | ||
| 275 | return; | ||
| 276 | } | ||
| 277 | return "$json\n"; | ||
| 278 | }, | ||
| 279 | ]; | ||
| 280 | } | ||
| 281 | |||
| 282 | sub stream_set { | ||
| 283 | my $self = shift; | ||
| 284 | my (%params) = @_; | ||
| 285 | |||
| 286 | my $custom_opts = $params{custom_opts} // {}; | ||
| 287 | my $custom_query = $params{custom_query} // undef; | ||
| 288 | my $id_field = $params{id_field}; | ||
| 289 | my $limit = $params{initial_limit}; | ||
| 290 | my $method = $params{method}; | ||
| 291 | my $ord_field = $params{ord_field}; | ||
| 292 | my $parent = $params{parent}; | ||
| 293 | my $search_base = $params{search_base}; | ||
| 294 | my $sort_field = $params{sort_field}; | ||
| 295 | my $sort_dir = $params{sort_dir} // '-desc'; | ||
| 296 | my $timeout = $params{timeout}; | ||
| 297 | my $track = $params{track_status}; | ||
| 298 | my $type = $params{type}; | ||
| 299 | |||
| 300 | my $order_by = $params{order_by} // $sort_field ? {$sort_dir => $sort_field} : croak "Must specify either 'order_by' or 'sort_field'"; | ||
| 301 | |||
| 302 | my $items = $search_base->search($custom_query, {%$custom_opts, order_by => $order_by, $limit ? (rows => $limit) : ()}); | ||
| 303 | |||
| 304 | my $start = time; | ||
| 305 | my $ord; | ||
| 306 | my $incomplete = {}; | ||
| 307 | |||
| 308 | return [ | ||
| 309 | sub { | ||
| 310 | return 0 if $items; | ||
| 311 | return 0 if $track && keys %$incomplete; | ||
| 312 | |||
| 313 | return 1 if $parent && $parent->complete; | ||
| 314 | |||
| 315 | my $running = time - $start; | ||
| 316 | return 1 if $timeout && $running > $timeout; # Stop if they have been camping the page for 30 min. | ||
| 317 | |||
| 318 | return 0; | ||
| 319 | }, | ||
| 320 | sub { | ||
| 321 | unless ($items) { | ||
| 322 | my $val; | ||
| 323 | if (blessed($ord) && $ord->isa('DateTime')) { | ||
| 324 | my $schema = $self->{+CONFIG}->schema; | ||
| 325 | my $dtf = $schema->storage->datetime_parser; | ||
| 326 | $val = $dtf->format_datetime($ord); | ||
| 327 | } | ||
| 328 | else { | ||
| 329 | $val = $ord; | ||
| 330 | } | ||
| 331 | |||
| 332 | my $query = { | ||
| 333 | ($custom_query ? %$custom_query : ()), | ||
| 334 | $ord_field => {'>' => $val}, | ||
| 335 | }; | ||
| 336 | |||
| 337 | my @ids = $track ? keys %$incomplete : (); | ||
| 338 | $query = [$query, {"me.$id_field" => {'IN' => \@ids}}] if @ids; | ||
| 339 | |||
| 340 | $items = $search_base->search( | ||
| 341 | $query, | ||
| 342 | {%$custom_opts, order_by => $order_by} | ||
| 343 | ); | ||
| 344 | } | ||
| 345 | |||
| 346 | my @batch; | ||
| 347 | while (my $item = $items->next()) { | ||
| 348 | $ord = $item->$ord_field; | ||
| 349 | |||
| 350 | my $update = JSON::PP::false; | ||
| 351 | if ($track) { | ||
| 352 | my $id = $item->$id_field; | ||
| 353 | if (my $old = $incomplete->{$id}) { | ||
| 354 | $update = JSON::PP::true; | ||
| 355 | # Nothing has changed, no need to send it. | ||
| 356 | next if $old->sig eq $item->sig; | ||
| 357 | } | ||
| 358 | |||
| 359 | if ($item->complete) { | ||
| 360 | delete $incomplete->{$id}; | ||
| 361 | } | ||
| 362 | else { | ||
| 363 | $incomplete->{$id} = $item; | ||
| 364 | } | ||
| 365 | } | ||
| 366 | |||
| 367 | my $data = $method ? $item->$method : $item->TO_JSON; | ||
| 368 | my $json; | ||
| 369 | local $@; | ||
| 370 | unless (eval { $json = encode_json({type => $type, update => $update, data => $data}); 1 }) { | ||
| 371 | warn "Failed to encode $type: $@"; | ||
| 372 | next; | ||
| 373 | } | ||
| 374 | push @batch => "$json\n"; | ||
| 375 | last if @batch >= 50; | ||
| 376 | } | ||
| 377 | |||
| 378 | return join('', @batch) if @batch; | ||
| 379 | |||
| 380 | $items = undef; | ||
| 381 | return; | ||
| 382 | }, | ||
| 383 | ]; | ||
| 384 | } | ||
| 385 | |||
| 386 | 1; | ||
| 387 | |||
| 388 | __END__ |