Mojolicious/Plugin/BoardStreams.pm untested

filter: Statementstmt 0.0% 0 / 548 Branchbran 0.0% 0 / 160 Conditioncond 0.0% 0 / 164 MC/DCmcdc 0.0% 0 / 94 Subroutinesub 0.0% 0 / 26 Podpod 0.0% 0 / 4 totaltotal 0.0% 0 / 996 CC 82 SCAR 85.2
CC
82
Cov
0%
register
6642.0
stop_gracefully
6.0
get_pg_channel_name
2.0
CRAP 5028.6
1 package Mojolicious::Plugin::BoardStreams;
2  
3 use Mojo::Base 'Mojolicious::Plugin', -signatures, -async_await;
4  
5 use BoardStreams::Registry;
6 use BoardStreams::DBUtil qw/
7 query_throwing_exception_object_p exists_p
8 row_exists query_throwing_exception_object
9 /;
10 use BoardStreams::Util qw/
11 trim :bool unique_id hashify next_tick_p sleep_p
12 encode_json decode_json
13 /;
14 use BoardStreams::Exceptions qw/ jsonrpc_error /;
15 use BoardStreams::REs;
16 use BoardStreams::DBMigrations;
17  
18 use Mojo::IOLoop;
19 use RxPerl::Mojo ':all';
20 use Syntax::Keyword::Try;
21 use Syntax::Keyword::Dynamically;
22 use Safe::Isa;
23 use Text::Trim;
24 use Struct::Diff 'diff';
25 use List::AllUtils 'pairs', 'min';
26 use Crypt::Digest::SHA256 'sha256_b64u';
27 use Storable 'dclone';
28 use Carp 'croak';
29  
30 no autovivification;
31  
32 use experimental 'isa';
33  
34 use constant UNIQUE_STREAM_NAME_INDEX => 'bs_streams_uidx_name';
35 use constant {
36 DEFAULT_HEARTBEAT_INTERVAL => 5,
37 DEFAULT_REPAIR_INTERVAL => 40,
38 DEFAULT_PING_INTERVAL => 15,
39 DEFAULT_NOTIFY_SIZE_LIMIT => 8_000,
40 MAX_WEBSOCKET_SIZE => 262_144,
41 };
42  
43 our $VERSION = "v0.0.36";
44  
45 has _listener_observables => sub { +{} };
46  
47 my $MAX_WEBSOCKET_SIZE = MAX_WEBSOCKET_SIZE;
48  
49 our @CARP_NOT;
50  
51 sub register ($self, $app, $config) {
52 # config options
53 my $HEARTBEAT_INTERVAL = $config->{heartbeat_interval} // DEFAULT_HEARTBEAT_INTERVAL;
54 my $REPAIR_INTERVAL = $config->{repair_interval} // DEFAULT_REPAIR_INTERVAL;
55 my $PING_INTERVAL = $config->{ping_interval} // DEFAULT_PING_INTERVAL;
56 my $NOTIFY_SIZE_LIMIT = ($config->{notify_size_limit} // DEFAULT_NOTIFY_SIZE_LIMIT) - 1;
57  
58 my $pg = $config->{Pg};
59 my $registry = BoardStreams::Registry->new;
60 my $worker_id; # UUID
61 my $am_repairer; # is this worker process the repairer?
62 my $is_finished; # is this worker process shutting down?
63 my $can_accept_clients = 0;
64  
65 # Migrate database to newest schema
66 BoardStreams::DBMigrations->apply_migrations($pg);
67  
68 $app->helper('bs.worker_id' => sub ($c) { $worker_id });
69  
70 my sub stop_gracefully {
71 Mojo::IOLoop->stop_gracefully unless $is_finished;
72 }
73  
74 my sub get_pg_channel_name ($stream_name) {
75 return sha256_b64u("boardstreams.$stream_name");
76 }
77  
78 my $on_client_finish_sub = async sub ($c) {
79 $registry->is_conn_registered($c) or return;
80 $registry->unregister_conn($c);
81 await first_value_from(
82 $registry->pending_joins->{$c}->pipe(
83 op_filter(sub { $_ == 0 }),
84 ),
85 );
86 await next_tick_p; # to allow pending join handlers to send their return values
87 my $streams_and_counts_of_conn = $registry->get_streams_and_counts_of_conn($c) or return;
88  
89 my $db = dynamically $BoardStreams::db = $pg->db;
90  
91 PAIR:
92 foreach my $stream_name (keys %$streams_and_counts_of_conn) {
93 $registry->is_member_of($c, $stream_name) or next PAIR;
94  
95 my $tx = $db->begin;
96  
97 await exists_p(
98 $db,
99 'bs_streams',
100 { name => $stream_name },
101 { for => 'update' },
102 ) or next PAIR;
103  
104 $registry->is_member_of($c, $stream_name) or next PAIR;
105  
106 if (my $leave_sub = $registry->get_leave($stream_name)) {
107 my $left_completely;
108 while (! $left_completely) {
109 $left_completely = $registry->remove_membership($c, $stream_name)
110 and (delete $registry->conn_subscriptions->{$c}{$stream_name})->unsubscribe;
111  
112 try {
113 my $result = $leave_sub->($c, $stream_name);
114 await $result if $result->$_can('then');
115 } catch ($e) {
116 $c->log->fatal("Couldn't execute leave sub, stopping worker: $e");
117 # so as to not leave the streams in an inconsistent state
118 stop_gracefully();
119 };
120 }
121 } else {
122 1 until $registry->remove_membership($c, $stream_name);
123 (delete $registry->conn_subscriptions->{$c}{$stream_name})->unsubscribe;
124 }
125  
126 $tx->commit;
127 }
128 };
129  
130 # all workers execute this at the beginning
131 Mojo::IOLoop->next_tick(sub {
132 $worker_id = unique_id;
133  
134 # set $can_accept_clients
135 {
136 my $pubsub_connected_o = rx_merge(
137 rx_from_event($pg->pubsub, 'reconnect')->pipe(op_map_to(1)),
138 rx_from_event($pg->pubsub, 'disconnect')->pipe(op_map_to(0)),
139 )->pipe(op_start_with(0));
140  
141 # stop & kick everyone on these events
142 rx_merge(
143 # if disconnect from pg
144 $pubsub_connected_o->pipe(
145 op_pairwise(),
146 op_filter(sub {
147 my ($prev, $curr) = @$_;
148 return $prev && ! $curr;
149 })
150 ),
151  
152 # if not connected to pg the first three seconds
153 rx_timer(3)->pipe(
154 op_take_until(
155 $pubsub_connected_o->pipe(op_filter(sub { $_ })),
156 ),
157 ),
158 )->subscribe(sub {
159 $can_accept_clients = 0;
160 stop_gracefully();
161 });
162  
163 # allow connections on...
164 $pubsub_connected_o->pipe(
165 op_filter(sub { $_ }),
166 )->subscribe(sub { $can_accept_clients = 1 });
167 }
168  
169 # this is to... (?)
170 $pg->pubsub->listen('boardstreams.dummy' => sub {});
171  
172 # create worker row
173 $pg->db->insert(
174 'bs_workers',
175 {
176 id => $worker_id,
177 dt_heartbeat => \'CURRENT_TIMESTAMP',
178 },
179 { on_conflict => undef },
180 )->rows or stop_gracefully();
181  
182 # store heartbeat, shutdown if worker row is missing
183 rx_timer(rand($HEARTBEAT_INTERVAL), $HEARTBEAT_INTERVAL)->subscribe(async sub {
184 # update heartbeat or stop and finish
185 (await $pg->db->update_p(
186 'bs_workers',
187 { dt_heartbeat => \'CURRENT_TIMESTAMP' },
188 { id => $worker_id },
189 ))->rows or stop_gracefully();
190 });
191  
192 # elect repairer if needed, and repair
193 rx_timer(rand($REPAIR_INTERVAL), $REPAIR_INTERVAL)->subscribe(async sub {
194 my $db = $pg->db;
195  
196 # revolt against absent ruler
197 await $db->delete_p(
198 'bs_workers',
199 {
200 is_repairer => 1,
201 dt_heartbeat => { '<', \["NOW() - INTERVAL '1 SECOND' * ?", 2 * $HEARTBEAT_INTERVAL] }
202 }
203 );
204  
205 my $repairer_row = (await $db->select_p(
206 'bs_workers',
207 'id',
208 { is_repairer => 1 },
209 ))->hashes->[0];
210  
211 if ($repairer_row) {
212 $am_repairer = $repairer_row->{id} eq $worker_id;
213 } else {
214 $am_repairer = 1 if (await $db->update_p(
215 'bs_workers',
216 { is_repairer => 1 },
217 { id => $worker_id },
218 { on_conflict => undef },
219 ))->rows;
220 }
221  
222 if ($am_repairer) {
223 await $app->bs->repair_p;
224 }
225 });
226  
227 # on ioloop finish, make clients leave and remove worker row
228 rx_from_event(Mojo::IOLoop->singleton, 'finish')->pipe(
229 op_take(1),
230 )->subscribe(async sub {
231 $is_finished = 1;
232 # this is to make clients leave their streams before before deleting worker row, to avoid
233 # having the repairer repairing w/o reason
234 $_->finish foreach $registry->get_all_conns->@*;
235 foreach my $conn ($registry->get_all_conns->@*) {
236 await $on_client_finish_sub->($conn);
237 };
238  
239 await $pg->db->delete_p(
240 'bs_workers',
241 { id => $worker_id },
242 );
243 });
244 });
245  
246 # repair all streams that need repairing
247 $app->helper('bs.repair_p' => async sub ($c) {
248 my $db = dynamically $BoardStreams::db = $pg->db;
249  
250 await $db->delete_p(
251 'bs_workers',
252 { dt_heartbeat => { '<', \["NOW() - INTERVAL '1 SECOND' * ?", 2 * $HEARTBEAT_INTERVAL] } },
253 );
254  
255 my @dead_worker_ids = (await $db->select_p(
256 [
257 'bs_guards',
258 [-left, 'bs_workers', id => 'worker_id'],
259 ],
260 [\'DISTINCT bs_guards.worker_id'],
261 { 'bs_workers.id' => undef },
262 ))->hashes->map(sub {$_->{worker_id}})->@* or return;
263  
264 while (1) {
265 my $tx = $db->begin;
266  
267 # lock one stream that needs repair
268 my $stream_row = (await $db->select_p(
269 [
270 'bs_guards',
271 ['bs_streams', id => 'stream_id'],
272 ],
273 ['bs_streams.id', 'bs_streams.name'],
274 { 'bs_guards.worker_id' => {-in, \@dead_worker_ids} },
275 {
276 limit => 1,
277 for => 'update',
278 },
279 ))->hashes->[0] or last;
280 my ($stream_id, $stream_name) = $stream_row->@{qw/ id name /};
281  
282 my @new_dead_worker_ids_for_stream;
283 my $get_dead_worker_ids = async sub {
284 my $results = await $db->select_p(
285 [
286 'bs_guards',
287 [-left, 'bs_workers', id => 'worker_id'],
288 ],
289 [\'DISTINCT bs_guards.worker_id'],
290 {
291 'bs_guards.stream_id' => $stream_id,
292 'bs_workers.id' => undef,
293 },
294 );
295 @new_dead_worker_ids_for_stream = $results->arrays->map(sub {$_->[0]})->@*;
296 return { map {( $_ => 1 )} @new_dead_worker_ids_for_stream };
297 };
298  
299 # repair stream
300 if (my $stream_repair_sub = $registry->get_repair($stream_name)) {
301 my $ret = $stream_repair_sub->($c, $stream_name, $get_dead_worker_ids);
302 await $ret if $ret->$_can('then');
303 }
304  
305 # delete guards pointing to this stream
306 await $db->delete_p(
307 'bs_guards',
308 {
309 stream_id => $stream_id,
310 worker_id => {-in, \@new_dead_worker_ids_for_stream},
311 },
312 ) if @new_dead_worker_ids_for_stream;
313  
314 $tx->commit;
315 }
316 });
317  
318 # send JSON, but only if transaction is not destroyed
319 async sub _send_p ($c, $data) {
320 my sub get_max_size {
321 my $tx = $c->tx or return $MAX_WEBSOCKET_SIZE;
322 return min($tx->max_websocket_size, $MAX_WEBSOCKET_SIZE);
323 }
324  
325 my $message = encode_json $data;
326 my $whole_length = length $message;
327 if ($whole_length <= get_max_size) {
328 $c->tx or return !!0; # check if transaction is destroyed
329 $c->send({ binary => $message });
330 return 1;
331 }
332  
333 my $identifier = $c->stash->{'boardstreams.outgoing_uuid'}++;
334  
335 for (my ($i, $cursor, $sent_ending) = (0, 0, 0); ! $sent_ending; $i++) {
336 my $bytes_prefix;
337 my $ending_prefix = ":$identifier $i\$: ";
338 my $max_size = get_max_size;
339 if (length($ending_prefix) + $whole_length - $cursor <= $max_size) {
340 $bytes_prefix = $ending_prefix;
341 $sent_ending = 1;
342 } else {
343 $bytes_prefix = ":$identifier $i: ";
344 }
345  
346 my $max_sublength = $max_size - length $bytes_prefix;
347 my $substring = $cursor <= $whole_length ? substr($message, $cursor, $max_sublength) : '';
348 $cursor += $max_sublength;
349  
350 $c->tx or return !!0; # check if transaction is destroyed
351 $c->send({ binary => $bytes_prefix . $substring });
352  
353 # don't cause other threads to hang if message is very large
354 await next_tick_p unless $sent_ending;
355 }
356  
357 return 1;
358 }
359  
360 $app->helper('bs.init_client_p' => async sub ($c) {
361 $can_accept_clients or return $c->rendered(503);
362  
363 $registry->conn_subscriptions->{$c} = {};
364 $registry->pending_joins->{$c} = rx_behavior_subject->new(0);
365 $registry->register_conn($c);
366 $c->stash->{'boardstreams.outgoing_uuid'} = 'a';
367  
368 $c->on(finish => async sub ($_c, @) {
369 await $on_client_finish_sub->($_c);
370 });
371  
372 await sleep_p(0.25); # mojo issue 1895
373  
374 $c->on(text => async sub ($_c, $bytes) {
375 my $data = decode_json $bytes;
376  
377 my $id;
378  
379 # pong and return on ping
380 if (($data->{type} // '') eq 'ping') {
381 await _send_p($_c, { type => 'pong' });
382 return;
383 }
384  
385 try {
386 defined $data->{jsonrpc} and $data->{jsonrpc} eq '2.0'
387 or die 'incoming message is not jsonrpc 2.0';
388  
389 (my $method, my $params, $id) = $data->@{qw/ method params id /};
390  
391 ! $is_finished or die jsonrpc_error 503, 'this server worker stopped, please try again';
392 $registry->is_conn_registered($_c)
393 or die jsonrpc_error 503, "can't receive because connection is closing";
394  
395 if ($method eq 'doAction') {
396 # params
397 my ($stream_name, $action_name, $payload) = @$params;
398  
399 # validation
400 ! defined $id or die "action '$action_name' on stream '$stream_name' contains extra id ($id)\n";
401 $registry->is_member_of($_c, $stream_name)
402 or die "Connection has not joined '$stream_name' but tried to do action '$action_name'\n"
403 unless $stream_name eq '!open';
404  
405 # fetch + act
406 my $action_sub = $registry->get_action($stream_name, $action_name)
407 or die "invalid action '$action_name' on stream '$stream_name'\n";
408  
409 my $ret = $action_sub->($_c, $stream_name, $payload);
410 await $ret if $ret->$_can('then');
411 } elsif ($method eq 'doRequest') {
412 # params
413 my ($stream_name, $request_name, $payload) = @$params;
414  
415 # validation
416 defined $id and ! length ref $id
417 or die "request '$request_name' on stream '$stream_name' has missing or invalid id ("
418 . ($id // 'undef') . ")\n";
419 $registry->is_member_of($_c, $stream_name)
420 or die jsonrpc_error(
421 403,
422 "Connection has not joined '$stream_name' but tried to do request '$request_name'"
423 )
424 unless $stream_name eq '!open';
425  
426 # fetch + act
427 my $request_sub = $registry->get_request($stream_name, $request_name)
428 or die "invalid request '$request_name' on stream '$stream_name'\n";
429  
430 my $result = $request_sub->($_c, $stream_name, $payload);
431 $result = await $result if $result->$_can('then');
432  
433 # respond
434 await _send_p($_c, {
435 jsonrpc => '2.0',
436 result => $result,
437 id => $id,
438 });
439 } else {
440 die jsonrpc_error -32_601, 'invalid method', { method => $method };
441 }
442 } catch ($e) {
443 $_c->log->error(trim "$e");
444 if (defined $id) {
445 my $jsonrpc_error =
446 $e isa 'BoardStreams::Error::JSONRPC' ? $e
447 : jsonrpc_error 500, trim("$e");
448  
449 await _send_p($_c, {
450 jsonrpc => '2.0',
451 error => $jsonrpc_error,
452 id => $id,
453 });
454 }
455 };
456 }) if $c->tx;
457  
458 await _send_p($c, {
459 type => 'config',
460 data => {
461 pingInterval => 0 + $PING_INTERVAL,
462 },
463 });
464 });
465  
466 $app->hook(around_action => async sub ($next, $c, $action, $last) {
467 if ($last and $c->stash->{'boardstreams.endpoint'}) {
468 $c->render_later;
469 try {
470 my $ret = $next->();
471 await $ret if $ret->$_can('then');
472 return await $c->bs->init_client_p;
473 } catch ($e) {
474 await sleep_p(1.5);
475 await _send_p($c, { type => 'connection failure', requestId => scalar eval {$c->req->request_id} });
476 $c->finish if $c->tx;
477 $c->log->error($e);
478 die $e;
479 };
480 } else {
481 return $next->();
482 }
483 });
484  
485 $app->helper('bs.create_stream_p' => async sub ($c, $stream_name, $starting_state, %opts) {
486 local @CARP_NOT = qw/ Mojolicious::Renderer /,
487 croak 'Not in an event loop, using bs->create_stream_p; use bs->create_stream instead'
488 unless Mojo::IOLoop->is_running;
489  
490 # opts can be: type, keep_events
491 my $type = $opts{type};
492 my $keep_events = exists $opts{keep_events} ? int(!!$opts{keep_events}) : 1;
493  
494 # validate params
495 $stream_name =~ $BoardStreams::REs::STREAM_NAME
496 or croak "invalid stream name: '$stream_name'";
497 defined $starting_state
498 or croak "starting state not defined";
499  
500 my $db = $BoardStreams::db // $pg->db;
501  
502 # TODO: Consider locking this row for update
503 return !!0 if await exists_p($db, 'bs_streams', { name => $stream_name });
504  
505 my $savepoint_name = unique_id;
506 my ($in_txn) = eval {
507 await $db->query_p(qq{SAVEPOINT "$savepoint_name"});
508 1
509 };
510  
511 try {
512 await query_throwing_exception_object_p($db, 'insert_p', [
513 'bs_streams',
514 {
515 name => $stream_name,
516 state => { -json => $starting_state },
517 type => $type,
518 keep_events => $keep_events,
519 },
520 ]);
521  
522 return 1;
523 } catch ($e) {
524 die $e
525 unless $e isa 'BoardStreams::Error::DB::Duplicate'
526 and $e->data->{key_name} eq UNIQUE_STREAM_NAME_INDEX;
527  
528 await $db->query_p(qq{ROLLBACK TO "$savepoint_name"}) if $in_txn;
529 return !!0;
530 };
531 });
532  
533 $app->helper('bs.create_stream' => async sub ($c, $stream_name, $starting_state, %opts) {
534 local @CARP_NOT = qw/ Mojolicious::Renderer /,
535 croak 'In an event loop, using bs->create_stream; use await bs->create_stream_p instead'
536 if Mojo::IOLoop->is_running;
537  
538 # opts can be: type, keep_events
539 my $type = $opts{type};
540 my $keep_events = exists $opts{keep_events} ? int(!!$opts{keep_events}) : 1;
541  
542 # validate params
543 $stream_name =~ $BoardStreams::REs::STREAM_NAME
544 or croak "invalid stream name: '$stream_name'";
545 defined $starting_state
546 or croak "starting state not defined";
547  
548 my $db = $BoardStreams::db // $pg->db;
549  
550 # TODO: Consider locking this row for update
551 return !!0 if row_exists($db, 'bs_streams', { name => $stream_name });
552  
553 my $savepoint_name = unique_id;
554 my ($in_txn) = eval { $db->query(qq{SAVEPOINT "$savepoint_name"}); 1 };
555  
556 try {
557 query_throwing_exception_object($db, 'insert', [
558 'bs_streams',
559 {
560 name => $stream_name,
561 state => { -json => $starting_state },
562 type => $type,
563 keep_events => $keep_events,
564 },
565 ]);
566  
567 return 1;
568 } catch ($e) {
569 die $e
570 unless $e isa 'BoardStreams::Error::DB::Duplicate'
571 and $e->data->{key_name} eq UNIQUE_STREAM_NAME_INDEX;
572  
573 $db->query(qq{ROLLBACK TO "$savepoint_name"}) if $in_txn;
574 return !!0;
575 };
576 });
577  
578 $app->helper('bs.lock_stream_p' => async sub ($c, $stream_name, $sub) {
579 my $is_array = ref $stream_name eq 'ARRAY';
580 my $stream_names = $is_array ? $stream_name : [$stream_name];
581 await $c->bs->do_txn_p(async sub ($db) {
582 my $stream_rows = (await $db->select_p(
583 'bs_streams',
584 [qw/ id name state keep_events /],
585 { name => {-in, $stream_names} },
586 { for => 'update' },
587 ))->expand->hashes;
588 my $streams = hashify($stream_rows, ['name']);
589 my ($missing_stream) = grep { not exists $streams->{$_} } @$stream_names;
590 ! defined $missing_stream or die "Couldn't find stream $missing_stream in database to lock\n";
591 # my ($stream_id, $old_state, $keep_events) = $stream_row->@{qw/ id state keep_events /};
592  
593 my @old_states = map $_->{state}, $streams->@{@$stream_names};
594 @old_states = map { (dclone [$_])->[0] } @old_states;
595  
596 my @ret = $sub->(map $_->{state}, $streams->@{@$stream_names});
597 @ret = await $ret[0] if $ret[0]->$_can('then');
598 $is_array or @ret = ([@ret]);
599  
600 foreach my $sname (@$stream_names) {
601 my $old_state = shift @old_states;
602 my $stream_id = $streams->{$sname}{id};
603 my ($new_event, $new_state, $extra_guards) = (shift @ret)->@*;
604  
605 my $new_event_id;
606 $new_event_id = (await $db->insert_p(
607 'bs_events',
608 {
609 stream_id => $stream_id,
610 data => { -json => $new_event },
611 },
612 { returning => 'id' },
613 ))->hashes->[0]{id} if $streams->{$sname}{keep_events} and defined $new_event;
614  
615 if (defined $new_event or defined $new_state) {
616 $new_event_id //= (await $db->query_p("SELECT nextval('bs_events_id_seq')"))->arrays->[0][0];
617  
618 await $db->update_p(
619 'bs_streams',
620 {
621 defined $new_state ? (state => { -json => $new_state }) : (),
622 event_id => $new_event_id,
623 last_dt => \'CURRENT_TIMESTAMP',
624 },
625 { id => $stream_id },
626 ) if (defined $new_event and $streams->{$sname}{keep_events}) or defined $new_state;
627 }
628  
629 my $diff;
630 if (defined $new_state) {
631 delete $old_state->{_secret} if ref $old_state eq 'HASH';
632 delete $new_state->{_secret} if ref $new_state eq 'HASH';
633 $diff = diff($old_state, $new_state, noO => 1, noU => 1);
634 $diff = undef if ! %$diff;
635 }
636  
637 if (defined $extra_guards) {
638 if ($extra_guards > 0) {
639 await $db->insert_p(
640 'bs_guards',
641 {
642 worker_id => $worker_id,
643 stream_id => $stream_id,
644 count => $extra_guards,
645 },
646 {
647 on_conflict => [
648 [ 'worker_id', 'stream_id' ] => { count => \'bs_guards.count + EXCLUDED.count' },
649 ],
650 }
651 );
652 } elsif ($extra_guards < 0) {
653 my $guard_row = (await $db->update_p(
654 'bs_guards',
655 { count => \['"count" - ?', abs($extra_guards)] },
656 {
657 worker_id => $worker_id,
658 stream_id => $stream_id,
659 },
660 { returning => 'count' },
661 ))->hashes->[0] or croak "missing guard row";
662  
663 if (! $guard_row->{count}) {
664 await $db->delete_p(
665 'bs_guards',
666 {
667 worker_id => $worker_id,
668 stream_id => $stream_id,
669 count => 0, # because user may have forgotten to start a txn
670 },
671 );
672 }
673 }
674 }
675  
676 if (defined $diff or defined $new_event) {
677 my $pg_channel = get_pg_channel_name($sname);
678 my $notification = encode_json({
679 id => int $new_event_id,
680 event => $new_event,
681 patch => $diff,
682 });
683 my $whole_length = length $notification;
684 if ($whole_length <= $NOTIFY_SIZE_LIMIT) {
685 $db->notify($pg_channel, $notification);
686 } else {
687 # send notification in chunks
688 for (my ($i, $cursor, $sent_ending) = (0, 0, 0); ! $sent_ending; $i++) {
689 my $bytes_prefix;
690 my $ending_bytes_prefix = ":$new_event_id $i\$: ";
691 # this assumes that length $ending_bytes_prefix < $NOTIFY_SIZE_LIMIT
692 if (length($ending_bytes_prefix) + $whole_length - $cursor <= $NOTIFY_SIZE_LIMIT) {
693 $bytes_prefix = $ending_bytes_prefix;
694 $sent_ending = 1;
695 } else {
696 $bytes_prefix = ":$new_event_id $i: ";
697 }
698  
699 my $max_sublength = $NOTIFY_SIZE_LIMIT - length $bytes_prefix;
700 my $substring = $cursor <= $whole_length ? substr($notification, $cursor, $max_sublength) : '';
701 $cursor += $max_sublength;
702  
703 $db->notify($pg_channel, $bytes_prefix . $substring);
704  
705 # don't cause other threads to hang if notification is very large
706 await next_tick_p unless $stream_names->[-1] eq $sname and $sent_ending;
707 }
708 }
709 }
710 }
711 });
712  
713 return undef;
714 });
715  
716 $app->helper('bs.set_action' => sub ($c, $stream_name, $action_name, $sub) {
717 $registry->set_action_request(
718 action => $stream_name, $action_name, $sub
719 );
720 });
721  
722 $app->helper('bs.set_request' => sub ($c, $stream_name, $request_name, $sub) {
723 $registry->set_action_request(
724 request => $stream_name, $request_name, $sub
725 );
726 });
727  
728 $app->helper('bs.set_join' => sub ($c, $stream_name, $sub) {
729 $registry->set_action_request(
730 join_leave => $stream_name, 'join', $sub
731 );
732 });
733  
734 $app->helper('bs.set_leave' => sub ($c, $stream_name, $sub) {
735 $registry->set_action_request(
736 join_leave => $stream_name, 'leave', $sub
737 );
738 });
739  
740 $app->helper('bs.set_repair' => sub ($c, $stream_name, $sub) {
741 $registry->set_action_request(
742 join_leave => $stream_name, 'repair', $sub
743 );
744 });
745  
746 $app->helper('bs.get_state_p' => async sub ($c, $stream_name) {
747 my $db = $BoardStreams::db // $pg->db;
748  
749 my $stream_row = (await $db->select_p(
750 'bs_streams',
751 [qw/ state /],
752 { name => $stream_name },
753 { for => 'update' },
754 ))->expand->hashes->[0] or return undef;
755  
756 return $stream_row->{state};
757 });
758  
759 # join
760 $app->bs->set_request('!open', 'join', async sub ($c, $, $payload) {
761 my ($stream_name, $last_id) = $payload->@{qw/ name last_id /};
762  
763 # fetch + act
764 my $join_sub = $registry->get_join($stream_name)
765 or die "stream '$stream_name' has no join method\n";
766  
767 my $db = dynamically $BoardStreams::db = $pg->db;
768 my $tx = $db->begin;
769  
770 await exists_p(
771 $db,
772 'bs_streams',
773 { name => $stream_name },
774 { for => 'update' },
775 ) or die "stream '$stream_name' does not exist\n";
776  
777 $registry->inc_pending_joins_by($c, 1);
778 my $result = do {
779 try {
780 my $result = $join_sub->($c, $stream_name, {
781 is_reconnect => defined $last_id,
782 });
783 $result = await $result if $result->$_can('then');
784 $result;
785 } catch ($e) {
786 $registry->inc_pending_joins_by($c, -1);
787 die $e;
788 };
789 };
790  
791 $result or die jsonrpc_error 403, 'joining not allowed';
792  
793 return do {
794 try {
795 if (my $is_first_join = $registry->add_membership($c, $stream_name)) {
796 my $o = $c->bs->_get_listener_observable($stream_name);
797 my $s = $o->subscribe();
798 $registry->conn_subscriptions->{$c}{$stream_name} = $s;
799 }
800  
801 my $limit = eval { $result->{limit} } // 0;
802  
803 my $stream_row = (await $db->select_p(
804 'bs_streams',
805 [qw/ id event_id state /],
806 { name => $stream_name },
807 ))->expand->hashes->[0] or die "Stream $stream_name does not exist in database";
808 my ($stream_id, $stream_event_id, $stream_state) = $stream_row->@{qw/ id event_id state /};
809 my $past_event_rows = !$limit ? [] : (await $db->select_p(
810 'bs_events',
811 [qw/ id data /],
812 {
813 stream_id => $stream_id,
814 id => { '<=', $stream_event_id },
815 defined($last_id) ? (id => { '>', $last_id }) : (),
816 },
817 {
818 order_by => { -desc, 'id' },
819 $limit ne 'all' ? (limit => $limit) : (),
820 }
821 ))->expand->hashes->reverse;
822  
823 $tx->commit;
824  
825 delete $stream_state->{_secret} if ref $stream_state eq 'HASH';
826  
827 +{
828 state => {
829 id => int $stream_event_id,
830 data => $stream_state,
831 },
832 events => [
833 map +{
834 id => int $_->{id},
835 event => $_->{data},
836 }, @$past_event_rows
837 ],
838 };
839 } catch ($e) {
840 my $is_last_leave = $registry->remove_membership($c, $stream_name);
841 if ($is_last_leave and exists $registry->conn_subscriptions->{$c}{$stream_name}) {
842 (delete $registry->conn_subscriptions->{$c}{$stream_name})->unsubscribe();
843 }
844 die $e;
845 } finally {
846 $registry->inc_pending_joins_by($c, -1);
847 };
848 };
849 });
850  
851 $app->bs->set_request('!open', 'leave', async sub ($c, $, $payload) {
852 my $stream_name = $payload;
853  
854 my $db = dynamically $BoardStreams::db = $pg->db;
855 my $tx = $db->begin;
856  
857 await exists_p(
858 $db,
859 'bs_streams',
860 { name => $stream_name },
861 { for => 'update' },
862 ) or return;
863  
864 if (my $leave_sub = $registry->get_leave($stream_name)) {
865 if (my $is_last_leave = $registry->remove_membership($c, $stream_name)) {
866 (delete $registry->conn_subscriptions->{$c}{$stream_name})->unsubscribe;
867 }
868  
869 try {
870 my $result = $leave_sub->($c, $stream_name);
871 await $result if $result->$_can('then');
872 } catch ($e) {
873 $c->log->fatal("Couldn't execute leave sub, stopping worker: $e");
874 # so as to not leave the streams in an inconsistent state
875 stop_gracefully();
876 die "$e";
877 };
878  
879 $tx->commit;
880 } else {
881 if (my $is_last_leave = $registry->remove_membership($c, $stream_name)) {
882 (delete $registry->conn_subscriptions->{$c}{$stream_name})->unsubscribe;
883 }
884 }
885  
886 return 'ok';
887 });
888  
889 $app->helper('bs._get_listener_observable' => sub ($c, $stream_name) {
890 return $self->_listener_observables->{$stream_name} //= rx_observable->new(sub ($subscriber) {
891 my $pg_channel = get_pg_channel_name($stream_name);
892 my ($acc_event_id, $acc_string, $acc_i) = (0, '', -1);
893 my $cb = $pg->pubsub->listen($pg_channel => sub ($, $payload) {
894 my $msg;
895 if ($payload =~ s/^\:([0-9]+) ([0-9]+)(\$)?\: //) {
896 my ($event_id, $i, $is_final) = ($1, $2, $3);
897 $event_id >= $acc_event_id or die 'event_id < acc_event_id';
898 $event_id == $acc_event_id or ($acc_event_id, $acc_string, $acc_i) = ($event_id, '', -1);
899 $i == $acc_i + 1 or return; # on listen, maybe we receive second part of old message
900 $acc_string .= $payload;
901 if ($is_final) {
902 $msg = decode_json $acc_string;
903 ($acc_string, $acc_i) = ('', -2); # -2 is closed to appends
904 } else {
905 $acc_i = $i;
906 }
907 } else {
908 $msg = decode_json $payload;
909 my ($event_id) = $msg->{id};
910 $event_id > $acc_event_id or die 'event_id <= acc_event_id';
911 ($acc_event_id, $acc_string, $acc_i) = ($event_id, '', -2);
912 }
913 $subscriber->next($msg) if defined $msg;
914 });
915  
916 return sub {
917 $pg->pubsub->unlisten($pg_channel => $cb);
918 delete $self->_listener_observables->{$stream_name};
919 };
920 })->pipe(
921 # wait until all clients have been sent their message before going on to the next msg
922 op_concat_map(sub ($payload, @) {
923 my @conns = $registry->get_conns_of_stream($stream_name)->@*;
924 my $msg = {
925 type => 'eventPatch',
926 stream => $stream_name,
927 # payload is +{id, event, patch}
928 %$payload,
929 };
930 return rx_merge(
931 map rx_from(_send_p($_, $msg)), @conns
932 );
933 }),
934 op_share(),
935 );
936 });
937  
938 $app->helper("bs.delete_events_p", async sub ($c, $stream_name, %opts) {
939 # opts can be: keep_num and/or keep_dur
940 my @until_ids;
941  
942 my $db = $pg->db;
943  
944 my $stream_row = (await $db->select_p(
945 'bs_streams',
946 'id',
947 { name => $stream_name },
948 ))->hashes->[0] or return;
949 my $stream_id = $stream_row->{id};
950  
951 if (defined $opts{keep_num}) {
952 my $until_row = (await $db->select_p(
953 'bs_events',
954 'id',
955 { stream_id => $stream_id },
956 {
957 order_by => { -desc, 'id' },
958 offset => $opts{keep_num},
959 limit => 1,
960 },
961 ))->hashes->[0];
962 my $until_id = $until_row ? $until_row->{id} : 0;
963 push @until_ids, $until_id;
964 }
965  
966 if (defined $opts{keep_dur}) {
967 my $until_row = (await $db->select_p(
968 'bs_events',
969 'id',
970 {
971 stream_id => $stream_id,
972 datetime => { '<', \["CURRENT_TIMESTAMP - INTERVAL '1 SECOND' * ?", $opts{keep_dur}] },
973 },
974 {
975 order_by => { -desc, 'datetime' },
976 limit => 1,
977 }
978 ))->hashes->[0];
979 my $until_id = $until_row ? $until_row->{id} : 0;
980 push @until_ids, $until_id;
981 }
982  
983 my $until_id = min @until_ids;
984  
985 await $db->delete_p(
986 'bs_events',
987 {
988 stream_id => $stream_id,
989 defined($until_id) ? (id => {'<=', $until_id}) : (),
990 },
991 );
992 });
993  
994 my $deleting_streams = {};
995 $app->helper("bs.delete_stream_p", async sub ($c, $stream_name) {
996 return 0 if $deleting_streams->{$stream_name};
997 dynamically $deleting_streams->{$stream_name} = 1;
998  
999 await $c->bs->do_txn_p(async sub ($db) {
1000 await exists_p(
1001 $db,
1002 'bs_streams',
1003 { name => $stream_name },
1004 { for => 'update' },
1005 ) or return 0;
1006  
1007  
1008 if (my $leave_sub = $registry->get_leave($stream_name)) {
1009 foreach my $conn ($registry->get_conns_of_stream($stream_name)->@*) {
1010 my $left_completely;
1011 while (! $left_completely) {
1012 $left_completely = $registry->remove_membership($conn, $stream_name)
1013 and (delete $registry->conn_subscriptions->{$conn}{$stream_name})->unsubscribe;
1014  
1015 try {
1016 my $result = $leave_sub->($conn, $stream_name);
1017 await $result if $result->$_can('then');
1018 } catch ($e) {
1019 $c->log->fatal("Couldn't execute leave sub, stopping worker: $e");
1020 # so as to not leave the streams in an inconsistent state
1021 stop_gracefully();
1022 };
1023 }
1024 }
1025 } else {
1026 foreach my $conn ($registry->get_conns_of_stream($stream_name)->@*) {
1027 1 until $registry->remove_membership($conn, $stream_name);
1028 (delete $registry->conn_subscriptions->{$conn}{$stream_name})->unsubscribe;
1029 }
1030 }
1031  
1032 await $db->delete_p(
1033 'bs_streams',
1034 { name => $stream_name },
1035 );
1036 });
1037  
1038 delete $deleting_streams->{$stream_name};
1039  
1040 return 1;
1041 });
1042  
1043 $app->helper('bs.do_txn_p', async sub ($c, $sub) {
1044 my $tx;
1045 dynamically $BoardStreams::db = do { my $db = $pg->db; $tx = $db->begin; $db }
1046 if ! $BoardStreams::db;
1047 my $db = $BoardStreams::db;
1048  
1049 my $ret = $sub->($db);
1050 await $ret if $ret->$_can('then');
1051  
1052 $tx->commit if $tx;
1053 });
1054 }
1055  
1056 1;