blib/lib/AnyEvent/PgRecvlogical.pm

filter: Statementstmt 77.7% 7 / 9 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 3 / 3 Podpod n/a totaltotal 83.3% 10 / 12 CC 1 SCAR 0.9
CC
1
Cov
78%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.1
↑ Summary
1 package AnyEvent::PgRecvlogical;
2 $AnyEvent::PgRecvlogical::VERSION = '1.04';
3 # ABSTRACT: perl port of pg_recvlogical
4  
5 =pod
6  
7 =head1 NAME
8  
9 AnyEvent::PgRecvlogical - perl port of pg_recvlogical
10  
11 =for html
12 <a href="https://travis-ci.org/mydimension/AnyEvent-PgRecvlogical"><img src="https://travis-ci.org/mydimension/AnyEvent-PgRecvlogical.svg?branch=master" /></a>
13 <a href='https://coveralls.io/github/mydimension/AnyEvent-PgRecvlogical?branch=master'><img src='https://coveralls.io/repos/github/mydimension/AnyEvent-PgRecvlogical/badge.svg?branch=master' alt='Coverage Status' /></a>
14 <a href="https://badge.fury.io/pl/AnyEvent-PgRecvlogical"><img src="https://badge.fury.io/pl/AnyEvent-PgRecvlogical.svg" alt="CPAN version" height="18"></a>
15  
16 =head1 SYNOPSIS
17  
18 use AnyEvent::PgRecvlogical;
19  
20 my $recv = AnyEvent::PgRecvlogical->new(
21 dbname => 'mydb',
22 slot => 'myreplslot',
23 on_message => sub {
24 my ($record, $guard) = @_;
25  
26 process($record);
27  
28 undef $guard; # declare done with $record
29 }
30 );
31  
32 $recv->start;
33  
34 =head1 DESCRIPTION
35  
36 C<AnyEvent::PgRecvlogical> provides perl bindings of similar functionality to that of
37 L<pg_recvlogical|https://www.postgresql.org/docs/current/static/app-pgrecvlogical.html>.
38 The reasoning being that C<pg_recvlogical> does afford the consuming process the opportunity to emit feedback to
39 PostgreSQL. This results is potentially being sent more data than you can handle in a timely fashion.
40  
41 =cut
42  
43 2 ▶use Moo;
Subroutine BEGIN: called CC=1 SCAR=0.0
44 2 ▶use DBI;
Subroutine BEGIN: called CC=1 SCAR=0.0
45 0 ▶use DBD::Pg 3.7.0 ':async';
Subroutine BEGIN: called CC=1 SCAR=2.6
46 use AnyEvent;
47 use AnyEvent::Util 'guard';
48 use Promises 0.99 backend => ['AnyEvent'], qw(deferred);
49 use Types::Standard ':all';
50 use Try::Tiny;
51 use Carp 'croak';
52 use curry;
53  
54 use constant {
55 AWAIT_INTERVAL => 1,
56 USECS => 1_000_000,
57 PG_MIN_VERSION => 9_04_00,
58 PG_MIN_NOEXPORT => 10_00_00,
59 PG_STATE_DUPEOBJ => '42710',
60 PG_EPOCH_DELTA => 946_684_800,
61 XLOGDATA => 'Aq>3a*',
62 PRIMARY_HEARTBEAT => 'Aq>2b',
63 STANDBY_HEARTBEAT => 'Aq>4b',
64 };
65  
66 use namespace::clean;
67  
68 my $DBH = (InstanceOf ['DBI::db'])->create_child_type(
69 constraint => sub {
70 $_->{Driver}->{Name} eq 'Pg'
71 and $_->{pg_server_version} >= PG_MIN_VERSION
72 and $_->{Name} =~ /replication=/;
73 },
74 message => sub {
75 my $parent_check = (InstanceOf ['DBI::db'])->validate($_);
76 return $parent_check if $parent_check;
77 return "$_ is not a DBD::Pg handle" unless $_->{Driver}->{Name} eq 'Pg';
78 return "$_ is connected to an old postgres version ($_->{pg_server_version} < 9.4.0)"
79 unless $_->{pg_server_version} >= PG_MIN_VERSION;
80 return "$_ is not a replication connection: $_->{Name}" unless $_->{Name} =~ /replication=/;
81 }
82 );
83  
84 my $LSNStr = Str->where(sub { m{[0-9A-F]{1,8}/[0-9A-F]{1,8}} })
85 ->plus_coercions(Int() => sub { sprintf '%X/%X', (($_ >> 32) & 0xffff_ffff), ($_ & 0xffff_ffff) });
86  
87 my $LSN = Int->plus_coercions(
88 $LSNStr => sub {
89 my ($h, $l) = map { hex } split m{/}; ($h << 32) | $l;
90 }
91 );
92  
93 =head1 ATTRIBUTES
94  
95 =over
96  
97 =item C<dbname>
98  
99 =over
100  
101 =item L<Str|Types::Standard/Str>, Required
102  
103 =back
104  
105 Database name to connect to.
106  
107 =item C<slot>
108  
109 =over
110  
111 =item L<Str|Types::Standard/Str>, Required
112  
113 =back
114  
115 Name of the replication slot to use (and/or create, see L</do_create_slot> and L</slot_exists_ok>)
116  
117 =item C<host>
118  
119 =over
120  
121 =item L<Str|Types::Standard/Str>
122  
123 =back
124  
125 =item C<port>
126  
127 =over
128  
129 =item L<Int|Types::Standard/SInt>
130  
131 =back
132  
133 =item C<username>
134  
135 =over
136  
137 =item L<Str|Types::Standard/Str>
138  
139 =back
140  
141 =item C<password>
142  
143 =over
144  
145 =item L<Str|Types::Standard/Str>
146  
147 =back
148  
149 Standard PostgreSQL connection parameters, see L<DBD::Pg/connect>.
150  
151 =item C<do_create_slot>
152  
153 =over
154  
155 =item L<Bool|Types::Standard/Bool>, Default: C<0>
156  
157 =back
158  
159 If true, the L</slot> will be be created upon connection. Otherwise, it's assumed it already exists. If it does not,
160 PostgreSQL will raise an exception.
161  
162 =item C<slot_exists_ok>
163  
164 =over
165  
166 =item L<Bool|Types::Standard/Bool>, Default: C<0>
167  
168 =back
169  
170 If true, and if L</do_create_slot> is also true, then no exception will be raised if the L</slot> already exists.
171 Otherwise, one will be raised.
172  
173 =item C<reconnect>
174  
175 =over
176  
177 =item L<Bool|Types::Standard/Bool>, Default: C<1>
178  
179 =back
180  
181 If true, will attempt to reconnect to the server and resume logical replication in the event the connection fails.
182 Otherwise, the connection will gracefully be allowed to close.
183  
184 =item C<reconnect_delay>
185  
186 =over
187  
188 =item L<Int|Types::Standard/Int>, Default: C<5>
189  
190 =back
191  
192 Time, in seconds, to wait before reconnecting.
193  
194 =item C<reconnect_limit>
195  
196 =over
197  
198 =item L<Int|Types::Standard/Int>, Default: C<1>
199  
200 =back
201  
202 Number of times to attempt reconnecting. If this limit is exceded, an exception will be thrown.
203  
204 =item C<heartbeat>
205  
206 =over
207  
208 =item L<Int|Types::Standard/Int>, Default: C<10>
209  
210 =back
211  
212 Interval, in seconds, to report our progress to the PostgreSQL server.
213  
214 =item C<plugin>
215  
216 =over
217  
218 =item L<Str|Types::Standard/Str>, Default: L<test_decoding|https://www.postgresql.org/docs/current/static/test-decoding.html>
219  
220 =back
221  
222 The server-sider plugin used to decode the WAL file before being sent to this connection. Only required when
223 L</create_slot> is true.
224  
225 =item C<options>
226  
227 =over
228  
229 =item L<HashRef|Types::Standard/HashRef>, Default: C<{}>
230  
231 =back
232  
233 Key-value pairs sent to the server-side L</plugin>. Keys with a value of C<undef> are sent as the keyword only.
234  
235 =item C<startpos>
236  
237 =over
238  
239 =item L<LSN|https://www.postgresql.org/docs/current/static/datatype-pg-lsn.html>, Default: C<0/0>
240  
241 =back
242  
243 Start replication from the given LSN. Also accepts the integer form, but that is considered advanced usage.
244  
245 =item C<received_lsn>
246  
247 =over
248  
249 =item L<LSN|https://www.postgresql.org/docs/current/static/datatype-pg-lsn.html>, Default: C<0/0>, Read Only
250  
251 =back
252  
253 Holds the last LSN position received from the server.
254  
255 =item C<flushed_lsn>
256  
257 =over
258  
259 =item L<LSN|https://www.postgresql.org/docs/current/static/datatype-pg-lsn.html>, Default: C<0/0>, Read Only
260  
261 =back
262  
263 Holds the last LSN signaled to handled by the client (see: L</on_message>)
264  
265 =item C<on_error>
266  
267 =over
268  
269 =item L<CodeRef|Types::Standard/CodeRef>, Default: L<croak|Carp/croak>
270  
271 =back
272  
273 Callback in the event of an error.
274  
275 =item C<on_message>
276  
277 =over
278  
279 =item L<CodeRef|Types::Standard/CodeRef>, Required
280  
281 =back
282  
283 Callback to receive the replication payload from the server. This is the raw output from the L</plugin>.
284  
285 The callback is passed the C<$payload> received and a C<$guard> object. Hang onto the C<$guard> until you have handled
286 the payload. Once it is released, the server will be informed that the WAL position has been "flushed."
287  
288 =back
289  
290 =cut
291  
292 has dbname => (is => 'ro', isa => Str, required => 1);
293 has host => (is => 'ro', isa => Str, predicate => 1);
294 has port => (is => 'ro', isa => Int, predicate => 1);
295 has username => (is => 'ro', isa => Str, default => q{});
296 has password => (is => 'ro', isa => Str, default => q{});
297 has slot => (is => 'ro', isa => Str, required => 1);
298  
299 has dbh => (is => 'lazy', isa => $DBH, clearer => 1, init_arg => undef);
300 has do_create_slot => (is => 'ro', isa => Bool, default => 0);
301 has slot_exists_ok => (is => 'ro', isa => Bool, default => 0);
302 has reconnect => (is => 'ro', isa => Bool, default => 1);
303 has reconnect_delay => (is => 'ro', isa => Int, default => 5);
304 has reconnect_limit => (is => 'ro', isa => Int, predicate => 1);
305 has _reconnect_counter => (is => 'rw', isa => Int, default => 0);
306 has heartbeat => (is => 'ro', isa => Int, default => 10);
307 has plugin => (is => 'ro', isa => Str, default => 'test_decoding');
308 has options => (is => 'ro', isa => HashRef, default => sub { {} });
309 has startpos => (is => 'rwp', isa => $LSN, default => 0, coerce => 1);
310 has received_lsn => (is => 'rwp', isa => $LSN, default => 0, clearer => 1, init_arg => undef, lazy => 1);
311 has flushed_lsn => (is => 'rwp', isa => $LSN, default => 0, clearer => 1, init_arg => undef, lazy => 1);
312  
313 has on_message => (is => 'ro', isa => CodeRef, required => 1);
314 has on_error => (is => 'ro', isa => CodeRef, default => sub { \&croak });
315  
316 has _fh_watch => (is => 'lazy', isa => Ref, clearer => 1, predicate => 1);
317 has _timer => (is => 'lazy', isa => Ref, clearer => 1);
318  
319 =head1 CONSTRUCTOR
320  
321 All the L</"ATTRIBUTES"> above are accepted by the constructor, with a few exceptions:
322  
323 L</"received_lsn"> and L<"flushed_lsn"> are read-only and not accepted by the constructor.
324  
325 L</"dbname">, L</"slot"> and L</"on_message"> are required.
326  
327 Note, that logical replication will not automatically commence upon construction. One must call L</"start"> first.
328  
329 =cut
330  
331 sub _dsn {
332 my $self = shift;
333  
334 my %dsn = (replication => 'database', client_encoding => 'sql_ascii');
335 foreach (qw(host port dbname)) {
336 my $x = "has_$_";
337 next if $self->can($x) and not $self->$x;
338  
339 $dsn{$_} = $self->$_;
340 }
341  
342 return 'dbi:Pg:' . join q{;}, map { "$_=$dsn{$_}" } sort keys %dsn;
343 }
344  
345 sub _build_dbh {
346 my $self = shift;
347 my $dbh = DBI->connect($self->_dsn, $self->username, $self->password, { PrintError => 0 },);
348  
349 croak $DBI::errstr unless $dbh;
350  
351 return $dbh;
352 }
353  
354 sub _build__fh_watch {
355 my $self = shift;
356  
357 my $w = AE::io $self->dbh->{pg_socket}, 0, $self->curry::weak::_read_copydata;
358 if ($AnyEvent::MODEL and $AnyEvent::MODEL eq 'AnyEvent::Impl::EV') {
359 $w->priority($w->priority - 1); # be a little less aggressive
360 }
361  
362 return $w;
363 }
364  
365 sub _build__timer {
366 my $self = shift;
367 if ($AnyEvent::MODEL and $AnyEvent::MODEL eq 'AnyEvent::Impl::EV') {
368 my $w = EV::periodic(0, $self->heartbeat, 0, $self->curry::weak::_heartbeat);
369 $w->priority(&EV::MAXPRI);
370 return $w;
371 } else {
372 return AE::timer $self->heartbeat, $self->heartbeat, $self->curry::weak::_heartbeat;
373 }
374 }
375  
376 =head1 METHODS
377  
378 All L</"ATTRIBUTES"> are also accesible via methods. They are all read-only.
379  
380 =over
381  
382 =item start
383  
384 Initialize the logical replication process asyncronously and return immediately. This performs the following steps:
385  
386 =over
387  
388 =item 1. L</"identify_system">
389  
390 =item 2. L</"create_slot"> (if requested)
391  
392 =item 3. L</"start_replication">
393  
394 =item 4. heartbeat timer
395  
396 =back
397  
398 This method wraps the above steps for convenience. Should you desire to modify the
399 L<replication startup protocol|https://www.postgresql.org/docs/current/static/protocol-replication.html> (which you
400 shouldn't), the methods are described in detail below.
401  
402 Returns: L<Promises::Promise>
403  
404 =cut
405  
406 sub start {
407 my $self = shift;
408  
409 $self->_post_init(
410 deferred {
411 shift->chain($self->curry::identify_system, $self->curry::create_slot, $self->curry::start_replication);
412 }
413 );
414 }
415  
416 sub _post_init {
417 my ($self, $d) = @_;
418  
419 return $d->then(
420 sub {
421 $self->_fh_watch;
422 $self->_timer;
423 },
424 $self->on_error,
425 );
426 }
427  
428 =item identify_system
429  
430 Issues the C<IDENTIFY_SYSTEM> command to the server to put the connection in repliction mode.
431  
432 Returns: L<Promises::Promise>
433  
434 =cut
435  
436 sub identify_system {
437 my $self = shift;
438 $self->dbh->do('IDENTIFY_SYSTEM', { pg_async => PG_ASYNC });
439 return _async_await($self->dbh)->catch(
440 sub {
441 my @error = @_;
442 unshift @error, $DBI::errstr if $DBI::errstr;
443  
444 croak @error;
445 }
446 );
447 }
448  
449 =item create_slot
450  
451 Issues the appropriate C<CREATE_REPLICATION_SLOT> command to the server, if requested.
452  
453 Returns: L<Promises::Promise>
454  
455 =cut
456  
457 sub create_slot {
458 my $self = shift;
459  
460 return deferred->resolve unless $self->do_create_slot;
461  
462 my $dbh = $self->dbh;
463 $dbh->do(
464 sprintf(
465 'CREATE_REPLICATION_SLOT %s LOGICAL %s%s',
466 $dbh->quote_identifier($self->slot),
467 $dbh->quote_identifier($self->plugin),
468 ($dbh->{pg_server_version} >= PG_MIN_NOEXPORT ? ' NOEXPORT_SNAPSHOT' : '') # uncoverable branch true
469 ),
470 { pg_async => PG_ASYNC }
471 );
472  
473 return _async_await($dbh)->catch(
474 sub {
475 croak @_ unless $dbh->state eq PG_STATE_DUPEOBJ and $self->slot_exists_ok;
476 }
477 );
478 }
479  
480 sub _option_string {
481 my $self = shift;
482  
483 my @opts;
484 while (my ($k, $v) = each %{ $self->options }) {
485 push @opts, $self->dbh->quote_identifier($k);
486 defined $v and $opts[-1] .= sprintf ' %s', $self->dbh->quote($v); # uncoverable branch false
487 }
488  
489 return @opts ? sprintf('(%s)', join q{, }, @opts) : q{}; # uncoverable branch false
490 }
491  
492 =item start_replication
493  
494 Issues the C<START_REPLICATION SLOT> command and immediately returns. The connection will then start receiving
495 logical replication payloads.
496  
497 =cut
498  
499 sub start_replication {
500 my $self = shift;
501  
502 $self->dbh->do(
503 sprintf(
504 'START_REPLICATION SLOT %s LOGICAL %s%s',
505 $self->dbh->quote_identifier($self->slot),
506 $LSNStr->coerce($self->startpos),
507 $self->_option_string
508 )
509 );
510 }
511  
512 =item pause
513  
514 Pauses reading from the database. Useful for throttling the inbound flow of data so as to not overwhelm your
515 application. It is safe, albeit redundant, to call this method multiple time in a row without unpausing.
516  
517 =cut
518  
519 sub pause { shift->_clear_fh_watch; return; }
520  
521 =item unpause
522  
523 Resume reading from the database. After a successful L</pause>, this will pick right back reciving data and sending it
524 to the provided L</callback>. It is safe, albeit redundant, to call this method multiple time in a row without pausing.
525  
526 =cut
527  
528 sub unpause { shift->_fh_watch; return; }
529  
530 =item is_paused
531  
532 Returns the current pause state.
533  
534 Returns: boolean
535  
536 =cut
537  
538 sub is_paused { return !shift->_has_fh_watch }
539  
540 sub _read_copydata {
541 my $self = shift;
542  
543 my ($n, $msg);
544 my $ok = try {
545 $n = $self->dbh->pg_getcopydata_async($msg);
546 1;
547 } catch {
548 # uncoverable statement count:2
549 AE::postpone { $self->_handle_disconnect };
550 0;
551 };
552  
553 # exception thrown, going to reconnect
554 return unless $ok; # uncoverable branch true
555  
556 # nothing waiting
557 # watcher will re-enter until $n == 0
558 return if $n == 0;
559  
560 if ($n == -1) {
561 AE::postpone { $self->_handle_disconnect };
562 return;
563 }
564  
565 # uncoverable branch true
566 if ($n == -2) {
567 # error reading
568 # uncoverable statement
569 $self->on_error->('could not read COPY data: ' . $self->dbh->errstr);
570 }
571  
572 my $type = substr $msg, 0, 1;
573  
574 if ('k' eq $type) {
575 # server keepalive
576 my (undef, $lsnpos, $ts, $reply) = unpack PRIMARY_HEARTBEAT, $msg;
577  
578 $self->_set_received_lsn($lsnpos) if $lsnpos > $self->received_lsn;
579  
580 # only interested in the request-reply bit
581 # uncoverable branch true
582 if ($reply) {
583 # uncoverable statement
584 AE::postpone { $self->_heartbeat };
585 }
586  
587 # an inbound heartbeat is proof enough of successful reconnect
588 $self->_reconnect_counter(0) if $self->_reconnect_counter;
589  
590 return;
591 }
592  
593 # uncoverable branch true
594 unless ('w' eq $type) {
595 # uncoverable statement
596 $self->on_error->("unrecognized streaming header: '$type'");
597 return;
598 }
599  
600 my (undef, $startlsn, $endlsn, $ts, $record) = unpack XLOGDATA, $msg;
601  
602 $self->_set_received_lsn($startlsn) if $startlsn > $self->received_lsn;
603  
604 my $guard = $self->$curry::weak(
605 sub {
606 my $self = shift;
607 $self->_set_flushed_lsn($startlsn) if $startlsn > $self->flushed_lsn;
608 }
609 );
610  
611 $self->on_message->($record, guard(\&$guard));
612  
613 return;
614 }
615  
616 =item stop
617  
618 Stop receiving replication payloads and disconnect from the PostgreSQL server.
619  
620 =back
621  
622 =cut
623  
624 sub stop {
625 my $self = shift;
626  
627 $self->_clear_fh_watch;
628 $self->_clear_timer;
629 $self->clear_dbh;
630 }
631  
632 sub _handle_disconnect {
633 my $self = shift;
634  
635 $self->stop;
636  
637 return unless $self->reconnect;
638  
639 if ( $self->has_reconnect_limit
640 and $self->_reconnect_counter($self->_reconnect_counter + 1) > $self->reconnect_limit) {
641 $self->on_error->('reconnect limit reached: ' . $self->reconnect_limit);
642 return;
643 }
644  
645 $self->_set_startpos($self->flushed_lsn);
646 $self->clear_received_lsn;
647 $self->clear_flushed_lsn;
648  
649 my $w; $w = AE::timer $self->reconnect_delay, 0, sub {
650 undef $w;
651 $self->_post_init(deferred { $self->start_replication });
652 };
653 }
654  
655 sub _heartbeat {
656 my ($self, $req_reply) = @_;
657 $req_reply = !!$req_reply || 0; #uncoverable condition right
658  
659 my $status = pack STANDBY_HEARTBEAT, 'r', # receiver status update
660 $self->received_lsn, # last WAL received
661 $self->flushed_lsn, # last WAL flushed
662 0, # last WAL applied
663 int((AE::now - PG_EPOCH_DELTA) * USECS), # ms since 2000-01-01
664 $req_reply; # request heartbeat
665  
666 $self->dbh->pg_putcopydata($status);
667 }
668  
669 sub _async_await {
670 my ($dbh) = @_;
671  
672 my $d = deferred;
673  
674 # no async operation in progress
675 return $d->reject if $dbh->{pg_async_status} == 0; # uncoverable branch true
676  
677 my $w; $w = AE::timer 0, AWAIT_INTERVAL, sub {
678 return unless $dbh->pg_ready;
679 try {
680 $d->resolve($dbh->pg_result);
681 } catch {
682 $d->reject($_);
683 };
684 undef $w;
685 };
686  
687 return $d->promise;
688 }
689  
690 =head1 AUTHOR
691  
692 William Cox (cpan:MYDMNSN) <mydimension@gmail.com>
693  
694 =head1 COPYRIGHT
695  
696 Copyright (c) 2017-2018 William Cox
697  
698 =head1 LICENSE
699  
700 This library is free software and may be distributed under the same terms as perl itself.
701 See L<http://dev.perl.org/licenses/>.
702  
703 =cut
704  
705 1;
↑ Summary