blib/lib/Kafka/Producer/Avro.pm

filter: Statementstmt 20.3% 35 / 172 Branchbran 0.0% 0 / 88 Conditioncond 0.0% 0 / 97 MC/DCmcdc 0.0% 0 / 38 Subroutinesub 50.0% 11 / 22 Podpod 100% 5 / 5 totaltotal 12.0% 51 / 422 CC 69 SCAR 62.6
CC
69
Cov
10%
send
992.0
bulk_send
600.0
_get_avro_schema
156.0
CRAP 523.8
↑ Summary
1 package Kafka::Producer::Avro;
2  
3 =pod
4  
5 =head1 NAME
6  
7 Kafka::Producer::Avro - Avro message producer for Apache Kafka.
8  
9 =head1 SYNOPSIS
10  
11 use Kafka::Connection;
12 use Kafka::Producer::Avro;
13
14 my $connection = Kafka::Connection->new( host => 'localhost' );
15
16 my $producer = Kafka::Producer::Avro->new( Connection => $connection , SchemaRegistry => Confluent::SchemaRegistry->new() );
17
18 # Set Avro schema for message key (valid JSON-string)
19 my $key_schema = <<KEY_SCHEMA;
20 {
21 "type": "long",
22 "name": "_id"
23 }
24 KEY_SCHEMA
25 # Set Avro schema for message value (payload) (valid JSON-string)
26 my $value_schema = <<VALUE_SCHEMA;
27 {
28 "type": "record",
29 "name": "myrecord",
30 "fields": [
31 {
32 "name": "f1",
33 "type": "string"
34 }
35 ]
36 }
37 VALUE_SCHEMA
38
39 # Sending a single message
40 my $response = $producer->send(
41 'mytopic', # topic
42 0, # partition
43 'Single message', # message
44 undef, # key
45 undef, # compression_codec
46 undef, # timestamps
47 $key_schema, # key_schema
48 $value_schema # value_schema
49 );
50
51 # Sending a series of messages
52 $response = $producer->send(
53 'mytopic', # topic
54 0, # partition
55 [ # messages
56 'The first message',
57 'The second message',
58 'The third message',
59 ],
60 undef, # key(s)
61 undef, # compression_codec
62 undef, # timestamp(s)
63 $key_schema, # key_schema
64 $value_schema # value_schema
65 );
66
67 # ...or use named parameters
68
69 $producer->send(
70 topic => $topic,
71 partition => $partition,
72 messages => $messages,
73 keys => $keys,
74 compression_codec => $compression_codec,
75 timestamps => $timestamps,
76 key_schema => $key_schema,
77 value_schema => $value_schema
78 );
79
80 # Closes the producer and cleans up
81 undef $producer;
82 $connection->close;
83 undef $connection;
84  
85 =head1 DESCRIPTION
86  
87 C<Kafka::Producer::Avro> main feature is to provide object-oriented API to
88 produce messages according to I<Confluent SchemaRegistry> and I<Avro> serialization.
89  
90 C<Kafka::Producer::Avro> inerhits from and extends L<Kafka::Producer|Kafka::Producer>.
91  
92 =cut
93  
94 1 ▶use 5.010;
Subroutine BEGIN: called CC=1 SCAR=0.0
95 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
96 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
97  
98 1 ▶use JSON::XS;
Subroutine BEGIN: called CC=1 SCAR=0.0
99  
100 1 ▶use base 'Kafka::Producer';
Subroutine BEGIN: called CC=1 SCAR=0.0
101  
102 1 ▶use Avro::BinaryEncoder;
Subroutine BEGIN: called CC=1 SCAR=0.0
103 1 ▶use Avro::Schema;
Subroutine BEGIN: called CC=1 SCAR=0.0
104 1 ▶use Confluent::SchemaRegistry;
Subroutine BEGIN: called CC=1 SCAR=0.0
105  
106 1 ▶use Memoize;
Subroutine BEGIN: called CC=1 SCAR=0.0
107  
108 1 ▶use constant MAGIC_BYTE => 0;
Subroutine BEGIN: called CC=1 SCAR=0.0
109  
110 1 ▶use version; our $VERSION = version->declare('v1.0.1');
Subroutine BEGIN: called CC=1 SCAR=0.0
111  
112 =head1 INSTALL
113  
114 Installation of C<Kafka::Producer::Avro> is a canonical:
115  
116 perl Makefile.PL
117 make
118 make test
119 make install
120  
121 =head2 TEST NOTES
122  
123 Tests are focused on verifying Avro-formatted messages and theirs interactions with Confluent Schema Registry and are intended to extend C<Kafka::Producer> test suite.
124  
125 They expect that in the target are listening Apache Kafka and Schema Registry services, respectively listening on C<localhost:9092> and C<http://localhost:8081>.
126  
127 You can alternatively set a different URLs by exporting the following environment variable:
128  
129 =over 3
130  
131 =item C<KAFKA_HOST>
132  
133 =item C<KAFKA_PORT>
134  
135 =item C<CONFLUENT_SCHEMA_REGISTY_URL>
136  
137 =back
138  
139 For example:
140  
141 export KAFKA_HOST=my-kafka-host.my-domain.org
142 export FALFA_PORT=9092
143 export CONFLUENT_SCHEMA_REGISTY_URL=http://my-schema-registry-host.my-domain.org
144  
145 =head1 USAGE
146  
147 =head2 CONSTRUCTOR
148  
149 =head3 C<new>
150  
151 Creates new producer client object.
152  
153 C<new()> takes arguments in key-value pairs as described in L<Kafka::Producer|Kafka::Producer> from which it inherits.
154  
155 In addition, takes in the following arguments:
156  
157 =over 3
158  
159 =item C<SchemaRegistry =E<gt> $schema_registry> (B<mandatory>)
160  
161 Is a L<Confluent::SchemaRegistry|Confluent::SchemaRegistry> instance.
162  
163 =back
164  
165 =cut
166  
167 0 ▶sub new {
Subroutine new: not called CC=4 SCAR=30.0
Pod documented
168 0 my $this = shift;
169 0 ▶ my $class = ref($this) || $this;
Condition0%0/3
ref $this || $this
l!l&&r!l&&!r
000
MC/DC0%0/2
ref $this || $this
ref $this $this
Truth table0%0/3
A || B
A= ref $thisB= $this
ABresult
1X1
011
000
170 0 my $schema_registry_class = 'Confluent::SchemaRegistry';
171 0 my %params = @_;
172  
173 # Check SchemaRegistry param
174 die "Missing SchemaRegistry param"
175 0 ▶ unless exists $params{SchemaRegistry};
Branch: 1 branch
BranchTrueFalseTotal
unless exists $params{"SchemaRegistry"} 0 0 0
176 die "SchemaRegistry param must be a $schema_registry_class instance object"
177 0 ▶ unless ref($params{SchemaRegistry}) eq $schema_registry_class;
Branch: 1 branch
BranchTrueFalseTotal
unless ref $params{"SchemaRegistry"} eq $schema_registry_class 0 0 0
178 0 my $schema_registry = delete $params{SchemaRegistry};
179
180 # Use parent class constructor
181 0 my $self = $class->SUPER::new(%params);
182
183 # Add ans internal reference to SchemaRegistry
184 0 $self->{__SCHEMA_REGISTRY} = $schema_registry;
185
186 0 return bless($self, $class);
187 }
188  
189  
190  
191 ##### Class methods
192  
193 # Encode $payload in Avro format according to an Avro schema
194 0 ▶sub _encode {
Subroutine _encode: not called CC=2 SCAR=17.9
195 0 my $schema_ref = shift;
196 0 my $payload = shift;
197 return undef
198 0 ▶ unless defined $payload;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $payload 0 0 0
199 0 my $encoded = pack('bN', &MAGIC_BYTE, $schema_ref->{id});
200 Avro::BinaryEncoder->encode(
201 schema => $schema_ref->{schema},
202 data => $payload,
203 0 ▶ emit_cb => sub {
Subroutine __ANON__: not called CC=1 SCAR=6.9
204 0 $encoded .= ${ $_[0] };
205 }
206 0 );
207 0 return $encoded;
208 }
209  
210  
211 ##### Private methods
212  
213 0 ▶sub _clear_error { $_[0]->_set_error() }
Subroutine _clear_error: not called CC=1 SCAR=6.9
214 0 ▶sub _set_error { $_[0]->{__ERROR} = $_[1] }
Subroutine _set_error: not called CC=1 SCAR=6.9
215 0 ▶sub _get_error { $_[0]->{__ERROR} }
Subroutine _get_error: not called CC=1 SCAR=6.9
216  
217 # Interact with Schema Registry
218 1 memoize('_get_avro_schema');
219 0 ▶sub _get_avro_schema {
Subroutine _get_avro_schema: not called CC=12 SCAR=50.5
220 0 my $self = shift;
221 0 my $topic = shift;
222 0 my $type = shift;
223 0 my $supplied_schema = shift;
224
225 0 my $subject = $topic . '-' . $type;
226 0 my $sr = $self->schema_registry();
227 0 my ($schema_id, $avro_schema);
228
229 # If a schema is supplied...
230 0 ▶ if ($supplied_schema) {
Branch: 1 branch
BranchTrueFalseTotal
if ($supplied_schema) { } 0 0 0
231
232 # If the subject exixts...
233 0 my $subjects = $sr->get_subjects();
234 0 ▶ if (grep(/^$subject$/, @$subjects) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (grep /^$subject$/, @$subjects) 0 0 0
235
236 # ...check if it already exists in registry
237 0 my $schema_info = $sr->check_schema(
238 SUBJECT => $topic,
239 TYPE => $type,
240 SCHEMA => $supplied_schema
241 );
242 0 ▶ if ( defined $schema_info ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $schema_info) { } 0 0 0
243
244 0 $schema_id = $schema_info->{id};
245 0 $avro_schema = $schema_info->{schema};
246  
247 # ...if it does not already exist in the registry....
248 } else {
249
250 # ...test new schema compliancy against latest version
251 0 my $compliant = $sr->test_schema(
252 SUBJECT => $topic,
253 TYPE => $type,
254 SCHEMA => $supplied_schema
255 );
256 0 ▶ $self->_set_error('Schema not compliant with latest one from registry') &&
Branch: 1 branch
BranchTrueFalseTotal
unless $compliant 0 0 0
Condition0%0/5
$compliant || $self->_set_error('Schema not compliant with latest one from registry') && (return undef)
l!l&&r!l&&!r
000
$self->_set_error("Schema not compliant with latest one from registry") and return undef
!ll
00
MC/DC0%0/2
$compliant || $self->_set_error('Schema not compliant with latest one from registry') && (return undef)
$compliant $self->_set_error("Schema not compliant with latest one from registry")
Truth table0%0/3
A || B
A= $compliantB= $self->_set_error("Schema not compliant with latest one from registry")
ABresult
1X1
011
000
257 return undef
258 unless $compliant;
259
260 }
261
262 }
263
264 # ...if a previous id for the schema is not available, try to add the one supplied to the registry....
265 0 ▶ unless ($schema_id) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($schema_id) 0 0 0
266
267 # ...procede adding it to the registry
268 0 $schema_id = $sr->add_schema(
269 SUBJECT => $topic,
270 TYPE => $type,
271 SCHEMA => $supplied_schema
272 );
273 0 ▶ $self->_set_error('Error adding schema to registry: ' . encode_json($sr->get_error())) &&
Branch: 1 branch
BranchTrueFalseTotal
unless $schema_id 0 0 0
Condition0%0/5
$schema_id || $self->_set_error('Error adding schema to registry: ' . &encode_json(scalar $sr->get_error)) && (return undef)
l!l&&r!l&&!r
000
$self->_set_error('Error adding schema to registry: ' . &encode_json(scalar $sr->get_error)) and return undef
!ll
00
MC/DC0%0/2
$schema_id || $self->_set_error('Error adding schema to registry: ' . &encode_json(scalar $sr->get_error)) && (return undef)
$schema_id $self->_set_error('Error adding schema to registry: ' . &encode_json(scalar $sr->get_error))
Truth table0%0/3
A || B
A= $schema_idB= $self->_set_error('Error adding schema to registry: ' . &encode_json(scalar $sr->get_error))
ABresult
1X1
011
000
274 return undef
275 unless $schema_id;
276
277 # ...and bless new schema into an Avro schema object
278 0 $avro_schema = Avro::Schema->parse($supplied_schema);
279
280 }
281
282 } else {
283
284 # retreive latest schema for the topic value
285 0 my $schema_info = $sr->get_schema(
286 SUBJECT => $topic,
287 TYPE => $type
288 );
289 0 ▶ if ( defined $schema_info ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $schema_info) { } 0 0 0
290
291 0 $schema_id = $schema_info->{id};
292 0 $avro_schema = $schema_info->{schema};
293
294 } else {
295 0 ▶ $self->_set_error("No schema in registry for subject " . $topic . '-' . 'value') &&
Branch: 1 branch
BranchTrueFalseTotal
unless $schema_info 0 0 0
Condition0%0/5
$schema_info || $self->_set_error('No schema in registry for subject ' . $topic . '-' . 'value') && (return undef)
l!l&&r!l&&!r
000
$self->_set_error('No schema in registry for subject ' . $topic . '-' . "value") and return undef
!ll
00
MC/DC0%0/2
$schema_info || $self->_set_error('No schema in registry for subject ' . $topic . '-' . 'value') && (return undef)
$schema_info $self->_set_error('No schema in registry for subject ' . $topic . '-' . "value")
Truth table0%0/3
A || B
A= $schema_infoB= $self->_set_error('No schema in registry for subject ' . $topic . '-' . "value")
ABresult
1X1
011
000
296 return undef
297 unless $schema_info;
298 }
299
300 }
301
302 0 return ($schema_id, $avro_schema);
303 }
304  
305  
306  
307  
308  
309 ##### Public methods
310  
311 =head2 METHODS
312  
313 The following methods are defined for the C<Kafka::Avro::Producer> class:
314  
315 =cut
316  
317  
318 =head3 C<schema_registry>()
319  
320 Returns the L<Confluent::SchemaRegistry|Confluent::SchemaRegistry> instance supplied to the construcor.
321  
322 =cut
323  
324 0 ▶sub schema_registry { $_[0]->{__SCHEMA_REGISTRY} }
Subroutine schema_registry: not called CC=1 SCAR=6.9
Pod documented
325  
326  
327 =head3 C<get_error>()
328  
329 Returns a string containing last error message.
330  
331 =cut
332  
333 0 ▶sub get_error { $_[0]->_get_error() }
Subroutine get_error: not called CC=1 SCAR=6.9
Pod documented
334  
335  
336 =head3 C<send( $topic, $partition, $messages, $keys, $compression_codec, $timestamps, $key_schema, $value_schema )>
337  
338 =head3 C<send( %named_params )>
339  
340 Sends Avro-formatted messages on a L<Kafka::Connection|Kafka::Connection> object.
341  
342 Returns a non-blank value (a reference to a hash with server response description)
343 if the message is successfully sent.
344  
345 In order to handle Avro format, C<Kafka::Producer|Kafka::Producer> C<send()> method is extended
346 with two more positional arguments, C<$key_schema> and C<$value_schema>:
347  
348 $producer->send(
349 $topic, # scalar
350 $partition, # scalar
351 $messages, # scalar | array
352 $keys, # (optional) undef | scalar | array
353 $compression_codec, # (optional) undef | scalar
354 $timestamps, # (optional) undef | scalar | array
355 $key_schema, # (optional) undef | JSON-string
356 $value_schema # (optional) undef | JSON-string
357 );
358  
359 Both C<$key_schema> and C<$value_schema> parameters are optional and must provide JSON strings that
360 represent Avro schemas to use to validate and serialize key(s) and value(s).
361  
362 These schemas are validated against C<$schema_registry> and, if compliant, they are added to the registry
363 under the C<$topic+'key'> or C<$topic+'value'> Schema Registry's subjects.
364  
365 If an expected schema isn't provided, latest version from Schema Registry is used accordingly to the
366 (topic + key/value) subject.
367  
368 Alternatively, for ease of use, the C<send()> method may be also used by suggesting named parameters:
369  
370 $producer->send(
371 topic => $topic, # scalar
372 partition => $partition, # scalar
373 messages => $messages, # scalar | array
374 keys => $keys, # (optional) undef | scalar | array
375 compression_codec => $compression_codec, # (optional) undef | scalar
376 timestamps => $timestamps, # (optional) undef | scalar | array
377 key_schema => $key_schema, # (optional) undef | JSON-string
378 value_schema => $value_schema # (optional) undef | JSON-string
379 );
380  
381 =cut
382  
383 0 ▶sub send {
Subroutine send: not called CC=31 SCAR=69.0
Pod documented
384 0 my $self = shift;
385 0 my %params = (
386 'topic' => undef,
387 'partition' => undef,
388 'messages' => undef,
389 'keys' => undef, # optional in Kafka::Producer
390 'compression_codec' => undef, # optional in Kafka::Producer
391 'timestamps' => undef, # optional in Kafka::Producer
392 'key_schema' => undef, # optional in Kafka::Producer::Avro
393 'value_schema' => undef # optional in Kafka::Producer::Avro
394 );
395 0 my @p = @_;
396 0 my $is_positional = 0;
397 0 ▶ if ( scalar(@p) % 2 == 0 ) {
Branch: 1 branch
BranchTrueFalseTotal
if (scalar(@p) % 2 == 0) { } 0 0 0
398 0 ▶ for ( my $i = 0 ; $i < scalar(@p)-1 ; $i += 2 ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($i < scalar(@p) - 1) 0 0 0
399 0 ▶ if (ref($p[$i])) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $p[$i]) 0 0 0
400 0 $is_positional = 1;
401 0 last;
402 }
403 0 ▶ if ( grep /^$p[$i]$/, keys(%params) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (grep /^$p[$i]$/, keys %params) 0 0 0
404 0 $params{ $p[$i] } = $p[ $i + 1 ];
405 }
406 }
407 } else {
408 0 $is_positional = 1;
409 }
410 0 ▶ if ($is_positional) {
Branch: 1 branch
BranchTrueFalseTotal
if ($is_positional) 0 0 0
411 0 $params{topic} = shift @p;
412 0 $params{partition} = shift @p;
413 0 $params{messages} = shift @p;
414 0 $params{keys} = shift @p;
415 0 $params{compression_codec} = shift @p;
416 0 $params{timestamps} = shift @p;
417 0 $params{key_schema} = shift @p;
418 0 $params{value_schema} = shift @p;
419 }
420 0 my $avro_schemas = {
421 key => {
422 id => undef,
423 schema => undef
424 },
425 value => {
426 id => undef,
427 schema => undef
428 }
429 };
430 0 my ($keys, $messages);
431  
432 0 $self->_clear_error();
433
434 $self->_set_error('Missing topic param')
435 and return undef
436 0 ▶ unless defined $params{topic};
Branch: 1 branch
BranchTrueFalseTotal
unless defined $params{"topic"} 0 0 0
Condition0%0/5
defined $params{'topic'} || $self->_set_error('Missing topic param') && (return undef)
l!l&&r!l&&!r
000
$self->_set_error("Missing topic param") and return undef
!ll
00
MC/DC0%0/2
defined $params{'topic'} || $self->_set_error('Missing topic param') && (return undef)
defined $params{'topic'} $self->_set_error("Missing topic param")
Truth table0%0/3
A || B
A= defined $params{'topic'}B= $self->_set_error("Missing topic param")
ABresult
1X1
011
000
437 $self->_set_error('Missing partition param')
438 and return undef
439 0 ▶ unless defined $params{partition};
Branch: 1 branch
BranchTrueFalseTotal
unless defined $params{"partition"} 0 0 0
Condition0%0/5
defined $params{'partition'} || $self->_set_error('Missing partition param') && (return undef)
l!l&&r!l&&!r
000
$self->_set_error("Missing partition param") and return undef
!ll
00
MC/DC0%0/2
defined $params{'partition'} || $self->_set_error('Missing partition param') && (return undef)
defined $params{'partition'} $self->_set_error("Missing partition param")
Truth table0%0/3
A || B
A= defined $params{'partition'}B= $self->_set_error("Missing partition param")
ABresult
1X1
011
000
440 $self->_set_error('Missing messages')
441 and return undef
442 0 ▶ unless defined $params{messages};
Branch: 1 branch
BranchTrueFalseTotal
unless defined $params{"messages"} 0 0 0
Condition0%0/5
defined $params{'messages'} || $self->_set_error('Missing messages') && (return undef)
l!l&&r!l&&!r
000
$self->_set_error("Missing messages") and return undef
!ll
00
MC/DC0%0/2
defined $params{'messages'} || $self->_set_error('Missing messages') && (return undef)
defined $params{'messages'} $self->_set_error("Missing messages")
Truth table0%0/3
A || B
A= defined $params{'messages'}B= $self->_set_error("Missing messages")
ABresult
1X1
011
000
443
444 # Get Avro schema for keys and values
445 0 foreach my $type (qw/key value/) {
446 0 ($avro_schemas->{$type}->{id}, $avro_schemas->{$type}->{schema}) = $self->_get_avro_schema($params{topic}, $type, $params{"${type}_schema"});
447 return undef
448 0 ▶ unless $avro_schemas->{$type}->{id};
Branch: 1 branch
BranchTrueFalseTotal
unless $avro_schemas->{$type}{"id"} 0 0 0
449 }
450  
451 0 ▶ if ($params{keys}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($params{"keys"}) 0 0 0
452
453 # Return if key Avro schema was not found
454 $self->_set_error('No key Avro schema found')
455 and return undef
456 0 ▶ unless defined $avro_schemas->{key}->{id} && defined $avro_schemas->{key}->{schema};
Branch: 1 branch
BranchTrueFalseTotal
unless defined $avro_schemas->{"key"}{"id"} and defined $avro_schemas->{"key"}{"schema"} 0 0 0
Condition0%0/8
defined $avro_schemas->{'key'}{'id'} && defined $avro_schemas->{'key'}{'schema'} || $self->_set_error('No key Avro schema found') && (return undef)
l!l&&r!l&&!r
000
defined $avro_schemas->{"key"}{"id"} and defined $avro_schemas->{"key"}{"schema"}
!ll&&!rl&&r
000
$self->_set_error("No key Avro schema found") and return undef
!ll
00
MC/DC0%0/3
defined $avro_schemas->{'key'}{'id'} && defined $avro_schemas->{'key'}{'schema'} || $self->_set_error('No key Avro schema found') && (return undef)
defined $avro_schemas->{"key"}{"id"} defined $avro_schemas->{"key"}{"schema"} $self->_set_error("No key Avro schema found")
Truth table0%0/5
A and B || C
A= defined $avro_schemas->{"key"}{"id"}B= defined $avro_schemas->{"key"}{"schema"}C= $self->_set_error("No key Avro schema found")
ABCresult
11X1
0X11
1011
0X00
1000
457
458 # Avro encoding of keys
459 0 ▶ if (ref($params{keys}) eq 'ARRAY') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $params{'keys'} eq 'ARRAY') { } 0 0 0
460 $keys = [
461 map {
462 0 _encode($avro_schemas->{key}, $_);
463 0 } @{$params{keys}}
464 ];
465 } else {
466 0 $keys = _encode($avro_schemas->{key}, $params{keys});
467 }
468
469 }
470  
471 # Return if value Avro schema was not found
472 $self->_set_error('No value Avro schema found')
473 and return undef
474 0 ▶ unless defined $avro_schemas->{value}->{id} && defined $avro_schemas->{value}->{schema};
Branch: 1 branch
BranchTrueFalseTotal
unless defined $avro_schemas->{"value"}{"id"} and defined $avro_schemas->{"value"}{"schema"} 0 0 0
Condition0%0/8
defined $avro_schemas->{'value'}{'id'} && defined $avro_schemas->{'value'}{'schema'} || $self->_set_error('No value Avro schema found') && (return undef)
l!l&&r!l&&!r
000
defined $avro_schemas->{"value"}{"id"} and defined $avro_schemas->{"value"}{"schema"}
!ll&&!rl&&r
000
$self->_set_error("No value Avro schema found") and return undef
!ll
00
MC/DC0%0/3
defined $avro_schemas->{'value'}{'id'} && defined $avro_schemas->{'value'}{'schema'} || $self->_set_error('No value Avro schema found') && (return undef)
defined $avro_schemas->{"value"}{"id"} defined $avro_schemas->{"value"}{"schema"} $self->_set_error("No value Avro schema found")
Truth table0%0/5
A and B || C
A= defined $avro_schemas->{"value"}{"id"}B= defined $avro_schemas->{"value"}{"schema"}C= $self->_set_error("No value Avro schema found")
ABCresult
11X1
0X11
1011
0X00
1000
475
476 # Avro encoding of messages
477 0 ▶ if (ref($params{messages}) eq 'ARRAY') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $params{'messages'} eq 'ARRAY') { } 0 0 0
478 $messages = [
479 map {
480 0 _encode($avro_schemas->{value}, $_);
481 0 } @{$params{messages}}
482 ];
483 } else {
484 0 $messages = _encode($avro_schemas->{value}, $params{messages});
485 }
486
487 # Return in case of mismatch between keys and messages
488 0 ▶ if ($avro_schemas->{key}->{id}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($avro_schemas->{"key"}{"id"}) 0 0 0
489 0 ▶ $self->_set_error('Keys/messages format mismatch')
Branch: 1 branch
BranchTrueFalseTotal
if ref $keys eq "ARRAY" and ref $messages ne "ARRAY" 0 0 0
Condition0%0/8
ref $keys eq 'ARRAY' && ref $messages ne 'ARRAY' && ($self->_set_error('Keys/messages format mismatch') && (return undef))
!ll&&!rl&&r
000
ref $keys eq "ARRAY" and ref $messages ne "ARRAY"
!ll&&!rl&&r
000
$self->_set_error("Keys/messages format mismatch") and return undef
!ll
00
MC/DC0%0/3
ref $keys eq 'ARRAY' && ref $messages ne 'ARRAY' && ($self->_set_error('Keys/messages format mismatch') && (return undef))
ref $keys eq "ARRAY" ref $messages ne "ARRAY" $self->_set_error("Keys/messages format mismatch")
Truth table0%0/4
A and B && C
A= ref $keys eq "ARRAY"B= ref $messages ne "ARRAY"C= $self->_set_error("Keys/messages format mismatch")
ABCresult
0XX0
10X0
1100
1111
490 and return undef
491 if ref($keys) eq 'ARRAY' && ref($messages) ne 'ARRAY';
492 0 ▶ $self->_set_error('Keys/messages count mismatch')
Branch: 1 branch
BranchTrueFalseTotal
if ref $keys eq "ARRAY" and ref $messages eq "ARRAY" and $keys->$#* != $messages->$#* 0 0 0
Condition0%0/11
ref $keys eq 'ARRAY' && ref $messages eq 'ARRAY' && $keys->$#* != $messages->$#* && ($self->_set_error('Keys/messages count mismatch') && (return undef))
!ll&&!rl&&r
000
ref $keys eq "ARRAY" and ref $messages eq "ARRAY" and $keys->$#* != $messages->$#*
!ll&&!rl&&r
000
ref $keys eq "ARRAY" and ref $messages eq "ARRAY"
!ll&&!rl&&r
000
$self->_set_error("Keys/messages count mismatch") and return undef
!ll
00
MC/DC0%0/4
ref $keys eq 'ARRAY' && ref $messages eq 'ARRAY' && $keys->$#* != $messages->$#* && ($self->_set_error('Keys/messages count mismatch') && (return undef))
ref $keys eq "ARRAY" ref $messages eq "ARRAY" $keys->$#* != $messages->$#* $self->_set_error("Keys/messages count mismatch")
Truth table0%0/5
A and B and C && D
A= ref $keys eq "ARRAY"B= ref $messages eq "ARRAY"C= $keys->$#* != $messages->$#*D= $self->_set_error("Keys/messages count mismatch")
ABCDresult
0XXX0
10XX0
110X0
11100
11111
493 and return undef
494 if ref($keys) eq 'ARRAY' && ref($messages) eq 'ARRAY' && $#$keys != $#$messages;
495 }
496
497 # Send messages through Kafka::Producer parent class
498 return $self->SUPER::send(
499 $params{topic},
500 $params{partition},
501 $messages,
502 $keys,
503 $params{compression_codec},
504 $params{timestamps}
505 0 );
506
507 }
508  
509  
510 =head3 C<bulk_send( %params )>
511  
512 Similar to C<send> but uses bulks to avoid memory leaking.
513  
514 Extra named parameters are expected:
515  
516 =over 3
517  
518 =item C<size =E<gt> $size>
519  
520 The size of the bulk
521  
522 =item C<on_before_send_bulk =E<gt> sub {...} > (optional)
523  
524 A code block that will be executed before the sending of each bulk.
525  
526 The block will receive the following positional parameters:
527  
528 =over 3
529  
530 =item C<$bulk_num> the number of the bulk
531  
532 =item C<$bulk_messages> the number of messages in the bulk
533  
534 =item C<$bulk_keys> the number of keys in the bulk
535  
536 =item C<$index_from> the absolute index of the first message in the bulk
537  
538 =item C<$index_to> the absolute index of the last message in the bulk
539  
540 =back
541  
542 =item C<on_after_send_bulk =E<gt> sub {...} > (optional)
543  
544 A code block that will be executed after the sending of each bulk.
545  
546 The block will receive the following positional parameters:
547  
548 =over 3
549  
550 =item C<$sent> the number of sent messages in the bulk
551  
552 =item C<$total_sent> the total number of messages sent
553  
554 =back
555  
556 =item C<on_init =E<gt> sub {...} > (optional)
557  
558 A code block that will be executed only once before at the beginning of the cycle.
559  
560 The block will receive the following positional parameters:
561  
562 =over 3
563  
564 =item C<$to_send> the total number of messages to send
565  
566 =item C<$bulk_size> the size of the bulk
567  
568 =back
569  
570 =item C<on_complete =E<gt> sub {...} > (optional)
571  
572 A code block that will be executed only once after the end of the cycle.
573  
574 The block will receive the following positional parameters:
575  
576 =over 3
577  
578 =item C<$to_send> the total number of messages to send
579  
580 =item C<$total_sent> the total number of messages sent
581  
582 =item C<$errors> the number bulks sent with errors
583  
584 =back
585  
586 =item C<on_send_error =E<gt> sub {...} > (optional)
587  
588 A code block that will be executed when a bulk registers an error.
589  
590 =back
591  
592 =cut
593  
594 0 ▶sub bulk_send {
Subroutine bulk_send: not called CC=24 SCAR=64.0
Pod documented
595 0 my $self = shift;
596 0 my %params = @_;
597
598 0 $self->_clear_error();
599
600 # If there is no an array of messages use send()
601 return $self->send(%params)
602 0 ▶ unless ref($params{messages}) eq 'ARRAY';
Branch: 1 branch
BranchTrueFalseTotal
unless ref $params{"messages"} eq "ARRAY" 0 0 0
603
604 $self->_set_error('Missing bulk size')
605 and return undef
606 0 ▶ unless defined $params{size};
Branch: 1 branch
BranchTrueFalseTotal
unless defined $params{"size"} 0 0 0
Condition0%0/5
defined $params{'size'} || $self->_set_error('Missing bulk size') && (return undef)
l!l&&r!l&&!r
000
$self->_set_error("Missing bulk size") and return undef
!ll
00
MC/DC0%0/2
defined $params{'size'} || $self->_set_error('Missing bulk size') && (return undef)
defined $params{'size'} $self->_set_error("Missing bulk size")
Truth table0%0/3
A || B
A= defined $params{'size'}B= $self->_set_error("Missing bulk size")
ABresult
1X1
011
000
607 $self->_set_error('Bad bulk size')
608 and return undef
609 0 ▶ unless $params{size} =~ /^\d+$/;
Branch: 1 branch
BranchTrueFalseTotal
unless $params{"size"} =~ /^\d+$/ 0 0 0
Condition0%0/5
$params{'size'} =~ /^\d+$/ || $self->_set_error('Bad bulk size') && (return undef)
l!l&&r!l&&!r
000
$self->_set_error("Bad bulk size") and return undef
!ll
00
MC/DC0%0/2
$params{'size'} =~ /^\d+$/ || $self->_set_error('Bad bulk size') && (return undef)
$params{'size'} =~ /^\d+$/ $self->_set_error("Bad bulk size")
Truth table0%0/3
A || B
A= $params{'size'} =~ /^\d+$/B= $self->_set_error("Bad bulk size")
ABresult
1X1
011
000
610  
611 0 my $messages = $params{messages};
612 0 my $keys = $params{keys};
613 0 ▶ $self->_set_error('Keys/messages format mismatch')
Branch: 1 branch
BranchTrueFalseTotal
if ref $keys eq "ARRAY" and ref $messages ne "ARRAY" 0 0 0
Condition0%0/8
ref $keys eq 'ARRAY' && ref $messages ne 'ARRAY' && ($self->_set_error('Keys/messages format mismatch') && (return undef))
!ll&&!rl&&r
000
ref $keys eq "ARRAY" and ref $messages ne "ARRAY"
!ll&&!rl&&r
000
$self->_set_error("Keys/messages format mismatch") and return undef
!ll
00
MC/DC0%0/3
ref $keys eq 'ARRAY' && ref $messages ne 'ARRAY' && ($self->_set_error('Keys/messages format mismatch') && (return undef))
ref $keys eq "ARRAY" ref $messages ne "ARRAY" $self->_set_error("Keys/messages format mismatch")
Truth table0%0/4
A and B && C
A= ref $keys eq "ARRAY"B= ref $messages ne "ARRAY"C= $self->_set_error("Keys/messages format mismatch")
ABCresult
0XX0
10X0
1100
1111
614 and return undef
615 if ref($keys) eq 'ARRAY' && ref($messages) ne 'ARRAY';
616 0 ▶ $self->_set_error('Keys/messages count mismatch')
Branch: 1 branch
BranchTrueFalseTotal
if ref $keys eq "ARRAY" and ref $messages eq "ARRAY" and $keys->$#* != $messages->$#* 0 0 0
Condition0%0/11
ref $keys eq 'ARRAY' && ref $messages eq 'ARRAY' && $keys->$#* != $messages->$#* && ($self->_set_error('Keys/messages count mismatch') && (return undef))
!ll&&!rl&&r
000
ref $keys eq "ARRAY" and ref $messages eq "ARRAY" and $keys->$#* != $messages->$#*
!ll&&!rl&&r
000
ref $keys eq "ARRAY" and ref $messages eq "ARRAY"
!ll&&!rl&&r
000
$self->_set_error("Keys/messages count mismatch") and return undef
!ll
00
MC/DC0%0/4
ref $keys eq 'ARRAY' && ref $messages eq 'ARRAY' && $keys->$#* != $messages->$#* && ($self->_set_error('Keys/messages count mismatch') && (return undef))
ref $keys eq "ARRAY" ref $messages eq "ARRAY" $keys->$#* != $messages->$#* $self->_set_error("Keys/messages count mismatch")
Truth table0%0/5
A and B and C && D
A= ref $keys eq "ARRAY"B= ref $messages eq "ARRAY"C= $keys->$#* != $messages->$#*D= $self->_set_error("Keys/messages count mismatch")
ABCDresult
0XXX0
10XX0
110X0
11100
11111
617 and return undef
618 if ref($keys) eq 'ARRAY' && ref($messages) eq 'ARRAY' && $#$keys != $#$messages;
619
620 0 my @messages = @$messages; # duplicate messages array to preserve external array
621 # Check for key(s) format (ARRAY vs. SCALAR)
622 0 my (@keys, $key);
623 0 ▶ if (ref($params{keys}) eq 'ARRAY') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $params{'keys'} eq 'ARRAY') { } 0 0 0
624 0 @keys = @$keys;
625 } else {
626 0 $key = $keys;
627 }
628 0 my $message_count = scalar(@messages);
629 0 my $bulk_size = $params{size};
630 0 my $sent = 0;
631 0 my $errors = 0;
632 0 my $bulk_num = 0;
633
634 0 ▶ if (ref($params{on_init}) eq 'CODE') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $params{"on_init"} eq "CODE") 0 0 0
635 0 $params{on_init}->($message_count, $bulk_size);
636 }
637 0 ▶ for (my $i=1; $i<=$message_count; $i+=$bulk_size) {
Branch: 1 branch
BranchTrueFalseTotal
if ($i <= $message_count) 0 0 0
638 0 $bulk_num++;
639 0 my @bulk = splice @messages, 0, $bulk_size;
640 0 my $bulk_keys = $key;
641 0 ▶ unless ($bulk_keys) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($bulk_keys) 0 0 0
642 0 $bulk_keys = [ splice(@keys, 0, $bulk_size) ];
643 }
644 0 ▶ if (ref($params{on_before_send_bulk}) eq 'CODE') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $params{"on_before_send_bulk"} eq "CODE") 0 0 0
645 0 ▶ $params{on_before_send_bulk}->($bulk_num, \@bulk, $bulk_keys, $i, ($i+$bulk_size<$message_count ? $i+$bulk_size-1 : $message_count));
Branch: 1 branch
BranchTrueFalseTotal
$i + $bulk_size < $message_count ? : 0 0 0
646 }
647 my $res = $self->send(
648 'topic' => $params{topic},
649 'partition' => 0,
650 'messages' => [ @bulk ],
651 'keys' => $bulk_keys,
652 'compressione_codec' => undef,
653 'key_schema' => $params{key_schema},
654 'value_schema' => $params{value_schema},
655 'timestamps' => $params{timestamps}
656 0 );
657 0 ▶ if (defined $res) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $res) { } 0 0 0
658 0 $sent += scalar(@bulk);
659 0 ▶ if (ref($params{on_after_send_bulk}) eq 'CODE') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $params{"on_after_send_bulk"} eq "CODE") 0 0 0
660 0 $params{on_after_send_bulk}->(scalar(@bulk), $sent);
661 }
662 } else {
663 0 $errors++;
664 0 ▶ if (ref($params{on_send_error}) eq 'CODE') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $params{"on_send_error"} eq "CODE") 0 0 0
665 0 ▶ $params{on_send_error}->($self->get_error(), $bulk_num, \@bulk, $bulk_keys, $i, ($i+$bulk_size<$message_count ? $i+$bulk_size-1 : $message_count));
Branch: 1 branch
BranchTrueFalseTotal
$i + $bulk_size < $message_count ? : 0 0 0
666 }
667 }
668 }
669 0 ▶ if (ref($params{on_complete}) eq 'CODE') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $params{"on_complete"} eq "CODE") 0 0 0
670 0 $params{on_complete}->($message_count, $sent, $errors);
671 }
672 0 return $sent;
673
674 }
675  
676 =head1 AUTHOR
677  
678 Alvaro Livraghi, E<lt>alvarol@cpan.orgE<gt>
679  
680 =head1 CONTRIBUTE
681  
682 L<https://github.com/alivraghi/Kafka-Producer-Avro>
683  
684 =head1 BUGS
685  
686 Please use GitHub project link above to report problems or contact authors.
687  
688 =head1 COPYRIGHT AND LICENSE
689  
690 Copyright 2018 by Alvaro Livraghi
691  
692 This program is free software; you can redistribute it and/or modify it under the same terms as Perl itself.
693  
694 =cut
695  
696 1 1;
↑ Summary