LongSteps/Storage/DynamoDB.pm untested

filter: Statementstmt 0.0% 0 / 252 Branchbran 0.0% 0 / 40 Conditioncond 0.0% 0 / 15 MC/DCmcdc 0.0% 0 / 8 Subroutinesub 0.0% 0 / 33 Podpod 0.0% 0 / 17 totaltotal 0.0% 0 / 365 CC 21 SCAR 22.0
CC
21
Cov
0%
destroy_table
30.0
_process_from_attrmap
20.0
table_status
12.0
CRAP 9.0
1 package Schedule::LongSteps::Storage::DynamoDB;
2 $Schedule::LongSteps::Storage::DynamoDB::VERSION = '0.023';
3 use Moose;
4 extends qw/Schedule::LongSteps::Storage/;
5  
6 use Compress::Zlib;
7 use DateTime::Format::ISO8601;
8 use DateTime;
9 use JSON qw//;
10 use Log::Any qw/$log/;
11 use MIME::Base64;
12 use Scalar::Util;
13  
14 my $TIME_MAX = '9999-12-31T23:59:59.999Z';
15  
16 =head1 NAME
17  
18 Schedule::LongSteps::Storage::DynamoDB - A DynamoDB backed longstep storage.
19  
20 =cut
21  
22 =head1 SYNOPSIS
23  
24  
25 my $dynamo_db = Paws->service('DynamoDB', ...); # see Paws
26 # You can also look in t/storage-dynamodb.t for a working example.
27  
28 my $storage = Schedule::LongSteps::Storage::DynamoDB->new({ dynamo_db => $dynamo_db, table_prefix => 'my_app_longsteps' });
29  
30 # Call that only once as part of your persistent data management:
31 $storage->vivify_table();
32  
33 my $sl = Schedule::LongSteps->new({ storage => $storage, ... }, ... );
34  
35 =cut
36  
37 has 'dynamo_db' => (is => 'ro', isa => 'Paws::DynamoDB' , required => 1 );
38 has 'table_prefix' => ( is => 'ro', isa => 'Str', required => 1);
39  
40 has 'table_name' => ( is => 'ro', isa => 'Str', lazy_build => 1);
41  
42 has 'creation_wait_time' => ( is => 'ro', isa => 'Int', default => 2 );
43  
44 sub _build_table_name{
45 my ($self) = @_;
46 my $package = __PACKAGE__;
47 $package =~ s/::/_/g;
48 return $self->table_prefix().'_'.$package;
49 }
50  
51 =head2 table_active
52  
53 The remote DynamoDB table exists and is active.
54  
55 =cut
56  
57 sub table_active{
58 my ($self) = @_;
59 my $status = $self->table_status();
60 unless( defined( $status ) ){ $status = 'NOMATCH'; }
61 return $status eq 'ACTIVE';
62 }
63  
64 =head2 table_exists
65  
66 The remote DynamoDB table exists.
67  
68 =cut
69  
70 sub table_exists{
71 return defined( shift->table_status() );
72 }
73  
74 =head2 table_status
75  
76 Returns the table status (or undef if the table doens't exists at all)
77  
78 Usage:
79  
80 if( my $status = $self->table_status() ){ .. }
81  
82 Returned status can be one of those described here: L<https://metacpan.org/pod/Paws::DynamoDB::TableDescription>
83  
84 =cut
85  
86 sub table_status{
87 my ($self) = @_;
88 my $desc_table = eval{ $self->dynamo_db()->DescribeTable( TableName => $self->table_name() ) };
89 if( my $err = $@ ){
90 if( Scalar::Util::blessed($err) &&
91 $err->isa('Paws::Exception') &&
92 $err->code() eq 'ResourceNotFoundException'
93 ){
94 $log->debug("No table ".$self->table_name());
95 return undef;
96 }
97 # Rethrow any other error.
98 confess( $err );
99 }
100 return $desc_table->Table()->TableStatus();
101 }
102  
103 =head2 vivify_table
104  
105 Vivifies the remote DynamoDB table to support this storage.
106  
107 You need to call that at least once as part of your persistent data
108 management process, or at the beginning of your application.
109  
110 =cut
111  
112 sub vivify_table{
113 my ($self) = @_;
114  
115 my $table_name = $self->table_name();
116 if( $self->table_exists() ){
117 $log->warn("Table $table_name already exists in remote DynamoDB. Not creating it");
118 return;
119 }
120 # Get all tables and check this one is not there already.
121 $log->info("Creating Table ".$table_name." in dynamoDB");
122  
123 my $creation = $self->dynamo_db()->CreateTable(
124 TableName => $table_name,
125 AttributeDefinitions => [
126 { AttributeName => 'id', AttributeType => 'S' },
127 ## Note those are only there for reference.
128 ## as AttributeDefinition must only defined attributes
129 ## used in the KeySchema
130 # { AttributeName => 'process_class', AttributeType => 'S' },
131 # { AttributeName => 'mstatus', AttributeType => 'S' },
132 # { AttributeName => 'what', AttributeType => 'S' },
133 { AttributeName => 'run_at_day' , AttributeType => 'S' }, # ISO8601 YYYY-MM-DD
134 { AttributeName => 'run_at', AttributeType => 'S' }, # ISO8601 YYYY-MM-DDTHH:MM:SS.000
135 # { AttributeName => 'run_id', AttributeType => 'S' }, # The current run_id
136 # { AttributeName => 'mstate', AttributeType => 'S' },
137 # { AttributeName => 'merror', AttributeType => 'S' },
138 ],
139 KeySchema => [
140 { AttributeName => 'id', KeyType => 'HASH' },
141 ],
142 GlobalSecondaryIndexes => [
143 {
144 IndexName => 'by_run_at_day',
145 KeySchema => [
146 { AttributeName => 'run_at_day', KeyType => 'HASH' },
147 { AttributeName => 'run_at', KeyType => 'RANGE' },
148 ],
149 Projection => {
150 NonKeyAttributes => [ 'run_id' ],
151 ProjectionType => 'INCLUDE'
152 },
153 ProvisionedThroughput => {
154 ReadCapacityUnits => 2,
155 WriteCapacityUnits => 2,
156 }
157 }
158 ],
159 ProvisionedThroughput => {
160 ReadCapacityUnits => 2,
161 WriteCapacityUnits => 2,
162 # This is low to avoid having a large bill in case
163 # of tests failure.
164 # Note that we can change that AFTER the fact
165 },
166 );
167  
168 while(! $self->table_active() ){
169 $log->info("Table $table_name not active yet. Waiting ".$self->creation_wait_time()." second");
170 if( $self->creation_wait_time() ){
171 sleep($self->creation_wait_time());
172 }
173 }
174 $log->info("Table $table_name ACTIVE. All is fine");
175 return $creation;
176 }
177  
178 =head2 prepare_due_processes
179  
180 See L<Schedule::LongSteps::Storage>
181  
182 =cut
183  
184 sub prepare_due_processes{
185 my ($self, $options ) = @_;
186  
187 $options ||= {};
188  
189 my $now = DateTime->now();
190 my $run_at_day = $now->clone()->truncate( to => 'day' );
191  
192 my @found_items = ();
193 my $query_output;
194 do{
195 # Query the by_run_at_day index
196 $query_output = $self->dynamo_db()->Query(
197 TableName => $self->table_name(),
198 IndexName => 'by_run_at_day',
199 ConsistentRead => 0, # Consistent read are not supported on secondary indices.
200 ExpressionAttributeValues => {
201 ":run_at_day" => { "S" => substr( $run_at_day->iso8601() , 0 , 10 ) },
202 ":now" => { "S" => $now->iso8601().'Z' },
203 ":null" => { "S" => 'NULL' },
204 },
205 Limit => 20,
206 KeyConditionExpression => 'run_at_day = :run_at_day AND run_at <= :now',
207 FilterExpression => 'run_id = :null',
208 ProjectionExpression => 'id',
209 );
210  
211 # Next we are going to look at the day before
212 $run_at_day->subtract( days => 1 )->truncate( to => 'day' );
213  
214 $log->info("Got ".$query_output->Count()." results back for run_at_day = $run_at_day");
215 push @found_items , @{ $query_output->Items() };
216  
217 } while( $query_output->Count() );
218  
219 # Of all the items found, we need to inject a run_id in those who dont have any yet
220 # and return only those ones.
221 my $run_id = $self->uuid()->create_str();
222 $log->info("Will set run_id=$run_id on due items");
223 my @locked_processes;
224  
225 ( $options->{concurrent_fiddle} || sub{} )->();
226  
227 foreach my $item ( @found_items ){
228 my $id = $item->Map()->{id}->S();
229 my $update_output = eval{
230 $self->dynamo_db()->UpdateItem(
231 TableName => $self->table_name(),
232 ExpressionAttributeValues => {
233 ":run_id" => { S => $run_id },
234 ":null" => { "S" => 'NULL' },
235 },
236 Key => { id => { S => $id } },
237 UpdateExpression => 'SET run_id = :run_id',
238 ConditionExpression => 'run_id = :null',
239 ReturnValues => 'ALL_NEW',
240 );
241 };
242 if( my $err = $@ ){
243 # This error is expected in case of race condition.
244 if( $err =~ m/^The conditional request failed/ ){
245 $log->info("INVALID ITEM Item ".$id." has not been updated with our run_id. It was taken by another run id");
246 next;
247 }
248 # This is a non-expected error.
249 confess( $err );
250 }
251  
252 my $attributes = $update_output->Attributes()->Map();
253 my $new_run_id = $attributes->{run_id}->S();
254 $log->info("VALID ITEM Updated item ".$id." with our run_id=".$run_id);
255 push @locked_processes, $self->_process_from_attrmap( $update_output->Attributes() );
256 }
257  
258 return @locked_processes;
259 }
260  
261 =head2 create_process
262  
263 See L<Schedule::LongSteps::Storage>
264  
265 =cut
266  
267 sub create_process{
268 my ($self, $properties) = @_;
269 my $o = Schedule::LongSteps::Storage::DynamoDB::Process->new({
270 storage => $self,
271 id => $self->uuid()->create_str(),
272 %{$properties}
273 });
274 $o->_insert();
275 return $o;
276 }
277  
278 =head2 find_process
279  
280 See L<Schedule::LongSteps::Storage>
281  
282 =cut
283  
284 sub find_process{
285 my ($self, $pid) = @_;
286 my $dynamo_item = $self->dynamo_db()->GetItem(
287 TableName => $self->table_name(),
288 ConsistentRead => 1,
289 Key => {
290 id => { S => $pid }
291 }
292 )->Item();
293 unless( $dynamo_item ){
294 return undef;
295 }
296 return $self->_process_from_attrmap( $dynamo_item );
297 }
298  
299 sub _state_decode{
300 my ($self, $dynamo_state) = @_;
301 if( $dynamo_state =~ /^{/ ){
302 # Assume JSON
303 return JSON::from_json( $dynamo_state );
304 }
305  
306 # Assume base64 encoded memGunzip
307 return JSON::from_json( Compress::Zlib::memGunzip( MIME::Base64::decode_base64( $dynamo_state ) ) );
308 }
309  
310 sub _process_from_attrmap{
311 my ($self, $dynamoItem) = @_;
312 my $map = $dynamoItem->Map();
313  
314 my $run_at = $map->{run_at}->S();
315 if( $run_at eq $TIME_MAX ){
316 $run_at = undef;
317 }else{
318 $run_at = DateTime::Format::ISO8601->parse_datetime( $run_at );
319 }
320 my $run_id = $map->{run_id}->S();
321 if( $run_id eq 'NULL' ){
322 $run_id = undef;
323 }
324 my $state = $self->_state_decode( $map->{mstate}->S() );
325 my $error = $map->{merror}->S();
326 if( $error eq 'NULL' ){
327 $error = undef;
328 }
329  
330 return Schedule::LongSteps::Storage::DynamoDB::Process->new({
331 storage => $self,
332 id => $map->{id}->S(),
333 process_class => $map->{process_class}->S(),
334 status => $map->{mstatus}->S(),
335 what => $map->{what}->S(),
336 run_at => $run_at,
337 run_id => $run_id,
338 state => $state,
339 error => $error,
340 });
341 }
342  
343  
344 =head2 destroy_table
345  
346 Destroys this table. Mainly so tests don't leave some crap behind.
347  
348 Use that responsibly. Which is never except in tests.
349  
350 Note that this blocks until the table has effectively been deleted remotely.
351  
352 =cut
353  
354 sub destroy_table{
355 my ($self, $am_i_sure) = @_;
356 unless( $am_i_sure eq 'I am very sure and I am not insane' ){
357 confess("Sorry we cannot let you do that");
358 }
359 unless( $self->table_active() ){
360 confess("Sorry this table is not ACTIVE. Too early to destroy");
361 }
362 my $table_name = $self->table_name();
363 unless( $table_name =~ /^testdelete/ ){
364 confess("Sorry this is not a test table (from the test suite. Destroy manually if you are sure");
365 }
366 $log->warn("Will destroy $table_name from dynamoDB");
367 my $deletion = $self->dynamo_db()->DeleteTable(TableName => $table_name);
368 # Wait until the table is effectively destroyed.
369 while( $self->table_exists() ){
370 $log->warn("Table $table_name not destroyed yet. Waiting ".$self->creation_wait_time()." second");
371 if( $self->creation_wait_time() ){
372 sleep( $self->creation_wait_time() );
373 }
374 }
375 $log->warn("Table $table_name DESTROYED");
376 return $deletion;
377 }
378  
379 __PACKAGE__->meta()->make_immutable();
380  
381 package Schedule::LongSteps::Storage::DynamoDB::Process;
382 $Schedule::LongSteps::Storage::DynamoDB::Process::VERSION = '0.023';
383 use Moose;
384  
385 use Compress::Zlib;
386 use Data::Dumper;
387 use DateTime::Format::ISO8601;
388 use DateTime;
389 use JSON qw//;
390 use Log::Any qw/$log/;
391 use MIME::Base64;
392  
393 has 'storage' => ( is => 'ro', isa => 'Schedule::LongSteps::Storage::DynamoDB', required => 1 );
394 has 'id' => ( is => 'ro', isa => 'Str', required => 1 );
395  
396 has 'process_class' => ( is => 'rw', isa => 'Str', required => 1); # rw only for test. Should not changed ever.
397 has 'status' => ( is => 'rw', isa => 'Str', default => 'pending' );
398 has 'what' => ( is => 'rw' , isa => 'Str', required => 1);
399 has 'run_at' => ( is => 'rw', isa => 'Maybe[DateTime]', default => sub{ undef; } );
400 has 'run_id' => ( is => 'rw', isa => 'Maybe[Str]', default => sub{ undef; } );
401 has 'state' => ( is => 'rw', default => sub{ {}; });
402 has 'error' => ( is => 'rw', isa => 'Maybe[Str]', default => sub{ undef; } );
403  
404 my $MAX_DYNAMO_ITEM_SIZE = 350_000;
405  
406 # Local to Dynamo Items.
407 my $MEMORY_TO_DYNAMO = {
408 id => [ 'id' ],
409 process_class => [ 'process_class' ],
410 status => [ 'mstatus' ], # My status
411 what => [ 'what' ],
412 run_at => [ 'run_at', 'run_at_day' ],
413 run_id => [ 'run_id' ],
414 state => [ 'mstate' ], # My state
415 error => [ 'merror' ], # My error
416 };
417  
418  
419 =head2 update
420  
421 Updates via upsert.
422  
423 =cut
424  
425 sub update{
426 my ($self, $attributes) = @_;
427  
428 foreach my $key ( keys %$attributes ){
429 $self->$key( $attributes->{$key} );
430 }
431 my $dynamo_item = $self->_to_dynamo_item();
432 my $to_update = {};
433 # Only keep attributes that were updated.
434 foreach my $attr ( keys %$attributes ){
435 foreach my $dynamo_attr ( @{ $MEMORY_TO_DYNAMO->{$attr} } ){
436 $to_update->{$dynamo_attr} = $dynamo_item->{$dynamo_attr};
437 }
438 }
439  
440 $log->info("Updating Item ID = ".$self->id()." with ".Dumper( $to_update ) );
441  
442 my @update_keys = keys %$to_update;
443 my $expression_attributes = {
444 map { ':'.$_ => $to_update->{$_} } @update_keys
445 };
446 # $log->debug("ExpressionAttributes = ".Dumper( $expression_attributes ) );
447 my $update_expression = 'SET '.join(', ', map { $_.' = :'.$_ } @update_keys );
448 # $log->debug("UpdateExpression = ".$update_expression);
449  
450 $self->storage()->dynamo_db()->UpdateItem(
451 TableName => $self->storage()->table_name(),
452 Key => { id => { S => $self->id() } },
453 ExpressionAttributeValues => $expression_attributes,
454 UpdateExpression => $update_expression
455 );
456  
457 return $self;
458 }
459  
460 =head2 discard_changes
461  
462 Updates what is updatable.
463  
464 =cut
465  
466 sub discard_changes{
467 my ($self) = @_;
468 my $fresh_one = $self->storage()->find_process( $self->id() );
469 foreach my $rw_attr ( qw/process_class status what run_at run_id state error/ ){
470 $self->$rw_attr( $fresh_one->$rw_attr() );
471 }
472 return $self;
473 }
474  
475 sub _insert{
476 my ($self) = @_;
477 my $table_name = $self->storage()->table_name();
478  
479 $log->info("Inserting ".ref($self)." , id = ".$self->id()." in Dynamo table ".$table_name);
480 $self->storage()->dynamo_db->PutItem(
481 TableName => $table_name,
482 Item => $self->_to_dynamo_item(),
483 );
484 return $self;
485 }
486  
487 sub _state_encode{
488 my ($self) = @_;
489 my $state_json = JSON::to_json(
490 $self->state(),
491 { ascii => 1 }
492 );
493 unless( length( $state_json ) > $MAX_DYNAMO_ITEM_SIZE ){
494 $log->debug("Encoded state is ".substr( $state_json, 0 , 2000 ));
495 return $state_json;
496 }
497  
498 $log->debug("State JSON is over 350KB, compressing");
499 my $state_b64zjs = MIME::Base64::encode_base64(
500 Compress::Zlib::memGzip( $state_json ) );
501 if( length( $state_b64zjs ) > $MAX_DYNAMO_ITEM_SIZE ){
502 confess("Compressed state is too large (over 350000 bytes)");
503 }
504 $log->debug("Encoded state is ".substr( $state_b64zjs , 0, 1000 ) .'...' );
505 return $state_b64zjs;
506 }
507  
508 sub _error_trim{
509 my ($self) = @_;
510 unless( $self->error() ){
511 return undef;
512 }
513 return substr( $self->error() , 0 , 2000 );
514 }
515  
516 sub _to_dynamo_item{
517 my ($self) = @_;
518 my $run_at_str = $TIME_MAX;
519 if( my $run_at = $self->run_at() ){
520 $run_at_str = $run_at->iso8601().'Z';
521 }
522 my $run_at_str_day = substr( $run_at_str, 0, 10 );
523  
524  
525 return {
526 id => { S => $self->id() },
527 process_class => { S => $self->process_class() },
528 mstatus => { S => $self->status() },
529 what => { S => $self->what() },
530 run_at_day => { S => $run_at_str_day },
531 run_at => { S => $run_at_str },
532 run_id => { S => $self->run_id() || 'NULL' },
533 mstate => { S => $self->_state_encode() },
534 merror => { S => $self->_error_trim() || 'NULL' },
535 }
536 }
537  
538 __PACKAGE__->meta()->make_immutable();