Eulerian/EDW/Peer/Rest.pm untested

filter: Statementstmt 0.0% 0 / 247 Branchbran 0.0% 0 / 64 Conditioncond 0.0% 0 / 19 MC/DCmcdc 0.0% 0 / 12 Subroutinesub 0.0% 0 / 33 Podpod 0.0% 0 / 21 totaltotal 0.0% 0 / 396 CC 33 SCAR 24.3
CC
33
Cov
0%
download
30.0
setup
20.0
url
20.0
CRAP 11.4
1 #/usr/bin/env perl
2 ###############################################################################
3 #
4 # @file Rest.pm
5 #
6 # @brief Eulerian Data Warehouse REST Peer Module definition.
7 #
8 # This module is aimed to provide access to Eulerian Data Warehouse
9 # Analytics Analysis Through REST Protocol.
10 #
11 # @author Thorillon Xavier:x.thorillon@eulerian.com
12 #
13 # @date 26/11/2021
14 #
15 # @version 1.0
16 #
17 ###############################################################################
18 #
19 # Setup module name
20 #
21 package API::Eulerian::EDW::Peer::Rest;
22  
23 use strict;
24  
25 use JSON();
26 use POSIX();
27 use Sys::Hostname();
28  
29 use API::Eulerian::EDW::Peer();
30 use API::Eulerian::EDW::File();
31 use API::Eulerian::EDW::Bench();
32 use API::Eulerian::EDW::Status();
33 use API::Eulerian::EDW::Authority();
34 use API::Eulerian::EDW::Parser::JSON();
35 use API::Eulerian::EDW::Parser::CSV();
36  
37 our @ISA = qw/ API::Eulerian::EDW::Peer /;
38  
39 #
40 # Define default platforms endpoint
41 #
42  
43 my $DEFAULT_PLATFORM = 'fr';
44 my %PLATFORMS = (
45 'fr' => 'edw.ea.eulerian.com',
46 'ca' => 'edw.ea.eulerian.ca'
47 );
48  
49 #
50 # Defines Parser class name matching format.
51 #
52 my %PARSERS = (
53 'json' => 'API::Eulerian::EDW::Parser::JSON',
54 'csv' => 'API::Eulerian::EDW::Parser::CSV',
55 );
56 #
57 # @brief Allocate and initialize a new Eulerian Data Warehouse Rest Peer.
58 #
59 # @param $class - Eulerian Data Warehouse Rest Peer class.
60 # @param $setup - Setup attributes.
61 #
62 # @return Eulerian Data Warehouse Peer.
63 #
64 sub new
65 {
66 my $proto = shift();
67 my $class = ref($proto) || $proto;
68 my $setup = shift() || {};
69  
70 my $self = $class->SUPER::new();
71  
72 # Setup Rest Peer Default attributes values
73 $self->{ _ACCEPT } = $setup->{accept} || 'application/json';
74 $self->{ _ENCODING } = 'gzip';
75
76 my $wdir = $setup->{wdir};
77 if ( !defined $wdir ) {
78 foreach my $twdir ( '/opt/tmp-fs', '/tmp' ) {
79 if ( -e $twdir && -w $twdir ) {
80 $wdir = $twdir;
81 last;
82 }
83 }
84 }
85 $self->{ _WDIR } = $wdir;
86
87 $self->{ _UUID } = 0;
88  
89 # Setup Rest Peer Attributes
90 $self->setup( $setup );
91  
92 return bless($self, $class);
93 }
94 #
95 # @brief UUID attribute accessors.
96 #
97 # @param $self - Eulerian Data Warehouse Peer.
98 # @param $uuid - Request UUID.
99 #
100 # @return UUID.
101 #
102 sub uuid
103 {
104 my ( $self, $uuid ) = @_;
105 $self->{ _UUID } = $uuid if defined( $uuid );
106 return $self->{ _UUID };
107 }
108 #
109 # @brief Encoding attribute accessors.
110 #
111 # @param $self - Eulerian Data Warehouse Peer.
112 # @param $encoding - Encoding.
113 #
114 # @return Encoding.
115 #
116 sub encoding
117 {
118 my ( $self, $encoding ) = @_;
119 $self->{ _ENCODING } = $encoding if defined( $encoding );
120 return $self->{ _ENCODING };
121 }
122 #
123 # @brief Accept attribute accessors.
124 #
125 # @param $self - Eulerian Data Warehouse Peer.
126 # @param $accept - Accept.
127 #
128 # @return Accept.
129 #
130 sub accept
131 {
132 my ( $self, $accept ) = @_;
133 $self->{ _ACCEPT } = $accept if defined( $accept );
134 return $self->{ _ACCEPT };
135 }
136 #
137 # @brief Working directory attribute accessors.
138 #
139 # @param $self - Eulerian Data Warehouse Peer.
140 # @param $wdir - Working directory.
141 #
142 # @return Working Directory.
143 #
144 sub wdir
145 {
146 my ( $self, $wdir ) = @_;
147 $self->{ _WDIR } = $wdir if defined( $wdir );
148 return $self->{ _WDIR };
149 }
150 #
151 # @brief Setup Eulerian Data Warehouse Peer.
152 #
153 # @param $self - Eulerian Data Warehouse Peer.
154 # @param $setup - Setup entries.
155 #
156 sub setup
157 {
158 my ( $self, $setup ) = @_;
159  
160 # Setup base interface values
161 $self->SUPER::setup( $setup );
162  
163 # Setup Rest specifics options
164 $self->accept( $setup->{ accept } ) if exists( $setup->{ accept } );
165 $self->encoding( $setup->{ encoding } ) if exists( $setup->{ encoding } );
166 $self->wdir( $setup->{ wdir } ) if exists( $setup->{ wdir } );
167  
168 return $self;
169 }
170 #
171 # @brief Dump Eulerian Data Warehouse Peer setup.
172 #
173 # @param $self - Eulerian Data Warehouse Peer.
174 #
175 sub dump
176 {
177 my $self = shift;
178 my $dump = '';
179 $self->SUPER::dump();
180 $dump .= 'Accept : ' . $self->accept() . "\n";
181 $dump .= 'Encoding : ' . $self->encoding() . "\n";
182 $dump .= 'WorkDir : ' . $self->wdir() . "\n\n";
183 print( $dump );
184 return $self;
185 }
186 #
187 # @brief Get remote URL to Eulerian Data Warehouse Platform.
188 #
189 # @param $self - Eulerian Data Warehouse Peer.
190 #
191 # @return Remote URL to Eulerian Data Warehouse Platform.
192 #
193 sub url
194 {
195 my $self = shift;
196 my $platform;
197 my $host;
198 my $url;
199  
200 $url = $self->secure() ? 'https://' : 'http://';
201 $platform = $self->platform();
202 $host = $self->host();
203 if( $host ) {
204 $url .= $host . ':';
205 $url .= $self->ports()->[ $self->secure() ];
206 } else {
207 if ( !exists $PLATFORMS{$platform} ) {
208 $platform = $DEFAULT_PLATFORM;
209 }
210 $url .= $PLATFORMS{$platform};
211 }
212  
213 return $url;
214 }
215 #
216 # @brief Get HTTP Request Body used to send command to Eulerian Data Warehouse
217 # Platform.
218 #
219 # @param $self - Eulerian Data Warehouse Rest Peer.
220 # @param $command - Eulerian Data Warehouse Command.
221 #
222 # @return HTTP Request Body.
223 #
224 sub body
225 {
226 my ( $self, $command ) = @_;
227 $command =~ s/\n//gm;
228 return JSON::encode_json({
229 kind => 'edw#request',
230 query => $command,
231 creationTime => POSIX::strftime( '%d/%m/%Y %H:%M:%S', gmtime() ),
232 location => Sys::Hostname::hostname(),
233 expiration => undef,
234 });
235 };
236 #
237 # @brief Setup HTTP Request Headers.
238 #
239 # @param $self - Eulerian Data Warehouse Peer.
240 #
241 # @return HTTP Headers.
242 #
243 sub headers
244 {
245 my $self = shift;
246 my $status = $self->SUPER::headers();
247 my $headers;
248  
249 if( ! $status->error() ) {
250 $headers = $status->{ headers };
251 $headers->push_header( 'Content-Type', 'application/json' );
252 $headers->push_header( 'Accept', $self->accept() );
253 $headers->push_header( 'Accept-Encoding', $self->encoding() );
254 }
255  
256 return $status;
257 }
258 #
259 # @brief Create a new JOB on Eulerian Data Warehouse Rest Platform.
260 #
261 # @param $self - Eulerian Data Warehouse Peer.
262 # @param $command - Eulerian Data Warehouse Command.
263 #
264 # @return Reply content.
265 #
266 sub create
267 {
268 my ( $self, $command ) = @_;
269 my $response;
270 my $status;
271  
272 # Create headers
273 $status = $self->headers();
274 if( ! $status->error() ) {
275 my $url = $self->url() . '/edw/jobs';
276  
277 # Post Job create request to remote host
278 $status = API::Eulerian::EDW::Request->post(
279 $url, $status->{ headers }, $self->body( $command )
280 );
281 if( ! $status->error() ) {
282 $self->uuid(
283 API::Eulerian::EDW::Request->json(
284 $status->{ response }
285 )->{ data }->[ 0 ]
286 );
287 }
288  
289 }
290  
291 return $status;
292 }
293 #
294 # @brief Get Eulerian Data Warehouse Job Status.
295 #
296 # @param $self - Eulerian Data Warehouse Rest Peer.
297 # @param $reply - Eulerian Data Warehouse Platform Reply.
298 #
299 # @return Job Reply status.
300 #
301 sub status
302 {
303 my ( $self, $status ) = @_;
304 my $response = $status->{ response };
305 my $url = API::Eulerian::EDW::Request->json(
306 $response )->{ data }->[ 1 ];
307  
308 $status = $self->headers();
309 if( ! $status->error() ) {
310 $status = API::Eulerian::EDW::Request->get(
311 $url, $status->{ headers }
312 );
313 }
314  
315 return $status;
316 }
317 #
318 # @brief Test if Job status is 'Running';
319 #
320 # @param $self - API::Eulerian::EDW::Rest instance.
321 # @param $rc - Return context.
322 #
323 # @return 0 - Not running.
324 # @return 1 - Running.
325 #
326 sub running
327 {
328 my ( $self, $status ) = @_;
329 return API::Eulerian::EDW::Request->json(
330 $status->{ response }
331 )->{ status } eq 'Running';
332 }
333 #
334 # @brief Test if Job status is 'Done'.
335 #
336 # @param $self - API::Eulerian::EDW::Rest instance.
337 # @param $rc - Return context.
338 #
339 # @return 0 - Not Done.
340 # @return 1 - Done.
341 #
342 sub done
343 {
344 my ( $self, $status ) = @_;
345 return ! $status->{ error } ?
346 API::Eulerian::EDW::Request->json(
347 $status->{ response }
348 )->{ status } eq 'Done' :
349 0;
350 }
351 #
352 # @brief Get Path to local filepath.
353 #
354 # @param $self - API::Eulerian::EDW::Rest instance.
355 #
356 # @return Local file path.
357 #
358 sub path
359 {
360 my ( $self, $response ) = @_;
361 my $json = API::Eulerian::EDW::Request->json( $response );
362 my $pattern = '([0-9]*)\.(json|csv|parquet)';
363 my $status = API::Eulerian::EDW::Status->new();
364 my $url = $json->{ data }->[ 1 ];
365 my $wdir = $self->wdir();
366 my %rc = ();
367  
368 if( ! $wdir ) {
369 $status->error( 1 );
370 $status->code( 400 );
371 $status->msg( "Working directory isn't set" );
372 } elsif( ! API::Eulerian::EDW::File->writable( $wdir ) ) {
373 $status->error( 1 );
374 $status->code( 400 );
375 $status->msg( "Working directory isn't writable" );
376 } elsif( ! ( $url =~ m/$pattern/ ) ) {
377 $status->error( 1 );
378 $status->code( 400 );
379 $status->msg( "Unknown local file name" );
380 } else {
381 my $path = $wdir. '/' . "$1.$2";
382 $status->{ url } = $url;
383 $status->{ path } = $path;
384 }
385  
386 return $status;
387 }
388 #
389 # @brief Unzip given file.
390 #
391 # @param $self - API::Eulerian::EDW::Rest instance.
392 # @param $zipped - Path to zipped file.
393 #
394 # @return Path to unzipped file.
395 #
396 sub unzip
397 {
398 my( $self, $zipped ) = @_;
399 my $unzipped;
400  
401 # Parse zipped file
402 $zipped =~ /(.*)\.gz/;
403  
404 # Unzipped file name
405 $unzipped = $1;
406  
407 # Gunzip zipped file into unzipped file
408 IO::Uncompress::Gunzip::gunzip(
409 $zipped, $unzipped, BinModeOut => 1
410 );
411  
412 # Remove zipped file
413 unlink $zipped;
414  
415 # Return path to unzipped file
416 return $unzipped;
417 }
418 #
419 # @brief Download Job reply file.
420 #
421 # @param $self - Eulerian Data Warehouse Rest Peer.
422 # @param $rc - Reply context.
423 #
424 # @return Reply context
425 #
426 sub download
427 {
428 my ( $self, $status ) = @_;
429  
430 # From Last status message compute local file path
431 $status = $self->path( $status->{ response } );
432  
433 # If no error
434 if( ! $status->{ error } ) {
435 my $path = $status->{ path };
436 my $url = $status->{ url };
437
438 # Get HTTP request headers
439 $status = $self->headers();
440  
441 if( ! $status->{ error } ) {
442 # Send Download request to remote host, reply is
443 # writen into $path file
444 $status = API::Eulerian::EDW::Request->get(
445 $url, $status->{ headers }, $path
446 );
447 # Handle errors
448 if( ! $status->error() ) {
449 my $response = $status->{ response };
450 my $headers = $response->{ _headers };
451 my $encoding = $headers->{ 'content-encoding' };
452
453 if( defined( $encoding ) && ( $encoding eq 'gzip' ) ) {
454 rename $path, "$path.gz";
455 $status->{ path } = $self->unzip( "$path.gz" );
456 } else {
457 $status->{ path } = $path;
458 }
459 } else {
460 print "ERROR : " . $status->{ _CODE } . "\n";
461 }
462 }
463  
464 }
465  
466 return $status;
467 }
468 #
469 # @brief Parse local file path and invoke hook handlers.
470 #
471 # @param $self - API::Eulerian::EDW::Rest instance.
472 # @param $rc - Reply context.
473 #
474 # @return Reply context.
475 #
476 sub parse
477 {
478 my ( $self, $status ) = @_;
479 my $path = $status->{ path };
480 my $parser;
481 my $name;
482 my %rc;
483  
484 my $pattern = '[0-9]*\.(json|csv|parquet)';
485  
486 # Parse file path, get file type
487 if( ( $path =~ m/$pattern/ ) ) {
488 # Lookup for parser matching file type
489 if( ( $name = $PARSERS{ $1 } ) ) {
490 # Create new instance of Parser
491 if( ( $parser = $name->new( $path, $self->uuid() ) ) ) {
492 # Parse reply file raise callback hook
493 $parser->do( $self->hook() );
494 } else {
495 $status->error( 1 );
496 $status->msg( "Failed to create Parser" );
497 $status->code( 401 );
498 }
499 } else {
500 $status->error( 1 );
501 $status->msg( "Unknown Parser" );
502 $status->code( 501 );
503 }
504 } else {
505 $status->error( 1 );
506 $status->msg( "Unknown file format" );
507 $status->code( 401 );
508 }
509  
510 return $status;
511 }
512 #
513 # @brief Do Request on Eulerian Data Warehouse Platform.
514 #
515 # @param $self - Eulerian Data Warehouse Peer.
516 # @param $command - Eulerian Data Warehouse Command.
517 #
518 use Data::Dumper;
519 sub request
520 {
521 my ( $self, $command ) = @_;
522 my $bench = new API::Eulerian::EDW::Bench();
523 my $response;
524 my $status;
525 my $json;
526  
527 # Create Job on Eulerian Data Warehouse Platform
528 $bench->start();
529 $status = $self->create( $command );
530 $bench->stage( 'create' );
531  
532 # Wait end of Job
533 $bench->start();
534 while( ! $status->error() && $self->running( $status ) ) {
535 $status = $self->status( $status );
536 sleep( 2 );
537 }
538 $bench->stage( 'running' );
539  
540 # If Done, download reply file
541 if( ! $status->error() && $self->done( $status ) ) {
542 $bench->start();
543 $status = $self->download( $status );
544 $bench->stage( 'download' );
545 if( ! $status->error() ) {
546 # Parse reply file, call hooks
547 $bench->start();
548 $status = $self->parse( $status );
549 $bench->stage( 'parse' );
550 }
551 $status->{ bench } = $bench;
552 } else {
553 print "ERROR : " . Dumper( $status ) . "\n";
554 }
555  
556 return $status;
557 }
558 #
559 # @brief Cancel Job on Eulerian Data Warehouse Platform.
560 #
561 # @param $self - API::Eulerian::EDW::Rest instance.
562 # @param $rc - Reply context.
563 #
564 sub cancel
565 {
566 my ( $self ) = @_;
567 my $status;
568  
569 # Get HTTP request headers
570 $status = $self->headers();
571 if( ! $status->error() ) {
572 my $headers = $status->{ headers };
573  
574 delete $status->{ headers };
575 # Create cancel job url
576 if( ! $self->uuid() ) {
577 $status->error( 1 );
578 $status->msg( "Failed to cancel Job. Unknown UUID" );
579 $status->code( 404 );
580 } else {
581 my $url = $self->url() . '/edw/jobs/';
582  
583 $url .= $self->uuid() . '/cancel';
584 # Send Cancel request to remote host
585 $status = API::Eulerian::EDW::Request->get( $url, $headers );
586  
587 }
588 }
589  
590 return $status;
591 }
592 #
593 # @brief Drop Eulerian Data Warehouse Object Events.
594 #
595 # @param $self - Peer.
596 # @param $what - Object Path.
597 # @param $site - Site.
598 # @param $from - Day begin.
599 # @param $end - Day end.
600 #
601 sub drop
602 {
603 my ( $self, $what, $site, $from, $to ) = @_;
604 my $status;
605  
606 # Get HTTP request headers
607 $status = $self->headers();
608 if( ! $status->error() ) {
609 my $headers = $status->{ headers };
610 my $url = $self->url() . '/edw/store';
611 my $cmd = "DROP $what\@$site FROM $from TO $to;";
612 delete $status->{ headers };
613 # Send Cancel request to remote host
614 $status = API::Eulerian::EDW::Request->post(
615 $url, $headers, $self->body( $cmd )
616 );
617 }
618  
619 return $status;
620 }
621 #
622 # End Up module properly
623 #
624 1;