| 1 | package Amazon::API::Role::Botocore; | ||
| 2 | |||
| 3 | use strict; | ||
| 4 | use warnings; | ||
| 5 | |||
| 6 | use Amazon::API::Constants qw($PADDING $EMPTY $SLASH $TRUE $QUESTION_MARK); | ||
| 7 | use Amazon::API::Utils qw(slurp slurp_json choose); | ||
| 8 | use Amazon::API::BuildInfo; | ||
| 9 | use Amazon::API::Paginator::Compiler; | ||
| 10 | use Amazon::API::EndpointContext::Compiler; | ||
| 11 | use Amazon::API::Template qw(fetch_template to_template_var render_template); | ||
| 12 | |||
| 13 | use Carp; | ||
| 14 | use Cwd qw(getcwd abs_path); | ||
| 15 | use Data::Dumper; | ||
| 16 | use Digest::SHA qw(sha256_hex); | ||
| 17 | use English qw(-no_match_vars); | ||
| 18 | use File::Basename qw(basename dirname); | ||
| 19 | use File::ShareDir qw(dist_file); | ||
| 20 | use File::Path qw(make_path); | ||
| 21 | use JSON; | ||
| 22 | use File::Find; | ||
| 23 | use Storable qw(nstore dclone retrieve); | ||
| 24 | |||
| 25 | use Role::Tiny; | ||
| 26 | |||
| 27 | use Readonly; | ||
| 28 | Readonly::Scalar our $REQUEST_CLASS_TEMPLATE => 'Amazon::API::Botocore::Shape::%s::%sRequest'; | ||
| 29 | Readonly::Scalar our $BOTO_PATH_OFFSET => 3; | ||
| 30 | |||
| 31 | ######################################################################## | ||
| 32 | sub get_service_descriptions { | ||
| 33 | ######################################################################## | ||
| 34 | my ( $self, $boto_services, @services ) = @_; | ||
| 35 | |||
| 36 | my @descriptions; | ||
| 37 | |||
| 38 | if ( !@services ) { | ||
| 39 | @services = sort keys %{$boto_services}; | ||
| 40 | } | ||
| 41 | |||
| 42 | my $botocore_path = $self->_check_botocore_path( $ENV{BOTOCORE_PATH} // $self->get_botocore_path ); | ||
| 43 | |||
| 44 | foreach my $s ( map {lc} @services ) { | ||
| 45 | |||
| 46 | croak "ERROR: no such service: $s\n" | ||
| 47 | if !$boto_services->{$s}; | ||
| 48 | |||
| 49 | my $service_path = sprintf '%s/botocore/data/%s/%s', $botocore_path, $s, $boto_services->{$s}->{date}; | ||
| 50 | |||
| 51 | my $service_file = "$service_path/service-2.json"; | ||
| 52 | my $paginators_file = "$service_path/paginators-1.json"; | ||
| 53 | my $endpoint_rule_set_file = "$service_path/endpoint-rule-set-1.json"; | ||
| 54 | |||
| 55 | my $paginators = eval { return $self->fetch_paginators($paginators_file); }; | ||
| 56 | my $endpoint_rule_set = eval { return $self->fetch_endpoint_rule_set($endpoint_rule_set_file); }; | ||
| 57 | |||
| 58 | my $service_description = $self->fetch_service_description($service_file); | ||
| 59 | |||
| 60 | if ( $service_description->{operations} ) { | ||
| 61 | my $operations = $service_description->{operations}; | ||
| 62 | my $shapes = $service_description->{shapes}; | ||
| 63 | my $metadata = $service_description->{metadata}; | ||
| 64 | my $client_context_params = $service_description->{clientContextParams} // {}; | ||
| 65 | my $service_name = $metadata->{signingName} || $metadata->{endpointPrefix}; | ||
| 66 | |||
| 67 | my $botocore_data = { | ||
| 68 | $s => { | ||
| 69 | actions => [ keys %{$operations} ], | ||
| 70 | client_context_params => $client_context_params, | ||
| 71 | documentation => $service_description->{documentation}, | ||
| 72 | endpoint_prefix => $metadata->{endpointPrefix}, | ||
| 73 | json_version => $metadata->{jsonVersion}, | ||
| 74 | metadata => $metadata, | ||
| 75 | metadata_keys => [ keys %{$metadata} ], | ||
| 76 | operations => $operations, | ||
| 77 | paginators => $paginators, | ||
| 78 | endpoint_rule_set => $endpoint_rule_set, | ||
| 79 | protocol => $metadata->{protocol}, | ||
| 80 | service_name => $service_name, | ||
| 81 | shapes => $shapes, | ||
| 82 | target_prefix => $metadata->{targetPrefix}, | ||
| 83 | version => $boto_services->{$s}->{date}, | ||
| 84 | service_2_digest => sha256_hex( slurp($service_file) ), | ||
| 85 | paginators_1_digest => -e $paginators_file ? sha256_hex( slurp($paginators_file) ) : 'no paginators', | ||
| 86 | endpoint_rule_set_1_digest => -e $endpoint_rule_set_file | ||
| 87 | ? sha256_hex( slurp($endpoint_rule_set_file) ) | ||
| 88 | : 'no endpoint rule set', | ||
| 89 | } | ||
| 90 | }; | ||
| 91 | |||
| 92 | push @descriptions, $botocore_data; | ||
| 93 | } | ||
| 94 | } | ||
| 95 | |||
| 96 | return \@descriptions; | ||
| 97 | } | ||
| 98 | |||
| 99 | ######################################################################## | ||
| 100 | sub fetch_json_file { | ||
| 101 | ######################################################################## | ||
| 102 | my ( $self, $path ) = @_; | ||
| 103 | |||
| 104 | my $json; | ||
| 105 | |||
| 106 | open my $fh, '<', $path | ||
| 107 | or croak 'ERROR: could not open ' . $path; | ||
| 108 | |||
| 109 | { | ||
| 110 | local $RS = undef; | ||
| 111 | $json = JSON->new->utf8->decode(<$fh>); | ||
| 112 | } | ||
| 113 | |||
| 114 | close $fh | ||
| 115 | or croak 'ERROR: could not close ' . $path; | ||
| 116 | |||
| 117 | return $json; | ||
| 118 | } | ||
| 119 | |||
| 120 | ######################################################################## | ||
| 121 | sub fetch_service_description { goto &fetch_json_file; } | ||
| 122 | sub fetch_paginators { goto &fetch_json_file; } | ||
| 123 | sub fetch_endpoint_rule_set { goto &fetch_json_file; } | ||
| 124 | ######################################################################## | ||
| 125 | |||
| 126 | # collect paths to most recent service-2.json files for all AWS services | ||
| 127 | ######################################################################## | ||
| 128 | sub find_latest_services { | ||
| 129 | ######################################################################## | ||
| 130 | my ( $self, $boto_services ) = @_; | ||
| 131 | |||
| 132 | my $file = $_; | ||
| 133 | |||
| 134 | my $dir = $File::Find::name; | ||
| 135 | |||
| 136 | return | ||
| 137 | if $dir !~ qr/botocore\N{SOLIDUS}botocore/xsm; | ||
| 138 | |||
| 139 | return | ||
| 140 | if $file !~ qr/service\N{HYPHEN-MINUS}2\N{FULL STOP}json/xsm; | ||
| 141 | |||
| 142 | my (@path) = split /\N{SOLIDUS}/xsm, $dir; | ||
| 143 | |||
| 144 | my $boto_path = -1; | ||
| 145 | |||
| 146 | for my $i ( 0 .. $#path ) { | ||
| 147 | if ( $path[$i] =~ /botocore/xsm ) { | ||
| 148 | $boto_path = $i; | ||
| 149 | last; | ||
| 150 | } | ||
| 151 | } | ||
| 152 | |||
| 153 | # this should not happen... | ||
| 154 | croak 'no botocore in path ' . $dir | ||
| 155 | if $boto_path < 0; | ||
| 156 | |||
| 157 | $boto_path += $BOTO_PATH_OFFSET; | ||
| 158 | |||
| 159 | my ( $service, $date ) = @path[ $boto_path, $boto_path + 1, ]; | ||
| 160 | |||
| 161 | $boto_services->{$service}->{date} = $boto_services->{$service}->{date} // $EMPTY; | ||
| 162 | |||
| 163 | if ( $date gt $boto_services->{$service}->{date} ) { | ||
| 164 | $boto_services->{$service} = { | ||
| 165 | date => $date, | ||
| 166 | path => \@path, | ||
| 167 | file => join q{/}, | ||
| 168 | @path, | ||
| 169 | }; | ||
| 170 | } | ||
| 171 | |||
| 172 | return $file; | ||
| 173 | } | ||
| 174 | |||
| 175 | ######################################################################## | ||
| 176 | sub create_stub { | ||
| 177 | ######################################################################## | ||
| 178 | my ( $self, %args ) = @_; | ||
| 179 | |||
| 180 | my ( $output_path, $module_name, $service ) = @args{qw(output_path module_name service)}; | ||
| 181 | |||
| 182 | croak "ERROR: usage: create_stub(output_path => path, module_name => module-name, service => service\n" | ||
| 183 | if !$output_path || !$module_name || !$service; | ||
| 184 | |||
| 185 | my $boto_services = $self->fetch_boto_services; | ||
| 186 | |||
| 187 | my $path = sprintf '%s/Amazon/API', $output_path; | ||
| 188 | |||
| 189 | croak "ERROR: could not create $path" | ||
| 190 | if !-d $path && !make_path($path); | ||
| 191 | |||
| 192 | my $package_name = sprintf 'Amazon::API::%s', $module_name; | ||
| 193 | |||
| 194 | my $description = $self->get_service_descriptions( $boto_services, $service ); | ||
| 195 | |||
| 196 | my $parameters = $description->[0]->{$service}; | ||
| 197 | |||
| 198 | $parameters->{package_name} = $package_name; | ||
| 199 | |||
| 200 | # bake the parsed request URI into each operation so the runtime | ||
| 201 | # (Amazon::API rest-json/rest-xml dispatch) can build request URIs | ||
| 202 | # from the stored metadata without re-parsing. Must run before the | ||
| 203 | # metadata is stored below. | ||
| 204 | foreach my $op ( values %{ $parameters->{operations} } ) { | ||
| 205 | my $http = $op->{http}; | ||
| 206 | |||
| 207 | if ( $http && $http->{requestUri} ) { | ||
| 208 | my ( $request_uri_tpl, $args ) = $self->_parse_request_uri( $http->{requestUri} ); | ||
| 209 | |||
| 210 | $http->{parsed_request_uri} = { | ||
| 211 | request_uri_tpl => $request_uri_tpl, | ||
| 212 | parameters => $args, | ||
| 213 | }; | ||
| 214 | } | ||
| 215 | } | ||
| 216 | |||
| 217 | # documentation is developer-facing only (rendered on demand by | ||
| 218 | # Amazon::API::Help) and is never read by the runtime. Strip it from | ||
| 219 | # the shipped metadata. | ||
| 220 | my $operations = $self->_strip_documentation( dclone $parameters->{operations} ); | ||
| 221 | my $shapes = $self->_strip_documentation( dclone $parameters->{shapes} ); | ||
| 222 | |||
| 223 | my $endpoint_context_compiler = Amazon::API::EndpointContext::Compiler->new; | ||
| 224 | |||
| 225 | my $compiled_endpoint_context = $endpoint_context_compiler->compile($operations); | ||
| 226 | |||
| 227 | my $compiler = Amazon::API::Paginator::Compiler->new( | ||
| 228 | operations => $operations, | ||
| 229 | shapes => $shapes, | ||
| 230 | ); | ||
| 231 | |||
| 232 | my $compiled_paginators = $compiler->compile( $parameters->{paginators} ); | ||
| 233 | |||
| 234 | local $Storable::canonical = $TRUE; | ||
| 235 | |||
| 236 | my $meta; | ||
| 237 | |||
| 238 | nstore( | ||
| 239 | { compiled_endpoint_context => $compiled_endpoint_context, | ||
| 240 | compiled_paginators => $compiled_paginators, | ||
| 241 | metadata => $parameters->{metadata}, | ||
| 242 | client_context_params => $parameters->{client_context_params}, | ||
| 243 | operations => $operations, | ||
| 244 | shapes => $shapes, | ||
| 245 | paginators => $parameters->{paginators}, | ||
| 246 | endpoint_rule_set => $parameters->{endpoint_rule_set}, | ||
| 247 | service_2_digest => $parameters->{service_2_digest}, | ||
| 248 | paginators_1_digest => $parameters->{paginators_1_digest}, | ||
| 249 | endpoint_rule_set_1_digest => $parameters->{endpoint_rule_set_1_digest}, | ||
| 250 | }, | ||
| 251 | \$meta | ||
| 252 | ); | ||
| 253 | |||
| 254 | require IO::Compress::Gzip; | ||
| 255 | IO::Compress::Gzip::gzip( \$meta, sprintf '%s/Amazon/API/%s.api.gz', $output_path, $module_name ); | ||
| 256 | |||
| 257 | my @actions = @{ $parameters->{actions} }; | ||
| 258 | |||
| 259 | $parameters->{actions} = $PADDING . join "\n ", sort @actions; | ||
| 260 | |||
| 261 | $parameters->{service} | ||
| 262 | = $parameters->{service_name} || $parameters->{endpoint_prefix}; | ||
| 263 | |||
| 264 | if ( $parameters->{protocol} eq 'query' ) { | ||
| 265 | $parameters->{content_type} = 'application/x-www-form-urlencoded'; | ||
| 266 | } | ||
| 267 | |||
| 268 | if ( $parameters->{protocol} eq 'json' ) { | ||
| 269 | $parameters->{content_type} = 'application/x-amz-json-' . $parameters->{json_version}; | ||
| 270 | } | ||
| 271 | |||
| 272 | # for rest-json protocol we need a method and and a query uri in | ||
| 273 | # addition to the payload | ||
| 274 | if ( $parameters->{protocol} eq 'rest-json' ) { | ||
| 275 | $parameters->{content_type} = 'application/json'; | ||
| 276 | } | ||
| 277 | |||
| 278 | my $version = $parameters->{version}; | ||
| 279 | $parameters->{service_version} = $version; | ||
| 280 | |||
| 281 | ###################################################################### | ||
| 282 | # Botocore version | ||
| 283 | ###################################################################### | ||
| 284 | my $api_version = Amazon::API::BuildInfo->version(); | ||
| 285 | $parameters->{botocore_version} = $api_version->{botocore_version}->{version}; | ||
| 286 | $parameters->{botocore_commit} = $api_version->{botocore_version}->{commit}; | ||
| 287 | |||
| 288 | ###################################################################### | ||
| 289 | # resolve stub template | ||
| 290 | ###################################################################### | ||
| 291 | my @template_vars = qw( | ||
| 292 | content_type | ||
| 293 | endpoint_prefix | ||
| 294 | package_name | ||
| 295 | paginators_1_digest | ||
| 296 | protocol | ||
| 297 | service | ||
| 298 | service_2_digest | ||
| 299 | target_prefix | ||
| 300 | botocore_version | ||
| 301 | botocore_commit | ||
| 302 | service_version | ||
| 303 | ); | ||
| 304 | |||
| 305 | foreach my $var (@template_vars) { | ||
| 306 | $parameters->{ to_template_var($var) } = $parameters->{$var}; | ||
| 307 | } | ||
| 308 | |||
| 309 | my ($stub) = grep { defined $_ && -e $_ } ( '../share/stub.pm.tmpl', eval { dist_file( 'Amazon-API', 'stub.pm.tmpl' ) } ); | ||
| 310 | |||
| 311 | croak sprintf "ERROR: could not find stub.pm.tmpl (%s)\n", getcwd | ||
| 312 | if !$stub; | ||
| 313 | |||
| 314 | my $module = $self->render_stub( | ||
| 315 | template => fetch_template($stub), | ||
| 316 | parameters => $parameters, | ||
| 317 | ); | ||
| 318 | |||
| 319 | my $file = sprintf '%s/%s.pm', $path, $module_name; | ||
| 320 | |||
| 321 | if ( -e $file ) { | ||
| 322 | rename $file, "$file.bak"; | ||
| 323 | } | ||
| 324 | |||
| 325 | open my $fh, '>', $file | ||
| 326 | or croak "ERROR: could not open $file for writing\n$OS_ERROR"; | ||
| 327 | |||
| 328 | print {$fh} $module; | ||
| 329 | |||
| 330 | close $fh | ||
| 331 | or croak 'ERROR: could not close file'; | ||
| 332 | |||
| 333 | return; | ||
| 334 | } | ||
| 335 | |||
| 336 | ######################################################################## | ||
| 337 | sub fetch_boto_services { | ||
| 338 | ######################################################################## | ||
| 339 | my ( $self, $want_metadata ) = @_; | ||
| 340 | |||
| 341 | my $botocore_path = $ENV{BOTOCORE_PATH} // $self->get_botocore_path; | ||
| 342 | |||
| 343 | croak "ERROR: No Botocore path found\n" | ||
| 344 | if !$botocore_path || !-d "$botocore_path/botocore"; | ||
| 345 | |||
| 346 | my %boto_services; | ||
| 347 | |||
| 348 | find( { wanted => sub { $self->find_latest_services( \%boto_services ); }, follow => $TRUE }, "$botocore_path/botocore" ); | ||
| 349 | |||
| 350 | croak 'ERROR: no services found in path ' . $botocore_path | ||
| 351 | if !keys %boto_services; | ||
| 352 | |||
| 353 | return \%boto_services | ||
| 354 | if !$want_metadata; | ||
| 355 | |||
| 356 | foreach my $service ( keys %boto_services ) { | ||
| 357 | my $service_2_file = $boto_services{$service}->{file}; | ||
| 358 | my $service_2 = slurp($service_2_file); | ||
| 359 | |||
| 360 | $boto_services{$service}->{metadata} = decode_json($service_2); | ||
| 361 | |||
| 362 | # calculate digests for service, paginators | ||
| 363 | $boto_services{$service}->{service_2_digest} = sha256_hex($service_2); | ||
| 364 | |||
| 365 | my $paginators_file = dirname($service_2_file) . '/paginators-1.json'; | ||
| 366 | if ( -e $paginators_file ) { | ||
| 367 | $boto_services{$service}->{paginators_1_digest} = sha256_hex( slurp($paginators_file) ); | ||
| 368 | } | ||
| 369 | else { | ||
| 370 | $boto_services{$service}->{paginators_1_digest} = 'no paginators'; | ||
| 371 | } | ||
| 372 | |||
| 373 | my $endpoint_rule_set_file = dirname($service_2_file) . '/endpoint-rule-set-1.json'; | ||
| 374 | if ( -e $endpoint_rule_set_file ) { | ||
| 375 | $boto_services{$service}->{endpoint_rule_set_1_digest} = sha256_hex( slurp($endpoint_rule_set_file) ); | ||
| 376 | } | ||
| 377 | else { | ||
| 378 | $boto_services{$service}->{endpoint_rule_set_1_digest} = 'no endpoint rule set'; | ||
| 379 | } | ||
| 380 | } | ||
| 381 | |||
| 382 | return \%boto_services; | ||
| 383 | } | ||
| 384 | |||
| 385 | ######################################################################## | ||
| 386 | sub _check_botocore_path { | ||
| 387 | ######################################################################## | ||
| 388 | my ( $self, $path ) = @_; | ||
| 389 | |||
| 390 | croak "ERROR: no botocore path\n" | ||
| 391 | if !$path; | ||
| 392 | |||
| 393 | my $abs_path = abs_path($path); | ||
| 394 | |||
| 395 | my $candidate = choose { | ||
| 396 | |||
| 397 | return $abs_path | ||
| 398 | if -d "$abs_path/botocore/data"; | ||
| 399 | |||
| 400 | return "../$abs_path" | ||
| 401 | if -d "../$abs_path/botocore/data"; | ||
| 402 | |||
| 403 | return getcwd | ||
| 404 | if -d sprintf '%s/botocore/data', getcwd; | ||
| 405 | |||
| 406 | return q{}; | ||
| 407 | }; | ||
| 408 | |||
| 409 | # final sanity check | ||
| 410 | croak sprintf "ERROR: %s is not a botocore repository\n", $abs_path | ||
| 411 | if !$candidate || !-d $candidate || !-r sprintf '%s/botocore/data/ec2/2016-11-15/service-2.json', $candidate; | ||
| 412 | |||
| 413 | return $candidate; | ||
| 414 | } | ||
| 415 | |||
| 416 | { | ||
| 417 | my $botocore_metadata; | ||
| 418 | |||
| 419 | ###################################################################### | ||
| 420 | sub _botocore_metadata { | ||
| 421 | ###################################################################### | ||
| 422 | my ($self) = @_; | ||
| 423 | |||
| 424 | return $botocore_metadata | ||
| 425 | if $botocore_metadata; | ||
| 426 | |||
| 427 | my @candidates = ( 'botocore-metadata.api', eval { dist_file( 'Amazon-API', 'botocore-metadata.api' ) } ); | ||
| 428 | |||
| 429 | my ($metadata_file) = grep { defined $_ && -e $_ } @candidates; | ||
| 430 | |||
| 431 | croak "ERROR: could not find 'botocore-metadata.api'\n" | ||
| 432 | if !$metadata_file; | ||
| 433 | |||
| 434 | $botocore_metadata = retrieve($metadata_file); | ||
| 435 | |||
| 436 | return $botocore_metadata; | ||
| 437 | } | ||
| 438 | } | ||
| 439 | |||
| 440 | ######################################################################## | ||
| 441 | sub module_name { | ||
| 442 | ######################################################################## | ||
| 443 | my ( $self, $service ) = @_; | ||
| 444 | |||
| 445 | return $self->_botocore_metadata->{$service}{metadata}{metadata}{perl_module_name}; | ||
| 446 | } | ||
| 447 | |||
| 448 | ######################################################################## | ||
| 449 | sub _strip_documentation { | ||
| 450 | ######################################################################## | ||
| 451 | my ( $self, $data ) = @_; | ||
| 452 | |||
| 453 | my $ref = ref $data; | ||
| 454 | |||
| 455 | if ( $ref eq 'HASH' ) { | ||
| 456 | delete $data->{documentation}; | ||
| 457 | _strip_documentation($_) for values %{$data}; | ||
| 458 | } | ||
| 459 | elsif ( $ref eq 'ARRAY' ) { | ||
| 460 | _strip_documentation($_) for @{$data}; | ||
| 461 | } | ||
| 462 | |||
| 463 | return $data; | ||
| 464 | } | ||
| 465 | ######################################################################## | ||
| 466 | sub render_stub { | ||
| 467 | ######################################################################## | ||
| 468 | my ( $self, %args ) = @_; | ||
| 469 | |||
| 470 | my $template = $args{template}; | ||
| 471 | my $parameters = $args{parameters}; | ||
| 472 | |||
| 473 | $parameters->{ to_template_var('git_sha') } = $Amazon::API::BuildInfo::GIT_SHA; | ||
| 474 | |||
| 475 | return render_template( $template, $parameters ); | ||
| 476 | } | ||
| 477 | |||
| 478 | ######################################################################## | ||
| 479 | sub _parse_request_uri { | ||
| 480 | ######################################################################## | ||
| 481 | my ( $self, $uri ) = @_; | ||
| 482 | |||
| 483 | my ( $path, $query_string ) = split /\N{QUESTION MARK}/xsm, $uri; | ||
| 484 | |||
| 485 | my @path_parts = split /\N{SOLIDUS}/xsm, $path; | ||
| 486 | |||
| 487 | my @path_args; | ||
| 488 | |||
| 489 | foreach my $idx ( 1 .. $#path_parts + 1 ) { | ||
| 490 | if ( $path_parts[$idx] && $path_parts[$idx] =~ /^[{](.*)[}]$/xsm ) { | ||
| 491 | push @path_args, ucfirst $1; | ||
| 492 | $path_parts[$idx] = '%s'; | ||
| 493 | } | ||
| 494 | } | ||
| 495 | |||
| 496 | my $request_uri = join $SLASH, @path_parts; | ||
| 497 | |||
| 498 | if ($query_string) { | ||
| 499 | $request_uri .= $QUESTION_MARK . $query_string; | ||
| 500 | } | ||
| 501 | |||
| 502 | return ( $request_uri || $uri, \@path_args ); | ||
| 503 | } | ||
| 504 | |||
| 505 | 1; | ||
| 506 | |||
| 507 | __END__ |