| line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
|
1
|
|
|
|
|
|
|
package Amazon::S3::Lite; |
|
2
|
|
|
|
|
|
|
|
|
3
|
2
|
|
|
2
|
|
334023
|
use strict; |
|
|
2
|
|
|
|
|
3
|
|
|
|
2
|
|
|
|
|
55
|
|
|
4
|
2
|
|
|
2
|
|
6
|
use warnings; |
|
|
2
|
|
|
|
|
2
|
|
|
|
2
|
|
|
|
|
99
|
|
|
5
|
|
|
|
|
|
|
|
|
6
|
|
|
|
|
|
|
our $VERSION = '1.0.1'; |
|
7
|
|
|
|
|
|
|
|
|
8
|
2
|
|
|
2
|
|
848
|
use AWS::Signature4; |
|
|
2
|
|
|
|
|
47372
|
|
|
|
2
|
|
|
|
|
62
|
|
|
9
|
2
|
|
|
2
|
|
736
|
use Amazon::S3::Lite::Credentials; |
|
|
2
|
|
|
|
|
7
|
|
|
|
2
|
|
|
|
|
47
|
|
|
10
|
2
|
|
|
2
|
|
608
|
use Amazon::S3::Lite::Logger; |
|
|
2
|
|
|
|
|
4
|
|
|
|
2
|
|
|
|
|
53
|
|
|
11
|
2
|
|
|
2
|
|
7
|
use Carp qw(croak carp); |
|
|
2
|
|
|
|
|
3
|
|
|
|
2
|
|
|
|
|
84
|
|
|
12
|
2
|
|
|
2
|
|
459
|
use Data::Dumper; |
|
|
2
|
|
|
|
|
5866
|
|
|
|
2
|
|
|
|
|
85
|
|
|
13
|
2
|
|
|
2
|
|
7
|
use Digest::MD5 qw(md5_base64 md5); |
|
|
2
|
|
|
|
|
4
|
|
|
|
2
|
|
|
|
|
72
|
|
|
14
|
2
|
|
|
2
|
|
9
|
use Digest::SHA qw(sha256_hex); |
|
|
2
|
|
|
|
|
16
|
|
|
|
2
|
|
|
|
|
55
|
|
|
15
|
2
|
|
|
2
|
|
821
|
use HTTP::Request; |
|
|
2
|
|
|
|
|
23185
|
|
|
|
2
|
|
|
|
|
62
|
|
|
16
|
2
|
|
|
2
|
|
1160
|
use HTTP::Tiny; |
|
|
2
|
|
|
|
|
60737
|
|
|
|
2
|
|
|
|
|
82
|
|
|
17
|
2
|
|
|
2
|
|
739
|
use MIME::Base64 qw(encode_base64); |
|
|
2
|
|
|
|
|
785
|
|
|
|
2
|
|
|
|
|
113
|
|
|
18
|
2
|
|
|
2
|
|
9
|
use Scalar::Util qw(blessed openhandle reftype); |
|
|
2
|
|
|
|
|
4
|
|
|
|
2
|
|
|
|
|
79
|
|
|
19
|
2
|
|
|
2
|
|
7
|
use URI::Escape qw(uri_escape_utf8); |
|
|
2
|
|
|
|
|
4
|
|
|
|
2
|
|
|
|
|
92
|
|
|
20
|
2
|
|
|
2
|
|
1111
|
use XML::LibXML; |
|
|
2
|
|
|
|
|
59109
|
|
|
|
2
|
|
|
|
|
8
|
|
|
21
|
|
|
|
|
|
|
|
|
22
|
|
|
|
|
|
|
######################################################################## |
|
23
|
|
|
|
|
|
|
sub new { |
|
24
|
|
|
|
|
|
|
######################################################################## |
|
25
|
16
|
|
|
16
|
1
|
176481
|
my ( $class, $args ) = @_; |
|
26
|
|
|
|
|
|
|
|
|
27
|
16
|
|
50
|
|
|
41
|
$args //= {}; |
|
28
|
|
|
|
|
|
|
|
|
29
|
16
|
50
|
|
|
|
38
|
croak 'new() requires a hashref' |
|
30
|
|
|
|
|
|
|
if ref $args ne 'HASH'; |
|
31
|
|
|
|
|
|
|
|
|
32
|
|
|
|
|
|
|
croak 'region is required' |
|
33
|
16
|
100
|
|
|
|
174
|
if !$args->{region}; |
|
34
|
|
|
|
|
|
|
|
|
35
|
15
|
|
|
|
|
24
|
my $self = bless {}, $class; |
|
36
|
|
|
|
|
|
|
|
|
37
|
15
|
|
|
|
|
52
|
$self->{region} = $args->{region}; |
|
38
|
15
|
|
50
|
|
|
61
|
$self->{host} = $args->{host} // 's3.amazonaws.com'; |
|
39
|
15
|
|
50
|
|
|
40
|
$self->{secure} = $args->{secure} // 1; |
|
40
|
15
|
|
50
|
|
|
33
|
$self->{timeout} = $args->{timeout} // 30; |
|
41
|
|
|
|
|
|
|
|
|
42
|
15
|
|
|
|
|
47
|
$self->_init_logger( $args->{logger} ); |
|
43
|
15
|
|
|
|
|
43
|
$self->_init_credentials($args); |
|
44
|
13
|
|
|
|
|
27
|
$self->_init_ua; |
|
45
|
|
|
|
|
|
|
|
|
46
|
13
|
|
|
|
|
33
|
return $self; |
|
47
|
|
|
|
|
|
|
} |
|
48
|
|
|
|
|
|
|
|
|
49
|
|
|
|
|
|
|
######################################################################## |
|
50
|
|
|
|
|
|
|
# Logger setup |
|
51
|
|
|
|
|
|
|
# Priority: caller-supplied object -> Log::Log4perl (if available) -> |
|
52
|
|
|
|
|
|
|
# minimal STDERR logger |
|
53
|
|
|
|
|
|
|
######################################################################## |
|
54
|
|
|
|
|
|
|
sub _init_logger { |
|
55
|
|
|
|
|
|
|
######################################################################## |
|
56
|
15
|
|
|
15
|
|
39
|
my ( $self, $logger ) = @_; |
|
57
|
|
|
|
|
|
|
|
|
58
|
15
|
100
|
|
|
|
32
|
if ($logger) { |
|
59
|
|
|
|
|
|
|
# Validate it quacks like a logger |
|
60
|
1
|
|
|
|
|
2
|
for my $method (qw(trace debug info warn error)) { |
|
61
|
5
|
50
|
|
|
|
13
|
croak "logger object must implement '$method'" |
|
62
|
|
|
|
|
|
|
if !$logger->can($method); |
|
63
|
|
|
|
|
|
|
} |
|
64
|
1
|
|
|
|
|
2
|
$self->{logger} = $logger; |
|
65
|
1
|
|
|
|
|
2
|
return; |
|
66
|
|
|
|
|
|
|
} |
|
67
|
|
|
|
|
|
|
|
|
68
|
14
|
50
|
|
|
|
16
|
if ( eval { require Log::Log4perl; 1 } ) { |
|
|
14
|
|
|
|
|
1285
|
|
|
|
0
|
|
|
|
|
0
|
|
|
69
|
0
|
0
|
|
|
|
0
|
if ( !Log::Log4perl->initialized ) { |
|
70
|
0
|
|
|
|
|
0
|
Log::Log4perl->easy_init($Log::Log4perl::WARN); |
|
71
|
|
|
|
|
|
|
} |
|
72
|
0
|
|
|
|
|
0
|
$self->{logger} = Log::Log4perl->get_logger(__PACKAGE__); |
|
73
|
0
|
|
|
|
|
0
|
return; |
|
74
|
|
|
|
|
|
|
} |
|
75
|
|
|
|
|
|
|
|
|
76
|
|
|
|
|
|
|
# Fall back to minimal STDERR logger |
|
77
|
14
|
|
|
|
|
62
|
$self->{logger} = Amazon::S3::Lite::Logger->new; |
|
78
|
|
|
|
|
|
|
|
|
79
|
14
|
|
|
|
|
25
|
return; |
|
80
|
|
|
|
|
|
|
} |
|
81
|
|
|
|
|
|
|
|
|
82
|
|
|
|
|
|
|
######################################################################## |
|
83
|
|
|
|
|
|
|
# Credential resolution |
|
84
|
|
|
|
|
|
|
# Priority: explicit credentials object -> constructor args -> |
|
85
|
|
|
|
|
|
|
# environment variables -> Amazon::Credentials (if available) |
|
86
|
|
|
|
|
|
|
######################################################################## |
|
87
|
|
|
|
|
|
|
sub _init_credentials { |
|
88
|
|
|
|
|
|
|
######################################################################## |
|
89
|
14
|
|
|
14
|
|
17
|
my ( $self, $args ) = @_; |
|
90
|
|
|
|
|
|
|
|
|
91
|
|
|
|
|
|
|
# 1. Caller-supplied credentials object (duck-typed) |
|
92
|
14
|
100
|
|
|
|
61
|
if ( my $creds = $args->{credentials} ) { |
|
93
|
2
|
50
|
|
|
|
5
|
croak "credential object is not blessed.\n" |
|
94
|
|
|
|
|
|
|
if !blessed $creds; |
|
95
|
|
|
|
|
|
|
|
|
96
|
2
|
|
|
|
|
5
|
foreach (qw(aws_access_key_id aws_secret_access_key token)) { |
|
97
|
5
|
|
66
|
|
|
24
|
my $sub = $creds->can($_) // $creds->can("get_$_"); |
|
98
|
|
|
|
|
|
|
|
|
99
|
5
|
100
|
|
|
|
131
|
croak "credentials object must implement $_ or get_$_\n" |
|
100
|
|
|
|
|
|
|
if !$sub; |
|
101
|
|
|
|
|
|
|
} |
|
102
|
|
|
|
|
|
|
|
|
103
|
1
|
|
|
|
|
2
|
$self->{credentials} = $creds; |
|
104
|
|
|
|
|
|
|
|
|
105
|
1
|
|
|
|
|
2
|
return; |
|
106
|
|
|
|
|
|
|
} |
|
107
|
|
|
|
|
|
|
|
|
108
|
|
|
|
|
|
|
# 2. Explicit constructor args |
|
109
|
12
|
50
|
66
|
|
|
40
|
if ( $args->{aws_access_key_id} && $args->{aws_secret_access_key} ) { |
|
110
|
|
|
|
|
|
|
$self->{credentials} = Amazon::S3::Lite::Credentials->new( |
|
111
|
|
|
|
|
|
|
aws_access_key_id => $args->{aws_access_key_id}, |
|
112
|
|
|
|
|
|
|
aws_secret_access_key => $args->{aws_secret_access_key}, |
|
113
|
|
|
|
|
|
|
token => $args->{token}, |
|
114
|
11
|
|
|
|
|
54
|
); |
|
115
|
11
|
|
|
|
|
21
|
return; |
|
116
|
|
|
|
|
|
|
} |
|
117
|
|
|
|
|
|
|
|
|
118
|
|
|
|
|
|
|
# 3. Environment variables |
|
119
|
1
|
50
|
33
|
|
|
6
|
if ( $ENV{AWS_ACCESS_KEY_ID} && $ENV{AWS_SECRET_ACCESS_KEY} ) { |
|
120
|
|
|
|
|
|
|
$self->{credentials} = Amazon::S3::Lite::Credentials->new( |
|
121
|
|
|
|
|
|
|
aws_access_key_id => $ENV{AWS_ACCESS_KEY_ID}, |
|
122
|
|
|
|
|
|
|
aws_secret_access_key => $ENV{AWS_SECRET_ACCESS_KEY}, |
|
123
|
|
|
|
|
|
|
token => $ENV{AWS_SESSION_TOKEN}, |
|
124
|
1
|
|
|
|
|
5
|
); |
|
125
|
1
|
|
|
|
|
2
|
return; |
|
126
|
|
|
|
|
|
|
} |
|
127
|
|
|
|
|
|
|
|
|
128
|
|
|
|
|
|
|
# 4. Amazon::Credentials (covers IAM roles, ECS task roles, |
|
129
|
|
|
|
|
|
|
# ~/.aws/credentials, etc.) |
|
130
|
0
|
0
|
|
|
|
0
|
if ( eval { require Amazon::Credentials; 1 } ) { |
|
|
0
|
|
|
|
|
0
|
|
|
|
0
|
|
|
|
|
0
|
|
|
131
|
0
|
|
|
|
|
0
|
$self->{credentials} = Amazon::Credentials->new; |
|
132
|
0
|
|
|
|
|
0
|
return; |
|
133
|
|
|
|
|
|
|
} |
|
134
|
|
|
|
|
|
|
|
|
135
|
0
|
|
|
|
|
0
|
croak 'No AWS credentials found. Supply aws_access_key_id/' |
|
136
|
|
|
|
|
|
|
. 'aws_secret_access_key, set AWS_ACCESS_KEY_ID/' |
|
137
|
|
|
|
|
|
|
. 'AWS_SECRET_ACCESS_KEY environment variables, ' |
|
138
|
|
|
|
|
|
|
. 'or install Amazon::Credentials for IAM role support.'; |
|
139
|
|
|
|
|
|
|
} |
|
140
|
|
|
|
|
|
|
|
|
141
|
|
|
|
|
|
|
######################################################################## |
|
142
|
|
|
|
|
|
|
# HTTP::Tiny instance - one per object, keep-alive enabled |
|
143
|
|
|
|
|
|
|
######################################################################## |
|
144
|
|
|
|
|
|
|
sub _init_ua { |
|
145
|
|
|
|
|
|
|
######################################################################## |
|
146
|
13
|
|
|
13
|
|
18
|
my ($self) = @_; |
|
147
|
|
|
|
|
|
|
|
|
148
|
|
|
|
|
|
|
$self->{ua} = HTTP::Tiny->new( |
|
149
|
|
|
|
|
|
|
timeout => $self->{timeout}, |
|
150
|
|
|
|
|
|
|
verify_SSL => $self->{secure}, |
|
151
|
13
|
|
|
|
|
53
|
); |
|
152
|
|
|
|
|
|
|
|
|
153
|
13
|
|
|
|
|
964
|
return; |
|
154
|
|
|
|
|
|
|
} |
|
155
|
|
|
|
|
|
|
|
|
156
|
|
|
|
|
|
|
######################################################################## |
|
157
|
|
|
|
|
|
|
# Accessors |
|
158
|
|
|
|
|
|
|
######################################################################## |
|
159
|
1
|
|
|
1
|
1
|
9
|
sub logger { return $_[0]->{logger} } |
|
160
|
0
|
|
|
0
|
0
|
0
|
sub ua { return $_[0]->{ua} } |
|
161
|
2
|
|
|
2
|
1
|
2517
|
sub region { return $_[0]->{region} } |
|
162
|
27
|
|
|
27
|
1
|
52
|
sub host { return $_[0]->{host} } |
|
163
|
3
|
|
|
3
|
1
|
16
|
sub credentials { return $_[0]->{credentials} } |
|
164
|
|
|
|
|
|
|
|
|
165
|
|
|
|
|
|
|
######################################################################## |
|
166
|
|
|
|
|
|
|
# Build a fresh signer from current credentials. |
|
167
|
|
|
|
|
|
|
# Called per-request so that rotating credentials (Lambda IAM roles) |
|
168
|
|
|
|
|
|
|
# are always current. |
|
169
|
|
|
|
|
|
|
######################################################################## |
|
170
|
|
|
|
|
|
|
sub _signer { |
|
171
|
|
|
|
|
|
|
######################################################################## |
|
172
|
0
|
|
|
0
|
|
0
|
my ($self) = @_; |
|
173
|
|
|
|
|
|
|
|
|
174
|
0
|
|
|
|
|
0
|
my $creds = $self->credentials; |
|
175
|
|
|
|
|
|
|
|
|
176
|
0
|
|
0
|
|
|
0
|
return AWS::Signature4->new( |
|
|
|
|
0
|
|
|
|
|
|
177
|
|
|
|
|
|
|
-access_key => $creds->get_aws_access_key_id // $creds->aws_access_key_id, |
|
178
|
|
|
|
|
|
|
-secret_key => $creds->get_aws_secret_access_key // $creds->get_aws_secret_access_key, |
|
179
|
|
|
|
|
|
|
); |
|
180
|
|
|
|
|
|
|
} |
|
181
|
|
|
|
|
|
|
|
|
182
|
|
|
|
|
|
|
######################################################################## |
|
183
|
|
|
|
|
|
|
# Build the endpoint URL for a bucket/key |
|
184
|
|
|
|
|
|
|
######################################################################## |
|
185
|
|
|
|
|
|
|
sub _endpoint { |
|
186
|
|
|
|
|
|
|
######################################################################## |
|
187
|
26
|
|
|
26
|
|
51
|
my ( $self, $bucket, $key ) = @_; |
|
188
|
|
|
|
|
|
|
|
|
189
|
26
|
50
|
|
|
|
72
|
my $scheme = $self->{secure} ? 'https' : 'http'; |
|
190
|
26
|
|
|
|
|
54
|
my $host = $self->host; |
|
191
|
|
|
|
|
|
|
|
|
192
|
|
|
|
|
|
|
# Path-style URL: https://s3.amazonaws.com/bucket/key |
|
193
|
|
|
|
|
|
|
# (virtual-hosted style omitted for simplicity; path-style works |
|
194
|
|
|
|
|
|
|
# everywhere and avoids SSL cert issues with dotted bucket names) |
|
195
|
26
|
|
|
|
|
45
|
my $url = "$scheme://$host"; |
|
196
|
|
|
|
|
|
|
|
|
197
|
26
|
100
|
66
|
|
|
88
|
$url .= "/$bucket" if defined $bucket && length $bucket; |
|
198
|
26
|
100
|
66
|
|
|
78
|
$url .= '/' . _encode_key($key) if defined $key && length $key; |
|
199
|
|
|
|
|
|
|
|
|
200
|
26
|
|
|
|
|
675
|
return $url; |
|
201
|
|
|
|
|
|
|
} |
|
202
|
|
|
|
|
|
|
|
|
203
|
|
|
|
|
|
|
######################################################################## |
|
204
|
|
|
|
|
|
|
# URI-encode an S3 key, preserving '/' separators |
|
205
|
|
|
|
|
|
|
######################################################################## |
|
206
|
|
|
|
|
|
|
sub _encode_key { |
|
207
|
|
|
|
|
|
|
######################################################################## |
|
208
|
23
|
|
|
23
|
|
35
|
my ($key) = @_; |
|
209
|
|
|
|
|
|
|
|
|
210
|
23
|
|
|
|
|
65
|
return join '/', map { uri_escape_utf8( $_, '^A-Za-z0-9\-._~' ) } |
|
|
30
|
|
|
|
|
247
|
|
|
211
|
|
|
|
|
|
|
split m{/}, $key, -1; |
|
212
|
|
|
|
|
|
|
} |
|
213
|
|
|
|
|
|
|
######################################################################## |
|
214
|
|
|
|
|
|
|
# Core request method: |
|
215
|
|
|
|
|
|
|
# 1. Build HTTP::Request for signing |
|
216
|
|
|
|
|
|
|
# 2. Sign it (fresh signer = current credentials) |
|
217
|
|
|
|
|
|
|
# 3. Extract auth headers |
|
218
|
|
|
|
|
|
|
# 4. Execute via HTTP::Tiny |
|
219
|
|
|
|
|
|
|
# |
|
220
|
|
|
|
|
|
|
# Returns raw HTTP::Tiny response hashref. |
|
221
|
|
|
|
|
|
|
######################################################################## |
|
222
|
|
|
|
|
|
|
sub _request { |
|
223
|
|
|
|
|
|
|
######################################################################## |
|
224
|
0
|
|
|
0
|
|
0
|
my ( $self, $method, $url, $headers, $content, $extra, $region ) = @_; |
|
225
|
|
|
|
|
|
|
|
|
226
|
0
|
|
0
|
|
|
0
|
$region //= $self->region; |
|
227
|
|
|
|
|
|
|
|
|
228
|
0
|
|
0
|
|
|
0
|
$headers //= {}; |
|
229
|
0
|
|
0
|
|
|
0
|
$content //= q{}; |
|
230
|
0
|
|
|
|
|
0
|
my $content_is_coderef = ref $content eq 'CODE'; |
|
231
|
|
|
|
|
|
|
|
|
232
|
0
|
|
0
|
|
|
0
|
$extra //= {}; |
|
233
|
|
|
|
|
|
|
|
|
234
|
|
|
|
|
|
|
# Build an HTTP::Request just for signing |
|
235
|
0
|
|
|
|
|
0
|
my $req = HTTP::Request->new( $method, $url ); |
|
236
|
0
|
0
|
0
|
|
|
0
|
$req->content($content) if length $content && !$content_is_coderef; |
|
237
|
|
|
|
|
|
|
|
|
238
|
0
|
|
|
|
|
0
|
for my $name ( keys %{$headers} ) { |
|
|
0
|
|
|
|
|
0
|
|
|
239
|
0
|
|
|
|
|
0
|
$req->header( $name => $headers->{$name} ); |
|
240
|
|
|
|
|
|
|
} |
|
241
|
|
|
|
|
|
|
|
|
242
|
0
|
|
0
|
|
|
0
|
my $token_sub = $self->credentials->can('get_token') // $self->credentials->can('token'); |
|
243
|
0
|
0
|
|
|
|
0
|
my $token = $token_sub ? $token_sub->( $self->credentials ) : undef; |
|
244
|
|
|
|
|
|
|
|
|
245
|
0
|
0
|
0
|
|
|
0
|
if ( defined $token && length $token ) { |
|
246
|
0
|
|
|
|
|
0
|
$req->header( 'x-amz-security-token' => $token ); |
|
247
|
|
|
|
|
|
|
} |
|
248
|
|
|
|
|
|
|
|
|
249
|
|
|
|
|
|
|
# add before $signer->sign(...) |
|
250
|
0
|
0
|
|
|
|
0
|
if ( !$content_is_coderef ) { |
|
251
|
0
|
0
|
0
|
|
|
0
|
my $hash = sha256_hex( ref $content eq 'SCALAR' ? ${$content} : $content // q{} ); |
|
|
0
|
|
|
|
|
0
|
|
|
252
|
0
|
|
|
|
|
0
|
$req->header( 'x-amz-content-sha256' => $hash ); |
|
253
|
|
|
|
|
|
|
} |
|
254
|
|
|
|
|
|
|
|
|
255
|
|
|
|
|
|
|
# Sign — mutates $req with Authorization and x-amz-date |
|
256
|
0
|
|
|
|
|
0
|
my $signer = $self->_signer; |
|
257
|
|
|
|
|
|
|
|
|
258
|
0
|
0
|
|
|
|
0
|
my $payload_hash = $content_is_coderef ? 'UNSIGNED-PAYLOAD' : undef; |
|
259
|
0
|
|
|
|
|
0
|
$signer->sign( $req, $region, $payload_hash ); |
|
260
|
|
|
|
|
|
|
|
|
261
|
|
|
|
|
|
|
# Pull signed headers back out for HTTP::Tiny |
|
262
|
0
|
|
|
|
|
0
|
my %signed_headers; |
|
263
|
0
|
|
|
|
|
0
|
for my $name ( $req->headers->header_field_names ) { |
|
264
|
0
|
|
|
|
|
0
|
$signed_headers{$name} = $req->header($name); |
|
265
|
|
|
|
|
|
|
} |
|
266
|
|
|
|
|
|
|
|
|
267
|
|
|
|
|
|
|
# HTTP::Tiny sets Host itself from the URL and rejects it as an explicit |
|
268
|
|
|
|
|
|
|
# header option. The host value is already covered by the Authorization |
|
269
|
|
|
|
|
|
|
# SignedHeaders list, so removing it here is safe. |
|
270
|
0
|
|
|
|
|
0
|
delete $signed_headers{Host}; |
|
271
|
|
|
|
|
|
|
|
|
272
|
0
|
|
|
|
|
0
|
$self->logger->debug("$method $url"); |
|
273
|
|
|
|
|
|
|
|
|
274
|
|
|
|
|
|
|
# Execute via HTTP::Tiny |
|
275
|
0
|
|
|
|
|
0
|
my $options = { headers => \%signed_headers }; |
|
276
|
|
|
|
|
|
|
|
|
277
|
|
|
|
|
|
|
# For HTTP::Tiny — code ref is valid content |
|
278
|
0
|
0
|
0
|
|
|
0
|
$options->{content} = $content if length $content || $content_is_coderef; |
|
279
|
|
|
|
|
|
|
|
|
280
|
|
|
|
|
|
|
# Pass through data_callback for streaming downloads |
|
281
|
0
|
0
|
|
|
|
0
|
if ( $extra->{data_callback} ) { |
|
282
|
0
|
|
|
|
|
0
|
$options->{data_callback} = $extra->{data_callback}; |
|
283
|
|
|
|
|
|
|
} |
|
284
|
|
|
|
|
|
|
|
|
285
|
0
|
|
|
|
|
0
|
my $response = $self->ua->request( $method, $url, $options ); |
|
286
|
|
|
|
|
|
|
|
|
287
|
0
|
|
|
|
|
0
|
$self->logger->debug( sprintf 'Response: %s %s', $response->{status}, $response->{reason} ); |
|
288
|
|
|
|
|
|
|
|
|
289
|
0
|
|
|
|
|
0
|
return $response; |
|
290
|
|
|
|
|
|
|
} |
|
291
|
|
|
|
|
|
|
|
|
292
|
|
|
|
|
|
|
######################################################################## |
|
293
|
|
|
|
|
|
|
# head_object( $bucket, $key ) |
|
294
|
|
|
|
|
|
|
# |
|
295
|
|
|
|
|
|
|
# Fetches metadata for an object without retrieving the body. |
|
296
|
|
|
|
|
|
|
# Returns undef if the key does not exist (404). |
|
297
|
|
|
|
|
|
|
# Returns a hashref with content_type, content_length, etag, |
|
298
|
|
|
|
|
|
|
# last_modified, and metadata (x-amz-meta-* headers). |
|
299
|
|
|
|
|
|
|
######################################################################## |
|
300
|
|
|
|
|
|
|
sub head_object { |
|
301
|
|
|
|
|
|
|
######################################################################## |
|
302
|
4
|
|
|
4
|
1
|
4239
|
my ( $self, $bucket, $key ) = @_; |
|
303
|
|
|
|
|
|
|
|
|
304
|
4
|
100
|
66
|
|
|
185
|
croak 'bucket is required' if !defined $bucket || !length $bucket; |
|
305
|
3
|
100
|
66
|
|
|
182
|
croak 'key is required' if !defined $key || !length $key; |
|
306
|
|
|
|
|
|
|
|
|
307
|
2
|
|
|
|
|
7
|
my $url = $self->_endpoint( $bucket, $key ); |
|
308
|
2
|
|
|
|
|
7
|
my $response = $self->_request( 'HEAD', $url ); |
|
309
|
|
|
|
|
|
|
|
|
310
|
|
|
|
|
|
|
return undef ## no critic (Subroutines::ProhibitExplicitReturnUndef) |
|
311
|
2
|
100
|
|
|
|
19
|
if _is_not_found($response); |
|
312
|
|
|
|
|
|
|
|
|
313
|
1
|
|
|
|
|
6
|
$self->_croak_on_error( $response, 'head_object' ); |
|
314
|
|
|
|
|
|
|
|
|
315
|
1
|
|
|
|
|
5
|
return $self->_extract_object_metadata( $response->{headers} ); |
|
316
|
|
|
|
|
|
|
} |
|
317
|
|
|
|
|
|
|
|
|
318
|
|
|
|
|
|
|
######################################################################## |
|
319
|
|
|
|
|
|
|
# Extract the standard object metadata hashref from a response headers |
|
320
|
|
|
|
|
|
|
# hash. Used by both head_object and get_object. |
|
321
|
|
|
|
|
|
|
######################################################################## |
|
322
|
|
|
|
|
|
|
sub _extract_object_metadata { |
|
323
|
|
|
|
|
|
|
######################################################################## |
|
324
|
4
|
|
|
4
|
|
8
|
my ( $self, $headers ) = @_; |
|
325
|
|
|
|
|
|
|
|
|
326
|
4
|
|
|
|
|
7
|
my $etag = $headers->{etag}; |
|
327
|
4
|
50
|
|
|
|
33
|
$etag =~ s/\A"|"\z//gxsm if defined $etag; |
|
328
|
|
|
|
|
|
|
|
|
329
|
|
|
|
|
|
|
# Collect x-amz-meta-* headers, stripping the prefix from the key |
|
330
|
4
|
|
|
|
|
5
|
my %metadata; |
|
331
|
4
|
|
|
|
|
5
|
for my $name ( keys %{$headers} ) { |
|
|
4
|
|
|
|
|
12
|
|
|
332
|
15
|
100
|
|
|
|
27
|
if ( $name =~ /^x-amz-meta-(.+)$/xsm ) { |
|
333
|
1
|
|
|
|
|
4
|
$metadata{$1} = $headers->{$name}; |
|
334
|
|
|
|
|
|
|
} |
|
335
|
|
|
|
|
|
|
} |
|
336
|
|
|
|
|
|
|
|
|
337
|
|
|
|
|
|
|
return { |
|
338
|
|
|
|
|
|
|
content_type => $headers->{'content-type'}, |
|
339
|
|
|
|
|
|
|
content_length => $headers->{'content-length'} + 0, |
|
340
|
|
|
|
|
|
|
etag => $etag, |
|
341
|
4
|
|
|
|
|
40
|
last_modified => $headers->{'last-modified'}, |
|
342
|
|
|
|
|
|
|
metadata => \%metadata, |
|
343
|
|
|
|
|
|
|
}; |
|
344
|
|
|
|
|
|
|
} |
|
345
|
|
|
|
|
|
|
|
|
346
|
|
|
|
|
|
|
######################################################################## |
|
347
|
|
|
|
|
|
|
# get_object( $bucket, $key, %options ) |
|
348
|
|
|
|
|
|
|
# |
|
349
|
|
|
|
|
|
|
# Fetches an object from S3. Options: |
|
350
|
|
|
|
|
|
|
# range => 'bytes=0-1023' partial fetch |
|
351
|
|
|
|
|
|
|
# filename => '/tmp/foo' stream body to disk; omits content key |
|
352
|
|
|
|
|
|
|
# |
|
353
|
|
|
|
|
|
|
# Returns undef on 404. |
|
354
|
|
|
|
|
|
|
# Returns a hashref with content_type, content_length, etag, |
|
355
|
|
|
|
|
|
|
# last_modified, metadata, and content (unless filename is used). |
|
356
|
|
|
|
|
|
|
######################################################################## |
|
357
|
|
|
|
|
|
|
sub get_object { |
|
358
|
|
|
|
|
|
|
######################################################################## |
|
359
|
4
|
|
|
4
|
1
|
3354
|
my ( $self, $bucket, $key, %options ) = @_; |
|
360
|
|
|
|
|
|
|
|
|
361
|
4
|
50
|
33
|
|
|
16
|
croak 'bucket is required' if !defined $bucket || !length $bucket; |
|
362
|
4
|
50
|
33
|
|
|
13
|
croak 'key is required' if !defined $key || !length $key; |
|
363
|
|
|
|
|
|
|
|
|
364
|
4
|
|
|
|
|
8
|
my $url = $self->_endpoint( $bucket, $key ); |
|
365
|
|
|
|
|
|
|
|
|
366
|
4
|
|
|
|
|
7
|
my %headers; |
|
367
|
4
|
100
|
|
|
|
9
|
$headers{Range} = $options{range} if defined $options{range}; |
|
368
|
|
|
|
|
|
|
|
|
369
|
4
|
|
|
|
|
5
|
my $filename = $options{filename}; |
|
370
|
4
|
|
|
|
|
7
|
my $extra = {}; |
|
371
|
|
|
|
|
|
|
|
|
372
|
4
|
100
|
|
|
|
27
|
if ( defined $filename ) { |
|
373
|
|
|
|
|
|
|
# Open the destination file before making the request so we catch |
|
374
|
|
|
|
|
|
|
# permission errors early, before network round-trip |
|
375
|
1
|
50
|
|
|
|
62
|
open my $fh, '>', $filename |
|
376
|
|
|
|
|
|
|
or croak "cannot open '$filename' for writing: $!"; |
|
377
|
|
|
|
|
|
|
|
|
378
|
|
|
|
|
|
|
$extra->{data_callback} = sub { |
|
379
|
2
|
|
|
2
|
|
10
|
my ($data) = @_; |
|
380
|
2
|
50
|
|
|
|
3
|
print {$fh} $data |
|
|
2
|
|
|
|
|
18
|
|
|
381
|
|
|
|
|
|
|
or croak "write to '$filename' failed: $!"; |
|
382
|
1
|
|
|
|
|
6
|
}; |
|
383
|
|
|
|
|
|
|
|
|
384
|
1
|
|
|
|
|
5
|
my $response = $self->_request( 'GET', $url, \%headers, q{}, $extra ); |
|
385
|
|
|
|
|
|
|
|
|
386
|
1
|
50
|
|
|
|
69
|
close $fh |
|
387
|
|
|
|
|
|
|
or croak "close of '$filename' failed: $!"; |
|
388
|
|
|
|
|
|
|
|
|
389
|
|
|
|
|
|
|
return undef ## no critic (Subroutines::ProhibitExplicitReturnUndef) |
|
390
|
1
|
50
|
|
|
|
3
|
if _is_not_found($response); |
|
391
|
|
|
|
|
|
|
|
|
392
|
1
|
|
|
|
|
3
|
$self->_croak_on_error( $response, 'get_object' ); |
|
393
|
|
|
|
|
|
|
|
|
394
|
|
|
|
|
|
|
# Return metadata only — content is on disk |
|
395
|
1
|
|
|
|
|
2
|
return $self->_extract_object_metadata( $response->{headers} ); |
|
396
|
|
|
|
|
|
|
} |
|
397
|
|
|
|
|
|
|
|
|
398
|
|
|
|
|
|
|
# In-memory path |
|
399
|
3
|
|
|
|
|
9
|
my $response = $self->_request( 'GET', $url, \%headers ); |
|
400
|
|
|
|
|
|
|
|
|
401
|
|
|
|
|
|
|
return undef ## no critic (Subroutines::ProhibitExplicitReturnUndef) |
|
402
|
3
|
100
|
|
|
|
44
|
if _is_not_found($response); |
|
403
|
|
|
|
|
|
|
|
|
404
|
2
|
|
|
|
|
6
|
$self->_croak_on_error( $response, 'get_object' ); |
|
405
|
|
|
|
|
|
|
|
|
406
|
2
|
|
|
|
|
5
|
my $result = $self->_extract_object_metadata( $response->{headers} ); |
|
407
|
2
|
|
|
|
|
4
|
$result->{content} = $response->{content}; |
|
408
|
|
|
|
|
|
|
|
|
409
|
2
|
|
|
|
|
7
|
return $result; |
|
410
|
|
|
|
|
|
|
} |
|
411
|
|
|
|
|
|
|
|
|
412
|
|
|
|
|
|
|
######################################################################## |
|
413
|
|
|
|
|
|
|
# delete_object( $bucket, $key, %options ) |
|
414
|
|
|
|
|
|
|
# |
|
415
|
|
|
|
|
|
|
# Deletes an object from S3. Options: |
|
416
|
|
|
|
|
|
|
# version_id => $vid delete a specific version |
|
417
|
|
|
|
|
|
|
# |
|
418
|
|
|
|
|
|
|
# Returns true on success. Note S3 returns 204 for both successful |
|
419
|
|
|
|
|
|
|
# deletes and deletes of non-existent keys — no distinction is made. |
|
420
|
|
|
|
|
|
|
# Croaks on network or server errors. |
|
421
|
|
|
|
|
|
|
######################################################################## |
|
422
|
|
|
|
|
|
|
sub delete_object { |
|
423
|
|
|
|
|
|
|
######################################################################## |
|
424
|
6
|
|
|
6
|
1
|
5213
|
my ( $self, $bucket, $key, %options ) = @_; |
|
425
|
|
|
|
|
|
|
|
|
426
|
6
|
100
|
66
|
|
|
154
|
croak 'bucket is required' if !defined $bucket || !length $bucket; |
|
427
|
5
|
100
|
66
|
|
|
184
|
croak 'key is required' if !defined $key || !length $key; |
|
428
|
|
|
|
|
|
|
|
|
429
|
4
|
|
|
|
|
13
|
my $url = $self->_endpoint( $bucket, $key ); |
|
430
|
|
|
|
|
|
|
|
|
431
|
4
|
100
|
|
|
|
15
|
if ( defined $options{version_id} ) { |
|
432
|
2
|
|
|
|
|
8
|
$url .= '?versionId=' . uri_escape_utf8( $options{version_id} ); |
|
433
|
|
|
|
|
|
|
} |
|
434
|
|
|
|
|
|
|
|
|
435
|
4
|
|
|
|
|
121
|
my $response = $self->_request( 'DELETE', $url ); |
|
436
|
|
|
|
|
|
|
|
|
437
|
4
|
|
|
|
|
48
|
$self->_croak_on_error( $response, 'delete_object' ); |
|
438
|
|
|
|
|
|
|
|
|
439
|
3
|
|
|
|
|
10
|
return 1; |
|
440
|
|
|
|
|
|
|
} |
|
441
|
|
|
|
|
|
|
|
|
442
|
|
|
|
|
|
|
######################################################################## |
|
443
|
|
|
|
|
|
|
# list_buckets() |
|
444
|
|
|
|
|
|
|
# |
|
445
|
|
|
|
|
|
|
# Lists all buckets owned by the authenticated user. |
|
446
|
|
|
|
|
|
|
# |
|
447
|
|
|
|
|
|
|
# Note: ListBuckets is a global S3 operation and must always be signed |
|
448
|
|
|
|
|
|
|
# against us-east-1 regardless of the region the object was constructed |
|
449
|
|
|
|
|
|
|
# with. We pass the region override directly to the signer. |
|
450
|
|
|
|
|
|
|
# |
|
451
|
|
|
|
|
|
|
# Returns a hashref: |
|
452
|
|
|
|
|
|
|
# { |
|
453
|
|
|
|
|
|
|
# owner_id => '...', |
|
454
|
|
|
|
|
|
|
# owner_name => '...', |
|
455
|
|
|
|
|
|
|
# buckets => [ |
|
456
|
|
|
|
|
|
|
# { name => '...', creation_date => '...' }, |
|
457
|
|
|
|
|
|
|
# ... |
|
458
|
|
|
|
|
|
|
# ], |
|
459
|
|
|
|
|
|
|
# } |
|
460
|
|
|
|
|
|
|
######################################################################## |
|
461
|
|
|
|
|
|
|
sub list_buckets { |
|
462
|
|
|
|
|
|
|
######################################################################## |
|
463
|
2
|
|
|
2
|
1
|
26
|
my ($self) = @_; |
|
464
|
|
|
|
|
|
|
|
|
465
|
2
|
|
|
|
|
5
|
my $url = $self->_endpoint; # https://s3.amazonaws.com |
|
466
|
|
|
|
|
|
|
|
|
467
|
2
|
|
|
|
|
6
|
my $response = $self->_request( 'GET', $url, {}, q{}, {}, 'us-east-1' ); |
|
468
|
|
|
|
|
|
|
|
|
469
|
2
|
|
|
|
|
25
|
$self->_croak_on_error( $response, 'list_buckets' ); |
|
470
|
|
|
|
|
|
|
|
|
471
|
1
|
|
|
|
|
4
|
return $self->_parse_list_buckets( $response->{content} ); |
|
472
|
|
|
|
|
|
|
} |
|
473
|
|
|
|
|
|
|
|
|
474
|
|
|
|
|
|
|
######################################################################## |
|
475
|
|
|
|
|
|
|
# Parse ListAllMyBucketsResult XML |
|
476
|
|
|
|
|
|
|
######################################################################## |
|
477
|
|
|
|
|
|
|
sub _parse_list_buckets { |
|
478
|
|
|
|
|
|
|
######################################################################## |
|
479
|
1
|
|
|
1
|
|
2
|
my ( $self, $xml ) = @_; |
|
480
|
|
|
|
|
|
|
|
|
481
|
1
|
|
|
|
|
8
|
my $doc = XML::LibXML->load_xml( string => $xml ); |
|
482
|
1
|
|
|
|
|
321
|
my $xpc = _xpc($doc); |
|
483
|
1
|
|
|
|
|
1
|
my $root = '/s3:ListAllMyBucketsResult'; |
|
484
|
|
|
|
|
|
|
|
|
485
|
1
|
|
|
|
|
2
|
my @buckets; |
|
486
|
|
|
|
|
|
|
|
|
487
|
1
|
|
|
|
|
5
|
for my $node ( $xpc->findnodes("$root/s3:Buckets/s3:Bucket") ) { |
|
488
|
2
|
|
|
|
|
274
|
push @buckets, |
|
489
|
|
|
|
|
|
|
{ |
|
490
|
|
|
|
|
|
|
name => $xpc->findvalue( 's3:Name', $node ), |
|
491
|
|
|
|
|
|
|
creation_date => $xpc->findvalue( 's3:CreationDate', $node ), |
|
492
|
|
|
|
|
|
|
}; |
|
493
|
|
|
|
|
|
|
} |
|
494
|
|
|
|
|
|
|
|
|
495
|
|
|
|
|
|
|
return { |
|
496
|
1
|
|
|
|
|
110
|
owner_id => $xpc->findvalue("$root/s3:Owner/s3:ID"), |
|
497
|
|
|
|
|
|
|
owner_name => $xpc->findvalue("$root/s3:Owner/s3:DisplayName"), |
|
498
|
|
|
|
|
|
|
buckets => \@buckets, |
|
499
|
|
|
|
|
|
|
}; |
|
500
|
|
|
|
|
|
|
} |
|
501
|
|
|
|
|
|
|
|
|
502
|
|
|
|
|
|
|
######################################################################## |
|
503
|
|
|
|
|
|
|
# copy_object( %args ) |
|
504
|
|
|
|
|
|
|
# |
|
505
|
|
|
|
|
|
|
# Copies an object within or between buckets, entirely server-side. |
|
506
|
|
|
|
|
|
|
# Required: src_bucket, src_key, dst_bucket, dst_key |
|
507
|
|
|
|
|
|
|
# |
|
508
|
|
|
|
|
|
|
# Note: S3 can return HTTP 200 with an XML error body for copy operations |
|
509
|
|
|
|
|
|
|
# that fail mid-transfer. This method detects and croaks on that case. |
|
510
|
|
|
|
|
|
|
# |
|
511
|
|
|
|
|
|
|
# Returns a hashref: { etag => '...', last_modified => '...' } |
|
512
|
|
|
|
|
|
|
######################################################################## |
|
513
|
|
|
|
|
|
|
sub copy_object { |
|
514
|
|
|
|
|
|
|
######################################################################## |
|
515
|
7
|
|
|
7
|
1
|
8952
|
my ( $self, %args ) = @_; |
|
516
|
|
|
|
|
|
|
|
|
517
|
7
|
|
|
|
|
21
|
for my $required (qw( src_bucket src_key dst_bucket dst_key )) { |
|
518
|
|
|
|
|
|
|
croak "$required is required" |
|
519
|
22
|
100
|
66
|
|
|
684
|
if !defined $args{$required} || !length $args{$required}; |
|
520
|
|
|
|
|
|
|
} |
|
521
|
|
|
|
|
|
|
|
|
522
|
3
|
|
|
|
|
11
|
my $url = $self->_endpoint( $args{dst_bucket}, $args{dst_key} ); |
|
523
|
|
|
|
|
|
|
|
|
524
|
|
|
|
|
|
|
# x-amz-copy-source: /src-bucket/encoded-key |
|
525
|
3
|
|
|
|
|
9
|
my $copy_source = '/' . $args{src_bucket} . '/' . _encode_key( $args{src_key} ); |
|
526
|
|
|
|
|
|
|
|
|
527
|
3
|
|
|
|
|
63
|
my %headers = ( |
|
528
|
|
|
|
|
|
|
'x-amz-copy-source' => $copy_source, |
|
529
|
|
|
|
|
|
|
'x-amz-tagging-directive' => 'COPY', |
|
530
|
|
|
|
|
|
|
'Content-Length' => 0, |
|
531
|
|
|
|
|
|
|
); |
|
532
|
|
|
|
|
|
|
|
|
533
|
3
|
|
|
|
|
11
|
my $response = $self->_request( 'PUT', $url, \%headers ); |
|
534
|
|
|
|
|
|
|
|
|
535
|
3
|
|
|
|
|
34
|
$self->_croak_on_error( $response, 'copy_object' ); |
|
536
|
|
|
|
|
|
|
|
|
537
|
|
|
|
|
|
|
# S3 can return HTTP 200 with an XML error body for copies that fail |
|
538
|
|
|
|
|
|
|
# after the headers have been sent. Detect this by checking the root |
|
539
|
|
|
|
|
|
|
# element — a success response has , an error has . |
|
540
|
3
|
|
|
|
|
9
|
return $self->_parse_copy_response( $response->{content}, 'copy_object' ); |
|
541
|
|
|
|
|
|
|
} |
|
542
|
|
|
|
|
|
|
|
|
543
|
|
|
|
|
|
|
######################################################################## |
|
544
|
|
|
|
|
|
|
# Parse CopyObjectResult XML, detecting the 200-with-error edge case |
|
545
|
|
|
|
|
|
|
######################################################################## |
|
546
|
|
|
|
|
|
|
sub _parse_copy_response { |
|
547
|
|
|
|
|
|
|
######################################################################## |
|
548
|
3
|
|
|
3
|
|
5
|
my ( $self, $xml, $context ) = @_; |
|
549
|
|
|
|
|
|
|
|
|
550
|
3
|
|
|
|
|
16
|
my $doc = XML::LibXML->load_xml( string => $xml ); |
|
551
|
3
|
|
|
|
|
680
|
my $root = $doc->documentElement->localname; |
|
552
|
|
|
|
|
|
|
|
|
553
|
3
|
100
|
|
|
|
11
|
if ( $root eq 'Error' ) { |
|
554
|
1
|
|
|
|
|
51
|
my $code = $doc->findvalue('/Error/Code'); |
|
555
|
1
|
|
|
|
|
67
|
my $msg = $doc->findvalue('/Error/Message'); |
|
556
|
1
|
|
|
|
|
174
|
croak sprintf '%s failed: %s - %s', $context, $code, $msg; |
|
557
|
|
|
|
|
|
|
} |
|
558
|
|
|
|
|
|
|
|
|
559
|
2
|
|
|
|
|
72
|
my $xpc = _xpc($doc); |
|
560
|
2
|
|
|
|
|
4
|
my $result_root = '/s3:CopyObjectResult'; |
|
561
|
|
|
|
|
|
|
|
|
562
|
2
|
|
|
|
|
10
|
my $etag = $xpc->findvalue("$result_root/s3:ETag"); |
|
563
|
2
|
50
|
|
|
|
138
|
$etag =~ s/\A"|"\z//gxsm if defined $etag; |
|
564
|
|
|
|
|
|
|
|
|
565
|
|
|
|
|
|
|
return { |
|
566
|
2
|
|
|
|
|
34
|
etag => $etag, |
|
567
|
|
|
|
|
|
|
last_modified => $xpc->findvalue("$result_root/s3:LastModified"), |
|
568
|
|
|
|
|
|
|
}; |
|
569
|
|
|
|
|
|
|
} |
|
570
|
|
|
|
|
|
|
|
|
571
|
|
|
|
|
|
|
######################################################################## |
|
572
|
|
|
|
|
|
|
# put_object( $bucket, $key, $data, %options ) |
|
573
|
|
|
|
|
|
|
# |
|
574
|
|
|
|
|
|
|
# Stores an object in S3. $data may be a scalar string, a reference to |
|
575
|
|
|
|
|
|
|
# a scalar, or an open filehandle / IO::File object. |
|
576
|
|
|
|
|
|
|
# |
|
577
|
|
|
|
|
|
|
# Options: content_type, content_length, metadata (hashref), acl |
|
578
|
|
|
|
|
|
|
# |
|
579
|
|
|
|
|
|
|
# Returns the ETag of the stored object. Croaks on failure. |
|
580
|
|
|
|
|
|
|
######################################################################## |
|
581
|
|
|
|
|
|
|
sub put_object { |
|
582
|
|
|
|
|
|
|
######################################################################## |
|
583
|
8
|
|
|
8
|
1
|
7738
|
my ( $self, $bucket, $key, $data, %options ) = @_; |
|
584
|
|
|
|
|
|
|
|
|
585
|
8
|
100
|
66
|
|
|
134
|
croak 'bucket is required' if !defined $bucket || !length $bucket; |
|
586
|
7
|
100
|
66
|
|
|
109
|
croak 'key is required' if !defined $key || !length $key; |
|
587
|
6
|
100
|
|
|
|
102
|
croak 'data is required' if !defined $data; |
|
588
|
|
|
|
|
|
|
|
|
589
|
5
|
|
|
|
|
13
|
my $url = $self->_endpoint( $bucket, $key ); |
|
590
|
|
|
|
|
|
|
|
|
591
|
5
|
|
|
|
|
7
|
my %headers; |
|
592
|
5
|
|
100
|
|
|
16
|
$headers{'Content-Type'} = $options{content_type} // 'application/octet-stream'; |
|
593
|
|
|
|
|
|
|
|
|
594
|
|
|
|
|
|
|
# x-amz-acl header |
|
595
|
5
|
100
|
|
|
|
11
|
if ( $options{acl} ) { |
|
596
|
1
|
|
|
|
|
3
|
$headers{'x-amz-acl'} = $options{acl}; |
|
597
|
|
|
|
|
|
|
} |
|
598
|
|
|
|
|
|
|
|
|
599
|
|
|
|
|
|
|
# User metadata — prefix bare keys with x-amz-meta- |
|
600
|
5
|
100
|
|
|
|
10
|
if ( my $meta = $options{metadata} ) { |
|
601
|
1
|
|
|
|
|
2
|
for my $k ( keys %{$meta} ) { |
|
|
1
|
|
|
|
|
2
|
|
|
602
|
1
|
50
|
|
|
|
5
|
my $header = $k =~ /^x-amz-meta-/xsm ? $k : "x-amz-meta-$k"; |
|
603
|
1
|
|
|
|
|
3
|
$headers{$header} = $meta->{$k}; |
|
604
|
|
|
|
|
|
|
} |
|
605
|
|
|
|
|
|
|
} |
|
606
|
|
|
|
|
|
|
|
|
607
|
5
|
|
|
|
|
5
|
my $body; |
|
608
|
|
|
|
|
|
|
|
|
609
|
5
|
100
|
33
|
|
|
26
|
if ( openhandle($data) || ( blessed($data) && $data->can('read') ) ) { |
|
|
|
100
|
66
|
|
|
|
|
|
610
|
|
|
|
|
|
|
# --- Filehandle path --- |
|
611
|
2
|
|
|
|
|
8
|
my $content_length = $options{content_length}; |
|
612
|
|
|
|
|
|
|
|
|
613
|
2
|
50
|
|
|
|
5
|
if ( !defined $content_length ) { |
|
614
|
|
|
|
|
|
|
# Try to stat the handle for real files; suppress warning on |
|
615
|
|
|
|
|
|
|
# in-memory handles (IO::Scalar etc.) that have no underlying fd |
|
616
|
2
|
|
|
|
|
3
|
my @st = eval { stat $data }; |
|
|
2
|
|
|
|
|
55
|
|
|
617
|
2
|
100
|
66
|
|
|
9
|
$content_length = $st[7] if @st && defined $st[7]; |
|
618
|
|
|
|
|
|
|
} |
|
619
|
|
|
|
|
|
|
|
|
620
|
2
|
100
|
|
|
|
183
|
croak 'content_length is required for in-memory filehandles' |
|
621
|
|
|
|
|
|
|
if !defined $content_length; |
|
622
|
|
|
|
|
|
|
|
|
623
|
1
|
|
|
|
|
23
|
$headers{'Content-Length'} = $content_length; |
|
624
|
|
|
|
|
|
|
|
|
625
|
|
|
|
|
|
|
# Wrap filehandle in a code ref for HTTP::Tiny streaming |
|
626
|
1
|
|
|
|
|
2
|
my $chunk_size = 1024 * 64; # 64KB chunks |
|
627
|
|
|
|
|
|
|
$body = sub { |
|
628
|
0
|
|
|
0
|
|
0
|
my $buf; |
|
629
|
0
|
|
|
|
|
0
|
my $n = read( $data, $buf, $chunk_size ); |
|
630
|
0
|
0
|
|
|
|
0
|
return $buf if $n; |
|
631
|
0
|
|
|
|
|
0
|
return q{}; |
|
632
|
1
|
|
|
|
|
4
|
}; |
|
633
|
|
|
|
|
|
|
} |
|
634
|
|
|
|
|
|
|
elsif ( ref $data eq 'SCALAR' ) { |
|
635
|
|
|
|
|
|
|
# --- Scalar ref path --- |
|
636
|
1
|
|
|
|
|
2
|
$body = ${$data}; |
|
|
1
|
|
|
|
|
2
|
|
|
637
|
1
|
|
|
|
|
3
|
$headers{'Content-Length'} = length $body; |
|
638
|
1
|
|
|
|
|
6
|
$headers{'Content-MD5'} = encode_base64( md5($body), q{} ); |
|
639
|
|
|
|
|
|
|
} |
|
640
|
|
|
|
|
|
|
else { |
|
641
|
|
|
|
|
|
|
# --- Plain scalar path --- |
|
642
|
2
|
|
|
|
|
3
|
$body = $data; |
|
643
|
2
|
|
|
|
|
3
|
$headers{'Content-Length'} = length $body; |
|
644
|
2
|
|
|
|
|
14
|
$headers{'Content-MD5'} = encode_base64( md5($body), q{} ); |
|
645
|
|
|
|
|
|
|
} |
|
646
|
|
|
|
|
|
|
|
|
647
|
4
|
|
|
|
|
12
|
my $response = $self->_request( 'PUT', $url, \%headers, $body ); |
|
648
|
|
|
|
|
|
|
|
|
649
|
4
|
|
|
|
|
51
|
$self->_croak_on_error( $response, 'put_object' ); |
|
650
|
|
|
|
|
|
|
|
|
651
|
4
|
|
|
|
|
5
|
my $etag = $response->{headers}{etag}; |
|
652
|
4
|
50
|
|
|
|
22
|
$etag =~ s/\A"|"\z//gxsm if defined $etag; |
|
653
|
|
|
|
|
|
|
|
|
654
|
4
|
|
|
|
|
11
|
return $etag; |
|
655
|
|
|
|
|
|
|
} |
|
656
|
|
|
|
|
|
|
|
|
657
|
|
|
|
|
|
|
######################################################################## |
|
658
|
|
|
|
|
|
|
# S3 XML namespace — must be registered for all XPath queries |
|
659
|
|
|
|
|
|
|
######################################################################## |
|
660
|
2
|
|
|
2
|
|
5562
|
use constant S3_NS => 'http://s3.amazonaws.com/doc/2006-03-01/'; |
|
|
2
|
|
|
|
|
3
|
|
|
|
2
|
|
|
|
|
1703
|
|
|
661
|
|
|
|
|
|
|
|
|
662
|
|
|
|
|
|
|
######################################################################## |
|
663
|
|
|
|
|
|
|
# Build a namespace-aware XPath context for a parsed document |
|
664
|
|
|
|
|
|
|
######################################################################## |
|
665
|
|
|
|
|
|
|
sub _xpc { |
|
666
|
|
|
|
|
|
|
######################################################################## |
|
667
|
4
|
|
|
4
|
|
7
|
my ($doc) = @_; |
|
668
|
|
|
|
|
|
|
|
|
669
|
4
|
|
|
|
|
31
|
my $xpc = XML::LibXML::XPathContext->new($doc); |
|
670
|
4
|
|
|
|
|
22
|
$xpc->registerNs( 's3', S3_NS ); |
|
671
|
|
|
|
|
|
|
|
|
672
|
4
|
|
|
|
|
6
|
return $xpc; |
|
673
|
|
|
|
|
|
|
} |
|
674
|
|
|
|
|
|
|
|
|
675
|
|
|
|
|
|
|
######################################################################## |
|
676
|
|
|
|
|
|
|
# list_objects_v2( $bucket, %options ) |
|
677
|
|
|
|
|
|
|
# |
|
678
|
|
|
|
|
|
|
# Lists objects in a bucket using the S3 ListObjectsV2 API. |
|
679
|
|
|
|
|
|
|
# Returns a hashref with keys: bucket, prefix, key_count, max_keys, |
|
680
|
|
|
|
|
|
|
# is_truncated, next_continuation_token, objects, common_prefixes. |
|
681
|
|
|
|
|
|
|
######################################################################## |
|
682
|
|
|
|
|
|
|
sub list_objects_v2 { |
|
683
|
|
|
|
|
|
|
######################################################################## |
|
684
|
3
|
|
|
3
|
1
|
7574
|
my ( $self, $bucket, %options ) = @_; |
|
685
|
|
|
|
|
|
|
|
|
686
|
3
|
100
|
66
|
|
|
114
|
croak 'bucket is required' |
|
687
|
|
|
|
|
|
|
if !defined $bucket || !length $bucket; |
|
688
|
|
|
|
|
|
|
|
|
689
|
|
|
|
|
|
|
# Map our option names to S3 query parameter names |
|
690
|
2
|
|
|
|
|
11
|
my %param_map = ( |
|
691
|
|
|
|
|
|
|
prefix => 'prefix', |
|
692
|
|
|
|
|
|
|
delimiter => 'delimiter', |
|
693
|
|
|
|
|
|
|
max_keys => 'max-keys', |
|
694
|
|
|
|
|
|
|
continuation_token => 'continuation-token', |
|
695
|
|
|
|
|
|
|
start_after => 'start-after', |
|
696
|
|
|
|
|
|
|
); |
|
697
|
|
|
|
|
|
|
|
|
698
|
2
|
|
|
|
|
2
|
my %params = ( 'list-type' => '2' ); |
|
699
|
|
|
|
|
|
|
|
|
700
|
2
|
|
|
|
|
7
|
for my $opt ( keys %param_map ) { |
|
701
|
10
|
100
|
|
|
|
13
|
if ( defined $options{$opt} ) { |
|
702
|
1
|
|
|
|
|
3
|
$params{ $param_map{$opt} } = $options{$opt}; |
|
703
|
|
|
|
|
|
|
} |
|
704
|
|
|
|
|
|
|
} |
|
705
|
|
|
|
|
|
|
|
|
706
|
|
|
|
|
|
|
# Build query string |
|
707
|
2
|
|
|
|
|
8
|
my $query = join q{&}, map { uri_escape_utf8($_) . q{=} . uri_escape_utf8( $params{$_} ) } |
|
|
3
|
|
|
|
|
63
|
|
|
708
|
|
|
|
|
|
|
sort keys %params; |
|
709
|
|
|
|
|
|
|
|
|
710
|
2
|
|
|
|
|
63
|
my $url = $self->_endpoint($bucket) . q{?} . $query; |
|
711
|
|
|
|
|
|
|
|
|
712
|
2
|
|
|
|
|
8
|
my $response = $self->_request( 'GET', $url ); |
|
713
|
|
|
|
|
|
|
|
|
714
|
|
|
|
|
|
|
return undef ## no critic (Subroutines::ProhibitExplicitReturnUndef) |
|
715
|
2
|
100
|
|
|
|
17
|
if _is_not_found($response); |
|
716
|
|
|
|
|
|
|
|
|
717
|
1
|
|
|
|
|
2
|
$self->_croak_on_error( $response, 'list_objects_v2' ); |
|
718
|
|
|
|
|
|
|
|
|
719
|
1
|
|
|
|
|
3
|
return $self->_parse_list_objects_v2( $response->{content} ); |
|
720
|
|
|
|
|
|
|
} |
|
721
|
|
|
|
|
|
|
|
|
722
|
|
|
|
|
|
|
######################################################################## |
|
723
|
|
|
|
|
|
|
# Parse the XML body of a ListObjectsV2 response |
|
724
|
|
|
|
|
|
|
######################################################################## |
|
725
|
|
|
|
|
|
|
sub _parse_list_objects_v2 { |
|
726
|
|
|
|
|
|
|
######################################################################## |
|
727
|
1
|
|
|
1
|
|
3
|
my ( $self, $xml ) = @_; |
|
728
|
|
|
|
|
|
|
|
|
729
|
1
|
|
|
|
|
4
|
my $doc = XML::LibXML->load_xml( string => $xml ); |
|
730
|
1
|
|
|
|
|
243
|
my $xpc = _xpc($doc); |
|
731
|
1
|
|
|
|
|
1
|
my $root = '/s3:ListBucketResult'; |
|
732
|
|
|
|
|
|
|
|
|
733
|
1
|
50
|
|
|
|
5
|
my $is_truncated = $xpc->findvalue("$root/s3:IsTruncated") eq 'true' ? 1 : 0; |
|
734
|
|
|
|
|
|
|
|
|
735
|
1
|
50
|
|
|
|
101
|
my $next_token |
|
736
|
|
|
|
|
|
|
= $is_truncated |
|
737
|
|
|
|
|
|
|
? $xpc->findvalue("$root/s3:NextContinuationToken") |
|
738
|
|
|
|
|
|
|
: undef; |
|
739
|
|
|
|
|
|
|
|
|
740
|
1
|
|
|
|
|
45
|
my @objects; |
|
741
|
|
|
|
|
|
|
|
|
742
|
1
|
|
|
|
|
3
|
for my $node ( $xpc->findnodes("$root/s3:Contents") ) { |
|
743
|
2
|
|
|
|
|
224
|
my $etag = $xpc->findvalue( 's3:ETag', $node ); |
|
744
|
2
|
50
|
|
|
|
87
|
$etag =~ s/\A"|"\z//gxsm if defined $etag; # strip surrounding quotes |
|
745
|
|
|
|
|
|
|
|
|
746
|
2
|
|
|
|
|
30
|
push @objects, |
|
747
|
|
|
|
|
|
|
{ |
|
748
|
|
|
|
|
|
|
key => $xpc->findvalue( 's3:Key', $node ), |
|
749
|
|
|
|
|
|
|
size => $xpc->findvalue( 's3:Size', $node ) + 0, |
|
750
|
|
|
|
|
|
|
last_modified => $xpc->findvalue( 's3:LastModified', $node ), |
|
751
|
|
|
|
|
|
|
etag => $etag, |
|
752
|
|
|
|
|
|
|
storage_class => $xpc->findvalue( 's3:StorageClass', $node ), |
|
753
|
|
|
|
|
|
|
}; |
|
754
|
|
|
|
|
|
|
} |
|
755
|
|
|
|
|
|
|
|
|
756
|
1
|
|
|
|
|
147
|
my @common_prefixes; |
|
757
|
|
|
|
|
|
|
|
|
758
|
1
|
|
|
|
|
12
|
for my $node ( $xpc->findnodes("$root/s3:CommonPrefixes") ) { |
|
759
|
0
|
|
|
|
|
0
|
push @common_prefixes, $xpc->findvalue( 's3:Prefix', $node ); |
|
760
|
|
|
|
|
|
|
} |
|
761
|
|
|
|
|
|
|
|
|
762
|
|
|
|
|
|
|
return { |
|
763
|
1
|
|
|
|
|
23
|
bucket => $xpc->findvalue("$root/s3:Name"), |
|
764
|
|
|
|
|
|
|
prefix => $xpc->findvalue("$root/s3:Prefix"), |
|
765
|
|
|
|
|
|
|
key_count => $xpc->findvalue("$root/s3:KeyCount") + 0, |
|
766
|
|
|
|
|
|
|
max_keys => $xpc->findvalue("$root/s3:MaxKeys") + 0, |
|
767
|
|
|
|
|
|
|
is_truncated => $is_truncated, |
|
768
|
|
|
|
|
|
|
next_continuation_token => $next_token, |
|
769
|
|
|
|
|
|
|
objects => \@objects, |
|
770
|
|
|
|
|
|
|
common_prefixes => \@common_prefixes, |
|
771
|
|
|
|
|
|
|
}; |
|
772
|
|
|
|
|
|
|
} |
|
773
|
|
|
|
|
|
|
|
|
774
|
|
|
|
|
|
|
######################################################################## |
|
775
|
|
|
|
|
|
|
# list_all_objects_v2( $bucket, %options ) |
|
776
|
|
|
|
|
|
|
# |
|
777
|
|
|
|
|
|
|
# Convenience wrapper that auto-paginates list_objects_v2 and returns |
|
778
|
|
|
|
|
|
|
# a flat list of all matching object hashrefs. |
|
779
|
|
|
|
|
|
|
# delimiter is ignored — use list_objects_v2 directly for that. |
|
780
|
|
|
|
|
|
|
######################################################################## |
|
781
|
|
|
|
|
|
|
sub list_all_objects_v2 { |
|
782
|
|
|
|
|
|
|
######################################################################## |
|
783
|
2
|
|
|
2
|
1
|
3161
|
my ( $self, $bucket, %options ) = @_; |
|
784
|
|
|
|
|
|
|
|
|
785
|
|
|
|
|
|
|
# delimiter is meaningless here — silently remove it |
|
786
|
2
|
|
|
|
|
6
|
delete $options{delimiter}; |
|
787
|
|
|
|
|
|
|
|
|
788
|
2
|
|
|
|
|
4
|
my @all_objects; |
|
789
|
|
|
|
|
|
|
my $continuation_token; |
|
790
|
|
|
|
|
|
|
|
|
791
|
2
|
|
|
|
|
5
|
while (1) { |
|
792
|
3
|
100
|
|
|
|
12
|
$options{continuation_token} = $continuation_token |
|
793
|
|
|
|
|
|
|
if defined $continuation_token; |
|
794
|
|
|
|
|
|
|
|
|
795
|
3
|
|
|
|
|
13
|
my $result = $self->list_objects_v2( $bucket, %options ); |
|
796
|
|
|
|
|
|
|
|
|
797
|
3
|
50
|
|
|
|
1709
|
last if !$result; # 404 / empty bucket |
|
798
|
|
|
|
|
|
|
|
|
799
|
3
|
|
|
|
|
7
|
push @all_objects, @{ $result->{objects} }; |
|
|
3
|
|
|
|
|
8
|
|
|
800
|
|
|
|
|
|
|
|
|
801
|
3
|
100
|
|
|
|
16
|
last if !$result->{is_truncated}; |
|
802
|
|
|
|
|
|
|
|
|
803
|
1
|
|
|
|
|
4
|
$continuation_token = $result->{next_continuation_token}; |
|
804
|
|
|
|
|
|
|
} |
|
805
|
|
|
|
|
|
|
|
|
806
|
2
|
|
|
|
|
31
|
return @all_objects; |
|
807
|
|
|
|
|
|
|
} |
|
808
|
|
|
|
|
|
|
|
|
809
|
|
|
|
|
|
|
######################################################################## |
|
810
|
|
|
|
|
|
|
# Error checking helpers |
|
811
|
|
|
|
|
|
|
######################################################################## |
|
812
|
|
|
|
|
|
|
sub _is_success { |
|
813
|
18
|
|
|
18
|
|
122
|
return $_[0]->{status} =~ /\A2\d{2}\z/; |
|
814
|
|
|
|
|
|
|
} |
|
815
|
|
|
|
|
|
|
|
|
816
|
|
|
|
|
|
|
sub _is_not_found { |
|
817
|
8
|
|
|
8
|
|
39
|
return $_[0]->{status} == 404; |
|
818
|
|
|
|
|
|
|
} |
|
819
|
|
|
|
|
|
|
|
|
820
|
|
|
|
|
|
|
sub _croak_on_error { |
|
821
|
18
|
|
|
18
|
|
41
|
my ( $self, $response, $context ) = @_; |
|
822
|
|
|
|
|
|
|
|
|
823
|
18
|
100
|
|
|
|
25
|
return if _is_success($response); |
|
824
|
|
|
|
|
|
|
|
|
825
|
2
|
|
|
|
|
5
|
my $status = $response->{status}; |
|
826
|
2
|
|
|
|
|
3
|
my $reason = $response->{reason}; |
|
827
|
|
|
|
|
|
|
|
|
828
|
|
|
|
|
|
|
# Attempt to extract S3 error message from XML body |
|
829
|
2
|
|
|
|
|
3
|
my $detail = q{}; |
|
830
|
2
|
50
|
33
|
|
|
6
|
if ( $response->{content} && $response->{content} =~ /<\?xml/xsm ) { |
|
831
|
0
|
|
|
|
|
0
|
eval { |
|
832
|
0
|
|
|
|
|
0
|
my $doc = XML::LibXML->load_xml( string => $response->{content} ); |
|
833
|
0
|
|
|
|
|
0
|
my $msg = $doc->findvalue('/Error/Message'); |
|
834
|
0
|
|
|
|
|
0
|
my $code = $doc->findvalue('/Error/Code'); |
|
835
|
0
|
0
|
0
|
|
|
0
|
$detail = " - $code: $msg" if $code || $msg; |
|
836
|
|
|
|
|
|
|
}; |
|
837
|
|
|
|
|
|
|
} |
|
838
|
|
|
|
|
|
|
|
|
839
|
2
|
|
|
|
|
215
|
croak sprintf '%s failed: HTTP %s %s%s', $context, $status, $reason, $detail; |
|
840
|
|
|
|
|
|
|
} |
|
841
|
|
|
|
|
|
|
|
|
842
|
|
|
|
|
|
|
1; |
|
843
|
|
|
|
|
|
|
|
|
844
|
|
|
|
|
|
|
__END__ |