line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
#!/usr/bin/perl |
2
|
|
|
|
|
|
|
package MogileFS::Client; |
3
|
|
|
|
|
|
|
|
4
|
|
|
|
|
|
|
=head1 NAME |
5
|
|
|
|
|
|
|
|
6
|
|
|
|
|
|
|
MogileFS::Client - Client library for the MogileFS distributed file system. |
7
|
|
|
|
|
|
|
|
8
|
|
|
|
|
|
|
=head1 SYNOPSIS |
9
|
|
|
|
|
|
|
|
10
|
|
|
|
|
|
|
use MogileFS::Client; |
11
|
|
|
|
|
|
|
|
12
|
|
|
|
|
|
|
# create client object w/ server-configured namespace |
13
|
|
|
|
|
|
|
# and IPs of trackers |
14
|
|
|
|
|
|
|
$mogc = MogileFS::Client->new(domain => "foo.com::my_namespace", |
15
|
|
|
|
|
|
|
hosts => ['10.0.0.2:7001', '10.0.0.3:7001']); |
16
|
|
|
|
|
|
|
|
17
|
|
|
|
|
|
|
# create a file |
18
|
|
|
|
|
|
|
# mogile is a flat namespace. no paths. |
19
|
|
|
|
|
|
|
$key = "image_of_userid:$userid"; |
20
|
|
|
|
|
|
|
# must be configured on server |
21
|
|
|
|
|
|
|
$class = "user_images"; |
22
|
|
|
|
|
|
|
$fh = $mogc->new_file($key, $class); |
23
|
|
|
|
|
|
|
|
24
|
|
|
|
|
|
|
print $fh $data; |
25
|
|
|
|
|
|
|
|
26
|
|
|
|
|
|
|
unless ($fh->close) { |
27
|
|
|
|
|
|
|
die "Error writing file: " . $mogc->errcode . ": " . $mogc->errstr; |
28
|
|
|
|
|
|
|
} |
29
|
|
|
|
|
|
|
|
30
|
|
|
|
|
|
|
# Find the URLs that the file was replicated to. |
31
|
|
|
|
|
|
|
# May change over time. |
32
|
|
|
|
|
|
|
@urls = $mogc->get_paths($key); |
33
|
|
|
|
|
|
|
|
34
|
|
|
|
|
|
|
# no longer want it? |
35
|
|
|
|
|
|
|
$mogc->delete($key); |
36
|
|
|
|
|
|
|
|
37
|
|
|
|
|
|
|
=head1 DESCRIPTION |
38
|
|
|
|
|
|
|
|
39
|
|
|
|
|
|
|
This module is a client library for the MogileFS distributed file system. The class method 'new' creates a client object against a |
40
|
|
|
|
|
|
|
particular mogilefs tracker and domain. This object may then be used to store and retrieve content easily from MogileFS. |
41
|
|
|
|
|
|
|
|
42
|
|
|
|
|
|
|
=cut |
43
|
|
|
|
|
|
|
|
44
|
4
|
|
|
4
|
|
106959
|
use strict; |
|
4
|
|
|
|
|
10
|
|
|
4
|
|
|
|
|
156
|
|
45
|
4
|
|
|
4
|
|
22
|
use Carp; |
|
4
|
|
|
|
|
7
|
|
|
4
|
|
|
|
|
337
|
|
46
|
4
|
|
|
4
|
|
4181
|
use IO::WrapTie; |
|
4
|
|
|
|
|
76360
|
|
|
4
|
|
|
|
|
218
|
|
47
|
4
|
|
|
4
|
|
6467
|
use LWP::UserAgent; |
|
4
|
|
|
|
|
340895
|
|
|
4
|
|
|
|
|
200
|
|
48
|
|
|
|
|
|
|
use fields ( |
49
|
4
|
|
|
|
|
28
|
'domain', # scalar: the MogileFS domain (namespace). |
50
|
|
|
|
|
|
|
'backend', # MogileFS::Backend object |
51
|
|
|
|
|
|
|
'readonly', # bool: if set, client won't permit write actions/etc. just reads. |
52
|
|
|
|
|
|
|
'hooks', # hash: hookname -> coderef |
53
|
4
|
|
|
4
|
|
4985
|
); |
|
4
|
|
|
|
|
7057
|
|
54
|
4
|
|
|
4
|
|
4741
|
use Time::HiRes (); |
|
4
|
|
|
|
|
10136
|
|
|
4
|
|
|
|
|
134
|
|
55
|
4
|
|
|
4
|
|
3159
|
use MogileFS::Backend; |
|
4
|
|
|
|
|
15
|
|
|
4
|
|
|
|
|
144
|
|
56
|
4
|
|
|
4
|
|
2763
|
use MogileFS::NewHTTPFile; |
|
4
|
|
|
|
|
20
|
|
|
4
|
|
|
|
|
135
|
|
57
|
4
|
|
|
4
|
|
2586
|
use MogileFS::ClientHTTPFile; |
|
4
|
|
|
|
|
13
|
|
|
4
|
|
|
|
|
12419
|
|
58
|
|
|
|
|
|
|
|
59
|
|
|
|
|
|
|
our $VERSION = '1.17'; |
60
|
|
|
|
|
|
|
|
61
|
|
|
|
|
|
|
our $AUTOLOAD; |
62
|
|
|
|
|
|
|
|
63
|
|
|
|
|
|
|
=head1 METHODS |
64
|
|
|
|
|
|
|
|
65
|
|
|
|
|
|
|
=head2 new |
66
|
|
|
|
|
|
|
|
67
|
|
|
|
|
|
|
$client = MogileFS::Client->new( %OPTIONS ); |
68
|
|
|
|
|
|
|
|
69
|
|
|
|
|
|
|
Creates a new MogileFS::Client object. |
70
|
|
|
|
|
|
|
|
71
|
|
|
|
|
|
|
Returns MogileFS::Client object on success, or dies on failure. |
72
|
|
|
|
|
|
|
|
73
|
|
|
|
|
|
|
OPTIONS: |
74
|
|
|
|
|
|
|
|
75
|
|
|
|
|
|
|
=over |
76
|
|
|
|
|
|
|
|
77
|
|
|
|
|
|
|
=item hosts |
78
|
|
|
|
|
|
|
|
79
|
|
|
|
|
|
|
Arrayref of 'host:port' strings to connect to as backend trackers in this client. |
80
|
|
|
|
|
|
|
|
81
|
|
|
|
|
|
|
=item domain |
82
|
|
|
|
|
|
|
|
83
|
|
|
|
|
|
|
String representing the mogile domain which this MogileFS client is associated with. (All create/delete/fetch operations |
84
|
|
|
|
|
|
|
will be performed against this mogile domain). See the mogadm shell command and its 'domain' category of operations for |
85
|
|
|
|
|
|
|
information on manipulating the list of possible domains on a MogileFS system. |
86
|
|
|
|
|
|
|
|
87
|
|
|
|
|
|
|
=back |
88
|
|
|
|
|
|
|
|
89
|
|
|
|
|
|
|
=cut |
90
|
|
|
|
|
|
|
|
91
|
|
|
|
|
|
|
sub new { |
92
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
93
|
0
|
0
|
|
|
|
0
|
$self = fields::new($self) unless ref $self; |
94
|
|
|
|
|
|
|
|
95
|
0
|
|
|
|
|
0
|
return $self->_init(@_); |
96
|
|
|
|
|
|
|
} |
97
|
|
|
|
|
|
|
|
98
|
|
|
|
|
|
|
=head2 reload |
99
|
|
|
|
|
|
|
|
100
|
|
|
|
|
|
|
$mogc->reload( %OPTIONS ) |
101
|
|
|
|
|
|
|
|
102
|
|
|
|
|
|
|
Re-init the object, like you'd just reconstructed it with 'new', but change it in-place instead. Useful if you have a system which reloads a config file, and you want to update a singleton $mogc handle's config value. |
103
|
|
|
|
|
|
|
|
104
|
|
|
|
|
|
|
=cut |
105
|
|
|
|
|
|
|
|
106
|
|
|
|
|
|
|
sub reload { |
107
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
108
|
0
|
0
|
|
|
|
0
|
return undef unless $self; |
109
|
|
|
|
|
|
|
|
110
|
0
|
|
|
|
|
0
|
return $self->_init(@_); |
111
|
|
|
|
|
|
|
} |
112
|
|
|
|
|
|
|
|
113
|
|
|
|
|
|
|
sub _init { |
114
|
0
|
|
|
0
|
|
0
|
my MogileFS::Client $self = shift; |
115
|
|
|
|
|
|
|
|
116
|
0
|
|
|
|
|
0
|
my %args = @_; |
117
|
|
|
|
|
|
|
|
118
|
|
|
|
|
|
|
# FIXME: add actual validation |
119
|
|
|
|
|
|
|
{ |
120
|
|
|
|
|
|
|
# by default, set readonly off |
121
|
0
|
0
|
|
|
|
0
|
$self->{readonly} = $args{readonly} ? 1 : 0; |
|
0
|
|
|
|
|
0
|
|
122
|
|
|
|
|
|
|
|
123
|
|
|
|
|
|
|
# get domain (required) |
124
|
0
|
0
|
|
|
|
0
|
$self->{domain} = $args{domain} or |
125
|
|
|
|
|
|
|
_fail("constructor requires parameter 'domain'"); |
126
|
|
|
|
|
|
|
|
127
|
|
|
|
|
|
|
# create a new backend object if there's not one already, |
128
|
|
|
|
|
|
|
# otherwise call a reload on the existing one |
129
|
0
|
0
|
|
|
|
0
|
if ($self->{backend}) { |
130
|
0
|
|
|
|
|
0
|
$self->{backend}->reload( hosts => $args{hosts} ); |
131
|
|
|
|
|
|
|
} else { |
132
|
0
|
|
|
|
|
0
|
$self->{backend} = MogileFS::Backend->new( hosts => $args{hosts}, |
133
|
|
|
|
|
|
|
timeout => $args{timeout}, |
134
|
|
|
|
|
|
|
); |
135
|
|
|
|
|
|
|
} |
136
|
0
|
0
|
|
|
|
0
|
_fail("cannot instantiate MogileFS::Backend") unless $self->{backend}; |
137
|
|
|
|
|
|
|
} |
138
|
|
|
|
|
|
|
|
139
|
0
|
|
|
|
|
0
|
_debug("MogileFS object: [$self]", $self); |
140
|
|
|
|
|
|
|
|
141
|
0
|
|
|
|
|
0
|
return $self; |
142
|
|
|
|
|
|
|
} |
143
|
|
|
|
|
|
|
|
144
|
|
|
|
|
|
|
=head2 last_tracker |
145
|
|
|
|
|
|
|
|
146
|
|
|
|
|
|
|
Returns a scalar of form "ip:port", representing the last mogilefsd |
147
|
|
|
|
|
|
|
'tracker' server which was talked to. |
148
|
|
|
|
|
|
|
|
149
|
|
|
|
|
|
|
=cut |
150
|
|
|
|
|
|
|
|
151
|
|
|
|
|
|
|
sub last_tracker { |
152
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
153
|
0
|
|
|
|
|
0
|
return $self->{backend}->last_tracker; |
154
|
|
|
|
|
|
|
} |
155
|
|
|
|
|
|
|
|
156
|
|
|
|
|
|
|
=head2 errstr |
157
|
|
|
|
|
|
|
|
158
|
|
|
|
|
|
|
Returns string representation of the last error that occurred. It |
159
|
|
|
|
|
|
|
includes the error code (same as method 'errcode') and a space before |
160
|
|
|
|
|
|
|
the optional English error message. |
161
|
|
|
|
|
|
|
|
162
|
|
|
|
|
|
|
This isn't necessarily guaranteed to reset after a successful |
163
|
|
|
|
|
|
|
operation. Only call it after another operation returns an error. |
164
|
|
|
|
|
|
|
|
165
|
|
|
|
|
|
|
|
166
|
|
|
|
|
|
|
=cut |
167
|
|
|
|
|
|
|
|
168
|
|
|
|
|
|
|
sub errstr { |
169
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
170
|
0
|
|
|
|
|
0
|
return $self->{backend}->errstr; |
171
|
|
|
|
|
|
|
} |
172
|
|
|
|
|
|
|
|
173
|
|
|
|
|
|
|
=head2 errcode |
174
|
|
|
|
|
|
|
|
175
|
|
|
|
|
|
|
Returns an error code. Not a number, but a string identifier |
176
|
|
|
|
|
|
|
(e.g. "no_domain") which is stable for use in error handling logic. |
177
|
|
|
|
|
|
|
|
178
|
|
|
|
|
|
|
This isn't necessarily guaranteed to reset after a successful |
179
|
|
|
|
|
|
|
operation. Only call it after another operation returns an error. |
180
|
|
|
|
|
|
|
|
181
|
|
|
|
|
|
|
=cut |
182
|
|
|
|
|
|
|
|
183
|
|
|
|
|
|
|
sub errcode { |
184
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
185
|
0
|
|
|
|
|
0
|
return $self->{backend}->errcode; |
186
|
|
|
|
|
|
|
} |
187
|
|
|
|
|
|
|
|
188
|
|
|
|
|
|
|
=head2 force_disconnect |
189
|
|
|
|
|
|
|
|
190
|
|
|
|
|
|
|
Forces the client to disconnect from the tracker, causing it to reconnect |
191
|
|
|
|
|
|
|
when the next request is made. It will reconnect to a different tracker if |
192
|
|
|
|
|
|
|
possible. A paranoid application may wish to do to this before retrying a |
193
|
|
|
|
|
|
|
failed command, on the off chance that another tracker may be working better. |
194
|
|
|
|
|
|
|
|
195
|
|
|
|
|
|
|
=cut |
196
|
|
|
|
|
|
|
|
197
|
|
|
|
|
|
|
sub force_disconnect { |
198
|
1
|
|
|
1
|
1
|
1525
|
my MogileFS::Client $self = shift; |
199
|
1
|
|
|
|
|
10
|
return $self->{backend}->force_disconnect(); |
200
|
|
|
|
|
|
|
} |
201
|
|
|
|
|
|
|
|
202
|
|
|
|
|
|
|
=head2 readonly |
203
|
|
|
|
|
|
|
|
204
|
|
|
|
|
|
|
$is_readonly = $mogc->readonly |
205
|
|
|
|
|
|
|
$mogc->readonly(1) |
206
|
|
|
|
|
|
|
|
207
|
|
|
|
|
|
|
Getter/setter to mark this client object as read-only. Purely a local |
208
|
|
|
|
|
|
|
operation/restriction, doesn't do a network operation to the mogilefsd |
209
|
|
|
|
|
|
|
server. |
210
|
|
|
|
|
|
|
|
211
|
|
|
|
|
|
|
=cut |
212
|
|
|
|
|
|
|
|
213
|
|
|
|
|
|
|
sub readonly { |
214
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
215
|
0
|
0
|
|
|
|
0
|
return $self->{readonly} = $_[0] ? 1 : 0 if @_; |
|
|
0
|
|
|
|
|
|
216
|
0
|
|
|
|
|
0
|
return $self->{readonly}; |
217
|
|
|
|
|
|
|
} |
218
|
|
|
|
|
|
|
|
219
|
|
|
|
|
|
|
=head2 new_file |
220
|
|
|
|
|
|
|
|
221
|
|
|
|
|
|
|
$mogc->new_file($key) |
222
|
|
|
|
|
|
|
$mogc->new_file($key, $class) |
223
|
|
|
|
|
|
|
$mogc->new_file($key, $class, $content_length) |
224
|
|
|
|
|
|
|
$mogc->new_file($key, $class, $content_length , $opts_hashref) |
225
|
|
|
|
|
|
|
|
226
|
|
|
|
|
|
|
Start creating a new filehandle with the given key, and option given |
227
|
|
|
|
|
|
|
class and options. |
228
|
|
|
|
|
|
|
|
229
|
|
|
|
|
|
|
Returns a filehandle you should then print to, and later close to |
230
|
|
|
|
|
|
|
complete the operation. B check the return value from close! |
231
|
|
|
|
|
|
|
If your close didn't succeed, the file didn't get saved! |
232
|
|
|
|
|
|
|
|
233
|
|
|
|
|
|
|
$opts_hashref can contain keys: |
234
|
|
|
|
|
|
|
|
235
|
|
|
|
|
|
|
=over |
236
|
|
|
|
|
|
|
|
237
|
|
|
|
|
|
|
=item fid |
238
|
|
|
|
|
|
|
|
239
|
|
|
|
|
|
|
Explicitly specify the fid number to use, rather than it being automatically allocated. |
240
|
|
|
|
|
|
|
|
241
|
|
|
|
|
|
|
=item create_open_args |
242
|
|
|
|
|
|
|
|
243
|
|
|
|
|
|
|
Hashref of extra key/value pairs to send to mogilefsd in create_open phase. |
244
|
|
|
|
|
|
|
|
245
|
|
|
|
|
|
|
=item create_close_args |
246
|
|
|
|
|
|
|
|
247
|
|
|
|
|
|
|
Hashref of extra key/value pairs to send to mogilefsd in create_close phase. |
248
|
|
|
|
|
|
|
|
249
|
|
|
|
|
|
|
=item largefile |
250
|
|
|
|
|
|
|
|
251
|
|
|
|
|
|
|
Use MogileFS::ClientHTTPFile which will not load the entire file into memory |
252
|
|
|
|
|
|
|
like the default MogileFS::NewHTTPFile but requires that the storage node |
253
|
|
|
|
|
|
|
HTTP servers support the Content-Range header in PUT requests and is a little |
254
|
|
|
|
|
|
|
slower. |
255
|
|
|
|
|
|
|
|
256
|
|
|
|
|
|
|
=back |
257
|
|
|
|
|
|
|
|
258
|
|
|
|
|
|
|
=cut |
259
|
|
|
|
|
|
|
|
260
|
|
|
|
|
|
|
# returns MogileFS::NewHTTPFile object, or undef if no device |
261
|
|
|
|
|
|
|
# available for writing |
262
|
|
|
|
|
|
|
# ARGS: ( key, class, bytes?, opts? ) |
263
|
|
|
|
|
|
|
# where bytes is optional and the length of the file and opts is also optional |
264
|
|
|
|
|
|
|
# and is a hashref of options. supported options: fid = unique file id to use |
265
|
|
|
|
|
|
|
# instead of just picking one in the database. |
266
|
|
|
|
|
|
|
sub new_file { |
267
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
268
|
0
|
0
|
|
|
|
0
|
return undef if $self->{readonly}; |
269
|
|
|
|
|
|
|
|
270
|
0
|
|
|
|
|
0
|
my ($key, $class, $bytes, $opts) = @_; |
271
|
0
|
|
|
|
|
0
|
$bytes += 0; |
272
|
0
|
|
0
|
|
|
0
|
$opts ||= {}; |
273
|
|
|
|
|
|
|
|
274
|
|
|
|
|
|
|
# Extra args to be passed along with the create_open and create_close commands. |
275
|
|
|
|
|
|
|
# Any internally generated args of the same name will overwrite supplied ones in |
276
|
|
|
|
|
|
|
# these hashes. |
277
|
0
|
|
0
|
|
|
0
|
my $create_open_args = $opts->{create_open_args} || {}; |
278
|
0
|
|
0
|
|
|
0
|
my $create_close_args = $opts->{create_close_args} || {}; |
279
|
|
|
|
|
|
|
|
280
|
0
|
|
|
|
|
0
|
$self->run_hook('new_file_start', $self, $key, $class, $opts); |
281
|
|
|
|
|
|
|
|
282
|
0
|
0
|
0
|
|
|
0
|
my $res = $self->{backend}->do_request |
283
|
|
|
|
|
|
|
("create_open", { |
284
|
|
|
|
|
|
|
%$create_open_args, |
285
|
|
|
|
|
|
|
domain => $self->{domain}, |
286
|
|
|
|
|
|
|
class => $class, |
287
|
|
|
|
|
|
|
key => $key, |
288
|
|
|
|
|
|
|
fid => $opts->{fid} || 0, # fid should be specified, or pass 0 meaning to auto-generate one |
289
|
|
|
|
|
|
|
multi_dest => 1, |
290
|
|
|
|
|
|
|
}) or return undef; |
291
|
|
|
|
|
|
|
|
292
|
0
|
|
|
|
|
0
|
my $dests = []; # [ [devid,path], [devid,path], ... ] |
293
|
|
|
|
|
|
|
|
294
|
|
|
|
|
|
|
# determine old vs. new format to populate destinations |
295
|
0
|
0
|
|
|
|
0
|
unless (exists $res->{dev_count}) { |
296
|
0
|
|
|
|
|
0
|
push @$dests, [ $res->{devid}, $res->{path} ]; |
297
|
|
|
|
|
|
|
} else { |
298
|
0
|
|
|
|
|
0
|
for my $i (1..$res->{dev_count}) { |
299
|
0
|
|
|
|
|
0
|
push @$dests, [ $res->{"devid_$i"}, $res->{"path_$i"} ]; |
300
|
|
|
|
|
|
|
} |
301
|
|
|
|
|
|
|
} |
302
|
|
|
|
|
|
|
|
303
|
0
|
|
|
|
|
0
|
my $main_dest = shift @$dests; |
304
|
0
|
|
|
|
|
0
|
my ($main_devid, $main_path) = ($main_dest->[0], $main_dest->[1]); |
305
|
|
|
|
|
|
|
|
306
|
|
|
|
|
|
|
# create a MogileFS::NewHTTPFile object, based off of IO::File |
307
|
0
|
0
|
|
|
|
0
|
unless ($main_path =~ m!^http://!) { |
308
|
0
|
|
|
|
|
0
|
Carp::croak("This version of MogileFS::Client no longer supports non-http storage URLs.\n"); |
309
|
|
|
|
|
|
|
} |
310
|
|
|
|
|
|
|
|
311
|
0
|
|
|
|
|
0
|
$self->run_hook('new_file_end', $self, $key, $class, $opts); |
312
|
|
|
|
|
|
|
|
313
|
0
|
0
|
|
|
|
0
|
return IO::WrapTie::wraptie( ( $opts->{largefile} |
314
|
|
|
|
|
|
|
? 'MogileFS::ClientHTTPFile' |
315
|
|
|
|
|
|
|
: 'MogileFS::NewHTTPFile' ), |
316
|
|
|
|
|
|
|
mg => $self, |
317
|
|
|
|
|
|
|
fid => $res->{fid}, |
318
|
|
|
|
|
|
|
path => $main_path, |
319
|
|
|
|
|
|
|
devid => $main_devid, |
320
|
|
|
|
|
|
|
backup_dests => $dests, |
321
|
|
|
|
|
|
|
class => $class, |
322
|
|
|
|
|
|
|
key => $key, |
323
|
|
|
|
|
|
|
content_length => $bytes+0, |
324
|
|
|
|
|
|
|
create_close_args => $create_close_args, |
325
|
|
|
|
|
|
|
overwrite => 1, |
326
|
|
|
|
|
|
|
); |
327
|
|
|
|
|
|
|
} |
328
|
|
|
|
|
|
|
|
329
|
|
|
|
|
|
|
=head2 edit_file |
330
|
|
|
|
|
|
|
|
331
|
|
|
|
|
|
|
$mogc->edit_file($key, $opts_hashref) |
332
|
|
|
|
|
|
|
|
333
|
|
|
|
|
|
|
Edit the file with the the given key. |
334
|
|
|
|
|
|
|
|
335
|
|
|
|
|
|
|
|
336
|
|
|
|
|
|
|
B edit_file is currently EXPERIMENTAL and not recommended for |
337
|
|
|
|
|
|
|
production use. MogileFS is primarily designed for storing files |
338
|
|
|
|
|
|
|
for later retrieval, rather than editing. Use of this function may lead to |
339
|
|
|
|
|
|
|
poor performance and, until it has been proven mature, should be |
340
|
|
|
|
|
|
|
considered to also potentially cause data loss. |
341
|
|
|
|
|
|
|
|
342
|
|
|
|
|
|
|
B use of this function requires support for the DAV 'MOVE' |
343
|
|
|
|
|
|
|
verb and partial PUT (i.e. Content-Range in PUT) on the back-end |
344
|
|
|
|
|
|
|
storage servers (e.g. apache with mod_dav). |
345
|
|
|
|
|
|
|
|
346
|
|
|
|
|
|
|
Returns a seekable filehandle you can read/write to. Calling this |
347
|
|
|
|
|
|
|
function may invalidate some or all URLs you currently have for this |
348
|
|
|
|
|
|
|
key, so you should call ->get_paths again afterwards if you need |
349
|
|
|
|
|
|
|
them. |
350
|
|
|
|
|
|
|
|
351
|
|
|
|
|
|
|
On close of the filehandle, the new file contents will replace the |
352
|
|
|
|
|
|
|
previous contents (and again invalidate any existing URLs). |
353
|
|
|
|
|
|
|
|
354
|
|
|
|
|
|
|
By default, the file contents are preserved on open, but you may |
355
|
|
|
|
|
|
|
specify the overwrite option to zero the file first. The seek position |
356
|
|
|
|
|
|
|
is at the beginning of the file, but you may seek to the end to append. |
357
|
|
|
|
|
|
|
|
358
|
|
|
|
|
|
|
$opts_hashref can contain keys: |
359
|
|
|
|
|
|
|
|
360
|
|
|
|
|
|
|
=over |
361
|
|
|
|
|
|
|
|
362
|
|
|
|
|
|
|
=item overwrite |
363
|
|
|
|
|
|
|
|
364
|
|
|
|
|
|
|
The edit will overwrite the file, equivalent to opening with '>'. |
365
|
|
|
|
|
|
|
Default: false. |
366
|
|
|
|
|
|
|
|
367
|
|
|
|
|
|
|
=back |
368
|
|
|
|
|
|
|
|
369
|
|
|
|
|
|
|
=cut |
370
|
|
|
|
|
|
|
|
371
|
|
|
|
|
|
|
sub edit_file { |
372
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
373
|
0
|
0
|
|
|
|
0
|
return undef if $self->{readonly}; |
374
|
|
|
|
|
|
|
|
375
|
0
|
|
|
|
|
0
|
my($key, $opts) = @_; |
376
|
|
|
|
|
|
|
|
377
|
0
|
0
|
|
|
|
0
|
my $res = $self->{backend}->do_request |
378
|
|
|
|
|
|
|
("edit_file", { |
379
|
|
|
|
|
|
|
domain => $self->{domain}, |
380
|
|
|
|
|
|
|
key => $key, |
381
|
|
|
|
|
|
|
}) or return undef; |
382
|
|
|
|
|
|
|
|
383
|
0
|
|
|
|
|
0
|
my $moveReq = HTTP::Request->new('MOVE', $res->{oldpath}); |
384
|
0
|
|
|
|
|
0
|
$moveReq->header(Destination => $res->{newpath}); |
385
|
0
|
|
|
|
|
0
|
my $ua = LWP::UserAgent->new; |
386
|
0
|
|
|
|
|
0
|
my $resp = $ua->request($moveReq); |
387
|
0
|
0
|
|
|
|
0
|
unless ($resp->is_success) { |
388
|
0
|
|
|
|
|
0
|
warn "Failed to MOVE $res->{oldpath} to $res->{newpath}"; |
389
|
0
|
|
|
|
|
0
|
return undef; |
390
|
|
|
|
|
|
|
} |
391
|
|
|
|
|
|
|
|
392
|
0
|
|
|
|
|
0
|
return IO::WrapTie::wraptie('MogileFS::ClientHTTPFile', |
393
|
|
|
|
|
|
|
mg => $self, |
394
|
|
|
|
|
|
|
fid => $res->{fid}, |
395
|
|
|
|
|
|
|
path => $res->{newpath}, |
396
|
|
|
|
|
|
|
devid => $res->{devid}, |
397
|
|
|
|
|
|
|
class => $res->{class}, |
398
|
|
|
|
|
|
|
key => $key, |
399
|
|
|
|
|
|
|
overwrite => $opts->{overwrite}, |
400
|
|
|
|
|
|
|
); |
401
|
|
|
|
|
|
|
} |
402
|
|
|
|
|
|
|
|
403
|
|
|
|
|
|
|
=head2 read_file |
404
|
|
|
|
|
|
|
|
405
|
|
|
|
|
|
|
$mogc->read_file($key) |
406
|
|
|
|
|
|
|
|
407
|
|
|
|
|
|
|
Read the file with the the given key. |
408
|
|
|
|
|
|
|
|
409
|
|
|
|
|
|
|
Returns a seekable filehandle you can read() from. Note that you cannot |
410
|
|
|
|
|
|
|
read line by line using <$fh> notation. |
411
|
|
|
|
|
|
|
|
412
|
|
|
|
|
|
|
Takes the same options as get_paths (which is called internally to get |
413
|
|
|
|
|
|
|
the URIs to read from). |
414
|
|
|
|
|
|
|
|
415
|
|
|
|
|
|
|
=cut |
416
|
|
|
|
|
|
|
|
417
|
|
|
|
|
|
|
sub read_file { |
418
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
419
|
|
|
|
|
|
|
|
420
|
0
|
|
|
|
|
0
|
my @paths = $self->get_paths(@_); |
421
|
|
|
|
|
|
|
|
422
|
0
|
|
|
|
|
0
|
my $path = shift @paths; |
423
|
|
|
|
|
|
|
|
424
|
0
|
0
|
|
|
|
0
|
return if !$path; |
425
|
|
|
|
|
|
|
|
426
|
0
|
|
|
|
|
0
|
my @backup_dests = map { [ undef, $_ ] } @paths; |
|
0
|
|
|
|
|
0
|
|
427
|
|
|
|
|
|
|
|
428
|
0
|
|
|
|
|
0
|
return IO::WrapTie::wraptie('MogileFS::ClientHTTPFile', |
429
|
|
|
|
|
|
|
path => $path, |
430
|
|
|
|
|
|
|
backup_dests => \@backup_dests, |
431
|
|
|
|
|
|
|
readonly => 1, |
432
|
|
|
|
|
|
|
); |
433
|
|
|
|
|
|
|
} |
434
|
|
|
|
|
|
|
|
435
|
|
|
|
|
|
|
=head2 store_file |
436
|
|
|
|
|
|
|
|
437
|
|
|
|
|
|
|
$mogc->store_file($key, $class, $fh_or_filename[, $opts_hashref]) |
438
|
|
|
|
|
|
|
|
439
|
|
|
|
|
|
|
Wrapper around new_file, print, and close. |
440
|
|
|
|
|
|
|
|
441
|
|
|
|
|
|
|
Given a key, class, and a filehandle or filename, stores the file |
442
|
|
|
|
|
|
|
contents in MogileFS. Returns the number of bytes stored on success, |
443
|
|
|
|
|
|
|
undef on failure. |
444
|
|
|
|
|
|
|
|
445
|
|
|
|
|
|
|
$opts_hashref can contain keys for new_file, and also the following: |
446
|
|
|
|
|
|
|
|
447
|
|
|
|
|
|
|
=over |
448
|
|
|
|
|
|
|
|
449
|
|
|
|
|
|
|
=item chunk_size |
450
|
|
|
|
|
|
|
|
451
|
|
|
|
|
|
|
Number of bytes to read and write and write at once out of the larger file. |
452
|
|
|
|
|
|
|
Defaults to 8192 bytes. Increasing this can increase performance at the cost |
453
|
|
|
|
|
|
|
of more memory used while uploading the file. |
454
|
|
|
|
|
|
|
Note that this mostly helps when using largefile => 1 |
455
|
|
|
|
|
|
|
|
456
|
|
|
|
|
|
|
=back |
457
|
|
|
|
|
|
|
|
458
|
|
|
|
|
|
|
=cut |
459
|
|
|
|
|
|
|
|
460
|
|
|
|
|
|
|
sub store_file { |
461
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
462
|
0
|
0
|
|
|
|
0
|
return undef if $self->{readonly}; |
463
|
|
|
|
|
|
|
|
464
|
0
|
|
|
|
|
0
|
my($key, $class, $file, $opts) = @_; |
465
|
0
|
|
|
|
|
0
|
$self->run_hook('store_file_start', $self, $key, $class, $opts); |
466
|
|
|
|
|
|
|
|
467
|
0
|
|
0
|
|
|
0
|
my $chunk_size = $opts->{chunk_size} || 8192; |
468
|
0
|
0
|
|
|
|
0
|
my $fh = $self->new_file($key, $class, undef, $opts) or return; |
469
|
0
|
|
|
|
|
0
|
my $fh_from; |
470
|
0
|
0
|
|
|
|
0
|
if (ref($file)) { |
471
|
0
|
|
|
|
|
0
|
$fh_from = $file; |
472
|
|
|
|
|
|
|
} else { |
473
|
0
|
0
|
|
|
|
0
|
open $fh_from, $file or return; |
474
|
|
|
|
|
|
|
} |
475
|
0
|
|
|
|
|
0
|
my $bytes; |
476
|
0
|
|
|
|
|
0
|
while (my $len = read $fh_from, my($chunk), $chunk_size) { |
477
|
0
|
|
|
|
|
0
|
$fh->print($chunk); |
478
|
0
|
|
|
|
|
0
|
$bytes += $len; |
479
|
|
|
|
|
|
|
} |
480
|
|
|
|
|
|
|
|
481
|
0
|
|
|
|
|
0
|
$self->run_hook('store_file_end', $self, $key, $class, $opts); |
482
|
|
|
|
|
|
|
|
483
|
0
|
0
|
|
|
|
0
|
close $fh_from unless ref $file; |
484
|
0
|
0
|
|
|
|
0
|
$fh->close or return; |
485
|
0
|
|
|
|
|
0
|
$bytes; |
486
|
|
|
|
|
|
|
} |
487
|
|
|
|
|
|
|
|
488
|
|
|
|
|
|
|
=head2 store_content |
489
|
|
|
|
|
|
|
|
490
|
|
|
|
|
|
|
$mogc->store_content($key, $class, $content[, $opts]); |
491
|
|
|
|
|
|
|
|
492
|
|
|
|
|
|
|
Wrapper around new_file, print, and close. Given a key, class, and |
493
|
|
|
|
|
|
|
file contents (scalar or scalarref), stores the file contents in |
494
|
|
|
|
|
|
|
MogileFS. Returns the number of bytes stored on success, undef on |
495
|
|
|
|
|
|
|
failure. |
496
|
|
|
|
|
|
|
|
497
|
|
|
|
|
|
|
=cut |
498
|
|
|
|
|
|
|
|
499
|
|
|
|
|
|
|
sub store_content { |
500
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
501
|
0
|
0
|
|
|
|
0
|
return undef if $self->{readonly}; |
502
|
|
|
|
|
|
|
|
503
|
0
|
|
|
|
|
0
|
my($key, $class, $content, $opts) = @_; |
504
|
|
|
|
|
|
|
|
505
|
0
|
|
|
|
|
0
|
$self->run_hook('store_content_start', $self, $key, $class, $opts); |
506
|
|
|
|
|
|
|
|
507
|
0
|
0
|
|
|
|
0
|
my $fh = $self->new_file($key, $class, undef, $opts) or return; |
508
|
0
|
0
|
|
|
|
0
|
$content = ref($content) eq 'SCALAR' ? $$content : $content; |
509
|
0
|
|
|
|
|
0
|
$fh->print($content); |
510
|
|
|
|
|
|
|
|
511
|
0
|
|
|
|
|
0
|
$self->run_hook('store_content_end', $self, $key, $class, $opts); |
512
|
|
|
|
|
|
|
|
513
|
0
|
0
|
|
|
|
0
|
$fh->close or return; |
514
|
0
|
|
|
|
|
0
|
length($content); |
515
|
|
|
|
|
|
|
} |
516
|
|
|
|
|
|
|
|
517
|
|
|
|
|
|
|
=head2 get_paths |
518
|
|
|
|
|
|
|
|
519
|
|
|
|
|
|
|
@paths = $mogc->get_paths($key) |
520
|
|
|
|
|
|
|
@paths = $mogc->get_paths($key, $no_verify_bool); # old way |
521
|
|
|
|
|
|
|
@paths = $mogc->get_paths($key, { noverify => $bool }); # new way |
522
|
|
|
|
|
|
|
|
523
|
|
|
|
|
|
|
Given a key, returns an array of all the locations (HTTP URLs) that |
524
|
|
|
|
|
|
|
the file has been replicated to. |
525
|
|
|
|
|
|
|
|
526
|
|
|
|
|
|
|
=over |
527
|
|
|
|
|
|
|
|
528
|
|
|
|
|
|
|
=item noverify |
529
|
|
|
|
|
|
|
|
530
|
|
|
|
|
|
|
If the "no verify" option is set, the mogilefsd tracker doesn't verify |
531
|
|
|
|
|
|
|
that the first item returned in the list is up/alive. Skipping that |
532
|
|
|
|
|
|
|
check is faster, so use "noverify" if your application can do it |
533
|
|
|
|
|
|
|
faster/smarter. For instance, when giving L a list of URLs |
534
|
|
|
|
|
|
|
to reproxy to, Perlbal can intelligently find one that's alive, so use |
535
|
|
|
|
|
|
|
noverify and get out of mod_perl or whatever as soon as possible. |
536
|
|
|
|
|
|
|
|
537
|
|
|
|
|
|
|
=item zone |
538
|
|
|
|
|
|
|
|
539
|
|
|
|
|
|
|
If the zone option is set to 'alt', the mogilefsd tracker will use the |
540
|
|
|
|
|
|
|
alternative IP for each host if available, while constructing the paths. |
541
|
|
|
|
|
|
|
|
542
|
|
|
|
|
|
|
=item pathcount |
543
|
|
|
|
|
|
|
|
544
|
|
|
|
|
|
|
If the pathcount option is set to a positive integer greater than 2, the |
545
|
|
|
|
|
|
|
mogilefsd tracker will attempt to return that many different paths (if |
546
|
|
|
|
|
|
|
available) to the same file. If not present or out of range, this value |
547
|
|
|
|
|
|
|
defaults to 2. |
548
|
|
|
|
|
|
|
|
549
|
|
|
|
|
|
|
=back |
550
|
|
|
|
|
|
|
|
551
|
|
|
|
|
|
|
=cut |
552
|
|
|
|
|
|
|
|
553
|
|
|
|
|
|
|
# old style calling: |
554
|
|
|
|
|
|
|
# get_paths(key, noverify) |
555
|
|
|
|
|
|
|
# new style calling: |
556
|
|
|
|
|
|
|
# get_paths(key, { noverify => 0/1, zone => "alt", pathcount => 2..N }); |
557
|
|
|
|
|
|
|
# but with both, second parameter is optional |
558
|
|
|
|
|
|
|
# |
559
|
|
|
|
|
|
|
# returns list of URLs that key can be found at, or the empty |
560
|
|
|
|
|
|
|
# list on either error or no paths |
561
|
|
|
|
|
|
|
sub get_paths { |
562
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
563
|
0
|
|
|
|
|
0
|
my ($key, $opts) = @_; |
564
|
|
|
|
|
|
|
|
565
|
|
|
|
|
|
|
# handle parameters, if any |
566
|
0
|
|
|
|
|
0
|
my ($noverify, $zone); |
567
|
0
|
0
|
|
|
|
0
|
unless (ref $opts) { |
568
|
0
|
|
|
|
|
0
|
$opts = { noverify => $opts }; |
569
|
|
|
|
|
|
|
} |
570
|
0
|
|
|
|
|
0
|
my %extra_args; |
571
|
|
|
|
|
|
|
|
572
|
0
|
0
|
|
|
|
0
|
$noverify = 1 if $opts->{noverify}; |
573
|
0
|
|
|
|
|
0
|
$zone = $opts->{zone}; |
574
|
|
|
|
|
|
|
|
575
|
0
|
0
|
|
|
|
0
|
if (my $pathcount = delete $opts->{pathcount}) { |
576
|
0
|
|
|
|
|
0
|
$extra_args{pathcount} = $pathcount; |
577
|
|
|
|
|
|
|
} |
578
|
|
|
|
|
|
|
|
579
|
0
|
|
|
|
|
0
|
$self->run_hook('get_paths_start', $self, $key, $opts); |
580
|
|
|
|
|
|
|
|
581
|
0
|
0
|
|
|
|
0
|
my $res = $self->{backend}->do_request |
|
|
0
|
|
|
|
|
|
582
|
|
|
|
|
|
|
("get_paths", { |
583
|
|
|
|
|
|
|
domain => $self->{domain}, |
584
|
|
|
|
|
|
|
key => $key, |
585
|
|
|
|
|
|
|
noverify => $noverify ? 1 : 0, |
586
|
|
|
|
|
|
|
zone => $zone, |
587
|
|
|
|
|
|
|
%extra_args, |
588
|
|
|
|
|
|
|
}) or return (); |
589
|
|
|
|
|
|
|
|
590
|
0
|
|
|
|
|
0
|
my @paths = map { $res->{"path$_"} } (1..$res->{paths}); |
|
0
|
|
|
|
|
0
|
|
591
|
|
|
|
|
|
|
|
592
|
0
|
|
|
|
|
0
|
$self->run_hook('get_paths_end', $self, $key, $opts); |
593
|
|
|
|
|
|
|
|
594
|
0
|
|
|
|
|
0
|
return @paths; |
595
|
|
|
|
|
|
|
} |
596
|
|
|
|
|
|
|
|
597
|
|
|
|
|
|
|
=head2 get_file_data |
598
|
|
|
|
|
|
|
|
599
|
|
|
|
|
|
|
$dataref = $mogc->get_file_data($key) |
600
|
|
|
|
|
|
|
|
601
|
|
|
|
|
|
|
Wrapper around get_paths & LWP, which returns scalarref of file |
602
|
|
|
|
|
|
|
contents in a scalarref. |
603
|
|
|
|
|
|
|
|
604
|
|
|
|
|
|
|
Don't use for large data, as it all comes back to you in one string. |
605
|
|
|
|
|
|
|
|
606
|
|
|
|
|
|
|
=cut |
607
|
|
|
|
|
|
|
|
608
|
|
|
|
|
|
|
# given a key, returns a scalar reference pointing at a string containing |
609
|
|
|
|
|
|
|
# the contents of the file. takes one parameter; a scalar key to get the |
610
|
|
|
|
|
|
|
# data for the file. |
611
|
|
|
|
|
|
|
sub get_file_data { |
612
|
|
|
|
|
|
|
# given a key, load some paths and get data |
613
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = $_[0]; |
614
|
0
|
|
|
|
|
0
|
my ($key, $timeout) = ($_[1], $_[2]); |
615
|
|
|
|
|
|
|
|
616
|
0
|
|
|
|
|
0
|
my @paths = $self->get_paths($key, 1); |
617
|
0
|
0
|
|
|
|
0
|
return undef unless @paths; |
618
|
|
|
|
|
|
|
|
619
|
|
|
|
|
|
|
# iterate over each |
620
|
0
|
|
|
|
|
0
|
foreach my $path (@paths) { |
621
|
0
|
0
|
|
|
|
0
|
next unless defined $path; |
622
|
0
|
0
|
|
|
|
0
|
if ($path =~ m!^http://!) { |
623
|
|
|
|
|
|
|
# try via HTTP |
624
|
0
|
|
|
|
|
0
|
my $ua = new LWP::UserAgent; |
625
|
0
|
|
0
|
|
|
0
|
$ua->timeout($timeout || 10); |
626
|
|
|
|
|
|
|
|
627
|
0
|
|
|
|
|
0
|
my $res = $ua->get($path); |
628
|
0
|
0
|
|
|
|
0
|
if ($res->is_success) { |
629
|
0
|
|
|
|
|
0
|
my $contents = $res->content; |
630
|
0
|
|
|
|
|
0
|
return \$contents; |
631
|
|
|
|
|
|
|
} |
632
|
|
|
|
|
|
|
|
633
|
|
|
|
|
|
|
} else { |
634
|
|
|
|
|
|
|
# open the file from disk and just grab it all |
635
|
0
|
0
|
|
|
|
0
|
open FILE, "<$path" or next; |
636
|
0
|
|
|
|
|
0
|
my $contents; |
637
|
0
|
|
|
|
|
0
|
{ local $/ = undef; $contents = ; } |
|
0
|
|
|
|
|
0
|
|
|
0
|
|
|
|
|
0
|
|
638
|
0
|
|
|
|
|
0
|
close FILE; |
639
|
0
|
0
|
|
|
|
0
|
return \$contents if $contents; |
640
|
|
|
|
|
|
|
} |
641
|
|
|
|
|
|
|
} |
642
|
0
|
|
|
|
|
0
|
return undef; |
643
|
|
|
|
|
|
|
} |
644
|
|
|
|
|
|
|
|
645
|
|
|
|
|
|
|
=head2 delete |
646
|
|
|
|
|
|
|
|
647
|
|
|
|
|
|
|
$mogc->delete($key); |
648
|
|
|
|
|
|
|
|
649
|
|
|
|
|
|
|
Delete a key from MogileFS. |
650
|
|
|
|
|
|
|
|
651
|
|
|
|
|
|
|
=cut |
652
|
|
|
|
|
|
|
|
653
|
|
|
|
|
|
|
# this method returns undef only on a fatal error such as inability to actually |
654
|
|
|
|
|
|
|
# delete a resource and inability to contact the server. attempting to delete |
655
|
|
|
|
|
|
|
# something that doesn't exist counts as success, as it doesn't exist. |
656
|
|
|
|
|
|
|
sub delete { |
657
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
658
|
0
|
0
|
|
|
|
0
|
return undef if $self->{readonly}; |
659
|
|
|
|
|
|
|
|
660
|
0
|
|
|
|
|
0
|
my $key = shift; |
661
|
|
|
|
|
|
|
|
662
|
0
|
|
|
|
|
0
|
my $rv = $self->{backend}->do_request |
663
|
|
|
|
|
|
|
("delete", { |
664
|
|
|
|
|
|
|
domain => $self->{domain}, |
665
|
|
|
|
|
|
|
key => $key, |
666
|
|
|
|
|
|
|
}); |
667
|
|
|
|
|
|
|
|
668
|
|
|
|
|
|
|
# if it's unknown_key, not an error |
669
|
0
|
0
|
0
|
|
|
0
|
return undef unless defined $rv || |
670
|
|
|
|
|
|
|
$self->{backend}->{lasterr} eq 'unknown_key'; |
671
|
|
|
|
|
|
|
|
672
|
0
|
|
|
|
|
0
|
return 1; |
673
|
|
|
|
|
|
|
} |
674
|
|
|
|
|
|
|
|
675
|
|
|
|
|
|
|
=head2 rename |
676
|
|
|
|
|
|
|
|
677
|
|
|
|
|
|
|
$mogc->rename($oldkey, $newkey); |
678
|
|
|
|
|
|
|
|
679
|
|
|
|
|
|
|
Rename file (key) in MogileFS from oldkey to newkey. Returns true on |
680
|
|
|
|
|
|
|
success, failure otherwise. |
681
|
|
|
|
|
|
|
|
682
|
|
|
|
|
|
|
=cut |
683
|
|
|
|
|
|
|
|
684
|
|
|
|
|
|
|
# this method renames a file. it returns an undef on error (only a fatal error |
685
|
|
|
|
|
|
|
# is considered as undef; "file didn't exist" isn't an error). |
686
|
|
|
|
|
|
|
sub rename { |
687
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
688
|
0
|
0
|
|
|
|
0
|
return undef if $self->{readonly}; |
689
|
|
|
|
|
|
|
|
690
|
0
|
|
|
|
|
0
|
my ($fkey, $tkey) = @_; |
691
|
|
|
|
|
|
|
|
692
|
0
|
|
|
|
|
0
|
my $rv = $self->{backend}->do_request |
693
|
|
|
|
|
|
|
("rename", { |
694
|
|
|
|
|
|
|
domain => $self->{domain}, |
695
|
|
|
|
|
|
|
from_key => $fkey, |
696
|
|
|
|
|
|
|
to_key => $tkey, |
697
|
|
|
|
|
|
|
}); |
698
|
|
|
|
|
|
|
|
699
|
|
|
|
|
|
|
# if it's unknown_key, not an error |
700
|
0
|
0
|
0
|
|
|
0
|
return undef unless defined $rv || |
701
|
|
|
|
|
|
|
$self->{backend}->{lasterr} eq 'unknown_key'; |
702
|
|
|
|
|
|
|
|
703
|
0
|
|
|
|
|
0
|
return 1; |
704
|
|
|
|
|
|
|
} |
705
|
|
|
|
|
|
|
|
706
|
|
|
|
|
|
|
=head2 file_debug |
707
|
|
|
|
|
|
|
|
708
|
|
|
|
|
|
|
my $info_gob = $mogc->file_debug(fid => $fid); |
709
|
|
|
|
|
|
|
... or ... |
710
|
|
|
|
|
|
|
my $info_gob = $mogc->file_debug(key => $key); |
711
|
|
|
|
|
|
|
|
712
|
|
|
|
|
|
|
Thoroughly search for any database notes about a particular fid. Searchable by |
713
|
|
|
|
|
|
|
raw fidid, or by domain and key. B |
714
|
|
|
|
|
|
|
database numerous times, and if you're using it in production something is |
715
|
|
|
|
|
|
|
likely very wrong. |
716
|
|
|
|
|
|
|
|
717
|
|
|
|
|
|
|
To be used with troubleshooting broken/odd files and errors from mogilefsd. |
718
|
|
|
|
|
|
|
|
719
|
|
|
|
|
|
|
=cut |
720
|
|
|
|
|
|
|
|
721
|
|
|
|
|
|
|
sub file_debug { |
722
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
723
|
0
|
|
|
|
|
0
|
my %opts = @_; |
724
|
0
|
0
|
|
|
|
0
|
$opts{domain} = $self->{domain} unless exists $opts{domain}; |
725
|
|
|
|
|
|
|
|
726
|
0
|
0
|
|
|
|
0
|
my $res = $self->{backend}->do_request |
727
|
|
|
|
|
|
|
("file_debug", { |
728
|
|
|
|
|
|
|
%opts, |
729
|
|
|
|
|
|
|
}) or return undef; |
730
|
0
|
|
|
|
|
0
|
return $res; |
731
|
|
|
|
|
|
|
} |
732
|
|
|
|
|
|
|
|
733
|
|
|
|
|
|
|
=head2 file_info |
734
|
|
|
|
|
|
|
|
735
|
|
|
|
|
|
|
my $fid = $mogc->file_info($key, { devices => 0 }); |
736
|
|
|
|
|
|
|
|
737
|
|
|
|
|
|
|
Used to return metadata about a file. Returns the domain, class, expected |
738
|
|
|
|
|
|
|
length, devcount, etc. Optionally device ids (not paths) can be returned as |
739
|
|
|
|
|
|
|
well. |
740
|
|
|
|
|
|
|
|
741
|
|
|
|
|
|
|
Should be used for informational purposes, and not usually for dynamically |
742
|
|
|
|
|
|
|
serving files. |
743
|
|
|
|
|
|
|
|
744
|
|
|
|
|
|
|
=cut |
745
|
|
|
|
|
|
|
|
746
|
|
|
|
|
|
|
sub file_info { |
747
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
748
|
0
|
|
|
|
|
0
|
my ($key, $opts) = @_; |
749
|
|
|
|
|
|
|
|
750
|
0
|
|
|
|
|
0
|
my %extra = (); |
751
|
0
|
|
|
|
|
0
|
$extra{devices} = delete $opts->{devices}; |
752
|
0
|
0
|
|
|
|
0
|
die "Unknown arguments: " . join(', ', keys %$opts) if keys %$opts; |
753
|
|
|
|
|
|
|
|
754
|
0
|
0
|
|
|
|
0
|
my $res = $self->{backend}->do_request |
755
|
|
|
|
|
|
|
("file_info", { |
756
|
|
|
|
|
|
|
domain => $self->{domain}, |
757
|
|
|
|
|
|
|
key => $key, |
758
|
|
|
|
|
|
|
%extra, |
759
|
|
|
|
|
|
|
}) or return undef; |
760
|
0
|
|
|
|
|
0
|
return $res; |
761
|
|
|
|
|
|
|
} |
762
|
|
|
|
|
|
|
|
763
|
|
|
|
|
|
|
=head2 list_keys |
764
|
|
|
|
|
|
|
|
765
|
|
|
|
|
|
|
$keys = $mogc->list_keys($prefix, $after[, $limit]); |
766
|
|
|
|
|
|
|
($after, $keys) = $mogc->list_keys($prefix, $after[, $limit]); |
767
|
|
|
|
|
|
|
|
768
|
|
|
|
|
|
|
Used to get a list of keys matching a certain prefix. |
769
|
|
|
|
|
|
|
|
770
|
|
|
|
|
|
|
$prefix specifies what you want to get a list of. $after is the item |
771
|
|
|
|
|
|
|
specified as a return value from this function last time you called |
772
|
|
|
|
|
|
|
it. $limit is optional and defaults to 1000 keys returned. |
773
|
|
|
|
|
|
|
|
774
|
|
|
|
|
|
|
In list context, returns ($after, $keys). In scalar context, returns |
775
|
|
|
|
|
|
|
arrayref of keys. The value $after is to be used as $after when you |
776
|
|
|
|
|
|
|
call this function again. |
777
|
|
|
|
|
|
|
|
778
|
|
|
|
|
|
|
When there are no more keys in the list, you will get back undef or |
779
|
|
|
|
|
|
|
an empty list. |
780
|
|
|
|
|
|
|
|
781
|
|
|
|
|
|
|
=cut |
782
|
|
|
|
|
|
|
|
783
|
|
|
|
|
|
|
sub list_keys { |
784
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
785
|
0
|
|
|
|
|
0
|
my ($prefix, $after, $limit) = @_; |
786
|
|
|
|
|
|
|
|
787
|
0
|
0
|
|
|
|
0
|
my $res = $self->{backend}->do_request |
788
|
|
|
|
|
|
|
("list_keys", { |
789
|
|
|
|
|
|
|
domain => $self->{domain}, |
790
|
|
|
|
|
|
|
prefix => $prefix, |
791
|
|
|
|
|
|
|
after => $after, |
792
|
|
|
|
|
|
|
limit => $limit, |
793
|
|
|
|
|
|
|
}) or return undef; |
794
|
|
|
|
|
|
|
|
795
|
|
|
|
|
|
|
# construct our list of keys and the new after value |
796
|
0
|
|
|
|
|
0
|
my $resafter = $res->{next_after}; |
797
|
0
|
|
|
|
|
0
|
my $reslist = []; |
798
|
0
|
|
|
|
|
0
|
for (my $i = 1; $i <= $res->{key_count}+0; $i++) { |
799
|
0
|
|
|
|
|
0
|
push @$reslist, $res->{"key_$i"}; |
800
|
|
|
|
|
|
|
} |
801
|
0
|
0
|
|
|
|
0
|
return wantarray ? ($resafter, $reslist) : $reslist; |
802
|
|
|
|
|
|
|
} |
803
|
|
|
|
|
|
|
|
804
|
|
|
|
|
|
|
=head2 foreach_key |
805
|
|
|
|
|
|
|
|
806
|
|
|
|
|
|
|
$mogc->foreach_key( %OPTIONS, sub { my $key = shift; ... } ); |
807
|
|
|
|
|
|
|
$mogc->foreach_key( prefix => "foo:", sub { my $key = shift; ... } ); |
808
|
|
|
|
|
|
|
|
809
|
|
|
|
|
|
|
|
810
|
|
|
|
|
|
|
Functional interface/wrapper around list_keys. |
811
|
|
|
|
|
|
|
|
812
|
|
|
|
|
|
|
Given some %OPTIONS (currently only one, "prefix"), calls your callback |
813
|
|
|
|
|
|
|
for each key matching the provided prefix. |
814
|
|
|
|
|
|
|
|
815
|
|
|
|
|
|
|
=cut |
816
|
|
|
|
|
|
|
|
817
|
|
|
|
|
|
|
sub foreach_key { |
818
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
819
|
0
|
|
|
|
|
0
|
my $callback = pop; |
820
|
0
|
0
|
|
|
|
0
|
Carp::croak("Last parameter not a subref") unless ref $callback eq "CODE"; |
821
|
0
|
|
|
|
|
0
|
my %opts = @_; |
822
|
0
|
|
|
|
|
0
|
my $prefix = delete $opts{prefix}; |
823
|
0
|
0
|
|
|
|
0
|
Carp::croak("Unknown option(s): " . join(", ", keys %opts)) if %opts; |
824
|
|
|
|
|
|
|
|
825
|
0
|
|
|
|
|
0
|
my $last = ""; |
826
|
0
|
|
|
|
|
0
|
my $max = 1000; |
827
|
0
|
|
|
|
|
0
|
my $count = $max; |
828
|
|
|
|
|
|
|
|
829
|
0
|
|
|
|
|
0
|
while ($count == $max) { |
830
|
0
|
0
|
|
|
|
0
|
my $res = $self->{backend}->do_request |
831
|
|
|
|
|
|
|
("list_keys", { |
832
|
|
|
|
|
|
|
domain => $self->{domain}, |
833
|
|
|
|
|
|
|
prefix => $prefix, |
834
|
|
|
|
|
|
|
after => $last, |
835
|
|
|
|
|
|
|
limit => $max, |
836
|
|
|
|
|
|
|
}) or return undef; |
837
|
0
|
|
|
|
|
0
|
$count = $res->{key_count}+0; |
838
|
0
|
|
|
|
|
0
|
for (my $i = 1; $i <= $count; $i++) { |
839
|
0
|
|
|
|
|
0
|
$callback->($res->{"key_$i"}); |
840
|
|
|
|
|
|
|
} |
841
|
0
|
|
|
|
|
0
|
$last = $res->{"key_$count"}; |
842
|
|
|
|
|
|
|
} |
843
|
0
|
|
|
|
|
0
|
return 1; |
844
|
|
|
|
|
|
|
} |
845
|
|
|
|
|
|
|
|
846
|
|
|
|
|
|
|
# just makes some sleeping happen. first and only argument is number of |
847
|
|
|
|
|
|
|
# seconds to instruct backend thread to sleep for. |
848
|
|
|
|
|
|
|
sub sleep { |
849
|
0
|
|
|
0
|
0
|
0
|
my MogileFS::Client $self = shift; |
850
|
0
|
|
|
|
|
0
|
my $duration = shift; |
851
|
|
|
|
|
|
|
|
852
|
0
|
0
|
|
|
|
0
|
$self->{backend}->do_request("sleep", { duration => $duration + 0 }) |
853
|
|
|
|
|
|
|
or return undef; |
854
|
|
|
|
|
|
|
|
855
|
0
|
|
|
|
|
0
|
return 1; |
856
|
|
|
|
|
|
|
} |
857
|
|
|
|
|
|
|
|
858
|
|
|
|
|
|
|
=head2 update_class |
859
|
|
|
|
|
|
|
|
860
|
|
|
|
|
|
|
$mogc->update_class($key, $newclass); |
861
|
|
|
|
|
|
|
|
862
|
|
|
|
|
|
|
Update the replication class of a pre-existing file, causing |
863
|
|
|
|
|
|
|
the file to become more or less replicated. |
864
|
|
|
|
|
|
|
|
865
|
|
|
|
|
|
|
=cut |
866
|
|
|
|
|
|
|
|
867
|
|
|
|
|
|
|
sub update_class { |
868
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
869
|
0
|
|
|
|
|
0
|
my ($key, $class) = @_; |
870
|
0
|
0
|
|
|
|
0
|
my $res = $self->{backend}->do_request |
871
|
|
|
|
|
|
|
("updateclass", { |
872
|
|
|
|
|
|
|
domain => $self->{domain}, |
873
|
|
|
|
|
|
|
key => $key, |
874
|
|
|
|
|
|
|
class => $class, |
875
|
|
|
|
|
|
|
}) or return undef; |
876
|
0
|
|
|
|
|
0
|
return $res; |
877
|
|
|
|
|
|
|
} |
878
|
|
|
|
|
|
|
|
879
|
|
|
|
|
|
|
=head2 set_pref_ip |
880
|
|
|
|
|
|
|
|
881
|
|
|
|
|
|
|
$mogc->set_pref_ip({ "10.0.0.2" => "10.2.0.2" }); |
882
|
|
|
|
|
|
|
|
883
|
|
|
|
|
|
|
Weird option for old, weird network architecture. Sets a mapping |
884
|
|
|
|
|
|
|
table of preferred alternate IPs, if reachable. For instance, if |
885
|
|
|
|
|
|
|
trying to connect to 10.0.0.2 in the above example, the module would |
886
|
|
|
|
|
|
|
instead try to connect to 10.2.0.2 quickly first, then then fall back |
887
|
|
|
|
|
|
|
to 10.0.0.2 if 10.2.0.2 wasn't reachable. |
888
|
|
|
|
|
|
|
|
889
|
|
|
|
|
|
|
=cut |
890
|
|
|
|
|
|
|
|
891
|
|
|
|
|
|
|
# expects as argument a hashref of "standard-ip" => "preferred-ip" |
892
|
|
|
|
|
|
|
sub set_pref_ip { |
893
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
894
|
0
|
0
|
|
|
|
0
|
$self->{backend}->set_pref_ip(shift) |
895
|
|
|
|
|
|
|
if $self->{backend}; |
896
|
|
|
|
|
|
|
} |
897
|
|
|
|
|
|
|
|
898
|
|
|
|
|
|
|
=head1 PLUGIN METHODS |
899
|
|
|
|
|
|
|
|
900
|
|
|
|
|
|
|
WRITEME |
901
|
|
|
|
|
|
|
|
902
|
|
|
|
|
|
|
=cut |
903
|
|
|
|
|
|
|
|
904
|
|
|
|
|
|
|
# used to support plugins that have modified the server, this builds things into |
905
|
|
|
|
|
|
|
# an argument list and passes them back to the server |
906
|
|
|
|
|
|
|
# TODO: there is no readonly protection here? does it matter? should we check |
907
|
|
|
|
|
|
|
# with the server to see what methods they support? (and if they should be disallowed |
908
|
|
|
|
|
|
|
# when the client is in readonly mode?) |
909
|
|
|
|
|
|
|
sub AUTOLOAD { |
910
|
|
|
|
|
|
|
# remove everything up to the last colon, so we only have the method name left |
911
|
0
|
|
|
0
|
|
0
|
my $method = $AUTOLOAD; |
912
|
0
|
|
|
|
|
0
|
$method =~ s/^.*://; |
913
|
|
|
|
|
|
|
|
914
|
0
|
0
|
|
|
|
0
|
return if $method eq 'DESTROY'; |
915
|
|
|
|
|
|
|
|
916
|
|
|
|
|
|
|
# let this work |
917
|
4
|
|
|
4
|
|
44
|
no strict 'refs'; |
|
4
|
|
|
|
|
9
|
|
|
4
|
|
|
|
|
2070
|
|
918
|
|
|
|
|
|
|
|
919
|
|
|
|
|
|
|
# create a method to pass this on back |
920
|
0
|
|
|
|
|
0
|
*{$AUTOLOAD} = sub { |
921
|
0
|
|
|
0
|
|
0
|
my MogileFS::Client $self = shift; |
922
|
|
|
|
|
|
|
# pre-assemble the arguments into a hashref |
923
|
0
|
|
|
|
|
0
|
my $ct = 0; |
924
|
0
|
|
|
|
|
0
|
my $args = {}; |
925
|
0
|
|
|
|
|
0
|
$args->{"arg" . ++$ct} = shift() while @_; |
926
|
0
|
|
|
|
|
0
|
$args->{"argcount"} = $ct; |
927
|
|
|
|
|
|
|
|
928
|
|
|
|
|
|
|
# now put in standard args |
929
|
0
|
|
|
|
|
0
|
$args->{"domain"} = $self->{domain}; |
930
|
|
|
|
|
|
|
|
931
|
|
|
|
|
|
|
# now call and return whatever we get back from the backend |
932
|
0
|
|
|
|
|
0
|
return $self->{backend}->do_request("plugin_$method", $args); |
933
|
0
|
|
|
|
|
0
|
}; |
934
|
|
|
|
|
|
|
|
935
|
|
|
|
|
|
|
# now pass through |
936
|
0
|
|
|
|
|
0
|
goto &$AUTOLOAD; |
937
|
|
|
|
|
|
|
} |
938
|
|
|
|
|
|
|
|
939
|
|
|
|
|
|
|
=head1 HOOKS |
940
|
|
|
|
|
|
|
|
941
|
|
|
|
|
|
|
=head2 add_hook |
942
|
|
|
|
|
|
|
|
943
|
|
|
|
|
|
|
WRITEME |
944
|
|
|
|
|
|
|
|
945
|
|
|
|
|
|
|
=cut |
946
|
|
|
|
|
|
|
|
947
|
|
|
|
|
|
|
sub add_hook { |
948
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
949
|
0
|
|
0
|
|
|
0
|
my $hookname = shift || return; |
950
|
|
|
|
|
|
|
|
951
|
0
|
0
|
|
|
|
0
|
if (@_) { |
952
|
0
|
|
|
|
|
0
|
$self->{hooks}->{$hookname} = shift; |
953
|
|
|
|
|
|
|
} else { |
954
|
0
|
|
|
|
|
0
|
delete $self->{hooks}->{$hookname}; |
955
|
|
|
|
|
|
|
} |
956
|
|
|
|
|
|
|
} |
957
|
|
|
|
|
|
|
|
958
|
|
|
|
|
|
|
sub run_hook { |
959
|
0
|
|
|
0
|
0
|
0
|
my MogileFS::Client $self = shift; |
960
|
0
|
|
0
|
|
|
0
|
my $hookname = shift || return; |
961
|
|
|
|
|
|
|
|
962
|
0
|
|
|
|
|
0
|
my $hook = $self->{hooks}->{$hookname}; |
963
|
0
|
0
|
|
|
|
0
|
return unless $hook; |
964
|
|
|
|
|
|
|
|
965
|
0
|
|
|
|
|
0
|
eval { $hook->(@_) }; |
|
0
|
|
|
|
|
0
|
|
966
|
|
|
|
|
|
|
|
967
|
0
|
0
|
|
|
|
0
|
warn "MogileFS::Client hook '$hookname' threw error: $@\n" if $@; |
968
|
|
|
|
|
|
|
} |
969
|
|
|
|
|
|
|
|
970
|
|
|
|
|
|
|
=head2 add_backend_hook |
971
|
|
|
|
|
|
|
|
972
|
|
|
|
|
|
|
WRITEME |
973
|
|
|
|
|
|
|
|
974
|
|
|
|
|
|
|
=cut |
975
|
|
|
|
|
|
|
|
976
|
|
|
|
|
|
|
sub add_backend_hook { |
977
|
0
|
|
|
0
|
1
|
0
|
my MogileFS::Client $self = shift; |
978
|
0
|
|
|
|
|
0
|
my $backend = $self->{backend}; |
979
|
|
|
|
|
|
|
|
980
|
0
|
|
|
|
|
0
|
$backend->add_hook(@_); |
981
|
|
|
|
|
|
|
} |
982
|
|
|
|
|
|
|
|
983
|
|
|
|
|
|
|
|
984
|
|
|
|
|
|
|
################################################################################ |
985
|
|
|
|
|
|
|
# MogileFS class methods |
986
|
|
|
|
|
|
|
# |
987
|
|
|
|
|
|
|
|
988
|
|
|
|
|
|
|
sub _fail { |
989
|
0
|
|
|
0
|
|
0
|
croak "MogileFS: $_[0]"; |
990
|
|
|
|
|
|
|
} |
991
|
|
|
|
|
|
|
|
992
|
|
|
|
|
|
|
sub _debug { |
993
|
1
|
50
|
|
1
|
|
15
|
return 1 unless $MogileFS::DEBUG; |
994
|
|
|
|
|
|
|
|
995
|
0
|
|
|
|
|
|
my $msg = shift; |
996
|
0
|
|
|
|
|
|
my $ref = shift; |
997
|
0
|
|
|
|
|
|
chomp $msg; |
998
|
|
|
|
|
|
|
|
999
|
0
|
|
|
|
|
|
eval "use Data::Dumper;"; |
1000
|
0
|
|
|
|
|
|
print STDERR "$msg\n" . Dumper($ref) . "\n"; |
1001
|
0
|
|
|
|
|
|
return 1; |
1002
|
|
|
|
|
|
|
} |
1003
|
|
|
|
|
|
|
|
1004
|
|
|
|
|
|
|
|
1005
|
|
|
|
|
|
|
1; |
1006
|
|
|
|
|
|
|
__END__ |