line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
1
|
|
|
1
|
|
780811
|
use 5.006; |
|
1
|
|
|
|
|
3
|
|
2
|
1
|
|
|
1
|
|
3
|
use strict; |
|
1
|
|
|
|
|
1
|
|
|
1
|
|
|
|
|
16
|
|
3
|
1
|
|
|
1
|
|
3
|
use warnings; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
53
|
|
4
|
|
|
|
|
|
|
|
5
|
|
|
|
|
|
|
package Metabase::Archive::SQLite::Sharded; |
6
|
|
|
|
|
|
|
# ABSTRACT: Metabase storage using multiple SQLite databases |
7
|
|
|
|
|
|
|
|
8
|
|
|
|
|
|
|
our $VERSION = '1.001'; |
9
|
|
|
|
|
|
|
|
10
|
1
|
|
|
1
|
|
4
|
use Moose; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
9
|
|
11
|
1
|
|
|
1
|
|
5095
|
use Data::Stream::Bulk::Callback; |
|
1
|
|
|
|
|
165129
|
|
|
1
|
|
|
|
|
53
|
|
12
|
1
|
|
|
1
|
|
10
|
use Moose::Util::TypeConstraints; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
13
|
|
13
|
1
|
|
|
1
|
|
2831
|
use Metabase::Archive::SQLite; |
|
1
|
|
|
|
|
4
|
|
|
1
|
|
|
|
|
742
|
|
14
|
|
|
|
|
|
|
|
15
|
|
|
|
|
|
|
with 'Metabase::Backend::SQLite'; |
16
|
|
|
|
|
|
|
with 'Metabase::Archive'; |
17
|
|
|
|
|
|
|
|
18
|
|
|
|
|
|
|
subtype 'ShardSize', |
19
|
|
|
|
|
|
|
as 'Int', |
20
|
|
|
|
|
|
|
where { $_ > 0 && $_ < 8 }, |
21
|
|
|
|
|
|
|
message { "The number you provided, $_, was not between 1 and 8" }; |
22
|
|
|
|
|
|
|
|
23
|
|
|
|
|
|
|
has shard_digits => ( |
24
|
|
|
|
|
|
|
is => 'ro', |
25
|
|
|
|
|
|
|
isa => 'ShardSize', |
26
|
|
|
|
|
|
|
default => 2, |
27
|
|
|
|
|
|
|
); |
28
|
|
|
|
|
|
|
|
29
|
|
|
|
|
|
|
has _shards => ( |
30
|
|
|
|
|
|
|
is => 'ro', |
31
|
|
|
|
|
|
|
traits => ['Hash'], |
32
|
|
|
|
|
|
|
isa => 'HashRef[Object]', |
33
|
|
|
|
|
|
|
default => sub { return {} }, |
34
|
|
|
|
|
|
|
handles => { |
35
|
|
|
|
|
|
|
'_get_shard' => 'get', |
36
|
|
|
|
|
|
|
'_set_shard' => 'set', |
37
|
|
|
|
|
|
|
'_all_shards' => 'values', |
38
|
|
|
|
|
|
|
}, |
39
|
|
|
|
|
|
|
); |
40
|
|
|
|
|
|
|
|
41
|
|
|
|
3
|
0
|
|
sub initialize { } |
42
|
|
|
|
|
|
|
|
43
|
|
|
|
|
|
|
sub _create_shard { |
44
|
5
|
|
|
5
|
|
12
|
my ($self, $key) = @_; |
45
|
|
|
|
|
|
|
|
46
|
5
|
|
|
|
|
211
|
my $filename = $self->filename; |
47
|
5
|
|
|
|
|
192
|
my ($basename, $ext) = $self->filename =~ m{^(.*)\.([^.]+)$}; |
48
|
5
|
50
|
33
|
|
|
293
|
if ( $basename && $ext ) { |
49
|
0
|
|
|
|
|
0
|
$filename = "$basename\_$key.$ext"; |
50
|
|
|
|
|
|
|
} |
51
|
|
|
|
|
|
|
else { |
52
|
5
|
|
|
|
|
20
|
$filename .= "_$key"; |
53
|
|
|
|
|
|
|
} |
54
|
5
|
50
|
|
|
|
365
|
my %args = ( |
|
|
50
|
|
|
|
|
|
55
|
|
|
|
|
|
|
filename => $filename, |
56
|
|
|
|
|
|
|
($self->has_page_size ? ( page_size => $self->page_size ) : ()), |
57
|
|
|
|
|
|
|
($self->has_cache_size ? ( cache_size => $self->cache_size ) : ()), |
58
|
|
|
|
|
|
|
); |
59
|
5
|
|
|
|
|
63
|
my $archive = Metabase::Archive::SQLite->new( %args ); |
60
|
5
|
|
|
|
|
17326
|
$archive->initialize; |
61
|
5
|
|
|
|
|
296
|
$self->_set_shard($key, $archive); |
62
|
5
|
|
|
|
|
43
|
return $archive; |
63
|
|
|
|
|
|
|
} |
64
|
|
|
|
|
|
|
|
65
|
|
|
|
|
|
|
sub _shard_key { |
66
|
7
|
|
|
7
|
|
16
|
my ($self, $guid) = @_; |
67
|
7
|
|
|
|
|
330
|
return substr $guid, (7-$self->shard_digits), $self->shard_digits; |
68
|
|
|
|
|
|
|
} |
69
|
|
|
|
|
|
|
|
70
|
|
|
|
|
|
|
sub store { |
71
|
5
|
|
|
5
|
0
|
3919
|
my ( $self, $fact_struct ) = @_; |
72
|
5
|
|
|
|
|
29
|
my $key = $self->_shard_key( $fact_struct->{metadata}{core}{guid} ); |
73
|
5
|
|
33
|
|
|
257
|
my $archive = $self->_get_shard($key) || $self->_create_shard($key); |
74
|
5
|
|
|
|
|
33
|
return $archive->store($fact_struct); |
75
|
|
|
|
|
|
|
} |
76
|
|
|
|
|
|
|
|
77
|
|
|
|
|
|
|
|
78
|
|
|
|
|
|
|
sub extract { |
79
|
1
|
|
|
1
|
0
|
1480
|
my ( $self, $guid ) = @_; |
80
|
1
|
|
|
|
|
7
|
my $key = $self->_shard_key( $guid ); |
81
|
1
|
|
33
|
|
|
62
|
my $archive = $self->_get_shard($key) || $self->_create_shard($key); |
82
|
1
|
|
|
|
|
8
|
return $archive->extract($guid); |
83
|
|
|
|
|
|
|
} |
84
|
|
|
|
|
|
|
|
85
|
|
|
|
|
|
|
sub delete { |
86
|
1
|
|
|
1
|
0
|
962
|
my ( $self, $guid ) = @_; |
87
|
1
|
|
|
|
|
5
|
my $key = $self->_shard_key( $guid ); |
88
|
1
|
|
33
|
|
|
52
|
my $archive = $self->_get_shard($key) || $self->_create_shard($key); |
89
|
1
|
|
|
|
|
8
|
return $archive->delete($guid); |
90
|
|
|
|
|
|
|
} |
91
|
|
|
|
|
|
|
|
92
|
|
|
|
|
|
|
sub iterator { |
93
|
2
|
|
|
2
|
0
|
1498
|
my ($self) = @_; |
94
|
|
|
|
|
|
|
|
95
|
2
|
|
|
|
|
105
|
my @iters = map { $_->iterator } $self->_all_shards; |
|
4
|
|
|
|
|
1061
|
|
96
|
|
|
|
|
|
|
|
97
|
|
|
|
|
|
|
return Data::Stream::Bulk::Callback->new( |
98
|
|
|
|
|
|
|
callback => sub { |
99
|
6
|
50
|
|
6
|
|
708
|
if ( @iters ) { |
100
|
6
|
|
|
|
|
13
|
my $s = shift @iters; |
101
|
6
|
100
|
|
|
|
27
|
if ( my $next = $s->next ) { |
102
|
4
|
|
|
|
|
39
|
push @iters, $s; # round-robin |
103
|
4
|
|
|
|
|
16
|
return $next; |
104
|
|
|
|
|
|
|
} |
105
|
|
|
|
|
|
|
} else { |
106
|
0
|
|
|
|
|
|
return; # done |
107
|
|
|
|
|
|
|
} |
108
|
|
|
|
|
|
|
}, |
109
|
2
|
|
|
|
|
629
|
); |
110
|
|
|
|
|
|
|
} |
111
|
|
|
|
|
|
|
|
112
|
|
|
|
|
|
|
1; |
113
|
|
|
|
|
|
|
|
114
|
|
|
|
|
|
|
__END__ |
115
|
|
|
|
|
|
|
|
116
|
|
|
|
|
|
|
=pod |
117
|
|
|
|
|
|
|
|
118
|
|
|
|
|
|
|
=encoding UTF-8 |
119
|
|
|
|
|
|
|
|
120
|
|
|
|
|
|
|
=head1 NAME |
121
|
|
|
|
|
|
|
|
122
|
|
|
|
|
|
|
Metabase::Archive::SQLite::Sharded - Metabase storage using multiple SQLite databases |
123
|
|
|
|
|
|
|
|
124
|
|
|
|
|
|
|
=head1 VERSION |
125
|
|
|
|
|
|
|
|
126
|
|
|
|
|
|
|
version 1.001 |
127
|
|
|
|
|
|
|
|
128
|
|
|
|
|
|
|
=head1 SYNOPSIS |
129
|
|
|
|
|
|
|
|
130
|
|
|
|
|
|
|
use Metabase::Archive::SQLite::Sharded; |
131
|
|
|
|
|
|
|
|
132
|
|
|
|
|
|
|
my $archive = Metabase::Archive::SQLite::Sharded->new( |
133
|
|
|
|
|
|
|
filename => $sqlite_file, |
134
|
|
|
|
|
|
|
shard_digits => 2, |
135
|
|
|
|
|
|
|
); |
136
|
|
|
|
|
|
|
|
137
|
|
|
|
|
|
|
=head1 DESCRIPTION |
138
|
|
|
|
|
|
|
|
139
|
|
|
|
|
|
|
This is an implementation of the L<Metabase::Archive::SQL> role using SQLite |
140
|
|
|
|
|
|
|
shards. |
141
|
|
|
|
|
|
|
|
142
|
|
|
|
|
|
|
SQLite stores a database entirely in a single file. That starts to become |
143
|
|
|
|
|
|
|
slow as the size of the file gets large. This Metabase::Archive shards |
144
|
|
|
|
|
|
|
facts across multiple SQLite files. |
145
|
|
|
|
|
|
|
|
146
|
|
|
|
|
|
|
It takes the same options as L<Metabase::Archive::SQLite>, with one additional |
147
|
|
|
|
|
|
|
option, C<shard_digits>. The C<shard_digits> attribute defines how many digits |
148
|
|
|
|
|
|
|
of the GUID to use as a shard key. Each digit is a hexadecimal number, so |
149
|
|
|
|
|
|
|
digits increase the number of shards as a power of 16. E.g., "1" means 16 |
150
|
|
|
|
|
|
|
shards, "2" means 256 shards and so on. |
151
|
|
|
|
|
|
|
|
152
|
|
|
|
|
|
|
The shard key is inserted to the database C<filename> parameter either before |
153
|
|
|
|
|
|
|
the final period or at the end. E.g. for C<shard_digits> of "2" and |
154
|
|
|
|
|
|
|
C<filename> "db.sqlite3", the shards would be "db_00.slite3", "db_01.sqlite3", |
155
|
|
|
|
|
|
|
and so on. |
156
|
|
|
|
|
|
|
|
157
|
|
|
|
|
|
|
=for Pod::Coverage::TrustPod store extract delete iterator initialize |
158
|
|
|
|
|
|
|
|
159
|
|
|
|
|
|
|
=head1 AUTHORS |
160
|
|
|
|
|
|
|
|
161
|
|
|
|
|
|
|
=over 4 |
162
|
|
|
|
|
|
|
|
163
|
|
|
|
|
|
|
=item * |
164
|
|
|
|
|
|
|
|
165
|
|
|
|
|
|
|
David Golden <dagolden@cpan.org> |
166
|
|
|
|
|
|
|
|
167
|
|
|
|
|
|
|
=item * |
168
|
|
|
|
|
|
|
|
169
|
|
|
|
|
|
|
Leon Brocard <acme@astray.org> |
170
|
|
|
|
|
|
|
|
171
|
|
|
|
|
|
|
=back |
172
|
|
|
|
|
|
|
|
173
|
|
|
|
|
|
|
=head1 COPYRIGHT AND LICENSE |
174
|
|
|
|
|
|
|
|
175
|
|
|
|
|
|
|
This software is Copyright (c) 2011 by David Golden. |
176
|
|
|
|
|
|
|
|
177
|
|
|
|
|
|
|
This is free software, licensed under: |
178
|
|
|
|
|
|
|
|
179
|
|
|
|
|
|
|
The Apache License, Version 2.0, January 2004 |
180
|
|
|
|
|
|
|
|
181
|
|
|
|
|
|
|
=cut |