File Coverage

blib/lib/Metabase/Archive/SQLite/Sharded.pm
Criterion Covered Total %
statement 53 55 96.3
branch 6 10 60.0
condition 4 12 33.3
subroutine 15 15 100.0
pod 0 5 0.0
total 78 97 80.4


line stmt bran cond sub pod time code
1 1     1   780811 use 5.006;
  1         3  
2 1     1   3 use strict;
  1         1  
  1         16  
3 1     1   3 use warnings;
  1         2  
  1         53  
4              
5             package Metabase::Archive::SQLite::Sharded;
6             # ABSTRACT: Metabase storage using multiple SQLite databases
7              
8             our $VERSION = '1.001';
9              
10 1     1   4 use Moose;
  1         2  
  1         9  
11 1     1   5095 use Data::Stream::Bulk::Callback;
  1         165129  
  1         53  
12 1     1   10 use Moose::Util::TypeConstraints;
  1         2  
  1         13  
13 1     1   2831 use Metabase::Archive::SQLite;
  1         4  
  1         742  
14              
15             with 'Metabase::Backend::SQLite';
16             with 'Metabase::Archive';
17              
18             subtype 'ShardSize',
19             as 'Int',
20             where { $_ > 0 && $_ < 8 },
21             message { "The number you provided, $_, was not between 1 and 8" };
22              
23             has shard_digits => (
24             is => 'ro',
25             isa => 'ShardSize',
26             default => 2,
27             );
28              
29             has _shards => (
30             is => 'ro',
31             traits => ['Hash'],
32             isa => 'HashRef[Object]',
33             default => sub { return {} },
34             handles => {
35             '_get_shard' => 'get',
36             '_set_shard' => 'set',
37             '_all_shards' => 'values',
38             },
39             );
40              
41       3 0   sub initialize { }
42              
43             sub _create_shard {
44 5     5   12 my ($self, $key) = @_;
45              
46 5         211 my $filename = $self->filename;
47 5         192 my ($basename, $ext) = $self->filename =~ m{^(.*)\.([^.]+)$};
48 5 50 33     293 if ( $basename && $ext ) {
49 0         0 $filename = "$basename\_$key.$ext";
50             }
51             else {
52 5         20 $filename .= "_$key";
53             }
54 5 50       365 my %args = (
    50          
55             filename => $filename,
56             ($self->has_page_size ? ( page_size => $self->page_size ) : ()),
57             ($self->has_cache_size ? ( cache_size => $self->cache_size ) : ()),
58             );
59 5         63 my $archive = Metabase::Archive::SQLite->new( %args );
60 5         17326 $archive->initialize;
61 5         296 $self->_set_shard($key, $archive);
62 5         43 return $archive;
63             }
64              
65             sub _shard_key {
66 7     7   16 my ($self, $guid) = @_;
67 7         330 return substr $guid, (7-$self->shard_digits), $self->shard_digits;
68             }
69              
70             sub store {
71 5     5 0 3919 my ( $self, $fact_struct ) = @_;
72 5         29 my $key = $self->_shard_key( $fact_struct->{metadata}{core}{guid} );
73 5   33     257 my $archive = $self->_get_shard($key) || $self->_create_shard($key);
74 5         33 return $archive->store($fact_struct);
75             }
76              
77              
78             sub extract {
79 1     1 0 1480 my ( $self, $guid ) = @_;
80 1         7 my $key = $self->_shard_key( $guid );
81 1   33     62 my $archive = $self->_get_shard($key) || $self->_create_shard($key);
82 1         8 return $archive->extract($guid);
83             }
84              
85             sub delete {
86 1     1 0 962 my ( $self, $guid ) = @_;
87 1         5 my $key = $self->_shard_key( $guid );
88 1   33     52 my $archive = $self->_get_shard($key) || $self->_create_shard($key);
89 1         8 return $archive->delete($guid);
90             }
91              
92             sub iterator {
93 2     2 0 1498 my ($self) = @_;
94              
95 2         105 my @iters = map { $_->iterator } $self->_all_shards;
  4         1061  
96              
97             return Data::Stream::Bulk::Callback->new(
98             callback => sub {
99 6 50   6   708 if ( @iters ) {
100 6         13 my $s = shift @iters;
101 6 100       27 if ( my $next = $s->next ) {
102 4         39 push @iters, $s; # round-robin
103 4         16 return $next;
104             }
105             } else {
106 0           return; # done
107             }
108             },
109 2         629 );
110             }
111              
112             1;
113              
114             __END__
115              
116             =pod
117              
118             =encoding UTF-8
119              
120             =head1 NAME
121              
122             Metabase::Archive::SQLite::Sharded - Metabase storage using multiple SQLite databases
123              
124             =head1 VERSION
125              
126             version 1.001
127              
128             =head1 SYNOPSIS
129              
130             use Metabase::Archive::SQLite::Sharded;
131              
132             my $archive = Metabase::Archive::SQLite::Sharded->new(
133             filename => $sqlite_file,
134             shard_digits => 2,
135             );
136              
137             =head1 DESCRIPTION
138              
139             This is an implementation of the L<Metabase::Archive::SQL> role using SQLite
140             shards.
141              
142             SQLite stores a database entirely in a single file. That starts to become
143             slow as the size of the file gets large. This Metabase::Archive shards
144             facts across multiple SQLite files.
145              
146             It takes the same options as L<Metabase::Archive::SQLite>, with one additional
147             option, C<shard_digits>. The C<shard_digits> attribute defines how many digits
148             of the GUID to use as a shard key. Each digit is a hexadecimal number, so
149             digits increase the number of shards as a power of 16. E.g., "1" means 16
150             shards, "2" means 256 shards and so on.
151              
152             The shard key is inserted to the database C<filename> parameter either before
153             the final period or at the end. E.g. for C<shard_digits> of "2" and
154             C<filename> "db.sqlite3", the shards would be "db_00.slite3", "db_01.sqlite3",
155             and so on.
156              
157             =for Pod::Coverage::TrustPod store extract delete iterator initialize
158              
159             =head1 AUTHORS
160              
161             =over 4
162              
163             =item *
164              
165             David Golden <dagolden@cpan.org>
166              
167             =item *
168              
169             Leon Brocard <acme@astray.org>
170              
171             =back
172              
173             =head1 COPYRIGHT AND LICENSE
174              
175             This software is Copyright (c) 2011 by David Golden.
176              
177             This is free software, licensed under:
178              
179             The Apache License, Version 2.0, January 2004
180              
181             =cut