ShardedKV/Storage/MySQL/ActiveKeyMigration.pm untested

filter: Statementstmt 0.0% 0 / 82 Branchbran 0.0% 0 / 14 Conditioncond 0.0% 0 / 13 MC/DCmcdc 0.0% 0 / 7 Subroutinesub 0.0% 0 / 6 Podpod 0.0% 0 / 1 totaltotal 0.0% 0 / 123 CC 8 SCAR 38.1
CC
8
Cov
0%
migrate_to_additional_storage
72.0
BEGIN
2.0
BEGIN
2.0
CRAP 45.1
1 package ShardedKV::Storage::MySQL::ActiveKeyMigration;
2 $ShardedKV::Storage::MySQL::ActiveKeyMigration::VERSION = '0.20';
3 use strict;
4 use warnings;
5 # ABSTRACT: Simple-minded active key migration for the MySQL storage back-end
6 use Scalar::Util qw(blessed);
7 use Carp qw(croak);
8 use Time::HiRes qw(sleep);
9  
10  
11  
12 # MySQL storage only!
13 sub migrate_to_additional_storage {
14 my %args = @_;
15  
16 my $skv = $args{shardedkv};
17 if (not blessed($skv) or not $skv->isa("ShardedKV")) {
18 croak("Need ShardedKV object as first parameter");
19 }
20 my $chunksize = $args{chunksize} || 1000;
21 my $chunksleep = defined($args{chunksleep}) ? $args{chunksleep} : 1;
22  
23 my $storages = $skv->storages;
24  
25 my $orig_continuum = $skv->continuum;
26 my $migr_continuum = $skv->migration_continuum;
27 my @orig_buckets = $orig_continuum->get_bucket_names;
28 my @migr_buckets = $migr_continuum->get_bucket_names;
29  
30 my %new_buckets = map {$_ => 1} @migr_buckets;
31 delete $new_buckets{$_} for @orig_buckets;
32  
33 die "No new storages?"
34 if not keys %new_buckets;
35  
36 my %dest_dbhs = map {$_ => $storages->{$_}->mysql_connection} keys %new_buckets;
37 my %new_insert_fragments;
38 foreach my $bname (keys %dest_dbhs) {
39 my $storage = $storages->{$bname};
40 my ($dest_tbl, $dest_key, $dest_val) = map $storage->$_(),
41 qw(table_name key_col_name value_col_name);
42 my $insert_frag = qq{
43 INSERT IGNORE INTO $dest_tbl
44 ($dest_key, $dest_val)
45 VALUES
46 };
47 $new_insert_fragments{$bname} = $insert_frag;
48 }
49  
50 foreach my $storage_name (@orig_buckets) {
51 warn "Fetching from '$storage_name'";
52 my $src_storage = $storages->{$storage_name}
53 or die "Invalid bucket name '$storage_name'";
54 my $src_dbh = $src_storage->mysql_connection;
55 my ($src_tbl, $src_key, $src_val) = map $src_storage->$_(),
56 qw(table_name key_col_name value_col_name);
57  
58 my $src_prep_first = $src_dbh->prepare(qq{
59 SELECT $src_key, $src_val FROM $src_tbl
60 ORDER BY $src_key LIMIT $chunksize
61 });
62 my $src_prep = $src_dbh->prepare(qq{
63 SELECT $src_key, $src_val FROM $src_tbl
64 WHERE $src_key > ? ORDER BY $src_key LIMIT $chunksize
65 });
66  
67 my $del_fragment = qq{
68 DELETE FROM $src_tbl WHERE $src_key IN
69 };
70  
71 my $cur_key = $src_dbh->selectcol_arrayref("SELECT MIN($src_key) FROM $src_tbl")->[0];
72  
73 my $first = 1;
74 while (1) {
75 my $sth;
76 if ($first) {
77 $sth = $src_prep_first;
78 $sth->execute();
79 $first = 0;
80 }
81 else {
82 $sth = $src_prep;
83 $sth->execute($cur_key);
84 }
85 my $rows = $sth->fetchall_arrayref;
86 last if not @$rows;
87 $cur_key = $rows->[-1][0];
88  
89 my %new_storage_rows;
90 my @to_delete;
91 foreach my $row (@$rows) {
92 my $sname = $migr_continuum->choose($row->[0]);
93 if (exists $new_buckets{$sname}) {
94 push(@to_delete, $row->[0]);
95 push(@{$new_storage_rows{$sname}||=[]}, $row);
96 }
97 }
98  
99 if (@to_delete) {
100 foreach my $new_sname (keys %new_storage_rows) {
101 my $insert_sql = $new_insert_fragments{$new_sname};
102 my $insert_data = $new_storage_rows{$new_sname};
103 my $qstr = join ',', ( ("(?,?)") x scalar(@$insert_data) );
104 $dest_dbhs{$new_sname}->do(qq{$insert_sql $qstr}, undef, (map @$_, @$insert_data))
105 or die "Failed to insert migrated keys into new storage";
106 }
107  
108 # remove from old storage
109 my $nto_remove = @to_delete;
110 my $qstr = join ',', ( ('?') x $nto_remove );
111 $src_dbh->do(qq{$del_fragment ($qstr)}, undef, @to_delete)
112 or die "Failed to remove migrated keys from old storage";
113 }
114  
115 warn "Sleeping before next chunk...";
116 sleep($chunksleep);
117 }
118 } # end foreach old/src storage
119 }
120  
121 1;
122  
123 __END__