| 1 | package ShardedKV::Storage::MySQL::ActiveKeyMigration; | ||
| 2 | $ShardedKV::Storage::MySQL::ActiveKeyMigration::VERSION = '0.20'; | ||
| 3 | use strict; | ||
| 4 | use warnings; | ||
| 5 | # ABSTRACT: Simple-minded active key migration for the MySQL storage back-end | ||
| 6 | use Scalar::Util qw(blessed); | ||
| 7 | use Carp qw(croak); | ||
| 8 | use Time::HiRes qw(sleep); | ||
| 9 | |||
| 10 | |||
| 11 | |||
| 12 | # MySQL storage only! | ||
| 13 | sub migrate_to_additional_storage { | ||
| 14 | my %args = @_; | ||
| 15 | |||
| 16 | my $skv = $args{shardedkv}; | ||
| 17 | if (not blessed($skv) or not $skv->isa("ShardedKV")) { | ||
| 18 | croak("Need ShardedKV object as first parameter"); | ||
| 19 | } | ||
| 20 | my $chunksize = $args{chunksize} || 1000; | ||
| 21 | my $chunksleep = defined($args{chunksleep}) ? $args{chunksleep} : 1; | ||
| 22 | |||
| 23 | my $storages = $skv->storages; | ||
| 24 | |||
| 25 | my $orig_continuum = $skv->continuum; | ||
| 26 | my $migr_continuum = $skv->migration_continuum; | ||
| 27 | my @orig_buckets = $orig_continuum->get_bucket_names; | ||
| 28 | my @migr_buckets = $migr_continuum->get_bucket_names; | ||
| 29 | |||
| 30 | my %new_buckets = map {$_ => 1} @migr_buckets; | ||
| 31 | delete $new_buckets{$_} for @orig_buckets; | ||
| 32 | |||
| 33 | die "No new storages?" | ||
| 34 | if not keys %new_buckets; | ||
| 35 | |||
| 36 | my %dest_dbhs = map {$_ => $storages->{$_}->mysql_connection} keys %new_buckets; | ||
| 37 | my %new_insert_fragments; | ||
| 38 | foreach my $bname (keys %dest_dbhs) { | ||
| 39 | my $storage = $storages->{$bname}; | ||
| 40 | my ($dest_tbl, $dest_key, $dest_val) = map $storage->$_(), | ||
| 41 | qw(table_name key_col_name value_col_name); | ||
| 42 | my $insert_frag = qq{ | ||
| 43 | INSERT IGNORE INTO $dest_tbl | ||
| 44 | ($dest_key, $dest_val) | ||
| 45 | VALUES | ||
| 46 | }; | ||
| 47 | $new_insert_fragments{$bname} = $insert_frag; | ||
| 48 | } | ||
| 49 | |||
| 50 | foreach my $storage_name (@orig_buckets) { | ||
| 51 | warn "Fetching from '$storage_name'"; | ||
| 52 | my $src_storage = $storages->{$storage_name} | ||
| 53 | or die "Invalid bucket name '$storage_name'"; | ||
| 54 | my $src_dbh = $src_storage->mysql_connection; | ||
| 55 | my ($src_tbl, $src_key, $src_val) = map $src_storage->$_(), | ||
| 56 | qw(table_name key_col_name value_col_name); | ||
| 57 | |||
| 58 | my $src_prep_first = $src_dbh->prepare(qq{ | ||
| 59 | SELECT $src_key, $src_val FROM $src_tbl | ||
| 60 | ORDER BY $src_key LIMIT $chunksize | ||
| 61 | }); | ||
| 62 | my $src_prep = $src_dbh->prepare(qq{ | ||
| 63 | SELECT $src_key, $src_val FROM $src_tbl | ||
| 64 | WHERE $src_key > ? ORDER BY $src_key LIMIT $chunksize | ||
| 65 | }); | ||
| 66 | |||
| 67 | my $del_fragment = qq{ | ||
| 68 | DELETE FROM $src_tbl WHERE $src_key IN | ||
| 69 | }; | ||
| 70 | |||
| 71 | my $cur_key = $src_dbh->selectcol_arrayref("SELECT MIN($src_key) FROM $src_tbl")->[0]; | ||
| 72 | |||
| 73 | my $first = 1; | ||
| 74 | while (1) { | ||
| 75 | my $sth; | ||
| 76 | if ($first) { | ||
| 77 | $sth = $src_prep_first; | ||
| 78 | $sth->execute(); | ||
| 79 | $first = 0; | ||
| 80 | } | ||
| 81 | else { | ||
| 82 | $sth = $src_prep; | ||
| 83 | $sth->execute($cur_key); | ||
| 84 | } | ||
| 85 | my $rows = $sth->fetchall_arrayref; | ||
| 86 | last if not @$rows; | ||
| 87 | $cur_key = $rows->[-1][0]; | ||
| 88 | |||
| 89 | my %new_storage_rows; | ||
| 90 | my @to_delete; | ||
| 91 | foreach my $row (@$rows) { | ||
| 92 | my $sname = $migr_continuum->choose($row->[0]); | ||
| 93 | if (exists $new_buckets{$sname}) { | ||
| 94 | push(@to_delete, $row->[0]); | ||
| 95 | push(@{$new_storage_rows{$sname}||=[]}, $row); | ||
| 96 | } | ||
| 97 | } | ||
| 98 | |||
| 99 | if (@to_delete) { | ||
| 100 | foreach my $new_sname (keys %new_storage_rows) { | ||
| 101 | my $insert_sql = $new_insert_fragments{$new_sname}; | ||
| 102 | my $insert_data = $new_storage_rows{$new_sname}; | ||
| 103 | my $qstr = join ',', ( ("(?,?)") x scalar(@$insert_data) ); | ||
| 104 | $dest_dbhs{$new_sname}->do(qq{$insert_sql $qstr}, undef, (map @$_, @$insert_data)) | ||
| 105 | or die "Failed to insert migrated keys into new storage"; | ||
| 106 | } | ||
| 107 | |||
| 108 | # remove from old storage | ||
| 109 | my $nto_remove = @to_delete; | ||
| 110 | my $qstr = join ',', ( ('?') x $nto_remove ); | ||
| 111 | $src_dbh->do(qq{$del_fragment ($qstr)}, undef, @to_delete) | ||
| 112 | or die "Failed to remove migrated keys from old storage"; | ||
| 113 | } | ||
| 114 | |||
| 115 | warn "Sleeping before next chunk..."; | ||
| 116 | sleep($chunksleep); | ||
| 117 | } | ||
| 118 | } # end foreach old/src storage | ||
| 119 | } | ||
| 120 | |||
| 121 | 1; | ||
| 122 | |||
| 123 | __END__ |