BatchChunker.pm

filter: Statementstmt 94.8% 440 / 464 Branchbran 83.6% 199 / 238 Conditioncond 72.7% 115 / 158 MC/DCmcdc 42.8% 39 / 91 Subroutinesub 94.5% 52 / 55 Podpod 75.0% 3 / 4 totaltotal 83.9% 848 / 1010 CC 180 SCAR 29.0
CC
180
Cov
88%
__ANON__
41.7
_process_past_max_checker
33.2
execute
23.3
CRAP 18.3
↑ Summary BatchChunker/LoopState.pm »
1 package DBIx::BatchChunker;
2  
3 9 our $AUTHORITY = 'cpan:GSG';
4 # ABSTRACT: Run large database changes safely
5 9 ▶use version;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 9 our $VERSION = 'v1.0.3'; # VERSION
7  
8 9 ▶use Moo;
Subroutine BEGIN: called CC=1 SCAR=0.0
9 9 ▶use MooX::StrictConstructor;
Subroutine BEGIN: called CC=1 SCAR=0.0
10  
11 9 ▶use CLDR::Number;
Subroutine BEGIN: called CC=1 SCAR=0.0
12  
13 9 ▶use Types::Standard qw( Str Bool Undef ArrayRef HashRef CodeRef InstanceOf Tuple Maybe Optional slurpy );
Subroutine BEGIN: called CC=1 SCAR=0.0
14 9 ▶use Types::Numbers qw( NumRange UnsignedInt PerlSafeInt PositiveInt PositiveOrZeroNum );
Subroutine BEGIN: called CC=1 SCAR=0.0
15 9 ▶use Type::Utils;
Subroutine BEGIN: called CC=1 SCAR=0.0
16  
17 9 ▶use List::Util 1.33 (qw( min max sum any first )); # has any/all/etc.
Subroutine BEGIN: called CC=1 SCAR=0.0
18 9 ▶use Math::BigInt upgrade => 'Math::BigFloat';
Subroutine BEGIN: called CC=1 SCAR=0.0
19 9 ▶use Math::BigFloat;
Subroutine BEGIN: called CC=1 SCAR=0.0
20 9 ▶use POSIX qw( ceil );
Subroutine BEGIN: called CC=1 SCAR=0.0
21 9 ▶use Scalar::Util qw( blessed weaken );
Subroutine BEGIN: called CC=1 SCAR=0.0
22 9 ▶use Term::ProgressBar 2.14; # with silent option
Subroutine BEGIN: called CC=1 SCAR=0.0
23 9 ▶use Time::HiRes qw( time sleep );
Subroutine BEGIN: called CC=1 SCAR=0.0
24  
25 9 ▶use DBIx::BatchChunker::LoopState;
Subroutine BEGIN: called CC=1 SCAR=0.0
26  
27 # Don't export the above, but don't conflict with StrictConstructor, either
28 9 ▶use namespace::clean -except => [qw< new meta >];
Subroutine BEGIN: called CC=1 SCAR=0.0
29  
30 # This is now an unused, dummy variable
31 9 our $DB_MAX_ID = ~0;
32  
33 #pod =encoding utf8
34 #pod
35 #pod =head1 SYNOPSIS
36 #pod
37 #pod use DBIx::BatchChunker;
38 #pod
39 #pod my $account_rs = $schema->resultset('Account')->search({
40 #pod account_type => 'deprecated',
41 #pod });
42 #pod
43 #pod my %params = (
44 #pod chunk_size => 5000,
45 #pod target_time => 5,
46 #pod
47 #pod rs => $account_rs,
48 #pod id_name => 'account_id',
49 #pod
50 #pod coderef => sub { $_[1]->delete },
51 #pod sleep => 1,
52 #pod verbose => 1,
53 #pod
54 #pod progress_name => 'Deleting deprecated accounts',
55 #pod process_past_max => 1,
56 #pod );
57 #pod
58 #pod # EITHER:
59 #pod # 1) Automatically construct and execute the changes:
60 #pod
61 #pod DBIx::BatchChunker->construct_and_execute(%params);
62 #pod
63 #pod # OR
64 #pod # 2) Manually construct and execute the changes:
65 #pod
66 #pod my $batch_chunker = DBIx::BatchChunker->new(%params);
67 #pod
68 #pod $batch_chunker->calculate_ranges;
69 #pod $batch_chunker->execute;
70 #pod
71 #pod =head1 DESCRIPTION
72 #pod
73 #pod This utility class is for running a large batch of DB changes in a manner that doesn't
74 #pod cause huge locks, outages, and missed transactions. It's highly flexible to allow for
75 #pod many different kinds of change operations, and dynamically adjusts chunks to its
76 #pod workload.
77 #pod
78 #pod It works by splitting up DB operations into smaller chunks within a loop. These chunks
79 #pod are transactionalized, either naturally as single-operation bulk work or by the loop
80 #pod itself. The full range is calculated beforehand to get the right start/end points.
81 #pod A L<progress bar|/Progress Bar Attributes> will be created to let the deployer know the
82 #pod processing status.
83 #pod
84 #pod There are two ways to use this class: call the automatic constructor and executor
85 #pod (L</construct_and_execute>) or manually construct the object and call its methods. See
86 #pod L</SYNOPSIS> for examples of both.
87 #pod
88 #pod B<DISCLAIMER:> You should not rely on this class to magically fix any and all locking
89 #pod problems the DB might experience just because it's being used. Thorough testing and
90 #pod best practices are still required.
91 #pod
92 #pod =head2 Processing Modes
93 #pod
94 #pod This class has several different modes of operation, depending on what was passed to
95 #pod the constructor:
96 #pod
97 #pod =head3 DBIC Processing
98 #pod
99 #pod If both L</rs> and L</coderef> are passed, a chunk ResultSet is built from the base
100 #pod ResultSet, to add in a C<BETWEEN> clause, and the new ResultSet is passed into the
101 #pod coderef. The coderef should run some sort of active ResultSet operation from there.
102 #pod
103 #pod An L</id_name> should be provided, but if it is missing it will be looked up based on
104 #pod the primary key of the ResultSource.
105 #pod
106 #pod If L</single_rows> is also enabled, then each chunk is wrapped in a transaction and the
107 #pod coderef is called for each row in the chunk. In this case, the coderef is passed a
108 #pod Result object instead of the chunk ResultSet.
109 #pod
110 #pod Note that whether L</single_rows> is enabled or not, the coderef execution is encapsulated
111 #pod in DBIC's retry logic, so any failures will re-connect and retry the coderef. Because of
112 #pod this, any changes you make within the coderef should be idempotent, or should at least be
113 #pod able to skip over any already-processed rows.
114 #pod
115 #pod =head3 Active DBI Processing
116 #pod
117 #pod If an L</stmt> (DBI statement handle args) is passed without a L</coderef>, the statement
118 #pod handle is merely executed on each iteration with the start and end IDs. It is assumed
119 #pod that the SQL for the statement handle contains exactly two placeholders for a C<BETWEEN>
120 #pod clause. For example:
121 #pod
122 #pod my $update_stmt = q{
123 #pod UPDATE
124 #pod accounts a
125 #pod JOIN account_updates au USING (account_id)
126 #pod SET
127 #pod a.time_stamp = au.time_stamp
128 #pod WHERE
129 #pod a.account_id BETWEEN ? AND ? AND
130 #pod a.time_stamp != au.time_stamp
131 #pod });
132 #pod
133 #pod The C<BETWEEN> clause should, of course, match the IDs being used in the loop.
134 #pod
135 #pod The statement is ran with L</dbi_connector> for retry protection. Therefore, the
136 #pod statement should also be idempotent.
137 #pod
138 #pod =head3 Query DBI Processing
139 #pod
140 #pod If both a L</stmt> and a L</coderef> are passed, the statement handle is prepared and
141 #pod executed. Like the L</Active DBI Processing> mode, the SQL for the statement should
142 #pod contain exactly two placeholders for a C<BETWEEN> clause. Then the C<$sth> is passed to
143 #pod the coderef. It's up to the coderef to extract data from the executed statement handle,
144 #pod and do something with it.
145 #pod
146 #pod If C<single_rows> is enabled, each chunk is wrapped in a transaction and the coderef is
147 #pod called for each row in the chunk. In this case, the coderef is passed a hashref of the
148 #pod row instead of the executed C<$sth>, with lowercase alias names used as keys.
149 #pod
150 #pod Note that in both cases, the coderef execution is encapsulated in a L<DBIx::Connector::Retry>
151 #pod call to either C<run> or C<txn> (using L</dbi_connector>), so any failures will
152 #pod re-connect and retry the coderef. Because of this, any changes you make within the
153 #pod coderef should be idempotent, or should at least be able to skip over any
154 #pod already-processed rows.
155 #pod
156 #pod =head3 DIY Processing
157 #pod
158 #pod If a L</coderef> is passed but neither a C<stmt> nor a C<rs> are passed, then the
159 #pod multiplier loop does not touch the database. The coderef is merely passed the start and
160 #pod end IDs for each chunk. It is expected that the coderef will run through all database
161 #pod operations using those start and end points.
162 #pod
163 #pod It's still valid to include L</min_stmt>, L</max_stmt>, and/or L</count_stmt> in the
164 #pod constructor to enable features like L<max ID recalculation|/process_past_max> or
165 #pod L<chunk resizing|/min_chunk_percent>.
166 #pod
167 #pod If you're not going to include any min/max statements for L</calculate_ranges>, you will
168 #pod need to set L</min_id> and L</max_id> yourself, either in the constructor or before the
169 #pod L</execute> call. Using L</construct_and_execute> is also not an option in this case, as
170 #pod this tries to call L</calculate_ranges> without a way to do so.
171 #pod
172 #pod =head3 TL;DR Version
173 #pod
174 #pod $stmt = Active DBI Processing
175 #pod $stmt + $coderef = Query DBI Processing | $bc->$coderef($executed_sth)
176 #pod $stmt + $coderef + single_rows=>1 = Query DBI Processing | $bc->$coderef($row_hashref)
177 #pod $rs + $coderef = DBIC Processing | $bc->$coderef($chunk_rs)
178 #pod $rs + $coderef + single_rows=>1 = DBIC Processing | $bc->$coderef($result)
179 #pod $coderef = DIY Processing | $bc->$coderef($start, $end)
180 #pod
181 #pod =head1 ATTRIBUTES
182 #pod
183 #pod See the L</METHODS> section for more in-depth descriptions of these attributes and their
184 #pod usage.
185 #pod
186 #pod =head2 DBIC Processing Attributes
187 #pod
188 #pod =head3 rs
189 #pod
190 #pod A L<DBIx::Class::ResultSet>. This is used by all methods as the base ResultSet onto which
191 #pod the DB changes will be applied. Required for DBIC processing.
192 #pod
193 #pod =cut
194  
195 9 has rs => (
196 is => 'ro',
197 isa => InstanceOf['DBIx::Class::ResultSet'],
198 required => 0,
199 );
200  
201 #pod =head3 rsc
202 #pod
203 #pod A L<DBIx::Class::ResultSetColumn>. This is only used to override L</rs> for min/max
204 #pod calculations. Optional.
205 #pod
206 #pod =cut
207  
208 9 has rsc => (
209 is => 'ro',
210 isa => InstanceOf['DBIx::Class::ResultSetColumn'],
211 required => 0,
212 );
213  
214 #pod =head3 count_rs
215 #pod
216 #pod A L<DBIx::Class::ResultSet>, only used to override L</rs> for row counting calculations.
217 #pod For 99.9% of cases, you do not need to set this. Though, it could be used for the rare
218 #pod case where the original Resulset would run into indexing problems with its row counting
219 #pod statement and needs something broader to compensate.
220 #pod
221 #pod B<WARNING:> Do not set this unless you know what you're doing. Having a different C<COUNT>
222 #pod ResultSet from the base ResultSet means that the row counts to size up the chunk workload
223 #pod will be different from the workload itself. If the row counts are too high, you may end
224 #pod up with workloads that are too quick and L<runtime targeting|/target_time> may compensate
225 #pod with an overly large chunk size, anyway. If the row counts are too low, you risk having
226 #pod a oversized chunk that gets processed and locks rows for too long, and chunk resizing may
227 #pod even skip blocks that it thinks have no rows to process.
228 #pod
229 #pod =cut
230  
231 9 has count_rs => (
232 is => 'ro',
233 isa => InstanceOf['DBIx::Class::ResultSet'],
234 required => 0,
235 );
236  
237 #pod =head3 dbic_retry_opts
238 #pod
239 #pod A hashref of DBIC retry options. These options control how retry protection works within
240 #pod DBIC. So far, there are two supported options:
241 #pod
242 #pod max_attempts = Number of times to retry
243 #pod retry_handler = Coderef that returns true to continue to retry or false to re-throw
244 #pod the last exception
245 #pod
246 #pod The default is to let the DBIC storage engine handle its own protection, which will retry
247 #pod once if the DB connection was disconnected. If you specify any options, even a blank
248 #pod hashref, BatchChunker will fill in a default C<max_attempts> of 10, and an always-true
249 #pod C<retry_handler>. This is similar to L<DBIx::Connector::Retry>'s defaults.
250 #pod
251 #pod Under the hood, these are options that are passed to the as-yet-undocumented
252 #pod L<DBIx::Class::Storage::BlockRunner>. The C<retry_handler> has access to the same
253 #pod BlockRunner object (passed as its only argument) and its methods/accessors, such as C<storage>,
254 #pod C<failed_attempt_count>, and C<last_exception>.
255 #pod
256 #pod =cut
257  
258 9 has dbic_retry_opts => (
259 is => 'ro',
260 isa => HashRef,
261 required => 0,
262 predicate => '_has_dbic_retry_opts',
263 );
264  
265 866 ▶sub _dbic_block_runner {
Subroutine _dbic_block_runner: called CC=4 SCAR=13.9
266 866 my ($self, $method, $coderef) = @_;
267  
268 866 my $storage = $self->dbic_storage;
269  
270 # Block running disabled
271 866 ▶ unless ($self->_has_dbic_retry_opts) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($self->_has_dbic_retry_opts) 694 172 866
272 694 ▶ return $storage->txn_do($coderef) if $method eq 'txn';
Branch: 1 branch
BranchTrueFalseTotal
if $method eq "txn" 78 616 694
273 616 return $storage->dbh_do($coderef);
274 }
275  
276 # A very light wrapper around BlockRunner. No need to load BlockRunner, since DBIC
277 # loads it in before us if we're using this method.
278 DBIx::Class::Storage::BlockRunner->new(
279 # in case they are not defined with a custom dbic_retry_opts
280 max_attempts => 10,
281 106 ▶ retry_handler => sub { 1 },
Subroutine __ANON__: called CC=1 SCAR=0.0
282  
283 # never overrides the important ones below
284 172 ▶ %{ $self->dbic_retry_opts },
Branch: 1 branch
BranchTrueFalseTotal
$method eq 'txn' ? : 48 124 172
285  
286 storage => $storage,
287 wrap_txn => ($method eq 'txn' ? 1 : 0),
288 )->run($coderef);
289 }
290  
291 #pod =head2 DBI Processing Attributes
292 #pod
293 #pod =head3 dbi_connector
294 #pod
295 #pod A L<DBIx::Connector::Retry> object. Instead of L<DBI> statement handles, this is the
296 #pod recommended way for BatchChunker to interface with the DBI, as it handles retries on
297 #pod failures. The connection mode used is whatever default is set within the object.
298 #pod
299 #pod Required for DBI Processing, unless L</dbic_storage> is specified.
300 #pod
301 #pod =cut
302  
303 9 has dbi_connector => (
304 is => 'ro',
305 isa => InstanceOf['DBIx::Connector::Retry'],
306 required => 0,
307 );
308  
309 #pod =head3 dbic_storage
310 #pod
311 #pod A DBIC storage object, as an alternative for L</dbi_connector>. There may be times when
312 #pod you want to run plain DBI statements, but are still using DBIC. In these cases, you
313 #pod don't have to create a L<DBIx::Connector::Retry> object to run those statements.
314 #pod
315 #pod This uses a BlockRunner object for retry protection, so the options in
316 #pod L</dbic_retry_opts> would apply here.
317 #pod
318 #pod Required for DBI Processing, unless L</dbi_connector> is specified.
319 #pod
320 #pod =cut
321  
322 9 has dbic_storage => (
323 is => 'ro',
324 isa => InstanceOf['DBIx::Class::Storage::DBI'],
325 required => 0,
326 );
327  
328 #pod =head3 min_stmt
329 #pod
330 #pod =head3 max_stmt
331 #pod
332 #pod SQL statement strings or an arrayref of parameters for L<DBI/selectrow_array>.
333 #pod
334 #pod When executed, these statements should each return a single value, either the minimum or
335 #pod maximum ID that will be affected by the DB changes. These are used by
336 #pod L</calculate_ranges>. Required if using either type of DBI Processing.
337 #pod
338 #pod =cut
339  
340 my $SQLStringOrSTHArgs_type = Type::Utils::declare(
341 name => 'SQLStringOrSTHArgs',
342 # Allow an SQL string, an optional hashref/undef, and any number of strings/undefs
343 parent => Tuple->parameterize(Str, Optional[Maybe[HashRef]], slurpy ArrayRef[Maybe[Str]]),
344 30 ▶ coercion => sub { $_ = [ $_ ] if Str->check($_); $_ },
Subroutine __ANON__: called CC=2 SCAR=6.9
Branch: 1 branch
BranchTrueFalseTotal
if &Str()->check($_) 29 1 30
345 1 ▶ message => sub { 'Must be either an SQL string or an arrayref of parameters for $sth creation (SQL + hashref/undef + binds)' },
Subroutine __ANON__: called CC=1 SCAR=0.0
346 9 );
347  
348 9 has min_stmt => (
349 is => 'ro',
350 isa => $SQLStringOrSTHArgs_type,
351 required => 0,
352 coerce => 1,
353 );
354  
355 9 has max_stmt => (
356 is => 'ro',
357 isa => $SQLStringOrSTHArgs_type,
358 required => 0,
359 coerce => 1,
360 );
361  
362 #pod =head3 stmt
363 #pod
364 #pod A SQL statement string or an arrayref of parameters for L<DBI/prepare> + binds.
365 #pod
366 #pod If using L</Active DBI Processing> (no coderef), this is a L<do-able|DBI/do> statement
367 #pod (usually DML like C<INSERT/UPDATE/DELETE>). If using L</Query DBI Processing> (with
368 #pod coderef), this is a passive DQL (C<SELECT>) statement.
369 #pod
370 #pod In either case, the statement should contain C<BETWEEN> placeholders, which will be
371 #pod executed with the start/end ID points. If there are already bind placeholders in the
372 #pod arrayref, then make sure the C<BETWEEN> bind points are last on the list.
373 #pod
374 #pod Required for DBI Processing.
375 #pod
376 #pod =cut
377  
378 9 has stmt => (
379 is => 'ro',
380 isa => $SQLStringOrSTHArgs_type,
381 required => 0,
382 coerce => 1,
383 );
384  
385 #pod =head3 count_stmt
386 #pod
387 #pod A C<SELECT COUNT> SQL statement string or an arrayref of parameters for
388 #pod L<DBI/selectrow_array>.
389 #pod
390 #pod Like L</stmt>, it should contain C<BETWEEN> placeholders. In fact, the SQL should look
391 #pod exactly like the L</stmt> query, except with C<COUNT(*)> instead of the column list.
392 #pod
393 #pod Used only for L</Query DBI Processing>. Optional, but recommended for
394 #pod L<chunk resizing|/min_chunk_percent>.
395 #pod
396 #pod =cut
397  
398 9 has count_stmt => (
399 is => 'ro',
400 isa => $SQLStringOrSTHArgs_type,
401 required => 0,
402 coerce => 1,
403 );
404  
405 #pod =head2 Progress Bar Attributes
406 #pod
407 #pod =head3 progress_bar
408 #pod
409 #pod The progress bar used for all methods. This can be specified right before the method
410 #pod call to override the default used for that method. Unlike most attributes, this one
411 #pod is read-write, so it can be switched on-the-fly.
412 #pod
413 #pod Don't forget to remove or switch to a different progress bar if you want to use a
414 #pod different one for another method:
415 #pod
416 #pod $batch_chunker->progress_bar( $calc_pb );
417 #pod $batch_chunker->calculate_ranges;
418 #pod $batch_chunker->progress_bar( $loop_pb );
419 #pod $batch_chunker->execute;
420 #pod
421 #pod All of this is optional. If the progress bar isn't specified, the method will create
422 #pod a default one. If the terminal isn't interactive, the default L<Term::ProgressBar> will
423 #pod be set to C<silent> to naturally skip the output.
424 #pod
425 #pod =cut
426  
427 9 has progress_bar => (
428 is => 'rw',
429 isa => InstanceOf['Term::ProgressBar'],
430 );
431  
432 #pod =head3 progress_name
433 #pod
434 #pod A string used by L</execute> to assist in creating a progress bar. Ignored if
435 #pod L</progress_bar> is already specified.
436 #pod
437 #pod This is the preferred way of customizing the progress bar without having to create one
438 #pod from scratch.
439 #pod
440 #pod =cut
441  
442 has progress_name => (
443 is => 'rw',
444 isa => Str,
445 required => 0,
446 lazy => 1,
447 36 ▶ default => sub {
Subroutine __ANON__: called CC=2 SCAR=6.9
448 36 my $rs = shift->rs;
449 36 ▶ 'Processing'.(defined $rs ? ' '.$rs->result_source->name : '');
Branch: 1 branch
BranchTrueFalseTotal
defined $rs ? : 16 20 36
450 },
451 9 );
452  
453 #pod =head3 cldr
454 #pod
455 #pod A L<CLDR::Number> object. English speakers that use a typical C<1,234.56> format would
456 #pod probably want to leave it at the default. Otherwise, you should provide your own.
457 #pod
458 #pod =cut
459  
460 has cldr => (
461 is => 'rw',
462 isa => InstanceOf['CLDR::Number'],
463 required => 0,
464 lazy => 1,
465 3 ▶ default => sub { CLDR::Number->new(locale => 'en') },
Subroutine __ANON__: called CC=1 SCAR=0.0
466 9 );
467  
468 #pod =head3 verbose
469 #pod
470 #pod Boolean. By default, this is on, which displays timing stats on each chunk, as well as
471 #pod total numbers. This is still subject to non-interactivity checks from L</progress_bar>.
472 #pod
473 #pod (This was previously defaulted to off, and called C<debug>, prior to v1.0.0.)
474 #pod
475 #pod =for Pod::Coverage debug
476 #pod
477 #pod =cut
478  
479 9 has verbose => (
480 is => 'rw',
481 isa => Bool,
482 required => 0,
483 default => 1,
484 );
485  
486 # Backwards-compatibility
487 9 *debug = \&verbose;
488  
489 #pod =head2 Common Attributes
490 #pod
491 #pod =head3 id_name
492 #pod
493 #pod The column name used as the iterator in the processing loops. This should be a primary
494 #pod key or integer-based (indexed) key, tied to the L<resultset|/rs>.
495 #pod
496 #pod Optional. Used mainly in DBIC processing. If not specified, it will look up
497 #pod the first primary key column from L</rs> and use that.
498 #pod
499 #pod This can still be specified for other processing modes to use in progress bars.
500 #pod
501 #pod =cut
502  
503 9 has id_name => (
504 is => 'rw',
505 isa => Str,
506 required => 0,
507 trigger => \&_fix_id_name,
508 );
509  
510 76 ▶sub _fix_id_name {
Subroutine _fix_id_name: called CC=4 SCAR=13.9
511 76 my ($self, $id_name) = @_;
512 76 ▶ return if !$id_name || $id_name =~ /\./ || !defined $self->rs; # prevent an infinite trigger loop
Branch: 1 branch
BranchTrueFalseTotal
if not $id_name or $id_name =~ /\./ or not defined $self->rs 60 16 76
Condition100%6/6
not $id_name or $id_name =~ /\./ or not defined $self->rs
l!l&&r!l&&!r
342616
not $id_name or $id_name =~ /\./
l!l&&r!l&&!r
181642
MC/DC100%3/3
not $id_name or $id_name =~ /\./ or not defined $self->rs
not $id_name $id_name =~ /\./ not defined $self->rs
Truth table100%4/4
not(A) or B or not(C)
A= not $id_nameB= $id_name =~ /\./C= not defined $self->rs
ABCresult
1XX1
01X1
0011
0000
513 16 $self->id_name( $self->rs->current_source_alias.".$id_name" );
514 }
515  
516 #pod =head3 coderef
517 #pod
518 #pod The coderef that will be called either on each chunk or each row, depending on how
519 #pod L</single_rows> is set. The first input is always the BatchChunker object. The rest
520 #pod vary depending on the processing mode:
521 #pod
522 #pod $stmt + $coderef = Query DBI Processing | $bc->$coderef($executed_sth)
523 #pod $stmt + $coderef + single_rows=>1 = Query DBI Processing | $bc->$coderef($row_hashref)
524 #pod $rs + $coderef = DBIC Processing | $bc->$coderef($chunk_rs)
525 #pod $rs + $coderef + single_rows=>1 = DBIC Processing | $bc->$coderef($result)
526 #pod $coderef = DIY Processing | $bc->$coderef($start, $end)
527 #pod
528 #pod The loop does not monitor the return values from the coderef.
529 #pod
530 #pod Required for all processing modes except L</Active DBI Processing>.
531 #pod
532 #pod =cut
533  
534 9 has coderef => (
535 is => 'ro',
536 isa => CodeRef,
537 required => 0,
538 );
539  
540 #pod =head3 chunk_size
541 #pod
542 #pod The amount of rows to be processed in each loop.
543 #pod
544 #pod This figure should be sized to keep per-chunk processing time at around 5 seconds. If
545 #pod this is too large, rows may lock for too long. If it's too small, processing may be
546 #pod unnecessarily slow.
547 #pod
548 #pod Default is 1 row, which is only appropriate if L</target_time> (on by default) is
549 #pod enabled. This will cause the processing to slowly ramp up to the target time as
550 #pod BatchChunker gathers more data.
551 #pod
552 #pod Otherwise, if you using static chunk sizes with C<target_time> turned off, figure out
553 #pod the right chunk size with a few test runs and set it here.
554 #pod
555 #pod (This was previously defaulted to 1000 rows, prior to v1.0.0.)
556 #pod
557 #pod =cut
558  
559 9 has chunk_size => (
560 is => 'rw',
561 isa => PositiveInt,
562 required => 0,
563 default => 1,
564 );
565  
566 #pod =head3 id_increment
567 #pod
568 #pod The expected distance between consecutive IDs, such as MySQL's C<auto_increment_increment>
569 #pod setting. Each chunk covers a range of C<< chunk_size * id_increment >> IDs, so that a
570 #pod chunk holds roughly L</chunk_size> rows.
571 #pod
572 #pod For example, a Galera cluster that hands out every tenth ID only has 500 rows in every
573 #pod 5000 IDs. Without this setting, every chunk would start out at 10% of the expected row
574 #pod count, and L<chunk resizing|/min_chunk_percent> would need several extra C<COUNT> queries
575 #pod to expand each chunk to a reasonable size.
576 #pod
577 #pod This only changes the size of the ID range. L</chunk_size> and L</min_chunk_percent>
578 #pod still count rows, and L</target_time> still resizes chunks by rows. The setting is a
579 #pod starting estimate, not a guarantee: chunk resizing still corrects ranges with more or
580 #pod fewer rows than expected, such as rows that were inserted under a different increment.
581 #pod
582 #pod Default is 1, which assumes that IDs are contiguous.
583 #pod
584 #pod =cut
585  
586 9 has id_increment => (
587 is => 'ro',
588 isa => PositiveInt,
589 required => 0,
590 default => 1,
591 );
592  
593 #pod =head3 target_time
594 #pod
595 #pod The target runtime (in seconds) that chunk processing should strive to achieve, not
596 #pod including L</sleep>. If the chunk processing times are too high or too low, this will
597 #pod dynamically adjust L</chunk_size> to try to match the target.
598 #pod
599 #pod BatchChunker will still use the initial C<chunk_size>, and it will need at least one
600 #pod chunk processed, before it makes adjustments. If the starting chunk size is grossly
601 #pod inaccurate to the workload, you could end up with several chunks in the beginning causing
602 #pod long-lasting locks before the runtime targeting reduces them down to a reasonable size.
603 #pod
604 #pod (Chunk size reductions are prioritized before increases, so it should re-size as soon as
605 #pod it finds the problem. But, one bad chunk could be all it takes to cause an outage.)
606 #pod
607 #pod Default is 5 seconds. Set this to zero to turn off runtime targeting. (This was
608 #pod previously defaulted to off prior to v0.92, and set to 15 in v0.92.)
609 #pod
610 #pod =cut
611  
612 9 has target_time => (
613 is => 'ro',
614 isa => PositiveOrZeroNum,
615 required => 0,
616 default => 5,
617 );
618  
619 #pod =head3 sleep
620 #pod
621 #pod The number of seconds to sleep after each chunk. It uses L<Time::HiRes>'s version, so
622 #pod fractional numbers are allowed.
623 #pod
624 #pod Default is 0.5 seconds, which is fine for most operations. You can likely get away with
625 #pod zero for smaller operations, but test it out first. If processing is going to take up a
626 #pod lot of disk I/O, you may want to consider a higher setting. If the database server
627 #pod spends too much time on processing, the replicas may have a hard time keeping up with
628 #pod standard load.
629 #pod
630 #pod This will increase the overall processing time of the loop, so try to find a balance
631 #pod between the two.
632 #pod
633 #pod (This was previously defaulted to 0 seconds, prior to v1.0.0.)
634 #pod
635 #pod =cut
636  
637 9 has 'sleep' => (
638 is => 'ro',
639 isa => PositiveOrZeroNum,
640 required => 0,
641 default => 0.5,
642 );
643  
644 #pod =head3 max_runtime
645 #pod
646 #pod The number of seconds that the entire process is allowed to run. If you have a
647 #pod long-running I<and idempotent> operation that you don't want to run for days, you can set
648 #pod this attribute, execute the operation, and run it again at a later date. The L</min_id>
649 #pod will be set to the last-used ID, so the operation can be continued with another
650 #pod L</execute> call. Or you can use this to figure out if it finished or not.
651 #pod
652 #pod Turned off by default. If you use this, you should add in extra multipler operations to
653 #pod separate out the time math, like C<6 * 60 * 60> for 6 hours.
654 #pod
655 #pod =cut
656  
657 9 has max_runtime => (
658 is => 'ro',
659 isa => PositiveOrZeroNum,
660 required => 0,
661 default => 0,
662 );
663  
664 #pod =head3 process_past_max
665 #pod
666 #pod Boolean that controls whether to check past the L</max_id> during the loop. If the loop
667 #pod hits the end point, it will run another maximum ID check in the DB, and adjust C<max_id>
668 #pod accordingly. If it somehow cannot run a DB check (no L</rs> or L</max_stmt> available,
669 #pod for example), the last chunk will just be one at the end of
670 #pod C<< max_id + chunk_size * id_increment >>.
671 #pod
672 #pod This is useful if the entire table is expected to be processed, and you don't want to
673 #pod miss any new rows that come up between L</calculate_ranges> and the end of the loop.
674 #pod
675 #pod Turned off by default.
676 #pod
677 #pod =cut
678  
679 9 has process_past_max => (
680 is => 'ro',
681 isa => Bool,
682 required => 0,
683 default => 0,
684 );
685  
686 #pod =head3 single_rows
687 #pod
688 #pod Boolean that controls whether single rows are passed to the L</coderef> or the chunk's
689 #pod ResultSets/statement handle is passed.
690 #pod
691 #pod Since running single-row operations in a DB is painfully slow (compared to bulk
692 #pod operations), this also controls whether the entire set of coderefs are encapsulated into
693 #pod a DB transaction. Transactionalizing the entire chunk brings the speed, and atomicity,
694 #pod back to what a bulk operation would be. (Bulk operations are still faster, but you can't
695 #pod do anything you want in a single DML statement.)
696 #pod
697 #pod Used only by L</DBIC Processing> and L</Query DBI Processing>.
698 #pod
699 #pod =cut
700  
701 9 has single_rows => (
702 is => 'ro',
703 isa => Bool,
704 required => 0,
705 default => 0,
706 );
707  
708 #pod =head3 min_chunk_percent
709 #pod
710 #pod The minimum row count, as a percentage of L</chunk_size>. This value is actually
711 #pod expressed in decimal form, i.e.: between 0 and 1.
712 #pod
713 #pod This value will be used to determine when to process, skip, or expand a block, based on
714 #pod a count query. The default is C<0.5> or 50%, which means that it will try to expand the
715 #pod block to a larger size if the row count is less than 50% of the chunk size. Zero-sized
716 #pod blocks will be skipped entirely.
717 #pod
718 #pod This "chunk resizing" is useful for large regions of the table that have been deleted, or
719 #pod when the incrementing ID has large gaps in it for other reasons. Wasting time on
720 #pod numerical gaps that span millions can slow down the processing considerably, especially
721 #pod if L</sleep> is enabled.
722 #pod
723 #pod If the gaps are regular, such as an auto-increment step larger than 1, set
724 #pod L</id_increment> instead, so that chunks start at the right size and need no resizing.
725 #pod
726 #pod If this needs to be disabled, set this to 0. The maximum chunk percentage does not have
727 #pod a setting and is hard-coded at C<< 100% + min_chunk_percent >>.
728 #pod
729 #pod If DBIC processing isn't used, L</count_stmt> is also required to enable chunk resizing.
730 #pod
731 #pod =cut
732  
733 has min_chunk_percent => (
734 is => 'ro',
735 isa => Type::Utils::declare(
736 name => 'PositiveZeroToOneNum',
737 parent => NumRange->parameterize(0, 1),
738 0 ▶ message => sub { 'Must be a number between 0 and 1' },
Subroutine __ANON__: not called CC=1 SCAR=6.9
739 9 ),
740 required => 0,
741 default => 0.5,
742 );
743  
744 #pod =head3 min_id
745 #pod
746 #pod =head3 max_id
747 #pod
748 #pod Used by L</execute> to figure out the main start and end points. Calculated by
749 #pod L</calculate_ranges>.
750 #pod
751 #pod Manually setting this is not recommended, as each database is different and the
752 #pod information may have changed between the DB change development and deployment. Instead,
753 #pod use L</calculate_ranges> to fill in these values right before running the loop.
754 #pod
755 #pod When the operation is finished, C<min_id> will be set to the last processed ID, just in
756 #pod case it was stopped early and needs to be restarted (eg: L</max_runtime> is set).
757 #pod Alternately, you can run L</calculate_ranges> again to confirm from the database.
758 #pod
759 #pod =cut
760  
761 9 has min_id => (
762 is => 'rw',
763 isa => UnsignedInt,
764 );
765  
766 9 has max_id => (
767 is => 'rw',
768 isa => UnsignedInt,
769 );
770  
771 # Big number handling
772 9 has _use_bignums => (
773 is => 'rw',
774 isa => Bool,
775 default => 0,
776 trigger => \&_upgrade_attrs_to_bigint,
777 );
778  
779 9 my @BIGNUM_BC_ATTRS = (qw< chunk_size min_id max_id >);
780 9 my @BIGNUM_LS_ATTRS = (qw< start end prev_end multiplier_range multiplier_step chunk_size chunk_count >);
781  
782 853 ▶sub _check_bignums {
Subroutine _check_bignums: called CC=16 SCAR=28.0
783 853 my ($self) = shift;
784 853 ▶ return 1 if $self->_use_bignums; # already checked these
Branch: 1 branch
BranchTrueFalseTotal
if $self->_use_bignums 144 709 853
785  
786 # Auto-set _use_bignums if we detect that we need it
787 709 my $set_bignums = 0;
788  
789 # If other values are passed, check those, too
790 709 foreach my $val (@_) {
791 667 ▶ next unless defined $val;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $val 174 493 667
792 493 ▶ $set_bignums = 1 if blessed $val || !PerlSafeInt->check($val);
Branch: 1 branch
BranchTrueFalseTotal
if builtin::blessed($val) or not &PerlSafeInt()->check($val) 0 493 493
Condition33%1/3
builtin::blessed($val) or not &PerlSafeInt()->check($val)
l!l&&r!l&&!r
00493
MC/DC0%0/2
builtin::blessed($val) or not &PerlSafeInt()->check($val)
builtin::blessed($val) not &PerlSafeInt()->check($val)
Truth table33%1/3
A or not(B)
A= builtin::blessed($val)B= not &PerlSafeInt()->check($val)
ABresult
1X1
011
000
793 }
794  
795 # Check BatchChunker attributes
796 709 foreach my $attr (@BIGNUM_BC_ATTRS) {
797 2127 my $val = $self->$attr();
798 2127 ▶ next unless defined $val;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $val 140 1987 2127
799 1987 ▶ $set_bignums = 1 if blessed $val || !PerlSafeInt->check($val);
Branch: 1 branch
BranchTrueFalseTotal
if builtin::blessed($val) or not &PerlSafeInt()->check($val) 2 1985 1987
Condition66%2/3
builtin::blessed($val) or not &PerlSafeInt()->check($val)
l!l&&r!l&&!r
201985
MC/DC50%1/2
builtin::blessed($val) or not &PerlSafeInt()->check($val)
builtin::blessed($val) not &PerlSafeInt()->check($val)
Truth table66%2/3
A or not(B)
A= builtin::blessed($val)B= not &PerlSafeInt()->check($val)
ABresult
1X1
011
000
800 }
801  
802 # Check LoopState attributes
803 709 ▶ if (my $ls = $self->loop_state) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $ls = $self->loop_state) 605 104 709
804 605 foreach my $attr (@BIGNUM_LS_ATTRS) {
805 4235 my $val = $ls->$attr();
806 4235 ▶ next unless defined $val;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $val 667 3568 4235
807 3568 ▶ $set_bignums = 1 if blessed $val || !PerlSafeInt->check($val);
Branch: 1 branch
BranchTrueFalseTotal
if builtin::blessed($val) or not &PerlSafeInt()->check($val) 6 3562 3568
Condition66%2/3
builtin::blessed($val) or not &PerlSafeInt()->check($val)
l!l&&r!l&&!r
063562
MC/DC50%1/2
builtin::blessed($val) or not &PerlSafeInt()->check($val)
builtin::blessed($val) not &PerlSafeInt()->check($val)
Truth table66%2/3
A or not(B)
A= builtin::blessed($val)B= not &PerlSafeInt()->check($val)
ABresult
1X1
011
000
808 }
809 }
810  
811 709 ▶ $self->_use_bignums(1) if $set_bignums;
Branch: 1 branch
BranchTrueFalseTotal
if $set_bignums 5 704 709
812 709 return $set_bignums;
813 }
814  
815 5 ▶sub _upgrade_attrs_to_bigint {
Subroutine _upgrade_attrs_to_bigint: called CC=9 SCAR=22.0
816 5 my ($self, $is_on) = @_;
817 5 ▶ return unless $is_on;
Branch: 1 branch
BranchTrueFalseTotal
unless $is_on 0 5 5
818  
819 # Fix BatchChunker attributes
820 5 foreach my $attr (@BIGNUM_BC_ATTRS) {
821 15 my $val = $self->$attr();
822 15 ▶ next unless defined $val; # nothing to upgrade
Branch: 1 branch
BranchTrueFalseTotal
unless defined $val 4 11 15
823 11 ▶ next if blessed $val; # already upgraded
Branch: 1 branch
BranchTrueFalseTotal
if builtin::blessed($val) 2 9 11
824 9 $self->$attr( Math::BigInt->new($val) );
825 }
826  
827 # Fix LoopState attributes
828 5 my $ls = $self->loop_state;
829 5 ▶ return unless $ls;
Branch: 1 branch
BranchTrueFalseTotal
unless $ls 2 3 5
830 3 foreach my $attr (@BIGNUM_LS_ATTRS) {
831 21 my $val = $ls->$attr();
832 21 ▶ next unless defined $val; # nothing to upgrade
Branch: 1 branch
BranchTrueFalseTotal
unless defined $val 6 15 21
833 15 ▶ next if blessed $val; # already upgraded
Branch: 1 branch
BranchTrueFalseTotal
if builtin::blessed($val) 0 15 15
834 15 $ls->$attr( Math::BigInt->new($val) );
835 }
836 }
837  
838 #pod =head3 loop_state
839 #pod
840 #pod A L<DBIx::BatchChunker::LoopState> object designed to hold variables during the
841 #pod processing loop. The object will be cleared out after use. Most of the complexity is
842 #pod needed for chunk resizing.
843 #pod
844 #pod =cut
845  
846 9 has loop_state => (
847 is => 'rw',
848 isa => InstanceOf['DBIx::BatchChunker::LoopState'],
849 required => 0,
850 init_arg => undef,
851 clearer => 'clear_loop_state',
852 );
853  
854 # Backwards-compatibility
855 9 *_loop_state = \&loop_state;
856  
857 #pod =for Pod::Coverage BUILDARGS BUILD
858 #pod
859 #pod =cut
860  
861 47 ▶around BUILDARGS => sub {
Subroutine __ANON__: called CC=39 SCAR=37.3
862 47 my $next = shift;
863 47 my $class = shift;
864  
865 0 ▶ my %args = @_ == 1 ? %{ $_[0] } : @_;
Branch: 1 branch
BranchTrueFalseTotal
@_ == 1 ? : 0 47 47
866  
867 # debug -> verbose
868 47 ▶ $args{verbose} //= delete $args{debug} if exists $args{debug};
Branch: 1 branch
BranchTrueFalseTotal
if exists $args{"debug"} 8 39 47
Condition33%1/3
$args{"verbose"} //= delete $args{"debug"}
l!l&&r!l&&!r
080
MC/DC0%0/2
$args{"verbose"} //= delete $args{"debug"}
$args{"verbose"} delete $args{"debug"}
Truth table33%1/3
A //= B
A= $args{"verbose"}B= delete $args{"debug"}
ABresult
1X1
011
000
869  
870 # Auto-building of rsc and id_name can be a weird dependency dance, so it's better to
871 # handle it here.
872 47 my ($rsc, $rs, $id_name) = @args{qw< rsc rs id_name >};
873 47 ▶ if (defined $rsc && !$id_name) {
Branch: 3 branches
BranchTrueFalseTotal
if (defined $rsc and not $id_name) { } 6 41 47
elsif (not defined $rsc and $id_name and defined $rs) { } 2 39 41
elsif (not defined $rsc || $id_name and defined $rs) { } 14 26 40
Condition66%2/3
defined $rsc and not $id_name
!ll&&!rl&&r
4106
MC/DC50%1/2
defined $rsc and not $id_name
defined $rsc not $id_name
Truth table66%2/3
A and not(B)
A= defined $rscB= not $id_name
ABresult
0X0
100
111
874 6 $args{id_name} = $rsc->{_as};
875 }
876 ▶ elsif (!defined $rsc && $id_name && defined $rs) {
Condition83%5/6
not defined $rsc and $id_name and defined $rs
!ll&&!rl&&r
3812
not defined $rsc and $id_name
!ll&&!rl&&r
0383
MC/DC66%2/3
not defined $rsc and $id_name and defined $rs
not defined $rsc $id_name defined $rs
Truth table75%3/4
not(A) and B and C
A= not defined $rscB= $id_nameC= defined $rs
ABCresult
0XX0
10X0
1100
1111
877 2 $args{rsc} = $rs->get_column( $args{id_name} );
878 }
879 ▶ elsif (!defined $rsc && !$id_name && defined $rs) {
Condition83%5/6
not defined $rsc || $id_name and defined $rs
!ll&&!rl&&r
12414
defined $rsc || $id_name
l!l&&r!l&&!r
0138
MC/DC66%2/3
not defined $rsc || $id_name and defined $rs
defined $rsc $id_name defined $rs
Truth table75%3/4
not(A || B) and C
A= defined $rscB= $id_nameC= defined $rs
ABCresult
1XX0
01X0
0000
0011
880 14 $args{id_name} = ($rs->result_source->primary_columns)[0];
881 14 $args{rsc} = $rs->get_column( $args{id_name} );
882 }
883 47 $rsc = $args{rsc};
884  
885 # Default count_rs is just $rs
886 47 ▶ $args{count_rs} //= $rs if defined $rs;
Branch: 1 branch
BranchTrueFalseTotal
if defined $rs 18 29 47
Condition66%2/3
$args{"count_rs"} //= $rs
l!l&&r!l&&!r
1170
MC/DC0%0/2
$args{"count_rs"} //= $rs
$args{"count_rs"} $rs
Truth table66%2/3
A //= B
A= $args{"count_rs"}B= $rs
ABresult
1X1
011
000
887  
888 # Auto-add dbic_storage, if available
889 47 ▶ if (!defined $args{dbic_storage} && (defined $rs || defined $rsc)) {
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $args{"dbic_storage"} and defined $rs || defined $rsc) 21 26 47
Condition100%6/6
not defined $args{"dbic_storage"} and defined $rs || defined $rsc
!ll&&!rl&&r
62021
defined $rs or defined $rsc
l!l&&r!l&&!r
18320
MC/DC100%3/3
not defined $args{"dbic_storage"} and defined $rs || defined $rsc
not defined $args{"dbic_storage"} defined $rs defined $rsc
Truth table100%4/4
not(A) and B or C
A= not defined $args{"dbic_storage"}B= defined $rsC= defined $rsc
ABCresult
0XX0
1000
11X1
1011
890 21 ▶ $args{dbic_storage} = defined $rs ? $rs->result_source->storage : $rsc->_resultset->result_source->storage;
Branch: 1 branch
BranchTrueFalseTotal
defined $rs ? : 18 3 21
891 }
892  
893 # Find something to use as a dbi_connector, if it doesn't already exist
894 47 my @old_attrs = qw< sth min_sth max_sth count_sth >;
895 47 my @new_attrs = map { my $k = $_; $k =~ s/sth$/stmt/; $k } @old_attrs;
896 47 ▶ my $example_key = first { $args{$_} } @old_attrs;
Subroutine __ANON__: called CC=1 SCAR=0.0
897 47 ▶ if ($example_key && !defined $args{dbi_connector}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($example_key and not defined $args{"dbi_connector"}) 6 41 47
Condition66%2/3
$example_key and not defined $args{"dbi_connector"}
!ll&&!rl&&r
4106
MC/DC50%1/2
$example_key and not defined $args{"dbi_connector"}
$example_key not defined $args{"dbi_connector"}
Truth table66%2/3
A and not(B)
A= $example_keyB= not defined $args{"dbi_connector"}
ABresult
0X0
100
111
898 6 warn join "\n",
899 'The sth/*_sth options are now considered legacy usage in DBIx::BatchChunker. Because there is no',
900 'way to re-acquire the password, any attempt to reconnect will fail. Please use dbi_connector and',
901 'stmt/*_stmt instead for reconnection support.',
902 ''
903 ;
904  
905 # NOTE: There was a way to monkey-patch _connect to use $dbh->clone, but I've considered it
906 # too intrusive of a solution to use. Better to demand that the user switch to the new
907 # attributes, but have something that still works in most cases.
908  
909 # Attempt to build some sort of Connector object
910 6 require DBIx::Connector::Retry;
911 6 my $dbh = $args{$example_key}->{Database};
912  
913 my $conn = DBIx::Connector::Retry->new(
914 connect_info => [
915 join(':', 'dbi', $dbh->{Driver}{Name}, $dbh->{Name}),
916 $dbh->{Username},
917 6 '', # XXX: Can't acquire the password
918 # Sane %attr defaults on the off-chance that it actually re-connects
919 { AutoCommit => 1, RaiseError => 1 },
920 ],
921  
922 # Do not disconnect on DESTROY. The $dbh might still be used post-run.
923 disconnect_on_destroy => 0,
924 );
925  
926 # Pretend $conn->_connect was called and store our pre-existing $dbh
927 6 $conn->{_pid} = $$;
928 6 ▶ $conn->{_tid} = threads->tid if $INC{'threads.pm'};
Branch: 1 branch
BranchTrueFalseTotal
if $INC{"threads.pm"} 0 6 6
929 6 $conn->{_dbh} = $dbh;
930 6 $conn->driver;
931  
932 6 $args{dbi_connector} = $conn;
933 }
934  
935 # Handle legacy options for sth/*_sth
936 47 foreach my $old_attr (grep { $args{$_} } @old_attrs) {
937 16 my $new_attr = $old_attr;
938 16 $new_attr =~ s/sth$/stmt/;
939  
940 16 my $sth = delete $args{$old_attr};
941 16 ▶ $args{$new_attr} ||= [ $sth->{Statement} ];
Condition50%1/2
$args{$new_attr} ||= [$sth->{"Statement"}]
l!l
016
MC/DC0%0/1
$args{$new_attr} ||= [$sth->{"Statement"}]
$args{$new_attr}
Truth table50%1/2
A
A= $args{$new_attr}
Aresult
11
00
942 }
943  
944 # Now check to make sure dbi_connector is available for DBI processing
945 die 'DBI processing requires a dbi_connector or dbic_storage attribute!' if (
946 !(defined $args{dbi_connector} || defined $args{dbic_storage}) &&
947 47 ▶ (defined first { $args{$_} } @new_attrs)
Subroutine __ANON__: called CC=1 SCAR=0.1
Branch: 1 branch
BranchTrueFalseTotal
if not defined $args{"dbi_connector"} || defined $args{"dbic_storage"} and defined &first(sub { $args{$_}; } , @new_attrs) 0 47 47
Condition83%5/6
not defined $args{"dbi_connector"} || defined $args{"dbic_storage"} and defined &first(sub { $args{$_}; } , @new_attrs)
!ll&&!rl&&r
4160
defined $args{'dbi_connector'} || defined $args{'dbic_storage'}
l!l&&r!l&&!r
16256
MC/DC0%0/3
not defined $args{"dbi_connector"} || defined $args{"dbic_storage"} and defined &first(sub { $args{$_}; } , @new_attrs)
defined $args{'dbi_connector'} defined $args{'dbic_storage'} defined &first(sub { $args{$_}; } , @new_attrs)
Truth table75%3/4
not(A || B) and C
A= defined $args{'dbi_connector'}B= defined $args{'dbic_storage'}C= defined &first(sub { $args{$_}; } , @new_attrs)
ABCresult
1XX0
01X0
0000
0011
948 );
949  
950 # Other sanity checks
951 die 'Range calculations require one of these attr sets: rsc, rs, or dbi_connector|dbic_storage + min_stmt + max_stmt' unless (
952 defined $args{rsc} ||
953 (defined $args{min_stmt} && defined $args{max_stmt}) ||
954 (!defined $args{dbi_connector} && !defined $args{dbic_storage} && defined $args{coderef}) # DIY mode is exempt
955 47 ▶ );
Branch: 1 branch
BranchTrueFalseTotal
unless defined $args{"rsc"} or defined $args{"min_stmt"} and defined $args{"max_stmt"} or not defined $args{"dbi_connector"} || defined $args{"dbic_storage"} and defined $args{"coderef"} 4 43 47
Condition93%14/15
defined $args{"rsc"} or defined $args{"min_stmt"} and defined $args{"max_stmt"} or not defined $args{"dbi_connector"} || defined $args{"dbic_storage"} and defined $args{"coderef"}
l!l&&r!l&&!r
3944
defined $args{"rsc"} or defined $args{"min_stmt"} and defined $args{"max_stmt"}
l!l&&r!l&&!r
22178
defined $args{"min_stmt"} and defined $args{"max_stmt"}
!ll&&!rl&&r
6217
not defined $args{"dbi_connector"} || defined $args{"dbic_storage"} and defined $args{"coderef"}
!ll&&!rl&&r
224
defined $args{'dbi_connector'} || defined $args{'dbic_storage'}
l!l&&r!l&&!r
206
MC/DC66%4/6
defined $args{"rsc"} or defined $args{"min_stmt"} and defined $args{"max_stmt"} or not defined $args{"dbi_connector"} || defined $args{"dbic_storage"} and defined $args{"coderef"}
defined $args{"rsc"} defined $args{"min_stmt"} defined $args{"max_stmt"} defined $args{'dbi_connector'} defined $args{'dbic_storage'} defined $args{"coderef"}
Truth table50%5/10
A or B and C or not(D || E) and F
A= defined $args{"rsc"}B= defined $args{"min_stmt"}C= defined $args{"max_stmt"}D= defined $args{'dbi_connector'}E= defined $args{'dbic_storage'}F= defined $args{"coderef"}
ABCDEFresult
1XXXXX1
011XXX1
00X0011
0100011
00X1XX0
00X01X0
00X0000
0101XX0
01001X0
0100000
956  
957 die 'Block execution requires one of these attr sets: dbi_connector|dbic_storage + stmt, rs + coderef, or coderef' unless (
958 $args{stmt} ||
959 (defined $args{rs} && $args{coderef}) ||
960 $args{coderef}
961 43 ▶ );
Branch: 1 branch
BranchTrueFalseTotal
unless $args{"stmt"} or defined $args{"rs"} and $args{"coderef"} or $args{"coderef"} 1 42 43
Condition100%9/9
$args{"stmt"} or defined $args{"rs"} and $args{"coderef"} or $args{"coderef"}
l!l&&r!l&&!r
3481
$args{"stmt"} or defined $args{"rs"} and $args{"coderef"}
l!l&&r!l&&!r
17179
defined $args{"rs"} and $args{"coderef"}
!ll&&!rl&&r
8117
MC/DC50%2/4
$args{"stmt"} or defined $args{"rs"} and $args{"coderef"} or $args{"coderef"}
$args{"stmt"} defined $args{"rs"} $args{"coderef"} $args{"coderef"}
Truth table66%4/6
A or B and C or D
A= $args{"stmt"}B= defined $args{"rs"}C= $args{"coderef"}D= $args{"coderef"}
ABCDresult
1XXX1
011X1
00X11
01011
00X00
01000
962  
963 42 ▶ if (exists $args{target_time} && $args{target_time} == 0 && !$args{chunk_size}) {
Branch: 1 branch
BranchTrueFalseTotal
if (exists $args{"target_time"} and $args{"target_time"} == 0 and not $args{"chunk_size"}) 0 42 42
Condition83%5/6
exists $args{"target_time"} and $args{"target_time"} == 0 and not $args{"chunk_size"}
!ll&&!rl&&r
12300
exists $args{"target_time"} and $args{"target_time"} == 0
!ll&&!rl&&r
8430
MC/DC0%0/3
exists $args{"target_time"} and $args{"target_time"} == 0 and not $args{"chunk_size"}
exists $args{"target_time"} $args{"target_time"} == 0 not $args{"chunk_size"}
Truth table75%3/4
A and B and not(C)
A= exists $args{"target_time"}B= $args{"target_time"} == 0C= not $args{"chunk_size"}
ABCresult
0XX0
10X0
1100
1111
964 0 warn join "\n",
965 'Dynamic chunk resizing is turned off and the chunk_size is still set to its default of 1.',
966 'This is probably not desirable, and you should find an appropriate static chunk size for',
967 'your workload.',
968 ''
969 ;
970 }
971  
972 42 $class->$next( %args );
973 9 };
974  
975 39 ▶sub BUILD {
Subroutine BUILD: called CC=1 SCAR=0.0
Pod undocumented
976 39 my $self = shift;
977 # Make sure id_name gets fixed at the right time
978 39 $self->_fix_id_name( $self->id_name );
979 39 $self->_check_bignums;
980 }
981  
982 #pod =head1 CONSTRUCTORS
983 #pod
984 #pod See L</ATTRIBUTES> for information on what can be passed into these constructors.
985 #pod
986 #pod =head2 new
987 #pod
988 #pod my $batch_chunker = DBIx::BatchChunker->new(...);
989 #pod
990 #pod A standard object constructor. If you use this constructor, you will need to
991 #pod manually call L</calculate_ranges> and L</execute> to execute the DB changes.
992 #pod
993 #pod =head2 construct_and_execute
994 #pod
995 #pod my $batch_chunker = DBIx::BatchChunker->construct_and_execute(...);
996 #pod
997 #pod Constructs a DBIx::BatchChunker object and automatically calls
998 #pod L</calculate_ranges> and L</execute> on it. Anything passed to this method will be passed
999 #pod through to the constructor.
1000 #pod
1001 #pod Returns the constructed object, post-execution. This is typically only useful if you want
1002 #pod to inspect the attributes after the process has finished. Otherwise, it's safe to just
1003 #pod ignore the return and throw away the object immediately.
1004 #pod
1005 #pod =cut
1006  
1007 2 ▶sub construct_and_execute {
Subroutine construct_and_execute: called CC=1 SCAR=0.0
Pod documented
1008 2 my $class = shift;
1009 2 my $db_change = $class->new(@_);
1010  
1011 1 $db_change->calculate_ranges;
1012 1 $db_change->execute;
1013  
1014 1 return $db_change;
1015 }
1016  
1017 #pod =head1 METHODS
1018 #pod
1019 #pod =head2 calculate_ranges
1020 #pod
1021 #pod my $batch_chunker = DBIx::BatchChunker->new(
1022 #pod rsc => $account_rsc, # a ResultSetColumn
1023 #pod ### OR ###
1024 #pod rs => $account_rs, # a ResultSet
1025 #pod id_name => 'account_id', # can be looked up if not provided
1026 #pod ### OR ###
1027 #pod dbi_connector => $conn, # DBIx::Connector::Retry object
1028 #pod min_stmt => $min_stmt, # a SQL statement or DBI $sth args
1029 #pod max_stmt => $max_stmt, # ditto
1030 #pod
1031 #pod ### Optional but recommended ###
1032 #pod id_name => 'account_id', # will also be added into the progress bar title
1033 #pod chunk_size => 20_000, # default is 1000
1034 #pod
1035 #pod ### Optional ###
1036 #pod progress_bar => $progress, # defaults to a 2-count 'Calculating ranges' bar
1037 #pod
1038 #pod # ...other attributes for execute...
1039 #pod );
1040 #pod
1041 #pod my $has_data_to_process = $batch_chunker->calculate_ranges;
1042 #pod
1043 #pod Given a L<DBIx::Class::ResultSetColumn>, L<DBIx::Class::ResultSet>, or L<DBI> statement
1044 #pod argument set, this method calculates the min/max IDs of those objects. It fills in the
1045 #pod L</min_id> and L</max_id> attributes, based on the ID data, and then returns 1.
1046 #pod
1047 #pod If either of the min/max statements don't return any ID data, this method will return 0.
1048 #pod
1049 #pod =cut
1050  
1051 33 ▶sub calculate_ranges {
Subroutine calculate_ranges: called CC=10 SCAR=23.2
Pod documented
1052 33 my $self = shift;
1053  
1054 33 ▶ my $column_name = $self->id_name || '';
Condition100%2/2
$self->id_name || ''
l!l
2013
MC/DC100%1/1
$self->id_name || ''
$self->id_name
Truth table100%2/2
A
A= $self->id_name
Aresult
11
00
1055 33 $column_name =~ s/^\w+\.//;
1056  
1057 33 ▶ my $progress = $self->progress_bar || Term::ProgressBar->new({
Branch: 1 branch
BranchTrueFalseTotal
$column_name ? : 20 13 33
Condition33%2/6
$self->progress_bar || 'Term::ProgressBar'->new({'name', 'Calculating ranges' . ($column_name ? " for $column_name" : ''), 'count', 2, 'ETA', 'linear', 'silent', !(-t *STDERR && -t *STDIN)})
l!l&&r!l&&!r
0330
-t *STDERR && -t *STDIN
!ll&&!rl&&r
3300
MC/DC0%0/2
$self->progress_bar || 'Term::ProgressBar'->new({'name', 'Calculating ranges' . ($column_name ? " for $column_name" : ''), 'count', 2, 'ETA', 'linear', 'silent', !(-t *STDERR && -t *STDIN)})
$self->progress_bar 'Term::ProgressBar'->new({'name', 'Calculating ranges' . ($column_name ? " for $column_name" : ''), 'count', 2, 'ETA', 'linear', 'silent', !(-t *STDERR && -t *STDIN)})
MC/DC0%0/2
-t *STDERR && -t *STDIN
-t *STDERR -t *STDIN
Truth table33%1/3
A || B
A= $self->progress_barB= 'Term::ProgressBar'->new({'name', 'Calculating ranges' . ($column_name ? " for $column_name" : ''), 'count', 2, 'ETA', 'linear', 'silent', !(-t *STDERR && -t *STDIN)})
ABresult
1X1
011
000
Truth table33%1/3
A && B
A= -t *STDERRB= -t *STDIN
ABresult
0X0
100
111
1058 name => 'Calculating ranges'.($column_name ? " for $column_name" : ''),
1059 count => 2,
1060 ETA => 'linear',
1061 silent => !(-t *STDERR && -t *STDIN), # STDERR is what {fh} is set to use
1062 });
1063  
1064 # Actually run the statements
1065 33 my ($min_id, $max_id);
1066 33 ▶ if ($self->rsc) {
Branch: 2 branches
BranchTrueFalseTotal
if ($self->rsc) { } 19 14 33
elsif ($self->dbic_storage) { } 5 9 14
1067 19 ▶ $self->_dbic_block_runner( run => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
1068 # In case the sub is retried
1069 19 $progress->update(0);
1070  
1071 19 $min_id = $self->rsc->min;
1072 19 $progress->update(1);
1073  
1074 19 $max_id = $self->rsc->max;
1075 19 $progress->update(2);
1076 19 });
1077 }
1078 elsif ($self->dbic_storage) {
1079 5 ▶ $self->_dbic_block_runner( run => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
1080 5 my $dbh = $self->dbic_storage->dbh;
1081  
1082 # In case the sub is retried
1083 5 $progress->update(0);
1084  
1085 5 ($min_id) = $dbh->selectrow_array(@{ $self->min_stmt });
1086 5 $progress->update(1);
1087  
1088 5 ($max_id) = $dbh->selectrow_array(@{ $self->max_stmt });
1089 5 $progress->update(2);
1090 5 });
1091 }
1092 else {
1093 9 ▶ $self->dbi_connector->run(sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
1094 9 my $dbh = $_;
1095  
1096 # In case the sub is retried
1097 9 $progress->update(0);
1098  
1099 9 ($min_id) = $dbh->selectrow_array(@{ $self->min_stmt });
1100 9 $progress->update(1);
1101  
1102 9 ($max_id) = $dbh->selectrow_array(@{ $self->max_stmt });
1103 9 $progress->update(2);
1104 9 });
1105 }
1106  
1107 # Set the ranges and return
1108 33 ▶ return 0 unless defined $min_id && defined $max_id;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $min_id and defined $max_id 0 33 33
Condition33%1/3
defined $min_id and defined $max_id
!ll&&!rl&&r
0033
MC/DC0%0/2
defined $min_id and defined $max_id
defined $min_id defined $max_id
Truth table33%1/3
A and B
A= defined $min_idB= defined $max_id
ABresult
0X0
100
111
1109  
1110 # This would be the primary spot where we notice we need to upgrade, so check the values before
1111 # we attempt to mangle them.
1112 33 ▶ if ($self->_check_bignums($min_id, $max_id)) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->_check_bignums($min_id, $max_id)) { } 2 31 33
1113 2 $min_id = Math::BigFloat->new($min_id)->as_int;
1114 2 $max_id = Math::BigFloat->new($max_id)->as_int;
1115 }
1116 else {
1117 31 $min_id = int $min_id;
1118 31 $max_id = int $max_id;
1119 }
1120  
1121 33 $self->min_id($min_id);
1122 33 $self->max_id($max_id);
1123  
1124 33 return 1;
1125 }
1126  
1127 #pod =head2 execute
1128 #pod
1129 #pod my $batch_chunker = DBIx::BatchChunker->new(
1130 #pod # ...other attributes for calculate_ranges...
1131 #pod
1132 #pod dbi_connector => $conn, # DBIx::Connector::Retry object
1133 #pod stmt => $do_stmt, # INSERT/UPDATE/DELETE $stmt with BETWEEN placeholders
1134 #pod ### OR ###
1135 #pod dbi_connector => $conn, # DBIx::Connector::Retry object
1136 #pod stmt => $select_stmt, # SELECT $stmt with BETWEEN placeholders
1137 #pod count_stmt => $count_stmt, # SELECT COUNT $stmt to be used for min_chunk_percent; optional
1138 #pod coderef => $coderef, # called code that does the actual work
1139 #pod ### OR ###
1140 #pod rs => $account_rs, # base ResultSet, which gets filtered with -between later on
1141 #pod id_name => 'account_id', # can be looked up if not provided
1142 #pod coderef => $coderef, # called code that does the actual work
1143 #pod ### OR ###
1144 #pod coderef => $coderef, # DIY database work; just pass the $start/$end IDs
1145 #pod
1146 #pod ### Optional but recommended ###
1147 #pod sleep => 0.25, # number of seconds to sleep each chunk; defaults to 0
1148 #pod process_past_max => 1, # use this if processing the whole table
1149 #pod single_rows => 1, # does $coderef get a single $row or the whole $chunk_rs / $stmt
1150 #pod min_chunk_percent => 0.25, # minimum row count of chunk size percentage; defaults to 0.5 (or 50%)
1151 #pod target_time => 5, # target runtime for dynamic chunk size scaling; default is 5 seconds
1152 #pod id_increment => 10, # distance between consecutive IDs; defaults to 1
1153 #pod max_runtime => 12 * 60 * 60, # stop processing after 12 hours
1154 #pod
1155 #pod progress_name => 'Updating Accounts', # easier than creating your own progress_bar
1156 #pod
1157 #pod ### Optional ###
1158 #pod progress_bar => $progress, # defaults to "Processing $source_name" bar
1159 #pod verbose => 1, # displays timing stats on each chunk
1160 #pod );
1161 #pod
1162 #pod $batch_chunker->execute if $batch_chunker->calculate_ranges;
1163 #pod
1164 #pod Applies the configured DB changes in chunks. Runs through the loop, processing a
1165 #pod statement handle, ResultSet, and/or coderef as it goes. Each loop iteration processes a
1166 #pod chunk of work, determined by L</chunk_size>.
1167 #pod
1168 #pod The L</calculate_ranges> method should be run first to fill in L</min_id> and L</max_id>.
1169 #pod If either of these are missing, the function will assume L</calculate_ranges> couldn't
1170 #pod find them and warn about it.
1171 #pod
1172 #pod More details can be found in the L</Processing Modes> and L</ATTRIBUTES> sections.
1173 #pod
1174 #pod =cut
1175  
1176 36 ▶sub execute {
Subroutine execute: called CC=21 SCAR=31.5
Pod documented
1177 36 my $self = shift;
1178 36 $self->_check_bignums;
1179  
1180 36 my $count;
1181 36 ▶ if (defined $self->min_id && defined $self->max_id) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $self->min_id and defined $self->max_id) 35 1 36
Condition66%2/3
defined $self->min_id and defined $self->max_id
!ll&&!rl&&r
1035
MC/DC50%1/2
defined $self->min_id and defined $self->max_id
defined $self->min_id defined $self->max_id
Truth table66%2/3
A and B
A= defined $self->min_idB= defined $self->max_id
ABresult
0X0
100
111
1182 35 $count = $self->max_id - $self->min_id + 1;
1183 }
1184  
1185 # Fire up the progress bar
1186 36 ▶ my $progress = $self->progress_bar || Term::ProgressBar->new({
Condition50%4/8
$self->progress_bar || 'Term::ProgressBar'->new({'name', $self->progress_name, 'count', $count || 1, 'ETA', 'linear', 'silent', !(-t *STDERR && -t *STDIN)})
l!l&&r!l&&!r
0360
$count || 1
l!l
351
-t *STDERR && -t *STDIN
!ll&&!rl&&r
3600
MC/DC0%0/2
$self->progress_bar || 'Term::ProgressBar'->new({'name', $self->progress_name, 'count', $count || 1, 'ETA', 'linear', 'silent', !(-t *STDERR && -t *STDIN)})
$self->progress_bar 'Term::ProgressBar'->new({'name', $self->progress_name, 'count', $count || 1, 'ETA', 'linear', 'silent', !(-t *STDERR && -t *STDIN)})
MC/DC100%1/1
$count || 1
$count
MC/DC0%0/2
-t *STDERR && -t *STDIN
-t *STDERR -t *STDIN
Truth table33%1/3
A || B
A= $self->progress_barB= 'Term::ProgressBar'->new({'name', $self->progress_name, 'count', $count || 1, 'ETA', 'linear', 'silent', !(-t *STDERR && -t *STDIN)})
ABresult
1X1
011
000
Truth table100%2/2
A
A= $count
Aresult
11
00
Truth table33%1/3
A && B
A= -t *STDERRB= -t *STDIN
ABresult
0X0
100
111
1187 name => $self->progress_name,
1188 count => $count || 1,
1189 ETA => 'linear',
1190 silent => !(-t *STDERR && -t *STDIN), # STDERR is what {fh} is set to use
1191 });
1192  
1193 36 ▶ unless ($count) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($count) 1 35 36
1194 1 $progress->message('No chunks; nothing to process...');
1195 1 return;
1196 }
1197  
1198 35 ▶ if ($self->verbose) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->verbose) 1 34 35
1199 $progress->message(
1200 sprintf "(%s total chunks; %s total rows)",
1201 1 map { $self->cldr->decimal_formatter->format($_) } ( ceil($count / ($self->chunk_size * $self->id_increment)), $count)
1202 );
1203 }
1204  
1205 # Loop state setup
1206 35 $self->clear_loop_state;
1207 35 my $ls = $self->loop_state( DBIx::BatchChunker::LoopState->new({
1208 batch_chunker => $self,
1209 progress_bar => $progress,
1210 }) );
1211  
1212 # Da loop
1213 35 ▶ while (
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $ls->prev_end or defined $ls->start or $ls->prev_end < $self->max_id) 740 34 774
Condition100%6/6
not defined $ls->prev_end or defined $ls->start or $ls->prev_end < $self->max_id
l!l&&r!l&&!r
16957134
not defined $ls->prev_end or defined $ls->start
l!l&&r!l&&!r
66103605
MC/DC100%3/3
not defined $ls->prev_end or defined $ls->start or $ls->prev_end < $self->max_id
not defined $ls->prev_end defined $ls->start $ls->prev_end < $self->max_id
Truth table100%4/4
not(A) or B or C
A= not defined $ls->prev_endB= defined $ls->startC= $ls->prev_end < $self->max_id
ABCresult
1XX1
01X1
0011
0000
1214 !defined $ls->prev_end || # first chunk
1215 defined $ls->start || # still in the middle of chunk resizing
1216 $ls->prev_end < $self->max_id # processed chunk, but still more to go
1217 ) {
1218 740 $ls->multiplier_range($ls->multiplier_range + $ls->multiplier_step);
1219  
1220 # this could be already set, if chunk resizing
1221 740 ▶ $ls->start(
Branch: 3 branches
BranchTrueFalseTotal
unless defined $ls->start 571 169 740
defined $ls->prev_end ? : 571 0 571
defined $ls->min_id ? : 0 0 0
1222 defined $ls->prev_end ? $ls->prev_end + 1 :
1223 defined $ls->min_id ? $ls->min_id :
1224 1
1225 ) unless defined $ls->start;
1226  
1227 740 $ls->end(
1228 min(
1229 $ls->start + ceil($ls->multiplier_range * $ls->chunk_size * $self->id_increment) - 1, # ceil, because multiplier_* could be fractional
1230 $self->max_id, # ensure we never exceed max_id
1231 )
1232 );
1233 740 $ls->chunk_count(undef);
1234  
1235 # Early loop exit because of maximum run time
1236 740 ▶ if ($self->max_runtime && time - $ls->total_timer > $self->max_runtime) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->max_runtime and &DBIx::BatchChunker::time() - $ls->total_timer > $self->max_runtime) 1 739 740
Condition100%3/3
$self->max_runtime and &DBIx::BatchChunker::time() - $ls->total_timer > $self->max_runtime
!ll&&!rl&&r
73631
MC/DC100%2/2
$self->max_runtime and &DBIx::BatchChunker::time() - $ls->total_timer > $self->max_runtime
$self->max_runtime &DBIx::BatchChunker::time() - $ls->total_timer > $self->max_runtime
Truth table100%3/3
A and B
A= $self->max_runtimeB= &DBIx::BatchChunker::time() - $ls->total_timer > $self->max_runtime
ABresult
0X0
100
111
1237 1 $progress->message('Ran past the maximum run time');
1238 1 last;
1239 }
1240  
1241 739 ▶ next unless $self->_process_past_max_checker;
Branch: 1 branch
BranchTrueFalseTotal
unless $self->_process_past_max_checker 0 739 739
1242  
1243 # The actual DB processing
1244 739 ▶ next unless $self->_process_block;
Branch: 1 branch
BranchTrueFalseTotal
unless $self->_process_block 169 570 739
1245  
1246 # Record the time quickly
1247 570 $ls->prev_runtime(time - $ls->chunk_timer);
1248  
1249 # Give the DB a little bit of breathing room
1250 570 ▶ sleep $self->sleep if $self->sleep;
Branch: 1 branch
BranchTrueFalseTotal
if $self->sleep 570 0 570
1251  
1252 570 $self->_print_chunk_status('processed');
1253 570 $self->_increment_progress;
1254 570 $self->_runtime_checker;
1255  
1256 # End-of-loop activities (skipped by early next)
1257 570 $ls->_reset_chunk_state;
1258 }
1259  
1260 # Re-set min_id and clear the loop state
1261 35 ▶ $self->min_id( $ls->prev_end ) if defined $ls->prev_end;
Branch: 1 branch
BranchTrueFalseTotal
if defined $ls->prev_end 35 0 35
1262 35 $self->clear_loop_state;
1263  
1264 # Keep the finished time from the progress bar, in case there are other loops or output
1265 35 ▶ unless ($progress->silent) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($progress->silent) 0 35 35
1266 0 $progress->update( $progress->target );
1267 0 print "\n";
1268 }
1269 }
1270  
1271 #pod =head1 PRIVATE METHODS
1272 #pod
1273 #pod =head2 _process_block
1274 #pod
1275 #pod Runs the DB work and passes it to the coderef. Its return value determines whether the
1276 #pod block should be processed or not.
1277 #pod
1278 #pod =cut
1279  
1280 739 ▶sub _process_block {
Subroutine _process_block: called CC=18 SCAR=28.9
1281 739 my ($self) = @_;
1282  
1283 739 my $ls = $self->loop_state;
1284 739 my $conn = $self->dbi_connector;
1285 739 my $coderef = $self->coderef;
1286 739 my $rs = $self->rs;
1287 739 ▶ my $count_rs = $self->count_rs // $rs;
Condition66%2/3
$self->count_rs // $rs
l!l&&r!l&&!r
3580381
MC/DC50%1/2
$self->count_rs // $rs
$self->count_rs $rs
Truth table66%2/3
A // B
A= $self->count_rsB= $rs
ABresult
1X1
011
000
1288  
1289 # Figure out if the row count is worth the work
1290 739 my $chunk_rs;
1291 739 my $count_stmt = $self->count_stmt;
1292 739 my $chunk_count;
1293 739 ▶ if ($count_stmt && defined $self->dbic_storage) {
Branch: 3 branches
BranchTrueFalseTotal
if ($count_stmt and defined $self->dbic_storage) { } 97 642 739
elsif ($count_stmt) { } 110 532 642
elsif (defined $count_rs) { } 358 174 532
Condition100%3/3
$count_stmt and defined $self->dbic_storage
!ll&&!rl&&r
53211097
MC/DC100%2/2
$count_stmt and defined $self->dbic_storage
$count_stmt defined $self->dbic_storage
Truth table100%3/3
A and B
A= $count_stmtB= defined $self->dbic_storage
ABresult
0X0
100
111
1294 97 ▶ $self->_dbic_block_runner( run => sub {
Subroutine __ANON__: called CC=2 SCAR=6.9
1295 97 ▶ $chunk_count = $self->dbic_storage->dbh->selectrow_array(
Branch: 1 branch
BranchTrueFalseTotal
@$count_stmt == 1 ? : 29 68 97
1296 @$count_stmt,
1297 (@$count_stmt == 1 ? undef : ()),
1298 $ls->start, $ls->end,
1299 );
1300 97 });
1301 }
1302 elsif ($count_stmt) {
1303 110 ▶ $chunk_count = $conn->run(sub {
Subroutine __ANON__: called CC=2 SCAR=6.9
1304 110 ▶ $_->selectrow_array(
Branch: 1 branch
BranchTrueFalseTotal
@$count_stmt == 1 ? : 72 38 110
1305 @$count_stmt,
1306 (@$count_stmt == 1 ? undef : ()),
1307 $ls->start, $ls->end,
1308 );
1309 110 });
1310 }
1311 elsif (defined $count_rs) {
1312 358 $chunk_rs = $count_rs->search({
1313 $self->id_name => { -between => [$ls->start, $ls->end] },
1314 });
1315  
1316 358 ▶ $self->_dbic_block_runner( run => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
1317 358 $chunk_count = $chunk_rs->count;
1318 358 });
1319 }
1320  
1321 739 ▶ $chunk_count = Math::BigInt->new($chunk_count) if $self->_check_bignums($chunk_count);
Branch: 1 branch
BranchTrueFalseTotal
if $self->_check_bignums($chunk_count) 141 598 739
1322 739 $ls->chunk_count($chunk_count);
1323  
1324 739 ▶ return unless $self->_chunk_count_checker;
Branch: 1 branch
BranchTrueFalseTotal
unless $self->_chunk_count_checker 169 570 739
1325  
1326 # NOTE: Try to minimize the amount of closures by using $self as much as possible
1327 # inside coderefs.
1328  
1329 # Do the work
1330 570 ▶ if (my $stmt = $self->stmt) {
Branch: 2 branches
BranchTrueFalseTotal
if (my $stmt = $self->stmt) { } 218 352 570
elsif (defined $rs and $coderef) { } 283 69 352
1331 ### Statement handle
1332 218 ▶ my @prepare_args = @$stmt > 2 ? @$stmt[0..1] : @$stmt;
Branch: 1 branch
BranchTrueFalseTotal
@$stmt > 2 ? : 62 156 218
1333 218 ▶ my @execute_args = (
Branch: 1 branch
BranchTrueFalseTotal
@$stmt > 2 ? : 62 156 218
1334 (@$stmt > 2 ? @$stmt[2..$#$stmt] : ()),
1335 $ls->start, $ls->end,
1336 );
1337  
1338 218 ▶ if ($self->single_rows && $coderef) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->single_rows and $coderef) { } 69 149 218
Condition66%2/3
$self->single_rows and $coderef
!ll&&!rl&&r
149069
MC/DC50%1/2
$self->single_rows and $coderef
$self->single_rows $coderef
Truth table66%2/3
A and B
A= $self->single_rowsB= $coderef
ABresult
0X0
100
111
1339 # Transactional work
1340 69 ▶ if ($self->dbic_storage) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->dbic_storage) { } 45 24 69
1341 50 ▶ $self->_dbic_block_runner( txn => sub {
Subroutine __ANON__: called CC=2 SCAR=6.9
1342 50 $self->loop_state->_mark_chunk_timer; # reset timer on retries
1343  
1344 50 my $sth = $self->dbic_storage->dbh->prepare(@prepare_args);
1345 50 $sth->execute(@execute_args);
1346  
1347 50 ▶ while (my $row = $sth->fetchrow_hashref('NAME_lc')) { $self->coderef->($self, $row) }
Branch: 1 branch
BranchTrueFalseTotal
if (my $row = $sth->fetchrow_hashref("NAME_lc")) 73 45 118
1348 45 });
1349 }
1350 else {
1351 87 ▶ $conn->txn(sub {
Subroutine __ANON__: called CC=2 SCAR=6.9
1352 87 $self->loop_state->_mark_chunk_timer; # reset timer on retries
1353  
1354 87 my $sth = $_->prepare(@prepare_args);
1355 87 $sth->execute(@execute_args);
1356  
1357 29 ▶ while (my $row = $sth->fetchrow_hashref('NAME_lc')) { $self->coderef->($self, $row) }
Branch: 1 branch
BranchTrueFalseTotal
if (my $row = $sth->fetchrow_hashref("NAME_lc")) 22 24 46
1358 24 });
1359 }
1360 }
1361 else {
1362 # Bulk work (or DML)
1363 149 ▶ if ($self->dbic_storage) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->dbic_storage) { } 53 96 149
1364 101 ▶ $self->_dbic_block_runner( run => sub {
Subroutine __ANON__: called CC=2 SCAR=6.9
1365 101 $self->loop_state->_mark_chunk_timer; # reset timer on retries
1366  
1367 101 my $sth = $self->dbic_storage->dbh->prepare(@prepare_args);
1368 101 $sth->execute(@execute_args);
1369  
1370 53 ▶ $self->coderef->($self, $sth) if $self->coderef;
Branch: 1 branch
BranchTrueFalseTotal
if $self->coderef 5 48 53
1371 53 });
1372 }
1373 else {
1374 144 ▶ $conn->run(sub {
Subroutine __ANON__: called CC=2 SCAR=6.9
1375 144 $self->loop_state->_mark_chunk_timer; # reset timer on retries
1376  
1377 144 my $sth = $_->prepare(@prepare_args);
1378 144 $sth->execute(@execute_args);
1379  
1380 96 ▶ $self->coderef->($self, $sth) if $self->coderef;
Branch: 1 branch
BranchTrueFalseTotal
if $self->coderef 10 86 96
1381 96 });
1382 }
1383 }
1384 }
1385 ▶ elsif (defined $rs && $coderef) {
Condition66%2/3
defined $rs and $coderef
!ll&&!rl&&r
690283
MC/DC50%1/2
defined $rs and $coderef
defined $rs $coderef
Truth table66%2/3
A and B
A= defined $rsB= $coderef
ABresult
0X0
100
111
1386 ### ResultSet with coderef
1387  
1388 283 ▶ if ($self->single_rows) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->single_rows) { } 81 202 283
1389 # Transactional work
1390 86 ▶ $self->_dbic_block_runner( txn => sub {
Subroutine __ANON__: called CC=2 SCAR=6.9
1391 # reset timer/$rs on retries
1392 86 $self->loop_state->_mark_chunk_timer;
1393 86 $chunk_rs->reset;
1394  
1395 86 ▶ while (my $row = $chunk_rs->next) { $self->coderef->($self, $row) }
Branch: 1 branch
BranchTrueFalseTotal
if (my $row = $chunk_rs->next) 76 81 157
1396 81 });
1397 }
1398 else {
1399 # Bulk work
1400 250 ▶ $self->_dbic_block_runner( run => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
1401 # reset timer/$rs on retries
1402 250 $self->loop_state->_mark_chunk_timer;
1403 250 $chunk_rs->reset;
1404  
1405 250 $self->coderef->($self, $chunk_rs);
1406 202 });
1407 }
1408 }
1409 else {
1410 ### Something a bit more free-form
1411  
1412 69 $self->$coderef($ls->start, $ls->end);
1413 }
1414  
1415 570 return 1;
1416 }
1417  
1418 #pod =head2 _process_past_max_checker
1419 #pod
1420 #pod Checks to make sure the current endpoint is actually the end, by checking the database.
1421 #pod Its return value determines whether the block should be processed or not.
1422 #pod
1423 #pod See L</process_past_max>.
1424 #pod
1425 #pod =cut
1426  
1427 739 ▶sub _process_past_max_checker {
Subroutine _process_past_max_checker: called CC=17 SCAR=35.0
1428 739 my ($self) = @_;
1429 739 my $ls = $self->loop_state;
1430 739 my $progress = $ls->progress_bar;
1431  
1432 739 ▶ return 1 unless $self->process_past_max;
Branch: 1 branch
BranchTrueFalseTotal
unless $self->process_past_max 648 91 739
1433 91 ▶ return 1 unless $ls->end >= $self->max_id;
Branch: 1 branch
BranchTrueFalseTotal
unless $ls->end >= $self->max_id 84 7 91
1434  
1435 # No checks for DIY, if they didn't include a max_stmt
1436 7 ▶ unless (defined $self->rsc || $self->max_stmt) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $self->rsc or $self->max_stmt) 1 6 7
Condition66%2/3
defined $self->rsc or $self->max_stmt
l!l&&r!l&&!r
601
MC/DC50%1/2
defined $self->rsc or $self->max_stmt
defined $self->rsc $self->max_stmt
Truth table66%2/3
A or B
A= defined $self->rscB= $self->max_stmt
ABresult
1X1
011
000
1437 # There's no way to size this, so add one more chunk
1438 1 $ls->end($self->max_id + $ls->chunk_size * $self->id_increment);
1439 1 return 1;
1440 }
1441  
1442 # Run another MAX check
1443 6 ▶ $progress->message('Reached end; re-checking max ID') if $self->verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $self->verbose 0 6 6
1444 6 my $new_max_id;
1445 6 ▶ if (defined( my $rsc = $self->rsc )) {
Branch: 2 branches
BranchTrueFalseTotal
if (defined(my $rsc = $self->rsc)) { } 6 0 6
elsif ($self->dbic_storage) { } 0 0 0
1446 6 ▶ $self->_dbic_block_runner( run => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
1447 6 $new_max_id = $rsc->max;
1448 6 });
1449 }
1450 elsif ($self->dbic_storage) {
1451 0 ▶ $self->_dbic_block_runner( run => sub {
Subroutine __ANON__: not called CC=1 SCAR=6.9
1452 0 ($new_max_id) = $self->dbic_storage->dbh->selectrow_array(@{ $self->max_stmt });
1453 0 });
1454 }
1455 else {
1456 0 ▶ ($new_max_id) = $self->dbi_connector->run(sub {
Subroutine __ANON__: not called CC=1 SCAR=6.9
1457 0 $_->selectrow_array(@{ $self->max_stmt });
1458 0 });
1459 }
1460 6 $ls->_mark_chunk_timer; # the above query shouldn't impact runtimes
1461  
1462 # Convert $new_max_id if necessary
1463 6 ▶ $new_max_id = Math::BigInt->new($new_max_id) if $self->_check_bignums($new_max_id);
Branch: 1 branch
BranchTrueFalseTotal
if $self->_check_bignums($new_max_id) 2 4 6
1464  
1465 6 ▶ if (!$new_max_id || $new_max_id eq '0E0') {
Branch: 3 branches
BranchTrueFalseTotal
if (not $new_max_id or $new_max_id eq '0E0') { } 0 6 6
elsif ($new_max_id > $self->max_id) { } 2 4 6
elsif ($new_max_id == $self->max_id) { } 4 0 4
Condition33%1/3
not $new_max_id or $new_max_id eq "0E0"
l!l&&r!l&&!r
006
MC/DC0%0/2
not $new_max_id or $new_max_id eq "0E0"
not $new_max_id $new_max_id eq "0E0"
Truth table33%1/3
not(A) or B
A= not $new_max_idB= $new_max_id eq "0E0"
ABresult
1X1
011
000
1466 # No max: No affected rows to change
1467 0 ▶ $progress->message('No max ID found; nothing left to process...') if $self->verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $self->verbose 0 0 0
1468 0 $ls->end($self->max_id);
1469  
1470 0 $ls->prev_check('no max');
1471 0 return 0;
1472 }
1473 elsif ($new_max_id > $self->max_id) {
1474 # New max ID
1475 2 ▶ $progress->message( sprintf 'New max ID set from %s to %s', $self->max_id, $new_max_id ) if $self->verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $self->verbose 0 2 2
1476 2 $self->max_id($new_max_id);
1477 2 $progress->target( $new_max_id - $self->min_id + 1 );
1478 2 $progress->update( $progress->last_update );
1479 }
1480 elsif ($new_max_id == $self->max_id) {
1481 # Same max ID
1482 4 ▶ $progress->message( sprintf 'Found max ID %s; same as end', $new_max_id ) if $self->verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $self->verbose 0 4 4
1483 }
1484 else {
1485 # Max too low
1486 0 ▶ $progress->message( sprintf 'Found max ID %s; ignoring...', $new_max_id ) if $self->verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $self->verbose 0 0 0
1487 }
1488  
1489 # Run another boundary check with the new max_id value
1490 6 $ls->end( min($ls->end, $self->max_id) );
1491  
1492 6 return 1;
1493 }
1494  
1495 #pod =head2 _chunk_count_checker
1496 #pod
1497 #pod Checks the chunk count to make sure it's properly sized. If not, it will try to shrink
1498 #pod or expand the current chunk (in C<< chunk_size * id_increment >> increments) as
1499 #pod necessary. Its return value determines whether the block should be processed or not.
1500 #pod
1501 #pod See L</min_chunk_percent>.
1502 #pod
1503 #pod This is not to be confused with the L</_runtime_checker>, which adjusts C<chunk_size>
1504 #pod after processing, based on previous run times.
1505 #pod
1506 #pod =cut
1507  
1508 739 ▶sub _chunk_count_checker {
Subroutine _chunk_count_checker: called CC=13 SCAR=25.7
1509 739 my ($self) = @_;
1510 739 my $ls = $self->loop_state;
1511 739 my $progress = $ls->progress_bar;
1512  
1513 # Chunk sizing is essentially disabled, so bounce out of here
1514 739 ▶ if ($self->min_chunk_percent <= 0 || !defined $ls->chunk_count) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->min_chunk_percent <= 0 or not defined $ls->chunk_count) 457 282 739
Condition100%3/3
$self->min_chunk_percent <= 0 or not defined $ls->chunk_count
l!l&&r!l&&!r
331126282
MC/DC100%2/2
$self->min_chunk_percent <= 0 or not defined $ls->chunk_count
$self->min_chunk_percent <= 0 not defined $ls->chunk_count
Truth table100%3/3
A or not(B)
A= $self->min_chunk_percent <= 0B= not defined $ls->chunk_count
ABresult
1X1
011
000
1515 457 $ls->prev_check('disabled');
1516 457 return 1;
1517 }
1518  
1519 282 my $chunk_percent = $ls->chunk_count / $ls->chunk_size;
1520 282 my $count_check_time = time - $ls->chunk_timer; # should only include the COUNT time at this point
1521 282 $ls->checked_count( $ls->checked_count + 1 );
1522  
1523 282 ▶ if ($ls->chunk_count == 0 && $self->min_chunk_percent > 0) {
Branch: 7 branches
BranchTrueFalseTotal
if ($ls->chunk_count == 0 and $self->min_chunk_percent > 0) { } 35 247 282
elsif ($ls->end - $ls->start <= 0) { } 10 237 247
elsif ($chunk_percent > 1 + $self->min_chunk_percent) { } 27 210 237
elsif ($self->target_time and $count_check_time > $self->target_time * 1.05) { } 31 179 210
elsif ($ls->checked_count > 10) { } 0 179 179
elsif ($ls->end >= $self->max_id) { } 13 166 179
elsif ($chunk_percent < $self->min_chunk_percent) { } 76 90 166
Condition66%2/3
$ls->chunk_count == 0 and $self->min_chunk_percent > 0
!ll&&!rl&&r
247035
MC/DC50%1/2
$ls->chunk_count == 0 and $self->min_chunk_percent > 0
$ls->chunk_count == 0 $self->min_chunk_percent > 0
Truth table66%2/3
A and B
A= $ls->chunk_count == 0B= $self->min_chunk_percent > 0
ABresult
0X0
100
111
1524 # No rows: Skip the block entirely, and accelerate the stepping
1525 35 $self->_print_chunk_status('skipped');
1526  
1527 35 $self->_increment_progress;
1528  
1529 35 my $step = $ls->multiplier_step;
1530 35 $ls->_reset_chunk_state;
1531 35 $ls->multiplier_step( $step * 2 );
1532  
1533 35 $ls->prev_check('skipped rows');
1534 35 return 0;
1535 }
1536 elsif ($ls->end - $ls->start <= 0) {
1537 # Down to a single ID: We _have_ to process it
1538 10 $ls->prev_check('at a single ID');
1539  
1540 # Complain, because this can be dangerous with a wild enough Row:ID ratio
1541 10 ▶ if ($ls->chunk_count > 1) {
Branch: 1 branch
BranchTrueFalseTotal
if ($ls->chunk_count > 1) 9 1 10
1542 9 $progress->message('WARNING: Processing a single ID with many rows attached because resizing cannot proceed any further.');
1543 9 $progress->message('Consider flipping the relationship so that IDs and row counts are 1:1.');
1544 }
1545  
1546 10 return 1;
1547 }
1548 elsif ($chunk_percent > 1 + $self->min_chunk_percent) {
1549 # Too many rows: Backtrack to the previous range and try to bisect
1550 27 $self->_print_chunk_status('shrunk');
1551 27 $ls->_mark_chunk_timer;
1552 27 $ls->_decrease_multiplier;
1553 27 $ls->prev_check('too many rows');
1554 27 return 0;
1555 }
1556 ▶ elsif ($self->target_time && $count_check_time > $self->target_time * 1.05) {
Condition100%3/3
$self->target_time and $count_check_time > $self->target_time * "1.05"
!ll&&!rl&&r
1572231
MC/DC100%2/2
$self->target_time and $count_check_time > $self->target_time * "1.05"
$self->target_time $count_check_time > $self->target_time * "1.05"
Truth table100%3/3
A and B
A= $self->target_timeB= $count_check_time > $self->target_time * "1.05"
ABresult
0X0
100
111
1557 # COUNT statement too slow: Backtrack to the previous range and try to bisect
1558  
1559 # This is a rare failure, so print a warning
1560 31 my $integer = $self->cldr->decimal_formatter;
1561 31 my $decimal = $self->cldr->decimal_formatter(
1562 minimum_fraction_digits => 2,
1563 maximum_fraction_digits => 2,
1564 );
1565 31 $progress->message( sprintf(
1566 'WARNING: COUNT statement was too slow; took %5s sec to return %s rows.',
1567 $decimal->format($count_check_time),
1568 $integer->format( $ls->chunk_count )
1569 ) );
1570  
1571 31 $self->_print_chunk_status('shrunk');
1572 31 $ls->_mark_chunk_timer;
1573 31 $ls->_decrease_multiplier;
1574 31 $ls->prev_check('COUNT too slow');
1575 31 return 0;
1576 }
1577  
1578 # The above four are more important than skipping the count checks. Better to
1579 # have too few rows than too many. The single ID check prevents infinite loops
1580 # from bisecting, though.
1581  
1582 elsif ($ls->checked_count > 10) {
1583 # Checked too many times: Just process it
1584 0 $ls->prev_check('too many checks');
1585 0 return 1;
1586 }
1587 elsif ($ls->end >= $self->max_id) {
1588 # At the end: Just process it
1589 13 $ls->prev_check('at max_id');
1590 13 return 1;
1591 }
1592 elsif ($chunk_percent < $self->min_chunk_percent) {
1593 # Too few rows: Keep the start ID and accelerate towards a better endpoint
1594 76 $self->_print_chunk_status('expanded');
1595 76 $ls->_mark_chunk_timer;
1596 76 $ls->_increase_multiplier;
1597 76 $ls->prev_check('too few rows');
1598 76 return 0;
1599 }
1600  
1601 90 $ls->prev_check('nothing wrong');
1602 90 return 1;
1603 }
1604  
1605 #pod =head2 _runtime_checker
1606 #pod
1607 #pod Stores the previously processed chunk's runtime, and then adjusts C<chunk_size> as
1608 #pod necessary.
1609 #pod
1610 #pod See L</target_time>.
1611 #pod
1612 #pod =cut
1613  
1614 570 ▶sub _runtime_checker {
Subroutine _runtime_checker: called CC=17 SCAR=29.4
1615 570 my ($self) = @_;
1616 570 my $ls = $self->loop_state;
1617 570 ▶ return unless $self->target_time;
Branch: 1 branch
BranchTrueFalseTotal
unless $self->target_time 431 139 570
1618 139 ▶ return unless $ls->chunk_size && $ls->prev_runtime; # prevent DIV/0
Branch: 1 branch
BranchTrueFalseTotal
unless $ls->chunk_size and $ls->prev_runtime 0 139 139
Condition33%1/3
$ls->chunk_size and $ls->prev_runtime
!ll&&!rl&&r
00139
MC/DC0%0/2
$ls->chunk_size and $ls->prev_runtime
$ls->chunk_size $ls->prev_runtime
Truth table33%1/3
A and B
A= $ls->chunk_sizeB= $ls->prev_runtime
ABresult
0X0
100
111
1619  
1620 139 my $timings = $ls->last_timings;
1621  
1622 139 ▶ my $new_timing = {
Condition66%2/3
$ls->chunk_count || $ls->chunk_size
l!l&&r!l&&!r
84550
MC/DC0%0/2
$ls->chunk_count || $ls->chunk_size
$ls->chunk_count $ls->chunk_size
Truth table66%2/3
A || B
A= $ls->chunk_countB= $ls->chunk_size
ABresult
1X1
011
000
1623 runtime => $ls->prev_runtime,
1624 chunk_count => $ls->chunk_count || $ls->chunk_size,
1625 };
1626 139 $new_timing->{chunk_per} = $new_timing->{chunk_count} / $ls->chunk_size;
1627  
1628 # Rowtime: a measure of how much of the chunk_size actually impacted the runtime
1629 139 $new_timing->{rowtime} = $new_timing->{runtime} / $new_timing->{chunk_per};
1630  
1631 # Store the last five processing times
1632 139 push @$timings, $new_timing;
1633 139 ▶ shift @$timings if @$timings > 5;
Branch: 1 branch
BranchTrueFalseTotal
if @$timings > 5 80 59 139
1634  
1635 # Figure out the averages and adjustment factor
1636 139 my $ttl = scalar @$timings;
1637 139 my $avg_rowtime = sum(map { $_->{rowtime} } @$timings) / $ttl;
1638 139 my $adjust_factor = $self->target_time / $avg_rowtime;
1639  
1640 139 my $new_target_chunk_size = $ls->chunk_size;
1641 139 my $adjective;
1642 139 ▶ if ($adjust_factor > 1.05) {
Branch: 2 branches
BranchTrueFalseTotal
if ($adjust_factor > 1.05) { } 49 90 139
elsif ($adjust_factor < 0.95) { } 90 0 90
1643 # Too fast: Raise the chunk size
1644  
1645 49 ▶ return unless $ttl >= 5; # must have a full set of timings
Branch: 1 branch
BranchTrueFalseTotal
unless $ttl >= 5 41 8 49
1646 8 ▶ return if any { $_->{runtime} >= $self->target_time } @$timings; # must ALL have low runtimes
Subroutine __ANON__: called CC=1 SCAR=0.2
Branch: 1 branch
BranchTrueFalseTotal
if &any(sub { $_->{"runtime"} >= $self->target_time; } , @$timings) 0 8 8
1647  
1648 8 $new_target_chunk_size *= min(2, $adjust_factor); # never more than double
1649 8 $adjective = 'fast';
1650 }
1651 elsif ($adjust_factor < 0.95) {
1652 # Too slow: Lower the chunk size
1653  
1654 90 ▶ return unless $ls->prev_runtime > $self->target_time; # last runtime must actually be too high
Branch: 1 branch
BranchTrueFalseTotal
unless $ls->prev_runtime > $self->target_time 21 69 90
1655  
1656 69 ▶ $new_target_chunk_size *=
Branch: 1 branch
BranchTrueFalseTotal
$ls->prev_runtime < $self->target_time * 3 ? : 0 69 69
1657 ($ls->prev_runtime < $self->target_time * 3) ?
1658 max(0.5, $adjust_factor) : # never less than half...
1659 $adjust_factor # ...unless the last runtime was waaaay off
1660 ;
1661 69 ▶ $new_target_chunk_size = 1 if $new_target_chunk_size < 1;
Branch: 1 branch
BranchTrueFalseTotal
if $new_target_chunk_size < 1 69 0 69
1662 69 $adjective = 'slow';
1663 }
1664  
1665 77 $new_target_chunk_size = int $new_target_chunk_size;
1666 77 ▶ return if $new_target_chunk_size == $ls->chunk_size; # either nothing changed or it's too miniscule
Branch: 1 branch
BranchTrueFalseTotal
if $new_target_chunk_size == $ls->chunk_size 68 9 77
1667 9 ▶ return if $new_target_chunk_size < 1;
Branch: 1 branch
BranchTrueFalseTotal
if $new_target_chunk_size < 1 0 9 9
1668  
1669 # Print out a processing line, if enabled
1670 9 ▶ if ($self->verbose) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->verbose) 0 9 9
1671 # CLDR number formatters
1672 0 my $integer = $self->cldr->decimal_formatter;
1673 0 my $percent = $self->cldr->percent_formatter;
1674  
1675 0 $ls->{progress_bar}->message( sprintf(
1676 "Processing too %s, avg %4s of target time, adjusting chunk size from %s to %s",
1677 $adjective,
1678 $percent->format( 1 / $adjust_factor ),
1679 $integer->format( $ls->chunk_size ),
1680 $integer->format( $new_target_chunk_size ),
1681 ) );
1682 }
1683  
1684 # Change it!
1685 9 $ls->chunk_size($new_target_chunk_size);
1686 9 ▶ $ls->_reset_last_timings if $adjective eq 'fast'; # never snowball too quickly
Branch: 1 branch
BranchTrueFalseTotal
if $adjective eq "fast" 8 1 9
1687 9 return 1;
1688 }
1689  
1690 #pod =head2 _increment_progress
1691 #pod
1692 #pod Increments the progress bar.
1693 #pod
1694 #pod =cut
1695  
1696 605 ▶sub _increment_progress {
Subroutine _increment_progress: called CC=2 SCAR=6.9
1697 605 my ($self) = @_;
1698 605 my $ls = $self->loop_state;
1699 605 my $progress = $ls->progress_bar;
1700  
1701 605 my $so_far = $ls->end - $self->min_id + 1;
1702 605 ▶ $progress->target($so_far+1) if $ls->end > $self->max_id;
Branch: 1 branch
BranchTrueFalseTotal
if $ls->end > $self->max_id 1 604 605
1703 605 $progress->update($so_far);
1704 }
1705  
1706 #pod =head2 _print_chunk_status
1707 #pod
1708 #pod Prints out a standard chunk status line, if L</verbose> is enabled. What it prints is
1709 #pod generally uniform, but it depends on the processing action. Most of the data is
1710 #pod pulled from L</loop_state>.
1711 #pod
1712 #pod =cut
1713  
1714 740 ▶sub _print_chunk_status {
Subroutine _print_chunk_status: called CC=13 SCAR=28.6
1715 740 my ($self, $action) = @_;
1716 740 ▶ return unless $self->verbose;
Branch: 1 branch
BranchTrueFalseTotal
unless $self->verbose 725 15 740
1717  
1718 15 my $ls = $self->loop_state;
1719 15 ▶ my $sleep = $self->sleep || 0;
Condition50%1/2
$self->sleep || 0
l!l
150
MC/DC0%0/1
$self->sleep || 0
$self->sleep
Truth table50%1/2
A
A= $self->sleep
Aresult
11
00
1720  
1721 # CLDR number formatters
1722 15 my $integer = $self->cldr->decimal_formatter;
1723 15 my $percent = $self->cldr->percent_formatter;
1724 15 my $decimal = $self->cldr->decimal_formatter(
1725 minimum_fraction_digits => 2,
1726 maximum_fraction_digits => 2,
1727 );
1728  
1729 15 my $chunk_count = $ls->chunk_count;
1730 15 ▶ my $count_display = defined $chunk_count
Branch: 2 branches
BranchTrueFalseTotal
defined $chunk_count ? : 14 1 15
$ls->start < 1000000000 && $ls->end < 1000000000 ? : 14 0 14
Condition33%1/3
$ls->start < 1000000000 and $ls->end < 1000000000
!ll&&!rl&&r
0014
MC/DC0%0/2
$ls->start < 1000000000 and $ls->end < 1000000000
$ls->start < 1000000000 $ls->end < 1000000000
Truth table33%1/3
A and B
A= $ls->start < 1000000000B= $ls->end < 1000000000
ABresult
0X0
100
111
1731 ? ($ls->start < 1_000_000_000 && $ls->end < 1_000_000_000
1732 ? $integer->format($chunk_count)
1733 : $chunk_count)
1734 : '?';
1735  
1736 15 my $message;
1737 15 ▶ if ($ls->start < 1_000_000_000 && $ls->end < 1_000_000_000) {
Branch: 1 branch
BranchTrueFalseTotal
if ($ls->start < 1000000000 and $ls->end < 1000000000) { } 15 0 15
Condition33%1/3
$ls->start < 1000000000 and $ls->end < 1000000000
!ll&&!rl&&r
0015
MC/DC0%0/2
$ls->start < 1000000000 and $ls->end < 1000000000
$ls->start < 1000000000 $ls->end < 1000000000
Truth table33%1/3
A and B
A= $ls->start < 1000000000B= $ls->end < 1000000000
ABresult
0X0
100
111
1738 15 $message = sprintf(
1739 'IDs %6u to %6u %9s, %9s rows found',
1740 $ls->start, $ls->end, $action,
1741 $count_display,
1742 );
1743 }
1744 else {
1745 0 $message = sprintf(
1746 'IDs %s to %s %s, %s rows found',
1747 $ls->start, $ls->end, $action,
1748 $count_display,
1749 );
1750 }
1751  
1752 15 ▶ $message .= sprintf(
Branch: 1 branch
BranchTrueFalseTotal
if $ls->chunk_count 12 3 15
1753 ' (%4s of chunk size)',
1754 $percent->format( $ls->chunk_count / $ls->chunk_size ),
1755 ) if $ls->chunk_count;
1756  
1757 15 ▶ if ($action eq 'processed') {
Branch: 1 branch
BranchTrueFalseTotal
if ($action eq "processed") 6 9 15
1758 6 ▶ $message .= $sleep ?
Branch: 1 branch
BranchTrueFalseTotal
$sleep ? : 6 0 6
1759 sprintf(
1760 ', %5s+%s sec runtime+sleep',
1761 $decimal->format( $ls->prev_runtime ),
1762 $decimal->format( $sleep )
1763 ) :
1764 sprintf(
1765 ', %5s sec runtime',
1766 $decimal->format( $ls->prev_runtime ),
1767 )
1768 ;
1769 }
1770  
1771 # Reduce spacing if the numbers are too large
1772 15 ▶ if ($ls->start > 1_000_000_000 || $ls->end > 1_000_000_000) {
Branch: 1 branch
BranchTrueFalseTotal
if ($ls->start > 1000000000 or $ls->end > 1000000000) 0 15 15
Condition33%1/3
$ls->start > 1000000000 or $ls->end > 1000000000
l!l&&r!l&&!r
0015
MC/DC0%0/2
$ls->start > 1000000000 or $ls->end > 1000000000
$ls->start > 1000000000 $ls->end > 1000000000
Truth table33%1/3
A or B
A= $ls->start > 1000000000B= $ls->end > 1000000000
ABresult
1X1
011
000
1773 0 $message =~ s/\s+/ /g;
1774 0 $message =~ s/\(\s+/\(/g;
1775 }
1776  
1777 15 return $ls->progress_bar->message($message);
1778 }
1779  
1780 #pod =head1 CAVEATS
1781 #pod
1782 #pod =head2 Big Number Support
1783 #pod
1784 #pod If the module detects that the ID numbers are no longer safe for standard Perl NV
1785 #pod storage, it will automatically switch to using L<Math::BigInt> and L<Math::BigFloat> for
1786 #pod big number support. If any blessed numbers are already being used to define the
1787 #pod attributes, this will also switch on the support.
1788 #pod
1789 #pod =head2 String-based IDs
1790 #pod
1791 #pod If you're working with C<VARCHAR> types or other string-based IDs to represent integers,
1792 #pod these may be subject to whatever string-based comparison rules your RDBMS uses when
1793 #pod calculating with C<MIN>/C<MAX> or using C<BETWEEN>. Row counting and chunk size scaling
1794 #pod will try to compensate, but will be mixing string-based comparisons from the RDBMS and
1795 #pod Perl-based integer math.
1796 #pod
1797 #pod Using the C<CAST> function may help, but it may also cause critical indexes to be
1798 #pod ignored, especially if the function is used on the left-hand side against the column.
1799 #pod Strings with the exact same length may be safe from comparison weirdness, but YMMV.
1800 #pod
1801 #pod Non-integer inputs from ID columns, such as GUIDs or other alphanumeric strings, are not
1802 #pod currently supported. They would have to be converted to integers via SQL, and doing so
1803 #pod may run into a similar risk of having your RDBMS ignore indexes.
1804 #pod
1805 #pod =head1 SEE ALSO
1806 #pod
1807 #pod L<DBIx::BulkLoader::Mysql>, L<DBIx::Class::BatchUpdate>, L<DBIx::BulkUtil>
1808 #pod
1809 #pod =cut
1810  
1811 9 1;
1812  
1813 __END__
↑ Summary BatchChunker/LoopState.pm »