blib/lib/Coro/DataPipe.pm

filter: Statementstmt 95.2% 100 / 105 Branchbran 61.1% 22 / 36 Conditioncond 78.2% 18 / 23 MC/DCmcdc 60.0% 9 / 15 Subroutinesub 100% 21 / 21 Podpod 0.0% 0 / 11 totaltotal 80.5% 170 / 211 CC 30 SCAR 18.5
CC
30
Cov
85%
pipeline
13.1
run_pipes
11.5
new
4.1
CRAP 6.4
↑ Summary
1 package Coro::DataPipe;
2  
3 1 our $VERSION='0.02';
4 1 ▶use 5.006; # Perl::MinimumVersion says that
Subroutine BEGIN: called CC=1 SCAR=0.0
5  
6 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
7 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
8 1 ▶use List::Util qw(first);
Subroutine BEGIN: called CC=1 SCAR=0.0
9 1 ▶use Coro; #qw(schedule async);
Subroutine BEGIN: called CC=1 SCAR=0.0
10  
11 # run is single parallel pipe which is trivial case of pipeline
12 1 ▶sub run {
Subroutine run: called CC=1 SCAR=0.0
Pod undocumented
13 1 pipeline(@_);
14 }
15  
16 2 ▶sub pipeline {
Subroutine pipeline: called CC=10 SCAR=25.7
Pod undocumented
17 2 my $class=shift;
18 2 ▶ if (ref($class) eq 'HASH') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $class eq "HASH") 2 0 2
19 2 unshift @_, $class;
20 2 $class = __PACKAGE__;
21 }
22 2 my @pipes;
23 # init pipes
24 my $default_input;
25 2 for my $param (@_) {
26 3 ▶ unless (exists $param->{input}) {
Branch: 1 branch
BranchTrueFalseTotal
unless (exists $param->{"input"}) 1 2 3
27 1 ▶ $param->{input} = $default_input or die "You have to specify input for the first pipe";
Branch: 1 branch
BranchTrueFalseTotal
$param->{"input"} = $default_input or die "You have to specify input for the first pipe" 1 0 1
28 }
29 3 my $pipe = $class->new($param);
30 3 ▶ if (ref($pipe->{output}) eq 'ARRAY') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $pipe->{"output"} eq "ARRAY") 3 0 3
31 3 $default_input = $pipe->{output};
32 }
33 3 push @pipes, $pipe;
34 }
35 2 run_pipes(0,@pipes);
36 2 my $result = $pipes[$#pipes]->{output};
37 # @pipes=() kills parent
38 # as well as its implicit destroying
39 # destroy pipes one by one if you want to survive!!!
40 2 undef $_ for @pipes;
41 2 ▶ return unless defined(wantarray);
Branch: 1 branch
BranchTrueFalseTotal
unless defined wantarray 2 0 2
42 0 ▶ return unless $result;
Branch: 1 branch
BranchTrueFalseTotal
unless $result 0 0 0
43 0 ▶ return wantarray?@$result:$result;
Branch: 1 branch
BranchTrueFalseTotal
wantarray ? : 0 0 0
44 }
45  
46 31 ▶sub run_pipes {
Subroutine run_pipes: called CC=11 SCAR=24.4
Pod undocumented
47 31 my ($prev_busy,$me,@next) = @_;
48 31 while (1) {
49 79 my $data_loaded = $me->load_data;
50 79 ▶ my $me_busy = $data_loaded || $me->busy_processors;
Condition100%3/3
$data_loaded || $me->busy_processors
l!l&&r!l&&!r
7261
MC/DC100%2/2
$data_loaded || $me->busy_processors
$data_loaded $me->busy_processors
Truth table100%3/3
A || B
A= $data_loadedB= $me->busy_processors
ABresult
1X1
011
000
51 # get processed data
52 79 ▶ schedule if $me_busy;
Branch: 1 branch
BranchTrueFalseTotal
if $me_busy 78 1 79
53 # push it to next pipe
54 79 ▶ $me_busy = $data_loaded || $me->busy_processors;
Condition100%3/3
$data_loaded || $me->busy_processors
l!l&&r!l&&!r
7243
MC/DC100%2/2
$data_loaded || $me->busy_processors
$data_loaded $me->busy_processors
Truth table100%3/3
A || B
A= $data_loadedB= $me->busy_processors
ABresult
1X1
011
000
55 79 ▶ my $next_busy = @next && run_pipes($prev_busy || $me_busy, @next);
Condition66%4/6
@next && run_pipes($prev_busy || $me_busy, @next)
!ll&&!rl&&r
50290
$prev_busy || $me_busy
l!l&&r!l&&!r
0281
MC/DC0%0/2
@next && run_pipes($prev_busy || $me_busy, @next)
@next run_pipes($prev_busy || $me_busy, @next)
MC/DC50%1/2
$prev_busy || $me_busy
$prev_busy $me_busy
Truth table66%2/3
A && B
A= @nextB= run_pipes($prev_busy || $me_busy, @next)
ABresult
0X0
100
111
Truth table66%2/3
A || B
A= $prev_busyB= $me_busy
ABresult
1X1
011
000
56 # I am busy either when I am already busy or my child are busy
57 79 ▶ $me_busy ||= $next_busy;
Condition66%2/3
$me_busy ||= $next_busy
l!l&&r!l&&!r
7603
MC/DC50%1/2
$me_busy ||= $next_busy
$me_busy $next_busy
Truth table66%2/3
A ||= B
A= $me_busyB= $next_busy
ABresult
1X1
011
000
58 # pipeline is free if every pipe is free and no more data to process
59 79 ▶ return 0 unless $me_busy || $data_loaded;
Branch: 1 branch
BranchTrueFalseTotal
unless $me_busy or $data_loaded 3 76 79
Condition66%2/3
$me_busy or $data_loaded
l!l&&r!l&&!r
7603
MC/DC50%1/2
$me_busy or $data_loaded
$me_busy $data_loaded
Truth table66%2/3
A or B
A= $me_busyB= $data_loaded
ABresult
1X1
011
000
60 # get data from pipe if we have free_processors
61 76 ▶ return $me_busy if $prev_busy && $me->free_processors;
Branch: 1 branch
BranchTrueFalseTotal
if $prev_busy and $me->free_processors 28 48 76
Condition66%2/3
$prev_busy and $me->free_processors
!ll&&!rl&&r
48028
MC/DC50%1/2
$prev_busy and $me->free_processors
$prev_busy $me->free_processors
Truth table66%2/3
A and B
A= $prev_busyB= $me->free_processors
ABresult
0X0
100
111
62 }
63 0 return 0;
64 }
65  
66 3 ▶sub set_input_iterator {
Subroutine set_input_iterator: called CC=3 SCAR=11.3
Pod undocumented
67 3 my ($self,$param) = @_;
68 3 my ($input_iterator) = extract_param($param, qw(input));
69 3 ▶ if (ref($input_iterator) ne 'CODE') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $input_iterator ne "CODE") 3 0 3
70 3 ▶ die "array or code reference expected for input_iterator" unless ref($input_iterator) eq 'ARRAY';
Branch: 1 branch
BranchTrueFalseTotal
unless ref $input_iterator eq "ARRAY" 0 3 3
71 3 my $queue = $input_iterator;
72 3 ▶ $input_iterator = sub {$queue?shift(@$queue):undef};
Subroutine __ANON__: called CC=2 SCAR=7.2
Branch: 1 branch
BranchTrueFalseTotal
$queue ? : 25013 0 25013
73 }
74 3 $self->{input_iterator} = $input_iterator;
75 }
76  
77 3 ▶sub set_output_iterator {
Subroutine set_output_iterator: called CC=3 SCAR=11.0
Pod undocumented
78 3 my ($self,$param) = @_;
79 3 my ($output_iterator) = extract_param($param, qw(output));
80 3 ▶ if (ref($output_iterator) ne 'CODE') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $output_iterator ne "CODE") 3 0 3
81 3 ▶ my $queue = $output_iterator || [];
Condition100%2/2
$output_iterator || []
l!l
21
MC/DC100%1/1
$output_iterator || []
$output_iterator
Truth table100%2/2
A
A= $output_iterator
Aresult
11
00
82 3 $self->{output} = $queue;
83 3 ▶ $output_iterator = sub {push @$queue,$_};
Subroutine __ANON__: called CC=1 SCAR=0.0
84 }
85 3 $self->{output_iterator} = $output_iterator;
86 }
87  
88 3 ▶sub set_process_iterator {
Subroutine set_process_iterator: called CC=1 SCAR=0.0
Pod undocumented
89 3 my ($self,$param) = @_;
90 3 my $process_data_callback = extract_param($param,qw(process));
91 3 my $main = $Coro::current;
92 25000 ▶ $self->{process_iterator} = sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
93 25000 my $data = shift;
94 25000 my $item_number = $self->{item_number}++;
95 25000 $self->{busy}++;
96 25000 ▶ my $coro = async {
Subroutine __ANON__: called CC=1 SCAR=0.0
97 25000 local $_ = $data;
98 25000 $_ = $process_data_callback->($data);
99 25000 $self->{output_iterator}->($_,$item_number);
100 25000 $self->{busy}--;
101 25000 $main->ready;
102 25000 };
103 3 };
104 }
105  
106 # loads all free processor with data from input
107 # return the number of loaded processors
108 79 ▶sub load_data {
Subroutine load_data: called CC=3 SCAR=11.0
Pod undocumented
109 79 my $self = shift;
110 79 my $result = 0;
111 79 ▶ while ($self->free_processors) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->free_processors) 25013 66 25079
112 25013 my $data = $self->{input_iterator}->();
113 25013 ▶ return $result unless defined($data);
Branch: 1 branch
BranchTrueFalseTotal
unless defined $data 13 25000 25013
114 25000 $self->{process_iterator}->($data);
115 25000 $result++;
116 }
117 66 return $result;
118 }
119  
120 12 ▶sub extract_param {
Subroutine extract_param: called CC=1 SCAR=0.0
Pod undocumented
121 12 my ($param, @alias) = @_;
122 12 ▶ return first {defined($_)} map delete($param->{$_}), @alias;
Subroutine __ANON__: called CC=1 SCAR=0.0
123 }
124  
125 14 ▶sub busy_processors {
Subroutine busy_processors: called CC=1 SCAR=0.0
Pod undocumented
126 14 my $self = shift;
127 14 return $self->{busy};
128 }
129  
130 25107 ▶sub free_processors {
Subroutine free_processors: called CC=1 SCAR=0.0
Pod undocumented
131 25107 my $self = shift;
132 25107 return $self->{busy} < $self->{number_of_data_processors};
133 }
134  
135 3 ▶sub new {
Subroutine new: called CC=4 SCAR=14.2
Pod undocumented
136 3 my ($class, $param) = @_;
137 3 my $self = {};
138 3 bless $self,$class;
139 # this is cooperative, so it's better to set explicit number of processor - your better know when it wins
140 3 my $number_of_data_processors = extract_param($param,'number_of_data_processors');
141 3 ▶ unless ($number_of_data_processors) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($number_of_data_processors) 0 3 3
142 0 $number_of_data_processors = 2;
143 0 warn "number_of_data_processors set to $number_of_data_processors";
144 }
145 3 $self->{number_of_data_processors} = $number_of_data_processors;
146 # item_number & busy
147 3 $self->{$_} = 0 for qw(item_number busy);
148 3 $self->set_input_iterator($param);
149 3 $self->set_output_iterator($param);
150 3 $self->set_process_iterator($param);
151 3 my $not_supported = join ", ", keys %$param;
152 3 ▶ die "Parameters are redundant or not supported:". $not_supported if $not_supported;
Branch: 1 branch
BranchTrueFalseTotal
if $not_supported 0 3 3
153 3 return $self;
154 }
155  
156 1 1;
157  
158 =head1 NAME
159  
160 C<Coro::DataPipe> - parallel data processing conveyor
161  
162 =encoding utf-8
163  
164 =head1 SYNOPSIS
165  
166 use Coro::AnyEvent;
167 use Coro::DataPipe;
168 Coro::DataPipe::run {
169 input => [1..100],
170 process => sub { Coro::AnyEvent::sleep(1);$_*2 },
171 number_of_data_processors => 100,
172 output => sub { print "$_\n" },
173 };
174  
175 time perl test.t >/dev/null
176 # 1 second, not 100!
177  
178 =head1 DESCRIPTION
179  
180 This is implementation of L<Parallel::DataPipe> algorithm using cooperative threads (Coro).
181 See description of alorithm and subroutines there.
182  
183 This module uses cooperative threads, so all threads share the same memory and no forks are used.
184 Good use case is when you make some long lasting queries to database/www and then process data
185 and want to do it asynchronosuly.
186 In that case even if you have one processor you will win because processor will be always busy thanks to Coro.
187  
188 =head1 SEE ALSO
189  
190 L<Coro>
191  
192 =head1 DEPENDENCIES
193  
194 It requires Coro 6.31. Alhough may be it will work with more old version - it is just one I use for myself.
195  
196 =head1 BUGS
197  
198 For all bugs please send an email to okharch@gmail.com.
199  
200 =head1 SOURCE REPOSITORY
201  
202 See the git source on github
203 L<https://github.com/okharch/Coro-DataPipe>
204  
205 =head1 COPYRIGHT
206  
207 Copyright (c) 2013 Oleksandr Kharchenko <okharch@gmail.com>
208  
209 All right reserved. This library is free software; you can redistribute it
210 and/or modify it under the same terms as Perl itself.
211  
212 =head1 AUTHOR
213  
214 Oleksandr Kharchenko <okharch@gmail.com>
215  
216 =cut
↑ Summary