MrGamoo/Xfer.pm untested

filter: Statementstmt 0.0% 0 / 177 Branchbran 0.0% 0 / 48 Conditioncond 0.0% 0 / 6 MC/DCmcdc 0.0% 0 / 4 Subroutinesub 0.0% 0 / 22 Podpod 0.0% 0 / 11 totaltotal 0.0% 0 / 268 CC 25 SCAR 30.8
CC
25
Cov
0%
_get_file
56.0
periodic
42.0
_sendfile
30.0
CRAP 21.7
1 # -*- perl -*-
2  
3 # Copyright (c) 2009 AdCopy
4 # Author: Jeff Weisberg
5 # Created: 2009-Dec-14 17:49 (EST)
6 # Function: get file from remote scriblr
7 #
8 # $Id: Xfer.pm,v 1.7 2011/01/14 22:24:30 jaw Exp $
9  
10 package AC::MrGamoo::Xfer;
11 use AC::MrGamoo::Debug 'xfer';
12 use AC::MrGamoo::Config;
13 use AC::MrGamoo::Protocol;
14 use AC::MrGamoo::EUConsole;
15 use AC::MrGamoo::PeerList;
16 use Digest::SHA1;
17 use File::Path;
18 use Socket;
19 use strict;
20  
21 require 'AC/protobuf/std_reply.pl';
22 require 'AC/protobuf/scrible.pl';
23  
24 our @ISA = 'AC::DC::IO::Forked';
25  
26 my $TIMEOUT = 3600; # QQQ
27 my $BUFSIZ = 65536;
28 my $STATUSTIME = 5;
29 my $MAXRUNNING = 4; # QQQ - configurable?
30 my %REGISTRY;
31  
32 ################################################################
33  
34 # schedule periodic "cronjob"
35 AC::DC::Sched->new(
36 info => "xfer periodic",
37 freq => 5,
38 func => \&periodic,
39 );
40  
41 ################################################################
42  
43  
44 sub new {
45 my $class = shift;
46 my $srcname = shift; # on remote system
47 my $dstname = shift; # on local system
48 my $loc = shift;
49 my $req = shift; # APCMRMFileXfer
50  
51 if( $REGISTRY{$req->{copyid}} ){
52 verbose("ignoring duplicate xfer $req->{copyid}");
53 return $REGISTRY{$req->{copyid}};
54 }
55  
56 $dstname = conf_value('basedir') . '/' . $dstname;
57 my $tmpfile = $dstname . ".$$";
58  
59 # mkpath
60 my($dir) = $dstname =~ m|^(.+)/[^/]+$|;
61 eval{ mkpath($dir, undef, 0755) };
62  
63 my $me = $class->SUPER::new( \&_run_child,
64 [ $srcname, $dstname, $tmpfile, $loc, $req ],
65 info => "xfer $loc:$srcname",
66 request => $req,
67 rbufsize => 65536,
68 );
69  
70 return unless $me;
71 $REGISTRY{$req->{copyid}} = $me;
72 debug("xfer requesting $loc:$srcname => $dstname, id $req->{copyid}");
73  
74 return $me;
75 }
76  
77 sub start {
78 my $me = shift;
79  
80 my $nrun = 0;
81 for my $t (values %REGISTRY){
82 $nrun ++ if $t->{fd};
83 }
84  
85 if( $nrun >= $MAXRUNNING ){
86 $me->{_queueprio} = $^T;
87 debug("queue xfer $me->{request}{copyid}");
88 return 1;
89 }
90  
91 $me->_start();
92  
93 }
94  
95 sub _start {
96 my $me = shift;
97  
98 debug("start xfer $me->{request}{copyid}");
99  
100 $me->timeout_rel($TIMEOUT);
101 $me->set_callback('timeout', \&timeout);
102 $me->set_callback('shutdown', \&shutdown);
103  
104 $me->SUPER::start();
105 }
106  
107 sub _run_child {
108 my $srcname = shift; # on remote system
109 my $dstname = shift; # on local system
110 my $tmpfile = shift;
111 my $loc = shift;
112 my $req = shift;
113  
114  
115 exit(0) if -f $dstname;
116 my $con = AC::MrGamoo::EUConsole->new( $req->{jobid}, $req->{console} );
117 $con->send_msg('debug', "retrieving file $loc:$srcname");
118  
119 # RSN - remove scriblr::client
120 my $ok;
121 if( get_peer_addr_from_id($loc) ){
122 $ok = _get_file( $req, $loc, $srcname, $tmpfile );
123 }else{
124 verbose("cannot locate server: $loc");
125 }
126  
127 if( $ok ){
128 rename $tmpfile, $dstname;
129 exit 0;
130 }
131 exit 1;
132 }
133  
134 sub timeout {
135 my $me = shift;
136 debug("xfer timeout");
137 $me->shut();
138 }
139  
140 sub shutdown {
141 my $me = shift;
142  
143 delete $REGISTRY{$me->{request}{copyid}};
144  
145 debug("exitval = $me->{exitval}");
146 if( !$me->{exitval} ){
147 $me->run_callback('on_success');
148 }else{
149 $me->run_callback('on_failure');
150 }
151  
152 periodic(1); # try to start another xfer
153 }
154  
155 sub _send_status_update {
156 my $req = shift;
157  
158 debug('send xfer status');
159 AC::MrGamoo::API::Xfer::tell_master( $req, 100, 'Working...' );
160 }
161  
162  
163 sub _get_file {
164 my $oreq = shift;
165 my $loc = shift;
166 my $srcname = shift; # on remote system
167 my $tmpfile = shift;
168  
169  
170 my($addr, $port) = get_peer_addr_from_id( $loc );
171 unless( $addr ){
172 debug("cannot find addr for $loc");
173 return;
174 }
175  
176 debug("connecting to $addr:$port");
177  
178 my $req = AC::MrGamoo::Protocol->encode_request( {
179 type => 'scribl_get',
180 msgidno => $$,
181 want_reply => 1,
182 }, { filename => $srcname } );
183  
184 my $p;
185 eval {
186 # connect
187 my $s = AC::MrGamoo::Protocol->connect_to_server( inet_aton($addr), $port );
188 return unless $s;
189  
190 # send req
191 AC::MrGamoo::Protocol->write_request($s, $req);
192  
193 # get response
194 my $buf = AC::MrGamoo::Protocol->read_data($s, AC::MrGamoo::Protocol->header_size(), 30);
195 $p = AC::MrGamoo::Protocol->decode_header($buf);
196 $p->{data} = AC::MrGamoo::Protocol->read_data($s, $p->{data_length}, 1);
197 $p->{data} = AC::MrGamoo::Protocol->decode_reply($p);
198  
199 debug("recvd response $p->{data}{status_code}");
200 return unless $p->{data}{status_code} == 200;
201  
202 # stream file to disk
203 my $size = $p->{content_length};
204 debug("recving file ($size B)");
205  
206 my $fd;
207 unless( open( $fd, "> $tmpfile" ) ){
208 verbose("cannot open output file '$tmpfile': $!");
209 return;
210 }
211  
212 my $chk = _sendfile($oreq, $fd, $s, $size);
213 my $sha1 = $p->{data}{hash_sha1};
214 die "SHA1 check failed\n" if $sha1 && $sha1 ne $chk;
215 };
216 if(my $e=$@){
217 debug("error: $e");
218 return;
219 }
220  
221 return $p;
222 }
223  
224 sub _sendfile {
225 my $req = shift;
226 my $out = shift;
227 my $in = shift;
228 my $size = shift;
229  
230 my $t;
231  
232 my $sha1 = Digest::SHA1->new();
233  
234 while($size){
235 my $buf;
236 my $len = $size > $BUFSIZ ? $BUFSIZ : $size;
237 alarm( 1 );
238 my $i = sysread($in, $buf, $len);
239 die "read failed: $!\n" unless $i > 0;
240 $size -= $i;
241 $i = syswrite($out, $buf);
242 die "write failed: $!\n" unless $i > 0;
243 $sha1->add($buf);
244  
245 # periodically tell master we are still copying
246 if( time() - $t > $STATUSTIME ){
247 _send_status_update( $req );
248 $t = time();
249 }
250 }
251 alarm(0);
252  
253 return $sha1->b64digest();
254 }
255  
256 sub report {
257  
258 my $txt;
259  
260 for my $t (values %REGISTRY){
261 my $status = $t->{fd} ? 'running' : 'queued';
262 $txt .= "$t->{request}{copyid} $status\n";
263 }
264  
265 return $txt;
266 }
267  
268 sub periodic {
269 my $quick = shift;
270  
271 # how many xfers are running?
272 my $nrun = 0;
273 for my $t (values %REGISTRY){
274 $nrun ++ if $t->{fd};
275 }
276  
277 return if $quick && $nrun >= $MAXRUNNING;
278  
279 # queued? send status, maybe start
280  
281 for my $t (sort { $a->{_queueprio} <=> $b->{_queueprio} } values %REGISTRY){
282 next if $t->{fd};
283  
284 _send_status_update( $t->{request} ) unless $quick;
285  
286 if( $nrun < $MAXRUNNING ){
287 $t->_start();
288 $nrun ++;
289 }
290 }
291 }
292  
293  
294 1;