|  line  | 
 stmt  | 
 bran  | 
 cond  | 
 sub  | 
 pod  | 
 time  | 
 code  | 
| 
1
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 # mt-aws-glacier - Amazon Glacier sync client  | 
| 
2
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 # Copyright (C) 2012-2014  Victor Efimov  | 
| 
3
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 # http://mt-aws.com (also http://vs-dev.com) vs@vs-dev.com  | 
| 
4
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 # License: GPLv3  | 
| 
5
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #  | 
| 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 # This file is part of "mt-aws-glacier"  | 
| 
7
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #  | 
| 
8
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #    mt-aws-glacier is free software: you can redistribute it and/or modify  | 
| 
9
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #    it under the terms of the GNU General Public License as published by  | 
| 
10
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #    the Free Software Foundation, either version 3 of the License, or  | 
| 
11
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #    (at your option) any later version.  | 
| 
12
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #  | 
| 
13
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #    mt-aws-glacier is distributed in the hope that it will be useful,  | 
| 
14
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #    but WITHOUT ANY WARRANTY; without even the implied warranty of  | 
| 
15
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the  | 
| 
16
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #    GNU General Public License for more details.  | 
| 
17
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #  | 
| 
18
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #    You should have received a copy of the GNU General Public License  | 
| 
19
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.  | 
| 
20
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
    | 
| 
21
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 package App::MtAws::QueueJob::FetchAndDownload;  | 
| 
22
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
    | 
| 
23
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 our $VERSION = '1.114_2';  | 
| 
24
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
    | 
| 
25
 | 
6
 | 
 
 | 
 
 | 
  
6
  
 | 
 
 | 
452
 | 
 use strict;  | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
8
 | 
    | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
141
 | 
    | 
| 
26
 | 
6
 | 
 
 | 
 
 | 
  
6
  
 | 
 
 | 
19
 | 
 use warnings;  | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
7
 | 
    | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
108
 | 
    | 
| 
27
 | 
6
 | 
 
 | 
 
 | 
  
6
  
 | 
 
 | 
19
 | 
 use Carp;  | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
9
 | 
    | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
229
 | 
    | 
| 
28
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
    | 
| 
29
 | 
6
 | 
 
 | 
 
 | 
  
6
  
 | 
 
 | 
27
 | 
 use App::MtAws::QueueJobResult;  | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
11
 | 
    | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
286
 | 
    | 
| 
30
 | 
6
 | 
 
 | 
 
 | 
  
6
  
 | 
 
 | 
2424
 | 
 use App::MtAws::QueueJob::Download;  | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
11
 | 
    | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
135
 | 
    | 
| 
31
 | 
6
 | 
 
 | 
 
 | 
  
6
  
 | 
 
 | 
2532
 | 
 use App::MtAws::QueueJob::Iterator;  | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
7
 | 
    | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
134
 | 
    | 
| 
32
 | 
6
 | 
 
 | 
 
 | 
  
6
  
 | 
 
 | 
2417
 | 
 use App::MtAws::Glacier::ListJobs;  | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
14
 | 
    | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
180
 | 
    | 
| 
33
 | 
6
 | 
 
 | 
 
 | 
  
6
  
 | 
 
 | 
31
 | 
 use base 'App::MtAws::QueueJob';  | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
8
 | 
    | 
| 
 
 | 
6
 | 
 
 | 
 
 | 
 
 | 
 
 | 
2630
 | 
    | 
| 
34
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
    | 
| 
35
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 sub init  | 
| 
36
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 {  | 
| 
37
 | 
27
 | 
 
 | 
 
 | 
  
27
  
 | 
  
0
  
 | 
46
 | 
 	my ($self) = @_;  | 
| 
38
 | 
27
 | 
  
100
  
 | 
 
 | 
 
 | 
 
 | 
239
 | 
 	$self->{archives}||confess;  | 
| 
39
 | 
26
 | 
  
100
  
 | 
 
 | 
 
 | 
 
 | 
174
 | 
 	$self->{file_downloads}||confess;  | 
| 
40
 | 
25
 | 
 
 | 
 
 | 
 
 | 
 
 | 
54
 | 
 	$self->{downloads} = [];  | 
| 
41
 | 
25
 | 
 
 | 
 
 | 
 
 | 
 
 | 
49
 | 
 	$self->{marker} = undef;  | 
| 
42
 | 
25
 | 
 
 | 
 
 | 
 
 | 
 
 | 
89
 | 
 	$self->enter("list");  | 
| 
43
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 }  | 
| 
44
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
    | 
| 
45
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 sub on_list  | 
| 
46
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 {  | 
| 
47
 | 
192
 | 
 
 | 
 
 | 
  
192
  
 | 
  
0
  
 | 
166
 | 
 	my ($self) = @_;  | 
| 
48
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 	return state "wait", task "retrieval_fetch_job", {  marker => $self->{marker} } => sub {  | 
| 
49
 | 
192
 | 
 
 | 
 
 | 
  
192
  
 | 
 
 | 
218
 | 
 		my ($args) = @_;  | 
| 
50
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
    | 
| 
51
 | 
192
 | 
 
 | 
  
 33
  
 | 
 
 | 
 
 | 
882
 | 
 		my ($marker, @jobs) = App::MtAws::Glacier::ListJobs->new( $args->{response} || confess )->get_archive_entries();  | 
| 
52
 | 
192
 | 
 
 | 
 
 | 
 
 | 
 
 | 
975
 | 
 		for (@jobs) {  | 
| 
53
 | 
1680
 | 
  
100
  
 | 
 
 | 
 
 | 
 
 | 
2808
 | 
 			if (my $a = $self->{archives}{ $_->{ArchiveId} }) {  | 
| 
54
 | 
144
 | 
  
100
  
 | 
 
 | 
 
 | 
 
 | 
315
 | 
 				unless ($a->{seen}++) { # TODO: better take latest archives in case of duplicates  | 
| 
55
 | 
96
 | 
 
 | 
 
 | 
 
 | 
 
 | 
204
 | 
 					$a->{jobid} = $_->{JobId};  | 
| 
56
 | 
96
 | 
 
 | 
 
 | 
 
 | 
 
 | 
70
 | 
 					push @{ $self->{downloads} }, $a;  | 
| 
 
 | 
96
 | 
 
 | 
 
 | 
 
 | 
 
 | 
156
 | 
    | 
| 
57
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 				}  | 
| 
58
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 			}  | 
| 
59
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 		}  | 
| 
60
 | 
192
 | 
  
100
  
 | 
 
 | 
 
 | 
 
 | 
296
 | 
 		if ($marker) {  | 
| 
61
 | 
168
 | 
 
 | 
 
 | 
 
 | 
 
 | 
213
 | 
 			$self->{marker} = $marker;  | 
| 
62
 | 
168
 | 
 
 | 
 
 | 
 
 | 
 
 | 
372
 | 
 			return state 'list';  | 
| 
63
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 		} else {  | 
| 
64
 | 
24
 | 
 
 | 
 
 | 
 
 | 
 
 | 
56
 | 
 			return state 'download'; # TODO: or if pending archive list is empty  | 
| 
65
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 		}  | 
| 
66
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 		  | 
| 
67
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 	}  | 
| 
68
 | 
192
 | 
 
 | 
 
 | 
 
 | 
 
 | 
379
 | 
 }  | 
| 
69
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
    | 
| 
70
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 sub next_download  | 
| 
71
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 {  | 
| 
72
 | 
120
 | 
 
 | 
 
 | 
  
120
  
 | 
  
0
  
 | 
106
 | 
 	my ($self) = @_;  | 
| 
73
 | 
120
 | 
  
100
  
 | 
 
 | 
 
 | 
 
 | 
96
 | 
 	if (my $rec = shift @{ $self->{downloads}}) {  | 
| 
 
 | 
120
 | 
 
 | 
 
 | 
 
 | 
 
 | 
261
 | 
    | 
| 
74
 | 
672
 | 
 
 | 
 
 | 
 
 | 
 
 | 
1178
 | 
 		return App::MtAws::QueueJob::Download->new( (map { $_ => $rec->{$_}} qw/archive_id filename relfilename size jobid mtime treehash/),  | 
| 
75
 | 
96
 | 
 
 | 
 
 | 
 
 | 
 
 | 
164
 | 
 			file_downloads => $self->{file_downloads} );  | 
| 
76
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 	} else {  | 
| 
77
 | 
24
 | 
 
 | 
 
 | 
 
 | 
 
 | 
69
 | 
 		return;  | 
| 
78
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 	}  | 
| 
79
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 }  | 
| 
80
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
    | 
| 
81
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 sub on_download  | 
| 
82
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 {  | 
| 
83
 | 
24
 | 
 
 | 
 
 | 
  
24
  
 | 
  
0
  
 | 
33
 | 
 	my ($self) = @_;  | 
| 
84
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 	  | 
| 
85
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 	return state("wait"),  | 
| 
86
 | 
120
 | 
 
 | 
 
 | 
  
120
  
 | 
 
 | 
211
 | 
 		job( App::MtAws::QueueJob::Iterator->new(iterator => sub { $self->next_download() }), sub {  | 
| 
87
 | 
24
 | 
 
 | 
 
 | 
  
24
  
 | 
 
 | 
60
 | 
 			state("done")  | 
| 
88
 | 
24
 | 
 
 | 
 
 | 
 
 | 
 
 | 
67
 | 
 		});  | 
| 
89
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 }  | 
| 
90
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
    | 
| 
91
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
    | 
| 
92
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 
 | 
 1;  |