DDG/Meta/Data.pm untested

filter: Statementstmt 0.0% 0 / 138 Branchbran 0.0% 0 / 52 Conditioncond 0.0% 0 / 27 MC/DCmcdc n/a Subroutinesub 0.0% 0 / 19 Podpod 0.0% 0 / 9 totaltotal 0.0% 0 / 245 CC 10 SCAR 22.4
CC
10
Cov
0%
filter_ias
30.0
get_ia
12.0
get_js
12.0
CRAP 9.4
1 package DDG::Meta::Data;
2 our $AUTHORITY = 'cpan:DDG';
3 # ABSTRACT: Metadata functions for instant answers
4 $DDG::Meta::Data::VERSION = '1017';
5 use JSON::XS qw'decode_json encode_json';
6 use Path::Class;
7 use File::ShareDir 'dist_file';
8 use LWP::UserAgent;
9 use File::Copy::Recursive 'pathmk';
10 use List::Util qw( all any );
11 use File::Copy;
12  
13 use strict;
14  
15 sub debug { 0 }
16 use if debug, 'Data::Printer';
17  
18 no warnings 'uninitialized';
19  
20 # $ia_metadata => {
21 # id => ...
22 # module => ...
23 # sharedir => ...
24 # }
25 my %ia_metadata;
26  
27 # Only build metadata once.
28 unless(%ia_metadata){
29  
30 my $tmpdir = $ENV{METADATA_TMP_DIR} || '/var/tmp/ddg-metadata';
31  
32 my $mdir = "$tmpdir-$>";
33 unless(-d $mdir){
34 pathmk $mdir or die "Failed to mkdir $mdir: $!";
35 }
36  
37 debug && warn "Processing metadata";
38  
39 my $f = "$mdir/metadata.json.bz2";
40 my $tmp_bak = "$f.bak";
41 my @timestamps = (stat $f)[8,9];
42 if(-e $f){
43 copy $f, $tmp_bak or die "Failed to copy $f to $tmp_bak: $!";;
44 }
45 utime @timestamps, $tmp_bak;
46 my $restore_backup = sub {
47 move $tmp_bak, $f or die "Failed to move backup $tmp_bak to $f: $!";
48 utime @timestamps, $f;
49 };
50  
51 unless($ENV{NO_METADATA_DOWNLOAD}){
52 my $ua = LWP::UserAgent->new;
53 $ua->timeout(5);
54 $ua->default_header('Accept-Encoding' => scalar HTTP::Message::decodable());
55 my $res = $ua->mirror('http://ddg-community.s3.amazonaws.com/metadata/repo_all.json.bz2', $f);
56 unless($res->is_success || $res->code == 304){
57 debug && warn "Failed to download metdata: " . $res->status_line . " . Restoring backup from $tmp_bak";
58 $restore_backup->();
59 }
60 }
61  
62 my $metadata;
63 while(!$metadata){
64 eval {
65 # Decompress to command-line
66 open my $fh, "bzip2 -dc $f |" or die "Failed to open file $f: $!";
67 # slurp into a single string
68 my $json = do { local $/; <$fh> };
69 $metadata = decode_json($json);
70 }
71 or do {
72 if(-e $tmp_bak){
73 debug && warn "Failed to process metadata $f: $@. Restoring backup from $tmp_bak";
74 $restore_backup->();
75 }
76 else{
77 die "Failed to to process metadata from $f: $@";
78 }
79 };
80 }
81 unlink $tmp_bak if -e $tmp_bak;
82  
83 # { "<id>": {
84 # "id": " "
85 # "signal" : " "
86 # ....
87 # }
88  
89 IA: while (my ($id, $ia) = each %{ $metadata }) {
90  
91 # 20150502 (zt) Can't filter like this yet as some tests depend on non-live IA metadata
92 #next unless $ia->{status} eq 'live';
93  
94 # warn if we run into a duplicate id. These should be unique within *and*
95 # across all IA types
96 if( $ia_metadata{id}{$id} ){
97 warn "Duplicate ID for IA with ID: $id";
98 }
99  
100 # Perl modules aren't strictly required, e.g. Fatheads and Longtails
101 # It's ok not to have every IA represented in the by_module view
102 # We may need to update the parameter to _js_callback_name in the future
103 if(my $perl_module = $ia->{perl_module}){
104 #add new ia to ia_metadata{module}. Multiple ias per module possible
105 push @{$ia_metadata{module}{$perl_module}}, $ia;
106 $ia->{js_callback_name} = _js_callback_name($perl_module);
107 }
108  
109 # Clean up/set some values
110 $ia->{signal_from} ||= $ia->{id};
111  
112 #add new ia to ia_metadata{id}
113 $ia_metadata{id}{$id} = $ia;
114 # by language for multilang wiki
115 if($ia->{repo} eq 'fathead'){
116 my $source = $ia->{src_id};
117 # by source number for fatheads
118 $ia_metadata{fathead_source}{$source} = $ia;
119 # by language for multi language wiki sources
120 # check that language is set since most fatheads don't have a language
121 if( my $lang = $ia->{src_options}{language}){
122 # By default use current source
123 my $want_src = $source;
124 if(exists $ia_metadata{fathead_lang}{$lang}){
125 my $prev_ia = $ia_metadata{fathead_source}{$ia_metadata{fathead_lang}{$lang}};
126 # Skip setting lang to source if existing src_id is lower
127 $want_src = 0 if $prev_ia->{src_id} < $ia->{src_id};
128 }
129 $ia_metadata{fathead_lang}{$lang} = $want_src if $want_src;
130 }
131 my $min_length = $ia->{src_options}{min_abstract_length};
132 $ia_metadata{fathead_min_length}{$source} = $min_length if $min_length;
133 }
134 }
135  
136 unless(%ia_metadata){
137 warn "[Error] No Instant Answer metadata loaded. Metadata will be downloaded\n",
138 "automatically and stored in $mdir if a network connection can be made\n",
139 "to https://duck.co.\n";
140 }
141 }
142  
143 sub get_ia {
144 my ($self, $by, $lookup) = @_;
145 warn 'Get IA obj lookup params: ', p($lookup) if debug;
146  
147 my $m = $ia_metadata{$by}{$lookup};
148 warn 'Returning IA ', p($m) if debug;
149 return $m;
150 }
151  
152 # filter_ias({ repo => 'goodies', dev_milestone => 'live'... })
153 # Lookups combine as an AND operation.
154 # Returns a list of IAs on `wantarray', otherwise an (id => ia) HASH ref.
155 #
156 # Each condition consists of a $key and $lookup.
157 # $lookup should be either a string, ARRAY ref, or CODE ref.
158 # If a string, then $lookup is compared with `eq'
159 # If a CODE ref, then $lookup is called with the IAs $key attribute and
160 # should return a boolean value.
161 # If an ARRAY ref, then the above two rules are used with each element, the
162 # IA only needs to satisfy one.
163 sub filter_ias {
164 my $lookups = $_[1];
165 my %ias = %{by_id()};
166 my %lookups = %$lookups;
167 my @by = keys %lookups;
168 # Ensure lookups are of the form (by => [lookup...])
169 map {
170 my $cond = $lookups{$_};
171 $lookups{$_} = [$cond] unless ref $cond eq 'ARRAY';
172 } @by;
173 while (my ($id, $ia) = each %ias) {
174 delete $ias{$id} unless all {
175 my ($by, $lookup) = ($_, $lookups{$_});
176 any {
177 ref $_ eq 'CODE' ? $_->($ia->{$by}) : $_ eq $ia->{$by};
178 } @$lookup;
179 } @by;
180 }
181 return wantarray ? (values %ias) : \%ias;
182 }
183  
184 sub get_js {
185 my ($self, $by, $lookup) = @_;
186 return unless $by =~ /id|source/;
187 my $ia = $self->get_ia($by => $lookup);
188 return unless $ia;
189  
190 my $id = $ia->{id};
191 my $metaj = eval { JSON::XS->new->ascii->encode($ia) } || return;
192 return qq(DDH.$id=DDH.$id||{};DDH.$id.meta=$metaj;);
193 }
194  
195 # return a hash of IA objects by id
196 sub by_id { $ia_metadata{id} }
197  
198 # return a hash of IA metadata for fatheads
199 sub fathead_by_source { $ia_metadata{fathead_source} }
200  
201 # return hash if IA metadata by language
202 sub fathead_by_lang { $ia_metadata{fathead_lang} }
203  
204 # fathead min lengths
205 sub fathead_by_length { $ia_metadata{fathead_min_length} }
206  
207 # Internal function.
208 sub _js_callback_name {
209 my $name = shift;
210 my $fn;
211 if(($fn) = $name =~ /^DDG::\w+::(\w.+)$/o){
212 $fn =~ s/::/_/og;
213 $fn =~ s/([a-z])([A-Z])/$1_$2/og;
214 $fn = lc $fn;
215 }
216 return $fn;
217 }
218  
219 1;
220  
221 __END__