DDG/Meta/Data.pm untested

filter: Statementstmt 0.0% 0 / 118 Branchbran 0.0% 0 / 46 Conditioncond 0.0% 0 / 18 MC/DCmcdc n/a Subroutinesub 0.0% 0 / 18 Podpod 0.0% 0 / 9 totaltotal 0.0% 0 / 209 SCAR 47.0
CC
10
Cov
0%
filter_ias
30.0
get_ia
12.0
get_js
12.0
CRAP 110.0
1 package DDG::Meta::Data;
2 our $AUTHORITY = 'cpan:DDG';
3 # ABSTRACT: Metadata functions for instant answers
4 $DDG::Meta::Data::VERSION = '1016';
5 use JSON::XS qw'decode_json encode_json';
6 use Path::Class;
7 use File::ShareDir 'dist_file';
8 use LWP::UserAgent;
9 use File::Copy::Recursive 'pathmk';
10 use List::Util qw( all any );
11  
12 use strict;
13  
14 sub debug { 0 }
15 use if debug, 'Data::Printer';
16  
17 no warnings 'uninitialized';
18  
19 # $ia_metadata => {
20 # id => ...
21 # module => ...
22 # sharedir => ...
23 # }
24 my %ia_metadata;
25  
26 # Only build metadata once.
27 unless(%ia_metadata){
28  
29 my $tmpdir = $ENV{METADATA_TMP_DIR} || '/var/tmp/ddg-metadata';
30  
31 my $mdir = "$tmpdir-$>";
32 unless(-d $mdir){
33 pathmk $mdir or die "Failed to mkdir $mdir: $!";
34 }
35  
36 debug && warn "Processing metadata";
37  
38 my $f = "$mdir/metadata.json.bz2";
39 unless($ENV{NO_METADATA_DOWNLOAD}){
40 my $ua = LWP::UserAgent->new;
41 $ua->timeout(5);
42 $ua->default_header('Accept-Encoding' => scalar HTTP::Message::decodable());
43 my $res = $ua->mirror('http://ddg-community.s3.amazonaws.com/metadata/repo_all.json.bz2', $f);
44 unless($res->is_success || $res->code == 304){
45 debug && warn "Failed to download metdata: " . $res->status_line;
46 }
47 }
48  
49 # Decompress to command-line
50 open my $fh, "bzip2 -dc $f |" or die "Failed to open file $f: $!";
51 # slurp into a single string
52 my $json = do { local $/; <$fh> };
53 close $fh;
54 my $metadata = decode_json($json);
55  
56 # { "<id>": {
57 # "id": " "
58 # "signal" : " "
59 # ....
60 # }
61  
62 IA: while (my ($id, $ia) = each %{ $metadata }) {
63  
64 # 20150502 (zt) Can't filter like this yet as some tests depend on non-live IA metadata
65 #next unless $ia->{status} eq 'live';
66  
67 # warn if we run into a duplicate id. These should be unique within *and*
68 # across all IA types
69 if( $ia_metadata{id}{$id} ){
70 warn "Duplicate ID for IA with ID: $id";
71 }
72  
73 # Perl modules aren't strictly required, e.g. Fatheads and Longtails
74 # It's ok not to have every IA represented in the by_module view
75 # We may need to update the parameter to _js_callback_name in the future
76 if(my $perl_module = $ia->{perl_module}){
77 #add new ia to ia_metadata{module}. Multiple ias per module possible
78 push @{$ia_metadata{module}{$perl_module}}, $ia;
79 $ia->{js_callback_name} = _js_callback_name($perl_module);
80 }
81  
82 # Clean up/set some values
83 $ia->{signal_from} ||= $ia->{id};
84  
85 #add new ia to ia_metadata{id}
86 $ia_metadata{id}{$id} = $ia;
87 # by language for multilang wiki
88 if($ia->{repo} eq 'fathead'){
89 my $source = $ia->{src_id};
90 # by source number for fatheads
91 $ia_metadata{fathead_source}{$source} = $ia;
92 # by language for multi language wiki sources
93 # check that language is set since most fatheads don't have a language
94 if( my $lang = $ia->{src_options}{language}){
95 # By default use current source
96 my $want_src = $source;
97 if(exists $ia_metadata{fathead_lang}{$lang}){
98 my $prev_ia = $ia_metadata{fathead_source}{$ia_metadata{fathead_lang}{$lang}};
99 # Skip setting lang to source if existing src_id is lower
100 $want_src = 0 if $prev_ia->{src_id} < $ia->{src_id};
101 }
102 $ia_metadata{fathead_lang}{$lang} = $want_src if $want_src;
103 }
104 my $min_length = $ia->{src_options}{min_abstract_length};
105 $ia_metadata{fathead_min_length}{$source} = $min_length if $min_length;
106 }
107 }
108  
109 unless(%ia_metadata){
110 warn "[Error] No Instant Answer metadata loaded. Metadata will be downloaded\n",
111 "automatically and stored in $mdir if a network connection can be made\n",
112 "to https://duck.co.\n";
113 }
114 }
115  
116 sub get_ia {
117 my ($self, $by, $lookup) = @_;
118 warn 'Get IA obj lookup params: ', p($lookup) if debug;
119  
120 my $m = $ia_metadata{$by}{$lookup};
121 warn 'Returning IA ', p($m) if debug;
122 return $m;
123 }
124  
125 # filter_ias({ repo => 'goodies', dev_milestone => 'live'... })
126 # Lookups combine as an AND operation.
127 # Returns a list of IAs on `wantarray', otherwise an (id => ia) HASH ref.
128 #
129 # Each condition consists of a $key and $lookup.
130 # $lookup should be either a string, ARRAY ref, or CODE ref.
131 # If a string, then $lookup is compared with `eq'
132 # If a CODE ref, then $lookup is called with the IAs $key attribute and
133 # should return a boolean value.
134 # If an ARRAY ref, then the above two rules are used with each element, the
135 # IA only needs to satisfy one.
136 sub filter_ias {
137 my $lookups = $_[1];
138 my %ias = %{by_id()};
139 my %lookups = %$lookups;
140 my @by = keys %lookups;
141 # Ensure lookups are of the form (by => [lookup...])
142 map {
143 my $cond = $lookups{$_};
144 $lookups{$_} = [$cond] unless ref $cond eq 'ARRAY';
145 } @by;
146 while (my ($id, $ia) = each %ias) {
147 delete $ias{$id} unless all {
148 my ($by, $lookup) = ($_, $lookups{$_});
149 any {
150 ref $_ eq 'CODE' ? $_->($ia->{$by}) : $_ eq $ia->{$by};
151 } @$lookup;
152 } @by;
153 }
154 return wantarray ? (values %ias) : \%ias;
155 }
156  
157 sub get_js {
158 my ($self, $by, $lookup) = @_;
159 return unless $by =~ /id|source/;
160 my $ia = $self->get_ia($by => $lookup);
161 return unless $ia;
162  
163 my $id = $ia->{id};
164 my $metaj = eval { JSON::XS->new->ascii->encode($ia) } || return;
165 return qq(DDH.$id=DDH.$id||{};DDH.$id.meta=$metaj;);
166 }
167  
168 # return a hash of IA objects by id
169 sub by_id { $ia_metadata{id} }
170  
171 # return a hash of IA metadata for fatheads
172 sub fathead_by_source { $ia_metadata{fathead_source} }
173  
174 # return hash if IA metadata by language
175 sub fathead_by_lang { $ia_metadata{fathead_lang} }
176  
177 # fathead min lengths
178 sub fathead_by_length { $ia_metadata{fathead_min_length} }
179  
180 # Internal function.
181 sub _js_callback_name {
182 my $name = shift;
183 my $fn;
184 if(($fn) = $name =~ /^DDG::\w+::(\w.+)$/o){
185 $fn =~ s/::/_/og;
186 $fn =~ s/([a-z])([A-Z])/$1_$2/og;
187 $fn = lc $fn;
188 }
189 return $fn;
190 }
191  
192 1;
193  
194 __END__