| 1 | package DDG::Meta::Data; | ||
| 2 | our $AUTHORITY = 'cpan:DDG'; | ||
| 3 | # ABSTRACT: Metadata functions for instant answers | ||
| 4 | $DDG::Meta::Data::VERSION = '1017'; | ||
| 5 | use JSON::XS qw'decode_json encode_json'; | ||
| 6 | use Path::Class; | ||
| 7 | use File::ShareDir 'dist_file'; | ||
| 8 | use LWP::UserAgent; | ||
| 9 | use File::Copy::Recursive 'pathmk'; | ||
| 10 | use List::Util qw( all any ); | ||
| 11 | use File::Copy; | ||
| 12 | |||
| 13 | use strict; | ||
| 14 | |||
| 15 | sub debug { 0 } | ||
| 16 | use if debug, 'Data::Printer'; | ||
| 17 | |||
| 18 | no warnings 'uninitialized'; | ||
| 19 | |||
| 20 | # $ia_metadata => { | ||
| 21 | # id => ... | ||
| 22 | # module => ... | ||
| 23 | # sharedir => ... | ||
| 24 | # } | ||
| 25 | my %ia_metadata; | ||
| 26 | |||
| 27 | # Only build metadata once. | ||
| 28 | unless(%ia_metadata){ | ||
| 29 | |||
| 30 | my $tmpdir = $ENV{METADATA_TMP_DIR} || '/var/tmp/ddg-metadata'; | ||
| 31 | |||
| 32 | my $mdir = "$tmpdir-$>"; | ||
| 33 | unless(-d $mdir){ | ||
| 34 | pathmk $mdir or die "Failed to mkdir $mdir: $!"; | ||
| 35 | } | ||
| 36 | |||
| 37 | debug && warn "Processing metadata"; | ||
| 38 | |||
| 39 | my $f = "$mdir/metadata.json.bz2"; | ||
| 40 | my $tmp_bak = "$f.bak"; | ||
| 41 | my @timestamps = (stat $f)[8,9]; | ||
| 42 | if(-e $f){ | ||
| 43 | copy $f, $tmp_bak or die "Failed to copy $f to $tmp_bak: $!";; | ||
| 44 | } | ||
| 45 | utime @timestamps, $tmp_bak; | ||
| 46 | my $restore_backup = sub { | ||
| 47 | move $tmp_bak, $f or die "Failed to move backup $tmp_bak to $f: $!"; | ||
| 48 | utime @timestamps, $f; | ||
| 49 | }; | ||
| 50 | |||
| 51 | unless($ENV{NO_METADATA_DOWNLOAD}){ | ||
| 52 | my $ua = LWP::UserAgent->new; | ||
| 53 | $ua->timeout(5); | ||
| 54 | $ua->default_header('Accept-Encoding' => scalar HTTP::Message::decodable()); | ||
| 55 | my $res = $ua->mirror('http://ddg-community.s3.amazonaws.com/metadata/repo_all.json.bz2', $f); | ||
| 56 | unless($res->is_success || $res->code == 304){ | ||
| 57 | debug && warn "Failed to download metdata: " . $res->status_line . " . Restoring backup from $tmp_bak"; | ||
| 58 | $restore_backup->(); | ||
| 59 | } | ||
| 60 | } | ||
| 61 | |||
| 62 | my $metadata; | ||
| 63 | while(!$metadata){ | ||
| 64 | eval { | ||
| 65 | # Decompress to command-line | ||
| 66 | open my $fh, "bzip2 -dc $f |" or die "Failed to open file $f: $!"; | ||
| 67 | # slurp into a single string | ||
| 68 | my $json = do { local $/; <$fh> }; | ||
| 69 | $metadata = decode_json($json); | ||
| 70 | } | ||
| 71 | or do { | ||
| 72 | if(-e $tmp_bak){ | ||
| 73 | debug && warn "Failed to process metadata $f: $@. Restoring backup from $tmp_bak"; | ||
| 74 | $restore_backup->(); | ||
| 75 | } | ||
| 76 | else{ | ||
| 77 | die "Failed to to process metadata from $f: $@"; | ||
| 78 | } | ||
| 79 | }; | ||
| 80 | } | ||
| 81 | unlink $tmp_bak if -e $tmp_bak; | ||
| 82 | |||
| 83 | # { "<id>": { | ||
| 84 | # "id": " " | ||
| 85 | # "signal" : " " | ||
| 86 | # .... | ||
| 87 | # } | ||
| 88 | |||
| 89 | IA: while (my ($id, $ia) = each %{ $metadata }) { | ||
| 90 | |||
| 91 | # 20150502 (zt) Can't filter like this yet as some tests depend on non-live IA metadata | ||
| 92 | #next unless $ia->{status} eq 'live'; | ||
| 93 | |||
| 94 | # warn if we run into a duplicate id. These should be unique within *and* | ||
| 95 | # across all IA types | ||
| 96 | if( $ia_metadata{id}{$id} ){ | ||
| 97 | warn "Duplicate ID for IA with ID: $id"; | ||
| 98 | } | ||
| 99 | |||
| 100 | # Perl modules aren't strictly required, e.g. Fatheads and Longtails | ||
| 101 | # It's ok not to have every IA represented in the by_module view | ||
| 102 | # We may need to update the parameter to _js_callback_name in the future | ||
| 103 | if(my $perl_module = $ia->{perl_module}){ | ||
| 104 | #add new ia to ia_metadata{module}. Multiple ias per module possible | ||
| 105 | push @{$ia_metadata{module}{$perl_module}}, $ia; | ||
| 106 | $ia->{js_callback_name} = _js_callback_name($perl_module); | ||
| 107 | } | ||
| 108 | |||
| 109 | # Clean up/set some values | ||
| 110 | $ia->{signal_from} ||= $ia->{id}; | ||
| 111 | |||
| 112 | #add new ia to ia_metadata{id} | ||
| 113 | $ia_metadata{id}{$id} = $ia; | ||
| 114 | # by language for multilang wiki | ||
| 115 | if($ia->{repo} eq 'fathead'){ | ||
| 116 | my $source = $ia->{src_id}; | ||
| 117 | # by source number for fatheads | ||
| 118 | $ia_metadata{fathead_source}{$source} = $ia; | ||
| 119 | # by language for multi language wiki sources | ||
| 120 | # check that language is set since most fatheads don't have a language | ||
| 121 | if( my $lang = $ia->{src_options}{language}){ | ||
| 122 | # By default use current source | ||
| 123 | my $want_src = $source; | ||
| 124 | if(exists $ia_metadata{fathead_lang}{$lang}){ | ||
| 125 | my $prev_ia = $ia_metadata{fathead_source}{$ia_metadata{fathead_lang}{$lang}}; | ||
| 126 | # Skip setting lang to source if existing src_id is lower | ||
| 127 | $want_src = 0 if $prev_ia->{src_id} < $ia->{src_id}; | ||
| 128 | } | ||
| 129 | $ia_metadata{fathead_lang}{$lang} = $want_src if $want_src; | ||
| 130 | } | ||
| 131 | my $min_length = $ia->{src_options}{min_abstract_length}; | ||
| 132 | $ia_metadata{fathead_min_length}{$source} = $min_length if $min_length; | ||
| 133 | } | ||
| 134 | } | ||
| 135 | |||
| 136 | unless(%ia_metadata){ | ||
| 137 | warn "[Error] No Instant Answer metadata loaded. Metadata will be downloaded\n", | ||
| 138 | "automatically and stored in $mdir if a network connection can be made\n", | ||
| 139 | "to https://duck.co.\n"; | ||
| 140 | } | ||
| 141 | } | ||
| 142 | |||
| 143 | sub get_ia { | ||
| 144 | my ($self, $by, $lookup) = @_; | ||
| 145 | warn 'Get IA obj lookup params: ', p($lookup) if debug; | ||
| 146 | |||
| 147 | my $m = $ia_metadata{$by}{$lookup}; | ||
| 148 | warn 'Returning IA ', p($m) if debug; | ||
| 149 | return $m; | ||
| 150 | } | ||
| 151 | |||
| 152 | # filter_ias({ repo => 'goodies', dev_milestone => 'live'... }) | ||
| 153 | # Lookups combine as an AND operation. | ||
| 154 | # Returns a list of IAs on `wantarray', otherwise an (id => ia) HASH ref. | ||
| 155 | # | ||
| 156 | # Each condition consists of a $key and $lookup. | ||
| 157 | # $lookup should be either a string, ARRAY ref, or CODE ref. | ||
| 158 | # If a string, then $lookup is compared with `eq' | ||
| 159 | # If a CODE ref, then $lookup is called with the IAs $key attribute and | ||
| 160 | # should return a boolean value. | ||
| 161 | # If an ARRAY ref, then the above two rules are used with each element, the | ||
| 162 | # IA only needs to satisfy one. | ||
| 163 | sub filter_ias { | ||
| 164 | my $lookups = $_[1]; | ||
| 165 | my %ias = %{by_id()}; | ||
| 166 | my %lookups = %$lookups; | ||
| 167 | my @by = keys %lookups; | ||
| 168 | # Ensure lookups are of the form (by => [lookup...]) | ||
| 169 | map { | ||
| 170 | my $cond = $lookups{$_}; | ||
| 171 | $lookups{$_} = [$cond] unless ref $cond eq 'ARRAY'; | ||
| 172 | } @by; | ||
| 173 | while (my ($id, $ia) = each %ias) { | ||
| 174 | delete $ias{$id} unless all { | ||
| 175 | my ($by, $lookup) = ($_, $lookups{$_}); | ||
| 176 | any { | ||
| 177 | ref $_ eq 'CODE' ? $_->($ia->{$by}) : $_ eq $ia->{$by}; | ||
| 178 | } @$lookup; | ||
| 179 | } @by; | ||
| 180 | } | ||
| 181 | return wantarray ? (values %ias) : \%ias; | ||
| 182 | } | ||
| 183 | |||
| 184 | sub get_js { | ||
| 185 | my ($self, $by, $lookup) = @_; | ||
| 186 | return unless $by =~ /id|source/; | ||
| 187 | my $ia = $self->get_ia($by => $lookup); | ||
| 188 | return unless $ia; | ||
| 189 | |||
| 190 | my $id = $ia->{id}; | ||
| 191 | my $metaj = eval { JSON::XS->new->ascii->encode($ia) } || return; | ||
| 192 | return qq(DDH.$id=DDH.$id||{};DDH.$id.meta=$metaj;); | ||
| 193 | } | ||
| 194 | |||
| 195 | # return a hash of IA objects by id | ||
| 196 | sub by_id { $ia_metadata{id} } | ||
| 197 | |||
| 198 | # return a hash of IA metadata for fatheads | ||
| 199 | sub fathead_by_source { $ia_metadata{fathead_source} } | ||
| 200 | |||
| 201 | # return hash if IA metadata by language | ||
| 202 | sub fathead_by_lang { $ia_metadata{fathead_lang} } | ||
| 203 | |||
| 204 | # fathead min lengths | ||
| 205 | sub fathead_by_length { $ia_metadata{fathead_min_length} } | ||
| 206 | |||
| 207 | # Internal function. | ||
| 208 | sub _js_callback_name { | ||
| 209 | my $name = shift; | ||
| 210 | my $fn; | ||
| 211 | if(($fn) = $name =~ /^DDG::\w+::(\w.+)$/o){ | ||
| 212 | $fn =~ s/::/_/og; | ||
| 213 | $fn =~ s/([a-z])([A-Z])/$1_$2/og; | ||
| 214 | $fn = lc $fn; | ||
| 215 | } | ||
| 216 | return $fn; | ||
| 217 | } | ||
| 218 | |||
| 219 | 1; | ||
| 220 | |||
| 221 | __END__ |