| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## File: Lingua::TT::Dict::JSON.pm | ||
| 3 | ## Author: Bryan Jurish <TT/IO.pm> | ||
| 4 | ## Descript: TT Utils: dictionary: JSON | ||
| 5 | |||
| 6 | package Lingua::TT::Dict::JSON; | ||
| 7 | use Lingua::TT::Dict; | ||
| 8 | use Lingua::TT::IO; | ||
| 9 | use JSON::XS; | ||
| 10 | use Carp; | ||
| 11 | use strict; | ||
| 12 | |||
| 13 | ##============================================================================== | ||
| 14 | ## Globals & Constants | ||
| 15 | |||
| 16 | our @ISA = qw(Lingua::TT::Dict); | ||
| 17 | |||
| 18 | ##============================================================================== | ||
| 19 | ## Constructors etc. | ||
| 20 | |||
| 21 | ## $dict = CLASS_OR_OBJECT->new(%opts) | ||
| 22 | ## + %opts, %$dict: | ||
| 23 | ## dict => \%key2val, ##-- dict data; values are refs (encoded/decoded via JSON) | ||
| 24 | sub new { | ||
| 25 | my $that = shift; | ||
| 26 | return $that->SUPER::new(@_); | ||
| 27 | } | ||
| 28 | |||
| 29 | ##============================================================================== | ||
| 30 | ## Methods: Access and Manipulation | ||
| 31 | |||
| 32 | ## $jxs = $obj->jsonxs() | ||
| 33 | sub jsonxs { | ||
| 34 | return $_[0]{jxs} if (ref($_[0]) && defined($_[0]{jxs})); | ||
| 35 | return $_[0]{jxs} = JSON::XS->new->utf8(0)->allow_nonref; | ||
| 36 | } | ||
| 37 | |||
| 38 | ##============================================================================== | ||
| 39 | ## Methods: merge | ||
| 40 | |||
| 41 | ## $dict = $dict->merge($dict2, %opts) | ||
| 42 | ## + include $dict2 entries in $dict, destructively alters $dict | ||
| 43 | ## + %opts: | ||
| 44 | ## append => $bool, ##-- if true, $dict2 values are appended (dict clobber) to $dict1 values | ||
| 45 | sub merge { | ||
| 46 | my ($d1,$d2,%opts) = @_; | ||
| 47 | if (!$opts{append}) { | ||
| 48 | @{$d1->{dict}}{CORE::keys %{$d2->{dict}}} = CORE::values %{$d2->{dict}}; ##-- clobber | ||
| 49 | } else { | ||
| 50 | my $h1 = $d1->{dict}; | ||
| 51 | my $h2 = $d2->{dict}; | ||
| 52 | my $jxs = $d1->jsonxs; | ||
| 53 | my ($key,$val1,$val2); | ||
| 54 | while (($key,$val2)=each %$h2) { | ||
| 55 | if (!defined($val1=$h1->{$key})) { | ||
| 56 | $h1->{$key} = $val2; | ||
| 57 | } | ||
| 58 | elsif (ref($val1) eq 'HASH' && ref($val2) eq 'HASH') { | ||
| 59 | @$val1{keys %$val2} = values %$val2; | ||
| 60 | } | ||
| 61 | elsif (ref($val1) eq 'ARRAY' && ref($val2) eq 'ARRAY') { | ||
| 62 | push(@$val1, @$val2); | ||
| 63 | } | ||
| 64 | else { | ||
| 65 | warn(ref($d1)."::merge(): cannot merge values $val1, $val2 for key '$key'"); | ||
| 66 | $h1->{$key} = $val2; | ||
| 67 | next; | ||
| 68 | } | ||
| 69 | } | ||
| 70 | } | ||
| 71 | return $d1; | ||
| 72 | } | ||
| 73 | |||
| 74 | ##============================================================================== | ||
| 75 | ## Methods: Apply | ||
| 76 | |||
| 77 | ## \&apply = $dict->applySub(%opts) | ||
| 78 | ## + returns a CODE-ref for applying dictionary analysis to a single item | ||
| 79 | ## + returned sub is called without arguments | ||
| 80 | ## - data line to be analyzed (chomped) is in $_ | ||
| 81 | ## - output for current data line should be stored in $_ | ||
| 82 | sub applySub { | ||
| 83 | my ($dict,%opts) = @_; | ||
| 84 | my $jxs = $dict->jsonxs; | ||
| 85 | my $dh = $dict->{dict}; | ||
| 86 | #my $include_empty = $opts{allow_empty}; | ||
| 87 | my ($text,$a_in,$a_dict); | ||
| 88 | return sub { | ||
| 89 | ($text,$a_in) = split(/\t/,$_,2); | ||
| 90 | $a_in = $jxs->decode($a_in) if (defined($a_in)); | ||
| 91 | $a_dict = $dh->{$text}; | ||
| 92 | if (!defined($a_dict)) { | ||
| 93 | ##-- +in, -dict | ||
| 94 | ; | ||
| 95 | } | ||
| 96 | elsif (!defined($a_in)) { | ||
| 97 | ##-- -in, +dict | ||
| 98 | $a_in = $a_dict; | ||
| 99 | } | ||
| 100 | elsif (ref($a_in) eq 'HASH' && ref($a_dict) eq 'HASH') { | ||
| 101 | ##-- +in, +dict: HASH | ||
| 102 | @$a_in{keys %$a_dict} = values %$a_dict; | ||
| 103 | } | ||
| 104 | elsif (ref($a_in) eq 'ARRAY' && ref($a_dict) eq 'ARRAY') { | ||
| 105 | ##-- +in, +dict: ARRAY | ||
| 106 | push(@$a_in, @$a_dict); | ||
| 107 | } | ||
| 108 | else { | ||
| 109 | ##-- +in, +dict: OTHER | ||
| 110 | warn(ref($dict)."::applySub(): cannot merge values $a_in, $a_dict for key '$text'"); | ||
| 111 | $a_in = $a_dict; | ||
| 112 | } | ||
| 113 | $_ = join("\t", $text, (defined($a_in) ? $jxs->encode($a_in) : qw()))."\n"; | ||
| 114 | }; | ||
| 115 | } | ||
| 116 | |||
| 117 | ## $bool = $dict->apply($infh,$outfh,%opts) | ||
| 118 | ## + apply dict to filehandle $fh | ||
| 119 | ## + %opts: | ||
| 120 | ## allow_empty => $bool, ##-- include empty analyses? (default=0) | ||
| 121 | |||
| 122 | ##============================================================================== | ||
| 123 | ## Methods: I/O | ||
| 124 | |||
| 125 | ##-------------------------------------------------------------- | ||
| 126 | ## Methods: I/O: generic | ||
| 127 | |||
| 128 | ## $bool = $dict->setFhLayers($fh,%opts) | ||
| 129 | sub setFhLayers { | ||
| 130 | binmode($_[1],':utf8'); | ||
| 131 | } | ||
| 132 | |||
| 133 | ##-------------------------------------------------------------- | ||
| 134 | ## Methods: I/O: Native | ||
| 135 | |||
| 136 | ## $bool = $dict->saveNativeFh($fh,%opts) | ||
| 137 | ## + saves to filehandle | ||
| 138 | ## + %opts: (none) | ||
| 139 | sub saveNativeFh { | ||
| 140 | my ($dict,$fh,%opts) = @_; | ||
| 141 | binmode($fh,":utf8"); | ||
| 142 | my $jxs = $dict->jsonxs(); | ||
| 143 | my ($key,$val); | ||
| 144 | while (($key,$val)=each(%{$dict->{dict}})) { | ||
| 145 | $fh->print($key, "\t", $jxs->encode($val), "\n"); | ||
| 146 | } | ||
| 147 | return $dict; | ||
| 148 | } | ||
| 149 | |||
| 150 | ## $bool = $dict->loadNativeFh($fh) | ||
| 151 | ## + loads from handle | ||
| 152 | ## + %opts | ||
| 153 | ## encoding => $enc, ##-- sets $fh :encoding flag if defined; default: none | ||
| 154 | ## append => $bool, ##-- if true, multiple entries for a single key will be appended (and maybe promoted to ARRAY) | ||
| 155 | ## merge => $bool, ##-- if true, multiple HASH-entries for a single key will be merged | ||
| 156 | sub loadNativeFh { | ||
| 157 | my ($dict,$fh,%opts) = @_; | ||
| 158 | binmode($fh,":utf8"); | ||
| 159 | $dict = $dict->new() if (!ref($dict)); | ||
| 160 | my $dh = $dict->{dict}; | ||
| 161 | my $jxs = $dict->jsonxs; | ||
| 162 | my $merge = $opts{merge}; | ||
| 163 | my ($line,$key,$val); | ||
| 164 | if ($opts{append} || $opts{merge}) { | ||
| 165 | ##-- append mode | ||
| 166 | my ($oldval); | ||
| 167 | while (defined($line=<$fh>)) { | ||
| 168 | chomp($line); | ||
| 169 | next if ($line =~ /^\s*$/ || $line =~ /^%%/); | ||
| 170 | ($key,$val) = split(/\t/,$line,2); | ||
| 171 | next if (!defined($val)); ##-- don't store keys for undef values (but do for empty string) | ||
| 172 | $val = $jxs->decode($val); | ||
| 173 | if (!defined($oldval=$dh->{$key})) { | ||
| 174 | ##-- new key | ||
| 175 | $dh->{$key} = $val; | ||
| 176 | } | ||
| 177 | elsif ($merge && (ref($oldval)//'') eq 'HASH' && (ref($val)//'') eq 'HASH') { | ||
| 178 | ##-- merge multiple HASH values | ||
| 179 | @$oldval{keys %$val} = values %$val; | ||
| 180 | } | ||
| 181 | else { | ||
| 182 | ##-- append / promote to ARRAY values | ||
| 183 | $oldval = $dh->{$key} = [$oldval] if (!UNIVERSAL::isa($oldval,'ARRAY')); | ||
| 184 | push(@$oldval, UNIVERSAL::isa($val,'ARRAY') ? @$val : $val); | ||
| 185 | } | ||
| 186 | } | ||
| 187 | } else { | ||
| 188 | ##-- clobber mode (default) | ||
| 189 | while (defined($line=<$fh>)) { | ||
| 190 | chomp($line); | ||
| 191 | next if ($line =~ /^\s*$/ || $line =~ /^%%/); | ||
| 192 | ($key,$val) = split(/\t/,$line,2); | ||
| 193 | next if (!defined($val)); ##-- don't store keys for undef values (but do for empty string) | ||
| 194 | $dh->{$key} = $jxs->decode($val); | ||
| 195 | } | ||
| 196 | } | ||
| 197 | return $dict; | ||
| 198 | } | ||
| 199 | |||
| 200 | ##-------------------------------------------------------------- | ||
| 201 | ## Methods: I/O: Bin | ||
| 202 | |||
| 203 | ## ($serialized_string,\@other_refs) = STORABLE_freeze($obj, $cloning_flag) | ||
| 204 | |||
| 205 | ## $obj = STORABLE_thaw($obj, $cloning_flag, $serialized_string, @other_refs) | ||
| 206 | |||
| 207 | ##============================================================================== | ||
| 208 | ## Footer | ||
| 209 | 1; | ||
| 210 | |||
| 211 | __END__ |