| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer::Dict::Json.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: generic analysis dictionary API using JSON values | ||
| 6 | |||
| 7 | package DTA::CAB::Analyzer::Dict::Json; | ||
| 8 | use DTA::CAB::Analyzer ':child'; | ||
| 9 | use DTA::CAB::Analyzer::Dict; | ||
| 10 | use JSON::XS; | ||
| 11 | use IO::File; | ||
| 12 | use Carp; | ||
| 13 | use Encode qw(encode decode); | ||
| 14 | |||
| 15 | use strict; | ||
| 16 | |||
| 17 | ##============================================================================== | ||
| 18 | ## Globals | ||
| 19 | ##============================================================================== | ||
| 20 | |||
| 21 | our @ISA = qw(DTA::CAB::Analyzer::Dict); | ||
| 22 | |||
| 23 | our $CODE_DEFAULT = | ||
| 24 | ('return if (!defined($val=$dhash->{'._am_xlit('$_').'}));' | ||
| 25 | .' $val=$jxs->decode($val);' | ||
| 26 | .' @$_{keys %$val}=values %$val;'); | ||
| 27 | |||
| 28 | ##============================================================================== | ||
| 29 | ## Constructors etc. | ||
| 30 | ##============================================================================== | ||
| 31 | |||
| 32 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 33 | ## + object structure: | ||
| 34 | ## ( | ||
| 35 | ## ##-- Filename Options | ||
| 36 | ## dictFile => $filename, ##-- filename (default=undef): should be TT-dict with JSON-encoded hash values | ||
| 37 | ## | ||
| 38 | ## ##-- Analysis Output | ||
| 39 | ## label => $lab, ##-- analyzer label | ||
| 40 | ## analyzeCode => $code, ##-- pseudo-accessor to perform actual analysis for token ($_); see DTA::CAB::Analyzer::Dict for details | ||
| 41 | ## | ||
| 42 | ## ##-- Analysis Options | ||
| 43 | ## encoding => $enc, ##-- encoding of db file (default='UTF-8'): clobbers $dba{encoding} ; uses DB filters | ||
| 44 | ## | ||
| 45 | ## ##-- Analysis objects | ||
| 46 | ## dbf => $dbf, ##-- underlying Lingua::TT::DBFile object (default=undef) | ||
| 47 | ## dba => \%dba, ##-- args for Lingua::TT::DBFile->new() | ||
| 48 | ## #={ | ||
| 49 | ## # mode => $mode, ##-- default: 0644 | ||
| 50 | ## # dbflags => $flags, ##-- default: O_RDONLY | ||
| 51 | ## # type => $type, ##-- one of 'HASH', 'BTREE', 'RECNO', 'GUESS' (default: 'GUESS') | ||
| 52 | ## # dbinfo => \%dbinfo, ##-- default: "DB_File::${type}INFO"->new(); | ||
| 53 | ## # dbopts => \%opts, ##-- db options (e.g. cachesize,bval,...) -- defaults to none (uses DB_File defaults) | ||
| 54 | ## # } | ||
| 55 | ## ) | ||
| 56 | sub new { | ||
| 57 | my $that = shift; | ||
| 58 | my $dic = $that->SUPER::new( | ||
| 59 | ##-- filenames | ||
| 60 | dictFile => undef, | ||
| 61 | |||
| 62 | ##-- options | ||
| 63 | encoding => 'UTF-8', | ||
| 64 | |||
| 65 | ##-- analysis output | ||
| 66 | label => 'dict_json', | ||
| 67 | analyzeCode => $CODE_DEFAULT, | ||
| 68 | |||
| 69 | ##-- JSON parser (segfaults: see jsonxs() method, below) | ||
| 70 | #jxs => __PACKAGE__->jsonxs, | ||
| 71 | |||
| 72 | ##-- user args | ||
| 73 | @_ | ||
| 74 | ); | ||
| 75 | return $dic; | ||
| 76 | } | ||
| 77 | |||
| 78 | |||
| 79 | |||
| 80 | ##============================================================================== | ||
| 81 | ## Methods: Embedded API | ||
| 82 | ##============================================================================== | ||
| 83 | |||
| 84 | ##-------------------------------------------------------------- | ||
| 85 | ## Methods: I/O: Input: all | ||
| 86 | |||
| 87 | ## $bool = $dic->ensureLoaded() | ||
| 88 | ## + ensures analyzer data is loaded from default files | ||
| 89 | sub ensureLoaded { | ||
| 90 | my $dic = shift; | ||
| 91 | my $rc = 1; | ||
| 92 | if ( defined($dic->{dictFile}) && !$dic->dictOk ) { | ||
| 93 | $dic->info("loading dictionary file '$dic->{dictFile}'"); | ||
| 94 | $rc &&= $dic->{ttd}->loadFile($dic->{dictFile}, encoding=>$dic->{encoding}); | ||
| 95 | |||
| 96 | ##-- json with utf8(1) wants utf8-encoded octets, so we munge them here | ||
| 97 | foreach (values %{$dic->{ttd}{dict}}) { | ||
| 98 | #$_ = Encode::encode_utf8($_) if (utf8::is_utf8($_)); | ||
| 99 | utf8::encode($_) if (utf8::is_utf8($_)); | ||
| 100 | } | ||
| 101 | } | ||
| 102 | return $rc; | ||
| 103 | } | ||
| 104 | |||
| 105 | ## $dic = $dic->decodeDictValues() | ||
| 106 | ## + runs $jxs->decode() on all dict values (for in-memory dicts) | ||
| 107 | sub decodeDictValues { | ||
| 108 | my $dic = shift; | ||
| 109 | return $dic if (!$dic->{ttd} || !$dic->{ttd}{dict}); | ||
| 110 | |||
| 111 | my $dict = $dic->{ttd}{dict}; | ||
| 112 | my $jxs = $dic->jsonxs; | ||
| 113 | foreach (keys %$dict) { | ||
| 114 | $dict->{$_} = $jxs->decode($dict->{$_}) if (!ref($dict->{$_})); | ||
| 115 | } | ||
| 116 | return $dic; | ||
| 117 | } | ||
| 118 | |||
| 119 | |||
| 120 | ##============================================================================== | ||
| 121 | ## Methods: Persistence | ||
| 122 | ##============================================================================== | ||
| 123 | |||
| 124 | ##====================================================================== | ||
| 125 | ## Methods: Persistence: Perl | ||
| 126 | |||
| 127 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 128 | ## + returns list of keys not to be saved | ||
| 129 | sub noSaveKeys { | ||
| 130 | my $that = shift; | ||
| 131 | return ($that->SUPER::noSaveKeys, qw(jxs)); | ||
| 132 | } | ||
| 133 | |||
| 134 | ## @keys = $class_or_obj->noSaveBinKeys() | ||
| 135 | ## + returns list of keys not to be saved in binary mode | ||
| 136 | sub noSaveBinKeys { | ||
| 137 | my $that = shift; | ||
| 138 | return ($that->SUPER::noSaveBinKeys, qw(jxs)); | ||
| 139 | } | ||
| 140 | |||
| 141 | |||
| 142 | ##============================================================================== | ||
| 143 | ## Methods: Analysis | ||
| 144 | ##============================================================================== | ||
| 145 | |||
| 146 | ##------------------------------------------------------------------------ | ||
| 147 | ## Methods: Analysis: v1.x: API | ||
| 148 | |||
| 149 | ## $doc = $anl->analyzeTypes($doc,\%types,\%opts) | ||
| 150 | ## + perform type-wise analysis of all (text) types in $doc->{types} | ||
| 151 | ## + INHERITED from Dict | ||
| 152 | |||
| 153 | ##------------------------------------------------------------------------ | ||
| 154 | ## Methods: Analysis: Utils | ||
| 155 | |||
| 156 | ## $jxs = $dict->jsonxs() | ||
| 157 | ## $jxs = $CLASS->jsonxs() | ||
| 158 | ## + returns underlying JSON::XS object or creates appropriate new object | ||
| 159 | ## + DISABLED (b/c of segfaults): caches if not already defined for object call | ||
| 160 | ##-- | ||
| 161 | ## Program received signal SIGSEGV, Segmentation fault. | ||
| 162 | ## 0xb7a03584 in XS_JSON__XS_DESTROY () from /usr/lib/perl5/auto/JSON/XS/XS.so | ||
| 163 | ## (gdb) bt | ||
| 164 | ## #0 0xb7a03584 in XS_JSON__XS_DESTROY () from /usr/lib/perl5/auto/JSON/XS/XS.so | ||
| 165 | ## #1 0x080d5d7b in Perl_pp_entersub () | ||
| 166 | ## #2 0x08078bb8 in Perl_call_sv () | ||
| 167 | ## #3 0x080e8090 in Perl_sv_clear () | ||
| 168 | ## #4 0x080e87da in Perl_sv_free2 () | ||
| 169 | ## #5 0x080dd8d9 in ?? () | ||
| 170 | ## #6 0x080dd939 in Perl_sv_clean_objs () | ||
| 171 | ## #7 0x0807dcaf in perl_destruct () | ||
| 172 | ## #8 0x080642a5 in main () | ||
| 173 | ##-- | ||
| 174 | sub jsonxs { | ||
| 175 | return $_[0]{jxs} if (ref($_[0]) && defined($_[0]{jxs})); | ||
| 176 | my $jxs = JSON::XS->new->utf8(1)->relaxed(1)->shrink(1)->allow_blessed(1)->convert_blessed(1); | ||
| 177 | #return $_[0]{jxs} = $jxs if (ref($_[0])); | ||
| 178 | return $jxs; | ||
| 179 | } | ||
| 180 | |||
| 181 | ## $prefix = $dict->analyzePre() | ||
| 182 | sub analyzePre { | ||
| 183 | my $dic = shift; | ||
| 184 | return 'my $jxs=$anl->jsonxs; '.$dic->SUPER::analyzePre(); | ||
| 185 | } | ||
| 186 | |||
| 187 | ## $coderef = $dict->analyzeCode() | ||
| 188 | ## $coderef = $dict->analyzeCode($code) | ||
| 189 | ## + inherited | ||
| 190 | |||
| 191 | |||
| 192 | |||
| 193 | 1; ##-- be happy | ||
| 194 | |||
| 195 | __END__ |