| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer::LangId.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: Lingua::LangId::Map wrapper | ||
| 6 | |||
| 7 | package DTA::CAB::Analyzer::LangId; | ||
| 8 | use DTA::CAB::Analyzer; | ||
| 9 | use DTA::CAB::Datum ':all'; | ||
| 10 | use Lingua::LangId::Map; | ||
| 11 | |||
| 12 | use Encode qw(encode decode); | ||
| 13 | use IO::File; | ||
| 14 | use Carp; | ||
| 15 | |||
| 16 | use strict; | ||
| 17 | |||
| 18 | ##============================================================================== | ||
| 19 | ## Globals | ||
| 20 | ##============================================================================== | ||
| 21 | |||
| 22 | our @ISA = qw(DTA::CAB::Analyzer); | ||
| 23 | |||
| 24 | ##============================================================================== | ||
| 25 | ## Constructors etc. | ||
| 26 | ##============================================================================== | ||
| 27 | |||
| 28 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 29 | ## + object structure: | ||
| 30 | ## ( | ||
| 31 | ## ##-- Filename Options | ||
| 32 | ## mapFile => $filename, ##-- default: none (REQUIRED) | ||
| 33 | ## | ||
| 34 | ## ##-- Analysis Options | ||
| 35 | ## analyzeWhich => $which, ##-- one of 'token', 'sentence', 'document'; default='document' | ||
| 36 | ## vlabel => $label, ##-- verbose destination key (default='langid') | ||
| 37 | ## label => $label, ##-- simple destination key (default='lang') | ||
| 38 | ## | ||
| 39 | ## | ||
| 40 | ## ##-- Analysis Objects | ||
| 41 | ## map => $map, ##-- a Lingua::LangId::Map object | ||
| 42 | ## ) | ||
| 43 | sub new { | ||
| 44 | my $that = shift; | ||
| 45 | my $lid = $that->SUPER::new( | ||
| 46 | ##-- filenames | ||
| 47 | mapFile => undef, | ||
| 48 | |||
| 49 | ##-- options | ||
| 50 | analyzeWhich => 'document', | ||
| 51 | vlabel => 'langid', | ||
| 52 | label => 'lang', | ||
| 53 | |||
| 54 | ##-- analysis objects | ||
| 55 | #map => undef, | ||
| 56 | |||
| 57 | ##-- user args | ||
| 58 | @_ | ||
| 59 | ); | ||
| 60 | return $lid; | ||
| 61 | } | ||
| 62 | |||
| 63 | ## $lid = $lid->clear() | ||
| 64 | sub clear { | ||
| 65 | my $lid = shift; | ||
| 66 | |||
| 67 | ##-- analysis sub(s) | ||
| 68 | $lid->dropClosures(); | ||
| 69 | |||
| 70 | ##-- analysis objects | ||
| 71 | delete($lid->{map}); | ||
| 72 | |||
| 73 | return $lid; | ||
| 74 | } | ||
| 75 | |||
| 76 | ##============================================================================== | ||
| 77 | ## Methods: Generic | ||
| 78 | ##============================================================================== | ||
| 79 | |||
| 80 | ## $bool = $lid->mapOk() | ||
| 81 | ## + should return false iff map is undefined or "empty" | ||
| 82 | ## + default version checks for non-empty 'map' and 'sigs' | ||
| 83 | sub mapOk { | ||
| 84 | return defined($_[0]{map}) && %{$_[0]{map}{sigs}}; | ||
| 85 | } | ||
| 86 | |||
| 87 | ## @keys = $anl->typeKeys(\%opts) | ||
| 88 | ## + returns list of type-wise keys to be expanded for this analyzer by expandTypes() | ||
| 89 | ## + default returns @{$anl->{typeKeys}} if defined, otherwise ($anl->{label}) | ||
| 90 | sub typeKeys { | ||
| 91 | return ($_[0]{typeKeys} ? @{$_[0]{typeKeys}} : qw()); | ||
| 92 | } | ||
| 93 | |||
| 94 | ##============================================================================== | ||
| 95 | ## Methods: I/O | ||
| 96 | ##============================================================================== | ||
| 97 | |||
| 98 | ##-------------------------------------------------------------- | ||
| 99 | ## Methods: I/O: Input: all | ||
| 100 | |||
| 101 | ## $bool = $lid->ensureLoaded() | ||
| 102 | ## + ensures model data is loaded from default files (if available) | ||
| 103 | sub ensureLoaded { | ||
| 104 | my $lid = shift; | ||
| 105 | ##-- ensure: map | ||
| 106 | if ( defined($lid->{mapFile}) && !$lid->mapOk ) { | ||
| 107 | return $lid->loadMap($lid->{mapFile}); | ||
| 108 | } | ||
| 109 | return 1; ##-- allow empty models | ||
| 110 | } | ||
| 111 | |||
| 112 | ##-------------------------------------------------------------- | ||
| 113 | ## Methods: I/O: Input: Map | ||
| 114 | |||
| 115 | ## $lid = $lid->loadMap($map_file) | ||
| 116 | sub loadMap { | ||
| 117 | my ($lid,$mapfile) = @_; | ||
| 118 | $lid->info("loading map file '$mapfile'"); | ||
| 119 | if (!defined($lid->{map})) { | ||
| 120 | $lid->{map} = Lingua::LangId::Map->new() | ||
| 121 | or $lid->logconfess("could not create map object: $!"); | ||
| 122 | } | ||
| 123 | $lid->{map}->loadBinFile($mapfile) | ||
| 124 | or $lid->logconfess("loadMap(): load failed for '$mapfile': $!"); | ||
| 125 | $lid->dropClosures(); | ||
| 126 | return $lid; | ||
| 127 | } | ||
| 128 | |||
| 129 | ##============================================================================== | ||
| 130 | ## Methods: Persistence | ||
| 131 | ##============================================================================== | ||
| 132 | |||
| 133 | ##====================================================================== | ||
| 134 | ## Methods: Persistence: Perl | ||
| 135 | |||
| 136 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 137 | ## + returns list of keys not to be saved | ||
| 138 | sub noSaveKeys { | ||
| 139 | my $that = shift; | ||
| 140 | return ($that->SUPER::noSaveKeys, qw(map)); | ||
| 141 | } | ||
| 142 | |||
| 143 | ## $saveRef = $obj->savePerlRef() | ||
| 144 | ## + inherited from DTA::CAB::Persistent | ||
| 145 | |||
| 146 | ## $loadedObj = $CLASS_OR_OBJ->loadPerlRef($ref) | ||
| 147 | ## + implicitly calls $obj->clear() | ||
| 148 | sub loadPerlRef { | ||
| 149 | my ($that,$ref) = @_; | ||
| 150 | my $obj = $that->SUPER::loadPerlRef($ref); | ||
| 151 | $obj->clear(); | ||
| 152 | return $obj; | ||
| 153 | } | ||
| 154 | |||
| 155 | ##============================================================================== | ||
| 156 | ## Methods: Analysis | ||
| 157 | ##============================================================================== | ||
| 158 | |||
| 159 | ##------------------------------------------------------------------------ | ||
| 160 | ## Methods: Analysis: Generic | ||
| 161 | |||
| 162 | ## $bool = $anl->canAnalyze() | ||
| 163 | ## + returns true if analyzer can perform its function (e.g. data is loaded & non-empty) | ||
| 164 | sub canAnalyze { | ||
| 165 | return $_[0]->mapOk(); | ||
| 166 | } | ||
| 167 | |||
| 168 | ##------------------------------------------------------------------------ | ||
| 169 | ## Methods: Analysis: Generic | ||
| 170 | |||
| 171 | ## $thingy = $lid->analyzeThingy($thingy, \$str, \%opts) | ||
| 172 | sub analyzeThingy { | ||
| 173 | my ($lid,$thingy,$ref,$opts) = @_; | ||
| 174 | my $details = $lid->{map}->applyString($ref); | ||
| 175 | $thingy->{$lid->{vlabel}} = $details if (defined($lid->{vlabel})); | ||
| 176 | if (defined($lid->{label})) { | ||
| 177 | my $best = (sort {$details->{$a}{kldp} <=> $details->{$b}{kldp}} grep {$details->{$_}{match}} keys %$details)[0]; | ||
| 178 | $thingy->{$lid->{label}} = $best if (defined($best)); | ||
| 179 | } | ||
| 180 | return $thingy; | ||
| 181 | } | ||
| 182 | |||
| 183 | ##------------------------------------------------------------------------ | ||
| 184 | ## Methods: Analysis: v1.x: API | ||
| 185 | |||
| 186 | ## $doc = $anl->analyzeDocument($doc,\%opts) | ||
| 187 | ## + analyze a DTA::CAB::Document $doc | ||
| 188 | ## + top-level API routine | ||
| 189 | sub analyzeLocal { | ||
| 190 | my ($anl,$doc,$opts) = @_; | ||
| 191 | return undef if (!$anl->ensureLoaded()); ##-- uh-oh... | ||
| 192 | return $doc if (!$anl->canAnalyze); ##-- ok... | ||
| 193 | |||
| 194 | my $xlit = sub { $_[0]{xlit} ? $_[0]{xlit}{latin1Text} : $_[0]{text} }; | ||
| 195 | my ($str,$x); | ||
| 196 | if ($anl->{analyzeWhich} eq 'document') { | ||
| 197 | $str = join(' ', map {$xlit->($_)} map {@{$_->{tokens}}} @{$doc->{body}}); | ||
| 198 | $anl->analyzeThingy($doc,\$str,$opts); | ||
| 199 | } | ||
| 200 | elsif ($anl->{analyzeWhich} eq 'sentence') { | ||
| 201 | foreach $x (@{$doc->{body}}) { | ||
| 202 | $str = join(' ', map {$xlit->($_)} @{$x->{tokens}}); | ||
| 203 | $anl->analyzeThingy($x,\$str,$opts); | ||
| 204 | } | ||
| 205 | } | ||
| 206 | elsif ($anl->{analyzeWhich} eq 'token' || $anl->{analyzeWhich} eq 'type') { | ||
| 207 | foreach $x (map {@{$_->{tokens}}} @{$doc->{body}}) { | ||
| 208 | $str = $xlit->($x); | ||
| 209 | $anl->analyzeThingy($x,\$str,$opts); | ||
| 210 | } | ||
| 211 | } | ||
| 212 | else { | ||
| 213 | $anl->logconfess("analyzeDocument(): unknown {analyzeWhich}='$anl->{analyzeWhich}'"); | ||
| 214 | } | ||
| 215 | return $doc; | ||
| 216 | } | ||
| 217 | |||
| 218 | 1; ##-- be happy | ||
| 219 | |||
| 220 | __END__ |