| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Format::Raw::Waste.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: Datum parser: raw untokenized text (using moot/waste) | ||
| 6 | |||
| 7 | package DTA::CAB::Format::Raw::Waste; | ||
| 8 | 1 | ▶ | use DTA::CAB::Format; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 9 | 1 | ▶ | use DTA::CAB::Format::TT; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 10 | 1 | ▶ | use DTA::CAB::Format::Raw::Base; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 11 | 1 | ▶ | use DTA::CAB::Datum ':all'; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 12 | 1 | ▶ | use DTA::CAB::Utils qw(file_mtime timestamp_str); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 13 | 1 | ▶ | use IO::File; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 14 | 1 | ▶ | use Cwd qw(abs_path); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 15 | 1 | ▶ | use Encode qw(encode decode); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 16 | 0 | ▶ | use Moot::Waste; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 17 | use Carp; | ||
| 18 | use strict; | ||
| 19 | |||
| 20 | ##============================================================================== | ||
| 21 | ## Globals | ||
| 22 | ##============================================================================== | ||
| 23 | |||
| 24 | our @ISA = qw(DTA::CAB::Format::Raw::Base); | ||
| 25 | |||
| 26 | BEGIN { | ||
| 27 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>'raw-waste', filenameRegex=>qr/\.(?i:raw-waste|txt-waste)$/); | ||
| 28 | } | ||
| 29 | |||
| 30 | our @DEFAULT_WASTERC_PATHS = | ||
| 31 | ( | ||
| 32 | ($ENV{TOKWRAP_RCDIR} ? "$ENV{TOKWRAP_RCDIR}/waste/waste.rc" : qw()), | ||
| 33 | (defined($DTA::TokWrap::Version::VERSION) ? "$DTA::TokWrap::Version::RCDIR/waste/waste.rc" : qw()), | ||
| 34 | (defined $ENV{HOME} ? "$ENV{HOME}/.wasterc" : qw()), | ||
| 35 | "/etc/wasterc", | ||
| 36 | "/etc/default/wasterc" | ||
| 37 | ); | ||
| 38 | |||
| 39 | ## $logLoad : default logLoad option | ||
| 40 | our $logLoad = 'trace'; | ||
| 41 | |||
| 42 | ## $logCache : default logCache option | ||
| 43 | our $logCache = undef; | ||
| 44 | |||
| 45 | ## $logRun : default logRun option | ||
| 46 | our $logRun = undef; | ||
| 47 | |||
| 48 | ##============================================================================== | ||
| 49 | ## Constructors etc. | ||
| 50 | ##============================================================================== | ||
| 51 | |||
| 52 | ## $fmt = CLASS_OR_OBJ->new(%args) | ||
| 53 | ## + object structure: assumed HASH | ||
| 54 | ## { | ||
| 55 | ## ##-- Input | ||
| 56 | ## doc => $doc, ##-- buffered input document | ||
| 57 | ## wasterc => $rcFile, ##-- waste .rc file; default: "$HOME/.wasterc" || "/etc/wasterc" || "/etc/default/waste" | ||
| 58 | ## tloc => $attr, ##-- pased to Format::TT::parseTokenizerString(): if non-empty, sets $w->{$attr}="$off $len"; default=0 | ||
| 59 | ## | ||
| 60 | ## ##-- Runtime | ||
| 61 | ## wmodel => \%wmodel ##-- waste model; %wmodel=( | ||
| 62 | ## # config => \%config, #-- parsed rcfile (see loadModelConfig()) | ||
| 63 | ## # loaded => $time, #-- unix timestamp of last model load | ||
| 64 | ## # wscanner => $scanner, #-- waste scanner | ||
| 65 | ## # wlexer => $lexer, #-- waste lexer | ||
| 66 | ## # wtagger => $tagger, #-- waste tagger | ||
| 67 | ## # wdecoder => $decoder, #-- waste decoder | ||
| 68 | ## # wannotator => $wannot, #-- waste annotator | ||
| 69 | ## # wwriter => $wwriter, #-- native-format writer (hack) | ||
| 70 | ## # ) | ||
| 71 | ## | ||
| 72 | ## ##-- logging (in order of increasing verbosity) | ||
| 73 | ## logLoad => $level, # log-level for model loading (default=$logLoad) | ||
| 74 | ## logCache => $level, # cache operation log-level (default=$logCache) | ||
| 75 | ## logRun => $level, # runtime operation log-level (default=$logRun) | ||
| 76 | ## | ||
| 77 | ## ##-- Common | ||
| 78 | ## #utf8 => $bool, ##-- utf8 mode always on | ||
| 79 | sub new { | ||
| 80 | my $that = shift; | ||
| 81 | my $fmt = bless({ | ||
| 82 | ##-- common | ||
| 83 | utf8 => 1, | ||
| 84 | |||
| 85 | ##-- input | ||
| 86 | doc => undef, | ||
| 87 | wasterc => undef, | ||
| 88 | tloc => undef, | ||
| 89 | |||
| 90 | ##-- runtime | ||
| 91 | wmodel => undef, | ||
| 92 | |||
| 93 | ##-- logging | ||
| 94 | logLoad => $logLoad, | ||
| 95 | logCache => $logCache, | ||
| 96 | logRun => $logRun, | ||
| 97 | |||
| 98 | ##-- user args | ||
| 99 | @_ | ||
| 100 | }, ref($that)||$that); | ||
| 101 | return $fmt; | ||
| 102 | } | ||
| 103 | |||
| 104 | ##============================================================================== | ||
| 105 | ## Methods: Persistence | ||
| 106 | ##============================================================================== | ||
| 107 | |||
| 108 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 109 | ## + returns list of keys not to be saved | ||
| 110 | ## + default just returns empty list | ||
| 111 | sub noSaveKeys { | ||
| 112 | return (shift->SUPER::noSaveKeys(), qw(doc wmodel wscanner wlexer wtagger wdecoder wannotator wwriter)); | ||
| 113 | } | ||
| 114 | |||
| 115 | ##============================================================================== | ||
| 116 | ## Methods: Local: log levels | ||
| 117 | |||
| 118 | ## $logLoad = $CLASS_OR_OBJECT->logLevelLoad() | ||
| 119 | sub logLevelLoad { return ref($_[0]) ? $_[0]{logLoad} : $logLoad; } | ||
| 120 | |||
| 121 | ## $logCache = $CLASS_OR_OBJECT->logLevelCache() | ||
| 122 | sub logLevelCache { return ref($_[0]) ? $_[0]{logCache} : $logCache; } | ||
| 123 | |||
| 124 | ## $logRun = $CLASS_OR_OBJECT->logLevelRun() | ||
| 125 | sub logLevelRun { return ref($_[0]) ? $_[0]{logRun} : $logRun; } | ||
| 126 | |||
| 127 | |||
| 128 | ##============================================================================== | ||
| 129 | ## Methods: Local: model caching | ||
| 130 | |||
| 131 | ## %MODELS : cached models ("$wasterc_abspath:$PID" => \%wmodel) | ||
| 132 | our %MODELS = qw(); | ||
| 133 | |||
| 134 | END { | ||
| 135 | ##-- END block clears cached waste models for this PID | ||
| 136 | my @cached = grep {/:${$}$/} keys %MODELS; | ||
| 137 | __PACKAGE__->vlog(__PACKAGE__->logLevelCache, "clearing model cache for PID $$: ", join(' ', @cached)); | ||
| 138 | delete @MODELS{@cached}; | ||
| 139 | } | ||
| 140 | |||
| 141 | ## \%wmodel_or_undef = $fmt->ensureModel() | ||
| 142 | ## \%wmodel_or_undef = $fmt->ensureModel($wasterc) | ||
| 143 | ## \%wmodel_or_undef = CLASS->ensureModel($wasterc) | ||
| 144 | ## + loads cached model if available; otherwise populates cache | ||
| 145 | sub ensureModel { | ||
| 146 | my ($fmt,$rcfile) = @_; | ||
| 147 | $rcfile = $fmt->{rcfile} if (ref($fmt) && !$rcfile); | ||
| 148 | |||
| 149 | ##-- get rc file | ||
| 150 | if (!$rcfile) { | ||
| 151 | $rcfile = (grep {-f $_} @DEFAULT_WASTERC_PATHS)[0]; | ||
| 152 | $fmt->logconfess("getModel(): no wasterc specified!") if (!$rcfile); | ||
| 153 | } | ||
| 154 | |||
| 155 | ##-- cache lookup | ||
| 156 | my $mkey = abs_path($rcfile).":$$"; | ||
| 157 | my $wmodel = $MODELS{$mkey}; | ||
| 158 | ##-- debug | ||
| 159 | if (defined($wmodel)) { | ||
| 160 | $fmt->vlog($fmt->logLevelCache, "found cached waste model: $mkey"); | ||
| 161 | |||
| 162 | my $modeltime = int(file_mtime($rcfile)); | ||
| 163 | if ($wmodel->{loaded} >= $modeltime) { | ||
| 164 | $fmt->vlog($fmt->logLevelCache, "using cached waste model: $mkey"); | ||
| 165 | return $wmodel; | ||
| 166 | } else { | ||
| 167 | $fmt->vlog($fmt->logLevelCache, | ||
| 168 | "cached waste model is stale (".timestamp_str($wmodel->{loaded}).' < '.timestamp_str($modeltime)."): $mkey"); | ||
| 169 | } | ||
| 170 | } else { | ||
| 171 | $fmt->vlog($fmt->logLevelCache, "no cached waste model found: $mkey"); | ||
| 172 | } | ||
| 173 | |||
| 174 | ##-- not cached or stale: create a new model & update the cache | ||
| 175 | $fmt->vlog($fmt->logLevelCache, "updating waste model cache: $mkey"); | ||
| 176 | my $config = $fmt->loadModelConfig($rcfile); | ||
| 177 | $wmodel = $MODELS{$mkey} = | ||
| 178 | { | ||
| 179 | config => $config, | ||
| 180 | loaded => time(), | ||
| 181 | wscanner => Moot::Waste::Scanner->new( $Moot::ioFormat{text}|$Moot::ioFormat{location} ), | ||
| 182 | wlexer => Moot::Waste::Lexer->new( $Moot::ioFormat{wd}|$Moot::ioFormat{location} ), | ||
| 183 | wtagger => Moot::HMM->new(), | ||
| 184 | wdecoder => Moot::Waste::Decoder->new( $Moot::ioFormat{m}|$Moot::ioFormat{location} ), | ||
| 185 | wannotator => Moot::Waste::Annotator->new( $Moot::ioFormat{mr}|$Moot::ioFormat{location} ), | ||
| 186 | wwriter => Moot::TokenWriter::Native->new( $Moot::ioFormat{mr}|$Moot::ioFormat{location} ), | ||
| 187 | }; | ||
| 188 | |||
| 189 | ##-- load model from configuration options | ||
| 190 | $wmodel->{wlexer}->abbrevs->load($config->{abbrevs}) if ($config->{abbrevs}); | ||
| 191 | $wmodel->{wlexer}->conjunctions->load($config->{conjunctions}) if ($config->{conjunctions}); | ||
| 192 | $wmodel->{wlexer}->stopwords->load($config->{stopwords}) if ($config->{stopwords}); | ||
| 193 | $wmodel->{wlexer}->dehyphenate($config->{dehyphenate} ? 1 : 0) if (exists $config->{dehyphenate}); | ||
| 194 | if ($config->{hmm}) { | ||
| 195 | $wmodel->{wtagger}->load($config->{hmm}) or $fmt->logconfess("failed to load waste model '$config->{hmm}'"); | ||
| 196 | } else { | ||
| 197 | $fmt->logconfess("no 'hmm' key specified in waste rc-file '$rcfile'"); | ||
| 198 | } | ||
| 199 | |||
| 200 | return $wmodel; | ||
| 201 | } | ||
| 202 | |||
| 203 | ## \%config = CLASS_OR_OBJECT->loadModelConfig($wasterc) | ||
| 204 | ## + loads rc-file with keys qw(abbrevs conjunctions stopwords dehyphenate hmm) | ||
| 205 | sub loadModelConfig { | ||
| 206 | my ($fmt,$rcfile) = @_; | ||
| 207 | $fmt->vlog($fmt->logLevelCache, "loading waste model configuration $rcfile"); | ||
| 208 | |||
| 209 | open(my $rc,"<$rcfile") | ||
| 210 | or $fmt->logconfess("open failed for waste-rc $rcfile: $!"); | ||
| 211 | my $config = {rcfile=>$rcfile}; | ||
| 212 | while (defined($_=<$rc>)) { | ||
| 213 | next if (/^\#/ || /^\s*$/); | ||
| 214 | chomp; | ||
| 215 | my ($opt,$val) = split(/\s/,$_,2); | ||
| 216 | if ($opt =~ /^abbr/) { $config->{abbrevs}=$val } | ||
| 217 | elsif ($opt =~ /^conj/) { $config->{conjunctions}=$val; } | ||
| 218 | elsif ($opt =~ /^stop/) { $config->{stopwords}=$val; } | ||
| 219 | elsif ($opt =~ /^dehyph/) { $config->{dehyphenate}=$val; } | ||
| 220 | elsif ($opt =~ /^no-dehyph/) { $config->{dehyphenate}=$val; } | ||
| 221 | elsif ($opt =~ /^(?:hmm|model)/) { $config->{hmm}=$val; } | ||
| 222 | else { | ||
| 223 | ; ##-- ignore other options | ||
| 224 | } | ||
| 225 | } | ||
| 226 | close($rc); | ||
| 227 | |||
| 228 | $fmt->vlog($fmt->logLevelLoad, "loaded waste model configuration $rcfile"); | ||
| 229 | return $config; | ||
| 230 | } | ||
| 231 | |||
| 232 | ##============================================================================== | ||
| 233 | ## Methods: Model I/O | ||
| 234 | |||
| 235 | ## $fmt_or_undef = $fmt->ensureLoaded() | ||
| 236 | sub ensureLoaded { | ||
| 237 | my $fmt = shift; | ||
| 238 | return $fmt if ($fmt->{wmodel} && $fmt->{wmodel}{wtagger}); | ||
| 239 | |||
| 240 | ##-- get rc file | ||
| 241 | if (!$fmt->{wasterc}) { | ||
| 242 | $fmt->{wasterc} = (grep {-f $_} @DEFAULT_WASTERC_PATHS)[0]; | ||
| 243 | $fmt->logconfess("cannot tokenize without a model -- specify wasterc!") if (!$fmt->{wasterc}); | ||
| 244 | } | ||
| 245 | #$fmt->vlog($fmt->logLevelRun, "using waste model configuration $fmt->{wasterc}"); | ||
| 246 | |||
| 247 | return $fmt->loadModel(); | ||
| 248 | } | ||
| 249 | |||
| 250 | ## $fmt_or_undef = $fmt->loadModel() | ||
| 251 | ## $fmt_or_undef = $fmt->loadModel($rcfile) | ||
| 252 | ## + backwards-compatible | ||
| 253 | sub loadModel { | ||
| 254 | my ($fmt,$rcfile) = @_; | ||
| 255 | $rcfile //= $fmt->{wasterc}; | ||
| 256 | $fmt->{wasterc} = $rcfile; | ||
| 257 | $fmt->vlog($fmt->logLevelRun, "using waste model $rcfile"); | ||
| 258 | $fmt->{wmodel} = $fmt->ensureModel($rcfile) | ||
| 259 | or $fmt->logconfess("failed to load waste model '$rcfile': $!"); | ||
| 260 | |||
| 261 | return $fmt; | ||
| 262 | } | ||
| 263 | |||
| 264 | ##============================================================================== | ||
| 265 | ## Methods: Input | ||
| 266 | ##============================================================================== | ||
| 267 | |||
| 268 | ##-------------------------------------------------------------- | ||
| 269 | ## Methods: Input: Input selection | ||
| 270 | |||
| 271 | ## $fmt = $fmt->close() | ||
| 272 | sub close { | ||
| 273 | delete($_[0]{doc}); | ||
| 274 | return $_[0]->SUPER::close(@_[1..$#_]); | ||
| 275 | } | ||
| 276 | |||
| 277 | ## $fmt = $fmt->fromString( $string) | ||
| 278 | ## $fmt = $fmt->fromString(\$string) | ||
| 279 | ## + select input from string $string | ||
| 280 | ## + default calls fromFh() | ||
| 281 | sub fromString { | ||
| 282 | my $fmt = shift; | ||
| 283 | my $ref = ref($_[0]) ? $_[0] : \$_[0]; | ||
| 284 | return $fmt->SUPER::fromString(@_) if (!utf8::is_utf8($$ref)); | ||
| 285 | |||
| 286 | my $raw = $$ref; | ||
| 287 | utf8::encode($raw); | ||
| 288 | return $fmt->SUPER::fromString(\$raw); | ||
| 289 | } | ||
| 290 | |||
| 291 | ## $fmt = $fmt->fromFh($fh) | ||
| 292 | sub fromFh { | ||
| 293 | my ($fmt,$fh) = @_; | ||
| 294 | $fmt->ensureLoaded(); | ||
| 295 | |||
| 296 | my ($ttstr); | ||
| 297 | my $wmodel = $fmt->{wmodel}; | ||
| 298 | $wmodel->{wlexer}->close(); | ||
| 299 | $wmodel->{wscanner}->close(); | ||
| 300 | $wmodel->{wscanner}->from_fh($fh); | ||
| 301 | $wmodel->{wlexer}->scanner($wmodel->{wscanner}); | ||
| 302 | $wmodel->{wwriter}->to_string($ttstr); | ||
| 303 | $wmodel->{wdecoder}->sink($wmodel->{wannotator}); | ||
| 304 | $wmodel->{wannotator}->sink($wmodel->{wwriter}); | ||
| 305 | $wmodel->{wtagger}->tag_stream($wmodel->{wlexer},$wmodel->{wdecoder}); | ||
| 306 | $wmodel->{wdecoder}->close(); | ||
| 307 | $wmodel->{wannotator}->close(); | ||
| 308 | $wmodel->{wwriter}->close(); | ||
| 309 | |||
| 310 | ##-- construct & buffer document | ||
| 311 | utf8::decode($ttstr) if (!utf8::is_utf8($ttstr)); | ||
| 312 | $fmt->{doc} = DTA::CAB::Format::TT->parseTokenizerString(\$ttstr, {tloc=>$fmt->{tloc}}); | ||
| 313 | return $fmt; | ||
| 314 | } | ||
| 315 | |||
| 316 | ##-------------------------------------------------------------- | ||
| 317 | ## Methods: Input: Generic API | ||
| 318 | |||
| 319 | ## $doc = $fmt->parseDocument() | ||
| 320 | sub parseDocument { | ||
| 321 | return $_[0]{doc}; | ||
| 322 | } | ||
| 323 | |||
| 324 | |||
| 325 | ##============================================================================== | ||
| 326 | ## Methods: Output | ||
| 327 | ## + output not supported | ||
| 328 | ##============================================================================== | ||
| 329 | |||
| 330 | ##-------------------------------------------------------------- | ||
| 331 | ## Methods: Output: Generic | ||
| 332 | |||
| 333 | ## $type = $fmt->mimeType() | ||
| 334 | ## + default returns text/plain | ||
| 335 | sub mimeType { return 'text/plain'; } | ||
| 336 | |||
| 337 | ## $ext = $fmt->defaultExtension() | ||
| 338 | ## + returns default filename extension for this format | ||
| 339 | sub defaultExtension { return '.raw'; } | ||
| 340 | |||
| 341 | |||
| 342 | 1; ##-- be happy | ||
| 343 | |||
| 344 | __END__ | ||