CAB/Format/Raw/Waste.pm

filter: Statementstmt 92.5% 25 / 27 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 9 / 9 Podpod n/a totaltotal 94.4% 34 / 36 CC 1 SCAR 0.3
CC
1
Cov
93%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« CAB/Common.pm ↑ Summary CAB/Format/Builtin.pm »
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Format::Raw::Waste.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: Datum parser: raw untokenized text (using moot/waste)
6  
7 package DTA::CAB::Format::Raw::Waste;
8 1 ▶use DTA::CAB::Format;
Subroutine BEGIN: called CC=1 SCAR=0.0
9 1 ▶use DTA::CAB::Format::TT;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 1 ▶use DTA::CAB::Format::Raw::Base;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 1 ▶use DTA::CAB::Datum ':all';
Subroutine BEGIN: called CC=1 SCAR=0.0
12 1 ▶use DTA::CAB::Utils qw(file_mtime timestamp_str);
Subroutine BEGIN: called CC=1 SCAR=0.0
13 1 ▶use IO::File;
Subroutine BEGIN: called CC=1 SCAR=0.0
14 1 ▶use Cwd qw(abs_path);
Subroutine BEGIN: called CC=1 SCAR=0.0
15 1 ▶use Encode qw(encode decode);
Subroutine BEGIN: called CC=1 SCAR=0.0
16 0 ▶use Moot::Waste;
Subroutine BEGIN: called CC=1 SCAR=2.6
17 use Carp;
18 use strict;
19  
20 ##==============================================================================
21 ## Globals
22 ##==============================================================================
23  
24 our @ISA = qw(DTA::CAB::Format::Raw::Base);
25  
26 BEGIN {
27 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>'raw-waste', filenameRegex=>qr/\.(?i:raw-waste|txt-waste)$/);
28 }
29  
30 our @DEFAULT_WASTERC_PATHS =
31 (
32 ($ENV{TOKWRAP_RCDIR} ? "$ENV{TOKWRAP_RCDIR}/waste/waste.rc" : qw()),
33 (defined($DTA::TokWrap::Version::VERSION) ? "$DTA::TokWrap::Version::RCDIR/waste/waste.rc" : qw()),
34 (defined $ENV{HOME} ? "$ENV{HOME}/.wasterc" : qw()),
35 "/etc/wasterc",
36 "/etc/default/wasterc"
37 );
38  
39 ## $logLoad : default logLoad option
40 our $logLoad = 'trace';
41  
42 ## $logCache : default logCache option
43 our $logCache = undef;
44  
45 ## $logRun : default logRun option
46 our $logRun = undef;
47  
48 ##==============================================================================
49 ## Constructors etc.
50 ##==============================================================================
51  
52 ## $fmt = CLASS_OR_OBJ->new(%args)
53 ## + object structure: assumed HASH
54 ## {
55 ## ##-- Input
56 ## doc => $doc, ##-- buffered input document
57 ## wasterc => $rcFile, ##-- waste .rc file; default: "$HOME/.wasterc" || "/etc/wasterc" || "/etc/default/waste"
58 ## tloc => $attr, ##-- pased to Format::TT::parseTokenizerString(): if non-empty, sets $w->{$attr}="$off $len"; default=0
59 ##
60 ## ##-- Runtime
61 ## wmodel => \%wmodel ##-- waste model; %wmodel=(
62 ## # config => \%config, #-- parsed rcfile (see loadModelConfig())
63 ## # loaded => $time, #-- unix timestamp of last model load
64 ## # wscanner => $scanner, #-- waste scanner
65 ## # wlexer => $lexer, #-- waste lexer
66 ## # wtagger => $tagger, #-- waste tagger
67 ## # wdecoder => $decoder, #-- waste decoder
68 ## # wannotator => $wannot, #-- waste annotator
69 ## # wwriter => $wwriter, #-- native-format writer (hack)
70 ## # )
71 ##
72 ## ##-- logging (in order of increasing verbosity)
73 ## logLoad => $level, # log-level for model loading (default=$logLoad)
74 ## logCache => $level, # cache operation log-level (default=$logCache)
75 ## logRun => $level, # runtime operation log-level (default=$logRun)
76 ##
77 ## ##-- Common
78 ## #utf8 => $bool, ##-- utf8 mode always on
79 sub new {
80 my $that = shift;
81 my $fmt = bless({
82 ##-- common
83 utf8 => 1,
84  
85 ##-- input
86 doc => undef,
87 wasterc => undef,
88 tloc => undef,
89  
90 ##-- runtime
91 wmodel => undef,
92  
93 ##-- logging
94 logLoad => $logLoad,
95 logCache => $logCache,
96 logRun => $logRun,
97  
98 ##-- user args
99 @_
100 }, ref($that)||$that);
101 return $fmt;
102 }
103  
104 ##==============================================================================
105 ## Methods: Persistence
106 ##==============================================================================
107  
108 ## @keys = $class_or_obj->noSaveKeys()
109 ## + returns list of keys not to be saved
110 ## + default just returns empty list
111 sub noSaveKeys {
112 return (shift->SUPER::noSaveKeys(), qw(doc wmodel wscanner wlexer wtagger wdecoder wannotator wwriter));
113 }
114  
115 ##==============================================================================
116 ## Methods: Local: log levels
117  
118 ## $logLoad = $CLASS_OR_OBJECT->logLevelLoad()
119 sub logLevelLoad { return ref($_[0]) ? $_[0]{logLoad} : $logLoad; }
120  
121 ## $logCache = $CLASS_OR_OBJECT->logLevelCache()
122 sub logLevelCache { return ref($_[0]) ? $_[0]{logCache} : $logCache; }
123  
124 ## $logRun = $CLASS_OR_OBJECT->logLevelRun()
125 sub logLevelRun { return ref($_[0]) ? $_[0]{logRun} : $logRun; }
126  
127  
128 ##==============================================================================
129 ## Methods: Local: model caching
130  
131 ## %MODELS : cached models ("$wasterc_abspath:$PID" => \%wmodel)
132 our %MODELS = qw();
133  
134 END {
135 ##-- END block clears cached waste models for this PID
136 my @cached = grep {/:${$}$/} keys %MODELS;
137 __PACKAGE__->vlog(__PACKAGE__->logLevelCache, "clearing model cache for PID $$: ", join(' ', @cached));
138 delete @MODELS{@cached};
139 }
140  
141 ## \%wmodel_or_undef = $fmt->ensureModel()
142 ## \%wmodel_or_undef = $fmt->ensureModel($wasterc)
143 ## \%wmodel_or_undef = CLASS->ensureModel($wasterc)
144 ## + loads cached model if available; otherwise populates cache
145 sub ensureModel {
146 my ($fmt,$rcfile) = @_;
147 $rcfile = $fmt->{rcfile} if (ref($fmt) && !$rcfile);
148  
149 ##-- get rc file
150 if (!$rcfile) {
151 $rcfile = (grep {-f $_} @DEFAULT_WASTERC_PATHS)[0];
152 $fmt->logconfess("getModel(): no wasterc specified!") if (!$rcfile);
153 }
154  
155 ##-- cache lookup
156 my $mkey = abs_path($rcfile).":$$";
157 my $wmodel = $MODELS{$mkey};
158 ##-- debug
159 if (defined($wmodel)) {
160 $fmt->vlog($fmt->logLevelCache, "found cached waste model: $mkey");
161  
162 my $modeltime = int(file_mtime($rcfile));
163 if ($wmodel->{loaded} >= $modeltime) {
164 $fmt->vlog($fmt->logLevelCache, "using cached waste model: $mkey");
165 return $wmodel;
166 } else {
167 $fmt->vlog($fmt->logLevelCache,
168 "cached waste model is stale (".timestamp_str($wmodel->{loaded}).' < '.timestamp_str($modeltime)."): $mkey");
169 }
170 } else {
171 $fmt->vlog($fmt->logLevelCache, "no cached waste model found: $mkey");
172 }
173  
174 ##-- not cached or stale: create a new model & update the cache
175 $fmt->vlog($fmt->logLevelCache, "updating waste model cache: $mkey");
176 my $config = $fmt->loadModelConfig($rcfile);
177 $wmodel = $MODELS{$mkey} =
178 {
179 config => $config,
180 loaded => time(),
181 wscanner => Moot::Waste::Scanner->new( $Moot::ioFormat{text}|$Moot::ioFormat{location} ),
182 wlexer => Moot::Waste::Lexer->new( $Moot::ioFormat{wd}|$Moot::ioFormat{location} ),
183 wtagger => Moot::HMM->new(),
184 wdecoder => Moot::Waste::Decoder->new( $Moot::ioFormat{m}|$Moot::ioFormat{location} ),
185 wannotator => Moot::Waste::Annotator->new( $Moot::ioFormat{mr}|$Moot::ioFormat{location} ),
186 wwriter => Moot::TokenWriter::Native->new( $Moot::ioFormat{mr}|$Moot::ioFormat{location} ),
187 };
188  
189 ##-- load model from configuration options
190 $wmodel->{wlexer}->abbrevs->load($config->{abbrevs}) if ($config->{abbrevs});
191 $wmodel->{wlexer}->conjunctions->load($config->{conjunctions}) if ($config->{conjunctions});
192 $wmodel->{wlexer}->stopwords->load($config->{stopwords}) if ($config->{stopwords});
193 $wmodel->{wlexer}->dehyphenate($config->{dehyphenate} ? 1 : 0) if (exists $config->{dehyphenate});
194 if ($config->{hmm}) {
195 $wmodel->{wtagger}->load($config->{hmm}) or $fmt->logconfess("failed to load waste model '$config->{hmm}'");
196 } else {
197 $fmt->logconfess("no 'hmm' key specified in waste rc-file '$rcfile'");
198 }
199  
200 return $wmodel;
201 }
202  
203 ## \%config = CLASS_OR_OBJECT->loadModelConfig($wasterc)
204 ## + loads rc-file with keys qw(abbrevs conjunctions stopwords dehyphenate hmm)
205 sub loadModelConfig {
206 my ($fmt,$rcfile) = @_;
207 $fmt->vlog($fmt->logLevelCache, "loading waste model configuration $rcfile");
208  
209 open(my $rc,"<$rcfile")
210 or $fmt->logconfess("open failed for waste-rc $rcfile: $!");
211 my $config = {rcfile=>$rcfile};
212 while (defined($_=<$rc>)) {
213 next if (/^\#/ || /^\s*$/);
214 chomp;
215 my ($opt,$val) = split(/\s/,$_,2);
216 if ($opt =~ /^abbr/) { $config->{abbrevs}=$val }
217 elsif ($opt =~ /^conj/) { $config->{conjunctions}=$val; }
218 elsif ($opt =~ /^stop/) { $config->{stopwords}=$val; }
219 elsif ($opt =~ /^dehyph/) { $config->{dehyphenate}=$val; }
220 elsif ($opt =~ /^no-dehyph/) { $config->{dehyphenate}=$val; }
221 elsif ($opt =~ /^(?:hmm|model)/) { $config->{hmm}=$val; }
222 else {
223 ; ##-- ignore other options
224 }
225 }
226 close($rc);
227  
228 $fmt->vlog($fmt->logLevelLoad, "loaded waste model configuration $rcfile");
229 return $config;
230 }
231  
232 ##==============================================================================
233 ## Methods: Model I/O
234  
235 ## $fmt_or_undef = $fmt->ensureLoaded()
236 sub ensureLoaded {
237 my $fmt = shift;
238 return $fmt if ($fmt->{wmodel} && $fmt->{wmodel}{wtagger});
239  
240 ##-- get rc file
241 if (!$fmt->{wasterc}) {
242 $fmt->{wasterc} = (grep {-f $_} @DEFAULT_WASTERC_PATHS)[0];
243 $fmt->logconfess("cannot tokenize without a model -- specify wasterc!") if (!$fmt->{wasterc});
244 }
245 #$fmt->vlog($fmt->logLevelRun, "using waste model configuration $fmt->{wasterc}");
246  
247 return $fmt->loadModel();
248 }
249  
250 ## $fmt_or_undef = $fmt->loadModel()
251 ## $fmt_or_undef = $fmt->loadModel($rcfile)
252 ## + backwards-compatible
253 sub loadModel {
254 my ($fmt,$rcfile) = @_;
255 $rcfile //= $fmt->{wasterc};
256 $fmt->{wasterc} = $rcfile;
257 $fmt->vlog($fmt->logLevelRun, "using waste model $rcfile");
258 $fmt->{wmodel} = $fmt->ensureModel($rcfile)
259 or $fmt->logconfess("failed to load waste model '$rcfile': $!");
260  
261 return $fmt;
262 }
263  
264 ##==============================================================================
265 ## Methods: Input
266 ##==============================================================================
267  
268 ##--------------------------------------------------------------
269 ## Methods: Input: Input selection
270  
271 ## $fmt = $fmt->close()
272 sub close {
273 delete($_[0]{doc});
274 return $_[0]->SUPER::close(@_[1..$#_]);
275 }
276  
277 ## $fmt = $fmt->fromString( $string)
278 ## $fmt = $fmt->fromString(\$string)
279 ## + select input from string $string
280 ## + default calls fromFh()
281 sub fromString {
282 my $fmt = shift;
283 my $ref = ref($_[0]) ? $_[0] : \$_[0];
284 return $fmt->SUPER::fromString(@_) if (!utf8::is_utf8($$ref));
285  
286 my $raw = $$ref;
287 utf8::encode($raw);
288 return $fmt->SUPER::fromString(\$raw);
289 }
290  
291 ## $fmt = $fmt->fromFh($fh)
292 sub fromFh {
293 my ($fmt,$fh) = @_;
294 $fmt->ensureLoaded();
295  
296 my ($ttstr);
297 my $wmodel = $fmt->{wmodel};
298 $wmodel->{wlexer}->close();
299 $wmodel->{wscanner}->close();
300 $wmodel->{wscanner}->from_fh($fh);
301 $wmodel->{wlexer}->scanner($wmodel->{wscanner});
302 $wmodel->{wwriter}->to_string($ttstr);
303 $wmodel->{wdecoder}->sink($wmodel->{wannotator});
304 $wmodel->{wannotator}->sink($wmodel->{wwriter});
305 $wmodel->{wtagger}->tag_stream($wmodel->{wlexer},$wmodel->{wdecoder});
306 $wmodel->{wdecoder}->close();
307 $wmodel->{wannotator}->close();
308 $wmodel->{wwriter}->close();
309  
310 ##-- construct & buffer document
311 utf8::decode($ttstr) if (!utf8::is_utf8($ttstr));
312 $fmt->{doc} = DTA::CAB::Format::TT->parseTokenizerString(\$ttstr, {tloc=>$fmt->{tloc}});
313 return $fmt;
314 }
315  
316 ##--------------------------------------------------------------
317 ## Methods: Input: Generic API
318  
319 ## $doc = $fmt->parseDocument()
320 sub parseDocument {
321 return $_[0]{doc};
322 }
323  
324  
325 ##==============================================================================
326 ## Methods: Output
327 ## + output not supported
328 ##==============================================================================
329  
330 ##--------------------------------------------------------------
331 ## Methods: Output: Generic
332  
333 ## $type = $fmt->mimeType()
334 ## + default returns text/plain
335 sub mimeType { return 'text/plain'; }
336  
337 ## $ext = $fmt->defaultExtension()
338 ## + returns default filename extension for this format
339 sub defaultExtension { return '.raw'; }
340  
341  
342 1; ##-- be happy
343  
344 __END__
« CAB/Common.pm ↑ Summary CAB/Format/Builtin.pm »