blib/lib/GermaNet/Flat.pm

filter: Statementstmt 33.3% 1 / 3 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 1 / 1 Podpod n/a totaltotal 50.0% 2 / 4 CC 1 SCAR 2.6
CC
1
Cov
33%
BEGIN
1.3
CRAP 1.3
↑ Summary
1 ##-*- Mode: CPerl -*-
2  
3 package GermaNet::Flat;
4 0 ▶use XML::Parser;
Subroutine BEGIN: called CC=1 SCAR=2.6
5 use IO::File;
6 use Fcntl;
7 use File::Basename qw(basename dirname);
8 use Carp;
9 use strict;
10 use 5.010; ##-- v5.10.0: for // operator
11  
12 ##==============================================================================
13 ## Constants
14  
15 our $VERSION = 0.04;
16  
17 ##-- constants: verbosity levels
18 our $vl_silent = 0;
19 our $vl_error = 1;
20 our $vl_warn = 2;
21 our $vl_info = 3;
22 our $vl_progress = 4;
23 our $vl_debug = 5;
24  
25 ##==============================================================================
26  
27 ## Constructors etc
28 sub new {
29 my $that = shift;
30 return bless({
31 verbose => $vl_info,
32 rel=>{}, ##-- "${relation}:${arg}" => join(' ',@vals), ...
33 #dbf=>undef,
34  
35 @_,
36 },
37 ref($that)||$that);
38 }
39  
40 ## $gn = $gn->clear();
41 sub clear {
42 my $gn = shift;
43 %{$gn->{rel}} = qw();
44 delete @$gn{qw(dbf)};
45 return $gn;
46 }
47  
48 ##==============================================================================
49 ## Verbosity
50  
51 sub vmsg0 {
52 my $gn = shift;
53 my $level = shift;
54 print STDERR @_ if (ref($gn) && ($gn->{verbose}//0) >= $level);
55 }
56 sub vmsg {
57 $_[0]->vmsg0($_[1], (ref($_[0])||$_[0]).": ", @_[2..$#_], "\n");
58 }
59 sub vpdo {
60 my ($gn,$lvl,$msg,$sub) = @_;
61 $gn->vmsg0($lvl, (ref($gn)||$gn), ": $msg");
62 $sub->() if ($sub);
63 $gn->vmsg0($lvl, " done.\n");
64 }
65  
66 ##==============================================================================
67 ## Utils
68  
69 ## \@array_uniq = auniq(\@array)
70 ## \@array_uniq = $gn->auniq(\@array)
71 sub auniq {
72 shift if (@_ && UNIVERSAL::isa($_[0],__PACKAGE__));
73 my ($prev);
74 @{$_[0]} = grep {($prev//'') eq ($_//'') ? qw() : ($prev=$_)} sort @{$_[0]};
75 return $_[0];
76 }
77  
78 ## @uniq = luniq(@list)
79 ## @uniq = $gn->luniq(@list)
80 sub luniq {
81 shift if (@_ && UNIVERSAL::isa($_[0],__PACKAGE__));
82 my ($prev);
83 return grep {($prev//'') eq ($_//'') ? qw() : ($prev=$_)} sort @_;
84 }
85  
86 ## $gn = $gn->sanitize()
87 sub sanitize {
88 my $gn = shift;
89 foreach (values %{$gn->{rel}}) {
90 $_ = join(' ', luniq split(' ',$_));
91 }
92 return $gn;
93 }
94  
95 ##==============================================================================
96 ## API: Relations: Generic
97  
98 ## \@vals = $gn->relation($rel, $arg)
99 ## \@vals = $gn->relation($rel, \@args)
100 ## + not necessarily unique values
101 sub relation {
102 #my ($gn,$rel,$arg) = @_;
103 return [map {split(' ',($_[0]->{rel}{"$_[1]:$_"}//''))} (UNIVERSAL::isa($_[2],'ARRAY') ? @{$_[2]} : ($_[2]//''))];
104 }
105  
106  
107  
108 ##----------------------------------------------------------------------
109 ## API: Relations: Common
110  
111 ## \&CODE = relationWrapper($relation)
112 ## + returns unique values only
113 sub relationWrapper {
114 my $rel = shift;
115 return sub {
116 return auniq( $_[0]->relation($rel,@_[1..$#_]) );
117 };
118 }
119  
120 ## \@lexids = $gn->orth2lex($lemma)
121 ## \@lemmas = $gn->lex2orth($lemma)
122 ## \@synids = $gn->lex2syn($lexid)
123 ## \@lexids = $gn->syn2lex($synid)
124 ## \@subids = $gn->hypernyms($synid) = $gn->hyperonyms($synid)
125 ## \@supids = $gn->hyponyms($synid)
126 BEGIN {
127 *orth2lex = relationWrapper('orth2lex');
128 *lex2orth = relationWrapper('lex2orth');
129 *lex2syn = relationWrapper('lex2syn');
130 *syn2lex = relationWrapper('syn2lex');
131 *hyperonyms = *hypernyms = relationWrapper('has_hypernym');
132 *hyponyms = relationWrapper('has_hyponym');
133 }
134  
135 ## $dbversion = $gn->dbversion()
136 sub dbversion {
137 return $_[0]->relation('dbversion')->[0];
138 }
139  
140 ##----------------------------------------------------------------------
141 ## API: compat
142  
143 ## \@synsets_or_undef = $gn->get_synsets($lemma);
144 ## + uniqueness not guaranteed
145 sub get_synsets {
146 return $_[0]->relation('lex2syn',$_[0]->relation('orth2lex',$_[1]));
147 }
148  
149 ## \@terms_or_undef = $gn->synset_terms($synset);
150 ## + uniqueness not guaranteed
151 sub synset_terms {
152 return $_[0]->relation('lex2orth',$_[0]->relation('syn2lex',$_[1]));
153 }
154  
155 ##==============================================================================
156 ## Input: generic
157  
158 ## $gn = $CLASS_OR_OBJECT->load($filename_or_xmldirname)
159 sub load {
160 my ($gn,$file) = @_;
161 $gn = $gn->new() if (!ref($gn));
162  
163 return $gn->loadXmlDir($file) if (-d $file);
164 return $gn->loadBin($file) if ($file =~ /\.(?:bin|sto)$/i);
165 return $gn->loadDB($file) if ($file =~ /\.(?:b?)db$/i);
166 return $gn->loadCDB($file) if ($file =~ /\.cdb$/i);
167 return $gn->loadText($file);
168 }
169  
170  
171 ##==============================================================================
172 ## Input: XML
173  
174 ## $gn = CLASS_OR_OBJECT->loadXmlDir($directoryx)
175 ## + loaded data appended to existing relations, no implicit clear()
176 sub loadXmlDir {
177 my ($gn,$dir) = @_;
178 $dir =~ s/\/+$//;
179 $gn = $gn->new() if (!ref($gn));
180  
181 ##-- try to load db version
182 my $dbversion = "unknown";
183 if (-r "$dir/VERSION") {
184 open(my $fh,"<$dir/VERSION")
185 or confess(__PACKAGE__ . "::loadXmlDir(): could not open $dir/VERSION: $!");
186 $dbversion = <$fh>;
187 close $fh;
188 } elsif (basename($dir) =~ /[\.\-](\d[\d\.\-]*$)/) {
189 $dbversion = $1;
190 }
191 chomp($dbversion);
192 $gn->{rel}{"dbversion:"} = $dbversion;
193 $gn->vmsg($vl_progress, "loadXmlDir(): set dbversion = ", ($dbversion//'-undef-'));
194  
195 ##-- load guts
196 return $gn->loadXml(glob("$dir/*.xml"));
197 }
198  
199  
200 ## $gn = CLASS_OR_OBJECT->loadXml(@xml_filenames_or_handles)
201 ## + loaded data appended to existing relations, no implicit clear()
202 sub loadXml {
203 my $gn = shift;
204 $gn = $gn->new() if (!ref($gn));
205 $gn->{inmode} = 'xml';
206  
207 ##--------------------------------------------------
208 ## XML: callbacks
209 my ($_xp, $_elt, %attrs);
210 my ($dir,$synset,$lexid,$isorth,$oform);
211 my $rel = $gn->{rel};
212 my %xml_relations = (con_rel=>1, lex_rel=>1);
213  
214 ##--------------------------------------------
215 ## undef = cb_start($expat, $elt,%attrs)
216 my %rel_xlate = (hyperonymy=>'has_hypernym', 'hyponymy'=>'has_hyponym');
217 my ($relname);
218 my $cb_start = sub {
219 ($_xp,$_elt,%attrs) = @_;
220 if ($_elt eq 'synset') {
221 $synset = $attrs{id};
222 } elsif ($_elt eq 'lexUnit') {
223 $lexid = $attrs{id};
224 } elsif ($_elt eq 'orthForm') {
225 $isorth = 1;
226 $oform = '';
227 } elsif ($xml_relations{$_elt}) {
228 $relname = $rel_xlate{$attrs{name}} // $attrs{name};
229 $rel->{"$relname:$attrs{from}"} .= "$attrs{to} ";
230 $dir = $attrs{dir}//'one';
231 if ($dir eq 'revert' && $attrs{inv}) {
232 $relname = $rel_xlate{$attrs{inv}} // $attrs{inv};
233 $rel->{"$relname:$attrs{to}"} .= "$attrs{from} ";
234 } elsif ($dir eq 'both') {
235 $rel->{"$relname:$attrs{to}"} .= "$attrs{from} ";
236 }
237 }
238 };
239  
240 ##--------------------------------------------
241 ## undef = cb_end($expat,$elt)
242 my $cb_end = sub {
243 if (defined($synset) && $_[1] eq 'orthForm') {
244 $oform =~ s/\s/_/g;
245 $rel->{"syn2lex:$synset"} .= "$lexid ";
246 $rel->{"lex2syn:$lexid"} .= "$synset ";
247 $rel->{"orth2lex:$oform"} .= "$lexid ";
248 $rel->{"lex2orth:$lexid"} .= "$oform ";
249 }
250 elsif ($_[1] eq 'lexUnit') {
251 undef $lexid;
252 }
253 elsif ($_[1] eq 'synset') {
254 undef $synset;
255 }
256 };
257  
258 ##--------------------------------------------
259 ## undef = cb_char($expat,$str)
260 my $cb_char = sub {
261 $oform .= $_[1] if ($isorth);
262 };
263  
264 ##--------------------------------------------------
265 ## XML::Parser object
266 my $xp = XML::Parser->new(
267 ErrorContext => 1,
268 ProtocolEncoding => 'UTF-8',
269 #ParseParamEnt => '???',
270 Handlers => {
271 #Init => $cb_init,
272 Char => $cb_char,
273 Start => $cb_start,
274 End => $cb_end,
275 #Default => $cb_default,
276 #Final => $cb_final,
277 },
278 )
279 or confess(ref($gn)."::loadXml(): ERROR: couldn't create XML::Parser");
280  
281 ##-- actual load
282 foreach my $xmlfile (@_) {
283 $gn->vmsg($vl_progress, "loadXml(): parsing $xmlfile ...");
284 if ($xmlfile eq '-') {
285 $xp->parse(\*STDIN);
286 }
287 elsif (ref($xmlfile)) {
288 $xp->parse($xmlfile);
289 }
290 else {
291 $xp->parsefile($xmlfile);
292 }
293 }
294  
295 $gn->vmsg($vl_progress, "sanitizing ...");
296 return $gn->sanitize;
297 }
298  
299 ##==============================================================================
300 ## I/O: Text
301  
302 ##----------------------------------------------------------------------
303 ## $bool = $gn->saveText($filename_or_fh)
304 sub saveText {
305 my ($gn,$file) = @_;
306 my $fh = ref($file) ? $file : IO::File->new(">$file");
307 confess(__PACKAGE__."::saveText(): open failed for '$file'") if (!$fh);
308 binmode($fh,':utf8');
309  
310 my ($k,$v);
311 while (($k,$v)=each %{$gn->{rel}}) {
312 $fh->print("$k\t$v\n");
313 }
314  
315 $fh->close() if (!ref($file));
316 return $gn;
317 }
318  
319 ##----------------------------------------------------------------------
320 ## $bool = $CLASS_OR_OBJECT->loadText($filename_or_fh)
321 ## + loaded data clobbers existing relations, but no implicit clear()
322 sub loadText {
323 my ($gn,$file) = @_;
324 $gn = $gn->new() if (!ref($gn));
325 $gn->{inmode} = 'text';
326 my $fh = ref($file) ? $file : IO::File->new("<$file");
327 confess(__PACKAGE__."::loadText(): open failed for '$file'") if (!$fh);
328 binmode($fh,':utf8');
329  
330 my ($k,$v);
331 while (<$fh>) {
332 chomp;
333 next if (/^\%\%/ || /^\s*$/);
334 ($k,$v) = split(/\t/,$_,2);
335 $gn->{rel}{$k} = $v;
336 }
337  
338 $fh->close() if (!ref($file));
339 return $gn;
340 }
341  
342 ##==============================================================================
343 ## I/O: DB
344  
345 ##----------------------------------------------------------------------
346 ## $gn = $gn->saveDB($dbfilename)
347 sub saveDB {
348 require Lingua::TT::DBFile;
349 my ($gn,$dbfile) = @_;
350  
351 ##-- test
352 if ($dbfile eq '-') {
353 $gn->vmsg($vl_warn, "saveDB(): WARNING: dbfile '-' maps to 'GermaNet-Flat.db'");
354 $dbfile = 'GermaNet-Flat.db';
355 }
356  
357 ##-- get db
358 my %dbf = ();
359 my $dbf = Lingua::TT::DBFile->new(type=>'BTREE',
360 flags=>O_RDWR|O_CREAT|O_TRUNC,
361 encoding=>'utf8',
362 dbopts=>{cachesize=>'32M'},
363 )
364 or confess(__PACKAGE__."::saveDB(): could not create Lingua::TT::DBFile: $!");
365  
366 $dbf->open($dbfile)
367 or confess(__PACKAGE__."::saveDB(): could not open DB file '$dbfile': $!");
368  
369 ##-- copy data
370 my $rel = $gn->{rel};
371 my $tied = $dbf->{tied};
372 foreach (sort keys %$rel) {
373 $tied->put($_,$rel->{$_});
374 }
375  
376 return $gn;
377 }
378  
379 ##----------------------------------------------------------------------
380 ## $bool = $CLASS_OR_OBJECT->loadDB($dbfile)
381 ## + implicitly replaces $gn->{rel} with tied data
382 sub loadDB {
383 require Lingua::TT::DBFile;
384 my ($gn,$dbfile) = @_;
385 $gn = $gn->new() if (!ref($gn));
386 $gn->{inmode} = 'db';
387  
388 ##-- get db
389 my %dbf = ();
390 my $dbf = Lingua::TT::DBFile->new(type=>'BTREE',
391 flags=>O_RDONLY,
392 encoding=>'utf8',
393 dbopts=>{cachesize=>'32M'},
394 )
395 or confess(__PACKAGE__."::loadDB(): could not create Lingua::TT::DBFile: $!");
396  
397 $dbf->open($dbfile)
398 or confess(__PACKAGE__."::loadDB(): could not open DB file '$dbfile': $!");
399  
400 ##-- attach data
401 $gn->{rel} = $dbf->{data};
402 $gn->{dbf} = $dbf;
403  
404 return $gn;
405 }
406  
407  
408 ##==============================================================================
409 ## I/O: CDB
410  
411 ##----------------------------------------------------------------------
412 ## $gn = $gn->saveCDB($dbfilename)
413 ## + utf-8 broken!
414 sub saveCDB {
415 require Lingua::TT::CDBFile;
416 my ($gn,$dbfile) = @_;
417  
418 ##-- test
419 if ($dbfile eq '-') {
420 $gn->vmsg($vl_warn, "saveCDB(): WARNING: cdbfile '-' maps to 'GermaNet-Flat.cdb'");
421 $dbfile = 'GermaNet-Flat.cdb';
422 }
423  
424 ##-- get db
425 my %dbf = ();
426 my $dbf = Lingua::TT::CDBFile->new(mode=>'w')
427 or confess(__PACKAGE__."::saveCDB(): could not create Lingua::TT::CDBFile: $!");
428  
429 $dbf->open($dbfile)
430 or confess(__PACKAGE__."::saveCDB(): could not open CDB file '$dbfile': $!");
431  
432 ##-- copy data
433 my $writer = $dbf->{writer};
434 my ($k,$v);
435 while (($k,$v)=each %{$gn->{rel}}) {
436 $writer->insert($k,$v);
437 }
438  
439 ##-- cleanup
440 undef $writer;
441 $dbf->close();
442  
443 return $gn;
444 }
445  
446 ##----------------------------------------------------------------------
447 ## $bool = $CLASS_OR_OBJECT->loadCDB($dbfile)
448 ## + implicitly replaces $gn->{rel} with tied data
449 ## + utf-8 broken!
450 sub loadCDB {
451 require Lingua::TT::CDBFile;
452 my ($gn,$dbfile) = @_;
453 $gn = $gn->new() if (!ref($gn));
454 $gn->{inmode} = 'cdb';
455  
456 ##-- get db
457 my %dbf = ();
458 my $dbf = Lingua::TT::CDBFile->new(mode=>'r')
459 or confess(__PACKAGE__."::loadCDB(): could not create Lingua::TT::CDBFile: $!");
460  
461 $dbf->open($dbfile)
462 or confess(__PACKAGE__."::loadCDB(): could not open CDB file '$dbfile': $!");
463  
464 ##-- attach data
465 $gn->{rel} = $dbf->{data};
466 $gn->{dbf} = $dbf;
467  
468 return $gn;
469 }
470  
471 ##==============================================================================
472 ## I/O: Storable
473  
474 ##----------------------------------------------------------------------
475 ## $bool = $gn->saveBin($filename_or_fh)
476 sub saveBin {
477 require Storable;
478 my ($gn,$file) = @_;
479 my $fh = ref($file) ? $file : IO::File->new(">$file");
480 confess(__PACKAGE__."::saveBin(): open failed for '$file'") if (!$fh);
481 binmode($fh,':raw');
482 my $obj = $gn;
483 if ($gn->{dbf}) {
484 $obj = $gn->new(%$gn, rel=>{ %{$gn->{rel}} });
485 delete @$obj{qw(dbf inmode)};
486 }
487 my $rc = Storable::nstore_fd($obj, $fh);
488 $fh->close() if (!ref($file));
489 return $rc;
490 }
491  
492 ##----------------------------------------------------------------------
493 ## $bool = $gn->loadBin($filename_or_fh)
494 sub loadBin {
495 require Storable;
496 my ($gn,$file) = @_;
497 my $fh = ref($file) ? $file : IO::File->new("<$file");
498 confess(__PACKAGE__."::loadBin(): open failed for '$file'") if (!$fh);
499 binmode($fh,':raw');
500 my $loaded = Storable::fd_retrieve($fh);
501 $fh->close() if (!ref($file));
502 return $loaded if (!ref($gn));
503 %$gn = %$loaded;
504 $gn->{inmode} = 'bin';
505 return $gn;
506 }
↑ Summary