| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## File: Lingua::TT::DBFile.pm | ||
| 3 | ## Author: Bryan Jurish <TT/IO.pm> | ||
| 4 | ## Descript: TT I/O: Berkely DB: tied Files | ||
| 5 | |||
| 6 | |||
| 7 | package Lingua::TT::DBFile; | ||
| 8 | 1 | ▶ | use 5.010; ##-- for // operator |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 9 | 1 | ▶ | use Lingua::TT::Persistent; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 10 | 0 | ▶ | use DB_File; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 11 | use Fcntl; | ||
| 12 | use Carp; | ||
| 13 | use IO::File; | ||
| 14 | use File::Copy qw(); | ||
| 15 | use Encode qw(encode decode); | ||
| 16 | use strict; | ||
| 17 | |||
| 18 | ##============================================================================== | ||
| 19 | ## Globals & Constants | ||
| 20 | |||
| 21 | our @ISA = qw(Lingua::TT::Persistent); | ||
| 22 | |||
| 23 | ## $DEFAULT_TYPE | ||
| 24 | ## + default file type if otherwise unspecified | ||
| 25 | our $DEFAULT_TYPE = 'BTREE'; | ||
| 26 | |||
| 27 | ##============================================================================== | ||
| 28 | ## Constructors etc. | ||
| 29 | |||
| 30 | ## $dbf = CLASS_OR_OBJECT->new(%opts) | ||
| 31 | ## + %opts, %$doc: | ||
| 32 | ## ##-- user options | ||
| 33 | ## file => $filename, ##-- default: undef (none) | ||
| 34 | ## mode => $mode, ##-- default: 0666 & ~umask | ||
| 35 | ## flags => $flags, ##-- default: O_RDWR|O_CREAT | ||
| 36 | ## type => $type, ##-- one of 'HASH', 'BTREE', 'RECNO', 'GUESS' (default: 'GUESS' for read, $DEFAULT_TYPE otherwise) | ||
| 37 | ## dbinfo => \%dbinfo, ##-- default: "DB_File::${type}INFO"->new(); | ||
| 38 | ## dbopts => \%opts, ##-- db options (e.g. cachesize,bval,...) -- defaults to none (uses DB_File defaults) | ||
| 39 | ## encoding => $enc, ##-- if defined, $enc will be used to store db data (uses Encode and DB filters); default=undef (raw bytes) | ||
| 40 | ## pack_key => $packas, ##-- if defined, $packas will be used to (un)pack db keys | ||
| 41 | ## pack_val => $packas, ##-- if defined, $packas will be used to (un)pack db values | ||
| 42 | ## pack_sep => $str, ##-- separator string for multi-value pack/unpack (default="\t") | ||
| 43 | ## ## | ||
| 44 | ## ##-- low-level data | ||
| 45 | ## data => $thingy, ##-- tied data (hash or array) | ||
| 46 | ## tied => $ref, ##-- reference returned by tie() | ||
| 47 | sub new { | ||
| 48 | my $that = shift; | ||
| 49 | my $db = bless({ | ||
| 50 | file => undef, | ||
| 51 | encoding => undef, | ||
| 52 | pack_key => undef, | ||
| 53 | pack_val => undef, | ||
| 54 | pack_sep => "\t", | ||
| 55 | mode => (0644 & ~umask), | ||
| 56 | flags => (O_RDWR|O_CREAT), | ||
| 57 | type => undef, ##-- no default type (guess) | ||
| 58 | dbinfo => undef, | ||
| 59 | dbopts => {}, | ||
| 60 | data => undef, | ||
| 61 | tied => undef, | ||
| 62 | @_ | ||
| 63 | }, ref($that)||$that); | ||
| 64 | #$db->{dbinfo} = ("DB_File::".uc($db->{type})."INFO")->new() if (!defined($db->{dbinfo})); | ||
| 65 | return $db->open($db->{file}) if (defined($db->{file})); | ||
| 66 | return $db; | ||
| 67 | } | ||
| 68 | |||
| 69 | ## undef = $dbf->clear() | ||
| 70 | ## + clears data (if any) | ||
| 71 | sub clear { | ||
| 72 | my $dbf = shift; | ||
| 73 | return if (!$dbf->opened); | ||
| 74 | if (uc($dbf->{type}) eq 'RECNO') { | ||
| 75 | $dbf->{tied}->splice(0,scalar(@{$dbf->{data}})); | ||
| 76 | } else { | ||
| 77 | %{$dbf->{data}} = qw(); | ||
| 78 | } | ||
| 79 | return $dbf; | ||
| 80 | } | ||
| 81 | |||
| 82 | ##============================================================================== | ||
| 83 | ## Class Methods | ||
| 84 | |||
| 85 | ## $typeString = $that->guessFileType($filename_or_handle) | ||
| 86 | ## + attempts to guess type of $filename; returns undef on failure | ||
| 87 | ## + this will die() if $filename doesn't already exist | ||
| 88 | ## + magic codes grabbed from /usr/share/file/magic | ||
| 89 | sub guessFileType { | ||
| 90 | my ($that,$file) = @_; | ||
| 91 | my $fh = ref($file) ? $file : IO::File->new("<$file"); | ||
| 92 | my $pos = ref($file) ? tell($fh) : undef; | ||
| 93 | die(__PACKAGE__ . "::guessFileType(): open failed for '$file': $!") if (!$fh); | ||
| 94 | CORE::binmode($fh); | ||
| 95 | my ($buf,$typ,$magic,$fmt); | ||
| 96 | seek($fh,12,SEEK_SET) or return undef; | ||
| 97 | read($fh,$buf,4)==4 or return undef; | ||
| 98 | foreach $fmt (qw(L N V)) { | ||
| 99 | $magic = unpack($fmt,$buf); | ||
| 100 | if ($magic == 0x00053162) { $typ='BTREE'; last; } | ||
| 101 | elsif ($magic == 0x00061561) { $typ='HASH'; last; } | ||
| 102 | } | ||
| 103 | if (ref($file)) { | ||
| 104 | seek($fh,$pos,SEEK_SET); ##-- return handle to original position | ||
| 105 | } else { | ||
| 106 | $fh->close(); | ||
| 107 | } | ||
| 108 | return $typ; | ||
| 109 | } | ||
| 110 | |||
| 111 | |||
| 112 | ##============================================================================== | ||
| 113 | ## Methods: low-level utilities | ||
| 114 | |||
| 115 | ##============================================================================== | ||
| 116 | ## Methods: I/O | ||
| 117 | |||
| 118 | ## $bool = $dbf->opened() | ||
| 119 | sub opened { | ||
| 120 | return defined($_[0]{tied}); | ||
| 121 | } | ||
| 122 | |||
| 123 | ## $dbf = $dbf->close() | ||
| 124 | sub close { | ||
| 125 | my $dbf = shift; | ||
| 126 | return $dbf if (!$dbf->opened); | ||
| 127 | $dbf->{tied} = undef; | ||
| 128 | if (uc($dbf->{type}) eq 'RECNO') { | ||
| 129 | untie(@{$dbf->{data}}); | ||
| 130 | } else { | ||
| 131 | untie(%{$dbf->{data}}); | ||
| 132 | } | ||
| 133 | return $dbf; | ||
| 134 | } | ||
| 135 | |||
| 136 | ## $sizeInt = parseSize($sizeString) | ||
| 137 | sub parseSize { | ||
| 138 | my ($dbf,$str) = @_; | ||
| 139 | if (defined($str) && $str =~ /^\s*([\d\.\+\-eE]*)\s*([BKMGT]?)\s*$/i) { | ||
| 140 | my ($size,$suff) = ($1,$2); | ||
| 141 | $suff = 'B' if (!defined($suff)); | ||
| 142 | $suff = uc($suff); | ||
| 143 | $size *= 1024 if ($suff eq 'K'); | ||
| 144 | $size *= 1024**2 if ($suff eq 'M'); | ||
| 145 | $size *= 1024**3 if ($suff eq 'G'); | ||
| 146 | $size *= 1024**4 if ($suff eq 'T'); | ||
| 147 | return $size; | ||
| 148 | } | ||
| 149 | return $str; | ||
| 150 | } | ||
| 151 | |||
| 152 | ## \%opts = $dbf->parseOptions(\%opts=$dbf->{dbopts}) | ||
| 153 | sub parseOptions { | ||
| 154 | my ($dbf,$dbopts) = @_; | ||
| 155 | $dbopts = $dbf->{dbopts} if (!$dbopts); | ||
| 156 | $dbopts = $dbf->{dbopts} = {} if (!$dbopts); | ||
| 157 | |||
| 158 | ##-- parse: size arguments | ||
| 159 | foreach (qw(cachesize psize)) { | ||
| 160 | $dbopts->{$_} = $dbf->parseSize($dbopts->{$_}) if (defined($dbopts->{$_})); | ||
| 161 | } | ||
| 162 | |||
| 163 | ##-- delete undef arguments | ||
| 164 | delete @$dbopts{grep {!defined($dbopts->{$_})} keys %$dbopts}; | ||
| 165 | |||
| 166 | return $dbopts; | ||
| 167 | } | ||
| 168 | |||
| 169 | |||
| 170 | ## $dbf = $dbf->open($file,%opts) | ||
| 171 | ## + %opts are as for new() | ||
| 172 | ## + $file defaults to $dbf->{file} | ||
| 173 | sub open { | ||
| 174 | my ($dbf,$file,%opts) = @_; | ||
| 175 | $dbf->close() if ($dbf->opened); | ||
| 176 | @$dbf{keys %opts} = values(%opts); | ||
| 177 | $file = $dbf->{file} if (!defined($file)); | ||
| 178 | $dbf->{file} = $file; | ||
| 179 | $dbf->parseOptions(); | ||
| 180 | |||
| 181 | ##-- truncate file here if user specified O_TRUNC, since DB_File doesn't | ||
| 182 | if (($dbf->{flags} & O_TRUNC) && defined($dbf->{file}) && -e $dbf->{file}) { | ||
| 183 | $dbf->truncate() | ||
| 184 | or confess(ref($dbf)."::open(O_TRUNC): could not unlink file '$dbf->{file}': $!"); | ||
| 185 | } | ||
| 186 | |||
| 187 | ##-- guess file type | ||
| 188 | if (!defined($dbf->{type}) || uc($dbf->{type}) eq 'GUESS') { | ||
| 189 | if (-e $dbf->{file}) { | ||
| 190 | $dbf->{type} = $dbf->guessFileType($dbf->{file}) || 'RECNO'; | ||
| 191 | } | ||
| 192 | $dbf->{type} = $DEFAULT_TYPE if (!defined($dbf->{type}) || uc($dbf->{type}) eq 'GUESS'); ##-- last-ditch effort | ||
| 193 | } | ||
| 194 | |||
| 195 | ##-- parse bval (octal, hex, and unicode escapes) | ||
| 196 | if ($dbf->{dbopts}{bval} && length($dbf->{dbopts}{bval}) != 1) { | ||
| 197 | $dbf->{dbopts}{bval} =~ s{\\([0-7]{1,3})}{chr(oct($1))}ge; | ||
| 198 | $dbf->{dbopts}{bval} =~ s{\\x([0-9a-f]{1,2})}{chr(hex($1))}gie; | ||
| 199 | $dbf->{dbopts}{bval} =~ s{\\u([0-9a-f]{1,4})}{chr(hex($1))}gie; | ||
| 200 | $dbf->{dbopts}{bval} =~ s{\\n}{\n}g; | ||
| 201 | $dbf->{dbopts}{bval} =~ s{\\r}{\r}g; | ||
| 202 | $dbf->{dbopts}{bval} =~ s{\\t}{\t}g; | ||
| 203 | $dbf->{dbopts}{bval} =~ s{\\v}{\x0b}g; | ||
| 204 | } | ||
| 205 | |||
| 206 | ##-- setup info | ||
| 207 | $dbf->{dbinfo} = ("DB_File::".uc($dbf->{type})."INFO")->new(); | ||
| 208 | @{$dbf->{dbinfo}}{keys %{$dbf->{dbopts}}} = values %{$dbf->{dbopts}}; | ||
| 209 | |||
| 210 | if (uc($dbf->{type}) eq 'RECNO') { | ||
| 211 | ##-- tie: recno (array) | ||
| 212 | $dbf->{dbinfo}{flags} |= R_FIXEDLEN if (defined($dbf->{dbinfo}{reclen})); | ||
| 213 | $dbf->{data} = []; | ||
| 214 | $dbf->{tied} = tie(@{$dbf->{data}}, 'DB_File', $dbf->{file}, $dbf->{flags}, $dbf->{mode}, $dbf->{dbinfo}) | ||
| 215 | or confess(ref($dbf).":open(): tie() failed for ARRAY file '$dbf->{file}': $!"); | ||
| 216 | } else { | ||
| 217 | ##-- tie: btree or hash (hash) | ||
| 218 | $dbf->{data} = {}; | ||
| 219 | $dbf->{tied} = tie(%{$dbf->{data}}, 'DB_File', $dbf->{file}, $dbf->{flags}, $dbf->{mode}, $dbf->{dbinfo}) | ||
| 220 | or confess(ref($dbf).":open(): tie() failed for $dbf->{type} file '$dbf->{file}': $!"); | ||
| 221 | } | ||
| 222 | |||
| 223 | ##-- maybe install encoding filters | ||
| 224 | if (defined($dbf->{encoding}) && $dbf->{encoding} ne 'raw') { | ||
| 225 | my $ffetch = $dbf->encFilterFetch(); | ||
| 226 | my $fstore = $dbf->encFilterStore(); | ||
| 227 | if (uc($dbf->{type}) ne 'RECNO') { | ||
| 228 | $dbf->{tied}->filter_fetch_key($ffetch); | ||
| 229 | $dbf->{tied}->filter_store_key($fstore); | ||
| 230 | } | ||
| 231 | $dbf->{tied}->filter_fetch_value($ffetch); | ||
| 232 | $dbf->{tied}->filter_store_value($fstore); | ||
| 233 | } | ||
| 234 | else { | ||
| 235 | ##-- maybe install pack filters | ||
| 236 | if (defined($dbf->{pack_key}) && $dbf->{pack_key} ne 'raw' && uc($dbf->{type}) ne 'RECNO') { | ||
| 237 | $dbf->{tied}->filter_fetch_key($dbf->packFilterFetch($dbf->{pack_key})); | ||
| 238 | $dbf->{tied}->filter_store_key($dbf->packFilterStore($dbf->{pack_key})); | ||
| 239 | } | ||
| 240 | if (defined($dbf->{pack_val}) && $dbf->{pack_val} ne 'raw') { | ||
| 241 | $dbf->{tied}->filter_fetch_value($dbf->packFilterFetch($dbf->{pack_val})); | ||
| 242 | $dbf->{tied}->filter_store_value($dbf->packFilterStore($dbf->{pack_val})); | ||
| 243 | } | ||
| 244 | } | ||
| 245 | |||
| 246 | return $dbf; | ||
| 247 | } | ||
| 248 | |||
| 249 | ## $bool = $dbf->truncate() | ||
| 250 | ## $bool = $CLASS_OR_OBJ->truncate($file) | ||
| 251 | ## + actually calls unlink($file) | ||
| 252 | ## + no-op if $file and $dbf->{file} are both undef | ||
| 253 | sub truncate { | ||
| 254 | my ($dbf,$file) = @_; | ||
| 255 | $file = $dbf->{file} if (!defined($file)); | ||
| 256 | return if (!defined($file)); | ||
| 257 | unlink($file); | ||
| 258 | } | ||
| 259 | |||
| 260 | ## $bool = $dbf->sync() | ||
| 261 | ## $bool = $dbf->sync($flags) | ||
| 262 | sub sync { | ||
| 263 | my $dbf = shift; | ||
| 264 | return 1 if (!$dbf->opened); | ||
| 265 | return $dbf->{tied}->sync(@_) == 0; | ||
| 266 | } | ||
| 267 | |||
| 268 | ## $bool = $dbf->copy($file2) | ||
| 269 | ## $bool = PACKAGE::copy($file1,$file2) | ||
| 270 | ## + copies database data to $file2 | ||
| 271 | sub copy { | ||
| 272 | my ($dbf,$file2) = @_; | ||
| 273 | my $that = ref($dbf) || __PACKAGE__; | ||
| 274 | my $file1 = ref($dbf) ? $dbf->{file} : $dbf; | ||
| 275 | confess("${that}::copy(): no source specified!") if (!defined($file1)); | ||
| 276 | confess("${that}::copy(): no destination specified!") if (!defined($file2)); | ||
| 277 | if (ref($dbf)) { $dbf->sync() or confess("${that}::copy(): sync failed: $!"); } | ||
| 278 | File::Copy::copy($file1, $file2) | ||
| 279 | or confess("${that}::copy() failed from '$file1' to '$file2': $!"); | ||
| 280 | return 1; | ||
| 281 | } | ||
| 282 | |||
| 283 | ##============================================================================== | ||
| 284 | ## Methods: TT::Persistent | ||
| 285 | |||
| 286 | ## @keys = $dbf->noSaveKeys() | ||
| 287 | sub noSaveKeys { | ||
| 288 | return qw(dbinfo dbopts data tied); | ||
| 289 | } | ||
| 290 | |||
| 291 | ##============================================================================== | ||
| 292 | ## Utils: Filters | ||
| 293 | |||
| 294 | ## \&filter_sub = $db->encFilterFetch() | ||
| 295 | ## \&filter_sub = $db->encFilterFetch($encoding=$db->{encoding}) | ||
| 296 | ## + returns a DB FETCH-filter sub for transparent decoding of DB-data from $encoding | ||
| 297 | sub encFilterFetch { | ||
| 298 | my $enc = defined($_[1]) ? $_[1] : $_[0]{encoding}; | ||
| 299 | return undef if (!$enc || $enc eq 'raw'); | ||
| 300 | return sub { | ||
| 301 | $_ = decode($enc,$_); | ||
| 302 | }; | ||
| 303 | } | ||
| 304 | |||
| 305 | ## \&filter_sub = $db->encFilterStore() | ||
| 306 | ## \&filter_sub = $db->encFilterStore($encoding=$db->{encoding}) | ||
| 307 | ## + returns a DB STORE-filter sub for transparent encoding of DB-data to $encoding | ||
| 308 | sub encFilterStore { | ||
| 309 | my $enc = defined($_[1]) ? $_[1] : $_[0]{encoding}; | ||
| 310 | return undef if (!$enc || $enc eq 'raw'); | ||
| 311 | return sub { | ||
| 312 | $_ = encode($enc,$_); | ||
| 313 | }; | ||
| 314 | } | ||
| 315 | |||
| 316 | ## \&filter_sub = $dbf->packFilterFetch($packas) | ||
| 317 | ## + returns a DB FETCH-filter sub for transparent unpacking of DB-data from $packas | ||
| 318 | sub packFilterFetch { | ||
| 319 | my ($dbf,$packas) = @_; | ||
| 320 | my $packsep = $dbf->{pack_sep} // "\t"; | ||
| 321 | return undef if (!$packas || $packas eq 'raw'); | ||
| 322 | if (length($packas)==1) { | ||
| 323 | return sub { | ||
| 324 | $_ = unpack($packas,$_); | ||
| 325 | }; | ||
| 326 | } else { | ||
| 327 | return sub { | ||
| 328 | $_ = join($packsep,unpack($packas,$_)); | ||
| 329 | } | ||
| 330 | } | ||
| 331 | } | ||
| 332 | |||
| 333 | ## \&filter_sub = $db->packFilterStore($packas) | ||
| 334 | ## + returns a DB STORE-filter sub for transparent encoding of DB-data to $encoding | ||
| 335 | sub packFilterStore { | ||
| 336 | my ($dbf,$packas) = @_; | ||
| 337 | my $packsep = $dbf->{packsep} // "\t"; | ||
| 338 | return undef if (!$packas || $packas eq 'raw'); | ||
| 339 | if (length($packas)==1) { | ||
| 340 | return sub { | ||
| 341 | $_ = pack($packas,$_); | ||
| 342 | }; | ||
| 343 | } else { | ||
| 344 | return sub { | ||
| 345 | $_ = pack($packas,ref($_) ? @$_ : split($packsep,$_)); | ||
| 346 | }; | ||
| 347 | } | ||
| 348 | } | ||
| 349 | |||
| 350 | |||
| 351 | ##============================================================================== | ||
| 352 | ## Footer | ||
| 353 | 1; | ||
| 354 | |||
| 355 | __END__ | ||