| 1 | package AmberDB::Base::Encoder; | ||
| 2 | |||
| 3 | use 5.016; | ||
| 4 | use warnings; | ||
| 5 | use Carp qw(croak cluck); | ||
| 6 | use MIME::Base64 qw(encode_base64 decode_base64); | ||
| 7 | |||
| 8 | our $VERSION = '5.26.6'; | ||
| 9 | my $CREATED = '2026-09-06'; | ||
| 10 | |||
| 11 | # Cross-platform 64-bit integer packing: | ||
| 12 | # Native Q> on 64-bit Perl; transparent high/low 32-bit (NN) fallback on 32-bit Perls (e.g. armv6l) | ||
| 13 | our $HAS_QUAD = eval { my $p = pack( "Q>", 1 ); length($p) == 8 } ? 1 : 0; | ||
| 14 | |||
| 15 | sub _pack_ids { | ||
| 16 | return pack( "(Q>)*", @_ ) if $HAS_QUAD; | ||
| 17 | return pack( "(NN)*", map { ( int( $_ / 4294967296 ), $_ % 4294967296 ) } @_ ); | ||
| 18 | } | ||
| 19 | |||
| 20 | sub _unpack_ids { | ||
| 21 | my ($buf) = @_; | ||
| 22 | return () unless defined $buf && length($buf); | ||
| 23 | return unpack( "(Q>)*", $buf ) if $HAS_QUAD; | ||
| 24 | my @raw = unpack( "(NN)*", $buf ); | ||
| 25 | my @ids; | ||
| 26 | for ( my $i = 0; $i < @raw; $i += 2 ) { | ||
| 27 | push @ids, ( $raw[$i] ? ( $raw[$i] * 4294967296 + $raw[$i + 1] ) : $raw[$i + 1] ); | ||
| 28 | } | ||
| 29 | return @ids; | ||
| 30 | } | ||
| 31 | |||
| 32 | sub _pack_single_id { | ||
| 33 | my ($id) = @_; | ||
| 34 | return pack( "Q>", $id ) if $HAS_QUAD; | ||
| 35 | return pack( "NN", int( $id / 4294967296 ), $id % 4294967296 ); | ||
| 36 | } | ||
| 37 | |||
| 38 | # ===================================================================== | ||
| 39 | # RECORD ENCODING / DECODING — ABR v5 (AmberDB Binary Record) | ||
| 40 | # Native Pure Perl Binary Format with Zero CPAN Dependencies | ||
| 41 | # Format Specification: | ||
| 42 | # Header: \x00 A B R \x05 (5 bytes: NUL + Magic "ABR" + Version 5) | ||
| 43 | # Mode: 1 byte (0x00 = multiple fields, 0x01 = single root reference) | ||
| 44 | # Payload: | ||
| 45 | # Mode 0x00: 2 bytes unsigned short "n" (field count) + field nodes | ||
| 46 | # Mode 0x01: single root node | ||
| 47 | # Node Types (1 byte tag): | ||
| 48 | # 0x00 -> UNDEF | ||
| 49 | # 0x01 -> SCALAR_RAW (4-byte "N" length + raw octets, numbers/ASCII/binary) | ||
| 50 | # 0x02 -> SCALAR_UTF8 (4-byte "N" length + UTF-8 octets, decoded with utf8::decode) | ||
| 51 | # 0x03 -> ARRAY (2-byte "n" count + child nodes) | ||
| 52 | # 0x04 -> HASH (2-byte "n" pair count + 2-byte "n" key len + UTF-8 key + child value node) | ||
| 53 | # ===================================================================== | ||
| 54 | |||
| 55 | sub _abr_encode_node { | ||
| 56 | my ( $self, $node, $depth ) = @_; | ||
| 57 | die "AmberDB ABR: Max nesting depth exceeded (>32)\n" if ( $depth // 0 ) > 32; | ||
| 58 | |||
| 59 | if ( !defined $node ) { | ||
| 60 | return "\x00"; | ||
| 61 | } | ||
| 62 | |||
| 63 | my $ref = ref($node); | ||
| 64 | if ( !$ref ) { | ||
| 65 | my $is_utf8 = utf8::is_utf8($node); | ||
| 66 | my $bytes = "$node"; | ||
| 67 | utf8::encode($bytes) if $is_utf8; | ||
| 68 | return ( $is_utf8 ? "\x02" : "\x01" ) . pack( "N", length($bytes) ) . $bytes; | ||
| 69 | } | ||
| 70 | elsif ( $ref eq 'ARRAY' ) { | ||
| 71 | my $cnt = scalar @$node; | ||
| 72 | my $out = "\x03" . pack( "n", $cnt ); | ||
| 73 | for my $item (@$node) { | ||
| 74 | $out .= $self->_abr_encode_node( $item, ( $depth // 0 ) + 1 ); | ||
| 75 | } | ||
| 76 | return $out; | ||
| 77 | } | ||
| 78 | elsif ( $ref eq 'HASH' ) { | ||
| 79 | my @keys = sort keys %$node; | ||
| 80 | my $cnt = scalar @keys; | ||
| 81 | my $out = "\x04" . pack( "n", $cnt ); | ||
| 82 | for my $k (@keys) { | ||
| 83 | my $k_bytes = "$k"; | ||
| 84 | my $k_utf8 = utf8::is_utf8($k_bytes); | ||
| 85 | utf8::encode($k_bytes) if $k_utf8; | ||
| 86 | $out .= pack( "n", length($k_bytes) ) . $k_bytes; | ||
| 87 | $out .= $self->_abr_encode_node( $node->{$k}, ( $depth // 0 ) + 1 ); | ||
| 88 | } | ||
| 89 | return $out; | ||
| 90 | } | ||
| 91 | else { | ||
| 92 | my $str = "$node"; | ||
| 93 | return "\x01" . pack( "N", length($str) ) . $str; | ||
| 94 | } | ||
| 95 | } | ||
| 96 | |||
| 97 | sub _abr_decode_node { | ||
| 98 | my ( $self, $dref, $pref, $depth ) = @_; | ||
| 99 | die "AmberDB ABR: Max nesting depth exceeded (>32)\n" if ( $depth // 0 ) > 32; | ||
| 100 | |||
| 101 | return undef if $$pref >= length($$dref); | ||
| 102 | my $tag = substr( $$dref, $$pref++, 1 ); | ||
| 103 | return undef if !defined $tag || $tag eq "\x00"; | ||
| 104 | |||
| 105 | if ( $tag eq "\x01" || $tag eq "\x02" ) { | ||
| 106 | # SCALAR_RAW / SCALAR_UTF8 | ||
| 107 | return undef if $$pref + 4 > length($$dref); | ||
| 108 | my $len = unpack( "N", substr( $$dref, $$pref, 4 ) ); | ||
| 109 | $$pref += 4; | ||
| 110 | return undef unless defined $len; | ||
| 111 | return undef if $$pref + $len > length($$dref); | ||
| 112 | my $val = substr( $$dref, $$pref, $len ); | ||
| 113 | $$pref += $len; | ||
| 114 | utf8::decode($val) if $tag eq "\x02"; | ||
| 115 | return $val; | ||
| 116 | } | ||
| 117 | elsif ( $tag eq "\x03" ) { | ||
| 118 | # ARRAY | ||
| 119 | return undef if $$pref + 2 > length($$dref); | ||
| 120 | my $cnt = unpack( "n", substr( $$dref, $$pref, 2 ) ); | ||
| 121 | $$pref += 2; | ||
| 122 | return undef unless defined $cnt; | ||
| 123 | my @arr; | ||
| 124 | for ( 1 .. $cnt ) { | ||
| 125 | push @arr, $self->_abr_decode_node( $dref, $pref, ( $depth // 0 ) + 1 ); | ||
| 126 | } | ||
| 127 | return \@arr; | ||
| 128 | } | ||
| 129 | elsif ( $tag eq "\x04" ) { | ||
| 130 | # HASH | ||
| 131 | return undef if $$pref + 2 > length($$dref); | ||
| 132 | my $cnt = unpack( "n", substr( $$dref, $$pref, 2 ) ); | ||
| 133 | $$pref += 2; | ||
| 134 | return undef unless defined $cnt; | ||
| 135 | my %h; | ||
| 136 | for ( 1 .. $cnt ) { | ||
| 137 | return undef if $$pref + 2 > length($$dref); | ||
| 138 | my $klen = unpack( "n", substr( $$dref, $$pref, 2 ) ); | ||
| 139 | $$pref += 2; | ||
| 140 | return undef unless defined $klen; | ||
| 141 | return undef if $$pref + $klen > length($$dref); | ||
| 142 | my $k = substr( $$dref, $$pref, $klen ); | ||
| 143 | $$pref += $klen; | ||
| 144 | utf8::decode($k); | ||
| 145 | $h{$k} = $self->_abr_decode_node( $dref, $pref, ( $depth // 0 ) + 1 ); | ||
| 146 | } | ||
| 147 | return \%h; | ||
| 148 | } | ||
| 149 | return undef; | ||
| 150 | } | ||
| 151 | |||
| 152 | # my $record = $adb->db_encode(@fields); | ||
| 153 | # my $record = $adb->db_encode(\%hash_data); | ||
| 154 | # ------------------------------------------------ | ||
| 155 | sub db_encode { | ||
| 156 | my ( $self, @fields ) = @_; | ||
| 157 | return unless @fields; | ||
| 158 | |||
| 159 | # ROOT CHECK: If single item passed and it is a reference | ||
| 160 | if ( @fields == 1 && ref( $fields[0] ) ) { | ||
| 161 | return "\x00ABR\x05\x01" . $self->_abr_encode_node( $fields[0], 0 ); | ||
| 162 | } | ||
| 163 | |||
| 164 | my $out = "\x00ABR\x05\x00" . pack( "n", scalar @fields ); | ||
| 165 | for my $f (@fields) { | ||
| 166 | $out .= $self->_abr_encode_node( $f, 0 ); | ||
| 167 | } | ||
| 168 | return $out; | ||
| 169 | } | ||
| 170 | |||
| 171 | # my @fields = $adb->db_decode($record); | ||
| 172 | # my $hash_ref = $adb->db_decode($record); | ||
| 173 | # ------------------------------------------------ | ||
| 174 | sub db_decode { | ||
| 175 | my ( $self, $record ) = @_; | ||
| 176 | return unless defined $record && length $record; | ||
| 177 | |||
| 178 | # ABR Binary Check (5-byte magic signature \x00ABR\x05 or \x00ABR\x01) | ||
| 179 | if ( length($record) >= 7 && substr( $record, 0, 4 ) eq "\x00ABR" && ( substr( $record, 4, 1 ) eq "\x05" || substr( $record, 4, 1 ) eq "\x01" ) ) { | ||
| 180 | my $mode = substr( $record, 5, 1 ); | ||
| 181 | my $pos = 6; | ||
| 182 | |||
| 183 | if ( $mode eq "\x01" ) { | ||
| 184 | # Single root reference | ||
| 185 | return $self->_abr_decode_node( \$record, \$pos, 0 ); | ||
| 186 | } | ||
| 187 | elsif ( $mode eq "\x00" ) { | ||
| 188 | # Multiple fields | ||
| 189 | my $fcnt = unpack( "n", substr( $record, $pos, 2 ) ); | ||
| 190 | $pos += 2; | ||
| 191 | my @fields; | ||
| 192 | for ( 1 .. $fcnt ) { | ||
| 193 | push @fields, $self->_abr_decode_node( \$record, \$pos, 0 ); | ||
| 194 | } | ||
| 195 | return wantarray ? @fields : ( @fields == 1 ? $fields[0] : \@fields ); | ||
| 196 | } | ||
| 197 | } | ||
| 198 | |||
| 199 | # TRANSPARENT FALLBACK: Legacy Text Format Decoding | ||
| 200 | return $self->tsv_decode($record); | ||
| 201 | } | ||
| 202 | |||
| 203 | # ------------------------------------------------ | ||
| 204 | # tsv_decode: Multi-era decoder for historical and text-based records (v1 - v4) | ||
| 205 | # ------------------------------------------------ | ||
| 206 | sub tsv_decode { | ||
| 207 | my ( $self, $record, $expected_rid ) = @_; | ||
| 208 | return () unless defined $record && length($record); | ||
| 209 | |||
| 210 | # If already ABR binary, decode directly via db_decode | ||
| 211 | if ( length($record) >= 7 && substr( $record, 0, 4 ) eq "\x00ABR" ) { | ||
| 212 | return $self->db_decode($record); | ||
| 213 | } | ||
| 214 | |||
| 215 | if ( $self && ref($self) ) { | ||
| 216 | $record = $self->utf_decode($record); | ||
| 217 | } | ||
| 218 | |||
| 219 | # drop line endings: chomp | ||
| 220 | $record =~ s/\R$//; | ||
| 221 | |||
| 222 | # FAST-PATH: Plain TSV record without escapes, entities, or nested tags | ||
| 223 | if ( index($record, "\\") == -1 && index($record, "<TAB") == -1 && index($record, "&#") == -1 && index($record, "ARRAY:") == -1 && index($record, "HASH:") == -1 ) { | ||
| 224 | my @fields = split( /\t/, $record, -1 ); | ||
| 225 | return wantarray ? @fields : ( @fields == 1 ? $fields[0] : \@fields ); | ||
| 226 | } | ||
| 227 | |||
| 228 | # 1. ERA 2019 - 2025: <TAB> Hierarchy (<TAB0>, <TAB1>, <TAB2>, <TAB3>) | ||
| 229 | if ( $record =~ /<TAB[0-9]+>/ ) { | ||
| 230 | my $white_decode = sub { | ||
| 231 | return map { | ||
| 232 | my $s = $_; | ||
| 233 | if ( defined $s ) { | ||
| 234 | $s =~ s/\\(.)/$1 eq "t" ? "\t" : $1 eq "n" ? "\n" : $1 eq "r" ? "\r" : $1 eq "T" ? "\\T" : $1/eg; | ||
| 235 | } | ||
| 236 | $s; | ||
| 237 | } @_; | ||
| 238 | }; | ||
| 239 | |||
| 240 | my $rid_prefix; | ||
| 241 | if ( $record =~ /^([a-zA-Z0-9_\-\.]+)(?:<TAB0>|\t)(.*)$/s ) { | ||
| 242 | my ( $candidate_rid, $rest ) = ( $1, $2 ); | ||
| 243 | if ( !defined $expected_rid || $candidate_rid eq $expected_rid ) { | ||
| 244 | $rid_prefix = $candidate_rid; | ||
| 245 | $record = $rest; | ||
| 246 | } | ||
| 247 | } | ||
| 248 | |||
| 249 | my @fields = $record =~ /<TAB0>/ ? split( /<TAB0>/, $record, -1 ) : split( /\t/, $record, -1 ); | ||
| 250 | @fields = $white_decode->(@fields); | ||
| 251 | |||
| 252 | for my $f1 (@fields) { | ||
| 253 | if ( defined $f1 && $f1 =~ /<TAB1>/ ) { | ||
| 254 | my @sub1 = map { $_ eq '-' ? '' : $_ } split( /<TAB1>/, $f1, -1 ); | ||
| 255 | @sub1 = $white_decode->(@sub1); | ||
| 256 | for my $f2 (@sub1) { | ||
| 257 | if ( defined $f2 && $f2 =~ /<TAB2>/ ) { | ||
| 258 | my @sub2 = map { $_ eq '-' ? '' : $_ } split( /<TAB2>/, $f2, -1 ); | ||
| 259 | @sub2 = $white_decode->(@sub2); | ||
| 260 | for my $f3 (@sub2) { | ||
| 261 | if ( defined $f3 && $f3 =~ /<TAB3>/ ) { | ||
| 262 | my @sub3 = map { $_ eq '-' ? '' : $_ } split( /<TAB3>/, $f3, -1 ); | ||
| 263 | $f3 = [ $white_decode->(@sub3) ]; | ||
| 264 | } | ||
| 265 | } | ||
| 266 | $f2 = \@sub2; | ||
| 267 | } | ||
| 268 | } | ||
| 269 | $f1 = \@sub1; | ||
| 270 | } | ||
| 271 | } | ||
| 272 | |||
| 273 | if ( defined $rid_prefix ) { | ||
| 274 | unshift @fields, $rid_prefix; | ||
| 275 | } | ||
| 276 | return wantarray ? @fields : ( @fields == 1 ? $fields[0] : \@fields ); | ||
| 277 | } | ||
| 278 | |||
| 279 | # 2. ERA 2026: HTML entities (&, |, =) + ARRAY: / HASH: | ||
| 280 | if ( $record =~ /(?:ARRAY:|HASH:|&#(?:38|61|124|92|30);)/ ) { | ||
| 281 | my $unescape_chars = sub { | ||
| 282 | my ($str) = @_; | ||
| 283 | return "" unless defined $str; | ||
| 284 | $str =~ s/\\\\/\\/g; | ||
| 285 | $str =~ s/\\([nrt])/$1 eq 'n' ? "\n" : $1 eq 'r' ? "\r" : "\t"/eg; | ||
| 286 | $str =~ s/=/=/g; | ||
| 287 | $str =~ s/|/|/g; | ||
| 288 | $str =~ s//\x1e/g; | ||
| 289 | $str =~ s/\/\\/g; | ||
| 290 | $str =~ s/&/&/g; | ||
| 291 | return $str; | ||
| 292 | }; | ||
| 293 | |||
| 294 | my $decode_node; | ||
| 295 | $decode_node = sub { | ||
| 296 | my ($field) = @_; | ||
| 297 | return "" unless defined $field; | ||
| 298 | |||
| 299 | if ( $field =~ /^ARRAY:(.*)/s ) { | ||
| 300 | my $payload = $1; | ||
| 301 | return [] if $payload eq ""; | ||
| 302 | return [ map { $decode_node->( $unescape_chars->($_) ) } split( /\|/, $payload, -1 ) ]; | ||
| 303 | } | ||
| 304 | elsif ( $field =~ /^HASH:(.*)/s ) { | ||
| 305 | my $payload = $1; | ||
| 306 | my %h; | ||
| 307 | if ( $payload ne "" ) { | ||
| 308 | for my $pair ( split( /\|/, $payload, -1 ) ) { | ||
| 309 | my ( $k, $v ) = split( /=/, $pair, 2 ); | ||
| 310 | $h{ $unescape_chars->($k) } = $decode_node->( $unescape_chars->( $v // '' ) ); | ||
| 311 | } | ||
| 312 | } | ||
| 313 | return \%h; | ||
| 314 | } | ||
| 315 | elsif ( $field =~ /\\T/ ) { | ||
| 316 | return [ map { $unescape_chars->($_) } split( /\\T/, $field, -1 ) ]; | ||
| 317 | } | ||
| 318 | else { | ||
| 319 | return $unescape_chars->($field); | ||
| 320 | } | ||
| 321 | }; | ||
| 322 | |||
| 323 | my @raw = split( /\t/, $record, -1 ); | ||
| 324 | if ( @raw == 1 && $raw[0] =~ /^(?:ARRAY|HASH):/ ) { | ||
| 325 | my $res = $decode_node->( $raw[0] ); | ||
| 326 | return wantarray ? ($res) : $res; | ||
| 327 | } | ||
| 328 | my @res = map { $decode_node->( $unescape_chars->($_) ) } @raw; | ||
| 329 | return wantarray ? @res : ( @res == 1 ? $res[0] : \@res ); | ||
| 330 | } | ||
| 331 | |||
| 332 | # 3. ERA 2004 - 2006 & 2003: \t root, \T array delimiter, standard escapes | ||
| 333 | my $unescape_basic = sub { | ||
| 334 | my ($s) = @_; | ||
| 335 | return "" unless defined $s; | ||
| 336 | $s =~ s/\\(.)/$1 eq "t" ? "\t" : $1 eq "n" ? "\n" : $1 eq "r" ? "\r" : $1 eq "T" ? "\\T" : $1 eq "\\" ? "\\" : $1/eg; | ||
| 337 | return $s; | ||
| 338 | }; | ||
| 339 | |||
| 340 | my @raw_fields = split( /\t/, $record, -1 ); | ||
| 341 | @raw_fields = map { $unescape_basic->($_) } @raw_fields; | ||
| 342 | |||
| 343 | for my $line (@raw_fields) { | ||
| 344 | if ( defined $line && $line =~ /\\T/ ) { | ||
| 345 | my @parts = split( /\\T/, $line, -1 ); | ||
| 346 | @parts = map { $unescape_basic->($_) } @parts; | ||
| 347 | $line = \@parts; | ||
| 348 | } | ||
| 349 | } | ||
| 350 | |||
| 351 | return wantarray ? @raw_fields : ( @raw_fields == 1 ? $raw_fields[0] : \@raw_fields ); | ||
| 352 | } | ||
| 353 | |||
| 354 | # ------------------------------------------------ | ||
| 355 | # tsv_encode: Legacy text encoder for CSV exports and text-mode pipelines | ||
| 356 | # ------------------------------------------------ | ||
| 357 | sub tsv_encode { | ||
| 358 | my ( $self, @fields ) = @_; | ||
| 359 | return unless @fields; | ||
| 360 | |||
| 361 | my $encode_node; | ||
| 362 | $encode_node = sub { | ||
| 363 | my ($node) = @_; | ||
| 364 | |||
| 365 | if ( ref($node) eq "ARRAY" ) { | ||
| 366 | my @escaped = map { ref($_) ? $encode_node->($_) : $self->char_escape($_) } @$node; | ||
| 367 | return "ARRAY:" . join( "|", @escaped ); | ||
| 368 | } | ||
| 369 | elsif ( ref($node) eq "HASH" ) { | ||
| 370 | my @escaped_pairs; | ||
| 371 | foreach my $k ( sort keys %$node ) { | ||
| 372 | my $safe_k = $self->char_escape($k); | ||
| 373 | my $safe_v = ref( $node->{$k} ) ? $encode_node->( $node->{$k} ) : $self->char_escape( $node->{$k} ); | ||
| 374 | push @escaped_pairs, "$safe_k=$safe_v"; | ||
| 375 | } | ||
| 376 | return "HASH:" . join( "|", @escaped_pairs ); | ||
| 377 | } | ||
| 378 | else { | ||
| 379 | return $self->char_escape($node); | ||
| 380 | } | ||
| 381 | }; | ||
| 382 | |||
| 383 | if ( @fields == 1 && ref( $fields[0] ) ) { | ||
| 384 | return $encode_node->( $fields[0] ); | ||
| 385 | } | ||
| 386 | |||
| 387 | my @encoded = map { ref($_) ? $encode_node->($_) : $self->char_escape($_) } @fields; | ||
| 388 | my $result = join( "\t", @encoded ); | ||
| 389 | utf8::encode($result) if utf8::is_utf8($result); | ||
| 390 | |||
| 391 | return $result; | ||
| 392 | } | ||
| 393 | |||
| 394 | # ------------------------------------------------ | ||
| 395 | sub char_escape { | ||
| 396 | my ( $self, $str ) = @_; | ||
| 397 | return "" unless defined $str; | ||
| 398 | |||
| 399 | # WARNING: Escape ampersand & first! (Double escaping logic) | ||
| 400 | $str =~ s/&/&/g; # ampersand | ||
| 401 | $str =~ s/\\/\/g; # backslash | ||
| 402 | $str =~ s/\|/|/g; # pipe (Array/Hash delimiter) | ||
| 403 | $str =~ s/=/=/g; # equals (Hash key-value delimiter) | ||
| 404 | $str =~ s/\x1e//g; # record separator (Transaction journal delimiter) | ||
| 405 | $str =~ s/\t/\\t/g; # tab | ||
| 406 | $str =~ s/\n/\\n/g; # newline | ||
| 407 | $str =~ s/\r/\\r/g; # carriage return | ||
| 408 | return $str; | ||
| 409 | } | ||
| 410 | |||
| 411 | # ------------------------------------------------ | ||
| 412 | sub char_unescape { | ||
| 413 | my ( $self, $str ) = @_; | ||
| 414 | return "" unless defined $str; | ||
| 415 | |||
| 416 | $str =~ s{ (\\\\) | \\([nrt]) | &\#(92|61|124|38|30); }{ | ||
| 417 | defined $1 ? "\\" : | ||
| 418 | defined $2 ? ( $2 eq 'n' ? "\n" : $2 eq 'r' ? "\r" : "\t" ) : | ||
| 419 | chr($3) | ||
| 420 | }gex; | ||
| 421 | |||
| 422 | return $str; | ||
| 423 | } | ||
| 424 | |||
| 425 | # encode like cgi escape | ||
| 426 | # my $sifresiz = $adb->uri_encode("sifreli"); | ||
| 427 | # ------------------------------------------------ | ||
| 428 | sub uri_encode { | ||
| 429 | my ( $self, $str ) = @_; | ||
| 430 | |||
| 431 | $str =~ s/([^A-Za-z0-9\-_.~])/sprintf("%%%02X", ord($1))/ge; | ||
| 432 | |||
| 433 | return $str; | ||
| 434 | } | ||
| 435 | |||
| 436 | # decode like cgi unescape | ||
| 437 | # my $sifresiz = $adb->uri_decode("sifresiz"); | ||
| 438 | # ------------------------------------------------ | ||
| 439 | sub uri_decode { | ||
| 440 | my ( $self, $str ) = @_; | ||
| 441 | |||
| 442 | $str =~ s/%([0-9A-Fa-f]{2})/chr(hex($1))/ge; | ||
| 443 | |||
| 444 | return $str; | ||
| 445 | } | ||
| 446 | |||
| 447 | # my $key_escape = $adb->key_encode($key); | ||
| 448 | # ------------------------------------------------ | ||
| 449 | sub key_encode { | ||
| 450 | my ( $self, $key ) = @_; | ||
| 451 | |||
| 452 | my $key_escape = "$key"; | ||
| 453 | |||
| 454 | if ( $key_escape =~ /[^\w]/ ) { | ||
| 455 | if ( $self && ref($self) ) { | ||
| 456 | $key_escape = $self->to_ascii($key_escape); | ||
| 457 | } | ||
| 458 | $key_escape =~ s/[^\w]//g; | ||
| 459 | } | ||
| 460 | |||
| 461 | return $key_escape; | ||
| 462 | } | ||
| 463 | |||
| 464 | # ===================================================================== | ||
| 465 | # FORMAT DETECTION & LEGACY DECODING (2003-2026 formats) | ||
| 466 | # ===================================================================== | ||
| 467 | |||
| 468 | sub detect_tsv_format { | ||
| 469 | my ( $self, $record ) = @_; | ||
| 470 | return 'v5' if !defined $record || $record eq ''; | ||
| 471 | |||
| 472 | if ( length($record) >= 7 && substr( $record, 0, 4 ) eq "\x00ABR" ) { | ||
| 473 | return 'v5'; | ||
| 474 | } | ||
| 475 | if ( $record =~ /(?:ARRAY:|HASH:|&#(?:38|61|124|92|30);)/ ) { | ||
| 476 | return 'v4'; | ||
| 477 | } | ||
| 478 | if ( $record =~ /<TAB[0-9]+>/ ) { | ||
| 479 | return 'v3'; | ||
| 480 | } | ||
| 481 | if ( $record =~ /\\T/ ) { | ||
| 482 | return 'v2'; | ||
| 483 | } | ||
| 484 | return 'v1'; | ||
| 485 | } | ||
| 486 | |||
| 487 | # ===================================================================== | ||
| 488 | # BINARY INDEX PACKING (64-bit Big-Endian Packed Identifiers) | ||
| 489 | # Pure binary buffer operations: 0 hashes, 0 regexes, 0 splits. | ||
| 490 | # ===================================================================== | ||
| 491 | |||
| 492 | # $adb->bin_encode(\@rids) | ||
| 493 | # Encodes list of numeric record IDs into 8-byte packed binary format (64-bit uint Q>*). | ||
| 494 | # ------------------------------------------------ | ||
| 495 | sub bin_encode { | ||
| 496 | my ( $self, $rids ) = @_; | ||
| 497 | |||
| 498 | return '' unless ref($rids) eq 'ARRAY' && @$rids; | ||
| 499 | |||
| 500 | # Fast numeric filter: positive numbers only (no regex overhead) | ||
| 501 | my @valid = grep { defined $_ && $_ > 0 } @$rids; | ||
| 502 | return '' unless @valid; | ||
| 503 | |||
| 504 | return _pack_ids( @valid ); | ||
| 505 | } | ||
| 506 | |||
| 507 | # $adb->bin_decode($binary_buffer, $offset, $limit, $dir) | ||
| 508 | # Decodes 8-byte binary buffer (64-bit uint Q>*) with O(1) substr slicing. | ||
| 509 | # Returns ($total_count, @sliced_ids) | ||
| 510 | # ------------------------------------------------ | ||
| 511 | sub bin_decode { | ||
| 512 | my ( $self, $buffer, $offset, $limit, $dir ) = @_; | ||
| 513 | |||
| 514 | return ( 0, () ) unless defined $buffer && length($buffer) >= 8; | ||
| 515 | |||
| 516 | $offset = 0 if !defined $offset || $offset < 0; | ||
| 517 | $limit ||= 0; | ||
| 518 | my $rec_size = 8; | ||
| 519 | my $total = int( length($buffer) / $rec_size ); | ||
| 520 | return ( $total, () ) if $offset >= $total; | ||
| 521 | |||
| 522 | $dir = ( defined $dir && lc($dir) eq 'asc' ) ? 'asc' : 'desc'; | ||
| 523 | |||
| 524 | if ( $total - $offset < $limit || $limit <= 0 ) { | ||
| 525 | $limit = $total - $offset; | ||
| 526 | } | ||
| 527 | |||
| 528 | if ( $dir eq 'desc' ) { | ||
| 529 | my $real_start = $total - $offset - $limit; | ||
| 530 | my $slice = substr( $buffer, $real_start * $rec_size, $limit * $rec_size ); | ||
| 531 | return ( $total, reverse _unpack_ids( $slice ) ); | ||
| 532 | } | ||
| 533 | |||
| 534 | my $slice = substr( $buffer, $offset * $rec_size, $limit * $rec_size ); | ||
| 535 | return ( $total, _unpack_ids( $slice ) ); | ||
| 536 | } | ||
| 537 | |||
| 538 | # $idx = $adb->bin_search($buffer, $target) | ||
| 539 | # Binary search on 8-byte big-endian packed buffer. | ||
| 540 | # Returns 0-based record index (0, 1, 2, ...) if found, -1 if not found. | ||
| 541 | # ------------------------------------------------ | ||
| 542 | sub bin_search { | ||
| 543 | my ( $self, $buffer, $target ) = @_; | ||
| 544 | |||
| 545 | return -1 unless defined $buffer && length($buffer) >= 8 && defined $target && $target > 0; | ||
| 546 | |||
| 547 | my $len = int( length($buffer) / 8 ); | ||
| 548 | my ( $low, $high ) = ( 0, $len - 1 ); | ||
| 549 | |||
| 550 | while ( $low <= $high ) { | ||
| 551 | my $mid = int( ( $low + $high ) / 2 ); | ||
| 552 | my $val = $HAS_QUAD | ||
| 553 | ? unpack( "Q>", substr( $buffer, $mid * 8, 8 ) ) | ||
| 554 | : do { | ||
| 555 | my ( $hi, $lo ) = unpack( "NN", substr( $buffer, $mid * 8, 8 ) ); | ||
| 556 | $hi ? ( $hi * 4294967296 + $lo ) : $lo; | ||
| 557 | }; | ||
| 558 | return $mid if $val == $target; | ||
| 559 | if ( $val < $target ) { | ||
| 560 | $low = $mid + 1; | ||
| 561 | } | ||
| 562 | else { | ||
| 563 | $high = $mid - 1; | ||
| 564 | } | ||
| 565 | } | ||
| 566 | |||
| 567 | return -1; | ||
| 568 | } | ||
| 569 | |||
| 570 | # $adb->bin_crop($buffer, $liste, $option) | ||
| 571 | # Ultra-fast binary intersection / cropping without unpacking the buffer into hashes. | ||
| 572 | # $buffer: Packed 8-byte binary buffer (64-bit uint Q>*) | ||
| 573 | # $liste: ARRAY ref of integer IDs OR another 8-byte packed binary buffer | ||
| 574 | # $option: 0 (or undef/empty) -> no sorting (fastest) | ||
| 575 | # 1 -> preserves $buffer order | ||
| 576 | # 2 -> preserves $liste order | ||
| 577 | # Returns: | ||
| 578 | # List context: Array of matching integer IDs | ||
| 579 | # Scalar context: Packed 8-byte binary buffer ((Q>)*) | ||
| 580 | # ------------------------------------------------ | ||
| 581 | sub bin_crop { | ||
| 582 | my ( $self, $buffer, $liste, $option ) = @_; | ||
| 583 | |||
| 584 | return () unless defined $buffer && length($buffer) >= 8; | ||
| 585 | return () unless defined $liste; | ||
| 586 | |||
| 587 | $option ||= 0; | ||
| 588 | my @result; | ||
| 589 | |||
| 590 | if ( ref($liste) eq 'ARRAY' ) { | ||
| 591 | return () unless @$liste; | ||
| 592 | |||
| 593 | if ( $option == 2 ) { | ||
| 594 | # 2: Listenin sıralaması korunur | ||
| 595 | for my $id (@$liste) { | ||
| 596 | next unless defined $id && $id > 0; | ||
| 597 | push @result, $id if $self->bin_search( $buffer, $id ) >= 0; | ||
| 598 | } | ||
| 599 | } | ||
| 600 | elsif ( $option == 1 ) { | ||
| 601 | # 1: Buffer'ın sıralaması korunur (indeks pozisyonuna göre sırala: 0 hash) | ||
| 602 | my @matches; | ||
| 603 | for my $id (@$liste) { | ||
| 604 | next unless defined $id && $id > 0; | ||
| 605 | my $pos = $self->bin_search( $buffer, $id ); | ||
| 606 | push @matches, [ $pos, $id ] if $pos >= 0; | ||
| 607 | } | ||
| 608 | @result = map { $_->[1] } sort { $a->[0] <=> $b->[0] } @matches; | ||
| 609 | } | ||
| 610 | else { | ||
| 611 | # 0: Sıralama önemsiz (en hızlı yol) | ||
| 612 | for my $id (@$liste) { | ||
| 613 | next unless defined $id && $id > 0; | ||
| 614 | push @result, $id if $self->bin_search( $buffer, $id ) >= 0; | ||
| 615 | } | ||
| 616 | } | ||
| 617 | } | ||
| 618 | elsif ( !ref($liste) && length($liste) >= 8 ) { | ||
| 619 | # $liste de ikili bir tampon ise | ||
| 620 | if ( $option == 2 ) { | ||
| 621 | # 2: $liste tamponunun sırası | ||
| 622 | my @l_ids = _unpack_ids( $liste ); | ||
| 623 | @result = grep { $self->bin_search( $buffer, $_ ) >= 0 } @l_ids; | ||
| 624 | } | ||
| 625 | elsif ( $option == 1 ) { | ||
| 626 | # 1: $buffer tamponunun sırası (indeks pozisyonuna göre sırala: 0 hash) | ||
| 627 | my @l_ids = _unpack_ids( $liste ); | ||
| 628 | my @matches; | ||
| 629 | for my $id (@l_ids) { | ||
| 630 | my $pos = $self->bin_search( $buffer, $id ); | ||
| 631 | push @matches, [ $pos, $id ] if $pos >= 0; | ||
| 632 | } | ||
| 633 | @result = map { $_->[1] } sort { $a->[0] <=> $b->[0] } @matches; | ||
| 634 | } | ||
| 635 | else { | ||
| 636 | # 0: Sıralama önemsiz: küçük olanı büyükte ikili ara | ||
| 637 | my ( $small, $large ) = length($buffer) <= length($liste) ? ( $buffer, $liste ) : ( $liste, $buffer ); | ||
| 638 | my @s_ids = _unpack_ids( $small ); | ||
| 639 | @result = grep { $self->bin_search( $large, $_ ) >= 0 } @s_ids; | ||
| 640 | } | ||
| 641 | } | ||
| 642 | |||
| 643 | return wantarray ? @result : ( @result ? _pack_ids( @result ) : '' ); | ||
| 644 | } | ||
| 645 | |||
| 646 | # $adb->bin_union(\@buffers, $dir) | ||
| 647 | # High-performance binary union across 8-byte packed buffers without Perl %seen hash. | ||
| 648 | # In list context: returns deduplicated list of IDs sorted by $dir (default 'asc'). | ||
| 649 | # In scalar context: returns 8-byte packed binary buffer ((Q>)*) sorted ascending. | ||
| 650 | # ------------------------------------------------ | ||
| 651 | sub bin_union { | ||
| 652 | my ( $self, $buffers, $dir ) = @_; | ||
| 653 | |||
| 654 | return () unless $buffers; | ||
| 655 | |||
| 656 | my @bufs = ref($buffers) eq 'ARRAY' ? @$buffers : ($buffers); | ||
| 657 | @bufs = grep { defined $_ && length($_) >= 8 } @bufs; | ||
| 658 | return () unless @bufs; | ||
| 659 | |||
| 660 | # Single buffer fast-path | ||
| 661 | if ( @bufs == 1 ) { | ||
| 662 | if (wantarray) { | ||
| 663 | my ( undef, @ids ) = $self->bin_decode( $bufs[0], 0, 0, $dir // 'asc' ); | ||
| 664 | return @ids; | ||
| 665 | } | ||
| 666 | return $bufs[0]; | ||
| 667 | } | ||
| 668 | |||
| 669 | # Multiple buffers: concatenate binary streams in C (zero hash allocation) | ||
| 670 | my $combined = join( '', @bufs ); | ||
| 671 | return () unless length($combined) >= 8; | ||
| 672 | |||
| 673 | my @all_ids = _unpack_ids( $combined ); | ||
| 674 | @all_ids = ( defined $dir && lc($dir) eq 'desc' ) ? sort { $b <=> $a } @all_ids : sort { $a <=> $b } @all_ids; | ||
| 675 | |||
| 676 | # O(N) linear deduplication of adjacent elements (no hash table!) | ||
| 677 | my $prev = -1; | ||
| 678 | my @uniq = grep { my $dup = ($_ == $prev); $prev = $_; !$dup } @all_ids; | ||
| 679 | |||
| 680 | return wantarray ? @uniq : ( @uniq ? _pack_ids( @uniq ) : '' ); | ||
| 681 | } | ||
| 682 | |||
| 683 | # $updated_buf = $adb->bin_add($buffer, $rids) | ||
| 684 | # Adds one or more record IDs into 8-byte packed binary buffer without duplicates. | ||
| 685 | # Pure binary buffer operation: uses binary search to check existence and maintains sort order. | ||
| 686 | # ------------------------------------------------ | ||
| 687 | sub bin_add { | ||
| 688 | my ( $self, $buffer, $new_rids ) = @_; | ||
| 689 | |||
| 690 | return $buffer // '' unless defined $new_rids; | ||
| 691 | |||
| 692 | $buffer //= ''; | ||
| 693 | |||
| 694 | my @ids = ref($new_rids) eq 'ARRAY' | ||
| 695 | ? @$new_rids | ||
| 696 | : ( !ref($new_rids) && length($new_rids) >= 8 && length($new_rids) % 8 == 0 | ||
| 697 | ? _unpack_ids( $new_rids ) | ||
| 698 | : ($new_rids) ); | ||
| 699 | |||
| 700 | @ids = grep { defined $_ && $_ > 0 } @ids; | ||
| 701 | return $buffer unless @ids; | ||
| 702 | |||
| 703 | # Deduplicate input IDs without a hash | ||
| 704 | my @sorted_in = sort { $a <=> $b } @ids; | ||
| 705 | my $prev_in = -1; | ||
| 706 | @ids = grep { my $d = ($_ == $prev_in); $prev_in = $_; !$d } @sorted_in; | ||
| 707 | |||
| 708 | if ( length($buffer) < 8 ) { | ||
| 709 | return _pack_ids( @ids ); | ||
| 710 | } | ||
| 711 | |||
| 712 | my @to_add; | ||
| 713 | for my $id (@ids) { | ||
| 714 | push @to_add, $id if $self->bin_search( $buffer, $id ) < 0; | ||
| 715 | } | ||
| 716 | return $buffer unless @to_add; | ||
| 717 | |||
| 718 | $buffer .= _pack_ids( @to_add ); | ||
| 719 | return $self->bin_sort($buffer); | ||
| 720 | } | ||
| 721 | |||
| 722 | # $updated_buf = $adb->bin_punch($buffer, $del_rids) | ||
| 723 | # Removes one or more record IDs from 8-byte packed binary buffer. | ||
| 724 | # Pure binary buffer operation: O(log N) binary search and surgical 8-byte excision. | ||
| 725 | # ------------------------------------------------ | ||
| 726 | sub bin_punch { | ||
| 727 | my ( $self, $buffer, $del_rids ) = @_; | ||
| 728 | |||
| 729 | return '' unless defined $buffer && length($buffer) >= 8; | ||
| 730 | return $buffer unless defined $del_rids; | ||
| 731 | |||
| 732 | my @del_list = ref($del_rids) eq 'ARRAY' | ||
| 733 | ? @$del_rids | ||
| 734 | : ( !ref($del_rids) && length($del_rids) >= 8 && length($del_rids) % 8 == 0 | ||
| 735 | ? _unpack_ids( $del_rids ) | ||
| 736 | : ($del_rids) ); | ||
| 737 | |||
| 738 | @del_list = grep { defined $_ && $_ > 0 } @del_list; | ||
| 739 | return $buffer unless @del_list; | ||
| 740 | |||
| 741 | for my $del_id (@del_list) { | ||
| 742 | my $len = length($buffer); | ||
| 743 | last if $len < 8; | ||
| 744 | |||
| 745 | my $mid = $self->bin_search( $buffer, $del_id ); | ||
| 746 | if ( $mid >= 0 ) { | ||
| 747 | substr( $buffer, $mid * 8, 8, "" ); | ||
| 748 | next; | ||
| 749 | } | ||
| 750 | |||
| 751 | # Fallback for unsorted buffer: 8-byte boundary index search | ||
| 752 | my $target_bytes = _pack_single_id( $del_id ); | ||
| 753 | my $pos = index( $buffer, $target_bytes ); | ||
| 754 | while ( $pos >= 0 ) { | ||
| 755 | if ( $pos % 8 == 0 ) { | ||
| 756 | substr( $buffer, $pos, 8, "" ); | ||
| 757 | last; | ||
| 758 | } | ||
| 759 | $pos = index( $buffer, $target_bytes, $pos + 1 ); | ||
| 760 | } | ||
| 761 | } | ||
| 762 | |||
| 763 | return $buffer; | ||
| 764 | } | ||
| 765 | |||
| 766 | # $found = $adb->bin_find($buffer, $rid) | ||
| 767 | # Returns 1 if $rid exists in 8-byte binary buffer, 0 otherwise. | ||
| 768 | # Pure O(log N) binary search on 8-byte chunks. | ||
| 769 | # ------------------------------------------------ | ||
| 770 | sub bin_find { | ||
| 771 | my ( $self, $buffer, $rid ) = @_; | ||
| 772 | |||
| 773 | return 0 unless defined $buffer && length($buffer) >= 8 && defined $rid && $rid > 0; | ||
| 774 | |||
| 775 | return 1 if $self->bin_search( $buffer, $rid ) >= 0; | ||
| 776 | |||
| 777 | # Fallback for unsorted buffer: 8-byte boundary index search | ||
| 778 | my $target_bytes = _pack_single_id( $rid ); | ||
| 779 | my $pos = index( $buffer, $target_bytes ); | ||
| 780 | while ( $pos >= 0 ) { | ||
| 781 | return 1 if $pos % 8 == 0; | ||
| 782 | $pos = index( $buffer, $target_bytes, $pos + 1 ); | ||
| 783 | } | ||
| 784 | |||
| 785 | return 0; | ||
| 786 | } | ||
| 787 | |||
| 788 | # $sorted_buf = $adb->bin_sort($buffer, $dir) | ||
| 789 | # Sorts 8-byte big-endian binary buffer in ascending (or descending) order. | ||
| 790 | # Pure C-level string sort of 8-byte big-endian chunks (0 Perl hash / 0 unpack to array). | ||
| 791 | # ------------------------------------------------ | ||
| 792 | sub bin_sort { | ||
| 793 | my ( $self, $buffer, $dir ) = @_; | ||
| 794 | |||
| 795 | return '' unless defined $buffer && length($buffer) >= 8; | ||
| 796 | return $buffer if length($buffer) == 8; | ||
| 797 | |||
| 798 | if ( defined $dir && lc($dir) eq 'desc' ) { | ||
| 799 | return join( '', sort { $b cmp $a } unpack( "(a8)*", $buffer ) ); | ||
| 800 | } | ||
| 801 | return join( '', sort unpack( "(a8)*", $buffer ) ); | ||
| 802 | } | ||
| 803 | |||
| 804 | # $count = $adb->bin_count($buffer) | ||
| 805 | # Returns record count in 8-byte binary buffer. | ||
| 806 | # ------------------------------------------------ | ||
| 807 | sub bin_count { | ||
| 808 | my ( $self, $buffer ) = @_; | ||
| 809 | |||
| 810 | return 0 unless defined $buffer && length($buffer) >= 8; | ||
| 811 | return int( length($buffer) / 8 ); | ||
| 812 | } | ||
| 813 | |||
| 814 | # ===================================================================== | ||
| 815 | # JOURNAL ENCODING / DECODING (TSV + Base64 Delta Specification) | ||
| 816 | # Standard 8-field Journal Entry Format: | ||
| 817 | # $epoch \t $type \t $tableid \t $file_path \t $key \t $action \t $pos \t $base64_payload \n | ||
| 818 | # - $type: 'recs' or 'index' | ||
| 819 | # - $action: 'add', 'edit', 'del', 'append', 'punch', 'patch', 'put' | ||
| 820 | # - $pos: exact byte offset (e.g. 0, 4800000, 8000000) or '' if unpositioned | ||
| 821 | # - $payload: Base64-encoded raw octets, or '__NULL__' if undef | ||
| 822 | # ===================================================================== | ||
| 823 | |||
| 824 | sub journal_encode { | ||
| 825 | my ( $self, $type, $tableid, $file_path, $key, $action, $pos, $payload, $epoch ) = @_; | ||
| 826 | |||
| 827 | $epoch //= time(); | ||
| 828 | $type //= 'recs'; | ||
| 829 | $tableid //= ''; | ||
| 830 | $file_path //= ''; | ||
| 831 | $key //= ''; | ||
| 832 | $action //= 'put'; | ||
| 833 | $pos = ( defined $pos && $pos ne '' ) ? "$pos" : ''; | ||
| 834 | |||
| 835 | my $b64; | ||
| 836 | if ( !defined $payload || $payload eq '__NULL__' ) { | ||
| 837 | $b64 = '__NULL__'; | ||
| 838 | } | ||
| 839 | else { | ||
| 840 | $b64 = encode_base64( $payload, '' ); | ||
| 841 | } | ||
| 842 | |||
| 843 | return join( "\t", $epoch, $type, $tableid, $file_path, $key, $action, $pos, $b64 ); | ||
| 844 | } | ||
| 845 | |||
| 846 | sub journal_decode { | ||
| 847 | my ( $self, $line ) = @_; | ||
| 848 | |||
| 849 | return unless defined $line; | ||
| 850 | chomp $line; | ||
| 851 | return if $line eq ''; | ||
| 852 | |||
| 853 | my ( $epoch, $type, $tableid, $file_path, $key, $action, $pos, $b64 ) = split /\t/, $line, 8; | ||
| 854 | |||
| 855 | my $payload; | ||
| 856 | if ( !defined $b64 || $b64 eq '__NULL__' ) { | ||
| 857 | $payload = undef; | ||
| 858 | } | ||
| 859 | else { | ||
| 860 | $payload = decode_base64( $b64 ); | ||
| 861 | } | ||
| 862 | |||
| 863 | return { | ||
| 864 | epoch => $epoch, | ||
| 865 | type => $type, | ||
| 866 | tableid => $tableid, | ||
| 867 | file_path => $file_path, | ||
| 868 | key => $key, | ||
| 869 | action => $action, | ||
| 870 | pos => ( defined $pos && $pos ne '' ) ? ( 0 + $pos ) : undef, | ||
| 871 | payload => $payload, | ||
| 872 | }; | ||
| 873 | } | ||
| 874 | |||
| 875 | 1; |