blib/lib/AmberDB/Tools/Index.pm untested

filter: Statementstmt 0.0% 0 / 805 Branchbran 0.0% 0 / 384 Conditioncond 0.0% 0 / 359 MC/DCmcdc 0.0% 0 / 211 Subroutinesub 0.0% 0 / 28 Podpod 0.0% 0 / 13 totaltotal 0.0% 0 / 1800 CC 193 SCAR 64.5
CC
193
Cov
0%
set_search
1260.0
set_fields
992.0
set_sort
812.0
CRAP 631.6
1 package AmberDB::Tools::Index;
2  
3 use 5.016;
4 use warnings;
5 use strict;
6 use Carp qw(croak cluck);
7 use File::Spec;
8  
9 our $VERSION = '5.26.6';
10 my $CREATED = '2018-10-08';
11  
12 sub new {
13  
14 my $class = shift;
15 my $self = {};
16  
17 require AmberDB;
18  
19 my ( $adb, %inputs );
20  
21 if ( ref( $_[0] ) ) {
22 $adb = shift;
23 %inputs = @_;
24 }
25 else {
26 %inputs = @_;
27 $adb = AmberDB->new(%inputs);
28 }
29  
30 $self->{_adb} = $adb;
31  
32 foreach my $in ( keys %inputs ) {
33 $self->{ uc($in) } = $inputs{$in};
34 }
35 $self->{say} = "";
36  
37 bless $self, $class;
38 return $self;
39 }
40  
41 # my $ok = $tools->set_index($tableid, @records);
42 # ------------------------------------------------
43 sub set_index {
44  
45 my ( $self, $tableid, @records ) = @_;
46 my $adb = $self->{_adb} or return;
47  
48 my $table_info = $adb->table_info($tableid) or return;
49 if ( $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' ) ) {
50 $self->{say} .= " - Table '$tableid' is in simple mode, skipping index generation.\n";
51 return 1;
52 }
53 my $table_path = $adb->table_path($tableid);
54 return unless ( -e "$table_path.$adb->{db_ext}" );
55  
56 # 1. Clean up previous index files on physical storage and RAM-disk before rebuilding
57 my @idx_exts = qw( inx src fld fac slg unq ajw ajn );
58 for my $ext (@idx_exts) {
59 my $p_file = "$table_path.$ext";
60 if ( -e $p_file ) {
61 $adb->table_close($p_file);
62 unlink $p_file;
63 }
64 }
65 if ( $adb->ramdisk_is_mounted() ) {
66 my $rm_path = $adb->ramdisk_path($tableid);
67 if ($rm_path) {
68 for my $ext (@idx_exts) {
69 my $rm_file = "$rm_path.$ext";
70 if ( -e $rm_file ) {
71 $adb->table_close($rm_file);
72 unlink $rm_file;
73 }
74 }
75 }
76 }
77 $adb->clear_cache($tableid);
78  
79 # 2. Read records if list empty.
80 if ( !@records ) {
81 @records = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc');
82 }
83 return unless scalar @records;
84  
85 # Pre-fetch foreign records for junk_rules in one batch before sub-indexers run
86 if ( $table_info->{use_junk} ) {
87 $self->{say} .= " - Pre-fetching relational foreign records for junk rules...\n";
88 $adb->prefetch_junk_rdbm( $table_info, \@records );
89 }
90  
91 # 3. Create readall index
92 if ( exists( $table_info->{record_index} ) ) {
93 $self->{say} .= " - Rebuilding readall index (.inx)...\n";
94 my $ok = $self->set_readall( $tableid, @records );
95 }
96  
97 # 4. Create search index
98 if ( exists( $table_info->{search_block} ) ) {
99 $self->{say} .= " - Rebuilding search index (.src)...\n";
100 my $ok = $self->set_search( $tableid, @records );
101 }
102  
103 # 5. Create fetch field index.
104 if ( exists( $table_info->{match_block} ) ) {
105 $self->{say} .= " - Rebuilding field index (.fld)...\n";
106 my $ok = $self->set_fields( $tableid, @records );
107 }
108  
109 # 6. Create facet index.
110 if ( exists( $table_info->{use_facet} ) ) {
111 $self->{say} .= " - Rebuilding facet index (.fac)...\n";
112 my $ok = $self->set_filters( $tableid, @records );
113 }
114  
115 # 7. Create sort index.
116 if ( exists( $table_info->{sort_block} ) ) {
117 $self->{say} .= " - Rebuilding sort index (.inx)...\n";
118 my $ok = $self->set_sort( $tableid, @records );
119 }
120  
121 # 8. Create URL slug index
122 if ( exists( $table_info->{slug_block} ) ) {
123 $self->{say} .= " - Rebuilding slug index (.slg)...\n";
124 my $ok = $self->set_rwlnkall( $tableid, @records );
125 }
126  
127 # 9. Create unique index if schema defines unique blocks
128 my $has_unique = 0;
129 if ( $table_info->{blocks} && ref($table_info->{blocks}) eq 'ARRAY' ) {
130 for my $b (@{ $table_info->{blocks} }) {
131 if ( ref($b) eq 'HASH' && defined $b->{valid} && $b->{valid} =~ /unique/i ) {
132 $has_unique = 1;
133 last;
134 }
135 }
136 }
137 if ($has_unique) {
138 $adb->unique_add( $table_path, $table_info, \@records );
139 }
140  
141 # 10. Create suggest index (.ajw, .ajn)
142 if ( exists( $table_info->{suggest_block} ) ) {
143 $self->{say} .= " - Rebuilding suggest index (.ajw, .ajn)...\n";
144 my $ok = $self->set_suggest( $tableid, @records );
145 }
146  
147 # 10. If RAM-disk is active, sync freshly built index files to RAM-disk
148 if ( $adb->ramdisk_is_mounted() ) {
149 my $rm_path = $adb->ramdisk_path($tableid);
150 if ($rm_path) {
151 require File::Copy;
152 for my $ext (@idx_exts) {
153 my $src_file = "$table_path.$ext";
154 my $dst_file = "$rm_path.$ext";
155 if ( -e $src_file ) {
156 $adb->table_close($src_file);
157 $adb->table_close($dst_file);
158 unlink $dst_file if -e $dst_file;
159 File::Copy::copy( $src_file, $dst_file );
160 chmod 0777, $dst_file if -e $dst_file;
161 }
162 }
163 }
164 }
165  
166 return 1;
167 }
168  
169 # Rebuilds Read All index.
170 # my $ok = $tools->set_readall($tableid);
171 # my $ok = $tools->set_readall($tableid, @records);
172 # ------------------------------------------------
173 sub set_readall {
174  
175 my ( $self, $tableid, @records ) = @_;
176 my $adb = $self->{_adb} or return;
177  
178 $tableid or return;
179 my $table_path = $adb->table_path($tableid);
180  
181 my $file_path = "$table_path.$adb->{db_ext}";
182 my $index_path = "$table_path.inx";
183 my $tmp_path = "$index_path.tmp";
184 return unless -e $file_path;
185  
186 my $table_info = $adb->table_info($tableid);
187 return unless $table_info;
188 return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' );
189  
190 # Read records from table if absent
191 if ( !scalar @records ) {
192 @records = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc');
193 }
194 scalar @records or return;
195 my @all_records = ref( $records[0] ) eq 'ARRAY' ? ( map { $_->[0] } @records ) : @records;
196 @all_records = grep { defined $_ && $_ > 0 } @all_records;
197  
198 my ( @active_records, @junk_records );
199 my $has_junk = $table_info->{use_junk} ? 1 : 0;
200  
201 # Accept both full record arrayrefs [ $id, @data ] and scalar ID lists
202 if ( ref $records[0] eq "ARRAY" ) {
203 if ($has_junk) {
204 my $rdbm_recs = $adb->prefetch_junk_rdbm( $table_info, \@records );
205 for my $rec (@records) {
206 my $rid = $rec->[0];
207 next unless defined $rid && $rid > 0;
208 if ( $adb->junk_rules($table_info, $rec, $rdbm_recs) ) {
209 push @junk_records, $rid;
210 }
211 else {
212 push @active_records, $rid;
213 }
214 }
215 }
216 else {
217 for my $rec (@records) {
218 my $rid = $rec->[0];
219 push @active_records, $rid if defined $rid && $rid > 0;
220 }
221 }
222 }
223 else {
224 my @clean_rids = grep { defined $_ && $_ > 0 } @records;
225 if ($has_junk) {
226 for my $rid (@clean_rids) {
227 my @rec = $adb->table_readid($file_path, $rid);
228 if ( $adb->junk_rules($table_info, @rec) ) {
229 push @junk_records, $rid;
230 }
231 else {
232 push @active_records, $rid;
233 }
234 }
235 }
236 else {
237 @active_records = @clean_rids;
238 }
239 }
240  
241 # Deduplicate in original order
242 my %seen_act;
243 @active_records = grep { defined $_ && $_ > 0 && !$seen_act{$_}++ } @active_records;
244 my %seen_junk;
245 @junk_records = grep { defined $_ && $_ > 0 && !$seen_junk{$_}++ } @junk_records;
246  
247  
248 my $cur_max = 0;
249 if ( @all_records && $all_records[-1] > $cur_max ) {
250 $cur_max = $all_records[-1];
251 }
252  
253 my $del_path = "$table_path.del";
254 if ( -e $del_path && $adb->table_read($del_path) ) {
255 $adb->recs_scan( $del_path, sub {
256 my ($k) = @_;
257 $cur_max = $k if $k > $cur_max;
258 } );
259 $adb->table_close($del_path);
260 }
261  
262 my $old_lastid = $adb->table_lastid($tableid) // 0;
263 my $last_id = $cur_max > $old_lastid ? $cur_max : $old_lastid;
264  
265 if ( $adb->table_write($tmp_path) ) {
266 my $cnt = scalar @all_records;
267 $adb->index_put( $tmp_path, "keys", \@all_records, "ids" );
268 $adb->index_put( $tmp_path, "count", $cnt, "raw" );
269 $adb->index_put( $tmp_path, "lastid", $last_id, "raw" ) if defined $last_id;
270  
271 if ($has_junk) {
272 $adb->index_put( $tmp_path, "A:keys", \@active_records, "ids" );
273 $adb->index_put( $tmp_path, "A:count", scalar @active_records, "raw" );
274 $adb->index_put( $tmp_path, "B:keys", \@junk_records, "ids" );
275 $adb->index_put( $tmp_path, "B:count", scalar @junk_records, "raw" );
276 }
277  
278 $adb->table_close($tmp_path);
279 $adb->table_close($index_path);
280 unlink($index_path);
281 rename( $tmp_path, $index_path ) or do {
282 require File::Copy;
283 File::Copy::move( $tmp_path, $index_path );
284 };
285 $self->{say} .= " - Readall records created: \n";
286 $self->{say} .= " * $table_path.inx created ($cnt total records, " . scalar(@active_records) . " active, " . scalar(@junk_records) . " junk)\n";
287 }
288  
289 return 1;
290 }
291  
292 # Rebuilds search word index.
293 # my $ok = $tools->set_search($tableid, @records);
294 # ------------------------------------------------
295 sub set_search {
296  
297 my ( $self, $tableid, @records ) = @_;
298 my $adb = $self->{_adb} or return;
299  
300 return unless $tableid;
301 my $table_path = $adb->table_path($tableid);
302 my $table_info = $adb->table_info($tableid);
303 return unless $table_info;
304 return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' );
305 return unless exists( $table_info->{search_block} );
306  
307 # Read records from table if absent
308 if ( !scalar @records ) {
309 @records = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc');
310 }
311 return unless scalar @records;
312  
313 # 1. Identify RDBM blocks and prepare pre-fetch
314 my %rdbm_blocks; # field => { table => ..., display => ... }
315 for my $blk ( @{ $table_info->{search_block} } ) {
316 my ( $field, $src_table, $src_display ) =
317 ref($blk) eq 'ARRAY' ? ( $blk->[0], $blk->[1], $blk->[2] )
318 : ( $blk, undef, undef );
319 if ( !$src_table ) {
320 ( $src_table, $src_display ) = $adb->rdbm_target( $table_info, $field );
321 }
322 if ( $src_table ) {
323 $src_display //= 1;
324 $rdbm_blocks{$field} = { table => $src_table, display => $src_display };
325 }
326 }
327  
328 # 2. Collect unique foreign IDs across all records
329 my %string;
330 if ( %rdbm_blocks ) {
331 for my $rec (@records) {
332 for my $field ( keys %rdbm_blocks ) {
333 my $val = $rec->[$field];
334 next unless defined $val && $val ne '';
335 my $target_table = $rdbm_blocks{$field}->{table};
336 for my $id ( split /[,;]/, $val ) {
337 $id =~ s/^\s+|\s+$//g;
338 $string{$target_table}->{$id} = 1 if $id =~ /^\d+$/;
339 }
340 }
341 }
342 }
343  
344 # 3. Batch pre-fetch foreign tables into in-memory lookup map and official cache
345 my %rdbm_recs; # table => { id => \@target_record }
346 for my $table ( keys %string ) {
347 my @needed_ids = grep { !$adb->get_cache( $table, $_ ) } keys %{ $string{$table} };
348 if (@needed_ids) {
349 my @target_recs = $adb->read_list( $table, \@needed_ids );
350 for my $trec (@target_recs) {
351 $adb->set_cache( $table, $trec->[0], $trec );
352 }
353 }
354 $rdbm_recs{$table} = $adb->get_cache($table) || {};
355 }
356  
357 # 4. Tokenize and index words
358 my ( %search, %act_search, %junk_search );
359 $self->{say} .= " - Search word kayitlari olusturuluyor: \n";
360 my $has_junk = $table_info->{use_junk} ? 1 : 0;
361 my $junk_rdbm = $has_junk ? $adb->prefetch_junk_rdbm( $table_info, \@records ) : {};
362 foreach my $line (@records) {
363 my @fields = @$line;
364 my $rid = $fields[0];
365 next unless defined $rid && $rid > 0;
366 my $is_junk = $has_junk ? $adb->junk_rules( $table_info, \@fields, $junk_rdbm ) : 0;
367 foreach my $blk ( @{ $table_info->{search_block} } ) {
368 my $b_idx = ref($blk) eq "ARRAY" ? $blk->[0] : $blk;
369 $b_idx =~ /^\d+$/ or next;
370  
371 my $val = $fields[$b_idx];
372 if ( my $rdbm_info = $rdbm_blocks{$b_idx} ) {
373 if ( defined $val && $val ne '' ) {
374 my $table = $rdbm_info->{table};
375 my $disp = $rdbm_info->{display};
376 my @parts;
377 for my $id ( split /[,;]/, $val ) {
378 $id =~ s/^\s+|\s+$//g;
379 if ( my $rec = $rdbm_recs{$table}->{$id} ) {
380 my $name = $rec->[$disp];
381 push @parts, $name if defined $name && length($name);
382 }
383 }
384 $val = join( ' ', @parts );
385 }
386 }
387  
388 next unless defined $val && $val ne '';
389 my %recsearch = $adb->get_words( $val, "write", $tableid );
390  
391 foreach my $key ( keys %recsearch ) {
392 push @{ $search{$b_idx}{$key} }, $rid;
393 if ($has_junk) {
394 if ($is_junk) {
395 push @{ $junk_search{$b_idx}{$key} }, $rid;
396 }
397 else {
398 push @{ $act_search{$b_idx}{$key} }, $rid;
399 }
400 }
401 }
402 }
403 }
404  
405 my $file_path = "${table_path}.src";
406 my $tmp_path = "$file_path.tmp";
407  
408 unlink($tmp_path);
409  
410 my %batch_put;
411 foreach my $blk ( @{ $table_info->{search_block} } ) {
412 my $b_idx = ref($blk) eq "ARRAY" ? $blk->[0] : $blk;
413 $b_idx =~ /^\d+$/ or next;
414  
415 next unless $search{$b_idx};
416 for my $key ( keys %{ $search{$b_idx} } ) {
417 my $all_ids = $search{$b_idx}{$key};
418 $batch_put{"$b_idx:$key"} = $all_ids if @$all_ids;
419  
420 if ($has_junk) {
421 if ( my $act_recs = $act_search{$b_idx}{$key} ) {
422 $batch_put{"A:$b_idx:$key"} = $act_recs if @$act_recs;
423 }
424 if ( my $junk_recs = $junk_search{$b_idx}{$key} ) {
425 $batch_put{"B:$b_idx:$key"} = $junk_recs if @$junk_recs;
426 }
427 }
428 }
429 }
430  
431 if ( %batch_put ) {
432 $adb->table_write($tmp_path)
433 or do {
434 cluck "[DB_TOOL] $tmp_path can't open.\n";
435 return;
436 };
437 $adb->index_put( $tmp_path, \%batch_put, "ids" );
438 $adb->table_close($tmp_path);
439 $adb->table_close($file_path);
440 unlink($file_path);
441 rename( $tmp_path, $file_path ) or do {
442 require File::Copy;
443 File::Copy::move( $tmp_path, $file_path );
444 };
445 $self->{say} .= " * $file_path created.\n";
446 }
447 else {
448 $adb->table_close($file_path);
449 unlink($file_path);
450 }
451  
452 return 1;
453 }
454  
455 # Rebuilds block index.
456 # my $ok = $tools->set_fields("tableid", @records);
457 # ------------------------------------------------
458 sub set_fields {
459  
460 my ( $self, $tableid, @records ) = @_;
461 my $adb = $self->{_adb} or return;
462  
463 my $table_path = $adb->table_path($tableid);
464 my $table_info = $adb->table_info($tableid);
465 return unless $table_info;
466 return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' );
467 return unless exists( $table_info->{match_block} );
468 return unless -e "$table_path.$adb->{db_ext}";
469  
470 # Read records from table if absent
471 if ( !scalar @records ) {
472 @records = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc');
473 }
474 return unless scalar @records;
475  
476 my ( %fields, %act_fields, %junk_fields );
477  
478 my $unq_path = "${table_path}.unq";
479 my $unq_opened = 0;
480 if ( !$adb->{_db}->{$unq_path} ) {
481 $adb->table_write($unq_path);
482 $unq_opened = 1;
483 }
484  
485 # Invert loops: outer loop @records (single pass), inner loop match_block
486 my @match_blks = grep { /^\d+$/ } @{ $table_info->{match_block} };
487 my $has_junk = $table_info->{use_junk} ? 1 : 0;
488 my $rdbm_recs = $has_junk ? $adb->prefetch_junk_rdbm( $table_info, \@records ) : {};
489  
490 my $table_name = $table_info->{table} || $table_info->{id} || $tableid;
491  
492 # Preload unq into in-memory cache to eliminate per-record BDB disk lookups
493 if ( $adb->{_db}->{$unq_path} ) {
494 $adb->recs_scan(
495 $unq_path,
496 sub {
497 my ( $k, $v ) = @_;
498 if ( $k =~ /^(\d+):s:(.*)$/s ) {
499 $adb->set_cache( $table_name, "$1:$2", $v );
500 }
501 elsif ( $k =~ /^(\d+):lastid$/ ) {
502 $adb->set_cache( $table_name, "$1:lastid", $v );
503 }
504 }
505 );
506 }
507  
508 my $total_rec = scalar @records;
509 my $prog_step = int($total_rec / 10) || 1000;
510 my $count = 0;
511  
512 foreach my $record (@records) {
513 $count++;
514 if ( $total_rec > 1000 && $count % $prog_step == 0 ) {
515 $self->{say} .= " ... processed $count / $total_rec records for field index\n";
516 }
517 my @fields_arr = @$record;
518 my $rid = $fields_arr[0];
519 next unless defined $rid && $rid > 0;
520 my $is_junk = $has_junk ? $adb->junk_rules( $table_info, \@fields_arr, $rdbm_recs ) : 0;
521  
522 foreach my $line (@match_blks) {
523 my $val = $fields_arr[$line];
524 next unless defined $val && $val ne '';
525  
526 my @num_ids;
527 if ( !ref($val) && $val =~ /^\d+$/ && length($val) < 20 ) {
528 @num_ids = ($val);
529 }
530 elsif ( !ref($val) && $val =~ /^[\d,;\s]+$/ ) {
531 ( my $clean = $val ) =~ s/\s+//g;
532 @num_ids = grep { /^\d+$/ } split /[,;]/, $clean;
533 }
534 else {
535 @num_ids = $adb->field_to_list( $val, $table_path, $table_info, $line );
536 }
537  
538 my %seen_nid;
539 foreach my $nid (@num_ids) {
540 next if $seen_nid{$nid}++;
541 push @{ $fields{$line}{$nid} }, $rid;
542 if ($has_junk) {
543 if ($is_junk) {
544 push @{ $junk_fields{$line}{$nid} }, $rid;
545 }
546 else {
547 push @{ $act_fields{$line}{$nid} }, $rid;
548 }
549 }
550 }
551 }
552 }
553  
554 if ($unq_opened) {
555 $adb->table_close($unq_path);
556 }
557  
558 $self->{say} .= " - Fields fetch kayitlari olusturuluyor: \n";
559  
560 my $file_path = "${table_path}.fld";
561 my $tmp_path = "$file_path.tmp";
562  
563 unlink($tmp_path);
564  
565 my %batch_put;
566 foreach my $line (@match_blks) {
567 next unless $fields{$line};
568 for my $val ( keys %{ $fields{$line} } ) {
569 my $all_ids = $fields{$line}{$val};
570 $batch_put{"$line:$val"} = $all_ids if @$all_ids;
571  
572 if ($has_junk) {
573 if ( my $act_recs = $act_fields{$line}{$val} ) {
574 $batch_put{"A:$line:$val"} = $act_recs if @$act_recs;
575 }
576 if ( my $junk_recs = $junk_fields{$line}{$val} ) {
577 $batch_put{"B:$line:$val"} = $junk_recs if @$junk_recs;
578 }
579 }
580 }
581 }
582  
583 if ( %batch_put ) {
584 $adb->table_write($tmp_path)
585 or do {
586 cluck "[DB_TOOL] $tmp_path can't open for write.\n";
587 return;
588 };
589 $adb->index_put( $tmp_path, \%batch_put, "ids" );
590 $adb->table_close($tmp_path);
591 $adb->table_close($file_path);
592 unlink($file_path);
593 rename( $tmp_path, $file_path ) or do {
594 require File::Copy;
595 File::Copy::move( $tmp_path, $file_path );
596 };
597  
598 $self->{say} .= " * $file_path \n";
599 }
600 else {
601 $adb->table_close($file_path);
602 unlink($file_path);
603 }
604  
605 return 1;
606 }
607  
608 # Rebuilds facet forward index.
609 # Includes active_flag and "active" key.
610 # my $ok = $tools->set_filters("tableid", @records);
611 # ------------------------------------------------
612 sub set_filters {
613  
614 my ( $self, $tableid, @records ) = @_;
615 my $adb = $self->{_adb} or return;
616  
617 return unless $tableid;
618 my $table_path = $adb->table_path($tableid);
619 return unless -e "$table_path.$adb->{db_ext}";
620  
621 my $table_info = $adb->table_info($tableid);
622 return unless $table_info;
623 return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' );
624 return unless exists( $table_info->{match_block} );
625 return unless exists( $table_info->{use_facet} );
626  
627 my @fblocks = @{ $table_info->{match_block} };
628 my $has_active_rule = exists $table_info->{facet_rules};
629 my $fac_path = "$table_path.fac";
630 my $tmp_path = "$fac_path.tmp";
631  
632 # Read records from table if absent
633 if ( !scalar @records ) {
634 @records = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc');
635 }
636 scalar @records or return;
637  
638 # Columnar facet architecture with dictionary (.unq) support
639 if ( exists $table_info->{facet_block} && ref($table_info->{facet_block}) eq 'ARRAY' && @{ $table_info->{facet_block} } ) {
640 $self->{say} .= " - Columnar facet index (.fac, .unq) olusturuluyor: \n";
641 unlink $fac_path if -e $fac_path;
642 unlink "$table_path.unq" if -e "$table_path.unq";
643  
644 my $batch_size = 5000;
645 for ( my $i = 0; $i < @records; $i += $batch_size ) {
646 my $end = $i + $batch_size - 1;
647 $end = $#records if $end > $#records;
648 my @batch = @records[ $i .. $end ];
649 $adb->facet_add( $table_path, $table_info, \@batch );
650 }
651 $self->{say} .= " * $fac_path \n";
652 $self->{say} .= " * $table_path.unq \n" if -e "$table_path.unq";
653 return 1;
654 }
655  
656 unlink($tmp_path);
657  
658 $adb->table_write($tmp_path)
659 or do {
660 cluck "[DB_TOOL] $tmp_path can't open for write.\n";
661 return;
662 };
663  
664 $self->{say} .= " - Facet forward index olusturuluyor: \n";
665  
666 my @active_ids;
667 my %batch_facets;
668  
669 foreach my $record (@records) {
670 my @fields = @$record;
671 $fields[0] or next;
672 my @pairs;
673 foreach my $blk (@fblocks) {
674 $blk =~ /^\d+$/ or next;
675 next unless defined $fields[$blk] && $fields[$blk] ne '';
676 my @vals;
677 if ( ref $fields[$blk] eq "ARRAY" ) { @vals = @{ $fields[$blk] } }
678 elsif ( $fields[$blk] =~ /[,;]/ ) { @vals = split /\s*[,;]\s*/, $fields[$blk] }
679 else { @vals = ( $fields[$blk] ) }
680 push @pairs, "$blk:$_" for @vals;
681 }
682 next unless @pairs;
683 my $is_active = $adb->facet_rules( $table_info, @fields );
684 push @active_ids, $fields[0] if $is_active && $has_active_rule;
685 $batch_facets{ $fields[0] } = join( "\t", $is_active, @pairs );
686 }
687  
688 if ( %batch_facets ) {
689 $adb->index_put( $tmp_path, \%batch_facets, "raw" );
690 }
691  
692 # "active" key: write all active IDs if facet_rules defined
693 if ( $has_active_rule && @active_ids ) {
694 $adb->index_put( $tmp_path, "active", \@active_ids, "ids" );
695 }
696  
697 $adb->table_close($tmp_path);
698 $adb->table_close($fac_path);
699 unlink($fac_path);
700 rename( $tmp_path, $fac_path ) or do {
701 require File::Copy;
702 File::Copy::move( $tmp_path, $fac_path );
703 };
704  
705 $self->{say} .= " * $fac_path \n";
706  
707 return 1;
708 }
709  
710 # Rebuilds ReWrite links.
711 # my $ok = $tools->set_rwlnkall($tableid);
712 # my $ok = $tools->set_rwlnkall($tableid, @records);
713 # ------------------------------------------------
714 sub set_rwlnkall {
715  
716 my ( $self, $tableid, @records ) = @_;
717 my $adb = $self->{_adb} or return;
718  
719 my $table_path = $adb->table_path($tableid);
720 return unless -e "$table_path.$adb->{db_ext}";
721  
722 my $table_info = $adb->table_info($tableid);
723 return unless $table_info;
724 return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' );
725 return unless $table_info->{slug_block};
726  
727 # Read records from table if input is empty
728 if ( !scalar @records ) {
729 (@records) = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc');
730 @records = grep { ref($_) eq 'ARRAY' } @records;
731 if ( !scalar @records ) {
732 cluck "[DB_TOOL] No records found for ReWrite in $tableid.\n";
733 return;
734 }
735 }
736 else {
737 @records = grep { ref($_) eq 'ARRAY' } @records;
738 }
739  
740 # Rebuild unified slug file.
741 my $slg_path = "${table_path}.slg";
742 my $tmp_path = "$slg_path.tmp";
743  
744 unlink($tmp_path);
745  
746 my @slg_records;
747 my %seen_links;
748  
749 @records = $adb->db_sortid( $tableid, @records );
750 foreach my $record (@records) {
751 my $rw_link = $adb->set_slug( $tableid, $record );
752 next unless defined $rw_link && length $rw_link;
753 if ( exists $seen_links{$rw_link} ) {
754 $rw_link .= "-$record->[0]";
755 }
756 $seen_links{$rw_link} = $record->[0];
757 push @slg_records, [ "0:$record->[0]", $rw_link ];
758 push @slg_records, [ "1:$rw_link", $record->[0] ];
759 }
760  
761 if ( @slg_records ) {
762 $adb->table_write($tmp_path)
763 or do {
764 cluck "[DB_TOOL] $tmp_path can't be written.\n";
765 return 0;
766 };
767 $adb->recs_put( $tmp_path, @slg_records );
768 $adb->table_close($tmp_path);
769 $adb->table_close($slg_path);
770 unlink($slg_path);
771 rename( $tmp_path, $slg_path ) or do {
772 require File::Copy;
773 File::Copy::move( $tmp_path, $slg_path );
774 };
775  
776 $self->{say} .= " - Slug indexes are being created.: \n";
777 $self->{say} .= " * $slg_path \n";
778 }
779 else {
780 $adb->table_close($slg_path);
781 unlink($slg_path);
782 }
783  
784 return 1;
785 }
786  
787 # Rebuilds sort index within .inx ($blk:keys).
788 # my $ok = $tools->set_sort($tableid, @records);
789 # ------------------------------------------------
790 sub set_sort {
791  
792 my ( $self, $tableid, @records ) = @_;
793 my $adb = $self->{_adb} or return;
794  
795 return unless $tableid;
796 my $table_path = $adb->table_path($tableid);
797 my $table_info = $adb->table_info($tableid);
798 return unless $table_info;
799 return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' );
800 return unless exists $table_info->{sort_block};
801  
802 if ( !@records ) {
803 @records = $adb->read_all( $tableid, 0, 0, no_index => 1, dir => 'asc' );
804 }
805  
806 my $has_junk = $table_info->{use_junk};
807 my $junk_rdbm = $has_junk ? $adb->prefetch_junk_rdbm( $table_info, \@records ) : {};
808  
809 my ( %keys_batch, %raw_batch );
810 foreach my $cfg ( @{ $table_info->{sort_block} } ) {
811 my ( $blk, $type, $len ) = ref($cfg) eq 'HASH'
812 ? ( $cfg->{blk}, $cfg->{type}, $cfg->{len} // 8 )
813 : ( $cfg, undef, 8 );
814  
815 if ( ( !defined $type || $type eq '' || $type eq 'auto' ) && $table_info->{blocks} ) {
816 if ( ref($table_info->{blocks}) eq 'ARRAY' && ref($table_info->{blocks}[$blk]) eq 'HASH' ) {
817 $type = $table_info->{blocks}[$blk]{type};
818 }
819 }
820 $type ||= 'string';
821  
822 my %map;
823 my ( %act_map, %junk_map );
824 my %uniq_vals;
825 my ( %act_uniq, %junk_uniq );
826 foreach my $rec (@records) {
827 next unless ref($rec) eq 'ARRAY' && defined $rec->[0] && $rec->[0] > 0;
828 my $rid = $rec->[0];
829 my $val = $rec->[$blk];
830 my $norm = $adb->normalize_sort_key( $val, $type, $len );
831 $map{$rid} = $norm;
832  
833 if ( defined $val && $val ne '' ) {
834 $uniq_vals{$val} = 1;
835 }
836  
837 if ($has_junk) {
838 my $is_junk = $adb->junk_rules( $table_info, $rec, $junk_rdbm );
839 if ($is_junk) {
840 $junk_map{$rid} = $norm;
841 $junk_uniq{$val} = 1 if defined $val && $val ne '';
842 }
843 else {
844 $act_map{$rid} = $norm;
845 $act_uniq{$val} = 1 if defined $val && $val ne '';
846 }
847 }
848 }
849  
850 # Sort all keys in-memory with deterministic tie-breaker (strictly numeric IDs)
851 my @sorted_ids = grep { defined $_ && $_ > 0 } sort {
852 ( ( $map{$a} // '' ) cmp ( $map{$b} // '' ) )
853 || ( $a <=> $b )
854 } keys %map;
855  
856 $keys_batch{"$blk:keys"} = \@sorted_ids;
857 foreach my $k ( keys %map ) {
858 $raw_batch{"$blk:$k"} = $map{$k};
859 }
860  
861 my $sort_vals = sub {
862 my ($href) = @_;
863 my $is_num = ( $type eq 'num' || $type eq 'decimal' ) ? 1 : 0;
864 if ( !$is_num && %$href ) {
865 $is_num = 1;
866 for my $k ( keys %$href ) {
867 if ( $k !~ /^-?[0-9]+(?:\.[0-9]+)?$/ ) {
868 $is_num = 0;
869 last;
870 }
871 }
872 }
873 my @sv = $is_num ? ( sort { $a <=> $b } keys %$href ) : ( sort { $a cmp $b } keys %$href );
874 return join("\t", @sv);
875 };
876 $raw_batch{"$blk:vals"} = $sort_vals->(\%uniq_vals) if %uniq_vals;
877  
878 if ($has_junk) {
879 my @sorted_act = grep { defined $_ && $_ > 0 } sort {
880 ( ( $act_map{$a} // '' ) cmp ( $act_map{$b} // '' ) )
881 || ( $a <=> $b )
882 } keys %act_map;
883 $keys_batch{"A:$blk:keys"} = \@sorted_act;
884 $raw_batch{"A:$blk:vals"} = $sort_vals->(\%act_uniq) if %act_uniq;
885  
886 my @sorted_junk = grep { defined $_ && $_ > 0 } sort {
887 ( ( $junk_map{$a} // '' ) cmp ( $junk_map{$b} // '' ) )
888 || ( $a <=> $b )
889 } keys %junk_map;
890 $keys_batch{"B:$blk:keys"} = \@sorted_junk;
891 $raw_batch{"B:$blk:vals"} = $sort_vals->(\%junk_uniq) if %junk_uniq;
892 }
893  
894 }
895  
896 if ( %keys_batch || %raw_batch ) {
897 my $inx_path = "${table_path}.inx";
898 if ( $adb->table_write($inx_path) ) {
899 $adb->index_put( $inx_path, \%keys_batch, "ids" ) if %keys_batch;
900 $adb->index_put( $inx_path, \%raw_batch, "raw" ) if %raw_batch;
901 $adb->table_close($inx_path);
902 }
903 }
904  
905 $self->{say} .= " - Sort indexes created in ${table_path}.inx for table $tableid.\n";
906 return 1;
907 }
908  
909 # Rebuilds index for all tables
910 # my $ok = $tools->index_alltables();
911 # print $self->{say};
912 # ------------------------------------------------
913 sub index_alltables {
914  
915 my ($self) = @_;
916 my $adb = $self->{_adb} or return;
917  
918 my @tables;
919  
920 # 1. Locate tables.
921 my $tables_hash = ( $self->can('all_tables') ?
922 $self->all_tables() :
923 do {
924 require AmberDB::Tools::Maintain;
925 AmberDB::Tools::Maintain->new($self->{_adb})->all_tables();
926 }
927 );
928 foreach my $dbase ( keys %$tables_hash ) {
929 $dbase =~ /^[a-z0-9]+$/ or next;
930 foreach my $tableid ( keys %{ $tables_hash->{$dbase} } ) {
931 $tableid =~ /^[a-z0-9_]+$/ or next;
932 my $table_path = $adb->table_path($tableid);
933 push @tables, [ $tableid, "$table_path.$adb->{db_ext}" ];
934 }
935 }
936  
937 # 2. Read table records and enter indexing loop
938 foreach my $table_entry (@tables) {
939 my $tbl = $table_entry->[0];
940 my @records = $adb->read_all($tbl, 0, 0, no_index => 1, dir => 'asc');
941  
942 my $count = scalar @records;
943 $self->set_index( $tbl, @records );
944  
945 $self->{say} .=
946 "Table: $tbl, Record count: $count\n";
947 }
948 }
949  
950 # Verifies validity of Read All index
951 # my $diff = $tools->check_readall($tableid, @records);
952 # ------------------------------------------------
953 sub check_readall {
954  
955 my ( $self, $tableid, @records ) = @_;
956 my $adb = $self->{_adb} or return;
957  
958 return unless $tableid;
959 my $table_path = $adb->table_path($tableid);
960 my $table_info = $adb->table_info($tableid);
961  
962 my ( %diff, %recs );
963 foreach my $rec (@records) {
964 $rec = $rec->[0] if ref $rec eq "ARRAY";
965 next unless defined $rec && $rec ne "";
966 $recs{keys}->{$rec} = 1;
967 $recs{lastid} ||= $rec;
968 $rec > $recs{lastid} and $recs{lastid} = $rec;
969 }
970  
971 my (%inds);
972 my $inx_path = "$table_path.inx";
973 my ( $total_keys, @keys ) = $adb->index_get( $inx_path, "keys", "ids" );
974 $adb->table_close($inx_path);
975 foreach my $rec (@keys) {
976 $inds{keys}->{$rec} = 1;
977 $inds{lastid} ||= $rec;
978 $rec > $inds{lastid} and $inds{lastid} = $rec;
979 }
980  
981 if ( ( $recs{lastid} // "" ) ne ( $inds{lastid} // "" ) ) {
982 $diff{lastid}->{recs} = $recs{lastid};
983 $diff{lastid}->{inds} = $inds{lastid};
984 }
985  
986 my $diffs = $adb->hash_diff( $recs{keys}, $inds{keys} );
987  
988 if ( $diffs->{hash1} ) {
989 $diff{keys}->{recs} = $diffs->{hash1};
990 }
991 if ( $diffs->{hash2} ) {
992 $diff{keys}->{inds} = $diffs->{hash2};
993 }
994  
995 return \%diff;
996 }
997  
998 # Rebuilds search word index.
999 # my $ok = $tools->check_search($tableid, @records);
1000 # ------------------------------------------------
1001 sub check_search {
1002  
1003 my ( $self, $tableid, @records ) = @_;
1004 my $adb = $self->{_adb} or return;
1005  
1006 my %diff = ();
1007 return unless $tableid;
1008 scalar @records or return \%diff;
1009 my $table_path = $adb->table_path($tableid);
1010 my $table_info = $adb->table_info($tableid);
1011 exists( $table_info->{search_block} ) or return \%diff;
1012  
1013 foreach my $line (@records) {
1014 my @fields = @$line;
1015 foreach my $src ( @{ $table_info->{search_block} } ) {
1016 $src =~ /^[0-9]+$/ or next;
1017 my %words = $adb->get_words( $fields[$src], "write" );
1018  
1019 foreach my $word ( keys %words ) {
1020 $diff{recs}->{$src}->{$word}->{ $fields[0] } = 1;
1021 }
1022 }
1023 }
1024  
1025 my $unified_src = "${table_path}.src";
1026 if ( -e $unified_src ) {
1027 if ( $adb->table_read($unified_src) ) {
1028 $adb->recs_scan(
1029 $unified_src,
1030 sub {
1031 my ( $k, $records ) = @_;
1032 my ( $src, $word ) = split( /:/, $k, 2 );
1033 return unless defined $src && defined $word;
1034 my ( undef, @rids ) = $adb->bin_decode($records);
1035 foreach my $rid (@rids) {
1036 if ( exists( $diff{recs}->{$src}->{$word}->{$rid} ) ) {
1037 delete( $diff{recs}->{$src}->{$word}->{$rid} );
1038 }
1039 else {
1040 $diff{inds}->{$src}->{$word}->{$rid} = 1;
1041 }
1042 }
1043 }
1044 );
1045 $adb->table_close($unified_src);
1046 }
1047 }
1048  
1049 return \%diff;
1050 }
1051  
1052 # $ok = $self->set_suggest( $tableid, @records );
1053 # Rebuilds .ajw and .ajn suggest index files from records
1054 # ------------------------------------------------
1055 sub set_suggest {
1056 my ( $self, $tableid, @records ) = @_;
1057  
1058 my $adb = $self->{_adb};
1059 my $table_info = $adb->table_info($tableid);
1060 return unless $table_info && exists $table_info->{suggest_block};
1061  
1062 my $table_path = $adb->table_path($tableid);
1063 my $ajw_path = "${table_path}.ajw";
1064 my $ajn_path = "${table_path}.ajn";
1065 my $tmp_ajw = "$ajw_path.tmp";
1066 my $tmp_ajn = "$ajn_path.tmp";
1067  
1068 unlink $tmp_ajw, $tmp_ajn;
1069  
1070 # 1. Identify RDBM blocks and prepare pre-fetch
1071 my %rdbm_blocks;
1072 for my $blk ( @{ $table_info->{suggest_block} } ) {
1073 my ( $real_blk, $src_table, $src_display ) =
1074 ref($blk) eq 'ARRAY' ? ( $blk->[0], $blk->[1], $blk->[2] )
1075 : ( $blk, undef, undef );
1076  
1077 if ( !$src_table ) {
1078 ( $src_table, $src_display ) = $adb->rdbm_target( $table_info, $real_blk );
1079 }
1080 if ( $src_table ) {
1081 $src_display //= 1;
1082 $rdbm_blocks{$real_blk} = { table => $src_table, display => $src_display };
1083 }
1084 }
1085  
1086 # 2. Collect unique foreign IDs across records
1087 my ( %foreign_ids, %rdbm_recs );
1088 if ( %rdbm_blocks ) {
1089 for my $rec (@records) {
1090 for my $real_blk ( keys %rdbm_blocks ) {
1091 my $val = $rec->[$real_blk];
1092 next unless defined $val && $val ne '';
1093 my $target_table = $rdbm_blocks{$real_blk}->{table};
1094 for my $id ( split /[,;]/, $val ) {
1095 $id =~ s/^\s+|\s+$//g;
1096 $foreign_ids{$target_table}->{$id} = 1 if $id =~ /^\d+$/;
1097 }
1098 }
1099 }
1100 for my $target_table ( keys %foreign_ids ) {
1101 my @ids = keys %{ $foreign_ids{$target_table} };
1102 next unless @ids;
1103 my @target_records = $adb->read_list( $target_table, \@ids );
1104 $rdbm_recs{$target_table} = { map { $_->[0] => $_ } @target_records };
1105 }
1106 }
1107  
1108 # 3. Extract sentences across all records
1109 my @all_sentences;
1110 for my $rec (@records) {
1111 push @all_sentences, $adb->_extract_suggest_sentences( $table_info, $rec, \%rdbm_recs, \%rdbm_blocks );
1112 }
1113 return unless @all_sentences;
1114  
1115 my ( $word_freq, $transitions ) = $adb->_extract_words_and_transitions( @all_sentences );
1116  
1117 # 4. Build .ajw (Word prefix index)
1118 my %ajw_buckets;
1119 for my $norm ( keys %$word_freq ) {
1120 my $disp = $word_freq->{$norm}{disp};
1121 my $cnt = $word_freq->{$norm}{cnt};
1122 my $max_len = length($norm) < 12 ? length($norm) : 12;
1123 for ( my $len = 2; $len <= $max_len; $len++ ) {
1124 my $pfx = substr($norm, 0, $len);
1125 $ajw_buckets{$pfx}{$norm} = [ $disp, $cnt ];
1126 }
1127 my $ascii_norm = $adb->to_ascii($norm);
1128 if ( defined $ascii_norm && $ascii_norm ne $norm ) {
1129 my $a_len = length($ascii_norm) < 12 ? length($ascii_norm) : 12;
1130 for ( my $len = 2; $len <= $a_len; $len++ ) {
1131 my $a_pfx = substr($ascii_norm, 0, $len);
1132 $ajw_buckets{$a_pfx}{$norm} = [ $disp, $cnt ];
1133 }
1134 }
1135 }
1136  
1137 my @ajw_records;
1138 for my $pfx ( keys %ajw_buckets ) {
1139 my $bucket = $ajw_buckets{$pfx};
1140 my @sorted = sort {
1141 $bucket->{$b}->[1] <=> $bucket->{$a}->[1]
1142 || $a cmp $b
1143 } keys %$bucket;
1144 @sorted = @sorted[0 .. 9] if @sorted > 10;
1145 my @payload = map { $bucket->{$_} } @sorted;
1146 push @ajw_records, [ $pfx, $adb->db_encode(\@payload) ];
1147 }
1148  
1149 if (@ajw_records) {
1150 $adb->table_write($tmp_ajw) or return 0;
1151 $adb->recs_put( $tmp_ajw, @ajw_records );
1152 $adb->table_close($tmp_ajw);
1153 $adb->table_close($ajw_path);
1154 unlink $ajw_path;
1155 rename( $tmp_ajw, $ajw_path ) or do {
1156 require File::Copy;
1157 File::Copy::move( $tmp_ajw, $ajw_path );
1158 };
1159 $self->{say} .= " * $ajw_path \n";
1160 }
1161  
1162 # 5. Build .ajn (Next word transitions)
1163 my @ajn_records;
1164 for my $norm1 ( keys %$transitions ) {
1165 my $targets = $transitions->{$norm1};
1166 my @sorted = sort {
1167 $targets->{$b}->{cnt} <=> $targets->{$a}->{cnt}
1168 || $a cmp $b
1169 } keys %$targets;
1170 @sorted = @sorted[0 .. 299] if @sorted > 300;
1171 my @payload = map {
1172 [ $targets->{$_}->{disp}, $targets->{$_}->{cnt}, $_ ]
1173 } @sorted;
1174 push @ajn_records, [ $norm1, $adb->db_encode(\@payload) ];
1175 }
1176  
1177 if (@ajn_records) {
1178 $adb->table_write($tmp_ajn) or return 0;
1179 $adb->recs_put( $tmp_ajn, @ajn_records );
1180 $adb->table_close($tmp_ajn);
1181 $adb->table_close($ajn_path);
1182 unlink $ajn_path;
1183 rename( $tmp_ajn, $ajn_path ) or do {
1184 require File::Copy;
1185 File::Copy::move( $tmp_ajn, $ajn_path );
1186 };
1187 $self->{say} .= " * $ajn_path \n";
1188 }
1189  
1190 return 1;
1191 }
1192  
1193 sub reindex_suggest {
1194 my ( $self, $tableid ) = @_;
1195 my $adb = $self->{_adb};
1196 my @records = $adb->read_all($tableid);
1197 return $self->set_suggest( $tableid, @records );
1198 }
1199  
1200 1;