| 1 | package AmberDB::Tools::Index; | ||
| 2 | |||
| 3 | use 5.016; | ||
| 4 | use warnings; | ||
| 5 | use strict; | ||
| 6 | use Carp qw(croak cluck); | ||
| 7 | use File::Spec; | ||
| 8 | |||
| 9 | our $VERSION = '5.26.6'; | ||
| 10 | my $CREATED = '2018-10-08'; | ||
| 11 | |||
| 12 | sub new { | ||
| 13 | |||
| 14 | my $class = shift; | ||
| 15 | my $self = {}; | ||
| 16 | |||
| 17 | require AmberDB; | ||
| 18 | |||
| 19 | my ( $adb, %inputs ); | ||
| 20 | |||
| 21 | if ( ref( $_[0] ) ) { | ||
| 22 | $adb = shift; | ||
| 23 | %inputs = @_; | ||
| 24 | } | ||
| 25 | else { | ||
| 26 | %inputs = @_; | ||
| 27 | $adb = AmberDB->new(%inputs); | ||
| 28 | } | ||
| 29 | |||
| 30 | $self->{_adb} = $adb; | ||
| 31 | |||
| 32 | foreach my $in ( keys %inputs ) { | ||
| 33 | $self->{ uc($in) } = $inputs{$in}; | ||
| 34 | } | ||
| 35 | $self->{say} = ""; | ||
| 36 | |||
| 37 | bless $self, $class; | ||
| 38 | return $self; | ||
| 39 | } | ||
| 40 | |||
| 41 | # my $ok = $tools->set_index($tableid, @records); | ||
| 42 | # ------------------------------------------------ | ||
| 43 | sub set_index { | ||
| 44 | |||
| 45 | my ( $self, $tableid, @records ) = @_; | ||
| 46 | my $adb = $self->{_adb} or return; | ||
| 47 | |||
| 48 | my $table_info = $adb->table_info($tableid) or return; | ||
| 49 | if ( $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' ) ) { | ||
| 50 | $self->{say} .= " - Table '$tableid' is in simple mode, skipping index generation.\n"; | ||
| 51 | return 1; | ||
| 52 | } | ||
| 53 | my $table_path = $adb->table_path($tableid); | ||
| 54 | return unless ( -e "$table_path.$adb->{db_ext}" ); | ||
| 55 | |||
| 56 | # 1. Clean up previous index files on physical storage and RAM-disk before rebuilding | ||
| 57 | my @idx_exts = qw( inx src fld fac slg unq ajw ajn ); | ||
| 58 | for my $ext (@idx_exts) { | ||
| 59 | my $p_file = "$table_path.$ext"; | ||
| 60 | if ( -e $p_file ) { | ||
| 61 | $adb->table_close($p_file); | ||
| 62 | unlink $p_file; | ||
| 63 | } | ||
| 64 | } | ||
| 65 | if ( $adb->ramdisk_is_mounted() ) { | ||
| 66 | my $rm_path = $adb->ramdisk_path($tableid); | ||
| 67 | if ($rm_path) { | ||
| 68 | for my $ext (@idx_exts) { | ||
| 69 | my $rm_file = "$rm_path.$ext"; | ||
| 70 | if ( -e $rm_file ) { | ||
| 71 | $adb->table_close($rm_file); | ||
| 72 | unlink $rm_file; | ||
| 73 | } | ||
| 74 | } | ||
| 75 | } | ||
| 76 | } | ||
| 77 | $adb->clear_cache($tableid); | ||
| 78 | |||
| 79 | # 2. Read records if list empty. | ||
| 80 | if ( !@records ) { | ||
| 81 | @records = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc'); | ||
| 82 | } | ||
| 83 | return unless scalar @records; | ||
| 84 | |||
| 85 | # Pre-fetch foreign records for junk_rules in one batch before sub-indexers run | ||
| 86 | if ( $table_info->{use_junk} ) { | ||
| 87 | $self->{say} .= " - Pre-fetching relational foreign records for junk rules...\n"; | ||
| 88 | $adb->prefetch_junk_rdbm( $table_info, \@records ); | ||
| 89 | } | ||
| 90 | |||
| 91 | # 3. Create readall index | ||
| 92 | if ( exists( $table_info->{record_index} ) ) { | ||
| 93 | $self->{say} .= " - Rebuilding readall index (.inx)...\n"; | ||
| 94 | my $ok = $self->set_readall( $tableid, @records ); | ||
| 95 | } | ||
| 96 | |||
| 97 | # 4. Create search index | ||
| 98 | if ( exists( $table_info->{search_block} ) ) { | ||
| 99 | $self->{say} .= " - Rebuilding search index (.src)...\n"; | ||
| 100 | my $ok = $self->set_search( $tableid, @records ); | ||
| 101 | } | ||
| 102 | |||
| 103 | # 5. Create fetch field index. | ||
| 104 | if ( exists( $table_info->{match_block} ) ) { | ||
| 105 | $self->{say} .= " - Rebuilding field index (.fld)...\n"; | ||
| 106 | my $ok = $self->set_fields( $tableid, @records ); | ||
| 107 | } | ||
| 108 | |||
| 109 | # 6. Create facet index. | ||
| 110 | if ( exists( $table_info->{use_facet} ) ) { | ||
| 111 | $self->{say} .= " - Rebuilding facet index (.fac)...\n"; | ||
| 112 | my $ok = $self->set_filters( $tableid, @records ); | ||
| 113 | } | ||
| 114 | |||
| 115 | # 7. Create sort index. | ||
| 116 | if ( exists( $table_info->{sort_block} ) ) { | ||
| 117 | $self->{say} .= " - Rebuilding sort index (.inx)...\n"; | ||
| 118 | my $ok = $self->set_sort( $tableid, @records ); | ||
| 119 | } | ||
| 120 | |||
| 121 | # 8. Create URL slug index | ||
| 122 | if ( exists( $table_info->{slug_block} ) ) { | ||
| 123 | $self->{say} .= " - Rebuilding slug index (.slg)...\n"; | ||
| 124 | my $ok = $self->set_rwlnkall( $tableid, @records ); | ||
| 125 | } | ||
| 126 | |||
| 127 | # 9. Create unique index if schema defines unique blocks | ||
| 128 | my $has_unique = 0; | ||
| 129 | if ( $table_info->{blocks} && ref($table_info->{blocks}) eq 'ARRAY' ) { | ||
| 130 | for my $b (@{ $table_info->{blocks} }) { | ||
| 131 | if ( ref($b) eq 'HASH' && defined $b->{valid} && $b->{valid} =~ /unique/i ) { | ||
| 132 | $has_unique = 1; | ||
| 133 | last; | ||
| 134 | } | ||
| 135 | } | ||
| 136 | } | ||
| 137 | if ($has_unique) { | ||
| 138 | $adb->unique_add( $table_path, $table_info, \@records ); | ||
| 139 | } | ||
| 140 | |||
| 141 | # 10. Create suggest index (.ajw, .ajn) | ||
| 142 | if ( exists( $table_info->{suggest_block} ) ) { | ||
| 143 | $self->{say} .= " - Rebuilding suggest index (.ajw, .ajn)...\n"; | ||
| 144 | my $ok = $self->set_suggest( $tableid, @records ); | ||
| 145 | } | ||
| 146 | |||
| 147 | # 10. If RAM-disk is active, sync freshly built index files to RAM-disk | ||
| 148 | if ( $adb->ramdisk_is_mounted() ) { | ||
| 149 | my $rm_path = $adb->ramdisk_path($tableid); | ||
| 150 | if ($rm_path) { | ||
| 151 | require File::Copy; | ||
| 152 | for my $ext (@idx_exts) { | ||
| 153 | my $src_file = "$table_path.$ext"; | ||
| 154 | my $dst_file = "$rm_path.$ext"; | ||
| 155 | if ( -e $src_file ) { | ||
| 156 | $adb->table_close($src_file); | ||
| 157 | $adb->table_close($dst_file); | ||
| 158 | unlink $dst_file if -e $dst_file; | ||
| 159 | File::Copy::copy( $src_file, $dst_file ); | ||
| 160 | chmod 0777, $dst_file if -e $dst_file; | ||
| 161 | } | ||
| 162 | } | ||
| 163 | } | ||
| 164 | } | ||
| 165 | |||
| 166 | return 1; | ||
| 167 | } | ||
| 168 | |||
| 169 | # Rebuilds Read All index. | ||
| 170 | # my $ok = $tools->set_readall($tableid); | ||
| 171 | # my $ok = $tools->set_readall($tableid, @records); | ||
| 172 | # ------------------------------------------------ | ||
| 173 | sub set_readall { | ||
| 174 | |||
| 175 | my ( $self, $tableid, @records ) = @_; | ||
| 176 | my $adb = $self->{_adb} or return; | ||
| 177 | |||
| 178 | $tableid or return; | ||
| 179 | my $table_path = $adb->table_path($tableid); | ||
| 180 | |||
| 181 | my $file_path = "$table_path.$adb->{db_ext}"; | ||
| 182 | my $index_path = "$table_path.inx"; | ||
| 183 | my $tmp_path = "$index_path.tmp"; | ||
| 184 | return unless -e $file_path; | ||
| 185 | |||
| 186 | my $table_info = $adb->table_info($tableid); | ||
| 187 | return unless $table_info; | ||
| 188 | return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' ); | ||
| 189 | |||
| 190 | # Read records from table if absent | ||
| 191 | if ( !scalar @records ) { | ||
| 192 | @records = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc'); | ||
| 193 | } | ||
| 194 | scalar @records or return; | ||
| 195 | my @all_records = ref( $records[0] ) eq 'ARRAY' ? ( map { $_->[0] } @records ) : @records; | ||
| 196 | @all_records = grep { defined $_ && $_ > 0 } @all_records; | ||
| 197 | |||
| 198 | my ( @active_records, @junk_records ); | ||
| 199 | my $has_junk = $table_info->{use_junk} ? 1 : 0; | ||
| 200 | |||
| 201 | # Accept both full record arrayrefs [ $id, @data ] and scalar ID lists | ||
| 202 | if ( ref $records[0] eq "ARRAY" ) { | ||
| 203 | if ($has_junk) { | ||
| 204 | my $rdbm_recs = $adb->prefetch_junk_rdbm( $table_info, \@records ); | ||
| 205 | for my $rec (@records) { | ||
| 206 | my $rid = $rec->[0]; | ||
| 207 | next unless defined $rid && $rid > 0; | ||
| 208 | if ( $adb->junk_rules($table_info, $rec, $rdbm_recs) ) { | ||
| 209 | push @junk_records, $rid; | ||
| 210 | } | ||
| 211 | else { | ||
| 212 | push @active_records, $rid; | ||
| 213 | } | ||
| 214 | } | ||
| 215 | } | ||
| 216 | else { | ||
| 217 | for my $rec (@records) { | ||
| 218 | my $rid = $rec->[0]; | ||
| 219 | push @active_records, $rid if defined $rid && $rid > 0; | ||
| 220 | } | ||
| 221 | } | ||
| 222 | } | ||
| 223 | else { | ||
| 224 | my @clean_rids = grep { defined $_ && $_ > 0 } @records; | ||
| 225 | if ($has_junk) { | ||
| 226 | for my $rid (@clean_rids) { | ||
| 227 | my @rec = $adb->table_readid($file_path, $rid); | ||
| 228 | if ( $adb->junk_rules($table_info, @rec) ) { | ||
| 229 | push @junk_records, $rid; | ||
| 230 | } | ||
| 231 | else { | ||
| 232 | push @active_records, $rid; | ||
| 233 | } | ||
| 234 | } | ||
| 235 | } | ||
| 236 | else { | ||
| 237 | @active_records = @clean_rids; | ||
| 238 | } | ||
| 239 | } | ||
| 240 | |||
| 241 | # Deduplicate in original order | ||
| 242 | my %seen_act; | ||
| 243 | @active_records = grep { defined $_ && $_ > 0 && !$seen_act{$_}++ } @active_records; | ||
| 244 | my %seen_junk; | ||
| 245 | @junk_records = grep { defined $_ && $_ > 0 && !$seen_junk{$_}++ } @junk_records; | ||
| 246 | |||
| 247 | |||
| 248 | my $cur_max = 0; | ||
| 249 | if ( @all_records && $all_records[-1] > $cur_max ) { | ||
| 250 | $cur_max = $all_records[-1]; | ||
| 251 | } | ||
| 252 | |||
| 253 | my $del_path = "$table_path.del"; | ||
| 254 | if ( -e $del_path && $adb->table_read($del_path) ) { | ||
| 255 | $adb->recs_scan( $del_path, sub { | ||
| 256 | my ($k) = @_; | ||
| 257 | $cur_max = $k if $k > $cur_max; | ||
| 258 | } ); | ||
| 259 | $adb->table_close($del_path); | ||
| 260 | } | ||
| 261 | |||
| 262 | my $old_lastid = $adb->table_lastid($tableid) // 0; | ||
| 263 | my $last_id = $cur_max > $old_lastid ? $cur_max : $old_lastid; | ||
| 264 | |||
| 265 | if ( $adb->table_write($tmp_path) ) { | ||
| 266 | my $cnt = scalar @all_records; | ||
| 267 | $adb->index_put( $tmp_path, "keys", \@all_records, "ids" ); | ||
| 268 | $adb->index_put( $tmp_path, "count", $cnt, "raw" ); | ||
| 269 | $adb->index_put( $tmp_path, "lastid", $last_id, "raw" ) if defined $last_id; | ||
| 270 | |||
| 271 | if ($has_junk) { | ||
| 272 | $adb->index_put( $tmp_path, "A:keys", \@active_records, "ids" ); | ||
| 273 | $adb->index_put( $tmp_path, "A:count", scalar @active_records, "raw" ); | ||
| 274 | $adb->index_put( $tmp_path, "B:keys", \@junk_records, "ids" ); | ||
| 275 | $adb->index_put( $tmp_path, "B:count", scalar @junk_records, "raw" ); | ||
| 276 | } | ||
| 277 | |||
| 278 | $adb->table_close($tmp_path); | ||
| 279 | $adb->table_close($index_path); | ||
| 280 | unlink($index_path); | ||
| 281 | rename( $tmp_path, $index_path ) or do { | ||
| 282 | require File::Copy; | ||
| 283 | File::Copy::move( $tmp_path, $index_path ); | ||
| 284 | }; | ||
| 285 | $self->{say} .= " - Readall records created: \n"; | ||
| 286 | $self->{say} .= " * $table_path.inx created ($cnt total records, " . scalar(@active_records) . " active, " . scalar(@junk_records) . " junk)\n"; | ||
| 287 | } | ||
| 288 | |||
| 289 | return 1; | ||
| 290 | } | ||
| 291 | |||
| 292 | # Rebuilds search word index. | ||
| 293 | # my $ok = $tools->set_search($tableid, @records); | ||
| 294 | # ------------------------------------------------ | ||
| 295 | sub set_search { | ||
| 296 | |||
| 297 | my ( $self, $tableid, @records ) = @_; | ||
| 298 | my $adb = $self->{_adb} or return; | ||
| 299 | |||
| 300 | return unless $tableid; | ||
| 301 | my $table_path = $adb->table_path($tableid); | ||
| 302 | my $table_info = $adb->table_info($tableid); | ||
| 303 | return unless $table_info; | ||
| 304 | return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' ); | ||
| 305 | return unless exists( $table_info->{search_block} ); | ||
| 306 | |||
| 307 | # Read records from table if absent | ||
| 308 | if ( !scalar @records ) { | ||
| 309 | @records = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc'); | ||
| 310 | } | ||
| 311 | return unless scalar @records; | ||
| 312 | |||
| 313 | # 1. Identify RDBM blocks and prepare pre-fetch | ||
| 314 | my %rdbm_blocks; # field => { table => ..., display => ... } | ||
| 315 | for my $blk ( @{ $table_info->{search_block} } ) { | ||
| 316 | my ( $field, $src_table, $src_display ) = | ||
| 317 | ref($blk) eq 'ARRAY' ? ( $blk->[0], $blk->[1], $blk->[2] ) | ||
| 318 | : ( $blk, undef, undef ); | ||
| 319 | if ( !$src_table ) { | ||
| 320 | ( $src_table, $src_display ) = $adb->rdbm_target( $table_info, $field ); | ||
| 321 | } | ||
| 322 | if ( $src_table ) { | ||
| 323 | $src_display //= 1; | ||
| 324 | $rdbm_blocks{$field} = { table => $src_table, display => $src_display }; | ||
| 325 | } | ||
| 326 | } | ||
| 327 | |||
| 328 | # 2. Collect unique foreign IDs across all records | ||
| 329 | my %string; | ||
| 330 | if ( %rdbm_blocks ) { | ||
| 331 | for my $rec (@records) { | ||
| 332 | for my $field ( keys %rdbm_blocks ) { | ||
| 333 | my $val = $rec->[$field]; | ||
| 334 | next unless defined $val && $val ne ''; | ||
| 335 | my $target_table = $rdbm_blocks{$field}->{table}; | ||
| 336 | for my $id ( split /[,;]/, $val ) { | ||
| 337 | $id =~ s/^\s+|\s+$//g; | ||
| 338 | $string{$target_table}->{$id} = 1 if $id =~ /^\d+$/; | ||
| 339 | } | ||
| 340 | } | ||
| 341 | } | ||
| 342 | } | ||
| 343 | |||
| 344 | # 3. Batch pre-fetch foreign tables into in-memory lookup map and official cache | ||
| 345 | my %rdbm_recs; # table => { id => \@target_record } | ||
| 346 | for my $table ( keys %string ) { | ||
| 347 | my @needed_ids = grep { !$adb->get_cache( $table, $_ ) } keys %{ $string{$table} }; | ||
| 348 | if (@needed_ids) { | ||
| 349 | my @target_recs = $adb->read_list( $table, \@needed_ids ); | ||
| 350 | for my $trec (@target_recs) { | ||
| 351 | $adb->set_cache( $table, $trec->[0], $trec ); | ||
| 352 | } | ||
| 353 | } | ||
| 354 | $rdbm_recs{$table} = $adb->get_cache($table) || {}; | ||
| 355 | } | ||
| 356 | |||
| 357 | # 4. Tokenize and index words | ||
| 358 | my ( %search, %act_search, %junk_search ); | ||
| 359 | $self->{say} .= " - Search word kayitlari olusturuluyor: \n"; | ||
| 360 | my $has_junk = $table_info->{use_junk} ? 1 : 0; | ||
| 361 | my $junk_rdbm = $has_junk ? $adb->prefetch_junk_rdbm( $table_info, \@records ) : {}; | ||
| 362 | foreach my $line (@records) { | ||
| 363 | my @fields = @$line; | ||
| 364 | my $rid = $fields[0]; | ||
| 365 | next unless defined $rid && $rid > 0; | ||
| 366 | my $is_junk = $has_junk ? $adb->junk_rules( $table_info, \@fields, $junk_rdbm ) : 0; | ||
| 367 | foreach my $blk ( @{ $table_info->{search_block} } ) { | ||
| 368 | my $b_idx = ref($blk) eq "ARRAY" ? $blk->[0] : $blk; | ||
| 369 | $b_idx =~ /^\d+$/ or next; | ||
| 370 | |||
| 371 | my $val = $fields[$b_idx]; | ||
| 372 | if ( my $rdbm_info = $rdbm_blocks{$b_idx} ) { | ||
| 373 | if ( defined $val && $val ne '' ) { | ||
| 374 | my $table = $rdbm_info->{table}; | ||
| 375 | my $disp = $rdbm_info->{display}; | ||
| 376 | my @parts; | ||
| 377 | for my $id ( split /[,;]/, $val ) { | ||
| 378 | $id =~ s/^\s+|\s+$//g; | ||
| 379 | if ( my $rec = $rdbm_recs{$table}->{$id} ) { | ||
| 380 | my $name = $rec->[$disp]; | ||
| 381 | push @parts, $name if defined $name && length($name); | ||
| 382 | } | ||
| 383 | } | ||
| 384 | $val = join( ' ', @parts ); | ||
| 385 | } | ||
| 386 | } | ||
| 387 | |||
| 388 | next unless defined $val && $val ne ''; | ||
| 389 | my %recsearch = $adb->get_words( $val, "write", $tableid ); | ||
| 390 | |||
| 391 | foreach my $key ( keys %recsearch ) { | ||
| 392 | push @{ $search{$b_idx}{$key} }, $rid; | ||
| 393 | if ($has_junk) { | ||
| 394 | if ($is_junk) { | ||
| 395 | push @{ $junk_search{$b_idx}{$key} }, $rid; | ||
| 396 | } | ||
| 397 | else { | ||
| 398 | push @{ $act_search{$b_idx}{$key} }, $rid; | ||
| 399 | } | ||
| 400 | } | ||
| 401 | } | ||
| 402 | } | ||
| 403 | } | ||
| 404 | |||
| 405 | my $file_path = "${table_path}.src"; | ||
| 406 | my $tmp_path = "$file_path.tmp"; | ||
| 407 | |||
| 408 | unlink($tmp_path); | ||
| 409 | |||
| 410 | my %batch_put; | ||
| 411 | foreach my $blk ( @{ $table_info->{search_block} } ) { | ||
| 412 | my $b_idx = ref($blk) eq "ARRAY" ? $blk->[0] : $blk; | ||
| 413 | $b_idx =~ /^\d+$/ or next; | ||
| 414 | |||
| 415 | next unless $search{$b_idx}; | ||
| 416 | for my $key ( keys %{ $search{$b_idx} } ) { | ||
| 417 | my $all_ids = $search{$b_idx}{$key}; | ||
| 418 | $batch_put{"$b_idx:$key"} = $all_ids if @$all_ids; | ||
| 419 | |||
| 420 | if ($has_junk) { | ||
| 421 | if ( my $act_recs = $act_search{$b_idx}{$key} ) { | ||
| 422 | $batch_put{"A:$b_idx:$key"} = $act_recs if @$act_recs; | ||
| 423 | } | ||
| 424 | if ( my $junk_recs = $junk_search{$b_idx}{$key} ) { | ||
| 425 | $batch_put{"B:$b_idx:$key"} = $junk_recs if @$junk_recs; | ||
| 426 | } | ||
| 427 | } | ||
| 428 | } | ||
| 429 | } | ||
| 430 | |||
| 431 | if ( %batch_put ) { | ||
| 432 | $adb->table_write($tmp_path) | ||
| 433 | or do { | ||
| 434 | cluck "[DB_TOOL] $tmp_path can't open.\n"; | ||
| 435 | return; | ||
| 436 | }; | ||
| 437 | $adb->index_put( $tmp_path, \%batch_put, "ids" ); | ||
| 438 | $adb->table_close($tmp_path); | ||
| 439 | $adb->table_close($file_path); | ||
| 440 | unlink($file_path); | ||
| 441 | rename( $tmp_path, $file_path ) or do { | ||
| 442 | require File::Copy; | ||
| 443 | File::Copy::move( $tmp_path, $file_path ); | ||
| 444 | }; | ||
| 445 | $self->{say} .= " * $file_path created.\n"; | ||
| 446 | } | ||
| 447 | else { | ||
| 448 | $adb->table_close($file_path); | ||
| 449 | unlink($file_path); | ||
| 450 | } | ||
| 451 | |||
| 452 | return 1; | ||
| 453 | } | ||
| 454 | |||
| 455 | # Rebuilds block index. | ||
| 456 | # my $ok = $tools->set_fields("tableid", @records); | ||
| 457 | # ------------------------------------------------ | ||
| 458 | sub set_fields { | ||
| 459 | |||
| 460 | my ( $self, $tableid, @records ) = @_; | ||
| 461 | my $adb = $self->{_adb} or return; | ||
| 462 | |||
| 463 | my $table_path = $adb->table_path($tableid); | ||
| 464 | my $table_info = $adb->table_info($tableid); | ||
| 465 | return unless $table_info; | ||
| 466 | return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' ); | ||
| 467 | return unless exists( $table_info->{match_block} ); | ||
| 468 | return unless -e "$table_path.$adb->{db_ext}"; | ||
| 469 | |||
| 470 | # Read records from table if absent | ||
| 471 | if ( !scalar @records ) { | ||
| 472 | @records = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc'); | ||
| 473 | } | ||
| 474 | return unless scalar @records; | ||
| 475 | |||
| 476 | my ( %fields, %act_fields, %junk_fields ); | ||
| 477 | |||
| 478 | my $unq_path = "${table_path}.unq"; | ||
| 479 | my $unq_opened = 0; | ||
| 480 | if ( !$adb->{_db}->{$unq_path} ) { | ||
| 481 | $adb->table_write($unq_path); | ||
| 482 | $unq_opened = 1; | ||
| 483 | } | ||
| 484 | |||
| 485 | # Invert loops: outer loop @records (single pass), inner loop match_block | ||
| 486 | my @match_blks = grep { /^\d+$/ } @{ $table_info->{match_block} }; | ||
| 487 | my $has_junk = $table_info->{use_junk} ? 1 : 0; | ||
| 488 | my $rdbm_recs = $has_junk ? $adb->prefetch_junk_rdbm( $table_info, \@records ) : {}; | ||
| 489 | |||
| 490 | my $table_name = $table_info->{table} || $table_info->{id} || $tableid; | ||
| 491 | |||
| 492 | # Preload unq into in-memory cache to eliminate per-record BDB disk lookups | ||
| 493 | if ( $adb->{_db}->{$unq_path} ) { | ||
| 494 | $adb->recs_scan( | ||
| 495 | $unq_path, | ||
| 496 | sub { | ||
| 497 | my ( $k, $v ) = @_; | ||
| 498 | if ( $k =~ /^(\d+):s:(.*)$/s ) { | ||
| 499 | $adb->set_cache( $table_name, "$1:$2", $v ); | ||
| 500 | } | ||
| 501 | elsif ( $k =~ /^(\d+):lastid$/ ) { | ||
| 502 | $adb->set_cache( $table_name, "$1:lastid", $v ); | ||
| 503 | } | ||
| 504 | } | ||
| 505 | ); | ||
| 506 | } | ||
| 507 | |||
| 508 | my $total_rec = scalar @records; | ||
| 509 | my $prog_step = int($total_rec / 10) || 1000; | ||
| 510 | my $count = 0; | ||
| 511 | |||
| 512 | foreach my $record (@records) { | ||
| 513 | $count++; | ||
| 514 | if ( $total_rec > 1000 && $count % $prog_step == 0 ) { | ||
| 515 | $self->{say} .= " ... processed $count / $total_rec records for field index\n"; | ||
| 516 | } | ||
| 517 | my @fields_arr = @$record; | ||
| 518 | my $rid = $fields_arr[0]; | ||
| 519 | next unless defined $rid && $rid > 0; | ||
| 520 | my $is_junk = $has_junk ? $adb->junk_rules( $table_info, \@fields_arr, $rdbm_recs ) : 0; | ||
| 521 | |||
| 522 | foreach my $line (@match_blks) { | ||
| 523 | my $val = $fields_arr[$line]; | ||
| 524 | next unless defined $val && $val ne ''; | ||
| 525 | |||
| 526 | my @num_ids; | ||
| 527 | if ( !ref($val) && $val =~ /^\d+$/ && length($val) < 20 ) { | ||
| 528 | @num_ids = ($val); | ||
| 529 | } | ||
| 530 | elsif ( !ref($val) && $val =~ /^[\d,;\s]+$/ ) { | ||
| 531 | ( my $clean = $val ) =~ s/\s+//g; | ||
| 532 | @num_ids = grep { /^\d+$/ } split /[,;]/, $clean; | ||
| 533 | } | ||
| 534 | else { | ||
| 535 | @num_ids = $adb->field_to_list( $val, $table_path, $table_info, $line ); | ||
| 536 | } | ||
| 537 | |||
| 538 | my %seen_nid; | ||
| 539 | foreach my $nid (@num_ids) { | ||
| 540 | next if $seen_nid{$nid}++; | ||
| 541 | push @{ $fields{$line}{$nid} }, $rid; | ||
| 542 | if ($has_junk) { | ||
| 543 | if ($is_junk) { | ||
| 544 | push @{ $junk_fields{$line}{$nid} }, $rid; | ||
| 545 | } | ||
| 546 | else { | ||
| 547 | push @{ $act_fields{$line}{$nid} }, $rid; | ||
| 548 | } | ||
| 549 | } | ||
| 550 | } | ||
| 551 | } | ||
| 552 | } | ||
| 553 | |||
| 554 | if ($unq_opened) { | ||
| 555 | $adb->table_close($unq_path); | ||
| 556 | } | ||
| 557 | |||
| 558 | $self->{say} .= " - Fields fetch kayitlari olusturuluyor: \n"; | ||
| 559 | |||
| 560 | my $file_path = "${table_path}.fld"; | ||
| 561 | my $tmp_path = "$file_path.tmp"; | ||
| 562 | |||
| 563 | unlink($tmp_path); | ||
| 564 | |||
| 565 | my %batch_put; | ||
| 566 | foreach my $line (@match_blks) { | ||
| 567 | next unless $fields{$line}; | ||
| 568 | for my $val ( keys %{ $fields{$line} } ) { | ||
| 569 | my $all_ids = $fields{$line}{$val}; | ||
| 570 | $batch_put{"$line:$val"} = $all_ids if @$all_ids; | ||
| 571 | |||
| 572 | if ($has_junk) { | ||
| 573 | if ( my $act_recs = $act_fields{$line}{$val} ) { | ||
| 574 | $batch_put{"A:$line:$val"} = $act_recs if @$act_recs; | ||
| 575 | } | ||
| 576 | if ( my $junk_recs = $junk_fields{$line}{$val} ) { | ||
| 577 | $batch_put{"B:$line:$val"} = $junk_recs if @$junk_recs; | ||
| 578 | } | ||
| 579 | } | ||
| 580 | } | ||
| 581 | } | ||
| 582 | |||
| 583 | if ( %batch_put ) { | ||
| 584 | $adb->table_write($tmp_path) | ||
| 585 | or do { | ||
| 586 | cluck "[DB_TOOL] $tmp_path can't open for write.\n"; | ||
| 587 | return; | ||
| 588 | }; | ||
| 589 | $adb->index_put( $tmp_path, \%batch_put, "ids" ); | ||
| 590 | $adb->table_close($tmp_path); | ||
| 591 | $adb->table_close($file_path); | ||
| 592 | unlink($file_path); | ||
| 593 | rename( $tmp_path, $file_path ) or do { | ||
| 594 | require File::Copy; | ||
| 595 | File::Copy::move( $tmp_path, $file_path ); | ||
| 596 | }; | ||
| 597 | |||
| 598 | $self->{say} .= " * $file_path \n"; | ||
| 599 | } | ||
| 600 | else { | ||
| 601 | $adb->table_close($file_path); | ||
| 602 | unlink($file_path); | ||
| 603 | } | ||
| 604 | |||
| 605 | return 1; | ||
| 606 | } | ||
| 607 | |||
| 608 | # Rebuilds facet forward index. | ||
| 609 | # Includes active_flag and "active" key. | ||
| 610 | # my $ok = $tools->set_filters("tableid", @records); | ||
| 611 | # ------------------------------------------------ | ||
| 612 | sub set_filters { | ||
| 613 | |||
| 614 | my ( $self, $tableid, @records ) = @_; | ||
| 615 | my $adb = $self->{_adb} or return; | ||
| 616 | |||
| 617 | return unless $tableid; | ||
| 618 | my $table_path = $adb->table_path($tableid); | ||
| 619 | return unless -e "$table_path.$adb->{db_ext}"; | ||
| 620 | |||
| 621 | my $table_info = $adb->table_info($tableid); | ||
| 622 | return unless $table_info; | ||
| 623 | return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' ); | ||
| 624 | return unless exists( $table_info->{match_block} ); | ||
| 625 | return unless exists( $table_info->{use_facet} ); | ||
| 626 | |||
| 627 | my @fblocks = @{ $table_info->{match_block} }; | ||
| 628 | my $has_active_rule = exists $table_info->{facet_rules}; | ||
| 629 | my $fac_path = "$table_path.fac"; | ||
| 630 | my $tmp_path = "$fac_path.tmp"; | ||
| 631 | |||
| 632 | # Read records from table if absent | ||
| 633 | if ( !scalar @records ) { | ||
| 634 | @records = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc'); | ||
| 635 | } | ||
| 636 | scalar @records or return; | ||
| 637 | |||
| 638 | # Columnar facet architecture with dictionary (.unq) support | ||
| 639 | if ( exists $table_info->{facet_block} && ref($table_info->{facet_block}) eq 'ARRAY' && @{ $table_info->{facet_block} } ) { | ||
| 640 | $self->{say} .= " - Columnar facet index (.fac, .unq) olusturuluyor: \n"; | ||
| 641 | unlink $fac_path if -e $fac_path; | ||
| 642 | unlink "$table_path.unq" if -e "$table_path.unq"; | ||
| 643 | |||
| 644 | my $batch_size = 5000; | ||
| 645 | for ( my $i = 0; $i < @records; $i += $batch_size ) { | ||
| 646 | my $end = $i + $batch_size - 1; | ||
| 647 | $end = $#records if $end > $#records; | ||
| 648 | my @batch = @records[ $i .. $end ]; | ||
| 649 | $adb->facet_add( $table_path, $table_info, \@batch ); | ||
| 650 | } | ||
| 651 | $self->{say} .= " * $fac_path \n"; | ||
| 652 | $self->{say} .= " * $table_path.unq \n" if -e "$table_path.unq"; | ||
| 653 | return 1; | ||
| 654 | } | ||
| 655 | |||
| 656 | unlink($tmp_path); | ||
| 657 | |||
| 658 | $adb->table_write($tmp_path) | ||
| 659 | or do { | ||
| 660 | cluck "[DB_TOOL] $tmp_path can't open for write.\n"; | ||
| 661 | return; | ||
| 662 | }; | ||
| 663 | |||
| 664 | $self->{say} .= " - Facet forward index olusturuluyor: \n"; | ||
| 665 | |||
| 666 | my @active_ids; | ||
| 667 | my %batch_facets; | ||
| 668 | |||
| 669 | foreach my $record (@records) { | ||
| 670 | my @fields = @$record; | ||
| 671 | $fields[0] or next; | ||
| 672 | my @pairs; | ||
| 673 | foreach my $blk (@fblocks) { | ||
| 674 | $blk =~ /^\d+$/ or next; | ||
| 675 | next unless defined $fields[$blk] && $fields[$blk] ne ''; | ||
| 676 | my @vals; | ||
| 677 | if ( ref $fields[$blk] eq "ARRAY" ) { @vals = @{ $fields[$blk] } } | ||
| 678 | elsif ( $fields[$blk] =~ /[,;]/ ) { @vals = split /\s*[,;]\s*/, $fields[$blk] } | ||
| 679 | else { @vals = ( $fields[$blk] ) } | ||
| 680 | push @pairs, "$blk:$_" for @vals; | ||
| 681 | } | ||
| 682 | next unless @pairs; | ||
| 683 | my $is_active = $adb->facet_rules( $table_info, @fields ); | ||
| 684 | push @active_ids, $fields[0] if $is_active && $has_active_rule; | ||
| 685 | $batch_facets{ $fields[0] } = join( "\t", $is_active, @pairs ); | ||
| 686 | } | ||
| 687 | |||
| 688 | if ( %batch_facets ) { | ||
| 689 | $adb->index_put( $tmp_path, \%batch_facets, "raw" ); | ||
| 690 | } | ||
| 691 | |||
| 692 | # "active" key: write all active IDs if facet_rules defined | ||
| 693 | if ( $has_active_rule && @active_ids ) { | ||
| 694 | $adb->index_put( $tmp_path, "active", \@active_ids, "ids" ); | ||
| 695 | } | ||
| 696 | |||
| 697 | $adb->table_close($tmp_path); | ||
| 698 | $adb->table_close($fac_path); | ||
| 699 | unlink($fac_path); | ||
| 700 | rename( $tmp_path, $fac_path ) or do { | ||
| 701 | require File::Copy; | ||
| 702 | File::Copy::move( $tmp_path, $fac_path ); | ||
| 703 | }; | ||
| 704 | |||
| 705 | $self->{say} .= " * $fac_path \n"; | ||
| 706 | |||
| 707 | return 1; | ||
| 708 | } | ||
| 709 | |||
| 710 | # Rebuilds ReWrite links. | ||
| 711 | # my $ok = $tools->set_rwlnkall($tableid); | ||
| 712 | # my $ok = $tools->set_rwlnkall($tableid, @records); | ||
| 713 | # ------------------------------------------------ | ||
| 714 | sub set_rwlnkall { | ||
| 715 | |||
| 716 | my ( $self, $tableid, @records ) = @_; | ||
| 717 | my $adb = $self->{_adb} or return; | ||
| 718 | |||
| 719 | my $table_path = $adb->table_path($tableid); | ||
| 720 | return unless -e "$table_path.$adb->{db_ext}"; | ||
| 721 | |||
| 722 | my $table_info = $adb->table_info($tableid); | ||
| 723 | return unless $table_info; | ||
| 724 | return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' ); | ||
| 725 | return unless $table_info->{slug_block}; | ||
| 726 | |||
| 727 | # Read records from table if input is empty | ||
| 728 | if ( !scalar @records ) { | ||
| 729 | (@records) = $adb->read_all($tableid, 0, 0, no_index => 1, dir => 'asc'); | ||
| 730 | @records = grep { ref($_) eq 'ARRAY' } @records; | ||
| 731 | if ( !scalar @records ) { | ||
| 732 | cluck "[DB_TOOL] No records found for ReWrite in $tableid.\n"; | ||
| 733 | return; | ||
| 734 | } | ||
| 735 | } | ||
| 736 | else { | ||
| 737 | @records = grep { ref($_) eq 'ARRAY' } @records; | ||
| 738 | } | ||
| 739 | |||
| 740 | # Rebuild unified slug file. | ||
| 741 | my $slg_path = "${table_path}.slg"; | ||
| 742 | my $tmp_path = "$slg_path.tmp"; | ||
| 743 | |||
| 744 | unlink($tmp_path); | ||
| 745 | |||
| 746 | my @slg_records; | ||
| 747 | my %seen_links; | ||
| 748 | |||
| 749 | @records = $adb->db_sortid( $tableid, @records ); | ||
| 750 | foreach my $record (@records) { | ||
| 751 | my $rw_link = $adb->set_slug( $tableid, $record ); | ||
| 752 | next unless defined $rw_link && length $rw_link; | ||
| 753 | if ( exists $seen_links{$rw_link} ) { | ||
| 754 | $rw_link .= "-$record->[0]"; | ||
| 755 | } | ||
| 756 | $seen_links{$rw_link} = $record->[0]; | ||
| 757 | push @slg_records, [ "0:$record->[0]", $rw_link ]; | ||
| 758 | push @slg_records, [ "1:$rw_link", $record->[0] ]; | ||
| 759 | } | ||
| 760 | |||
| 761 | if ( @slg_records ) { | ||
| 762 | $adb->table_write($tmp_path) | ||
| 763 | or do { | ||
| 764 | cluck "[DB_TOOL] $tmp_path can't be written.\n"; | ||
| 765 | return 0; | ||
| 766 | }; | ||
| 767 | $adb->recs_put( $tmp_path, @slg_records ); | ||
| 768 | $adb->table_close($tmp_path); | ||
| 769 | $adb->table_close($slg_path); | ||
| 770 | unlink($slg_path); | ||
| 771 | rename( $tmp_path, $slg_path ) or do { | ||
| 772 | require File::Copy; | ||
| 773 | File::Copy::move( $tmp_path, $slg_path ); | ||
| 774 | }; | ||
| 775 | |||
| 776 | $self->{say} .= " - Slug indexes are being created.: \n"; | ||
| 777 | $self->{say} .= " * $slg_path \n"; | ||
| 778 | } | ||
| 779 | else { | ||
| 780 | $adb->table_close($slg_path); | ||
| 781 | unlink($slg_path); | ||
| 782 | } | ||
| 783 | |||
| 784 | return 1; | ||
| 785 | } | ||
| 786 | |||
| 787 | # Rebuilds sort index within .inx ($blk:keys). | ||
| 788 | # my $ok = $tools->set_sort($tableid, @records); | ||
| 789 | # ------------------------------------------------ | ||
| 790 | sub set_sort { | ||
| 791 | |||
| 792 | my ( $self, $tableid, @records ) = @_; | ||
| 793 | my $adb = $self->{_adb} or return; | ||
| 794 | |||
| 795 | return unless $tableid; | ||
| 796 | my $table_path = $adb->table_path($tableid); | ||
| 797 | my $table_info = $adb->table_info($tableid); | ||
| 798 | return unless $table_info; | ||
| 799 | return 1 if $adb->config('use_simple') || $table_info->{use_simple} || ( $table_info->{id_type} && $table_info->{id_type} eq 'ascii' ); | ||
| 800 | return unless exists $table_info->{sort_block}; | ||
| 801 | |||
| 802 | if ( !@records ) { | ||
| 803 | @records = $adb->read_all( $tableid, 0, 0, no_index => 1, dir => 'asc' ); | ||
| 804 | } | ||
| 805 | |||
| 806 | my $has_junk = $table_info->{use_junk}; | ||
| 807 | my $junk_rdbm = $has_junk ? $adb->prefetch_junk_rdbm( $table_info, \@records ) : {}; | ||
| 808 | |||
| 809 | my ( %keys_batch, %raw_batch ); | ||
| 810 | foreach my $cfg ( @{ $table_info->{sort_block} } ) { | ||
| 811 | my ( $blk, $type, $len ) = ref($cfg) eq 'HASH' | ||
| 812 | ? ( $cfg->{blk}, $cfg->{type}, $cfg->{len} // 8 ) | ||
| 813 | : ( $cfg, undef, 8 ); | ||
| 814 | |||
| 815 | if ( ( !defined $type || $type eq '' || $type eq 'auto' ) && $table_info->{blocks} ) { | ||
| 816 | if ( ref($table_info->{blocks}) eq 'ARRAY' && ref($table_info->{blocks}[$blk]) eq 'HASH' ) { | ||
| 817 | $type = $table_info->{blocks}[$blk]{type}; | ||
| 818 | } | ||
| 819 | } | ||
| 820 | $type ||= 'string'; | ||
| 821 | |||
| 822 | my %map; | ||
| 823 | my ( %act_map, %junk_map ); | ||
| 824 | my %uniq_vals; | ||
| 825 | my ( %act_uniq, %junk_uniq ); | ||
| 826 | foreach my $rec (@records) { | ||
| 827 | next unless ref($rec) eq 'ARRAY' && defined $rec->[0] && $rec->[0] > 0; | ||
| 828 | my $rid = $rec->[0]; | ||
| 829 | my $val = $rec->[$blk]; | ||
| 830 | my $norm = $adb->normalize_sort_key( $val, $type, $len ); | ||
| 831 | $map{$rid} = $norm; | ||
| 832 | |||
| 833 | if ( defined $val && $val ne '' ) { | ||
| 834 | $uniq_vals{$val} = 1; | ||
| 835 | } | ||
| 836 | |||
| 837 | if ($has_junk) { | ||
| 838 | my $is_junk = $adb->junk_rules( $table_info, $rec, $junk_rdbm ); | ||
| 839 | if ($is_junk) { | ||
| 840 | $junk_map{$rid} = $norm; | ||
| 841 | $junk_uniq{$val} = 1 if defined $val && $val ne ''; | ||
| 842 | } | ||
| 843 | else { | ||
| 844 | $act_map{$rid} = $norm; | ||
| 845 | $act_uniq{$val} = 1 if defined $val && $val ne ''; | ||
| 846 | } | ||
| 847 | } | ||
| 848 | } | ||
| 849 | |||
| 850 | # Sort all keys in-memory with deterministic tie-breaker (strictly numeric IDs) | ||
| 851 | my @sorted_ids = grep { defined $_ && $_ > 0 } sort { | ||
| 852 | ( ( $map{$a} // '' ) cmp ( $map{$b} // '' ) ) | ||
| 853 | || ( $a <=> $b ) | ||
| 854 | } keys %map; | ||
| 855 | |||
| 856 | $keys_batch{"$blk:keys"} = \@sorted_ids; | ||
| 857 | foreach my $k ( keys %map ) { | ||
| 858 | $raw_batch{"$blk:$k"} = $map{$k}; | ||
| 859 | } | ||
| 860 | |||
| 861 | my $sort_vals = sub { | ||
| 862 | my ($href) = @_; | ||
| 863 | my $is_num = ( $type eq 'num' || $type eq 'decimal' ) ? 1 : 0; | ||
| 864 | if ( !$is_num && %$href ) { | ||
| 865 | $is_num = 1; | ||
| 866 | for my $k ( keys %$href ) { | ||
| 867 | if ( $k !~ /^-?[0-9]+(?:\.[0-9]+)?$/ ) { | ||
| 868 | $is_num = 0; | ||
| 869 | last; | ||
| 870 | } | ||
| 871 | } | ||
| 872 | } | ||
| 873 | my @sv = $is_num ? ( sort { $a <=> $b } keys %$href ) : ( sort { $a cmp $b } keys %$href ); | ||
| 874 | return join("\t", @sv); | ||
| 875 | }; | ||
| 876 | $raw_batch{"$blk:vals"} = $sort_vals->(\%uniq_vals) if %uniq_vals; | ||
| 877 | |||
| 878 | if ($has_junk) { | ||
| 879 | my @sorted_act = grep { defined $_ && $_ > 0 } sort { | ||
| 880 | ( ( $act_map{$a} // '' ) cmp ( $act_map{$b} // '' ) ) | ||
| 881 | || ( $a <=> $b ) | ||
| 882 | } keys %act_map; | ||
| 883 | $keys_batch{"A:$blk:keys"} = \@sorted_act; | ||
| 884 | $raw_batch{"A:$blk:vals"} = $sort_vals->(\%act_uniq) if %act_uniq; | ||
| 885 | |||
| 886 | my @sorted_junk = grep { defined $_ && $_ > 0 } sort { | ||
| 887 | ( ( $junk_map{$a} // '' ) cmp ( $junk_map{$b} // '' ) ) | ||
| 888 | || ( $a <=> $b ) | ||
| 889 | } keys %junk_map; | ||
| 890 | $keys_batch{"B:$blk:keys"} = \@sorted_junk; | ||
| 891 | $raw_batch{"B:$blk:vals"} = $sort_vals->(\%junk_uniq) if %junk_uniq; | ||
| 892 | } | ||
| 893 | |||
| 894 | } | ||
| 895 | |||
| 896 | if ( %keys_batch || %raw_batch ) { | ||
| 897 | my $inx_path = "${table_path}.inx"; | ||
| 898 | if ( $adb->table_write($inx_path) ) { | ||
| 899 | $adb->index_put( $inx_path, \%keys_batch, "ids" ) if %keys_batch; | ||
| 900 | $adb->index_put( $inx_path, \%raw_batch, "raw" ) if %raw_batch; | ||
| 901 | $adb->table_close($inx_path); | ||
| 902 | } | ||
| 903 | } | ||
| 904 | |||
| 905 | $self->{say} .= " - Sort indexes created in ${table_path}.inx for table $tableid.\n"; | ||
| 906 | return 1; | ||
| 907 | } | ||
| 908 | |||
| 909 | # Rebuilds index for all tables | ||
| 910 | # my $ok = $tools->index_alltables(); | ||
| 911 | # print $self->{say}; | ||
| 912 | # ------------------------------------------------ | ||
| 913 | sub index_alltables { | ||
| 914 | |||
| 915 | my ($self) = @_; | ||
| 916 | my $adb = $self->{_adb} or return; | ||
| 917 | |||
| 918 | my @tables; | ||
| 919 | |||
| 920 | # 1. Locate tables. | ||
| 921 | my $tables_hash = ( $self->can('all_tables') ? | ||
| 922 | $self->all_tables() : | ||
| 923 | do { | ||
| 924 | require AmberDB::Tools::Maintain; | ||
| 925 | AmberDB::Tools::Maintain->new($self->{_adb})->all_tables(); | ||
| 926 | } | ||
| 927 | ); | ||
| 928 | foreach my $dbase ( keys %$tables_hash ) { | ||
| 929 | $dbase =~ /^[a-z0-9]+$/ or next; | ||
| 930 | foreach my $tableid ( keys %{ $tables_hash->{$dbase} } ) { | ||
| 931 | $tableid =~ /^[a-z0-9_]+$/ or next; | ||
| 932 | my $table_path = $adb->table_path($tableid); | ||
| 933 | push @tables, [ $tableid, "$table_path.$adb->{db_ext}" ]; | ||
| 934 | } | ||
| 935 | } | ||
| 936 | |||
| 937 | # 2. Read table records and enter indexing loop | ||
| 938 | foreach my $table_entry (@tables) { | ||
| 939 | my $tbl = $table_entry->[0]; | ||
| 940 | my @records = $adb->read_all($tbl, 0, 0, no_index => 1, dir => 'asc'); | ||
| 941 | |||
| 942 | my $count = scalar @records; | ||
| 943 | $self->set_index( $tbl, @records ); | ||
| 944 | |||
| 945 | $self->{say} .= | ||
| 946 | "Table: $tbl, Record count: $count\n"; | ||
| 947 | } | ||
| 948 | } | ||
| 949 | |||
| 950 | # Verifies validity of Read All index | ||
| 951 | # my $diff = $tools->check_readall($tableid, @records); | ||
| 952 | # ------------------------------------------------ | ||
| 953 | sub check_readall { | ||
| 954 | |||
| 955 | my ( $self, $tableid, @records ) = @_; | ||
| 956 | my $adb = $self->{_adb} or return; | ||
| 957 | |||
| 958 | return unless $tableid; | ||
| 959 | my $table_path = $adb->table_path($tableid); | ||
| 960 | my $table_info = $adb->table_info($tableid); | ||
| 961 | |||
| 962 | my ( %diff, %recs ); | ||
| 963 | foreach my $rec (@records) { | ||
| 964 | $rec = $rec->[0] if ref $rec eq "ARRAY"; | ||
| 965 | next unless defined $rec && $rec ne ""; | ||
| 966 | $recs{keys}->{$rec} = 1; | ||
| 967 | $recs{lastid} ||= $rec; | ||
| 968 | $rec > $recs{lastid} and $recs{lastid} = $rec; | ||
| 969 | } | ||
| 970 | |||
| 971 | my (%inds); | ||
| 972 | my $inx_path = "$table_path.inx"; | ||
| 973 | my ( $total_keys, @keys ) = $adb->index_get( $inx_path, "keys", "ids" ); | ||
| 974 | $adb->table_close($inx_path); | ||
| 975 | foreach my $rec (@keys) { | ||
| 976 | $inds{keys}->{$rec} = 1; | ||
| 977 | $inds{lastid} ||= $rec; | ||
| 978 | $rec > $inds{lastid} and $inds{lastid} = $rec; | ||
| 979 | } | ||
| 980 | |||
| 981 | if ( ( $recs{lastid} // "" ) ne ( $inds{lastid} // "" ) ) { | ||
| 982 | $diff{lastid}->{recs} = $recs{lastid}; | ||
| 983 | $diff{lastid}->{inds} = $inds{lastid}; | ||
| 984 | } | ||
| 985 | |||
| 986 | my $diffs = $adb->hash_diff( $recs{keys}, $inds{keys} ); | ||
| 987 | |||
| 988 | if ( $diffs->{hash1} ) { | ||
| 989 | $diff{keys}->{recs} = $diffs->{hash1}; | ||
| 990 | } | ||
| 991 | if ( $diffs->{hash2} ) { | ||
| 992 | $diff{keys}->{inds} = $diffs->{hash2}; | ||
| 993 | } | ||
| 994 | |||
| 995 | return \%diff; | ||
| 996 | } | ||
| 997 | |||
| 998 | # Rebuilds search word index. | ||
| 999 | # my $ok = $tools->check_search($tableid, @records); | ||
| 1000 | # ------------------------------------------------ | ||
| 1001 | sub check_search { | ||
| 1002 | |||
| 1003 | my ( $self, $tableid, @records ) = @_; | ||
| 1004 | my $adb = $self->{_adb} or return; | ||
| 1005 | |||
| 1006 | my %diff = (); | ||
| 1007 | return unless $tableid; | ||
| 1008 | scalar @records or return \%diff; | ||
| 1009 | my $table_path = $adb->table_path($tableid); | ||
| 1010 | my $table_info = $adb->table_info($tableid); | ||
| 1011 | exists( $table_info->{search_block} ) or return \%diff; | ||
| 1012 | |||
| 1013 | foreach my $line (@records) { | ||
| 1014 | my @fields = @$line; | ||
| 1015 | foreach my $src ( @{ $table_info->{search_block} } ) { | ||
| 1016 | $src =~ /^[0-9]+$/ or next; | ||
| 1017 | my %words = $adb->get_words( $fields[$src], "write" ); | ||
| 1018 | |||
| 1019 | foreach my $word ( keys %words ) { | ||
| 1020 | $diff{recs}->{$src}->{$word}->{ $fields[0] } = 1; | ||
| 1021 | } | ||
| 1022 | } | ||
| 1023 | } | ||
| 1024 | |||
| 1025 | my $unified_src = "${table_path}.src"; | ||
| 1026 | if ( -e $unified_src ) { | ||
| 1027 | if ( $adb->table_read($unified_src) ) { | ||
| 1028 | $adb->recs_scan( | ||
| 1029 | $unified_src, | ||
| 1030 | sub { | ||
| 1031 | my ( $k, $records ) = @_; | ||
| 1032 | my ( $src, $word ) = split( /:/, $k, 2 ); | ||
| 1033 | return unless defined $src && defined $word; | ||
| 1034 | my ( undef, @rids ) = $adb->bin_decode($records); | ||
| 1035 | foreach my $rid (@rids) { | ||
| 1036 | if ( exists( $diff{recs}->{$src}->{$word}->{$rid} ) ) { | ||
| 1037 | delete( $diff{recs}->{$src}->{$word}->{$rid} ); | ||
| 1038 | } | ||
| 1039 | else { | ||
| 1040 | $diff{inds}->{$src}->{$word}->{$rid} = 1; | ||
| 1041 | } | ||
| 1042 | } | ||
| 1043 | } | ||
| 1044 | ); | ||
| 1045 | $adb->table_close($unified_src); | ||
| 1046 | } | ||
| 1047 | } | ||
| 1048 | |||
| 1049 | return \%diff; | ||
| 1050 | } | ||
| 1051 | |||
| 1052 | # $ok = $self->set_suggest( $tableid, @records ); | ||
| 1053 | # Rebuilds .ajw and .ajn suggest index files from records | ||
| 1054 | # ------------------------------------------------ | ||
| 1055 | sub set_suggest { | ||
| 1056 | my ( $self, $tableid, @records ) = @_; | ||
| 1057 | |||
| 1058 | my $adb = $self->{_adb}; | ||
| 1059 | my $table_info = $adb->table_info($tableid); | ||
| 1060 | return unless $table_info && exists $table_info->{suggest_block}; | ||
| 1061 | |||
| 1062 | my $table_path = $adb->table_path($tableid); | ||
| 1063 | my $ajw_path = "${table_path}.ajw"; | ||
| 1064 | my $ajn_path = "${table_path}.ajn"; | ||
| 1065 | my $tmp_ajw = "$ajw_path.tmp"; | ||
| 1066 | my $tmp_ajn = "$ajn_path.tmp"; | ||
| 1067 | |||
| 1068 | unlink $tmp_ajw, $tmp_ajn; | ||
| 1069 | |||
| 1070 | # 1. Identify RDBM blocks and prepare pre-fetch | ||
| 1071 | my %rdbm_blocks; | ||
| 1072 | for my $blk ( @{ $table_info->{suggest_block} } ) { | ||
| 1073 | my ( $real_blk, $src_table, $src_display ) = | ||
| 1074 | ref($blk) eq 'ARRAY' ? ( $blk->[0], $blk->[1], $blk->[2] ) | ||
| 1075 | : ( $blk, undef, undef ); | ||
| 1076 | |||
| 1077 | if ( !$src_table ) { | ||
| 1078 | ( $src_table, $src_display ) = $adb->rdbm_target( $table_info, $real_blk ); | ||
| 1079 | } | ||
| 1080 | if ( $src_table ) { | ||
| 1081 | $src_display //= 1; | ||
| 1082 | $rdbm_blocks{$real_blk} = { table => $src_table, display => $src_display }; | ||
| 1083 | } | ||
| 1084 | } | ||
| 1085 | |||
| 1086 | # 2. Collect unique foreign IDs across records | ||
| 1087 | my ( %foreign_ids, %rdbm_recs ); | ||
| 1088 | if ( %rdbm_blocks ) { | ||
| 1089 | for my $rec (@records) { | ||
| 1090 | for my $real_blk ( keys %rdbm_blocks ) { | ||
| 1091 | my $val = $rec->[$real_blk]; | ||
| 1092 | next unless defined $val && $val ne ''; | ||
| 1093 | my $target_table = $rdbm_blocks{$real_blk}->{table}; | ||
| 1094 | for my $id ( split /[,;]/, $val ) { | ||
| 1095 | $id =~ s/^\s+|\s+$//g; | ||
| 1096 | $foreign_ids{$target_table}->{$id} = 1 if $id =~ /^\d+$/; | ||
| 1097 | } | ||
| 1098 | } | ||
| 1099 | } | ||
| 1100 | for my $target_table ( keys %foreign_ids ) { | ||
| 1101 | my @ids = keys %{ $foreign_ids{$target_table} }; | ||
| 1102 | next unless @ids; | ||
| 1103 | my @target_records = $adb->read_list( $target_table, \@ids ); | ||
| 1104 | $rdbm_recs{$target_table} = { map { $_->[0] => $_ } @target_records }; | ||
| 1105 | } | ||
| 1106 | } | ||
| 1107 | |||
| 1108 | # 3. Extract sentences across all records | ||
| 1109 | my @all_sentences; | ||
| 1110 | for my $rec (@records) { | ||
| 1111 | push @all_sentences, $adb->_extract_suggest_sentences( $table_info, $rec, \%rdbm_recs, \%rdbm_blocks ); | ||
| 1112 | } | ||
| 1113 | return unless @all_sentences; | ||
| 1114 | |||
| 1115 | my ( $word_freq, $transitions ) = $adb->_extract_words_and_transitions( @all_sentences ); | ||
| 1116 | |||
| 1117 | # 4. Build .ajw (Word prefix index) | ||
| 1118 | my %ajw_buckets; | ||
| 1119 | for my $norm ( keys %$word_freq ) { | ||
| 1120 | my $disp = $word_freq->{$norm}{disp}; | ||
| 1121 | my $cnt = $word_freq->{$norm}{cnt}; | ||
| 1122 | my $max_len = length($norm) < 12 ? length($norm) : 12; | ||
| 1123 | for ( my $len = 2; $len <= $max_len; $len++ ) { | ||
| 1124 | my $pfx = substr($norm, 0, $len); | ||
| 1125 | $ajw_buckets{$pfx}{$norm} = [ $disp, $cnt ]; | ||
| 1126 | } | ||
| 1127 | my $ascii_norm = $adb->to_ascii($norm); | ||
| 1128 | if ( defined $ascii_norm && $ascii_norm ne $norm ) { | ||
| 1129 | my $a_len = length($ascii_norm) < 12 ? length($ascii_norm) : 12; | ||
| 1130 | for ( my $len = 2; $len <= $a_len; $len++ ) { | ||
| 1131 | my $a_pfx = substr($ascii_norm, 0, $len); | ||
| 1132 | $ajw_buckets{$a_pfx}{$norm} = [ $disp, $cnt ]; | ||
| 1133 | } | ||
| 1134 | } | ||
| 1135 | } | ||
| 1136 | |||
| 1137 | my @ajw_records; | ||
| 1138 | for my $pfx ( keys %ajw_buckets ) { | ||
| 1139 | my $bucket = $ajw_buckets{$pfx}; | ||
| 1140 | my @sorted = sort { | ||
| 1141 | $bucket->{$b}->[1] <=> $bucket->{$a}->[1] | ||
| 1142 | || $a cmp $b | ||
| 1143 | } keys %$bucket; | ||
| 1144 | @sorted = @sorted[0 .. 9] if @sorted > 10; | ||
| 1145 | my @payload = map { $bucket->{$_} } @sorted; | ||
| 1146 | push @ajw_records, [ $pfx, $adb->db_encode(\@payload) ]; | ||
| 1147 | } | ||
| 1148 | |||
| 1149 | if (@ajw_records) { | ||
| 1150 | $adb->table_write($tmp_ajw) or return 0; | ||
| 1151 | $adb->recs_put( $tmp_ajw, @ajw_records ); | ||
| 1152 | $adb->table_close($tmp_ajw); | ||
| 1153 | $adb->table_close($ajw_path); | ||
| 1154 | unlink $ajw_path; | ||
| 1155 | rename( $tmp_ajw, $ajw_path ) or do { | ||
| 1156 | require File::Copy; | ||
| 1157 | File::Copy::move( $tmp_ajw, $ajw_path ); | ||
| 1158 | }; | ||
| 1159 | $self->{say} .= " * $ajw_path \n"; | ||
| 1160 | } | ||
| 1161 | |||
| 1162 | # 5. Build .ajn (Next word transitions) | ||
| 1163 | my @ajn_records; | ||
| 1164 | for my $norm1 ( keys %$transitions ) { | ||
| 1165 | my $targets = $transitions->{$norm1}; | ||
| 1166 | my @sorted = sort { | ||
| 1167 | $targets->{$b}->{cnt} <=> $targets->{$a}->{cnt} | ||
| 1168 | || $a cmp $b | ||
| 1169 | } keys %$targets; | ||
| 1170 | @sorted = @sorted[0 .. 299] if @sorted > 300; | ||
| 1171 | my @payload = map { | ||
| 1172 | [ $targets->{$_}->{disp}, $targets->{$_}->{cnt}, $_ ] | ||
| 1173 | } @sorted; | ||
| 1174 | push @ajn_records, [ $norm1, $adb->db_encode(\@payload) ]; | ||
| 1175 | } | ||
| 1176 | |||
| 1177 | if (@ajn_records) { | ||
| 1178 | $adb->table_write($tmp_ajn) or return 0; | ||
| 1179 | $adb->recs_put( $tmp_ajn, @ajn_records ); | ||
| 1180 | $adb->table_close($tmp_ajn); | ||
| 1181 | $adb->table_close($ajn_path); | ||
| 1182 | unlink $ajn_path; | ||
| 1183 | rename( $tmp_ajn, $ajn_path ) or do { | ||
| 1184 | require File::Copy; | ||
| 1185 | File::Copy::move( $tmp_ajn, $ajn_path ); | ||
| 1186 | }; | ||
| 1187 | $self->{say} .= " * $ajn_path \n"; | ||
| 1188 | } | ||
| 1189 | |||
| 1190 | return 1; | ||
| 1191 | } | ||
| 1192 | |||
| 1193 | sub reindex_suggest { | ||
| 1194 | my ( $self, $tableid ) = @_; | ||
| 1195 | my $adb = $self->{_adb}; | ||
| 1196 | my @records = $adb->read_all($tableid); | ||
| 1197 | return $self->set_suggest( $tableid, @records ); | ||
| 1198 | } | ||
| 1199 | |||
| 1200 | 1; |