| 1 | package MCP::Server::KnowledgeStore::Command::rebuild_vectors; | ||
| 2 | 1 | ▶ | use v5.38; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 3 | 0 | ▶ | use Mojo::Base 'Mojolicious::Command', -signatures; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 4 | |||
| 5 | our $VERSION = '0.001'; | ||
| 6 | |||
| 7 | use MCP::Server::KnowledgeStore::TermWeight; | ||
| 8 | |||
| 9 | # ABSTRACT: Rebuild the TF-IDF vectors used by search tools | ||
| 10 | |||
| 11 | has description => | ||
| 12 | 'Rebuild TF-IDF vectors for latest revisions (hybrid search)'; | ||
| 13 | has usage => <<~'USAGE'; | ||
| 14 | Usage: APPLICATION rebuild_vectors [OPTIONS] | ||
| 15 | |||
| 16 | ./bin/mcp-ks rebuild_vectors | ||
| 17 | ./bin/mcp-ks rebuild_vectors --kind=memory | ||
| 18 | ./bin/mcp-ks rebuild_vectors --kind=skill | ||
| 19 | ./bin/mcp-ks rebuild_vectors --all | ||
| 20 | |||
| 21 | Rebuilds the TF-IDF vectors used by search_* tools. | ||
| 22 | By default rebuilds vectors for all kinds (memory, skill, spec). | ||
| 23 | Only processes LATEST REVISIONS (head_id). | ||
| 24 | |||
| 25 | This should be run after initial deployment or when you want to | ||
| 26 | refresh the vector cache. | ||
| 27 | |||
| 28 | USAGE | ||
| 29 | |||
| 30 | sub run ($self, @args) { | ||
| 31 | my $kind = undef; | ||
| 32 | my $all = 0; | ||
| 33 | |||
| 34 | require Getopt::Long; | ||
| 35 | Getopt::Long::GetOptions( | ||
| 36 | 'kind=s' => \$kind, | ||
| 37 | 'all' => \$all, | ||
| 38 | ) or die $self->usage; | ||
| 39 | |||
| 40 | die $self->usage if @args; | ||
| 41 | |||
| 42 | my $store = $self->app->store; | ||
| 43 | my @kinds = $all ? qw(memory skill spec) : ($kind // qw(memory skill spec)); | ||
| 44 | |||
| 45 | for my $k (@kinds) { | ||
| 46 | $self->app->log->info( | ||
| 47 | "Rebuilding vectors for kind '$k' (latest revisions only)..."); | ||
| 48 | |||
| 49 | # Clear existing cache for this kind | ||
| 50 | $store->db->query(<<~"SQL", $k); | ||
| 51 | DELETE FROM document_tfidf | ||
| 52 | WHERE revision_id IN ( | ||
| 53 | SELECT r.id | ||
| 54 | FROM revisions r | ||
| 55 | JOIN entries e ON e.head_id = r.id | ||
| 56 | WHERE e.kind = ? | ||
| 57 | ) | ||
| 58 | SQL | ||
| 59 | $store->db->query('DELETE FROM corpus_idf WHERE kind = ?', $k); | ||
| 60 | |||
| 61 | # Get all LATEST revisions | ||
| 62 | my $entries = $store->db->query(<<~'SQL', $k)->hashes->to_array; | ||
| 63 | SELECT e.id AS entry_id, e.name, r.id AS revision_id, | ||
| 64 | r.body, r.description | ||
| 65 | FROM entries e | ||
| 66 | JOIN revisions r ON r.id = e.head_id | ||
| 67 | WHERE e.kind = ? AND e.archived_at IS NULL | ||
| 68 | SQL | ||
| 69 | |||
| 70 | $self->app->log->info( | ||
| 71 | "Found " . scalar(@$entries) . " latest revisions for $k"); | ||
| 72 | |||
| 73 | next unless @$entries; | ||
| 74 | |||
| 75 | # Compute corpus IDF from latest revisions | ||
| 76 | my $tw = MCP::Server::KnowledgeStore::TermWeight->new; | ||
| 77 | my @texts | ||
| 78 | = map { join ' ', $_->{name}, $_->{body}, ($_->{description} // '') } | ||
| 79 | @$entries; | ||
| 80 | my $idf = $tw->compute_idf(\@texts); | ||
| 81 | |||
| 82 | # Store corpus IDF | ||
| 83 | my $json_idf = { json => $idf }; | ||
| 84 | my $document_count = scalar @$entries; | ||
| 85 | my @idf_bind | ||
| 86 | = ($k, $json_idf, $document_count, $json_idf, $document_count); | ||
| 87 | $store->db->query(<<~'SQL', @idf_bind); | ||
| 88 | INSERT INTO corpus_idf (kind, idf, document_count) | ||
| 89 | VALUES (?, ?, ?) | ||
| 90 | ON CONFLICT (kind) DO UPDATE SET idf = ?, document_count = ?, updated_at = now() | ||
| 91 | SQL | ||
| 92 | |||
| 93 | # Compute and store document vectors | ||
| 94 | my $count = 0; | ||
| 95 | for my $entry (@$entries) { | ||
| 96 | my $text = join ' ', $entry->{name}, $entry->{body}, | ||
| 97 | ($entry->{description} // ''); | ||
| 98 | my $tf = $tw->compute_tf($text); | ||
| 99 | my $tfidf | ||
| 100 | = { map { $_ => ($tf->{$_} // 0) * ($idf->{$_} // 0) } keys %$tf }; | ||
| 101 | |||
| 102 | my $json_tfidf = { json => $tfidf }; | ||
| 103 | my $term_count = scalar keys %$tfidf; | ||
| 104 | my @tfidf_bind | ||
| 105 | = ($entry->{revision_id}, $json_tfidf, $term_count, $json_tfidf); | ||
| 106 | $store->db->query(<<~'SQL', @tfidf_bind); | ||
| 107 | INSERT INTO document_tfidf (revision_id, tfidf_vector, term_count) | ||
| 108 | VALUES (?, ?, ?) | ||
| 109 | ON CONFLICT (revision_id) DO UPDATE SET tfidf_vector = ?, updated_at = now() | ||
| 110 | SQL | ||
| 111 | $count++; | ||
| 112 | } | ||
| 113 | |||
| 114 | $self->app->log->info("Rebuild complete for $k: $count vectors stored"); | ||
| 115 | } | ||
| 116 | |||
| 117 | $self->app->log->info('All vector rebuilds complete!'); | ||
| 118 | return 0; | ||
| 119 | } | ||
| 120 | |||
| 121 | 1; | ||
| 122 | |||
| 123 | __END__ | ||