KnowledgeStore/Command/rebuild_vectors.pm

filter: Statementstmt 60.0% 3 / 5 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 2 / 2 Podpod n/a totaltotal 71.4% 5 / 7 CC 1 SCAR 1.4
CC
1
Cov
60%
BEGIN
1.3
BEGIN
1.0
CRAP 1.1
« KnowledgeStore/Command/agent.pm ↑ Summary KnowledgeStore/Command/token.pm »
1 package MCP::Server::KnowledgeStore::Command::rebuild_vectors;
2 1 ▶use v5.38;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 0 ▶use Mojo::Base 'Mojolicious::Command', -signatures;
Subroutine BEGIN: called CC=1 SCAR=2.6
4  
5 our $VERSION = '0.001';
6  
7 use MCP::Server::KnowledgeStore::TermWeight;
8  
9 # ABSTRACT: Rebuild the TF-IDF vectors used by search tools
10  
11 has description =>
12 'Rebuild TF-IDF vectors for latest revisions (hybrid search)';
13 has usage => <<~'USAGE';
14 Usage: APPLICATION rebuild_vectors [OPTIONS]
15  
16 ./bin/mcp-ks rebuild_vectors
17 ./bin/mcp-ks rebuild_vectors --kind=memory
18 ./bin/mcp-ks rebuild_vectors --kind=skill
19 ./bin/mcp-ks rebuild_vectors --all
20  
21 Rebuilds the TF-IDF vectors used by search_* tools.
22 By default rebuilds vectors for all kinds (memory, skill, spec).
23 Only processes LATEST REVISIONS (head_id).
24  
25 This should be run after initial deployment or when you want to
26 refresh the vector cache.
27  
28 USAGE
29  
30 sub run ($self, @args) {
31 my $kind = undef;
32 my $all = 0;
33  
34 require Getopt::Long;
35 Getopt::Long::GetOptions(
36 'kind=s' => \$kind,
37 'all' => \$all,
38 ) or die $self->usage;
39  
40 die $self->usage if @args;
41  
42 my $store = $self->app->store;
43 my @kinds = $all ? qw(memory skill spec) : ($kind // qw(memory skill spec));
44  
45 for my $k (@kinds) {
46 $self->app->log->info(
47 "Rebuilding vectors for kind '$k' (latest revisions only)...");
48  
49 # Clear existing cache for this kind
50 $store->db->query(<<~"SQL", $k);
51 DELETE FROM document_tfidf
52 WHERE revision_id IN (
53 SELECT r.id
54 FROM revisions r
55 JOIN entries e ON e.head_id = r.id
56 WHERE e.kind = ?
57 )
58 SQL
59 $store->db->query('DELETE FROM corpus_idf WHERE kind = ?', $k);
60  
61 # Get all LATEST revisions
62 my $entries = $store->db->query(<<~'SQL', $k)->hashes->to_array;
63 SELECT e.id AS entry_id, e.name, r.id AS revision_id,
64 r.body, r.description
65 FROM entries e
66 JOIN revisions r ON r.id = e.head_id
67 WHERE e.kind = ? AND e.archived_at IS NULL
68 SQL
69  
70 $self->app->log->info(
71 "Found " . scalar(@$entries) . " latest revisions for $k");
72  
73 next unless @$entries;
74  
75 # Compute corpus IDF from latest revisions
76 my $tw = MCP::Server::KnowledgeStore::TermWeight->new;
77 my @texts
78 = map { join ' ', $_->{name}, $_->{body}, ($_->{description} // '') }
79 @$entries;
80 my $idf = $tw->compute_idf(\@texts);
81  
82 # Store corpus IDF
83 my $json_idf = { json => $idf };
84 my $document_count = scalar @$entries;
85 my @idf_bind
86 = ($k, $json_idf, $document_count, $json_idf, $document_count);
87 $store->db->query(<<~'SQL', @idf_bind);
88 INSERT INTO corpus_idf (kind, idf, document_count)
89 VALUES (?, ?, ?)
90 ON CONFLICT (kind) DO UPDATE SET idf = ?, document_count = ?, updated_at = now()
91 SQL
92  
93 # Compute and store document vectors
94 my $count = 0;
95 for my $entry (@$entries) {
96 my $text = join ' ', $entry->{name}, $entry->{body},
97 ($entry->{description} // '');
98 my $tf = $tw->compute_tf($text);
99 my $tfidf
100 = { map { $_ => ($tf->{$_} // 0) * ($idf->{$_} // 0) } keys %$tf };
101  
102 my $json_tfidf = { json => $tfidf };
103 my $term_count = scalar keys %$tfidf;
104 my @tfidf_bind
105 = ($entry->{revision_id}, $json_tfidf, $term_count, $json_tfidf);
106 $store->db->query(<<~'SQL', @tfidf_bind);
107 INSERT INTO document_tfidf (revision_id, tfidf_vector, term_count)
108 VALUES (?, ?, ?)
109 ON CONFLICT (revision_id) DO UPDATE SET tfidf_vector = ?, updated_at = now()
110 SQL
111 $count++;
112 }
113  
114 $self->app->log->info("Rebuild complete for $k: $count vectors stored");
115 }
116  
117 $self->app->log->info('All vector rebuilds complete!');
118 return 0;
119 }
120  
121 1;
122  
123 __END__
« KnowledgeStore/Command/agent.pm ↑ Summary KnowledgeStore/Command/token.pm »