| 1 | package MCP::Server::KnowledgeStore::TermWeight; | ||
| 2 | 1 | ▶ | use v5.38; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 3 | 0 | ▶ | use Mojo::Base -base; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 4 | |||
| 5 | our $VERSION = '0.001'; | ||
| 6 | |||
| 7 | # ABSTRACT: TermWeight vector operations using Lingua::TermWeight | ||
| 8 | |||
| 9 | use Lingua::TermWeight; | ||
| 10 | use Lingua::TermWeight::WordSegmenter::SplitBySpace; | ||
| 11 | use Carp qw(croak); | ||
| 12 | |||
| 13 | # Singleton calculator instance | ||
| 14 | my $Calc = do { | ||
| 15 | my $segmenter = Lingua::TermWeight::WordSegmenter::SplitBySpace->new; | ||
| 16 | Lingua::TermWeight->new(word_segmenter => $segmenter); | ||
| 17 | }; | ||
| 18 | |||
| 19 | # Compute TF for a single document | ||
| 20 | # Returns: {term => normalized_frequency} | ||
| 21 | sub compute_tf { | ||
| 22 | my ($self, $text, $normalize) = @_; | ||
| 23 | $normalize //= 1; | ||
| 24 | croak 'text is required' unless defined $text; | ||
| 25 | return $Calc->tf(document => $text, normalize => $normalize); | ||
| 26 | } | ||
| 27 | |||
| 28 | # Compute IDF for a corpus of documents | ||
| 29 | # Arguments: arrayref of document texts | ||
| 30 | # Returns: {term => idf_weight} | ||
| 31 | sub compute_idf { | ||
| 32 | my ($self, $documents) = @_; | ||
| 33 | croak 'documents is required' | ||
| 34 | unless defined $documents && ref $documents eq 'ARRAY'; | ||
| 35 | croak 'documents must not be empty' unless @$documents; | ||
| 36 | return $Calc->idf(documents => $documents); | ||
| 37 | } | ||
| 38 | |||
| 39 | # Compute TF-IDF for a single document given pre-computed IDF | ||
| 40 | # Arguments: document text, idf hashref | ||
| 41 | # Returns: {term => tfidf_weight} | ||
| 42 | sub compute_tfidf { | ||
| 43 | my ($self, $text, $idf) = @_; | ||
| 44 | croak 'text is required' unless defined $text; | ||
| 45 | croak 'idf is required' unless defined $idf && ref $idf eq 'HASH'; | ||
| 46 | |||
| 47 | my $tf = $self->compute_tf($text); | ||
| 48 | return { | ||
| 49 | map { $_ => ($tf->{$_} // 0) * ($idf->{$_} // 0) } keys %$tf, | ||
| 50 | keys %$idf | ||
| 51 | }; | ||
| 52 | } | ||
| 53 | |||
| 54 | # Compute TF-IDF for multiple documents given pre-computed IDF | ||
| 55 | # Arguments: arrayref of document texts, idf hashref | ||
| 56 | # Returns: arrayref of {term => tfidf_weight} | ||
| 57 | sub compute_tfidf_batch { | ||
| 58 | my ($self, $documents, $idf) = @_; | ||
| 59 | croak 'documents is required' | ||
| 60 | unless defined $documents && ref $documents eq 'ARRAY'; | ||
| 61 | croak 'idf is required' unless defined $idf && ref $idf eq 'HASH'; | ||
| 62 | |||
| 63 | my @tfidf; | ||
| 64 | for my $doc (@$documents) { | ||
| 65 | push @tfidf, $self->compute_tfidf($doc, $idf); | ||
| 66 | } | ||
| 67 | return \@tfidf; | ||
| 68 | } | ||
| 69 | |||
| 70 | # Compute cosine similarity between two vectors | ||
| 71 | # Arguments: two hashrefs {term => weight} | ||
| 72 | # Returns: similarity score (0-1) | ||
| 73 | sub cosine_similarity { | ||
| 74 | my ($self, $vec1, $vec2) = @_; | ||
| 75 | croak 'vec1 is required' unless defined $vec1 && ref $vec1 eq 'HASH'; | ||
| 76 | croak 'vec2 is required' unless defined $vec2 && ref $vec2 eq 'HASH'; | ||
| 77 | |||
| 78 | my $dot = 0; | ||
| 79 | my $mag1 = 0; | ||
| 80 | my $mag2 = 0; | ||
| 81 | |||
| 82 | my %all_terms = map { $_ => 1 } keys %$vec1, keys %$vec2; | ||
| 83 | for my $term (keys %all_terms) { | ||
| 84 | my $v1 = $vec1->{$term} // 0; | ||
| 85 | my $v2 = $vec2->{$term} // 0; | ||
| 86 | $dot += $v1 * $v2; | ||
| 87 | $mag1 += $v1**2; | ||
| 88 | $mag2 += $v2**2; | ||
| 89 | } | ||
| 90 | |||
| 91 | return 0 unless $mag1 && $mag2; | ||
| 92 | return $dot / (sqrt($mag1) * sqrt($mag2)); | ||
| 93 | } | ||
| 94 | |||
| 95 | # Batch cosine similarity: compute similarity of query vector against multiple document vectors | ||
| 96 | # Arguments: query_vector, arrayref of document_vectors | ||
| 97 | # Returns: arrayref of similarity scores | ||
| 98 | sub batch_cosine_similarity { | ||
| 99 | my ($self, $query_vec, $doc_vectors) = @_; | ||
| 100 | croak 'query_vec is required' | ||
| 101 | unless defined $query_vec && ref $query_vec eq 'HASH'; | ||
| 102 | croak 'doc_vectors is required' | ||
| 103 | unless defined $doc_vectors && ref $doc_vectors eq 'ARRAY'; | ||
| 104 | |||
| 105 | return [map { $self->cosine_similarity($query_vec, $_) } @$doc_vectors]; | ||
| 106 | } | ||
| 107 | |||
| 108 | 1; | ||
| 109 | |||
| 110 | __END__ | ||