KnowledgeStore/TermWeight.pm

filter: Statementstmt 60.0% 3 / 5 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 2 / 2 Podpod n/a totaltotal 71.4% 5 / 7 CC 1 SCAR 1.4
CC
1
Cov
60%
BEGIN
1.3
BEGIN
1.0
CRAP 1.1
« KnowledgeStore/Store.pm ↑ Summary KnowledgeStore/Tools.pm »
1 package MCP::Server::KnowledgeStore::TermWeight;
2 1 ▶use v5.38;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 0 ▶use Mojo::Base -base;
Subroutine BEGIN: called CC=1 SCAR=2.6
4  
5 our $VERSION = '0.001';
6  
7 # ABSTRACT: TermWeight vector operations using Lingua::TermWeight
8  
9 use Lingua::TermWeight;
10 use Lingua::TermWeight::WordSegmenter::SplitBySpace;
11 use Carp qw(croak);
12  
13 # Singleton calculator instance
14 my $Calc = do {
15 my $segmenter = Lingua::TermWeight::WordSegmenter::SplitBySpace->new;
16 Lingua::TermWeight->new(word_segmenter => $segmenter);
17 };
18  
19 # Compute TF for a single document
20 # Returns: {term => normalized_frequency}
21 sub compute_tf {
22 my ($self, $text, $normalize) = @_;
23 $normalize //= 1;
24 croak 'text is required' unless defined $text;
25 return $Calc->tf(document => $text, normalize => $normalize);
26 }
27  
28 # Compute IDF for a corpus of documents
29 # Arguments: arrayref of document texts
30 # Returns: {term => idf_weight}
31 sub compute_idf {
32 my ($self, $documents) = @_;
33 croak 'documents is required'
34 unless defined $documents && ref $documents eq 'ARRAY';
35 croak 'documents must not be empty' unless @$documents;
36 return $Calc->idf(documents => $documents);
37 }
38  
39 # Compute TF-IDF for a single document given pre-computed IDF
40 # Arguments: document text, idf hashref
41 # Returns: {term => tfidf_weight}
42 sub compute_tfidf {
43 my ($self, $text, $idf) = @_;
44 croak 'text is required' unless defined $text;
45 croak 'idf is required' unless defined $idf && ref $idf eq 'HASH';
46  
47 my $tf = $self->compute_tf($text);
48 return {
49 map { $_ => ($tf->{$_} // 0) * ($idf->{$_} // 0) } keys %$tf,
50 keys %$idf
51 };
52 }
53  
54 # Compute TF-IDF for multiple documents given pre-computed IDF
55 # Arguments: arrayref of document texts, idf hashref
56 # Returns: arrayref of {term => tfidf_weight}
57 sub compute_tfidf_batch {
58 my ($self, $documents, $idf) = @_;
59 croak 'documents is required'
60 unless defined $documents && ref $documents eq 'ARRAY';
61 croak 'idf is required' unless defined $idf && ref $idf eq 'HASH';
62  
63 my @tfidf;
64 for my $doc (@$documents) {
65 push @tfidf, $self->compute_tfidf($doc, $idf);
66 }
67 return \@tfidf;
68 }
69  
70 # Compute cosine similarity between two vectors
71 # Arguments: two hashrefs {term => weight}
72 # Returns: similarity score (0-1)
73 sub cosine_similarity {
74 my ($self, $vec1, $vec2) = @_;
75 croak 'vec1 is required' unless defined $vec1 && ref $vec1 eq 'HASH';
76 croak 'vec2 is required' unless defined $vec2 && ref $vec2 eq 'HASH';
77  
78 my $dot = 0;
79 my $mag1 = 0;
80 my $mag2 = 0;
81  
82 my %all_terms = map { $_ => 1 } keys %$vec1, keys %$vec2;
83 for my $term (keys %all_terms) {
84 my $v1 = $vec1->{$term} // 0;
85 my $v2 = $vec2->{$term} // 0;
86 $dot += $v1 * $v2;
87 $mag1 += $v1**2;
88 $mag2 += $v2**2;
89 }
90  
91 return 0 unless $mag1 && $mag2;
92 return $dot / (sqrt($mag1) * sqrt($mag2));
93 }
94  
95 # Batch cosine similarity: compute similarity of query vector against multiple document vectors
96 # Arguments: query_vector, arrayref of document_vectors
97 # Returns: arrayref of similarity scores
98 sub batch_cosine_similarity {
99 my ($self, $query_vec, $doc_vectors) = @_;
100 croak 'query_vec is required'
101 unless defined $query_vec && ref $query_vec eq 'HASH';
102 croak 'doc_vectors is required'
103 unless defined $doc_vectors && ref $doc_vectors eq 'ARRAY';
104  
105 return [map { $self->cosine_similarity($query_vec, $_) } @$doc_vectors];
106 }
107  
108 1;
109  
110 __END__
« KnowledgeStore/Store.pm ↑ Summary KnowledgeStore/Tools.pm »