blib/lib/LucyX/Suggester.pm

filter: Statementstmt 17.7% 20 / 113 Branchbran 0.0% 0 / 62 Conditioncond 0.0% 0 / 23 MC/DCmcdc 0.0% 0 / 13 Subroutinesub 66.6% 6 / 9 Podpod 100% 2 / 2 totaltotal 12.6% 28 / 222 CC 50 SCAR 68.2
CC
50
Cov
10%
suggest
1406.0
new
110.0
_analyze_terms
30.0
CRAP 918.6
↑ Summary
1 package LucyX::Suggester;
2 3 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 3 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 3 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 3 ▶use Data::Dump qw( dump );
Subroutine BEGIN: called CC=1 SCAR=0.0
6 3 ▶use Search::Tools;
Subroutine BEGIN: called CC=1 SCAR=0.0
7 3 ▶use Lucy;
Subroutine BEGIN: called CC=1 SCAR=0.0
8  
9 3 our $VERSION = '0.005';
10  
11 =head1 NAME
12  
13 LucyX::Suggester - suggest terms for Apache Lucy search
14  
15 =head1 SYNOPSIS
16  
17 use LucyX::Suggester;
18 my $suggester = LucyX::Suggester->new(
19 fields => [qw( foo bar )],
20 indexes => $list_of_indexes,
21 spellcheck => $search_tools_spellcheck,
22 limit => 10,
23 use_regex => 0,
24 max_length => 64,
25 );
26 my $suggestions = $suggester->suggest('quiK brwn fox');
27  
28 =head1 DESCRIPTION
29  
30 Inspired by the Solr Suggester feature, LucyX::Suggester
31 will return a list of suggested terms based on
32 actual terms in the specified index(es). Spellchecking
33 on query terms is performed with Search::Tools::SpellCheck,
34 which uses Text::Aspell.
35  
36 =head1 METHODS
37  
38 =head2 new( I<params> )
39  
40 Returns a new Suggester object. Supported I<params> include:
41  
42 =over
43  
44 =item fields I<arrayref>
45  
46 List of fields to limit Lexicon scans to.
47  
48 =item indexes I<arrayref>
49  
50 List of indexes to search within.
51  
52 =item spellcheck I<search_tools_spellcheck>
53  
54 An instance of L<Search::Tools::SpellCheck>. Set
55 this to indicate custom values for language, dictionary,
56 and other params to L<Search::Tools::SpellCheck>.
57  
58 =item limit I<n>
59  
60 Maximum number of suggestions to return. Defaults to 10.
61  
62 =item use_regex 1|0
63  
64 Use a simple regex when comparing terms. Defaults to false (0),
65 preferring index(). If your analyzer results in terms containing
66 multiple words (e.g. phrases) then B<use_regex> is probably what you want.
67  
68 =item max_length I<n>
69  
70 Set a max term length beyond which suggestions are trimmed
71 with substr(). Default is 64 characters. Set to 0 to disable.
72  
73 =back
74  
75 =cut
76  
77 0 ▶sub new {
Subroutine new: not called CC=10 SCAR=47.0
Pod documented
78 0 my $class = shift;
79 0 my %args = @_;
80 0 ▶ my $fields = delete $args{fields} || [];
Condition0%0/2
delete $args{'fields'} || []
l!l
00
MC/DC0%0/1
delete $args{'fields'} || []
delete $args{'fields'}
Truth table0%0/2
A
A= delete $args{'fields'}
Aresult
11
00
81 0 ▶ my $indexes = delete $args{indexes} or croak "indexes required";
Branch: 1 branch
BranchTrueFalseTotal
my $indexes = delete $args{"indexes"} or croak("indexes required") 0 0 0
82 0 my $spellcheck = delete $args{spellcheck};
83 0 ▶ my $limit = delete $args{limit} || 10;
Condition0%0/2
delete $args{'limit'} || 10
l!l
00
MC/DC0%0/1
delete $args{'limit'} || 10
delete $args{'limit'}
Truth table0%0/2
A
A= delete $args{'limit'}
Aresult
11
00
84 0 ▶ my $debug = delete $args{debug} || $ENV{LUCYX_DEBUG} || 0;
Condition0%0/5
delete $args{'debug'} || $ENV{'LUCYX_DEBUG'} || 0
l!l
00
delete $args{"debug"} or $ENV{"LUCYX_DEBUG"}
l!l&&r!l&&!r
000
MC/DC0%0/2
delete $args{'debug'} || $ENV{'LUCYX_DEBUG'} || 0
delete $args{"debug"} $ENV{"LUCYX_DEBUG"}
Truth table0%0/3
A or B
A= delete $args{"debug"}B= $ENV{"LUCYX_DEBUG"}
ABresult
1X1
011
000
85 0 ▶ my $use_regex = delete $args{use_regex} || 0;
Condition0%0/2
delete $args{'use_regex'} || 0
l!l
00
MC/DC0%0/1
delete $args{'use_regex'} || 0
delete $args{'use_regex'}
Truth table0%0/2
A
A= delete $args{'use_regex'}
Aresult
11
00
86 0 my $max_length = delete $args{max_length};
87 0 ▶ $max_length = 64 unless defined $max_length;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $max_length 0 0 0
88  
89 0 ▶ if (%args) {
Branch: 1 branch
BranchTrueFalseTotal
if (%args) 0 0 0
90 0 croak "Too many arguments to new(): " . dump( \%args );
91 }
92 0 ▶ if ( ref $indexes ne 'ARRAY' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $indexes ne "ARRAY") 0 0 0
93 0 croak "indexes must be an ARRAY ref";
94 }
95 0 return bless(
96 { fields => $fields,
97 indexes => $indexes,
98 spellcheck => $spellcheck,
99 limit => $limit,
100 debug => $debug,
101 use_regex => $use_regex,
102 max_length => $max_length,
103 },
104 $class
105 );
106 }
107  
108 =head2 suggest( I<query> )
109  
110 Returns arrayref of terms that match I<query>.
111  
112 =cut
113  
114 #
115 # I tried re-writing this to use a PolySearcher
116 # instead, in order to preserve phrases, etc
117 # but it was up to 3x slower.
118 #
119  
120 0 ▶sub suggest {
Subroutine suggest: not called CC=37 SCAR=72.5
Pod documented
121 0 my $self = shift;
122 0 my $query = shift;
123  
124 0 ▶ croak "query required" unless defined $query;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $query 0 0 0
125  
126 0 my $debug = $self->{debug};
127  
128 0 my $spellchecker = $self->{spellcheck};
129 0 ▶ if ( !$spellchecker ) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($spellchecker) 0 0 0
130 0 my $qparser = Search::Tools->parser( debug => $debug, );
131 0 $spellchecker = Search::Tools->spellcheck(
132 debug => $debug,
133 query_parser => $qparser,
134 );
135 }
136  
137 0 my $suggestions = $spellchecker->suggest($query);
138 0 my %terms;
139 0 for my $s (@$suggestions) {
140  
141 0 ▶ if ( !$s->{suggestions} ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not $s->{'suggestions'}) { } 0 0 0
142 0 $terms{ $s->{word} }++;
143 }
144 else {
145 0 for my $suggest ( @{ $s->{suggestions} } ) {
146 0 $terms{$suggest}++;
147 }
148 }
149 }
150  
151 # must analyze each query term and suggestion
152 # per-field, which we cache for performance
153 0 my %analyzed;
154  
155 # suggestions
156 my %matches;
157  
158 0 my @my_fields = @{ $self->{fields} };
159 0 my $use_regex = $self->{use_regex};
160 0 my $maxl = $self->{max_length};
161  
162 0 INDEX: for my $invindex ( @{ $self->{indexes} } ) {
163 0 my $reader = Lucy::Index::IndexReader->open( index => $invindex );
164 0 my $schema = $reader->get_schema();
165 0 my $fields;
166 0 ▶ if (@my_fields) {
Branch: 1 branch
BranchTrueFalseTotal
if (@my_fields) { } 0 0 0
167 0 $fields = \@my_fields;
168 }
169 else {
170 0 $fields = $schema->all_fields();
171 }
172 0 my $seg_readers = $reader->seg_readers;
173 0 SEG: for my $seg_reader (@$seg_readers) {
174 0 my $lex_reader
175 = $seg_reader->obtain('Lucy::Index::LexiconReader');
176 0 FIELD: for my $field (@$fields) {
177 0 $self->_analyze_terms( $schema, $field, \%analyzed, \%terms );
178  
179 # sort in order to seek() below.
180 0 my @to_check = sort keys %{ $analyzed{$field} };
181  
182 0 ▶ $debug and warn "$field=" . dump \@to_check;
Branch: 1 branch
BranchTrueFalseTotal
$debug and warn "$field=" . LucyX::Suggester::dump(\@to_check) 0 0 0
183  
184 0 my $lexicon = $lex_reader->lexicon( field => $field );
185 0 ▶ next FIELD unless $lexicon;
Branch: 1 branch
BranchTrueFalseTotal
unless $lexicon 0 0 0
186  
187 0 CHECK: for my $check_term (@to_check) {
188  
189 0 my $check_initial = substr( $check_term, 0, 1 );
190  
191 0 ▶ $debug and warn " seek($check_term)";
Branch: 1 branch
BranchTrueFalseTotal
$debug and warn " seek($check_term)" 0 0 0
192 0 $lexicon->seek($check_term);
193  
194 0 ▶ TERM: while ( defined( my $term = $lexicon->get_term ) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $term = $lexicon->get_term)) 0 0 0
195  
196 0 ▶ $debug and warn " $check_term -> $term";
Branch: 1 branch
BranchTrueFalseTotal
$debug and warn " $check_term -> $term" 0 0 0
197  
198 0 my $initial = substr( $term, 0, 1 );
199 0 ▶ if ( $initial and $initial gt $check_initial ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($initial and $initial gt $check_initial) 0 0 0
Condition0%0/3
$initial and $initial gt $check_initial
!ll&&!rl&&r
000
MC/DC0%0/2
$initial and $initial gt $check_initial
$initial $initial gt $check_initial
Truth table0%0/3
A and B
A= $initialB= $initial gt $check_initial
ABresult
0X0
100
111
200 0 ▶ $debug
Branch: 1 branch
BranchTrueFalseTotal
$debug and warn " reset: initial=$initial > check_initial=$check_initial" 0 0 0
201 and warn
202 " reset: initial=$initial > check_initial=$check_initial";
203 0 next CHECK;
204 }
205  
206 # TODO better weighting than simple freq?
207  
208 0 ▶ if ( !$use_regex
Branch: 2 branches
BranchTrueFalseTotal
if (not $use_regex and index($term, $check_term, 0) == 0) { } 0 0 0
elsif ($use_regex and $term =~ /\b\Q$check_term\E/) { } 0 0 0
Condition0%0/3
not $use_regex and index($term, $check_term, 0) == 0
!ll&&!rl&&r
000
MC/DC0%0/2
not $use_regex and index($term, $check_term, 0) == 0
not $use_regex index($term, $check_term, 0) == 0
Truth table0%0/3
not(A) and B
A= not $use_regexB= index($term, $check_term, 0) == 0
ABresult
0X0
100
111
209 && index( $term, $check_term, 0 ) == 0 )
210 {
211 0 my $freq = $lex_reader->doc_freq(
212 field => $field,
213 term => $term,
214 );
215 0 ▶ $debug and warn "ok term=$term [$freq]";
Branch: 1 branch
BranchTrueFalseTotal
$debug and warn "ok term=$term [$freq]" 0 0 0
216 $matches{
217 0 ▶ $maxl
Branch: 1 branch
BranchTrueFalseTotal
$maxl ? : 0 0 0
218 ? substr( $term, 0, $maxl )
219 : $term
220 } += $freq;
221  
222 # abort everything if we've hit our limit
223 0 ▶ if ( scalar( keys %matches ) >= $self->{limit} ) {
Branch: 1 branch
BranchTrueFalseTotal
if (scalar keys %matches >= $self->{"limit"}) 0 0 0
224 0 last INDEX;
225 }
226  
227 }
228 ▶ elsif ( $use_regex && $term =~ m/\b\Q$check_term/ ) {
Condition0%0/3
$use_regex and $term =~ /\b\Q$check_term\E/
!ll&&!rl&&r
000
MC/DC0%0/2
$use_regex and $term =~ /\b\Q$check_term\E/
$use_regex $term =~ /\b\Q$check_term\E/
Truth table0%0/3
A and B
A= $use_regexB= $term =~ /\b\Q$check_term\E/
ABresult
0X0
100
111
229 0 my $freq = $lex_reader->doc_freq(
230 field => $field,
231 term => $term,
232 );
233 0 ▶ $debug and warn "ok term=$term [$freq]";
Branch: 1 branch
BranchTrueFalseTotal
$debug and warn "ok term=$term [$freq]" 0 0 0
234 $matches{
235 0 ▶ $maxl
Branch: 1 branch
BranchTrueFalseTotal
$maxl ? : 0 0 0
236 ? substr(
237 $term, index( $term, $check_term, 0 ),
238 $maxl
239 )
240 : $term
241 } += $freq;
242  
243 # abort everything if we've hit our limit
244 0 ▶ if ( scalar( keys %matches ) >= $self->{limit} ) {
Branch: 1 branch
BranchTrueFalseTotal
if (scalar keys %matches >= $self->{"limit"}) 0 0 0
245 0 last INDEX;
246 }
247 }
248 else {
249 0 ▶ $debug
Branch: 1 branch
BranchTrueFalseTotal
$debug and warn " No match - skipping to next CHECK term" 0 0 0
250 and warn
251 " No match - skipping to next CHECK term";
252 0 next CHECK;
253 }
254  
255 0 ▶ last TERM unless $lexicon->next;
Branch: 1 branch
BranchTrueFalseTotal
unless $lexicon->next 0 0 0
256  
257 }
258 }
259 }
260 }
261 }
262  
263 # boost phrase matches
264 0 for my $m ( keys %matches ) {
265 0 ▶ next unless $m =~ m/ /;
Branch: 1 branch
BranchTrueFalseTotal
unless $m =~ / / 0 0 0
266 0 for my $m2 ( keys %matches ) {
267 0 ▶ if ( $m =~ m/\b\Q$m2\E\b/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($m =~ /\b\Q$m2\E\b/) 0 0 0
268 0 $matches{$m} += $matches{$m2};
269 }
270 }
271 }
272  
273 0 ▶ $debug and warn "matches=" . dump( \%matches );
Branch: 1 branch
BranchTrueFalseTotal
$debug and warn 'matches=' . LucyX::Suggester::dump(\%matches) 0 0 0
274  
275 return [
276 0 ▶ sort { $matches{$b} <=> $matches{$a} || $a cmp $b }
Condition0%0/3
$matches{$b} <=> $matches{$a} || $a cmp $b
l!l&&r!l&&!r
000
MC/DC0%0/2
$matches{$b} <=> $matches{$a} || $a cmp $b
$matches{$b} <=> $matches{$a} $a cmp $b
Truth table0%0/3
A || B
A= $matches{$b} <=> $matches{$a}B= $a cmp $b
ABresult
1X1
011
000
277 keys %matches
278 ];
279 }
280  
281 0 ▶sub _analyze_terms {
Subroutine _analyze_terms: not called CC=5 SCAR=34.0
282 0 my ( $self, $schema, $field, $analyzed, $terms ) = @_;
283  
284 0 ▶ return if exists $analyzed->{$field};
Branch: 1 branch
BranchTrueFalseTotal
if exists $analyzed->{$field} 0 0 0
285  
286 0 my $analyzer = $schema->fetch_analyzer($field);
287 0 for my $t ( keys %$terms ) {
288 0 ▶ my $baked = $analyzer ? $analyzer->split($t)->[0] : $t;
Branch: 1 branch
BranchTrueFalseTotal
$analyzer ? : 0 0 0
289 0 ▶ next if length $baked == 1; # too much noise
Branch: 1 branch
BranchTrueFalseTotal
if length $baked == 1 0 0 0
290 0 $analyzed->{$field}->{$baked} = $t;
291 }
292 }
293  
294 3 1;
295  
296 __END__
↑ Summary