buildlib/Lucy/Build/Binding/Analysis.pm

filter: Statementstmt 100% 127 / 127 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 14 / 14 Podpod 0.0% 0 / 12 totaltotal 92.1% 141 / 153 CC 2 SCAR 1.3
CC
2
Cov
100%
bind_token
2.0
BEGIN
1.0
BEGIN
1.0
CRAP 1.1
« buildlib/Lucy/Build/Binding/Store.pm ↑ Summary autogen/include/Clownfish/Class.h »
1 # Licensed to the Apache Software Foundation (ASF) under one or more
2 # contributor license agreements. See the NOTICE file distributed with
3 # this work for additional information regarding copyright ownership.
4 # The ASF licenses this file to You under the Apache License, Version 2.0
5 # (the "License"); you may not use this file except in compliance with
6 # the License. You may obtain a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 package Lucy::Build::Binding::Analysis;
16 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
17 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
18  
19 1 our $VERSION = '0.008000';
20 1 $VERSION = eval $VERSION;
21  
22 1 ▶sub bind_all {
Subroutine bind_all: called CC=1 SCAR=0.0
Pod undocumented
23 1 my $class = shift;
24 1 $class->bind_analyzer;
25 1 $class->bind_casefolder;
26 1 $class->bind_easyanalyzer;
27 1 $class->bind_inversion;
28 1 $class->bind_normalizer;
29 1 $class->bind_polyanalyzer;
30 1 $class->bind_regextokenizer;
31 1 $class->bind_snowballstemmer;
32 1 $class->bind_snowballstopfilter;
33 1 $class->bind_standardtokenizer;
34 1 $class->bind_token;
35 }
36  
37 1 ▶sub bind_analyzer {
Subroutine bind_analyzer: called CC=1 SCAR=0.0
Pod undocumented
38 1 my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
39 1 my $synopsis = <<'END_SYNOPSIS';
40 my $strings = $analyzer->split($text);
41  
42 package SimpleAnalyzer;
43 use base qw( Lucy::Analysis::Analyzer );
44 sub new {
45 return shift->SUPER::new;
46 }
47 sub equals {
48 my ( $self, $other ) = @_;
49 return $other->isa(__PACKAGE__);
50 }
51 sub transform {
52 my ( $self, $inversion ) = @_;
53 while ( my $token = $inversion->next ) {
54 my $text = $token->get_text;
55 # Transform text...
56 $token->set_text($text);
57 }
58 $inversion->reset;
59 return $inversion;
60 }
61  
62 package AnalyzerWithMemberVars;
63 use base qw( Lucy::Analysis::Analyzer );
64 our %foo;
65 sub new {
66 my $self = shift->SUPER::new;
67 return $self->init( { @_ } );
68 }
69 sub init {
70 my ( $self, $args ) = @_;
71 $foo{$$self} = $args->{foo};
72 return $self;
73 }
74 sub DESTROY {
75 my $self = shift;
76 delete $foo{$$self};
77 $self->SUPER::DESTROY;
78 }
79 sub equals {
80 my ( $self, $other ) = @_;
81 return $other->isa(__PACKAGE__)
82 && $foo{$$self} eq $foo{$$other};
83 }
84 sub dump {
85 my $self = shift;
86 my $dump = $self->SUPER::dump;
87 $dump->{foo} = $foo{$$self};
88 return $dump;
89 }
90 sub load {
91 my ( $self, $dump ) = @_;
92 my $loaded = $self->SUPER::load($dump);
93 return $loaded->init($dump);
94 }
95 END_SYNOPSIS
96 1 my $constructor = <<'END_CONSTRUCTOR';
97 =head2 new
98  
99 package MyAnalyzer;
100 use base qw( Lucy::Analysis::Analyzer );
101 our %foo;
102 sub new {
103 my $self = shift->SUPER::new;
104 my %args = @_;
105 $foo{$$self} = $args{foo};
106 return $self;
107 }
108  
109 Abstract constructor. Takes no arguments.
110 END_CONSTRUCTOR
111 1 $pod_spec->set_synopsis($synopsis);
112 1 $pod_spec->add_constructor( pod => $constructor );
113  
114 1 my $binding = Clownfish::CFC::Binding::Perl::Class->new(
115 parcel => "Lucy",
116 class_name => "Lucy::Analysis::Analyzer",
117 );
118 1 $binding->set_pod_spec($pod_spec);
119  
120 1 Clownfish::CFC::Binding::Perl::Class->register($binding);
121 }
122  
123 1 ▶sub bind_casefolder {
Subroutine bind_casefolder: called CC=1 SCAR=0.0
Pod undocumented
124 1 my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
125 1 my $synopsis = <<'END_SYNOPSIS';
126 my $case_folder = Lucy::Analysis::CaseFolder->new;
127  
128 my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
129 analyzers => [ $tokenizer, $case_folder, $stemmer ],
130 );
131 END_SYNOPSIS
132 1 my $constructor = <<'END_CONSTRUCTOR';
133 my $case_folder = Lucy::Analysis::CaseFolder->new;
134 END_CONSTRUCTOR
135 1 $pod_spec->set_synopsis($synopsis);
136 1 $pod_spec->add_constructor( alias => 'new', sample => $constructor );
137  
138 1 my $binding = Clownfish::CFC::Binding::Perl::Class->new(
139 parcel => "Lucy",
140 class_name => "Lucy::Analysis::CaseFolder",
141 );
142 1 $binding->set_pod_spec($pod_spec);
143 1 $binding->add_class_alias('KinoSearch::Analysis::CaseFolder');
144 1 $binding->add_class_alias('KinoSearch::Analysis::LCNormalizer');
145  
146 1 Clownfish::CFC::Binding::Perl::Class->register($binding);
147 }
148  
149 1 ▶sub bind_easyanalyzer {
Subroutine bind_easyanalyzer: called CC=1 SCAR=0.0
Pod undocumented
150 1 my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
151 1 my $synopsis = <<'END_SYNOPSIS';
152 my $schema = Lucy::Plan::Schema->new;
153 my $analyzer = Lucy::Analysis::EasyAnalyzer->new(
154 language => 'en',
155 );
156 my $type = Lucy::Plan::FullTextType->new(
157 analyzer => $analyzer,
158 );
159 $schema->spec_field( name => 'title', type => $type );
160 $schema->spec_field( name => 'content', type => $type );
161 END_SYNOPSIS
162 1 my $constructor = <<'END_CONSTRUCTOR';
163 my $analyzer = Lucy::Analysis::EasyAnalyzer->new(
164 language => 'es',
165 );
166 END_CONSTRUCTOR
167 1 $pod_spec->set_synopsis($synopsis);
168 1 $pod_spec->add_constructor( alias => 'new', sample => $constructor, );
169  
170 1 my $binding = Clownfish::CFC::Binding::Perl::Class->new(
171 parcel => "Lucy",
172 class_name => "Lucy::Analysis::EasyAnalyzer",
173 );
174 1 $binding->set_pod_spec($pod_spec);
175  
176 1 Clownfish::CFC::Binding::Perl::Class->register($binding);
177 }
178  
179 1 ▶sub bind_inversion {
Subroutine bind_inversion: called CC=1 SCAR=0.0
Pod undocumented
180 1 my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
181 1 my $synopsis = <<'END_SYNOPSIS';
182 my $result = Lucy::Analysis::Inversion->new;
183  
184 while (my $token = $inversion->next) {
185 $result->append($token);
186 }
187 END_SYNOPSIS
188 1 my $constructor = <<'END_CONSTRUCTOR';
189 my $inversion = Lucy::Analysis::Inversion->new(
190 $seed, # optional
191 );
192 END_CONSTRUCTOR
193 1 $pod_spec->set_synopsis($synopsis);
194 1 $pod_spec->add_constructor( alias => 'new', sample => $constructor, );
195  
196 1 my $xs = <<'END_XS';
197 MODULE = Lucy PACKAGE = Lucy::Analysis::Inversion
198  
199 SV*
200 new(...)
201 CODE:
202 {
203 static const XSBind_ParamSpec param_specs[1] = {
204 XSBIND_PARAM("text", false)
205 };
206 int32_t locations[1];
207 SV *text_sv = NULL;
208 lucy_Token *starter_token = NULL;
209  
210 XSBind_locate_args(aTHX_ &ST(0), 1, items, param_specs, locations, 1);
211  
212 text_sv = locations[0] < items ? ST(locations[0]) : NULL;
213 if (XSBind_sv_defined(aTHX_ text_sv)) {
214 STRLEN len;
215 char *text = SvPVutf8(text_sv, len);
216 STRLEN length = utf8_length((U8*)text, (U8*)text + len);
217 starter_token = lucy_Token_new(text, len, 0, length, 1.0, 1);
218 }
219  
220 RETVAL = CFISH_OBJ_TO_SV_NOINC(lucy_Inversion_new(starter_token));
221 CFISH_DECREF(starter_token);
222 }
223 OUTPUT: RETVAL
224 END_XS
225  
226 1 my $binding = Clownfish::CFC::Binding::Perl::Class->new(
227 parcel => "Lucy",
228 class_name => "Lucy::Analysis::Inversion",
229 );
230 1 $binding->set_pod_spec($pod_spec);
231 1 $binding->append_xs($xs);
232  
233 1 Clownfish::CFC::Binding::Perl::Class->register($binding);
234 }
235  
236 1 ▶sub bind_normalizer {
Subroutine bind_normalizer: called CC=1 SCAR=0.0
Pod undocumented
237 1 my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
238 1 my $synopsis = <<'END_SYNOPSIS';
239 my $normalizer = Lucy::Analysis::Normalizer->new;
240
241 my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
242 analyzers => [ $tokenizer, $normalizer, $stemmer ],
243 );
244 END_SYNOPSIS
245 1 my $constructor = <<'END_CONSTRUCTOR';
246 my $normalizer = Lucy::Analysis::Normalizer->new(
247 normalization_form => 'NFKC',
248 case_fold => 1,
249 strip_accents => 0,
250 );
251 END_CONSTRUCTOR
252 1 $pod_spec->set_synopsis($synopsis);
253 1 $pod_spec->add_constructor( alias => 'new', sample => $constructor );
254  
255 1 my $binding = Clownfish::CFC::Binding::Perl::Class->new(
256 parcel => "Lucy",
257 class_name => "Lucy::Analysis::Normalizer",
258 );
259 1 $binding->set_pod_spec($pod_spec);
260  
261 1 Clownfish::CFC::Binding::Perl::Class->register($binding);
262 }
263  
264 1 ▶sub bind_polyanalyzer {
Subroutine bind_polyanalyzer: called CC=1 SCAR=0.0
Pod undocumented
265 1 my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
266 1 my $synopsis = <<'END_SYNOPSIS';
267 my $schema = Lucy::Plan::Schema->new;
268 my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
269 analyzers => \@analyzers,
270 );
271 my $type = Lucy::Plan::FullTextType->new(
272 analyzer => $polyanalyzer,
273 );
274 $schema->spec_field( name => 'title', type => $type );
275 $schema->spec_field( name => 'content', type => $type );
276 END_SYNOPSIS
277 1 my $constructor = <<'END_CONSTRUCTOR';
278 my $tokenizer = Lucy::Analysis::StandardTokenizer->new;
279 my $normalizer = Lucy::Analysis::Normalizer->new;
280 my $stemmer = Lucy::Analysis::SnowballStemmer->new( language => 'en' );
281 my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
282 analyzers => [ $tokenizer, $normalizer, $stemmer, ], );
283 END_CONSTRUCTOR
284 1 $pod_spec->set_synopsis($synopsis);
285 1 $pod_spec->add_constructor( alias => 'new', sample => $constructor );
286  
287 1 my $binding = Clownfish::CFC::Binding::Perl::Class->new(
288 parcel => "Lucy",
289 class_name => "Lucy::Analysis::PolyAnalyzer",
290 );
291 1 $binding->set_pod_spec($pod_spec);
292 1 $binding->add_class_alias('KinoSearch::Analysis::PolyAnalyzer');
293  
294 1 Clownfish::CFC::Binding::Perl::Class->register($binding);
295 }
296  
297 1 ▶sub bind_regextokenizer {
Subroutine bind_regextokenizer: called CC=1 SCAR=0.0
Pod undocumented
298 1 my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
299 1 my $synopsis = <<'END_SYNOPSIS';
300 my $whitespace_tokenizer
301 = Lucy::Analysis::RegexTokenizer->new( pattern => '\S+' );
302  
303 # or...
304 my $word_char_tokenizer
305 = Lucy::Analysis::RegexTokenizer->new( pattern => '\w+' );
306  
307 # or...
308 my $apostrophising_tokenizer = Lucy::Analysis::RegexTokenizer->new;
309  
310 # Then... once you have a tokenizer, put it into a PolyAnalyzer:
311 my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
312 analyzers => [ $word_char_tokenizer, $normalizer, $stemmer ], );
313 END_SYNOPSIS
314 1 my $constructor = <<'END_CONSTRUCTOR';
315 my $word_char_tokenizer = Lucy::Analysis::RegexTokenizer->new(
316 pattern => '\w+', # required
317 );
318 END_CONSTRUCTOR
319 1 $pod_spec->set_synopsis($synopsis);
320 1 $pod_spec->add_constructor( alias => 'new', sample => $constructor );
321  
322 1 my $binding = Clownfish::CFC::Binding::Perl::Class->new(
323 parcel => "Lucy",
324 class_name => "Lucy::Analysis::RegexTokenizer",
325 );
326 1 $binding->bind_constructor( alias => '_new' );
327 1 $binding->set_pod_spec($pod_spec);
328 1 $binding->add_class_alias('KinoSearch::Analysis::Tokenizer');
329  
330 1 Clownfish::CFC::Binding::Perl::Class->register($binding);
331 }
332  
333 1 ▶sub bind_snowballstemmer {
Subroutine bind_snowballstemmer: called CC=1 SCAR=0.0
Pod undocumented
334 1 my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
335 1 my $synopsis = <<'END_SYNOPSIS';
336 my $stemmer = Lucy::Analysis::SnowballStemmer->new( language => 'es' );
337
338 my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
339 analyzers => [ $tokenizer, $normalizer, $stemmer ],
340 );
341  
342 This class is a wrapper around the Snowball stemming library, so it supports
343 the same languages.
344 END_SYNOPSIS
345 1 my $constructor = <<'END_CONSTRUCTOR';
346 my $stemmer = Lucy::Analysis::SnowballStemmer->new( language => 'es' );
347 END_CONSTRUCTOR
348 1 $pod_spec->set_synopsis($synopsis);
349 1 $pod_spec->add_constructor( alias => 'new', sample => $constructor );
350  
351 1 my $binding = Clownfish::CFC::Binding::Perl::Class->new(
352 parcel => "Lucy",
353 class_name => "Lucy::Analysis::SnowballStemmer",
354 );
355 1 $binding->set_pod_spec($pod_spec);
356 1 $binding->add_class_alias('KinoSearch::Analysis::Stemmer');
357  
358 1 Clownfish::CFC::Binding::Perl::Class->register($binding);
359 }
360  
361 1 ▶sub bind_snowballstopfilter {
Subroutine bind_snowballstopfilter: called CC=1 SCAR=0.0
Pod undocumented
362 1 my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
363 1 my $synopsis = <<'END_SYNOPSIS';
364 my $stopfilter = Lucy::Analysis::SnowballStopFilter->new(
365 language => 'fr',
366 );
367 my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
368 analyzers => [ $tokenizer, $normalizer, $stopfilter, $stemmer ],
369 );
370 END_SYNOPSIS
371 1 my $constructor = <<'END_CONSTRUCTOR';
372 my $stopfilter = Lucy::Analysis::SnowballStopFilter->new(
373 language => 'de',
374 );
375
376 # or...
377 my $stopfilter = Lucy::Analysis::SnowballStopFilter->new(
378 stoplist => \%stoplist,
379 );
380 END_CONSTRUCTOR
381 1 $pod_spec->set_synopsis($synopsis);
382 1 $pod_spec->add_constructor( alias => 'new', sample => $constructor );
383  
384 1 my $binding = Clownfish::CFC::Binding::Perl::Class->new(
385 parcel => "Lucy",
386 class_name => "Lucy::Analysis::SnowballStopFilter",
387 );
388 1 $binding->set_pod_spec($pod_spec);
389 1 $binding->add_class_alias('KinoSearch::Analysis::Stopalizer');
390  
391 1 Clownfish::CFC::Binding::Perl::Class->register($binding);
392 }
393  
394 1 ▶sub bind_standardtokenizer {
Subroutine bind_standardtokenizer: called CC=1 SCAR=0.0
Pod undocumented
395 1 my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
396 1 my $synopsis = <<'END_SYNOPSIS';
397 my $tokenizer = Lucy::Analysis::StandardTokenizer->new;
398  
399 # Then... once you have a tokenizer, put it into a PolyAnalyzer:
400 my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
401 analyzers => [ $tokenizer, $normalizer, $stemmer ], );
402 END_SYNOPSIS
403 1 my $constructor = <<'END_CONSTRUCTOR';
404 my $tokenizer = Lucy::Analysis::StandardTokenizer->new;
405 END_CONSTRUCTOR
406 1 $pod_spec->set_synopsis($synopsis);
407 1 $pod_spec->add_constructor( alias => 'new', sample => $constructor );
408  
409 1 my $binding = Clownfish::CFC::Binding::Perl::Class->new(
410 parcel => "Lucy",
411 class_name => "Lucy::Analysis::StandardTokenizer",
412 );
413 1 $binding->set_pod_spec($pod_spec);
414  
415 1 Clownfish::CFC::Binding::Perl::Class->register($binding);
416 }
417  
418 1 ▶sub bind_token {
Subroutine bind_token: called CC=2 SCAR=6.9
Pod undocumented
419 1 my @hand_rolled = qw(
420 Set_Text
421 Get_Text
422 );
423  
424 1 my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
425 1 my $synopsis = <<'END_SYNOPSIS';
426 my $token = Lucy::Analysis::Token->new(
427 text => 'blind',
428 start_offset => 8,
429 end_offset => 13,
430 );
431  
432 $token->set_text('mice');
433 END_SYNOPSIS
434 1 my $constructor_pod = <<'END_CONSTRUCTOR_POD';
435 =head2 new
436  
437 my $token = Lucy::Analysis::Token->new(
438 text => $text, # required
439 start_offset => $start_offset, # required
440 end_offset => $end_offset, # required
441 boost => 1.0, # optional
442 pos_inc => 1, # optional
443 );
444  
445 =over
446  
447 =item *
448  
449 B<text> - A string.
450  
451 =item *
452  
453 B<start_offset> - Start offset into the original document in Unicode
454 code points.
455  
456 =item *
457  
458 B<start_offset> - End offset into the original document in Unicode
459 code points.
460  
461 =item *
462  
463 B<boost> - Per-token weight.
464  
465 =item *
466  
467 B<pos_inc> - Position increment for phrase matching.
468  
469 =back
470 END_CONSTRUCTOR_POD
471 1 my $get_text_pod = <<'END_GET_TEXT_POD';
472 =head2 get_text
473  
474 my $text = $token->get_text;
475  
476 Get the token's text.
477 END_GET_TEXT_POD
478 1 my $set_text_pod = <<'END_SET_TEXT_POD';
479 =head2 set_text
480  
481 $token->set_text($text);
482  
483 Set the token's text.
484 END_SET_TEXT_POD
485 1 $pod_spec->set_synopsis($synopsis);
486 1 $pod_spec->add_constructor( alias => 'new', pod => $constructor_pod );
487 1 $pod_spec->add_method( alias => 'Get_Text', pod => $get_text_pod);
488 1 $pod_spec->add_method( alias => 'Set_Text', pod => $set_text_pod);
489  
490 1 my $xs = <<'END_XS';
491 MODULE = Lucy PACKAGE = Lucy::Analysis::Token
492  
493 SV*
494 new(either_sv, ...)
495 SV *either_sv;
496 CODE:
497 {
498 static const XSBind_ParamSpec param_specs[5] = {
499 XSBIND_PARAM("text", true),
500 XSBIND_PARAM("start_offset", true),
501 XSBIND_PARAM("end_offset", true),
502 XSBIND_PARAM("pos_inc", false),
503 XSBIND_PARAM("boost", false)
504 };
505 int32_t locations[5];
506 uint32_t start_off = 0;
507 uint32_t end_off = 0;
508 int32_t pos_inc = 1;
509 float boost = 1.0f;
510 STRLEN len = 0;
511 char *text = NULL;
512 lucy_Token *self = NULL;
513  
514 XSBind_locate_args(aTHX_ &ST(0), 1, items, param_specs, locations, 5);
515  
516 text = SvPVutf8(ST(locations[0]), len);
517 start_off = (uint32_t)SvUV(ST(locations[1]));
518 end_off = (uint32_t)SvUV(ST(locations[2]));
519 pos_inc = locations[3] < items ? (int32_t)SvIV(ST(locations[3])) : 1;
520 boost = locations[4] < items ? (float)SvNV(ST(locations[4])) : 1.0f;
521  
522 self = (lucy_Token*)XSBind_new_blank_obj(aTHX_ either_sv);
523 lucy_Token_init(self, text, len, start_off, end_off, boost,
524 pos_inc);
525 RETVAL = CFISH_OBJ_TO_SV_NOINC(self);
526 }
527 OUTPUT: RETVAL
528  
529 SV*
530 get_text(self)
531 lucy_Token *self;
532 CODE:
533 RETVAL = newSVpvn(LUCY_Token_Get_Text(self), LUCY_Token_Get_Len(self));
534 SvUTF8_on(RETVAL);
535 OUTPUT: RETVAL
536  
537 void
538 set_text(self, sv)
539 lucy_Token *self;
540 SV *sv;
541 PPCODE:
542 {
543 STRLEN len;
544 char *ptr = SvPVutf8(sv, len);
545 LUCY_Token_Set_Text(self, ptr, len);
546 }
547 END_XS
548  
549 1 my $binding = Clownfish::CFC::Binding::Perl::Class->new(
550 parcel => "Lucy",
551 class_name => "Lucy::Analysis::Token",
552 );
553 1 $binding->set_pod_spec($pod_spec);
554 1 $binding->append_xs($xs);
555 1 $binding->exclude_method($_) for @hand_rolled;
556 1 $binding->exclude_constructor;
557  
558 1 Clownfish::CFC::Binding::Perl::Class->register($binding);
559 }
560  
561 1 1;
« buildlib/Lucy/Build/Binding/Store.pm ↑ Summary autogen/include/Clownfish/Class.h »