line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package Plucene::Analysis::WhitespaceTokenizer; |
2
|
|
|
|
|
|
|
|
3
|
|
|
|
|
|
|
=head1 NAME |
4
|
|
|
|
|
|
|
|
5
|
|
|
|
|
|
|
Plucene::Analysis::WhitespaceTokenizer - white space tokenizer |
6
|
|
|
|
|
|
|
|
7
|
|
|
|
|
|
|
=head1 SYNOPSIS |
8
|
|
|
|
|
|
|
|
9
|
|
|
|
|
|
|
# isa Plucene::Analysis::CharTokenizer |
10
|
|
|
|
|
|
|
|
11
|
|
|
|
|
|
|
=head1 DESCRIPTION |
12
|
|
|
|
|
|
|
|
13
|
|
|
|
|
|
|
A WhitespaceTokenizer is a tokenizer that divides text at whitespace. |
14
|
|
|
|
|
|
|
Adjacent sequences of non-Whitespace characters form tokens. |
15
|
|
|
|
|
|
|
|
16
|
|
|
|
|
|
|
=head1 METHODS |
17
|
|
|
|
|
|
|
|
18
|
|
|
|
|
|
|
=cut |
19
|
|
|
|
|
|
|
|
20
|
5
|
|
|
5
|
|
94
|
use strict; |
|
5
|
|
|
|
|
11
|
|
|
5
|
|
|
|
|
161
|
|
21
|
5
|
|
|
5
|
|
28
|
use warnings; |
|
5
|
|
|
|
|
12
|
|
|
5
|
|
|
|
|
127
|
|
22
|
|
|
|
|
|
|
|
23
|
5
|
|
|
5
|
|
26
|
use base 'Plucene::Analysis::CharTokenizer'; |
|
5
|
|
|
|
|
16
|
|
|
5
|
|
|
|
|
619
|
|
24
|
|
|
|
|
|
|
|
25
|
596
|
|
|
596
|
1
|
2925
|
sub token_re { qr/\S+/ } |
26
|
|
|
|
|
|
|
|
27
|
|
|
|
|
|
|
1; |