line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package Plucene::Index::DocumentWriter; |
2
|
|
|
|
|
|
|
|
3
|
|
|
|
|
|
|
=head1 NAME |
4
|
|
|
|
|
|
|
|
5
|
|
|
|
|
|
|
Plucene::Index::DocumentWriter - the document writer |
6
|
|
|
|
|
|
|
|
7
|
|
|
|
|
|
|
=head1 SYNOPSIS |
8
|
|
|
|
|
|
|
|
9
|
|
|
|
|
|
|
my $writer = Plucene::Index::DocumentWriter |
10
|
|
|
|
|
|
|
->new($directory, $analyser, $max_field_length); |
11
|
|
|
|
|
|
|
|
12
|
|
|
|
|
|
|
$writer->add_document($segment, $doc); |
13
|
|
|
|
|
|
|
|
14
|
|
|
|
|
|
|
=head1 DESCRIPTION |
15
|
|
|
|
|
|
|
|
16
|
|
|
|
|
|
|
This is the document writer class. |
17
|
|
|
|
|
|
|
|
18
|
|
|
|
|
|
|
=head2 METHODS |
19
|
|
|
|
|
|
|
|
20
|
|
|
|
|
|
|
=cut |
21
|
|
|
|
|
|
|
|
22
|
17
|
|
|
17
|
|
102
|
use strict; |
|
17
|
|
|
|
|
54
|
|
|
17
|
|
|
|
|
629
|
|
23
|
17
|
|
|
17
|
|
95
|
use warnings; |
|
17
|
|
|
|
|
42
|
|
|
17
|
|
|
|
|
566
|
|
24
|
|
|
|
|
|
|
|
25
|
17
|
|
|
17
|
|
994
|
use File::Slurp; |
|
17
|
|
|
|
|
13393
|
|
|
17
|
|
|
|
|
1213
|
|
26
|
17
|
|
|
17
|
|
627
|
use Plucene::Index::FieldInfos; |
|
17
|
|
|
|
|
39
|
|
|
17
|
|
|
|
|
403
|
|
27
|
17
|
|
|
17
|
|
12506
|
use Plucene::Index::FieldsWriter; |
|
17
|
|
|
|
|
50
|
|
|
17
|
|
|
|
|
561
|
|
28
|
17
|
|
|
17
|
|
751
|
use Plucene::Index::Term; |
|
17
|
|
|
|
|
40
|
|
|
17
|
|
|
|
|
174
|
|
29
|
17
|
|
|
17
|
|
2194
|
use Plucene::Index::TermInfo; |
|
17
|
|
|
|
|
36
|
|
|
17
|
|
|
|
|
131
|
|
30
|
17
|
|
|
17
|
|
1094
|
use Plucene::Index::TermInfosWriter; |
|
17
|
|
|
|
|
42
|
|
|
17
|
|
|
|
|
351
|
|
31
|
17
|
|
|
17
|
|
10460
|
use Plucene::Search::Similarity; |
|
17
|
|
|
|
|
47
|
|
|
17
|
|
|
|
|
446
|
|
32
|
17
|
|
|
17
|
|
109
|
use Plucene::Store::OutputStream; |
|
17
|
|
|
|
|
31
|
|
|
17
|
|
|
|
|
520
|
|
33
|
|
|
|
|
|
|
|
34
|
17
|
|
|
17
|
|
15275
|
use IO::Scalar; |
|
17
|
|
|
|
|
77335
|
|
|
17
|
|
|
|
|
25593
|
|
35
|
|
|
|
|
|
|
|
36
|
|
|
|
|
|
|
=head2 new |
37
|
|
|
|
|
|
|
|
38
|
|
|
|
|
|
|
my $writer = Plucene::Index::DocumentWriter |
39
|
|
|
|
|
|
|
->new($directory, $analyser, $max_field_length); |
40
|
|
|
|
|
|
|
|
41
|
|
|
|
|
|
|
This will create a new Plucene::Index::DocumentWriter object with the passed |
42
|
|
|
|
|
|
|
in arguments. |
43
|
|
|
|
|
|
|
|
44
|
|
|
|
|
|
|
=cut |
45
|
|
|
|
|
|
|
|
46
|
|
|
|
|
|
|
sub new { |
47
|
257
|
|
|
257
|
1
|
667
|
my ($self, $d, $a, $mfl) = @_; |
48
|
257
|
|
|
|
|
2196
|
bless { |
49
|
|
|
|
|
|
|
directory => $d, |
50
|
|
|
|
|
|
|
analyzer => $a, |
51
|
|
|
|
|
|
|
max_field_length => $mfl, |
52
|
|
|
|
|
|
|
postings => {}, |
53
|
|
|
|
|
|
|
}, $self; |
54
|
|
|
|
|
|
|
} |
55
|
|
|
|
|
|
|
|
56
|
|
|
|
|
|
|
=head2 add_document |
57
|
|
|
|
|
|
|
|
58
|
|
|
|
|
|
|
$writer->add_document($segment, $doc); |
59
|
|
|
|
|
|
|
|
60
|
|
|
|
|
|
|
=cut |
61
|
|
|
|
|
|
|
|
62
|
|
|
|
|
|
|
sub add_document { |
63
|
257
|
|
|
257
|
1
|
516
|
my ($self, $segment, $doc) = @_; |
64
|
257
|
|
|
|
|
2348
|
my $fi = Plucene::Index::FieldInfos->new(); |
65
|
257
|
|
|
|
|
1067
|
$fi->add($doc); |
66
|
257
|
|
|
|
|
1904
|
$fi->write("$self->{directory}/$segment.fnm"); |
67
|
257
|
|
|
|
|
77024
|
$self->{field_infos} = $fi; |
68
|
|
|
|
|
|
|
|
69
|
257
|
|
|
|
|
2582
|
my $fw = |
70
|
|
|
|
|
|
|
Plucene::Index::FieldsWriter->new($self->{directory}, $segment, $fi); |
71
|
257
|
|
|
|
|
1609
|
$fw->add_document($doc); |
72
|
257
|
|
|
|
|
878
|
$self->{postings} = {}; |
73
|
257
|
|
|
|
|
11799
|
$self->{field_lengths} = []; |
74
|
257
|
|
|
|
|
925
|
$self->_invert_document($doc); |
75
|
295670
|
50
|
|
|
|
926892
|
my @postings = sort { |
76
|
257
|
|
|
|
|
8736
|
$a->{term}->{field} cmp $b->{term}->{field} |
77
|
|
|
|
|
|
|
|| $a->{term}->{text} cmp $b->{term}->{text} |
78
|
257
|
|
|
|
|
697
|
} values %{ $self->{postings} }; |
79
|
|
|
|
|
|
|
|
80
|
257
|
|
|
|
|
1661
|
$self->_write_postings($segment, @postings); |
81
|
257
|
|
|
|
|
1257
|
$self->_write_norms($doc, $segment); |
82
|
|
|
|
|
|
|
} |
83
|
|
|
|
|
|
|
|
84
|
|
|
|
|
|
|
sub _invert_document { |
85
|
257
|
|
|
257
|
|
457
|
my ($self, $doc) = @_; |
86
|
257
|
|
|
|
|
1140
|
for my $field (grep $_->is_indexed, $doc->fields) { |
87
|
370
|
|
|
|
|
2637
|
my $name = $field->name; |
88
|
370
|
|
|
|
|
2609
|
my $fn = $self->{field_infos}->field_number($name); |
89
|
370
|
|
|
|
|
2912
|
my $pos = $self->{field_lengths}->[$fn]; |
90
|
370
|
100
|
|
|
|
1182
|
if (!$field->is_tokenized) { |
91
|
48
|
|
|
|
|
339
|
$self->_add_position($name, $field->string, $pos++); |
92
|
|
|
|
|
|
|
} else { |
93
|
322
|
|
33
|
|
|
2483
|
my $reader = $field->reader |
94
|
|
|
|
|
|
|
|| IO::Scalar->new(\$field->{string}); |
95
|
322
|
|
|
|
|
48826
|
my $stream = $self->{analyzer}->tokenstream({ |
96
|
|
|
|
|
|
|
field => $name, |
97
|
|
|
|
|
|
|
reader => $reader |
98
|
|
|
|
|
|
|
}); |
99
|
322
|
|
|
|
|
5087
|
while (my $t = $stream->next) { |
100
|
143694
|
|
|
|
|
388036
|
$self->_add_position($name, $t->text, $pos++); |
101
|
143694
|
50
|
|
|
|
1659140
|
last if $pos > $self->{max_field_length}; |
102
|
|
|
|
|
|
|
} |
103
|
|
|
|
|
|
|
} |
104
|
370
|
|
|
|
|
11938
|
$self->{field_lengths}->[$fn] = $pos; |
105
|
|
|
|
|
|
|
} |
106
|
|
|
|
|
|
|
} |
107
|
|
|
|
|
|
|
|
108
|
|
|
|
|
|
|
sub _add_position { |
109
|
143742
|
|
|
143742
|
|
844155
|
my ($self, $field, $text, $pos) = @_; |
110
|
143742
|
|
|
|
|
397962
|
my $ti = $self->{postings}->{"$field\0$text"}; |
111
|
143742
|
100
|
|
|
|
314918
|
if ($ti) { |
112
|
110866
|
|
|
|
|
376041
|
$ti->{positions}->[ $ti->freq ] = $pos; |
113
|
110866
|
|
|
|
|
601425
|
$ti->{freq}++; |
114
|
110866
|
|
|
|
|
205863
|
return; |
115
|
|
|
|
|
|
|
} |
116
|
32876
|
|
|
|
|
159919
|
$self->{postings}->{"$field\0$text"} = Plucene::Index::Posting->new({ |
117
|
|
|
|
|
|
|
term => Plucene::Index::Term->new({ field => $field, text => $text }), |
118
|
|
|
|
|
|
|
positions => [$pos], |
119
|
|
|
|
|
|
|
freq => 1, |
120
|
|
|
|
|
|
|
}); |
121
|
|
|
|
|
|
|
} |
122
|
|
|
|
|
|
|
|
123
|
|
|
|
|
|
|
sub _write_postings { |
124
|
257
|
|
|
257
|
|
4464
|
my ($self, $segment, @postings) = @_; |
125
|
257
|
|
|
|
|
388
|
my (@freqs, @proxs); |
126
|
257
|
|
|
|
|
3009
|
my $tis = |
127
|
|
|
|
|
|
|
Plucene::Index::TermInfosWriter->new($self->{directory}, $segment, |
128
|
|
|
|
|
|
|
$self->{field_infos}); |
129
|
257
|
|
|
|
|
1590
|
my $ti = Plucene::Index::TermInfo->new(); |
130
|
|
|
|
|
|
|
|
131
|
257
|
|
|
|
|
2937
|
for my $posting (@postings) { |
132
|
32876
|
|
|
|
|
116267
|
$ti->doc_freq(1); |
133
|
32876
|
|
|
|
|
219712
|
$ti->freq_pointer(scalar @freqs); |
134
|
32876
|
|
|
|
|
199531
|
$ti->prox_pointer(scalar @proxs); |
135
|
|
|
|
|
|
|
|
136
|
32876
|
|
|
|
|
201924
|
$tis->add($posting->term, $ti); |
137
|
32876
|
|
|
|
|
87526
|
my $f = $posting->freq; |
138
|
32876
|
100
|
|
|
|
188632
|
push @freqs, ($f == 1) ? 1 : (0, $f); |
139
|
32876
|
|
|
|
|
39655
|
my $last_pos = 0; |
140
|
32876
|
|
|
|
|
85168
|
my $positions = $posting->positions; |
141
|
32876
|
|
|
|
|
167254
|
for my $j (0 .. $f - 1) { |
142
|
143742
|
|
100
|
|
|
311526
|
my $pos = $positions->[$j] || 0; |
143
|
143742
|
|
|
|
|
182502
|
push @proxs, $pos - $last_pos; |
144
|
143742
|
|
|
|
|
251355
|
$last_pos = $pos; |
145
|
|
|
|
|
|
|
} |
146
|
|
|
|
|
|
|
} |
147
|
|
|
|
|
|
|
|
148
|
257
|
|
|
|
|
13739
|
write_file("$self->{directory}/$segment.frq" => pack('(w)*', @freqs)); |
149
|
257
|
|
|
|
|
116111
|
write_file("$self->{directory}/$segment.prx" => pack('(w)*', @proxs)); |
150
|
257
|
|
|
|
|
55470
|
$tis->break_ref; |
151
|
|
|
|
|
|
|
} |
152
|
|
|
|
|
|
|
|
153
|
|
|
|
|
|
|
sub _write_norms { |
154
|
257
|
|
|
257
|
|
713
|
my ($self, $doc, $segment) = @_; |
155
|
257
|
|
|
|
|
1337
|
for my $field (grep $_->is_indexed, $doc->fields) { |
156
|
370
|
|
|
|
|
3381
|
my $fn = $self->{field_infos}->field_number($field->name); |
157
|
370
|
50
|
|
|
|
3239
|
warn "Couldn't find field @{[ $field->name ]} in list [ @{[ map |
|
0
|
|
|
|
|
0
|
|
|
0
|
|
|
|
|
0
|
|
158
|
|
|
|
|
|
|
$_->name, $self->{field_infos}->fields]}]" unless $fn >= 0; |
159
|
370
|
|
|
|
|
3145
|
my $norm = |
160
|
|
|
|
|
|
|
Plucene::Store::OutputStream->new("$self->{directory}/$segment.f$fn"); |
161
|
370
|
|
|
|
|
1166
|
my $val = $self->{field_lengths}[$fn]; |
162
|
370
|
|
|
|
|
3359
|
my $norm_val = Plucene::Search::Similarity->norm($val); |
163
|
370
|
|
|
|
|
1672
|
$norm->print(chr($norm_val)); |
164
|
|
|
|
|
|
|
} |
165
|
|
|
|
|
|
|
} |
166
|
|
|
|
|
|
|
|
167
|
|
|
|
|
|
|
package Plucene::Index::Posting; |
168
|
|
|
|
|
|
|
|
169
|
17
|
|
|
17
|
|
188
|
use base 'Class::Accessor::Fast'; |
|
17
|
|
|
|
|
120
|
|
|
17
|
|
|
|
|
2473
|
|
170
|
|
|
|
|
|
|
|
171
|
|
|
|
|
|
|
__PACKAGE__->mk_accessors(qw( term freq positions )); |
172
|
|
|
|
|
|
|
|
173
|
|
|
|
|
|
|
1; |