Indexer.pm untested

filter: Statementstmt 0.0% 0 / 180 Branchbran 0.0% 0 / 66 Conditioncond 0.0% 0 / 29 MC/DCmcdc 0.0% 0 / 17 Subroutinesub 0.0% 0 / 30 Podpod 0.0% 0 / 24 totaltotal 0.0% 0 / 346 CC 34 SCAR 37.3
CC
34
Cov
0%
source2eval
156.0
index
56.0
mknorm
20.0
CRAP 41.6
1 package MARC::Indexer;
2  
3 use strict;
4 use warnings;
5  
6 $MARC::Indexer::VERSION = '0.04';
7  
8 use MARC::Indexer::Config;
9 use MARC::Loop qw(marcparse TAG VALREF);
10 use Unicode::Normalize;
11 use POSIX qw(strftime);
12  
13 sub new {
14 my $cls = shift;
15 bless { @_ }, $cls;
16 }
17  
18 sub compile {
19 my ($self) = @_;
20 if (!$self->{'is_compiled'}) {
21 my %source2eval;
22 $self->{'source2eval'} = \%source2eval;
23 foreach my $term (values %{ $self->{'terms'} }) {
24 my ($name, $source, $derived, $norm, $match, $repeat)
25 = @$term{qw(name source derived norm match repeat)};
26 my $eval;
27 if ($derived) {
28 $eval = derived2eval($term);
29 }
30 else {
31 $eval = $source2eval{$source} ||= source2eval($source);
32 $self->{'want'}{source2tag($source)} = 1;
33 }
34 $term->{'eval'} = $eval;
35 $term->{'norm'} = mknorm(@$norm);
36 }
37 }
38 $self->{'is_compiled'} = 1;
39 }
40  
41 sub mknorm {
42 return sub { $_ } if !@_ || !defined $_[0];
43 my @subs = map { __PACKAGE__->can('norm_'.trim($_)) || die "Unknown normalizer: $_" } @_;
44 return sub {
45 my ($v) = @_;
46 return if !defined $v;
47 foreach my $sub (@subs) {
48 $v = $sub->($v);
49 return if !defined $v;
50 }
51 return $v;
52 };
53 }
54  
55 sub derived2eval {
56 my ($term) = @_;
57 if ($term->{'permuted'}) {
58  
59 }
60 else {
61  
62 }
63 }
64  
65 sub source2tag {
66 return 'M' if $_[0] =~ /^::/;
67 return 'L' if $_[0] =~ /^(?:L|leader)\b/;
68 return $1 if $_[0] =~ /^raw:(...)/;
69 return substr($_[0], 0, 3);
70 }
71  
72 sub index {
73 my ($self, $marc) = @_;
74 my $marcref = ref($marc) ? $marc : \$marc;
75 # 1. Compile the indexing code if needed
76 $self->compile if !$self->{'is_compiled'};
77 # 2. Gather all fields used for indexing
78 my $data = marcdata($marcref, $self->{'want'});
79 # 3. Index each term
80 my %index;
81 while (my ($name, $term) = each %{ $self->{'terms'} }) {
82 next if $term->{'derived'};
83 my $tag = source2tag($term->{'source'});
84 my @vals = @{ $data->{$tag} or next };
85 my $eval = $term->{'eval'} or next;
86 my $norm = $term->{'norm'};
87 @vals = map { $eval->($_) } @vals;
88 @vals = map { $norm->($_) } @vals if $norm;
89 next if !@vals;
90 splice @vals, 1 if !$term->{'repeat'};
91 push @{ $index{$name} ||= [] }, @vals;
92 }
93 return \%index;
94 }
95  
96 sub marcdata {
97 my ($marcref, $want) = @_;
98 my %data;
99 $data{'M'} = [$marcref];
100 $data{'L'} = [substr($$marcref, 0, 24)] if $want->{'L'};
101 my $baseaddr = substr($$marcref, 12, 5) + 0;
102 pos($$marcref) = 24;
103 while ($$marcref =~ /\G([0-9A-Za-z]{3})([0-9]{4})([0-9]{5})/gc) {
104 push @{ $data{$1} ||= [] }, substr($$marcref, $baseaddr + $3, $2 - 1) if $want->{$1};
105 }
106 return \%data;
107 }
108  
109 sub source2eval {
110 my ($source) = @_;
111 if ($source =~ m{^(?:L|leader)/(\d+)$}) {
112 my $pos = $1;
113 return sub { substr($_, $pos, 1) };
114 }
115 elsif ($source =~ m{^(00[0-9A-Za-z])(?:/(\d+)(?:-(\d+))?)?$}) {
116 my ($tag, $b1, $bn) = ($1, $2, $3);
117 return sub { substr($_, $b1,$bn-$b1+1) } if defined $bn;
118 return sub { substr($_, $b1, 1) } if defined $b1;
119 return sub { $_ };
120 }
121 elsif ($source =~ m{^::(\w+)}) {
122 my $sub = __PACKAGE__->can('src_'.$1) || die "Unknown source function: $1";
123 return sub {
124 my ($marcref) = @_;
125 return $sub->($marcref);
126 }
127 }
128 elsif ($source =~ m{^(raw:)?([0-9A-Za-z]{3})\$(.+)$}) {
129 my ($raw, $tag, @subs) = ($1, $2, split //, $3);
130 my %want = map { $_ => 1 } @subs;
131 if ($raw) {
132 return sub {
133 my @pieces = ( substr($_, 0, 2) );
134 pos($_) = 2;
135 while (/(\x1f([^\x1f])[^\x1f]+)/gc) {
136 push @pieces, $1 if $want{$2};
137 }
138 return if @pieces < 2;
139 return join('', @pieces);
140 };
141 }
142 else {
143 return sub {
144 pos($_) = 2;
145 my @subs;
146 while (/\x1f([^\x1f])([^\x1f]+)/gc) {
147 push @subs, $2 if $want{$1};
148 }
149 return if !@subs;
150 return join(' ', @subs);
151 };
152 }
153 }
154 }
155  
156 sub trim {
157 local $_ = shift;
158 s/^\s+|\s+$//g;
159 return $_;
160 }
161  
162 sub norm_blank2hash {
163 local $_ = shift;
164 tr/ /#/;
165 return $_;
166 }
167  
168 sub norm_date2unix {
169 local $_ = shift;
170 /^([0-9]{4})-?([0-9]{2})-?([0-9]{2})T?([0-9]{2}):?([0-9]{2}):?([0-9]{2})/ or return 0;
171 return strftime('%s', $6, $5, $4, $3, $2-1, $1-1900);
172 }
173  
174 sub norm_nfc {
175 NFC(shift);
176 }
177  
178 sub norm_nfd {
179 NFD(shift);
180 }
181  
182 sub norm_alpha {
183 local $_ = shift;
184 tr{'}{}d;
185 tr{-=./()[],:;"?!}{ };
186 s/[^[:alpha] ]//g;
187 tr{ }{}s;
188 s/^ | $//g;
189 return $_
190 }
191  
192 sub norm_uppercase {
193 uc shift;
194 }
195  
196 sub norm_lowercase {
197 lc shift;
198 }
199  
200 sub norm_numeric {
201 local $_ = shift;
202 die if !/^[0-9]+$/;
203 $_ + 0; # This strips leading zeroes
204 }
205  
206 sub norm_trim {
207 local $_ = shift;
208 tr/ / /s;
209 s/^ | $//g;
210 return $_;
211 }
212  
213 sub norm_isbn {
214 local $_ = shift;
215 tr/-//d;
216 my $n = length;
217 die if $n != 10 && $n != 13;
218 norm_trim(uc $_);
219 }
220  
221 sub norm_delnf1 { unshift @_, 1; goto &_remove_non_filing_chars }
222 sub norm_delnf2 { unshift @_, 2; goto &_remove_non_filing_chars }
223 sub norm_cook {
224 local $_ = shift;
225 s/^..\x1f.//;
226 s/\x1f./ /g;
227 return $_;
228 }
229  
230 sub src_pub_date {
231 my ($marcref) = @_;
232 my ($leader, $fields) = marcparse($marcref);
233 my ($f008) = grep { $_->[TAG] eq '008' } @$fields;
234 return ($1) if defined($f008) && ${ $f008->[VALREF] } =~ /^.{7}([0-9]{4})/;
235 my ($f260) = grep { $_->[TAG] eq '260' } @$fields;
236 return ($1) if defined($f260) && ${ $f260->[VALREF] } =~ /\x1fc([0-9]{4})/;
237 return ();
238 }
239  
240 sub _remove_non_filing_chars {
241 my $ind = shift;
242 local $_ = shift;
243 my $n = substr($_, $ind-1, 1);
244 if ($n =~ /[1-9]/) {
245 s/(?<=\x1fa)([^\x1f]+)/length($1) <= $n ? $1 : substr($1, $n)/e;
246 }
247 return $_;
248 }
249  
250 1;
251  
252 =pod
253  
254 =head1 NAME
255  
256 MARC::Indexer - index MARC records
257  
258 =head1 SYNOPSIS
259  
260 $indexer = MARC::Indexer->new(
261 'index_points' => [
262 { 'name' => 'ctrlnum',
263 'tag' => '001' },
264 { 'name' => 'title',
265 'tag' => '245',
266 'extract' => 'subfields:abcdnp',
267 'normalize' => [qw(text nfd uc)],
268 },
269 { 'name' => 'subjtopic',
270 'tag' => '650',
271 'repeatable' => 1,
272 ...
273 },
274 ...
275 ],
276 );
277 $idx = $indexer->index($marc);
278 $idx_title = $idx->{'title'};
279  
280 =cut
281