| 1 | package MARC::Matcher; | ||
| 2 | |||
| 3 | use strict; | ||
| 4 | use warnings; | ||
| 5 | |||
| 6 | use Text::LevenshteinXS qw(distance); | ||
| 7 | use MARC::Loop qw(marcparse marcfield marcbuild TAG VALREF SUBS SUB_ID SUB_VALREF); | ||
| 8 | |||
| 9 | use vars qw($VERSION $a $b); # Silly warning... | ||
| 10 | |||
| 11 | $VERSION = '0.02'; | ||
| 12 | |||
| 13 | # $matcher = MARC::Matcher->new( | ||
| 14 | # 'matchpoints' => $file_or_file_handle_or_hash, | ||
| 15 | # ); | ||
| 16 | # $matcher->target($marc); | ||
| 17 | # while (<STDIN>) { | ||
| 18 | # ($score, @details) = $matcher->match($_); | ||
| 19 | # print if $score > 50; | ||
| 20 | # } | ||
| 21 | |||
| 22 | sub new { | ||
| 23 | my $cls = shift; | ||
| 24 | bless { @_ }, $cls; | ||
| 25 | } | ||
| 26 | |||
| 27 | sub target { | ||
| 28 | my $self = shift; | ||
| 29 | return ${ $self->{'tref'} } if !@_; | ||
| 30 | delete $self->{'mkey2tvals'}; | ||
| 31 | my $marc = shift; | ||
| 32 | $self->{'tref'} = ref($marc) ? $marc : \$marc; | ||
| 33 | return $marc; | ||
| 34 | } | ||
| 35 | |||
| 36 | sub matchpoints { | ||
| 37 | my $self = shift; | ||
| 38 | my $m = $self->{'matchpoints'} ||= []; | ||
| 39 | return @$m if !@_; | ||
| 40 | @$m = (@_ == 1 && ref($_[0]) eq 'ARRAY') ? @{ shift() } : @_; | ||
| 41 | return @$m; | ||
| 42 | } | ||
| 43 | |||
| 44 | sub prepare { | ||
| 45 | my ($self) = @_; | ||
| 46 | my $m = $self->{'matchpoints'} || die "No matchpoints"; | ||
| 47 | foreach (@$m) { | ||
| 48 | $_->{'threshold'} = 0.0 if !defined $_->{'threshold'}; | ||
| 49 | $_->{'weight'} or die; | ||
| 50 | $_->{'minscore'} = 0 if !defined $_->{'minscore'}; | ||
| 51 | $_->{'maxscore'} = 2**31 if !defined $_->{'maxscore'}; | ||
| 52 | $_->{'comparison'} ||= | ||
| 53 | $_->{'exact'} | ||
| 54 | ? sub { $a cmp $b } | ||
| 55 | : sub { | ||
| 56 | my $alen = length $a; | ||
| 57 | my $blen = length $b; | ||
| 58 | return 1.0 if !$alen || !$blen; | ||
| 59 | my $d = distance($a, $b); | ||
| 60 | return $d / ($alen > $blen ? $alen : $blen); | ||
| 61 | }; | ||
| 62 | next if defined $_->{'key'}; | ||
| 63 | my ($tdef, $cdef) = @$_{qw(target candidate)}; | ||
| 64 | $_->{'key'} = join('::', $_->{'target'}{'key'}, $_->{'candidate'}{'key'}); | ||
| 65 | } | ||
| 66 | return $self; | ||
| 67 | } | ||
| 68 | |||
| 69 | sub match { | ||
| 70 | my $self = shift; | ||
| 71 | my ($cleader, $cfields); | ||
| 72 | if (@_ == 2) { | ||
| 73 | ($cleader, $cfields) = @_; | ||
| 74 | } | ||
| 75 | elsif (@_ == 1) { | ||
| 76 | my $cand = shift; | ||
| 77 | my $ref = ref $cand; | ||
| 78 | my ($cleader, $cfields) = | ||
| 79 | $ref eq '' ? marcparse(\$cand) : | ||
| 80 | $ref eq 'SCALAR' ? marcparse($cand) : | ||
| 81 | $ref eq 'ARRAY' ? @$cand : | ||
| 82 | die "Can't match a $ref" | ||
| 83 | ; | ||
| 84 | } | ||
| 85 | else { | ||
| 86 | die; | ||
| 87 | } | ||
| 88 | my $mkey2tvals = $self->{'mkey2tvals'} ||= $self->build_target_values; | ||
| 89 | my $score = 0; | ||
| 90 | my @details; | ||
| 91 | foreach my $m ($self->matchpoints) { | ||
| 92 | my ($mkey, $tdef, $cdef, $thr, $w, $smin, $smax, $cmp) | ||
| 93 | = @$m{qw(key target candidate threshold weight minscore maxscore comparison)}; | ||
| 94 | my @cvals = $self->build_values($cdef, $cleader, $cfields); | ||
| 95 | my $tbest = $tdef->{'best'}; | ||
| 96 | my $cbest = $cdef->{'best'}; | ||
| 97 | my @results; | ||
| 98 | foreach my $tval (@{ $mkey2tvals->{$mkey} }) { | ||
| 99 | foreach my $cval (@cvals) { | ||
| 100 | local $a = $tval; | ||
| 101 | local $b = $cval; | ||
| 102 | my $d = my $raw_distance = $cmp->(); | ||
| 103 | next if !defined $d; | ||
| 104 | # abs($d) == 0 -> exactly equal | ||
| 105 | # abs($d) == 1 -> totally dissimilar | ||
| 106 | $d = abs $d; # Map [-1,0) to (0,1] | ||
| 107 | $d = 1 if $d > 1; # Pin to [0,1] | ||
| 108 | my $s = 1 - $d; # Similarity = 1 - distance | ||
| 109 | next if $s < $thr; # Skip if too small | ||
| 110 | push @results, { | ||
| 111 | 'score' => $w * $s, | ||
| 112 | 'raw_distance' => $raw_distance, | ||
| 113 | 'target' => $tval, | ||
| 114 | 'candidate' => $cval, | ||
| 115 | }; | ||
| 116 | } | ||
| 117 | } | ||
| 118 | if (defined $tbest) { | ||
| 119 | # TODO: use the best n values | ||
| 120 | die "Not yet implemented"; | ||
| 121 | } | ||
| 122 | if (defined $cbest) { | ||
| 123 | # TODO: use the best n values | ||
| 124 | die "Not yet implemented"; | ||
| 125 | } | ||
| 126 | # Comparison is done for the purposes of this matchpoint | ||
| 127 | my $mscore = 0; | ||
| 128 | foreach my $r (@results) { | ||
| 129 | my ($s, $t, $c) = @$r{qw(score target candidate)}; | ||
| 130 | $mscore += $s; | ||
| 131 | } | ||
| 132 | $mscore = $smax if $mscore > $smax; | ||
| 133 | my $drop = ( $mscore < $smin ); | ||
| 134 | push @details, { | ||
| 135 | 'matchpoint' => $m, | ||
| 136 | 'score' => $mscore, | ||
| 137 | 'results' => \@results, | ||
| 138 | 'dropped' => $drop, | ||
| 139 | }; | ||
| 140 | next if $drop; | ||
| 141 | $score += $mscore; | ||
| 142 | } | ||
| 143 | return $score, @details; | ||
| 144 | } | ||
| 145 | |||
| 146 | sub build_target_values { | ||
| 147 | my ($self) = @_; | ||
| 148 | $self->prepare; | ||
| 149 | my $tref = $self->{'tref'} || die "No target specified"; | ||
| 150 | my ($tleader, $tfields) = marcparse($tref); | ||
| 151 | my %tvals; | ||
| 152 | foreach my $m ($self->matchpoints) { | ||
| 153 | my ($mkey, $tdef, $cdef) = @$m{qw(key target candidate)}; | ||
| 154 | my @values = $self->build_values($tdef, $tleader, $tfields); | ||
| 155 | $tvals{$mkey} = \@values; | ||
| 156 | } | ||
| 157 | return \%tvals; | ||
| 158 | } | ||
| 159 | |||
| 160 | sub build_values { | ||
| 161 | my ($self, $valdef, $leader, $fields) = @_; | ||
| 162 | my $field = $valdef->{'field'}; | ||
| 163 | my @norm = @{ $valdef->{'normalize'} || [] }; | ||
| 164 | my @values; | ||
| 165 | if ($field eq 'leader') { | ||
| 166 | my ($b, $n) = @{ $valdef->{'leader'} }; | ||
| 167 | @values = ( substr($leader, $b, $n) ); | ||
| 168 | } | ||
| 169 | else { | ||
| 170 | my @fields = grep { $_->[TAG] eq $field } @$fields; | ||
| 171 | return if !@fields; | ||
| 172 | my $index = $valdef->{'index'}; | ||
| 173 | if ($index && $index ne '*') { | ||
| 174 | if ($index > 0) { | ||
| 175 | # First $index instance(s) | ||
| 176 | splice @fields, $index; | ||
| 177 | } | ||
| 178 | else { | ||
| 179 | # Last -$index instance(s) | ||
| 180 | splice @fields, 0, $index; | ||
| 181 | } | ||
| 182 | } | ||
| 183 | my $subseq = $valdef->{'subfields'}; | ||
| 184 | my (@subs, %want_sub); | ||
| 185 | if (defined $subseq && length $subseq) { | ||
| 186 | @subs = split //, $subseq; | ||
| 187 | %want_sub = map { $_ => 1 } @subs; | ||
| 188 | } | ||
| 189 | foreach my $f (@fields) { | ||
| 190 | if ($field lt '010') { | ||
| 191 | my $val = ${ $f->[VALREF] }; | ||
| 192 | $val = $_->($val) for @norm; | ||
| 193 | push @values, $val; | ||
| 194 | } | ||
| 195 | else { | ||
| 196 | my $val; | ||
| 197 | if (%want_sub) { | ||
| 198 | my %subval; | ||
| 199 | foreach my $sub (@$f[SUBS..$#$f]) { | ||
| 200 | my ($id, $valref) = @$sub; | ||
| 201 | next if !$want_sub{$id}; | ||
| 202 | $subval{$id} = $$valref; | ||
| 203 | } | ||
| 204 | $val = join(' ', grep { defined && length } map { $subval{$_} } @subs); | ||
| 205 | $val = $_->($val) for @norm; | ||
| 206 | } | ||
| 207 | else { | ||
| 208 | $val = ${ $f->[VALREF] }; | ||
| 209 | $val =~ s/^..\x1f.//; | ||
| 210 | $val =~ s/\x1f./ /g; | ||
| 211 | $val = $_->($val) for @norm; | ||
| 212 | } | ||
| 213 | push @values, $val if defined $val && length $val; | ||
| 214 | } | ||
| 215 | } | ||
| 216 | } | ||
| 217 | return @values; | ||
| 218 | } | ||
| 219 | |||
| 220 | 1; |