SubjectMap.pm

filter: Statementstmt 88.8% 16 / 18 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 6 / 6 Podpod n/a totaltotal 91.6% 22 / 24 CC 1 SCAR 0.5
CC
1
Cov
89%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« SubjectMap/Rules.pm ↑ Summary
1 package MARC::SubjectMap;
2  
3 6 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 6 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 6 ▶use Carp qw( croak );
Subroutine BEGIN: called CC=1 SCAR=0.0
6 6 ▶use MARC::Field;
Subroutine BEGIN: called CC=1 SCAR=0.0
7 6 ▶use MARC::SubjectMap::XML qw( startTag endTag element comment );
Subroutine BEGIN: called CC=1 SCAR=0.0
8 0 ▶use MARC::SubjectMap::Rules;
Subroutine BEGIN: called CC=1 SCAR=2.6
9 use MARC::SubjectMap::Handler;
10 use XML::SAX::ParserFactory;
11 use IO::File;
12  
13 our $VERSION = '0.93';
14  
15 =head1 NAME
16  
17 MARC::SubjectMap - framework for translating subject headings
18  
19 =head1 SYNOPSIS
20  
21 use MARC::SubjectMap;
22 my $map = MARC::SubjectMap->newFromConfig( "config.xml" );
23  
24 my $batch = MARC::Batch->new( 'USMARC', 'batch.dat' );
25 while ( my $record = $batch->next() ) {
26 my $new = $map->translateRecord( $record );
27 ...
28 }
29  
30 =head1 DESCRIPTION
31  
32 MARC::SubjectMap is a framework for providing translations of subject
33 headings. MARC::SubjectMap is essentially a configuration which contains
34 a list of fields/subfields to translate or copy, and a list of rules
35 for translating one field/subfield value into another.
36  
37 Typical usage of the framework will be to use the C<subjmap-template>
38 command line application to generate a template XML configuration from a
39 batch of MARC records. You tell C<subjmap-template> the fields you'd like
40 to translate and/or copy and it will look through the records and extract
41 and add rule templates for the unique values. For example:
42  
43 subjmap-template --in=marc.dat --out=config.xml --translate=650ab
44  
45 Once the template configuration has been filled in with translations,
46 the MARC batch file can be run through another command line utility called
47 C<subjmap> which will add new subject headings where possible using
48 the configuration file. If a subject headings can't be translated it will be
49 logged to a file so that the configuration file can be improved if necessary.
50
51 subjmap --in=marc.dat --out=new.dat --config=config.xml --log=log.txt
52  
53 The idea is that all the configuration is done in the XML file, and the
54 command line programs take care of driving these modules for you. Methods
55 and related modules are listed below for the sake of completeness, and if
56 you want to write your own driving program for some reason.
57  
58 =head1 METHODS
59  
60 =head2 new()
61  
62 The constructor which accepts no arguments.
63  
64 =cut
65  
66 sub new {
67 my ($class) = @_;
68 my $self = {
69 fields => [],
70 sourceLanguage => '',
71 error => '',
72 stats => { recordsProcessed=>0, fieldsAdded=>0, errors=>0 }
73 };
74 return bless $self, ref($class) || $class;
75 }
76  
77 =head2 newFromConfig()
78  
79 Factory method for creating a MARC::SubjectMap object from an XML
80 configuration. If there is an error you will get it on STDERR.
81  
82 my $mapper = MARC::SubjectMap->new( 'config.xml' );
83  
84 =cut
85  
86 sub newFromConfig {
87 my ($package,$file) = @_;
88 my $handler = MARC::SubjectMap::Handler->new();
89 my $parser = XML::SAX::ParserFactory->parser( Handler => $handler );
90 eval { $parser->parse_uri( $file ) };
91 croak( "invalid configuration file: $file: $@" ) if $@;
92 return $handler->config();
93 }
94  
95 =head2 writeConfig()
96  
97 Serializes the configuration to disk as XML.
98  
99 =cut
100  
101 sub writeConfig {
102 my ($self,$file) = @_;
103 my $fh = IO::File->new( ">$file" )
104 or croak( "unable to write to file $file: $! " );
105 $self->toXML($fh);
106 }
107  
108 =head2 addField()
109  
110 Adds a field specification to the configuration. Each specification defines the
111 fields and subfields to look for and copy/translate in MARC data. The
112 information is bundled up in a MARC::SubjectMap::Field object.
113  
114 =cut
115  
116 sub addField {
117 my ($self,$field) = @_;
118 croak( "must supply MARC::SubjectMap::Field object" )
119 if ref($field) ne 'MARC::SubjectMap::Field';
120 push( @{ $self->{fields} }, $field );
121 }
122  
123 =head2 fields()
124  
125 Returns a list of MARC::SubjectMap::Field objects which specify the
126 fields/subfields in MARC data that will be copied and/or translated.
127  
128 =cut
129  
130 sub fields {
131 my ($self) = @_;
132 return @{ $self->{fields} };
133 }
134  
135 =head2 rules()
136  
137 Get/set the rules being used in this configuration. You should pass
138 in a MARC::SubjectMap::Rules object if you are setting the rules.
139  
140 $map->rules( $rules );
141  
142 The reason why a sepearte object is used to hold the Rules as opposed to the
143 fields being contained in the MARC::SubjectMap object is that there can be
144 many (thousands perhaps) of rules -- which need to be stored differently than
145 the handful of fields.
146  
147 =cut
148  
149 sub rules {
150 my ($self,$rules) = @_;
151 croak( "must supply MARC::SubjectMap::Rules object if setting rules" )
152 if $rules and ref($rules) ne 'MARC::SubjectMap::Rules';
153 $self->{rules} = $rules if $rules;
154 return $self->{rules};
155 }
156  
157 =head2 sourceLanguage()
158  
159 Option for specifying the three digit language code to be expected in
160 translation records. If a record is passed is translated that is not of the
161 expected source language then a log message will be generated.
162  
163 =cut
164  
165 sub sourceLanguage {
166 my ($self,$lang) = @_;
167 $self->{sourceLanguage} = $lang if defined $lang;
168 return $self->{sourceLanguage};
169 }
170  
171 =head2 translateRecord()
172  
173 Accepts a MARC::Record object and returns a translated version of it
174 if there were any translations that could be performed. If no translations
175 were possible undef will be returned.
176  
177 =cut
178  
179 sub translateRecord {
180 my ($self,$record) = @_;
181 croak( "must supply MARC::Record object to translateRecord()" )
182 if ! ref($record) or ! $record->isa( 'MARC::Record' );
183  
184 my $record_id = $record->field('001') ? $record->field('001')->data() : '';
185 $record_id =~ s/ +$//;
186  
187 $self->{stats}{recordsProcessed}++;
188  
189 ## log message if the record isn't the expected language
190 if ( language($record) ne $self->sourceLanguage() ) {
191 $self->log( sprintf( "record language=%s instead of %s",
192 language($record), $self->sourceLanguage() ) );
193 }
194  
195 ## create a copy of the record to add to
196 my $clone = $record->clone();
197 my $found = 0;
198  
199 # process each field that we need to look at
200 foreach my $field ( $self->fields() ) {
201  
202 my @marcFields = $record->field( $field->tag() );
203 my $fieldCount = 0;
204  
205 foreach my $marcField ( @marcFields ) {
206 $fieldCount++;
207  
208 # do the translation
209 my $new = $self->translateField( $marcField, $field );
210 my $error = $self->error();
211  
212 if ( $new ) {
213 $clone->insert_grouped_field($new);
214 $self->{stats}{fieldsAdded}++;
215 $found = 1;
216 $self->log("record $record_id: translated \"" .
217 $marcField->as_string() . '" to "' .
218 $new->as_string() . '"') ;
219 }
220 elsif ( $error ) {
221 $self->log("record $record_id: $error");
222 }
223 else {
224 # the field didn't match subfield filters or
225 # it only had copy actions and no translations
226 # so we just continue along
227 }
228 }
229 }
230 return $clone if $found;
231 return;
232 }
233  
234 # you won't want to call this directly so there's no POD for it
235 # warning: subroutine that's longer than your console window alert
236 # TODO: break this up
237  
238 sub translateField {
239 # args are MARC::SubjectMap object, the MARC::Field to translate
240 # and the MARC::SubjectMap::Field object which defines how we translate
241 my ($self,$field,$fieldConfig) = @_;
242 croak( "must supply MARC::Field object to translateField()" )
243 if !ref($field) or !$field->isa('MARC::Field');
244 croak( "must pass in MARC::SubjectMap::Field" )
245 if !ref($fieldConfig) or !$fieldConfig->isa('MARC::SubjectMap::Field');
246  
247 # make sure error flag is undef
248 $self->error( undef );
249  
250 ## subfields with subfield 2 already present are not translated
251 if ($field->subfield(2)) {
252 $self->error( "subfield 2 already present" );
253 return;
254 }
255  
256 ## don't bother translating if it doesn't meet indicator criteria
257 ## no error set here since it really isn't an error just a filter
258 my $indicator1 = $fieldConfig->indicator1();
259 my $indicator2 = $fieldConfig->indicator2();
260 return if defined $indicator1 and $indicator1 ne $field->indicator(1) ;
261 return if defined $indicator2 and $indicator2 ne $field->indicator(2) ;
262  
263 ## these are subfields we can copy wholesale
264 my @copySubfields = $fieldConfig->copy();
265  
266 my (@subfields,%sources,$lastSource,$didTranslation);
267 foreach my $subfield ( $field->subfields() ) {
268 my ($subfieldCode,$subfieldValue) = @$subfield;
269  
270 ## remove trailing period if present
271 $subfieldValue =~ s|\.$||;
272  
273 ## if we just copy this subfield lets do it and move on
274 if ( grep /^$subfieldCode$/, @copySubfields ) {
275 push( @subfields, $subfieldCode, $subfieldValue );
276 next;
277 }
278  
279 ## remove trailing whitespace since all rules have had their
280 ## original , but remember it so we can add it
281 ## back on to the translated subfield
282 my $trailingSpaces = '';
283 if ( $subfieldValue =~ /( +)$/ ) {
284 $trailingSpaces = $1;
285 }
286
287 ## look up the rule!
288 my $rule = $self->{rules}->getRule(
289 field => $field->tag(),
290 subfield => $subfieldCode,
291 original => $subfieldValue, );
292  
293 ## if we have a matching rule
294 if ( $rule ) {
295 if ( $rule->translation() ) {
296 $didTranslation = 1;
297 push( @subfields, $subfieldCode,
298 $rule->translation() . $trailingSpaces );
299 } else {
300 $self->{stats}{errors}++;
301 $self->error("missing translation for rule: ".$rule->toString);
302 return;
303 }
304
305 ## if a subfield a store away the source
306 $sources{ $subfieldCode } = $rule->source();
307 $lastSource = $rule->source();
308 }
309  
310 ## uhoh we don't know what to do with this subfield
311 else {
312 $self->{stats}{errors}++;
313 $self->error(
314 sprintf(
315 'could not translate "%s" from %s $%s',
316 $subfieldValue, $field->tag(), $subfieldCode
317 )
318 );
319 return;
320 }
321 }
322  
323 ## if we didn't translate anything no need to make a new field
324 ## note we dont' set an error message since this isn't really an error
325 return if ! $didTranslation;
326  
327 ## if the last subfield doesn't end in a <.> or a <)> add a period
328 $subfields[-1] .= '.' if ( $subfields[-1] !~ /[.)]/ );
329  
330 ## the configuration determines what subfield should have precedence
331 ## in determining the source of the subfield.
332 my $sourceSubfield = $fieldConfig->sourceSubfield();
333 if ( exists $sources{ $sourceSubfield } ) {
334 push( @subfields, '2', $sources{ $sourceSubfield } );
335 } elsif ( defined $lastSource ) {
336 push( @subfields, '2', $lastSource );
337 } else {
338 $self->{stats}{errors}++;
339 $self->log( "missing source for new field: ".join('', @subfields ) );
340 }
341
342 return MARC::Field->new($field->tag(),$field->indicator(1),7,@subfields);
343 }
344  
345 =head2 stats()
346  
347 Returns a hash of statistics for conversions performed by a MARC::SubjectMap
348 object.
349  
350 =cut
351  
352 sub stats {
353 return %{ shift->{stats} };
354 }
355  
356 =head2 setLog()
357  
358 Set a file to send diagnostic messages to. If unspecified messages will go to
359 STDERR. Alternatively you can pass in a IO::Handle object.
360  
361 =cut
362  
363 ## logging methods
364  
365 sub setLog {
366 my ($self,$f) = @_;
367 if ( ref($f) ) {
368 $self->{log} = $f;
369 } else {
370 $self->{log} = IO::File->new( ">$f" );
371 }
372 }
373  
374 sub log {
375 my ($self,$msg) = @_;
376 $msg .= "\n";
377 if ( $self->{log} ) {
378 $self->{log}->print( $msg );
379 } else {
380 print STDERR $msg;
381 }
382 }
383  
384 # returns entire object as XML
385 # this is essentially the configuration
386 # since it can be big a filehandle must be passed in
387  
388 sub toXML {
389 my ($self,$fh) = @_;
390 print $fh qq(<?xml version="1.0" encoding="ISO-8859-1"?>\n);
391 print $fh startTag( "config" ),"\n\n";
392
393 # language limiter if present
394 my $lang = $self->sourceLanguage() || '';
395 print $fh comment( "the original language" ), "\n";
396 print $fh element( "sourceLanguage", $self->sourceLanguage() ), "\n\n";
397  
398 ## add fields
399 print $fh comment( "the fields and subfields to be processed" )."\n";
400 print $fh startTag( "fields" ), "\n\n";
401 foreach my $field ( $self->fields() ) {
402 print $fh $field->toXML(), "\n";
403 }
404 print $fh endTag( "fields" ), "\n\n";
405  
406 ## add rules
407 if ( $self->rules() ) {
408 $self->rules()->toXML( $fh );
409 }
410  
411 print $fh "\n", endTag( "config" ), "\n";
412 }
413  
414 # helper to extract the language code from the 008
415  
416 sub language {
417 my $r = shift;
418 my $f008 = $r->field('008');
419 return '' if ! $f008;
420 return substr( $f008->data(), 35, 3 );
421 }
422  
423  
424 # helper to store a single error message, not really for public use
425  
426 sub error {
427 my ($self,$msg) = @_;
428 if ( $msg ) { $self->{error} = $msg; }
429 return $self->{error};
430 }
431  
432 sub DESTROY {
433 my $self = shift;
434 ## close log file handle if its open
435 $self->{log}->close() if exists( $self->{log} );
436 }
437  
438 =head1 SEE ALSO
439  
440 =over 4
441  
442 =item * L<MARC::SubjectMap::Rules>
443  
444 =item * L<MARC::SubjectMap::Rule>
445  
446 =item * L<MARC::SubjectMap::Field>
447  
448 =back
449  
450 =head1 AUTHORS
451  
452 =over 4
453  
454 =item * Ed Summers <ehs@pobox.com>
455  
456 =back
457  
458 =cut
459  
460 1;
« SubjectMap/Rules.pm ↑ Summary