blib/lib/MARC/Loop.pm untested

filter: Statementstmt 0.0% 0 / 238 Branchbran 0.0% 0 / 94 Conditioncond 0.0% 0 / 39 MC/DCmcdc 0.0% 0 / 25 Subroutinesub 0.0% 0 / 21 Podpod 0.0% 0 / 6 totaltotal 0.0% 0 / 423 CC 48 SCAR 59.0
CC
48
Cov
0%
marcloop
812.0
marcparse
110.0
marcindicators
56.0
CRAP 363.8
1 package MARC::Loop;
2  
3 use strict;
4 use warnings;
5  
6 require Exporter;
7  
8 use vars qw($VERSION @ISA @EXPORT_OK);
9  
10 $VERSION = '0.01';
11  
12 sub marcloop(&;$%);
13  
14 use constant TAG => 0;
15 use constant VALREF => 1;
16 use constant DELETE => 2;
17 use constant IND1 => 3;
18 use constant IND2 => 4;
19 use constant SUBS => 5;
20  
21 use constant SUB_ID => 0;
22 use constant SUB_VALREF => 1;
23  
24 use constant RECORD_TERMINATOR => "\x1d";
25 use constant FIELD_TERMINATOR => "\x1e";
26 use constant SUBFIELD_DELIMITER => "\x1f";
27  
28 @ISA = qw(Exporter);
29 @EXPORT_OK = qw(marcloop marcparse marcfield marcindicators marcbuild TAG VALREF DELETE IND1 IND2 SUBS SUB_ID SUB_VALREF RECORD_TERMINATOR FIELD_TERMINATOR SUBFIELD_DELIMITER);
30  
31 sub marcloop(&;$%) {
32 my ($sub, $f, %arg) = @_;
33 my $fh;
34 $arg{'print'} = 1 if $arg{'print_all'};
35 my (%drop, %only);
36 if ($arg{'drop'}) {
37 %drop = map { $_ => 1 } @{ $arg{'drop'} };
38 }
39 if ($arg{'only'}) {
40 %only = map { $_ => 1 } @{ $arg{'only'} };
41 die "drop and only are mutually exclusive" if %drop && %only;
42 }
43 if (defined $f) {
44 if (ref $f) {
45 $fh = $f;
46 }
47 elsif ($f eq '-') {
48 $fh = \*STDIN;
49 }
50 else {
51 open $fh, '<', $f or die "Can't open file '$f': $!";
52 }
53 }
54 else {
55 $f = '-';
56 $fh = \*STDIN;
57 }
58 # --- Set up variables that identify the current record
59 my $str;
60 my $n = 0;
61 my $byte_pos = 0;
62 my $bibid;
63  
64 my $warning = sub {
65 my ($msg) = @_;
66 chomp $msg;
67 printf STDERR "Warning: %s at record %d (offset %d, bib ID %s) of file %s\n",
68 $msg, $n, $byte_pos, (defined $bibid ? $bibid : 'unknown'), $f;
69 };
70 my $error = $arg{'error'} || sub {
71 my ($msg) = @_;
72 chomp $msg;
73 printf STDERR "Error: %s at record %d (offset %d, bib ID %s) of file %s\n",
74 $msg, $n, $byte_pos, (defined $bibid ? $bibid : 'unknown'), $f;
75 die "MARC::Loop - $msg";
76 };
77  
78 RECORD:
79 while (1) {
80 # --- Read the next MARC record
81 {
82 local $/ = RECORD_TERMINATOR;
83 $str = <$fh>;
84 }
85 last if !defined $str;
86 $n++;
87 # --- Parse and process it
88 eval {
89 if ($str !~ /
90 \A
91 # bytes description
92 # ----- -------------------------------------------
93 (\d{5}) # 00-04 rec length
94 (....) # 05-08 rec status, rec type, bib level, type of control
95 (.) # 09 character coding
96 (..) # 10-11 indicator count, subfield code count
97 (\d{5}) # 12-16 base address = length of leader + directory
98 (.{7}) # 17-23 other stuff
99 /x) {
100 $error->("Not a USMARC record: pathological leader");
101 }
102 my ($reclen, $baseaddr) = ($1, $5);
103 if ($reclen != length $str) {
104 $error->("Incorrect record length");
105 }
106 my $leader = substr($str, 0, 24);
107 my $directory = substr($str, 24, $baseaddr - 24);
108 if (length($directory) % 12 != 1) {
109 $error->("Directory length not a multiple of 12 bytes");
110 }
111 if (substr($directory, -1, 1) ne FIELD_TERMINATOR) {
112 $error->("Directory not terminated");
113 }
114 my ($field, @fields);
115 # --- Loop through the fields
116 while ($directory =~ /(...)(....)(.....)/gc) {
117 my ($tag, $len, $ofs) = ($1, $2, $3);
118 next if $drop{$tag} || (%only && !$only{$tag});
119 my $value = substr($str, $baseaddr + $ofs, $len);
120 # --- Make sure the field ends in the field terminator
121 if (substr($value, -1) ne FIELD_TERMINATOR) {
122 $error->("Field $tag not terminated");
123 }
124 else {
125 $value = substr($value, 0, -1);
126 }
127 if ($tag lt '010') {
128 # --- Control field
129 push @fields, [ $tag, \$value ];
130 if ($tag eq '001') {
131 $bibid = $value;
132 }
133 }
134 else {
135 # --- Data field
136 my ($i1, $i2) = ($value =~ /\G(.)(.)/gc);
137 my @subfields;
138 pos($value) = 2; # Shouldn't have to do this to skip past indicators :-(
139 while ($value =~ /\G(?:\x1f([^\x1d-\x1f])([^\x1d-\x1f]*))/gc) {
140 my $subval = $2;
141 push @subfields, [ $1, \$subval ];
142 }
143 if (@subfields == 0) {
144 $error->("Empty field '$tag'");
145 }
146 push @fields, [ $tag, \$value, undef, $i1, $i2, @subfields ];
147 }
148 }
149 if ($arg{'test_build_record'} && $str ne marcbuild($leader, \@fields)) {
150 $error->("INTERNAL ERROR: marcbuild() failed");
151 }
152 eval {
153 $sub->($leader, \@fields, \$str);
154 };
155 if ($@) {
156 chomp $@;
157 $error->($@);
158 last if $arg{'strict'};
159 }
160 elsif ($arg{'print'}) {
161 my $new_str = marcbuild($leader, \@fields);
162 if ($arg{'print_all'} || $new_str ne $str) {
163 print $new_str;
164 }
165 }
166 };
167 if ($@) {
168 # --- Handle exceptions
169 last if $arg{'strict'};
170 }
171 }
172 continue {
173 $byte_pos += length($str);
174 }
175  
176 # --- Report results
177 if ($arg{'show_results'}) {
178 printf STDERR "Results: %d of %d records printed\n"
179 }
180 }
181  
182  
183 # --- Functions
184  
185 sub marcindicators {
186 if (@_ == 0) {
187 return (' ', ' ');
188 }
189 elsif (@_ == 1 && length($_[0] eq 2)) {
190 return split //, $_[0];
191 }
192 elsif (@_ != 2) {
193 die "Wrong number of indicators";
194 }
195 my $i = 1;
196 for (@_) {
197 die "Bad indicator $i" unless defined;
198 die "Indicator $i too long" if length > 1;
199 $_ = ' ' if length == 0;
200 $i++;
201 }
202 return @_;
203 }
204  
205 sub marcfield {
206 # marcfield('001', 1234567);
207 # marcfield('245',
208 # marcindicators(' ', ' '),
209 # 'a' => 'Blah blah',
210 # 'c' => 'Foo B. Arrrr',
211 # ...
212 # );
213 my $tag = shift;
214 my @field;
215 $field[TAG] = $tag;
216 $field[DELETE] = undef;
217 if ($tag lt '010') {
218 my $val = shift;
219 $field[VALREF] = \$val;
220 }
221 else {
222 $field[VALREF] = undef;
223 $field[IND1] = shift;
224 $field[IND2] = shift;
225 die "Odd number of subfield (id, val) tuples" if @_ % 2;
226 while (@_) {
227 my ($id, $val) = splice @_, 0, 2;
228 push @field, [ $id, \$val ];
229 }
230 }
231 return \@field;
232 }
233  
234 sub marcbuild {
235 my ($leader, $fields) = @_;
236 my $directory = '';
237 my $body = '';
238 my $ofs = 0;
239 foreach my $f (@$fields) {
240 my ($tag, $valref, $delete, $i1, $i2, @subfields) = @$f;
241 next if $delete;
242 my $bodyprevlen = length($body);
243 if ($tag lt '010') {
244 $body .= $$valref;
245 }
246 else {
247 $body .= $i1 . $i2;
248 foreach my $s (@subfields) {
249 my ($code, $content) = @$s;
250 $body .= SUBFIELD_DELIMITER . $code . $$content if defined $$content;
251 }
252 }
253 $body .= FIELD_TERMINATOR;
254 my $fstrlen = length($body) - $bodyprevlen;
255 $directory .= sprintf('%3.3s%04d%05d', $tag, $fstrlen, $ofs);
256 $ofs += $fstrlen;
257 }
258 $directory .= FIELD_TERMINATOR;
259 my $dirlen = length $directory;
260 $body .= RECORD_TERMINATOR;
261 substr($leader, 0, 5) = sprintf('%05d', 24 + $dirlen + length $body);
262 substr($leader, 12, 5) = sprintf('%05d', 24 + $dirlen);
263 return $leader . $directory . $body;
264 }
265  
266 sub marcparse {
267 my ($strref, %args) = @_;
268 my $warning = $args{'warning'} || sub {
269 my ($msg) = @_;
270 print STDERR "WARNING: $msg\n";
271 };
272 my $error = $args{'error'} || sub {
273 my ($msg) = @_;
274 die "marcloop: error: $msg\n";
275 };
276 my %drop = %{ $args{'drop'} || {} };
277 my %only = %{ $args{'only'} || {} };
278 my $bibid;
279 if ($$strref !~ /
280 \A
281 # bytes description
282 # ----- -------------------------------------------
283 (\d{5}) # 00-04 rec length
284 (....) # 05-08 rec status, rec type, bib level, type of control
285 (.) # 09 character coding
286 (..) # 10-11 indicator count, subfield code count
287 (\d{5}) # 12-16 base address = length of leader + directory
288 (.{7}) # 17-23 other stuff
289 /x) {
290 $error->("Not a USMARC record: pathological leader");
291 }
292 my ($reclen, $baseaddr) = ($1, $5);
293 if ($reclen != length $$strref) {
294 $error->("Incorrect record length");
295 }
296 my $leader = substr($$strref, 0, 24);
297 my $directory = substr($$strref, 24, $baseaddr - 24);
298 if (length($directory) % 12 != 1) {
299 $error->("Directory length not a multiple of 12 bytes");
300 }
301 if (substr($directory, -1, 1) ne FIELD_TERMINATOR) {
302 $error->("Directory not terminated");
303 }
304 my ($field, @fields);
305 # --- Loop through the fields
306 while ($directory =~ /(...)(....)(.....)/gc) {
307 my ($tag, $len, $ofs) = ($1, $2, $3);
308 next if $drop{$tag} || (%only && !$only{$tag});
309 my $value = substr($$strref, $baseaddr + $ofs, $len);
310 # --- Make sure the field ends in the field terminator
311 if (substr($value, -1) ne FIELD_TERMINATOR) {
312 $error->("Field $tag not terminated");
313 }
314 # --- Strip the field terminator
315 $value = substr($value, 0, -1);
316 if ($tag lt '010') {
317 # --- Control field
318 push @fields, [ $tag, \$value ];
319 if ($tag eq '001') {
320 $bibid = $value;
321 }
322 }
323 else {
324 # --- Data field
325 my ($i1, $i2) = ($value =~ /\G(.)(.)/gc);
326 my @subfields;
327 pos($value) = 2; # Shouldn't have to do this to skip past indicators :-(
328 while ($value =~ /\G(?:\x1f([^\x1d-\x1f])([^\x1d-\x1f]*))/gc) {
329 my $subval = $2;
330 push @subfields, [ $1, \$subval ];
331 }
332 if (@subfields == 0) {
333 $error->("Empty field '$tag'");
334 }
335 push @fields, [ $tag, \$value, undef, $i1, $i2, @subfields ];
336 }
337 }
338 return ($leader, \@fields);
339 }
340  
341 =head1 NAME
342  
343 MARC::Loop - process a batch of MARC21 records
344  
345 =head1 SYNOPSIS
346  
347 use MARC::Loop qw(marcloop marcbuild TAG VALREF DEL);
348 my $filehandle = \*STDIN;
349 my $deleted999 = 0;
350 my $fixed035 = 0;
351 marcloop {
352 my ($leader, $fields, $raw_marc_ref) = @_;
353 my $changed;
354 foreach my $field (@$fields) {
355 if ($field->[TAG] eq '035') {
356 # Normalize OCLC numbers
357 my $valref = $field->[VALREF];
358 $$valref =~ s/\(OCoLC\)oc[mn]0*/(OCoLC)/g;
359 $fixed035++;
360 $changed = 1;
361 }
362 elsif ($field->[TAG] eq '999') {
363 # Delete 999 fields
364 $field->[DEL] = 1;
365 $deleted999++;
366 $changed = 1;
367 }
368 }
369 # Print only changed records
370 print marcbuild($leader, $fields) if $changed;
371 } $filehandle;
372 print STDERR "$deleted999 999 fields were deleted\n",
373 "$fixed035 035 fields were fixed\n";
374  
375 =head1 DESCRIPTION
376  
377 MARC::Loop is an alternative to L<MARC::File|MARC::File> and
378 L<MARC::Record|MARC::Record> that eschews an object-oriented approach in favor
379 of a bare-bones procedural one.
380  
381 =head1 FUNCTIONS
382  
383 All of these functions are exported upon request.
384  
385 =head2 marcloop
386  
387 # This example prints a MARC record in human-readable form, using a single
388 # line for each field no matter how many subfields it might have.
389 use MARC::Loop qw(marcloop);
390 marcloop {
391 my ($leader, $fields, $raw_marc_ref) = @_;
392 foreach my $field (@$fields) {
393 if ($field->[TAG] lt '010) {
394 # Control field
395 my $valref = $field->[VALREF];
396 print $field->[TAG], ' ', $$valref, "\n";
397 }
398 else {
399 # Data field
400 my ($i1, $i2) = ($field->[IND1], $field->[IND2]);
401 print "$field->[TAG] $i1$i2";
402 my @subfields = @{ $field->[SUBS..$#$field] };
403 foreach my $subfield (@subfields) {
404 my ($code, $valref) = @$subfield;
405 print '$', $code, ' ', $$valref;
406 }
407 print "\n";
408 }
409 }
410 } $filehandle, %options;
411  
412 Options:
413  
414 =over 4
415  
416 =item B<print_all>
417  
418 All MARC records will be printed to STDOUT. Any changes you make will be
419 reflected in what is printed.
420  
421 =item B<drop> I<array_ref>
422  
423 'drop' => [ '001', '090' ]
424  
425 Tags of fields to be dropped unconditionally.
426  
427 =item B<only> I<array_ref>
428  
429 'only' => [ qw(001 004 008 852 856) ]
430  
431 Tags of fields to be preserved; all other fields will be dropped. (The leader
432 is never dropped, of course.)
433  
434 =item B<error>
435  
436 A code reference to call when there is an error.
437  
438 'error' => sub { exit -1 }
439  
440 =item B<strict>
441  
442 'strict' => 1
443  
444 Halt processing when an ill-formed MARC record is encountered.
445  
446 =back
447  
448 Options will be more fully documented in a future release; in the meantime,
449 read the source code.
450  
451 =head2 marcparse
452  
453 ($leader, $fields) = marcparse(\$string, %options);
454  
455 Parses a MARC record into the same data structure that B<marcloop> uses.
456 Options are mostly the same as for B<marcloop>.
457  
458 =head2 marcbuild
459  
460 print marcbuild($leader, $fields);
461  
462 Builds a raw MARC record from the same data structure that B<marcloop> uses and
463 that B<marcparse> produces.
464  
465 =head2 marcfield
466  
467 # Control field
468 push $@fields, marcfield(
469 '001', # Tag
470 1234567, # Content
471 );
472  
473 # Data field
474 push $@fields, marcfield(
475 '245', # Tag
476 ' ', ' ', # Indicators
477 'a' => 'Blah blah', # Subfield $a
478 'c' => 'Amy Emu', # Subfield $c
479 ... # More subfields
480 );
481  
482 A convenience function to build a data structure representing a single MARC21
483 field.
484  
485 =head1 BUGS
486  
487 The documentation is woefully lacking and probably just plain wrong. Read the
488 source code -- or, better yet, step through it in the Perl debugger -- to get a
489 better understanding of what this code does.
490  
491 Unhandled errors cause ill-formed records to be dropped without notice. You
492 must specify an B<error> option to B<marcloop> to get around this.
493  
494 =head1 AUTHOR
495  
496 Paul Hoffman E<lt>paul@flo.orgE<gt>.
497  
498 =head1 COPYRIGHT
499  
500 Copyright 2009-2010 Fenway Libraries Online. Released under the GNU Public
501 License, version 3.