| 1 | package MARC::Loop; | ||
| 2 | |||
| 3 | use strict; | ||
| 4 | use warnings; | ||
| 5 | |||
| 6 | require Exporter; | ||
| 7 | |||
| 8 | use vars qw($VERSION @ISA @EXPORT_OK); | ||
| 9 | |||
| 10 | $VERSION = '0.01'; | ||
| 11 | |||
| 12 | sub marcloop(&;$%); | ||
| 13 | |||
| 14 | use constant TAG => 0; | ||
| 15 | use constant VALREF => 1; | ||
| 16 | use constant DELETE => 2; | ||
| 17 | use constant IND1 => 3; | ||
| 18 | use constant IND2 => 4; | ||
| 19 | use constant SUBS => 5; | ||
| 20 | |||
| 21 | use constant SUB_ID => 0; | ||
| 22 | use constant SUB_VALREF => 1; | ||
| 23 | |||
| 24 | use constant RECORD_TERMINATOR => "\x1d"; | ||
| 25 | use constant FIELD_TERMINATOR => "\x1e"; | ||
| 26 | use constant SUBFIELD_DELIMITER => "\x1f"; | ||
| 27 | |||
| 28 | @ISA = qw(Exporter); | ||
| 29 | @EXPORT_OK = qw(marcloop marcparse marcfield marcindicators marcbuild TAG VALREF DELETE IND1 IND2 SUBS SUB_ID SUB_VALREF RECORD_TERMINATOR FIELD_TERMINATOR SUBFIELD_DELIMITER); | ||
| 30 | |||
| 31 | sub marcloop(&;$%) { | ||
| 32 | my ($sub, $f, %arg) = @_; | ||
| 33 | my $fh; | ||
| 34 | $arg{'print'} = 1 if $arg{'print_all'}; | ||
| 35 | my (%drop, %only); | ||
| 36 | if ($arg{'drop'}) { | ||
| 37 | %drop = map { $_ => 1 } @{ $arg{'drop'} }; | ||
| 38 | } | ||
| 39 | if ($arg{'only'}) { | ||
| 40 | %only = map { $_ => 1 } @{ $arg{'only'} }; | ||
| 41 | die "drop and only are mutually exclusive" if %drop && %only; | ||
| 42 | } | ||
| 43 | if (defined $f) { | ||
| 44 | if (ref $f) { | ||
| 45 | $fh = $f; | ||
| 46 | } | ||
| 47 | elsif ($f eq '-') { | ||
| 48 | $fh = \*STDIN; | ||
| 49 | } | ||
| 50 | else { | ||
| 51 | open $fh, '<', $f or die "Can't open file '$f': $!"; | ||
| 52 | } | ||
| 53 | } | ||
| 54 | else { | ||
| 55 | $f = '-'; | ||
| 56 | $fh = \*STDIN; | ||
| 57 | } | ||
| 58 | # --- Set up variables that identify the current record | ||
| 59 | my $str; | ||
| 60 | my $n = 0; | ||
| 61 | my $byte_pos = 0; | ||
| 62 | my $bibid; | ||
| 63 | |||
| 64 | my $warning = sub { | ||
| 65 | my ($msg) = @_; | ||
| 66 | chomp $msg; | ||
| 67 | printf STDERR "Warning: %s at record %d (offset %d, bib ID %s) of file %s\n", | ||
| 68 | $msg, $n, $byte_pos, (defined $bibid ? $bibid : 'unknown'), $f; | ||
| 69 | }; | ||
| 70 | my $error = $arg{'error'} || sub { | ||
| 71 | my ($msg) = @_; | ||
| 72 | chomp $msg; | ||
| 73 | printf STDERR "Error: %s at record %d (offset %d, bib ID %s) of file %s\n", | ||
| 74 | $msg, $n, $byte_pos, (defined $bibid ? $bibid : 'unknown'), $f; | ||
| 75 | die "MARC::Loop - $msg"; | ||
| 76 | }; | ||
| 77 | |||
| 78 | RECORD: | ||
| 79 | while (1) { | ||
| 80 | # --- Read the next MARC record | ||
| 81 | { | ||
| 82 | local $/ = RECORD_TERMINATOR; | ||
| 83 | $str = <$fh>; | ||
| 84 | } | ||
| 85 | last if !defined $str; | ||
| 86 | $n++; | ||
| 87 | # --- Parse and process it | ||
| 88 | eval { | ||
| 89 | if ($str !~ / | ||
| 90 | \A | ||
| 91 | # bytes description | ||
| 92 | # ----- ------------------------------------------- | ||
| 93 | (\d{5}) # 00-04 rec length | ||
| 94 | (....) # 05-08 rec status, rec type, bib level, type of control | ||
| 95 | (.) # 09 character coding | ||
| 96 | (..) # 10-11 indicator count, subfield code count | ||
| 97 | (\d{5}) # 12-16 base address = length of leader + directory | ||
| 98 | (.{7}) # 17-23 other stuff | ||
| 99 | /x) { | ||
| 100 | $error->("Not a USMARC record: pathological leader"); | ||
| 101 | } | ||
| 102 | my ($reclen, $baseaddr) = ($1, $5); | ||
| 103 | if ($reclen != length $str) { | ||
| 104 | $error->("Incorrect record length"); | ||
| 105 | } | ||
| 106 | my $leader = substr($str, 0, 24); | ||
| 107 | my $directory = substr($str, 24, $baseaddr - 24); | ||
| 108 | if (length($directory) % 12 != 1) { | ||
| 109 | $error->("Directory length not a multiple of 12 bytes"); | ||
| 110 | } | ||
| 111 | if (substr($directory, -1, 1) ne FIELD_TERMINATOR) { | ||
| 112 | $error->("Directory not terminated"); | ||
| 113 | } | ||
| 114 | my ($field, @fields); | ||
| 115 | # --- Loop through the fields | ||
| 116 | while ($directory =~ /(...)(....)(.....)/gc) { | ||
| 117 | my ($tag, $len, $ofs) = ($1, $2, $3); | ||
| 118 | next if $drop{$tag} || (%only && !$only{$tag}); | ||
| 119 | my $value = substr($str, $baseaddr + $ofs, $len); | ||
| 120 | # --- Make sure the field ends in the field terminator | ||
| 121 | if (substr($value, -1) ne FIELD_TERMINATOR) { | ||
| 122 | $error->("Field $tag not terminated"); | ||
| 123 | } | ||
| 124 | else { | ||
| 125 | $value = substr($value, 0, -1); | ||
| 126 | } | ||
| 127 | if ($tag lt '010') { | ||
| 128 | # --- Control field | ||
| 129 | push @fields, [ $tag, \$value ]; | ||
| 130 | if ($tag eq '001') { | ||
| 131 | $bibid = $value; | ||
| 132 | } | ||
| 133 | } | ||
| 134 | else { | ||
| 135 | # --- Data field | ||
| 136 | my ($i1, $i2) = ($value =~ /\G(.)(.)/gc); | ||
| 137 | my @subfields; | ||
| 138 | pos($value) = 2; # Shouldn't have to do this to skip past indicators :-( | ||
| 139 | while ($value =~ /\G(?:\x1f([^\x1d-\x1f])([^\x1d-\x1f]*))/gc) { | ||
| 140 | my $subval = $2; | ||
| 141 | push @subfields, [ $1, \$subval ]; | ||
| 142 | } | ||
| 143 | if (@subfields == 0) { | ||
| 144 | $error->("Empty field '$tag'"); | ||
| 145 | } | ||
| 146 | push @fields, [ $tag, \$value, undef, $i1, $i2, @subfields ]; | ||
| 147 | } | ||
| 148 | } | ||
| 149 | if ($arg{'test_build_record'} && $str ne marcbuild($leader, \@fields)) { | ||
| 150 | $error->("INTERNAL ERROR: marcbuild() failed"); | ||
| 151 | } | ||
| 152 | eval { | ||
| 153 | $sub->($leader, \@fields, \$str); | ||
| 154 | }; | ||
| 155 | if ($@) { | ||
| 156 | chomp $@; | ||
| 157 | $error->($@); | ||
| 158 | last if $arg{'strict'}; | ||
| 159 | } | ||
| 160 | elsif ($arg{'print'}) { | ||
| 161 | my $new_str = marcbuild($leader, \@fields); | ||
| 162 | if ($arg{'print_all'} || $new_str ne $str) { | ||
| 163 | print $new_str; | ||
| 164 | } | ||
| 165 | } | ||
| 166 | }; | ||
| 167 | if ($@) { | ||
| 168 | # --- Handle exceptions | ||
| 169 | last if $arg{'strict'}; | ||
| 170 | } | ||
| 171 | } | ||
| 172 | continue { | ||
| 173 | $byte_pos += length($str); | ||
| 174 | } | ||
| 175 | |||
| 176 | # --- Report results | ||
| 177 | if ($arg{'show_results'}) { | ||
| 178 | printf STDERR "Results: %d of %d records printed\n" | ||
| 179 | } | ||
| 180 | } | ||
| 181 | |||
| 182 | |||
| 183 | # --- Functions | ||
| 184 | |||
| 185 | sub marcindicators { | ||
| 186 | if (@_ == 0) { | ||
| 187 | return (' ', ' '); | ||
| 188 | } | ||
| 189 | elsif (@_ == 1 && length($_[0] eq 2)) { | ||
| 190 | return split //, $_[0]; | ||
| 191 | } | ||
| 192 | elsif (@_ != 2) { | ||
| 193 | die "Wrong number of indicators"; | ||
| 194 | } | ||
| 195 | my $i = 1; | ||
| 196 | for (@_) { | ||
| 197 | die "Bad indicator $i" unless defined; | ||
| 198 | die "Indicator $i too long" if length > 1; | ||
| 199 | $_ = ' ' if length == 0; | ||
| 200 | $i++; | ||
| 201 | } | ||
| 202 | return @_; | ||
| 203 | } | ||
| 204 | |||
| 205 | sub marcfield { | ||
| 206 | # marcfield('001', 1234567); | ||
| 207 | # marcfield('245', | ||
| 208 | # marcindicators(' ', ' '), | ||
| 209 | # 'a' => 'Blah blah', | ||
| 210 | # 'c' => 'Foo B. Arrrr', | ||
| 211 | # ... | ||
| 212 | # ); | ||
| 213 | my $tag = shift; | ||
| 214 | my @field; | ||
| 215 | $field[TAG] = $tag; | ||
| 216 | $field[DELETE] = undef; | ||
| 217 | if ($tag lt '010') { | ||
| 218 | my $val = shift; | ||
| 219 | $field[VALREF] = \$val; | ||
| 220 | } | ||
| 221 | else { | ||
| 222 | $field[VALREF] = undef; | ||
| 223 | $field[IND1] = shift; | ||
| 224 | $field[IND2] = shift; | ||
| 225 | die "Odd number of subfield (id, val) tuples" if @_ % 2; | ||
| 226 | while (@_) { | ||
| 227 | my ($id, $val) = splice @_, 0, 2; | ||
| 228 | push @field, [ $id, \$val ]; | ||
| 229 | } | ||
| 230 | } | ||
| 231 | return \@field; | ||
| 232 | } | ||
| 233 | |||
| 234 | sub marcbuild { | ||
| 235 | my ($leader, $fields) = @_; | ||
| 236 | my $directory = ''; | ||
| 237 | my $body = ''; | ||
| 238 | my $ofs = 0; | ||
| 239 | foreach my $f (@$fields) { | ||
| 240 | my ($tag, $valref, $delete, $i1, $i2, @subfields) = @$f; | ||
| 241 | next if $delete; | ||
| 242 | my $bodyprevlen = length($body); | ||
| 243 | if ($tag lt '010') { | ||
| 244 | $body .= $$valref; | ||
| 245 | } | ||
| 246 | else { | ||
| 247 | $body .= $i1 . $i2; | ||
| 248 | foreach my $s (@subfields) { | ||
| 249 | my ($code, $content) = @$s; | ||
| 250 | $body .= SUBFIELD_DELIMITER . $code . $$content if defined $$content; | ||
| 251 | } | ||
| 252 | } | ||
| 253 | $body .= FIELD_TERMINATOR; | ||
| 254 | my $fstrlen = length($body) - $bodyprevlen; | ||
| 255 | $directory .= sprintf('%3.3s%04d%05d', $tag, $fstrlen, $ofs); | ||
| 256 | $ofs += $fstrlen; | ||
| 257 | } | ||
| 258 | $directory .= FIELD_TERMINATOR; | ||
| 259 | my $dirlen = length $directory; | ||
| 260 | $body .= RECORD_TERMINATOR; | ||
| 261 | substr($leader, 0, 5) = sprintf('%05d', 24 + $dirlen + length $body); | ||
| 262 | substr($leader, 12, 5) = sprintf('%05d', 24 + $dirlen); | ||
| 263 | return $leader . $directory . $body; | ||
| 264 | } | ||
| 265 | |||
| 266 | sub marcparse { | ||
| 267 | my ($strref, %args) = @_; | ||
| 268 | my $warning = $args{'warning'} || sub { | ||
| 269 | my ($msg) = @_; | ||
| 270 | print STDERR "WARNING: $msg\n"; | ||
| 271 | }; | ||
| 272 | my $error = $args{'error'} || sub { | ||
| 273 | my ($msg) = @_; | ||
| 274 | die "marcloop: error: $msg\n"; | ||
| 275 | }; | ||
| 276 | my %drop = %{ $args{'drop'} || {} }; | ||
| 277 | my %only = %{ $args{'only'} || {} }; | ||
| 278 | my $bibid; | ||
| 279 | if ($$strref !~ / | ||
| 280 | \A | ||
| 281 | # bytes description | ||
| 282 | # ----- ------------------------------------------- | ||
| 283 | (\d{5}) # 00-04 rec length | ||
| 284 | (....) # 05-08 rec status, rec type, bib level, type of control | ||
| 285 | (.) # 09 character coding | ||
| 286 | (..) # 10-11 indicator count, subfield code count | ||
| 287 | (\d{5}) # 12-16 base address = length of leader + directory | ||
| 288 | (.{7}) # 17-23 other stuff | ||
| 289 | /x) { | ||
| 290 | $error->("Not a USMARC record: pathological leader"); | ||
| 291 | } | ||
| 292 | my ($reclen, $baseaddr) = ($1, $5); | ||
| 293 | if ($reclen != length $$strref) { | ||
| 294 | $error->("Incorrect record length"); | ||
| 295 | } | ||
| 296 | my $leader = substr($$strref, 0, 24); | ||
| 297 | my $directory = substr($$strref, 24, $baseaddr - 24); | ||
| 298 | if (length($directory) % 12 != 1) { | ||
| 299 | $error->("Directory length not a multiple of 12 bytes"); | ||
| 300 | } | ||
| 301 | if (substr($directory, -1, 1) ne FIELD_TERMINATOR) { | ||
| 302 | $error->("Directory not terminated"); | ||
| 303 | } | ||
| 304 | my ($field, @fields); | ||
| 305 | # --- Loop through the fields | ||
| 306 | while ($directory =~ /(...)(....)(.....)/gc) { | ||
| 307 | my ($tag, $len, $ofs) = ($1, $2, $3); | ||
| 308 | next if $drop{$tag} || (%only && !$only{$tag}); | ||
| 309 | my $value = substr($$strref, $baseaddr + $ofs, $len); | ||
| 310 | # --- Make sure the field ends in the field terminator | ||
| 311 | if (substr($value, -1) ne FIELD_TERMINATOR) { | ||
| 312 | $error->("Field $tag not terminated"); | ||
| 313 | } | ||
| 314 | # --- Strip the field terminator | ||
| 315 | $value = substr($value, 0, -1); | ||
| 316 | if ($tag lt '010') { | ||
| 317 | # --- Control field | ||
| 318 | push @fields, [ $tag, \$value ]; | ||
| 319 | if ($tag eq '001') { | ||
| 320 | $bibid = $value; | ||
| 321 | } | ||
| 322 | } | ||
| 323 | else { | ||
| 324 | # --- Data field | ||
| 325 | my ($i1, $i2) = ($value =~ /\G(.)(.)/gc); | ||
| 326 | my @subfields; | ||
| 327 | pos($value) = 2; # Shouldn't have to do this to skip past indicators :-( | ||
| 328 | while ($value =~ /\G(?:\x1f([^\x1d-\x1f])([^\x1d-\x1f]*))/gc) { | ||
| 329 | my $subval = $2; | ||
| 330 | push @subfields, [ $1, \$subval ]; | ||
| 331 | } | ||
| 332 | if (@subfields == 0) { | ||
| 333 | $error->("Empty field '$tag'"); | ||
| 334 | } | ||
| 335 | push @fields, [ $tag, \$value, undef, $i1, $i2, @subfields ]; | ||
| 336 | } | ||
| 337 | } | ||
| 338 | return ($leader, \@fields); | ||
| 339 | } | ||
| 340 | |||
| 341 | =head1 NAME | ||
| 342 | |||
| 343 | MARC::Loop - process a batch of MARC21 records | ||
| 344 | |||
| 345 | =head1 SYNOPSIS | ||
| 346 | |||
| 347 | use MARC::Loop qw(marcloop marcbuild TAG VALREF DEL); | ||
| 348 | my $filehandle = \*STDIN; | ||
| 349 | my $deleted999 = 0; | ||
| 350 | my $fixed035 = 0; | ||
| 351 | marcloop { | ||
| 352 | my ($leader, $fields, $raw_marc_ref) = @_; | ||
| 353 | my $changed; | ||
| 354 | foreach my $field (@$fields) { | ||
| 355 | if ($field->[TAG] eq '035') { | ||
| 356 | # Normalize OCLC numbers | ||
| 357 | my $valref = $field->[VALREF]; | ||
| 358 | $$valref =~ s/\(OCoLC\)oc[mn]0*/(OCoLC)/g; | ||
| 359 | $fixed035++; | ||
| 360 | $changed = 1; | ||
| 361 | } | ||
| 362 | elsif ($field->[TAG] eq '999') { | ||
| 363 | # Delete 999 fields | ||
| 364 | $field->[DEL] = 1; | ||
| 365 | $deleted999++; | ||
| 366 | $changed = 1; | ||
| 367 | } | ||
| 368 | } | ||
| 369 | # Print only changed records | ||
| 370 | print marcbuild($leader, $fields) if $changed; | ||
| 371 | } $filehandle; | ||
| 372 | print STDERR "$deleted999 999 fields were deleted\n", | ||
| 373 | "$fixed035 035 fields were fixed\n"; | ||
| 374 | |||
| 375 | =head1 DESCRIPTION | ||
| 376 | |||
| 377 | MARC::Loop is an alternative to L<MARC::File|MARC::File> and | ||
| 378 | L<MARC::Record|MARC::Record> that eschews an object-oriented approach in favor | ||
| 379 | of a bare-bones procedural one. | ||
| 380 | |||
| 381 | =head1 FUNCTIONS | ||
| 382 | |||
| 383 | All of these functions are exported upon request. | ||
| 384 | |||
| 385 | =head2 marcloop | ||
| 386 | |||
| 387 | # This example prints a MARC record in human-readable form, using a single | ||
| 388 | # line for each field no matter how many subfields it might have. | ||
| 389 | use MARC::Loop qw(marcloop); | ||
| 390 | marcloop { | ||
| 391 | my ($leader, $fields, $raw_marc_ref) = @_; | ||
| 392 | foreach my $field (@$fields) { | ||
| 393 | if ($field->[TAG] lt '010) { | ||
| 394 | # Control field | ||
| 395 | my $valref = $field->[VALREF]; | ||
| 396 | print $field->[TAG], ' ', $$valref, "\n"; | ||
| 397 | } | ||
| 398 | else { | ||
| 399 | # Data field | ||
| 400 | my ($i1, $i2) = ($field->[IND1], $field->[IND2]); | ||
| 401 | print "$field->[TAG] $i1$i2"; | ||
| 402 | my @subfields = @{ $field->[SUBS..$#$field] }; | ||
| 403 | foreach my $subfield (@subfields) { | ||
| 404 | my ($code, $valref) = @$subfield; | ||
| 405 | print '$', $code, ' ', $$valref; | ||
| 406 | } | ||
| 407 | print "\n"; | ||
| 408 | } | ||
| 409 | } | ||
| 410 | } $filehandle, %options; | ||
| 411 | |||
| 412 | Options: | ||
| 413 | |||
| 414 | =over 4 | ||
| 415 | |||
| 416 | =item B<print_all> | ||
| 417 | |||
| 418 | All MARC records will be printed to STDOUT. Any changes you make will be | ||
| 419 | reflected in what is printed. | ||
| 420 | |||
| 421 | =item B<drop> I<array_ref> | ||
| 422 | |||
| 423 | 'drop' => [ '001', '090' ] | ||
| 424 | |||
| 425 | Tags of fields to be dropped unconditionally. | ||
| 426 | |||
| 427 | =item B<only> I<array_ref> | ||
| 428 | |||
| 429 | 'only' => [ qw(001 004 008 852 856) ] | ||
| 430 | |||
| 431 | Tags of fields to be preserved; all other fields will be dropped. (The leader | ||
| 432 | is never dropped, of course.) | ||
| 433 | |||
| 434 | =item B<error> | ||
| 435 | |||
| 436 | A code reference to call when there is an error. | ||
| 437 | |||
| 438 | 'error' => sub { exit -1 } | ||
| 439 | |||
| 440 | =item B<strict> | ||
| 441 | |||
| 442 | 'strict' => 1 | ||
| 443 | |||
| 444 | Halt processing when an ill-formed MARC record is encountered. | ||
| 445 | |||
| 446 | =back | ||
| 447 | |||
| 448 | Options will be more fully documented in a future release; in the meantime, | ||
| 449 | read the source code. | ||
| 450 | |||
| 451 | =head2 marcparse | ||
| 452 | |||
| 453 | ($leader, $fields) = marcparse(\$string, %options); | ||
| 454 | |||
| 455 | Parses a MARC record into the same data structure that B<marcloop> uses. | ||
| 456 | Options are mostly the same as for B<marcloop>. | ||
| 457 | |||
| 458 | =head2 marcbuild | ||
| 459 | |||
| 460 | print marcbuild($leader, $fields); | ||
| 461 | |||
| 462 | Builds a raw MARC record from the same data structure that B<marcloop> uses and | ||
| 463 | that B<marcparse> produces. | ||
| 464 | |||
| 465 | =head2 marcfield | ||
| 466 | |||
| 467 | # Control field | ||
| 468 | push $@fields, marcfield( | ||
| 469 | '001', # Tag | ||
| 470 | 1234567, # Content | ||
| 471 | ); | ||
| 472 | |||
| 473 | # Data field | ||
| 474 | push $@fields, marcfield( | ||
| 475 | '245', # Tag | ||
| 476 | ' ', ' ', # Indicators | ||
| 477 | 'a' => 'Blah blah', # Subfield $a | ||
| 478 | 'c' => 'Amy Emu', # Subfield $c | ||
| 479 | ... # More subfields | ||
| 480 | ); | ||
| 481 | |||
| 482 | A convenience function to build a data structure representing a single MARC21 | ||
| 483 | field. | ||
| 484 | |||
| 485 | =head1 BUGS | ||
| 486 | |||
| 487 | The documentation is woefully lacking and probably just plain wrong. Read the | ||
| 488 | source code -- or, better yet, step through it in the Perl debugger -- to get a | ||
| 489 | better understanding of what this code does. | ||
| 490 | |||
| 491 | Unhandled errors cause ill-formed records to be dropped without notice. You | ||
| 492 | must specify an B<error> option to B<marcloop> to get around this. | ||
| 493 | |||
| 494 | =head1 AUTHOR | ||
| 495 | |||
| 496 | Paul Hoffman E<lt>paul@flo.orgE<gt>. | ||
| 497 | |||
| 498 | =head1 COPYRIGHT | ||
| 499 | |||
| 500 | Copyright 2009-2010 Fenway Libraries Online. Released under the GNU Public | ||
| 501 | License, version 3. |