Net/Z3950/FOLIO/Config.pm

filter: Statementstmt 77.6% 87 / 112 Branchbran 61.3% 27 / 44 Conditioncond 16.6% 2 / 12 MC/DCmcdc 12.5% 1 / 8 Subroutinesub 100% 17 / 17 Podpod 0.0% 0 / 4 totaltotal 68.0% 134 / 197 CC 32 SCAR 20.7
CC
32
Cov
69%
_expandScalarVariableReference
17.8
_compileConfigFile
13.1
_compileConfig
12.0
CRAP 7.9
« Net/Z3950/FOLIO/Record.pm ↑ Summary Net/Z3950/FOLIO/PostProcess/MARC.pm »
1 package Net::Z3950::FOLIO::Config;
2  
3 1 ▶use 5.008000;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
6  
7 1 ▶use IO::File;
Subroutine BEGIN: called CC=1 SCAR=0.0
8 1 ▶use Cpanel::JSON::XS qw(decode_json);
Subroutine BEGIN: called CC=1 SCAR=0.0
9  
10  
11 # Possible values of $missingAction
12 5 ▶sub MISSING_ERROR { 0 }
Subroutine MISSING_ERROR: called CC=1 SCAR=0.0
Pod undocumented
13 5 ▶sub MISSING_TENANT { 1 }
Subroutine MISSING_TENANT: called CC=1 SCAR=0.0
Pod undocumented
14 5 ▶sub MISSING_FILTER { 2 }
Subroutine MISSING_FILTER: called CC=1 SCAR=0.0
Pod undocumented
15  
16  
17 5 ▶sub new {
Subroutine new: called CC=1 SCAR=0.0
Pod undocumented
18 5 my $class = shift();
19 5 my($cfgbase, @extras) = @_;
20  
21 5 my $cfg = _compileConfig($cfgbase, @extras);
22 5 return bless $cfg, $class;
23 }
24  
25  
26 5 ▶sub _compileConfig {
Subroutine _compileConfig: called CC=9 SCAR=24.8
27 5 my($cfgbase, @extras) = @_;
28  
29 5 my $cfg = _compileConfigFile($cfgbase, undef, MISSING_ERROR);
30  
31 5 my $isFirst = 1;
32 5 ▶ while (@extras) {
Branch: 1 branch
BranchTrueFalseTotal
if (@extras) 10 5 15
33 10 my $extra = shift @extras;
34 10 ▶ my $overlay = _compileConfigFile($cfgbase, $extra, $isFirst ? MISSING_TENANT : MISSING_FILTER);
Branch: 1 branch
BranchTrueFalseTotal
$isFirst ? : 5 5 10
35 10 $isFirst = 0;
36 10 _mergeConfig($cfg, $overlay);
37 }
38  
39 5 my $path = $cfgbase;
40 5 ▶ if ($path =~ /\//) {
Branch: 1 branch
BranchTrueFalseTotal
if ($path =~ m[/]) { } 5 0 5
41 5 $path =~ s/(.*)?\/.*/$1/;
42 } else {
43 0 $path = '.';
44 }
45  
46 my $gqlfile = $cfg->{graphqlQuery}
47 5 ▶ or die "$0: no GraphQL query file defined";
Branch: 1 branch
BranchTrueFalseTotal
my $gqlfile = $cfg->{"graphqlQuery"} or die "$0: no GraphQL query file defined" 5 0 5
48 5 ▶ my $fh = new IO::File("<$path/$gqlfile")
Branch: 1 branch
BranchTrueFalseTotal
my $fh = "IO::File"->new("<$path/$gqlfile") or die "$0: can't open GraphQL query file '$path/${gqlfile}': $!" 5 0 5
49 or die "$0: can't open GraphQL query file '$path/$gqlfile': $!";
50 5 { local $/; $cfg->{graphql} = <$fh> };
51 5 $fh->close();
52  
53 5 my $esets = $cfg->{xmlElementSets};
54 5 ▶ if ($esets) {
Branch: 1 branch
BranchTrueFalseTotal
if ($esets) 0 5 5
55 0 foreach my $setname (keys %$esets) {
56 0 my $filename = $esets->{$setname};
57 0 ▶ my $fh = new IO::File("<$path/$filename")
Branch: 1 branch
BranchTrueFalseTotal
my $fh = "IO::File"->new("<$path/$filename") or Net::Z3950::FOLIO::_throw(1, "can't open XSLT stylesheet file '$path/${filename}': $!") 0 0 0
58 or Net::Z3950::FOLIO::_throw(1, "can't open XSLT stylesheet file '$path/$filename': $!");
59 0 { local $/; $esets->{$setname} = <$fh> };
60 0 $fh->close();
61 }
62 }
63  
64 5 return $cfg;
65 }
66  
67  
68 15 ▶sub _compileConfigFile {
Subroutine _compileConfigFile: called CC=7 SCAR=25.7
69 15 my($cfgbase, $cfgsub, $missingAction) = @_;
70  
71 15 ▶ my $cfgname = $cfgbase . ($cfgsub ? ".$cfgsub" : '') . '.json';
Branch: 1 branch
BranchTrueFalseTotal
$cfgsub ? : 10 5 15
72 15 my $fh = new IO::File("<$cfgname");
73 15 ▶ if (!$fh) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($fh) 0 15 15
74 0 ▶ if ($! == 2 && $missingAction == MISSING_TENANT) {
Branch: 2 branches
BranchTrueFalseTotal
if ($! == 2 and $missingAction == MISSING_TENANT()) { } 0 0 0
elsif ($! == 2 and $missingAction == MISSING_FILTER()) { } 0 0 0
Condition0%0/3
$! == 2 and $missingAction == MISSING_TENANT()
!ll&&!rl&&r
000
MC/DC0%0/2
$! == 2 and $missingAction == MISSING_TENANT()
$! == 2 $missingAction == MISSING_TENANT()
Truth table0%0/3
A and B
A= $! == 2B= $missingAction == MISSING_TENANT()
ABresult
0X0
100
111
75 0 Net::Z3950::FOLIO::_throw(109, "$cfgsub");
76 ▶ } elsif ($! == 2 && $missingAction == MISSING_FILTER) {
Condition0%0/3
$! == 2 and $missingAction == MISSING_FILTER()
!ll&&!rl&&r
000
MC/DC0%0/2
$! == 2 and $missingAction == MISSING_FILTER()
$! == 2 $missingAction == MISSING_FILTER()
Truth table0%0/3
A and B
A= $! == 2B= $missingAction == MISSING_FILTER()
ABresult
0X0
100
111
77 0 Net::Z3950::FOLIO::_throw(1, "filter not configured: $cfgsub");
78 }
79 0 die "$0: can't open config file '$cfgbase.json': $!"
80 }
81  
82 15 my $json; { local $/; $json = <$fh> };
83 15 $fh->close();
84  
85 15 my $cfg = decode_json($json);
86 15 _expandVariableReferences($cfg);
87 15 return $cfg;
88 }
89  
90  
91 46 ▶sub _expandVariableReferences {
Subroutine _expandVariableReferences: called CC=2 SCAR=6.9
92 46 my($obj) = @_;
93  
94 46 foreach my $key (sort keys %$obj) {
95 119 $obj->{$key} = _expandSingleVariableReference($key, $obj->{$key});
96 }
97  
98 46 return $obj;
99 }
100  
101 125 ▶sub _expandSingleVariableReference {
Subroutine _expandSingleVariableReference: called CC=5 SCAR=16.2
102 125 my($key, $val) = @_;
103  
104 125 ▶ if (ref($val) eq 'HASH') {
Branch: 4 branches
BranchTrueFalseTotal
if (ref $val eq 'HASH') { } 31 94 125
elsif (ref $val eq 'ARRAY') { } 3 91 94
elsif (ref $val eq 'JSON::PP::Boolean') { } 3 88 91
elsif (not ref $val) { } 88 0 88
105 31 return _expandVariableReferences($val);
106 } elsif (ref($val) eq 'ARRAY') {
107 3 return [ map { _expandSingleVariableReference($key, $_) } @$val ];
108 } elsif (ref($val) eq 'JSON::PP::Boolean') {
109 3 return $val;
110 } elsif (!ref($val)) {
111 88 return _expandScalarVariableReference($key, $val);
112 } else {
113 1 ▶ use Data::Dumper;
Subroutine BEGIN: called CC=1 SCAR=0.0
114 0 die "non-hash, non-array, non-boolean, non-scalar configuration key '$key' = ", Dumper($val);
115 }
116 }
117  
118 88 ▶sub _expandScalarVariableReference {
Subroutine _expandScalarVariableReference: called CC=5 SCAR=28.8
119 88 my ($key, $val) = @_;
120  
121 88 my $orig = $val;
122 88 ▶ while ($val =~ /(.*?)\$\{(.*?)}(.*)/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($val =~ /(.*?)\$\{(.*?)}(.*)/) 0 88 88
123 0 my($pre, $inclusion, $post) = ($1, $2, $3);
124  
125 0 my($name, $default);
126 0 ▶ if ($inclusion =~ /(.*?)-(.*)/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($inclusion =~ /(.*?)-(.*)/) { } 0 0 0
127 0 $name = $1;
128 0 $default = $2;
129 } else {
130 0 $name = $inclusion;
131 0 $default = undef;
132 }
133  
134 0 ▶ my $env = $ENV{$name} || $default;
Condition0%0/3
$ENV{$name} || $default
l!l&&r!l&&!r
000
MC/DC0%0/2
$ENV{$name} || $default
$ENV{$name} $default
Truth table0%0/3
A || B
A= $ENV{$name}B= $default
ABresult
1X1
011
000
135 0 ▶ if (!defined $env) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $env) 0 0 0
136 0 warn "environment variable '$2' not defined for '$key'";
137 0 $env = '';
138 }
139 0 $val = "$pre$env$post";
140 }
141  
142 88 return $val;
143 }
144  
145  
146 10 ▶sub _mergeConfig {
Subroutine _mergeConfig: called CC=6 SCAR=17.9
147 10 my($base, $overlay) = @_;
148  
149 10 my %complex_keys = (
150 okapi => 1,
151 login => 1,
152 indexMap => 1,
153 marcHoldings => 1,
154 fieldDefinitions => {
155 holding => 1,
156 circulation => 1,
157 },
158 );
159  
160 10 foreach my $key (keys (%complex_keys)) {
161 50 ▶ if (defined $overlay->{$key}) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $overlay->{$key}) 9 41 50
162 9 ▶ if (ref $base->{$key} eq 'HASH') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $base->{$key} eq 'HASH') { } 6 3 9
163 6 _mergeHash($base->{$key}, $overlay->{$key}, $complex_keys{$key});
164 } else {
165 3 $base->{$key} = $overlay->{$key};
166 }
167 }
168 }
169  
170 10 foreach my $key (sort keys %$overlay) {
171 18 ▶ if (!$complex_keys{$key}) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($complex_keys{$key}) 9 9 18
172 9 $base->{$key} = $overlay->{$key};
173 }
174 }
175 }
176  
177  
178 7 ▶sub _mergeHash {
Subroutine _mergeHash: called CC=4 SCAR=13.9
179 7 my($base, $overlay, $keysOfsubStructures) = @_;
180  
181 7 foreach my $key (sort keys %$overlay) {
182 9 ▶ if (ref $keysOfsubStructures eq 'HASH' && $keysOfsubStructures->{$key}) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $keysOfsubStructures eq 'HASH' and $keysOfsubStructures->{$key}) { } 1 8 9
Condition66%2/3
ref $keysOfsubStructures eq "HASH" and $keysOfsubStructures->{$key}
!ll&&!rl&&r
801
MC/DC50%1/2
ref $keysOfsubStructures eq "HASH" and $keysOfsubStructures->{$key}
ref $keysOfsubStructures eq "HASH" $keysOfsubStructures->{$key}
Truth table66%2/3
A and B
A= ref $keysOfsubStructures eq "HASH"B= $keysOfsubStructures->{$key}
ABresult
0X0
100
111
183 1 _mergeHash($base->{$key}, $overlay->{$key}, $keysOfsubStructures->{$key});
184 } else {
185 8 $base->{$key} = $overlay->{$key};
186 }
187 }
188 }
189  
190 =encoding utf-8
191  
192 =head1 NAME
193  
194 Net::Z3950::FOLIO::Config - configuration file for the FOLIO Z39.50 gateway
195  
196 =head1 SYNOPSIS
197  
198 {
199 "okapi": {
200 "url": "https://folio-snapshot-okapi.dev.folio.org",
201 "tenant": "${OKAPI_TENANT-indexdata}"
202 },
203 "login": {
204 "username": "diku_admin",
205 "password": "${OKAPI_PASSWORD}"
206 },
207 "indexMap": {
208 "1": "author",
209 "7": "identifiers/@value/@identifierTypeId=\"8261054f-be78-422d-bd51-4ed9f33c3422\"",
210 "4": "title",
211 "12": {
212 "cql": "hrid",
213 "relation": "==",
214 "omitSortIndexModifiers": [ "missing", "case" ]
215 },
216 "21": "subject",
217 "1016": "author,title,hrid,subject"
218 },
219 "queryFilter": "source=marc",
220 "graphqlQuery": "instances.graphql-query",
221 "chunkSize": 5,
222 "fieldDefinitions": {
223 "circulation": {
224 "availableThru": "permanentLoanType"
225 }
226 },
227 "marcHoldings": {
228 "restrictToItem": 0,
229 "field": "952",
230 "indicators": [" ", " "],
231 "holdingsElements": {
232 "t": "copyNumber"
233 },
234 "itemElements": {
235 "b": "itemId",
236 "k": "_callNumberPrefix",
237 "h": "_callNumber",
238 "m": "_callNumberSuffix",
239 "v": "_volume",
240 "e": "_enumeration",
241 "y": "_yearCaption",
242 "c": "_chronology"
243 }
244 },
245 "postProcessing": {
246 "marc": {
247 "008": { "op": "regsub", "pattern": "([13579])", "replacement": "[$1]", "flags": "g" },
248 "245$a": [
249 { "op": "stripDiacritics" },
250 { "op": "regsub", "pattern": "[abc]", "replacement": "*", "flags": "g" }
251 ]
252 },
253 "opac": {
254 "holding": {
255 "callNumber": {
256 "op": "regsub",
257 "pattern": "(.*)",
258 "replacement": "%{_callNumberPrefix}$1%{_callNumberSuffix}"
259 }
260 }
261 }
262 },
263 "xmlElementSets": {
264 "testmarc": "xslt/folio2marcxml.xsl",
265 "testopac": "xslt/folio2opac.xsl"
266 }
267 }
268  
269 =head1 DESCRIPTION
270  
271 The FOLIO Z39.50 gateway C<z2folio> is configured by a stacking set of
272 JSON files whose basename is specified on the command-line. These
273 files specify how to connect to FOLIO, how to log in, and how to
274 search.
275  
276 The structure of each of these file is the same, and the mechanism by
277 which they are stacked is described below. The shared format is
278 simple. There are several top-level sections, each described in its own
279 section below, and each of them is an object with several keys that can
280 exist in it.
281  
282 If any string value contains sequences of the form C<${NAME}>, they
283 are each replaced by the values of the corresponding environment
284 variables C<$NAME>, providing a mechanism for injecting values into
285 the configuration. This is useful if, for example, it is necessary to
286 avoid embedding authentication secrets in the configuration file.
287  
288 When substituting environment variables, the bash-like fallback syntax
289 C<${NAME-VALUE}> is recognised. This evaluates to the value of the
290 environment variable C<$NAME> when defined, falling back to the
291 constant value C<VALUE> otherwise. In this way, the configuration can
292 include default values which may be overridden with environment
293 variables.
294  
295  
296 =head2 C<okapi>
297  
298 Contains three elements (two mandatory, one optional), all with string values:
299  
300 =over 4
301  
302 =item C<url>
303  
304 The full URL to the Okapi server that provides the gateway to the
305 FOLIO installation.
306  
307 =item C<graphqlUrl> (optional)
308  
309 Usually, the main Okapi URL is used for all interaction with FOLIO:
310 logging in, searching, retrieving records, etc. When the optional
311 C<graphqlUrl> configuration entry is provided, it is used for GraphQL
312 queries only. This provides a way of "side-loading" mod-graphql, which
313 is useful in at least two situations: when the FOLIO snapshot services
314 are unavailable (since the production services do not presently
315 included mod-graphql); and when you need to run against a development
316 version of mod-graphql so you can make changes to its behaviour.
317  
318 =item C<tenant>
319  
320 The name of the tenant within that FOLIO installation whose inventory
321 model should be queried.
322  
323 =back
324  
325 =head2 C<login>
326  
327 Contains two elements, both with string values:
328  
329 =over 4
330  
331 =item C<username>
332  
333 The name of the user to log in as, unless overridden by authentication information in the Z39.50 init request.
334  
335 =item C<password>
336  
337 The corresponding password, unless overridden by authentication information in the Z39.50 init request.
338  
339 =back
340  
341 =head2 C<nologin>
342  
343 If specified and set to 1, then no login is performed, and the
344 C<login> section need not be provided.
345  
346 =head2 C<indexMap>
347  
348 Contains any number of elements. The keys are the numbers of BIB-1 use
349 attributes, and the corresponding values contain instructions about
350 the indexes in the FOLIO instance record to map those access-points
351 to. The key C<default> is special, and is used for terms where no BIB-1
352 use attribute is specified.
353  
354 Each value may be either a string, in which case it is interpreted as
355 a CQL index to map to (see below for details), or an object. When the
356 object version is used, that object's C<cql> member contains the CQL
357 index mapping (see below), and any of the following additional members
358 may also be included:
359  
360 =over 4
361  
362 =item C<omitSortIndexModifiers>
363  
364 A bug in FOLIO's CQL query interpreter means that for some indexes,
365 query translation will fail if a sort-specification is provided that
366 requests certain valid behaviours, e.g. a case-sensitive search on the
367 HRID index. To work around this until it's fixed, an index's
368 C<omitSortIndexModifiers> allows you to specify a list of the
369 index-modifier types that they do not support, so that the server can
370 omit those qualifiers when creating sort-specifications. The valid
371 index-modifier types are C<missing>, C<relation> and C<case>.
372  
373 =item C<relation>
374  
375 If specified, the value is the relation that should be used instead of
376 C<=> by default when searching in this index. This is useful mostly
377 for defaulting to the strict-equality relation C<==> for indexes whose
378 values are atomic, such as identifiers.
379  
380 =back
381  
382 Each C<cql> value (or string value when the object form is not used)
383 may be a comma-separated list of multiple CQL indexes to be queried.
384  
385 Each CQL index specified as a value, or as one of the comma-separated
386 components of a value, may contain a forward slash. If it does, then
387 the part before the slash is used as the actual index name, and the
388 part after the slash as a CQL relation modifier. For example, if the
389 index map contains
390  
391 "999": "foo/bar=quux"
392  
393 Then a search for C<@attr 1=9 thrick> will be translated to the CQL
394 query C<foo =/bar=quux thrick>.
395  
396 =head2 C<queryFilter>
397  
398 If specified, this is a CQL query which is automatically C<and>ed with
399 every query submitted by the client, so it acts as a filter allowing
400 through only records that satisfy it. This might be used, for example,
401 to specify C<source=marc> to limit search result to only to those
402 FOLIO instance records that were translated from MARC imports.
403  
404 See the section below on B<Configuring filters>.
405  
406 =head2 C<graphqlQuery>
407  
408 The name of a file, in the same directory as the main configuration
409 file, which contains the text of the GraphQL query to be used to
410 obtain the instance, holdings and item data pertaining to the records
411 identified by the CQL query.
412  
413 =head2 C<chunkSize>
414  
415 An integer specifying how many records to fetch from FOLIO with each
416 search. This can be tweaked to tune performance. Setting it too low
417 will result in many requests with small numbers of records returned
418 each time; setting it too high will result in fetching and decoding
419 more records than are actually wanted.
420  
421 =head2 C<fieldDefinitions>
422  
423 An optional object specifying how the sources from which some
424 particular holdings fields should draw their data, overriding the
425 default sources.
426  
427 The keys of this object specify domains in which the fields can be
428 found: supported domains are C<holding> and C<circulation>}.
429  
430 Within the C<holdings> domain, no fields are presently supported;
431 within the C<circulation> domain, only the C<availableThru> field is
432 presently supported. Support for further fields may be added as
433 required.
434  
435 Each field within a domain is specified by its name as key, and the
436 corresponding value is a string specifying a path from which to fetch
437 the data. These strings follow the same syntax as L<the get function
438 in JavaScript's lodash library|https://docs-lodash.com/v4/get/>. It is
439 a dot-separate sequence of source field-names to navigate down, with
440 bracketed numbers indicating the selection of one element of an
441 array. For example, the path
442 C<temporaryLocation.servicePoints[0].discoveryDisplayName> fetches the
443 C<discoveryDisplayName> field from the first element of the
444 C<servicePoints> array within the C<temporaryLocation> structure. It
445 will work when running on data with the shape C<{ temporaryLocation: {
446 servicePoints: [ { discoveryDisplayName: "Main" } ] } }>.
447  
448  
449 =head2 C<marcHoldings>
450  
451 An optional object specifying how holdings and item-level data should
452 be mapped into MARC fields. It contains up to five elements:
453  
454 =over 4
455  
456 =item C<restrictToItem>
457  
458 If specified and set to 1, then the item-level holding information
459 included in MARC records is restricted to that which pertains to the
460 barcode mentioned in the search that yielded the record, if any. If
461 zero (the default), then information on all holdings and items is
462 included.
463  
464 =item C<field> (mandatory)
465  
466 A string specifying which MARC field should be used for holdings
467 information. When a record contains multiple holdings, a separate
468 instance of this MARC field is created for each holding.
469  
470 =item C<indicators> (mandatory)
471  
472 An array containing two strings, each of them specifying one of the
473 two indicators to be used in the MARC field that contains
474 holdings. There must be exactly two elements: blank indicators can
475 be specified as a single space.
476  
477 information.
478  
479 =item C<fieldPerItem>
480  
481 If specified and set to a true value, then a separate MARC field is
482 generated for each item in each holding. When this is absent or set to
483 a false value (the default), one MARC field is generated for each
484 holding, and multiple items within each holding are represented by
485 repeating sets of subfields within that field.
486  
487 This setting makes it simpler to access information about individual
488 items, at the cost of losing information about how they are grouped
489 into holdings.
490  
491 =item C<holdingsInEachItem>
492  
493 If specified and set to a true value, then holdings information is
494 included in each item's MARC field. When this is absent or set to a
495 false value (the default), only the first item in a given holding
496 includes the holdings fields.
497  
498 This setting makes it simpler to access all the information about any
499 single item, at the cost of wasting record size and network bandwidth
500 on redundant information.
501  
502 =item C<holdingsElements>
503  
504 An object specifying MARC subfields that should be set from
505 holdings-level data. The keys are the single-character names of the
506 subfields, and the corresponding values are the names of
507 holdings-level fields in the OPAC XML record structure.
508  
509 See C<itemElements> for detail of the structure.
510  
511 =item C<itemElements>
512  
513 An object specifying MARC subfields that should be set from item-level
514 data. The keys are the single-character names of the subfields, and
515 the corresponding values are the names of item-level fields in the
516 OPAC XML record structure. In addition to the standard field names,
517 several additional special fields are avaialable, not part of the OPAC
518 Z39.50 record, assigned names that begin with underscores:
519  
520 =over 4
521  
522 =item C<_enumeration>
523  
524 =item C<_chronology>
525  
526 =item C<_callNumber>
527  
528 =item C<_callNumberPrefix>
529  
530 =item C<_callNumberSuffix>
531  
532 =item C<_permanentLocation>
533  
534 =item C<_holdingsLocation>
535  
536 =item C<_volume>
537  
538 =item C<_yearCaption>
539  
540 =item C<_accessionNumber>
541  
542 =item C<_copyNumber>
543  
544 =item C<_descriptionOfPieces>
545  
546 =item C<_discoverySuppress>
547  
548 =item C<_hrid>
549  
550 =item C<_id>
551  
552 =item C<_itemIdentifier>
553  
554 =back
555  
556 Since there may be multiple items in a single holding, sets of these
557 fields can repeat, e.g. for a holding with two items each specifying
558 data that is encoded in the C<b>, C<e> and C<h> subfields, the field
559 would take the form
560  
561 $b 46243154 $e 1994/95 v.1 $h
562 $b 46243072 $e 1994/95 v.2 $h TD224.I3I58b
563  
564 =back
565  
566 =head2 C<postProcessing>
567  
568 Specifies sets of transformations to be applied to the values of
569 fields retrieved from the back-end. Two top-level keys are supported,
570 for post-processing of two different types of record, and they work
571 slightly differently:
572  
573 =over 4
574  
575 =item C<marc>
576  
577 Specifies transformations to be applied to MARC records (either
578 standalone or as part of OPAC records). The value is an object: keys
579 are the names of simple MARC fields, such as C<008>; or of complex
580 field$subfield combinations, such as C<245$a>. The corresponding
581 values specify the transformations that should be applied to the
582 values of these fields and subfields -- see below.
583  
584 =item C<opac>
585  
586 Specifies transformations to be applied to the holdings and
587 circulations parts of OPAC records. The value is an object which can
588 contain either or both C<holding> and C<circulation> keys; the values
589 corresponding to each of these keys are themselves objects which
590 specify which fields within the holdings records, or their circulation
591 subrecords, to transform.
592  
593 The keys within both of these subobjects are XML fieldnames, including
594 both those defined for the OPAC record and the "special fields"
595 beginning with underscores that are listed above in the
596 C<itemElements> section -- for example, C<_callNumberPrefix>. The
597 corresponding values specify the transformations that should be
598 applied to the values of these fields and subfields -- see below.
599  
600 =back
601  
602 In summary, the C<marc> section lists fieldnames (both simple and
603 complex) directly; whereas the C<opac> section contains C<holding> and
604 C<circulation> subsections which list the XML fieldnames of the two
605 sections separately.
606  
607 Post-processing transformation specifications are the same for MARC
608 and OPAC records. Each value in the specification may be either a
609 single transformation or an array o zero or more transformations which
610 will be applied in the specified order.
611  
612 Individual transformations are represented by objects. The fundamental
613 behaviour of each transformations is determined by an C<op> key whose
614 value specifies the required operation.
615  
616 The following transformation operations are supported:
617  
618 =over 4
619  
620 =item C<stripDiacritics>
621  
622 All diacritics are stripped from the value in the relevant field: for
623 example, C<délétère> becomes C<deletere>.
624  
625 =item C<regsub>
626  
627 A regular expression substitution is performed on the value in the
628 relevant field, as specified by the parameters in the transformation
629 object:
630  
631 =over 4
632  
633 =item C<pattern>
634  
635 A regular expression intended to matching some part of the field
636 value. This is Perl regular expression, as overviewed in
637 L<perlretut|https://perldoc.perl.org/perlretut>
638 and fully documented in
639 L<perlre|https://perldoc.perl.org/perlre>
640 and as such supports advanced facilities such as back-references.
641  
642 =item C<replacement>
643  
644 The string with which to replace the part of the field value that
645 matches the pattern. This may include numbered references C<$1>,
646 C<$2>, etc., to parenthesized sub-expressions in the pattern. (If this
647 statement means nothing to you, you need to
648 L<go and read about regular expressions|https://perldoc.perl.org/perlretut>.)
649  
650 Replacement strings may also include sequences of the form
651 I<%{fieldname}>, where I<fieldname> is interpreted in a manner
652 appropriate for the kind of record being post-processed. For MARC
653 records, I<fieldname> is either a simple control-field tag such as
654 C<001> or a field-and-subfield combination like C<245$a>; for OPAC
655 records, it is the name of an XML field from the same part of the
656 record (i.e. a holdings field if substituting holdings data, and a
657 circulation field if substituting circulation data).
658  
659 Such sequences cause the value of the specified field within the
660 current record to be interpolated, so that for example a replacement
661 string C<%{001}/%{245a}> in a MARC-record substitution will cause the
662 text that matches the regular expression to be replaced by the
663 contents of the C<001> and C<245$a> fields separated by a slash.
664  
665 This mechanism yields a powerful and general facility allowing
666 installations with complex requirements to generate exactly the detail
667 they need. For example, it can be used to implement fallbacks, as in
668 this case where if C<952$2> has no value it's replaced by the value in
669 C<952$b>. (This could be used for situations like reporting the
670 item-level copy-number if that's present, but falling back to the
671 holdings-level copy-number if not.)
672  
673 "952$2": [
674 { "op": "regsub", "pattern": "^$", "replacement": "%{952$b}" }
675 ]
676  
677 Or a location string could be built in the C<$z> subfield from
678 fragments in the C<$1>, C<$2> and C<$2> subfields:
679  
680 "952$z": [
681 { "op": "regsub", "pattern": ".*", "replacement": "%{952$1}/%{952$2}/%{952$3}" }
682 ]
683  
684  
685 =item C<flags>
686  
687 Optionally, a set of flags such as C<g> for global replacement, C<i>
688 for case-insensitivity, etc. See
689 L<Using regular expressions in Perl|https://perldoc.perl.org/perlretut#Using-regular-expressions-in-Perl>.
690  
691 =back
692  
693 =back
694  
695 For example, the MARC post-processing directive
696  
697 "245$a": [
698 { "op": "stripDiacritics" },
699 { "op": "regsub", "pattern": "[abc]", "replacement": "*", "flags": "g" }
700 ]
701  
702 Says first to remove all diacritics from the C<245$a> (title) field of
703 the MARC record (so that for example C<é> becomes C<e>), then to
704 replace all vowels with asterisks.
705  
706  
707 =head2 C<xmlElementSets>
708  
709 If provided, a mapping of XML element-set names to the names of XSLT 1.0 stylesheets, relative to the location of the top-level configuration file. These stylesheets must be provided as part of the broader configuration. When an XML record is requested in non-standard format (i.e. record-syntax is C<xml> and element-set name is not C<raw>, C<usmarc> or C<opac>), the raw XML record is processed using the nominated XSLT stylesheet and the reslt is returned. This facility can be used to provide clients with, for example, MODS records.
710  
711 B<NOTE.> Only XSLT 1.0 is supported: not XSLT 2.0. (There seems to be no Perl module that implements XSLT 2.0.)
712  
713  
714 =head1 CONFIGURATION STACKING
715  
716 To implement both multi-tenancy and per-database output tweaks that may be required for specific Z39.50 client application, it is necessary to allow flexibility in the configuration of the server, based both on Z39.50 database name and on further specifications. Three levels of configuration are therefore supported.
717  
718 =over 4
719  
720 =item 1
721  
722 A base configuration file is always used, and will typically provide the bulk of the configuration that is the same for all supported databases. Its base name is specified when the server is invoked -- for example, as the argument to the C<-c> command-line option of C<z2folio>: when the server is invoked as C<z2folio -c etc/config> the base configuration file is found at C<etc/config.json>.
723  
724 =item 2
725  
726 The Z39.50 database name provided in a search is used as a name of a sub-configuration specific to that database. So for example, if a Z39.50 search request come in for the database C<theo>, then the additional database-specific configuration file C<etc/config.theo.json> is also consulted. Often this file will specify the FOLIO tenant to be used for this database.
727  
728 Values provided in a database-specific configuration are added to those of the base configuration, overriding the base values when the same item is provided at both levels.
729  
730 =item 3
731  
732 One or more I<filters> may also be used to specify additional configuration. These are specified as part of the Z39.50 database name, separated from the main part of the name by pipe characters (C<|>). For example, if the database name C<theo|foo|bar> is used, then two additional filter-specific configuration files are also read, C<etc/config.foo.json> and C<etc/config.bar.json>. The values of filter-specific configurations override those of the base or database-specific configuration, and those of later filters override those of earlier filters.
733  
734 =back
735  
736 In the example used here, then, a server launched with C<z2folio -c etc/config> and serving a search against the Z39.50 database C<theo|foo|bar> will consult four configuration files:
737 C<etc/config.json>,
738 C<etc/config.theo.json> (if present),
739 C<etc/config.foo.json>
740 and
741 C<etc/config.bar.json>.
742  
743 This scheme allows us to handle several scenarios in a uniform way:
744  
745 =over 4
746  
747 =item *
748  
749 Basic configuration all in one place
750  
751 =item *
752  
753 Database-specific overrides, such as the FOLIO tenant and any customer-specific definition of ISBN searching (see issue ZF-24) in the database configuration, leaving the standard definition to apply to other tenants.
754  
755 =item *
756  
757 Application-specific overrides, such as those needed by the ABLE client (see issue ZF-25), specified only in a filter that is not used except when explicitly requested.
758  
759 =back
760  
761 =head1 CONFIGURING FILTERS
762  
763 By design, the FOLIO Z39.50 server follows L<the "Mechanism, Not Policy" approach|https://wiki.c2.com/?MechanismNotPolicy>: it is not opinionated about what kinds of records should be returned, how holdings should be encoded in MARC, etc. — instead, it invites institutions to configure it according to their preference.
764  
765 In some cases, that preference is for suppressed-from-discovery records to be omitted. So the configuration needs to be set up accordingly. The way to do this is with the C<queryFilter> configuration item, which is used to provide a query fragment that gets C<and>ed with every query submitted by the client.
766  
767 The configuration file could be modified to include, at the top level:
768  
769 "queryFilter": "cql.allRecords=1 NOT discoverySuppress=true"
770  
771 This could be done either in the top-level configuration, in a database-specific configuration, or in a filter configuration.
772  
773 When this was used this against a good-sized institution's test service, a search for "water" found 19395 records, as opposed to 39181 when the filter was not in place. Search time remained around one or two seconds.
774  
775 L<A more rigorous filter|https://wiki.folio.org/display/FOLIOtips/Searching> can be used:
776  
777 "queryFilter": "cql.allRecords=1 NOT discoverySuppress=true NOT holdingsItems.discoverySuppress=true NOT item.discoverySuppress=true"
778  
779 But as this is more complex, it has an impact on performance — as is the case when this query is used in the Inventory app's "query search". When this was used against the same institution's test service Chicago's test service, the "water" search came down to 19395 records, but it took nearly a minute to run.
780  
781 Or an intermediate version can be used that omits both suppressed instances and suppressed holdings, but not suppressed items. This got the result count down to 38791 — as one would expect, more than when only instances are suppress, but less then when items are also suppressed — and took two or three seconds.
782  
783 It's up to an individual installation to decide which trade-off best suits them. This trade-off may change as FOLIO's indexing changes: for example, if an index is added that makes the omit-instances-with-suppressed-items search much faster, that may become a more attractive option.
784  
785 =head1 SEE ALSO
786  
787 =over 4
788  
789 =item The C<z2folio> script conveniently launches the server.
790  
791 =item C<Net::Z3950::FOLIO> is the library that consumes this configuration.
792  
793 =item The C<Net::Z3950::SimpleServer> handles the Z39.50 service.
794  
795 =back
796  
797 =head1 AUTHOR
798  
799 Mike Taylor, E<lt>mike@indexdata.comE<gt>
800  
801 =head1 COPYRIGHT AND LICENSE
802  
803 Copyright (C) 2018-2022 The Open Library Foundation
804  
805 This software is distributed under the terms of the Apache License,
806 Version 2.0. See the file "LICENSE" for more information.
807  
808 =cut
809  
810 1 1;
« Net/Z3950/FOLIO/Record.pm ↑ Summary Net/Z3950/FOLIO/PostProcess/MARC.pm »