include/punk/punk_i18n.h

filter: Statementstmt 85.1% 161 / 189 Branchbran 86.2% 107 / 138 (18 uncoverable) Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 85.6% 268 / 327 (18 uncoverable) CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« include/punk/punk_response.h ↑ Summary include/punk/punk_static.h »
1 #ifndef PUNK_I18N_H
2 #define PUNK_I18N_H
3  
4 /* Punk::Plugin::I18n - translations and language negotiation.
5 *
6 * ---- the catalogue --------------------------------------------------------
7 *
8 * One JSON file per locale, named for the tag it holds:
9 *
10 * i18n/en.json i18n/en-GB.json i18n/fr.json
11 * { "greeting": "Hello, {name}" }
12 *
13 * Placeholders are `{name}` and not `%s`, because positional formats cannot
14 * be reordered and reordering is the entire reason a sentence needs
15 * translating: German puts the verb where English does not.
16 *
17 * ---- why this is not a hash of SVs ----------------------------------------
18 *
19 * Catalogues are read-only at request time, so reading them before the fork
20 * shares them across every worker. That is the argument, and it is wrong for
21 * Perl data: the first READ of an SV touches its refcount, the page is
22 * copied, and after a few minutes of traffic every worker holds a private
23 * copy of the catalogue. The sharing lasts about as long as it takes to serve
24 * the first request.
25 *
26 * So a catalogue is a block of bytes addressed by offset, built once at boot
27 * and never written again. A lookup compares bytes and returns a pointer
28 * INTO the block; nothing is allocated and no refcount is touched, so the
29 * pages stay shared for the life of the pool.
30 *
31 * That is also what lets the lookup serve both callers: `$c->locale` in a
32 * handler, and the per-request `locale` hash the templates read.
33 *
34 * ---- the block is Frozen's ------------------------------------------------
35 *
36 * It used to be hand-rolled here: a char buffer, an array of pi_entry, and a
37 * bucket array, three allocations per catalogue and two more for the arena.
38 * It is now ONE Frozen container holding { folded-tag => catalogue }, read
39 * through fz_abi.h - see punk_frozen.h for why that is the C table and not
40 * Frozen's Perl surface.
41 *
42 * The thing that changed is not the storage, it is that the block holds a
43 * TREE. The old form flattened { "a": { "b": x } } to the string "a.b" and
44 * faked the levels with a sentinel, so a literal key "a.b" was the same key -
45 * and which of the two answered was decided by the hash order of the process
46 * that loaded it. Two workers could disagree. A level and a leaf are now
47 * different nodes, so they are different keys, in every process.
48 *
49 * It costs one probe per level where the flat key cost one probe whatever
50 * its depth. That is the trade and it is not recoverable without giving the
51 * distinction back.
52 *
53 * ---- and not a cache ------------------------------------------------------
54 *
55 * Deliberately no `cache => 1` option. There is no per-request parse to cache
56 * away - the catalogue is resident from boot - and a cache lookup per key
57 * would be two orders of magnitude slower on the path that matters: a
58 * Punk::Cache::File hit is about 7.2us, of which 6.2us is the open syscall,
59 * so a page with forty translated strings would pay ~288us against a hash
60 * probe now.
61 */
62  
63 /* ---- why the files are read in Perl --------------------------------------
64 *
65 * Everything below builds the arena from an ALREADY DECODED hash. Finding the
66 * catalogues and parsing them happens in Punk::Plugin::I18n::register, in
67 * Perl, at boot, and that is deliberate twice over:
68 *
69 * - `opendir`/`readdir` in XS is the Win32 trap. There is no ambient DIR on
70 * native Windows and reaching for one costs a FindFirstFile shim; Punk
71 * builds there now, and a directory walk is not worth a second
72 * implementation of one.
73 * - a decode that croaks needs the FILENAME in the message, and catching a
74 * croak in C means JMPENV by hand, in a header that has to compile on
75 * every perl from 5.10. Two lines of eval in Perl say the same thing.
76 *
77 * It costs nothing at request time: this runs once, before the fork.
78 */
79  
80 #define PI_TAG_MAX 35 /* RFC 5646 says 35 is enough for any tag */
81 #define PI_KEY_MAX 512
82  
83 /* One catalogue: a language tag, and the node its translations hang off in
84 * the block. The whole arena is ONE Frozen container holding
85 * { folded-tag => catalogue }, so a catalogue is a handle rather than a
86 * structure, and a tag lookup is a probe on a hash node.
87 *
88 * The tag is a char array rather than a pointer into the block because
89 * _negotiate builds a pi_arena on the STACK, with tags and no container at
90 * all, to test the Accept-Language rules without a catalogue. A borrowed
91 * pointer would break that seam.
92 *
93 * `fz` is repeated per catalogue rather than reached through the arena so
94 * that pi_get and pi_is_prefix keep the signatures punk_lang.h calls them
95 * with. Eight bytes times the number of locales. */
96 typedef struct pi_cat {
97 char tag[PI_TAG_MAX + 1]; /* case-folded, NUL terminated */
98 STRLEN taglen;
99 fz_container *fz; /* NULL in the _negotiate seam */
100 uint32_t node; /* this catalogue's hash node */
101 } pi_cat;
102  
103 typedef struct pi_arena {
104 fz_container *c; /* owned; lives as long as the process */
105 uint32_t root; /* the hash of folded tag -> catalogue */
106 pi_cat *cat;
107 int ncat;
108 int def; /* index of the default locale, or -1 */
109 } pi_arena;
110  
111 /* ASCII fold. Language tags are case insensitive by RFC 5646 - `EN-gb` is
112 * `en-GB` - and folding once at load means the match rule never has to. */
113 2605 static char pi_fold(char c) {
114 2605 ▶ return (c >= 'A' && c <= 'Z') ? (char)(c - 'A' + 'a') : c;
Branch: 2 branches
BranchTrueFalseTotal
return (c >= 'A' && c <= 'Z') ? (char)(c - 'A' + 'a') : c; 2568 37 2605
return (c >= 'A' && c <= 'Z') ? (char)(c - 'A' + 'a') : c; 30 2538 2568
115 }
116  
117 919 static int pi_tag_eq(const char *a, STRLEN al, const char *b, STRLEN bl) {
118 STRLEN i;
119 919 ▶ if (al != bl) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (al != bl) return 0; 142 777 919
120 1206 ▶ for (i = 0; i < al; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < al; i++) 1014 192 1206
121 1014 ▶ if (pi_fold(a[i]) != pi_fold(b[i])) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (pi_fold(a[i]) != pi_fold(b[i])) return 0; 585 429 1014
122 192 return 1;
123 }
124  
125 /* A dotted key is a WALK now, not a joined string.
126 *
127 * `items.one` descends from the catalogue node through `items` to `one`,
128 * which costs one probe per segment where the flat key cost one probe
129 * whatever its depth. What it buys is that a level and a leaf are different
130 * nodes rather than the same entry wearing a sentinel, so { "a.b": x } and
131 * { "a": { "b": y } } stop being the same key. They were not merely
132 * ambiguous before: which of the two won was decided by Perl's per-process
133 * hash order, so two workers in one pool could answer differently.
134 *
135 * PUNK_FZ is read directly rather than passed in. It is a file-scope static
136 * in this translation unit, resolved once at boot by punk_fz(), and reading
137 * it here is what lets these two keep the signatures punk_lang.h calls them
138 * with - so the negotiation code did not have to change at all.
139 *
140 * ---- the rule Frozen does not enforce --------------------------------------
141 *
142 * A node is a uint32_t, and Frozen validates its SHAPE on every entry: the
143 * offset is in bounds, the alignment is right, the tag is a live kind. What
144 * it cannot validate is PROVENANCE. A node that is valid in one container
145 * names a different place in another, so reading it through the wrong
146 * container returns wrong data rather than failing.
147 *
148 * So a node never leaves the pi_cat it came from. That is why `fz` sits
149 * beside `node` in the struct instead of being reached through the arena:
150 * the pair travels together, and there is no call here that takes a node
151 * without the container it belongs to. */
152 50 static int pi_is_prefix(const pi_cat *c, const char *k, STRLEN kl) {
153 uint32_t node;
154 /*
155 # uncoverable branch true
156 # uncoverable condition left count:1 class:ignore_covered_err
157 # uncoverable condition left count:2 class:ignore_covered_err
158 # uncoverable condition left count:3 class:ignore_covered_err */
159 50 ▶ if (!c || !c->fz || !PUNK_FZ) return 0;
Branch: 3 branches
BranchTrueFalseTotal
if (!c || !c->fz || !PUNK_FZ) return 0; 50 0 50
if (!c || !c->fz || !PUNK_FZ) return 0; 50 0 50
if (!c || !c->fz || !PUNK_FZ) return 0; 0 50 50
160 50 node = (PUNK_FZ->path)(c->fz, c->node, k, kl, '.');
161 50 ▶ if (node == FZ_NOHANDLE) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (node == FZ_NOHANDLE) return 0; 48 2 50
162 2 return (PUNK_FZ->kind)(c->fz, node) == FZ_K_HASH;
163 }
164  
165 /* The value for a key, or NULL. Returns bytes INTO the block - the caller
166 * must not free them and must not hold them past the arena's life, which is
167 * the life of the process.
168 *
169 * `utf8` is an out-param because the block records the flag per string. The
170 * old arena did not, and _locale turned the flag on unconditionally, which
171 * was correct only because File::Raw::JSON happens to decode to UTF-8. Any
172 * other producer got mojibake. NULL is accepted for callers that do not
173 * care - the plural path and EXISTS. */
174 237 static const char *pi_get(const pi_cat *c, const char *k, STRLEN kl,
175 STRLEN *vlen, int *utf8) {
176 uint32_t node;
177 /*
178 # uncoverable branch true
179 # uncoverable condition left count:1 class:ignore_covered_err
180 # uncoverable condition left count:2 class:ignore_covered_err
181 # uncoverable condition left count:3 class:ignore_covered_err */
182 237 ▶ if (!c || !c->fz || !PUNK_FZ) return NULL;
Branch: 3 branches
BranchTrueFalseTotal
if (!c || !c->fz || !PUNK_FZ) return NULL; 237 0 237
if (!c || !c->fz || !PUNK_FZ) return NULL; 237 0 237
if (!c || !c->fz || !PUNK_FZ) return NULL; 0 237 237
183 237 node = (PUNK_FZ->path)(c->fz, c->node, k, kl, '.');
184 237 ▶ if (node == FZ_NOHANDLE) return NULL;
Branch: 1 branch
BranchTrueFalseTotal
if (node == FZ_NOHANDLE) return NULL; 54 183 237
185 /* No `kind` call before this. str answers NULL for anything that is not
186 * a string - it checks the tag and the bounds itself - so asking twice
187 * is a second call through the table for an answer already being
188 * computed. Two calls per lookup, not three, on the path that runs per
189 * translated string per request. */
190 183 return (PUNK_FZ->str)(c->fz, node, vlen, utf8);
191 }
192  
193 /* ---- loading -------------------------------------------------------------- */
194  
195 /* Free the arena. The block is one container, so this is one close plus the
196 * tag table. The container is closed through the table because it was opened
197 * through it, and (FZ->close) is PARENTHESISED: XSUB.h redefines `close` as a
198 * macro under PERL_IMPLICIT_SYS, which is the build failure Frozen 0.02 was
199 * released to fix in its own selftest. */
200 117 static void pi_arena_free(pTHX_ pi_arena *ar) {
201 /*
202 # uncoverable branch true
203 # uncoverable condition left count:1 class:ignore_covered_err */
204 117 ▶ if (!ar) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!ar) return; 0 117 117
205 /*
206 # uncoverable branch false
207 # uncoverable condition right count:1 class:ignore_covered_err
208 # uncoverable condition right count:2 class:ignore_covered_err */
209 117 ▶ if (ar->c && PUNK_FZ) (PUNK_FZ->close)(aTHX_ ar->c);
Branch: 2 branches
BranchTrueFalseTotal
if (ar->c && PUNK_FZ) (PUNK_FZ->close)(aTHX_ ar->c); 117 0 117
if (ar->c && PUNK_FZ) (PUNK_FZ->close)(aTHX_ ar->c); 117 0 117
210 117 ar->c = NULL;
211 117 Safefree(ar->cat);
212 117 Safefree(ar);
213 }
214  
215 /* What the boot walk found wrong, recorded rather than croaked.
216 *
217 * Nothing here croaks while the walk is running. Frozen's own fz_walk
218 * mallocs a segment stack and frees it after the walk returns, so a croak
219 * from inside longjmps past that free and leaks on every boot - a slow leak
220 * with no visible cause in a `punk dev` rebuild loop. The recursion below
221 * allocates nothing, but the same discipline is kept: the first fault is
222 * recorded and raised by the caller, so the rule holds if this ever goes
223 * back through the table. */
224 typedef struct pi_vcheck {
225 int err; /* 0 fine, else one of PI_V_* */
226 int plural; /* check plural categories? */
227 char path[PI_KEY_MAX + 1];
228 char what[32]; /* the offending kind, or category */
229 } pi_vcheck;
230  
231 #define PI_V_TYPE 1
232 #define PI_V_DOT 2
233 #define PI_V_PLURAL 3
234 #define PI_V_DEEP 4
235  
236 113 static const char *pi_kind_name(int k) {
237 /*
238 # uncoverable branch true count:2
239 # uncoverable branch true count:3
240 # uncoverable branch true count:4
241 # uncoverable branch true count:6
242 # uncoverable branch true count:7 */
243 113 switch (k) {
244 112 case FZ_K_UNDEF: return "null";
245 0 case FZ_K_FALSE: case FZ_K_TRUE: return "a boolean"; /* # uncoverable statement */
246 0 case FZ_K_INT: case FZ_K_UINT: return "a number"; /* # uncoverable statement */
247 0 case FZ_K_NUM: return "a number"; /* # uncoverable statement */
248 1 case FZ_K_ARRAY: return "an array";
249 0 case FZ_K_HASH: return "an object"; /* # uncoverable statement */
250 }
251 0 return "not a string"; /* # uncoverable statement */
252 }
253  
254 /* Walk a catalogue at boot, checking three things in one pass.
255 *
256 * fz_walk is not used, and the reason is not preference: it reports LEAVES
257 * only, so an array reaches the callback as its elements and the array node
258 * itself is never seen. An array is exactly one of the things that has to be
259 * refused, because nothing in the i18n surface can reach it - fz_path
260 * descends through hash nodes, so `list.0` stops at the array and answers
261 * absent. Left alone it is a silent miss that renders as the key.
262 *
263 * The path is built as it descends rather than kept as a segment stack, so
264 * the message has the joined path ready and the recursion allocates nothing.
265 */
266 1518 static void pi_walk_check(pi_vcheck *v, fz_container *fz, uint32_t node,
267 char *path, size_t plen, int depth) {
268 int kind;
269 uint32_t n, i;
270  
271 /*
272 # uncoverable branch true
273 # uncoverable condition left count:1 class:ignore_covered_err */
274 1518 ▶ if (v->err) return;
Branch: 1 branch
BranchTrueFalseTotal
if (v->err) return; 0 1518 1518
275 /*
276 # uncoverable branch true
277 # uncoverable condition left count:1 class:ignore_covered_err */
278 1518 ▶ if (depth > 64) { /* Frozen allows 256; a catalogue
Branch: 1 branch
BranchTrueFalseTotal
if (depth > 64) { /* Frozen allows 256; a catalogue 0 1518 1518
279 * that deep is a bug, not a
280 * translation */
281 0 v->err = PI_V_DEEP; /* # uncoverable statement */
282 0 path[plen] = '\0'; /* # uncoverable statement */
283 0 my_strlcpy(v->path, path, sizeof v->path); /* # uncoverable statement */
284 0 return; /* # uncoverable statement */
285 }
286  
287 1518 kind = (PUNK_FZ->kind)(fz, node);
288  
289 1518 ▶ if (kind == FZ_K_STR) {
Branch: 1 branch
BranchTrueFalseTotal
if (kind == FZ_K_STR) { 1136 382 1518
290 /* the last segment, for the plural rule */
291 1136 ▶ if (v->plural && depth >= 2) {
Branch: 2 branches
BranchTrueFalseTotal
if (v->plural && depth >= 2) { 6 1130 1136
if (v->plural && depth >= 2) { 2 4 6
292 2 size_t s = plen;
293 8 ▶ while (s > 0 && path[s - 1] != '.') s--; /* # uncoverable condition right count:1 class:ignore_covered_err */
Branch: 2 branches
BranchTrueFalseTotal
while (s > 0 && path[s - 1] != '.') s--; /* # uncoverable condition right count:1 class:ignore_covered_err */ 8 0 8
while (s > 0 && path[s - 1] != '.') s--; /* # uncoverable condition right count:1 class:ignore_covered_err */ 6 2 8
294 /*
295 # uncoverable branch false
296 # uncoverable condition right count:1 class:ignore_covered_err */
297 2 ▶ if (pi_cat_name(path + s, (STRLEN)(plen - s))) {
Branch: 1 branch
BranchTrueFalseTotal
if (pi_cat_name(path + s, (STRLEN)(plen - s))) { 2 0 2
298 2 path[plen] = '\0';
299 2 my_strlcpy(v->path, path, sizeof v->path);
300 2 my_strlcpy(v->what, path + s, sizeof v->what);
301 2 v->err = PI_V_PLURAL;
302 }
303 }
304 1136 return;
305 }
306  
307 382 ▶ if (kind != FZ_K_HASH) {
Branch: 1 branch
BranchTrueFalseTotal
if (kind != FZ_K_HASH) { 113 269 382
308 113 path[plen] = '\0';
309 113 my_strlcpy(v->path, path, sizeof v->path);
310 113 my_strlcpy(v->what, pi_kind_name(kind), sizeof v->what);
311 113 v->err = PI_V_TYPE;
312 113 return;
313 }
314  
315 269 n = (PUNK_FZ->count)(fz, node);
316 1588 ▶ for (i = 0; i < n && !v->err; i++) {
Branch: 2 branches
BranchTrueFalseTotal
for (i = 0; i < n && !v->err; i++) { 1431 157 1588
for (i = 0; i < n && !v->err; i++) { 1320 111 1431
317 1320 const char *k = NULL;
318 1320 STRLEN kl = 0;
319 uint32_t child;
320 size_t j, want;
321  
322 /*
323 # uncoverable branch true
324 # uncoverable condition left count:1 class:ignore_covered_err */
325 1320 ▶ if (!(PUNK_FZ->key_at)(fz, node, i, &k, &kl, NULL)) continue;
Branch: 1 branch
BranchTrueFalseTotal
if (!(PUNK_FZ->key_at)(fz, node, i, &k, &kl, NULL)) continue; 0 1320 1320
326  
327 5433 ▶ for (j = 0; j < (size_t)kl; j++) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = 0; j < (size_t)kl; j++) { 4114 1319 5433
328 4114 ▶ if (k[j] != '.') continue;
Branch: 1 branch
BranchTrueFalseTotal
if (k[j] != '.') continue; 4113 1 4114
329 1 path[plen] = '\0';
330 1 my_strlcpy(v->path, path, sizeof v->path);
331 {
332 1 size_t w = (size_t)kl < sizeof v->what ? (size_t)kl
333 1 : sizeof v->what - 1;
334 1 memcpy(v->what, k, w);
335 1 v->what[w] = '\0';
336 }
337 1 v->err = PI_V_DOT;
338 1 return;
339 }
340  
341 1319 ▶ want = plen + (plen ? 1 : 0) + (size_t)kl;
Branch: 1 branch
BranchTrueFalseTotal
want = plen + (plen ? 1 : 0) + (size_t)kl; 124 1195 1319
342 /*
343 # uncoverable branch true
344 # uncoverable condition left count:1 class:ignore_covered_err */
345 1319 ▶ if (want >= PI_KEY_MAX) { /* too long to name; refuse rather
Branch: 1 branch
BranchTrueFalseTotal
if (want >= PI_KEY_MAX) { /* too long to name; refuse rather 0 1319 1319
346 * than truncate into a message */
347 0 path[plen] = '\0'; /* # uncoverable statement */
348 0 my_strlcpy(v->path, path, sizeof v->path); /* # uncoverable statement */
349 0 my_strlcpy(v->what, "a key too long to address", sizeof v->what); /* # uncoverable statement */
350 0 v->err = PI_V_TYPE; /* # uncoverable statement */
351 0 return; /* # uncoverable statement */
352 }
353 1319 ▶ if (plen) path[plen] = '.';
Branch: 1 branch
BranchTrueFalseTotal
if (plen) path[plen] = '.'; 124 1195 1319
354 1319 ▶ memcpy(path + plen + (plen ? 1 : 0), k, kl);
Branch: 1 branch
BranchTrueFalseTotal
memcpy(path + plen + (plen ? 1 : 0), k, kl); 124 1195 1319
355  
356 1319 child = (PUNK_FZ->val_at)(fz, node, i);
357 1319 pi_walk_check(v, fz, child, path, want, depth + 1);
358 }
359 }
360  
361 199 static void pi_validate(pTHX_ pi_arena *ar, int idx, const char *where) {
362 199 pi_cat *c = &ar->cat[idx];
363 pi_vcheck v;
364 char path[PI_KEY_MAX + 2];
365  
366 199 memset(&v, 0, sizeof v);
367 /* Only the plural check is conditional. The type and dotted-key checks
368 * run for every catalogue, in every language - an earlier version
369 * returned early here when the language HAD a rule, which silently
370 * skipped both of them for en, fr and everything else with a rule. */
371 199 v.plural = (pi_rule_for(c->tag, c->taglen) == PR_NONE);
372  
373 199 path[0] = '\0';
374 /* The catalogue root is depth 0, so a TOP-LEVEL key's value is depth 1
375 * and a category - which must sit under another key - is depth 2. Off by
376 * one here and every top-level word named `one` or `few` is refused as a
377 * plural category in any language whose rule is unknown. */
378 199 pi_walk_check(&v, c->fz, c->node, path, 0, 0);
379  
380 199 ▶ if (!v.err) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!v.err) return; 83 116 199
381  
382 { /* Copied out BEFORE the arena is freed. `where` is the arena's own
383 * tag and v.path was built from bytes borrowed out of the block;
384 * pi_arena_free closes the container, so a croak reading either
385 * afterwards quotes freed memory - and reads as an empty string
386 * often enough to survive a test suite. */
387 char tag[PI_TAG_MAX + 1], p[PI_KEY_MAX + 1], w[32];
388 116 int err = v.err;
389 116 my_strlcpy(tag, where, sizeof tag);
390 116 my_strlcpy(p, v.path, sizeof p);
391 116 my_strlcpy(w, v.what, sizeof w);
392 116 pi_arena_free(aTHX_ ar);
393  
394 116 ▶ if (err == PI_V_PLURAL)
Branch: 1 branch
BranchTrueFalseTotal
if (err == PI_V_PLURAL) 2 114 116
395 2 croak("Punk::Plugin::I18n: '%s' uses the plural category '%s' in "
396 "'%s', and there is no plural rule for that language. A rule "
397 "is a grammar, not a preference: falling back to one/other "
398 "would be English's rule applied to a language that does not "
399 "have it, and it would read perfectly well while being wrong. "
400 "Add the rule to punk_plural.h, or use plain keys and choose "
401 "between them in the application", tag, w, p);
402 114 ▶ if (err == PI_V_DOT)
Branch: 1 branch
BranchTrueFalseTotal
if (err == PI_V_DOT) 1 113 114
403 1 ▶ croak("Punk::Plugin::I18n: '%s' has a key containing a dot, '%s', "
Branch: 1 branch
BranchTrueFalseTotal
croak("Punk::Plugin::I18n: '%s' has a key containing a dot, '%s', " 0 1 1
404 "under '%s'. A dot separates levels, so this key cannot be "
405 "reached - $c->locale would descend past it and a template "
406 "splits on dots before it ever gets here. Nest it instead",
407 tag, w, *p ? p : "the top level"); /* # uncoverable condition left count:1 class:ignore_covered_err */
408 /*
409 # uncoverable branch true
410 # uncoverable condition left count:1 class:ignore_covered_err */
411 113 ▶ if (err == PI_V_DEEP)
Branch: 1 branch
BranchTrueFalseTotal
if (err == PI_V_DEEP) 0 113 113
412 0 croak("Punk::Plugin::I18n: '%s' nests more than 64 levels deep at " /* # uncoverable statement */
413 "'%s', which is a mistake in the catalogue rather than a "
414 "translation", tag, p); /* # uncoverable statement */
415 113 croak("Punk::Plugin::I18n: '%s' has %s at '%s', and a translation is "
416 "a string. Quote it in the catalogue, or move the value into "
417 "the application - the lookup returns text and there is nothing "
418 "for it to return here", tag, w, p);
419 }
420 }
421  
422 /* Build the arena from { folded-tag => \%catalogue, ... }, decoded in Perl.
423 *
424 * The whole thing becomes ONE Frozen block through freeze_container, which
425 * builds and adopts in one step. Going through Frozen's Perl surface instead
426 * would mean building the block and then COPYING it, because attach copies -
427 * and it would make Punk depend on two contracts rather than one, since the
428 * Perl API carries no append-only guarantee and the ABI table does.
429 *
430 * No `flat` option. Frozen's flat index is only consulted for a path from
431 * the ROOT, and every catalogue lookup here starts one level down at a tag,
432 * so it would occupy bytes and never be read.
433 *
434 * Tags are folded in Perl, at derivation from the filename, so that EN.json
435 * and en.json produce the same block rather than two blocks that behave
436 * differently. */
437 152 static pi_arena *pi_arena_build(pTHX_ SV *cats, const char *def,
438 STRLEN deflen) {
439 pi_arena *ar;
440 152 const fz_abi *FZ = punk_fz(aTHX);
441 fz_container *c;
442 uint32_t root, n, i;
443 152 int err = 0;
444 char bad[PI_TAG_MAX + 1];
445  
446 /* FZ_F_STRINGIFY: a translation is text, and JSON's 5 decodes to an IV
447 * that would freeze as an integer - which `str` answers NULL for, so the
448 * lookup would find nothing and render the key. The flag makes the block
449 * hold strings only, which is also what lets the boot walk below say "a
450 * translation is text" as ONE rule rather than a list of exceptions. */
451 152 c = (FZ->freeze_container)(aTHX_ cats, FZ_F_STRINGIFY, NULL, &err);
452 /*
453 # uncoverable branch true
454 # uncoverable condition left count:1 class:ignore_covered_err */
455 151 ▶ if (!c)
Branch: 1 branch
BranchTrueFalseTotal
if (!c) 0 151 151
456 0 croak("Punk::Plugin::I18n: the catalogues could not be frozen: %s", /* # uncoverable statement */
457 (FZ->error)(err)); /* # uncoverable statement */
458  
459 151 root = (FZ->root)(c);
460 /*
461 # uncoverable branch true
462 # uncoverable condition left count:1 class:ignore_covered_err */
463 151 ▶ if ((FZ->kind)(c, root) != FZ_K_HASH) {
Branch: 1 branch
BranchTrueFalseTotal
if ((FZ->kind)(c, root) != FZ_K_HASH) { 0 151 151
464 0 (FZ->close)(aTHX_ c); /* # uncoverable statement */
465 0 croak("Punk::Plugin::I18n: the catalogues did not freeze to a map of " /* # uncoverable statement */
466 "tag to catalogue");
467 }
468 151 n = (FZ->count)(c, root);
469 /*
470 # uncoverable branch true
471 # uncoverable condition left count:1 class:ignore_covered_err */
472 151 ▶ if (!n) {
Branch: 1 branch
BranchTrueFalseTotal
if (!n) { 0 151 151
473 0 (FZ->close)(aTHX_ c); /* # uncoverable statement */
474 0 croak("Punk::Plugin::I18n: no catalogues - a plugin with nothing to " /* # uncoverable statement */
475 "translate is a typo in `dir`, not a working default");
476 }
477  
478 151 Newxz(ar, 1, pi_arena);
479 151 ar->c = c;
480 151 ar->root = root;
481 151 ar->def = -1;
482 151 Newxz(ar->cat, n, pi_cat);
483  
484 234 ▶ for (i = 0; i < n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < n; i++) { 199 35 234
485 199 const char *k = NULL;
486 199 STRLEN kl = 0;
487 199 int kutf8 = 0;
488 uint32_t node;
489 STRLEN j;
490  
491 /*
492 # uncoverable branch true
493 # uncoverable condition left count:1 class:ignore_covered_err */
494 199 ▶ if (!(FZ->key_at)(c, root, i, &k, &kl, &kutf8)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!(FZ->key_at)(c, root, i, &k, &kl, &kutf8)) { 0 199 199
495 0 pi_arena_free(aTHX_ ar); /* # uncoverable statement */
496 0 croak("Punk::Plugin::I18n: the frozen catalogue map is short"); /* # uncoverable statement */
497 }
498 /* fz_at is array-only, so a hash value by index is val_at - added in
499 * ABI v2 for exactly this. */
500 199 node = (FZ->val_at)(c, root, i);
501  
502 /* Copied before anything can free the block: `k` borrows from it,
503 * and every croak below closes the container first. */
504 {
505 199 STRLEN w = kl > PI_TAG_MAX ? PI_TAG_MAX : kl;
506 199 memcpy(bad, k, w);
507 199 bad[w] = '\0';
508 }
509  
510 /*
511 # uncoverable branch true
512 # uncoverable condition left count:1 class:ignore_covered_err
513 # uncoverable condition left count:2 class:ignore_covered_err */
514 199 ▶ if (kutf8 || kl > PI_TAG_MAX) {
Branch: 2 branches
BranchTrueFalseTotal
if (kutf8 || kl > PI_TAG_MAX) { 199 0 199
if (kutf8 || kl > PI_TAG_MAX) { 0 199 199
515 0 pi_arena_free(aTHX_ ar); /* # uncoverable statement */
516 0 croak("Punk::Plugin::I18n: '%s' is not a usable language tag - " /* # uncoverable statement */
517 "a tag is ASCII and at most %d bytes", bad, PI_TAG_MAX); /* # uncoverable statement */
518 }
519 /*
520 # uncoverable branch true
521 # uncoverable condition left count:1 class:ignore_covered_err */
522 199 ▶ if ((FZ->kind)(c, node) != FZ_K_HASH) {
Branch: 1 branch
BranchTrueFalseTotal
if ((FZ->kind)(c, node) != FZ_K_HASH) { 0 199 199
523 0 pi_arena_free(aTHX_ ar); /* # uncoverable statement */
524 0 croak("Punk::Plugin::I18n: the catalogue for '%s' is not a JSON " /* # uncoverable statement */
525 "object - a catalogue is a map of key to translation", bad); /* # uncoverable statement */
526 }
527 627 ▶ for (j = 0; j < kl; j++) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = 0; j < kl; j++) { 428 199 627
528 /*
529 # uncoverable branch true
530 # uncoverable condition left count:1 class:ignore_covered_err */
531 428 ▶ if (pi_fold(k[j]) != k[j]) {
Branch: 1 branch
BranchTrueFalseTotal
if (pi_fold(k[j]) != k[j]) { 0 428 428
532 0 pi_arena_free(aTHX_ ar); /* # uncoverable statement */
533 0 croak("Punk::Plugin::I18n: the tag '%s' reached the block " /* # uncoverable statement */
534 "unfolded - tags are folded when they are derived from "
535 "the filename", bad); /* # uncoverable statement */
536 }
537 428 ar->cat[i].tag[j] = k[j];
538 }
539 199 ar->cat[i].tag[kl] = '\0';
540 199 ar->cat[i].taglen = kl;
541 199 ar->cat[i].fz = c;
542 199 ar->cat[i].node = node;
543 199 ar->ncat++;
544  
545 199 pi_validate(aTHX_ ar, (int)i, ar->cat[i].tag);
546  
547 83 ▶ if (def && pi_tag_eq(ar->cat[i].tag, ar->cat[i].taglen, def, deflen)) /* # uncoverable condition right count:1 class:ignore_covered_err */
Branch: 2 branches
BranchTrueFalseTotal
if (def && pi_tag_eq(ar->cat[i].tag, ar->cat[i].taglen, def, deflen)) /* # uncoverable condition right count:1 class:ignore_covered_err */ 83 0 83
if (def && pi_tag_eq(ar->cat[i].tag, ar->cat[i].taglen, def, deflen)) /* # uncoverable condition right count:1 class:ignore_covered_err */ 36 47 83
548 36 ar->def = (int)i;
549 }
550  
551 35 ▶ if (def && ar->def < 0) { /* # uncoverable condition right count:1 class:ignore_covered_err */
Branch: 2 branches
BranchTrueFalseTotal
if (def && ar->def < 0) { /* # uncoverable condition right count:1 class:ignore_covered_err */ 35 0 35
if (def && ar->def < 0) { /* # uncoverable condition right count:1 class:ignore_covered_err */ 1 34 35
552 char want[PI_TAG_MAX + 1];
553 1 STRLEN j, w = deflen > PI_TAG_MAX ? PI_TAG_MAX : deflen;
554 3 ▶ for (j = 0; j < w; j++) want[j] = def[j];
Branch: 1 branch
BranchTrueFalseTotal
for (j = 0; j < w; j++) want[j] = def[j]; 2 1 3
555 1 want[w] = '\0';
556 1 pi_arena_free(aTHX_ ar);
557 1 croak("Punk::Plugin::I18n: the default locale '%s' has no catalogue - "
558 "the default is the answer when negotiation finds nothing, so "
559 "it is the one locale that must exist", want);
560 }
561 34 return ar;
562 }
563  
564 /* ---- the view a template reads --------------------------------------------
565 *
566 * `{% locale.welcome %}` is an ordinary dotted path through a hashref, so the
567 * template half needs no change to Stencil - the same seam CSP uses to make
568 * `{% csp_nonce %}` resolve with nothing passed by the handler.
569 *
570 * The hash is TIED, and both halves of that decision were paid for.
571 *
572 * A plain hash would have to be materialised: per request it copies the
573 * catalogue into SVs on every page, which is the cost the arena exists to
574 * avoid, and cached per catalogue it is a second copy of every translation as
575 * Perl data in whichever worker rendered in that locale. A tied hash copies
576 * nothing - FETCH reads the arena and builds one SV for the string actually
577 * asked for.
578 *
579 * It also keeps the missing-key rule true on BOTH paths. A template resolves
580 * a missing path to the empty string, so a plain hash would silently swallow
581 * `{% locale.typo %}` while `$c->locale('typo')` rendered the key - the same
582 * omission visible in a handler and invisible in a template, which is the
583 * worse half to lose.
584 *
585 * This needs Template::Stencil 0.10. Before it, a tied hash in a path
586 * resolved to nothing at all: its resolver read hashes with hv_common and a
587 * precomputed hash, which does not go through tie magic. That was fixed
588 * there rather than worked around here.
589 */
590 /* The tied object carries NODES, not just a path.
591 *
592 * It used to carry only the dotted path so far, so every FETCH rebuilt the
593 * joined key and looked it up from the catalogue root again: resolving
594 * {% locale.a.b.c %} cost 1 + 2 + 3 = six probes and three SV
595 * concatenations. Carrying the node it already resolved makes each FETCH one
596 * probe from where the last one stopped - three for that path, and no
597 * concatenation at all.
598 *
599 * PIT_DNODE is the same position in the DEFAULT catalogue, carried in step.
600 * Without it there is no way to tell "missing everywhere" from "present in
601 * the default but not translated here" once the joined key is gone, and the
602 * untranslated counter would stop being exact.
603 *
604 * PIT_PREFIX survives for one job only: the missing-key message, which
605 * renders the full dotted path a template asked for. */
606 #define PIT_ARENA 0 /* the arena, as an IV */
607 #define PIT_CAT 1 /* which catalogue, as an index */
608 #define PIT_PREFIX 2 /* the dotted path so far, for the miss message */
609 #define PIT_CFG 3 /* the plugin config, for the dev flag */
610 #define PIT_CTX 4 /* the context, for the logger */
611 #define PIT_NODE 5 /* where we are in the negotiated catalogue */
612 #define PIT_DNODE 6 /* and in the default one */
613 #define PIT_ITER 7 /* the each/keys cursor */
614 #define PIT_MAX 8
615  
616 /* ---- interpolation -------------------------------------------------------- */
617  
618 /* ---- what an application can see ------------------------------------------
619 *
620 * Three numbers, because they answer three different questions and rolling
621 * them together would answer none of them:
622 *
623 * missing - the key is in no catalogue at all. A bug.
624 * untranslated - the key is in the default catalogue but not in the
625 * negotiated one. NOT a bug: it is what a partly translated
626 * site is, and it is how an application measures coverage.
627 * warned - how many times the missing-key warning actually fired.
628 * Provable, so "it did not warn in production" is a number
629 * rather than the absence of one.
630 */
631 static UV pi_n_missing = 0;
632 static UV pi_n_untranslated = 0;
633 static UV pi_n_warned = 0;
634  
635 /* Keys already warned about, so the same page does not warn on every request.
636 * Twenty renders of one page must produce one line: noise is ignored, and
637 * being ignored is how the omission survives. */
638 static HV *pi_seen = NULL;
639  
640 /* `{name}` from a list of name => value pairs.
641 *
642 * Three rules, each of which exists because the alternative is worse:
643 *
644 * - an UNKNOWN placeholder stays literal. `{nmae}` renders as `{nmae}`,
645 * visibly wrong, rather than as a gap nobody notices;
646 * - a MISSING substitution is not an error - a translator adding a
647 * placeholder the caller does not pass must not take the page down;
648 * - a value containing `{other}` is NOT re-scanned. One pass, left to
649 * right, or a user whose name is `{admin}` reaches into the catalogue.
650 */
651 /* `utf8` is the flag the BLOCK recorded for the catalogue string, because
652 * `v` is a pointer and a length and carries none of its own.
653 *
654 * It has to be known here rather than applied by the caller afterwards. The
655 * catalogue string and the substituted values are two different encodings
656 * meeting in one buffer: appending a downgraded Latin-1 argument into a
657 * string of UTF-8 octets and then labelling the result UTF-8 puts a raw
658 * 0xE9 inside a string that claims to be UTF-8. That is corruption, it is
659 * reachable from a form field today through
660 * $c->locale($key, name => $value), and Frozen does not fix it - the block
661 * hands back exactly what it was given. It is ordinary broken code that has
662 * been sitting beside the interesting problem.
663 *
664 * So `out` is flagged UP FRONT and the values go in with sv_catsv, which is
665 * the primitive that knows both sides' flags and upgrades whichever needs
666 * it. sv_catpvn cannot: it sees bytes and a length, which is precisely what
667 * loses the information. The catalogue runs still use sv_catpvn, and that
668 * is correct - those octets already ARE out's encoding. */
669 /* A value made safe to put into markup: & < > " ' as entities, and the
670 * value's own encoding flag kept, since only ASCII bytes are rewritten.
671 * Mortal. */
672 2 static SV *pi_html_value(pTHX_ SV *val) {
673 STRLEN l;
674 2 const char *p = SvPV_const(val, l);
675 2 SV *out = sv_2mortal(newSVpvn("", 0));
676 2 ps_html_esc(aTHX_ out, p, l);
677 /*
678 # uncoverable branch true
679 # uncoverable condition left count:1 class:ignore_covered_err */
680 2 ▶ if (SvUTF8(val)) SvUTF8_on(out);
Branch: 1 branch
BranchTrueFalseTotal
if (SvUTF8(val)) SvUTF8_on(out); 0 2 2
681 2 return out;
682 }
683  
684 /* `html` escapes each substituted VALUE and leaves the catalogue's own bytes
685 * alone: a translation is authored and may carry a link or an emphasis, and
686 * a value is whatever the caller was handed. That is $c->locale_html. Plain
687 * $c->locale escapes nothing - its result is text, bound for a template that
688 * escapes, a JSON body, a log line - and escaping there would be once too
689 * often for every one of them. */
690 92 static SV *pi_interpolate(pTHX_ const char *v, STRLEN vl, int utf8,
691 SV **args, int nargs, int html) {
692 92 SV *out = newSVpvn("", 0);
693 92 STRLEN i = 0;
694  
695 92 ▶ if (utf8) SvUTF8_on(out);
Branch: 1 branch
BranchTrueFalseTotal
if (utf8) SvUTF8_on(out); 3 89 92
696  
697 179 ▶ while (i < vl) {
Branch: 1 branch
BranchTrueFalseTotal
while (i < vl) { 131 48 179
698 131 const char *open = (const char *)memchr(v + i, '{', vl - i);
699 STRLEN o, close;
700 131 int j, done = 0;
701  
702 131 ▶ if (!open) { sv_catpvn(out, v + i, vl - i); break; }
Branch: 1 branch
BranchTrueFalseTotal
if (!open) { sv_catpvn(out, v + i, vl - i); break; } 44 87 131
703 87 o = (STRLEN)(open - v);
704 87 sv_catpvn(out, v + i, o - i);
705  
706 /* find the close on the same run of name characters */
707 87 close = o + 1;
708 496 ▶ while (close < vl && v[close] != '}' && v[close] != '{') close++; /* # uncoverable condition right count:3 class:ignore_covered_err */
Branch: 3 branches
BranchTrueFalseTotal
while (close < vl && v[close] != '}' && v[close] != '{') close++; /* # uncoverable condition right count:3 class:ignore_covered_err */ 495 1 496
while (close < vl && v[close] != '}' && v[close] != '{') close++; /* # uncoverable condition right count:3 class:ignore_covered_err */ 409 86 495
while (close < vl && v[close] != '}' && v[close] != '{') close++; /* # uncoverable condition right count:3 class:ignore_covered_err */ 409 0 409
709 87 ▶ if (close >= vl || v[close] != '}') { /* # uncoverable condition left count:2 class:ignore_covered_err */
Branch: 2 branches
BranchTrueFalseTotal
if (close >= vl || v[close] != '}') { /* # uncoverable condition left count:2 class:ignore_covered_err */ 86 1 87
if (close >= vl || v[close] != '}') { /* # uncoverable condition left count:2 class:ignore_covered_err */ 0 86 86
710 1 sv_catpvn(out, v + o, 1); /* a lone brace is a lone brace */
711 1 i = o + 1;
712 1 continue;
713 }
714  
715 88 ▶ for (j = 0; j + 1 < nargs; j += 2) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = 0; j + 1 < nargs; j += 2) { 86 2 88
716 STRLEN nl;
717 86 const char *n = SvPV_const(args[j], nl);
718 86 ▶ if (nl == close - o - 1 && memcmp(n, v + o + 1, nl) == 0) { /* # uncoverable condition right count:1 class:ignore_covered_err */
Branch: 2 branches
BranchTrueFalseTotal
if (nl == close - o - 1 && memcmp(n, v + o + 1, nl) == 0) { /* # uncoverable condition right count:1 class:ignore_covered_err */ 86 0 86
if (nl == close - o - 1 && memcmp(n, v + o + 1, nl) == 0) { /* # uncoverable condition right count:1 class:ignore_covered_err */ 84 2 86
719 /* sv_catsv, not sv_catpvn(SvPV_const(...)): the argument
720 * carries its own flag and this is the only append that has
721 * to reconcile two encodings. */
722 84 ▶ if (SvOK(args[j + 1]))
Branch: 1 branch
BranchTrueFalseTotal
if (SvOK(args[j + 1])) 83 1 84
723 83 ▶ sv_catsv(out, html ? pi_html_value(aTHX_ args[j + 1])
Branch: 1 branch
BranchTrueFalseTotal
sv_catsv(out, html ? pi_html_value(aTHX_ args[j + 1]) 2 81 83
724 : args[j + 1]);
725 84 done = 1;
726 84 break;
727 }
728 }
729 86 ▶ if (!done) sv_catpvn(out, v + o, close - o + 1); /* left literal */
Branch: 1 branch
BranchTrueFalseTotal
if (!done) sv_catpvn(out, v + o, close - o + 1); /* left literal */ 2 84 86
730 86 i = close + 1;
731 }
732 92 return out;
733 }
734  
735 #endif /* PUNK_I18N_H */
« include/punk/punk_response.h ↑ Summary include/punk/punk_static.h »