include/punk/punk_i18n.h

filter: Statementstmt 85.1% 161 / 189 Branchbran 77.5% 107 / 138 Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 81.9% 268 / 327 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« xs/reqid.xs ↑ Summary include/punk/punk_log.h »
1 #ifndef PUNK_I18N_H
2 #define PUNK_I18N_H
3  
4 /* Punk::Plugin::I18n - translations and language negotiation.
5 *
6 * ---- the catalogue --------------------------------------------------------
7 *
8 * One JSON file per locale, named for the tag it holds:
9 *
10 * i18n/en.json i18n/en-GB.json i18n/fr.json
11 * { "greeting": "Hello, {name}" }
12 *
13 * Placeholders are `{name}` and not `%s`, because positional formats cannot
14 * be reordered and reordering is the entire reason a sentence needs
15 * translating: German puts the verb where English does not.
16 *
17 * ---- why this is not a hash of SVs ----------------------------------------
18 *
19 * Catalogues are read-only at request time, so reading them before the fork
20 * shares them across every worker. That is the argument, and it is wrong for
21 * Perl data: the first READ of an SV touches its refcount, the page is
22 * copied, and after a few minutes of traffic every worker holds a private
23 * copy of the catalogue. The sharing lasts about as long as it takes to serve
24 * the first request.
25 *
26 * So a catalogue is a block of bytes addressed by offset, built once at boot
27 * and never written again. A lookup compares bytes and returns a pointer
28 * INTO the block; nothing is allocated and no refcount is touched, so the
29 * pages stay shared for the life of the pool.
30 *
31 * That is also what lets the lookup serve both callers: `$c->locale` in a
32 * handler, and the per-request `locale` hash the templates read.
33 *
34 * ---- the block is Frozen's ------------------------------------------------
35 *
36 * It used to be hand-rolled here: a char buffer, an array of pi_entry, and a
37 * bucket array, three allocations per catalogue and two more for the arena.
38 * It is now ONE Frozen container holding { folded-tag => catalogue }, read
39 * through fz_abi.h - see punk_frozen.h for why that is the C table and not
40 * Frozen's Perl surface.
41 *
42 * The thing that changed is not the storage, it is that the block holds a
43 * TREE. The old form flattened { "a": { "b": x } } to the string "a.b" and
44 * faked the levels with a sentinel, so a literal key "a.b" was the same key -
45 * and which of the two answered was decided by the hash order of the process
46 * that loaded it. Two workers could disagree. A level and a leaf are now
47 * different nodes, so they are different keys, in every process.
48 *
49 * It costs one probe per level where the flat key cost one probe whatever
50 * its depth. That is the trade and it is not recoverable without giving the
51 * distinction back.
52 *
53 * ---- and not a cache ------------------------------------------------------
54 *
55 * Deliberately no `cache => 1` option. There is no per-request parse to cache
56 * away - the catalogue is resident from boot - and a cache lookup per key
57 * would be two orders of magnitude slower on the path that matters: a
58 * Punk::Cache::File hit is about 7.2us, of which 6.2us is the open syscall,
59 * so a page with forty translated strings would pay ~288us against a hash
60 * probe now.
61 */
62  
63 /* ---- why the files are read in Perl --------------------------------------
64 *
65 * Everything below builds the arena from an ALREADY DECODED hash. Finding the
66 * catalogues and parsing them happens in Punk::Plugin::I18n::register, in
67 * Perl, at boot, and that is deliberate twice over:
68 *
69 * - `opendir`/`readdir` in XS is the Win32 trap. There is no ambient DIR on
70 * native Windows and reaching for one costs a FindFirstFile shim; Punk
71 * builds there now, and a directory walk is not worth a second
72 * implementation of one.
73 * - a decode that croaks needs the FILENAME in the message, and catching a
74 * croak in C means JMPENV by hand, in a header that has to compile on
75 * every perl from 5.10. Two lines of eval in Perl say the same thing.
76 *
77 * It costs nothing at request time: this runs once, before the fork.
78 */
79  
80 #define PI_TAG_MAX 35 /* RFC 5646 says 35 is enough for any tag */
81 #define PI_KEY_MAX 512
82  
83 /* One catalogue: a language tag, and the node its translations hang off in
84 * the block. The whole arena is ONE Frozen container holding
85 * { folded-tag => catalogue }, so a catalogue is a handle rather than a
86 * structure, and a tag lookup is a probe on a hash node.
87 *
88 * The tag is a char array rather than a pointer into the block because
89 * _negotiate builds a pi_arena on the STACK, with tags and no container at
90 * all, to test the Accept-Language rules without a catalogue. A borrowed
91 * pointer would break that seam.
92 *
93 * `fz` is repeated per catalogue rather than reached through the arena so
94 * that pi_get and pi_is_prefix keep the signatures punk_lang.h calls them
95 * with. Eight bytes times the number of locales. */
96 typedef struct pi_cat {
97 char tag[PI_TAG_MAX + 1]; /* case-folded, NUL terminated */
98 STRLEN taglen;
99 fz_container *fz; /* NULL in the _negotiate seam */
100 uint32_t node; /* this catalogue's hash node */
101 } pi_cat;
102  
103 typedef struct pi_arena {
104 fz_container *c; /* owned; lives as long as the process */
105 uint32_t root; /* the hash of folded tag -> catalogue */
106 pi_cat *cat;
107 int ncat;
108 int def; /* index of the default locale, or -1 */
109 } pi_arena;
110  
111 /* ASCII fold. Language tags are case insensitive by RFC 5646 - `EN-gb` is
112 * `en-GB` - and folding once at load means the match rule never has to. */
113 2247 static char pi_fold(char c) {
114 2247 ▶ return (c >= 'A' && c <= 'Z') ? (char)(c - 'A' + 'a') : c;
Branch: 2 branches
BranchTrueFalseTotal
return (c >= 'A' && c <= 'Z') ? (char)(c - 'A' + 'a') : c; 2210 37 2247
return (c >= 'A' && c <= 'Z') ? (char)(c - 'A' + 'a') : c; 30 2180 2210
115 }
116  
117 774 static int pi_tag_eq(const char *a, STRLEN al, const char *b, STRLEN bl) {
118 STRLEN i;
119 774 ▶ if (al != bl) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (al != bl) return 0; 142 632 774
120 1003 ▶ for (i = 0; i < al; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < al; i++) 840 163 1003
121 840 ▶ if (pi_fold(a[i]) != pi_fold(b[i])) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (pi_fold(a[i]) != pi_fold(b[i])) return 0; 469 371 840
122 163 return 1;
123 }
124  
125 /* A dotted key is a WALK now, not a joined string.
126 *
127 * `items.one` descends from the catalogue node through `items` to `one`,
128 * which costs one probe per segment where the flat key cost one probe
129 * whatever its depth. What it buys is that a level and a leaf are different
130 * nodes rather than the same entry wearing a sentinel, so { "a.b": x } and
131 * { "a": { "b": y } } stop being the same key. They were not merely
132 * ambiguous before: which of the two won was decided by Perl's per-process
133 * hash order, so two workers in one pool could answer differently.
134 *
135 * PUNK_FZ is read directly rather than passed in. It is a file-scope static
136 * in this translation unit, resolved once at boot by punk_fz(), and reading
137 * it here is what lets these two keep the signatures punk_lang.h calls them
138 * with - so the negotiation code did not have to change at all.
139 *
140 * ---- the rule Frozen does not enforce --------------------------------------
141 *
142 * A node is a uint32_t, and Frozen validates its SHAPE on every entry: the
143 * offset is in bounds, the alignment is right, the tag is a live kind. What
144 * it cannot validate is PROVENANCE. A node that is valid in one container
145 * names a different place in another, so reading it through the wrong
146 * container returns wrong data rather than failing.
147 *
148 * So a node never leaves the pi_cat it came from. That is why `fz` sits
149 * beside `node` in the struct instead of being reached through the arena:
150 * the pair travels together, and there is no call here that takes a node
151 * without the container it belongs to. */
152 50 static int pi_is_prefix(const pi_cat *c, const char *k, STRLEN kl) {
153 uint32_t node;
154 50 ▶ if (!c || !c->fz || !PUNK_FZ) return 0;
Branch: 3 branches
BranchTrueFalseTotal
if (!c || !c->fz || !PUNK_FZ) return 0; 50 0 50
if (!c || !c->fz || !PUNK_FZ) return 0; 50 0 50
if (!c || !c->fz || !PUNK_FZ) return 0; 0 50 50
155 50 node = (PUNK_FZ->path)(c->fz, c->node, k, kl, '.');
156 50 ▶ if (node == FZ_NOHANDLE) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (node == FZ_NOHANDLE) return 0; 48 2 50
157 2 return (PUNK_FZ->kind)(c->fz, node) == FZ_K_HASH;
158 }
159  
160 /* The value for a key, or NULL. Returns bytes INTO the block - the caller
161 * must not free them and must not hold them past the arena's life, which is
162 * the life of the process.
163 *
164 * `utf8` is an out-param because the block records the flag per string. The
165 * old arena did not, and _locale turned the flag on unconditionally, which
166 * was correct only because File::Raw::JSON happens to decode to UTF-8. Any
167 * other producer got mojibake. NULL is accepted for callers that do not
168 * care - the plural path and EXISTS. */
169 181 static const char *pi_get(const pi_cat *c, const char *k, STRLEN kl,
170 STRLEN *vlen, int *utf8) {
171 uint32_t node;
172 181 ▶ if (!c || !c->fz || !PUNK_FZ) return NULL;
Branch: 3 branches
BranchTrueFalseTotal
if (!c || !c->fz || !PUNK_FZ) return NULL; 181 0 181
if (!c || !c->fz || !PUNK_FZ) return NULL; 181 0 181
if (!c || !c->fz || !PUNK_FZ) return NULL; 0 181 181
173 181 node = (PUNK_FZ->path)(c->fz, c->node, k, kl, '.');
174 181 ▶ if (node == FZ_NOHANDLE) return NULL;
Branch: 1 branch
BranchTrueFalseTotal
if (node == FZ_NOHANDLE) return NULL; 54 127 181
175 /* No `kind` call before this. str answers NULL for anything that is not
176 * a string - it checks the tag and the bounds itself - so asking twice
177 * is a second call through the table for an answer already being
178 * computed. Two calls per lookup, not three, on the path that runs per
179 * translated string per request. */
180 127 return (PUNK_FZ->str)(c->fz, node, vlen, utf8);
181 }
182  
183 /* ---- loading -------------------------------------------------------------- */
184  
185 /* Free the arena. The block is one container, so this is one close plus the
186 * tag table. The container is closed through the table because it was opened
187 * through it, and (FZ->close) is PARENTHESISED: XSUB.h redefines `close` as a
188 * macro under PERL_IMPLICIT_SYS, which is the build failure Frozen 0.02 was
189 * released to fix in its own selftest. */
190 117 static void pi_arena_free(pTHX_ pi_arena *ar) {
191 117 ▶ if (!ar) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!ar) return; 0 117 117
192 117 ▶ if (ar->c && PUNK_FZ) (PUNK_FZ->close)(aTHX_ ar->c);
Branch: 2 branches
BranchTrueFalseTotal
if (ar->c && PUNK_FZ) (PUNK_FZ->close)(aTHX_ ar->c); 117 0 117
if (ar->c && PUNK_FZ) (PUNK_FZ->close)(aTHX_ ar->c); 117 0 117
193 117 ar->c = NULL;
194 117 Safefree(ar->cat);
195 117 Safefree(ar);
196 }
197  
198 /* What the boot walk found wrong, recorded rather than croaked.
199 *
200 * Nothing here croaks while the walk is running. Frozen's own fz_walk
201 * mallocs a segment stack and frees it after the walk returns, so a croak
202 * from inside longjmps past that free and leaks on every boot - a slow leak
203 * with no visible cause in a `punk dev` rebuild loop. The recursion below
204 * allocates nothing, but the same discipline is kept: the first fault is
205 * recorded and raised by the caller, so the rule holds if this ever goes
206 * back through the table. */
207 typedef struct pi_vcheck {
208 int err; /* 0 fine, else one of PI_V_* */
209 int plural; /* check plural categories? */
210 char path[PI_KEY_MAX + 1];
211 char what[32]; /* the offending kind, or category */
212 } pi_vcheck;
213  
214 #define PI_V_TYPE 1
215 #define PI_V_DOT 2
216 #define PI_V_PLURAL 3
217 #define PI_V_DEEP 4
218  
219 113 static const char *pi_kind_name(int k) {
220 113 switch (k) {
221 112 case FZ_K_UNDEF: return "null";
222 0 case FZ_K_FALSE: case FZ_K_TRUE: return "a boolean";
223 0 case FZ_K_INT: case FZ_K_UINT: return "a number";
224 0 case FZ_K_NUM: return "a number";
225 1 case FZ_K_ARRAY: return "an array";
226 0 case FZ_K_HASH: return "an object";
227 }
228 0 return "not a string";
229 }
230  
231 /* Walk a catalogue at boot, checking three things in one pass.
232 *
233 * fz_walk is not used, and the reason is not preference: it reports LEAVES
234 * only, so an array reaches the callback as its elements and the array node
235 * itself is never seen. An array is exactly one of the things that has to be
236 * refused, because nothing in the i18n surface can reach it - fz_path
237 * descends through hash nodes, so `list.0` stops at the array and answers
238 * absent. Left alone it is a silent miss that renders as the key.
239 *
240 * The path is built as it descends rather than kept as a segment stack, so
241 * the message has the joined path ready and the recursion allocates nothing.
242 */
243 1488 static void pi_walk_check(pi_vcheck *v, fz_container *fz, uint32_t node,
244 char *path, size_t plen, int depth) {
245 int kind;
246 uint32_t n, i;
247  
248 1488 ▶ if (v->err) return;
Branch: 1 branch
BranchTrueFalseTotal
if (v->err) return; 0 1488 1488
249 1488 ▶ if (depth > 64) { /* Frozen allows 256; a catalogue
Branch: 1 branch
BranchTrueFalseTotal
if (depth > 64) { /* Frozen allows 256; a catalogue 0 1488 1488
250 * that deep is a bug, not a
251 * translation */
252 0 v->err = PI_V_DEEP;
253 0 path[plen] = '\0';
254 0 my_strlcpy(v->path, path, sizeof v->path);
255 0 return;
256 }
257  
258 1488 kind = (PUNK_FZ->kind)(fz, node);
259  
260 1488 ▶ if (kind == FZ_K_STR) {
Branch: 1 branch
BranchTrueFalseTotal
if (kind == FZ_K_STR) { 1116 372 1488
261 /* the last segment, for the plural rule */
262 1116 ▶ if (v->plural && depth >= 2) {
Branch: 2 branches
BranchTrueFalseTotal
if (v->plural && depth >= 2) { 6 1110 1116
if (v->plural && depth >= 2) { 2 4 6
263 2 size_t s = plen;
264 8 ▶ while (s > 0 && path[s - 1] != '.') s--;
Branch: 2 branches
BranchTrueFalseTotal
while (s > 0 && path[s - 1] != '.') s--; 8 0 8
while (s > 0 && path[s - 1] != '.') s--; 6 2 8
265 2 ▶ if (pi_cat_name(path + s, (STRLEN)(plen - s))) {
Branch: 1 branch
BranchTrueFalseTotal
if (pi_cat_name(path + s, (STRLEN)(plen - s))) { 2 0 2
266 2 path[plen] = '\0';
267 2 my_strlcpy(v->path, path, sizeof v->path);
268 2 my_strlcpy(v->what, path + s, sizeof v->what);
269 2 v->err = PI_V_PLURAL;
270 }
271 }
272 1116 return;
273 }
274  
275 372 ▶ if (kind != FZ_K_HASH) {
Branch: 1 branch
BranchTrueFalseTotal
if (kind != FZ_K_HASH) { 113 259 372
276 113 path[plen] = '\0';
277 113 my_strlcpy(v->path, path, sizeof v->path);
278 113 my_strlcpy(v->what, pi_kind_name(kind), sizeof v->what);
279 113 v->err = PI_V_TYPE;
280 113 return;
281 }
282  
283 259 n = (PUNK_FZ->count)(fz, node);
284 1553 ▶ for (i = 0; i < n && !v->err; i++) {
Branch: 2 branches
BranchTrueFalseTotal
for (i = 0; i < n && !v->err; i++) { 1406 147 1553
for (i = 0; i < n && !v->err; i++) { 1295 111 1406
285 1295 const char *k = NULL;
286 1295 STRLEN kl = 0;
287 uint32_t child;
288 size_t j, want;
289  
290 1295 ▶ if (!(PUNK_FZ->key_at)(fz, node, i, &k, &kl, NULL)) continue;
Branch: 1 branch
BranchTrueFalseTotal
if (!(PUNK_FZ->key_at)(fz, node, i, &k, &kl, NULL)) continue; 0 1295 1295
291  
292 5308 ▶ for (j = 0; j < (size_t)kl; j++) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = 0; j < (size_t)kl; j++) { 4014 1294 5308
293 4014 ▶ if (k[j] != '.') continue;
Branch: 1 branch
BranchTrueFalseTotal
if (k[j] != '.') continue; 4013 1 4014
294 1 path[plen] = '\0';
295 1 my_strlcpy(v->path, path, sizeof v->path);
296 {
297 1 size_t w = (size_t)kl < sizeof v->what ? (size_t)kl
298 1 : sizeof v->what - 1;
299 1 memcpy(v->what, k, w);
300 1 v->what[w] = '\0';
301 }
302 1 v->err = PI_V_DOT;
303 1 return;
304 }
305  
306 1294 ▶ want = plen + (plen ? 1 : 0) + (size_t)kl;
Branch: 1 branch
BranchTrueFalseTotal
want = plen + (plen ? 1 : 0) + (size_t)kl; 104 1190 1294
307 1294 ▶ if (want >= PI_KEY_MAX) { /* too long to name; refuse rather
Branch: 1 branch
BranchTrueFalseTotal
if (want >= PI_KEY_MAX) { /* too long to name; refuse rather 0 1294 1294
308 * than truncate into a message */
309 0 path[plen] = '\0';
310 0 my_strlcpy(v->path, path, sizeof v->path);
311 0 my_strlcpy(v->what, "a key too long to address", sizeof v->what);
312 0 v->err = PI_V_TYPE;
313 0 return;
314 }
315 1294 ▶ if (plen) path[plen] = '.';
Branch: 1 branch
BranchTrueFalseTotal
if (plen) path[plen] = '.'; 104 1190 1294
316 1294 ▶ memcpy(path + plen + (plen ? 1 : 0), k, kl);
Branch: 1 branch
BranchTrueFalseTotal
memcpy(path + plen + (plen ? 1 : 0), k, kl); 104 1190 1294
317  
318 1294 child = (PUNK_FZ->val_at)(fz, node, i);
319 1294 pi_walk_check(v, fz, child, path, want, depth + 1);
320 }
321 }
322  
323 194 static void pi_validate(pTHX_ pi_arena *ar, int idx, const char *where) {
324 194 pi_cat *c = &ar->cat[idx];
325 pi_vcheck v;
326 char path[PI_KEY_MAX + 2];
327  
328 194 memset(&v, 0, sizeof v);
329 /* Only the plural check is conditional. The type and dotted-key checks
330 * run for every catalogue, in every language - an earlier version
331 * returned early here when the language HAD a rule, which silently
332 * skipped both of them for en, fr and everything else with a rule. */
333 194 v.plural = (pi_rule_for(c->tag, c->taglen) == PR_NONE);
334  
335 194 path[0] = '\0';
336 /* The catalogue root is depth 0, so a TOP-LEVEL key's value is depth 1
337 * and a category - which must sit under another key - is depth 2. Off by
338 * one here and every top-level word named `one` or `few` is refused as a
339 * plural category in any language whose rule is unknown. */
340 194 pi_walk_check(&v, c->fz, c->node, path, 0, 0);
341  
342 194 ▶ if (!v.err) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!v.err) return; 78 116 194
343  
344 { /* Copied out BEFORE the arena is freed. `where` is the arena's own
345 * tag and v.path was built from bytes borrowed out of the block;
346 * pi_arena_free closes the container, so a croak reading either
347 * afterwards quotes freed memory - and reads as an empty string
348 * often enough to survive a test suite. */
349 char tag[PI_TAG_MAX + 1], p[PI_KEY_MAX + 1], w[32];
350 116 int err = v.err;
351 116 my_strlcpy(tag, where, sizeof tag);
352 116 my_strlcpy(p, v.path, sizeof p);
353 116 my_strlcpy(w, v.what, sizeof w);
354 116 pi_arena_free(aTHX_ ar);
355  
356 116 ▶ if (err == PI_V_PLURAL)
Branch: 1 branch
BranchTrueFalseTotal
if (err == PI_V_PLURAL) 2 114 116
357 2 croak("Punk::Plugin::I18n: '%s' uses the plural category '%s' in "
358 "'%s', and there is no plural rule for that language. A rule "
359 "is a grammar, not a preference: falling back to one/other "
360 "would be English's rule applied to a language that does not "
361 "have it, and it would read perfectly well while being wrong. "
362 "Add the rule to punk_plural.h, or use plain keys and choose "
363 "between them in the application", tag, w, p);
364 114 ▶ if (err == PI_V_DOT)
Branch: 1 branch
BranchTrueFalseTotal
if (err == PI_V_DOT) 1 113 114
365 1 ▶ croak("Punk::Plugin::I18n: '%s' has a key containing a dot, '%s', "
Branch: 1 branch
BranchTrueFalseTotal
croak("Punk::Plugin::I18n: '%s' has a key containing a dot, '%s', " 0 1 1
366 "under '%s'. A dot separates levels, so this key cannot be "
367 "reached - $c->locale would descend past it and a template "
368 "splits on dots before it ever gets here. Nest it instead",
369 tag, w, *p ? p : "the top level");
370 113 ▶ if (err == PI_V_DEEP)
Branch: 1 branch
BranchTrueFalseTotal
if (err == PI_V_DEEP) 0 113 113
371 0 croak("Punk::Plugin::I18n: '%s' nests more than 64 levels deep at "
372 "'%s', which is a mistake in the catalogue rather than a "
373 "translation", tag, p);
374 113 croak("Punk::Plugin::I18n: '%s' has %s at '%s', and a translation is "
375 "a string. Quote it in the catalogue, or move the value into "
376 "the application - the lookup returns text and there is nothing "
377 "for it to return here", tag, w, p);
378 }
379 }
380  
381 /* Build the arena from { folded-tag => \%catalogue, ... }, decoded in Perl.
382 *
383 * The whole thing becomes ONE Frozen block through freeze_container, which
384 * builds and adopts in one step. Going through Frozen's Perl surface instead
385 * would mean building the block and then COPYING it, because attach copies -
386 * and it would make Punk depend on two contracts rather than one, since the
387 * Perl API carries no append-only guarantee and the ABI table does.
388 *
389 * No `flat` option. Frozen's flat index is only consulted for a path from
390 * the ROOT, and every catalogue lookup here starts one level down at a tag,
391 * so it would occupy bytes and never be read.
392 *
393 * Tags are folded in Perl, at derivation from the filename, so that EN.json
394 * and en.json produce the same block rather than two blocks that behave
395 * differently. */
396 151 static pi_arena *pi_arena_build(pTHX_ SV *cats, const char *def,
397 STRLEN deflen) {
398 pi_arena *ar;
399 151 const fz_abi *FZ = punk_fz(aTHX);
400 fz_container *c;
401 uint32_t root, n, i;
402 151 int err = 0;
403 char bad[PI_TAG_MAX + 1];
404  
405 /* FZ_F_STRINGIFY: a translation is text, and JSON's 5 decodes to an IV
406 * that would freeze as an integer - which `str` answers NULL for, so the
407 * lookup would find nothing and render the key. The flag makes the block
408 * hold strings only, which is also what lets the boot walk below say "a
409 * translation is text" as ONE rule rather than a list of exceptions. */
410 151 c = (FZ->freeze_container)(aTHX_ cats, FZ_F_STRINGIFY, NULL, &err);
411 150 ▶ if (!c)
Branch: 1 branch
BranchTrueFalseTotal
if (!c) 0 150 150
412 0 croak("Punk::Plugin::I18n: the catalogues could not be frozen: %s",
413 (FZ->error)(err));
414  
415 150 root = (FZ->root)(c);
416 150 ▶ if ((FZ->kind)(c, root) != FZ_K_HASH) {
Branch: 1 branch
BranchTrueFalseTotal
if ((FZ->kind)(c, root) != FZ_K_HASH) { 0 150 150
417 0 (FZ->close)(aTHX_ c);
418 0 croak("Punk::Plugin::I18n: the catalogues did not freeze to a map of "
419 "tag to catalogue");
420 }
421 150 n = (FZ->count)(c, root);
422 150 ▶ if (!n) {
Branch: 1 branch
BranchTrueFalseTotal
if (!n) { 0 150 150
423 0 (FZ->close)(aTHX_ c);
424 0 croak("Punk::Plugin::I18n: no catalogues - a plugin with nothing to "
425 "translate is a typo in `dir`, not a working default");
426 }
427  
428 150 Newxz(ar, 1, pi_arena);
429 150 ar->c = c;
430 150 ar->root = root;
431 150 ar->def = -1;
432 150 Newxz(ar->cat, n, pi_cat);
433  
434 228 ▶ for (i = 0; i < n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < n; i++) { 194 34 228
435 194 const char *k = NULL;
436 194 STRLEN kl = 0;
437 194 int kutf8 = 0;
438 uint32_t node;
439 STRLEN j;
440  
441 194 ▶ if (!(FZ->key_at)(c, root, i, &k, &kl, &kutf8)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!(FZ->key_at)(c, root, i, &k, &kl, &kutf8)) { 0 194 194
442 0 pi_arena_free(aTHX_ ar);
443 0 croak("Punk::Plugin::I18n: the frozen catalogue map is short");
444 }
445 /* fz_at is array-only, so a hash value by index is val_at - added in
446 * ABI v2 for exactly this. */
447 194 node = (FZ->val_at)(c, root, i);
448  
449 /* Copied before anything can free the block: `k` borrows from it,
450 * and every croak below closes the container first. */
451 {
452 194 STRLEN w = kl > PI_TAG_MAX ? PI_TAG_MAX : kl;
453 194 memcpy(bad, k, w);
454 194 bad[w] = '\0';
455 }
456  
457 194 ▶ if (kutf8 || kl > PI_TAG_MAX) {
Branch: 2 branches
BranchTrueFalseTotal
if (kutf8 || kl > PI_TAG_MAX) { 194 0 194
if (kutf8 || kl > PI_TAG_MAX) { 0 194 194
458 0 pi_arena_free(aTHX_ ar);
459 0 croak("Punk::Plugin::I18n: '%s' is not a usable language tag - "
460 "a tag is ASCII and at most %d bytes", bad, PI_TAG_MAX);
461 }
462 194 ▶ if ((FZ->kind)(c, node) != FZ_K_HASH) {
Branch: 1 branch
BranchTrueFalseTotal
if ((FZ->kind)(c, node) != FZ_K_HASH) { 0 194 194
463 0 pi_arena_free(aTHX_ ar);
464 0 croak("Punk::Plugin::I18n: the catalogue for '%s' is not a JSON "
465 "object - a catalogue is a map of key to translation", bad);
466 }
467 612 ▶ for (j = 0; j < kl; j++) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = 0; j < kl; j++) { 418 194 612
468 418 ▶ if (pi_fold(k[j]) != k[j]) {
Branch: 1 branch
BranchTrueFalseTotal
if (pi_fold(k[j]) != k[j]) { 0 418 418
469 0 pi_arena_free(aTHX_ ar);
470 0 croak("Punk::Plugin::I18n: the tag '%s' reached the block "
471 "unfolded - tags are folded when they are derived from "
472 "the filename", bad);
473 }
474 418 ar->cat[i].tag[j] = k[j];
475 }
476 194 ar->cat[i].tag[kl] = '\0';
477 194 ar->cat[i].taglen = kl;
478 194 ar->cat[i].fz = c;
479 194 ar->cat[i].node = node;
480 194 ar->ncat++;
481  
482 194 pi_validate(aTHX_ ar, (int)i, ar->cat[i].tag);
483  
484 78 ▶ if (def && pi_tag_eq(ar->cat[i].tag, ar->cat[i].taglen, def, deflen))
Branch: 2 branches
BranchTrueFalseTotal
if (def && pi_tag_eq(ar->cat[i].tag, ar->cat[i].taglen, def, deflen)) 78 0 78
if (def && pi_tag_eq(ar->cat[i].tag, ar->cat[i].taglen, def, deflen)) 35 43 78
485 35 ar->def = (int)i;
486 }
487  
488 34 ▶ if (def && ar->def < 0) {
Branch: 2 branches
BranchTrueFalseTotal
if (def && ar->def < 0) { 34 0 34
if (def && ar->def < 0) { 1 33 34
489 char want[PI_TAG_MAX + 1];
490 1 STRLEN j, w = deflen > PI_TAG_MAX ? PI_TAG_MAX : deflen;
491 3 ▶ for (j = 0; j < w; j++) want[j] = def[j];
Branch: 1 branch
BranchTrueFalseTotal
for (j = 0; j < w; j++) want[j] = def[j]; 2 1 3
492 1 want[w] = '\0';
493 1 pi_arena_free(aTHX_ ar);
494 1 croak("Punk::Plugin::I18n: the default locale '%s' has no catalogue - "
495 "the default is the answer when negotiation finds nothing, so "
496 "it is the one locale that must exist", want);
497 }
498 33 return ar;
499 }
500  
501 /* ---- the view a template reads --------------------------------------------
502 *
503 * `{% locale.welcome %}` is an ordinary dotted path through a hashref, so the
504 * template half needs no change to Stencil - the same seam CSP uses to make
505 * `{% csp_nonce %}` resolve with nothing passed by the handler.
506 *
507 * The hash is TIED, and both halves of that decision were paid for.
508 *
509 * A plain hash would have to be materialised: per request it copies the
510 * catalogue into SVs on every page, which is the cost the arena exists to
511 * avoid, and cached per catalogue it is a second copy of every translation as
512 * Perl data in whichever worker rendered in that locale. A tied hash copies
513 * nothing - FETCH reads the arena and builds one SV for the string actually
514 * asked for.
515 *
516 * It also keeps the missing-key rule true on BOTH paths. A template resolves
517 * a missing path to the empty string, so a plain hash would silently swallow
518 * `{% locale.typo %}` while `$c->locale('typo')` rendered the key - the same
519 * omission visible in a handler and invisible in a template, which is the
520 * worse half to lose.
521 *
522 * This needs Template::Stencil 0.10. Before it, a tied hash in a path
523 * resolved to nothing at all: its resolver read hashes with hv_common and a
524 * precomputed hash, which does not go through tie magic. That was fixed
525 * there rather than worked around here.
526 */
527 /* The tied object carries NODES, not just a path.
528 *
529 * It used to carry only the dotted path so far, so every FETCH rebuilt the
530 * joined key and looked it up from the catalogue root again: resolving
531 * {% locale.a.b.c %} cost 1 + 2 + 3 = six probes and three SV
532 * concatenations. Carrying the node it already resolved makes each FETCH one
533 * probe from where the last one stopped - three for that path, and no
534 * concatenation at all.
535 *
536 * PIT_DNODE is the same position in the DEFAULT catalogue, carried in step.
537 * Without it there is no way to tell "missing everywhere" from "present in
538 * the default but not translated here" once the joined key is gone, and the
539 * untranslated counter would stop being exact.
540 *
541 * PIT_PREFIX survives for one job only: the missing-key message, which
542 * renders the full dotted path a template asked for. */
543 #define PIT_ARENA 0 /* the arena, as an IV */
544 #define PIT_CAT 1 /* which catalogue, as an index */
545 #define PIT_PREFIX 2 /* the dotted path so far, for the miss message */
546 #define PIT_CFG 3 /* the plugin config, for the dev flag */
547 #define PIT_CTX 4 /* the context, for the logger */
548 #define PIT_NODE 5 /* where we are in the negotiated catalogue */
549 #define PIT_DNODE 6 /* and in the default one */
550 #define PIT_ITER 7 /* the each/keys cursor */
551 #define PIT_MAX 8
552  
553 /* ---- interpolation -------------------------------------------------------- */
554  
555 /* ---- what an application can see ------------------------------------------
556 *
557 * Three numbers, because they answer three different questions and rolling
558 * them together would answer none of them:
559 *
560 * missing - the key is in no catalogue at all. A bug.
561 * untranslated - the key is in the default catalogue but not in the
562 * negotiated one. NOT a bug: it is what a partly translated
563 * site is, and it is how an application measures coverage.
564 * warned - how many times the missing-key warning actually fired.
565 * Provable, so "it did not warn in production" is a number
566 * rather than the absence of one.
567 */
568 static UV pi_n_missing = 0;
569 static UV pi_n_untranslated = 0;
570 static UV pi_n_warned = 0;
571  
572 /* Keys already warned about, so the same page does not warn on every request.
573 * Twenty renders of one page must produce one line: noise is ignored, and
574 * being ignored is how the omission survives. */
575 static HV *pi_seen = NULL;
576  
577 /* `{name}` from a list of name => value pairs.
578 *
579 * Three rules, each of which exists because the alternative is worse:
580 *
581 * - an UNKNOWN placeholder stays literal. `{nmae}` renders as `{nmae}`,
582 * visibly wrong, rather than as a gap nobody notices;
583 * - a MISSING substitution is not an error - a translator adding a
584 * placeholder the caller does not pass must not take the page down;
585 * - a value containing `{other}` is NOT re-scanned. One pass, left to
586 * right, or a user whose name is `{admin}` reaches into the catalogue.
587 */
588 /* `utf8` is the flag the BLOCK recorded for the catalogue string, because
589 * `v` is a pointer and a length and carries none of its own.
590 *
591 * It has to be known here rather than applied by the caller afterwards. The
592 * catalogue string and the substituted values are two different encodings
593 * meeting in one buffer: appending a downgraded Latin-1 argument into a
594 * string of UTF-8 octets and then labelling the result UTF-8 puts a raw
595 * 0xE9 inside a string that claims to be UTF-8. That is corruption, it is
596 * reachable from a form field today through
597 * $c->locale($key, name => $value), and Frozen does not fix it - the block
598 * hands back exactly what it was given. It is ordinary broken code that has
599 * been sitting beside the interesting problem.
600 *
601 * So `out` is flagged UP FRONT and the values go in with sv_catsv, which is
602 * the primitive that knows both sides' flags and upgrades whichever needs
603 * it. sv_catpvn cannot: it sees bytes and a length, which is precisely what
604 * loses the information. The catalogue runs still use sv_catpvn, and that
605 * is correct - those octets already ARE out's encoding. */
606 /* A value made safe to put into markup: & < > " ' as entities, and the
607 * value's own encoding flag kept, since only ASCII bytes are rewritten.
608 * Mortal. */
609 2 static SV *pi_html_value(pTHX_ SV *val) {
610 STRLEN l;
611 2 const char *p = SvPV_const(val, l);
612 2 SV *out = sv_2mortal(newSVpvn("", 0));
613 2 ps_html_esc(aTHX_ out, p, l);
614 2 ▶ if (SvUTF8(val)) SvUTF8_on(out);
Branch: 1 branch
BranchTrueFalseTotal
if (SvUTF8(val)) SvUTF8_on(out); 0 2 2
615 2 return out;
616 }
617  
618 /* `html` escapes each substituted VALUE and leaves the catalogue's own bytes
619 * alone: a translation is authored and may carry a link or an emphasis, and
620 * a value is whatever the caller was handed. That is $c->locale_html. Plain
621 * $c->locale escapes nothing - its result is text, bound for a template that
622 * escapes, a JSON body, a log line - and escaping there would be once too
623 * often for every one of them. */
624 64 static SV *pi_interpolate(pTHX_ const char *v, STRLEN vl, int utf8,
625 SV **args, int nargs, int html) {
626 64 SV *out = newSVpvn("", 0);
627 64 STRLEN i = 0;
628  
629 64 ▶ if (utf8) SvUTF8_on(out);
Branch: 1 branch
BranchTrueFalseTotal
if (utf8) SvUTF8_on(out); 3 61 64
630  
631 123 ▶ while (i < vl) {
Branch: 1 branch
BranchTrueFalseTotal
while (i < vl) { 103 20 123
632 103 const char *open = (const char *)memchr(v + i, '{', vl - i);
633 STRLEN o, close;
634 103 int j, done = 0;
635  
636 103 ▶ if (!open) { sv_catpvn(out, v + i, vl - i); break; }
Branch: 1 branch
BranchTrueFalseTotal
if (!open) { sv_catpvn(out, v + i, vl - i); break; } 44 59 103
637 59 o = (STRLEN)(open - v);
638 59 sv_catpvn(out, v + i, o - i);
639  
640 /* find the close on the same run of name characters */
641 59 close = o + 1;
642 328 ▶ while (close < vl && v[close] != '}' && v[close] != '{') close++;
Branch: 3 branches
BranchTrueFalseTotal
while (close < vl && v[close] != '}' && v[close] != '{') close++; 327 1 328
while (close < vl && v[close] != '}' && v[close] != '{') close++; 269 58 327
while (close < vl && v[close] != '}' && v[close] != '{') close++; 269 0 269
643 59 ▶ if (close >= vl || v[close] != '}') {
Branch: 2 branches
BranchTrueFalseTotal
if (close >= vl || v[close] != '}') { 58 1 59
if (close >= vl || v[close] != '}') { 0 58 58
644 1 sv_catpvn(out, v + o, 1); /* a lone brace is a lone brace */
645 1 i = o + 1;
646 1 continue;
647 }
648  
649 60 ▶ for (j = 0; j + 1 < nargs; j += 2) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = 0; j + 1 < nargs; j += 2) { 58 2 60
650 STRLEN nl;
651 58 const char *n = SvPV_const(args[j], nl);
652 58 ▶ if (nl == close - o - 1 && memcmp(n, v + o + 1, nl) == 0) {
Branch: 2 branches
BranchTrueFalseTotal
if (nl == close - o - 1 && memcmp(n, v + o + 1, nl) == 0) { 58 0 58
if (nl == close - o - 1 && memcmp(n, v + o + 1, nl) == 0) { 56 2 58
653 /* sv_catsv, not sv_catpvn(SvPV_const(...)): the argument
654 * carries its own flag and this is the only append that has
655 * to reconcile two encodings. */
656 56 ▶ if (SvOK(args[j + 1]))
Branch: 1 branch
BranchTrueFalseTotal
if (SvOK(args[j + 1])) 55 1 56
657 55 ▶ sv_catsv(out, html ? pi_html_value(aTHX_ args[j + 1])
Branch: 1 branch
BranchTrueFalseTotal
sv_catsv(out, html ? pi_html_value(aTHX_ args[j + 1]) 2 53 55
658 : args[j + 1]);
659 56 done = 1;
660 56 break;
661 }
662 }
663 58 ▶ if (!done) sv_catpvn(out, v + o, close - o + 1); /* left literal */
Branch: 1 branch
BranchTrueFalseTotal
if (!done) sv_catpvn(out, v + o, close - o + 1); /* left literal */ 2 56 58
664 58 i = close + 1;
665 }
666 64 return out;
667 }
668  
669 #endif /* PUNK_I18N_H */
« xs/reqid.xs ↑ Summary include/punk/punk_log.h »