| 1 |
|
| /* punk_sitemap.h - which of an application's routes are actually URLs. |
| 2 |
|
| * |
| 3 |
|
| * Punk compiles every route at boot, so the application already holds a |
| 4 |
|
| * complete list of what it serves. A sitemap is that list with a filter over |
| 5 |
|
| * it, and this is the filter. |
| 6 |
|
| * |
| 7 |
|
| * WHAT GOES IN, AND WHY THE RULES ARE THESE. |
| 8 |
|
| * |
| 9 |
|
| * A route is a candidate when its method is GET, its path holds no capture, |
| 10 |
|
| * it carries no guard, and it did not opt out. Everything else is out, and |
| 11 |
|
| * OUT IS THE SAFE DIRECTION: a page missing from a sitemap is still crawled |
| 12 |
|
| * if anything links to it, while a page wrongly present is a crawler fetching |
| 13 |
|
| * a 404 or a login redirect on a schedule, for as long as the site exists. |
| 14 |
|
| * |
| 15 |
|
| * A ROUTE WITH A CAPTURE IS NOT A URL. `/users/:id` names a shape, and the |
| 16 |
|
| * ids are not something the route table knows. Those come from the |
| 17 |
|
| * application through the `sitemap` keyword instead. |
| 18 |
|
| * |
| 19 |
|
| * A GUARDED ROUTE IS EXCLUDED WITHOUT ANYONE MAINTAINING A LIST. `guards` is |
| 20 |
|
| * on the route record and `under($prefix, $guard)` copies its chain into |
| 21 |
|
| * every route declared inside it, so the filter can see what a human writing |
| 22 |
|
| * a sitemap by hand has to remember. That is the whole argument for this |
| 23 |
|
| * living in the framework rather than in a script beside it. |
| 24 |
|
| * |
| 25 |
|
| * Must be included after punk_route.h (pr_router and the raw records), |
| 26 |
|
| * punk_names.h (the K_* keys) and punk_app.h (app_hv / app_av). |
| 27 |
|
| */ |
| 28 |
|
| |
| 29 |
|
| #ifndef PUNK_SITEMAP_H |
| 30 |
|
| #define PUNK_SITEMAP_H |
| 31 |
|
| |
| 32 |
|
| /* Is this path a concrete URL rather than a pattern? |
| 33 |
|
| * |
| 34 |
|
| * DELIBERATELY the same test punk_route.h's pr_build uses to decide whether a |
| 35 |
|
| * route is static: |
| 36 |
|
| * |
| 37 |
|
| * if (!memchr(p, ':', pl) && !memchr(p, '*', pl)) |
| 38 |
|
| * |
| 39 |
|
| * If the two ever disagree, the sitemap lists a path the router does not |
| 40 |
|
| * serve as written - so the rule is copied rather than reinvented, and this |
| 41 |
|
| * comment is the reminder to change both. */ |
| 42 |
60140 |
| static int pks_concrete(const char *p, STRLEN pl) { |
| 43 |
60140 |
▶ | if (!p || !pl || p[0] != '/') return 0; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!p || !pl || p[0] != '/') return 0; |
60140 |
0 |
60140 |
| if (!p || !pl || p[0] != '/') return 0; |
60140 |
0 |
60140 |
| if (!p || !pl || p[0] != '/') return 0; |
0 |
60140 |
60140 |
|
| 44 |
60140 |
▶ | return !memchr(p, ':', pl) && !memchr(p, '*', pl); |
Branch: 2 branches
| Branch | True | False | Total |
| return !memchr(p, ':', pl) && !memchr(p, '*', pl); |
60079 |
61 |
60140 |
| return !memchr(p, ':', pl) && !memchr(p, '*', pl); |
60077 |
2 |
60079 |
|
| 45 |
|
| } |
| 46 |
|
| |
| 47 |
|
| /* GET only. HEAD is served off GET and is not a page; anything else is not |
| 48 |
|
| * something a crawler should be fetching at all. `route`'s ANY form does |
| 49 |
|
| * include GET, so it counts. */ |
| 50 |
60206 |
| static int pks_method_ok(const char *m, STRLEN ml) { |
| 51 |
60206 |
▶ | if (!m) return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!m) return 0; |
0 |
60206 |
60206 |
|
| 52 |
60206 |
▶ | if (ml == 3 && memEQ(m, "GET", 3)) return 1; |
Branch: 2 branches
| Branch | True | False | Total |
| if (ml == 3 && memEQ(m, "GET", 3)) return 1; |
60202 |
4 |
60206 |
| if (ml == 3 && memEQ(m, "GET", 3)) return 1; |
60202 |
0 |
60202 |
|
| 53 |
4 |
▶ | if (ml == 3 && memEQ(m, "ANY", 3)) return 1; |
Branch: 2 branches
| Branch | True | False | Total |
| if (ml == 3 && memEQ(m, "ANY", 3)) return 1; |
0 |
4 |
4 |
| if (ml == 3 && memEQ(m, "ANY", 3)) return 1; |
0 |
0 |
0 |
|
| 54 |
4 |
| return 0; |
| 55 |
|
| } |
| 56 |
|
| |
| 57 |
|
| /* A guard chain of any length means the page is behind authentication. The |
| 58 |
|
| * chain is the DECLARED one here rather than the resolved one - `under` |
| 59 |
|
| * copies names in before anything resolves them - and emptiness is the only |
| 60 |
|
| * question being asked, which resolution does not change. */ |
| 61 |
60110 |
| static int pks_guarded(pTHX_ HV *rec) { |
| 62 |
60110 |
| SV **g = hv_fetchs(rec, K_GUARDS, 0); |
| 63 |
60110 |
▶ | if (!(g && *g && SvROK(*g) && SvTYPE(SvRV(*g)) == SVt_PVAV)) return 0; |
Branch: 4 branches
| Branch | True | False | Total |
| if (!(g && *g && SvROK(*g) && SvTYPE(SvRV(*g)) == SVt_PVAV)) return 0; |
60110 |
0 |
60110 |
| if (!(g && *g && SvROK(*g) && SvTYPE(SvRV(*g)) == SVt_PVAV)) return 0; |
60110 |
0 |
60110 |
| if (!(g && *g && SvROK(*g) && SvTYPE(SvRV(*g)) == SVt_PVAV)) return 0; |
60110 |
0 |
60110 |
| if (!(g && *g && SvROK(*g) && SvTYPE(SvRV(*g)) == SVt_PVAV)) return 0; |
0 |
60110 |
60110 |
|
| 64 |
60110 |
| return av_len((AV *)SvRV(*g)) >= 0; |
| 65 |
|
| } |
| 66 |
|
| |
| 67 |
|
| /* What did this method+path say about itself? |
| 68 |
|
| * |
| 69 |
|
| * -1 nothing |
| 70 |
|
| * 0 sitemap => 0, keep it out |
| 71 |
|
| * 1 sitemap => 1, put it in even though something above excluded it |
| 72 |
|
| * |
| 73 |
|
| * The records are the ones app.xs pushed at declaration time, the way it does |
| 74 |
|
| * for validate, compress and max_body. */ |
| 75 |
60327 |
| static int pks_override(pTHX_ HV *app, const char *m, STRLEN ml, |
| 76 |
|
| const char *p, STRLEN pl) { |
| 77 |
60327 |
| SV **x = hv_fetchs(app, K_SITEMAP_ROUTES, 0); |
| 78 |
|
| AV *av; |
| 79 |
|
| SSize_t i, n; |
| 80 |
60327 |
▶ | if (!(x && *x && SvROK(*x) && SvTYPE(SvRV(*x)) == SVt_PVAV)) return -1; |
Branch: 4 branches
| Branch | True | False | Total |
| if (!(x && *x && SvROK(*x) && SvTYPE(SvRV(*x)) == SVt_PVAV)) return -1; |
60327 |
0 |
60327 |
| if (!(x && *x && SvROK(*x) && SvTYPE(SvRV(*x)) == SVt_PVAV)) return -1; |
60327 |
0 |
60327 |
| if (!(x && *x && SvROK(*x) && SvTYPE(SvRV(*x)) == SVt_PVAV)) return -1; |
60327 |
0 |
60327 |
| if (!(x && *x && SvROK(*x) && SvTYPE(SvRV(*x)) == SVt_PVAV)) return -1; |
0 |
60327 |
60327 |
|
| 81 |
60327 |
| av = (AV *)SvRV(*x); |
| 82 |
60327 |
| n = av_len(av) + 1; |
| 83 |
180881 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
120708 |
60173 |
180881 |
|
| 84 |
120708 |
| SV **e = av_fetch(av, i, 0); |
| 85 |
|
| HV *r; |
| 86 |
|
| SV **rm, **rp; |
| 87 |
|
| STRLEN xl, yl; |
| 88 |
|
| const char *xm, *xp; |
| 89 |
120708 |
▶ | if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
Branch: 4 branches
| Branch | True | False | Total |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
120708 |
0 |
120708 |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
120708 |
0 |
120708 |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
120708 |
0 |
120708 |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
0 |
120708 |
120708 |
|
| 90 |
120708 |
| r = (HV *)SvRV(*e); |
| 91 |
120708 |
| rm = hv_fetchs(r, K_METHOD, 0); |
| 92 |
120708 |
| rp = hv_fetchs(r, K_PATH, 0); |
| 93 |
120708 |
▶ | if (!(rm && *rm && rp && *rp)) continue; |
Branch: 4 branches
| Branch | True | False | Total |
| if (!(rm && *rm && rp && *rp)) continue; |
120708 |
0 |
120708 |
| if (!(rm && *rm && rp && *rp)) continue; |
120708 |
0 |
120708 |
| if (!(rm && *rm && rp && *rp)) continue; |
120708 |
0 |
120708 |
| if (!(rm && *rm && rp && *rp)) continue; |
0 |
120708 |
120708 |
|
| 94 |
120708 |
| xm = SvPV_const(*rm, xl); |
| 95 |
120708 |
▶ | xp = SvPV_const(*rp, yl); |
Branch: 1 branch
| Branch | True | False | Total |
| xp = SvPV_const(*rp, yl); |
120708 |
0 |
120708 |
|
| 96 |
120708 |
▶ | if (xl == ml && yl == pl && memEQ(xm, m, ml) && memEQ(xp, p, pl)) { |
Branch: 4 branches
| Branch | True | False | Total |
| if (xl == ml && yl == pl && memEQ(xm, m, ml) && memEQ(xp, p, pl)) { |
120690 |
18 |
120708 |
| if (xl == ml && yl == pl && memEQ(xm, m, ml) && memEQ(xp, p, pl)) { |
238 |
120452 |
120690 |
| if (xl == ml && yl == pl && memEQ(xm, m, ml) && memEQ(xp, p, pl)) { |
238 |
0 |
238 |
| if (xl == ml && yl == pl && memEQ(xm, m, ml) && memEQ(xp, p, pl)) { |
154 |
84 |
238 |
|
| 97 |
154 |
| SV **f = hv_fetchs(r, K_SITEMAP, 0); |
| 98 |
154 |
▶ | return (f && *f && SvTRUE(*f)) ? 1 : 0; |
Branch: 3 branches
| Branch | True | False | Total |
| return (f && *f && SvTRUE(*f)) ? 1 : 0; |
154 |
0 |
154 |
| return (f && *f && SvTRUE(*f)) ? 1 : 0; |
154 |
0 |
154 |
| return (f && *f && SvTRUE(*f)) ? 1 : 0; |
6 |
148 |
154 |
|
| 99 |
|
| } |
| 100 |
|
| } |
| 101 |
60173 |
| return -1; |
| 102 |
|
| } |
| 103 |
|
| |
| 104 |
|
| /* The router's raw records: {method, path, target, guards}, one per declared |
| 105 |
|
| * route, in declaration order. |
| 106 |
|
| * |
| 107 |
|
| * The RAW list and not the compiled one, because the compiled records only |
| 108 |
|
| * exist after `compile` has run and this wants to be answerable from |
| 109 |
|
| * anywhere. Raw already carries the full path and the full guard chain - |
| 110 |
|
| * scopes concatenate the prefix and copy the guards in before `route` is |
| 111 |
|
| * reached - so nothing is missing for the questions above. */ |
| 112 |
64 |
| static AV *pks_raw(pTHX_ HV *app) { |
| 113 |
64 |
| SV *router = app_get(aTHX_ app, K_ROUTER); |
| 114 |
|
| pr_router *rt; |
| 115 |
64 |
▶ | if (!(router && SvROK(router) && SvIOK(SvRV(router)) && SvIV(SvRV(router)))) |
Branch: 4 branches
| Branch | True | False | Total |
| if (!(router && SvROK(router) && SvIOK(SvRV(router)) && SvIV(SvRV(router)))) |
64 |
0 |
64 |
| if (!(router && SvROK(router) && SvIOK(SvRV(router)) && SvIV(SvRV(router)))) |
64 |
0 |
64 |
| if (!(router && SvROK(router) && SvIOK(SvRV(router)) && SvIV(SvRV(router)))) |
64 |
0 |
64 |
| if (!(router && SvROK(router) && SvIOK(SvRV(router)) && SvIV(SvRV(router)))) |
0 |
64 |
64 |
|
| 116 |
0 |
| return NULL; |
| 117 |
64 |
| rt = INT2PTR(pr_router *, SvIV(SvRV(router))); |
| 118 |
64 |
▶ | return rt ? rt->raw : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| return rt ? rt->raw : NULL; |
64 |
0 |
64 |
|
| 119 |
|
| } |
| 120 |
|
| |
| 121 |
|
| /* Sort helper: paths compare as bytes, so two boots of one application |
| 122 |
|
| * produce a byte-identical list. Hash order would make the sitemap's ETag |
| 123 |
|
| * and Last-Modified move on a deploy that changed nothing, and would make a |
| 124 |
|
| * diff of two environments noise. */ |
| 125 |
131225 |
| static I32 pks_cmp(pTHX_ SV *a, SV *b) { |
| 126 |
|
| STRLEN al, bl; |
| 127 |
131225 |
▶ | const char *ap = SvPV_const(a, al); |
Branch: 1 branch
| Branch | True | False | Total |
| const char *ap = SvPV_const(a, al); |
131225 |
0 |
131225 |
|
| 128 |
131225 |
| const char *bp = SvPV_const(b, bl); |
| 129 |
131225 |
| STRLEN n = al < bl ? al : bl; |
| 130 |
131225 |
▶ | int c = n ? memcmp(ap, bp, n) : 0; |
Branch: 1 branch
| Branch | True | False | Total |
| int c = n ? memcmp(ap, bp, n) : 0; |
131225 |
0 |
131225 |
|
| 131 |
131225 |
▶ | if (c) return c < 0 ? -1 : 1; |
Branch: 2 branches
| Branch | True | False | Total |
| if (c) return c < 0 ? -1 : 1; |
117806 |
13419 |
131225 |
| if (c) return c < 0 ? -1 : 1; |
60034 |
57772 |
117806 |
|
| 132 |
13419 |
▶ | return al == bl ? 0 : (al < bl ? -1 : 1); |
Branch: 2 branches
| Branch | True | False | Total |
| return al == bl ? 0 : (al < bl ? -1 : 1); |
13419 |
0 |
13419 |
| return al == bl ? 0 : (al < bl ? -1 : 1); |
13415 |
4 |
13419 |
|
| 133 |
|
| } |
| 134 |
|
| |
| 135 |
|
| /* The same ordering, over [loc, lastmod] pairs. */ |
| 136 |
45 |
| static I32 pks_cmp_pair(pTHX_ SV *a, SV *b) { |
| 137 |
45 |
| SV **x = av_fetch((AV *)SvRV(a), 0, 0); |
| 138 |
45 |
| SV **y = av_fetch((AV *)SvRV(b), 0, 0); |
| 139 |
45 |
▶ | return pks_cmp(aTHX_ x ? *x : &PL_sv_undef, y ? *y : &PL_sv_undef); |
Branch: 2 branches
| Branch | True | False | Total |
| return pks_cmp(aTHX_ x ? *x : &PL_sv_undef, y ? *y : &PL_sv_undef); |
45 |
0 |
45 |
| return pks_cmp(aTHX_ x ? *x : &PL_sv_undef, y ? *y : &PL_sv_undef); |
45 |
0 |
45 |
|
| 140 |
|
| } |
| 141 |
|
| |
| 142 |
|
| /* The candidate paths, sorted and de-duplicated (+1). |
| 143 |
|
| * |
| 144 |
|
| * Paths, not URLs: the absolute form needs the configured base, and taking |
| 145 |
|
| * that from the request would be a host-header injection delivered to search |
| 146 |
|
| * engines. Joining happens where the base is known to be configuration. */ |
| 147 |
53 |
| static AV *pks_paths(pTHX_ SV *appsv) { |
| 148 |
53 |
| HV *app = app_hv(aTHX_ appsv); |
| 149 |
53 |
▶ | AV *raw = app ? pks_raw(aTHX_ app) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| AV *raw = app ? pks_raw(aTHX_ app) : NULL; |
53 |
0 |
53 |
|
| 150 |
53 |
| AV *out = newAV(); |
| 151 |
53 |
| HV *seen = (HV *)sv_2mortal((SV *)newHV()); |
| 152 |
|
| SSize_t i, n; |
| 153 |
|
| |
| 154 |
53 |
▶ | if (!raw) return out; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!raw) return out; |
0 |
53 |
53 |
|
| 155 |
53 |
| n = av_len(raw) + 1; |
| 156 |
60318 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
60265 |
53 |
60318 |
|
| 157 |
60265 |
| SV **e = av_fetch(raw, i, 0); |
| 158 |
|
| HV *rec; |
| 159 |
|
| SV **msv, **psv; |
| 160 |
|
| STRLEN ml, pl; |
| 161 |
|
| const char *m, *p; |
| 162 |
|
| |
| 163 |
60464 |
▶ | if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
Branch: 4 branches
| Branch | True | False | Total |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
60265 |
0 |
60265 |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
60265 |
0 |
60265 |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
60265 |
0 |
60265 |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
0 |
60265 |
60265 |
|
| 164 |
60265 |
| rec = (HV *)SvRV(*e); |
| 165 |
60265 |
| msv = hv_fetchs(rec, K_METHOD, 0); |
| 166 |
60265 |
| psv = hv_fetchs(rec, K_PATH, 0); |
| 167 |
60265 |
▶ | if (!(msv && *msv && psv && *psv)) continue; |
Branch: 4 branches
| Branch | True | False | Total |
| if (!(msv && *msv && psv && *psv)) continue; |
60265 |
0 |
60265 |
| if (!(msv && *msv && psv && *psv)) continue; |
60265 |
0 |
60265 |
| if (!(msv && *msv && psv && *psv)) continue; |
60265 |
0 |
60265 |
| if (!(msv && *msv && psv && *psv)) continue; |
0 |
60265 |
60265 |
|
| 168 |
60265 |
| m = SvPV_const(*msv, ml); |
| 169 |
60265 |
▶ | p = SvPV_const(*psv, pl); |
Branch: 1 branch
| Branch | True | False | Total |
| p = SvPV_const(*psv, pl); |
60265 |
0 |
60265 |
|
| 170 |
|
| |
| 171 |
|
| { |
| 172 |
60265 |
| int said = pks_override(aTHX_ app, m, ml, p, pl); |
| 173 |
60265 |
▶ | if (said == 0) continue; /* sitemap => 0 */ |
Branch: 1 branch
| Branch | True | False | Total |
| if (said == 0) continue; /* sitemap => 0 */ |
121 |
60144 |
60265 |
|
| 174 |
|
| |
| 175 |
|
| /* Method and shape are never overridable: a POST is not a page |
| 176 |
|
| * and `/users/:id` is not a URL, so `sitemap => 1` on either is a |
| 177 |
|
| * mistake rather than an instruction. The guard is the only |
| 178 |
|
| * exclusion an application can argue with, because it is the only |
| 179 |
|
| * one this cannot read the intent of. */ |
| 180 |
60144 |
▶ | if (!pks_method_ok(m, ml)) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!pks_method_ok(m, ml)) continue; |
4 |
60140 |
60144 |
|
| 181 |
60140 |
▶ | if (!pks_concrete(p, pl)) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!pks_concrete(p, pl)) continue; |
63 |
60077 |
60140 |
|
| 182 |
60077 |
▶ | if (said != 1 && pks_guarded(aTHX_ rec)) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (said != 1 && pks_guarded(aTHX_ rec)) continue; |
60075 |
2 |
60077 |
| if (said != 1 && pks_guarded(aTHX_ rec)) continue; |
11 |
60064 |
60075 |
|
| 183 |
|
| } |
| 184 |
60066 |
▶ | if (hv_exists(seen, p, (I32)pl)) continue; /* ANY + GET */ |
Branch: 1 branch
| Branch | True | False | Total |
| if (hv_exists(seen, p, (I32)pl)) continue; /* ANY + GET */ |
0 |
60066 |
60066 |
|
| 185 |
|
| |
| 186 |
|
| /* newSViv and not &PL_sv_yes: hv_store takes ownership of the value's |
| 187 |
|
| * refcount, and an immortal handed to it is decremented when the hash |
| 188 |
|
| * is freed - which before 5.20 is a real decrement of a value that |
| 189 |
|
| * has no protection, and eventually a free underneath the |
| 190 |
|
| * interpreter. */ |
| 191 |
60066 |
| (void)hv_store(seen, p, (I32)pl, newSViv(1), 0); |
| 192 |
60066 |
| av_push(out, newSVpvn(p, pl)); |
| 193 |
|
| } |
| 194 |
|
| |
| 195 |
53 |
▶ | if (av_len(out) > 0) |
Branch: 1 branch
| Branch | True | False | Total |
| if (av_len(out) > 0) |
15 |
38 |
53 |
|
| 196 |
15 |
| sortsv(AvARRAY(out), (size_t)(av_len(out) + 1), pks_cmp); |
| 197 |
53 |
| return out; |
| 198 |
|
| } |
| 199 |
|
| |
| 200 |
|
| /* ---- rendering ------------------------------------------------------------- |
| 201 |
|
| * |
| 202 |
|
| * The protocol caps a sitemap at 50,000 URLs AND 50MB uncompressed, and past |
| 203 |
|
| * either one crawlers reject the file - the whole file, not the excess. So |
| 204 |
|
| * both are counted as the document is written and whichever binds first |
| 205 |
|
| * starts a new part. |
| 206 |
|
| * |
| 207 |
|
| * The byte cap binds more often than people expect: 50,000 URLs at 1KB each |
| 208 |
|
| * is 50MB, and a long path inside its <url> element is not far off 200 bytes. |
| 209 |
|
| * Which is why it is counted as written rather than estimated. */ |
| 210 |
|
| #define PKS_MAX_URLS 50000 |
| 211 |
|
| #define PKS_MAX_BYTES (50UL * 1024 * 1024) |
| 212 |
|
| |
| 213 |
|
| /* XML escaping, over every emitted URL without exception. |
| 214 |
|
| * |
| 215 |
|
| * One unescaped `&` makes the document not well-formed, and a crawler |
| 216 |
|
| * rejects ALL of it rather than the offending entry - so this is a |
| 217 |
|
| * correctness problem before it is a security one. The route half rarely |
| 218 |
|
| * contains anything needing it; the dynamic half routinely will, because a |
| 219 |
|
| * slug with an ampersand in it is ordinary. */ |
| 220 |
60090 |
| static void pks_xml_cat(pTHX_ SV *out, const char *s, STRLEN l) { |
| 221 |
60090 |
| STRLEN i, start = 0; |
| 222 |
1611053 |
▶ | for (i = 0; i < l; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < l; i++) { |
1550963 |
60090 |
1611053 |
|
| 223 |
|
| const char *rep; |
| 224 |
1550963 |
| switch (s[i]) { |
| 225 |
0 |
| case '&': rep = "&"; break; |
| 226 |
0 |
| case '<': rep = "<"; break; |
| 227 |
0 |
| case '>': rep = ">"; break; |
| 228 |
0 |
| case '"': rep = """; break; |
| 229 |
0 |
| case '\'': rep = "'"; break; |
| 230 |
1550963 |
| default: continue; |
| 231 |
|
| } |
| 232 |
0 |
▶ | if (i > start) sv_catpvn(out, s + start, i - start); |
Branch: 1 branch
| Branch | True | False | Total |
| if (i > start) sv_catpvn(out, s + start, i - start); |
0 |
0 |
0 |
|
| 233 |
0 |
| sv_catpv(out, rep); |
| 234 |
0 |
| start = i + 1; |
| 235 |
|
| } |
| 236 |
60090 |
▶ | if (l > start) sv_catpvn(out, s + start, l - start); |
Branch: 1 branch
| Branch | True | False | Total |
| if (l > start) sv_catpvn(out, s + start, l - start); |
60090 |
0 |
60090 |
|
| 237 |
60090 |
| } |
| 238 |
|
| |
| 239 |
|
| /* Percent-encoding, which is a DIFFERENT job from escaping and comes first. |
| 240 |
|
| * |
| 241 |
|
| * A path holding a space or a non-ASCII byte has to be percent-encoded to be |
| 242 |
|
| * a URL at all; escaping alone would produce a well-formed document |
| 243 |
|
| * containing an invalid URL. Encoding first also means everything reaching |
| 244 |
|
| * the escaper is ASCII, so a route path that is not valid UTF-8 cannot make |
| 245 |
|
| * the document unparseable - the encoder turned it into %XX long before. |
| 246 |
|
| * |
| 247 |
|
| * Unreserved plus '/' is kept and everything else encoded. Over-encoding a |
| 248 |
|
| * sub-delimiter is legal and costs three bytes; under-encoding one is a |
| 249 |
|
| * different URL. */ |
| 250 |
60091 |
| static void pks_pct_cat(pTHX_ SV *out, const char *s, STRLEN l) { |
| 251 |
|
| static const char hex[] = "0123456789ABCDEF"; |
| 252 |
|
| STRLEN i; |
| 253 |
469571 |
▶ | for (i = 0; i < l; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < l; i++) { |
409480 |
60091 |
469571 |
|
| 254 |
409480 |
| unsigned char c = (unsigned char)s[i]; |
| 255 |
409480 |
▶ | if (isALNUM(c) || c == '-' || c == '.' || c == '_' || c == '~' |
Branch: 5 branches
| Branch | True | False | Total |
| if (isALNUM(c) || c == '-' || c == '.' || c == '_' || c == '~' |
60132 |
349348 |
409480 |
| if (isALNUM(c) || c == '-' || c == '.' || c == '_' || c == '~' |
60131 |
1 |
60132 |
| if (isALNUM(c) || c == '-' || c == '.' || c == '_' || c == '~' |
60131 |
0 |
60131 |
| if (isALNUM(c) || c == '-' || c == '.' || c == '_' || c == '~' |
60131 |
0 |
60131 |
| if (isALNUM(c) || c == '-' || c == '.' || c == '_' || c == '~' |
60131 |
0 |
60131 |
|
| 256 |
60131 |
▶ | || c == '/') { |
Branch: 1 branch
| Branch | True | False | Total |
| || c == '/') { |
60127 |
4 |
60131 |
|
| 257 |
409476 |
| sv_catpvn(out, (const char *)&c, 1); |
| 258 |
|
| } |
| 259 |
|
| else { |
| 260 |
|
| char e[3]; |
| 261 |
4 |
| e[0] = '%'; e[1] = hex[c >> 4]; e[2] = hex[c & 0xF]; |
| 262 |
4 |
| sv_catpvn(out, e, 3); |
| 263 |
|
| } |
| 264 |
|
| } |
| 265 |
60091 |
| } |
| 266 |
|
| |
| 267 |
|
| /* One <url> element onto `out`, base + path, encoded then escaped. */ |
| 268 |
|
| /* ---- the dynamic half ------------------------------------------------------- |
| 269 |
|
| * |
| 270 |
|
| * A section is application code returning data that goes straight into a |
| 271 |
|
| * structured document, so every field is checked on the way in. The route |
| 272 |
|
| * half needs none of this because its paths are declarations; a section's are |
| 273 |
|
| * database rows. |
| 274 |
|
| */ |
| 275 |
|
| |
| 276 |
|
| /* Is this a location this application may publish? |
| 277 |
|
| * |
| 278 |
|
| * Rooted, concrete, same-origin, and free of anything that would break out of |
| 279 |
|
| * the element it lands in: |
| 280 |
|
| * |
| 281 |
|
| * - it must start with '/', because the base supplies the origin; |
| 282 |
|
| * - it must NOT start with '//', which is protocol-relative and names |
| 283 |
|
| * another host - `//evil.example/x` in a sitemap is somebody else's pages |
| 284 |
|
| * published under your name; |
| 285 |
|
| * - it must hold no ':' or '*', which means a section returned the route |
| 286 |
|
| * PATTERN by mistake and a literal `:id` would go in the file; |
| 287 |
|
| * - no control bytes and no backslash, the bug class |
| 288 |
|
| * reference_reflected_request_bytes exists for. |
| 289 |
|
| * |
| 290 |
|
| * `$c->safe_path` encodes the same rules for redirects, and the reasoning is |
| 291 |
|
| * identical: bytes the application did not write, reaching somewhere |
| 292 |
|
| * structured. */ |
| 293 |
20 |
| static int pks_loc_ok(const char *p, STRLEN l) { |
| 294 |
|
| STRLEN i; |
| 295 |
20 |
▶ | if (!p || !l || p[0] != '/') return 0; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!p || !l || p[0] != '/') return 0; |
20 |
0 |
20 |
| if (!p || !l || p[0] != '/') return 0; |
20 |
0 |
20 |
| if (!p || !l || p[0] != '/') return 0; |
2 |
18 |
20 |
|
| 296 |
18 |
▶ | if (l > 1 && p[1] == '/') return 0; |
Branch: 2 branches
| Branch | True | False | Total |
| if (l > 1 && p[1] == '/') return 0; |
18 |
0 |
18 |
| if (l > 1 && p[1] == '/') return 0; |
1 |
17 |
18 |
|
| 297 |
17 |
▶ | if (memchr(p, ':', l) || memchr(p, '*', l)) return 0; |
Branch: 2 branches
| Branch | True | False | Total |
| if (memchr(p, ':', l) || memchr(p, '*', l)) return 0; |
16 |
1 |
17 |
| if (memchr(p, ':', l) || memchr(p, '*', l)) return 0; |
0 |
16 |
16 |
|
| 298 |
141 |
▶ | for (i = 0; i < l; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < l; i++) { |
127 |
14 |
141 |
|
| 299 |
127 |
| unsigned char c = (unsigned char)p[i]; |
| 300 |
127 |
▶ | if (c < 0x20 || c == 0x7F || c == '\\') return 0; |
Branch: 3 branches
| Branch | True | False | Total |
| if (c < 0x20 || c == 0x7F || c == '\\') return 0; |
126 |
1 |
127 |
| if (c < 0x20 || c == 0x7F || c == '\\') return 0; |
126 |
0 |
126 |
| if (c < 0x20 || c == 0x7F || c == '\\') return 0; |
1 |
125 |
126 |
|
| 301 |
|
| } |
| 302 |
14 |
| return 1; |
| 303 |
|
| } |
| 304 |
|
| |
| 305 |
|
| /* W3C Datetime, which is what <lastmod> takes: a date, optionally with a |
| 306 |
|
| * time. Parsed rather than passed through, because one unparseable value |
| 307 |
|
| * makes the whole document invalid and a crawler rejects all of it. Anything |
| 308 |
|
| * that does not fit is DROPPED and the URL still goes in - a missing lastmod |
| 309 |
|
| * costs nothing, and losing the page would. */ |
| 310 |
11 |
| static int pks_lastmod_ok(const char *s, STRLEN l) { |
| 311 |
|
| STRLEN i; |
| 312 |
11 |
▶ | if (l < 10) return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (l < 10) return 0; |
0 |
11 |
11 |
|
| 313 |
105 |
▶ | for (i = 0; i < 10; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < 10; i++) { |
96 |
9 |
105 |
|
| 314 |
96 |
▶ | if (i == 4 || i == 7) { if (s[i] != '-') return 0; } |
Branch: 3 branches
| Branch | True | False | Total |
| if (i == 4 || i == 7) { if (s[i] != '-') return 0; } |
86 |
10 |
96 |
| if (i == 4 || i == 7) { if (s[i] != '-') return 0; } |
9 |
77 |
86 |
| if (i == 4 || i == 7) { if (s[i] != '-') return 0; } |
1 |
18 |
19 |
|
| 315 |
77 |
▶ | else if (!isDIGIT(s[i])) return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| else if (!isDIGIT(s[i])) return 0; |
1 |
76 |
77 |
|
| 316 |
|
| } |
| 317 |
9 |
▶ | if (l == 10) return 1; |
Branch: 1 branch
| Branch | True | False | Total |
| if (l == 10) return 1; |
8 |
1 |
9 |
|
| 318 |
1 |
▶ | if (s[10] != 'T') return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (s[10] != 'T') return 0; |
0 |
1 |
1 |
|
| 319 |
|
| /* the rest is time and zone: digits, ':', '+', '-', '.', 'Z' */ |
| 320 |
10 |
▶ | for (i = 11; i < l; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 11; i < l; i++) { |
9 |
1 |
10 |
|
| 321 |
9 |
| char c = s[i]; |
| 322 |
9 |
▶ | if (!(isDIGIT(c) || c == ':' || c == '+' || c == '-' || c == '.' |
Branch: 6 branches
| Branch | True | False | Total |
| if (!(isDIGIT(c) || c == ':' || c == '+' || c == '-' || c == '.' |
3 |
6 |
9 |
| if (!(isDIGIT(c) || c == ':' || c == '+' || c == '-' || c == '.' |
1 |
2 |
3 |
| if (!(isDIGIT(c) || c == ':' || c == '+' || c == '-' || c == '.' |
1 |
0 |
1 |
| if (!(isDIGIT(c) || c == ':' || c == '+' || c == '-' || c == '.' |
1 |
0 |
1 |
| if (!(isDIGIT(c) || c == ':' || c == '+' || c == '-' || c == '.' |
1 |
0 |
1 |
| if (!(isDIGIT(c) || c == ':' || c == '+' || c == '-' || c == '.' |
0 |
1 |
1 |
|
| 323 |
0 |
| || c == 'Z')) return 0; |
| 324 |
|
| } |
| 325 |
1 |
| return 1; |
| 326 |
|
| } |
| 327 |
|
| |
| 328 |
|
| /* One <url> element, with the optional <lastmod>. */ |
| 329 |
|
| static void pks_url_cat_lm(pTHX_ SV *out, SV *base, const char *p, STRLEN pl, |
| 330 |
|
| SV *lastmod); |
| 331 |
|
| |
| 332 |
0 |
| static void pks_url_cat(pTHX_ SV *out, SV *base, const char *p, STRLEN pl) { |
| 333 |
0 |
| SV *loc = sv_2mortal(newSVpvn("", 0)); |
| 334 |
|
| STRLEN bl; |
| 335 |
0 |
| const char *bp = SvPV_const(base, bl); |
| 336 |
|
| /* A trailing slash on the base would double the one every path starts |
| 337 |
|
| * with, and `https://x//about` is a different URL to a crawler. */ |
| 338 |
0 |
▶ | while (bl && bp[bl - 1] == '/') bl--; |
Branch: 2 branches
| Branch | True | False | Total |
| while (bl && bp[bl - 1] == '/') bl--; |
0 |
0 |
0 |
| while (bl && bp[bl - 1] == '/') bl--; |
0 |
0 |
0 |
|
| 339 |
0 |
| sv_catpvn(loc, bp, bl); |
| 340 |
0 |
| pks_pct_cat(aTHX_ loc, p, pl); |
| 341 |
|
| |
| 342 |
0 |
| sv_catpvs(out, " <url><loc>"); |
| 343 |
|
| { |
| 344 |
|
| STRLEN ll; |
| 345 |
0 |
| const char *lp = SvPV_const(loc, ll); |
| 346 |
0 |
| pks_xml_cat(aTHX_ out, lp, ll); |
| 347 |
|
| } |
| 348 |
0 |
| sv_catpvs(out, "</loc></url>\n"); |
| 349 |
0 |
| } |
| 350 |
|
| |
| 351 |
60079 |
| static void pks_url_cat_lm(pTHX_ SV *out, SV *base, const char *p, STRLEN pl, |
| 352 |
|
| SV *lastmod) { |
| 353 |
60079 |
| SV *loc = sv_2mortal(newSVpvn("", 0)); |
| 354 |
|
| STRLEN bl; |
| 355 |
60079 |
| const char *bp = SvPV_const(base, bl); |
| 356 |
60081 |
▶ | while (bl && bp[bl - 1] == '/') bl--; |
Branch: 2 branches
| Branch | True | False | Total |
| while (bl && bp[bl - 1] == '/') bl--; |
60081 |
0 |
60081 |
| while (bl && bp[bl - 1] == '/') bl--; |
2 |
60079 |
60081 |
|
| 357 |
60079 |
| sv_catpvn(loc, bp, bl); |
| 358 |
60079 |
| pks_pct_cat(aTHX_ loc, p, pl); |
| 359 |
|
| |
| 360 |
60079 |
| sv_catpvs(out, " <url><loc>"); |
| 361 |
|
| { |
| 362 |
|
| STRLEN ll; |
| 363 |
60079 |
| const char *lp = SvPV_const(loc, ll); |
| 364 |
60079 |
| pks_xml_cat(aTHX_ out, lp, ll); |
| 365 |
|
| } |
| 366 |
60079 |
| sv_catpvs(out, "</loc>"); |
| 367 |
60079 |
▶ | if (lastmod && SvOK(lastmod)) { |
Branch: 2 branches
| Branch | True | False | Total |
| if (lastmod && SvOK(lastmod)) { |
60079 |
0 |
60079 |
| if (lastmod && SvOK(lastmod)) { |
11 |
60068 |
60079 |
|
| 368 |
|
| STRLEN ml; |
| 369 |
11 |
| const char *mp = SvPV_const(lastmod, ml); |
| 370 |
11 |
▶ | if (pks_lastmod_ok(mp, ml)) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (pks_lastmod_ok(mp, ml)) { |
9 |
2 |
11 |
|
| 371 |
9 |
| sv_catpvs(out, "<lastmod>"); |
| 372 |
9 |
| pks_xml_cat(aTHX_ out, mp, ml); |
| 373 |
9 |
| sv_catpvs(out, "</lastmod>"); |
| 374 |
|
| } |
| 375 |
|
| } |
| 376 |
60079 |
| sv_catpvs(out, "</url>\n"); |
| 377 |
60079 |
| } |
| 378 |
|
| |
| 379 |
|
| /* Run one section and collect what it returned as [loc, lastmod] pairs. |
| 380 |
|
| * |
| 381 |
|
| * G_EVAL: a section reads a database, and a section that died must not take |
| 382 |
|
| * the request with it. A failed section warns and contributes nothing, which |
| 383 |
|
| * degrades the sitemap rather than the site. */ |
| 384 |
10 |
| static void pks_run_section(pTHX_ SV *code, SV *name, AV *out) { |
| 385 |
10 |
| dSP; |
| 386 |
|
| int count, i; |
| 387 |
10 |
| AV *got = (AV *)sv_2mortal((SV *)newAV()); |
| 388 |
10 |
| IV kept = 0; |
| 389 |
|
| |
| 390 |
10 |
| ENTER; SAVETMPS; |
| 391 |
10 |
▶ | PUSHMARK(SP); |
Branch: 1 branch
| Branch | True | False | Total |
| PUSHMARK(SP); |
0 |
10 |
10 |
|
| 392 |
10 |
| PUTBACK; |
| 393 |
10 |
| count = call_sv(code, G_ARRAY | G_EVAL); |
| 394 |
10 |
| SPAGAIN; |
| 395 |
30 |
▶ | for (i = count - 1; i >= 0; i--) av_store(got, (SSize_t)i, newSVsv(POPs)); |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = count - 1; i >= 0; i--) av_store(got, (SSize_t)i, newSVsv(POPs)); |
20 |
10 |
30 |
|
| 396 |
10 |
| PUTBACK; |
| 397 |
10 |
▶ | if (SvTRUE(ERRSV)) |
Branch: 2 branches
| Branch | True | False | Total |
| if (SvTRUE(ERRSV)) |
10 |
0 |
10 |
| if (SvTRUE(ERRSV)) |
1 |
9 |
10 |
|
| 398 |
1 |
▶ | warn("Punk::Plugin::Sitemap: section '%" SVf "' died, and contributes " |
Branch: 1 branch
| Branch | True | False | Total |
| warn("Punk::Plugin::Sitemap: section '%" SVf "' died, and contributes " |
1 |
0 |
1 |
|
| 399 |
|
| "nothing: %" SVf, SVfARG(name), SVfARG(ERRSV)); |
| 400 |
10 |
▶ | FREETMPS; LEAVE; |
Branch: 1 branch
| Branch | True | False | Total |
| FREETMPS; LEAVE; |
10 |
0 |
10 |
|
| 401 |
|
| |
| 402 |
30 |
▶ | for (i = 0; i < count; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < count; i++) { |
20 |
10 |
30 |
|
| 403 |
20 |
| SV **e = av_fetch(got, (SSize_t)i, 0); |
| 404 |
20 |
| SV *loc = NULL, *lm = NULL; |
| 405 |
|
| STRLEN ll; |
| 406 |
|
| const char *lp; |
| 407 |
26 |
▶ | if (!(e && *e && SvOK(*e))) continue; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(e && *e && SvOK(*e))) continue; |
20 |
0 |
20 |
| if (!(e && *e && SvOK(*e))) continue; |
20 |
0 |
20 |
| if (!(e && *e && SvOK(*e))) continue; |
0 |
20 |
20 |
|
| 408 |
|
| |
| 409 |
20 |
▶ | if (SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV) { |
Branch: 2 branches
| Branch | True | False | Total |
| if (SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV) { |
7 |
13 |
20 |
| if (SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV) { |
7 |
0 |
7 |
|
| 410 |
7 |
| HV *r = (HV *)SvRV(*e); |
| 411 |
7 |
| SV **x = hv_fetchs(r, "loc", 0); |
| 412 |
7 |
| SV **m = hv_fetchs(r, "lastmod", 0); |
| 413 |
7 |
▶ | loc = (x && *x) ? *x : NULL; |
Branch: 2 branches
| Branch | True | False | Total |
| loc = (x && *x) ? *x : NULL; |
7 |
0 |
7 |
| loc = (x && *x) ? *x : NULL; |
7 |
0 |
7 |
|
| 414 |
7 |
▶ | lm = (m && *m && SvOK(*m)) ? *m : NULL; |
Branch: 3 branches
| Branch | True | False | Total |
| lm = (m && *m && SvOK(*m)) ? *m : NULL; |
5 |
2 |
7 |
| lm = (m && *m && SvOK(*m)) ? *m : NULL; |
5 |
0 |
5 |
| lm = (m && *m && SvOK(*m)) ? *m : NULL; |
5 |
0 |
5 |
|
| 415 |
|
| } |
| 416 |
13 |
| else loc = *e; /* a bare string is a loc */ |
| 417 |
|
| |
| 418 |
20 |
▶ | if (!(loc && SvOK(loc))) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!(loc && SvOK(loc))) continue; |
20 |
0 |
20 |
| if (!(loc && SvOK(loc))) continue; |
0 |
20 |
20 |
|
| 419 |
20 |
| lp = SvPV_const(loc, ll); |
| 420 |
|
| |
| 421 |
20 |
▶ | if (!pks_loc_ok(lp, ll)) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (!pks_loc_ok(lp, ll)) { |
6 |
14 |
20 |
|
| 422 |
6 |
| warn("Punk::Plugin::Sitemap: section '%" SVf "' returned a loc " |
| 423 |
|
| "that is not a rooted, same-origin path and is dropped: " |
| 424 |
|
| "'%" SVf "'", SVfARG(name), SVfARG(loc)); |
| 425 |
6 |
| continue; |
| 426 |
|
| } |
| 427 |
|
| /* A section walking a table that grew is how this becomes an |
| 428 |
|
| * out-of-memory at boot rather than a large file. */ |
| 429 |
14 |
▶ | if (kept >= PKS_MAX_URLS) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (kept >= PKS_MAX_URLS) { |
0 |
14 |
14 |
|
| 430 |
0 |
| warn("Punk::Plugin::Sitemap: section '%" SVf "' returned more " |
| 431 |
|
| "than %d locations and is truncated there", |
| 432 |
|
| SVfARG(name), (int)PKS_MAX_URLS); |
| 433 |
0 |
| break; |
| 434 |
|
| } |
| 435 |
|
| { |
| 436 |
14 |
| AV *pair = newAV(); |
| 437 |
14 |
| av_push(pair, newSVsv(loc)); |
| 438 |
14 |
▶ | av_push(pair, lm ? newSVsv(lm) : newSV(0)); |
Branch: 1 branch
| Branch | True | False | Total |
| av_push(pair, lm ? newSVsv(lm) : newSV(0)); |
5 |
9 |
14 |
|
| 439 |
14 |
| av_push(out, newRV_noinc((SV *)pair)); |
| 440 |
14 |
| kept++; |
| 441 |
|
| } |
| 442 |
|
| } |
| 443 |
10 |
| } |
| 444 |
|
| |
| 445 |
|
| /* The pages of every `markdown` mount, as [loc, lastmod] pairs sorted by loc. |
| 446 |
|
| * |
| 447 |
|
| * A mount is invisible to the route table, but a markdown mount is exactly the |
| 448 |
|
| * content a sitemap is for, and its page table is built by the time this runs |
| 449 |
|
| * (compile builds the mounts before it wraps the middleware). lastmod is the |
| 450 |
|
| * page's `updated:`, else its `date:`, as YYYY-MM-DD; an undated page goes in |
| 451 |
|
| * without one, as a route does. |
| 452 |
|
| * |
| 453 |
|
| * `robots:` naming noindex leaves a page out of the sitemap - and it is NOT |
| 454 |
|
| * added to robots.txt: a page a crawler may not fetch is a page whose noindex |
| 455 |
|
| * it never reads. `sitemap => 0` on the mount leaves out the whole tree. */ |
| 456 |
5 |
| static int pks_noindex(const char *s, STRLEN l) { |
| 457 |
|
| STRLEN i; |
| 458 |
5 |
▶ | for (i = 0; i + 7 <= l; i++) |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i + 7 <= l; i++) |
5 |
0 |
5 |
|
| 459 |
5 |
▶ | if (toLOWER(s[i]) == 'n' && toLOWER(s[i + 1]) == 'o' |
Branch: 4 branches
| Branch | True | False | Total |
| if (toLOWER(s[i]) == 'n' && toLOWER(s[i + 1]) == 'o' |
5 |
0 |
5 |
| if (toLOWER(s[i]) == 'n' && toLOWER(s[i + 1]) == 'o' |
5 |
0 |
5 |
| if (toLOWER(s[i]) == 'n' && toLOWER(s[i + 1]) == 'o' |
0 |
5 |
5 |
| if (toLOWER(s[i]) == 'n' && toLOWER(s[i + 1]) == 'o' |
5 |
0 |
5 |
|
| 460 |
5 |
▶ | && toLOWER(s[i + 2]) == 'i' && toLOWER(s[i + 3]) == 'n' |
Branch: 4 branches
| Branch | True | False | Total |
| && toLOWER(s[i + 2]) == 'i' && toLOWER(s[i + 3]) == 'n' |
5 |
0 |
5 |
| && toLOWER(s[i + 2]) == 'i' && toLOWER(s[i + 3]) == 'n' |
5 |
0 |
5 |
| && toLOWER(s[i + 2]) == 'i' && toLOWER(s[i + 3]) == 'n' |
0 |
5 |
5 |
| && toLOWER(s[i + 2]) == 'i' && toLOWER(s[i + 3]) == 'n' |
5 |
0 |
5 |
|
| 461 |
5 |
▶ | && toLOWER(s[i + 4]) == 'd' && toLOWER(s[i + 5]) == 'e' |
Branch: 4 branches
| Branch | True | False | Total |
| && toLOWER(s[i + 4]) == 'd' && toLOWER(s[i + 5]) == 'e' |
0 |
5 |
5 |
| && toLOWER(s[i + 4]) == 'd' && toLOWER(s[i + 5]) == 'e' |
5 |
0 |
5 |
| && toLOWER(s[i + 4]) == 'd' && toLOWER(s[i + 5]) == 'e' |
0 |
5 |
5 |
| && toLOWER(s[i + 4]) == 'd' && toLOWER(s[i + 5]) == 'e' |
5 |
0 |
5 |
|
| 462 |
5 |
▶ | && toLOWER(s[i + 6]) == 'x') |
Branch: 2 branches
| Branch | True | False | Total |
| && toLOWER(s[i + 6]) == 'x') |
0 |
5 |
5 |
| && toLOWER(s[i + 6]) == 'x') |
5 |
0 |
5 |
|
| 463 |
5 |
| return 1; |
| 464 |
0 |
| return 0; |
| 465 |
|
| } |
| 466 |
|
| |
| 467 |
47 |
| static AV *pks_md_entries(pTHX_ SV *appsv) { |
| 468 |
47 |
| HV *app = app_hv(aTHX_ appsv); |
| 469 |
47 |
▶ | SV *mp = app ? app_get(aTHX_ app, K_MOUNTS_C) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *mp = app ? app_get(aTHX_ app, K_MOUNTS_C) : NULL; |
47 |
0 |
47 |
|
| 470 |
47 |
| AV *out = newAV(); |
| 471 |
|
| AV *mounts; |
| 472 |
|
| SSize_t i, n; |
| 473 |
47 |
▶ | if (!(mp && SvROK(mp) && SvTYPE(SvRV(mp)) == SVt_PVAV)) return out; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(mp && SvROK(mp) && SvTYPE(SvRV(mp)) == SVt_PVAV)) return out; |
47 |
0 |
47 |
| if (!(mp && SvROK(mp) && SvTYPE(SvRV(mp)) == SVt_PVAV)) return out; |
47 |
0 |
47 |
| if (!(mp && SvROK(mp) && SvTYPE(SvRV(mp)) == SVt_PVAV)) return out; |
0 |
47 |
47 |
|
| 474 |
47 |
| mounts = (AV *)SvRV(mp); |
| 475 |
47 |
| n = av_len(mounts) + 1; |
| 476 |
55 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
8 |
47 |
55 |
|
| 477 |
8 |
| SV **e = av_fetch(mounts, i, 0); |
| 478 |
|
| HV *m; |
| 479 |
|
| SV **flag, **mapp, *opt; |
| 480 |
|
| AV *cap, *order; |
| 481 |
|
| HV *pages, *opts; |
| 482 |
|
| SV *prefix; |
| 483 |
|
| SSize_t j, pn; |
| 484 |
8 |
▶ | if (!(e && *e && SvROK(*e))) continue; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(e && *e && SvROK(*e))) continue; |
8 |
0 |
8 |
| if (!(e && *e && SvROK(*e))) continue; |
8 |
0 |
8 |
| if (!(e && *e && SvROK(*e))) continue; |
0 |
8 |
8 |
|
| 485 |
8 |
| m = (HV *)SvRV(*e); |
| 486 |
8 |
| flag = hv_fetchs(m, K_MD_MOUNT, 0); |
| 487 |
8 |
▶ | if (!(flag && *flag && SvTRUE(*flag))) continue; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(flag && *flag && SvTRUE(*flag))) continue; |
8 |
0 |
8 |
| if (!(flag && *flag && SvTRUE(*flag))) continue; |
8 |
0 |
8 |
| if (!(flag && *flag && SvTRUE(*flag))) continue; |
0 |
8 |
8 |
|
| 488 |
8 |
| mapp = hv_fetchs(m, K_APP, 0); |
| 489 |
8 |
▶ | cap = (mapp && *mapp && SvROK(*mapp) && SvTYPE(SvRV(*mapp)) == SVt_PVCV) |
Branch: 3 branches
| Branch | True | False | Total |
| cap = (mapp && *mapp && SvROK(*mapp) && SvTYPE(SvRV(*mapp)) == SVt_PVCV) |
8 |
0 |
8 |
| cap = (mapp && *mapp && SvROK(*mapp) && SvTYPE(SvRV(*mapp)) == SVt_PVCV) |
8 |
0 |
8 |
| cap = (mapp && *mapp && SvROK(*mapp) && SvTYPE(SvRV(*mapp)) == SVt_PVCV) |
8 |
0 |
8 |
|
| 490 |
16 |
▶ | ? punk_clos_cap(aTHX_ (CV *)SvRV(*mapp)) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| ? punk_clos_cap(aTHX_ (CV *)SvRV(*mapp)) : NULL; |
8 |
0 |
8 |
|
| 491 |
8 |
▶ | if (!cap) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!cap) continue; |
0 |
8 |
8 |
|
| 492 |
8 |
| opts = (HV *)SvRV(*av_fetch(cap, PMDC_OPTS, 0)); |
| 493 |
8 |
| opt = pmd_opt(aTHX_ opts, "sitemap"); |
| 494 |
8 |
▶ | if (opt && !SvTRUE(opt)) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (opt && !SvTRUE(opt)) continue; |
2 |
6 |
8 |
| if (opt && !SvTRUE(opt)) continue; |
2 |
0 |
2 |
|
| 495 |
6 |
| pages = (HV *)SvRV(*av_fetch(cap, PMDC_PAGES, 0)); |
| 496 |
6 |
| order = (AV *)SvRV(*av_fetch(cap, PMDC_ORDER, 0)); |
| 497 |
6 |
| prefix = *av_fetch(cap, PMDC_PREFIX, 0); |
| 498 |
6 |
| pn = av_len(order) + 1; |
| 499 |
37 |
▶ | for (j = 0; j < pn; j++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (j = 0; j < pn; j++) { |
31 |
6 |
37 |
|
| 500 |
31 |
| SV **u = av_fetch(order, j, 0); |
| 501 |
|
| HE *he; |
| 502 |
|
| AV *page, *pair; |
| 503 |
31 |
| SV **x, *loc, *lm = NULL; |
| 504 |
31 |
▶ | if (!(u && *u)) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!(u && *u)) continue; |
31 |
0 |
31 |
| if (!(u && *u)) continue; |
0 |
31 |
31 |
|
| 505 |
31 |
| he = hv_fetch_ent(pages, *u, 0, 0); |
| 506 |
31 |
▶ | if (!(he && HeVAL(he) && SvROK(HeVAL(he)))) continue; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(he && HeVAL(he) && SvROK(HeVAL(he)))) continue; |
31 |
0 |
31 |
| if (!(he && HeVAL(he) && SvROK(HeVAL(he)))) continue; |
31 |
0 |
31 |
| if (!(he && HeVAL(he) && SvROK(HeVAL(he)))) continue; |
0 |
31 |
31 |
|
| 507 |
31 |
| page = (AV *)SvRV(HeVAL(he)); |
| 508 |
31 |
| x = av_fetch(page, PMD_META, 0); |
| 509 |
31 |
▶ | if (x && *x && SvROK(*x)) { |
Branch: 3 branches
| Branch | True | False | Total |
| if (x && *x && SvROK(*x)) { |
31 |
0 |
31 |
| if (x && *x && SvROK(*x)) { |
31 |
0 |
31 |
| if (x && *x && SvROK(*x)) { |
31 |
0 |
31 |
|
| 510 |
31 |
| SV *r = pmd_opt(aTHX_ (HV *)SvRV(*x), "robots"); |
| 511 |
31 |
▶ | if (r) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (r) { |
5 |
26 |
31 |
|
| 512 |
|
| STRLEN rl; |
| 513 |
5 |
| const char *rp = SvPV_const(r, rl); |
| 514 |
5 |
▶ | if (pks_noindex(rp, rl)) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (pks_noindex(rp, rl)) continue; |
5 |
0 |
5 |
|
| 515 |
|
| } |
| 516 |
|
| } |
| 517 |
26 |
| loc = newSVsv(prefix); |
| 518 |
26 |
| sv_catsv(loc, *u); |
| 519 |
26 |
▶ | if (!SvCUR(loc)) sv_catpvs(loc, "/"); |
Branch: 1 branch
| Branch | True | False | Total |
| if (!SvCUR(loc)) sv_catpvs(loc, "/"); |
0 |
26 |
26 |
|
| 520 |
26 |
| x = av_fetch(page, PMD_UPDATED, 0); |
| 521 |
26 |
▶ | if (!(x && *x && SvOK(*x))) x = av_fetch(page, PMD_DATE, 0); |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(x && *x && SvOK(*x))) x = av_fetch(page, PMD_DATE, 0); |
26 |
0 |
26 |
| if (!(x && *x && SvOK(*x))) x = av_fetch(page, PMD_DATE, 0); |
26 |
0 |
26 |
| if (!(x && *x && SvOK(*x))) x = av_fetch(page, PMD_DATE, 0); |
16 |
10 |
26 |
|
| 522 |
26 |
▶ | if (x && *x && SvOK(*x)) { |
Branch: 3 branches
| Branch | True | False | Total |
| if (x && *x && SvOK(*x)) { |
26 |
0 |
26 |
| if (x && *x && SvOK(*x)) { |
26 |
0 |
26 |
| if (x && *x && SvOK(*x)) { |
10 |
16 |
26 |
|
| 523 |
10 |
| SV *iso = sv_2mortal(pmd_iso(aTHX_ SvIV(*x))); |
| 524 |
10 |
| lm = newSVpvn(SvPVX(iso), 10); /* YYYY-MM-DD */ |
| 525 |
|
| } |
| 526 |
26 |
| pair = newAV(); |
| 527 |
26 |
| av_push(pair, loc); |
| 528 |
26 |
▶ | av_push(pair, lm ? lm : newSV(0)); |
Branch: 1 branch
| Branch | True | False | Total |
| av_push(pair, lm ? lm : newSV(0)); |
16 |
10 |
26 |
|
| 529 |
26 |
| av_push(out, newRV_noinc((SV *)pair)); |
| 530 |
|
| } |
| 531 |
|
| } |
| 532 |
47 |
▶ | if (av_len(out) > 0) |
Branch: 1 branch
| Branch | True | False | Total |
| if (av_len(out) > 0) |
5 |
42 |
47 |
|
| 533 |
5 |
| sortsv(AvARRAY(out), (size_t)(av_len(out) + 1), pks_cmp_pair); |
| 534 |
47 |
| return out; |
| 535 |
|
| } |
| 536 |
|
| |
| 537 |
|
| /* Every entry that goes in the document: the route half first, then the |
| 538 |
|
| * markdown mounts' pages, then each section in declaration order. |
| 539 |
|
| * |
| 540 |
|
| * Both halves are sorted within themselves and de-duplicated across the |
| 541 |
|
| * whole, so a page that is both a static route and a section row appears |
| 542 |
|
| * once - a duplicated <url> is something crawlers dislike and nobody meant. |
| 543 |
|
| * Determinism survives the dynamic half this way: a section returning rows in |
| 544 |
|
| * database order would otherwise make every regeneration a different file. */ |
| 545 |
36 |
| static AV *pks_entries(pTHX_ SV *appsv) { |
| 546 |
36 |
| HV *app = app_hv(aTHX_ appsv); |
| 547 |
36 |
| AV *paths = (AV *)sv_2mortal((SV *)pks_paths(aTHX_ appsv)); |
| 548 |
36 |
| AV *out = newAV(); |
| 549 |
36 |
| HV *seen = (HV *)sv_2mortal((SV *)newHV()); |
| 550 |
|
| SV *secs; |
| 551 |
|
| SSize_t i, n; |
| 552 |
|
| |
| 553 |
36 |
| n = av_len(paths) + 1; |
| 554 |
60077 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
60041 |
36 |
60077 |
|
| 555 |
60041 |
| SV **e = av_fetch(paths, i, 0); |
| 556 |
|
| AV *pair; |
| 557 |
|
| STRLEN pl; |
| 558 |
|
| const char *p; |
| 559 |
60041 |
▶ | if (!(e && *e)) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!(e && *e)) continue; |
60041 |
0 |
60041 |
| if (!(e && *e)) continue; |
0 |
60041 |
60041 |
|
| 560 |
60041 |
| p = SvPV_const(*e, pl); |
| 561 |
60041 |
▶ | if (hv_exists(seen, p, (I32)pl)) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (hv_exists(seen, p, (I32)pl)) continue; |
0 |
60041 |
60041 |
|
| 562 |
60041 |
| (void)hv_store(seen, p, (I32)pl, newSViv(1), 0); |
| 563 |
60041 |
| pair = newAV(); |
| 564 |
60041 |
| av_push(pair, newSVsv(*e)); |
| 565 |
60041 |
| av_push(pair, newSV(0)); |
| 566 |
60041 |
| av_push(out, newRV_noinc((SV *)pair)); |
| 567 |
|
| } |
| 568 |
|
| |
| 569 |
|
| { |
| 570 |
36 |
| AV *md = (AV *)sv_2mortal((SV *)pks_md_entries(aTHX_ appsv)); |
| 571 |
36 |
| n = av_len(md) + 1; |
| 572 |
52 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
16 |
36 |
52 |
|
| 573 |
16 |
| SV **g = av_fetch(md, i, 0); |
| 574 |
|
| STRLEN pl; |
| 575 |
|
| const char *p; |
| 576 |
16 |
▶ | if (!(g && *g && SvROK(*g))) continue; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(g && *g && SvROK(*g))) continue; |
16 |
0 |
16 |
| if (!(g && *g && SvROK(*g))) continue; |
16 |
0 |
16 |
| if (!(g && *g && SvROK(*g))) continue; |
0 |
16 |
16 |
|
| 577 |
16 |
| p = SvPV_const(*av_fetch((AV *)SvRV(*g), 0, 0), pl); |
| 578 |
16 |
▶ | if (hv_exists(seen, p, (I32)pl)) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (hv_exists(seen, p, (I32)pl)) continue; |
0 |
16 |
16 |
|
| 579 |
16 |
| (void)hv_store(seen, p, (I32)pl, newSViv(1), 0); |
| 580 |
16 |
| av_push(out, newSVsv(*g)); |
| 581 |
|
| } |
| 582 |
|
| } |
| 583 |
|
| |
| 584 |
36 |
▶ | secs = app ? app_get(aTHX_ app, "sitemap_sections") : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| secs = app ? app_get(aTHX_ app, "sitemap_sections") : NULL; |
36 |
0 |
36 |
|
| 585 |
36 |
▶ | if (secs && SvROK(secs) && SvTYPE(SvRV(secs)) == SVt_PVAV) { |
Branch: 3 branches
| Branch | True | False | Total |
| if (secs && SvROK(secs) && SvTYPE(SvRV(secs)) == SVt_PVAV) { |
9 |
27 |
36 |
| if (secs && SvROK(secs) && SvTYPE(SvRV(secs)) == SVt_PVAV) { |
9 |
0 |
9 |
| if (secs && SvROK(secs) && SvTYPE(SvRV(secs)) == SVt_PVAV) { |
9 |
0 |
9 |
|
| 586 |
9 |
| AV *list = (AV *)SvRV(secs); |
| 587 |
9 |
| SSize_t si, sn = av_len(list) + 1; |
| 588 |
19 |
▶ | for (si = 0; si < sn; si++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (si = 0; si < sn; si++) { |
10 |
9 |
19 |
|
| 589 |
10 |
| SV **se = av_fetch(list, si, 0); |
| 590 |
|
| AV *rec; |
| 591 |
|
| AV *got; |
| 592 |
|
| SSize_t gi, gn; |
| 593 |
10 |
▶ | if (!(se && *se && SvROK(*se) && SvTYPE(SvRV(*se)) == SVt_PVAV)) |
Branch: 4 branches
| Branch | True | False | Total |
| if (!(se && *se && SvROK(*se) && SvTYPE(SvRV(*se)) == SVt_PVAV)) |
10 |
0 |
10 |
| if (!(se && *se && SvROK(*se) && SvTYPE(SvRV(*se)) == SVt_PVAV)) |
10 |
0 |
10 |
| if (!(se && *se && SvROK(*se) && SvTYPE(SvRV(*se)) == SVt_PVAV)) |
10 |
0 |
10 |
| if (!(se && *se && SvROK(*se) && SvTYPE(SvRV(*se)) == SVt_PVAV)) |
0 |
10 |
10 |
|
| 594 |
0 |
| continue; |
| 595 |
10 |
| rec = (AV *)SvRV(*se); |
| 596 |
10 |
| got = (AV *)sv_2mortal((SV *)newAV()); |
| 597 |
10 |
| pks_run_section(aTHX_ *av_fetch(rec, 1, 0), |
| 598 |
10 |
| *av_fetch(rec, 0, 0), got); |
| 599 |
10 |
▶ | if (av_len(got) > 0) |
Branch: 1 branch
| Branch | True | False | Total |
| if (av_len(got) > 0) |
4 |
6 |
10 |
|
| 600 |
4 |
| sortsv(AvARRAY(got), (size_t)(av_len(got) + 1), pks_cmp_pair); |
| 601 |
10 |
| gn = av_len(got) + 1; |
| 602 |
24 |
▶ | for (gi = 0; gi < gn; gi++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (gi = 0; gi < gn; gi++) { |
14 |
10 |
24 |
|
| 603 |
14 |
| SV **g = av_fetch(got, gi, 0); |
| 604 |
|
| AV *pair; |
| 605 |
|
| STRLEN pl; |
| 606 |
|
| const char *p; |
| 607 |
15 |
▶ | if (!(g && *g && SvROK(*g))) continue; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(g && *g && SvROK(*g))) continue; |
14 |
0 |
14 |
| if (!(g && *g && SvROK(*g))) continue; |
14 |
0 |
14 |
| if (!(g && *g && SvROK(*g))) continue; |
0 |
14 |
14 |
|
| 608 |
14 |
| pair = (AV *)SvRV(*g); |
| 609 |
14 |
| p = SvPV_const(*av_fetch(pair, 0, 0), pl); |
| 610 |
14 |
▶ | if (hv_exists(seen, p, (I32)pl)) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (hv_exists(seen, p, (I32)pl)) continue; |
1 |
13 |
14 |
|
| 611 |
13 |
| (void)hv_store(seen, p, (I32)pl, newSViv(1), 0); |
| 612 |
13 |
| av_push(out, newSVsv(*g)); |
| 613 |
|
| } |
| 614 |
|
| } |
| 615 |
|
| } |
| 616 |
36 |
| return out; |
| 617 |
|
| } |
| 618 |
|
| |
| 619 |
|
| #define PKS_HEAD "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" \ |
| 620 |
|
| "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n" |
| 621 |
|
| #define PKS_TAIL "</urlset>\n" |
| 622 |
|
| #define PKS_IHEAD "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" \ |
| 623 |
|
| "<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n" |
| 624 |
|
| #define PKS_ITAIL "</sitemapindex>\n" |
| 625 |
|
| |
| 626 |
|
| /* Render every candidate path into one or more <urlset> documents (+1 each). |
| 627 |
|
| * |
| 628 |
|
| * No <lastmod> on this half: "when did this static page last change" is not |
| 629 |
|
| * something the route table knows, and stamping boot time would tell crawlers |
| 630 |
|
| * every page changed on every deploy - the fastest way to have the file |
| 631 |
|
| * ignored. The dynamic half supplies its own, where the application does |
| 632 |
|
| * know. */ |
| 633 |
38 |
| static AV *pks_render_parts(pTHX_ AV *ents, SV *base) { |
| 634 |
38 |
| AV *parts = newAV(); |
| 635 |
38 |
| SSize_t i, n = av_len(ents) + 1; |
| 636 |
38 |
| SV *cur = NULL; |
| 637 |
38 |
| IV urls = 0; |
| 638 |
|
| |
| 639 |
60116 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
60078 |
38 |
60116 |
|
| 640 |
60078 |
| SV **e = av_fetch(ents, i, 0); |
| 641 |
|
| AV *pair; |
| 642 |
|
| SV *lm; |
| 643 |
|
| STRLEN pl; |
| 644 |
|
| const char *p; |
| 645 |
120118 |
▶ | if (!(e && *e && SvROK(*e))) continue; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(e && *e && SvROK(*e))) continue; |
60078 |
0 |
60078 |
| if (!(e && *e && SvROK(*e))) continue; |
60078 |
0 |
60078 |
| if (!(e && *e && SvROK(*e))) continue; |
0 |
60078 |
60078 |
|
| 646 |
60078 |
| pair = (AV *)SvRV(*e); |
| 647 |
60078 |
| p = SvPV_const(*av_fetch(pair, 0, 0), pl); |
| 648 |
60078 |
| lm = *av_fetch(pair, 1, 0); |
| 649 |
|
| |
| 650 |
60078 |
▶ | if (cur) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (cur) { |
60041 |
37 |
60078 |
|
| 651 |
|
| /* Would this one take the part past either cap? The URL is |
| 652 |
|
| * rendered into a scratch buffer so the byte cost is known |
| 653 |
|
| * rather than guessed. */ |
| 654 |
60041 |
| SV *probe = sv_2mortal(newSVpvn("", 0)); |
| 655 |
60041 |
| pks_url_cat_lm(aTHX_ probe, base, p, pl, lm); |
| 656 |
60041 |
▶ | if (urls + 1 > PKS_MAX_URLS |
Branch: 1 branch
| Branch | True | False | Total |
| if (urls + 1 > PKS_MAX_URLS |
60040 |
1 |
60041 |
|
| 657 |
60040 |
▶ | || SvCUR(cur) + SvCUR(probe) + (sizeof(PKS_TAIL) - 1) |
Branch: 1 branch
| Branch | True | False | Total |
| || SvCUR(cur) + SvCUR(probe) + (sizeof(PKS_TAIL) - 1) |
0 |
60040 |
60040 |
|
| 658 |
|
| > PKS_MAX_BYTES) { |
| 659 |
1 |
| sv_catpvs(cur, PKS_TAIL); |
| 660 |
1 |
| av_push(parts, cur); |
| 661 |
1 |
| cur = NULL; |
| 662 |
1 |
| urls = 0; |
| 663 |
|
| } |
| 664 |
|
| else { |
| 665 |
60040 |
| sv_catsv(cur, probe); |
| 666 |
60040 |
| urls++; |
| 667 |
60040 |
| continue; |
| 668 |
|
| } |
| 669 |
|
| } |
| 670 |
38 |
▶ | if (!cur) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (!cur) { |
38 |
0 |
38 |
|
| 671 |
38 |
| cur = newSVpvn(PKS_HEAD, sizeof(PKS_HEAD) - 1); |
| 672 |
38 |
| urls = 0; |
| 673 |
|
| } |
| 674 |
38 |
| pks_url_cat_lm(aTHX_ cur, base, p, pl, lm); |
| 675 |
38 |
| urls++; |
| 676 |
|
| } |
| 677 |
|
| |
| 678 |
38 |
▶ | if (cur) { sv_catpvs(cur, PKS_TAIL); av_push(parts, cur); } |
Branch: 1 branch
| Branch | True | False | Total |
| if (cur) { sv_catpvs(cur, PKS_TAIL); av_push(parts, cur); } |
37 |
1 |
38 |
|
| 679 |
|
| /* An application with no listable route still serves a valid, empty |
| 680 |
|
| * document rather than a 404 - a crawler asking gets an answer, and an |
| 681 |
|
| * empty urlset is a legitimate one. */ |
| 682 |
38 |
▶ | if (av_len(parts) < 0) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (av_len(parts) < 0) { |
1 |
37 |
38 |
|
| 683 |
1 |
| SV *empty = newSVpvn(PKS_HEAD, sizeof(PKS_HEAD) - 1); |
| 684 |
1 |
| sv_catpvs(empty, PKS_TAIL); |
| 685 |
1 |
| av_push(parts, empty); |
| 686 |
|
| } |
| 687 |
38 |
| return parts; |
| 688 |
|
| } |
| 689 |
|
| |
| 690 |
|
| /* The index over the parts (+1), when there is more than one. |
| 691 |
|
| * |
| 692 |
|
| * With one part there is no index and /sitemap.xml is that part: an index |
| 693 |
|
| * wrapping a single sitemap is a second fetch for nothing. */ |
| 694 |
1 |
| static SV *pks_render_index(pTHX_ SV *base, SSize_t nparts) { |
| 695 |
1 |
| SV *out = newSVpvn(PKS_IHEAD, sizeof(PKS_IHEAD) - 1); |
| 696 |
|
| STRLEN bl; |
| 697 |
1 |
| const char *bp = SvPV_const(base, bl); |
| 698 |
|
| SSize_t i; |
| 699 |
1 |
▶ | while (bl && bp[bl - 1] == '/') bl--; |
Branch: 2 branches
| Branch | True | False | Total |
| while (bl && bp[bl - 1] == '/') bl--; |
1 |
0 |
1 |
| while (bl && bp[bl - 1] == '/') bl--; |
0 |
1 |
1 |
|
| 700 |
3 |
▶ | for (i = 0; i < nparts; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < nparts; i++) { |
2 |
1 |
3 |
|
| 701 |
2 |
| SV *loc = sv_2mortal(newSVpvn(bp, bl)); |
| 702 |
2 |
| sv_catpvf(loc, "/sitemap/%" IVdf ".xml", (IV)(i + 1)); |
| 703 |
2 |
| sv_catpvs(out, " <sitemap><loc>"); |
| 704 |
|
| { |
| 705 |
|
| STRLEN ll; |
| 706 |
2 |
| const char *lp = SvPV_const(loc, ll); |
| 707 |
2 |
| pks_xml_cat(aTHX_ out, lp, ll); |
| 708 |
|
| } |
| 709 |
2 |
| sv_catpvs(out, "</loc></sitemap>\n"); |
| 710 |
|
| } |
| 711 |
1 |
| sv_catpvs(out, PKS_ITAIL); |
| 712 |
1 |
| return out; |
| 713 |
|
| } |
| 714 |
|
| |
| 715 |
|
| /* Render everything and stash it on the app: "sitemap_parts" is the AV of |
| 716 |
|
| * <urlset> documents, "sitemap_index" the index when there is more than one. |
| 717 |
|
| * |
| 718 |
|
| * Once, at to_app. The route half changes only when the routes do, which is |
| 719 |
|
| * never while the process runs, so a request for the document is a write of |
| 720 |
|
| * bytes that already exist. */ |
| 721 |
36 |
| static void pks_build(pTHX_ SV *appsv) { |
| 722 |
36 |
| HV *h = app_hv(aTHX_ appsv); |
| 723 |
36 |
▶ | SV *base = h ? app_get(aTHX_ h, "sitemap_base") : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *base = h ? app_get(aTHX_ h, "sitemap_base") : NULL; |
36 |
0 |
36 |
|
| 724 |
|
| AV *ents, *parts; |
| 725 |
36 |
▶ | if (!(h && base && SvOK(base))) return; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(h && base && SvOK(base))) return; |
36 |
0 |
36 |
| if (!(h && base && SvOK(base))) return; |
36 |
0 |
36 |
| if (!(h && base && SvOK(base))) return; |
0 |
36 |
36 |
|
| 726 |
|
| /* The entries are kept as well as the rendered parts: a request from an |
| 727 |
|
| * allowlisted tenant host renders a document naming itself from them |
| 728 |
|
| * (pks_doc_for), so the sections run once per TTL however many hosts |
| 729 |
|
| * ask. */ |
| 730 |
36 |
| ents = pks_entries(aTHX_ appsv); |
| 731 |
36 |
| (void)hv_stores(h, "sitemap_entries", newRV_noinc((SV *)ents)); |
| 732 |
36 |
| parts = pks_render_parts(aTHX_ ents, base); |
| 733 |
36 |
| (void)hv_stores(h, "sitemap_parts", newRV_noinc((SV *)parts)); |
| 734 |
36 |
▶ | (void)hv_stores(h, "sitemap_index", |
Branch: 1 branch
| Branch | True | False | Total |
| (void)hv_stores(h, "sitemap_index", |
1 |
35 |
36 |
|
| 735 |
|
| av_len(parts) > 0 |
| 736 |
|
| ? pks_render_index(aTHX_ base, av_len(parts) + 1) |
| 737 |
|
| : newSV(0)); |
| 738 |
36 |
| (void)hv_stores(h, "sitemap_built_at", newSVnv(pc_now(aTHX))); |
| 739 |
|
| } |
| 740 |
|
| |
| 741 |
|
| /* Does the document need rebuilding before it is served? |
| 742 |
|
| * |
| 743 |
|
| * The route half never does - routes cannot change while the process runs - |
| 744 |
|
| * so an application with no sections is built at to_app, served from memory |
| 745 |
|
| * and never rebuilt. |
| 746 |
|
| * |
| 747 |
|
| * A section reads a database, so its answer changes underneath us. Running it |
| 748 |
|
| * per request would be a query nobody is watching, on a schedule somebody |
| 749 |
|
| * else chooses; running it once at boot would produce a sitemap that is |
| 750 |
|
| * correct on the day of the deploy and progressively wrong afterwards. So it |
| 751 |
|
| * is rebuilt when the TTL has passed, and the document is stale by up to that |
| 752 |
|
| * long. |
| 753 |
|
| * |
| 754 |
|
| * That staleness is fine and is worth stating rather than engineering away: a |
| 755 |
|
| * crawler learning about a page an hour late is a crawler behaving normally, |
| 756 |
|
| * and no search engine promises to fetch a sitemap promptly anyway. |
| 757 |
|
| * |
| 758 |
|
| * There is no stampede to guard against inside a worker. Hyperman is |
| 759 |
|
| * single-threaded and a worker serves its requests one after another, so the |
| 760 |
|
| * second of two simultaneous crawler requests finds what the first built. The |
| 761 |
|
| * cost across a pool is one rebuild per worker per TTL. */ |
| 762 |
32 |
| static void pks_ensure(pTHX_ SV *appsv) { |
| 763 |
32 |
| HV *h = app_hv(aTHX_ appsv); |
| 764 |
|
| SV *secs, *at, *ttl; |
| 765 |
32 |
▶ | if (!h) return; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!h) return; |
0 |
32 |
32 |
|
| 766 |
32 |
| secs = app_get(aTHX_ h, "sitemap_sections"); |
| 767 |
32 |
▶ | if (!(secs && SvROK(secs) && av_len((AV *)SvRV(secs)) >= 0)) return; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(secs && SvROK(secs) && av_len((AV *)SvRV(secs)) >= 0)) return; |
15 |
17 |
32 |
| if (!(secs && SvROK(secs) && av_len((AV *)SvRV(secs)) >= 0)) return; |
15 |
0 |
15 |
| if (!(secs && SvROK(secs) && av_len((AV *)SvRV(secs)) >= 0)) return; |
0 |
15 |
15 |
|
| 768 |
|
| |
| 769 |
15 |
| at = app_get(aTHX_ h, "sitemap_built_at"); |
| 770 |
15 |
| ttl = app_get(aTHX_ h, "sitemap_ttl"); |
| 771 |
15 |
▶ | if (at && SvOK(at) && ttl && SvOK(ttl) |
Branch: 4 branches
| Branch | True | False | Total |
| if (at && SvOK(at) && ttl && SvOK(ttl) |
15 |
0 |
15 |
| if (at && SvOK(at) && ttl && SvOK(ttl) |
15 |
0 |
15 |
| if (at && SvOK(at) && ttl && SvOK(ttl) |
15 |
0 |
15 |
| if (at && SvOK(at) && ttl && SvOK(ttl) |
15 |
0 |
15 |
|
| 772 |
15 |
▶ | && pc_now(aTHX) - SvNV(at) < SvNV(ttl)) return; |
Branch: 1 branch
| Branch | True | False | Total |
| && pc_now(aTHX) - SvNV(at) < SvNV(ttl)) return; |
14 |
1 |
15 |
|
| 773 |
1 |
| pks_build(aTHX_ appsv); |
| 774 |
|
| } |
| 775 |
|
| |
| 776 |
|
| /* The document for another origin - a tenant host on the allowlist - |
| 777 |
|
| * rendered from the entries the last build kept. Costs the XML emission and |
| 778 |
|
| * no section run, and is NOT cached: a wildcard allow makes the set of hosts |
| 779 |
|
| * unbounded, and a cache keyed by hostnames a client chooses is a memory |
| 780 |
|
| * leak with a name. Mortal, or NULL when nothing has been built. */ |
| 781 |
2 |
| static SV *pks_doc_for(pTHX_ HV *h, SV *base, int is_part, IV n) { |
| 782 |
2 |
| SV *es = app_get(aTHX_ h, "sitemap_entries"); |
| 783 |
|
| AV *parts; |
| 784 |
2 |
| SV *doc = NULL; |
| 785 |
2 |
▶ | if (!(es && SvROK(es) && SvTYPE(SvRV(es)) == SVt_PVAV)) return NULL; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(es && SvROK(es) && SvTYPE(SvRV(es)) == SVt_PVAV)) return NULL; |
2 |
0 |
2 |
| if (!(es && SvROK(es) && SvTYPE(SvRV(es)) == SVt_PVAV)) return NULL; |
2 |
0 |
2 |
| if (!(es && SvROK(es) && SvTYPE(SvRV(es)) == SVt_PVAV)) return NULL; |
0 |
2 |
2 |
|
| 786 |
2 |
| parts = (AV *)sv_2mortal((SV *)pks_render_parts(aTHX_ (AV *)SvRV(es), |
| 787 |
|
| base)); |
| 788 |
2 |
▶ | if (!is_part) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (!is_part) { |
2 |
0 |
2 |
|
| 789 |
2 |
▶ | if (av_len(parts) > 0) |
Branch: 1 branch
| Branch | True | False | Total |
| if (av_len(parts) > 0) |
0 |
2 |
2 |
|
| 790 |
0 |
| return sv_2mortal(pks_render_index(aTHX_ base, av_len(parts) + 1)); |
| 791 |
2 |
| n = 1; |
| 792 |
|
| } |
| 793 |
2 |
▶ | if (n > 0) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (n > 0) { |
2 |
0 |
2 |
|
| 794 |
2 |
| SV **e = av_fetch(parts, (SSize_t)(n - 1), 0); |
| 795 |
2 |
▶ | if (e && *e) doc = sv_2mortal(newSVsv(*e)); |
Branch: 2 branches
| Branch | True | False | Total |
| if (e && *e) doc = sv_2mortal(newSVsv(*e)); |
2 |
0 |
2 |
| if (e && *e) doc = sv_2mortal(newSVsv(*e)); |
2 |
0 |
2 |
|
| 796 |
|
| } |
| 797 |
2 |
| return doc; |
| 798 |
|
| } |
| 799 |
|
| |
| 800 |
|
| /* Serve a rendered document. cap = [app, is_part]. |
| 801 |
|
| * |
| 802 |
|
| * The bytes already exist - the whole document was built at to_app - so a |
| 803 |
|
| * request is a write of something in memory and not a render. */ |
| 804 |
|
| XS_INTERNAL(pks_serve_cb); |
| 805 |
7 |
| XS_INTERNAL(pks_serve_cb) { |
| 806 |
7 |
| dXSARGS; |
| 807 |
7 |
| AV *cap = punk_clos_cap(aTHX_ cv); |
| 808 |
7 |
▶ | SV *app = cap ? *av_fetch(cap, 0, 0) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *app = cap ? *av_fetch(cap, 0, 0) : NULL; |
7 |
0 |
7 |
|
| 809 |
7 |
▶ | int is_part = cap ? (int)SvIV(*av_fetch(cap, 1, 0)) : 0; |
Branch: 1 branch
| Branch | True | False | Total |
| int is_part = cap ? (int)SvIV(*av_fetch(cap, 1, 0)) : 0; |
7 |
0 |
7 |
|
| 810 |
7 |
▶ | SV *c = items > 0 ? ST(0) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *c = items > 0 ? ST(0) : NULL; |
7 |
0 |
7 |
|
| 811 |
7 |
▶ | HV *h = app ? app_hv(aTHX_ app) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| HV *h = app ? app_hv(aTHX_ app) : NULL; |
7 |
0 |
7 |
|
| 812 |
7 |
| IV n = 0; |
| 813 |
7 |
| SV *doc = NULL; |
| 814 |
|
| AV *resp, *headers, *body; |
| 815 |
|
| |
| 816 |
7 |
▶ | if (!h) XSRETURN_EMPTY; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!h) XSRETURN_EMPTY; |
0 |
7 |
7 |
|
| 817 |
7 |
| pks_ensure(aTHX_ app); /* a section may have gone stale since the last hit */ |
| 818 |
|
| |
| 819 |
7 |
▶ | if (is_part && c) { |
Branch: 2 branches
| Branch | True | False | Total |
| if (is_part && c) { |
2 |
5 |
7 |
| if (is_part && c) { |
2 |
0 |
2 |
|
| 820 |
|
| /* Punk captures whole SEGMENTS, so the pattern is /sitemap/:n and the |
| 821 |
|
| * capture arrives as "1.xml" rather than "1". Parsed rather than |
| 822 |
|
| * handed to SvIV, which would take "9evil.xml" as 9 and serve a part |
| 823 |
|
| * for a URL nobody published. */ |
| 824 |
2 |
| SV *arg = sv_2mortal(newSVpvs("n")); |
| 825 |
2 |
| SV *v = pcx_call_meth(aTHX_ c, "param", &arg, 1, 1); |
| 826 |
2 |
▶ | if (v) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (v) { |
2 |
0 |
2 |
|
| 827 |
2 |
▶ | if (SvOK(v)) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (SvOK(v)) { |
2 |
0 |
2 |
|
| 828 |
|
| STRLEN vl; |
| 829 |
2 |
| const char *vp = SvPV_const(v, vl); |
| 830 |
2 |
| STRLEN d = 0; |
| 831 |
4 |
▶ | while (d < vl && isDIGIT(vp[d])) d++; |
Branch: 2 branches
| Branch | True | False | Total |
| while (d < vl && isDIGIT(vp[d])) d++; |
4 |
0 |
4 |
| while (d < vl && isDIGIT(vp[d])) d++; |
2 |
2 |
4 |
|
| 832 |
|
| /* at most nine digits: a longer number overflows the |
| 833 |
|
| * arithmetic below, and no sitemap has a billion parts */ |
| 834 |
2 |
▶ | if (d > 0 && d <= 9 && d < vl && vl - d == 4 |
Branch: 4 branches
| Branch | True | False | Total |
| if (d > 0 && d <= 9 && d < vl && vl - d == 4 |
2 |
0 |
2 |
| if (d > 0 && d <= 9 && d < vl && vl - d == 4 |
2 |
0 |
2 |
| if (d > 0 && d <= 9 && d < vl && vl - d == 4 |
2 |
0 |
2 |
| if (d > 0 && d <= 9 && d < vl && vl - d == 4 |
2 |
0 |
2 |
|
| 835 |
2 |
▶ | && memEQ(vp + d, ".xml", 4)) { |
Branch: 1 branch
| Branch | True | False | Total |
| && memEQ(vp + d, ".xml", 4)) { |
2 |
0 |
2 |
|
| 836 |
|
| STRLEN k; |
| 837 |
4 |
▶ | for (k = 0; k < d; k++) n = n * 10 + (vp[k] - '0'); |
Branch: 1 branch
| Branch | True | False | Total |
| for (k = 0; k < d; k++) n = n * 10 + (vp[k] - '0'); |
2 |
2 |
4 |
|
| 838 |
|
| } |
| 839 |
|
| } |
| 840 |
2 |
| SvREFCNT_dec(v); |
| 841 |
|
| } |
| 842 |
|
| } |
| 843 |
|
| |
| 844 |
|
| { /* A tenant host on the allowlist gets a document naming itself; |
| 845 |
|
| * the canonical host, an unknown one and a malformed one all get |
| 846 |
|
| * the frozen canonical document - never one naming the header. */ |
| 847 |
7 |
| int st = 0; |
| 848 |
7 |
▶ | SV *origin = c ? pk_origin_of(aTHX_ c, &st) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *origin = c ? pk_origin_of(aTHX_ c, &st) : NULL; |
7 |
0 |
7 |
|
| 849 |
7 |
▶ | if (origin) sv_2mortal(origin); |
Branch: 1 branch
| Branch | True | False | Total |
| if (origin) sv_2mortal(origin); |
4 |
3 |
7 |
|
| 850 |
7 |
▶ | if (st == 2) doc = pks_doc_for(aTHX_ h, origin, is_part, n); |
Branch: 1 branch
| Branch | True | False | Total |
| if (st == 2) doc = pks_doc_for(aTHX_ h, origin, is_part, n); |
2 |
5 |
7 |
|
| 851 |
|
| else { |
| 852 |
5 |
▶ | if (!is_part) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (!is_part) { |
3 |
2 |
5 |
|
| 853 |
3 |
| doc = app_get(aTHX_ h, "sitemap_index"); |
| 854 |
3 |
▶ | if (!(doc && SvOK(doc))) n = 1; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!(doc && SvOK(doc))) n = 1; |
3 |
0 |
3 |
| if (!(doc && SvOK(doc))) n = 1; |
3 |
0 |
3 |
|
| 855 |
|
| } |
| 856 |
5 |
▶ | if (n > 0) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (n > 0) { |
5 |
0 |
5 |
|
| 857 |
5 |
| SV *ps = app_get(aTHX_ h, "sitemap_parts"); |
| 858 |
5 |
▶ | AV *parts = (ps && SvROK(ps) && SvTYPE(SvRV(ps)) == SVt_PVAV) |
Branch: 2 branches
| Branch | True | False | Total |
| AV *parts = (ps && SvROK(ps) && SvTYPE(SvRV(ps)) == SVt_PVAV) |
5 |
0 |
5 |
| AV *parts = (ps && SvROK(ps) && SvTYPE(SvRV(ps)) == SVt_PVAV) |
5 |
0 |
5 |
|
| 859 |
10 |
▶ | ? (AV *)SvRV(ps) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| ? (AV *)SvRV(ps) : NULL; |
5 |
0 |
5 |
|
| 860 |
5 |
▶ | SV **e = (parts && n >= 1) |
Branch: 1 branch
| Branch | True | False | Total |
| SV **e = (parts && n >= 1) |
5 |
0 |
5 |
|
| 861 |
10 |
▶ | ? av_fetch(parts, (SSize_t)(n - 1), 0) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| ? av_fetch(parts, (SSize_t)(n - 1), 0) : NULL; |
5 |
0 |
5 |
|
| 862 |
5 |
▶ | doc = (e && *e) ? *e : NULL; |
Branch: 2 branches
| Branch | True | False | Total |
| doc = (e && *e) ? *e : NULL; |
4 |
1 |
5 |
| doc = (e && *e) ? *e : NULL; |
4 |
0 |
4 |
|
| 863 |
|
| } |
| 864 |
|
| } |
| 865 |
|
| } |
| 866 |
|
| |
| 867 |
|
| /* A part number nobody generated is a 404 and not an empty document: a |
| 868 |
|
| * crawler following a stale index should be told the part is gone. */ |
| 869 |
7 |
▶ | if (!(doc && SvOK(doc))) { |
Branch: 2 branches
| Branch | True | False | Total |
| if (!(doc && SvOK(doc))) { |
6 |
1 |
7 |
| if (!(doc && SvOK(doc))) { |
0 |
6 |
6 |
|
| 870 |
1 |
| SV *r = pcx_call_meth(aTHX_ c, "not_found", NULL, 0, 1); |
| 871 |
1 |
▶ | if (r) { ST(0) = sv_2mortal(r); XSRETURN(1); } |
Branch: 1 branch
| Branch | True | False | Total |
| if (r) { ST(0) = sv_2mortal(r); XSRETURN(1); } |
1 |
0 |
1 |
|
| 872 |
0 |
| XSRETURN_EMPTY; |
| 873 |
|
| } |
| 874 |
|
| |
| 875 |
6 |
| resp = newAV(); headers = newAV(); body = newAV(); |
| 876 |
6 |
| av_push(headers, newSVpvs("Content-Type")); |
| 877 |
6 |
| av_push(headers, newSVpvs("application/xml; charset=utf-8")); |
| 878 |
6 |
| av_push(headers, newSVpvs("Content-Length")); |
| 879 |
6 |
| av_push(headers, newSViv((IV)SvCUR(doc))); |
| 880 |
6 |
| av_push(body, newSVsv(doc)); |
| 881 |
6 |
| av_push(resp, newSViv(200)); |
| 882 |
6 |
| av_push(resp, newRV_noinc((SV *)headers)); |
| 883 |
6 |
| av_push(resp, newRV_noinc((SV *)body)); |
| 884 |
6 |
| ST(0) = sv_2mortal(newRV_noinc((SV *)resp)); |
| 885 |
6 |
| XSRETURN(1); |
| 886 |
|
| } |
| 887 |
|
| |
| 888 |
|
| static SV *pks_robots(pTHX_ SV *appsv); |
| 889 |
|
| static SV *pks_robots_for(pTHX_ SV *appsv, SV *base); |
| 890 |
|
| |
| 891 |
|
| /* Serve robots.txt. Rebuilt on demand like the sitemap, because a dynamic |
| 892 |
|
| * section cannot change it but a rebuilt document should not disagree with a |
| 893 |
|
| * stale robots either. */ |
| 894 |
|
| XS_INTERNAL(pks_robots_cb); |
| 895 |
3 |
| XS_INTERNAL(pks_robots_cb) { |
| 896 |
3 |
| dXSARGS; |
| 897 |
3 |
| AV *cap = punk_clos_cap(aTHX_ cv); |
| 898 |
3 |
▶ | SV *app = cap ? *av_fetch(cap, 0, 0) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *app = cap ? *av_fetch(cap, 0, 0) : NULL; |
3 |
0 |
3 |
|
| 899 |
3 |
▶ | SV *c = items > 0 ? ST(0) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *c = items > 0 ? ST(0) : NULL; |
3 |
0 |
3 |
|
| 900 |
|
| SV *body; |
| 901 |
|
| AV *resp, *headers, *bodyav; |
| 902 |
|
| |
| 903 |
3 |
▶ | if (!app) XSRETURN_EMPTY; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!app) XSRETURN_EMPTY; |
0 |
3 |
3 |
|
| 904 |
|
| { /* per tenant, like the sitemap it advertises */ |
| 905 |
3 |
| int st = 0; |
| 906 |
3 |
▶ | SV *origin = c ? pk_origin_of(aTHX_ c, &st) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *origin = c ? pk_origin_of(aTHX_ c, &st) : NULL; |
3 |
0 |
3 |
|
| 907 |
3 |
▶ | if (origin) sv_2mortal(origin); |
Branch: 1 branch
| Branch | True | False | Total |
| if (origin) sv_2mortal(origin); |
2 |
1 |
3 |
|
| 908 |
3 |
▶ | body = sv_2mortal(st == 2 ? pks_robots_for(aTHX_ app, origin) |
Branch: 1 branch
| Branch | True | False | Total |
| body = sv_2mortal(st == 2 ? pks_robots_for(aTHX_ app, origin) |
1 |
2 |
3 |
|
| 909 |
|
| : pks_robots(aTHX_ app)); |
| 910 |
|
| } |
| 911 |
|
| |
| 912 |
3 |
| resp = newAV(); headers = newAV(); bodyav = newAV(); |
| 913 |
3 |
| av_push(headers, newSVpvs("Content-Type")); |
| 914 |
3 |
| av_push(headers, newSVpvs("text/plain; charset=utf-8")); |
| 915 |
3 |
| av_push(headers, newSVpvs("Content-Length")); |
| 916 |
3 |
| av_push(headers, newSViv((IV)SvCUR(body))); |
| 917 |
3 |
| av_push(bodyav, newSVsv(body)); |
| 918 |
3 |
| av_push(resp, newSViv(200)); |
| 919 |
3 |
| av_push(resp, newRV_noinc((SV *)headers)); |
| 920 |
3 |
| av_push(resp, newRV_noinc((SV *)bodyav)); |
| 921 |
3 |
| ST(0) = sv_2mortal(newRV_noinc((SV *)resp)); |
| 922 |
3 |
| XSRETURN(1); |
| 923 |
|
| } |
| 924 |
|
| |
| 925 |
|
| /* The `sitemap` keyword. cap = [app]. |
| 926 |
|
| * |
| 927 |
|
| * sitemap 'users' => sub { ... }; |
| 928 |
|
| * |
| 929 |
|
| * NAMED, so a second declaration of the same name REPLACES rather than |
| 930 |
|
| * appends: a base class can declare a section and a subclass override it, and |
| 931 |
|
| * an application reloading its own definition does not end up with the |
| 932 |
|
| * section twice. Declaration order is the emission order, and a replaced |
| 933 |
|
| * section keeps its original position so the document does not reshuffle. */ |
| 934 |
|
| XS_INTERNAL(pks_kw_cb); |
| 935 |
10 |
| XS_INTERNAL(pks_kw_cb) { |
| 936 |
10 |
| dXSARGS; |
| 937 |
10 |
| AV *cap = punk_clos_cap(aTHX_ cv); |
| 938 |
10 |
▶ | SV *app = cap ? *av_fetch(cap, 0, 0) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *app = cap ? *av_fetch(cap, 0, 0) : NULL; |
10 |
0 |
10 |
|
| 939 |
10 |
▶ | HV *h = app ? app_hv(aTHX_ app) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| HV *h = app ? app_hv(aTHX_ app) : NULL; |
10 |
0 |
10 |
|
| 940 |
10 |
▶ | SV *name = items > 0 ? ST(0) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *name = items > 0 ? ST(0) : NULL; |
10 |
0 |
10 |
|
| 941 |
10 |
▶ | SV *code = items > 1 ? ST(1) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *code = items > 1 ? ST(1) : NULL; |
10 |
0 |
10 |
|
| 942 |
|
| AV *list; |
| 943 |
|
| SSize_t i, n; |
| 944 |
|
| |
| 945 |
10 |
▶ | if (!h) XSRETURN_EMPTY; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!h) XSRETURN_EMPTY; |
0 |
10 |
10 |
|
| 946 |
10 |
▶ | if (!(name && SvOK(name) && SvCUR(name))) |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(name && SvOK(name) && SvCUR(name))) |
10 |
0 |
10 |
| if (!(name && SvOK(name) && SvCUR(name))) |
10 |
0 |
10 |
| if (!(name && SvOK(name) && SvCUR(name))) |
0 |
10 |
10 |
|
| 947 |
0 |
| croak("Punk: sitemap needs a name - sitemap 'users' => sub { ... }"); |
| 948 |
10 |
▶ | if (!(code && SvROK(code) && SvTYPE(SvRV(code)) == SVt_PVCV)) |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(code && SvROK(code) && SvTYPE(SvRV(code)) == SVt_PVCV)) |
10 |
0 |
10 |
| if (!(code && SvROK(code) && SvTYPE(SvRV(code)) == SVt_PVCV)) |
10 |
0 |
10 |
| if (!(code && SvROK(code) && SvTYPE(SvRV(code)) == SVt_PVCV)) |
0 |
10 |
10 |
|
| 949 |
0 |
| croak("Punk: sitemap '%" SVf "' needs a code reference that returns " |
| 950 |
|
| "locations", SVfARG(name)); |
| 951 |
|
| |
| 952 |
10 |
| list = app_av(aTHX_ h, "sitemap_sections"); |
| 953 |
10 |
| n = av_len(list) + 1; |
| 954 |
11 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
2 |
9 |
11 |
|
| 955 |
2 |
| SV **e = av_fetch(list, i, 0); |
| 956 |
|
| AV *rec; |
| 957 |
2 |
▶ | if (!(e && *e && SvROK(*e))) continue; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(e && *e && SvROK(*e))) continue; |
2 |
0 |
2 |
| if (!(e && *e && SvROK(*e))) continue; |
2 |
0 |
2 |
| if (!(e && *e && SvROK(*e))) continue; |
0 |
2 |
2 |
|
| 958 |
2 |
| rec = (AV *)SvRV(*e); |
| 959 |
2 |
▶ | if (sv_eq(*av_fetch(rec, 0, 0), name)) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (sv_eq(*av_fetch(rec, 0, 0), name)) { |
1 |
1 |
2 |
|
| 960 |
1 |
| av_store(rec, 1, newSVsv(code)); /* replace, keep position */ |
| 961 |
1 |
| XSRETURN_EMPTY; |
| 962 |
|
| } |
| 963 |
|
| } |
| 964 |
|
| { |
| 965 |
9 |
| AV *rec = newAV(); |
| 966 |
9 |
| av_push(rec, newSVsv(name)); |
| 967 |
9 |
| av_push(rec, newSVsv(code)); |
| 968 |
9 |
| av_push(list, newRV_noinc((SV *)rec)); |
| 969 |
|
| } |
| 970 |
9 |
| XSRETURN_EMPTY; |
| 971 |
|
| } |
| 972 |
|
| |
| 973 |
|
| /* ---- robots.txt ------------------------------------------------------------- |
| 974 |
|
| * |
| 975 |
|
| * The same decision, made once, spelled twice. A sitemap says "crawl these" |
| 976 |
|
| * and robots.txt says "do not crawl those"; written apart they drift within a |
| 977 |
|
| * release, and the drift is silent in both directions - a path disallowed but |
| 978 |
|
| * listed is a crawler told two things, and a path excluded but not disallowed |
| 979 |
|
| * is crawled anyway, which is how a staging environment gets indexed. |
| 980 |
|
| * |
| 981 |
|
| * So both come out of one pass over the same records. |
| 982 |
|
| * |
| 983 |
|
| * ROBOTS.TXT IS NOT ACCESS CONTROL. It is a request to well-behaved crawlers, |
| 984 |
|
| * ignored by everything else, and a Disallow line is a signpost to anything |
| 985 |
|
| * hostile - a published list of the paths worth attacking. The guarding is |
| 986 |
|
| * auth_guard's job; this only tells Google not to bother. |
| 987 |
|
| */ |
| 988 |
|
| |
| 989 |
|
| /* Is this one of the routes the plugin registered for itself? */ |
| 990 |
32 |
| static int pks_is_own(pTHX_ HV *app, const char *p, STRLEN pl) { |
| 991 |
32 |
| SV *own = app_get(aTHX_ app, "sitemap_own"); |
| 992 |
|
| AV *av; |
| 993 |
|
| SSize_t i, n; |
| 994 |
32 |
▶ | if (!(own && SvROK(own) && SvTYPE(SvRV(own)) == SVt_PVAV)) return 0; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(own && SvROK(own) && SvTYPE(SvRV(own)) == SVt_PVAV)) return 0; |
32 |
0 |
32 |
| if (!(own && SvROK(own) && SvTYPE(SvRV(own)) == SVt_PVAV)) return 0; |
32 |
0 |
32 |
| if (!(own && SvROK(own) && SvTYPE(SvRV(own)) == SVt_PVAV)) return 0; |
0 |
32 |
32 |
|
| 995 |
32 |
| av = (AV *)SvRV(own); |
| 996 |
32 |
| n = av_len(av) + 1; |
| 997 |
84 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
74 |
10 |
84 |
|
| 998 |
74 |
| SV **e = av_fetch(av, i, 0); |
| 999 |
|
| STRLEN ol; |
| 1000 |
|
| const char *op; |
| 1001 |
74 |
▶ | if (!(e && *e)) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!(e && *e)) continue; |
74 |
0 |
74 |
| if (!(e && *e)) continue; |
0 |
74 |
74 |
|
| 1002 |
74 |
| op = SvPV_const(*e, ol); |
| 1003 |
74 |
▶ | if (ol == pl && memEQ(op, p, pl)) return 1; |
Branch: 2 branches
| Branch | True | False | Total |
| if (ol == pl && memEQ(op, p, pl)) return 1; |
35 |
39 |
74 |
| if (ol == pl && memEQ(op, p, pl)) return 1; |
22 |
13 |
35 |
|
| 1004 |
|
| } |
| 1005 |
10 |
| return 0; |
| 1006 |
|
| } |
| 1007 |
|
| |
| 1008 |
|
| /* How much of a path to disallow. |
| 1009 |
|
| * |
| 1010 |
|
| * A capture makes the rest of the path meaningless as a literal - nobody |
| 1011 |
|
| * benefits from `Disallow: /account/orders/:id` - so the path is truncated at |
| 1012 |
|
| * the first capture segment and the prefix stands for everything under it, |
| 1013 |
|
| * which is exactly the set the route matches. `/account/orders/:id` becomes |
| 1014 |
|
| * `/account/orders/`, and `/account/settings` stays as it is. */ |
| 1015 |
10 |
| static SV *pks_disallow_form(pTHX_ const char *p, STRLEN pl) { |
| 1016 |
10 |
| STRLEN i, cut = pl; |
| 1017 |
118 |
▶ | for (i = 0; i < pl; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < pl; i++) { |
110 |
8 |
118 |
|
| 1018 |
110 |
▶ | if (p[i] == ':' || p[i] == '*') { |
Branch: 2 branches
| Branch | True | False | Total |
| if (p[i] == ':' || p[i] == '*') { |
108 |
2 |
110 |
| if (p[i] == ':' || p[i] == '*') { |
0 |
108 |
108 |
|
| 1019 |
|
| /* back up to the segment boundary, keeping the slash */ |
| 1020 |
2 |
| cut = i; |
| 1021 |
2 |
▶ | while (cut > 1 && p[cut - 1] != '/') cut--; |
Branch: 2 branches
| Branch | True | False | Total |
| while (cut > 1 && p[cut - 1] != '/') cut--; |
2 |
0 |
2 |
| while (cut > 1 && p[cut - 1] != '/') cut--; |
0 |
2 |
2 |
|
| 1022 |
2 |
| break; |
| 1023 |
|
| } |
| 1024 |
|
| } |
| 1025 |
10 |
| return newSVpvn(p, cut); |
| 1026 |
|
| } |
| 1027 |
|
| |
| 1028 |
|
| /* Does `pfx` cover `path`? A robots prefix match: everything starting with |
| 1029 |
|
| * it. */ |
| 1030 |
38 |
| static int pks_covers(const char *pfx, STRLEN fl, const char *path, STRLEN pl) { |
| 1031 |
38 |
▶ | if (fl > pl) return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (fl > pl) return 0; |
30 |
8 |
38 |
|
| 1032 |
8 |
| return memEQ(pfx, path, fl); |
| 1033 |
|
| } |
| 1034 |
|
| |
| 1035 |
|
| /* Every prefix this application asks crawlers to leave alone (+1). |
| 1036 |
|
| * |
| 1037 |
|
| * Guarded routes and routes that said `sitemap => 0`, plus whatever the |
| 1038 |
|
| * `disallow` option added - then filtered against what the SITEMAP lists, so |
| 1039 |
|
| * a Disallow can never shadow a URL the same file is advertising. That filter |
| 1040 |
|
| * is the whole "they cannot disagree" property, and it is cheaper to enforce |
| 1041 |
|
| * than to test for afterwards. */ |
| 1042 |
11 |
| static AV *pks_disallows(pTHX_ SV *appsv) { |
| 1043 |
11 |
| HV *app = app_hv(aTHX_ appsv); |
| 1044 |
11 |
| AV *listed = (AV *)sv_2mortal((SV *)pks_paths(aTHX_ appsv)); |
| 1045 |
11 |
▶ | AV *raw = app ? pks_raw(aTHX_ app) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| AV *raw = app ? pks_raw(aTHX_ app) : NULL; |
11 |
0 |
11 |
|
| 1046 |
|
| { /* a markdown page the sitemap lists is listed, as a route is */ |
| 1047 |
11 |
| AV *md = (AV *)sv_2mortal((SV *)pks_md_entries(aTHX_ appsv)); |
| 1048 |
11 |
| SSize_t k, kn = av_len(md) + 1; |
| 1049 |
21 |
▶ | for (k = 0; k < kn; k++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (k = 0; k < kn; k++) { |
10 |
11 |
21 |
|
| 1050 |
10 |
| SV **g = av_fetch(md, k, 0); |
| 1051 |
10 |
▶ | if (g && *g && SvROK(*g)) |
Branch: 3 branches
| Branch | True | False | Total |
| if (g && *g && SvROK(*g)) |
10 |
0 |
10 |
| if (g && *g && SvROK(*g)) |
10 |
0 |
10 |
| if (g && *g && SvROK(*g)) |
10 |
0 |
10 |
|
| 1052 |
10 |
| av_push(listed, newSVsv(*av_fetch((AV *)SvRV(*g), 0, 0))); |
| 1053 |
|
| } |
| 1054 |
|
| } |
| 1055 |
11 |
| AV *cand = (AV *)sv_2mortal((SV *)newAV()); |
| 1056 |
11 |
| AV *out = newAV(); |
| 1057 |
11 |
| HV *seen = (HV *)sv_2mortal((SV *)newHV()); |
| 1058 |
|
| SV *extra; |
| 1059 |
|
| SSize_t i, n; |
| 1060 |
|
| |
| 1061 |
11 |
▶ | n = raw ? av_len(raw) + 1 : 0; |
Branch: 1 branch
| Branch | True | False | Total |
| n = raw ? av_len(raw) + 1 : 0; |
11 |
0 |
11 |
|
| 1062 |
73 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
62 |
11 |
73 |
|
| 1063 |
62 |
| SV **e = av_fetch(raw, i, 0); |
| 1064 |
|
| HV *rec; |
| 1065 |
|
| SV **msv, **psv; |
| 1066 |
|
| STRLEN ml, pl; |
| 1067 |
|
| const char *m, *p; |
| 1068 |
|
| int said; |
| 1069 |
|
| |
| 1070 |
114 |
▶ | if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
Branch: 4 branches
| Branch | True | False | Total |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
62 |
0 |
62 |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
62 |
0 |
62 |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
62 |
0 |
62 |
| if (!(e && *e && SvROK(*e) && SvTYPE(SvRV(*e)) == SVt_PVHV)) continue; |
0 |
62 |
62 |
|
| 1071 |
62 |
| rec = (HV *)SvRV(*e); |
| 1072 |
62 |
| msv = hv_fetchs(rec, K_METHOD, 0); |
| 1073 |
62 |
| psv = hv_fetchs(rec, K_PATH, 0); |
| 1074 |
62 |
▶ | if (!(msv && *msv && psv && *psv)) continue; |
Branch: 4 branches
| Branch | True | False | Total |
| if (!(msv && *msv && psv && *psv)) continue; |
62 |
0 |
62 |
| if (!(msv && *msv && psv && *psv)) continue; |
62 |
0 |
62 |
| if (!(msv && *msv && psv && *psv)) continue; |
62 |
0 |
62 |
| if (!(msv && *msv && psv && *psv)) continue; |
0 |
62 |
62 |
|
| 1075 |
62 |
| m = SvPV_const(*msv, ml); |
| 1076 |
62 |
▶ | p = SvPV_const(*psv, pl); |
Branch: 1 branch
| Branch | True | False | Total |
| p = SvPV_const(*psv, pl); |
62 |
0 |
62 |
|
| 1077 |
|
| |
| 1078 |
|
| /* Only what a crawler would have fetched. A guarded POST is not |
| 1079 |
|
| * something to disallow: crawlers do not POST, and the same path may |
| 1080 |
|
| * carry a public GET that must stay listed. */ |
| 1081 |
62 |
▶ | if (!pks_method_ok(m, ml)) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!pks_method_ok(m, ml)) continue; |
0 |
62 |
62 |
|
| 1082 |
62 |
| said = pks_override(aTHX_ app, m, ml, p, pl); |
| 1083 |
62 |
▶ | if (said == 1) continue; /* explicitly public */ |
Branch: 1 branch
| Branch | True | False | Total |
| if (said == 1) continue; /* explicitly public */ |
0 |
62 |
62 |
|
| 1084 |
62 |
▶ | if (!(said == 0 || pks_guarded(aTHX_ rec))) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!(said == 0 || pks_guarded(aTHX_ rec))) continue; |
35 |
27 |
62 |
| if (!(said == 0 || pks_guarded(aTHX_ rec))) continue; |
30 |
5 |
35 |
|
| 1085 |
|
| /* The plugin's OWN routes are kept out of the sitemap - a sitemap |
| 1086 |
|
| * listing itself is noise - but they are emphatically not "do not |
| 1087 |
|
| * crawl". Disallowing /sitemap.xml while the Sitemap: line below |
| 1088 |
|
| * advertises it is precisely the self-contradiction this file exists |
| 1089 |
|
| * to prevent, and /robots.txt disallowing itself is nonsense. */ |
| 1090 |
32 |
▶ | if (pks_is_own(aTHX_ app, p, pl)) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (pks_is_own(aTHX_ app, p, pl)) continue; |
22 |
10 |
32 |
|
| 1091 |
10 |
| av_push(cand, pks_disallow_form(aTHX_ p, pl)); |
| 1092 |
|
| } |
| 1093 |
|
| |
| 1094 |
11 |
▶ | extra = app ? app_get(aTHX_ app, "sitemap_disallow") : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| extra = app ? app_get(aTHX_ app, "sitemap_disallow") : NULL; |
11 |
0 |
11 |
|
| 1095 |
11 |
▶ | if (extra && SvROK(extra) && SvTYPE(SvRV(extra)) == SVt_PVAV) { |
Branch: 3 branches
| Branch | True | False | Total |
| if (extra && SvROK(extra) && SvTYPE(SvRV(extra)) == SVt_PVAV) { |
3 |
8 |
11 |
| if (extra && SvROK(extra) && SvTYPE(SvRV(extra)) == SVt_PVAV) { |
3 |
0 |
3 |
| if (extra && SvROK(extra) && SvTYPE(SvRV(extra)) == SVt_PVAV) { |
3 |
0 |
3 |
|
| 1096 |
3 |
| AV *ex = (AV *)SvRV(extra); |
| 1097 |
3 |
| SSize_t xi, xn = av_len(ex) + 1; |
| 1098 |
6 |
▶ | for (xi = 0; xi < xn; xi++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (xi = 0; xi < xn; xi++) { |
3 |
3 |
6 |
|
| 1099 |
3 |
| SV **e = av_fetch(ex, xi, 0); |
| 1100 |
3 |
▶ | if (e && *e && SvOK(*e)) av_push(cand, newSVsv(*e)); |
Branch: 3 branches
| Branch | True | False | Total |
| if (e && *e && SvOK(*e)) av_push(cand, newSVsv(*e)); |
3 |
0 |
3 |
| if (e && *e && SvOK(*e)) av_push(cand, newSVsv(*e)); |
3 |
0 |
3 |
| if (e && *e && SvOK(*e)) av_push(cand, newSVsv(*e)); |
3 |
0 |
3 |
|
| 1101 |
|
| } |
| 1102 |
|
| } |
| 1103 |
|
| |
| 1104 |
11 |
▶ | if (av_len(cand) > 0) |
Branch: 1 branch
| Branch | True | False | Total |
| if (av_len(cand) > 0) |
3 |
8 |
11 |
|
| 1105 |
3 |
| sortsv(AvARRAY(cand), (size_t)(av_len(cand) + 1), pks_cmp); |
| 1106 |
|
| |
| 1107 |
11 |
| n = av_len(cand) + 1; |
| 1108 |
24 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
13 |
11 |
24 |
|
| 1109 |
13 |
| SV **e = av_fetch(cand, i, 0); |
| 1110 |
|
| STRLEN fl; |
| 1111 |
|
| const char *f; |
| 1112 |
|
| SSize_t j, ln; |
| 1113 |
13 |
| int shadows = 0, covered = 0; |
| 1114 |
|
| |
| 1115 |
14 |
▶ | if (!(e && *e)) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!(e && *e)) continue; |
13 |
0 |
13 |
| if (!(e && *e)) continue; |
0 |
13 |
13 |
|
| 1116 |
13 |
| f = SvPV_const(*e, fl); |
| 1117 |
13 |
▶ | if (!fl || f[0] != '/') continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!fl || f[0] != '/') continue; |
13 |
0 |
13 |
| if (!fl || f[0] != '/') continue; |
0 |
13 |
13 |
|
| 1118 |
13 |
▶ | if (hv_exists(seen, f, (I32)fl)) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (hv_exists(seen, f, (I32)fl)) continue; |
0 |
13 |
13 |
|
| 1119 |
|
| |
| 1120 |
|
| /* Never disallow something the sitemap lists. */ |
| 1121 |
13 |
| ln = av_len(listed) + 1; |
| 1122 |
37 |
▶ | for (j = 0; j < ln; j++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (j = 0; j < ln; j++) { |
25 |
12 |
37 |
|
| 1123 |
25 |
| SV **l = av_fetch(listed, j, 0); |
| 1124 |
|
| STRLEN pl; |
| 1125 |
|
| const char *p; |
| 1126 |
25 |
▶ | if (!(l && *l)) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!(l && *l)) continue; |
25 |
0 |
25 |
| if (!(l && *l)) continue; |
0 |
25 |
25 |
|
| 1127 |
25 |
| p = SvPV_const(*l, pl); |
| 1128 |
25 |
▶ | if (pks_covers(f, fl, p, pl)) { shadows = 1; break; } |
Branch: 1 branch
| Branch | True | False | Total |
| if (pks_covers(f, fl, p, pl)) { shadows = 1; break; } |
1 |
24 |
25 |
|
| 1129 |
|
| } |
| 1130 |
13 |
▶ | if (shadows) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (shadows) continue; |
1 |
12 |
13 |
|
| 1131 |
|
| |
| 1132 |
|
| /* and drop what a shorter prefix already covers */ |
| 1133 |
12 |
| ln = av_len(out) + 1; |
| 1134 |
25 |
▶ | for (j = 0; j < ln; j++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (j = 0; j < ln; j++) { |
13 |
12 |
25 |
|
| 1135 |
13 |
| SV **o = av_fetch(out, j, 0); |
| 1136 |
|
| STRLEN ol; |
| 1137 |
|
| const char *op; |
| 1138 |
13 |
▶ | if (!(o && *o)) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!(o && *o)) continue; |
13 |
0 |
13 |
| if (!(o && *o)) continue; |
0 |
13 |
13 |
|
| 1139 |
13 |
| op = SvPV_const(*o, ol); |
| 1140 |
13 |
▶ | if (pks_covers(op, ol, f, fl)) { covered = 1; break; } |
Branch: 1 branch
| Branch | True | False | Total |
| if (pks_covers(op, ol, f, fl)) { covered = 1; break; } |
0 |
13 |
13 |
|
| 1141 |
|
| } |
| 1142 |
12 |
▶ | if (covered) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (covered) continue; |
0 |
12 |
12 |
|
| 1143 |
|
| |
| 1144 |
12 |
| (void)hv_store(seen, f, (I32)fl, newSViv(1), 0); |
| 1145 |
12 |
| av_push(out, newSVpvn(f, fl)); |
| 1146 |
|
| } |
| 1147 |
11 |
| return out; |
| 1148 |
|
| } |
| 1149 |
|
| |
| 1150 |
|
| /* The crawlers that never read the `*` group. |
| 1151 |
|
| * |
| 1152 |
|
| * AdsBot-Google, AdsBot-Google-Mobile and Mediapartners-Google (AdSense) obey |
| 1153 |
|
| * only a group naming their own token; Google documents the global user agent |
| 1154 |
|
| * as ignored for all three. So a Disallow written for `*` alone is not a |
| 1155 |
|
| * Disallow for them, and nothing says so: the file looks right, the sitemap |
| 1156 |
|
| * agrees with it, and the path is crawled anyway. It costs most where it |
| 1157 |
|
| * matters most, because the paths worth disallowing are the ones that should |
| 1158 |
|
| * not be fetched on a schedule by anybody. |
| 1159 |
|
| * |
| 1160 |
|
| * Each group is written out again in full rather than as extra User-agent |
| 1161 |
|
| * lines on one group. Several agents may share a group, but a parser that |
| 1162 |
|
| * reads a second User-agent line as the start of a new one would then leave |
| 1163 |
|
| * the `*` group with no rules in it - and of the two ways to be wrong, a |
| 1164 |
|
| * silently empty `*` group is the one that opens the whole site. |
| 1165 |
|
| * |
| 1166 |
|
| * `agents` replaces this list, and `agents => []` asks for the `*` group |
| 1167 |
|
| * alone. */ |
| 1168 |
|
| static const char *const pks_agents[] = { |
| 1169 |
|
| "AdsBot-Google", |
| 1170 |
|
| "AdsBot-Google-Mobile", |
| 1171 |
|
| "Mediapartners-Google", |
| 1172 |
|
| NULL |
| 1173 |
|
| }; |
| 1174 |
|
| |
| 1175 |
|
| /* One group: a blank line unless this is the first, the User-agent line, then |
| 1176 |
|
| * the rules every group shares. */ |
| 1177 |
25 |
| static void pks_group(pTHX_ SV *out, const char *agent, STRLEN alen, |
| 1178 |
|
| SV *rules) { |
| 1179 |
25 |
▶ | if (SvCUR(out)) sv_catpvs(out, "\n"); |
Branch: 1 branch
| Branch | True | False | Total |
| if (SvCUR(out)) sv_catpvs(out, "\n"); |
13 |
12 |
25 |
|
| 1180 |
25 |
| sv_catpvs(out, "User-agent: "); |
| 1181 |
25 |
| sv_catpvn(out, agent, alen); |
| 1182 |
25 |
| sv_catpvs(out, "\n"); |
| 1183 |
25 |
| sv_catsv(out, rules); |
| 1184 |
25 |
| } |
| 1185 |
|
| |
| 1186 |
|
| /* The robots.txt body (+1), advertising the sitemap at `base`. */ |
| 1187 |
12 |
| static SV *pks_robots_for(pTHX_ SV *appsv, SV *base) { |
| 1188 |
12 |
| HV *app = app_hv(aTHX_ appsv); |
| 1189 |
12 |
▶ | SV *all = app ? app_get(aTHX_ app, "sitemap_disallow_all") : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *all = app ? app_get(aTHX_ app, "sitemap_disallow_all") : NULL; |
12 |
0 |
12 |
|
| 1190 |
12 |
▶ | SV *ags = app ? app_get(aTHX_ app, "sitemap_agents") : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *ags = app ? app_get(aTHX_ app, "sitemap_agents") : NULL; |
12 |
0 |
12 |
|
| 1191 |
2 |
▶ | AV *agents = (ags && SvROK(ags) && SvTYPE(SvRV(ags)) == SVt_PVAV) |
Branch: 2 branches
| Branch | True | False | Total |
| AV *agents = (ags && SvROK(ags) && SvTYPE(SvRV(ags)) == SVt_PVAV) |
2 |
0 |
2 |
| AV *agents = (ags && SvROK(ags) && SvTYPE(SvRV(ags)) == SVt_PVAV) |
2 |
0 |
2 |
|
| 1192 |
14 |
▶ | ? (AV *)SvRV(ags) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| ? (AV *)SvRV(ags) : NULL; |
2 |
10 |
12 |
|
| 1193 |
12 |
| SV *rules = sv_2mortal(newSVpvs("")); |
| 1194 |
12 |
| SV *out = newSVpvs(""); |
| 1195 |
12 |
▶ | int staging = (all && SvTRUE(all)) ? 1 : 0; |
Branch: 2 branches
| Branch | True | False | Total |
| int staging = (all && SvTRUE(all)) ? 1 : 0; |
1 |
11 |
12 |
| int staging = (all && SvTRUE(all)) ? 1 : 0; |
1 |
0 |
1 |
|
| 1196 |
|
| |
| 1197 |
12 |
▶ | if (staging) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (staging) { |
1 |
11 |
12 |
|
| 1198 |
|
| /* A staging environment being indexed is routine, embarrassing and |
| 1199 |
|
| * slow to undo, and it is usually a robots.txt copied from |
| 1200 |
|
| * production. No Sitemap line below: advertising a sitemap while |
| 1201 |
|
| * disallowing everything says two opposite things. */ |
| 1202 |
1 |
| sv_catpvs(rules, "Disallow: /\n"); |
| 1203 |
|
| } |
| 1204 |
|
| else { |
| 1205 |
11 |
| AV *dis = (AV *)sv_2mortal((SV *)pks_disallows(aTHX_ appsv)); |
| 1206 |
11 |
| SSize_t i, n = av_len(dis) + 1; |
| 1207 |
23 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
12 |
11 |
23 |
|
| 1208 |
12 |
| SV **e = av_fetch(dis, i, 0); |
| 1209 |
|
| STRLEN l; |
| 1210 |
|
| const char *p; |
| 1211 |
12 |
▶ | if (!(e && *e)) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!(e && *e)) continue; |
12 |
0 |
12 |
| if (!(e && *e)) continue; |
0 |
12 |
12 |
|
| 1212 |
12 |
| p = SvPV_const(*e, l); |
| 1213 |
12 |
| sv_catpvs(rules, "Disallow: "); |
| 1214 |
|
| /* percent-encoded, so a path with a space or a newline in it |
| 1215 |
|
| * cannot become a second directive */ |
| 1216 |
12 |
| pks_pct_cat(aTHX_ rules, p, l); |
| 1217 |
12 |
| sv_catpvs(rules, "\n"); |
| 1218 |
|
| } |
| 1219 |
|
| } |
| 1220 |
|
| |
| 1221 |
12 |
| pks_group(aTHX_ out, "*", 1, rules); |
| 1222 |
|
| |
| 1223 |
|
| /* Nothing is disallowed, so there is nothing the named crawlers are |
| 1224 |
|
| * missing: a group naming one of them and carrying no rule says exactly |
| 1225 |
|
| * what their absence already says. */ |
| 1226 |
12 |
▶ | if (SvCUR(rules)) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (SvCUR(rules)) { |
6 |
6 |
12 |
|
| 1227 |
6 |
▶ | if (agents) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (agents) { |
2 |
4 |
6 |
|
| 1228 |
2 |
| SSize_t i, n = av_len(agents) + 1; |
| 1229 |
3 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
1 |
2 |
3 |
|
| 1230 |
1 |
| SV **e = av_fetch(agents, i, 0); |
| 1231 |
|
| STRLEN l; |
| 1232 |
|
| const char *a; |
| 1233 |
1 |
▶ | if (!(e && *e && SvOK(*e))) continue; |
Branch: 3 branches
| Branch | True | False | Total |
| if (!(e && *e && SvOK(*e))) continue; |
1 |
0 |
1 |
| if (!(e && *e && SvOK(*e))) continue; |
1 |
0 |
1 |
| if (!(e && *e && SvOK(*e))) continue; |
0 |
1 |
1 |
|
| 1234 |
1 |
| a = SvPV_const(*e, l); |
| 1235 |
1 |
▶ | if (!l) continue; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!l) continue; |
0 |
1 |
1 |
|
| 1236 |
1 |
| pks_group(aTHX_ out, a, l, rules); |
| 1237 |
|
| } |
| 1238 |
|
| } |
| 1239 |
|
| else { |
| 1240 |
|
| int i; |
| 1241 |
16 |
▶ | for (i = 0; pks_agents[i]; i++) |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; pks_agents[i]; i++) |
12 |
4 |
16 |
|
| 1242 |
12 |
| pks_group(aTHX_ out, pks_agents[i], |
| 1243 |
12 |
| strlen(pks_agents[i]), rules); |
| 1244 |
|
| } |
| 1245 |
|
| } |
| 1246 |
|
| |
| 1247 |
12 |
▶ | if (staging) return out; |
Branch: 1 branch
| Branch | True | False | Total |
| if (staging) return out; |
1 |
11 |
12 |
|
| 1248 |
|
| |
| 1249 |
|
| /* The line most hand-written robots.txt files are missing, and the one |
| 1250 |
|
| * that makes a sitemap discoverable without anybody being told. */ |
| 1251 |
11 |
▶ | if (base && SvOK(base)) { |
Branch: 2 branches
| Branch | True | False | Total |
| if (base && SvOK(base)) { |
11 |
0 |
11 |
| if (base && SvOK(base)) { |
11 |
0 |
11 |
|
| 1252 |
|
| STRLEN bl; |
| 1253 |
11 |
| const char *bp = SvPV_const(base, bl); |
| 1254 |
11 |
▶ | while (bl && bp[bl - 1] == '/') bl--; |
Branch: 2 branches
| Branch | True | False | Total |
| while (bl && bp[bl - 1] == '/') bl--; |
11 |
0 |
11 |
| while (bl && bp[bl - 1] == '/') bl--; |
0 |
11 |
11 |
|
| 1255 |
11 |
| sv_catpvs(out, "\nSitemap: "); |
| 1256 |
11 |
| sv_catpvn(out, bp, bl); |
| 1257 |
11 |
| sv_catpvs(out, "/sitemap.xml\n"); |
| 1258 |
|
| } |
| 1259 |
11 |
| return out; |
| 1260 |
|
| } |
| 1261 |
|
| |
| 1262 |
|
| /* ... at the canonical base */ |
| 1263 |
11 |
| static SV *pks_robots(pTHX_ SV *appsv) { |
| 1264 |
11 |
| HV *app = app_hv(aTHX_ appsv); |
| 1265 |
22 |
▶ | return pks_robots_for(aTHX_ appsv, |
Branch: 1 branch
| Branch | True | False | Total |
| return pks_robots_for(aTHX_ appsv, |
11 |
0 |
11 |
|
| 1266 |
11 |
| app ? app_get(aTHX_ app, "sitemap_base") : NULL); |
| 1267 |
|
| } |
| 1268 |
|
| |
| 1269 |
|
| /* Does this class have that method? A `can` through the ordinary lookup, so |
| 1270 |
|
| * inheritance and AUTOLOAD behave as they would anywhere else. */ |
| 1271 |
19 |
| static int pkc_can_meth(pTHX_ SV *obj, const char *meth) { |
| 1272 |
19 |
| SV *m = sv_2mortal(newSVpv(meth, 0)); |
| 1273 |
19 |
| SV *r = pcx_call_meth(aTHX_ obj, "can", &m, 1, 1); |
| 1274 |
19 |
▶ | int ok = (r && SvTRUE(r)) ? 1 : 0; |
Branch: 2 branches
| Branch | True | False | Total |
| int ok = (r && SvTRUE(r)) ? 1 : 0; |
19 |
0 |
19 |
| int ok = (r && SvTRUE(r)) ? 1 : 0; |
15 |
4 |
19 |
|
| 1275 |
19 |
▶ | if (r) SvREFCNT_dec(r); |
Branch: 1 branch
| Branch | True | False | Total |
| if (r) SvREFCNT_dec(r); |
19 |
0 |
19 |
|
| 1276 |
19 |
| return ok; |
| 1277 |
|
| } |
| 1278 |
|
| |
| 1279 |
|
| /* Install the `sitemap` keyword on this app. Idempotent: install_kw treats a |
| 1280 |
|
| * second install by the same owner as a no-op, so `use` and `plugin` can both |
| 1281 |
|
| * ask and only the first does anything. */ |
| 1282 |
52 |
| static void pks_install_kw(pTHX_ SV *app) { |
| 1283 |
52 |
| AV *cap = newAV(); |
| 1284 |
|
| SV *argv[3], *r; |
| 1285 |
52 |
| av_push(cap, newSVsv(app)); |
| 1286 |
52 |
| argv[0] = sv_2mortal(newSVpvs("sitemap")); |
| 1287 |
52 |
| argv[1] = sv_2mortal(punk_closure(aTHX_ pks_kw_cb, cap)); |
| 1288 |
52 |
| argv[2] = sv_2mortal(newSVpvs("Punk::Plugin::Sitemap")); |
| 1289 |
52 |
| r = pcx_call_meth(aTHX_ app, "install_kw", argv, 3, 1); |
| 1290 |
52 |
▶ | if (r) SvREFCNT_dec(r); |
Branch: 1 branch
| Branch | True | False | Total |
| if (r) SvREFCNT_dec(r); |
52 |
0 |
52 |
|
| 1291 |
52 |
| } |
| 1292 |
|
| |
| 1293 |
|
| /* Resolve the base, at to_app. An explicit `base` was stored by register and |
| 1294 |
|
| * wins; without one, the application's `host` is the same fact declared once |
| 1295 |
|
| * at the app level - and because this runs at compile rather than at the |
| 1296 |
|
| * `plugin` line, `host` may sit on either side of it in the package body. |
| 1297 |
|
| * |
| 1298 |
|
| * There is still no fallback to the request's Host header, and the croak |
| 1299 |
|
| * says why: a request-derived origin is attacker-supplied bytes delivered to |
| 1300 |
|
| * search engines. */ |
| 1301 |
37 |
| static void pks_resolve_base(pTHX_ SV *appsv) { |
| 1302 |
37 |
| HV *h = app_hv(aTHX_ appsv); |
| 1303 |
37 |
▶ | SV *b = h ? app_get(aTHX_ h, "sitemap_base") : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *b = h ? app_get(aTHX_ h, "sitemap_base") : NULL; |
37 |
0 |
37 |
|
| 1304 |
|
| SV *host; |
| 1305 |
37 |
▶ | if (!h || (b && SvOK(b) && SvCUR(b))) return; |
Branch: 4 branches
| Branch | True | False | Total |
| if (!h || (b && SvOK(b) && SvCUR(b))) return; |
37 |
0 |
37 |
| if (!h || (b && SvOK(b) && SvCUR(b))) return; |
30 |
7 |
37 |
| if (!h || (b && SvOK(b) && SvCUR(b))) return; |
30 |
0 |
30 |
| if (!h || (b && SvOK(b) && SvCUR(b))) return; |
30 |
0 |
30 |
|
| 1306 |
7 |
| host = app_get(aTHX_ h, "host"); |
| 1307 |
7 |
▶ | if (host && SvOK(host) && SvCUR(host)) { |
Branch: 3 branches
| Branch | True | False | Total |
| if (host && SvOK(host) && SvCUR(host)) { |
5 |
2 |
7 |
| if (host && SvOK(host) && SvCUR(host)) { |
5 |
0 |
5 |
| if (host && SvOK(host) && SvCUR(host)) { |
5 |
0 |
5 |
|
| 1308 |
5 |
| (void)hv_stores(h, "sitemap_base", newSVsv(host)); |
| 1309 |
5 |
| return; |
| 1310 |
|
| } |
| 1311 |
2 |
| croak("Punk::Plugin::Sitemap: `base` is required - declare the " |
| 1312 |
|
| "application's `host`, or pass base => 'https://...' - the " |
| 1313 |
|
| "sitemap protocol needs absolute URLs, and taking the host from " |
| 1314 |
|
| "the request would let a crawler be handed somebody else's " |
| 1315 |
|
| "hostname"); |
| 1316 |
|
| } |
| 1317 |
|
| |
| 1318 |
|
| /* The to_app seam. Punk calls every registered middleware wrapper while it |
| 1319 |
|
| * compiles, which is the one moment every route has been declared and nothing |
| 1320 |
|
| * has been served - so the document is built there and the wrapper hands the |
| 1321 |
|
| * application back untouched. */ |
| 1322 |
|
| XS_INTERNAL(pks_mw_cb); |
| 1323 |
37 |
| XS_INTERNAL(pks_mw_cb) { |
| 1324 |
37 |
| dXSARGS; |
| 1325 |
37 |
| AV *cap = punk_clos_cap(aTHX_ cv); |
| 1326 |
37 |
▶ | SV *app = cap ? *av_fetch(cap, 0, 0) : NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| SV *app = cap ? *av_fetch(cap, 0, 0) : NULL; |
37 |
0 |
37 |
|
| 1327 |
37 |
▶ | if (app) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (app) { |
37 |
0 |
37 |
|
| 1328 |
37 |
| pks_resolve_base(aTHX_ app); |
| 1329 |
35 |
| pks_build(aTHX_ app); |
| 1330 |
|
| } |
| 1331 |
35 |
▶ | if (items > 0) XSRETURN(1); /* ST(0) is the inner app, unchanged */ |
Branch: 1 branch
| Branch | True | False | Total |
| if (items > 0) XSRETURN(1); /* ST(0) is the inner app, unchanged */ |
35 |
0 |
35 |
|
| 1332 |
0 |
| XSRETURN_EMPTY; |
| 1333 |
|
| } |
| 1334 |
|
| |
| 1335 |
|
| #endif /* PUNK_SITEMAP_H */ |