| 1 |
|
| MODULE = Punk::Observe PACKAGE = Punk::Observe::WAL PREFIX = pow_ |
| 2 |
|
| |
| 3 |
|
| # The WAL surface. The ingest path in phase 3 calls the C directly with the |
| 4 |
|
| # record array the decoder produced; this exists so t/0004-wal.t can drive it, |
| 5 |
|
| # and so a forked child in that test can append and then die mid-frame. |
| 6 |
|
| |
| 7 |
|
| # --- CRC-32C --------------------------------------------------------------- |
| 8 |
|
| |
| 9 |
|
| UV |
| 10 |
|
| pow_crc32c(SV *buf) |
| 11 |
|
| CODE: |
| 12 |
|
| { |
| 13 |
|
| STRLEN len; const char *p; |
| 14 |
218 |
| p = SvPV(buf, len); |
| 15 |
218 |
▶ | RETVAL = (UV)po_crc32c(0, p, (size_t)len); |
Branch: 1 branch
| Branch | True | False | Total |
| RETVAL = (UV)po_crc32c(0, p, (size_t)len); |
215 |
3 |
218 |
|
| 16 |
|
| } |
| 17 |
|
| OUTPUT: |
| 18 |
|
| RETVAL |
| 19 |
|
| |
| 20 |
|
| # The table path, forced. The hardware path is what runs in production on any |
| 21 |
|
| # modern box, so without this the fallback would ship exercised only on the |
| 22 |
|
| # machines least able to report why it broke - and a WAL written with the |
| 23 |
|
| # instruction and replayed without it has to verify. |
| 24 |
|
| UV |
| 25 |
|
| pow_crc32c_table(SV *buf) |
| 26 |
|
| CODE: |
| 27 |
|
| { |
| 28 |
|
| STRLEN len; const char *p; |
| 29 |
218 |
| p = SvPV(buf, len); |
| 30 |
218 |
▶ | RETVAL = (UV)po_crc32c_table(0, p, (size_t)len); |
Branch: 1 branch
| Branch | True | False | Total |
| RETVAL = (UV)po_crc32c_table(0, p, (size_t)len); |
215 |
3 |
218 |
|
| 31 |
|
| } |
| 32 |
|
| OUTPUT: |
| 33 |
|
| RETVAL |
| 34 |
|
| |
| 35 |
|
| int |
| 36 |
|
| pow_crc32c_hardware(...) |
| 37 |
|
| CODE: |
| 38 |
1 |
| RETVAL = po_crc32c_hardware(); |
| 39 |
|
| OUTPUT: |
| 40 |
|
| RETVAL |
| 41 |
|
| |
| 42 |
|
| UV |
| 43 |
|
| pow_hdr_size(...) |
| 44 |
|
| CODE: |
| 45 |
3 |
▶ | RETVAL = (UV)PO_WAL_HDR; |
Branch: 1 branch
| Branch | True | False | Total |
| RETVAL = (UV)PO_WAL_HDR; |
0 |
3 |
3 |
|
| 46 |
|
| OUTPUT: |
| 47 |
|
| RETVAL |
| 48 |
|
| |
| 49 |
|
| # --- appending ------------------------------------------------------------- |
| 50 |
|
| |
| 51 |
|
| # Append a batch of records described from Perl. |
| 52 |
|
| # |
| 53 |
|
| # A record is the hash po_rec_hv produces, and every field of it is stored - |
| 54 |
|
| # see po_rec_from_hv for why that is not merely tidy. The WAL still does not |
| 55 |
|
| # need to UNDERSTAND a record; it does need to keep one whole, because it is |
| 56 |
|
| # the only copy until the compactor seals a segment. |
| 57 |
|
| SV * |
| 58 |
|
| pow_append(SV *path, SV *specs, SV *policy, SV *interval_ns) |
| 59 |
|
| CODE: |
| 60 |
|
| { |
| 61 |
|
| po_wal w; |
| 62 |
|
| po_arena ar; |
| 63 |
110 |
| po_rec *recs = NULL; |
| 64 |
|
| AV *av; |
| 65 |
|
| SSize_t n, i; |
| 66 |
|
| STRLEN plen; |
| 67 |
|
| const char *p; |
| 68 |
110 |
| po_u64 iv = 0; |
| 69 |
110 |
| int ok = 1; |
| 70 |
|
| HV *res; |
| 71 |
|
| |
| 72 |
110 |
| p = SvPV(path, plen); |
| 73 |
110 |
▶ | if (!SvROK(specs) || SvTYPE(SvRV(specs)) != SVt_PVAV) |
Branch: 2 branches
| Branch | True | False | Total |
| if (!SvROK(specs) || SvTYPE(SvRV(specs)) != SVt_PVAV) |
110 |
0 |
110 |
| if (!SvROK(specs) || SvTYPE(SvRV(specs)) != SVt_PVAV) |
0 |
110 |
110 |
|
| 74 |
0 |
| croak("specs must be an arrayref"); |
| 75 |
110 |
| av = (AV *)SvRV(specs); |
| 76 |
110 |
| n = av_len(av) + 1; |
| 77 |
|
| |
| 78 |
110 |
▶ | if (!po_sv_to_u64(aTHX_ interval_ns, &iv)) iv = 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!po_sv_to_u64(aTHX_ interval_ns, &iv)) iv = 0; |
0 |
110 |
110 |
|
| 79 |
110 |
▶ | if (!po_wal_open(&w, p, (int)SvIV(policy), iv)) |
Branch: 1 branch
| Branch | True | False | Total |
| if (!po_wal_open(&w, p, (int)SvIV(policy), iv)) |
0 |
110 |
110 |
|
| 80 |
0 |
| croak("open %s: %s", p, strerror(errno)); |
| 81 |
110 |
▶ | if (!po_arena_init(&ar, 256)) { po_wal_close(&w); croak("arena"); } |
Branch: 1 branch
| Branch | True | False | Total |
| if (!po_arena_init(&ar, 256)) { po_wal_close(&w); croak("arena"); } |
0 |
110 |
110 |
|
| 82 |
|
| |
| 83 |
110 |
▶ | if (n > 0) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (n > 0) { |
109 |
1 |
110 |
|
| 84 |
109 |
▶ | Newx(recs, n, po_rec); |
Branch: 1 branch
| Branch | True | False | Total |
| Newx(recs, n, po_rec); |
0 |
109 |
109 |
|
| 85 |
12537 |
▶ | for (i = 0; i < n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n; i++) { |
12428 |
109 |
12537 |
|
| 86 |
12428 |
| SV **e = av_fetch(av, i, 0); |
| 87 |
12428 |
| po_rec_zero(&recs[i]); |
| 88 |
12428 |
▶ | if (!e || !SvROK(*e)) continue; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!e || !SvROK(*e)) continue; |
12428 |
0 |
12428 |
| if (!e || !SvROK(*e)) continue; |
0 |
12428 |
12428 |
|
| 89 |
12428 |
▶ | if (!po_rec_from_hv(aTHX_ (HV *)SvRV(*e), &recs[i], &ar)) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (!po_rec_from_hv(aTHX_ (HV *)SvRV(*e), &recs[i], &ar)) { |
0 |
12428 |
12428 |
|
| 90 |
0 |
| ok = 0; |
| 91 |
0 |
| break; |
| 92 |
|
| } |
| 93 |
|
| } |
| 94 |
109 |
▶ | if (ok) ok = po_wal_append(&w, recs, (size_t)n, ar.base, ar.len); |
Branch: 1 branch
| Branch | True | False | Total |
| if (ok) ok = po_wal_append(&w, recs, (size_t)n, ar.base, ar.len); |
109 |
0 |
109 |
|
| 95 |
|
| } |
| 96 |
|
| |
| 97 |
110 |
| res = newHV(); |
| 98 |
110 |
| hv_stores(res, "ok", newSViv(ok ? 1 : 0)); |
| 99 |
110 |
| hv_stores(res, "frames", po_u64_to_sv(w.frames_written)); |
| 100 |
110 |
| hv_stores(res, "bytes", po_u64_to_sv(w.bytes_written)); |
| 101 |
110 |
| hv_stores(res, "fsyncs", po_u64_to_sv(w.fsyncs)); |
| 102 |
110 |
▶ | if (!ok) hv_stores(res, "errno", newSVpv(strerror(errno), 0)); |
Branch: 1 branch
| Branch | True | False | Total |
| if (!ok) hv_stores(res, "errno", newSVpv(strerror(errno), 0)); |
0 |
110 |
110 |
|
| 103 |
|
| |
| 104 |
110 |
| Safefree(recs); |
| 105 |
110 |
| po_arena_free(&ar); |
| 106 |
110 |
| po_wal_close(&w); |
| 107 |
110 |
| RETVAL = newRV_noinc((SV *)res); |
| 108 |
|
| } |
| 109 |
|
| OUTPUT: |
| 110 |
|
| RETVAL |
| 111 |
|
| |
| 112 |
|
| # Append several frames through ONE open handle, so the fsync policy can be |
| 113 |
|
| # observed across frames rather than one at a time. |
| 114 |
|
| SV * |
| 115 |
|
| pow_append_many(SV *path, SV *frames, SV *policy, SV *interval_ns) |
| 116 |
|
| CODE: |
| 117 |
|
| { |
| 118 |
|
| po_wal w; |
| 119 |
|
| STRLEN plen; |
| 120 |
|
| const char *p; |
| 121 |
12 |
| po_u64 iv = 0; |
| 122 |
12 |
| IV nframes = SvIV(frames); |
| 123 |
|
| IV k; |
| 124 |
12 |
| int ok = 1; |
| 125 |
|
| HV *res; |
| 126 |
|
| |
| 127 |
12 |
| p = SvPV(path, plen); |
| 128 |
12 |
▶ | if (!po_sv_to_u64(aTHX_ interval_ns, &iv)) iv = 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!po_sv_to_u64(aTHX_ interval_ns, &iv)) iv = 0; |
0 |
12 |
12 |
|
| 129 |
12 |
▶ | if (!po_wal_open(&w, p, (int)SvIV(policy), iv)) |
Branch: 1 branch
| Branch | True | False | Total |
| if (!po_wal_open(&w, p, (int)SvIV(policy), iv)) |
0 |
12 |
12 |
|
| 130 |
0 |
| croak("open %s: %s", p, strerror(errno)); |
| 131 |
|
| |
| 132 |
672 |
▶ | for (k = 0; k < nframes && ok; k++) { |
Branch: 2 branches
| Branch | True | False | Total |
| for (k = 0; k < nframes && ok; k++) { |
660 |
12 |
672 |
| for (k = 0; k < nframes && ok; k++) { |
660 |
0 |
660 |
|
| 133 |
|
| po_rec r; |
| 134 |
660 |
| po_rec_zero(&r); |
| 135 |
660 |
| r.kind = PO_SPAN; |
| 136 |
660 |
| r.t_unix_nano = (po_u64)1774224000000000000ULL + (po_u64)k; |
| 137 |
660 |
| ok = po_wal_append(&w, &r, 1, NULL, 0); |
| 138 |
|
| } |
| 139 |
|
| |
| 140 |
12 |
| res = newHV(); |
| 141 |
12 |
| hv_stores(res, "ok", newSViv(ok ? 1 : 0)); |
| 142 |
12 |
| hv_stores(res, "frames", po_u64_to_sv(w.frames_written)); |
| 143 |
12 |
| hv_stores(res, "fsyncs", po_u64_to_sv(w.fsyncs)); |
| 144 |
12 |
| po_wal_close(&w); |
| 145 |
12 |
| RETVAL = newRV_noinc((SV *)res); |
| 146 |
|
| } |
| 147 |
|
| OUTPUT: |
| 148 |
|
| RETVAL |
| 149 |
|
| |
| 150 |
|
| SV * |
| 151 |
|
| pow_seal(SV *path, SV *total) |
| 152 |
|
| CODE: |
| 153 |
|
| { |
| 154 |
|
| po_wal w; |
| 155 |
|
| STRLEN plen; |
| 156 |
92 |
| const char *p = SvPV(path, plen); |
| 157 |
92 |
| po_u64 t = 0; |
| 158 |
|
| int ok; |
| 159 |
92 |
| (void)po_sv_to_u64(aTHX_ total, &t); |
| 160 |
92 |
▶ | if (!po_wal_open(&w, p, PO_FSYNC_NEVER, 0)) |
Branch: 1 branch
| Branch | True | False | Total |
| if (!po_wal_open(&w, p, PO_FSYNC_NEVER, 0)) |
0 |
92 |
92 |
|
| 161 |
0 |
| croak("open %s: %s", p, strerror(errno)); |
| 162 |
92 |
| ok = po_wal_seal(&w, t); |
| 163 |
92 |
| po_wal_close(&w); |
| 164 |
92 |
| RETVAL = newSViv(ok ? 1 : 0); |
| 165 |
|
| } |
| 166 |
|
| OUTPUT: |
| 167 |
|
| RETVAL |
| 168 |
|
| |
| 169 |
|
| # --- replay ---------------------------------------------------------------- |
| 170 |
|
| |
| 171 |
|
| SV * |
| 172 |
|
| pow_replay(SV *buf) |
| 173 |
|
| CODE: |
| 174 |
|
| { |
| 175 |
|
| po_wal_replay rp; |
| 176 |
|
| STRLEN len; |
| 177 |
|
| const char *p; |
| 178 |
|
| HV *res; |
| 179 |
|
| |
| 180 |
2573 |
| p = SvPV(buf, len); |
| 181 |
2573 |
| po_wal_replay_buf(p, (size_t)len, &rp, NULL, NULL); |
| 182 |
|
| |
| 183 |
2573 |
| res = newHV(); |
| 184 |
2573 |
| hv_stores(res, "frames", po_u64_to_sv(rp.frames)); |
| 185 |
2573 |
| hv_stores(res, "records", po_u64_to_sv(rp.records)); |
| 186 |
2573 |
| hv_stores(res, "bytes_ok", po_u64_to_sv(rp.bytes_ok)); |
| 187 |
2573 |
| hv_stores(res, "bytes_truncated", po_u64_to_sv(rp.bytes_truncated)); |
| 188 |
2573 |
| hv_stores(res, "sealed", newSViv(rp.sealed)); |
| 189 |
2573 |
| hv_stores(res, "reason", |
| 190 |
|
| newSVpv(po_replay_reason(rp.stopped_reason), 0)); |
| 191 |
2573 |
| RETVAL = newRV_noinc((SV *)res); |
| 192 |
|
| } |
| 193 |
|
| OUTPUT: |
| 194 |
|
| RETVAL |
| 195 |
|
| |
| 196 |
|
| # Replay and hand back the records themselves - the same hash shape the |
| 197 |
|
| # decoder produces, so a replayed record and a freshly decoded one are |
| 198 |
|
| # indistinguishable to everything upstream. That is what lets the reader in |
| 199 |
|
| # Punk::Observe::Store query an unsealed log without a second row shape. |
| 200 |
|
| void |
| 201 |
|
| pow_replay_bodies(SV *buf) |
| 202 |
|
| PPCODE: |
| 203 |
|
| { |
| 204 |
|
| po_wal_replay rp; |
| 205 |
|
| STRLEN len; |
| 206 |
|
| const char *p; |
| 207 |
6 |
| AV *out = newAV(); |
| 208 |
|
| |
| 209 |
6 |
| p = SvPV(buf, len); |
| 210 |
|
| { |
| 211 |
6 |
| size_t off = 0; |
| 212 |
|
| /* Walk with the same rules as po_wal_replay_buf, collecting |
| 213 |
|
| * bodies. Kept here rather than as a callback so the XS does |
| 214 |
|
| * not have to smuggle a Perl context through a C function |
| 215 |
|
| * pointer. */ |
| 216 |
6 |
| po_wal_replay_buf(p, (size_t)len, &rp, NULL, NULL); |
| 217 |
5 |
| for (;;) { |
| 218 |
|
| uint32_t magic, frame_len, n_recs, arena_len; |
| 219 |
|
| uint16_t flags, version; |
| 220 |
|
| const char *h; |
| 221 |
|
| const po_rec *recs; |
| 222 |
|
| po_arena view; |
| 223 |
|
| size_t i; |
| 224 |
|
| |
| 225 |
11 |
▶ | if ((size_t)len - off < PO_WAL_HDR) break; |
Branch: 1 branch
| Branch | True | False | Total |
| if ((size_t)len - off < PO_WAL_HDR) break; |
5 |
6 |
11 |
|
| 226 |
6 |
| h = p + off; |
| 227 |
6 |
| memcpy(&magic, h, 4); magic = po_le32(magic); |
| 228 |
6 |
▶ | if (magic != PO_WAL_MAGIC) break; |
Branch: 1 branch
| Branch | True | False | Total |
| if (magic != PO_WAL_MAGIC) break; |
0 |
6 |
6 |
|
| 229 |
6 |
| memcpy(&frame_len, h + 4, 4); frame_len = po_le32(frame_len); |
| 230 |
6 |
| memcpy(&n_recs, h + 8, 4); n_recs = po_le32(n_recs); |
| 231 |
6 |
| memcpy(&arena_len, h + 12, 4); arena_len = po_le32(arena_len); |
| 232 |
6 |
| flags = (uint16_t)((unsigned char)h[36] |
| 233 |
6 |
| | ((unsigned char)h[37] << 8)); |
| 234 |
6 |
| version = (uint16_t)((unsigned char)h[38] |
| 235 |
6 |
| | ((unsigned char)h[39] << 8)); |
| 236 |
6 |
▶ | if (version != PO_WAL_VERSION) break; |
Branch: 1 branch
| Branch | True | False | Total |
| if (version != PO_WAL_VERSION) break; |
1 |
5 |
6 |
|
| 237 |
5 |
▶ | if (flags & PO_WAL_F_SEALED) break; |
Branch: 1 branch
| Branch | True | False | Total |
| if (flags & PO_WAL_F_SEALED) break; |
0 |
5 |
5 |
|
| 238 |
5 |
▶ | if ((size_t)len - off - PO_WAL_HDR < (size_t)frame_len) break; |
Branch: 1 branch
| Branch | True | False | Total |
| if ((size_t)len - off - PO_WAL_HDR < (size_t)frame_len) break; |
0 |
5 |
5 |
|
| 239 |
5 |
▶ | if (off + PO_WAL_HDR + frame_len > rp.bytes_ok) break; |
Branch: 1 branch
| Branch | True | False | Total |
| if (off + PO_WAL_HDR + frame_len > rp.bytes_ok) break; |
0 |
5 |
5 |
|
| 240 |
|
| |
| 241 |
5 |
| recs = (const po_rec *)(h + PO_WAL_HDR); |
| 242 |
|
| /* A read-only view onto the frame's own arena. po_rec_hv |
| 243 |
|
| * only ever reads base, and nothing here owns it, so this |
| 244 |
|
| * is never freed. */ |
| 245 |
5 |
| view.base = (char *)(h + PO_WAL_HDR |
| 246 |
5 |
| + (size_t)n_recs * sizeof(po_rec)); |
| 247 |
5 |
| view.len = (size_t)arena_len; |
| 248 |
5 |
| view.cap = (size_t)arena_len; |
| 249 |
|
| |
| 250 |
159 |
▶ | for (i = 0; i < n_recs; i++) |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < n_recs; i++) |
154 |
5 |
159 |
|
| 251 |
154 |
| av_push(out, po_rec_hv(aTHX_ &recs[i], &view)); |
| 252 |
5 |
| off += PO_WAL_HDR + frame_len; |
| 253 |
|
| } |
| 254 |
|
| } |
| 255 |
6 |
▶ | mXPUSHs(newRV_noinc((SV *)out)); |
Branch: 1 branch
| Branch | True | False | Total |
| mXPUSHs(newRV_noinc((SV *)out)); |
0 |
6 |
6 |
|
| 256 |
|
| } |