include/punk_observe/po_retain.h

filter: Statementstmt 84.7% 50 / 59 Branchbran 73.6% 28 / 38 Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 80.4% 78 / 97 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« include/punk_observe/po_time.h ↑ Summary include/punk_observe/po_wal.h »
1 /* po_retain.h - making the disk stop growing, without SIGBUS.
2 *
3 * TWO RULES, AND THE SECOND ONE IS WHY LOCK-FREE READERS WORK.
4 *
5 * 1. DELETION IS BY WHOLE BLOCK, NEVER BY RECORD. "Delete these logs" would
6 * require rewriting a compressed block, which means a segment is no longer
7 * immutable, which removes the property every reader depends on. A block
8 * is two hours; it expires as a unit. Per-record deletion, if a legal
9 * request ever needs it, is a separate feature that rewrites a block into
10 * a new generation - it does not get smuggled in here.
11 *
12 * 2. THE PRIMITIVE IS unlink(2), NEVER ftruncate(2).
13 *
14 * A reader holding an mmap of an UNLINKED file keeps valid pages on POSIX:
15 * the name is gone, the inode lives until the last mapping drops. That is
16 * what makes it safe to delete a segment somebody is reading.
17 *
18 * A reader holding an mmap of a TRUNCATED file takes SIGBUS on the next
19 * touch of a removed page. Not an error return - a signal, killing the
20 * worker, mid-request, for every connection it held.
21 *
22 * So there is no ftruncate in this file, and t/0082-retention.t fails the
23 * suite if one appears anywhere on a segment path. "Reclaim the tail of a
24 * partly-expired segment" is the optimisation that will be suggested, and
25 * this is the answer to it.
26 *
27 * AND THE THING NOBODY FINDS BY TESTING ON A LAPTOP: a deleted file that is
28 * still mapped is still on the disk. A 2GB segment unlinked an hour ago
29 * occupies 2GB until the last worker drops its mapping, so the space a
30 * retention policy promises and the space `df` reports diverge with no
31 * explanation. This file counts it.
32 */
33 #ifndef PO_RETAIN_H
34 #define PO_RETAIN_H
35  
36 #include "punk_observe/po_compat.h"
37 #include "punk_observe/po_seg.h"
38 #include "punk_observe/po_time.h"
39  
40 #ifndef _WIN32
41 # include <dirent.h>
42 # include <sys/stat.h>
43 #endif
44  
45 #define PO_RETAIN_MAX 4096
46  
47 typedef struct {
48 char path[PO_PATH_MAX];
49 po_u64 t_min, t_max;
50 po_u64 bytes;
51 int expired;
52 } po_seg_info;
53  
54 typedef struct {
55 po_seg_info *seg;
56 uint32_t n, cap;
57  
58 /* what the sweep did */
59 uint32_t unlinked;
60 po_u64 bytes_freed;
61 uint32_t kept;
62  
63 /* what is gone but still occupying the disk */
64 uint32_t mapped_deleted;
65 po_u64 mapped_deleted_bytes;
66  
67 uint32_t truncate_calls; /* MUST stay zero; the tests assert it */
68 } po_retain;
69  
70 7 static int po_retain_init(po_retain *r) {
71 7 memset(r, 0, sizeof(*r));
72 7 r->cap = 64;
73 7 r->seg = (po_seg_info *)calloc(r->cap, sizeof(po_seg_info));
74 7 return r->seg != NULL;
75 }
76  
77 7 static void po_retain_free(po_retain *r) {
78 7 free(r->seg); r->seg = NULL; r->n = r->cap = 0;
79 7 }
80  
81 17 static int po_retain_add(po_retain *r, const char *path) {
82 po_seg_r s;
83 po_seg_info *e;
84 #ifndef _WIN32
85 struct stat st;
86 #endif
87  
88 17 ▶ if (r->n == r->cap) {
Branch: 1 branch
BranchTrueFalseTotal
if (r->n == r->cap) { 0 17 17
89 0 uint32_t want = r->cap * 2;
90 0 po_seg_info *ns = (po_seg_info *)realloc(r->seg, want * sizeof(po_seg_info));
91 0 ▶ if (!ns) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!ns) return 0; 0 0 0
92 0 memset(ns + r->cap, 0, (want - r->cap) * sizeof(po_seg_info));
93 0 r->seg = ns; r->cap = want;
94 }
95 17 e = &r->seg[r->n];
96 17 memset(e, 0, sizeof(*e));
97 17 strncpy(e->path, path, sizeof(e->path) - 1);
98  
99 /* The footer carries the span, so expiry is decided WITHOUT reading the
100 * data - which is the same pruning a query uses. */
101 17 ▶ if (!po_seg_open(&s, path)) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!po_seg_open(&s, path)) return 0; 1 16 17
102 16 e->t_min = s.hdr.t_min;
103 16 e->t_max = s.hdr.t_max;
104 16 po_seg_close(&s);
105  
106 #ifndef _WIN32
107 16 ▶ if (stat(path, &st) == 0) e->bytes = (po_u64)st.st_size;
Branch: 1 branch
BranchTrueFalseTotal
if (stat(path, &st) == 0) e->bytes = (po_u64)st.st_size; 16 0 16
108 #endif
109 16 r->n++;
110 16 return 1;
111 }
112  
113 /* Mark everything whose LATEST record is older than the cutoff.
114 *
115 * t_max, not t_min: a segment is expired only when every record in it is,
116 * because a block is deleted as a unit and a partly-expired one keeps its
117 * whole span. Using t_min would delete data inside the retention window. */
118 7 static uint32_t po_retain_mark(po_retain *r, po_u64 cutoff) {
119 7 uint32_t i, n = 0;
120 23 ▶ for (i = 0; i < r->n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < r->n; i++) { 16 7 23
121 16 r->seg[i].expired = (r->seg[i].t_max < cutoff);
122 16 ▶ if (r->seg[i].expired) n++;
Branch: 1 branch
BranchTrueFalseTotal
if (r->seg[i].expired) n++; 12 4 16
123 }
124 7 return n;
125 }
126  
127 /* Delete the marked segments.
128 *
129 * unlink only. There is deliberately no code path here that shortens a file. */
130 7 static int po_retain_sweep(po_retain *r) {
131 uint32_t i;
132 23 ▶ for (i = 0; i < r->n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < r->n; i++) { 16 7 23
133 16 ▶ if (!r->seg[i].expired) { r->kept++; continue; }
Branch: 1 branch
BranchTrueFalseTotal
if (!r->seg[i].expired) { r->kept++; continue; } 4 12 16
134 12 ▶ if (unlink(r->seg[i].path) == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlink(r->seg[i].path) == 0) { 12 0 12
135 12 r->unlinked++;
136 12 r->bytes_freed += r->seg[i].bytes;
137 }
138 }
139 7 return 1;
140 }
141  
142 /* Account for segments that are unlinked but still mapped.
143 *
144 * `open_maps` is what the reader side reports: how many mappings of
145 * now-nameless segments are still held. Those bytes are NOT free, whatever
146 * the retention policy promised, and the status page shows this number so an
147 * operator can see why the disk has not shrunk. */
148 0 static void po_retain_account_mapped(po_retain *r, uint32_t open_maps,
149 po_u64 open_bytes) {
150 0 r->mapped_deleted = open_maps;
151 0 r->mapped_deleted_bytes = open_bytes;
152 0 }
153  
154 /* An empty block directory is NOT an expired one.
155 *
156 * A tenant with no traffic for two hours must not have its directory removed
157 * and recreated in a loop - that is churn with no benefit and it makes every
158 * sweep look like it did something. */
159 3 static int po_block_removable(uint32_t segments, int expired_all) {
160 3 ▶ return segments > 0 && expired_all;
Branch: 2 branches
BranchTrueFalseTotal
return segments > 0 && expired_all; 2 1 3
return segments > 0 && expired_all; 1 1 2
161 }
162  
163 /* ---- generations ----------------------------------------------------------
164 *
165 * A query reading generation N must keep reading it while retention moves on.
166 * Generations are reference-counted, the count lives with the reader, and a
167 * generation with a live reader is never the one whose segments are unlinked.
168 *
169 * (Unlinking one it IS reading would still be safe, by the mmap rule above.
170 * The reference count is about not losing the MANIFEST entry that tells the
171 * reader which segments belong together.) */
172 typedef struct {
173 po_u64 generation;
174 uint32_t readers;
175 } po_gen_ref;
176  
177 typedef struct {
178 po_gen_ref g[32];
179 int n;
180 } po_gen_table;
181  
182 3 static void po_gen_acquire(po_gen_table *t, po_u64 gen) {
183 int i;
184 3 ▶ for (i = 0; i < t->n; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < t->n; i++) 1 2 3
185 1 ▶ if (t->g[i].generation == gen) { t->g[i].readers++; return; }
Branch: 1 branch
BranchTrueFalseTotal
if (t->g[i].generation == gen) { t->g[i].readers++; return; } 1 0 1
186 2 ▶ if (t->n < 32) {
Branch: 1 branch
BranchTrueFalseTotal
if (t->n < 32) { 2 0 2
187 2 t->g[t->n].generation = gen;
188 2 t->g[t->n].readers = 1;
189 2 t->n++;
190 }
191 }
192  
193 3 static void po_gen_release(po_gen_table *t, po_u64 gen) {
194 int i;
195 3 ▶ for (i = 0; i < t->n; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < t->n; i++) 3 0 3
196 3 ▶ if (t->g[i].generation == gen && t->g[i].readers)
Branch: 2 branches
BranchTrueFalseTotal
if (t->g[i].generation == gen && t->g[i].readers) 3 0 3
if (t->g[i].generation == gen && t->g[i].readers) 3 0 3
197 3 { t->g[i].readers--; return; }
198 }
199  
200 7 static int po_gen_busy(const po_gen_table *t, po_u64 gen) {
201 int i;
202 8 ▶ for (i = 0; i < t->n; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < t->n; i++) 6 2 8
203 6 ▶ if (t->g[i].generation == gen) return t->g[i].readers > 0;
Branch: 1 branch
BranchTrueFalseTotal
if (t->g[i].generation == gen) return t->g[i].readers > 0; 5 1 6
204 2 return 0;
205 }
206  
207 #endif /* PO_RETAIN_H */
« include/punk_observe/po_time.h ↑ Summary include/punk_observe/po_wal.h »