include/punk_observe/po_log.h

filter: Statementstmt 95.8% 137 / 143 Branchbran 74.5% 76 / 102 Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 86.9% 213 / 245 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« include/punk_observe/po_traceidx.h ↑ Summary include/punk_observe/po_route.h »
1 /* po_log.h - log streams and compressed blocks.
2 *
3 * INDEX THE LABELS, BRUTE-FORCE THE TEXT, PRUNE WITH A BLOOM.
4 *
5 * A block is up to 1MB of lines from ONE stream, raw-deflated. 1MB because a
6 * block is the decompression unit: smaller means more directory and worse
7 * ratios, larger means a query for one minute inflates ten. 1MB of log lines
8 * is roughly 5,000-10,000 lines, which is seconds to minutes of a busy
9 * service - the granularity of the time ranges people actually ask for.
10 *
11 * RAW deflate (RFC 1951), not gzip: a gzip header and trailer on every block
12 * is 18 bytes of nothing, and the block directory already carries the length
13 * and the checksum. The dictionary is reset per block so blocks decompress
14 * INDEPENDENTLY, which is what makes the directory useful at all.
15 *
16 * STRUCTURED FIELDS, because OTLP logs are not strings. trace_id and span_id
17 * are first-class columns rather than text to be searched for, and that is
18 * what makes phase 9's `| logs` stage a lookup instead of a hunt for a hex
19 * string in a haystack.
20 */
21 #ifndef PO_LOG_H
22 #define PO_LOG_H
23  
24 #include "punk_observe/po_compat.h"
25 #include "punk_observe/po_rec.h"
26 #include "punk_observe/po_bloom.h"
27 #include "punk_observe/po_crc32c.h"
28  
29 #ifdef PO_HAVE_ZLIB
30 # include <zlib.h>
31 #endif
32  
33 #define PO_BLOCK_RAW_MAX (1024 * 1024)
34  
35 /* Block flags */
36 #define PO_BLK_STORED 0x01 /* not compressed: deflate did not help, or
37 * there is no zlib. Never a silent failure. */
38  
39 typedef struct {
40 po_u64 stream; /* the interned label set */
41 po_u64 t_min, t_max;
42 uint32_t lines;
43 uint32_t raw_len;
44 uint32_t comp_len;
45 uint32_t crc; /* over the RAW bytes, so a bad decompress is
46 * caught rather than fed onward */
47 uint32_t bloom_bits;
48 uint16_t flags;
49 } po_block_hdr;
50  
51 /* ---- writing --------------------------------------------------------------
52 *
53 * Lines accumulate raw, then the whole block is deflated once when sealed.
54 * Per-line compression would give up almost all the ratio: the redundancy in
55 * logs is BETWEEN lines, not within one.
56 */
57 typedef struct {
58 po_arena raw;
59 po_block_hdr h;
60 po_bloom bloom;
61 po_u64 t_last;
62 int have_bloom;
63 } po_block_w;
64  
65 34 static int po_block_w_init(po_block_w *b, po_u64 stream) {
66 34 memset(b, 0, sizeof(*b));
67 34 ▶ if (!po_arena_init(&b->raw, 8192)) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!po_arena_init(&b->raw, 8192)) return 0; 0 34 34
68 34 b->h.stream = stream;
69 34 b->h.t_min = PO_U64_MAX;
70 34 return 1;
71 }
72  
73 34 static void po_block_w_free(po_block_w *b) {
74 34 po_arena_free(&b->raw);
75 34 ▶ if (b->have_bloom) po_bloom_free(&b->bloom);
Branch: 1 branch
BranchTrueFalseTotal
if (b->have_bloom) po_bloom_free(&b->bloom); 34 0 34
76 34 }
77  
78 0 static int po_block_full(const po_block_w *b) {
79 0 return b->raw.len >= PO_BLOCK_RAW_MAX;
80 }
81  
82 /* One record on the wire inside a block:
83 *
84 * [varint dt][varint severity][varint body_len][body]
85 * [flags byte][trace_id 16][span_id 8] (ids only when flagged)
86 *
87 * The timestamp is a DELTA from the previous line, which for a log stream is
88 * usually small - lines cluster in time - and costs one or two bytes instead
89 * of eight.
90 */
91 12864 static void po_lputv(po_arena *a, po_u64 v) {
92 12864 char b[10]; size_t n = 0;
93 18890 do { unsigned char c = (unsigned char)(v & 0x7F); v >>= 7;
94 18890 ▶ if (v) c |= 0x80; b[n++] = (char)c; } while (v);
Branch: 2 branches
BranchTrueFalseTotal
if (v) c |= 0x80; b[n++] = (char)c; } while (v); 6026 12864 18890
if (v) c |= 0x80; b[n++] = (char)c; } while (v); 6026 12864 18890
95 12864 po_arena_put(a, b, n);
96 12864 }
97  
98 9123 static po_u64 po_lgetv(const unsigned char *b, size_t len, size_t *pos) {
99 9123 po_u64 v = 0; int sh = 0;
100 15133 ▶ while (*pos < len) {
Branch: 1 branch
BranchTrueFalseTotal
while (*pos < len) { 15133 0 15133
101 15133 unsigned char c = b[(*pos)++];
102 15133 v |= (po_u64)(c & 0x7F) << sh;
103 15133 ▶ if (!(c & 0x80)) return v;
Branch: 1 branch
BranchTrueFalseTotal
if (!(c & 0x80)) return v; 9123 6010 15133
104 6010 sh += 7;
105 6010 ▶ if (sh > 63) break;
Branch: 1 branch
BranchTrueFalseTotal
if (sh > 63) break; 0 6010 6010
106 }
107 0 return v;
108 }
109  
110 4288 static int po_block_add(po_block_w *b, po_u64 t, uint16_t severity,
111 const char *body, size_t body_len,
112 po_u64 tr_hi, po_u64 tr_lo, po_u64 span) {
113 4288 unsigned char flags = 0;
114 po_u64 dt;
115  
116 4288 ▶ if (b->h.lines == 0) { b->h.t_min = t; b->t_last = t; dt = 0; }
Branch: 1 branch
BranchTrueFalseTotal
if (b->h.lines == 0) { b->h.t_min = t; b->t_last = t; dt = 0; } 34 4254 4288
117 4254 ▶ else dt = t >= b->t_last ? t - b->t_last : 0; /* out of order: 0, and
Branch: 1 branch
BranchTrueFalseTotal
else dt = t >= b->t_last ? t - b->t_last : 0; /* out of order: 0, and 4252 2 4254
118 * t_min/t_max still bound
119 * the block correctly */
120  
121 4288 po_lputv(&b->raw, dt);
122 4288 po_lputv(&b->raw, (po_u64)severity);
123 4288 po_lputv(&b->raw, (po_u64)body_len);
124 4288 ▶ if (body_len && po_arena_put(&b->raw, body, body_len) == PO_ARENA_ERR)
Branch: 2 branches
BranchTrueFalseTotal
if (body_len && po_arena_put(&b->raw, body, body_len) == PO_ARENA_ERR) 4271 17 4288
if (body_len && po_arena_put(&b->raw, body, body_len) == PO_ARENA_ERR) 0 4271 4271
125 0 return 0;
126  
127 4288 ▶ if (po_trace_id_valid(tr_hi, tr_lo)) flags |= 1;
Branch: 2 branches
BranchTrueFalseTotal
if (po_trace_id_valid(tr_hi, tr_lo)) flags |= 1; 4287 1 4288
if (po_trace_id_valid(tr_hi, tr_lo)) flags |= 1; 0 4287 4287
128 4288 ▶ if (span) flags |= 2;
Branch: 1 branch
BranchTrueFalseTotal
if (span) flags |= 2; 2 4286 4288
129 4288 po_arena_put(&b->raw, (const char *)&flags, 1);
130 4288 ▶ if (flags & 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (flags & 1) { 1 4287 4288
131 char idb[16]; int i;
132 9 ▶ for (i = 0; i < 8; i++) idb[i] = (char)((tr_hi >> (56 - i * 8)) & 0xFF);
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 8; i++) idb[i] = (char)((tr_hi >> (56 - i * 8)) & 0xFF); 8 1 9
133 9 ▶ for (i = 0; i < 8; i++) idb[8 + i] = (char)((tr_lo >> (56 - i * 8)) & 0xFF);
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 8; i++) idb[8 + i] = (char)((tr_lo >> (56 - i * 8)) & 0xFF); 8 1 9
134 1 po_arena_put(&b->raw, idb, 16);
135 }
136 4288 ▶ if (flags & 2) {
Branch: 1 branch
BranchTrueFalseTotal
if (flags & 2) { 2 4286 4288
137 char sb[8]; int i;
138 18 ▶ for (i = 0; i < 8; i++) sb[i] = (char)((span >> (56 - i * 8)) & 0xFF);
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 8; i++) sb[i] = (char)((span >> (56 - i * 8)) & 0xFF); 16 2 18
139 2 po_arena_put(&b->raw, sb, 8);
140 }
141  
142 4288 ▶ if (t < b->h.t_min) b->h.t_min = t;
Branch: 1 branch
BranchTrueFalseTotal
if (t < b->h.t_min) b->h.t_min = t; 2 4286 4288
143 4288 ▶ if (t > b->h.t_max) b->h.t_max = t;
Branch: 1 branch
BranchTrueFalseTotal
if (t > b->h.t_max) b->h.t_max = t; 4285 3 4288
144 4288 b->t_last = t;
145 4288 b->h.lines++;
146 4288 return 1;
147 }
148  
149 /* Seal: build the bloom over the block's bodies, then deflate.
150 *
151 * The bloom is built at seal so it can be SIZED from the block's actual
152 * distinct-trigram count rather than from a guess made before any line
153 * arrived. */
154 34 static int po_block_seal(po_block_w *b, char **out, size_t *outlen) {
155 po_tri_count *tc;
156 uint32_t distinct;
157  
158 /* Count distinct trigrams over the whole raw buffer. Counting over the
159 * bodies only would be tighter, but the buffer is a superset and
160 * over-sizing a bloom costs bits, while under-sizing it costs the false
161 * -negative rate this file exists to keep at zero. */
162 34 tc = (po_tri_count *)calloc(1, sizeof(po_tri_count));
163 34 ▶ if (!tc) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!tc) return 0; 0 34 34
164 34 po_trigrams(b->raw.base, b->raw.len, po_tri_count_cb, tc);
165 34 distinct = tc->n;
166 34 free(tc);
167  
168 34 ▶ if (!po_bloom_init(&b->bloom, distinct)) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!po_bloom_init(&b->bloom, distinct)) return 0; 0 34 34
169 34 b->have_bloom = 1;
170 34 po_bloom_add_text(&b->bloom, b->raw.base, b->raw.len);
171 34 b->h.bloom_bits = b->bloom.nbits;
172  
173 34 b->h.raw_len = (uint32_t)b->raw.len;
174 34 b->h.crc = po_crc32c(0, b->raw.base, b->raw.len);
175  
176 #ifdef PO_HAVE_ZLIB
177 {
178 z_stream z;
179 uLongf cap;
180 char *cbuf;
181 34 memset(&z, 0, sizeof(z));
182 /* -15: RAW deflate, no zlib or gzip wrapper. */
183 34 ▶ if (deflateInit2(&z, 6, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK)
Branch: 1 branch
BranchTrueFalseTotal
if (deflateInit2(&z, 6, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK) 0 34 34
184 0 return 0;
185 34 cap = deflateBound(&z, (uLong)b->raw.len) + 16;
186 34 cbuf = (char *)malloc(cap);
187 34 ▶ if (!cbuf) { deflateEnd(&z); return 0; }
Branch: 1 branch
BranchTrueFalseTotal
if (!cbuf) { deflateEnd(&z); return 0; } 0 34 34
188 34 z.next_in = (Bytef *)b->raw.base;
189 34 z.avail_in = (uInt)b->raw.len;
190 34 z.next_out = (Bytef *)cbuf;
191 34 z.avail_out = (uInt)cap;
192 34 ▶ if (deflate(&z, Z_FINISH) != Z_STREAM_END) {
Branch: 1 branch
BranchTrueFalseTotal
if (deflate(&z, Z_FINISH) != Z_STREAM_END) { 0 34 34
193 0 deflateEnd(&z); free(cbuf); return 0;
194 }
195 34 b->h.comp_len = (uint32_t)z.total_out;
196 34 deflateEnd(&z);
197  
198 /* If deflate did not help, store raw and SAY SO. A block that claims
199 * to be compressed and is not would decompress to nothing. */
200 34 ▶ if (b->h.comp_len >= b->h.raw_len) {
Branch: 1 branch
BranchTrueFalseTotal
if (b->h.comp_len >= b->h.raw_len) { 19 15 34
201 19 free(cbuf);
202 19 b->h.flags |= PO_BLK_STORED;
203 19 b->h.comp_len = b->h.raw_len;
204 19 ▶ *out = (char *)malloc(b->raw.len ? b->raw.len : 1);
Branch: 1 branch
BranchTrueFalseTotal
*out = (char *)malloc(b->raw.len ? b->raw.len : 1); 19 0 19
205 19 ▶ if (!*out) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!*out) return 0; 0 19 19
206 19 memcpy(*out, b->raw.base, b->raw.len);
207 19 *outlen = b->raw.len;
208 19 return 1;
209 }
210 15 *out = cbuf;
211 15 *outlen = b->h.comp_len;
212 15 return 1;
213 }
214 #else
215 b->h.flags |= PO_BLK_STORED;
216 b->h.comp_len = b->h.raw_len;
217 *out = (char *)malloc(b->raw.len ? b->raw.len : 1);
218 if (!*out) return 0;
219 memcpy(*out, b->raw.base, b->raw.len);
220 *outlen = b->raw.len;
221 return 1;
222 #endif
223 }
224  
225 /* ---- reading -------------------------------------------------------------- */
226  
227 /* Inflate a block. The caller owns the result. The CRC over the RAW bytes is
228 * checked here, so corruption is caught before a single line is handed on. */
229 16 static int po_block_inflate(const po_block_hdr *h, const char *comp,
230 size_t comp_len, char **out) {
231 char *raw;
232  
233 16 ▶ if (h->flags & PO_BLK_STORED) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->flags & PO_BLK_STORED) { 2 14 16
234 2 ▶ if (comp_len != h->raw_len) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (comp_len != h->raw_len) return 0; 0 2 2
235 2 ▶ raw = (char *)malloc(h->raw_len ? h->raw_len : 1);
Branch: 1 branch
BranchTrueFalseTotal
raw = (char *)malloc(h->raw_len ? h->raw_len : 1); 2 0 2
236 2 ▶ if (!raw) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!raw) return 0; 0 2 2
237 2 memcpy(raw, comp, h->raw_len);
238 }
239 else {
240 #ifdef PO_HAVE_ZLIB
241 z_stream z;
242 14 memset(&z, 0, sizeof(z));
243 20 ▶ if (inflateInit2(&z, -15) != Z_OK) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (inflateInit2(&z, -15) != Z_OK) return 0; 0 14 14
244 14 ▶ raw = (char *)malloc(h->raw_len ? h->raw_len : 1);
Branch: 1 branch
BranchTrueFalseTotal
raw = (char *)malloc(h->raw_len ? h->raw_len : 1); 14 0 14
245 14 ▶ if (!raw) { inflateEnd(&z); return 0; }
Branch: 1 branch
BranchTrueFalseTotal
if (!raw) { inflateEnd(&z); return 0; } 0 14 14
246 14 z.next_in = (Bytef *)comp;
247 14 z.avail_in = (uInt)comp_len;
248 14 z.next_out = (Bytef *)raw;
249 14 z.avail_out = (uInt)h->raw_len;
250 14 ▶ if (inflate(&z, Z_FINISH) != Z_STREAM_END || z.total_out != h->raw_len) {
Branch: 2 branches
BranchTrueFalseTotal
if (inflate(&z, Z_FINISH) != Z_STREAM_END || z.total_out != h->raw_len) { 8 6 14
if (inflate(&z, Z_FINISH) != Z_STREAM_END || z.total_out != h->raw_len) { 0 8 8
251 6 inflateEnd(&z); free(raw); return 0;
252 }
253 8 inflateEnd(&z);
254 #else
255 return 0;
256 #endif
257 }
258  
259 10 ▶ if (po_crc32c(0, raw, h->raw_len) != h->crc) { free(raw); return 0; }
Branch: 1 branch
BranchTrueFalseTotal
if (po_crc32c(0, raw, h->raw_len) != h->crc) { free(raw); return 0; } 0 10 10
260 10 *out = raw;
261 10 return 1;
262 }
263  
264 /* Walk the lines of an inflated block. */
265 typedef struct {
266 const unsigned char *b;
267 size_t len, pos;
268 po_u64 t;
269 uint32_t i, n;
270 } po_line_r;
271  
272 9 static void po_line_r_init(po_line_r *r, const po_block_hdr *h,
273 const char *raw) {
274 9 memset(r, 0, sizeof(*r));
275 9 r->b = (const unsigned char *)raw;
276 9 r->len = h->raw_len;
277 9 r->n = h->lines;
278 9 r->t = h->t_min;
279 9 }
280  
281 typedef struct {
282 po_u64 t;
283 uint16_t severity;
284 const char *body;
285 size_t body_len;
286 po_u64 trace_hi, trace_lo, span_id;
287 int has_trace;
288 } po_line;
289  
290 3050 static int po_line_next(po_line_r *r, po_line *ln) {
291 po_u64 dt, sev, blen;
292 unsigned char flags;
293  
294 3050 ▶ if (r->i >= r->n || r->pos >= r->len) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (r->i >= r->n || r->pos >= r->len) return 0; 3041 9 3050
if (r->i >= r->n || r->pos >= r->len) return 0; 0 3041 3041
295  
296 3041 dt = po_lgetv(r->b, r->len, &r->pos);
297 3041 sev = po_lgetv(r->b, r->len, &r->pos);
298 3041 blen = po_lgetv(r->b, r->len, &r->pos);
299 3041 ▶ if (r->pos + blen > r->len) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (r->pos + blen > r->len) return 0; 0 3041 3041
300  
301 3041 ▶ r->t = (r->i == 0) ? r->t : r->t + dt;
Branch: 1 branch
BranchTrueFalseTotal
r->t = (r->i == 0) ? r->t : r->t + dt; 9 3032 3041
302  
303 3041 memset(ln, 0, sizeof(*ln));
304 3041 ln->t = r->t;
305 3041 ln->severity = (uint16_t)sev;
306 3041 ln->body = (const char *)r->b + r->pos;
307 3041 ln->body_len = (size_t)blen;
308 3041 r->pos += (size_t)blen;
309  
310 3041 ▶ if (r->pos >= r->len) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (r->pos >= r->len) return 0; 0 3041 3041
311 3041 flags = r->b[r->pos++];
312 3041 ▶ if (flags & 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (flags & 1) { 1 3040 3041
313 int i;
314 1 ▶ if (r->pos + 16 > r->len) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (r->pos + 16 > r->len) return 0; 0 1 1
315 9 ▶ for (i = 0; i < 8; i++) ln->trace_hi = (ln->trace_hi << 8) | r->b[r->pos + i];
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 8; i++) ln->trace_hi = (ln->trace_hi << 8) | r->b[r->pos + i]; 8 1 9
316 9 ▶ for (i = 0; i < 8; i++) ln->trace_lo = (ln->trace_lo << 8) | r->b[r->pos + 8 + i];
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 8; i++) ln->trace_lo = (ln->trace_lo << 8) | r->b[r->pos + 8 + i]; 8 1 9
317 1 r->pos += 16;
318 1 ln->has_trace = 1;
319 }
320 3041 ▶ if (flags & 2) {
Branch: 1 branch
BranchTrueFalseTotal
if (flags & 2) { 2 3039 3041
321 int i;
322 2 ▶ if (r->pos + 8 > r->len) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (r->pos + 8 > r->len) return 0; 0 2 2
323 18 ▶ for (i = 0; i < 8; i++) ln->span_id = (ln->span_id << 8) | r->b[r->pos + i];
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 8; i++) ln->span_id = (ln->span_id << 8) | r->b[r->pos + i]; 16 2 18
324 2 r->pos += 8;
325 }
326 3041 r->i++;
327 3041 return 1;
328 }
329  
330 #endif /* PO_LOG_H */
« include/punk_observe/po_traceidx.h ↑ Summary include/punk_observe/po_route.h »