| 1 |
|
| /* po_metric.h - metric chunks, series metadata, exemplars. |
| 2 |
|
| * |
| 3 |
|
| * A chunk is the unit of DECOMPRESSION, which is what sets its size: closed |
| 4 |
|
| * at 120 points or two hours, whichever comes first. Smaller means more |
| 5 |
|
| * directory and worse ratios; larger means a query for one minute |
| 6 |
|
| * decompresses ten. The chunk header carries the first timestamp and the |
| 7 |
|
| * point count uncompressed, so the index can skip a chunk without entering |
| 8 |
|
| * the bit stream at all. |
| 9 |
|
| */ |
| 10 |
|
| #ifndef PO_METRIC_H |
| 11 |
|
| #define PO_METRIC_H |
| 12 |
|
| |
| 13 |
|
| #include "punk_observe/po_compat.h" |
| 14 |
|
| #include "punk_observe/po_gorilla.h" |
| 15 |
|
| #include "punk_observe/po_rec.h" |
| 16 |
|
| |
| 17 |
|
| #define PO_CHUNK_POINTS 120 |
| 18 |
|
| #define PO_CHUNK_SPAN ((po_u64)2 * 3600 * 1000000000ULL) |
| 19 |
|
| |
| 20 |
|
| /* Chunk flags */ |
| 21 |
|
| #define PO_CH_INT 0x01 /* values are integers, not doubles */ |
| 22 |
|
| #define PO_CH_CUMULATIVE 0x02 |
| 23 |
|
| #define PO_CH_MONOTONIC 0x04 |
| 24 |
|
| #define PO_CH_RESET 0x08 /* a counter reset occurred inside */ |
| 25 |
|
| |
| 26 |
|
| typedef struct { |
| 27 |
|
| po_u64 t_first, t_last; |
| 28 |
|
| uint32_t count; |
| 29 |
|
| uint32_t bits; /* the bit stream's length, so a truncated |
| 30 |
|
| * chunk fails rather than decoding garbage */ |
| 31 |
|
| uint16_t flags; |
| 32 |
|
| } po_chunk_hdr; |
| 33 |
|
| |
| 34 |
|
| typedef struct { |
| 35 |
|
| po_gor_w g; |
| 36 |
|
| po_chunk_hdr h; |
| 37 |
|
| po_u64 v_last; /* for reset detection: the previous RAW value */ |
| 38 |
|
| int resets; |
| 39 |
|
| int is_int; |
| 40 |
|
| } po_chunk_w; |
| 41 |
|
| |
| 42 |
11 |
| static int po_chunk_w_init(po_chunk_w *c, int is_int, uint16_t flags) { |
| 43 |
11 |
| memset(c, 0, sizeof(*c)); |
| 44 |
11 |
| c->is_int = is_int; |
| 45 |
11 |
| c->h.flags = flags | (is_int ? PO_CH_INT : 0); |
| 46 |
11 |
| return po_gor_w_init(&c->g); |
| 47 |
|
| } |
| 48 |
|
| |
| 49 |
11 |
| static void po_chunk_w_free(po_chunk_w *c) { po_gor_w_free(&c->g); } |
| 50 |
|
| |
| 51 |
162 |
| static int po_chunk_full(const po_chunk_w *c, po_u64 t) { |
| 52 |
162 |
▶ | if (c->h.count >= PO_CHUNK_POINTS) return 1; |
Branch: 1 branch
| Branch | True | False | Total |
| if (c->h.count >= PO_CHUNK_POINTS) return 1; |
1 |
161 |
162 |
|
| 53 |
161 |
▶ | if (c->h.count && t >= c->h.t_first + PO_CHUNK_SPAN) return 1; |
Branch: 2 branches
| Branch | True | False | Total |
| if (c->h.count && t >= c->h.t_first + PO_CHUNK_SPAN) return 1; |
154 |
7 |
161 |
| if (c->h.count && t >= c->h.t_first + PO_CHUNK_SPAN) return 1; |
1 |
153 |
154 |
|
| 54 |
160 |
| return 0; |
| 55 |
|
| } |
| 56 |
|
| |
| 57 |
|
| /* A COUNTER RESET IS NOT A NEGATIVE RATE. |
| 58 |
|
| * |
| 59 |
|
| * A cumulative counter that goes backwards means the process restarted. The |
| 60 |
|
| * detection happens HERE, at encode time, and not in the query - because |
| 61 |
|
| * phase 10's rollups outlive the raw points, and a rate computed over a |
| 62 |
|
| * rolled-up range with an undetected reset inside it is simply wrong with |
| 63 |
|
| * nothing left to reveal it. */ |
| 64 |
220 |
| static int po_chunk_add(po_chunk_w *c, po_u64 t, po_u64 bits) { |
| 65 |
220 |
▶ | if (c->h.count == 0) c->h.t_first = t; |
Branch: 1 branch
| Branch | True | False | Total |
| if (c->h.count == 0) c->h.t_first = t; |
11 |
209 |
220 |
|
| 66 |
|
| |
| 67 |
220 |
▶ | if ((c->h.flags & PO_CH_MONOTONIC) && c->h.count) { |
Branch: 2 branches
| Branch | True | False | Total |
| if ((c->h.flags & PO_CH_MONOTONIC) && c->h.count) { |
28 |
192 |
220 |
| if ((c->h.flags & PO_CH_MONOTONIC) && c->h.count) { |
24 |
4 |
28 |
|
| 68 |
48 |
| int went_back = c->is_int |
| 69 |
6 |
| ? (bits < c->v_last) |
| 70 |
24 |
▶ | : (po_b2d(bits) < po_b2d(c->v_last)); |
Branch: 1 branch
| Branch | True | False | Total |
| : (po_b2d(bits) < po_b2d(c->v_last)); |
6 |
18 |
24 |
|
| 71 |
24 |
▶ | if (went_back) { c->h.flags |= PO_CH_RESET; c->resets++; } |
Branch: 1 branch
| Branch | True | False | Total |
| if (went_back) { c->h.flags |= PO_CH_RESET; c->resets++; } |
2 |
22 |
24 |
|
| 72 |
|
| } |
| 73 |
|
| |
| 74 |
220 |
| po_gor_put(&c->g, t, bits); |
| 75 |
220 |
| c->v_last = bits; |
| 76 |
220 |
| c->h.t_last = t; |
| 77 |
220 |
| c->h.count++; |
| 78 |
220 |
| c->h.bits = (uint32_t)c->g.bw.nbits; |
| 79 |
220 |
| return !c->g.bw.err; |
| 80 |
|
| } |
| 81 |
|
| |
| 82 |
|
| typedef struct { |
| 83 |
|
| po_gor_r g; |
| 84 |
|
| po_chunk_hdr h; |
| 85 |
|
| } po_chunk_r; |
| 86 |
|
| |
| 87 |
9 |
| static void po_chunk_r_init(po_chunk_r *c, const po_chunk_hdr *h, |
| 88 |
|
| const void *bits) { |
| 89 |
9 |
| memset(c, 0, sizeof(*c)); |
| 90 |
9 |
| c->h = *h; |
| 91 |
9 |
| po_gor_r_init(&c->g, bits, h->bits, h->count); |
| 92 |
9 |
| } |
| 93 |
|
| |
| 94 |
199 |
| static int po_chunk_next(po_chunk_r *c, po_u64 *t, po_u64 *bits) { |
| 95 |
199 |
| return po_gor_next(&c->g, t, bits); |
| 96 |
|
| } |
| 97 |
|
| |
| 98 |
|
| /* ---- exemplars ------------------------------------------------------------ |
| 99 |
|
| * |
| 100 |
|
| * (value, timestamp, trace_id, span_id) attached to a point. |
| 101 |
|
| * |
| 102 |
|
| * NOT a nicety and not optional: this is the ENTIRE mechanism behind the |
| 103 |
|
| * `| exemplars` stage, which is the cross-signal jump this project is being |
| 104 |
|
| * built for. A metric store that drops exemplars can never answer "what |
| 105 |
|
| * happened during that spike" without a human copying a trace id between |
| 106 |
|
| * tabs, which is the thing every other self-hosted stack makes you do. |
| 107 |
|
| */ |
| 108 |
|
| typedef struct { |
| 109 |
|
| po_u64 t; |
| 110 |
|
| po_u64 value_bits; |
| 111 |
|
| po_u64 trace_hi, trace_lo; |
| 112 |
|
| po_u64 span_id; |
| 113 |
|
| } po_exemplar; |
| 114 |
|
| |
| 115 |
|
| typedef struct { |
| 116 |
|
| po_exemplar *e; |
| 117 |
|
| uint32_t n, cap; |
| 118 |
|
| } po_exemplars; |
| 119 |
|
| |
| 120 |
2 |
| static int po_exemplars_init(po_exemplars *x) { |
| 121 |
2 |
| memset(x, 0, sizeof(*x)); |
| 122 |
2 |
| x->cap = 8; |
| 123 |
2 |
| x->e = (po_exemplar *)malloc(x->cap * sizeof(po_exemplar)); |
| 124 |
2 |
| return x->e != NULL; |
| 125 |
|
| } |
| 126 |
|
| |
| 127 |
2 |
| static void po_exemplars_free(po_exemplars *x) { |
| 128 |
2 |
| free(x->e); x->e = NULL; x->n = x->cap = 0; |
| 129 |
2 |
| } |
| 130 |
|
| |
| 131 |
503 |
| static int po_exemplar_add(po_exemplars *x, const po_exemplar *src) { |
| 132 |
|
| /* An exemplar with no trace id is useless: it points nowhere, and the |
| 133 |
|
| * whole point of the record is the jump. Dropped rather than stored. */ |
| 134 |
503 |
▶ | if (!po_trace_id_valid(src->trace_hi, src->trace_lo)) return 0; |
Branch: 2 branches
| Branch | True | False | Total |
| if (!po_trace_id_valid(src->trace_hi, src->trace_lo)) return 0; |
2 |
501 |
503 |
| if (!po_trace_id_valid(src->trace_hi, src->trace_lo)) return 0; |
1 |
1 |
2 |
|
| 135 |
502 |
▶ | if (x->n == x->cap) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (x->n == x->cap) { |
6 |
496 |
502 |
|
| 136 |
6 |
| uint32_t want = x->cap * 2; |
| 137 |
6 |
| po_exemplar *ne = (po_exemplar *)realloc(x->e, want * sizeof(po_exemplar)); |
| 138 |
6 |
▶ | if (!ne) return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!ne) return 0; |
0 |
6 |
6 |
|
| 139 |
6 |
| x->e = ne; x->cap = want; |
| 140 |
|
| } |
| 141 |
502 |
| x->e[x->n++] = *src; |
| 142 |
502 |
| return 1; |
| 143 |
|
| } |
| 144 |
|
| |
| 145 |
|
| #endif /* PO_METRIC_H */ |