include/punk_observe/po_alert.h

filter: Statementstmt 92.1% 117 / 127 Branchbran 70.2% 59 / 84 Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 83.4% 176 / 211 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« include/punk_observe/po_rec.h ↑ Summary include/punk_observe/po_manifest.h »
1 /* po_alert.h - rule evaluation, and the two states everybody forgets.
2 *
3 * A rule is an OQL string executed on the same path the dashboard uses, so
4 * an alert cannot fire on a different answer than the graph shows. That part
5 * is phases 8 and 9. This file is what happens to the ANSWER.
6 *
7 * STATE IS PER SERIES.
8 *
9 * A rule grouped `by service` produces many series and each carries its own
10 * state. One state per rule is the bug that makes an alert resolve because a
11 * DIFFERENT service recovered, and it is easy to write because a rule feels
12 * like one thing.
13 *
14 * THE TWO THAT GET MISSED, AND WHY EACH IS DANGEROUS.
15 *
16 * A VANISHED SERIES MUST NOT STAY FIRING. A pod is deleted, its series
17 * stops being reported, and the naive implementation leaves it red for
18 * ever. A permanently red dashboard is how alerting loses its audience,
19 * and once it has, the real alert is not read either.
20 *
21 * AN EVALUATION ERROR IS NOT "OK". A rule whose query fails - a store
22 * error, a budget refusal, a bad threshold - must go to `error` and
23 * notify, not report healthy. This one is most likely to be written by
24 * accident, because the natural code path treats "no rows" and "no answer"
25 * identically, and the result is a system that reports green because it
26 * could not look.
27 *
28 * So the evaluation input carries a STATUS as well as rows, and this file
29 * refuses to collapse the two.
30 */
31 #ifndef PO_ALERT_H
32 #define PO_ALERT_H
33  
34 #include "punk_observe/po_compat.h"
35 #include "punk_observe/po_time.h"
36  
37 #define PO_AL_OK 0
38 #define PO_AL_PENDING 1
39 #define PO_AL_FIRING 2
40 #define PO_AL_STALE 3
41 #define PO_AL_ERROR 4
42  
43 /* What an evaluation produced. NOT a boolean: see the header comment. */
44 #define PO_EVAL_OK 0 /* the query ran; the rows are what matched */
45 #define PO_EVAL_FAIL 1 /* the query did not run, or was refused */
46  
47 /* Comparison operators, spelled the same way the query language does. */
48 #define PO_AL_GT 1
49 #define PO_AL_GE 2
50 #define PO_AL_LT 3
51 #define PO_AL_LE 4
52 #define PO_AL_EQ 5
53 #define PO_AL_NE 6
54  
55 #define PO_AL_KEY_MAX 128
56 #define PO_AL_MAX_SERIES 1024
57  
58 /* What came out of a transition, for the router to deliver. */
59 #define PO_NOTE_FIRING 1
60 #define PO_NOTE_RESOLVED 2
61 #define PO_NOTE_VANISHED 3 /* resolved because the series stopped existing */
62 #define PO_NOTE_ERROR 4
63  
64 typedef struct {
65 int op;
66 double threshold;
67 po_u64 for_ns; /* how long the condition must hold CONTINUOUSLY */
68 po_u64 every_ns; /* how often this rule is evaluated */
69 } po_al_rule;
70  
71 typedef struct {
72 char key[PO_AL_KEY_MAX];
73 size_t key_len;
74 int state;
75 po_u64 since; /* when the condition began holding */
76 po_u64 fired_at; /* when it entered firing; part of the dedupe key */
77 po_u64 last_seen; /* the last evaluation that carried this series */
78 double last_value;
79 int notified_error; /* an error notifies ONCE, not every tick */
80 } po_al_series;
81  
82 typedef struct {
83 char key[PO_AL_KEY_MAX];
84 size_t key_len;
85 int kind;
86 int from, to;
87 po_u64 at;
88 po_u64 fired_at;
89 double value;
90 } po_al_note;
91  
92 typedef struct {
93 po_al_series s[PO_AL_MAX_SERIES];
94 uint32_t n;
95 po_al_note note[PO_AL_MAX_SERIES];
96 uint32_t nnote;
97 } po_al_state;
98  
99 /* One row of an evaluation result. */
100 typedef struct {
101 const char *key; size_t key_len;
102 double value;
103 } po_al_row;
104  
105 45 static void po_al_init(po_al_state *st) { memset(st, 0, sizeof(*st)); }
106  
107 80 static int po_al_cmp(double v, int op, double t) {
108 80 switch (op) {
109 68 case PO_AL_GT: return v > t;
110 4 case PO_AL_GE: return v >= t;
111 2 case PO_AL_LT: return v < t;
112 2 case PO_AL_LE: return v <= t;
113 2 case PO_AL_EQ: return v == t;
114 2 case PO_AL_NE: return v != t;
115 0 default: return 0;
116 }
117 }
118  
119 90 static po_al_series *po_al_find(po_al_state *st, const char *k, size_t klen,
120 int create) {
121 uint32_t i;
122 90 ▶ if (klen > PO_AL_KEY_MAX) klen = PO_AL_KEY_MAX;
Branch: 1 branch
BranchTrueFalseTotal
if (klen > PO_AL_KEY_MAX) klen = PO_AL_KEY_MAX; 0 90 90
123 96 ▶ for (i = 0; i < st->n; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < st->n; i++) 49 47 96
124 49 ▶ if (st->s[i].key_len == klen &&
Branch: 2 branches
BranchTrueFalseTotal
if (st->s[i].key_len == klen && 49 0 49
if (st->s[i].key_len == klen && 42 7 49
125 49 ▶ (klen == 0 || memcmp(st->s[i].key, k, klen) == 0)) return &st->s[i];
Branch: 1 branch
BranchTrueFalseTotal
(klen == 0 || memcmp(st->s[i].key, k, klen) == 0)) return &st->s[i]; 36 6 42
126 47 ▶ if (!create || st->n >= PO_AL_MAX_SERIES) return NULL;
Branch: 2 branches
BranchTrueFalseTotal
if (!create || st->n >= PO_AL_MAX_SERIES) return NULL; 47 0 47
if (!create || st->n >= PO_AL_MAX_SERIES) return NULL; 0 47 47
127 47 memset(&st->s[st->n], 0, sizeof(st->s[0]));
128 47 ▶ if (klen) memcpy(st->s[st->n].key, k, klen);
Branch: 1 branch
BranchTrueFalseTotal
if (klen) memcpy(st->s[st->n].key, k, klen); 26 21 47
129 47 st->s[st->n].key_len = klen;
130 47 st->s[st->n].state = PO_AL_OK;
131 47 return &st->s[st->n++];
132 }
133  
134 39 static void po_al_emit(po_al_state *st, const po_al_series *s, int kind,
135 int from, int to, po_u64 at, double value) {
136 po_al_note *n;
137 39 ▶ if (st->nnote >= PO_AL_MAX_SERIES) return;
Branch: 1 branch
BranchTrueFalseTotal
if (st->nnote >= PO_AL_MAX_SERIES) return; 0 39 39
138 39 n = &st->note[st->nnote++];
139 39 memset(n, 0, sizeof(*n));
140 39 memcpy(n->key, s->key, s->key_len);
141 39 n->key_len = s->key_len;
142 39 n->kind = kind;
143 39 n->from = from;
144 39 n->to = to;
145 39 n->at = at;
146 39 n->fired_at = s->fired_at;
147 39 n->value = value;
148 }
149  
150 /* One evaluation.
151 *
152 * `status` is PO_EVAL_OK or PO_EVAL_FAIL, and the difference is the whole
153 * point: a result with NO ROWS means every known series is absent, which is
154 * `stale`. A FAILED evaluation means nothing was learned about any of them,
155 * which is `error`. Collapsing the two is how an alerting system reports
156 * green because it could not look.
157 *
158 * Notes accumulate in `st->note`; the caller drains them. */
159 93 static void po_al_step(po_al_state *st, const po_al_rule *r,
160 int status, const po_al_row *rows, uint32_t nrows) {
161 93 po_u64 now = po_now_ns();
162 uint32_t i;
163  
164 93 st->nnote = 0;
165  
166 93 ▶ if (status == PO_EVAL_FAIL) {
Branch: 1 branch
BranchTrueFalseTotal
if (status == PO_EVAL_FAIL) { 15 78 93
167 /* Every series the rule knows about goes to error, and each notifies
168 * ONCE. A rule that pages every tick while a store is down is a rule
169 * that gets silenced, which is the same as not having it. */
170 20 ▶ for (i = 0; i < st->n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < st->n; i++) { 5 15 20
171 5 po_al_series *s = &st->s[i];
172 5 int from = s->state;
173 5 ▶ if (s->state != PO_AL_ERROR) {
Branch: 1 branch
BranchTrueFalseTotal
if (s->state != PO_AL_ERROR) { 4 1 5
174 4 s->state = PO_AL_ERROR;
175 4 s->since = now;
176 }
177 5 ▶ if (!s->notified_error) {
Branch: 1 branch
BranchTrueFalseTotal
if (!s->notified_error) { 4 1 5
178 4 s->notified_error = 1;
179 4 po_al_emit(st, s, PO_NOTE_ERROR, from, PO_AL_ERROR, now,
180 s->last_value);
181 }
182 }
183 /* A rule with no series yet still has to be able to report that it
184 * cannot evaluate, so a synthetic entry carries it. */
185 15 ▶ if (st->n == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (st->n == 0) { 10 5 15
186 10 po_al_series *s = po_al_find(st, "", 0, 1);
187 10 ▶ if (s && !s->notified_error) {
Branch: 2 branches
BranchTrueFalseTotal
if (s && !s->notified_error) { 10 0 10
if (s && !s->notified_error) { 10 0 10
188 10 s->state = PO_AL_ERROR;
189 10 s->since = now;
190 10 s->notified_error = 1;
191 10 po_al_emit(st, s, PO_NOTE_ERROR, PO_AL_OK, PO_AL_ERROR, now, 0);
192 }
193 }
194 15 return;
195 }
196  
197 /* Present series first. */
198 158 ▶ for (i = 0; i < nrows; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < nrows; i++) { 80 78 158
199 80 po_al_series *s = po_al_find(st, rows[i].key, rows[i].key_len, 1);
200 int cond, from;
201 80 ▶ if (!s) continue;
Branch: 1 branch
BranchTrueFalseTotal
if (!s) continue; 0 80 80
202 80 cond = po_al_cmp(rows[i].value, r->op, r->threshold);
203 80 from = s->state;
204 80 s->last_seen = now;
205 80 s->last_value = rows[i].value;
206  
207 /* A recovered evaluation clears the once-only error latch, so the
208 * NEXT failure notifies again rather than being swallowed. */
209 80 s->notified_error = 0;
210  
211 80 ▶ if (s->state == PO_AL_ERROR || s->state == PO_AL_STALE) {
Branch: 2 branches
BranchTrueFalseTotal
if (s->state == PO_AL_ERROR || s->state == PO_AL_STALE) { 79 1 80
if (s->state == PO_AL_ERROR || s->state == PO_AL_STALE) { 1 78 79
212 /* Back from the dead. Re-enter the machine from ok so the `for`
213 * window is measured afresh rather than from a stale `since`. */
214 2 s->state = PO_AL_OK;
215 2 s->since = 0;
216 2 from = s->state;
217 }
218  
219 80 switch (s->state) {
220 47 case PO_AL_OK:
221 47 ▶ if (cond) { s->state = PO_AL_PENDING; s->since = now; }
Branch: 1 branch
BranchTrueFalseTotal
if (cond) { s->state = PO_AL_PENDING; s->since = now; } 28 19 47
222 47 break;
223  
224 28 case PO_AL_PENDING:
225 28 ▶ if (!cond) {
Branch: 1 branch
BranchTrueFalseTotal
if (!cond) { 2 26 28
226 /* THE TRANSITION THAT SENDS NOTHING, and the entire
227 * purpose of `for`. Notifying here produces exactly the
228 * flapping the setting exists to prevent. */
229 2 s->state = PO_AL_OK;
230 2 s->since = 0;
231 }
232 26 ▶ else if (now - s->since >= r->for_ns) {
Branch: 1 branch
BranchTrueFalseTotal
else if (now - s->since >= r->for_ns) { 21 5 26
233 21 s->state = PO_AL_FIRING;
234 21 s->fired_at = now;
235 21 po_al_emit(st, s, PO_NOTE_FIRING, from, PO_AL_FIRING, now,
236 21 rows[i].value);
237 }
238 28 break;
239  
240 5 case PO_AL_FIRING:
241 5 ▶ if (!cond) {
Branch: 1 branch
BranchTrueFalseTotal
if (!cond) { 3 2 5
242 3 s->state = PO_AL_OK;
243 3 s->since = 0;
244 3 po_al_emit(st, s, PO_NOTE_RESOLVED, from, PO_AL_OK, now,
245 3 rows[i].value);
246 3 s->fired_at = 0;
247 }
248 5 break;
249  
250 0 default: break;
251 }
252 }
253  
254 /* Then the ones that were NOT in the result.
255 *
256 * Absent is not false. A series that stops being reported goes stale, and
257 * leaves firing when it does - the alternative is a dashboard that stays
258 * red for a pod that was deleted last Tuesday. */
259 162 ▶ for (i = 0; i < st->n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < st->n; i++) { 84 78 162
260 84 po_al_series *s = &st->s[i];
261 uint32_t j;
262 84 int present = 0;
263 90 ▶ for (j = 0; j < nrows; j++) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = 0; j < nrows; j++) { 86 4 90
264 86 size_t kl = rows[j].key_len > PO_AL_KEY_MAX
265 ? PO_AL_KEY_MAX : rows[j].key_len;
266 86 ▶ if (kl == s->key_len &&
Branch: 2 branches
BranchTrueFalseTotal
if (kl == s->key_len && 86 0 86
if (kl == s->key_len && 68 18 86
267 68 ▶ (kl == 0 || memcmp(rows[j].key, s->key, kl) == 0)) {
Branch: 1 branch
BranchTrueFalseTotal
(kl == 0 || memcmp(rows[j].key, s->key, kl) == 0)) { 62 6 68
268 80 present = 1; break;
269 }
270 }
271 84 ▶ if (present) continue;
Branch: 1 branch
BranchTrueFalseTotal
if (present) continue; 80 4 84
272  
273 4 ▶ if (s->state != PO_AL_STALE && s->state != PO_AL_OK) {
Branch: 2 branches
BranchTrueFalseTotal
if (s->state != PO_AL_STALE && s->state != PO_AL_OK) { 3 1 4
if (s->state != PO_AL_STALE && s->state != PO_AL_OK) { 3 0 3
274 3 s->state = PO_AL_STALE;
275 3 ▶ if (!s->since) s->since = now;
Branch: 1 branch
BranchTrueFalseTotal
if (!s->since) s->since = now; 0 3 3
276 3 continue;
277 }
278 1 ▶ if (s->state == PO_AL_STALE) {
Branch: 1 branch
BranchTrueFalseTotal
if (s->state == PO_AL_STALE) { 1 0 1
279 /* Two evaluation intervals of silence is the bound. It is a
280 * multiple of `every` rather than a constant, so a rule that runs
281 * hourly is not declared stale after a minute. */
282 1 ▶ po_u64 quiet = now > s->last_seen ? now - s->last_seen : 0;
Branch: 1 branch
BranchTrueFalseTotal
po_u64 quiet = now > s->last_seen ? now - s->last_seen : 0; 1 0 1
283 1 ▶ if (quiet >= 2 * r->every_ns) {
Branch: 1 branch
BranchTrueFalseTotal
if (quiet >= 2 * r->every_ns) { 1 0 1
284 1 int from = s->state;
285 1 int was_firing = s->fired_at != 0;
286 1 s->state = PO_AL_OK;
287 1 s->since = 0;
288 1 ▶ if (was_firing) {
Branch: 1 branch
BranchTrueFalseTotal
if (was_firing) { 1 0 1
289 /* A firing alert that simply stopped existing still owes
290 * the reader a resolution, and one that says WHY: this is
291 * not the same event as the condition clearing. */
292 1 po_al_emit(st, s, PO_NOTE_VANISHED, from, PO_AL_OK, now,
293 s->last_value);
294 1 s->fired_at = 0;
295 }
296 }
297 }
298 }
299 }
300  
301 /* A firing series must go stale the moment it is missing, which the loop
302 * above does. This is the question a UI asks. */
303 0 static int po_al_state_of(const po_al_state *st, const char *k, size_t klen) {
304 uint32_t i;
305 0 ▶ if (klen > PO_AL_KEY_MAX) klen = PO_AL_KEY_MAX;
Branch: 1 branch
BranchTrueFalseTotal
if (klen > PO_AL_KEY_MAX) klen = PO_AL_KEY_MAX; 0 0 0
306 0 ▶ for (i = 0; i < st->n; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < st->n; i++) 0 0 0
307 0 ▶ if (st->s[i].key_len == klen &&
Branch: 2 branches
BranchTrueFalseTotal
if (st->s[i].key_len == klen && 0 0 0
if (st->s[i].key_len == klen && 0 0 0
308 0 ▶ (klen == 0 || memcmp(st->s[i].key, k, klen) == 0))
Branch: 1 branch
BranchTrueFalseTotal
(klen == 0 || memcmp(st->s[i].key, k, klen) == 0)) 0 0 0
309 0 return st->s[i].state;
310 0 return -1;
311 }
312  
313 177 static const char *po_al_state_name(int s) {
314 177 switch (s) {
315 42 case PO_AL_OK: return "ok";
316 54 case PO_AL_PENDING: return "pending";
317 48 case PO_AL_FIRING: return "firing";
318 4 case PO_AL_STALE: return "stale";
319 29 case PO_AL_ERROR: return "error";
320 0 default: return "unknown";
321 }
322 }
323  
324 #endif /* PO_ALERT_H */
« include/punk_observe/po_rec.h ↑ Summary include/punk_observe/po_manifest.h »