| 1 |
|
| /* po_route.h - grouping, silences, and the outbox key. |
| 2 |
|
| * |
| 3 |
|
| * A DEPLOY BREAKS FORTY SERVICES AT ONCE. |
| 4 |
|
| * |
| 5 |
|
| * Forty firing series is one event, and delivering it as forty messages is |
| 6 |
|
| * how a channel gets muted. So notifications are grouped by a configured |
| 7 |
|
| * label set and held for `group_wait` before the first send: one message |
| 8 |
|
| * listing forty services rather than forty messages. |
| 9 |
|
| * |
| 10 |
|
| * That delay is deliberate and has to be SAID. Thirty seconds of latency on a |
| 11 |
|
| * page is a thing an operator needs to know in advance rather than discover |
| 12 |
|
| * during an incident, so the UI states it and this file makes it a |
| 13 |
|
| * configured number rather than a constant buried in a loop. |
| 14 |
|
| * |
| 15 |
|
| * A SILENCE SUPPRESSES NOTIFICATION AND NOT STATE. |
| 16 |
|
| * |
| 17 |
|
| * A silenced rule still reaches firing and still renders red; it just does |
| 18 |
|
| * not page. A silence that hides the state is how an incident is forgotten - |
| 19 |
|
| * somebody silences an alert to get through a deploy and the dashboard shows |
| 20 |
|
| * green for the rest of the week. |
| 21 |
|
| */ |
| 22 |
|
| #ifndef PO_ROUTE_H |
| 23 |
|
| #define PO_ROUTE_H |
| 24 |
|
| |
| 25 |
|
| #include "punk_observe/po_compat.h" |
| 26 |
|
| #include "punk_observe/po_time.h" |
| 27 |
|
| #include "punk_observe/po_alert.h" |
| 28 |
|
| |
| 29 |
|
| #define PO_GROUP_MAX 64 |
| 30 |
|
| #define PO_GROUP_MEMBERS 256 |
| 31 |
|
| #define PO_OUTBOX_MAX 4096 |
| 32 |
|
| |
| 33 |
|
| /* ---- the outbox key ------------------------------------------------------- |
| 34 |
|
| * |
| 35 |
|
| * (rule, series, fired_at). A retried send job recomputes the same key and is |
| 36 |
|
| * refused, so a delivery can never happen twice however many times the job |
| 37 |
|
| * runs. `fired_at` is in the key rather than a timestamp of the send, because |
| 38 |
|
| * a series that resolves and fires again is a NEW notification and must not |
| 39 |
|
| * be deduplicated against the old one. */ |
| 40 |
|
| typedef struct { |
| 41 |
|
| po_u64 rule; |
| 42 |
|
| char series[PO_AL_KEY_MAX]; |
| 43 |
|
| size_t series_len; |
| 44 |
|
| po_u64 fired_at; |
| 45 |
|
| int claimed; |
| 46 |
|
| } po_ob_ent; |
| 47 |
|
| |
| 48 |
|
| typedef struct { |
| 49 |
|
| po_ob_ent e[PO_OUTBOX_MAX]; |
| 50 |
|
| uint32_t n; |
| 51 |
|
| } po_outbox; |
| 52 |
|
| |
| 53 |
14 |
| static void po_outbox_init(po_outbox *o) { memset(o, 0, sizeof(*o)); } |
| 54 |
|
| |
| 55 |
101 |
| static po_ob_ent *po_outbox_find(po_outbox *o, po_u64 rule, |
| 56 |
|
| const char *s, size_t slen, po_u64 fired) { |
| 57 |
|
| uint32_t i; |
| 58 |
101 |
▶ | if (slen > PO_AL_KEY_MAX) slen = PO_AL_KEY_MAX; |
Branch: 1 branch
| Branch | True | False | Total |
| if (slen > PO_AL_KEY_MAX) slen = PO_AL_KEY_MAX; |
0 |
101 |
101 |
|
| 59 |
1713 |
▶ | for (i = 0; i < o->n; i++) |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < o->n; i++) |
1615 |
98 |
1713 |
|
| 60 |
1615 |
▶ | if (o->e[i].rule == rule && o->e[i].fired_at == fired |
Branch: 2 branches
| Branch | True | False | Total |
| if (o->e[i].rule == rule && o->e[i].fired_at == fired |
1614 |
1 |
1615 |
| if (o->e[i].rule == rule && o->e[i].fired_at == fired |
1613 |
1 |
1614 |
|
| 61 |
1613 |
▶ | && o->e[i].series_len == slen |
Branch: 1 branch
| Branch | True | False | Total |
| && o->e[i].series_len == slen |
1046 |
567 |
1613 |
|
| 62 |
1046 |
▶ | && (slen == 0 || memcmp(o->e[i].series, s, slen) == 0)) |
Branch: 2 branches
| Branch | True | False | Total |
| && (slen == 0 || memcmp(o->e[i].series, s, slen) == 0)) |
1046 |
0 |
1046 |
| && (slen == 0 || memcmp(o->e[i].series, s, slen) == 0)) |
3 |
1043 |
1046 |
|
| 63 |
3 |
| return &o->e[i]; |
| 64 |
98 |
| return NULL; |
| 65 |
|
| } |
| 66 |
|
| |
| 67 |
|
| /* Returns 1 when the notification was newly enqueued, 0 when the key was |
| 68 |
|
| * already present. Zero is the SUCCESS case for a retry: it means the work |
| 69 |
|
| * was already done, not that anything failed. */ |
| 70 |
101 |
| static int po_outbox_put(po_outbox *o, po_u64 rule, |
| 71 |
|
| const char *s, size_t slen, po_u64 fired) { |
| 72 |
|
| po_ob_ent *e; |
| 73 |
101 |
▶ | if (slen > PO_AL_KEY_MAX) slen = PO_AL_KEY_MAX; |
Branch: 1 branch
| Branch | True | False | Total |
| if (slen > PO_AL_KEY_MAX) slen = PO_AL_KEY_MAX; |
0 |
101 |
101 |
|
| 74 |
101 |
▶ | if (po_outbox_find(o, rule, s, slen, fired)) return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (po_outbox_find(o, rule, s, slen, fired)) return 0; |
3 |
98 |
101 |
|
| 75 |
98 |
▶ | if (o->n >= PO_OUTBOX_MAX) return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (o->n >= PO_OUTBOX_MAX) return 0; |
0 |
98 |
98 |
|
| 76 |
98 |
| e = &o->e[o->n++]; |
| 77 |
98 |
| memset(e, 0, sizeof(*e)); |
| 78 |
98 |
| e->rule = rule; |
| 79 |
98 |
▶ | if (slen) memcpy(e->series, s, slen); |
Branch: 1 branch
| Branch | True | False | Total |
| if (slen) memcpy(e->series, s, slen); |
98 |
0 |
98 |
|
| 80 |
98 |
| e->series_len = slen; |
| 81 |
98 |
| e->fired_at = fired; |
| 82 |
98 |
| return 1; |
| 83 |
|
| } |
| 84 |
|
| |
| 85 |
|
| /* Claim one unclaimed row. The database does this with FOR UPDATE SKIP |
| 86 |
|
| * LOCKED; the invariant either way is that a claimed row is not claimable by |
| 87 |
|
| * a second sender. */ |
| 88 |
126 |
| static po_ob_ent *po_outbox_claim(po_outbox *o) { |
| 89 |
|
| uint32_t i; |
| 90 |
1934 |
▶ | for (i = 0; i < o->n; i++) |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < o->n; i++) |
1906 |
28 |
1934 |
|
| 91 |
1906 |
▶ | if (!o->e[i].claimed) { o->e[i].claimed = 1; return &o->e[i]; } |
Branch: 1 branch
| Branch | True | False | Total |
| if (!o->e[i].claimed) { o->e[i].claimed = 1; return &o->e[i]; } |
98 |
1808 |
1906 |
|
| 92 |
28 |
| return NULL; |
| 93 |
|
| } |
| 94 |
|
| |
| 95 |
28 |
| static uint32_t po_outbox_pending(const po_outbox *o) { |
| 96 |
28 |
| uint32_t i, n = 0; |
| 97 |
224 |
▶ | for (i = 0; i < o->n; i++) if (!o->e[i].claimed) n++; |
Branch: 2 branches
| Branch | True | False | Total |
| for (i = 0; i < o->n; i++) if (!o->e[i].claimed) n++; |
98 |
98 |
196 |
| for (i = 0; i < o->n; i++) if (!o->e[i].claimed) n++; |
196 |
28 |
224 |
|
| 98 |
28 |
| return n; |
| 99 |
|
| } |
| 100 |
|
| |
| 101 |
|
| /* ---- grouping ------------------------------------------------------------- |
| 102 |
|
| * |
| 103 |
|
| * A group opens when its first member arrives and is DUE `group_wait` later. |
| 104 |
|
| * Members joining before it is due join the same message; one arriving after |
| 105 |
|
| * it has been sent opens a new group, which is what makes the forty-first |
| 106 |
|
| * service a second notification rather than a lost one. */ |
| 107 |
|
| typedef struct { |
| 108 |
|
| char key[PO_AL_KEY_MAX]; |
| 109 |
|
| size_t key_len; |
| 110 |
|
| char member[PO_GROUP_MEMBERS][PO_AL_KEY_MAX]; |
| 111 |
|
| size_t member_len[PO_GROUP_MEMBERS]; |
| 112 |
|
| uint32_t nmember; |
| 113 |
|
| uint32_t overflow; /* members past the cap, counted not dropped */ |
| 114 |
|
| po_u64 opened; |
| 115 |
|
| po_u64 last_sent; |
| 116 |
|
| int sent; |
| 117 |
|
| } po_ngroup; |
| 118 |
|
| |
| 119 |
|
| typedef struct { |
| 120 |
|
| po_ngroup g[PO_GROUP_MAX]; |
| 121 |
|
| uint32_t n; |
| 122 |
|
| po_u64 group_wait; |
| 123 |
|
| po_u64 repeat_interval; |
| 124 |
|
| } po_router; |
| 125 |
|
| |
| 126 |
14 |
| static void po_router_init(po_router *r, po_u64 wait, po_u64 repeat) { |
| 127 |
14 |
| memset(r, 0, sizeof(*r)); |
| 128 |
14 |
| r->group_wait = wait; |
| 129 |
14 |
| r->repeat_interval = repeat; |
| 130 |
14 |
| } |
| 131 |
|
| |
| 132 |
101 |
| static po_ngroup *po_group_for(po_router *r, const char *k, size_t klen) { |
| 133 |
|
| uint32_t i; |
| 134 |
101 |
| po_u64 now = po_now_ns(); |
| 135 |
101 |
▶ | if (klen > PO_AL_KEY_MAX) klen = PO_AL_KEY_MAX; |
Branch: 1 branch
| Branch | True | False | Total |
| if (klen > PO_AL_KEY_MAX) klen = PO_AL_KEY_MAX; |
0 |
101 |
101 |
|
| 136 |
118 |
▶ | for (i = 0; i < r->n; i++) |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < r->n; i++) |
95 |
23 |
118 |
|
| 137 |
95 |
▶ | if (r->g[i].key_len == klen && !r->g[i].sent |
Branch: 2 branches
| Branch | True | False | Total |
| if (r->g[i].key_len == klen && !r->g[i].sent |
94 |
1 |
95 |
| if (r->g[i].key_len == klen && !r->g[i].sent |
78 |
16 |
94 |
|
| 138 |
78 |
▶ | && (klen == 0 || memcmp(r->g[i].key, k, klen) == 0)) |
Branch: 2 branches
| Branch | True | False | Total |
| && (klen == 0 || memcmp(r->g[i].key, k, klen) == 0)) |
78 |
0 |
78 |
| && (klen == 0 || memcmp(r->g[i].key, k, klen) == 0)) |
78 |
0 |
78 |
|
| 139 |
78 |
| return &r->g[i]; |
| 140 |
23 |
▶ | if (r->n >= PO_GROUP_MAX) return NULL; |
Branch: 1 branch
| Branch | True | False | Total |
| if (r->n >= PO_GROUP_MAX) return NULL; |
0 |
23 |
23 |
|
| 141 |
23 |
| memset(&r->g[r->n], 0, sizeof(r->g[0])); |
| 142 |
23 |
▶ | if (klen) memcpy(r->g[r->n].key, k, klen); |
Branch: 1 branch
| Branch | True | False | Total |
| if (klen) memcpy(r->g[r->n].key, k, klen); |
23 |
0 |
23 |
|
| 143 |
23 |
| r->g[r->n].key_len = klen; |
| 144 |
23 |
| r->g[r->n].opened = now; |
| 145 |
23 |
| return &r->g[r->n++]; |
| 146 |
|
| } |
| 147 |
|
| |
| 148 |
|
| /* Add one firing series to its group. Silenced series are NOT added: the |
| 149 |
|
| * silence suppresses the notification, and the state machine has already |
| 150 |
|
| * recorded that it is firing. */ |
| 151 |
104 |
| static int po_route_add(po_router *r, const char *gkey, size_t gklen, |
| 152 |
|
| const char *series, size_t slen, int silenced) { |
| 153 |
|
| po_ngroup *g; |
| 154 |
104 |
▶ | if (silenced) return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (silenced) return 0; |
3 |
101 |
104 |
|
| 155 |
101 |
| g = po_group_for(r, gkey, gklen); |
| 156 |
101 |
▶ | if (!g) return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!g) return 0; |
0 |
101 |
101 |
|
| 157 |
101 |
▶ | if (slen > PO_AL_KEY_MAX) slen = PO_AL_KEY_MAX; |
Branch: 1 branch
| Branch | True | False | Total |
| if (slen > PO_AL_KEY_MAX) slen = PO_AL_KEY_MAX; |
0 |
101 |
101 |
|
| 158 |
101 |
▶ | if (g->nmember >= PO_GROUP_MEMBERS) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (g->nmember >= PO_GROUP_MEMBERS) { |
0 |
101 |
101 |
|
| 159 |
|
| /* Counted rather than dropped. A message saying "and 40 more" is an |
| 160 |
|
| * answer; a message silently listing 256 of 296 is a lie. */ |
| 161 |
0 |
| g->overflow++; |
| 162 |
0 |
| return 1; |
| 163 |
|
| } |
| 164 |
101 |
▶ | if (slen) memcpy(g->member[g->nmember], series, slen); |
Branch: 1 branch
| Branch | True | False | Total |
| if (slen) memcpy(g->member[g->nmember], series, slen); |
101 |
0 |
101 |
|
| 165 |
101 |
| g->member_len[g->nmember] = slen; |
| 166 |
101 |
| g->nmember++; |
| 167 |
101 |
| return 1; |
| 168 |
|
| } |
| 169 |
|
| |
| 170 |
|
| /* Is this group due to send? */ |
| 171 |
166 |
| static int po_group_due(const po_router *r, const po_ngroup *g) { |
| 172 |
166 |
| po_u64 now = po_now_ns(); |
| 173 |
166 |
▶ | if (g->sent) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (g->sent) { |
61 |
105 |
166 |
|
| 174 |
61 |
▶ | if (!r->repeat_interval) return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (!r->repeat_interval) return 0; |
57 |
4 |
61 |
|
| 175 |
4 |
| return now - g->last_sent >= r->repeat_interval; |
| 176 |
|
| } |
| 177 |
105 |
| return now - g->opened >= r->group_wait; |
| 178 |
|
| } |
| 179 |
|
| |
| 180 |
134 |
| static po_ngroup *po_router_next_due(po_router *r) { |
| 181 |
|
| uint32_t i; |
| 182 |
277 |
▶ | for (i = 0; i < r->n; i++) |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < r->n; i++) |
166 |
111 |
277 |
|
| 183 |
166 |
▶ | if (r->g[i].nmember && po_group_due(r, &r->g[i])) return &r->g[i]; |
Branch: 2 branches
| Branch | True | False | Total |
| if (r->g[i].nmember && po_group_due(r, &r->g[i])) return &r->g[i]; |
166 |
0 |
166 |
| if (r->g[i].nmember && po_group_due(r, &r->g[i])) return &r->g[i]; |
23 |
143 |
166 |
|
| 184 |
111 |
| return NULL; |
| 185 |
|
| } |
| 186 |
|
| |
| 187 |
23 |
| static void po_group_mark_sent(po_router *r, po_ngroup *g) { |
| 188 |
|
| (void)r; |
| 189 |
23 |
| g->sent = 1; |
| 190 |
23 |
| g->last_sent = po_now_ns(); |
| 191 |
23 |
| } |
| 192 |
|
| |
| 193 |
|
| /* ---- silences ------------------------------------------------------------- |
| 194 |
|
| * |
| 195 |
|
| * A silence matches a series by exact key or by prefix, and expires. An |
| 196 |
|
| * expired silence must not keep suppressing: a silence set for a deploy and |
| 197 |
|
| * forgotten is how a real page goes unsent for a month. */ |
| 198 |
|
| typedef struct { |
| 199 |
|
| char pat[PO_AL_KEY_MAX]; |
| 200 |
|
| size_t pat_len; |
| 201 |
|
| int prefix; |
| 202 |
|
| po_u64 until; |
| 203 |
|
| } po_silence; |
| 204 |
|
| |
| 205 |
|
| #define PO_SILENCE_MAX 128 |
| 206 |
|
| typedef struct { po_silence s[PO_SILENCE_MAX]; uint32_t n; } po_silences; |
| 207 |
|
| |
| 208 |
14 |
| static void po_silences_init(po_silences *s) { memset(s, 0, sizeof(*s)); } |
| 209 |
|
| |
| 210 |
3 |
| static int po_silence_add(po_silences *sl, const char *pat, size_t len, |
| 211 |
|
| int prefix, po_u64 until) { |
| 212 |
|
| po_silence *s; |
| 213 |
3 |
▶ | if (sl->n >= PO_SILENCE_MAX) return 0; |
Branch: 1 branch
| Branch | True | False | Total |
| if (sl->n >= PO_SILENCE_MAX) return 0; |
0 |
3 |
3 |
|
| 214 |
3 |
▶ | if (len > PO_AL_KEY_MAX) len = PO_AL_KEY_MAX; |
Branch: 1 branch
| Branch | True | False | Total |
| if (len > PO_AL_KEY_MAX) len = PO_AL_KEY_MAX; |
0 |
3 |
3 |
|
| 215 |
3 |
| s = &sl->s[sl->n++]; |
| 216 |
3 |
| memset(s, 0, sizeof(*s)); |
| 217 |
3 |
▶ | if (len) memcpy(s->pat, pat, len); |
Branch: 1 branch
| Branch | True | False | Total |
| if (len) memcpy(s->pat, pat, len); |
3 |
0 |
3 |
|
| 218 |
3 |
| s->pat_len = len; |
| 219 |
3 |
| s->prefix = prefix; |
| 220 |
3 |
| s->until = until; |
| 221 |
3 |
| return 1; |
| 222 |
|
| } |
| 223 |
|
| |
| 224 |
104 |
| static int po_is_silenced(const po_silences *sl, const char *k, size_t klen) { |
| 225 |
104 |
| po_u64 now = po_now_ns(); |
| 226 |
|
| uint32_t i; |
| 227 |
106 |
▶ | for (i = 0; i < sl->n; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < sl->n; i++) { |
5 |
101 |
106 |
|
| 228 |
5 |
| const po_silence *s = &sl->s[i]; |
| 229 |
5 |
▶ | if (s->until && now >= s->until) continue; /* expired */ |
Branch: 2 branches
| Branch | True | False | Total |
| if (s->until && now >= s->until) continue; /* expired */ |
2 |
3 |
5 |
| if (s->until && now >= s->until) continue; /* expired */ |
1 |
1 |
2 |
|
| 230 |
4 |
▶ | if (s->prefix) { |
Branch: 1 branch
| Branch | True | False | Total |
| if (s->prefix) { |
2 |
2 |
4 |
|
| 231 |
2 |
▶ | if (klen >= s->pat_len && memcmp(k, s->pat, s->pat_len) == 0) |
Branch: 2 branches
| Branch | True | False | Total |
| if (klen >= s->pat_len && memcmp(k, s->pat, s->pat_len) == 0) |
2 |
0 |
2 |
| if (klen >= s->pat_len && memcmp(k, s->pat, s->pat_len) == 0) |
1 |
1 |
2 |
|
| 232 |
1 |
| return 1; |
| 233 |
|
| } |
| 234 |
2 |
▶ | else if (klen == s->pat_len && memcmp(k, s->pat, klen) == 0) return 1; |
Branch: 2 branches
| Branch | True | False | Total |
| else if (klen == s->pat_len && memcmp(k, s->pat, klen) == 0) return 1; |
2 |
0 |
2 |
| else if (klen == s->pat_len && memcmp(k, s->pat, klen) == 0) return 1; |
2 |
0 |
2 |
|
| 235 |
|
| } |
| 236 |
101 |
| return 0; |
| 237 |
|
| } |
| 238 |
|
| |
| 239 |
|
| #endif /* PO_ROUTE_H */ |