askrobots / foxray Public

Foxray: a light error tracker for the Sentry SDKs. Runs inside Django (the admin is the UI) or on its own; SQLite or Postgres, nothing else. Not affiliated with Sentry.

0 downloads

foxray/tests/test_monitors.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/foxray.git
HTTPS
https://gitstable.com/git/askrobots/foxray.git
SSH
ssh://git@gitstable.com:2222/askrobots/foxray.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 0.1.2: the first PyPI release (v0.1.1 is already tagged at 05cc354, before the PyPI metadata) 58507e3 4 weeks ago History
test_monitors.py · 5.0 KB · 132 lines
1"""Cron monitors: check-ins, schedules, misses, timeouts, alerts."""
2
3import uuid
4
5from tests.helpers import auth_headers, envelope, make_stack
6
7
8def checkin(ing, p, slug, status, config=None, check_in_id=None, duration=None):
9 payload = {"check_in_id": check_in_id or uuid.uuid4().hex, "monitor_slug": slug, "status": status,
10 "environment": "production"}
11 if duration is not None:
12 payload["duration"] = duration
13 if config:
14 payload["monitor_config"] = config
15 r = ing.handle(p.id, "envelope", auth_headers(p), {}, envelope(({"type": "check_in"}, payload)))
16 assert r.status == 200
17 return payload["check_in_id"]
18
19
20HOURLY = {"schedule": {"type": "crontab", "value": "0 * * * *"}, "checkin_margin": 5, "max_runtime": 10}
21
22
23def alert_kinds(store):
24 return [a.kind for a in reversed(store.list_alerts())]
25
26
27def test_checkins_create_monitor_and_update_in_place(sqlite_store):
28 store = sqlite_store
29 ing, hk, _, _, clock = make_stack(store)
30 p = store.create_project("app")
31 cid = checkin(ing, p, "Nightly Backup", "in_progress", HOURLY)
32 clock.advance(seconds=42)
33 checkin(ing, p, "nightly-backup", "ok", check_in_id=cid, duration=42.0)
34 m = store.get_monitor(p.id, "nightly-backup")
35 assert (m.schedule_type, m.schedule_value, m.checkin_margin, m.max_runtime) == ("crontab", "0 * * * *", 5, 10)
36 assert m.status == "ok"
37 rows = store.list_checkins(m.id)
38 assert len(rows) == 1 and rows[0].status == "ok" and rows[0].duration == 42.0
39 # 12:00:42 -> next run 13:00, deadline 13:05
40 assert m.next_due_at.strftime("%H:%M") == "13:05"
41
42
43def test_missed_schedule_alerts_once_until_recovery(sqlite_store):
44 store = sqlite_store
45 ing, hk, _, _, clock = make_stack(store)
46 p = store.create_project("app")
47 checkin(ing, p, "hourly", "ok", HOURLY)
48 clock.advance(minutes=64) # 13:04: inside the margin
49 assert hk.check_monitors(force=True) == 0
50 clock.advance(minutes=2) # 13:06: missed
51 assert hk.check_monitors(force=True) == 1
52 m = store.get_monitor(p.id, "hourly")
53 assert m.status == "missed" and m.next_due_at.strftime("%H:%M") == "14:05"
54 assert [c.status for c in store.list_checkins(m.id)][0] == "missed"
55 clock.advance(hours=1) # 14:06: missed again, already failing: no new alert
56 assert hk.check_monitors(force=True) == 0
57 assert store.get_monitor(p.id, "hourly").status == "missed"
58 checkin(ing, p, "hourly", "ok")
59 assert store.get_monitor(p.id, "hourly").status == "ok"
60 assert alert_kinds(store) == ["monitor_missed"]
61
62
63def test_interval_schedule(sqlite_store):
64 store = sqlite_store
65 ing, hk, _, _, clock = make_stack(store)
66 p = store.create_project("app")
67 checkin(ing, p, "every-10", "ok", {"schedule": {"type": "interval", "value": 10, "unit": "minute"},
68 "checkin_margin": 1})
69 clock.advance(minutes=11)
70 assert hk.check_monitors(force=True) == 0
71 clock.advance(minutes=1)
72 assert hk.check_monitors(force=True) == 1
73
74
75def test_error_checkin_alerts_on_transition_only(sqlite_store):
76 store = sqlite_store
77 ing, *_ = make_stack(store)
78 p = store.create_project("app")
79 checkin(ing, p, "job", "ok", HOURLY)
80 checkin(ing, p, "job", "error")
81 checkin(ing, p, "job", "error")
82 assert alert_kinds(store) == ["monitor_error"]
83 checkin(ing, p, "job", "ok")
84 checkin(ing, p, "job", "error")
85 assert alert_kinds(store) == ["monitor_error", "monitor_error"]
86
87
88def test_in_progress_past_max_runtime_times_out(sqlite_store):
89 store = sqlite_store
90 ing, hk, _, _, clock = make_stack(store)
91 p = store.create_project("app")
92 checkin(ing, p, "slow", "in_progress", HOURLY)
93 clock.advance(minutes=9)
94 hk.check_monitors(force=True)
95 assert store.get_monitor(p.id, "slow").status != "timeout"
96 clock.advance(minutes=2)
97 hk.check_monitors(force=True)
98 m = store.get_monitor(p.id, "slow")
99 assert m.status == "timeout" and store.list_checkins(m.id)[0].status == "timeout"
100 assert alert_kinds(store) == ["monitor_timeout"]
101
102
103def test_monitor_without_schedule_never_misses(sqlite_store):
104 store = sqlite_store
105 ing, hk, _, _, clock = make_stack(store)
106 p = store.create_project("app")
107 checkin(ing, p, "adhoc", "ok")
108 clock.advance(days=3)
109 assert hk.check_monitors(force=True) == 0
110 assert store.get_monitor(p.id, "adhoc").next_due_at is None
111
112
113def test_two_checks_at_once_claim_a_miss_once(sqlite_store):
114 import threading
115 store = sqlite_store
116 ing, hk, _, _, clock = make_stack(store)
117 p = store.create_project("app")
118 for i in range(5):
119 checkin(ing, p, f"m{i}", "ok", HOURLY)
120 clock.advance(hours=2)
121 from foxray.core.monitors import check_monitors
122 results = []
123 barrier = threading.Barrier(4)
124
125 def run():
126 barrier.wait()
127 results.append(check_monitors(store, clock(), links=hk.links, channels_for=lambda p: ["test"]))
128 threads = [threading.Thread(target=run) for _ in range(4)]
129 [t.start() for t in threads]
130 [t.join() for t in threads]
131 assert sum(results) == 5
132 assert len(store.list_alerts()) == 5