Foxray: a light error tracker for the Sentry SDKs. Runs inside Django (the admin is the UI) or on its own; SQLite or Postgres, nothing else. Not affiliated with Sentry.
danq9
0.1.2: the first PyPI release (v0.1.1 is already tagged at 05cc354, before the PyPI metadata)
58507e3
4 weeks ago
History
test_monitors.py · 5.0 KB · 132 lines
| 1 | """Cron monitors: check-ins, schedules, misses, timeouts, alerts.""" |
| 2 | |
| 3 | import uuid |
| 4 | |
| 5 | from tests.helpers import auth_headers, envelope, make_stack |
| 6 | |
| 7 | |
| 8 | def checkin(ing, p, slug, status, config=None, check_in_id=None, duration=None): |
| 9 | payload = {"check_in_id": check_in_id or uuid.uuid4().hex, "monitor_slug": slug, "status": status, |
| 10 | "environment": "production"} |
| 11 | if duration is not None: |
| 12 | payload["duration"] = duration |
| 13 | if config: |
| 14 | payload["monitor_config"] = config |
| 15 | r = ing.handle(p.id, "envelope", auth_headers(p), {}, envelope(({"type": "check_in"}, payload))) |
| 16 | assert r.status == 200 |
| 17 | return payload["check_in_id"] |
| 18 | |
| 19 | |
| 20 | HOURLY = {"schedule": {"type": "crontab", "value": "0 * * * *"}, "checkin_margin": 5, "max_runtime": 10} |
| 21 | |
| 22 | |
| 23 | def alert_kinds(store): |
| 24 | return [a.kind for a in reversed(store.list_alerts())] |
| 25 | |
| 26 | |
| 27 | def test_checkins_create_monitor_and_update_in_place(sqlite_store): |
| 28 | store = sqlite_store |
| 29 | ing, hk, _, _, clock = make_stack(store) |
| 30 | p = store.create_project("app") |
| 31 | cid = checkin(ing, p, "Nightly Backup", "in_progress", HOURLY) |
| 32 | clock.advance(seconds=42) |
| 33 | checkin(ing, p, "nightly-backup", "ok", check_in_id=cid, duration=42.0) |
| 34 | m = store.get_monitor(p.id, "nightly-backup") |
| 35 | assert (m.schedule_type, m.schedule_value, m.checkin_margin, m.max_runtime) == ("crontab", "0 * * * *", 5, 10) |
| 36 | assert m.status == "ok" |
| 37 | rows = store.list_checkins(m.id) |
| 38 | assert len(rows) == 1 and rows[0].status == "ok" and rows[0].duration == 42.0 |
| 39 | # 12:00:42 -> next run 13:00, deadline 13:05 |
| 40 | assert m.next_due_at.strftime("%H:%M") == "13:05" |
| 41 | |
| 42 | |
| 43 | def test_missed_schedule_alerts_once_until_recovery(sqlite_store): |
| 44 | store = sqlite_store |
| 45 | ing, hk, _, _, clock = make_stack(store) |
| 46 | p = store.create_project("app") |
| 47 | checkin(ing, p, "hourly", "ok", HOURLY) |
| 48 | clock.advance(minutes=64) # 13:04: inside the margin |
| 49 | assert hk.check_monitors(force=True) == 0 |
| 50 | clock.advance(minutes=2) # 13:06: missed |
| 51 | assert hk.check_monitors(force=True) == 1 |
| 52 | m = store.get_monitor(p.id, "hourly") |
| 53 | assert m.status == "missed" and m.next_due_at.strftime("%H:%M") == "14:05" |
| 54 | assert [c.status for c in store.list_checkins(m.id)][0] == "missed" |
| 55 | clock.advance(hours=1) # 14:06: missed again, already failing: no new alert |
| 56 | assert hk.check_monitors(force=True) == 0 |
| 57 | assert store.get_monitor(p.id, "hourly").status == "missed" |
| 58 | checkin(ing, p, "hourly", "ok") |
| 59 | assert store.get_monitor(p.id, "hourly").status == "ok" |
| 60 | assert alert_kinds(store) == ["monitor_missed"] |
| 61 | |
| 62 | |
| 63 | def test_interval_schedule(sqlite_store): |
| 64 | store = sqlite_store |
| 65 | ing, hk, _, _, clock = make_stack(store) |
| 66 | p = store.create_project("app") |
| 67 | checkin(ing, p, "every-10", "ok", {"schedule": {"type": "interval", "value": 10, "unit": "minute"}, |
| 68 | "checkin_margin": 1}) |
| 69 | clock.advance(minutes=11) |
| 70 | assert hk.check_monitors(force=True) == 0 |
| 71 | clock.advance(minutes=1) |
| 72 | assert hk.check_monitors(force=True) == 1 |
| 73 | |
| 74 | |
| 75 | def test_error_checkin_alerts_on_transition_only(sqlite_store): |
| 76 | store = sqlite_store |
| 77 | ing, *_ = make_stack(store) |
| 78 | p = store.create_project("app") |
| 79 | checkin(ing, p, "job", "ok", HOURLY) |
| 80 | checkin(ing, p, "job", "error") |
| 81 | checkin(ing, p, "job", "error") |
| 82 | assert alert_kinds(store) == ["monitor_error"] |
| 83 | checkin(ing, p, "job", "ok") |
| 84 | checkin(ing, p, "job", "error") |
| 85 | assert alert_kinds(store) == ["monitor_error", "monitor_error"] |
| 86 | |
| 87 | |
| 88 | def test_in_progress_past_max_runtime_times_out(sqlite_store): |
| 89 | store = sqlite_store |
| 90 | ing, hk, _, _, clock = make_stack(store) |
| 91 | p = store.create_project("app") |
| 92 | checkin(ing, p, "slow", "in_progress", HOURLY) |
| 93 | clock.advance(minutes=9) |
| 94 | hk.check_monitors(force=True) |
| 95 | assert store.get_monitor(p.id, "slow").status != "timeout" |
| 96 | clock.advance(minutes=2) |
| 97 | hk.check_monitors(force=True) |
| 98 | m = store.get_monitor(p.id, "slow") |
| 99 | assert m.status == "timeout" and store.list_checkins(m.id)[0].status == "timeout" |
| 100 | assert alert_kinds(store) == ["monitor_timeout"] |
| 101 | |
| 102 | |
| 103 | def test_monitor_without_schedule_never_misses(sqlite_store): |
| 104 | store = sqlite_store |
| 105 | ing, hk, _, _, clock = make_stack(store) |
| 106 | p = store.create_project("app") |
| 107 | checkin(ing, p, "adhoc", "ok") |
| 108 | clock.advance(days=3) |
| 109 | assert hk.check_monitors(force=True) == 0 |
| 110 | assert store.get_monitor(p.id, "adhoc").next_due_at is None |
| 111 | |
| 112 | |
| 113 | def test_two_checks_at_once_claim_a_miss_once(sqlite_store): |
| 114 | import threading |
| 115 | store = sqlite_store |
| 116 | ing, hk, _, _, clock = make_stack(store) |
| 117 | p = store.create_project("app") |
| 118 | for i in range(5): |
| 119 | checkin(ing, p, f"m{i}", "ok", HOURLY) |
| 120 | clock.advance(hours=2) |
| 121 | from foxray.core.monitors import check_monitors |
| 122 | results = [] |
| 123 | barrier = threading.Barrier(4) |
| 124 | |
| 125 | def run(): |
| 126 | barrier.wait() |
| 127 | results.append(check_monitors(store, clock(), links=hk.links, channels_for=lambda p: ["test"])) |
| 128 | threads = [threading.Thread(target=run) for _ in range(4)] |
| 129 | [t.start() for t in threads] |
| 130 | [t.join() for t in threads] |
| 131 | assert sum(results) == 5 |
| 132 | assert len(store.list_alerts()) == 5 |