Foxray: a light error tracker for the Sentry SDKs. Runs inside Django (the admin is the UI) or on its own; SQLite or Postgres, nothing else. Not affiliated with Sentry.
danq9
0.1.2: the first PyPI release (v0.1.1 is already tagged at 05cc354, before the PyPI metadata)
58507e3
4 weeks ago
History
test_maintenance.py · 7.4 KB · 201 lines
| 1 | """Cleanup, the size cap, leases and giving space back.""" |
| 2 | |
| 3 | import json |
| 4 | import os |
| 5 | import threading |
| 6 | import time |
| 7 | |
| 8 | from foxray.core.maintenance import Maintenance, storage_report |
| 9 | from foxray.core.settings import Settings |
| 10 | from tests.helpers import auth_headers, envelope, make_event, make_stack |
| 11 | |
| 12 | |
| 13 | def send(ing, p, ev): |
| 14 | r = ing.handle(p.id, "envelope", auth_headers(p), {}, envelope(({"type": "event"}, ev))) |
| 15 | assert r.status == 200 |
| 16 | return r |
| 17 | |
| 18 | |
| 19 | def big_event(i, size=3000, **kw): |
| 20 | return make_event(value=f"v{i}", extra={"blob": "x" * 900, "more": ["y" * 900] * 3}, **kw) |
| 21 | |
| 22 | |
| 23 | def test_retention_limits(sqlite_store): |
| 24 | store = sqlite_store |
| 25 | s = Settings(events_per_issue=3, rate_limit_per_minute=0, sample_after=0) |
| 26 | ing, hk, _, _, clock = make_stack(store, s) |
| 27 | p = store.create_project("app") |
| 28 | for i in range(10): |
| 29 | send(ing, p, make_event()) |
| 30 | other = make_event("TypeError") |
| 31 | send(ing, p, other) |
| 32 | m = Maintenance(store, s, clock=clock) |
| 33 | res = m.run(force=True) |
| 34 | issues = {i.title: i for i in store.list_issues()[0]} |
| 35 | keep = issues["ValueError: boom"] |
| 36 | assert store.count_events(keep.id) == 3 and keep.times_seen == 10 |
| 37 | assert res["removed"]["events_over_issue_cap"] == 7 |
| 38 | |
| 39 | # bodies past the retention go; the issue and its counts stay |
| 40 | clock.advance(days=31) |
| 41 | send(ing, p, make_event("KeyError")) |
| 42 | m.run(force=True) |
| 43 | assert store.count_events(keep.id) == 0 |
| 44 | assert store.get_issue(keep.id).times_seen == 10 |
| 45 | assert sum(n for _, n in store.issue_days(keep.id)) == 10 |
| 46 | |
| 47 | # resolved and quiet for 90 days: the issue and its daily counts go |
| 48 | store.set_issue_status(keep.id, "resolved", now=clock()) |
| 49 | clock.advance(days=91) |
| 50 | m.run(force=True) |
| 51 | assert store.get_issue(keep.id) is None and store.issue_days(keep.id) == [] |
| 52 | assert store.get_issue(issues["TypeError: boom"].id) is not None # unresolved issues stay |
| 53 | |
| 54 | |
| 55 | def test_counters_rate_windows_alerts_and_checkins_are_trimmed(sqlite_store): |
| 56 | store = sqlite_store |
| 57 | s = Settings(checkins_per_monitor=5) |
| 58 | ing, hk, outbox, _, clock = make_stack(store, s) |
| 59 | p = store.create_project("app") |
| 60 | send(ing, p, make_event()) |
| 61 | outbox.flush() |
| 62 | ing.handle(p.id, "envelope", auth_headers(p), {}, envelope(({"type": "session"}, {}))) |
| 63 | for i in range(8): |
| 64 | ing.handle(p.id, "envelope", auth_headers(p), {}, |
| 65 | envelope(({"type": "check_in"}, {"monitor_slug": "job", "status": "ok"}))) |
| 66 | m = Maintenance(store, s, clock=clock) |
| 67 | m.run(force=True) |
| 68 | mon = store.get_monitor(p.id, "job") |
| 69 | assert len(store.list_checkins(mon.id, 100)) == 5 |
| 70 | clock.advance(days=3) |
| 71 | m.run(force=True) |
| 72 | counts = store.table_counts() |
| 73 | assert counts["foxray_counter"] == 0 and counts["foxray_rate"] == 0 |
| 74 | assert counts["foxray_alert"] == 1 # sent, kept 30 days |
| 75 | clock.advance(days=28) |
| 76 | m.run(force=True) |
| 77 | assert store.table_counts()["foxray_alert"] == 0 |
| 78 | |
| 79 | |
| 80 | def _fill(ing, p, n): |
| 81 | for i in range(n): |
| 82 | send(ing, p, make_event(f"E{i % 40}", f"v{i}", extra={"blob": "x" * 1000, "b2": "z" * 1000})) |
| 83 | |
| 84 | |
| 85 | def test_size_cap_prunes_shrinks_the_file_and_counts_survive(sqlite_store): |
| 86 | store = sqlite_store |
| 87 | s = Settings(max_db_mb=0.75, rate_limit_per_minute=0, sample_after=0, events_per_issue=10_000, |
| 88 | cleanup_budget_seconds=30) |
| 89 | ing, hk, _, _, clock = make_stack(store, s) |
| 90 | p = store.create_project("app") |
| 91 | _fill(ing, p, 600) |
| 92 | path = store.db.path |
| 93 | store.reclaim() |
| 94 | size_before = os.path.getsize(path) |
| 95 | total_before = sum(i.times_seen for i in store.list_issues(_all())[0]) |
| 96 | assert size_before > 0.9 * s.max_db_mb * 1048576 |
| 97 | res = Maintenance(store, s, clock=clock).run(force=True, budget=30) |
| 98 | assert res["removed"]["events_for_space"] > 0 |
| 99 | size_after = os.path.getsize(path) |
| 100 | assert size_after < size_before |
| 101 | assert res["used_after"] <= 0.8 * s.max_db_mb * 1048576 |
| 102 | assert res["mode"] == "normal" |
| 103 | assert sum(i.times_seen for i in store.list_issues(_all())[0]) == total_before == 600 |
| 104 | report = storage_report(store, s) |
| 105 | assert report["mode"] == "normal" and report["last_cleanup"]["removed"]["events_for_space"] > 0 |
| 106 | |
| 107 | |
| 108 | def _all(): |
| 109 | from foxray.core.sql import IssueQuery |
| 110 | return IssueQuery(status="all", limit=500) |
| 111 | |
| 112 | |
| 113 | def test_counts_only_mode_when_full_and_back(sqlite_store): |
| 114 | store = sqlite_store |
| 115 | # A cap so small that even with no bodies the store is over it. |
| 116 | s = Settings(max_db_mb=0.05, rate_limit_per_minute=0, sample_after=0) |
| 117 | ing, hk, outbox, channel, clock = make_stack(store, s) |
| 118 | p = store.create_project("app") |
| 119 | _fill(ing, p, 50) |
| 120 | res = Maintenance(store, s, global_channels=["test"], clock=clock).run(force=True, budget=10) |
| 121 | assert res["mode"] == "counts_only" |
| 122 | assert store.get_state_value("mode") == "counts_only" |
| 123 | # ingest still answers 200, still groups and counts, but keeps no new bodies |
| 124 | events_before = store.count_events() |
| 125 | r = send(ing, p, make_event("Fresh")) |
| 126 | assert r.status == 200 |
| 127 | fresh = [i for i in store.list_issues(_all())[0] if i.title.startswith("Fresh")][0] |
| 128 | assert fresh.times_seen == 1 and store.count_events() == events_before |
| 129 | assert {x[2]: x[3] for x in store.counters(p.id)}["counts_only"] == 1 |
| 130 | # one "store is full" alert, not one per pass |
| 131 | Maintenance(store, s, global_channels=["test"], clock=clock).run(force=True, budget=10) |
| 132 | assert [a.kind for a in store.list_alerts()].count("store_full") == 1 |
| 133 | # more room: back to normal |
| 134 | s.max_db_mb = 100 |
| 135 | res = Maintenance(store, s, clock=clock).run(force=True) |
| 136 | assert res["mode"] == "normal" |
| 137 | |
| 138 | |
| 139 | def test_two_cleanups_never_overlap(sqlite_store): |
| 140 | store = sqlite_store |
| 141 | s = Settings() |
| 142 | inside = [] |
| 143 | overlap = [] |
| 144 | lock = threading.Lock() |
| 145 | |
| 146 | def slow(): |
| 147 | with lock: |
| 148 | inside.append(1) |
| 149 | if len(inside) > 1: |
| 150 | overlap.append(1) |
| 151 | time.sleep(0.3) |
| 152 | with lock: |
| 153 | inside.pop() |
| 154 | |
| 155 | results = [] |
| 156 | barrier = threading.Barrier(5) |
| 157 | |
| 158 | def run(): |
| 159 | barrier.wait() |
| 160 | results.append(Maintenance(store, s, before_batch=slow).run(force=True)) |
| 161 | threads = [threading.Thread(target=run) for _ in range(5)] |
| 162 | [t.start() for t in threads] |
| 163 | [t.join() for t in threads] |
| 164 | assert not overlap |
| 165 | assert sum(r is not None for r in results) == 1 |
| 166 | |
| 167 | |
| 168 | def test_cleanup_runs_from_ingest_at_most_every_interval(sqlite_store): |
| 169 | store = sqlite_store |
| 170 | s = Settings(cleanup_interval_seconds=600) |
| 171 | ing, hk, _, _, clock = make_stack(store, s) |
| 172 | p = store.create_project("app") |
| 173 | r = send(ing, p, make_event()) |
| 174 | assert r.after is not None |
| 175 | r.after() |
| 176 | first = store.states()["cleanup"].last_run_at |
| 177 | clock.advance(minutes=5) |
| 178 | r = send(ing, p, make_event()) |
| 179 | if r.after: |
| 180 | r.after() |
| 181 | assert store.states()["cleanup"].last_run_at == first # not due yet |
| 182 | clock.advance(minutes=6) |
| 183 | r = send(ing, p, make_event()) |
| 184 | r.after() |
| 185 | assert store.states()["cleanup"].last_run_at == clock() |
| 186 | assert json.loads(store.states()["cleanup"].value)["mode"] == "normal" |
| 187 | |
| 188 | |
| 189 | def test_ingest_never_fails_when_cleanup_breaks(sqlite_store, monkeypatch): |
| 190 | store = sqlite_store |
| 191 | ing, hk, _, _, clock = make_stack(store) |
| 192 | p = store.create_project("app") |
| 193 | |
| 194 | def boom(*a, **k): |
| 195 | raise RuntimeError("disk on fire") |
| 196 | monkeypatch.setattr(store, "delete_events_before", boom) |
| 197 | r = send(ing, p, make_event()) |
| 198 | assert r.status == 200 |
| 199 | r.after() |
| 200 | assert "disk on fire" in json.loads(store.states()["cleanup"].value)["error"] |
| 201 | assert store.states()["cleanup"].lease_until is None |