askrobots / foxray Public

Foxray: a light error tracker for the Sentry SDKs. Runs inside Django (the admin is the UI) or on its own; SQLite or Postgres, nothing else. Not affiliated with Sentry.

0 downloads

foxray/tests/test_maintenance.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/foxray.git
HTTPS
https://gitstable.com/git/askrobots/foxray.git
SSH
ssh://git@gitstable.com:2222/askrobots/foxray.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 0.1.2: the first PyPI release (v0.1.1 is already tagged at 05cc354, before the PyPI metadata) 58507e3 4 weeks ago History
test_maintenance.py · 7.4 KB · 201 lines
1"""Cleanup, the size cap, leases and giving space back."""
2
3import json
4import os
5import threading
6import time
7
8from foxray.core.maintenance import Maintenance, storage_report
9from foxray.core.settings import Settings
10from tests.helpers import auth_headers, envelope, make_event, make_stack
11
12
13def send(ing, p, ev):
14 r = ing.handle(p.id, "envelope", auth_headers(p), {}, envelope(({"type": "event"}, ev)))
15 assert r.status == 200
16 return r
17
18
19def big_event(i, size=3000, **kw):
20 return make_event(value=f"v{i}", extra={"blob": "x" * 900, "more": ["y" * 900] * 3}, **kw)
21
22
23def test_retention_limits(sqlite_store):
24 store = sqlite_store
25 s = Settings(events_per_issue=3, rate_limit_per_minute=0, sample_after=0)
26 ing, hk, _, _, clock = make_stack(store, s)
27 p = store.create_project("app")
28 for i in range(10):
29 send(ing, p, make_event())
30 other = make_event("TypeError")
31 send(ing, p, other)
32 m = Maintenance(store, s, clock=clock)
33 res = m.run(force=True)
34 issues = {i.title: i for i in store.list_issues()[0]}
35 keep = issues["ValueError: boom"]
36 assert store.count_events(keep.id) == 3 and keep.times_seen == 10
37 assert res["removed"]["events_over_issue_cap"] == 7
38
39 # bodies past the retention go; the issue and its counts stay
40 clock.advance(days=31)
41 send(ing, p, make_event("KeyError"))
42 m.run(force=True)
43 assert store.count_events(keep.id) == 0
44 assert store.get_issue(keep.id).times_seen == 10
45 assert sum(n for _, n in store.issue_days(keep.id)) == 10
46
47 # resolved and quiet for 90 days: the issue and its daily counts go
48 store.set_issue_status(keep.id, "resolved", now=clock())
49 clock.advance(days=91)
50 m.run(force=True)
51 assert store.get_issue(keep.id) is None and store.issue_days(keep.id) == []
52 assert store.get_issue(issues["TypeError: boom"].id) is not None # unresolved issues stay
53
54
55def test_counters_rate_windows_alerts_and_checkins_are_trimmed(sqlite_store):
56 store = sqlite_store
57 s = Settings(checkins_per_monitor=5)
58 ing, hk, outbox, _, clock = make_stack(store, s)
59 p = store.create_project("app")
60 send(ing, p, make_event())
61 outbox.flush()
62 ing.handle(p.id, "envelope", auth_headers(p), {}, envelope(({"type": "session"}, {})))
63 for i in range(8):
64 ing.handle(p.id, "envelope", auth_headers(p), {},
65 envelope(({"type": "check_in"}, {"monitor_slug": "job", "status": "ok"})))
66 m = Maintenance(store, s, clock=clock)
67 m.run(force=True)
68 mon = store.get_monitor(p.id, "job")
69 assert len(store.list_checkins(mon.id, 100)) == 5
70 clock.advance(days=3)
71 m.run(force=True)
72 counts = store.table_counts()
73 assert counts["foxray_counter"] == 0 and counts["foxray_rate"] == 0
74 assert counts["foxray_alert"] == 1 # sent, kept 30 days
75 clock.advance(days=28)
76 m.run(force=True)
77 assert store.table_counts()["foxray_alert"] == 0
78
79
80def _fill(ing, p, n):
81 for i in range(n):
82 send(ing, p, make_event(f"E{i % 40}", f"v{i}", extra={"blob": "x" * 1000, "b2": "z" * 1000}))
83
84
85def test_size_cap_prunes_shrinks_the_file_and_counts_survive(sqlite_store):
86 store = sqlite_store
87 s = Settings(max_db_mb=0.75, rate_limit_per_minute=0, sample_after=0, events_per_issue=10_000,
88 cleanup_budget_seconds=30)
89 ing, hk, _, _, clock = make_stack(store, s)
90 p = store.create_project("app")
91 _fill(ing, p, 600)
92 path = store.db.path
93 store.reclaim()
94 size_before = os.path.getsize(path)
95 total_before = sum(i.times_seen for i in store.list_issues(_all())[0])
96 assert size_before > 0.9 * s.max_db_mb * 1048576
97 res = Maintenance(store, s, clock=clock).run(force=True, budget=30)
98 assert res["removed"]["events_for_space"] > 0
99 size_after = os.path.getsize(path)
100 assert size_after < size_before
101 assert res["used_after"] <= 0.8 * s.max_db_mb * 1048576
102 assert res["mode"] == "normal"
103 assert sum(i.times_seen for i in store.list_issues(_all())[0]) == total_before == 600
104 report = storage_report(store, s)
105 assert report["mode"] == "normal" and report["last_cleanup"]["removed"]["events_for_space"] > 0
106
107
108def _all():
109 from foxray.core.sql import IssueQuery
110 return IssueQuery(status="all", limit=500)
111
112
113def test_counts_only_mode_when_full_and_back(sqlite_store):
114 store = sqlite_store
115 # A cap so small that even with no bodies the store is over it.
116 s = Settings(max_db_mb=0.05, rate_limit_per_minute=0, sample_after=0)
117 ing, hk, outbox, channel, clock = make_stack(store, s)
118 p = store.create_project("app")
119 _fill(ing, p, 50)
120 res = Maintenance(store, s, global_channels=["test"], clock=clock).run(force=True, budget=10)
121 assert res["mode"] == "counts_only"
122 assert store.get_state_value("mode") == "counts_only"
123 # ingest still answers 200, still groups and counts, but keeps no new bodies
124 events_before = store.count_events()
125 r = send(ing, p, make_event("Fresh"))
126 assert r.status == 200
127 fresh = [i for i in store.list_issues(_all())[0] if i.title.startswith("Fresh")][0]
128 assert fresh.times_seen == 1 and store.count_events() == events_before
129 assert {x[2]: x[3] for x in store.counters(p.id)}["counts_only"] == 1
130 # one "store is full" alert, not one per pass
131 Maintenance(store, s, global_channels=["test"], clock=clock).run(force=True, budget=10)
132 assert [a.kind for a in store.list_alerts()].count("store_full") == 1
133 # more room: back to normal
134 s.max_db_mb = 100
135 res = Maintenance(store, s, clock=clock).run(force=True)
136 assert res["mode"] == "normal"
137
138
139def test_two_cleanups_never_overlap(sqlite_store):
140 store = sqlite_store
141 s = Settings()
142 inside = []
143 overlap = []
144 lock = threading.Lock()
145
146 def slow():
147 with lock:
148 inside.append(1)
149 if len(inside) > 1:
150 overlap.append(1)
151 time.sleep(0.3)
152 with lock:
153 inside.pop()
154
155 results = []
156 barrier = threading.Barrier(5)
157
158 def run():
159 barrier.wait()
160 results.append(Maintenance(store, s, before_batch=slow).run(force=True))
161 threads = [threading.Thread(target=run) for _ in range(5)]
162 [t.start() for t in threads]
163 [t.join() for t in threads]
164 assert not overlap
165 assert sum(r is not None for r in results) == 1
166
167
168def test_cleanup_runs_from_ingest_at_most_every_interval(sqlite_store):
169 store = sqlite_store
170 s = Settings(cleanup_interval_seconds=600)
171 ing, hk, _, _, clock = make_stack(store, s)
172 p = store.create_project("app")
173 r = send(ing, p, make_event())
174 assert r.after is not None
175 r.after()
176 first = store.states()["cleanup"].last_run_at
177 clock.advance(minutes=5)
178 r = send(ing, p, make_event())
179 if r.after:
180 r.after()
181 assert store.states()["cleanup"].last_run_at == first # not due yet
182 clock.advance(minutes=6)
183 r = send(ing, p, make_event())
184 r.after()
185 assert store.states()["cleanup"].last_run_at == clock()
186 assert json.loads(store.states()["cleanup"].value)["mode"] == "normal"
187
188
189def test_ingest_never_fails_when_cleanup_breaks(sqlite_store, monkeypatch):
190 store = sqlite_store
191 ing, hk, _, _, clock = make_stack(store)
192 p = store.create_project("app")
193
194 def boom(*a, **k):
195 raise RuntimeError("disk on fire")
196 monkeypatch.setattr(store, "delete_events_before", boom)
197 r = send(ing, p, make_event())
198 assert r.status == 200
199 r.after()
200 assert "disk on fire" in json.loads(store.states()["cleanup"].value)["error"]
201 assert store.states()["cleanup"].lease_until is None