askrobots / foxray Public

Foxray: a light error tracker for the Sentry SDKs. Runs inside Django (the admin is the UI) or on its own; SQLite or Postgres, nothing else. Not affiliated with Sentry.

0 downloads

foxray/tests/test_ingest.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/foxray.git
HTTPS
https://gitstable.com/git/askrobots/foxray.git
SSH
ssh://git@gitstable.com:2222/askrobots/foxray.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 0.1.2: the first PyPI release (v0.1.1 is already tagged at 05cc354, before the PyPI metadata) 58507e3 4 weeks ago History
test_ingest.py · 10.4 KB · 246 lines
1"""The ingest pipeline: answers to SDKs, rate limits, item types, alert rules."""
2
3import gzip
4import json
5from datetime import timedelta
6
7from foxray.core.settings import Settings
8from tests.helpers import auth_headers, envelope, make_event, make_stack
9
10
11def post(ing, project, body: bytes, kind="envelope", headers=None, gz=True):
12 h = auth_headers(project, **(headers or {}))
13 if gz:
14 h["content-encoding"] = "gzip"
15 body = gzip.compress(body)
16 return ing.handle(project.id, kind, h, {}, body)
17
18
19def event_env(ev):
20 return envelope(({"type": "event"}, ev), header={"event_id": ev["event_id"]})
21
22
23def test_store_and_group(sqlite_store):
24 store = sqlite_store
25 ing, *_ = make_stack(store)
26 p = store.create_project("app")
27 ids = []
28 for i in range(3):
29 ev = make_event(value=f"bad {i}")
30 ids.append(ev["event_id"])
31 r = post(ing, p, event_env(ev))
32 assert r.status == 200 and json.loads(r.body) == {"id": ev["event_id"]}
33 issues, total = store.list_issues()
34 assert total == 1 and issues[0].times_seen == 3
35 assert store.count_events(issues[0].id) == 3
36
37
38def test_auth_failures_and_unknown_project(sqlite_store):
39 ing, *_ = make_stack(sqlite_store)
40 p = sqlite_store.create_project("app")
41 body = event_env(make_event())
42 assert ing.handle(p.id, "envelope", {}, {}, body).status == 401
43 assert ing.handle(p.id, "envelope", {"x-sentry-auth": "Sentry sentry_key=wrong"}, {}, body).status == 403
44 assert ing.handle(999, "envelope", auth_headers(p), {}, body).status == 404
45 assert ing.handle(p.id, "envelope", {}, {"sentry_key": p.public_key}, body).status == 200
46
47
48def test_size_and_encoding_errors(sqlite_store):
49 ing, *_ = make_stack(sqlite_store, Settings(max_body_bytes=1000))
50 p = sqlite_store.create_project("app")
51 assert post(ing, p, b"x" * 2000, gz=False).status == 413
52 r = ing.handle(p.id, "envelope", auth_headers(p, **{"content-encoding": "zstd"}), {}, b"abc")
53 assert r.status == 415
54 assert post(ing, p, b"garbage\n", gz=False).status == 400
55
56
57def test_other_item_types_are_accepted_dropped_and_counted(sqlite_store):
58 store = sqlite_store
59 ing, *_ = make_stack(store)
60 p = store.create_project("app")
61 body = envelope(({"type": "transaction"}, {"x": 1}), ({"type": "session"}, {}), ({"type": "sessions"}, {}),
62 ({"type": "client_report"}, {}), ({"type": "attachment", "filename": "a"}, b"\x00\xff"),
63 ({"type": "profile"}, {}), ({"type": "replay_event"}, {}), ({"type": "statsd"}, b"a:1|c"),
64 ({"type": "brand_new_thing"}, {}))
65 r = post(ing, p, body)
66 assert r.status == 200
67 reasons = {row[2]: row[3] for row in store.counters(p.id)}
68 for t in ("transaction", "session", "sessions", "client_report", "attachment", "profile", "replay_event",
69 "statsd", "brand_new_thing"):
70 assert reasons["dropped:" + t] == 1
71 assert store.list_issues()[1] == 0
72
73
74def test_legacy_store_endpoint(sqlite_store):
75 ing, *_ = make_stack(sqlite_store)
76 p = sqlite_store.create_project("app")
77 ev = make_event()
78 r = post(ing, p, json.dumps(ev).encode(), kind="store")
79 assert r.status == 200 and json.loads(r.body)["id"] == ev["event_id"]
80 assert sqlite_store.list_issues()[1] == 1
81
82
83def test_duplicate_event_id_counts_once(sqlite_store):
84 ing, *_ = make_stack(sqlite_store)
85 p = sqlite_store.create_project("app")
86 ev = make_event()
87 assert post(ing, p, event_env(ev)).status == 200
88 assert post(ing, p, event_env(ev)).status == 200
89 assert sqlite_store.list_issues()[0][0].times_seen == 1
90
91
92def test_rate_limit_answers_429_with_headers(sqlite_store):
93 store = sqlite_store
94 ing, _, _, _, clock = make_stack(store, Settings(rate_limit_per_minute=3))
95 p = store.create_project("app")
96 statuses = [post(ing, p, event_env(make_event())) for _ in range(5)]
97 assert [r.status for r in statuses] == [200, 200, 200, 429, 429]
98 limited = statuses[3]
99 retry = int(limited.headers["Retry-After"])
100 assert 1 <= retry <= 60
101 assert limited.headers["X-Sentry-Rate-Limits"] == f"{retry}:error;default;security:project"
102 assert {r[2]: r[3] for r in store.counters(p.id)}["rate_limited"] == 2
103 clock.advance(minutes=1)
104 assert post(ing, p, event_env(make_event())).status == 200
105 # per-project override
106 store.update_project(p.id, rate_limit=0)
107 p = store.get_project(p.id)
108 assert all(post(ing, p, event_env(make_event())).status == 200 for _ in range(5))
109
110
111def test_filters_drop_with_200_and_count(sqlite_store):
112 store = sqlite_store
113 ing, *_ = make_stack(store)
114 p = store.create_project("app")
115 rule = store.add_filter(p.id, "environment", "development")
116 r = post(ing, p, event_env(make_event(environment="development")))
117 assert r.status == 200
118 assert store.list_issues()[1] == 0
119 assert {x[2]: x[3] for x in store.counters(p.id)} == {f"filter:environment#{rule.id}": 1}
120
121
122def test_sampling_keeps_count_drops_body(sqlite_store):
123 store = sqlite_store
124 ing, *_ = make_stack(store, Settings(sample_after=2))
125 p = store.create_project("app")
126 for _ in range(5):
127 post(ing, p, event_env(make_event()))
128 issue = store.list_issues()[0][0]
129 assert issue.times_seen == 5
130 assert store.count_events(issue.id) == 2
131 assert store.issue_days(issue.id)[0][1] == 5
132 assert {x[2]: x[3] for x in store.counters(p.id)}["sampled"] == 3
133
134
135# -- alert rules -----------------------------------------------------------
136
137def kinds(store):
138 return [a.kind for a in reversed(store.list_alerts())]
139
140
141def test_new_issue_alerts_repeats_do_not(sqlite_store):
142 store = sqlite_store
143 ing, *_ = make_stack(store)
144 p = store.create_project("app")
145 for _ in range(4):
146 post(ing, p, event_env(make_event()))
147 assert kinds(store) == ["new_issue"]
148 post(ing, p, event_env(make_event("TypeError")))
149 assert kinds(store) == ["new_issue", "new_issue"]
150
151
152def test_regression_alerts_after_resolve_respecting_cooldown(sqlite_store):
153 store = sqlite_store
154 ing, _, _, _, clock = make_stack(store, Settings(alert_cooldown_seconds=3600))
155 p = store.create_project("app")
156 post(ing, p, event_env(make_event()))
157 issue = store.list_issues()[0][0]
158 # resolved and back within the cooldown: reopened, but quiet
159 store.set_issue_status(issue.id, "resolved", now=clock())
160 clock.advance(minutes=10)
161 post(ing, p, event_env(make_event()))
162 assert store.get_issue(issue.id).status == "unresolved"
163 assert kinds(store) == ["new_issue"]
164 # resolved and back after the cooldown: a regression alert
165 store.set_issue_status(issue.id, "resolved", now=clock())
166 clock.advance(hours=2)
167 post(ing, p, event_env(make_event()))
168 got = store.get_issue(issue.id)
169 assert got.status == "unresolved" and got.reopened_at == clock()
170 assert kinds(store) == ["new_issue", "regression"]
171
172
173def test_ignore_until_count_and_until_date(sqlite_store):
174 store = sqlite_store
175 ing, _, _, _, clock = make_stack(store, Settings(alert_cooldown_seconds=0))
176 p = store.create_project("app")
177 post(ing, p, event_env(make_event()))
178 issue = store.list_issues()[0][0]
179 store.set_issue_status(issue.id, "ignored", now=clock(), ignore_more=3)
180 post(ing, p, event_env(make_event()))
181 post(ing, p, event_env(make_event()))
182 assert store.get_issue(issue.id).status == "ignored"
183 post(ing, p, event_env(make_event()))
184 assert store.get_issue(issue.id).status == "unresolved"
185 assert kinds(store)[-1] == "unignored"
186 store.set_issue_status(issue.id, "ignored", now=clock(), ignore_until=clock() + timedelta(days=1))
187 post(ing, p, event_env(make_event()))
188 assert store.get_issue(issue.id).status == "ignored"
189 clock.advance(days=2)
190 post(ing, p, event_env(make_event()))
191 assert store.get_issue(issue.id).status == "unresolved"
192 # plain ignore stays ignored forever and never alerts
193 store.set_issue_status(issue.id, "ignored", now=clock())
194 before = len(store.list_alerts())
195 for _ in range(3):
196 post(ing, p, event_env(make_event()))
197 assert store.get_issue(issue.id).status == "ignored" and len(store.list_alerts()) == before
198
199
200def test_alert_rules_level_environment_new_vs_regression(sqlite_store):
201 store = sqlite_store
202 ing, _, _, _, clock = make_stack(store, Settings(alert_cooldown_seconds=0))
203 p = store.create_project("app", alert_min_level="error", alert_environments="production", alert_regression=False)
204 post(ing, p, event_env(make_event("A", level="warning", environment="production")))
205 post(ing, p, event_env(make_event("B", environment="staging")))
206 post(ing, p, event_env(make_event("C", environment="production")))
207 assert [a.subject.split(": ", 1)[1] for a in store.list_alerts()] == ["C: boom"]
208 issue = [i for i in store.list_issues()[0] if i.title.startswith("C")][0]
209 store.set_issue_status(issue.id, "resolved", now=clock())
210 clock.advance(hours=1)
211 post(ing, p, event_env(make_event("C", environment="production")))
212 assert len(store.list_alerts()) == 1 # regressions switched off
213 store.update_project(p.id, alert_new=False, alert_regression=True)
214 p = store.get_project(p.id)
215 post(ing, p, event_env(make_event("D", environment="production")))
216 assert len(store.list_alerts()) == 1 # new issues switched off
217
218
219def test_outbox_sends_after_commit_and_retries(sqlite_store):
220 from tests.helpers import Recorder
221 store = sqlite_store
222 channel = Recorder(fail=True)
223 ing, hk, outbox, _, clock = make_stack(store, channel=channel)
224 p = store.create_project("app")
225 r = post(ing, p, event_env(make_event()))
226 assert r.after is not None
227 r.after() # what the server runs once the response is sent
228 alert = store.list_alerts()[0]
229 assert alert.status == "pending" and alert.attempts == 1 and "mail server down" in alert.last_error
230 assert outbox.flush()["sent"] == 0 # not due yet: backoff
231 channel.fail = False
232 clock.advance(minutes=5)
233 assert outbox.flush()["sent"] == 1
234 assert store.list_alerts()[0].status == "sent" and len(channel.sent) == 1
235 assert outbox.flush()["sent"] == 0 # never twice
236
237
238def test_outbox_gives_up_after_retention(sqlite_store):
239 from tests.helpers import Recorder
240 store = sqlite_store
241 ing, hk, outbox, _, clock = make_stack(store, channel=Recorder(fail=True))
242 p = store.create_project("app")
243 post(ing, p, event_env(make_event()))
244 clock.advance(days=8)
245 outbox.flush()
246 assert store.list_alerts()[0].status == "failed"