Foxray: a light error tracker for the Sentry SDKs. Runs inside Django (the admin is the UI) or on its own; SQLite or Postgres, nothing else. Not affiliated with Sentry.
danq9
0.1.2: the first PyPI release (v0.1.1 is already tagged at 05cc354, before the PyPI metadata)
58507e3
4 weeks ago
History
test_ingest.py · 10.4 KB · 246 lines
| 1 | """The ingest pipeline: answers to SDKs, rate limits, item types, alert rules.""" |
| 2 | |
| 3 | import gzip |
| 4 | import json |
| 5 | from datetime import timedelta |
| 6 | |
| 7 | from foxray.core.settings import Settings |
| 8 | from tests.helpers import auth_headers, envelope, make_event, make_stack |
| 9 | |
| 10 | |
| 11 | def post(ing, project, body: bytes, kind="envelope", headers=None, gz=True): |
| 12 | h = auth_headers(project, **(headers or {})) |
| 13 | if gz: |
| 14 | h["content-encoding"] = "gzip" |
| 15 | body = gzip.compress(body) |
| 16 | return ing.handle(project.id, kind, h, {}, body) |
| 17 | |
| 18 | |
| 19 | def event_env(ev): |
| 20 | return envelope(({"type": "event"}, ev), header={"event_id": ev["event_id"]}) |
| 21 | |
| 22 | |
| 23 | def test_store_and_group(sqlite_store): |
| 24 | store = sqlite_store |
| 25 | ing, *_ = make_stack(store) |
| 26 | p = store.create_project("app") |
| 27 | ids = [] |
| 28 | for i in range(3): |
| 29 | ev = make_event(value=f"bad {i}") |
| 30 | ids.append(ev["event_id"]) |
| 31 | r = post(ing, p, event_env(ev)) |
| 32 | assert r.status == 200 and json.loads(r.body) == {"id": ev["event_id"]} |
| 33 | issues, total = store.list_issues() |
| 34 | assert total == 1 and issues[0].times_seen == 3 |
| 35 | assert store.count_events(issues[0].id) == 3 |
| 36 | |
| 37 | |
| 38 | def test_auth_failures_and_unknown_project(sqlite_store): |
| 39 | ing, *_ = make_stack(sqlite_store) |
| 40 | p = sqlite_store.create_project("app") |
| 41 | body = event_env(make_event()) |
| 42 | assert ing.handle(p.id, "envelope", {}, {}, body).status == 401 |
| 43 | assert ing.handle(p.id, "envelope", {"x-sentry-auth": "Sentry sentry_key=wrong"}, {}, body).status == 403 |
| 44 | assert ing.handle(999, "envelope", auth_headers(p), {}, body).status == 404 |
| 45 | assert ing.handle(p.id, "envelope", {}, {"sentry_key": p.public_key}, body).status == 200 |
| 46 | |
| 47 | |
| 48 | def test_size_and_encoding_errors(sqlite_store): |
| 49 | ing, *_ = make_stack(sqlite_store, Settings(max_body_bytes=1000)) |
| 50 | p = sqlite_store.create_project("app") |
| 51 | assert post(ing, p, b"x" * 2000, gz=False).status == 413 |
| 52 | r = ing.handle(p.id, "envelope", auth_headers(p, **{"content-encoding": "zstd"}), {}, b"abc") |
| 53 | assert r.status == 415 |
| 54 | assert post(ing, p, b"garbage\n", gz=False).status == 400 |
| 55 | |
| 56 | |
| 57 | def test_other_item_types_are_accepted_dropped_and_counted(sqlite_store): |
| 58 | store = sqlite_store |
| 59 | ing, *_ = make_stack(store) |
| 60 | p = store.create_project("app") |
| 61 | body = envelope(({"type": "transaction"}, {"x": 1}), ({"type": "session"}, {}), ({"type": "sessions"}, {}), |
| 62 | ({"type": "client_report"}, {}), ({"type": "attachment", "filename": "a"}, b"\x00\xff"), |
| 63 | ({"type": "profile"}, {}), ({"type": "replay_event"}, {}), ({"type": "statsd"}, b"a:1|c"), |
| 64 | ({"type": "brand_new_thing"}, {})) |
| 65 | r = post(ing, p, body) |
| 66 | assert r.status == 200 |
| 67 | reasons = {row[2]: row[3] for row in store.counters(p.id)} |
| 68 | for t in ("transaction", "session", "sessions", "client_report", "attachment", "profile", "replay_event", |
| 69 | "statsd", "brand_new_thing"): |
| 70 | assert reasons["dropped:" + t] == 1 |
| 71 | assert store.list_issues()[1] == 0 |
| 72 | |
| 73 | |
| 74 | def test_legacy_store_endpoint(sqlite_store): |
| 75 | ing, *_ = make_stack(sqlite_store) |
| 76 | p = sqlite_store.create_project("app") |
| 77 | ev = make_event() |
| 78 | r = post(ing, p, json.dumps(ev).encode(), kind="store") |
| 79 | assert r.status == 200 and json.loads(r.body)["id"] == ev["event_id"] |
| 80 | assert sqlite_store.list_issues()[1] == 1 |
| 81 | |
| 82 | |
| 83 | def test_duplicate_event_id_counts_once(sqlite_store): |
| 84 | ing, *_ = make_stack(sqlite_store) |
| 85 | p = sqlite_store.create_project("app") |
| 86 | ev = make_event() |
| 87 | assert post(ing, p, event_env(ev)).status == 200 |
| 88 | assert post(ing, p, event_env(ev)).status == 200 |
| 89 | assert sqlite_store.list_issues()[0][0].times_seen == 1 |
| 90 | |
| 91 | |
| 92 | def test_rate_limit_answers_429_with_headers(sqlite_store): |
| 93 | store = sqlite_store |
| 94 | ing, _, _, _, clock = make_stack(store, Settings(rate_limit_per_minute=3)) |
| 95 | p = store.create_project("app") |
| 96 | statuses = [post(ing, p, event_env(make_event())) for _ in range(5)] |
| 97 | assert [r.status for r in statuses] == [200, 200, 200, 429, 429] |
| 98 | limited = statuses[3] |
| 99 | retry = int(limited.headers["Retry-After"]) |
| 100 | assert 1 <= retry <= 60 |
| 101 | assert limited.headers["X-Sentry-Rate-Limits"] == f"{retry}:error;default;security:project" |
| 102 | assert {r[2]: r[3] for r in store.counters(p.id)}["rate_limited"] == 2 |
| 103 | clock.advance(minutes=1) |
| 104 | assert post(ing, p, event_env(make_event())).status == 200 |
| 105 | # per-project override |
| 106 | store.update_project(p.id, rate_limit=0) |
| 107 | p = store.get_project(p.id) |
| 108 | assert all(post(ing, p, event_env(make_event())).status == 200 for _ in range(5)) |
| 109 | |
| 110 | |
| 111 | def test_filters_drop_with_200_and_count(sqlite_store): |
| 112 | store = sqlite_store |
| 113 | ing, *_ = make_stack(store) |
| 114 | p = store.create_project("app") |
| 115 | rule = store.add_filter(p.id, "environment", "development") |
| 116 | r = post(ing, p, event_env(make_event(environment="development"))) |
| 117 | assert r.status == 200 |
| 118 | assert store.list_issues()[1] == 0 |
| 119 | assert {x[2]: x[3] for x in store.counters(p.id)} == {f"filter:environment#{rule.id}": 1} |
| 120 | |
| 121 | |
| 122 | def test_sampling_keeps_count_drops_body(sqlite_store): |
| 123 | store = sqlite_store |
| 124 | ing, *_ = make_stack(store, Settings(sample_after=2)) |
| 125 | p = store.create_project("app") |
| 126 | for _ in range(5): |
| 127 | post(ing, p, event_env(make_event())) |
| 128 | issue = store.list_issues()[0][0] |
| 129 | assert issue.times_seen == 5 |
| 130 | assert store.count_events(issue.id) == 2 |
| 131 | assert store.issue_days(issue.id)[0][1] == 5 |
| 132 | assert {x[2]: x[3] for x in store.counters(p.id)}["sampled"] == 3 |
| 133 | |
| 134 | |
| 135 | # -- alert rules ----------------------------------------------------------- |
| 136 | |
| 137 | def kinds(store): |
| 138 | return [a.kind for a in reversed(store.list_alerts())] |
| 139 | |
| 140 | |
| 141 | def test_new_issue_alerts_repeats_do_not(sqlite_store): |
| 142 | store = sqlite_store |
| 143 | ing, *_ = make_stack(store) |
| 144 | p = store.create_project("app") |
| 145 | for _ in range(4): |
| 146 | post(ing, p, event_env(make_event())) |
| 147 | assert kinds(store) == ["new_issue"] |
| 148 | post(ing, p, event_env(make_event("TypeError"))) |
| 149 | assert kinds(store) == ["new_issue", "new_issue"] |
| 150 | |
| 151 | |
| 152 | def test_regression_alerts_after_resolve_respecting_cooldown(sqlite_store): |
| 153 | store = sqlite_store |
| 154 | ing, _, _, _, clock = make_stack(store, Settings(alert_cooldown_seconds=3600)) |
| 155 | p = store.create_project("app") |
| 156 | post(ing, p, event_env(make_event())) |
| 157 | issue = store.list_issues()[0][0] |
| 158 | # resolved and back within the cooldown: reopened, but quiet |
| 159 | store.set_issue_status(issue.id, "resolved", now=clock()) |
| 160 | clock.advance(minutes=10) |
| 161 | post(ing, p, event_env(make_event())) |
| 162 | assert store.get_issue(issue.id).status == "unresolved" |
| 163 | assert kinds(store) == ["new_issue"] |
| 164 | # resolved and back after the cooldown: a regression alert |
| 165 | store.set_issue_status(issue.id, "resolved", now=clock()) |
| 166 | clock.advance(hours=2) |
| 167 | post(ing, p, event_env(make_event())) |
| 168 | got = store.get_issue(issue.id) |
| 169 | assert got.status == "unresolved" and got.reopened_at == clock() |
| 170 | assert kinds(store) == ["new_issue", "regression"] |
| 171 | |
| 172 | |
| 173 | def test_ignore_until_count_and_until_date(sqlite_store): |
| 174 | store = sqlite_store |
| 175 | ing, _, _, _, clock = make_stack(store, Settings(alert_cooldown_seconds=0)) |
| 176 | p = store.create_project("app") |
| 177 | post(ing, p, event_env(make_event())) |
| 178 | issue = store.list_issues()[0][0] |
| 179 | store.set_issue_status(issue.id, "ignored", now=clock(), ignore_more=3) |
| 180 | post(ing, p, event_env(make_event())) |
| 181 | post(ing, p, event_env(make_event())) |
| 182 | assert store.get_issue(issue.id).status == "ignored" |
| 183 | post(ing, p, event_env(make_event())) |
| 184 | assert store.get_issue(issue.id).status == "unresolved" |
| 185 | assert kinds(store)[-1] == "unignored" |
| 186 | store.set_issue_status(issue.id, "ignored", now=clock(), ignore_until=clock() + timedelta(days=1)) |
| 187 | post(ing, p, event_env(make_event())) |
| 188 | assert store.get_issue(issue.id).status == "ignored" |
| 189 | clock.advance(days=2) |
| 190 | post(ing, p, event_env(make_event())) |
| 191 | assert store.get_issue(issue.id).status == "unresolved" |
| 192 | # plain ignore stays ignored forever and never alerts |
| 193 | store.set_issue_status(issue.id, "ignored", now=clock()) |
| 194 | before = len(store.list_alerts()) |
| 195 | for _ in range(3): |
| 196 | post(ing, p, event_env(make_event())) |
| 197 | assert store.get_issue(issue.id).status == "ignored" and len(store.list_alerts()) == before |
| 198 | |
| 199 | |
| 200 | def test_alert_rules_level_environment_new_vs_regression(sqlite_store): |
| 201 | store = sqlite_store |
| 202 | ing, _, _, _, clock = make_stack(store, Settings(alert_cooldown_seconds=0)) |
| 203 | p = store.create_project("app", alert_min_level="error", alert_environments="production", alert_regression=False) |
| 204 | post(ing, p, event_env(make_event("A", level="warning", environment="production"))) |
| 205 | post(ing, p, event_env(make_event("B", environment="staging"))) |
| 206 | post(ing, p, event_env(make_event("C", environment="production"))) |
| 207 | assert [a.subject.split(": ", 1)[1] for a in store.list_alerts()] == ["C: boom"] |
| 208 | issue = [i for i in store.list_issues()[0] if i.title.startswith("C")][0] |
| 209 | store.set_issue_status(issue.id, "resolved", now=clock()) |
| 210 | clock.advance(hours=1) |
| 211 | post(ing, p, event_env(make_event("C", environment="production"))) |
| 212 | assert len(store.list_alerts()) == 1 # regressions switched off |
| 213 | store.update_project(p.id, alert_new=False, alert_regression=True) |
| 214 | p = store.get_project(p.id) |
| 215 | post(ing, p, event_env(make_event("D", environment="production"))) |
| 216 | assert len(store.list_alerts()) == 1 # new issues switched off |
| 217 | |
| 218 | |
| 219 | def test_outbox_sends_after_commit_and_retries(sqlite_store): |
| 220 | from tests.helpers import Recorder |
| 221 | store = sqlite_store |
| 222 | channel = Recorder(fail=True) |
| 223 | ing, hk, outbox, _, clock = make_stack(store, channel=channel) |
| 224 | p = store.create_project("app") |
| 225 | r = post(ing, p, event_env(make_event())) |
| 226 | assert r.after is not None |
| 227 | r.after() # what the server runs once the response is sent |
| 228 | alert = store.list_alerts()[0] |
| 229 | assert alert.status == "pending" and alert.attempts == 1 and "mail server down" in alert.last_error |
| 230 | assert outbox.flush()["sent"] == 0 # not due yet: backoff |
| 231 | channel.fail = False |
| 232 | clock.advance(minutes=5) |
| 233 | assert outbox.flush()["sent"] == 1 |
| 234 | assert store.list_alerts()[0].status == "sent" and len(channel.sent) == 1 |
| 235 | assert outbox.flush()["sent"] == 0 # never twice |
| 236 | |
| 237 | |
| 238 | def test_outbox_gives_up_after_retention(sqlite_store): |
| 239 | from tests.helpers import Recorder |
| 240 | store = sqlite_store |
| 241 | ing, hk, outbox, _, clock = make_stack(store, channel=Recorder(fail=True)) |
| 242 | p = store.create_project("app") |
| 243 | post(ing, p, event_env(make_event())) |
| 244 | clock.advance(days=8) |
| 245 | outbox.flush() |
| 246 | assert store.list_alerts()[0].status == "failed" |