Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_object_rollups.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_object_rollups.py · 20.5 KB · 469 lines
1"""Unit tests for object_rollups.py -- plan/vocabulary/14-rollup-spec.md.
2
3Mirrors the direct schema/record-fixture style tests/test_object_daemon.py
4uses for process_compactions/process_stale_transitions: write a schema
5straight to data/schemas, create rows through object_records with
6base_dir=tmp_path and roots=[] (never touching the real packages/ tree),
7then exercise object_rollups directly.
8
9End-to-end package/permission tests (admin-only rollup_definitions, a
10target collection carrying its own public-read rule over a private
11source, and the worked orders-per-day example run through a real
12rollup_definitions row) live in tests/test_app_rollup_package.py instead.
13"""
14import json
15
16import pytest
17
18import object_records
19import object_rollups
20import object_schemas
21
22
23def write_schema(data_dir, collection, fields):
24 path = data_dir / "schemas" / f"{collection}.json"
25 path.parent.mkdir(parents=True, exist_ok=True)
26 path.write_text(json.dumps({"name": collection, "fields": fields}))
27 return path
28
29
30ORDERS_FIELDS = [
31 {"name": "id"},
32 {"name": "channel", "type": "text"},
33 {"name": "status", "type": "text"},
34 {"name": "total_cents", "type": "integer"},
35 {"name": "created_at", "type": "datetime"},
36]
37
38
39def _make_orders(data_dir, rows):
40 write_schema(data_dir, "orders", ORDERS_FIELDS)
41 for row in rows:
42 object_records.create_collection_record("orders", row, base_dir=data_dir, roots=[])
43
44
45def _definition(**overrides):
46 base = {
47 "id": "rollup_orders_by_day",
48 "name": "Orders per day",
49 "source_collection": "orders",
50 "filter": json.dumps({"status": "paid"}),
51 "group_by": json.dumps(["channel"]),
52 "time_bucket": json.dumps({"field": "created_at", "granularity": "day"}),
53 "metrics": json.dumps([
54 {"op": "count", "as": "order_count"},
55 {"op": "sum", "field": "total_cents", "as": "revenue_cents"},
56 {"op": "avg", "field": "total_cents", "as": "avg_order_cents"},
57 ]),
58 "target_collection": "rollup_orders_by_day",
59 "min_group_size": "",
60 "refresh_mode": "scheduled",
61 "refresh_interval_seconds": "3600",
62 "last_computed_at": "",
63 "enabled": "true",
64 }
65 base.update(overrides)
66 return base
67
68
69# --- parse_definition -----------------------------------------------------
70
71def test_parse_definition_happy_path(tmp_path):
72 data_dir = tmp_path / "data"
73 _make_orders(data_dir, [])
74
75 config = object_rollups.parse_definition(_definition(), base_dir=data_dir, roots=[])
76
77 assert config.definition_id == "rollup_orders_by_day"
78 assert config.source_collection == "orders"
79 assert config.target_collection == "rollup_orders_by_day"
80 assert config.filter == {"status": "paid"}
81 assert config.group_by == ("channel",)
82 assert config.time_bucket_field == "created_at"
83 assert config.time_bucket_granularity == "day"
84 assert [m.op for m in config.metrics] == ["count", "sum", "avg"]
85 assert config.metrics[0].as_name == "order_count"
86 assert config.metrics[1].target_type == "integer"
87 assert config.metrics[2].target_type == "number"
88 assert config.min_group_size is None
89 assert config.refresh_interval_seconds == 3600
90 assert config.enabled is True
91
92
93def test_parse_definition_defaults_group_by_and_filter_to_empty(tmp_path):
94 data_dir = tmp_path / "data"
95 _make_orders(data_dir, [])
96
97 config = object_rollups.parse_definition(
98 _definition(filter="", group_by="", **{"time_bucket": ""}),
99 base_dir=data_dir, roots=[],
100 )
101
102 assert config.filter == {}
103 assert config.group_by == ()
104 assert config.time_bucket_field is None
105
106
107def test_parse_definition_rejects_missing_source_collection(tmp_path):
108 data_dir = tmp_path / "data"
109 with pytest.raises(object_rollups.DefinitionError, match="source_collection"):
110 object_rollups.parse_definition(_definition(source_collection=""), base_dir=data_dir, roots=[])
111
112
113def test_parse_definition_rejects_nested_filter_value(tmp_path):
114 data_dir = tmp_path / "data"
115 _make_orders(data_dir, [])
116 bad = _definition(filter=json.dumps({"status": {"op": "eq", "value": "paid"}}))
117 with pytest.raises(object_rollups.DefinitionError, match="flat"):
118 object_rollups.parse_definition(bad, base_dir=data_dir, roots=[])
119
120
121def test_parse_definition_rejects_non_numeric_metric_field(tmp_path):
122 data_dir = tmp_path / "data"
123 _make_orders(data_dir, [])
124 bad = _definition(metrics=json.dumps([{"op": "sum", "field": "channel", "as": "x"}]))
125 with pytest.raises(object_rollups.DefinitionError, match="numeric"):
126 object_rollups.parse_definition(bad, base_dir=data_dir, roots=[])
127
128
129def test_parse_definition_rejects_sum_without_field(tmp_path):
130 data_dir = tmp_path / "data"
131 _make_orders(data_dir, [])
132 bad = _definition(metrics=json.dumps([{"op": "sum", "as": "x"}]))
133 with pytest.raises(object_rollups.DefinitionError, match="requires a field"):
134 object_rollups.parse_definition(bad, base_dir=data_dir, roots=[])
135
136
137def test_parse_definition_rejects_time_bucket_on_non_date_field(tmp_path):
138 data_dir = tmp_path / "data"
139 _make_orders(data_dir, [])
140 bad = _definition(time_bucket=json.dumps({"field": "channel", "granularity": "day"}))
141 with pytest.raises(object_rollups.DefinitionError, match="date/datetime"):
142 object_rollups.parse_definition(bad, base_dir=data_dir, roots=[])
143
144
145def test_parse_definition_requires_positive_refresh_interval(tmp_path):
146 data_dir = tmp_path / "data"
147 _make_orders(data_dir, [])
148 bad = _definition(refresh_interval_seconds="0")
149 with pytest.raises(object_rollups.DefinitionError, match="refresh_interval_seconds"):
150 object_rollups.parse_definition(bad, base_dir=data_dir, roots=[])
151
152
153def test_parse_definition_rejects_metric_name_colliding_with_group_by(tmp_path):
154 data_dir = tmp_path / "data"
155 _make_orders(data_dir, [])
156 bad = _definition(metrics=json.dumps([{"op": "count", "as": "channel"}]))
157 with pytest.raises(object_rollups.DefinitionError, match="collides"):
158 object_rollups.parse_definition(bad, base_dir=data_dir, roots=[])
159
160
161# --- derive_target_schema --------------------------------------------------
162
163def test_derive_target_schema_matches_spec_shape(tmp_path):
164 data_dir = tmp_path / "data"
165 _make_orders(data_dir, [])
166 config = object_rollups.parse_definition(_definition(), base_dir=data_dir, roots=[])
167
168 schema = object_rollups.derive_target_schema(config)
169
170 field_names = [f["name"] for f in schema["fields"]]
171 assert field_names == ["id", "channel", "bucket_start", "order_count", "revenue_cents", "avg_order_cents", "computed_at"]
172
173 by_name = {f["name"]: f for f in schema["fields"]}
174 assert by_name["channel"]["type"] == "text"
175 assert "computed" not in by_name["channel"]
176 assert by_name["bucket_start"]["type"] == "date"
177 assert by_name["order_count"] == {"name": "order_count", "type": "integer", "computed": True}
178 assert by_name["revenue_cents"] == {"name": "revenue_cents", "type": "integer", "computed": True}
179 assert by_name["avg_order_cents"] == {"name": "avg_order_cents", "type": "number", "computed": True}
180 assert by_name["computed_at"] == {"name": "computed_at", "type": "datetime", "computed": True}
181
182 assert schema["storage"] == "classic"
183 assert schema["views"]["list_mode"] == "table"
184 assert schema["views"]["list_fields"] == ["channel", "bucket_start", "order_count", "revenue_cents", "avg_order_cents"]
185
186
187def test_derive_target_schema_bucket_start_is_datetime_for_granularity_none_on_datetime_field(tmp_path):
188 data_dir = tmp_path / "data"
189 _make_orders(data_dir, [])
190 definition = _definition(
191 time_bucket=json.dumps({"field": "created_at", "granularity": "none"}),
192 group_by="[]",
193 metrics=json.dumps([{"op": "count", "as": "count"}]),
194 )
195 config = object_rollups.parse_definition(definition, base_dir=data_dir, roots=[])
196
197 schema = object_rollups.derive_target_schema(config)
198
199 by_name = {f["name"]: f for f in schema["fields"]}
200 assert by_name["bucket_start"]["type"] == "datetime"
201
202
203def test_derive_target_schema_omits_bucket_start_without_time_bucket(tmp_path):
204 data_dir = tmp_path / "data"
205 _make_orders(data_dir, [])
206 definition = _definition(time_bucket="", metrics=json.dumps([{"op": "count", "as": "count"}]))
207 config = object_rollups.parse_definition(definition, base_dir=data_dir, roots=[])
208
209 schema = object_rollups.derive_target_schema(config)
210
211 assert "bucket_start" not in [f["name"] for f in schema["fields"]]
212
213
214# --- compute_rollup ---------------------------------------------------
215
216def test_compute_rollup_groups_filters_and_computes_metrics(tmp_path):
217 data_dir = tmp_path / "data"
218 _make_orders(data_dir, [
219 {"id": "o1", "channel": "web", "status": "paid", "total_cents": "1000", "created_at": "2026-07-01T10:00:00Z"},
220 {"id": "o2", "channel": "web", "status": "paid", "total_cents": "2000", "created_at": "2026-07-01T18:00:00Z"},
221 {"id": "o3", "channel": "retail", "status": "paid", "total_cents": "500", "created_at": "2026-07-01T09:00:00Z"},
222 {"id": "o4", "channel": "web", "status": "draft", "total_cents": "9999", "created_at": "2026-07-01T09:00:00Z"},
223 ])
224
225 result = object_rollups.compute_rollup(_definition(), base_dir=data_dir, roots=[])
226
227 assert result["groups"] == 2
228 assert result["created"] == 2
229 assert result["updated"] == 0
230 assert result["deleted"] == 0
231
232 rows = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[])
233 by_channel = {r["channel"]: r for r in rows}
234
235 assert by_channel["web"]["bucket_start"] == "2026-07-01"
236 assert by_channel["web"]["order_count"] == "2"
237 assert by_channel["web"]["revenue_cents"] == "3000"
238 assert by_channel["web"]["avg_order_cents"] == "1500.0" # exact: 3000 / 2
239 assert by_channel["retail"]["order_count"] == "1"
240 assert by_channel["retail"]["revenue_cents"] == "500"
241 # the draft order (o4) never appears anywhere -- filter excluded it
242 assert all(r.get("id") for r in rows)
243 assert sum(int(r["order_count"]) for r in rows) == 3
244
245
246def test_compute_rollup_avg_is_exact_for_a_non_terminating_division(tmp_path):
247 data_dir = tmp_path / "data"
248 _make_orders(data_dir, [
249 {"id": "o1", "channel": "web", "status": "paid", "total_cents": "100", "created_at": "2026-07-01T00:00:00Z"},
250 {"id": "o2", "channel": "web", "status": "paid", "total_cents": "100", "created_at": "2026-07-01T00:00:00Z"},
251 {"id": "o3", "channel": "web", "status": "paid", "total_cents": "100", "created_at": "2026-07-01T00:00:00Z"},
252 ])
253
254 object_rollups.compute_rollup(_definition(), base_dir=data_dir, roots=[])
255
256 rows = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[])
257 row = rows[0]
258 assert row["revenue_cents"] == "300"
259 # sum (300) / count (3) computed once from exact integers, not accumulated.
260 assert float(row["avg_order_cents"]) == pytest.approx(100.0)
261
262
263@pytest.mark.parametrize(
264 "granularity,created_at,expected_bucket",
265 [
266 ("day", "2026-07-15T12:00:00Z", "2026-07-15"),
267 # Wednesday 2026-07-15 -> ISO week starts Monday 2026-07-13.
268 ("week", "2026-07-15T12:00:00Z", "2026-07-13"),
269 ("month", "2026-07-15T12:00:00Z", "2026-07-01"),
270 ],
271)
272def test_compute_rollup_time_bucket_granularities(tmp_path, granularity, created_at, expected_bucket):
273 data_dir = tmp_path / "data"
274 _make_orders(data_dir, [
275 {"id": "o1", "channel": "web", "status": "paid", "total_cents": "100", "created_at": created_at},
276 ])
277 definition = _definition(time_bucket=json.dumps({"field": "created_at", "granularity": granularity}))
278
279 object_rollups.compute_rollup(definition, base_dir=data_dir, roots=[])
280
281 rows = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[])
282 assert rows[0]["bucket_start"] == expected_bucket
283
284
285def test_compute_rollup_min_and_max_metrics(tmp_path):
286 data_dir = tmp_path / "data"
287 _make_orders(data_dir, [
288 {"id": "o1", "channel": "web", "status": "paid", "total_cents": "500", "created_at": "2026-07-01T00:00:00Z"},
289 {"id": "o2", "channel": "web", "status": "paid", "total_cents": "3000", "created_at": "2026-07-01T00:00:00Z"},
290 {"id": "o3", "channel": "web", "status": "paid", "total_cents": "1200", "created_at": "2026-07-01T00:00:00Z"},
291 ])
292 definition = _definition(metrics=json.dumps([
293 {"op": "min", "field": "total_cents", "as": "min_cents"},
294 {"op": "max", "field": "total_cents", "as": "max_cents"},
295 ]))
296
297 object_rollups.compute_rollup(definition, base_dir=data_dir, roots=[])
298
299 rows = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[])
300 assert rows[0]["min_cents"] == "500"
301 assert rows[0]["max_cents"] == "3000"
302
303
304def test_compute_rollup_deterministic_id_is_stable_across_recomputes(tmp_path):
305 data_dir = tmp_path / "data"
306 _make_orders(data_dir, [
307 {"id": "o1", "channel": "web", "status": "paid", "total_cents": "1000", "created_at": "2026-07-01T00:00:00Z"},
308 ])
309
310 first = object_rollups.compute_rollup(_definition(), base_dir=data_dir, roots=[])
311 rows_after_first = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[])
312 assert first["created"] == 1
313
314 second = object_rollups.compute_rollup(_definition(), base_dir=data_dir, roots=[])
315 rows_after_second = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[])
316
317 assert second["created"] == 0
318 assert second["updated"] == 1
319 assert {r["id"] for r in rows_after_first} == {r["id"] for r in rows_after_second}
320 # computed_at still advances even though nothing else about the row changed.
321 assert rows_after_second[0]["computed_at"] >= rows_after_first[0]["computed_at"]
322
323
324def test_compute_rollup_drops_stale_group_on_rewrite(tmp_path):
325 data_dir = tmp_path / "data"
326 _make_orders(data_dir, [
327 {"id": "o1", "channel": "web", "status": "paid", "total_cents": "1000", "created_at": "2026-07-01T00:00:00Z"},
328 {"id": "o2", "channel": "retail", "status": "paid", "total_cents": "500", "created_at": "2026-07-01T00:00:00Z"},
329 ])
330 object_rollups.compute_rollup(_definition(), base_dir=data_dir, roots=[])
331 rows = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[])
332 assert {r["channel"] for r in rows} == {"web", "retail"}
333
334 # The retail order un-pays (no longer matches the filter) -- its whole
335 # group should vanish from the target on the next recompute, not be
336 # left behind as a stale row.
337 object_records.update_collection_record("orders", "o2", {"status": "refunded"}, base_dir=data_dir, roots=[])
338
339 result = object_rollups.compute_rollup(_definition(), base_dir=data_dir, roots=[])
340 assert result["deleted"] == 1
341
342 rows_after = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[])
343 assert {r["channel"] for r in rows_after} == {"web"}
344
345
346def test_compute_rollup_min_group_size_suppresses_small_groups(tmp_path):
347 data_dir = tmp_path / "data"
348 _make_orders(data_dir, [
349 {"id": "o1", "channel": "web", "status": "paid", "total_cents": "100", "created_at": "2026-07-01T00:00:00Z"},
350 {"id": "o2", "channel": "web", "status": "paid", "total_cents": "100", "created_at": "2026-07-01T00:00:00Z"},
351 # "referral" has exactly one paid order today -- the textbook
352 # group-of-1 disclosure case 14's Permissions Posture names.
353 {"id": "o3", "channel": "referral", "status": "paid", "total_cents": "9999", "created_at": "2026-07-01T00:00:00Z"},
354 ])
355 definition = _definition(min_group_size="2")
356
357 result = object_rollups.compute_rollup(definition, base_dir=data_dir, roots=[])
358
359 assert result["suppressed"] == 1
360 assert result["groups"] == 1
361 rows = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[])
362 assert {r["channel"] for r in rows} == {"web"}
363 # Suppressed, not merged: no "referral" identity or its $99.99 value
364 # leaks into any row, including as part of an "other" bucket.
365 assert not any("referral" in json.dumps(r) for r in rows)
366
367
368def test_compute_rollup_empty_group_by_yields_one_row_per_bucket(tmp_path):
369 data_dir = tmp_path / "data"
370 _make_orders(data_dir, [
371 {"id": "o1", "channel": "web", "status": "paid", "total_cents": "100", "created_at": "2026-07-01T00:00:00Z"},
372 {"id": "o2", "channel": "retail", "status": "paid", "total_cents": "200", "created_at": "2026-07-01T00:00:00Z"},
373 ])
374 definition = _definition(group_by="[]")
375
376 object_rollups.compute_rollup(definition, base_dir=data_dir, roots=[])
377
378 rows = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[])
379 assert len(rows) == 1
380 assert rows[0]["order_count"] == "2"
381 assert rows[0]["revenue_cents"] == "300"
382
383
384def test_compute_rollup_regenerates_target_schema_with_computed_fields(tmp_path):
385 data_dir = tmp_path / "data"
386 _make_orders(data_dir, [
387 {"id": "o1", "channel": "web", "status": "paid", "total_cents": "100", "created_at": "2026-07-01T00:00:00Z"},
388 ])
389
390 object_rollups.compute_rollup(_definition(), base_dir=data_dir, roots=[])
391
392 schema = object_schemas.get_schema("rollup_orders_by_day", base_dir=data_dir)
393 by_name = {f["name"]: f for f in schema["fields"]}
394 assert by_name["order_count"].get("computed") is True
395 assert by_name["computed_at"].get("computed") is True
396
397 target_row_id = object_records.read_collection_records(
398 "rollup_orders_by_day", base_dir=data_dir, roots=[]
399 )[0]["id"]
400 # A client (ordinary create/update, no bypass flag) must still be
401 # unable to write a computed field -- this is what makes "the rollup
402 # pass is the only writer" a real guarantee, not just a convention.
403 with pytest.raises(object_records.InvalidRecordPayloadError):
404 object_records.update_collection_record(
405 "rollup_orders_by_day", target_row_id,
406 {"order_count": "999999"},
407 base_dir=data_dir, roots=[],
408 )
409
410
411def test_compute_rollup_missing_source_collection_raises(tmp_path):
412 data_dir = tmp_path / "data"
413 definition = _definition(source_collection="does_not_exist")
414 with pytest.raises(object_schemas.SchemaNotFoundError):
415 object_rollups.compute_rollup(definition, base_dir=data_dir, roots=[])
416
417
418# --- flags / due gate -------------------------------------------------
419
420def test_rollup_pass_enabled_defaults_true_with_no_feature_flags_collection(tmp_path):
421 assert object_rollups.rollup_pass_enabled(base_dir=tmp_path / "data") is True
422
423
424def test_rollup_pass_enabled_false_when_flag_explicitly_off(tmp_path):
425 data_dir = tmp_path / "data"
426 write_schema(data_dir, "feature_flags", [
427 {"name": "id"}, {"name": "flag", "type": "text"}, {"name": "value", "type": "text"},
428 ])
429 object_records.create_collection_record(
430 "feature_flags", {"id": "f1", "flag": "rollup_enabled", "value": "off"}, base_dir=data_dir, roots=[],
431 )
432 assert object_rollups.rollup_pass_enabled(base_dir=data_dir) is False
433
434
435def test_rollup_pass_enabled_true_when_flag_explicitly_on(tmp_path):
436 data_dir = tmp_path / "data"
437 write_schema(data_dir, "feature_flags", [
438 {"name": "id"}, {"name": "flag", "type": "text"}, {"name": "value", "type": "text"},
439 ])
440 object_records.create_collection_record(
441 "feature_flags", {"id": "f1", "flag": "rollup_enabled", "value": "on"}, base_dir=data_dir, roots=[],
442 )
443 assert object_rollups.rollup_pass_enabled(base_dir=data_dir) is True
444
445
446def test_is_definition_due_true_when_never_computed():
447 assert object_rollups.is_definition_due({"last_computed_at": "", "refresh_interval_seconds": "3600"}) is True
448
449
450def test_is_definition_due_false_within_interval():
451 import datetime as dt
452 now = dt.datetime(2026, 7, 18, 12, 0, 0, tzinfo=dt.timezone.utc)
453 record = {"last_computed_at": "2026-07-18T11:30:00Z", "refresh_interval_seconds": "3600"}
454 assert object_rollups.is_definition_due(record, now=now) is False
455
456
457def test_is_definition_due_true_after_interval_elapses():
458 import datetime as dt
459 now = dt.datetime(2026, 7, 18, 13, 0, 1, tzinfo=dt.timezone.utc)
460 record = {"last_computed_at": "2026-07-18T11:30:00Z", "refresh_interval_seconds": "3600"}
461 assert object_rollups.is_definition_due(record, now=now) is True
462
463
464def test_is_definition_enabled_defaults_true():
465 assert object_rollups.is_definition_enabled({}) is True
466
467
468def test_is_definition_enabled_false_when_explicitly_disabled():
469 assert object_rollups.is_definition_enabled({"enabled": "false"}) is False