askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_usage_metering.py · 10.2 KB · 238 lines
| 1 | """Metering: measure, fold, rate at close. |
| 2 | |
| 3 | The three failures this design exists to prevent, each with a test: |
| 4 | a retried request billed twice; an allowance that never applies because |
| 5 | each event was priced as it landed; and a bill a customer can dispute but |
| 6 | not check. |
| 7 | """ |
| 8 | |
| 9 | import json |
| 10 | import pathlib |
| 11 | |
| 12 | from conftest import stage_collection |
| 13 | |
| 14 | import object_billing |
| 15 | import object_execution |
| 16 | import object_records |
| 17 | import python_object_runtime |
| 18 | |
| 19 | REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] |
| 20 | PACKAGES = REPO_ROOT / "packages" |
| 21 | BILLING_OBJECTS = PACKAGES / "app-billing" / "objects" |
| 22 | RUNTIME = python_object_runtime.PythonObjectRuntime() |
| 23 | |
| 24 | PRICES = json.dumps({"ai.tokens": {"included": 100000, "unit_minor": 2}}) |
| 25 | |
| 26 | |
| 27 | def setup_env(tmp_path, monkeypatch, *, prices=PRICES, base_minor="4900"): |
| 28 | data_dir = tmp_path / "data" |
| 29 | for pkg, name in (("app-billing", "billing_plans"), ("app-billing", "subscriptions"), |
| 30 | ("app-billing", "usage_events"), ("app-billing", "usage_summaries"), |
| 31 | ("app-billing", "wallets"), ("app-billing", "wallet_entries"), |
| 32 | ("app-invoices", "invoices"), ("app-invoices", "invoice_lines"), |
| 33 | ("app-settings", "app_settings"), ("app-finance", "denominations")): |
| 34 | stage_collection(data_dir, pkg, name, seed=(name == "denominations")) |
| 35 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir)) |
| 36 | object_records.create_collection_record( |
| 37 | "billing_plans", |
| 38 | {"id": "plan-ai", "name": "AI Team", "code": "ai", "period": "month", |
| 39 | "base_minor": base_minor, "prices": prices, "is_active": "true", |
| 40 | "owner_id": "dan"}, base_dir=data_dir) |
| 41 | object_records.create_collection_record( |
| 42 | "subscriptions", |
| 43 | {"id": "sub-1", "customer_name": "Acme", "plan_id": "plan-ai", |
| 44 | "billing_mode": "hybrid", "status": "active", |
| 45 | "current_period_start": "2026-06-01", "current_period_end": "2026-06-30", |
| 46 | "owner_id": "dan"}, base_dir=data_dir) |
| 47 | return data_dir |
| 48 | |
| 49 | |
| 50 | def meter(data_dir, event_id, quantity, *, metric="ai.tokens", cost=0, eid=None): |
| 51 | return object_records.create_collection_record( |
| 52 | "usage_events", |
| 53 | {"id": eid or f"ue-{event_id}", "event_id": event_id, "subscription_id": "sub-1", |
| 54 | "metric": metric, "quantity": str(quantity), "unit": "tokens", |
| 55 | "cost_minor": str(cost), "occurred_at": "2026-06-15", "owner_id": "dan"}, |
| 56 | base_dir=data_dir) |
| 57 | |
| 58 | |
| 59 | def run(object_id, payload=None): |
| 60 | return object_execution.execute_object( |
| 61 | RUNTIME, |
| 62 | object_execution.ObjectExecutionRequest( |
| 63 | object_id, method="POST", payload=payload or {}), |
| 64 | roots=[BILLING_OBJECTS]).result |
| 65 | |
| 66 | |
| 67 | def hook(record): |
| 68 | return object_execution.execute_object( |
| 69 | RUNTIME, |
| 70 | object_execution.ObjectExecutionRequest( |
| 71 | "hook_usage_events", method="BEFORE_WRITE", |
| 72 | payload={"action": "create", "collection": "usage_events", "record": record}), |
| 73 | roots=[BILLING_OBJECTS]).result |
| 74 | |
| 75 | |
| 76 | def summaries(data_dir): |
| 77 | return object_records.read_collection_records("usage_summaries", base_dir=data_dir) |
| 78 | |
| 79 | |
| 80 | def lines(data_dir): |
| 81 | return object_records.read_collection_records("invoice_lines", base_dir=data_dir) |
| 82 | |
| 83 | |
| 84 | # --- pure rating ---------------------------------------------------------------- |
| 85 | |
| 86 | def test_the_allowance_applies_to_the_period_not_to_each_call(): |
| 87 | """Rating per event would charge the first call at retail and never |
| 88 | reach the included quantity -- the classic metered-billing bug.""" |
| 89 | spec = {"included": 100000, "unit_minor": 2} |
| 90 | assert object_billing.rate_metric(80000, spec)["amount_minor"] == 0 |
| 91 | assert object_billing.rate_metric(100000, spec)["amount_minor"] == 0 |
| 92 | # 20k over at 2 minor units each |
| 93 | assert object_billing.rate_metric(120000, spec)["amount_minor"] == 40000 |
| 94 | |
| 95 | |
| 96 | def test_tiers_price_bands_of_the_overage(): |
| 97 | spec = {"included": 100, "tiers": [{"upto": 1000, "unit_minor": 2}, |
| 98 | {"unit_minor": 1}]} |
| 99 | # 1600 used, 100 included -> 1500 overage: 1000 @2 + 500 @1 |
| 100 | assert object_billing.rate_metric(1600, spec)["amount_minor"] == 2500 |
| 101 | |
| 102 | |
| 103 | def test_rounding_happens_once_for_the_metric(): |
| 104 | spec = {"included": 0, "unit_minor": "0.4"} |
| 105 | # 5 * 0.4 = 2.0 exactly; 6 * 0.4 = 2.4 -> 2 (half-up on the total, not |
| 106 | # per unit, which would have rounded 0.4 to 0 six times and billed 0) |
| 107 | assert object_billing.rate_metric(5, spec)["amount_minor"] == 2 |
| 108 | assert object_billing.rate_metric(6, spec)["amount_minor"] == 2 |
| 109 | |
| 110 | |
| 111 | def test_a_metric_with_no_price_is_reported_not_silently_billed(): |
| 112 | rated = object_billing.rate_period( |
| 113 | [{"metric": "mystery", "quantity": "500"}], {"ai.tokens": {"unit_minor": 1}}) |
| 114 | assert rated["total_minor"] == 0 |
| 115 | assert rated["unpriced"] == ["mystery"] |
| 116 | |
| 117 | |
| 118 | def test_unparseable_plan_prices_yield_no_charges_not_a_crash(): |
| 119 | """A malformed plan must fail as 'nothing to charge', never as a |
| 120 | billing run that dies halfway through the customer base.""" |
| 121 | assert object_billing.parse_prices("{not json") == {} |
| 122 | assert object_billing.parse_prices(None) == {} |
| 123 | |
| 124 | |
| 125 | def test_margin_is_a_fold_over_stored_facts(): |
| 126 | m = object_billing.margin(revenue_minor=40000, cost_minor=15000) |
| 127 | assert m["gross_minor"] == 25000 and m["gross_pct"] == 62.5 |
| 128 | |
| 129 | |
| 130 | # --- the dedup gate -------------------------------------------------------------- |
| 131 | |
| 132 | def test_a_retried_request_is_not_billed_twice(tmp_path, monkeypatch): |
| 133 | data_dir = setup_env(tmp_path, monkeypatch) |
| 134 | meter(data_dir, "req-abc", 1000) |
| 135 | repeat = hook({"event_id": "req-abc", "metric": "ai.tokens", |
| 136 | "quantity": "1000", "owner_id": "dan"}) |
| 137 | assert repeat["status"] == 409 |
| 138 | assert "already recorded" in repeat["error"] |
| 139 | |
| 140 | |
| 141 | def test_negative_usage_is_refused(tmp_path, monkeypatch): |
| 142 | setup_env(tmp_path, monkeypatch) |
| 143 | verdict = hook({"event_id": "req-neg", "metric": "ai.tokens", |
| 144 | "quantity": "-5", "owner_id": "dan"}) |
| 145 | assert verdict["status"] == 400 |
| 146 | assert "correction is a credit" in verdict["error"] |
| 147 | |
| 148 | |
| 149 | # --- the fold --------------------------------------------------------------------- |
| 150 | |
| 151 | def test_events_fold_into_one_bucket_per_metric_and_period(tmp_path, monkeypatch): |
| 152 | data_dir = setup_env(tmp_path, monkeypatch) |
| 153 | meter(data_dir, "e1", 40000, cost=300) |
| 154 | meter(data_dir, "e2", 30000, cost=200) |
| 155 | result = run("system_usage_rollup", {"today": "2026-06-20"}) |
| 156 | assert result["events_folded"] == 2 and result["summaries_written"] == 1 |
| 157 | bucket = summaries(data_dir)[0] |
| 158 | assert bucket["quantity"] == "70000" |
| 159 | assert bucket["event_count"] == "2" |
| 160 | assert bucket["cost_minor"] == "500" # margin is a read, not a project |
| 161 | |
| 162 | |
| 163 | def test_folding_twice_counts_nothing_twice(tmp_path, monkeypatch): |
| 164 | data_dir = setup_env(tmp_path, monkeypatch) |
| 165 | meter(data_dir, "e1", 40000) |
| 166 | run("system_usage_rollup", {"today": "2026-06-20"}) |
| 167 | again = run("system_usage_rollup", {"today": "2026-06-20"}) |
| 168 | assert again["events_folded"] == 0 |
| 169 | assert summaries(data_dir)[0]["quantity"] == "40000" |
| 170 | |
| 171 | |
| 172 | def test_usage_with_no_subscription_stays_visible_and_unrated(tmp_path, monkeypatch): |
| 173 | """Unpriceable usage must not be folded somewhere wrong.""" |
| 174 | data_dir = setup_env(tmp_path, monkeypatch) |
| 175 | object_records.create_collection_record( |
| 176 | "usage_events", |
| 177 | {"id": "ue-orphan", "event_id": "orphan", "metric": "ai.tokens", |
| 178 | "quantity": "500", "owner_id": "dan"}, base_dir=data_dir) |
| 179 | result = run("system_usage_rollup", {"today": "2026-06-20"}) |
| 180 | assert result["orphaned_events"] == 1 and summaries(data_dir) == [] |
| 181 | |
| 182 | |
| 183 | # --- rating into the invoice -------------------------------------------------------- |
| 184 | |
| 185 | def test_overage_becomes_a_line_a_customer_can_check(tmp_path, monkeypatch): |
| 186 | data_dir = setup_env(tmp_path, monkeypatch) |
| 187 | meter(data_dir, "e1", 120000, cost=1200) # 20k over the 100k allowance |
| 188 | run("system_usage_rollup", {"today": "2026-06-20"}) |
| 189 | result = run("system_billing_runner", {"today": "2026-07-01"}) |
| 190 | assert result["invoiced"] == 1 |
| 191 | |
| 192 | invoice = object_records.read_collection_records("invoices", base_dir=data_dir)[0] |
| 193 | assert invoice["total_cents"] == "44900" # 4900 base + 40000 overage |
| 194 | |
| 195 | rows = sorted(lines(data_dir), key=lambda r: int(r["line_total_cents"])) |
| 196 | assert len(rows) == 2 |
| 197 | usage_line = rows[-1] |
| 198 | # The line shows used / included / billable, not one opaque number. |
| 199 | assert "120000 used" in usage_line["description"] |
| 200 | assert "100000 included" in usage_line["description"] |
| 201 | assert "20000 billable" in usage_line["description"] |
| 202 | assert usage_line["line_total_cents"] == "40000" |
| 203 | |
| 204 | |
| 205 | def test_usage_within_the_allowance_bills_only_the_base(tmp_path, monkeypatch): |
| 206 | data_dir = setup_env(tmp_path, monkeypatch) |
| 207 | meter(data_dir, "e1", 50000) |
| 208 | run("system_usage_rollup", {"today": "2026-06-20"}) |
| 209 | run("system_billing_runner", {"today": "2026-07-01"}) |
| 210 | invoice = object_records.read_collection_records("invoices", base_dir=data_dir)[0] |
| 211 | assert invoice["total_cents"] == "4900" |
| 212 | assert len(lines(data_dir)) == 1 |
| 213 | |
| 214 | |
| 215 | def test_a_summary_is_never_billed_in_two_periods(tmp_path, monkeypatch): |
| 216 | data_dir = setup_env(tmp_path, monkeypatch) |
| 217 | meter(data_dir, "e1", 120000) |
| 218 | run("system_usage_rollup", {"today": "2026-06-20"}) |
| 219 | run("system_billing_runner", {"today": "2026-07-01"}) |
| 220 | assert summaries(data_dir)[0]["invoiced"] == "true" |
| 221 | # Next period closes with no new usage: the base only, and the overage |
| 222 | # already billed does not ride along a second time. |
| 223 | run("system_billing_runner", {"today": "2026-08-01"}) |
| 224 | totals = sorted(int(i["total_cents"]) for i in |
| 225 | object_records.read_collection_records("invoices", base_dir=data_dir)) |
| 226 | assert totals == [4900, 44900] |
| 227 | |
| 228 | |
| 229 | def test_pure_metered_plan_with_no_base_still_bills_usage(tmp_path, monkeypatch): |
| 230 | """base 0 + usage is a real shape: pay-as-you-go with an invoice.""" |
| 231 | data_dir = setup_env(tmp_path, monkeypatch, base_minor="0") |
| 232 | meter(data_dir, "e1", 150000) |
| 233 | run("system_usage_rollup", {"today": "2026-06-20"}) |
| 234 | result = run("system_billing_runner", {"today": "2026-07-01"}) |
| 235 | assert result["invoiced"] == 1 |
| 236 | invoice = object_records.read_collection_records("invoices", base_dir=data_dir)[0] |
| 237 | assert invoice["total_cents"] == "100000" # 50k over at 2 |
| 238 | assert len(lines(data_dir)) == 1 |