Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_change_log_retention.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_change_log_retention.py · 12.6 KB · 287 lines
1"""Bounding an audit log without quietly losing accountability.
2
3The third unbounded log this server grew, after page_views and the package
4restore points. What makes this one different is that a change log IS the
5audit trail, so the rules are deliberately more conservative than the
6other two: nothing is pruned unless an operator asked for it.
7
8The churn that motivated it is fixed at the source instead -- a rollup
9rewriting derived rows no longer writes change entries at all -- and these
10tests pin that too, because an entry never written costs nothing to store,
11nothing to read past, and nothing to decide about later.
12"""
13
14import gzip
15import json
16
17import object_ids
18import object_record_changes
19import object_records
20
21
22def log_path(tmp_path, collection):
23 return object_record_changes.record_changes_file(collection, base_dir=tmp_path)
24
25
26def write_entries(tmp_path, collection, stamps):
27 path = log_path(tmp_path, collection)
28 path.parent.mkdir(parents=True, exist_ok=True)
29 with path.open("w", encoding="utf-8") as handle:
30 for i, stamp in enumerate(stamps):
31 handle.write(json.dumps({
32 "change_id": f"c{i}", "timestamp": stamp,
33 "collection": collection, "record_id": f"r{i}",
34 "action": "update", "actor": "dana",
35 }))
36 handle.write("\n")
37 return path
38
39
40def entries(tmp_path, collection):
41 path = log_path(tmp_path, collection)
42 return [json.loads(line) for line in
43 path.read_text(encoding="utf-8").splitlines() if line.strip()]
44
45
46# --- the policy is opt-in ------------------------------------------------------
47
48def test_no_policy_means_no_bound_at_all():
49 """An operator who has never thought about audit retention has not
50 thereby consented to losing audit history."""
51 policy = object_record_changes.retention_policy({})
52 assert policy["days"] == 0 and policy["max_entries"] == 0
53 assert object_record_changes.retention_policy(
54 {"DBBASIC_CHANGE_LOG_RETENTION_DAYS": "nonsense"})["days"] == 0
55
56
57def test_archiving_is_the_default_and_keeping_archives_forever_is_too():
58 """Somebody asking for a smaller hot file has not asked to lose the
59 history, and the default for evidence is never 'throw it away'."""
60 policy = object_record_changes.retention_policy(
61 {"DBBASIC_CHANGE_LOG_RETENTION_DAYS": "90"})
62 assert policy["archive"] is True
63 assert policy["keep_archives"] == 0 # 0 = keep every segment
64 assert object_record_changes.retention_policy(
65 {"DBBASIC_CHANGE_LOG_ARCHIVE": "off"})["archive"] is False
66
67
68def test_an_unconfigured_sweep_is_an_honest_no_op(tmp_path):
69 write_entries(tmp_path, "notes", ["2020-01-01T00:00:00Z"] * 5)
70 result = object_record_changes.prune_all_record_changes(base_dir=tmp_path)
71 assert result["removed"] == 0
72 assert result["skipped"] == "no policy set"
73 assert len(entries(tmp_path, "notes")) == 5
74
75
76# --- both bounds ----------------------------------------------------------------
77
78def test_age_prunes_what_is_older(tmp_path):
79 write_entries(tmp_path, "notes", [
80 "2020-01-01T00:00:00Z", "2020-06-01T00:00:00Z", "2026-07-20T00:00:00Z"])
81 result = object_record_changes.prune_record_changes(
82 "notes", keep_newer_than="2026-01-01T00:00:00Z", base_dir=tmp_path)
83 assert result["removed"] == 2 and result["entries_after"] == 1
84 assert entries(tmp_path, "notes")[0]["record_id"] == "r2"
85
86
87def test_a_count_bound_keeps_the_newest(tmp_path):
88 """Age alone bounds nothing: a window says how far back to keep and
89 nothing about how much can arrive inside it."""
90 write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(50)])
91 result = object_record_changes.prune_record_changes(
92 "notes", keep_last=10, base_dir=tmp_path)
93 assert result["entries_after"] == 10
94 kept = entries(tmp_path, "notes")
95 assert [e["record_id"] for e in kept] == [f"r{i}" for i in range(40, 50)]
96
97
98def test_both_bounds_compose(tmp_path):
99 write_entries(tmp_path, "notes",
100 ["2020-01-01T00:00:00Z"] * 5
101 + [f"2026-07-26T00:00:{i:02d}Z" for i in range(20)])
102 object_record_changes.prune_record_changes(
103 "notes", keep_newer_than="2026-01-01T00:00:00Z", keep_last=5,
104 base_dir=tmp_path)
105 kept = entries(tmp_path, "notes")
106 assert len(kept) == 5
107 assert all(e["timestamp"].startswith("2026") for e in kept)
108
109
110def test_an_undatable_entry_is_never_deleted(tmp_path):
111 """Never delete evidence we cannot date."""
112 write_entries(tmp_path, "notes", ["", "2020-01-01T00:00:00Z"])
113 object_record_changes.prune_record_changes(
114 "notes", keep_newer_than="2026-01-01T00:00:00Z", base_dir=tmp_path)
115 kept = entries(tmp_path, "notes")
116 assert [e["record_id"] for e in kept] == ["r0"]
117
118
119def test_a_missing_log_is_not_an_error(tmp_path):
120 result = object_record_changes.prune_record_changes(
121 "never_written", keep_last=10, base_dir=tmp_path)
122 assert result["pruned"] is False
123
124
125def test_the_sweep_covers_every_collection(tmp_path):
126 write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(30)])
127 write_entries(tmp_path, "invoices", [f"2026-07-26T00:00:{i:02d}Z" for i in range(30)])
128 result = object_record_changes.prune_all_record_changes(
129 base_dir=tmp_path, max_entries=5)
130 assert result["collections"] == 2 and result["removed"] == 50
131 assert len(entries(tmp_path, "notes")) == 5
132
133
134# --- the better fix: never written at all ----------------------------------------
135
136def test_a_derived_write_leaves_no_audit_entry(tmp_path):
137 """A rollup rewriting its own output describes nothing a person did,
138 and the value is recomputable from the source and the definition --
139 both still on disk. Left on, this produced a 944MB log for a
140 collection of 1818 rows."""
141 from conftest import stage_collection
142
143 stage_collection(tmp_path, "app-notes", "notes")
144 object_records.create_collection_record(
145 "notes", {"id": object_ids.new_uuid4(), "title": "Derived", "content": "x"},
146 base_dir=tmp_path, actor="system:rollup", record_changes=False)
147
148 assert not log_path(tmp_path, "notes").exists()
149
150
151def test_an_ordinary_write_is_still_traceable(tmp_path):
152 """The flag is for recomputes only. A write nobody can trace is the
153 thing the log exists to prevent."""
154 from conftest import stage_collection
155
156 stage_collection(tmp_path, "app-notes", "notes")
157 object_records.create_collection_record(
158 "notes", {"id": object_ids.new_uuid4(), "title": "By hand", "content": "x"},
159 base_dir=tmp_path, actor="dana")
160
161 logged = entries(tmp_path, "notes")
162 assert len(logged) == 1 and logged[0]["actor"] == "dana"
163
164
165# --- rotation, archival and removal are three different questions ----------------
166
167def test_rotated_entries_are_archived_not_destroyed(tmp_path):
168 """Rotating without archiving is deletion wearing a gentler word."""
169 write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(30)])
170 result = object_record_changes.prune_record_changes(
171 "notes", keep_last=5, base_dir=tmp_path)
172
173 assert result["removed"] == 25 and result["archived"]
174 assert len(entries(tmp_path, "notes")) == 5
175
176 segments = object_record_changes.list_archives("notes", base_dir=tmp_path)
177 assert len(segments) == 1
178 with gzip.open(segments[0], "rt", encoding="utf-8") as handle:
179 archived = [json.loads(line) for line in handle if line.strip()]
180 assert [e["record_id"] for e in archived] == [f"r{i}" for i in range(25)]
181
182
183def test_the_archive_and_the_hot_file_together_lose_nothing(tmp_path):
184 stamps = [f"2026-07-26T00:00:{i:02d}Z" for i in range(40)]
185 write_entries(tmp_path, "notes", stamps)
186 object_record_changes.prune_record_changes(
187 "notes", keep_last=8, base_dir=tmp_path)
188
189 kept = [e["record_id"] for e in entries(tmp_path, "notes")]
190 archived = []
191 for segment in object_record_changes.list_archives("notes", base_dir=tmp_path):
192 with gzip.open(segment, "rt", encoding="utf-8") as handle:
193 archived += [json.loads(line)["record_id"]
194 for line in handle if line.strip()]
195 assert sorted(archived + kept, key=lambda r: int(r[1:])) == [
196 f"r{i}" for i in range(40)]
197
198
199def test_duplicate_entries_are_counted_not_deduplicated(tmp_path):
200 """Two identical entries are two facts. Differencing by set membership
201 would archive one and lose the other."""
202 path = log_path(tmp_path, "notes")
203 path.parent.mkdir(parents=True, exist_ok=True)
204 row = json.dumps({"change_id": "c", "timestamp": "2026-07-26T00:00:00Z",
205 "collection": "notes", "record_id": "r", "action": "update",
206 "actor": "dana"})
207 path.write_text("\n".join([row] * 6) + "\n", encoding="utf-8")
208
209 object_record_changes.prune_record_changes("notes", keep_last=2, base_dir=tmp_path)
210 with gzip.open(object_record_changes.list_archives("notes", base_dir=tmp_path)[0],
211 "rt", encoding="utf-8") as handle:
212 archived = [line for line in handle if line.strip()]
213 assert len(archived) == 4 and len(entries(tmp_path, "notes")) == 2
214
215
216def test_archiving_can_be_turned_off_for_a_genuinely_disposable_log(tmp_path):
217 write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(20)])
218 result = object_record_changes.prune_record_changes(
219 "notes", keep_last=5, archive=False, base_dir=tmp_path)
220 assert result["archived"] is None
221 assert object_record_changes.list_archives("notes", base_dir=tmp_path) == []
222
223
224def test_old_archives_are_removed_only_when_asked(tmp_path):
225 """An archive nobody ever deletes is the original problem moved one
226 directory down -- but the default keeps everything."""
227 for _ in range(4):
228 write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(20)])
229 object_record_changes.prune_record_changes("notes", keep_last=1, base_dir=tmp_path)
230 assert len(object_record_changes.list_archives("notes", base_dir=tmp_path)) == 4
231
232 write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(20)])
233 result = object_record_changes.prune_record_changes(
234 "notes", keep_last=1, keep_archives=2, base_dir=tmp_path)
235 assert len(result["archives_removed"]) == 3
236 assert len(object_record_changes.list_archives("notes", base_dir=tmp_path)) == 2
237
238
239def test_compression_is_worth_doing(tmp_path):
240 """JSONL compresses roughly ten to twenty times; that ratio is the
241 whole argument for archiving instead of deleting."""
242 write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i % 60:02d}Z" for i in range(4000)])
243 plain = log_path(tmp_path, "notes").stat().st_size
244 object_record_changes.prune_record_changes("notes", keep_last=1, base_dir=tmp_path)
245 segment = object_record_changes.list_archives("notes", base_dir=tmp_path)[0]
246 assert segment.stat().st_size * 5 < plain
247
248
249# --- the scheduled pass -----------------------------------------------------------
250
251def test_the_daemon_pass_is_off_until_somebody_configures_it(tmp_path, monkeypatch):
252 """Deciding how much audit trail to keep is not a decision a daemon
253 makes on somebody's behalf."""
254 import object_daemon
255
256 monkeypatch.delenv("DBBASIC_CHANGE_LOG_RETENTION_DAYS", raising=False)
257 monkeypatch.delenv("DBBASIC_CHANGE_LOG_MAX_ENTRIES", raising=False)
258 write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(40)])
259
260 assert object_daemon.process_change_log_retention(base_dir=tmp_path) is None
261 assert len(entries(tmp_path, "notes")) == 40
262
263
264def test_the_daemon_pass_rotates_and_archives_when_asked(tmp_path, monkeypatch):
265 import object_daemon
266
267 monkeypatch.setenv("DBBASIC_CHANGE_LOG_MAX_ENTRIES", "10")
268 write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(40)])
269
270 result = object_daemon.process_change_log_retention(base_dir=tmp_path)
271 assert result["removed"] == 30
272 assert len(entries(tmp_path, "notes")) == 10
273 assert object_record_changes.list_archives("notes", base_dir=tmp_path)
274
275
276def test_the_pass_honours_its_interval(tmp_path, monkeypatch):
277 """Rewriting every collection's log took three minutes on the box that
278 motivated this; it is not something to do on a tick."""
279 import object_daemon
280
281 monkeypatch.setenv("DBBASIC_CHANGE_LOG_MAX_ENTRIES", "10")
282 write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(40)])
283
284 assert object_daemon.process_change_log_retention(base_dir=tmp_path) is not None
285 write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(40)])
286 assert object_daemon.process_change_log_retention(base_dir=tmp_path) is None
287 assert len(entries(tmp_path, "notes")) == 40 # untouched, not due