askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_change_log_retention.py · 12.6 KB · 287 lines
| 1 | """Bounding an audit log without quietly losing accountability. |
| 2 | |
| 3 | The third unbounded log this server grew, after page_views and the package |
| 4 | restore points. What makes this one different is that a change log IS the |
| 5 | audit trail, so the rules are deliberately more conservative than the |
| 6 | other two: nothing is pruned unless an operator asked for it. |
| 7 | |
| 8 | The churn that motivated it is fixed at the source instead -- a rollup |
| 9 | rewriting derived rows no longer writes change entries at all -- and these |
| 10 | tests pin that too, because an entry never written costs nothing to store, |
| 11 | nothing to read past, and nothing to decide about later. |
| 12 | """ |
| 13 | |
| 14 | import gzip |
| 15 | import json |
| 16 | |
| 17 | import object_ids |
| 18 | import object_record_changes |
| 19 | import object_records |
| 20 | |
| 21 | |
| 22 | def log_path(tmp_path, collection): |
| 23 | return object_record_changes.record_changes_file(collection, base_dir=tmp_path) |
| 24 | |
| 25 | |
| 26 | def write_entries(tmp_path, collection, stamps): |
| 27 | path = log_path(tmp_path, collection) |
| 28 | path.parent.mkdir(parents=True, exist_ok=True) |
| 29 | with path.open("w", encoding="utf-8") as handle: |
| 30 | for i, stamp in enumerate(stamps): |
| 31 | handle.write(json.dumps({ |
| 32 | "change_id": f"c{i}", "timestamp": stamp, |
| 33 | "collection": collection, "record_id": f"r{i}", |
| 34 | "action": "update", "actor": "dana", |
| 35 | })) |
| 36 | handle.write("\n") |
| 37 | return path |
| 38 | |
| 39 | |
| 40 | def entries(tmp_path, collection): |
| 41 | path = log_path(tmp_path, collection) |
| 42 | return [json.loads(line) for line in |
| 43 | path.read_text(encoding="utf-8").splitlines() if line.strip()] |
| 44 | |
| 45 | |
| 46 | # --- the policy is opt-in ------------------------------------------------------ |
| 47 | |
| 48 | def test_no_policy_means_no_bound_at_all(): |
| 49 | """An operator who has never thought about audit retention has not |
| 50 | thereby consented to losing audit history.""" |
| 51 | policy = object_record_changes.retention_policy({}) |
| 52 | assert policy["days"] == 0 and policy["max_entries"] == 0 |
| 53 | assert object_record_changes.retention_policy( |
| 54 | {"DBBASIC_CHANGE_LOG_RETENTION_DAYS": "nonsense"})["days"] == 0 |
| 55 | |
| 56 | |
| 57 | def test_archiving_is_the_default_and_keeping_archives_forever_is_too(): |
| 58 | """Somebody asking for a smaller hot file has not asked to lose the |
| 59 | history, and the default for evidence is never 'throw it away'.""" |
| 60 | policy = object_record_changes.retention_policy( |
| 61 | {"DBBASIC_CHANGE_LOG_RETENTION_DAYS": "90"}) |
| 62 | assert policy["archive"] is True |
| 63 | assert policy["keep_archives"] == 0 # 0 = keep every segment |
| 64 | assert object_record_changes.retention_policy( |
| 65 | {"DBBASIC_CHANGE_LOG_ARCHIVE": "off"})["archive"] is False |
| 66 | |
| 67 | |
| 68 | def test_an_unconfigured_sweep_is_an_honest_no_op(tmp_path): |
| 69 | write_entries(tmp_path, "notes", ["2020-01-01T00:00:00Z"] * 5) |
| 70 | result = object_record_changes.prune_all_record_changes(base_dir=tmp_path) |
| 71 | assert result["removed"] == 0 |
| 72 | assert result["skipped"] == "no policy set" |
| 73 | assert len(entries(tmp_path, "notes")) == 5 |
| 74 | |
| 75 | |
| 76 | # --- both bounds ---------------------------------------------------------------- |
| 77 | |
| 78 | def test_age_prunes_what_is_older(tmp_path): |
| 79 | write_entries(tmp_path, "notes", [ |
| 80 | "2020-01-01T00:00:00Z", "2020-06-01T00:00:00Z", "2026-07-20T00:00:00Z"]) |
| 81 | result = object_record_changes.prune_record_changes( |
| 82 | "notes", keep_newer_than="2026-01-01T00:00:00Z", base_dir=tmp_path) |
| 83 | assert result["removed"] == 2 and result["entries_after"] == 1 |
| 84 | assert entries(tmp_path, "notes")[0]["record_id"] == "r2" |
| 85 | |
| 86 | |
| 87 | def test_a_count_bound_keeps_the_newest(tmp_path): |
| 88 | """Age alone bounds nothing: a window says how far back to keep and |
| 89 | nothing about how much can arrive inside it.""" |
| 90 | write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(50)]) |
| 91 | result = object_record_changes.prune_record_changes( |
| 92 | "notes", keep_last=10, base_dir=tmp_path) |
| 93 | assert result["entries_after"] == 10 |
| 94 | kept = entries(tmp_path, "notes") |
| 95 | assert [e["record_id"] for e in kept] == [f"r{i}" for i in range(40, 50)] |
| 96 | |
| 97 | |
| 98 | def test_both_bounds_compose(tmp_path): |
| 99 | write_entries(tmp_path, "notes", |
| 100 | ["2020-01-01T00:00:00Z"] * 5 |
| 101 | + [f"2026-07-26T00:00:{i:02d}Z" for i in range(20)]) |
| 102 | object_record_changes.prune_record_changes( |
| 103 | "notes", keep_newer_than="2026-01-01T00:00:00Z", keep_last=5, |
| 104 | base_dir=tmp_path) |
| 105 | kept = entries(tmp_path, "notes") |
| 106 | assert len(kept) == 5 |
| 107 | assert all(e["timestamp"].startswith("2026") for e in kept) |
| 108 | |
| 109 | |
| 110 | def test_an_undatable_entry_is_never_deleted(tmp_path): |
| 111 | """Never delete evidence we cannot date.""" |
| 112 | write_entries(tmp_path, "notes", ["", "2020-01-01T00:00:00Z"]) |
| 113 | object_record_changes.prune_record_changes( |
| 114 | "notes", keep_newer_than="2026-01-01T00:00:00Z", base_dir=tmp_path) |
| 115 | kept = entries(tmp_path, "notes") |
| 116 | assert [e["record_id"] for e in kept] == ["r0"] |
| 117 | |
| 118 | |
| 119 | def test_a_missing_log_is_not_an_error(tmp_path): |
| 120 | result = object_record_changes.prune_record_changes( |
| 121 | "never_written", keep_last=10, base_dir=tmp_path) |
| 122 | assert result["pruned"] is False |
| 123 | |
| 124 | |
| 125 | def test_the_sweep_covers_every_collection(tmp_path): |
| 126 | write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(30)]) |
| 127 | write_entries(tmp_path, "invoices", [f"2026-07-26T00:00:{i:02d}Z" for i in range(30)]) |
| 128 | result = object_record_changes.prune_all_record_changes( |
| 129 | base_dir=tmp_path, max_entries=5) |
| 130 | assert result["collections"] == 2 and result["removed"] == 50 |
| 131 | assert len(entries(tmp_path, "notes")) == 5 |
| 132 | |
| 133 | |
| 134 | # --- the better fix: never written at all ---------------------------------------- |
| 135 | |
| 136 | def test_a_derived_write_leaves_no_audit_entry(tmp_path): |
| 137 | """A rollup rewriting its own output describes nothing a person did, |
| 138 | and the value is recomputable from the source and the definition -- |
| 139 | both still on disk. Left on, this produced a 944MB log for a |
| 140 | collection of 1818 rows.""" |
| 141 | from conftest import stage_collection |
| 142 | |
| 143 | stage_collection(tmp_path, "app-notes", "notes") |
| 144 | object_records.create_collection_record( |
| 145 | "notes", {"id": object_ids.new_uuid4(), "title": "Derived", "content": "x"}, |
| 146 | base_dir=tmp_path, actor="system:rollup", record_changes=False) |
| 147 | |
| 148 | assert not log_path(tmp_path, "notes").exists() |
| 149 | |
| 150 | |
| 151 | def test_an_ordinary_write_is_still_traceable(tmp_path): |
| 152 | """The flag is for recomputes only. A write nobody can trace is the |
| 153 | thing the log exists to prevent.""" |
| 154 | from conftest import stage_collection |
| 155 | |
| 156 | stage_collection(tmp_path, "app-notes", "notes") |
| 157 | object_records.create_collection_record( |
| 158 | "notes", {"id": object_ids.new_uuid4(), "title": "By hand", "content": "x"}, |
| 159 | base_dir=tmp_path, actor="dana") |
| 160 | |
| 161 | logged = entries(tmp_path, "notes") |
| 162 | assert len(logged) == 1 and logged[0]["actor"] == "dana" |
| 163 | |
| 164 | |
| 165 | # --- rotation, archival and removal are three different questions ---------------- |
| 166 | |
| 167 | def test_rotated_entries_are_archived_not_destroyed(tmp_path): |
| 168 | """Rotating without archiving is deletion wearing a gentler word.""" |
| 169 | write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(30)]) |
| 170 | result = object_record_changes.prune_record_changes( |
| 171 | "notes", keep_last=5, base_dir=tmp_path) |
| 172 | |
| 173 | assert result["removed"] == 25 and result["archived"] |
| 174 | assert len(entries(tmp_path, "notes")) == 5 |
| 175 | |
| 176 | segments = object_record_changes.list_archives("notes", base_dir=tmp_path) |
| 177 | assert len(segments) == 1 |
| 178 | with gzip.open(segments[0], "rt", encoding="utf-8") as handle: |
| 179 | archived = [json.loads(line) for line in handle if line.strip()] |
| 180 | assert [e["record_id"] for e in archived] == [f"r{i}" for i in range(25)] |
| 181 | |
| 182 | |
| 183 | def test_the_archive_and_the_hot_file_together_lose_nothing(tmp_path): |
| 184 | stamps = [f"2026-07-26T00:00:{i:02d}Z" for i in range(40)] |
| 185 | write_entries(tmp_path, "notes", stamps) |
| 186 | object_record_changes.prune_record_changes( |
| 187 | "notes", keep_last=8, base_dir=tmp_path) |
| 188 | |
| 189 | kept = [e["record_id"] for e in entries(tmp_path, "notes")] |
| 190 | archived = [] |
| 191 | for segment in object_record_changes.list_archives("notes", base_dir=tmp_path): |
| 192 | with gzip.open(segment, "rt", encoding="utf-8") as handle: |
| 193 | archived += [json.loads(line)["record_id"] |
| 194 | for line in handle if line.strip()] |
| 195 | assert sorted(archived + kept, key=lambda r: int(r[1:])) == [ |
| 196 | f"r{i}" for i in range(40)] |
| 197 | |
| 198 | |
| 199 | def test_duplicate_entries_are_counted_not_deduplicated(tmp_path): |
| 200 | """Two identical entries are two facts. Differencing by set membership |
| 201 | would archive one and lose the other.""" |
| 202 | path = log_path(tmp_path, "notes") |
| 203 | path.parent.mkdir(parents=True, exist_ok=True) |
| 204 | row = json.dumps({"change_id": "c", "timestamp": "2026-07-26T00:00:00Z", |
| 205 | "collection": "notes", "record_id": "r", "action": "update", |
| 206 | "actor": "dana"}) |
| 207 | path.write_text("\n".join([row] * 6) + "\n", encoding="utf-8") |
| 208 | |
| 209 | object_record_changes.prune_record_changes("notes", keep_last=2, base_dir=tmp_path) |
| 210 | with gzip.open(object_record_changes.list_archives("notes", base_dir=tmp_path)[0], |
| 211 | "rt", encoding="utf-8") as handle: |
| 212 | archived = [line for line in handle if line.strip()] |
| 213 | assert len(archived) == 4 and len(entries(tmp_path, "notes")) == 2 |
| 214 | |
| 215 | |
| 216 | def test_archiving_can_be_turned_off_for_a_genuinely_disposable_log(tmp_path): |
| 217 | write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(20)]) |
| 218 | result = object_record_changes.prune_record_changes( |
| 219 | "notes", keep_last=5, archive=False, base_dir=tmp_path) |
| 220 | assert result["archived"] is None |
| 221 | assert object_record_changes.list_archives("notes", base_dir=tmp_path) == [] |
| 222 | |
| 223 | |
| 224 | def test_old_archives_are_removed_only_when_asked(tmp_path): |
| 225 | """An archive nobody ever deletes is the original problem moved one |
| 226 | directory down -- but the default keeps everything.""" |
| 227 | for _ in range(4): |
| 228 | write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(20)]) |
| 229 | object_record_changes.prune_record_changes("notes", keep_last=1, base_dir=tmp_path) |
| 230 | assert len(object_record_changes.list_archives("notes", base_dir=tmp_path)) == 4 |
| 231 | |
| 232 | write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(20)]) |
| 233 | result = object_record_changes.prune_record_changes( |
| 234 | "notes", keep_last=1, keep_archives=2, base_dir=tmp_path) |
| 235 | assert len(result["archives_removed"]) == 3 |
| 236 | assert len(object_record_changes.list_archives("notes", base_dir=tmp_path)) == 2 |
| 237 | |
| 238 | |
| 239 | def test_compression_is_worth_doing(tmp_path): |
| 240 | """JSONL compresses roughly ten to twenty times; that ratio is the |
| 241 | whole argument for archiving instead of deleting.""" |
| 242 | write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i % 60:02d}Z" for i in range(4000)]) |
| 243 | plain = log_path(tmp_path, "notes").stat().st_size |
| 244 | object_record_changes.prune_record_changes("notes", keep_last=1, base_dir=tmp_path) |
| 245 | segment = object_record_changes.list_archives("notes", base_dir=tmp_path)[0] |
| 246 | assert segment.stat().st_size * 5 < plain |
| 247 | |
| 248 | |
| 249 | # --- the scheduled pass ----------------------------------------------------------- |
| 250 | |
| 251 | def test_the_daemon_pass_is_off_until_somebody_configures_it(tmp_path, monkeypatch): |
| 252 | """Deciding how much audit trail to keep is not a decision a daemon |
| 253 | makes on somebody's behalf.""" |
| 254 | import object_daemon |
| 255 | |
| 256 | monkeypatch.delenv("DBBASIC_CHANGE_LOG_RETENTION_DAYS", raising=False) |
| 257 | monkeypatch.delenv("DBBASIC_CHANGE_LOG_MAX_ENTRIES", raising=False) |
| 258 | write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(40)]) |
| 259 | |
| 260 | assert object_daemon.process_change_log_retention(base_dir=tmp_path) is None |
| 261 | assert len(entries(tmp_path, "notes")) == 40 |
| 262 | |
| 263 | |
| 264 | def test_the_daemon_pass_rotates_and_archives_when_asked(tmp_path, monkeypatch): |
| 265 | import object_daemon |
| 266 | |
| 267 | monkeypatch.setenv("DBBASIC_CHANGE_LOG_MAX_ENTRIES", "10") |
| 268 | write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(40)]) |
| 269 | |
| 270 | result = object_daemon.process_change_log_retention(base_dir=tmp_path) |
| 271 | assert result["removed"] == 30 |
| 272 | assert len(entries(tmp_path, "notes")) == 10 |
| 273 | assert object_record_changes.list_archives("notes", base_dir=tmp_path) |
| 274 | |
| 275 | |
| 276 | def test_the_pass_honours_its_interval(tmp_path, monkeypatch): |
| 277 | """Rewriting every collection's log took three minutes on the box that |
| 278 | motivated this; it is not something to do on a tick.""" |
| 279 | import object_daemon |
| 280 | |
| 281 | monkeypatch.setenv("DBBASIC_CHANGE_LOG_MAX_ENTRIES", "10") |
| 282 | write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(40)]) |
| 283 | |
| 284 | assert object_daemon.process_change_log_retention(base_dir=tmp_path) is not None |
| 285 | write_entries(tmp_path, "notes", [f"2026-07-26T00:00:{i:02d}Z" for i in range(40)]) |
| 286 | assert object_daemon.process_change_log_retention(base_dir=tmp_path) is None |
| 287 | assert len(entries(tmp_path, "notes")) == 40 # untouched, not due |