askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_record_changes.py · 20.5 KB · 557 lines
| 1 | """Append-only change logs for TSV-backed collection records. |
| 2 | |
| 3 | Record changes are the durable facts behind generated admin history screens and |
| 4 | record event publication. Events and listener delivery can fail or be retried; |
| 5 | this file records what actually changed. |
| 6 | """ |
| 7 | |
| 8 | from __future__ import annotations |
| 9 | |
| 10 | import gzip |
| 11 | import json |
| 12 | import os |
| 13 | import threading |
| 14 | from datetime import datetime, timedelta, timezone |
| 15 | from pathlib import Path |
| 16 | from typing import Any, Mapping |
| 17 | |
| 18 | import object_collections |
| 19 | import object_ids |
| 20 | import object_records |
| 21 | from object_versions import DEFAULT_DATA_DIR |
| 22 | |
| 23 | RECORD_CHANGES_DIR = "record_changes" |
| 24 | CHANGES_FILE = "changes.jsonl" |
| 25 | DEFAULT_CHANGE_LIMIT = 100 |
| 26 | MAX_CHANGE_LIMIT = 1000 |
| 27 | VALID_ACTIONS = {"create", "update", "delete"} |
| 28 | |
| 29 | _LOCKS: dict[str, threading.Lock] = {} |
| 30 | _LOCKS_GUARD = threading.Lock() |
| 31 | |
| 32 | |
| 33 | class InvalidRecordChangeError(ValueError): |
| 34 | """Raised when a record change entry is not safe to write or read.""" |
| 35 | |
| 36 | |
| 37 | def append_record_change( |
| 38 | *, |
| 39 | collection: str, |
| 40 | record_id: str, |
| 41 | action: str, |
| 42 | before: dict[str, Any] | None, |
| 43 | after: dict[str, Any] | None, |
| 44 | actor: str = "api", |
| 45 | message: str = "", |
| 46 | correlation_id: str | None = None, |
| 47 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 48 | ) -> dict[str, Any]: |
| 49 | """Append one record change and return the stored entry.""" |
| 50 | path = record_changes_file(collection, base_dir=base_dir) |
| 51 | if not object_records.validate_record_id(record_id): |
| 52 | raise object_records.InvalidRecordIdError(f"Invalid record id: {record_id}") |
| 53 | if action not in VALID_ACTIONS: |
| 54 | raise InvalidRecordChangeError(f"Invalid record change action: {action}") |
| 55 | |
| 56 | before_snapshot = _normalize_snapshot(before) |
| 57 | after_snapshot = _normalize_snapshot(after) |
| 58 | if before_snapshot is None and after_snapshot is None: |
| 59 | raise InvalidRecordChangeError("Record change must include before or after") |
| 60 | |
| 61 | timestamp = _utc_timestamp() |
| 62 | entry = { |
| 63 | "change_id": _change_id(timestamp, collection, record_id, action), |
| 64 | "timestamp": timestamp, |
| 65 | "collection": collection, |
| 66 | "record_id": record_id, |
| 67 | "action": action, |
| 68 | "actor": _clean_text(actor, default="api"), |
| 69 | "message": _clean_text(message, default=_default_message(action)), |
| 70 | "correlation_id": correlation_id or None, |
| 71 | "changed_fields": _changed_fields(before_snapshot, after_snapshot), |
| 72 | "before": before_snapshot, |
| 73 | "after": after_snapshot, |
| 74 | } |
| 75 | |
| 76 | with _file_lock(path): |
| 77 | path.parent.mkdir(parents=True, exist_ok=True) |
| 78 | with path.open("a", encoding="utf-8") as handle: |
| 79 | handle.write(json.dumps(entry, sort_keys=True, separators=(",", ":"))) |
| 80 | handle.write("\n") |
| 81 | |
| 82 | return entry |
| 83 | |
| 84 | |
| 85 | def list_record_changes( |
| 86 | collection: str, |
| 87 | *, |
| 88 | record_id: str | None = None, |
| 89 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 90 | limit: int = DEFAULT_CHANGE_LIMIT, |
| 91 | offset: int = 0, |
| 92 | tail_only: bool = False, |
| 93 | ) -> dict[str, Any]: |
| 94 | """Return newest-first record changes for one collection or record. |
| 95 | |
| 96 | `tail_only` reads just enough of the end of the log to satisfy |
| 97 | limit+offset instead of parsing the whole file. It is opt-in because |
| 98 | it makes `total` a count of what was READ rather than of what exists, |
| 99 | and a caller that shows "1 of 4,312" must not be handed "1 of 100" |
| 100 | without asking. A feed does not care; an audit page does. |
| 101 | """ |
| 102 | path = record_changes_file(collection, base_dir=base_dir) |
| 103 | if record_id is not None and not object_records.validate_record_id(record_id): |
| 104 | raise object_records.InvalidRecordIdError(f"Invalid record id: {record_id}") |
| 105 | _validate_page(limit=limit, offset=offset) |
| 106 | |
| 107 | changes = _read_changes( |
| 108 | path, record_id=record_id, |
| 109 | tail=(limit + offset if tail_only else None), |
| 110 | ) |
| 111 | total = len(changes) |
| 112 | window = changes[offset:offset + limit] |
| 113 | payload: dict[str, Any] = { |
| 114 | "collection": collection, |
| 115 | "changes": window, |
| 116 | "count": len(window), |
| 117 | "total": total, |
| 118 | "limit": limit, |
| 119 | "offset": offset, |
| 120 | "has_more": offset + len(window) < total, |
| 121 | } |
| 122 | if record_id is not None: |
| 123 | payload["record_id"] = record_id |
| 124 | return payload |
| 125 | |
| 126 | |
| 127 | def record_changes_file(collection: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> Path: |
| 128 | """Return the validated JSONL change-log path for a collection.""" |
| 129 | if not object_collections.validate_collection_name(collection): |
| 130 | raise object_collections.InvalidCollectionNameError( |
| 131 | f"Invalid collection name: {collection}" |
| 132 | ) |
| 133 | |
| 134 | root = Path(base_dir) / RECORD_CHANGES_DIR |
| 135 | path = root / collection / CHANGES_FILE |
| 136 | resolved_root = root.resolve(strict=False) |
| 137 | resolved_path = path.resolve(strict=False) |
| 138 | |
| 139 | try: |
| 140 | resolved_path.relative_to(resolved_root) |
| 141 | except ValueError as exc: |
| 142 | raise object_collections.InvalidCollectionNameError( |
| 143 | f"Record change path escapes change directory: {collection}" |
| 144 | ) from exc |
| 145 | |
| 146 | return path |
| 147 | |
| 148 | |
| 149 | # Reading a change log tail-first, in blocks, instead of parsing the whole |
| 150 | # file. A change log only ever grows, and the newest entries are the ones |
| 151 | # anybody asks for, so paying for the whole history to answer "what |
| 152 | # happened lately" is a cost that rises forever while the answer stays the |
| 153 | # same size. |
| 154 | # |
| 155 | # This was not theoretical. One collection's log reached 944MB -- a rollup |
| 156 | # rewriting 1818 derived rows every five minutes, each rewrite appending a |
| 157 | # row per record -- and the activity feed, which reads EVERY collection's |
| 158 | # log, sat parsing a gigabyte of JSON on a single core while the page said |
| 159 | # "loading..." forever. The function's own docstring had predicted it: |
| 160 | # "fine at current scale... future work if/when logs grow large." |
| 161 | _TAIL_BLOCK = 256 * 1024 |
| 162 | |
| 163 | |
| 164 | def _tail_lines(path: Path, wanted: int) -> list[str]: |
| 165 | """The last `wanted` non-empty lines, read backwards in blocks. |
| 166 | |
| 167 | Falls back to a whole-file read only for a file smaller than one |
| 168 | block, where seeking would be more code than it saves. |
| 169 | """ |
| 170 | try: |
| 171 | size = path.stat().st_size |
| 172 | except OSError: |
| 173 | return [] |
| 174 | if size <= _TAIL_BLOCK: |
| 175 | return path.read_text(encoding="utf-8").splitlines() |
| 176 | |
| 177 | chunks: list[bytes] = [] |
| 178 | newlines = 0 |
| 179 | with path.open("rb") as handle: |
| 180 | position = size |
| 181 | while position > 0 and newlines <= wanted: |
| 182 | step = min(_TAIL_BLOCK, position) |
| 183 | position -= step |
| 184 | handle.seek(position) |
| 185 | block = handle.read(step) |
| 186 | chunks.append(block) |
| 187 | newlines += block.count(b"\n") |
| 188 | text = b"".join(reversed(chunks)).decode("utf-8", "replace") |
| 189 | # The first line of the first block read is probably a fragment of a |
| 190 | # longer line; dropping it is why we read one block past `wanted`. |
| 191 | lines = text.splitlines() |
| 192 | return lines[1:] if position > 0 else lines |
| 193 | |
| 194 | |
| 195 | def _read_changes( |
| 196 | path: Path, *, record_id: str | None, tail: int | None = None |
| 197 | ) -> list[dict[str, Any]]: |
| 198 | if not path.exists(): |
| 199 | return [] |
| 200 | |
| 201 | if tail is not None and record_id is None: |
| 202 | source = _tail_lines(path, tail) |
| 203 | else: |
| 204 | # A record-scoped query still scans: the entries for one record can |
| 205 | # be anywhere in the file, and answering "show me this invoice's |
| 206 | # history" with only the recent tail would quietly lose the half |
| 207 | # that matters. |
| 208 | source = None |
| 209 | |
| 210 | changes: list[dict[str, Any]] = [] |
| 211 | with _file_lock(path): |
| 212 | for line in (source if source is not None |
| 213 | else path.read_text(encoding="utf-8").splitlines()): |
| 214 | if not line.strip(): |
| 215 | continue |
| 216 | try: |
| 217 | entry = json.loads(line) |
| 218 | except json.JSONDecodeError: |
| 219 | continue |
| 220 | if not isinstance(entry, dict): |
| 221 | continue |
| 222 | if record_id is not None and entry.get("record_id") != record_id: |
| 223 | continue |
| 224 | changes.append(entry) |
| 225 | |
| 226 | changes.reverse() |
| 227 | return changes |
| 228 | |
| 229 | |
| 230 | def _normalize_snapshot(record: dict[str, Any] | None) -> dict[str, Any] | None: |
| 231 | if record is None: |
| 232 | return None |
| 233 | if not isinstance(record, dict): |
| 234 | raise InvalidRecordChangeError("Record snapshot must be an object") |
| 235 | return {str(key): _json_safe_value(value) for key, value in record.items()} |
| 236 | |
| 237 | |
| 238 | def _json_safe_value(value: Any) -> Any: |
| 239 | if value is None or isinstance(value, str | int | float | bool): |
| 240 | return value |
| 241 | return str(value) |
| 242 | |
| 243 | |
| 244 | def _changed_fields( |
| 245 | before: dict[str, Any] | None, |
| 246 | after: dict[str, Any] | None, |
| 247 | ) -> list[str]: |
| 248 | before_values = before or {} |
| 249 | after_values = after or {} |
| 250 | names = set(before_values) | set(after_values) |
| 251 | return sorted(name for name in names if before_values.get(name) != after_values.get(name)) |
| 252 | |
| 253 | |
| 254 | def _validate_page(*, limit: int, offset: int) -> None: |
| 255 | if limit < 1: |
| 256 | raise ValueError("limit must be at least 1") |
| 257 | if limit > MAX_CHANGE_LIMIT: |
| 258 | raise ValueError(f"limit must be at most {MAX_CHANGE_LIMIT}") |
| 259 | if offset < 0: |
| 260 | raise ValueError("offset must be at least 0") |
| 261 | |
| 262 | |
| 263 | def _clean_text(value: str, *, default: str) -> str: |
| 264 | text = str(value).strip() |
| 265 | return text or default |
| 266 | |
| 267 | |
| 268 | def _default_message(action: str) -> str: |
| 269 | return { |
| 270 | "create": "Created record", |
| 271 | "update": "Updated record", |
| 272 | "delete": "Deleted record", |
| 273 | }[action] |
| 274 | |
| 275 | |
| 276 | def _change_id(timestamp: str, collection: str, record_id: str, action: str) -> str: |
| 277 | return object_ids.new_uuid4() |
| 278 | |
| 279 | |
| 280 | def _utc_timestamp() -> str: |
| 281 | return datetime.now(timezone.utc).isoformat() |
| 282 | |
| 283 | |
| 284 | def _file_lock(path: Path) -> threading.Lock: |
| 285 | key = str(path.resolve(strict=False)) |
| 286 | with _LOCKS_GUARD: |
| 287 | lock = _LOCKS.get(key) |
| 288 | if lock is None: |
| 289 | lock = threading.Lock() |
| 290 | _LOCKS[key] = lock |
| 291 | return lock |
| 292 | |
| 293 | |
| 294 | # --- retention ----------------------------------------------------------------- |
| 295 | # |
| 296 | # The third unbounded log this server has grown, after page_views and the |
| 297 | # package restore points, and the pattern is now unmistakable: a log |
| 298 | # nobody tells how big it may get is a log that fails on the worst day |
| 299 | # rather than an ordinary one. |
| 300 | # |
| 301 | # A change log is audit evidence, so pruning it is a genuinely different |
| 302 | # act from pruning analytics. Two consequences, both deliberate: |
| 303 | # |
| 304 | # * Retention is OFF unless an operator turns it on. Deleting audit |
| 305 | # history by default is not a performance optimisation, it is a |
| 306 | # decision about accountability, and it is not this module's to make |
| 307 | # quietly. |
| 308 | # * The bound is stated in BOTH dimensions, because age alone does not |
| 309 | # bound anything: a retention window says how far back to keep and |
| 310 | # nothing at all about how much can arrive inside it. |
| 311 | # |
| 312 | # The rollup churn that motivated this is fixed at the source instead |
| 313 | # (object_rollups writes derived rows with record_changes=False) -- which |
| 314 | # is the better fix, because a log entry never written costs nothing to |
| 315 | # store, nothing to read past, and nothing to decide about later. |
| 316 | |
| 317 | RETENTION_DAYS_ENV = "DBBASIC_CHANGE_LOG_RETENTION_DAYS" |
| 318 | MAX_ENTRIES_ENV = "DBBASIC_CHANGE_LOG_MAX_ENTRIES" |
| 319 | ARCHIVE_ENV = "DBBASIC_CHANGE_LOG_ARCHIVE" |
| 320 | KEEP_ARCHIVES_ENV = "DBBASIC_CHANGE_LOG_KEEP_ARCHIVES" |
| 321 | ARCHIVE_DIR = "archive" |
| 322 | |
| 323 | # Archival and retention are different questions, and conflating them is |
| 324 | # what made the first version of this uncomfortable. |
| 325 | # |
| 326 | # ROTATION bounds the file the server reads on every feed render. |
| 327 | # ARCHIVAL decides whether the entries leaving that file still exist. |
| 328 | # REMOVAL decides how long the archives themselves are kept. |
| 329 | # |
| 330 | # An audit trail wants all three, in that order. Rotating without |
| 331 | # archiving is deletion wearing a gentler word, and it is why retention |
| 332 | # had to be opt-in and slightly apologetic. Archiving first inverts that: |
| 333 | # pruning becomes cheap to turn on, because nothing is actually lost -- |
| 334 | # the entries move out of the hot read path into a compressed segment |
| 335 | # that is still on disk, still greppable, and still handable to an |
| 336 | # auditor. JSONL compresses roughly ten to twenty times, so the 944MB log |
| 337 | # that started all this becomes a segment measured in tens of megabytes. |
| 338 | # |
| 339 | # Removal still has to exist, because an archive nobody ever deletes is |
| 340 | # just the original problem moved one directory down. Hence |
| 341 | # KEEP_ARCHIVES: the same "keep the newest N" shape object_logs already |
| 342 | # uses for rotated server logs, so an operator learns the convention once. |
| 343 | # Its default keeps everything, because the default for evidence should |
| 344 | # never be "throw it away". |
| 345 | |
| 346 | |
| 347 | def retention_policy(env: Mapping[str, str] | None = None) -> dict[str, int]: |
| 348 | """{"days", "max_entries"} -- 0 for either means "no bound". |
| 349 | |
| 350 | Absent configuration is no bound at all, not a default window: an |
| 351 | operator who has never thought about audit retention has not thereby |
| 352 | consented to losing audit history. |
| 353 | """ |
| 354 | source = os.environ if env is None else env |
| 355 | |
| 356 | def _int(name: str) -> int: |
| 357 | try: |
| 358 | value = int(str(source.get(name, "")).strip()) |
| 359 | except (TypeError, ValueError): |
| 360 | return 0 |
| 361 | return value if value > 0 else 0 |
| 362 | |
| 363 | archive = str(source.get(ARCHIVE_ENV, "")).strip().lower() |
| 364 | return { |
| 365 | "days": _int(RETENTION_DAYS_ENV), |
| 366 | "max_entries": _int(MAX_ENTRIES_ENV), |
| 367 | # Archiving is the DEFAULT when a policy is set at all. Somebody |
| 368 | # asking for a smaller hot file has not asked to lose the history. |
| 369 | "archive": archive not in ("0", "false", "no", "off"), |
| 370 | "keep_archives": _int(KEEP_ARCHIVES_ENV), |
| 371 | } |
| 372 | |
| 373 | |
| 374 | def archive_dir(collection: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> Path: |
| 375 | """Where a collection's rotated change-log segments live.""" |
| 376 | return record_changes_file(collection, base_dir=base_dir).parent / ARCHIVE_DIR |
| 377 | |
| 378 | |
| 379 | def list_archives(collection: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> list[Path]: |
| 380 | """Archived segments for one collection, oldest first.""" |
| 381 | directory = archive_dir(collection, base_dir=base_dir) |
| 382 | if not directory.is_dir(): |
| 383 | return [] |
| 384 | return sorted(p for p in directory.glob("changes-*.jsonl.gz") if p.is_file()) |
| 385 | |
| 386 | |
| 387 | def _archive_entries(collection: str, lines: list[str], *, base_dir: Path | str) -> Path: |
| 388 | """Write expired entries to a new compressed segment and return it. |
| 389 | |
| 390 | A new segment per rotation rather than appending to one growing |
| 391 | archive: a gzip member you keep appending to is a file you can only |
| 392 | ever read whole, and the point of moving these out was to stop paying |
| 393 | for the whole history. Segments are timestamped and immutable, so an |
| 394 | auditor can take the three that cover a quarter and leave the rest. |
| 395 | """ |
| 396 | directory = archive_dir(collection, base_dir=base_dir) |
| 397 | directory.mkdir(parents=True, exist_ok=True) |
| 398 | stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S-%f") |
| 399 | path = directory / f"changes-{stamp}.jsonl.gz" |
| 400 | temp_path = path.with_name(f".{path.name}.tmp") |
| 401 | with gzip.open(temp_path, "wt", encoding="utf-8", compresslevel=6) as handle: |
| 402 | for line in lines: |
| 403 | handle.write(line) |
| 404 | handle.write("\n") |
| 405 | temp_path.replace(path) |
| 406 | return path |
| 407 | |
| 408 | |
| 409 | def _remove_stale_archives( |
| 410 | collection: str, *, keep: int, base_dir: Path | str |
| 411 | ) -> list[str]: |
| 412 | """Delete all but the newest `keep` segments; 0 keeps everything. |
| 413 | |
| 414 | This is the only place in this module that destroys anything, and it |
| 415 | is deliberately the last step of the chain rather than the first: by |
| 416 | the time a segment is dropped it has already been rotated out of the |
| 417 | hot file and compressed, so an operator who never sets this keeps |
| 418 | every entry the server ever wrote. |
| 419 | """ |
| 420 | if keep <= 0: |
| 421 | return [] |
| 422 | segments = list_archives(collection, base_dir=base_dir) |
| 423 | stale = segments[:-keep] if len(segments) > keep else [] |
| 424 | for path in stale: |
| 425 | path.unlink(missing_ok=True) |
| 426 | return [path.name for path in stale] |
| 427 | |
| 428 | |
| 429 | def prune_record_changes( |
| 430 | collection: str, |
| 431 | *, |
| 432 | keep_newer_than: str | None = None, |
| 433 | keep_last: int | None = None, |
| 434 | archive: bool = True, |
| 435 | keep_archives: int = 0, |
| 436 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 437 | ) -> dict[str, Any]: |
| 438 | """Trim one collection's change log to a time window and/or a count. |
| 439 | |
| 440 | Rewrites the file in one pass under the same lock every other reader |
| 441 | and writer takes, keeping entries newer than ``keep_newer_than`` |
| 442 | (ISO-8601) and then the newest ``keep_last`` of whatever survives. An |
| 443 | entry with a missing or unparseable timestamp is KEPT -- never delete |
| 444 | evidence we cannot date. |
| 445 | |
| 446 | Returns {"entries_before", "entries_after", "removed", "pruned"}. |
| 447 | Both bounds absent is an honest no-op rather than an error, so a |
| 448 | caller can pass an unconfigured policy straight through. |
| 449 | """ |
| 450 | path = record_changes_file(collection, base_dir=base_dir) |
| 451 | if not path.exists(): |
| 452 | return {"entries_before": 0, "entries_after": 0, "removed": 0, "pruned": False} |
| 453 | if not keep_newer_than and not keep_last: |
| 454 | return {"entries_before": 0, "entries_after": 0, "removed": 0, "pruned": False} |
| 455 | |
| 456 | with _file_lock(path): |
| 457 | lines = path.read_text(encoding="utf-8").splitlines() |
| 458 | kept: list[str] = [] |
| 459 | for line in lines: |
| 460 | if not line.strip(): |
| 461 | continue |
| 462 | try: |
| 463 | entry = json.loads(line) |
| 464 | except json.JSONDecodeError: |
| 465 | kept.append(line) # unreadable is not the same as expired |
| 466 | continue |
| 467 | stamp = str(entry.get("timestamp") or "") |
| 468 | if keep_newer_than and stamp and stamp < str(keep_newer_than): |
| 469 | continue |
| 470 | kept.append(line) |
| 471 | |
| 472 | if keep_last and len(kept) > keep_last: |
| 473 | # The log is append-ordered, so the tail is the newest. |
| 474 | kept = kept[len(kept) - keep_last:] |
| 475 | |
| 476 | before = len([line for line in lines if line.strip()]) |
| 477 | removed = before - len(kept) |
| 478 | if removed <= 0: |
| 479 | return {"entries_before": before, "entries_after": before, |
| 480 | "removed": 0, "pruned": False, "archived": None} |
| 481 | |
| 482 | archived_to = None |
| 483 | if archive: |
| 484 | # Difference by COUNT, not by identity or membership: two |
| 485 | # identical entries are two facts, and a set would archive one |
| 486 | # of them while the file lost both. |
| 487 | kept_counts: dict[str, int] = {} |
| 488 | for line in kept: |
| 489 | kept_counts[line] = kept_counts.get(line, 0) + 1 |
| 490 | expired = [] |
| 491 | for line in lines: |
| 492 | if not line.strip(): |
| 493 | continue |
| 494 | if kept_counts.get(line): |
| 495 | kept_counts[line] -= 1 |
| 496 | continue |
| 497 | expired.append(line) |
| 498 | if expired: |
| 499 | archived_to = _archive_entries( |
| 500 | collection, expired, base_dir=base_dir).name |
| 501 | |
| 502 | temp_path = path.with_name(f".{path.name}.tmp") |
| 503 | temp_path.write_text( |
| 504 | "".join(f"{line}\n" for line in kept), encoding="utf-8") |
| 505 | temp_path.replace(path) |
| 506 | |
| 507 | dropped = _remove_stale_archives( |
| 508 | collection, keep=keep_archives, base_dir=base_dir) |
| 509 | return {"entries_before": before, "entries_after": len(kept), |
| 510 | "removed": removed, "pruned": True, |
| 511 | "archived": archived_to, "archives_removed": dropped} |
| 512 | |
| 513 | |
| 514 | def prune_all_record_changes( |
| 515 | *, |
| 516 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 517 | days: int = 0, |
| 518 | max_entries: int = 0, |
| 519 | archive: bool = True, |
| 520 | keep_archives: int = 0, |
| 521 | ) -> dict[str, Any]: |
| 522 | """Apply one retention policy across every collection's change log. |
| 523 | |
| 524 | One bad or unreadable log never stops the sweep: this runs unattended |
| 525 | on a timer, and a single corrupt file must not mean every other log |
| 526 | grows forever. |
| 527 | """ |
| 528 | if not days and not max_entries: |
| 529 | return {"collections": 0, "removed": 0, "pruned": [], "skipped": "no policy set"} |
| 530 | |
| 531 | cutoff = None |
| 532 | if days: |
| 533 | cutoff = (datetime.now(timezone.utc) - timedelta(days=days) |
| 534 | ).isoformat().replace("+00:00", "Z") |
| 535 | |
| 536 | root = Path(base_dir) / RECORD_CHANGES_DIR |
| 537 | if not root.is_dir(): |
| 538 | return {"collections": 0, "removed": 0, "pruned": []} |
| 539 | |
| 540 | removed = 0 |
| 541 | pruned: list[dict[str, Any]] = [] |
| 542 | for entry in sorted(root.iterdir()): |
| 543 | if not entry.is_dir(): |
| 544 | continue |
| 545 | try: |
| 546 | result = prune_record_changes( |
| 547 | entry.name, keep_newer_than=cutoff, |
| 548 | keep_last=(max_entries or None), |
| 549 | archive=archive, keep_archives=keep_archives, |
| 550 | base_dir=base_dir, |
| 551 | ) |
| 552 | except Exception: # noqa: BLE001 -- one bad log must not stop the sweep |
| 553 | continue |
| 554 | if result.get("pruned"): |
| 555 | removed += result["removed"] |
| 556 | pruned.append({"collection": entry.name, **result}) |
| 557 | return {"collections": len(pruned), "removed": removed, "pruned": pruned} |