Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_record_changes.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_record_changes.py · 20.5 KB · 557 lines
1"""Append-only change logs for TSV-backed collection records.
2
3Record changes are the durable facts behind generated admin history screens and
4record event publication. Events and listener delivery can fail or be retried;
5this file records what actually changed.
6"""
7
8from __future__ import annotations
9
10import gzip
11import json
12import os
13import threading
14from datetime import datetime, timedelta, timezone
15from pathlib import Path
16from typing import Any, Mapping
17
18import object_collections
19import object_ids
20import object_records
21from object_versions import DEFAULT_DATA_DIR
22
23RECORD_CHANGES_DIR = "record_changes"
24CHANGES_FILE = "changes.jsonl"
25DEFAULT_CHANGE_LIMIT = 100
26MAX_CHANGE_LIMIT = 1000
27VALID_ACTIONS = {"create", "update", "delete"}
28
29_LOCKS: dict[str, threading.Lock] = {}
30_LOCKS_GUARD = threading.Lock()
31
32
33class InvalidRecordChangeError(ValueError):
34 """Raised when a record change entry is not safe to write or read."""
35
36
37def append_record_change(
38 *,
39 collection: str,
40 record_id: str,
41 action: str,
42 before: dict[str, Any] | None,
43 after: dict[str, Any] | None,
44 actor: str = "api",
45 message: str = "",
46 correlation_id: str | None = None,
47 base_dir: Path | str = DEFAULT_DATA_DIR,
48) -> dict[str, Any]:
49 """Append one record change and return the stored entry."""
50 path = record_changes_file(collection, base_dir=base_dir)
51 if not object_records.validate_record_id(record_id):
52 raise object_records.InvalidRecordIdError(f"Invalid record id: {record_id}")
53 if action not in VALID_ACTIONS:
54 raise InvalidRecordChangeError(f"Invalid record change action: {action}")
55
56 before_snapshot = _normalize_snapshot(before)
57 after_snapshot = _normalize_snapshot(after)
58 if before_snapshot is None and after_snapshot is None:
59 raise InvalidRecordChangeError("Record change must include before or after")
60
61 timestamp = _utc_timestamp()
62 entry = {
63 "change_id": _change_id(timestamp, collection, record_id, action),
64 "timestamp": timestamp,
65 "collection": collection,
66 "record_id": record_id,
67 "action": action,
68 "actor": _clean_text(actor, default="api"),
69 "message": _clean_text(message, default=_default_message(action)),
70 "correlation_id": correlation_id or None,
71 "changed_fields": _changed_fields(before_snapshot, after_snapshot),
72 "before": before_snapshot,
73 "after": after_snapshot,
74 }
75
76 with _file_lock(path):
77 path.parent.mkdir(parents=True, exist_ok=True)
78 with path.open("a", encoding="utf-8") as handle:
79 handle.write(json.dumps(entry, sort_keys=True, separators=(",", ":")))
80 handle.write("\n")
81
82 return entry
83
84
85def list_record_changes(
86 collection: str,
87 *,
88 record_id: str | None = None,
89 base_dir: Path | str = DEFAULT_DATA_DIR,
90 limit: int = DEFAULT_CHANGE_LIMIT,
91 offset: int = 0,
92 tail_only: bool = False,
93) -> dict[str, Any]:
94 """Return newest-first record changes for one collection or record.
95
96 `tail_only` reads just enough of the end of the log to satisfy
97 limit+offset instead of parsing the whole file. It is opt-in because
98 it makes `total` a count of what was READ rather than of what exists,
99 and a caller that shows "1 of 4,312" must not be handed "1 of 100"
100 without asking. A feed does not care; an audit page does.
101 """
102 path = record_changes_file(collection, base_dir=base_dir)
103 if record_id is not None and not object_records.validate_record_id(record_id):
104 raise object_records.InvalidRecordIdError(f"Invalid record id: {record_id}")
105 _validate_page(limit=limit, offset=offset)
106
107 changes = _read_changes(
108 path, record_id=record_id,
109 tail=(limit + offset if tail_only else None),
110 )
111 total = len(changes)
112 window = changes[offset:offset + limit]
113 payload: dict[str, Any] = {
114 "collection": collection,
115 "changes": window,
116 "count": len(window),
117 "total": total,
118 "limit": limit,
119 "offset": offset,
120 "has_more": offset + len(window) < total,
121 }
122 if record_id is not None:
123 payload["record_id"] = record_id
124 return payload
125
126
127def record_changes_file(collection: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> Path:
128 """Return the validated JSONL change-log path for a collection."""
129 if not object_collections.validate_collection_name(collection):
130 raise object_collections.InvalidCollectionNameError(
131 f"Invalid collection name: {collection}"
132 )
133
134 root = Path(base_dir) / RECORD_CHANGES_DIR
135 path = root / collection / CHANGES_FILE
136 resolved_root = root.resolve(strict=False)
137 resolved_path = path.resolve(strict=False)
138
139 try:
140 resolved_path.relative_to(resolved_root)
141 except ValueError as exc:
142 raise object_collections.InvalidCollectionNameError(
143 f"Record change path escapes change directory: {collection}"
144 ) from exc
145
146 return path
147
148
149# Reading a change log tail-first, in blocks, instead of parsing the whole
150# file. A change log only ever grows, and the newest entries are the ones
151# anybody asks for, so paying for the whole history to answer "what
152# happened lately" is a cost that rises forever while the answer stays the
153# same size.
154#
155# This was not theoretical. One collection's log reached 944MB -- a rollup
156# rewriting 1818 derived rows every five minutes, each rewrite appending a
157# row per record -- and the activity feed, which reads EVERY collection's
158# log, sat parsing a gigabyte of JSON on a single core while the page said
159# "loading..." forever. The function's own docstring had predicted it:
160# "fine at current scale... future work if/when logs grow large."
161_TAIL_BLOCK = 256 * 1024
162
163
164def _tail_lines(path: Path, wanted: int) -> list[str]:
165 """The last `wanted` non-empty lines, read backwards in blocks.
166
167 Falls back to a whole-file read only for a file smaller than one
168 block, where seeking would be more code than it saves.
169 """
170 try:
171 size = path.stat().st_size
172 except OSError:
173 return []
174 if size <= _TAIL_BLOCK:
175 return path.read_text(encoding="utf-8").splitlines()
176
177 chunks: list[bytes] = []
178 newlines = 0
179 with path.open("rb") as handle:
180 position = size
181 while position > 0 and newlines <= wanted:
182 step = min(_TAIL_BLOCK, position)
183 position -= step
184 handle.seek(position)
185 block = handle.read(step)
186 chunks.append(block)
187 newlines += block.count(b"\n")
188 text = b"".join(reversed(chunks)).decode("utf-8", "replace")
189 # The first line of the first block read is probably a fragment of a
190 # longer line; dropping it is why we read one block past `wanted`.
191 lines = text.splitlines()
192 return lines[1:] if position > 0 else lines
193
194
195def _read_changes(
196 path: Path, *, record_id: str | None, tail: int | None = None
197) -> list[dict[str, Any]]:
198 if not path.exists():
199 return []
200
201 if tail is not None and record_id is None:
202 source = _tail_lines(path, tail)
203 else:
204 # A record-scoped query still scans: the entries for one record can
205 # be anywhere in the file, and answering "show me this invoice's
206 # history" with only the recent tail would quietly lose the half
207 # that matters.
208 source = None
209
210 changes: list[dict[str, Any]] = []
211 with _file_lock(path):
212 for line in (source if source is not None
213 else path.read_text(encoding="utf-8").splitlines()):
214 if not line.strip():
215 continue
216 try:
217 entry = json.loads(line)
218 except json.JSONDecodeError:
219 continue
220 if not isinstance(entry, dict):
221 continue
222 if record_id is not None and entry.get("record_id") != record_id:
223 continue
224 changes.append(entry)
225
226 changes.reverse()
227 return changes
228
229
230def _normalize_snapshot(record: dict[str, Any] | None) -> dict[str, Any] | None:
231 if record is None:
232 return None
233 if not isinstance(record, dict):
234 raise InvalidRecordChangeError("Record snapshot must be an object")
235 return {str(key): _json_safe_value(value) for key, value in record.items()}
236
237
238def _json_safe_value(value: Any) -> Any:
239 if value is None or isinstance(value, str | int | float | bool):
240 return value
241 return str(value)
242
243
244def _changed_fields(
245 before: dict[str, Any] | None,
246 after: dict[str, Any] | None,
247) -> list[str]:
248 before_values = before or {}
249 after_values = after or {}
250 names = set(before_values) | set(after_values)
251 return sorted(name for name in names if before_values.get(name) != after_values.get(name))
252
253
254def _validate_page(*, limit: int, offset: int) -> None:
255 if limit < 1:
256 raise ValueError("limit must be at least 1")
257 if limit > MAX_CHANGE_LIMIT:
258 raise ValueError(f"limit must be at most {MAX_CHANGE_LIMIT}")
259 if offset < 0:
260 raise ValueError("offset must be at least 0")
261
262
263def _clean_text(value: str, *, default: str) -> str:
264 text = str(value).strip()
265 return text or default
266
267
268def _default_message(action: str) -> str:
269 return {
270 "create": "Created record",
271 "update": "Updated record",
272 "delete": "Deleted record",
273 }[action]
274
275
276def _change_id(timestamp: str, collection: str, record_id: str, action: str) -> str:
277 return object_ids.new_uuid4()
278
279
280def _utc_timestamp() -> str:
281 return datetime.now(timezone.utc).isoformat()
282
283
284def _file_lock(path: Path) -> threading.Lock:
285 key = str(path.resolve(strict=False))
286 with _LOCKS_GUARD:
287 lock = _LOCKS.get(key)
288 if lock is None:
289 lock = threading.Lock()
290 _LOCKS[key] = lock
291 return lock
292
293
294# --- retention -----------------------------------------------------------------
295#
296# The third unbounded log this server has grown, after page_views and the
297# package restore points, and the pattern is now unmistakable: a log
298# nobody tells how big it may get is a log that fails on the worst day
299# rather than an ordinary one.
300#
301# A change log is audit evidence, so pruning it is a genuinely different
302# act from pruning analytics. Two consequences, both deliberate:
303#
304# * Retention is OFF unless an operator turns it on. Deleting audit
305# history by default is not a performance optimisation, it is a
306# decision about accountability, and it is not this module's to make
307# quietly.
308# * The bound is stated in BOTH dimensions, because age alone does not
309# bound anything: a retention window says how far back to keep and
310# nothing at all about how much can arrive inside it.
311#
312# The rollup churn that motivated this is fixed at the source instead
313# (object_rollups writes derived rows with record_changes=False) -- which
314# is the better fix, because a log entry never written costs nothing to
315# store, nothing to read past, and nothing to decide about later.
316
317RETENTION_DAYS_ENV = "DBBASIC_CHANGE_LOG_RETENTION_DAYS"
318MAX_ENTRIES_ENV = "DBBASIC_CHANGE_LOG_MAX_ENTRIES"
319ARCHIVE_ENV = "DBBASIC_CHANGE_LOG_ARCHIVE"
320KEEP_ARCHIVES_ENV = "DBBASIC_CHANGE_LOG_KEEP_ARCHIVES"
321ARCHIVE_DIR = "archive"
322
323# Archival and retention are different questions, and conflating them is
324# what made the first version of this uncomfortable.
325#
326# ROTATION bounds the file the server reads on every feed render.
327# ARCHIVAL decides whether the entries leaving that file still exist.
328# REMOVAL decides how long the archives themselves are kept.
329#
330# An audit trail wants all three, in that order. Rotating without
331# archiving is deletion wearing a gentler word, and it is why retention
332# had to be opt-in and slightly apologetic. Archiving first inverts that:
333# pruning becomes cheap to turn on, because nothing is actually lost --
334# the entries move out of the hot read path into a compressed segment
335# that is still on disk, still greppable, and still handable to an
336# auditor. JSONL compresses roughly ten to twenty times, so the 944MB log
337# that started all this becomes a segment measured in tens of megabytes.
338#
339# Removal still has to exist, because an archive nobody ever deletes is
340# just the original problem moved one directory down. Hence
341# KEEP_ARCHIVES: the same "keep the newest N" shape object_logs already
342# uses for rotated server logs, so an operator learns the convention once.
343# Its default keeps everything, because the default for evidence should
344# never be "throw it away".
345
346
347def retention_policy(env: Mapping[str, str] | None = None) -> dict[str, int]:
348 """{"days", "max_entries"} -- 0 for either means "no bound".
349
350 Absent configuration is no bound at all, not a default window: an
351 operator who has never thought about audit retention has not thereby
352 consented to losing audit history.
353 """
354 source = os.environ if env is None else env
355
356 def _int(name: str) -> int:
357 try:
358 value = int(str(source.get(name, "")).strip())
359 except (TypeError, ValueError):
360 return 0
361 return value if value > 0 else 0
362
363 archive = str(source.get(ARCHIVE_ENV, "")).strip().lower()
364 return {
365 "days": _int(RETENTION_DAYS_ENV),
366 "max_entries": _int(MAX_ENTRIES_ENV),
367 # Archiving is the DEFAULT when a policy is set at all. Somebody
368 # asking for a smaller hot file has not asked to lose the history.
369 "archive": archive not in ("0", "false", "no", "off"),
370 "keep_archives": _int(KEEP_ARCHIVES_ENV),
371 }
372
373
374def archive_dir(collection: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> Path:
375 """Where a collection's rotated change-log segments live."""
376 return record_changes_file(collection, base_dir=base_dir).parent / ARCHIVE_DIR
377
378
379def list_archives(collection: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> list[Path]:
380 """Archived segments for one collection, oldest first."""
381 directory = archive_dir(collection, base_dir=base_dir)
382 if not directory.is_dir():
383 return []
384 return sorted(p for p in directory.glob("changes-*.jsonl.gz") if p.is_file())
385
386
387def _archive_entries(collection: str, lines: list[str], *, base_dir: Path | str) -> Path:
388 """Write expired entries to a new compressed segment and return it.
389
390 A new segment per rotation rather than appending to one growing
391 archive: a gzip member you keep appending to is a file you can only
392 ever read whole, and the point of moving these out was to stop paying
393 for the whole history. Segments are timestamped and immutable, so an
394 auditor can take the three that cover a quarter and leave the rest.
395 """
396 directory = archive_dir(collection, base_dir=base_dir)
397 directory.mkdir(parents=True, exist_ok=True)
398 stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S-%f")
399 path = directory / f"changes-{stamp}.jsonl.gz"
400 temp_path = path.with_name(f".{path.name}.tmp")
401 with gzip.open(temp_path, "wt", encoding="utf-8", compresslevel=6) as handle:
402 for line in lines:
403 handle.write(line)
404 handle.write("\n")
405 temp_path.replace(path)
406 return path
407
408
409def _remove_stale_archives(
410 collection: str, *, keep: int, base_dir: Path | str
411) -> list[str]:
412 """Delete all but the newest `keep` segments; 0 keeps everything.
413
414 This is the only place in this module that destroys anything, and it
415 is deliberately the last step of the chain rather than the first: by
416 the time a segment is dropped it has already been rotated out of the
417 hot file and compressed, so an operator who never sets this keeps
418 every entry the server ever wrote.
419 """
420 if keep <= 0:
421 return []
422 segments = list_archives(collection, base_dir=base_dir)
423 stale = segments[:-keep] if len(segments) > keep else []
424 for path in stale:
425 path.unlink(missing_ok=True)
426 return [path.name for path in stale]
427
428
429def prune_record_changes(
430 collection: str,
431 *,
432 keep_newer_than: str | None = None,
433 keep_last: int | None = None,
434 archive: bool = True,
435 keep_archives: int = 0,
436 base_dir: Path | str = DEFAULT_DATA_DIR,
437) -> dict[str, Any]:
438 """Trim one collection's change log to a time window and/or a count.
439
440 Rewrites the file in one pass under the same lock every other reader
441 and writer takes, keeping entries newer than ``keep_newer_than``
442 (ISO-8601) and then the newest ``keep_last`` of whatever survives. An
443 entry with a missing or unparseable timestamp is KEPT -- never delete
444 evidence we cannot date.
445
446 Returns {"entries_before", "entries_after", "removed", "pruned"}.
447 Both bounds absent is an honest no-op rather than an error, so a
448 caller can pass an unconfigured policy straight through.
449 """
450 path = record_changes_file(collection, base_dir=base_dir)
451 if not path.exists():
452 return {"entries_before": 0, "entries_after": 0, "removed": 0, "pruned": False}
453 if not keep_newer_than and not keep_last:
454 return {"entries_before": 0, "entries_after": 0, "removed": 0, "pruned": False}
455
456 with _file_lock(path):
457 lines = path.read_text(encoding="utf-8").splitlines()
458 kept: list[str] = []
459 for line in lines:
460 if not line.strip():
461 continue
462 try:
463 entry = json.loads(line)
464 except json.JSONDecodeError:
465 kept.append(line) # unreadable is not the same as expired
466 continue
467 stamp = str(entry.get("timestamp") or "")
468 if keep_newer_than and stamp and stamp < str(keep_newer_than):
469 continue
470 kept.append(line)
471
472 if keep_last and len(kept) > keep_last:
473 # The log is append-ordered, so the tail is the newest.
474 kept = kept[len(kept) - keep_last:]
475
476 before = len([line for line in lines if line.strip()])
477 removed = before - len(kept)
478 if removed <= 0:
479 return {"entries_before": before, "entries_after": before,
480 "removed": 0, "pruned": False, "archived": None}
481
482 archived_to = None
483 if archive:
484 # Difference by COUNT, not by identity or membership: two
485 # identical entries are two facts, and a set would archive one
486 # of them while the file lost both.
487 kept_counts: dict[str, int] = {}
488 for line in kept:
489 kept_counts[line] = kept_counts.get(line, 0) + 1
490 expired = []
491 for line in lines:
492 if not line.strip():
493 continue
494 if kept_counts.get(line):
495 kept_counts[line] -= 1
496 continue
497 expired.append(line)
498 if expired:
499 archived_to = _archive_entries(
500 collection, expired, base_dir=base_dir).name
501
502 temp_path = path.with_name(f".{path.name}.tmp")
503 temp_path.write_text(
504 "".join(f"{line}\n" for line in kept), encoding="utf-8")
505 temp_path.replace(path)
506
507 dropped = _remove_stale_archives(
508 collection, keep=keep_archives, base_dir=base_dir)
509 return {"entries_before": before, "entries_after": len(kept),
510 "removed": removed, "pruned": True,
511 "archived": archived_to, "archives_removed": dropped}
512
513
514def prune_all_record_changes(
515 *,
516 base_dir: Path | str = DEFAULT_DATA_DIR,
517 days: int = 0,
518 max_entries: int = 0,
519 archive: bool = True,
520 keep_archives: int = 0,
521) -> dict[str, Any]:
522 """Apply one retention policy across every collection's change log.
523
524 One bad or unreadable log never stops the sweep: this runs unattended
525 on a timer, and a single corrupt file must not mean every other log
526 grows forever.
527 """
528 if not days and not max_entries:
529 return {"collections": 0, "removed": 0, "pruned": [], "skipped": "no policy set"}
530
531 cutoff = None
532 if days:
533 cutoff = (datetime.now(timezone.utc) - timedelta(days=days)
534 ).isoformat().replace("+00:00", "Z")
535
536 root = Path(base_dir) / RECORD_CHANGES_DIR
537 if not root.is_dir():
538 return {"collections": 0, "removed": 0, "pruned": []}
539
540 removed = 0
541 pruned: list[dict[str, Any]] = []
542 for entry in sorted(root.iterdir()):
543 if not entry.is_dir():
544 continue
545 try:
546 result = prune_record_changes(
547 entry.name, keep_newer_than=cutoff,
548 keep_last=(max_entries or None),
549 archive=archive, keep_archives=keep_archives,
550 base_dir=base_dir,
551 )
552 except Exception: # noqa: BLE001 -- one bad log must not stop the sweep
553 continue
554 if result.get("pruned"):
555 removed += result["removed"]
556 pruned.append({"collection": entry.name, **result})
557 return {"collections": len(pruned), "removed": removed, "pruned": pruned}