askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_backup_index.py · 18.3 KB · 478 lines
| 1 | """Read-only inventory and on-demand creation for runtime backups. |
| 2 | |
| 3 | Backups are the tar/gzip archives written by ``object_backup`` under the |
| 4 | configured backups directory. This module lists them, creates a new |
| 5 | full-runtime backup on demand, and resolves a backup id to a safe path |
| 6 | for download. It is the data layer behind the admin backup endpoints. |
| 7 | |
| 8 | A backup contains the whole runtime data directory — records, identity, |
| 9 | credentials, service keys — so the HTTP surface that uses this module is |
| 10 | strictly admin-gated. Nothing here is public. |
| 11 | """ |
| 12 | |
| 13 | from __future__ import annotations |
| 14 | |
| 15 | import csv |
| 16 | import hashlib |
| 17 | import io |
| 18 | import json |
| 19 | import re |
| 20 | from datetime import datetime, timedelta, timezone |
| 21 | from pathlib import Path |
| 22 | |
| 23 | import object_backup |
| 24 | import object_collections |
| 25 | import object_records |
| 26 | |
| 27 | MANUAL_LABEL = "manual" |
| 28 | _ID_RE = re.compile(r"^[0-9A-Za-z][0-9A-Za-z._-]{0,255}\.tar\.gz$") |
| 29 | |
| 30 | |
| 31 | def backups_dir(data_dir: Path | str | None = None) -> Path: |
| 32 | """Return the configured backups directory (honors DBBASIC_BACKUPS_DIR).""" |
| 33 | return object_backup._backups_dir(None, data_dir=data_dir) |
| 34 | |
| 35 | |
| 36 | def validate_backup_id(backup_id: str) -> bool: |
| 37 | """Return True when a backup id is a safe archive filename (no traversal).""" |
| 38 | if not isinstance(backup_id, str) or "/" in backup_id or "\\" in backup_id: |
| 39 | return False |
| 40 | return bool(_ID_RE.fullmatch(backup_id)) |
| 41 | |
| 42 | |
| 43 | def backup_path(backup_id: str, *, data_dir: Path | str | None = None) -> Path: |
| 44 | """Resolve a backup id to its path, refusing anything outside the dir.""" |
| 45 | if not validate_backup_id(backup_id): |
| 46 | raise ValueError(f"invalid backup id: {backup_id!r}") |
| 47 | root = backups_dir(data_dir).resolve(strict=False) |
| 48 | path = (backups_dir(data_dir) / backup_id).resolve(strict=False) |
| 49 | try: |
| 50 | path.relative_to(root) |
| 51 | except ValueError as exc: |
| 52 | raise ValueError("backup path escapes the backups directory") from exc |
| 53 | return path |
| 54 | |
| 55 | |
| 56 | def list_backups(*, data_dir: Path | str | None = None) -> list[dict[str, object]]: |
| 57 | """Return backup metadata, newest first — never the archive contents.""" |
| 58 | directory = backups_dir(data_dir) |
| 59 | if not directory.is_dir(): |
| 60 | return [] |
| 61 | entries = [ |
| 62 | _entry(path) |
| 63 | for path in directory.glob("*.tar.gz") |
| 64 | if path.is_file() and validate_backup_id(path.name) |
| 65 | ] |
| 66 | entries.sort(key=lambda entry: entry["created_at"], reverse=True) |
| 67 | return entries |
| 68 | |
| 69 | |
| 70 | def create_backup(*, data_dir: Path | str | None = None) -> dict[str, object]: |
| 71 | """Create a full-runtime backup now and return its metadata.""" |
| 72 | summary = object_backup.create_runtime_restore_point(MANUAL_LABEL, data_dir=data_dir) |
| 73 | return _entry(Path(summary.path)) |
| 74 | |
| 75 | |
| 76 | def _entry(path: Path) -> dict[str, object]: |
| 77 | stat = path.stat() |
| 78 | name = path.name |
| 79 | stem = name[: -len(".tar.gz")] if name.endswith(".tar.gz") else name |
| 80 | parts = stem.split("-", 1) |
| 81 | label = parts[1] if len(parts) == 2 else stem |
| 82 | if label.startswith("package-"): |
| 83 | kind, scope = "package", label[len("package-"):] |
| 84 | elif label == MANUAL_LABEL: |
| 85 | kind, scope = "manual", "runtime" |
| 86 | else: |
| 87 | kind, scope = "restore-point", label |
| 88 | created_at = ( |
| 89 | datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc) |
| 90 | .isoformat() |
| 91 | .replace("+00:00", "Z") |
| 92 | ) |
| 93 | return { |
| 94 | "id": name, |
| 95 | "created_at": created_at, |
| 96 | "size": stat.st_size, |
| 97 | "kind": kind, |
| 98 | "scope": scope, |
| 99 | } |
| 100 | |
| 101 | |
| 102 | # Append-mode physical format (docs/storage-modes.md "Append"; the |
| 103 | # authoritative implementation is object_records.py's OP_FIELD/OP_UPSERT/ |
| 104 | # OP_DELETE and _fold_append_rows). Kept as plain local constants rather |
| 105 | # than importing object_records's, so this module's read of a records.tsv |
| 106 | # stays independent of that module's private helpers. |
| 107 | _OP_FIELD = "_op" |
| 108 | _OP_DELETE = "del" |
| 109 | |
| 110 | |
| 111 | def _drop_torn_tail(text: str) -> str: |
| 112 | """Drop an unterminated final physical line (a write caught mid-append). |
| 113 | |
| 114 | Append-mode writers only consider a row committed once it is followed |
| 115 | by a row-terminating "\\n" (see docs/append-only-storage-design.md, |
| 116 | Crash Safety), so anything after the last one is an in-flight write -- |
| 117 | not data -- on either the backup or the live side. |
| 118 | |
| 119 | QUOTE-AWARE, via the shared `object_records.committed_prefix_len`: a |
| 120 | "\\n" INSIDE a quoted multi-line field is content, never a row |
| 121 | boundary. |
| 122 | |
| 123 | THIS WAS THE BUG, and it is worth naming because of how it survived. |
| 124 | The body used to be `endswith("\\n") or rfind("\\n")` under a docstring |
| 125 | claiming it "mirrors object_records._drop_torn_tail" -- which had been |
| 126 | upgraded to be quote-aware while this copy was not, silently, for |
| 127 | months. Worse than the read-side original: a file ending with a "\\n" |
| 128 | that sits INSIDE an unclosed quote was returned untouched, so a |
| 129 | restored backup could resurrect a garbled row that a live read of the |
| 130 | same bytes handles correctly. It carried no test at all, which is |
| 131 | precisely why nothing caught the drift. |
| 132 | |
| 133 | It now CALLS the shared implementation instead of mirroring it. That is |
| 134 | a deliberate exception to this module's usual independence from |
| 135 | object_records' internals: keeping a private copy is exactly what |
| 136 | failed, `committed_prefix_len` is exported for this reason, and two |
| 137 | implementations of one invariant drift while one cannot. |
| 138 | """ |
| 139 | if text == "": |
| 140 | return text |
| 141 | return text[: object_records.committed_prefix_len(text)] |
| 142 | |
| 143 | |
| 144 | def _fold_append_rows(rows: list[dict[str, str]]) -> dict[str, dict[str, str]]: |
| 145 | """Fold physical append-mode rows into live records, last-wins per id. |
| 146 | |
| 147 | Minimal local replica of object_records._fold_append_rows's id |
| 148 | semantics, for backup preview/diff purposes only (docs/storage-modes.md |
| 149 | "Append"): a delete (`_op == "del"`) removes the id; any other row is |
| 150 | an upsert that (re)sets it — so an id updated multiple times, or |
| 151 | deleted and later re-created, ends up holding just its final live |
| 152 | values. `_op` itself is stripped from every returned row. Unlike |
| 153 | object_records's fold, this doesn't need to preserve list order (the |
| 154 | callers here only ever look records up by id for a diff), so it |
| 155 | returns a plain dict rather than an OrderedDict-backed list. |
| 156 | """ |
| 157 | folded: dict[str, dict[str, str]] = {} |
| 158 | for row in rows: |
| 159 | record_id = row.get("id") |
| 160 | if not record_id: |
| 161 | continue |
| 162 | if row.get(_OP_FIELD) == _OP_DELETE: |
| 163 | folded.pop(record_id, None) |
| 164 | continue |
| 165 | folded[record_id] = {key: value for key, value in row.items() if key != _OP_FIELD} |
| 166 | return folded |
| 167 | |
| 168 | |
| 169 | def _parse_append_tsv_by_id(text: str, header: list[str]) -> dict[str, dict[str, str]]: |
| 170 | """Parse+fold an append-mode records.tsv body into a dict keyed by id. |
| 171 | |
| 172 | `header` is the already-inspected physical header (first column |
| 173 | `_op`). A torn final line is dropped first (_drop_torn_tail); a row |
| 174 | that still fails to tokenize stops consumption at that point rather |
| 175 | than raising, matching object_records._parse_append_body's tolerance |
| 176 | of an in-flight write. |
| 177 | """ |
| 178 | field_count = len(header) |
| 179 | reader = csv.reader(io.StringIO(_drop_torn_tail(text)), delimiter="\t") |
| 180 | next(reader, None) # header, already parsed by the caller |
| 181 | |
| 182 | physical_rows: list[dict[str, str]] = [] |
| 183 | while True: |
| 184 | try: |
| 185 | row = next(reader) |
| 186 | except StopIteration: |
| 187 | break |
| 188 | except csv.Error: |
| 189 | break |
| 190 | if not row: |
| 191 | continue |
| 192 | physical_rows.append( |
| 193 | {header[i]: (row[i] if i < len(row) else "") for i in range(field_count)} |
| 194 | ) |
| 195 | |
| 196 | return _fold_append_rows(physical_rows) |
| 197 | |
| 198 | |
| 199 | def _parse_tsv_by_id(raw: bytes | None) -> dict[str, dict[str, str]]: |
| 200 | """Parse a records.tsv payload into a dict keyed by the "id" column. |
| 201 | |
| 202 | Dispatches on the physical header, exactly like object_records does |
| 203 | (docs/append-only-storage-design.md): a file whose first column is |
| 204 | `_op` is an append-mode log and is folded last-wins-by-id first (see |
| 205 | _fold_append_rows), so callers only ever see LIVE logical records — |
| 206 | never raw log rows, superseded values, or tombstoned ids. Classic |
| 207 | files (no `_op` header) go through the original DictReader-based |
| 208 | parse, unchanged. |
| 209 | """ |
| 210 | if not raw: |
| 211 | return {} |
| 212 | text = raw.decode("utf-8") |
| 213 | |
| 214 | header_row = next(csv.reader(io.StringIO(text), delimiter="\t"), None) |
| 215 | if header_row and header_row[0] == _OP_FIELD: |
| 216 | return _parse_append_tsv_by_id(text, header_row) |
| 217 | |
| 218 | reader = csv.DictReader(io.StringIO(text), delimiter="\t") |
| 219 | rows: dict[str, dict[str, str]] = {} |
| 220 | for row in reader: |
| 221 | record_id = row.get("id") |
| 222 | if record_id is None: |
| 223 | continue |
| 224 | rows[record_id] = dict(row) |
| 225 | return rows |
| 226 | |
| 227 | |
| 228 | def _records_tsv_member(collection: str) -> str: |
| 229 | return f"data/collections/{collection}/records.tsv" |
| 230 | |
| 231 | |
| 232 | def _live_records_tsv(collection: str, *, data_dir: Path | str | None) -> bytes | None: |
| 233 | path = object_backup._data_dir(data_dir) / "collections" / collection / "records.tsv" |
| 234 | if not path.is_file(): |
| 235 | return None |
| 236 | return path.read_bytes() |
| 237 | |
| 238 | |
| 239 | def _diff_records( |
| 240 | backup_rows: dict[str, dict[str, str]], |
| 241 | live_rows: dict[str, dict[str, str]], |
| 242 | ) -> dict[str, object]: |
| 243 | """Diff two id->row maps with the semantics "restoring makes live == backup".""" |
| 244 | added = sorted(set(backup_rows) - set(live_rows)) |
| 245 | removed = sorted(set(live_rows) - set(backup_rows)) |
| 246 | changed = [] |
| 247 | unchanged = 0 |
| 248 | for record_id in sorted(set(backup_rows) & set(live_rows)): |
| 249 | backup_row = backup_rows[record_id] |
| 250 | live_row = live_rows[record_id] |
| 251 | if backup_row == live_row: |
| 252 | unchanged += 1 |
| 253 | continue |
| 254 | fields = sorted( |
| 255 | field |
| 256 | for field in set(backup_row) | set(live_row) |
| 257 | if backup_row.get(field) != live_row.get(field) |
| 258 | ) |
| 259 | changed.append({"id": record_id, "fields": fields}) |
| 260 | |
| 261 | diff_hash = hashlib.sha256( |
| 262 | json.dumps({"added": added, "removed": removed, "changed": changed}, sort_keys=True).encode( |
| 263 | "utf-8" |
| 264 | ) |
| 265 | ).hexdigest() |
| 266 | |
| 267 | return { |
| 268 | "diff_hash": diff_hash, |
| 269 | "added": added, |
| 270 | "removed": removed, |
| 271 | "changed": changed, |
| 272 | "unchanged": unchanged, |
| 273 | } |
| 274 | |
| 275 | |
| 276 | def preview_collection( |
| 277 | backup_id: str, |
| 278 | collection: str, |
| 279 | *, |
| 280 | data_dir: Path | str | None = None, |
| 281 | ) -> dict[str, object]: |
| 282 | """Diff a collection's records between a backup and the live data dir. |
| 283 | |
| 284 | The semantics are "if you restore, live becomes the backup's version": |
| 285 | added ids would reappear, removed ids would be dropped, changed ids |
| 286 | would have the listed fields overwritten. Either side (or both) may be |
| 287 | an append-mode file, a classic file, or missing; _parse_tsv_by_id folds |
| 288 | append-mode logs to live records first, so this always diffs live |
| 289 | logical content regardless of physical storage mode. |
| 290 | """ |
| 291 | if not object_collections.validate_collection_name(collection): |
| 292 | raise ValueError(f"Invalid collection name: {collection}") |
| 293 | |
| 294 | archive_path = backup_path(backup_id, data_dir=data_dir) |
| 295 | member_name = _records_tsv_member(collection) |
| 296 | archived_raw = object_backup.read_backup_member(archive_path, member_name) |
| 297 | live_raw = _live_records_tsv(collection, data_dir=data_dir) |
| 298 | |
| 299 | backup_rows = _parse_tsv_by_id(archived_raw) |
| 300 | live_rows = _parse_tsv_by_id(live_raw) |
| 301 | diff = _diff_records(backup_rows, live_rows) |
| 302 | |
| 303 | return { |
| 304 | "target": {"kind": "collection", "name": collection}, |
| 305 | "backup_id": backup_id, |
| 306 | "present_in_backup": archived_raw is not None, |
| 307 | "diff_hash": diff["diff_hash"], |
| 308 | "added": diff["added"], |
| 309 | "removed": diff["removed"], |
| 310 | "changed": diff["changed"], |
| 311 | "unchanged": diff["unchanged"], |
| 312 | } |
| 313 | |
| 314 | |
| 315 | def preview_record( |
| 316 | backup_id: str, |
| 317 | collection: str, |
| 318 | record_id: str, |
| 319 | *, |
| 320 | data_dir: Path | str | None = None, |
| 321 | ) -> dict[str, object]: |
| 322 | """Return one record's backup vs. live presence, without diffing all rows. |
| 323 | |
| 324 | `record` is the FOLDED record when the source is append-mode: a |
| 325 | tombstoned id reports record=None/present_in_*=False exactly like an |
| 326 | id that was never written. |
| 327 | """ |
| 328 | if not object_collections.validate_collection_name(collection): |
| 329 | raise ValueError(f"Invalid collection name: {collection}") |
| 330 | |
| 331 | archive_path = backup_path(backup_id, data_dir=data_dir) |
| 332 | member_name = _records_tsv_member(collection) |
| 333 | archived_raw = object_backup.read_backup_member(archive_path, member_name) |
| 334 | live_raw = _live_records_tsv(collection, data_dir=data_dir) |
| 335 | |
| 336 | backup_rows = _parse_tsv_by_id(archived_raw) |
| 337 | live_rows = _parse_tsv_by_id(live_raw) |
| 338 | |
| 339 | return { |
| 340 | "collection": collection, |
| 341 | "id": record_id, |
| 342 | "record": backup_rows.get(record_id), |
| 343 | "present_in_backup": record_id in backup_rows, |
| 344 | "present_in_live": record_id in live_rows, |
| 345 | } |
| 346 | |
| 347 | |
| 348 | # === retention ============================================================== |
| 349 | # |
| 350 | # A restore point is created automatically before EVERY package install |
| 351 | # (object_server's create_restore_point, passed to install_package as |
| 352 | # before_write). Nothing ever removed them. On the box that motivated this |
| 353 | # there were 157 of them at ~91MB each -- 14.3GB against 4.0GB free, on a |
| 354 | # machine that also serves production, growing by one archive per install. |
| 355 | # 93 of them had been written in the preceding two days. |
| 356 | # |
| 357 | # That is the fourth instance of one pattern here: page_views, restore |
| 358 | # points, change logs, and now restore points again. A log nobody told how |
| 359 | # big it may get fails on the worst day rather than an ordinary one. |
| 360 | # |
| 361 | # WHAT MAKES A DEFAULT SAFE HERE, and why this differs from the change-log |
| 362 | # pass (which is off unless an operator asks): those archives are not the |
| 363 | # operator's data. Nobody asked for 157 of them; they are scaffolding the |
| 364 | # server produced on its own behalf, and the value of the hundredth-oldest |
| 365 | # one from three weeks ago is indistinguishable from zero. A change log, |
| 366 | # by contrast, IS the audit trail, which is why deciding how much of it to |
| 367 | # keep is not a decision a daemon makes for somebody. |
| 368 | # |
| 369 | # So the split is by KIND, which `_entry` already computes: |
| 370 | # |
| 371 | # package automatic, one per install -> prunable |
| 372 | # manual a person pressed a button -> NEVER touched |
| 373 | # restore-point a named point somebody chose -> NEVER touched |
| 374 | # |
| 375 | # Anything a human asked for survives forever regardless of policy. Only |
| 376 | # the machine's own scaffolding is swept. |
| 377 | |
| 378 | PRUNABLE_KINDS = ("package",) |
| 379 | |
| 380 | # Below this many prunable archives, do nothing at all. A handful of |
| 381 | # restore points is the feature working; sweeping down to the bone would |
| 382 | # remove the safety net this exists to provide. |
| 383 | MIN_KEEP = 5 |
| 384 | |
| 385 | # The ceiling that actually matters. A first cut of this used only |
| 386 | # keep_last plus an age window, and a dry run on the real box showed it |
| 387 | # freeing 78MB out of 14.3GB: 93 archives had been written in two days, so |
| 388 | # the age window protected nearly all of them -- and because the data |
| 389 | # directory grows, the RECENT ones are the large ones (~91MB each against |
| 390 | # ~1.7MB for the oldest). A policy expressed in COUNT and AGE cannot bound |
| 391 | # a quantity measured in BYTES. |
| 392 | # |
| 393 | # So the budget is the primary control and the window is a courtesy inside |
| 394 | # it: newest-first, keep while there is room. 2GB holds roughly the last |
| 395 | # twenty snapshots of a mid-size box and is a rounding error next to the |
| 396 | # disk it protects. |
| 397 | DEFAULT_MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 |
| 398 | |
| 399 | |
| 400 | def prune_backups( |
| 401 | *, |
| 402 | data_dir: Path | str | None = None, |
| 403 | keep_last: int = 20, |
| 404 | keep_newer_than_days: int = 0, |
| 405 | max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES, |
| 406 | dry_run: bool = False, |
| 407 | ) -> dict[str, object]: |
| 408 | """Remove old AUTOMATIC restore points. Never touches manual backups. |
| 409 | |
| 410 | `keep_last` -- how many of the newest automatic archives always |
| 411 | survive, whatever they weigh. The safety floor. |
| 412 | `max_total_bytes` -- the ceiling on everything kept. THE control that |
| 413 | actually bounds a disk; see DEFAULT_MAX_TOTAL_BYTES for why count and |
| 414 | age alone provably do not. 0 disables it. |
| 415 | `keep_newer_than_days` -- inside the budget, prefer keeping anything |
| 416 | younger than this, so a burst of installs in one afternoon does not |
| 417 | evict the morning's safety net. It can never push the total past the |
| 418 | ceiling. 0 disables the window. |
| 419 | |
| 420 | Returns what it did (or would do, with `dry_run`), never raising for |
| 421 | an individual file it cannot delete: a backup directory that is |
| 422 | partially unwritable should still be swept as far as it can be, and |
| 423 | the caller gets the errors rather than an exception mid-loop. |
| 424 | """ |
| 425 | keep_last = max(MIN_KEEP, int(keep_last)) |
| 426 | entries = list_backups(data_dir=data_dir) # newest first |
| 427 | |
| 428 | prunable = [e for e in entries if e["kind"] in PRUNABLE_KINDS] |
| 429 | protected = [e for e in entries if e["kind"] not in PRUNABLE_KINDS] |
| 430 | |
| 431 | cutoff = "" |
| 432 | if keep_newer_than_days and int(keep_newer_than_days) > 0: |
| 433 | cutoff = ( |
| 434 | datetime.now(tz=timezone.utc) |
| 435 | - timedelta(days=int(keep_newer_than_days)) |
| 436 | ).isoformat().replace("+00:00", "Z") |
| 437 | |
| 438 | # Newest-first: the floor is unconditional, then the window is honoured |
| 439 | # only while the running total stays under the ceiling. |
| 440 | kept_bytes = sum(int(e["size"]) for e in prunable[:keep_last]) |
| 441 | budget = int(max_total_bytes) |
| 442 | doomed = [] |
| 443 | for entry in prunable[keep_last:]: |
| 444 | size = int(entry["size"]) |
| 445 | in_window = bool(cutoff) and str(entry["created_at"]) >= cutoff |
| 446 | room = (not budget) or (kept_bytes + size <= budget) |
| 447 | if in_window and room: |
| 448 | kept_bytes += size |
| 449 | continue |
| 450 | doomed.append(entry) |
| 451 | |
| 452 | removed, freed, errors = [], 0, [] |
| 453 | if not dry_run: |
| 454 | directory = backups_dir(data_dir) |
| 455 | for entry in doomed: |
| 456 | path = directory / str(entry["id"]) |
| 457 | try: |
| 458 | size = path.stat().st_size |
| 459 | path.unlink() |
| 460 | except OSError as exc: |
| 461 | errors.append(f"{entry['id']}: {exc}") |
| 462 | continue |
| 463 | removed.append(str(entry["id"])) |
| 464 | freed += size |
| 465 | else: |
| 466 | removed = [str(entry["id"]) for entry in doomed] |
| 467 | freed = sum(int(entry["size"]) for entry in doomed) |
| 468 | |
| 469 | return { |
| 470 | "removed": len(removed), |
| 471 | "removed_ids": removed, |
| 472 | "freed_bytes": freed, |
| 473 | "kept": len(prunable) - len(removed), |
| 474 | "kept_bytes": kept_bytes, |
| 475 | "protected": len(protected), |
| 476 | "errors": errors, |
| 477 | "dry_run": bool(dry_run), |
| 478 | } |