Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_backup_index.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_backup_index.py · 18.3 KB · 478 lines
1"""Read-only inventory and on-demand creation for runtime backups.
2
3Backups are the tar/gzip archives written by ``object_backup`` under the
4configured backups directory. This module lists them, creates a new
5full-runtime backup on demand, and resolves a backup id to a safe path
6for download. It is the data layer behind the admin backup endpoints.
7
8A backup contains the whole runtime data directory — records, identity,
9credentials, service keys — so the HTTP surface that uses this module is
10strictly admin-gated. Nothing here is public.
11"""
12
13from __future__ import annotations
14
15import csv
16import hashlib
17import io
18import json
19import re
20from datetime import datetime, timedelta, timezone
21from pathlib import Path
22
23import object_backup
24import object_collections
25import object_records
26
27MANUAL_LABEL = "manual"
28_ID_RE = re.compile(r"^[0-9A-Za-z][0-9A-Za-z._-]{0,255}\.tar\.gz$")
29
30
31def backups_dir(data_dir: Path | str | None = None) -> Path:
32 """Return the configured backups directory (honors DBBASIC_BACKUPS_DIR)."""
33 return object_backup._backups_dir(None, data_dir=data_dir)
34
35
36def validate_backup_id(backup_id: str) -> bool:
37 """Return True when a backup id is a safe archive filename (no traversal)."""
38 if not isinstance(backup_id, str) or "/" in backup_id or "\\" in backup_id:
39 return False
40 return bool(_ID_RE.fullmatch(backup_id))
41
42
43def backup_path(backup_id: str, *, data_dir: Path | str | None = None) -> Path:
44 """Resolve a backup id to its path, refusing anything outside the dir."""
45 if not validate_backup_id(backup_id):
46 raise ValueError(f"invalid backup id: {backup_id!r}")
47 root = backups_dir(data_dir).resolve(strict=False)
48 path = (backups_dir(data_dir) / backup_id).resolve(strict=False)
49 try:
50 path.relative_to(root)
51 except ValueError as exc:
52 raise ValueError("backup path escapes the backups directory") from exc
53 return path
54
55
56def list_backups(*, data_dir: Path | str | None = None) -> list[dict[str, object]]:
57 """Return backup metadata, newest first — never the archive contents."""
58 directory = backups_dir(data_dir)
59 if not directory.is_dir():
60 return []
61 entries = [
62 _entry(path)
63 for path in directory.glob("*.tar.gz")
64 if path.is_file() and validate_backup_id(path.name)
65 ]
66 entries.sort(key=lambda entry: entry["created_at"], reverse=True)
67 return entries
68
69
70def create_backup(*, data_dir: Path | str | None = None) -> dict[str, object]:
71 """Create a full-runtime backup now and return its metadata."""
72 summary = object_backup.create_runtime_restore_point(MANUAL_LABEL, data_dir=data_dir)
73 return _entry(Path(summary.path))
74
75
76def _entry(path: Path) -> dict[str, object]:
77 stat = path.stat()
78 name = path.name
79 stem = name[: -len(".tar.gz")] if name.endswith(".tar.gz") else name
80 parts = stem.split("-", 1)
81 label = parts[1] if len(parts) == 2 else stem
82 if label.startswith("package-"):
83 kind, scope = "package", label[len("package-"):]
84 elif label == MANUAL_LABEL:
85 kind, scope = "manual", "runtime"
86 else:
87 kind, scope = "restore-point", label
88 created_at = (
89 datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
90 .isoformat()
91 .replace("+00:00", "Z")
92 )
93 return {
94 "id": name,
95 "created_at": created_at,
96 "size": stat.st_size,
97 "kind": kind,
98 "scope": scope,
99 }
100
101
102# Append-mode physical format (docs/storage-modes.md "Append"; the
103# authoritative implementation is object_records.py's OP_FIELD/OP_UPSERT/
104# OP_DELETE and _fold_append_rows). Kept as plain local constants rather
105# than importing object_records's, so this module's read of a records.tsv
106# stays independent of that module's private helpers.
107_OP_FIELD = "_op"
108_OP_DELETE = "del"
109
110
111def _drop_torn_tail(text: str) -> str:
112 """Drop an unterminated final physical line (a write caught mid-append).
113
114 Append-mode writers only consider a row committed once it is followed
115 by a row-terminating "\\n" (see docs/append-only-storage-design.md,
116 Crash Safety), so anything after the last one is an in-flight write --
117 not data -- on either the backup or the live side.
118
119 QUOTE-AWARE, via the shared `object_records.committed_prefix_len`: a
120 "\\n" INSIDE a quoted multi-line field is content, never a row
121 boundary.
122
123 THIS WAS THE BUG, and it is worth naming because of how it survived.
124 The body used to be `endswith("\\n") or rfind("\\n")` under a docstring
125 claiming it "mirrors object_records._drop_torn_tail" -- which had been
126 upgraded to be quote-aware while this copy was not, silently, for
127 months. Worse than the read-side original: a file ending with a "\\n"
128 that sits INSIDE an unclosed quote was returned untouched, so a
129 restored backup could resurrect a garbled row that a live read of the
130 same bytes handles correctly. It carried no test at all, which is
131 precisely why nothing caught the drift.
132
133 It now CALLS the shared implementation instead of mirroring it. That is
134 a deliberate exception to this module's usual independence from
135 object_records' internals: keeping a private copy is exactly what
136 failed, `committed_prefix_len` is exported for this reason, and two
137 implementations of one invariant drift while one cannot.
138 """
139 if text == "":
140 return text
141 return text[: object_records.committed_prefix_len(text)]
142
143
144def _fold_append_rows(rows: list[dict[str, str]]) -> dict[str, dict[str, str]]:
145 """Fold physical append-mode rows into live records, last-wins per id.
146
147 Minimal local replica of object_records._fold_append_rows's id
148 semantics, for backup preview/diff purposes only (docs/storage-modes.md
149 "Append"): a delete (`_op == "del"`) removes the id; any other row is
150 an upsert that (re)sets it — so an id updated multiple times, or
151 deleted and later re-created, ends up holding just its final live
152 values. `_op` itself is stripped from every returned row. Unlike
153 object_records's fold, this doesn't need to preserve list order (the
154 callers here only ever look records up by id for a diff), so it
155 returns a plain dict rather than an OrderedDict-backed list.
156 """
157 folded: dict[str, dict[str, str]] = {}
158 for row in rows:
159 record_id = row.get("id")
160 if not record_id:
161 continue
162 if row.get(_OP_FIELD) == _OP_DELETE:
163 folded.pop(record_id, None)
164 continue
165 folded[record_id] = {key: value for key, value in row.items() if key != _OP_FIELD}
166 return folded
167
168
169def _parse_append_tsv_by_id(text: str, header: list[str]) -> dict[str, dict[str, str]]:
170 """Parse+fold an append-mode records.tsv body into a dict keyed by id.
171
172 `header` is the already-inspected physical header (first column
173 `_op`). A torn final line is dropped first (_drop_torn_tail); a row
174 that still fails to tokenize stops consumption at that point rather
175 than raising, matching object_records._parse_append_body's tolerance
176 of an in-flight write.
177 """
178 field_count = len(header)
179 reader = csv.reader(io.StringIO(_drop_torn_tail(text)), delimiter="\t")
180 next(reader, None) # header, already parsed by the caller
181
182 physical_rows: list[dict[str, str]] = []
183 while True:
184 try:
185 row = next(reader)
186 except StopIteration:
187 break
188 except csv.Error:
189 break
190 if not row:
191 continue
192 physical_rows.append(
193 {header[i]: (row[i] if i < len(row) else "") for i in range(field_count)}
194 )
195
196 return _fold_append_rows(physical_rows)
197
198
199def _parse_tsv_by_id(raw: bytes | None) -> dict[str, dict[str, str]]:
200 """Parse a records.tsv payload into a dict keyed by the "id" column.
201
202 Dispatches on the physical header, exactly like object_records does
203 (docs/append-only-storage-design.md): a file whose first column is
204 `_op` is an append-mode log and is folded last-wins-by-id first (see
205 _fold_append_rows), so callers only ever see LIVE logical records —
206 never raw log rows, superseded values, or tombstoned ids. Classic
207 files (no `_op` header) go through the original DictReader-based
208 parse, unchanged.
209 """
210 if not raw:
211 return {}
212 text = raw.decode("utf-8")
213
214 header_row = next(csv.reader(io.StringIO(text), delimiter="\t"), None)
215 if header_row and header_row[0] == _OP_FIELD:
216 return _parse_append_tsv_by_id(text, header_row)
217
218 reader = csv.DictReader(io.StringIO(text), delimiter="\t")
219 rows: dict[str, dict[str, str]] = {}
220 for row in reader:
221 record_id = row.get("id")
222 if record_id is None:
223 continue
224 rows[record_id] = dict(row)
225 return rows
226
227
228def _records_tsv_member(collection: str) -> str:
229 return f"data/collections/{collection}/records.tsv"
230
231
232def _live_records_tsv(collection: str, *, data_dir: Path | str | None) -> bytes | None:
233 path = object_backup._data_dir(data_dir) / "collections" / collection / "records.tsv"
234 if not path.is_file():
235 return None
236 return path.read_bytes()
237
238
239def _diff_records(
240 backup_rows: dict[str, dict[str, str]],
241 live_rows: dict[str, dict[str, str]],
242) -> dict[str, object]:
243 """Diff two id->row maps with the semantics "restoring makes live == backup"."""
244 added = sorted(set(backup_rows) - set(live_rows))
245 removed = sorted(set(live_rows) - set(backup_rows))
246 changed = []
247 unchanged = 0
248 for record_id in sorted(set(backup_rows) & set(live_rows)):
249 backup_row = backup_rows[record_id]
250 live_row = live_rows[record_id]
251 if backup_row == live_row:
252 unchanged += 1
253 continue
254 fields = sorted(
255 field
256 for field in set(backup_row) | set(live_row)
257 if backup_row.get(field) != live_row.get(field)
258 )
259 changed.append({"id": record_id, "fields": fields})
260
261 diff_hash = hashlib.sha256(
262 json.dumps({"added": added, "removed": removed, "changed": changed}, sort_keys=True).encode(
263 "utf-8"
264 )
265 ).hexdigest()
266
267 return {
268 "diff_hash": diff_hash,
269 "added": added,
270 "removed": removed,
271 "changed": changed,
272 "unchanged": unchanged,
273 }
274
275
276def preview_collection(
277 backup_id: str,
278 collection: str,
279 *,
280 data_dir: Path | str | None = None,
281) -> dict[str, object]:
282 """Diff a collection's records between a backup and the live data dir.
283
284 The semantics are "if you restore, live becomes the backup's version":
285 added ids would reappear, removed ids would be dropped, changed ids
286 would have the listed fields overwritten. Either side (or both) may be
287 an append-mode file, a classic file, or missing; _parse_tsv_by_id folds
288 append-mode logs to live records first, so this always diffs live
289 logical content regardless of physical storage mode.
290 """
291 if not object_collections.validate_collection_name(collection):
292 raise ValueError(f"Invalid collection name: {collection}")
293
294 archive_path = backup_path(backup_id, data_dir=data_dir)
295 member_name = _records_tsv_member(collection)
296 archived_raw = object_backup.read_backup_member(archive_path, member_name)
297 live_raw = _live_records_tsv(collection, data_dir=data_dir)
298
299 backup_rows = _parse_tsv_by_id(archived_raw)
300 live_rows = _parse_tsv_by_id(live_raw)
301 diff = _diff_records(backup_rows, live_rows)
302
303 return {
304 "target": {"kind": "collection", "name": collection},
305 "backup_id": backup_id,
306 "present_in_backup": archived_raw is not None,
307 "diff_hash": diff["diff_hash"],
308 "added": diff["added"],
309 "removed": diff["removed"],
310 "changed": diff["changed"],
311 "unchanged": diff["unchanged"],
312 }
313
314
315def preview_record(
316 backup_id: str,
317 collection: str,
318 record_id: str,
319 *,
320 data_dir: Path | str | None = None,
321) -> dict[str, object]:
322 """Return one record's backup vs. live presence, without diffing all rows.
323
324 `record` is the FOLDED record when the source is append-mode: a
325 tombstoned id reports record=None/present_in_*=False exactly like an
326 id that was never written.
327 """
328 if not object_collections.validate_collection_name(collection):
329 raise ValueError(f"Invalid collection name: {collection}")
330
331 archive_path = backup_path(backup_id, data_dir=data_dir)
332 member_name = _records_tsv_member(collection)
333 archived_raw = object_backup.read_backup_member(archive_path, member_name)
334 live_raw = _live_records_tsv(collection, data_dir=data_dir)
335
336 backup_rows = _parse_tsv_by_id(archived_raw)
337 live_rows = _parse_tsv_by_id(live_raw)
338
339 return {
340 "collection": collection,
341 "id": record_id,
342 "record": backup_rows.get(record_id),
343 "present_in_backup": record_id in backup_rows,
344 "present_in_live": record_id in live_rows,
345 }
346
347
348# === retention ==============================================================
349#
350# A restore point is created automatically before EVERY package install
351# (object_server's create_restore_point, passed to install_package as
352# before_write). Nothing ever removed them. On the box that motivated this
353# there were 157 of them at ~91MB each -- 14.3GB against 4.0GB free, on a
354# machine that also serves production, growing by one archive per install.
355# 93 of them had been written in the preceding two days.
356#
357# That is the fourth instance of one pattern here: page_views, restore
358# points, change logs, and now restore points again. A log nobody told how
359# big it may get fails on the worst day rather than an ordinary one.
360#
361# WHAT MAKES A DEFAULT SAFE HERE, and why this differs from the change-log
362# pass (which is off unless an operator asks): those archives are not the
363# operator's data. Nobody asked for 157 of them; they are scaffolding the
364# server produced on its own behalf, and the value of the hundredth-oldest
365# one from three weeks ago is indistinguishable from zero. A change log,
366# by contrast, IS the audit trail, which is why deciding how much of it to
367# keep is not a decision a daemon makes for somebody.
368#
369# So the split is by KIND, which `_entry` already computes:
370#
371# package automatic, one per install -> prunable
372# manual a person pressed a button -> NEVER touched
373# restore-point a named point somebody chose -> NEVER touched
374#
375# Anything a human asked for survives forever regardless of policy. Only
376# the machine's own scaffolding is swept.
377
378PRUNABLE_KINDS = ("package",)
379
380# Below this many prunable archives, do nothing at all. A handful of
381# restore points is the feature working; sweeping down to the bone would
382# remove the safety net this exists to provide.
383MIN_KEEP = 5
384
385# The ceiling that actually matters. A first cut of this used only
386# keep_last plus an age window, and a dry run on the real box showed it
387# freeing 78MB out of 14.3GB: 93 archives had been written in two days, so
388# the age window protected nearly all of them -- and because the data
389# directory grows, the RECENT ones are the large ones (~91MB each against
390# ~1.7MB for the oldest). A policy expressed in COUNT and AGE cannot bound
391# a quantity measured in BYTES.
392#
393# So the budget is the primary control and the window is a courtesy inside
394# it: newest-first, keep while there is room. 2GB holds roughly the last
395# twenty snapshots of a mid-size box and is a rounding error next to the
396# disk it protects.
397DEFAULT_MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024
398
399
400def prune_backups(
401 *,
402 data_dir: Path | str | None = None,
403 keep_last: int = 20,
404 keep_newer_than_days: int = 0,
405 max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES,
406 dry_run: bool = False,
407) -> dict[str, object]:
408 """Remove old AUTOMATIC restore points. Never touches manual backups.
409
410 `keep_last` -- how many of the newest automatic archives always
411 survive, whatever they weigh. The safety floor.
412 `max_total_bytes` -- the ceiling on everything kept. THE control that
413 actually bounds a disk; see DEFAULT_MAX_TOTAL_BYTES for why count and
414 age alone provably do not. 0 disables it.
415 `keep_newer_than_days` -- inside the budget, prefer keeping anything
416 younger than this, so a burst of installs in one afternoon does not
417 evict the morning's safety net. It can never push the total past the
418 ceiling. 0 disables the window.
419
420 Returns what it did (or would do, with `dry_run`), never raising for
421 an individual file it cannot delete: a backup directory that is
422 partially unwritable should still be swept as far as it can be, and
423 the caller gets the errors rather than an exception mid-loop.
424 """
425 keep_last = max(MIN_KEEP, int(keep_last))
426 entries = list_backups(data_dir=data_dir) # newest first
427
428 prunable = [e for e in entries if e["kind"] in PRUNABLE_KINDS]
429 protected = [e for e in entries if e["kind"] not in PRUNABLE_KINDS]
430
431 cutoff = ""
432 if keep_newer_than_days and int(keep_newer_than_days) > 0:
433 cutoff = (
434 datetime.now(tz=timezone.utc)
435 - timedelta(days=int(keep_newer_than_days))
436 ).isoformat().replace("+00:00", "Z")
437
438 # Newest-first: the floor is unconditional, then the window is honoured
439 # only while the running total stays under the ceiling.
440 kept_bytes = sum(int(e["size"]) for e in prunable[:keep_last])
441 budget = int(max_total_bytes)
442 doomed = []
443 for entry in prunable[keep_last:]:
444 size = int(entry["size"])
445 in_window = bool(cutoff) and str(entry["created_at"]) >= cutoff
446 room = (not budget) or (kept_bytes + size <= budget)
447 if in_window and room:
448 kept_bytes += size
449 continue
450 doomed.append(entry)
451
452 removed, freed, errors = [], 0, []
453 if not dry_run:
454 directory = backups_dir(data_dir)
455 for entry in doomed:
456 path = directory / str(entry["id"])
457 try:
458 size = path.stat().st_size
459 path.unlink()
460 except OSError as exc:
461 errors.append(f"{entry['id']}: {exc}")
462 continue
463 removed.append(str(entry["id"]))
464 freed += size
465 else:
466 removed = [str(entry["id"]) for entry in doomed]
467 freed = sum(int(entry["size"]) for entry in doomed)
468
469 return {
470 "removed": len(removed),
471 "removed_ids": removed,
472 "freed_bytes": freed,
473 "kept": len(prunable) - len(removed),
474 "kept_bytes": kept_bytes,
475 "protected": len(protected),
476 "errors": errors,
477 "dry_run": bool(dry_run),
478 }