Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_restore_point_retention.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_restore_point_retention.py · 10.8 KB · 267 lines
1"""Restore points: swept, but only the ones nobody asked for.
2
3Found by looking at a production droplet: **157 restore points at ~91MB
4each — 14.3GB against 4.0GB free**, on the box that also serves
5dbbasic.com, with 93 of them written in the preceding two days. One is
6created automatically before every package install and nothing ever
7removed them. Left alone the disk fills in about a day, and a full disk
8there takes production down with it.
9
10That is the fourth appearance of one pattern in this project — page_views,
11restore points, change logs, restore points again. A log nobody told how
12big it may get fails on the worst day rather than an ordinary one.
13
14WHAT MAKES A DEFAULT SAFE HERE, and why this pass differs from the
15change-log one (which stays off unless an operator asks): those archives
16are not the operator's data. Nobody asked for 157 of them; they are
17scaffolding the server writes on its own behalf. A change log, by
18contrast, IS the audit trail — so how much of it to keep is not a
19decision a daemon makes for somebody.
20
21So the split is by kind, and everything a human asked for is untouchable.
22"""
23
24import time
25from datetime import datetime, timedelta, timezone
26from pathlib import Path
27
28import object_backup_index
29import object_daemon
30
31
32def make_archive(backups, label, *, age_days=0, size=1024):
33 """Write a file shaped like a real backup archive.
34
35 Named the way object_backup._backup_filename_timestamp names them, so
36 `_entry` classifies the kind from the label exactly as it does in
37 production — the classification IS the safety property here.
38 """
39 stamp = (datetime.now(tz=timezone.utc) - timedelta(days=age_days))
40 compact = stamp.isoformat().replace("-", "").replace(":", "").replace(".", "")
41 compact = compact.replace("+0000", "") + "Z"
42 path = backups / f"{compact}-{label}.tar.gz"
43 path.write_bytes(b"x" * size)
44 mtime = stamp.timestamp()
45 import os
46 os.utime(path, (mtime, mtime))
47 return path
48
49
50def backups_of(tmp_path):
51 directory = object_backup_index.backups_dir(tmp_path)
52 directory.mkdir(parents=True, exist_ok=True)
53 return directory
54
55
56# --- what is prunable, and what is never ---------------------------------------
57
58def test_a_manual_backup_is_never_swept_however_old(tmp_path):
59 """THE safety property. A person pressed a button; the daemon does not
60 get to decide that was a mistake."""
61 backups = backups_of(tmp_path)
62 make_archive(backups, "manual", age_days=400)
63 for n in range(40):
64 make_archive(backups, f"package-app-{n}", age_days=n + 10)
65
66 result = object_backup_index.prune_backups(data_dir=tmp_path, keep_last=5)
67
68 assert result["protected"] == 1
69 survivors = {entry["id"] for entry in
70 object_backup_index.list_backups(data_dir=tmp_path)}
71 assert any(name.endswith("-manual.tar.gz") for name in survivors)
72
73
74def test_a_named_restore_point_is_never_swept_either(tmp_path):
75 """`package-*` is the machine's scaffolding. A labelled point somebody
76 chose ('before-migration') is a decision, and decisions survive."""
77 backups = backups_of(tmp_path)
78 make_archive(backups, "before-the-big-migration", age_days=365)
79 for n in range(30):
80 make_archive(backups, f"package-app-{n}", age_days=n + 10)
81
82 object_backup_index.prune_backups(data_dir=tmp_path, keep_last=5)
83
84 survivors = {entry["id"] for entry in
85 object_backup_index.list_backups(data_dir=tmp_path)}
86 assert any("before-the-big-migration" in name for name in survivors)
87
88
89def test_the_newest_automatic_points_survive(tmp_path):
90 backups = backups_of(tmp_path)
91 for n in range(30):
92 make_archive(backups, f"package-app-{n}", age_days=n + 30)
93
94 result = object_backup_index.prune_backups(
95 data_dir=tmp_path, keep_last=10, keep_newer_than_days=0)
96
97 assert result["kept"] == 10
98 assert result["removed"] == 20
99 remaining = object_backup_index.list_backups(data_dir=tmp_path)
100 assert len(remaining) == 10
101 # Newest-first ordering means the survivors are the RECENT ones.
102 assert "package-app-0" in str(remaining[0]["id"])
103
104
105def test_a_burst_of_installs_cannot_evict_the_mornings_safety_net(tmp_path):
106 """The window exists for exactly this: twenty installs in one
107 afternoon would otherwise push out every point from before lunch,
108 which is when you actually want to roll back to."""
109 backups = backups_of(tmp_path)
110 for n in range(25):
111 make_archive(backups, f"package-burst-{n}", age_days=0)
112 make_archive(backups, "package-this-morning", age_days=1)
113
114 result = object_backup_index.prune_backups(
115 data_dir=tmp_path, keep_last=10, keep_newer_than_days=7)
116
117 assert result["removed"] == 0 # everything is inside the window
118 survivors = {entry["id"] for entry in
119 object_backup_index.list_backups(data_dir=tmp_path)}
120 assert any("this-morning" in name for name in survivors)
121
122
123def test_there_is_a_floor_so_the_safety_net_cannot_be_swept_to_nothing(tmp_path):
124 """keep_last=0 must not mean 'delete them all'. Sweeping to the bone
125 would remove the very thing this feature exists to provide."""
126 backups = backups_of(tmp_path)
127 for n in range(20):
128 make_archive(backups, f"package-app-{n}", age_days=n + 30)
129
130 result = object_backup_index.prune_backups(
131 data_dir=tmp_path, keep_last=0, keep_newer_than_days=0)
132
133 assert result["kept"] == object_backup_index.MIN_KEEP
134 assert result["kept"] >= 5
135
136
137def test_a_dry_run_reports_without_removing(tmp_path):
138 backups = backups_of(tmp_path)
139 for n in range(20):
140 make_archive(backups, f"package-app-{n}", age_days=n + 30)
141
142 result = object_backup_index.prune_backups(
143 data_dir=tmp_path, keep_last=5, keep_newer_than_days=0, dry_run=True)
144
145 assert result["dry_run"] is True
146 assert result["removed"] == 15
147 assert result["freed_bytes"] > 0
148 assert len(object_backup_index.list_backups(data_dir=tmp_path)) == 20
149
150
151def test_an_undeletable_archive_is_reported_rather_than_aborting_the_sweep(
152 tmp_path, monkeypatch):
153 """A partially unwritable backup directory should still be swept as
154 far as it can be. Raising mid-loop would leave the disk full AND the
155 sweep half-done, with no record of which half."""
156 backups = backups_of(tmp_path)
157 for n in range(20):
158 make_archive(backups, f"package-app-{n}", age_days=n + 30)
159 # Unlink permission on POSIX belongs to the DIRECTORY, not the file, so
160 # one unremovable archive cannot be built portably. Fail one unlink
161 # directly instead -- the property under test is that the loop
162 # continues and reports, not how the OS refused.
163 victim = sorted(p.name for p in backups.iterdir())[0]
164 real_unlink = Path.unlink
165
166 def flaky_unlink(self, *args, **kwargs):
167 if self.name == victim:
168 raise PermissionError("read-only file system")
169 return real_unlink(self, *args, **kwargs)
170
171 monkeypatch.setattr(Path, "unlink", flaky_unlink)
172 result = object_backup_index.prune_backups(
173 data_dir=tmp_path, keep_last=5, keep_newer_than_days=0)
174
175 assert result["errors"]
176 assert victim in result["errors"][0]
177 assert result["removed"] == 14 # the other fourteen still went
178
179
180# --- the daemon pass ------------------------------------------------------------
181
182def test_the_pass_is_on_by_default_unlike_the_change_log_one(tmp_path, monkeypatch):
183 """The change-log pass returns None with no policy configured, because
184 an audit trail is the operator's. This one runs unasked, because the
185 files are the server's own scaffolding."""
186 backups = backups_of(tmp_path)
187 for n in range(40):
188 make_archive(backups, f"package-app-{n}", age_days=n + 30)
189
190 result = object_daemon.process_restore_point_retention(base_dir=tmp_path)
191 assert result is not None
192 assert result["removed"] > 0
193
194
195def test_the_pass_respects_its_marker_and_does_not_run_every_tick(tmp_path):
196 backups = backups_of(tmp_path)
197 for n in range(40):
198 make_archive(backups, f"package-app-{n}", age_days=n + 30)
199
200 first = object_daemon.process_restore_point_retention(base_dir=tmp_path)
201 second = object_daemon.process_restore_point_retention(base_dir=tmp_path)
202 assert first is not None
203 assert second is None # marker suppresses the re-run
204
205
206def test_the_env_knobs_are_honoured(tmp_path, monkeypatch):
207 backups = backups_of(tmp_path)
208 for n in range(40):
209 make_archive(backups, f"package-app-{n}", age_days=n + 30)
210 monkeypatch.setenv(object_daemon.RESTORE_POINT_KEEP_LAST_ENV, "7")
211 monkeypatch.setenv(object_daemon.RESTORE_POINT_KEEP_DAYS_ENV, "0")
212
213 result = object_daemon.process_restore_point_retention(base_dir=tmp_path)
214 assert result["kept"] == 7
215
216
217# --- the budget -----------------------------------------------------------------
218#
219# The first cut of this policy had only keep_last and an age window. A dry
220# run against the real droplet freed 78MB out of 14.3GB: 93 archives had
221# been written in two days, so the window protected almost all of them --
222# and because the data directory grows, the RECENT archives are the large
223# ones (~91MB each against ~1.7MB for the oldest). A policy expressed in
224# COUNT and AGE cannot bound a quantity measured in BYTES.
225
226def test_the_budget_outranks_the_age_window(tmp_path):
227 """The exact shape that made the first policy useless: everything is
228 inside the window, and the window must not be able to blow the disk
229 budget to honour it."""
230 backups = backups_of(tmp_path)
231 for n in range(40): # 40 x 1MB, all written today
232 make_archive(backups, f"package-burst-{n}", age_days=0, size=1024 * 1024)
233
234 result = object_backup_index.prune_backups(
235 data_dir=tmp_path, keep_last=5, keep_newer_than_days=7,
236 max_total_bytes=10 * 1024 * 1024) # room for ~10
237
238 assert result["kept"] <= 11
239 assert result["removed"] >= 29
240 assert result["kept_bytes"] <= 11 * 1024 * 1024
241
242
243def test_the_floor_survives_even_a_budget_smaller_than_it(tmp_path):
244 """keep_last is unconditional: a budget set below what the newest few
245 weigh must not delete them. A rollback point you cannot afford is
246 still a rollback point."""
247 backups = backups_of(tmp_path)
248 for n in range(20):
249 make_archive(backups, f"package-app-{n}", age_days=n, size=1024 * 1024)
250
251 result = object_backup_index.prune_backups(
252 data_dir=tmp_path, keep_last=8, keep_newer_than_days=0,
253 max_total_bytes=1024) # 1KB: less than one archive
254
255 assert result["kept"] == 8
256
257
258def test_a_zero_budget_disables_the_ceiling(tmp_path):
259 backups = backups_of(tmp_path)
260 for n in range(30):
261 make_archive(backups, f"package-app-{n}", age_days=0, size=1024 * 1024)
262
263 result = object_backup_index.prune_backups(
264 data_dir=tmp_path, keep_last=5, keep_newer_than_days=7,
265 max_total_bytes=0)
266
267 assert result["removed"] == 0 # window keeps everything again