askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_restore_point_retention.py · 10.8 KB · 267 lines
| 1 | """Restore points: swept, but only the ones nobody asked for. |
| 2 | |
| 3 | Found by looking at a production droplet: **157 restore points at ~91MB |
| 4 | each — 14.3GB against 4.0GB free**, on the box that also serves |
| 5 | dbbasic.com, with 93 of them written in the preceding two days. One is |
| 6 | created automatically before every package install and nothing ever |
| 7 | removed them. Left alone the disk fills in about a day, and a full disk |
| 8 | there takes production down with it. |
| 9 | |
| 10 | That is the fourth appearance of one pattern in this project — page_views, |
| 11 | restore points, change logs, restore points again. A log nobody told how |
| 12 | big it may get fails on the worst day rather than an ordinary one. |
| 13 | |
| 14 | WHAT MAKES A DEFAULT SAFE HERE, and why this pass differs from the |
| 15 | change-log one (which stays off unless an operator asks): those archives |
| 16 | are not the operator's data. Nobody asked for 157 of them; they are |
| 17 | scaffolding the server writes on its own behalf. A change log, by |
| 18 | contrast, IS the audit trail — so how much of it to keep is not a |
| 19 | decision a daemon makes for somebody. |
| 20 | |
| 21 | So the split is by kind, and everything a human asked for is untouchable. |
| 22 | """ |
| 23 | |
| 24 | import time |
| 25 | from datetime import datetime, timedelta, timezone |
| 26 | from pathlib import Path |
| 27 | |
| 28 | import object_backup_index |
| 29 | import object_daemon |
| 30 | |
| 31 | |
| 32 | def make_archive(backups, label, *, age_days=0, size=1024): |
| 33 | """Write a file shaped like a real backup archive. |
| 34 | |
| 35 | Named the way object_backup._backup_filename_timestamp names them, so |
| 36 | `_entry` classifies the kind from the label exactly as it does in |
| 37 | production — the classification IS the safety property here. |
| 38 | """ |
| 39 | stamp = (datetime.now(tz=timezone.utc) - timedelta(days=age_days)) |
| 40 | compact = stamp.isoformat().replace("-", "").replace(":", "").replace(".", "") |
| 41 | compact = compact.replace("+0000", "") + "Z" |
| 42 | path = backups / f"{compact}-{label}.tar.gz" |
| 43 | path.write_bytes(b"x" * size) |
| 44 | mtime = stamp.timestamp() |
| 45 | import os |
| 46 | os.utime(path, (mtime, mtime)) |
| 47 | return path |
| 48 | |
| 49 | |
| 50 | def backups_of(tmp_path): |
| 51 | directory = object_backup_index.backups_dir(tmp_path) |
| 52 | directory.mkdir(parents=True, exist_ok=True) |
| 53 | return directory |
| 54 | |
| 55 | |
| 56 | # --- what is prunable, and what is never --------------------------------------- |
| 57 | |
| 58 | def test_a_manual_backup_is_never_swept_however_old(tmp_path): |
| 59 | """THE safety property. A person pressed a button; the daemon does not |
| 60 | get to decide that was a mistake.""" |
| 61 | backups = backups_of(tmp_path) |
| 62 | make_archive(backups, "manual", age_days=400) |
| 63 | for n in range(40): |
| 64 | make_archive(backups, f"package-app-{n}", age_days=n + 10) |
| 65 | |
| 66 | result = object_backup_index.prune_backups(data_dir=tmp_path, keep_last=5) |
| 67 | |
| 68 | assert result["protected"] == 1 |
| 69 | survivors = {entry["id"] for entry in |
| 70 | object_backup_index.list_backups(data_dir=tmp_path)} |
| 71 | assert any(name.endswith("-manual.tar.gz") for name in survivors) |
| 72 | |
| 73 | |
| 74 | def test_a_named_restore_point_is_never_swept_either(tmp_path): |
| 75 | """`package-*` is the machine's scaffolding. A labelled point somebody |
| 76 | chose ('before-migration') is a decision, and decisions survive.""" |
| 77 | backups = backups_of(tmp_path) |
| 78 | make_archive(backups, "before-the-big-migration", age_days=365) |
| 79 | for n in range(30): |
| 80 | make_archive(backups, f"package-app-{n}", age_days=n + 10) |
| 81 | |
| 82 | object_backup_index.prune_backups(data_dir=tmp_path, keep_last=5) |
| 83 | |
| 84 | survivors = {entry["id"] for entry in |
| 85 | object_backup_index.list_backups(data_dir=tmp_path)} |
| 86 | assert any("before-the-big-migration" in name for name in survivors) |
| 87 | |
| 88 | |
| 89 | def test_the_newest_automatic_points_survive(tmp_path): |
| 90 | backups = backups_of(tmp_path) |
| 91 | for n in range(30): |
| 92 | make_archive(backups, f"package-app-{n}", age_days=n + 30) |
| 93 | |
| 94 | result = object_backup_index.prune_backups( |
| 95 | data_dir=tmp_path, keep_last=10, keep_newer_than_days=0) |
| 96 | |
| 97 | assert result["kept"] == 10 |
| 98 | assert result["removed"] == 20 |
| 99 | remaining = object_backup_index.list_backups(data_dir=tmp_path) |
| 100 | assert len(remaining) == 10 |
| 101 | # Newest-first ordering means the survivors are the RECENT ones. |
| 102 | assert "package-app-0" in str(remaining[0]["id"]) |
| 103 | |
| 104 | |
| 105 | def test_a_burst_of_installs_cannot_evict_the_mornings_safety_net(tmp_path): |
| 106 | """The window exists for exactly this: twenty installs in one |
| 107 | afternoon would otherwise push out every point from before lunch, |
| 108 | which is when you actually want to roll back to.""" |
| 109 | backups = backups_of(tmp_path) |
| 110 | for n in range(25): |
| 111 | make_archive(backups, f"package-burst-{n}", age_days=0) |
| 112 | make_archive(backups, "package-this-morning", age_days=1) |
| 113 | |
| 114 | result = object_backup_index.prune_backups( |
| 115 | data_dir=tmp_path, keep_last=10, keep_newer_than_days=7) |
| 116 | |
| 117 | assert result["removed"] == 0 # everything is inside the window |
| 118 | survivors = {entry["id"] for entry in |
| 119 | object_backup_index.list_backups(data_dir=tmp_path)} |
| 120 | assert any("this-morning" in name for name in survivors) |
| 121 | |
| 122 | |
| 123 | def test_there_is_a_floor_so_the_safety_net_cannot_be_swept_to_nothing(tmp_path): |
| 124 | """keep_last=0 must not mean 'delete them all'. Sweeping to the bone |
| 125 | would remove the very thing this feature exists to provide.""" |
| 126 | backups = backups_of(tmp_path) |
| 127 | for n in range(20): |
| 128 | make_archive(backups, f"package-app-{n}", age_days=n + 30) |
| 129 | |
| 130 | result = object_backup_index.prune_backups( |
| 131 | data_dir=tmp_path, keep_last=0, keep_newer_than_days=0) |
| 132 | |
| 133 | assert result["kept"] == object_backup_index.MIN_KEEP |
| 134 | assert result["kept"] >= 5 |
| 135 | |
| 136 | |
| 137 | def test_a_dry_run_reports_without_removing(tmp_path): |
| 138 | backups = backups_of(tmp_path) |
| 139 | for n in range(20): |
| 140 | make_archive(backups, f"package-app-{n}", age_days=n + 30) |
| 141 | |
| 142 | result = object_backup_index.prune_backups( |
| 143 | data_dir=tmp_path, keep_last=5, keep_newer_than_days=0, dry_run=True) |
| 144 | |
| 145 | assert result["dry_run"] is True |
| 146 | assert result["removed"] == 15 |
| 147 | assert result["freed_bytes"] > 0 |
| 148 | assert len(object_backup_index.list_backups(data_dir=tmp_path)) == 20 |
| 149 | |
| 150 | |
| 151 | def test_an_undeletable_archive_is_reported_rather_than_aborting_the_sweep( |
| 152 | tmp_path, monkeypatch): |
| 153 | """A partially unwritable backup directory should still be swept as |
| 154 | far as it can be. Raising mid-loop would leave the disk full AND the |
| 155 | sweep half-done, with no record of which half.""" |
| 156 | backups = backups_of(tmp_path) |
| 157 | for n in range(20): |
| 158 | make_archive(backups, f"package-app-{n}", age_days=n + 30) |
| 159 | # Unlink permission on POSIX belongs to the DIRECTORY, not the file, so |
| 160 | # one unremovable archive cannot be built portably. Fail one unlink |
| 161 | # directly instead -- the property under test is that the loop |
| 162 | # continues and reports, not how the OS refused. |
| 163 | victim = sorted(p.name for p in backups.iterdir())[0] |
| 164 | real_unlink = Path.unlink |
| 165 | |
| 166 | def flaky_unlink(self, *args, **kwargs): |
| 167 | if self.name == victim: |
| 168 | raise PermissionError("read-only file system") |
| 169 | return real_unlink(self, *args, **kwargs) |
| 170 | |
| 171 | monkeypatch.setattr(Path, "unlink", flaky_unlink) |
| 172 | result = object_backup_index.prune_backups( |
| 173 | data_dir=tmp_path, keep_last=5, keep_newer_than_days=0) |
| 174 | |
| 175 | assert result["errors"] |
| 176 | assert victim in result["errors"][0] |
| 177 | assert result["removed"] == 14 # the other fourteen still went |
| 178 | |
| 179 | |
| 180 | # --- the daemon pass ------------------------------------------------------------ |
| 181 | |
| 182 | def test_the_pass_is_on_by_default_unlike_the_change_log_one(tmp_path, monkeypatch): |
| 183 | """The change-log pass returns None with no policy configured, because |
| 184 | an audit trail is the operator's. This one runs unasked, because the |
| 185 | files are the server's own scaffolding.""" |
| 186 | backups = backups_of(tmp_path) |
| 187 | for n in range(40): |
| 188 | make_archive(backups, f"package-app-{n}", age_days=n + 30) |
| 189 | |
| 190 | result = object_daemon.process_restore_point_retention(base_dir=tmp_path) |
| 191 | assert result is not None |
| 192 | assert result["removed"] > 0 |
| 193 | |
| 194 | |
| 195 | def test_the_pass_respects_its_marker_and_does_not_run_every_tick(tmp_path): |
| 196 | backups = backups_of(tmp_path) |
| 197 | for n in range(40): |
| 198 | make_archive(backups, f"package-app-{n}", age_days=n + 30) |
| 199 | |
| 200 | first = object_daemon.process_restore_point_retention(base_dir=tmp_path) |
| 201 | second = object_daemon.process_restore_point_retention(base_dir=tmp_path) |
| 202 | assert first is not None |
| 203 | assert second is None # marker suppresses the re-run |
| 204 | |
| 205 | |
| 206 | def test_the_env_knobs_are_honoured(tmp_path, monkeypatch): |
| 207 | backups = backups_of(tmp_path) |
| 208 | for n in range(40): |
| 209 | make_archive(backups, f"package-app-{n}", age_days=n + 30) |
| 210 | monkeypatch.setenv(object_daemon.RESTORE_POINT_KEEP_LAST_ENV, "7") |
| 211 | monkeypatch.setenv(object_daemon.RESTORE_POINT_KEEP_DAYS_ENV, "0") |
| 212 | |
| 213 | result = object_daemon.process_restore_point_retention(base_dir=tmp_path) |
| 214 | assert result["kept"] == 7 |
| 215 | |
| 216 | |
| 217 | # --- the budget ----------------------------------------------------------------- |
| 218 | # |
| 219 | # The first cut of this policy had only keep_last and an age window. A dry |
| 220 | # run against the real droplet freed 78MB out of 14.3GB: 93 archives had |
| 221 | # been written in two days, so the window protected almost all of them -- |
| 222 | # and because the data directory grows, the RECENT archives are the large |
| 223 | # ones (~91MB each against ~1.7MB for the oldest). A policy expressed in |
| 224 | # COUNT and AGE cannot bound a quantity measured in BYTES. |
| 225 | |
| 226 | def test_the_budget_outranks_the_age_window(tmp_path): |
| 227 | """The exact shape that made the first policy useless: everything is |
| 228 | inside the window, and the window must not be able to blow the disk |
| 229 | budget to honour it.""" |
| 230 | backups = backups_of(tmp_path) |
| 231 | for n in range(40): # 40 x 1MB, all written today |
| 232 | make_archive(backups, f"package-burst-{n}", age_days=0, size=1024 * 1024) |
| 233 | |
| 234 | result = object_backup_index.prune_backups( |
| 235 | data_dir=tmp_path, keep_last=5, keep_newer_than_days=7, |
| 236 | max_total_bytes=10 * 1024 * 1024) # room for ~10 |
| 237 | |
| 238 | assert result["kept"] <= 11 |
| 239 | assert result["removed"] >= 29 |
| 240 | assert result["kept_bytes"] <= 11 * 1024 * 1024 |
| 241 | |
| 242 | |
| 243 | def test_the_floor_survives_even_a_budget_smaller_than_it(tmp_path): |
| 244 | """keep_last is unconditional: a budget set below what the newest few |
| 245 | weigh must not delete them. A rollback point you cannot afford is |
| 246 | still a rollback point.""" |
| 247 | backups = backups_of(tmp_path) |
| 248 | for n in range(20): |
| 249 | make_archive(backups, f"package-app-{n}", age_days=n, size=1024 * 1024) |
| 250 | |
| 251 | result = object_backup_index.prune_backups( |
| 252 | data_dir=tmp_path, keep_last=8, keep_newer_than_days=0, |
| 253 | max_total_bytes=1024) # 1KB: less than one archive |
| 254 | |
| 255 | assert result["kept"] == 8 |
| 256 | |
| 257 | |
| 258 | def test_a_zero_budget_disables_the_ceiling(tmp_path): |
| 259 | backups = backups_of(tmp_path) |
| 260 | for n in range(30): |
| 261 | make_archive(backups, f"package-app-{n}", age_days=0, size=1024 * 1024) |
| 262 | |
| 263 | result = object_backup_index.prune_backups( |
| 264 | data_dir=tmp_path, keep_last=5, keep_newer_than_days=7, |
| 265 | max_total_bytes=0) |
| 266 | |
| 267 | assert result["removed"] == 0 # window keeps everything again |