askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_daemon_queue.py · 6.5 KB · 155 lines
| 1 | """The deferred-work queue that already existed (and the package that |
| 2 | finally turns it on). |
| 3 | |
| 4 | Worth pinning explicitly, because the capability was easy to overlook: the |
| 5 | daemon has had delay, priority, expiry and retry-with-backoff all along, |
| 6 | and the only reason none of it ran was a missing trigger object -- the same |
| 7 | one-file gap that left the scheduler passes disabled. A primitive nobody |
| 8 | can see is indistinguishable from a primitive nobody built. |
| 9 | """ |
| 10 | |
| 11 | import json |
| 12 | import pathlib |
| 13 | import time |
| 14 | |
| 15 | import object_daemon |
| 16 | import object_daemon_control |
| 17 | import object_packages |
| 18 | import object_state |
| 19 | import python_object_runtime |
| 20 | |
| 21 | REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] |
| 22 | PACKAGES_ROOT = REPO_ROOT / "packages" |
| 23 | |
| 24 | |
| 25 | def setup_env(tmp_path, monkeypatch, *, with_queue_object=True): |
| 26 | objects_dir = tmp_path / "objects" |
| 27 | (objects_dir / "action").mkdir(parents=True) |
| 28 | (objects_dir / "action" / "probe.py").write_text( |
| 29 | "import os\n" |
| 30 | "def POST(request):\n" |
| 31 | " path = os.environ['DBBASIC_DATA_DIR'] + '/probe-' + str(request.get('mark')) + '.txt'\n" |
| 32 | " open(path, 'w').write(str(request.get('mark')))\n" |
| 33 | " return {'ok': True, 'mark': request.get('mark')}\n") |
| 34 | (objects_dir / "action" / "boom.py").write_text( |
| 35 | "def POST(request):\n raise RuntimeError('job failed')\n") |
| 36 | if with_queue_object: |
| 37 | (objects_dir / "triggers").mkdir(parents=True) |
| 38 | (objects_dir / "triggers" / "queue.py").write_text( |
| 39 | (PACKAGES_ROOT / "system-triggers" / "objects" / "triggers" / "queue.py").read_text()) |
| 40 | data_dir = tmp_path / "data" |
| 41 | data_dir.mkdir() |
| 42 | monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(objects_dir)) |
| 43 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir)) |
| 44 | return objects_dir, data_dir |
| 45 | |
| 46 | |
| 47 | def pump(): |
| 48 | object_daemon.process_queue(python_object_runtime.PythonObjectRuntime()) |
| 49 | |
| 50 | |
| 51 | def messages(data_dir): |
| 52 | out = {} |
| 53 | for key, value in object_state.get_object_state("queue", data_dir).items(): |
| 54 | if key.startswith("msg_"): |
| 55 | row = json.loads(value) |
| 56 | out[row["id"]] = row |
| 57 | return out |
| 58 | |
| 59 | |
| 60 | def test_a_message_runs_on_the_next_pass(tmp_path, monkeypatch): |
| 61 | _, data_dir = setup_env(tmp_path, monkeypatch) |
| 62 | created = object_daemon_control.enqueue_message( |
| 63 | {"object_id": "action_probe", "method": "POST", "payload": {"mark": "now"}}, |
| 64 | base_dir=data_dir) |
| 65 | pump() |
| 66 | assert (data_dir / "probe-now.txt").exists() |
| 67 | assert messages(data_dir)[created["id"]]["status"] == "completed" |
| 68 | |
| 69 | |
| 70 | def test_visible_after_defers_work_until_its_time(tmp_path, monkeypatch): |
| 71 | """This is the 'delayed job' the system already had: enqueue now, run |
| 72 | later, no cron entry and no separate worker service.""" |
| 73 | _, data_dir = setup_env(tmp_path, monkeypatch) |
| 74 | later = int(time.time()) + 3600 |
| 75 | created = object_daemon_control.enqueue_message( |
| 76 | {"object_id": "action_probe", "payload": {"mark": "later"}, |
| 77 | "visible_after": later}, |
| 78 | base_dir=data_dir) |
| 79 | pump() |
| 80 | assert not (data_dir / "probe-later.txt").exists() |
| 81 | assert messages(data_dir)[created["id"]]["status"] == "pending" |
| 82 | |
| 83 | # Bring its time forward: the same message now runs, unchanged. |
| 84 | manager = object_state.ObjectStateManager("queue", base_dir=data_dir) |
| 85 | key = next(k for k in manager.get_all() if k.startswith("msg_")) |
| 86 | row = json.loads(manager.get(key)) |
| 87 | row["visible_after"] = int(time.time()) - 1 |
| 88 | manager.set(key, json.dumps(row)) |
| 89 | pump() |
| 90 | assert (data_dir / "probe-later.txt").exists() |
| 91 | assert messages(data_dir)[created["id"]]["status"] == "completed" |
| 92 | |
| 93 | |
| 94 | def test_priority_orders_the_batch(tmp_path, monkeypatch): |
| 95 | _, data_dir = setup_env(tmp_path, monkeypatch) |
| 96 | for mark, priority in (("low", 0), ("high", 9)): |
| 97 | object_daemon_control.enqueue_message( |
| 98 | {"object_id": "action_probe", "payload": {"mark": mark}, |
| 99 | "priority_level": priority}, base_dir=data_dir) |
| 100 | pump() |
| 101 | # Both ran; the point is that ordering is a first-class field rather |
| 102 | # than insertion order, which matters once a batch exceeds max_messages. |
| 103 | assert (data_dir / "probe-high.txt").exists() |
| 104 | assert (data_dir / "probe-low.txt").exists() |
| 105 | |
| 106 | |
| 107 | def test_failure_retries_with_backoff_then_parks_the_message(tmp_path, monkeypatch): |
| 108 | """A dead job stays inspectable instead of vanishing or looping.""" |
| 109 | _, data_dir = setup_env(tmp_path, monkeypatch) |
| 110 | created = object_daemon_control.enqueue_message( |
| 111 | {"object_id": "action_boom", "max_attempts": 2}, base_dir=data_dir) |
| 112 | pump() |
| 113 | row = messages(data_dir)[created["id"]] |
| 114 | assert row["status"] == "pending" and row["attempts"] == 1 |
| 115 | assert row["visible_after"] > int(time.time()) - 1 # backed off, not hammered |
| 116 | |
| 117 | row["visible_after"] = int(time.time()) - 1 |
| 118 | manager = object_state.ObjectStateManager("queue", base_dir=data_dir) |
| 119 | key = next(k for k in manager.get_all() if k.startswith("msg_")) |
| 120 | manager.set(key, json.dumps(row)) |
| 121 | pump() |
| 122 | dead = messages(data_dir)[created["id"]] |
| 123 | assert dead["status"] == "failed" and dead["attempts"] == 2 |
| 124 | |
| 125 | |
| 126 | def test_an_expired_message_is_dropped_not_run(tmp_path, monkeypatch): |
| 127 | _, data_dir = setup_env(tmp_path, monkeypatch) |
| 128 | created = object_daemon_control.enqueue_message( |
| 129 | {"object_id": "action_probe", "payload": {"mark": "stale"}, |
| 130 | "expires_at": int(time.time()) - 10}, base_dir=data_dir) |
| 131 | pump() |
| 132 | assert not (data_dir / "probe-stale.txt").exists() |
| 133 | assert messages(data_dir)[created["id"]]["status"] == "expired" |
| 134 | |
| 135 | |
| 136 | def test_without_the_trigger_object_nothing_runs_at_all(tmp_path, monkeypatch): |
| 137 | """The whole reason this package exists: the queue was fully built and |
| 138 | completely inert, because one file was missing.""" |
| 139 | _, data_dir = setup_env(tmp_path, monkeypatch, with_queue_object=False) |
| 140 | object_daemon_control.enqueue_message( |
| 141 | {"object_id": "action_probe", "payload": {"mark": "orphan"}}, base_dir=data_dir) |
| 142 | pump() |
| 143 | assert not (data_dir / "probe-orphan.txt").exists() |
| 144 | assert messages(data_dir) # the message is still there, just never read |
| 145 | |
| 146 | |
| 147 | def test_the_package_ships_both_trigger_objects(tmp_path): |
| 148 | package = object_packages.get_package("system-triggers", root=PACKAGES_ROOT) |
| 149 | assert {obj["id"] for obj in package["objects"]} == { |
| 150 | "triggers_scheduler", "triggers_queue"} |
| 151 | # The ids must map to triggers/<name>.py, which is where |
| 152 | # object_namespace.find_trigger_file looks -- get the path wrong and the |
| 153 | # daemon reports "no scheduler object" while the install says success. |
| 154 | for obj in package["objects"]: |
| 155 | assert obj["path"].startswith("objects/triggers/") |