Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
run_sweeper.py · 8.0 KB · 185 lines
1"""system_run_sweeper -- give the money back when a runner dies, and
2never, ever run the job again.
3
4A claimed run whose heartbeat has stopped is a run in an UNKNOWN state.
5The pass that owned it may have died before calling the provider, or
6after calling and before recording -- and in the second case the provider
7may have completed the work and billed us. There is no way to know from
8here, because the non-idempotent step belongs to somebody else's system.
9
10So the sweep does the only two things that are safe in both worlds:
11
12* mark the run `abandoned` -- a terminal state with no exits, declared so
13 in the schema's transition map, so nothing can quietly re-queue it
14* release the hold, in full, through the same idempotent settlement the
15 runner uses -- the user pays nothing for a job nobody can prove
16 happened, and the marker check means a runner that actually did settle
17 before dying is not paid back twice
18
19What it pointedly does NOT do is retry. Re-running a job that may already
20have been billed is paying twice for one thing. A retry is a NEW run,
21with a new hold, started by a person looking at the abandoned one --
22`provider_job_id`, recorded before the call where the provider supports
23it, is the thread that person pulls to ask "did they charge us".
24
25Staleness comes from `runner.stale_seconds` (default 900). Generous on
26purpose: this sweep is for processes that DIED, and a slow provider call
27is not a dead process. Sweeping a run that is merely slow abandons work
28the provider will finish and bill for -- so err long, and let genuinely
29long-running handlers push their own heartbeats when they arrive.
30"""
31
32import os
33from datetime import datetime, timezone
34
35import object_records
36import object_template_runs
37
38ACTOR = "system_run_sweeper"
39RUNS = "template_runs"
40
41STALE_SETTING = "runner.stale_seconds"
42MAX_RUN_SETTING = "runner.max_run_seconds"
43# Deliberately generous: this is not "how long should a job take" but
44# "how long before a machine gives up deciding and asks a person". Sora
45# budgets thirty minutes; an hour leaves room for a provider queue.
46DEFAULT_MAX_RUN_SECONDS = 3600
47DEFAULT_STALE_SECONDS = 900
48
49
50def _base_dir():
51 return os.environ.get("DBBASIC_DATA_DIR", "data")
52
53
54def _text(value):
55 return str(value if value is not None else "").strip()
56
57
58def _int(value, default=0):
59 try:
60 return int(_text(value) or default)
61 except (TypeError, ValueError):
62 return default
63
64
65def _now():
66 return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
67
68
69def _setting(base, key, default):
70 try:
71 for row in object_records.read_collection_records("app_settings",
72 base_dir=base):
73 if _text(row.get("key")) == key:
74 return _text(row.get("value")) or default
75 except Exception:
76 pass
77 return default
78
79
80def EVENT(request):
81 request = request or {}
82 base = _base_dir()
83 stale_seconds = max(60, _int(_setting(base, STALE_SETTING,
84 DEFAULT_STALE_SECONDS),
85 DEFAULT_STALE_SECONDS))
86 now = _text(request.get("now")) or _now()
87
88 try:
89 runs = object_records.read_collection_records(RUNS, base_dir=base)
90 except Exception:
91 return {"ok": True, "swept": 0,
92 "note": "No template_runs collection on this box."}
93
94 try:
95 entries = object_records.read_collection_records("wallet_entries",
96 base_dir=base)
97 except Exception:
98 entries = []
99
100 max_run_seconds = max(stale_seconds, _int(_setting(base, MAX_RUN_SETTING,
101 DEFAULT_MAX_RUN_SECONDS),
102 DEFAULT_MAX_RUN_SECONDS))
103
104 swept, review = [], []
105 # STEP 1 -- decide. Writing off a run is a COMPARE-AND-SET, because the
106 # sweeper races the runner it is sweeping: a runner that heartbeats or
107 # finishes between our staleness read and our write has changed the
108 # row, so our rev no longer matches and we correctly lose. Without the
109 # precondition this pass could write off a run that was completing and
110 # hand back money for work that succeeded.
111 #
112 # WHICH decision is object_template_runs.poll_disposition's, not this
113 # loop's, because the same rule has to bind the runner too. The rule
114 # that matters: a run carrying a provider_job_id is NEVER abandoned.
115 # Abandoning is what releases the hold, and a job the provider is
116 # already holding may already have cost everything -- handing the
117 # money back on a timer would be paying for it ourselves. Such a run
118 # is polled instead (the provider can simply be asked), and if polling
119 # never resolves it, it stops for a person.
120 for run in runs:
121 disposition = object_template_runs.poll_disposition(
122 run, now=now, stale_seconds=stale_seconds,
123 max_run_seconds=max_run_seconds)
124 if disposition not in ("abandon", "review"):
125 continue # "poll" belongs to the runner, not here
126 run_id = _text(run.get("id"))
127 if disposition == "abandon":
128 changes = {
129 "status": "abandoned",
130 "error": (f"Abandoned: no heartbeat for over {stale_seconds}s "
131 f"and nothing was submitted to a provider, so "
132 f"nothing was spent. The hold is released in full. "
133 f"NOT retried -- a retry is a new run, started by a "
134 f"person."),
135 "finished_at": now,
136 }
137 else:
138 changes = {
139 "status": object_template_runs.NEEDS_REVIEW_STATUS,
140 "error": (f"Submitted to the provider (job "
141 f"{_text(run.get('provider_job_id')) or 'unknown'}) "
142 f"and still unresolved after {max_run_seconds}s. "
143 f"The hold is deliberately NOT released: this job "
144 f"may already have been charged, so releasing would "
145 f"give away what we paid and charging would bill for "
146 f"work that may never have arrived. A person decides."),
147 }
148 try:
149 object_records.update_collection_record(
150 RUNS, run_id, changes, base_dir=base, actor=ACTOR,
151 expected_rev=object_records.compute_record_rev(run))
152 except object_records.VersionConflictError:
153 # It moved while we were looking at it -- alive after all.
154 continue
155 (swept if disposition == "abandon" else review).append(run_id)
156
157 # STEP 2 -- pay back, driven by STATE rather than by "did step 1 just
158 # run". Every terminal run whose hold has not been released gets it
159 # released, so a crash between the two steps is repaired by the next
160 # pass instead of leaving money reserved against a run that is over.
161 # Idempotent by the release marker, so this is a no-op for the runs
162 # the runner already settled itself.
163 released = []
164 for run in object_records.read_collection_records(RUNS, base_dir=base):
165 if _text(run.get("status")) not in object_template_runs.TERMINAL_STATUSES:
166 continue
167 run_id = _text(run.get("id"))
168 if object_template_runs.already_settled(run_id, entries):
169 continue
170 entries_written = object_template_runs.settlement(run, succeeded=False)
171 if not entries_written:
172 continue # a free run: nothing was held
173 for entry in entries_written:
174 object_records.create_collection_record(
175 "wallet_entries", entry, base_dir=base, actor=ACTOR)
176 entries.extend(entries_written) # so a second hit this pass skips
177 released.append(run_id)
178
179 return {"ok": True, "swept": len(swept), "runs": swept,
180 "needs_review": review,
181 "holds_released": released, "stale_seconds": stale_seconds,
182 "max_run_seconds": max_run_seconds}
183
184
185POST = EVENT