askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
run_sweeper.py · 8.0 KB · 185 lines
| 1 | """system_run_sweeper -- give the money back when a runner dies, and |
| 2 | never, ever run the job again. |
| 3 | |
| 4 | A claimed run whose heartbeat has stopped is a run in an UNKNOWN state. |
| 5 | The pass that owned it may have died before calling the provider, or |
| 6 | after calling and before recording -- and in the second case the provider |
| 7 | may have completed the work and billed us. There is no way to know from |
| 8 | here, because the non-idempotent step belongs to somebody else's system. |
| 9 | |
| 10 | So the sweep does the only two things that are safe in both worlds: |
| 11 | |
| 12 | * mark the run `abandoned` -- a terminal state with no exits, declared so |
| 13 | in the schema's transition map, so nothing can quietly re-queue it |
| 14 | * release the hold, in full, through the same idempotent settlement the |
| 15 | runner uses -- the user pays nothing for a job nobody can prove |
| 16 | happened, and the marker check means a runner that actually did settle |
| 17 | before dying is not paid back twice |
| 18 | |
| 19 | What it pointedly does NOT do is retry. Re-running a job that may already |
| 20 | have been billed is paying twice for one thing. A retry is a NEW run, |
| 21 | with a new hold, started by a person looking at the abandoned one -- |
| 22 | `provider_job_id`, recorded before the call where the provider supports |
| 23 | it, is the thread that person pulls to ask "did they charge us". |
| 24 | |
| 25 | Staleness comes from `runner.stale_seconds` (default 900). Generous on |
| 26 | purpose: this sweep is for processes that DIED, and a slow provider call |
| 27 | is not a dead process. Sweeping a run that is merely slow abandons work |
| 28 | the provider will finish and bill for -- so err long, and let genuinely |
| 29 | long-running handlers push their own heartbeats when they arrive. |
| 30 | """ |
| 31 | |
| 32 | import os |
| 33 | from datetime import datetime, timezone |
| 34 | |
| 35 | import object_records |
| 36 | import object_template_runs |
| 37 | |
| 38 | ACTOR = "system_run_sweeper" |
| 39 | RUNS = "template_runs" |
| 40 | |
| 41 | STALE_SETTING = "runner.stale_seconds" |
| 42 | MAX_RUN_SETTING = "runner.max_run_seconds" |
| 43 | # Deliberately generous: this is not "how long should a job take" but |
| 44 | # "how long before a machine gives up deciding and asks a person". Sora |
| 45 | # budgets thirty minutes; an hour leaves room for a provider queue. |
| 46 | DEFAULT_MAX_RUN_SECONDS = 3600 |
| 47 | DEFAULT_STALE_SECONDS = 900 |
| 48 | |
| 49 | |
| 50 | def _base_dir(): |
| 51 | return os.environ.get("DBBASIC_DATA_DIR", "data") |
| 52 | |
| 53 | |
| 54 | def _text(value): |
| 55 | return str(value if value is not None else "").strip() |
| 56 | |
| 57 | |
| 58 | def _int(value, default=0): |
| 59 | try: |
| 60 | return int(_text(value) or default) |
| 61 | except (TypeError, ValueError): |
| 62 | return default |
| 63 | |
| 64 | |
| 65 | def _now(): |
| 66 | return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") |
| 67 | |
| 68 | |
| 69 | def _setting(base, key, default): |
| 70 | try: |
| 71 | for row in object_records.read_collection_records("app_settings", |
| 72 | base_dir=base): |
| 73 | if _text(row.get("key")) == key: |
| 74 | return _text(row.get("value")) or default |
| 75 | except Exception: |
| 76 | pass |
| 77 | return default |
| 78 | |
| 79 | |
| 80 | def EVENT(request): |
| 81 | request = request or {} |
| 82 | base = _base_dir() |
| 83 | stale_seconds = max(60, _int(_setting(base, STALE_SETTING, |
| 84 | DEFAULT_STALE_SECONDS), |
| 85 | DEFAULT_STALE_SECONDS)) |
| 86 | now = _text(request.get("now")) or _now() |
| 87 | |
| 88 | try: |
| 89 | runs = object_records.read_collection_records(RUNS, base_dir=base) |
| 90 | except Exception: |
| 91 | return {"ok": True, "swept": 0, |
| 92 | "note": "No template_runs collection on this box."} |
| 93 | |
| 94 | try: |
| 95 | entries = object_records.read_collection_records("wallet_entries", |
| 96 | base_dir=base) |
| 97 | except Exception: |
| 98 | entries = [] |
| 99 | |
| 100 | max_run_seconds = max(stale_seconds, _int(_setting(base, MAX_RUN_SETTING, |
| 101 | DEFAULT_MAX_RUN_SECONDS), |
| 102 | DEFAULT_MAX_RUN_SECONDS)) |
| 103 | |
| 104 | swept, review = [], [] |
| 105 | # STEP 1 -- decide. Writing off a run is a COMPARE-AND-SET, because the |
| 106 | # sweeper races the runner it is sweeping: a runner that heartbeats or |
| 107 | # finishes between our staleness read and our write has changed the |
| 108 | # row, so our rev no longer matches and we correctly lose. Without the |
| 109 | # precondition this pass could write off a run that was completing and |
| 110 | # hand back money for work that succeeded. |
| 111 | # |
| 112 | # WHICH decision is object_template_runs.poll_disposition's, not this |
| 113 | # loop's, because the same rule has to bind the runner too. The rule |
| 114 | # that matters: a run carrying a provider_job_id is NEVER abandoned. |
| 115 | # Abandoning is what releases the hold, and a job the provider is |
| 116 | # already holding may already have cost everything -- handing the |
| 117 | # money back on a timer would be paying for it ourselves. Such a run |
| 118 | # is polled instead (the provider can simply be asked), and if polling |
| 119 | # never resolves it, it stops for a person. |
| 120 | for run in runs: |
| 121 | disposition = object_template_runs.poll_disposition( |
| 122 | run, now=now, stale_seconds=stale_seconds, |
| 123 | max_run_seconds=max_run_seconds) |
| 124 | if disposition not in ("abandon", "review"): |
| 125 | continue # "poll" belongs to the runner, not here |
| 126 | run_id = _text(run.get("id")) |
| 127 | if disposition == "abandon": |
| 128 | changes = { |
| 129 | "status": "abandoned", |
| 130 | "error": (f"Abandoned: no heartbeat for over {stale_seconds}s " |
| 131 | f"and nothing was submitted to a provider, so " |
| 132 | f"nothing was spent. The hold is released in full. " |
| 133 | f"NOT retried -- a retry is a new run, started by a " |
| 134 | f"person."), |
| 135 | "finished_at": now, |
| 136 | } |
| 137 | else: |
| 138 | changes = { |
| 139 | "status": object_template_runs.NEEDS_REVIEW_STATUS, |
| 140 | "error": (f"Submitted to the provider (job " |
| 141 | f"{_text(run.get('provider_job_id')) or 'unknown'}) " |
| 142 | f"and still unresolved after {max_run_seconds}s. " |
| 143 | f"The hold is deliberately NOT released: this job " |
| 144 | f"may already have been charged, so releasing would " |
| 145 | f"give away what we paid and charging would bill for " |
| 146 | f"work that may never have arrived. A person decides."), |
| 147 | } |
| 148 | try: |
| 149 | object_records.update_collection_record( |
| 150 | RUNS, run_id, changes, base_dir=base, actor=ACTOR, |
| 151 | expected_rev=object_records.compute_record_rev(run)) |
| 152 | except object_records.VersionConflictError: |
| 153 | # It moved while we were looking at it -- alive after all. |
| 154 | continue |
| 155 | (swept if disposition == "abandon" else review).append(run_id) |
| 156 | |
| 157 | # STEP 2 -- pay back, driven by STATE rather than by "did step 1 just |
| 158 | # run". Every terminal run whose hold has not been released gets it |
| 159 | # released, so a crash between the two steps is repaired by the next |
| 160 | # pass instead of leaving money reserved against a run that is over. |
| 161 | # Idempotent by the release marker, so this is a no-op for the runs |
| 162 | # the runner already settled itself. |
| 163 | released = [] |
| 164 | for run in object_records.read_collection_records(RUNS, base_dir=base): |
| 165 | if _text(run.get("status")) not in object_template_runs.TERMINAL_STATUSES: |
| 166 | continue |
| 167 | run_id = _text(run.get("id")) |
| 168 | if object_template_runs.already_settled(run_id, entries): |
| 169 | continue |
| 170 | entries_written = object_template_runs.settlement(run, succeeded=False) |
| 171 | if not entries_written: |
| 172 | continue # a free run: nothing was held |
| 173 | for entry in entries_written: |
| 174 | object_records.create_collection_record( |
| 175 | "wallet_entries", entry, base_dir=base, actor=ACTOR) |
| 176 | entries.extend(entries_written) # so a second hit this pass skips |
| 177 | released.append(run_id) |
| 178 | |
| 179 | return {"ok": True, "swept": len(swept), "runs": swept, |
| 180 | "needs_review": review, |
| 181 | "holds_released": released, "stale_seconds": stale_seconds, |
| 182 | "max_run_seconds": max_run_seconds} |
| 183 | |
| 184 | |
| 185 | POST = EVENT |