askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
scheduler.py · 14.5 KB · 363 lines
| 1 | """system_scheduler -- the Flower-style board for daemon-scheduled tasks. |
| 2 | |
| 3 | Dan's bar: "celery flower would do better than that" -- the raw /daemon/* |
| 4 | JSON (admin-token-gated, unreadable) is not an operator surface. This page |
| 5 | is: the task board (schedule, next run, last outcome, run count) joined |
| 6 | with the run history (scheduler_runs records the daemon appends per |
| 7 | execution -- result JSON, error + type, duration), plus the two controls |
| 8 | an operator actually reaches for: Run now and Pause/Resume. |
| 9 | |
| 10 | Identity posture: public execute like every page object; everything is |
| 11 | gated on an ADMIN identity inside (anonymous/non-admin get a prompt, no |
| 12 | data). POST actions mutate the scheduler trigger's task state directly -- |
| 13 | the same state the daemon reads -- so Run now simply sets next_run to now |
| 14 | and the next daemon poll executes it; results then appear here because the |
| 15 | daemon records them as scheduler_runs rows. Live: subscribes to |
| 16 | scheduler_runs over /ws (admin read permission gates the push), with a |
| 17 | 60s poll fallback. |
| 18 | """ |
| 19 | |
| 20 | import html |
| 21 | import json |
| 22 | import os |
| 23 | import time |
| 24 | from datetime import datetime, timezone |
| 25 | |
| 26 | import object_records |
| 27 | import object_state |
| 28 | |
| 29 | _logger = None |
| 30 | |
| 31 | _TASK_PREFIX = "task_" |
| 32 | _RUNS_SHOWN = 50 |
| 33 | |
| 34 | |
| 35 | def _base_dir(): |
| 36 | return os.environ.get("DBBASIC_DATA_DIR", "data") |
| 37 | |
| 38 | |
| 39 | def _tasks(): |
| 40 | rows, invalid = [], 0 |
| 41 | for key, value in object_state.get_object_state("scheduler", _base_dir()).items(): |
| 42 | if not key.startswith(_TASK_PREFIX): |
| 43 | continue |
| 44 | try: |
| 45 | task = json.loads(value) |
| 46 | except (ValueError, TypeError): |
| 47 | invalid += 1 |
| 48 | continue |
| 49 | if not isinstance(task, dict): |
| 50 | invalid += 1 |
| 51 | continue |
| 52 | task["_key"] = key |
| 53 | rows.append(task) |
| 54 | rows.sort(key=lambda t: str(t.get("id") or "")) |
| 55 | return rows, invalid |
| 56 | |
| 57 | |
| 58 | def _runs(): |
| 59 | try: |
| 60 | rows = object_records.read_collection_records("scheduler_runs", base_dir=_base_dir()) |
| 61 | except Exception: |
| 62 | return None # collection not installed |
| 63 | rows.sort(key=lambda r: r.get("started_at") or "", reverse=True) |
| 64 | return rows |
| 65 | |
| 66 | |
| 67 | def _when(epoch): |
| 68 | if not epoch: |
| 69 | return "—" |
| 70 | try: |
| 71 | return datetime.fromtimestamp(int(epoch), tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") |
| 72 | except (ValueError, TypeError, OSError): |
| 73 | return "—" |
| 74 | |
| 75 | |
| 76 | def _esc(value): |
| 77 | return html.escape(str(value or "")) |
| 78 | |
| 79 | |
| 80 | _DAYS = {"0": "Sunday", "1": "Monday", "2": "Tuesday", "3": "Wednesday", |
| 81 | "4": "Thursday", "5": "Friday", "6": "Saturday", "7": "Sunday", |
| 82 | "sun": "Sunday", "mon": "Monday", "tue": "Tuesday", "wed": "Wednesday", |
| 83 | "thu": "Thursday", "fri": "Friday", "sat": "Saturday"} |
| 84 | |
| 85 | |
| 86 | def describe_schedule(schedule, task_type=""): |
| 87 | """Render a schedule in plain English. |
| 88 | |
| 89 | "10 6 * * *" is precise and completely opaque to anyone who does not |
| 90 | write cron daily -- and an operator who cannot read WHEN a task runs |
| 91 | cannot tell whether it ran when it should have. So the board leads |
| 92 | with the English and keeps the cron string as the small print. |
| 93 | """ |
| 94 | text = str(schedule or "").strip() |
| 95 | if not text: |
| 96 | return "" |
| 97 | if str(task_type).strip() == "onetime": |
| 98 | return f"Once at {text[:16].replace('T', ' ')}" |
| 99 | |
| 100 | parts = text.split() |
| 101 | if len(parts) != 5: |
| 102 | return "" |
| 103 | minute, hour, dom, month, dow = parts |
| 104 | |
| 105 | def _every(field): |
| 106 | return field.startswith("*/") and field[2:].isdigit() |
| 107 | |
| 108 | if _every(minute) and hour == dom == month == dow == "*": |
| 109 | return f"Every {minute[2:]} minutes" |
| 110 | if minute.isdigit() and hour == "*" and dom == month == dow == "*": |
| 111 | return f"Hourly at :{int(minute):02d}" |
| 112 | if _every(hour) and minute.isdigit() and dom == month == dow == "*": |
| 113 | return f"Every {hour[2:]} hours at :{int(minute):02d}" |
| 114 | if not (minute.isdigit() and hour.isdigit()): |
| 115 | return "" |
| 116 | |
| 117 | at = f"{int(hour):02d}:{int(minute):02d} UTC" |
| 118 | if dom == month == dow == "*": |
| 119 | return f"Daily at {at}" |
| 120 | if dow != "*" and dom == "*": |
| 121 | # Only a single named day is describable: "1-5" is weekdays and |
| 122 | # "1,4" is twice a week -- calling either "Weekly" would be a |
| 123 | # confident wrong answer, so those fall back to the raw cron. |
| 124 | day = _DAYS.get(dow.lower()) |
| 125 | return f"Weekly on {day} at {at}" if day else "" |
| 126 | if dom.isdigit() and month == "*" and dow == "*": |
| 127 | return f"Monthly on day {int(dom)} at {at}" |
| 128 | if dom.isdigit() and month.isdigit() and dow == "*": |
| 129 | return f"Yearly on {int(month)}/{int(dom)} at {at}" |
| 130 | return "" |
| 131 | |
| 132 | |
| 133 | def _task_rows_html(tasks, latest_by_task): |
| 134 | if not tasks: |
| 135 | return '<tr><td colspan="8" class="muted">No task_* entries in the scheduler trigger state.</td></tr>' |
| 136 | out = [] |
| 137 | for task in tasks: |
| 138 | tid = str(task.get("id") or task.get("_key")) |
| 139 | last = latest_by_task.get(tid) |
| 140 | if last is None: |
| 141 | last_cell = '<span class="muted">never</span>' |
| 142 | else: |
| 143 | badge = ('<span class="ok">✓ ok</span>' if last.get("ok") == "true" |
| 144 | else '<span class="bad">✗ failed</span>') |
| 145 | last_cell = (f"{badge} <span class=\"muted\">{_esc(last.get('started_at', ''))[:16]}" |
| 146 | f" · {_esc(last.get('duration_ms'))}ms</span>") |
| 147 | status = str(task.get("status") or "") |
| 148 | toggle = ("pause" if status == "active" else "resume") |
| 149 | english = describe_schedule(task.get("schedule"), task.get("type")) |
| 150 | schedule_cell = ( |
| 151 | (f"<strong>{_esc(english)}</strong><br>" if english else "") |
| 152 | + f"<code class=\"muted\" title=\"minute hour day-of-month month day-of-week\">" |
| 153 | f"{_esc(task.get('schedule'))}</code>" |
| 154 | + (f" <span class=\"muted\">{_esc(task.get('type'))}</span>" if not english else "") |
| 155 | ) |
| 156 | # A package that declares a schedule says WHY it exists; showing it |
| 157 | # here is the difference between a board of cron strings and a board |
| 158 | # an operator can reason about at 3am. |
| 159 | description = str(task.get("description") or "") |
| 160 | out.append( |
| 161 | "<tr>" |
| 162 | f"<td><code>{_esc(tid)}</code>" |
| 163 | + (f"<br><span class=\"muted\">{_esc(description)}</span>" if description else "") |
| 164 | + "</td>" |
| 165 | f"<td><code>{_esc(task.get('object_id'))}</code>.{_esc(task.get('method', 'POST'))}</td>" |
| 166 | f"<td>{schedule_cell}</td>" |
| 167 | f"<td class=\"{'ok' if status == 'active' else 'warn'}\">{_esc(status)}</td>" |
| 168 | f"<td>{_when(task.get('next_run'))}</td>" |
| 169 | f"<td>{last_cell}</td>" |
| 170 | f"<td>{_esc(task.get('run_count') or 0)}</td>" |
| 171 | "<td>" |
| 172 | f"<button data-act=\"run_now\" data-key=\"{_esc(task['_key'])}\">Run now</button> " |
| 173 | f"<button data-act=\"{toggle}\" data-key=\"{_esc(task['_key'])}\">{toggle.capitalize()}</button>" |
| 174 | "</td>" |
| 175 | "</tr>" |
| 176 | ) |
| 177 | return "".join(out) |
| 178 | |
| 179 | |
| 180 | def _run_rows_html(runs): |
| 181 | if runs is None: |
| 182 | return ('<tr><td colspan="6" class="warn">scheduler_runs collection not installed ' |
| 183 | "— reinstall the system-dashboard package to record run history.</td></tr>") |
| 184 | if not runs: |
| 185 | return '<tr><td colspan="6" class="muted">No runs recorded yet.</td></tr>' |
| 186 | out = [] |
| 187 | for run in runs[:_RUNS_SHOWN]: |
| 188 | ok = run.get("ok") == "true" |
| 189 | detail = run.get("result") if ok else run.get("error") |
| 190 | detail_html = "" |
| 191 | if detail: |
| 192 | label = "result" if ok else _esc(run.get("error_type") or "error") |
| 193 | detail_html = (f"<details><summary>{label}</summary>" |
| 194 | f"<pre>{_esc(detail)}</pre></details>") |
| 195 | out.append( |
| 196 | "<tr>" |
| 197 | f"<td>{_esc(run.get('started_at', ''))[:19]}</td>" |
| 198 | f"<td><code>{_esc(run.get('task_id'))}</code></td>" |
| 199 | f"<td><code>{_esc(run.get('object_id'))}</code></td>" |
| 200 | f"<td>{'<span class=ok>✓ ok</span>' if ok else '<span class=bad>✗ failed</span>'}</td>" |
| 201 | f"<td>{_esc(run.get('duration_ms'))}ms</td>" |
| 202 | f"<td>{detail_html}</td>" |
| 203 | "</tr>" |
| 204 | ) |
| 205 | return "".join(out) |
| 206 | |
| 207 | |
| 208 | _SCRIPT = """ |
| 209 | document.addEventListener('click', async (e) => { |
| 210 | const btn = e.target.closest('button[data-act]'); |
| 211 | if (!btn) return; |
| 212 | btn.disabled = true; |
| 213 | const resp = await fetch('/scheduler', { |
| 214 | method: 'POST', |
| 215 | headers: {'Content-Type': 'application/json'}, |
| 216 | body: JSON.stringify({action: btn.dataset.act, key: btn.dataset.key}), |
| 217 | }); |
| 218 | if (!resp.ok) { alert('Action failed: ' + resp.status); btn.disabled = false; return; } |
| 219 | location.reload(); |
| 220 | }); |
| 221 | let reloadTimer = null; |
| 222 | function queueReload() { |
| 223 | if (reloadTimer) return; |
| 224 | reloadTimer = setTimeout(() => location.reload(), 400); |
| 225 | } |
| 226 | function trySubscribe() { |
| 227 | if (window.dbbasicSubscribe) { window.dbbasicSubscribe('scheduler_runs', queueReload); } |
| 228 | else { setTimeout(trySubscribe, 1500); } |
| 229 | } |
| 230 | trySubscribe(); |
| 231 | setTimeout(() => location.reload(), 60000); |
| 232 | """ |
| 233 | |
| 234 | |
| 235 | def _page(body, *, title="Scheduler"): |
| 236 | return { |
| 237 | "content_type": "text/html; charset=utf-8", |
| 238 | "body": f"""<!doctype html> |
| 239 | <html lang="en"> |
| 240 | <head> |
| 241 | <meta charset="utf-8"> |
| 242 | <meta name="viewport" content="width=device-width, initial-scale=1"> |
| 243 | <title>{title}</title> |
| 244 | <link rel="stylesheet" href="/style"> |
| 245 | <style> |
| 246 | .wrap {{ max-width: 1100px; margin: 0 auto; padding: 1.25rem; }} |
| 247 | table {{ width: 100%; border-collapse: collapse; }} |
| 248 | th, td {{ text-align: left; padding: 0.45rem 0.6rem; border-bottom: 1px solid var(--line, #333); vertical-align: top; }} |
| 249 | th {{ font-size: 0.72rem; text-transform: uppercase; letter-spacing: 0.05em; color: var(--muted, #999); }} |
| 250 | .muted {{ color: var(--muted, #999); }} |
| 251 | .ok {{ color: var(--positive, #52d273); }} |
| 252 | .bad {{ color: var(--danger, #ff6b6b); }} |
| 253 | .warn {{ color: var(--warning, #f1b747); }} |
| 254 | .tiles {{ display: flex; gap: 0.75rem; flex-wrap: wrap; margin: 1rem 0; }} |
| 255 | .tile {{ background: var(--panel, #1a1a22); border: 1px solid var(--line, #333); border-radius: 8px; padding: 0.6rem 1rem; min-width: 130px; }} |
| 256 | .tile .n {{ font-size: 1.3rem; font-weight: 600; }} |
| 257 | .tile .l {{ font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.05em; color: var(--muted, #999); }} |
| 258 | h2 {{ font-size: 0.85rem; text-transform: uppercase; letter-spacing: 0.05em; color: var(--muted, #999); margin: 1.5rem 0 0.5rem; }} |
| 259 | pre {{ white-space: pre-wrap; max-width: 480px; font-size: 0.75rem; margin: 0.3rem 0 0; }} |
| 260 | details summary {{ cursor: pointer; }} |
| 261 | button {{ cursor: pointer; }} |
| 262 | </style> |
| 263 | </head> |
| 264 | <body> |
| 265 | <script src="/nav"></script> |
| 266 | <div class="wrap"> |
| 267 | <h1>Scheduler</h1> |
| 268 | {body} |
| 269 | </div> |
| 270 | </body> |
| 271 | </html>""", |
| 272 | } |
| 273 | |
| 274 | |
| 275 | def GET(request): |
| 276 | identity = request.get("_identity") or {} |
| 277 | user_id = identity.get("user_id") |
| 278 | if "admin" not in (identity.get("roles") or []): |
| 279 | prompt = ('<a href="/login?next=/scheduler">Sign in</a> with an admin account to see ' |
| 280 | "scheduled tasks and their run history." |
| 281 | if not user_id else |
| 282 | "You are signed in, but the scheduler board needs an admin role.") |
| 283 | return _page(f'<p class="muted">{prompt}</p>') |
| 284 | |
| 285 | tasks, invalid = _tasks() |
| 286 | runs = _runs() |
| 287 | latest_by_task = {} |
| 288 | failures_24h = runs_24h = 0 |
| 289 | if runs: |
| 290 | cutoff = datetime.fromtimestamp(time.time() - 86400, tz=timezone.utc).isoformat() |
| 291 | for run in runs: |
| 292 | tid = str(run.get("task_id") or "") |
| 293 | if tid and tid not in latest_by_task: |
| 294 | latest_by_task[tid] = run # runs sorted newest-first |
| 295 | if (run.get("started_at") or "") >= cutoff: |
| 296 | runs_24h += 1 |
| 297 | if run.get("ok") != "true": |
| 298 | failures_24h += 1 |
| 299 | |
| 300 | now = int(time.time()) |
| 301 | due = sum(1 for t in tasks |
| 302 | if t.get("status") == "active" and (t.get("next_run") or 0) and int(t.get("next_run") or 0) <= now) |
| 303 | tiles = ( |
| 304 | f'<div class="tiles">' |
| 305 | f'<div class="tile"><div class="n">{sum(1 for t in tasks if t.get("status") == "active")}</div><div class="l">Active tasks</div></div>' |
| 306 | f'<div class="tile"><div class="n">{due}</div><div class="l">Due now</div></div>' |
| 307 | f'<div class="tile"><div class="n">{runs_24h}</div><div class="l">Runs (24h)</div></div>' |
| 308 | f'<div class="tile"><div class="n {"bad" if failures_24h else ""}">{failures_24h}</div><div class="l">Failures (24h)</div></div>' |
| 309 | + (f'<div class="tile"><div class="n warn">{invalid}</div><div class="l">Invalid entries</div></div>' if invalid else "") |
| 310 | + "</div>" |
| 311 | ) |
| 312 | body = f"""{tiles} |
| 313 | <h2>Tasks</h2> |
| 314 | <table> |
| 315 | <thead><tr><th>Task</th><th>Target</th><th>Schedule</th><th>Status</th><th>Next Run</th><th>Last Run</th><th>Runs</th><th>Actions</th></tr></thead> |
| 316 | <tbody>{_task_rows_html(tasks, latest_by_task)}</tbody> |
| 317 | </table> |
| 318 | <p class="muted" style="font-size:0.75rem;margin:0.5rem 0 0"> |
| 319 | Cron fields are <code>minute hour day-of-month month day-of-week</code> |
| 320 | (all times UTC). <code>*</code> means every, <code>*/5</code> means every 5th. |
| 321 | </p> |
| 322 | <h2>Recent Runs</h2> |
| 323 | <table> |
| 324 | <thead><tr><th>Started</th><th>Task</th><th>Target</th><th>Outcome</th><th>Duration</th><th>Detail</th></tr></thead> |
| 325 | <tbody>{_run_rows_html(runs)}</tbody> |
| 326 | </table> |
| 327 | <script>{_SCRIPT}</script>""" |
| 328 | return _page(body) |
| 329 | |
| 330 | |
| 331 | def POST(request): |
| 332 | identity = request.get("_identity") or {} |
| 333 | if "admin" not in (identity.get("roles") or []): |
| 334 | return {"status": 403, "error": "Scheduler actions require an admin session."} |
| 335 | |
| 336 | action = str(request.get("action") or "").strip() |
| 337 | key = str(request.get("key") or "").strip() |
| 338 | if action not in ("run_now", "pause", "resume") or not key.startswith(_TASK_PREFIX): |
| 339 | return {"status": 400, "error": "action must be run_now|pause|resume with a task_* key"} |
| 340 | |
| 341 | manager = object_state.ObjectStateManager("scheduler", base_dir=_base_dir()) |
| 342 | raw = manager.get(key) |
| 343 | if not raw: |
| 344 | return {"status": 404, "error": f"No such task entry: {key}"} |
| 345 | try: |
| 346 | task = json.loads(raw) |
| 347 | except (ValueError, TypeError): |
| 348 | return {"status": 409, "error": f"Task entry is not valid JSON: {key}"} |
| 349 | |
| 350 | if action == "run_now": |
| 351 | task["next_run"] = int(time.time()) |
| 352 | task["status"] = "active" |
| 353 | elif action == "pause": |
| 354 | task["status"] = "paused" |
| 355 | else: # resume |
| 356 | task["status"] = "active" |
| 357 | task["next_run"] = None # daemon recomputes from the schedule |
| 358 | manager.set(key, json.dumps(task)) |
| 359 | if _logger: |
| 360 | _logger.info("scheduler task action", action=action, key=key, |
| 361 | actor=identity.get("user_id")) |
| 362 | return {"status": 200, "ok": True, "action": action, "key": key, |
| 363 | "next_run": task.get("next_run"), "task_status": task.get("status")} |