askrobots / dbbasic-queue
Public
A job queue in TSV files for background tasks, with retries and exponential backoff.
queue.py · 5.8 KB · 208 lines
| 1 | """ |
| 2 | Core queue implementation (~50 lines) |
| 3 | """ |
| 4 | |
| 5 | import time |
| 6 | import json |
| 7 | import secrets |
| 8 | import os |
| 9 | |
| 10 | # NOTE: This is a stub implementation. In production, you would: |
| 11 | # pip install dbbasic-tsv |
| 12 | # from dbbasic_tsv import append, query, update |
| 13 | # |
| 14 | # For now, we'll implement basic TSV operations directly |
| 15 | |
| 16 | QUEUE_FILE = 'data/queue.tsv' |
| 17 | MAX_ATTEMPTS = 3 |
| 18 | |
| 19 | |
| 20 | def _get_queue_file(): |
| 21 | """Get the current queue file path""" |
| 22 | return QUEUE_FILE |
| 23 | |
| 24 | |
| 25 | def _ensure_queue_file(): |
| 26 | """Ensure queue file exists with headers""" |
| 27 | queue_file = _get_queue_file() |
| 28 | if not os.path.exists(queue_file): |
| 29 | os.makedirs(os.path.dirname(queue_file), exist_ok=True) |
| 30 | with open(queue_file, 'w') as f: |
| 31 | f.write('id\ttype\tpayload\tstatus\tcreated_at\trun_at\tattempts\terror\tresult\n') |
| 32 | |
| 33 | |
| 34 | def _read_tsv(): |
| 35 | """Read TSV file and return rows""" |
| 36 | _ensure_queue_file() |
| 37 | queue_file = _get_queue_file() |
| 38 | with open(queue_file, 'r') as f: |
| 39 | lines = f.readlines() |
| 40 | if not lines: |
| 41 | return [] |
| 42 | # Skip header and ensure all rows have 9 columns |
| 43 | rows = [] |
| 44 | for line in lines[1:]: |
| 45 | if line.strip(): |
| 46 | parts = line.strip().split('\t') |
| 47 | # Pad with empty strings to ensure 9 columns |
| 48 | while len(parts) < 9: |
| 49 | parts.append('') |
| 50 | rows.append(parts) |
| 51 | return rows |
| 52 | |
| 53 | |
| 54 | def _write_tsv(rows): |
| 55 | """Write rows to TSV file""" |
| 56 | _ensure_queue_file() |
| 57 | queue_file = _get_queue_file() |
| 58 | with open(queue_file, 'w') as f: |
| 59 | f.write('id\ttype\tpayload\tstatus\tcreated_at\trun_at\tattempts\terror\tresult\n') |
| 60 | for row in rows: |
| 61 | f.write('\t'.join(row) + '\n') |
| 62 | |
| 63 | |
| 64 | def _append_row(row): |
| 65 | """Append a row to TSV file""" |
| 66 | _ensure_queue_file() |
| 67 | queue_file = _get_queue_file() |
| 68 | with open(queue_file, 'a') as f: |
| 69 | f.write('\t'.join(row) + '\n') |
| 70 | |
| 71 | |
| 72 | def enqueue(job_type, payload, run_at=None): |
| 73 | """ |
| 74 | Add job to queue |
| 75 | |
| 76 | Args: |
| 77 | job_type (str): Job handler name (e.g., 'send_email') |
| 78 | payload (dict): Job parameters |
| 79 | run_at (int, optional): Unix timestamp to run job (default: now) |
| 80 | |
| 81 | Returns: |
| 82 | str: Unique job identifier |
| 83 | """ |
| 84 | job_id = secrets.token_hex(8) |
| 85 | now = int(time.time()) |
| 86 | run_at = run_at or now |
| 87 | |
| 88 | _append_row([ |
| 89 | job_id, |
| 90 | job_type, |
| 91 | json.dumps(payload), |
| 92 | 'pending', |
| 93 | str(now), |
| 94 | str(run_at), |
| 95 | '0', # attempts |
| 96 | '', # error |
| 97 | '' # result |
| 98 | ]) |
| 99 | return job_id |
| 100 | |
| 101 | |
| 102 | def process_jobs(handlers, max_attempts=MAX_ATTEMPTS): |
| 103 | """ |
| 104 | Process pending jobs (worker) |
| 105 | |
| 106 | Args: |
| 107 | handlers (dict): Map of job_type → handler function |
| 108 | max_attempts (int): Max retry attempts before marking failed |
| 109 | """ |
| 110 | now = int(time.time()) |
| 111 | |
| 112 | # Get pending jobs that are ready to run |
| 113 | rows = _read_tsv() |
| 114 | jobs = [row for row in rows if len(row) >= 9 and row[3] == 'pending' and int(row[5]) <= now] |
| 115 | |
| 116 | for job in jobs: |
| 117 | job_id, job_type, payload_json, status, created, run_at, attempts, error, result = job |
| 118 | attempts = int(attempts) |
| 119 | |
| 120 | # Update to processing |
| 121 | all_rows = _read_tsv() |
| 122 | for i, r in enumerate(all_rows): |
| 123 | if r[0] == job_id: |
| 124 | all_rows[i] = [r[0], r[1], r[2], 'processing', r[4], r[5], r[6], r[7], r[8]] |
| 125 | _write_tsv(all_rows) |
| 126 | |
| 127 | # Execute job |
| 128 | try: |
| 129 | handler = handlers.get(job_type) |
| 130 | if not handler: |
| 131 | raise Exception(f"No handler for {job_type}") |
| 132 | |
| 133 | payload = json.loads(payload_json) |
| 134 | result = handler(payload) |
| 135 | |
| 136 | # Success |
| 137 | all_rows = _read_tsv() |
| 138 | for i, r in enumerate(all_rows): |
| 139 | if r[0] == job_id: |
| 140 | all_rows[i] = [r[0], r[1], r[2], 'completed', r[4], r[5], str(attempts + 1), '', json.dumps(result) if result else ''] |
| 141 | _write_tsv(all_rows) |
| 142 | except Exception as e: |
| 143 | attempts += 1 |
| 144 | |
| 145 | all_rows = _read_tsv() |
| 146 | for i, r in enumerate(all_rows): |
| 147 | if r[0] == job_id: |
| 148 | if attempts >= max_attempts: |
| 149 | # Failed permanently |
| 150 | all_rows[i] = [r[0], r[1], r[2], 'failed', r[4], r[5], str(attempts), str(e), ''] |
| 151 | else: |
| 152 | # Retry with backoff |
| 153 | backoff = 60 * (2 ** attempts) # Exponential backoff |
| 154 | new_run_at = int(time.time()) + backoff |
| 155 | all_rows[i] = [r[0], r[1], r[2], 'pending', r[4], str(new_run_at), str(attempts), str(e), ''] |
| 156 | _write_tsv(all_rows) |
| 157 | |
| 158 | |
| 159 | def get_job(job_id): |
| 160 | """ |
| 161 | Get job details |
| 162 | |
| 163 | Args: |
| 164 | job_id (str): Job identifier |
| 165 | |
| 166 | Returns: |
| 167 | dict: Job details or None if not found |
| 168 | """ |
| 169 | rows = _read_tsv() |
| 170 | for row in rows: |
| 171 | if len(row) >= 9 and row[0] == job_id: |
| 172 | return { |
| 173 | 'id': row[0], |
| 174 | 'type': row[1], |
| 175 | 'payload': json.loads(row[2]) if row[2] else {}, |
| 176 | 'status': row[3], |
| 177 | 'created_at': int(row[4]), |
| 178 | 'run_at': int(row[5]), |
| 179 | 'attempts': int(row[6]), |
| 180 | 'error': row[7] or None, |
| 181 | 'result': json.loads(row[8]) if row[8] else None |
| 182 | } |
| 183 | return None |
| 184 | |
| 185 | |
| 186 | def cancel_job(job_id): |
| 187 | """ |
| 188 | Cancel pending job |
| 189 | |
| 190 | Args: |
| 191 | job_id (str): Job identifier |
| 192 | |
| 193 | Returns: |
| 194 | bool: True if cancelled, False if already processing/completed |
| 195 | """ |
| 196 | all_rows = _read_tsv() |
| 197 | cancelled = False |
| 198 | |
| 199 | for i, r in enumerate(all_rows): |
| 200 | if len(r) >= 9 and r[0] == job_id and r[3] == 'pending': |
| 201 | all_rows[i] = [r[0], r[1], r[2], 'cancelled', r[4], r[5], r[6], 'Cancelled by user', ''] |
| 202 | cancelled = True |
| 203 | break |
| 204 | |
| 205 | if cancelled: |
| 206 | _write_tsv(all_rows) |
| 207 | |
| 208 | return cancelled |