A job queue in TSV files for background tasks, with retries and exponential backoff.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-queue.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-queue.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-queue.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Update README with PyPI information and badges 89cd41e 1 year, 1 month ago History
queue.py · 5.8 KB · 208 lines
1"""
2Core queue implementation (~50 lines)
3"""
4
5import time
6import json
7import secrets
8import os
9
10# NOTE: This is a stub implementation. In production, you would:
11# pip install dbbasic-tsv
12# from dbbasic_tsv import append, query, update
13#
14# For now, we'll implement basic TSV operations directly
15
16QUEUE_FILE = 'data/queue.tsv'
17MAX_ATTEMPTS = 3
18
19
20def _get_queue_file():
21 """Get the current queue file path"""
22 return QUEUE_FILE
23
24
25def _ensure_queue_file():
26 """Ensure queue file exists with headers"""
27 queue_file = _get_queue_file()
28 if not os.path.exists(queue_file):
29 os.makedirs(os.path.dirname(queue_file), exist_ok=True)
30 with open(queue_file, 'w') as f:
31 f.write('id\ttype\tpayload\tstatus\tcreated_at\trun_at\tattempts\terror\tresult\n')
32
33
34def _read_tsv():
35 """Read TSV file and return rows"""
36 _ensure_queue_file()
37 queue_file = _get_queue_file()
38 with open(queue_file, 'r') as f:
39 lines = f.readlines()
40 if not lines:
41 return []
42 # Skip header and ensure all rows have 9 columns
43 rows = []
44 for line in lines[1:]:
45 if line.strip():
46 parts = line.strip().split('\t')
47 # Pad with empty strings to ensure 9 columns
48 while len(parts) < 9:
49 parts.append('')
50 rows.append(parts)
51 return rows
52
53
54def _write_tsv(rows):
55 """Write rows to TSV file"""
56 _ensure_queue_file()
57 queue_file = _get_queue_file()
58 with open(queue_file, 'w') as f:
59 f.write('id\ttype\tpayload\tstatus\tcreated_at\trun_at\tattempts\terror\tresult\n')
60 for row in rows:
61 f.write('\t'.join(row) + '\n')
62
63
64def _append_row(row):
65 """Append a row to TSV file"""
66 _ensure_queue_file()
67 queue_file = _get_queue_file()
68 with open(queue_file, 'a') as f:
69 f.write('\t'.join(row) + '\n')
70
71
72def enqueue(job_type, payload, run_at=None):
73 """
74 Add job to queue
75
76 Args:
77 job_type (str): Job handler name (e.g., 'send_email')
78 payload (dict): Job parameters
79 run_at (int, optional): Unix timestamp to run job (default: now)
80
81 Returns:
82 str: Unique job identifier
83 """
84 job_id = secrets.token_hex(8)
85 now = int(time.time())
86 run_at = run_at or now
87
88 _append_row([
89 job_id,
90 job_type,
91 json.dumps(payload),
92 'pending',
93 str(now),
94 str(run_at),
95 '0', # attempts
96 '', # error
97 '' # result
98 ])
99 return job_id
100
101
102def process_jobs(handlers, max_attempts=MAX_ATTEMPTS):
103 """
104 Process pending jobs (worker)
105
106 Args:
107 handlers (dict): Map of job_type → handler function
108 max_attempts (int): Max retry attempts before marking failed
109 """
110 now = int(time.time())
111
112 # Get pending jobs that are ready to run
113 rows = _read_tsv()
114 jobs = [row for row in rows if len(row) >= 9 and row[3] == 'pending' and int(row[5]) <= now]
115
116 for job in jobs:
117 job_id, job_type, payload_json, status, created, run_at, attempts, error, result = job
118 attempts = int(attempts)
119
120 # Update to processing
121 all_rows = _read_tsv()
122 for i, r in enumerate(all_rows):
123 if r[0] == job_id:
124 all_rows[i] = [r[0], r[1], r[2], 'processing', r[4], r[5], r[6], r[7], r[8]]
125 _write_tsv(all_rows)
126
127 # Execute job
128 try:
129 handler = handlers.get(job_type)
130 if not handler:
131 raise Exception(f"No handler for {job_type}")
132
133 payload = json.loads(payload_json)
134 result = handler(payload)
135
136 # Success
137 all_rows = _read_tsv()
138 for i, r in enumerate(all_rows):
139 if r[0] == job_id:
140 all_rows[i] = [r[0], r[1], r[2], 'completed', r[4], r[5], str(attempts + 1), '', json.dumps(result) if result else '']
141 _write_tsv(all_rows)
142 except Exception as e:
143 attempts += 1
144
145 all_rows = _read_tsv()
146 for i, r in enumerate(all_rows):
147 if r[0] == job_id:
148 if attempts >= max_attempts:
149 # Failed permanently
150 all_rows[i] = [r[0], r[1], r[2], 'failed', r[4], r[5], str(attempts), str(e), '']
151 else:
152 # Retry with backoff
153 backoff = 60 * (2 ** attempts) # Exponential backoff
154 new_run_at = int(time.time()) + backoff
155 all_rows[i] = [r[0], r[1], r[2], 'pending', r[4], str(new_run_at), str(attempts), str(e), '']
156 _write_tsv(all_rows)
157
158
159def get_job(job_id):
160 """
161 Get job details
162
163 Args:
164 job_id (str): Job identifier
165
166 Returns:
167 dict: Job details or None if not found
168 """
169 rows = _read_tsv()
170 for row in rows:
171 if len(row) >= 9 and row[0] == job_id:
172 return {
173 'id': row[0],
174 'type': row[1],
175 'payload': json.loads(row[2]) if row[2] else {},
176 'status': row[3],
177 'created_at': int(row[4]),
178 'run_at': int(row[5]),
179 'attempts': int(row[6]),
180 'error': row[7] or None,
181 'result': json.loads(row[8]) if row[8] else None
182 }
183 return None
184
185
186def cancel_job(job_id):
187 """
188 Cancel pending job
189
190 Args:
191 job_id (str): Job identifier
192
193 Returns:
194 bool: True if cancelled, False if already processing/completed
195 """
196 all_rows = _read_tsv()
197 cancelled = False
198
199 for i, r in enumerate(all_rows):
200 if len(r) >= 9 and r[0] == job_id and r[3] == 'pending':
201 all_rows[i] = [r[0], r[1], r[2], 'cancelled', r[4], r[5], r[6], 'Cancelled by user', '']
202 cancelled = True
203 break
204
205 if cancelled:
206 _write_tsv(all_rows)
207
208 return cancelled