A job queue in TSV files for background tasks, with retries and exponential backoff.

0 downloads

dbbasic-queue/tests/test_queue.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-queue.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-queue.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-queue.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Update README with PyPI information and badges 89cd41e 1 year, 1 month ago History
test_queue.py · 8.0 KB · 267 lines
1#!/usr/bin/env python3
2"""
3Unit tests for dbbasic-queue
4
5Run with:
6 python3 -m pytest tests/test_queue.py -v
7"""
8
9import sys
10import os
11import time
12import tempfile
13import shutil
14
15# Add parent directory to path
16sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
17
18from dbbasic_queue import enqueue, process_jobs, get_job, cancel_job
19import dbbasic_queue.queue as queue_module
20
21
22class TestQueue:
23 """Test suite for queue functionality"""
24
25 def setup_method(self):
26 """Setup test environment"""
27 # Create temporary data directory for tests
28 self.test_dir = tempfile.mkdtemp()
29 self.original_queue_file = queue_module.QUEUE_FILE
30 queue_module.QUEUE_FILE = os.path.join(self.test_dir, 'queue.tsv')
31
32 def teardown_method(self):
33 """Cleanup test environment"""
34 # Restore original queue file path
35 queue_module.QUEUE_FILE = self.original_queue_file
36 # Remove test directory
37 if os.path.exists(self.test_dir):
38 shutil.rmtree(self.test_dir)
39
40 def test_enqueue(self):
41 """Test enqueuing a job"""
42 job_id = enqueue('test_job', {'foo': 'bar'})
43
44 assert job_id is not None
45 assert len(job_id) == 16 # 8 bytes = 16 hex chars
46
47 job = get_job(job_id)
48 assert job is not None
49 assert job['type'] == 'test_job'
50 assert job['payload'] == {'foo': 'bar'}
51 assert job['status'] == 'pending'
52 assert job['attempts'] == 0
53
54 def test_enqueue_delayed(self):
55 """Test enqueuing a delayed job"""
56 future_time = int(time.time()) + 3600 # 1 hour from now
57 job_id = enqueue('delayed_job', {'data': 123}, run_at=future_time)
58
59 job = get_job(job_id)
60 assert job is not None
61 assert job['run_at'] == future_time
62 assert job['status'] == 'pending'
63
64 def test_process_job_success(self):
65 """Test processing a job successfully"""
66 results = []
67
68 def test_handler(payload):
69 results.append(payload)
70 return {'result': 'success'}
71
72 job_id = enqueue('test', {'data': 123})
73 process_jobs({'test': test_handler})
74
75 assert len(results) == 1
76 assert results[0] == {'data': 123}
77
78 job = get_job(job_id)
79 assert job['status'] == 'completed'
80 assert job['result'] == {'result': 'success'}
81 assert job['attempts'] == 1
82
83 def test_process_job_not_ready(self):
84 """Test that future jobs are not processed"""
85 results = []
86
87 def test_handler(payload):
88 results.append(payload)
89 return 'success'
90
91 # Queue job for 1 hour in future
92 future_time = int(time.time()) + 3600
93 job_id = enqueue('future_test', {'data': 456}, run_at=future_time)
94
95 # Try to process jobs
96 process_jobs({'future_test': test_handler})
97
98 # Should not be processed yet
99 assert len(results) == 0
100
101 job = get_job(job_id)
102 assert job['status'] == 'pending'
103
104 def test_process_job_failure_retry(self):
105 """Test job retry on failure"""
106 attempts = []
107
108 def failing_handler(payload):
109 attempts.append(1)
110 raise Exception("Test failure")
111
112 job_id = enqueue('fail_test', {'data': 789})
113
114 # First attempt
115 process_jobs({'fail_test': failing_handler}, max_attempts=3)
116 assert len(attempts) == 1
117
118 job = get_job(job_id)
119 assert job['status'] == 'pending' # Should be retried
120 assert job['attempts'] == 1
121
122 def test_process_job_permanent_failure(self):
123 """Test job marked as failed after max attempts"""
124 attempts = []
125
126 def failing_handler(payload):
127 attempts.append(1)
128 raise Exception("Test failure")
129
130 job_id = enqueue('fail_test', {})
131
132 # Process 3 times (max attempts)
133 # Need to manipulate the run_at time to force immediate retry
134 for i in range(3):
135 process_jobs({'fail_test': failing_handler}, max_attempts=3)
136
137 # After first and second failure, reset run_at to now so it can run immediately
138 if i < 2:
139 job = get_job(job_id)
140 if job and job['status'] == 'pending':
141 # Force the job to be ready to run now by updating run_at
142 import dbbasic_queue.queue as queue_module
143 rows = queue_module._read_tsv()
144 for idx, r in enumerate(rows):
145 if r[0] == job_id:
146 rows[idx][5] = str(int(time.time())) # Set run_at to now
147 queue_module._write_tsv(rows)
148
149 assert len(attempts) == 3
150
151 job = get_job(job_id)
152 assert job['status'] == 'failed'
153 assert job['attempts'] == 3
154 assert 'Test failure' in job['error']
155
156 def test_process_job_missing_handler(self):
157 """Test job fails if handler is missing"""
158 job_id = enqueue('unknown_job', {})
159
160 process_jobs({}, max_attempts=3)
161
162 job = get_job(job_id)
163 # Should still be pending or failed depending on implementation
164 assert job['attempts'] >= 1
165
166 def test_get_job_not_found(self):
167 """Test getting non-existent job"""
168 job = get_job('nonexistent')
169 assert job is None
170
171 def test_cancel_job_pending(self):
172 """Test cancelling pending job"""
173 job_id = enqueue('test', {'data': 'cancel_me'})
174
175 cancelled = cancel_job(job_id)
176 assert cancelled is True
177
178 job = get_job(job_id)
179 assert job['status'] == 'cancelled'
180 assert 'Cancelled by user' in job['error']
181
182 def test_cancel_job_completed(self):
183 """Test cannot cancel completed job"""
184 def test_handler(payload):
185 return 'done'
186
187 job_id = enqueue('test', {})
188 process_jobs({'test': test_handler})
189
190 # Try to cancel completed job
191 cancelled = cancel_job(job_id)
192 assert cancelled is False
193
194 def test_cancel_job_not_found(self):
195 """Test cancelling non-existent job"""
196 cancelled = cancel_job('nonexistent')
197 assert cancelled is False
198
199 def test_multiple_jobs(self):
200 """Test processing multiple jobs"""
201 results = []
202
203 def counter_handler(payload):
204 results.append(payload['count'])
205 return f"processed_{payload['count']}"
206
207 # Queue multiple jobs
208 job_ids = []
209 for i in range(5):
210 job_id = enqueue('counter', {'count': i})
211 job_ids.append(job_id)
212
213 # Process all
214 process_jobs({'counter': counter_handler})
215
216 # Check all were processed
217 assert len(results) == 5
218 assert sorted(results) == [0, 1, 2, 3, 4]
219
220 # Verify all completed
221 for job_id in job_ids:
222 job = get_job(job_id)
223 assert job['status'] == 'completed'
224
225 def test_job_payload_types(self):
226 """Test various payload types"""
227 payloads = [
228 {'string': 'test'},
229 {'number': 42},
230 {'float': 3.14},
231 {'boolean': True},
232 {'list': [1, 2, 3]},
233 {'nested': {'deep': {'value': 123}}},
234 {'mixed': {'str': 'test', 'num': 42, 'list': [1, 2]}}
235 ]
236
237 for i, payload in enumerate(payloads):
238 job_id = enqueue(f'test_{i}', payload)
239 job = get_job(job_id)
240 assert job['payload'] == payload
241
242
243if __name__ == '__main__':
244 # Run tests without pytest
245 import traceback
246
247 test = TestQueue()
248 test_methods = [m for m in dir(test) if m.startswith('test_')]
249
250 passed = 0
251 failed = 0
252
253 for method_name in test_methods:
254 try:
255 test.setup_method()
256 method = getattr(test, method_name)
257 method()
258 test.teardown_method()
259 print(f"✓ {method_name}")
260 passed += 1
261 except Exception as e:
262 print(f"✗ {method_name}")
263 traceback.print_exc()
264 failed += 1
265
266 print(f"\nPassed: {passed}, Failed: {failed}")
267 sys.exit(0 if failed == 0 else 1)