askrobots / dbbasic-queue
Public
A job queue in TSV files for background tasks, with retries and exponential backoff.
test_queue.py · 8.0 KB · 267 lines
| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | Unit tests for dbbasic-queue |
| 4 | |
| 5 | Run with: |
| 6 | python3 -m pytest tests/test_queue.py -v |
| 7 | """ |
| 8 | |
| 9 | import sys |
| 10 | import os |
| 11 | import time |
| 12 | import tempfile |
| 13 | import shutil |
| 14 | |
| 15 | # Add parent directory to path |
| 16 | sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| 17 | |
| 18 | from dbbasic_queue import enqueue, process_jobs, get_job, cancel_job |
| 19 | import dbbasic_queue.queue as queue_module |
| 20 | |
| 21 | |
| 22 | class TestQueue: |
| 23 | """Test suite for queue functionality""" |
| 24 | |
| 25 | def setup_method(self): |
| 26 | """Setup test environment""" |
| 27 | # Create temporary data directory for tests |
| 28 | self.test_dir = tempfile.mkdtemp() |
| 29 | self.original_queue_file = queue_module.QUEUE_FILE |
| 30 | queue_module.QUEUE_FILE = os.path.join(self.test_dir, 'queue.tsv') |
| 31 | |
| 32 | def teardown_method(self): |
| 33 | """Cleanup test environment""" |
| 34 | # Restore original queue file path |
| 35 | queue_module.QUEUE_FILE = self.original_queue_file |
| 36 | # Remove test directory |
| 37 | if os.path.exists(self.test_dir): |
| 38 | shutil.rmtree(self.test_dir) |
| 39 | |
| 40 | def test_enqueue(self): |
| 41 | """Test enqueuing a job""" |
| 42 | job_id = enqueue('test_job', {'foo': 'bar'}) |
| 43 | |
| 44 | assert job_id is not None |
| 45 | assert len(job_id) == 16 # 8 bytes = 16 hex chars |
| 46 | |
| 47 | job = get_job(job_id) |
| 48 | assert job is not None |
| 49 | assert job['type'] == 'test_job' |
| 50 | assert job['payload'] == {'foo': 'bar'} |
| 51 | assert job['status'] == 'pending' |
| 52 | assert job['attempts'] == 0 |
| 53 | |
| 54 | def test_enqueue_delayed(self): |
| 55 | """Test enqueuing a delayed job""" |
| 56 | future_time = int(time.time()) + 3600 # 1 hour from now |
| 57 | job_id = enqueue('delayed_job', {'data': 123}, run_at=future_time) |
| 58 | |
| 59 | job = get_job(job_id) |
| 60 | assert job is not None |
| 61 | assert job['run_at'] == future_time |
| 62 | assert job['status'] == 'pending' |
| 63 | |
| 64 | def test_process_job_success(self): |
| 65 | """Test processing a job successfully""" |
| 66 | results = [] |
| 67 | |
| 68 | def test_handler(payload): |
| 69 | results.append(payload) |
| 70 | return {'result': 'success'} |
| 71 | |
| 72 | job_id = enqueue('test', {'data': 123}) |
| 73 | process_jobs({'test': test_handler}) |
| 74 | |
| 75 | assert len(results) == 1 |
| 76 | assert results[0] == {'data': 123} |
| 77 | |
| 78 | job = get_job(job_id) |
| 79 | assert job['status'] == 'completed' |
| 80 | assert job['result'] == {'result': 'success'} |
| 81 | assert job['attempts'] == 1 |
| 82 | |
| 83 | def test_process_job_not_ready(self): |
| 84 | """Test that future jobs are not processed""" |
| 85 | results = [] |
| 86 | |
| 87 | def test_handler(payload): |
| 88 | results.append(payload) |
| 89 | return 'success' |
| 90 | |
| 91 | # Queue job for 1 hour in future |
| 92 | future_time = int(time.time()) + 3600 |
| 93 | job_id = enqueue('future_test', {'data': 456}, run_at=future_time) |
| 94 | |
| 95 | # Try to process jobs |
| 96 | process_jobs({'future_test': test_handler}) |
| 97 | |
| 98 | # Should not be processed yet |
| 99 | assert len(results) == 0 |
| 100 | |
| 101 | job = get_job(job_id) |
| 102 | assert job['status'] == 'pending' |
| 103 | |
| 104 | def test_process_job_failure_retry(self): |
| 105 | """Test job retry on failure""" |
| 106 | attempts = [] |
| 107 | |
| 108 | def failing_handler(payload): |
| 109 | attempts.append(1) |
| 110 | raise Exception("Test failure") |
| 111 | |
| 112 | job_id = enqueue('fail_test', {'data': 789}) |
| 113 | |
| 114 | # First attempt |
| 115 | process_jobs({'fail_test': failing_handler}, max_attempts=3) |
| 116 | assert len(attempts) == 1 |
| 117 | |
| 118 | job = get_job(job_id) |
| 119 | assert job['status'] == 'pending' # Should be retried |
| 120 | assert job['attempts'] == 1 |
| 121 | |
| 122 | def test_process_job_permanent_failure(self): |
| 123 | """Test job marked as failed after max attempts""" |
| 124 | attempts = [] |
| 125 | |
| 126 | def failing_handler(payload): |
| 127 | attempts.append(1) |
| 128 | raise Exception("Test failure") |
| 129 | |
| 130 | job_id = enqueue('fail_test', {}) |
| 131 | |
| 132 | # Process 3 times (max attempts) |
| 133 | # Need to manipulate the run_at time to force immediate retry |
| 134 | for i in range(3): |
| 135 | process_jobs({'fail_test': failing_handler}, max_attempts=3) |
| 136 | |
| 137 | # After first and second failure, reset run_at to now so it can run immediately |
| 138 | if i < 2: |
| 139 | job = get_job(job_id) |
| 140 | if job and job['status'] == 'pending': |
| 141 | # Force the job to be ready to run now by updating run_at |
| 142 | import dbbasic_queue.queue as queue_module |
| 143 | rows = queue_module._read_tsv() |
| 144 | for idx, r in enumerate(rows): |
| 145 | if r[0] == job_id: |
| 146 | rows[idx][5] = str(int(time.time())) # Set run_at to now |
| 147 | queue_module._write_tsv(rows) |
| 148 | |
| 149 | assert len(attempts) == 3 |
| 150 | |
| 151 | job = get_job(job_id) |
| 152 | assert job['status'] == 'failed' |
| 153 | assert job['attempts'] == 3 |
| 154 | assert 'Test failure' in job['error'] |
| 155 | |
| 156 | def test_process_job_missing_handler(self): |
| 157 | """Test job fails if handler is missing""" |
| 158 | job_id = enqueue('unknown_job', {}) |
| 159 | |
| 160 | process_jobs({}, max_attempts=3) |
| 161 | |
| 162 | job = get_job(job_id) |
| 163 | # Should still be pending or failed depending on implementation |
| 164 | assert job['attempts'] >= 1 |
| 165 | |
| 166 | def test_get_job_not_found(self): |
| 167 | """Test getting non-existent job""" |
| 168 | job = get_job('nonexistent') |
| 169 | assert job is None |
| 170 | |
| 171 | def test_cancel_job_pending(self): |
| 172 | """Test cancelling pending job""" |
| 173 | job_id = enqueue('test', {'data': 'cancel_me'}) |
| 174 | |
| 175 | cancelled = cancel_job(job_id) |
| 176 | assert cancelled is True |
| 177 | |
| 178 | job = get_job(job_id) |
| 179 | assert job['status'] == 'cancelled' |
| 180 | assert 'Cancelled by user' in job['error'] |
| 181 | |
| 182 | def test_cancel_job_completed(self): |
| 183 | """Test cannot cancel completed job""" |
| 184 | def test_handler(payload): |
| 185 | return 'done' |
| 186 | |
| 187 | job_id = enqueue('test', {}) |
| 188 | process_jobs({'test': test_handler}) |
| 189 | |
| 190 | # Try to cancel completed job |
| 191 | cancelled = cancel_job(job_id) |
| 192 | assert cancelled is False |
| 193 | |
| 194 | def test_cancel_job_not_found(self): |
| 195 | """Test cancelling non-existent job""" |
| 196 | cancelled = cancel_job('nonexistent') |
| 197 | assert cancelled is False |
| 198 | |
| 199 | def test_multiple_jobs(self): |
| 200 | """Test processing multiple jobs""" |
| 201 | results = [] |
| 202 | |
| 203 | def counter_handler(payload): |
| 204 | results.append(payload['count']) |
| 205 | return f"processed_{payload['count']}" |
| 206 | |
| 207 | # Queue multiple jobs |
| 208 | job_ids = [] |
| 209 | for i in range(5): |
| 210 | job_id = enqueue('counter', {'count': i}) |
| 211 | job_ids.append(job_id) |
| 212 | |
| 213 | # Process all |
| 214 | process_jobs({'counter': counter_handler}) |
| 215 | |
| 216 | # Check all were processed |
| 217 | assert len(results) == 5 |
| 218 | assert sorted(results) == [0, 1, 2, 3, 4] |
| 219 | |
| 220 | # Verify all completed |
| 221 | for job_id in job_ids: |
| 222 | job = get_job(job_id) |
| 223 | assert job['status'] == 'completed' |
| 224 | |
| 225 | def test_job_payload_types(self): |
| 226 | """Test various payload types""" |
| 227 | payloads = [ |
| 228 | {'string': 'test'}, |
| 229 | {'number': 42}, |
| 230 | {'float': 3.14}, |
| 231 | {'boolean': True}, |
| 232 | {'list': [1, 2, 3]}, |
| 233 | {'nested': {'deep': {'value': 123}}}, |
| 234 | {'mixed': {'str': 'test', 'num': 42, 'list': [1, 2]}} |
| 235 | ] |
| 236 | |
| 237 | for i, payload in enumerate(payloads): |
| 238 | job_id = enqueue(f'test_{i}', payload) |
| 239 | job = get_job(job_id) |
| 240 | assert job['payload'] == payload |
| 241 | |
| 242 | |
| 243 | if __name__ == '__main__': |
| 244 | # Run tests without pytest |
| 245 | import traceback |
| 246 | |
| 247 | test = TestQueue() |
| 248 | test_methods = [m for m in dir(test) if m.startswith('test_')] |
| 249 | |
| 250 | passed = 0 |
| 251 | failed = 0 |
| 252 | |
| 253 | for method_name in test_methods: |
| 254 | try: |
| 255 | test.setup_method() |
| 256 | method = getattr(test, method_name) |
| 257 | method() |
| 258 | test.teardown_method() |
| 259 | print(f"✓ {method_name}") |
| 260 | passed += 1 |
| 261 | except Exception as e: |
| 262 | print(f"✗ {method_name}") |
| 263 | traceback.print_exc() |
| 264 | failed += 1 |
| 265 | |
| 266 | print(f"\nPassed: {passed}, Failed: {failed}") |
| 267 | sys.exit(0 if failed == 0 else 1) |