askrobots / dbbasic-queue
Public
A job queue in TSV files for background tasks, with retries and exponential backoff.
demo.py · 4.3 KB · 159 lines
| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | Demo script showing dbbasic-queue in action |
| 4 | |
| 5 | Run this to see the queue system working end-to-end. |
| 6 | """ |
| 7 | |
| 8 | import sys |
| 9 | import os |
| 10 | import time |
| 11 | |
| 12 | # Add current directory to path |
| 13 | sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
| 14 | |
| 15 | from dbbasic_queue import enqueue, process_jobs, get_job |
| 16 | |
| 17 | print("=" * 60) |
| 18 | print("DBBasic Queue Demo") |
| 19 | print("=" * 60) |
| 20 | print() |
| 21 | |
| 22 | # Define some example handlers |
| 23 | def send_email_handler(payload): |
| 24 | """Simulate sending an email""" |
| 25 | print(f" 📧 Sending email to {payload['to']}") |
| 26 | print(f" Subject: {payload['subject']}") |
| 27 | time.sleep(0.1) # Simulate network delay |
| 28 | return {'sent_at': int(time.time()), 'to': payload['to']} |
| 29 | |
| 30 | def process_video_handler(payload): |
| 31 | """Simulate processing a video""" |
| 32 | print(f" 🎥 Processing video {payload['video_id']}") |
| 33 | time.sleep(0.2) # Simulate processing time |
| 34 | return {'output_url': f"https://cdn.example.com/videos/{payload['video_id']}.mp4"} |
| 35 | |
| 36 | def generate_report_handler(payload): |
| 37 | """Simulate generating a report""" |
| 38 | print(f" 📊 Generating report for user {payload['user_id']}") |
| 39 | time.sleep(0.15) # Simulate processing time |
| 40 | return {'report_url': f"https://example.com/reports/{payload['user_id']}.pdf"} |
| 41 | |
| 42 | # Step 1: Queue some jobs |
| 43 | print("Step 1: Queueing jobs") |
| 44 | print("-" * 60) |
| 45 | |
| 46 | job_ids = [] |
| 47 | |
| 48 | job_id = enqueue('send_email', { |
| 49 | 'to': 'user@example.com', |
| 50 | 'subject': 'Welcome!', |
| 51 | 'body': 'Thanks for signing up.' |
| 52 | }) |
| 53 | print(f"✓ Queued email job: {job_id}") |
| 54 | job_ids.append(('email', job_id)) |
| 55 | |
| 56 | job_id = enqueue('process_video', { |
| 57 | 'video_id': 42 |
| 58 | }) |
| 59 | print(f"✓ Queued video job: {job_id}") |
| 60 | job_ids.append(('video', job_id)) |
| 61 | |
| 62 | job_id = enqueue('generate_report', { |
| 63 | 'user_id': 99 |
| 64 | }) |
| 65 | print(f"✓ Queued report job: {job_id}") |
| 66 | job_ids.append(('report', job_id)) |
| 67 | |
| 68 | print() |
| 69 | |
| 70 | # Step 2: Check job status before processing |
| 71 | print("Step 2: Check job status (before processing)") |
| 72 | print("-" * 60) |
| 73 | |
| 74 | for name, job_id in job_ids: |
| 75 | job = get_job(job_id) |
| 76 | print(f" {name}: status={job['status']}, attempts={job['attempts']}") |
| 77 | |
| 78 | print() |
| 79 | |
| 80 | # Step 3: Process jobs |
| 81 | print("Step 3: Processing jobs") |
| 82 | print("-" * 60) |
| 83 | |
| 84 | handlers = { |
| 85 | 'send_email': send_email_handler, |
| 86 | 'process_video': process_video_handler, |
| 87 | 'generate_report': generate_report_handler, |
| 88 | } |
| 89 | |
| 90 | process_jobs(handlers, max_attempts=3) |
| 91 | |
| 92 | print() |
| 93 | |
| 94 | # Step 4: Check job status after processing |
| 95 | print("Step 4: Check job status (after processing)") |
| 96 | print("-" * 60) |
| 97 | |
| 98 | for name, job_id in job_ids: |
| 99 | job = get_job(job_id) |
| 100 | print(f" {name}: status={job['status']}, attempts={job['attempts']}") |
| 101 | if job['result']: |
| 102 | print(f" result: {job['result']}") |
| 103 | |
| 104 | print() |
| 105 | |
| 106 | # Step 5: Demonstrate failure and retry |
| 107 | print("Step 5: Demonstrate failure and retry") |
| 108 | print("-" * 60) |
| 109 | |
| 110 | def failing_handler(payload): |
| 111 | """Handler that fails on first two attempts""" |
| 112 | job = get_job(payload['job_id']) |
| 113 | attempts = job['attempts'] if job else 0 |
| 114 | |
| 115 | if attempts < 2: |
| 116 | print(f" ❌ Attempt {attempts + 1} failed") |
| 117 | raise Exception("Simulated failure") |
| 118 | else: |
| 119 | print(f" ✓ Attempt {attempts + 1} succeeded!") |
| 120 | return {'success': True} |
| 121 | |
| 122 | # Queue a job that will fail twice |
| 123 | fail_job_id = enqueue('failing_task', {'job_id': 'placeholder'}) |
| 124 | |
| 125 | # Update payload with actual job_id (normally you wouldn't do this) |
| 126 | import dbbasic_queue.queue as queue_module |
| 127 | rows = queue_module._read_tsv() |
| 128 | for i, r in enumerate(rows): |
| 129 | if r[0] == fail_job_id: |
| 130 | rows[i][2] = f'{{"job_id": "{fail_job_id}"}}' |
| 131 | queue_module._write_tsv(rows) |
| 132 | |
| 133 | handlers['failing_task'] = failing_handler |
| 134 | |
| 135 | # Try processing 3 times |
| 136 | for attempt in range(1, 4): |
| 137 | print(f"\nProcessing attempt {attempt}:") |
| 138 | process_jobs(handlers, max_attempts=3) |
| 139 | |
| 140 | # Reset run_at for next attempt (for demo purposes) |
| 141 | if attempt < 3: |
| 142 | job = get_job(fail_job_id) |
| 143 | if job and job['status'] == 'pending': |
| 144 | rows = queue_module._read_tsv() |
| 145 | for i, r in enumerate(rows): |
| 146 | if r[0] == fail_job_id: |
| 147 | rows[i][5] = str(int(time.time())) |
| 148 | queue_module._write_tsv(rows) |
| 149 | |
| 150 | final_job = get_job(fail_job_id) |
| 151 | print(f"\nFinal status: {final_job['status']}, attempts: {final_job['attempts']}") |
| 152 | |
| 153 | print() |
| 154 | print("=" * 60) |
| 155 | print("Demo completed!") |
| 156 | print() |
| 157 | print("The queue file is at: data/queue.tsv") |
| 158 | print("You can inspect it with: cat data/queue.tsv") |
| 159 | print("=" * 60) |