A job queue in TSV files for background tasks, with retries and exponential backoff.

0 downloads

dbbasic-queue/demo.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-queue.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-queue.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-queue.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Update README with PyPI information and badges 89cd41e 1 year, 1 month ago History
demo.py · 4.3 KB · 159 lines
1#!/usr/bin/env python3
2"""
3Demo script showing dbbasic-queue in action
4
5Run this to see the queue system working end-to-end.
6"""
7
8import sys
9import os
10import time
11
12# Add current directory to path
13sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
14
15from dbbasic_queue import enqueue, process_jobs, get_job
16
17print("=" * 60)
18print("DBBasic Queue Demo")
19print("=" * 60)
20print()
21
22# Define some example handlers
23def send_email_handler(payload):
24 """Simulate sending an email"""
25 print(f" 📧 Sending email to {payload['to']}")
26 print(f" Subject: {payload['subject']}")
27 time.sleep(0.1) # Simulate network delay
28 return {'sent_at': int(time.time()), 'to': payload['to']}
29
30def process_video_handler(payload):
31 """Simulate processing a video"""
32 print(f" 🎥 Processing video {payload['video_id']}")
33 time.sleep(0.2) # Simulate processing time
34 return {'output_url': f"https://cdn.example.com/videos/{payload['video_id']}.mp4"}
35
36def generate_report_handler(payload):
37 """Simulate generating a report"""
38 print(f" 📊 Generating report for user {payload['user_id']}")
39 time.sleep(0.15) # Simulate processing time
40 return {'report_url': f"https://example.com/reports/{payload['user_id']}.pdf"}
41
42# Step 1: Queue some jobs
43print("Step 1: Queueing jobs")
44print("-" * 60)
45
46job_ids = []
47
48job_id = enqueue('send_email', {
49 'to': 'user@example.com',
50 'subject': 'Welcome!',
51 'body': 'Thanks for signing up.'
52})
53print(f"✓ Queued email job: {job_id}")
54job_ids.append(('email', job_id))
55
56job_id = enqueue('process_video', {
57 'video_id': 42
58})
59print(f"✓ Queued video job: {job_id}")
60job_ids.append(('video', job_id))
61
62job_id = enqueue('generate_report', {
63 'user_id': 99
64})
65print(f"✓ Queued report job: {job_id}")
66job_ids.append(('report', job_id))
67
68print()
69
70# Step 2: Check job status before processing
71print("Step 2: Check job status (before processing)")
72print("-" * 60)
73
74for name, job_id in job_ids:
75 job = get_job(job_id)
76 print(f" {name}: status={job['status']}, attempts={job['attempts']}")
77
78print()
79
80# Step 3: Process jobs
81print("Step 3: Processing jobs")
82print("-" * 60)
83
84handlers = {
85 'send_email': send_email_handler,
86 'process_video': process_video_handler,
87 'generate_report': generate_report_handler,
88}
89
90process_jobs(handlers, max_attempts=3)
91
92print()
93
94# Step 4: Check job status after processing
95print("Step 4: Check job status (after processing)")
96print("-" * 60)
97
98for name, job_id in job_ids:
99 job = get_job(job_id)
100 print(f" {name}: status={job['status']}, attempts={job['attempts']}")
101 if job['result']:
102 print(f" result: {job['result']}")
103
104print()
105
106# Step 5: Demonstrate failure and retry
107print("Step 5: Demonstrate failure and retry")
108print("-" * 60)
109
110def failing_handler(payload):
111 """Handler that fails on first two attempts"""
112 job = get_job(payload['job_id'])
113 attempts = job['attempts'] if job else 0
114
115 if attempts < 2:
116 print(f" ❌ Attempt {attempts + 1} failed")
117 raise Exception("Simulated failure")
118 else:
119 print(f" ✓ Attempt {attempts + 1} succeeded!")
120 return {'success': True}
121
122# Queue a job that will fail twice
123fail_job_id = enqueue('failing_task', {'job_id': 'placeholder'})
124
125# Update payload with actual job_id (normally you wouldn't do this)
126import dbbasic_queue.queue as queue_module
127rows = queue_module._read_tsv()
128for i, r in enumerate(rows):
129 if r[0] == fail_job_id:
130 rows[i][2] = f'{{"job_id": "{fail_job_id}"}}'
131queue_module._write_tsv(rows)
132
133handlers['failing_task'] = failing_handler
134
135# Try processing 3 times
136for attempt in range(1, 4):
137 print(f"\nProcessing attempt {attempt}:")
138 process_jobs(handlers, max_attempts=3)
139
140 # Reset run_at for next attempt (for demo purposes)
141 if attempt < 3:
142 job = get_job(fail_job_id)
143 if job and job['status'] == 'pending':
144 rows = queue_module._read_tsv()
145 for i, r in enumerate(rows):
146 if r[0] == fail_job_id:
147 rows[i][5] = str(int(time.time()))
148 queue_module._write_tsv(rows)
149
150final_job = get_job(fail_job_id)
151print(f"\nFinal status: {final_job['status']}, attempts: {final_job['attempts']}")
152
153print()
154print("=" * 60)
155print("Demo completed!")
156print()
157print("The queue file is at: data/queue.tsv")
158print("You can inspect it with: cat data/queue.tsv")
159print("=" * 60)