A job queue in TSV files for background tasks, with retries and exponential backoff.

0 downloads

dbbasic-queue/workers/queue_worker.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-queue.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-queue.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-queue.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Update README with PyPI information and badges 89cd41e 1 year, 1 month ago History
queue_worker.py · 3.0 KB · 116 lines
1#!/usr/bin/env python3
2"""
3Example worker script for processing queue jobs
4
5This script should be run by cron every minute:
6* * * * * cd /app && python3 workers/queue_worker.py >> /var/log/queue.log 2>&1
7"""
8
9import sys
10import os
11import time
12
13# Add parent directory to path so we can import dbbasic_queue
14sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
15
16from dbbasic_queue import process_jobs
17
18
19def send_email_handler(payload):
20 """
21 Send email via SMTP
22
23 Args:
24 payload (dict): Email parameters
25 - to: Recipient email address
26 - subject: Email subject
27 - body: Email body
28
29 Returns:
30 dict: Result with sent_at timestamp
31 """
32 print(f"Sending email to {payload['to']}: {payload['subject']}")
33
34 # In production, use smtplib to send real emails:
35 # import smtplib
36 # from email.message import EmailMessage
37 #
38 # msg = EmailMessage()
39 # msg['From'] = 'noreply@example.com'
40 # msg['To'] = payload['to']
41 # msg['Subject'] = payload['subject']
42 # msg.set_content(payload['body'])
43 #
44 # with smtplib.SMTP('localhost') as smtp:
45 # smtp.send_message(msg)
46
47 # For demo purposes, just simulate sending
48 time.sleep(0.1) # Simulate network delay
49
50 return {'sent_at': int(time.time()), 'to': payload['to']}
51
52
53def process_video_handler(payload):
54 """
55 Process video (example)
56
57 Args:
58 payload (dict): Video parameters
59 - video_id: ID of video to process
60
61 Returns:
62 dict: Result with output URL
63 """
64 video_id = payload['video_id']
65 print(f"Processing video {video_id}")
66
67 # In production, use ffmpeg or similar:
68 # subprocess.run(['ffmpeg', '-i', input_path, '-c:v', 'libx264', output_path])
69
70 # For demo purposes, just simulate processing
71 time.sleep(1)
72
73 return {
74 'output_url': f'https://cdn.example.com/videos/{video_id}.mp4',
75 'processed_at': int(time.time())
76 }
77
78
79def generate_report_handler(payload):
80 """
81 Generate report (example)
82
83 Args:
84 payload (dict): Report parameters
85 - user_id: User ID to generate report for
86
87 Returns:
88 dict: Result with report URL
89 """
90 user_id = payload['user_id']
91 print(f"Generating report for user {user_id}")
92
93 # In production, generate actual report
94 time.sleep(0.5)
95
96 return {
97 'report_url': f'https://example.com/reports/{user_id}.pdf',
98 'generated_at': int(time.time())
99 }
100
101
102if __name__ == '__main__':
103 print(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting queue worker...")
104
105 handlers = {
106 'send_email': send_email_handler,
107 'process_video': process_video_handler,
108 'generate_report': generate_report_handler,
109 }
110
111 try:
112 process_jobs(handlers, max_attempts=3)
113 print(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Queue worker finished")
114 except Exception as e:
115 print(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: {e}")
116 sys.exit(1)