Python functions as REST objects, with clustering, load balancing and state replication; no containers or orchestration.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-primitive.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Q Consolidate duplicate QUICKSTARTs on the top-level file 503f830 7 months ago History
queue.py · 9.4 KB · 346 lines
1"""
2Queue Object - Message queue system for Object Primitives
3
4This object provides message queue capabilities for async task processing.
5Supports priority queues, visibility timeouts, and dead letter queues.
6
7Example usage:
8 # Enqueue message
9 POST /objects/queue
10 {
11 "queue_name": "email_queue",
12 "message": {"to": "user@example.com", "subject": "Welcome"},
13 "priority": "high"
14 }
15
16 # Dequeue message (get next message)
17 GET /objects/queue?queue_name=email_queue
18
19 # Acknowledge message (mark as completed)
20 DELETE /objects/queue?message_id=abc123
21
22 # Requeue message (return to queue on failure)
23 PUT /objects/queue?message_id=abc123
24
25 # Get queue status
26 GET /objects/queue?queue_name=email_queue&status=true
27"""
28
29import time
30import json
31import secrets
32from typing import Any, Dict, List, Optional
33
34# These will be injected by ObjectRuntime
35_logger = None
36_state_manager = None
37
38# Priority levels (higher number = higher priority)
39PRIORITY_LEVELS = {
40 'critical': 4,
41 'high': 3,
42 'normal': 2,
43 'low': 1,
44}
45
46# Default visibility timeout (seconds)
47DEFAULT_VISIBILITY_TIMEOUT = 300 # 5 minutes
48
49# Default message TTL (seconds)
50DEFAULT_TTL = 86400 # 24 hours
51
52
53def POST(request: Dict[str, Any]) -> Dict[str, Any]:
54 """Enqueue message"""
55 queue_name = request.get('queue_name', '').strip()
56 message = request.get('message', {})
57 priority = request.get('priority', 'normal').lower()
58 ttl = request.get('ttl', DEFAULT_TTL)
59
60 if not queue_name:
61 return {'status': 'error', 'message': 'queue_name is required'}
62
63 if priority not in PRIORITY_LEVELS:
64 return {'status': 'error', 'message': f'Invalid priority. Must be one of: {list(PRIORITY_LEVELS.keys())}'}
65
66 # Create message
67 message_id = secrets.token_hex(8)
68 timestamp = int(time.time())
69
70 queue_message = {
71 'id': message_id,
72 'queue_name': queue_name,
73 'message': message,
74 'priority': priority,
75 'priority_level': PRIORITY_LEVELS[priority],
76 'status': 'pending',
77 'created_at': timestamp,
78 'visible_after': timestamp, # When message becomes visible
79 'expires_at': timestamp + ttl,
80 'attempts': 0,
81 'max_attempts': 3,
82 }
83
84 # Save to queue
85 _save_message(queue_message)
86
87 if _logger:
88 _logger.info('Message enqueued',
89 message_id=message_id,
90 queue_name=queue_name,
91 priority=priority)
92
93 return {
94 'status': 'ok',
95 'message_id': message_id,
96 'queue_name': queue_name,
97 'message': 'Message enqueued',
98 }
99
100
101def GET(request: Dict[str, Any]) -> Dict[str, Any]:
102 """Dequeue message or get queue status"""
103 queue_name = request.get('queue_name', '').strip()
104
105 if not queue_name:
106 return {'status': 'error', 'message': 'queue_name is required'}
107
108 # Get queue status
109 if request.get('status'):
110 return _get_queue_status(queue_name)
111
112 # Dequeue message
113 visibility_timeout = request.get('visibility_timeout', DEFAULT_VISIBILITY_TIMEOUT)
114
115 try:
116 visibility_timeout = int(visibility_timeout)
117 except (ValueError, TypeError):
118 visibility_timeout = DEFAULT_VISIBILITY_TIMEOUT
119
120 message = _dequeue_message(queue_name, visibility_timeout)
121
122 if not message:
123 return {
124 'status': 'ok',
125 'message': 'No messages available',
126 'queue_name': queue_name,
127 }
128
129 if _logger:
130 _logger.info('Message dequeued',
131 message_id=message['id'],
132 queue_name=queue_name,
133 priority=message['priority'])
134
135 return {
136 'status': 'ok',
137 'message': message,
138 }
139
140
141def DELETE(request: Dict[str, Any]) -> Dict[str, Any]:
142 """Acknowledge message (mark as completed and remove from queue)"""
143 message_id = request.get('message_id', '').strip()
144
145 if not message_id:
146 return {'status': 'error', 'message': 'message_id is required'}
147
148 message = _get_message(message_id)
149 if not message:
150 return {'status': 'error', 'message': f'Message not found: {message_id}'}
151
152 # Mark as completed
153 message['status'] = 'completed'
154 message['completed_at'] = int(time.time())
155 _save_message(message)
156
157 if _logger:
158 _logger.info('Message acknowledged',
159 message_id=message_id,
160 queue_name=message['queue_name'])
161
162 return {
163 'status': 'ok',
164 'message': f'Message acknowledged: {message_id}',
165 }
166
167
168def PUT(request: Dict[str, Any]) -> Dict[str, Any]:
169 """Requeue message (return to queue, increment attempts)"""
170 message_id = request.get('message_id', '').strip()
171 delay = request.get('delay', 0) # Delay before message becomes visible again
172
173 if not message_id:
174 return {'status': 'error', 'message': 'message_id is required'}
175
176 message = _get_message(message_id)
177 if not message:
178 return {'status': 'error', 'message': f'Message not found: {message_id}'}
179
180 try:
181 delay = int(delay)
182 except (ValueError, TypeError):
183 delay = 0
184
185 # Increment attempts
186 message['attempts'] += 1
187
188 # Check max attempts
189 if message['attempts'] >= message.get('max_attempts', 3):
190 # Move to dead letter queue
191 message['status'] = 'failed'
192 message['failed_at'] = int(time.time())
193 _save_message(message)
194
195 if _logger:
196 _logger.error('Message failed (max attempts exceeded)',
197 message_id=message_id,
198 attempts=message['attempts'])
199
200 return {
201 'status': 'error',
202 'message': f'Message failed after {message["attempts"]} attempts',
203 'message_id': message_id,
204 }
205
206 # Requeue message
207 message['status'] = 'pending'
208 message['visible_after'] = int(time.time()) + delay
209 _save_message(message)
210
211 if _logger:
212 _logger.info('Message requeued',
213 message_id=message_id,
214 attempts=message['attempts'],
215 delay=delay)
216
217 return {
218 'status': 'ok',
219 'message': f'Message requeued: {message_id}',
220 'attempts': message['attempts'],
221 }
222
223
224# Helper functions
225
226def _save_message(message: Dict[str, Any]) -> None:
227 """Save message to state"""
228 if not _state_manager:
229 return
230
231 message_id = message['id']
232 queue_name = message['queue_name']
233 priority_level = message['priority_level']
234 created_at = message['created_at']
235
236 # Store with composite key for sorting: queue_priority_timestamp_id
237 # This ensures messages are retrieved in priority order, then FIFO
238 key = f'msg_{queue_name}_{priority_level}_{created_at}_{message_id}'
239 _state_manager.set(key, json.dumps(message))
240
241
242def _get_message(message_id: str) -> Optional[Dict[str, Any]]:
243 """Get message by ID"""
244 if not _state_manager:
245 return None
246
247 all_state = _state_manager.get_all()
248
249 for key, value in all_state.items():
250 if key.startswith('msg_') and message_id in key:
251 message = json.loads(value)
252 if message['id'] == message_id:
253 return message
254
255 return None
256
257
258def _dequeue_message(queue_name: str, visibility_timeout: int) -> Optional[Dict[str, Any]]:
259 """Get next available message from queue"""
260 if not _state_manager:
261 return None
262
263 now = int(time.time())
264 all_state = _state_manager.get_all()
265
266 # Find all pending messages for this queue
267 messages = []
268 prefix = f'msg_{queue_name}_'
269
270 for key, value in all_state.items():
271 if key.startswith(prefix):
272 message = json.loads(value)
273
274 # Only pending messages
275 if message['status'] != 'pending':
276 continue
277
278 # Check expiration
279 if message['expires_at'] < now:
280 message['status'] = 'expired'
281 _save_message(message)
282 continue
283
284 # Check visibility
285 if message['visible_after'] > now:
286 continue
287
288 messages.append(message)
289
290 # No messages available
291 if not messages:
292 return None
293
294 # Sort by priority (highest first), then timestamp (oldest first)
295 messages.sort(key=lambda m: (-m['priority_level'], m['created_at']))
296
297 # Get first message
298 message = messages[0]
299
300 # Mark as processing and set visibility timeout
301 message['status'] = 'processing'
302 message['visible_after'] = now + visibility_timeout
303 message['dequeued_at'] = now
304 _save_message(message)
305
306 return message
307
308
309def _get_queue_status(queue_name: str) -> Dict[str, Any]:
310 """Get queue status (counts by status)"""
311 if not _state_manager:
312 return {
313 'status': 'ok',
314 'queue_name': queue_name,
315 'total': 0,
316 'pending': 0,
317 'processing': 0,
318 'completed': 0,
319 'failed': 0,
320 }
321
322 all_state = _state_manager.get_all()
323 prefix = f'msg_{queue_name}_'
324
325 counts = {
326 'total': 0,
327 'pending': 0,
328 'processing': 0,
329 'completed': 0,
330 'failed': 0,
331 'expired': 0,
332 }
333
334 for key, value in all_state.items():
335 if key.startswith(prefix):
336 message = json.loads(value)
337 counts['total'] += 1
338 status = message.get('status', 'unknown')
339 if status in counts:
340 counts[status] += 1
341
342 return {
343 'status': 'ok',
344 'queue_name': queue_name,
345 **counts,
346 }