askrobots / dbbasic Public

Model-config web apps: describe the app in configuration instead of writing code. Built on Polars and DuckDB.

0 downloads

dbbasic/dbbasic_channels.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Add DBBasic crate engine logo and update README 0d628db 1 year, 2 months ago History
dbbasic_channels.py · 14.9 KB · 440 lines
1#!/usr/bin/env python3
2"""
3DBBasic Channels - Pub/Sub system for service orchestration
4Workers watch channels, pick up jobs, process them, and broadcast results
5Built on DuckDB for persistence + async for real-time
6"""
7
8import asyncio
9import json
10import uuid
11from datetime import datetime
12from typing import Dict, Any, List, Optional, Callable
13from collections import defaultdict
14import duckdb
15from enum import Enum
16
17class Channel(Enum):
18 """Standard channels for different job types"""
19 ORDERS = "orders"
20 SHIPPING = "shipping"
21 FRAUD = "fraud"
22 NOTIFICATIONS = "notifications"
23 ANALYTICS = "analytics"
24 CUSTOMER = "customer"
25 INVENTORY = "inventory"
26 PRICING = "pricing"
27 ALL = "*" # Special channel for monitoring everything
28
29class DBBasicChannels:
30 """
31 Pub/Sub system with channels for service orchestration.
32 - Publishers push jobs to channels
33 - Subscribers watch channels and process jobs
34 - Results broadcast back to interested listeners
35 """
36
37 def __init__(self, db_path: str = "dbbasic_channels.duckdb"):
38 self.db_path = db_path
39 self.conn = duckdb.connect(db_path)
40 self._init_db()
41
42 # In-memory subscriptions (channel -> list of callbacks)
43 self.subscribers: Dict[str, List[Callable]] = defaultdict(list)
44
45 # Active watchers (channel -> list of async tasks)
46 self.watchers: Dict[str, List[asyncio.Task]] = defaultdict(list)
47
48 # Event for new messages
49 self.new_message_event = asyncio.Event()
50
51 def _init_db(self):
52 """Initialize channel tables"""
53
54 # Messages table - the main pub/sub queue
55 self.conn.execute("""
56 CREATE TABLE IF NOT EXISTS messages (
57 id VARCHAR PRIMARY KEY,
58 channel VARCHAR NOT NULL,
59 message_type VARCHAR NOT NULL,
60 payload JSON,
61 status VARCHAR DEFAULT 'pending',
62 created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
63 processed_at TIMESTAMP,
64 processed_by VARCHAR,
65 result JSON,
66 error VARCHAR,
67 ttl_seconds INTEGER DEFAULT 3600
68 )
69 """)
70
71 # Channel registry
72 self.conn.execute("""
73 CREATE TABLE IF NOT EXISTS channels (
74 name VARCHAR PRIMARY KEY,
75 description VARCHAR,
76 created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
77 message_count INTEGER DEFAULT 0,
78 last_message_at TIMESTAMP,
79 active_subscribers INTEGER DEFAULT 0
80 )
81 """)
82
83 # Subscribers tracking
84 self.conn.execute("""
85 CREATE TABLE IF NOT EXISTS subscribers (
86 id VARCHAR PRIMARY KEY,
87 channel VARCHAR NOT NULL,
88 subscriber_name VARCHAR NOT NULL,
89 subscribed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
90 messages_processed INTEGER DEFAULT 0,
91 last_seen TIMESTAMP
92 )
93 """)
94
95 # Create indexes for performance
96 self.conn.execute("""
97 CREATE INDEX IF NOT EXISTS idx_messages_channel_status
98 ON messages(channel, status, created_at)
99 """)
100
101 # Initialize standard channels
102 for channel in Channel:
103 if channel != Channel.ALL:
104 self.conn.execute("""
105 INSERT OR IGNORE INTO channels (name, description)
106 VALUES (?, ?)
107 """, [channel.value, f"Channel for {channel.value} related messages"])
108
109 async def publish(self, channel: str, message_type: str, payload: Dict[str, Any],
110 ttl_seconds: int = 3600) -> str:
111 """
112 Publish a message to a channel.
113 Returns message ID.
114 """
115 message_id = str(uuid.uuid4())
116
117 self.conn.execute("""
118 INSERT INTO messages (id, channel, message_type, payload, ttl_seconds)
119 VALUES (?, ?, ?, ?, ?)
120 """, [message_id, channel, message_type, json.dumps(payload), ttl_seconds])
121
122 # Update channel stats
123 self.conn.execute("""
124 UPDATE channels
125 SET message_count = message_count + 1,
126 last_message_at = CURRENT_TIMESTAMP
127 WHERE name = ?
128 """, [channel])
129
130 # Notify watchers
131 self.new_message_event.set()
132
133 # Trigger in-memory subscribers
134 await self._trigger_subscribers(channel, {
135 'id': message_id,
136 'channel': channel,
137 'message_type': message_type,
138 'payload': payload,
139 'timestamp': datetime.now().isoformat()
140 })
141
142 print(f"📤 Published to {channel}: {message_type} (ID: {message_id[:8]}...)")
143 return message_id
144
145 async def subscribe(self, channel: str, callback: Callable,
146 subscriber_name: Optional[str] = None) -> str:
147 """
148 Subscribe to a channel with a callback function.
149 Returns subscriber ID.
150 """
151 subscriber_id = str(uuid.uuid4())
152 subscriber_name = subscriber_name or f"subscriber_{subscriber_id[:8]}"
153
154 # Register in database
155 self.conn.execute("""
156 INSERT INTO subscribers (id, channel, subscriber_name)
157 VALUES (?, ?, ?)
158 """, [subscriber_id, channel, subscriber_name])
159
160 # Add to in-memory subscribers
161 self.subscribers[channel].append(callback)
162
163 # Update channel subscriber count
164 self.conn.execute("""
165 UPDATE channels
166 SET active_subscribers = (
167 SELECT COUNT(*) FROM subscribers WHERE channel = ?
168 )
169 WHERE name = ?
170 """, [channel, channel])
171
172 print(f"📥 {subscriber_name} subscribed to {channel}")
173 return subscriber_id
174
175 async def watch_channel(self, channel: str, worker_name: str,
176 processor: Optional[Callable] = None):
177 """
178 Watch a channel and process incoming messages.
179 This is for workers that pull and process jobs.
180 """
181 print(f"👁️ {worker_name} watching channel: {channel}")
182
183 while True:
184 try:
185 # Get next pending message from channel
186 result = self.conn.execute("""
187 UPDATE messages
188 SET status = 'processing',
189 processed_by = ?,
190 processed_at = CURRENT_TIMESTAMP
191 WHERE id = (
192 SELECT id FROM messages
193 WHERE channel = ?
194 AND status = 'pending'
195 AND (created_at + INTERVAL '1 second' * ttl_seconds) > CURRENT_TIMESTAMP
196 ORDER BY created_at
197 LIMIT 1
198 )
199 RETURNING *
200 """, [worker_name, channel]).fetchone()
201
202 if result:
203 message = {
204 'id': result[0],
205 'channel': result[1],
206 'message_type': result[2],
207 'payload': json.loads(result[3]) if result[3] else {}
208 }
209
210 # Process the message
211 if processor:
212 try:
213 result_data = await processor(message)
214
215 # Mark as completed
216 self.conn.execute("""
217 UPDATE messages
218 SET status = 'completed',
219 result = ?
220 WHERE id = ?
221 """, [json.dumps(result_data) if result_data else None, message['id']])
222
223 # Update subscriber stats
224 self.conn.execute("""
225 UPDATE subscribers
226 SET messages_processed = messages_processed + 1,
227 last_seen = CURRENT_TIMESTAMP
228 WHERE subscriber_name = ?
229 """, [worker_name])
230
231 # Broadcast completion
232 await self._broadcast_completion(channel, message, result_data)
233
234 except Exception as e:
235 # Mark as failed
236 self.conn.execute("""
237 UPDATE messages
238 SET status = 'failed',
239 error = ?
240 WHERE id = ?
241 """, [str(e), message['id']])
242
243 print(f"❌ Error processing {message['id']}: {e}")
244
245 else:
246 # No messages, wait for signal
247 await asyncio.wait_for(
248 self.new_message_event.wait(),
249 timeout=1.0
250 )
251 self.new_message_event.clear()
252
253 except asyncio.TimeoutError:
254 # Periodic check even without new message signal
255 continue
256 except Exception as e:
257 print(f"❌ Watcher error: {e}")
258 await asyncio.sleep(1)
259
260 async def _trigger_subscribers(self, channel: str, message: Dict):
261 """Trigger all in-memory subscribers for a channel"""
262 # Direct channel subscribers
263 for callback in self.subscribers.get(channel, []):
264 try:
265 await callback(message)
266 except Exception as e:
267 print(f"❌ Subscriber error: {e}")
268
269 # Wildcard subscribers (monitoring all channels)
270 for callback in self.subscribers.get("*", []):
271 try:
272 await callback(message)
273 except Exception as e:
274 print(f"❌ Wildcard subscriber error: {e}")
275
276 async def _broadcast_completion(self, channel: str, message: Dict,
277 result: Any):
278 """Broadcast completion to interested listeners"""
279 completion_event = {
280 'type': 'completion',
281 'channel': channel,
282 'message_id': message['id'],
283 'message_type': message['message_type'],
284 'result': result,
285 'timestamp': datetime.now().isoformat()
286 }
287
288 await self._trigger_subscribers(channel, completion_event)
289
290 def get_channel_stats(self) -> Dict[str, Any]:
291 """Get statistics for all channels"""
292 stats = self.conn.execute("""
293 SELECT
294 c.name,
295 c.message_count,
296 c.active_subscribers,
297 COUNT(m.id) FILTER (WHERE m.status = 'pending') as pending,
298 COUNT(m.id) FILTER (WHERE m.status = 'processing') as processing,
299 COUNT(m.id) FILTER (WHERE m.status = 'completed') as completed,
300 COUNT(m.id) FILTER (WHERE m.status = 'failed') as failed
301 FROM channels c
302 LEFT JOIN messages m ON c.name = m.channel
303 GROUP BY c.name, c.message_count, c.active_subscribers
304 """).fetchall()
305
306 return {
307 row[0]: {
308 'total_messages': row[1],
309 'active_subscribers': row[2],
310 'pending': row[3] or 0,
311 'processing': row[4] or 0,
312 'completed': row[5] or 0,
313 'failed': row[6] or 0
314 }
315 for row in stats
316 }
317
318 async def cleanup_old_messages(self, max_age_seconds: int = 3600):
319 """Clean up old processed messages"""
320 deleted = self.conn.execute("""
321 DELETE FROM messages
322 WHERE status IN ('completed', 'failed')
323 AND (created_at + INTERVAL '1 second' * ?) < CURRENT_TIMESTAMP
324 RETURNING COUNT(*)
325 """, [max_age_seconds]).fetchone()
326
327 if deleted and deleted[0] > 0:
328 print(f"🧹 Cleaned up {deleted[0]} old messages")
329
330
331# Example worker processors for different channels
332async def process_order(message: Dict) -> Dict:
333 """Process order messages"""
334 payload = message['payload']
335 # Simulate order processing
336 await asyncio.sleep(0.01)
337 return {
338 'order_id': payload.get('order_id'),
339 'status': 'processed',
340 'total': payload.get('total', 0) * 1.1 # Add tax
341 }
342
343async def process_shipping(message: Dict) -> Dict:
344 """Process shipping messages"""
345 payload = message['payload']
346 # Simulate shipping calculation
347 await asyncio.sleep(0.01)
348 return {
349 'tracking_number': f"TRACK{uuid.uuid4().hex[:8].upper()}",
350 'carrier': 'UPS',
351 'estimated_days': 3
352 }
353
354async def process_fraud(message: Dict) -> Dict:
355 """Process fraud detection messages"""
356 payload = message['payload']
357 # Simulate fraud check
358 await asyncio.sleep(0.02)
359 risk_score = 25 if payload.get('amount', 0) < 1000 else 75
360 return {
361 'risk_score': risk_score,
362 'action': 'approve' if risk_score < 50 else 'review'
363 }
364
365
366# Demo
367async def demo_channels():
368 """Demonstrate the channel system"""
369
370 channels = DBBasicChannels()
371
372 print("\n🚀 DBBasic Channels Demo\n")
373
374 # Start workers watching different channels
375 print("Starting channel watchers...")
376
377 # Order processor
378 asyncio.create_task(
379 channels.watch_channel(Channel.ORDERS.value, "OrderWorker-1", process_order)
380 )
381
382 # Shipping processor
383 asyncio.create_task(
384 channels.watch_channel(Channel.SHIPPING.value, "ShippingWorker-1", process_shipping)
385 )
386
387 # Fraud processor
388 asyncio.create_task(
389 channels.watch_channel(Channel.FRAUD.value, "FraudWorker-1", process_fraud)
390 )
391
392 # Monitor all channels
393 async def monitor_callback(message):
394 print(f"📊 Monitor: {message.get('channel')} - {message.get('message_type')}")
395
396 await channels.subscribe("*", monitor_callback, "GlobalMonitor")
397
398 # Give workers time to start
399 await asyncio.sleep(1)
400
401 # Publish some messages
402 print("\nPublishing messages to channels...")
403
404 # Publish order
405 await channels.publish(
406 Channel.ORDERS.value,
407 "new_order",
408 {"order_id": "ORD123", "total": 150.00, "customer_id": "CUST456"}
409 )
410
411 # Publish shipping request
412 await channels.publish(
413 Channel.SHIPPING.value,
414 "calculate_shipping",
415 {"weight": 2.5, "destination": "90210"}
416 )
417
418 # Publish fraud check
419 await channels.publish(
420 Channel.FRAUD.value,
421 "check_transaction",
422 {"transaction_id": "TXN789", "amount": 2500.00}
423 )
424
425 # Wait for processing
426 await asyncio.sleep(2)
427
428 # Show stats
429 print("\n📊 Channel Statistics:")
430 stats = channels.get_channel_stats()
431 for channel, data in stats.items():
432 if data['total_messages'] > 0:
433 print(f" {channel}: {data}")
434
435 # Keep running for a bit to show real-time processing
436 await asyncio.sleep(5)
437
438
439if __name__ == "__main__":
440 asyncio.run(demo_channels())