Model-config web apps: describe the app in configuration instead of writing code. Built on Polars and DuckDB.
dbbasic_channels.py · 14.9 KB · 440 lines
| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | DBBasic Channels - Pub/Sub system for service orchestration |
| 4 | Workers watch channels, pick up jobs, process them, and broadcast results |
| 5 | Built on DuckDB for persistence + async for real-time |
| 6 | """ |
| 7 | |
| 8 | import asyncio |
| 9 | import json |
| 10 | import uuid |
| 11 | from datetime import datetime |
| 12 | from typing import Dict, Any, List, Optional, Callable |
| 13 | from collections import defaultdict |
| 14 | import duckdb |
| 15 | from enum import Enum |
| 16 | |
| 17 | class Channel(Enum): |
| 18 | """Standard channels for different job types""" |
| 19 | ORDERS = "orders" |
| 20 | SHIPPING = "shipping" |
| 21 | FRAUD = "fraud" |
| 22 | NOTIFICATIONS = "notifications" |
| 23 | ANALYTICS = "analytics" |
| 24 | CUSTOMER = "customer" |
| 25 | INVENTORY = "inventory" |
| 26 | PRICING = "pricing" |
| 27 | ALL = "*" # Special channel for monitoring everything |
| 28 | |
| 29 | class DBBasicChannels: |
| 30 | """ |
| 31 | Pub/Sub system with channels for service orchestration. |
| 32 | - Publishers push jobs to channels |
| 33 | - Subscribers watch channels and process jobs |
| 34 | - Results broadcast back to interested listeners |
| 35 | """ |
| 36 | |
| 37 | def __init__(self, db_path: str = "dbbasic_channels.duckdb"): |
| 38 | self.db_path = db_path |
| 39 | self.conn = duckdb.connect(db_path) |
| 40 | self._init_db() |
| 41 | |
| 42 | # In-memory subscriptions (channel -> list of callbacks) |
| 43 | self.subscribers: Dict[str, List[Callable]] = defaultdict(list) |
| 44 | |
| 45 | # Active watchers (channel -> list of async tasks) |
| 46 | self.watchers: Dict[str, List[asyncio.Task]] = defaultdict(list) |
| 47 | |
| 48 | # Event for new messages |
| 49 | self.new_message_event = asyncio.Event() |
| 50 | |
| 51 | def _init_db(self): |
| 52 | """Initialize channel tables""" |
| 53 | |
| 54 | # Messages table - the main pub/sub queue |
| 55 | self.conn.execute(""" |
| 56 | CREATE TABLE IF NOT EXISTS messages ( |
| 57 | id VARCHAR PRIMARY KEY, |
| 58 | channel VARCHAR NOT NULL, |
| 59 | message_type VARCHAR NOT NULL, |
| 60 | payload JSON, |
| 61 | status VARCHAR DEFAULT 'pending', |
| 62 | created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, |
| 63 | processed_at TIMESTAMP, |
| 64 | processed_by VARCHAR, |
| 65 | result JSON, |
| 66 | error VARCHAR, |
| 67 | ttl_seconds INTEGER DEFAULT 3600 |
| 68 | ) |
| 69 | """) |
| 70 | |
| 71 | # Channel registry |
| 72 | self.conn.execute(""" |
| 73 | CREATE TABLE IF NOT EXISTS channels ( |
| 74 | name VARCHAR PRIMARY KEY, |
| 75 | description VARCHAR, |
| 76 | created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, |
| 77 | message_count INTEGER DEFAULT 0, |
| 78 | last_message_at TIMESTAMP, |
| 79 | active_subscribers INTEGER DEFAULT 0 |
| 80 | ) |
| 81 | """) |
| 82 | |
| 83 | # Subscribers tracking |
| 84 | self.conn.execute(""" |
| 85 | CREATE TABLE IF NOT EXISTS subscribers ( |
| 86 | id VARCHAR PRIMARY KEY, |
| 87 | channel VARCHAR NOT NULL, |
| 88 | subscriber_name VARCHAR NOT NULL, |
| 89 | subscribed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, |
| 90 | messages_processed INTEGER DEFAULT 0, |
| 91 | last_seen TIMESTAMP |
| 92 | ) |
| 93 | """) |
| 94 | |
| 95 | # Create indexes for performance |
| 96 | self.conn.execute(""" |
| 97 | CREATE INDEX IF NOT EXISTS idx_messages_channel_status |
| 98 | ON messages(channel, status, created_at) |
| 99 | """) |
| 100 | |
| 101 | # Initialize standard channels |
| 102 | for channel in Channel: |
| 103 | if channel != Channel.ALL: |
| 104 | self.conn.execute(""" |
| 105 | INSERT OR IGNORE INTO channels (name, description) |
| 106 | VALUES (?, ?) |
| 107 | """, [channel.value, f"Channel for {channel.value} related messages"]) |
| 108 | |
| 109 | async def publish(self, channel: str, message_type: str, payload: Dict[str, Any], |
| 110 | ttl_seconds: int = 3600) -> str: |
| 111 | """ |
| 112 | Publish a message to a channel. |
| 113 | Returns message ID. |
| 114 | """ |
| 115 | message_id = str(uuid.uuid4()) |
| 116 | |
| 117 | self.conn.execute(""" |
| 118 | INSERT INTO messages (id, channel, message_type, payload, ttl_seconds) |
| 119 | VALUES (?, ?, ?, ?, ?) |
| 120 | """, [message_id, channel, message_type, json.dumps(payload), ttl_seconds]) |
| 121 | |
| 122 | # Update channel stats |
| 123 | self.conn.execute(""" |
| 124 | UPDATE channels |
| 125 | SET message_count = message_count + 1, |
| 126 | last_message_at = CURRENT_TIMESTAMP |
| 127 | WHERE name = ? |
| 128 | """, [channel]) |
| 129 | |
| 130 | # Notify watchers |
| 131 | self.new_message_event.set() |
| 132 | |
| 133 | # Trigger in-memory subscribers |
| 134 | await self._trigger_subscribers(channel, { |
| 135 | 'id': message_id, |
| 136 | 'channel': channel, |
| 137 | 'message_type': message_type, |
| 138 | 'payload': payload, |
| 139 | 'timestamp': datetime.now().isoformat() |
| 140 | }) |
| 141 | |
| 142 | print(f"📤 Published to {channel}: {message_type} (ID: {message_id[:8]}...)") |
| 143 | return message_id |
| 144 | |
| 145 | async def subscribe(self, channel: str, callback: Callable, |
| 146 | subscriber_name: Optional[str] = None) -> str: |
| 147 | """ |
| 148 | Subscribe to a channel with a callback function. |
| 149 | Returns subscriber ID. |
| 150 | """ |
| 151 | subscriber_id = str(uuid.uuid4()) |
| 152 | subscriber_name = subscriber_name or f"subscriber_{subscriber_id[:8]}" |
| 153 | |
| 154 | # Register in database |
| 155 | self.conn.execute(""" |
| 156 | INSERT INTO subscribers (id, channel, subscriber_name) |
| 157 | VALUES (?, ?, ?) |
| 158 | """, [subscriber_id, channel, subscriber_name]) |
| 159 | |
| 160 | # Add to in-memory subscribers |
| 161 | self.subscribers[channel].append(callback) |
| 162 | |
| 163 | # Update channel subscriber count |
| 164 | self.conn.execute(""" |
| 165 | UPDATE channels |
| 166 | SET active_subscribers = ( |
| 167 | SELECT COUNT(*) FROM subscribers WHERE channel = ? |
| 168 | ) |
| 169 | WHERE name = ? |
| 170 | """, [channel, channel]) |
| 171 | |
| 172 | print(f"📥 {subscriber_name} subscribed to {channel}") |
| 173 | return subscriber_id |
| 174 | |
| 175 | async def watch_channel(self, channel: str, worker_name: str, |
| 176 | processor: Optional[Callable] = None): |
| 177 | """ |
| 178 | Watch a channel and process incoming messages. |
| 179 | This is for workers that pull and process jobs. |
| 180 | """ |
| 181 | print(f"👁️ {worker_name} watching channel: {channel}") |
| 182 | |
| 183 | while True: |
| 184 | try: |
| 185 | # Get next pending message from channel |
| 186 | result = self.conn.execute(""" |
| 187 | UPDATE messages |
| 188 | SET status = 'processing', |
| 189 | processed_by = ?, |
| 190 | processed_at = CURRENT_TIMESTAMP |
| 191 | WHERE id = ( |
| 192 | SELECT id FROM messages |
| 193 | WHERE channel = ? |
| 194 | AND status = 'pending' |
| 195 | AND (created_at + INTERVAL '1 second' * ttl_seconds) > CURRENT_TIMESTAMP |
| 196 | ORDER BY created_at |
| 197 | LIMIT 1 |
| 198 | ) |
| 199 | RETURNING * |
| 200 | """, [worker_name, channel]).fetchone() |
| 201 | |
| 202 | if result: |
| 203 | message = { |
| 204 | 'id': result[0], |
| 205 | 'channel': result[1], |
| 206 | 'message_type': result[2], |
| 207 | 'payload': json.loads(result[3]) if result[3] else {} |
| 208 | } |
| 209 | |
| 210 | # Process the message |
| 211 | if processor: |
| 212 | try: |
| 213 | result_data = await processor(message) |
| 214 | |
| 215 | # Mark as completed |
| 216 | self.conn.execute(""" |
| 217 | UPDATE messages |
| 218 | SET status = 'completed', |
| 219 | result = ? |
| 220 | WHERE id = ? |
| 221 | """, [json.dumps(result_data) if result_data else None, message['id']]) |
| 222 | |
| 223 | # Update subscriber stats |
| 224 | self.conn.execute(""" |
| 225 | UPDATE subscribers |
| 226 | SET messages_processed = messages_processed + 1, |
| 227 | last_seen = CURRENT_TIMESTAMP |
| 228 | WHERE subscriber_name = ? |
| 229 | """, [worker_name]) |
| 230 | |
| 231 | # Broadcast completion |
| 232 | await self._broadcast_completion(channel, message, result_data) |
| 233 | |
| 234 | except Exception as e: |
| 235 | # Mark as failed |
| 236 | self.conn.execute(""" |
| 237 | UPDATE messages |
| 238 | SET status = 'failed', |
| 239 | error = ? |
| 240 | WHERE id = ? |
| 241 | """, [str(e), message['id']]) |
| 242 | |
| 243 | print(f"❌ Error processing {message['id']}: {e}") |
| 244 | |
| 245 | else: |
| 246 | # No messages, wait for signal |
| 247 | await asyncio.wait_for( |
| 248 | self.new_message_event.wait(), |
| 249 | timeout=1.0 |
| 250 | ) |
| 251 | self.new_message_event.clear() |
| 252 | |
| 253 | except asyncio.TimeoutError: |
| 254 | # Periodic check even without new message signal |
| 255 | continue |
| 256 | except Exception as e: |
| 257 | print(f"❌ Watcher error: {e}") |
| 258 | await asyncio.sleep(1) |
| 259 | |
| 260 | async def _trigger_subscribers(self, channel: str, message: Dict): |
| 261 | """Trigger all in-memory subscribers for a channel""" |
| 262 | # Direct channel subscribers |
| 263 | for callback in self.subscribers.get(channel, []): |
| 264 | try: |
| 265 | await callback(message) |
| 266 | except Exception as e: |
| 267 | print(f"❌ Subscriber error: {e}") |
| 268 | |
| 269 | # Wildcard subscribers (monitoring all channels) |
| 270 | for callback in self.subscribers.get("*", []): |
| 271 | try: |
| 272 | await callback(message) |
| 273 | except Exception as e: |
| 274 | print(f"❌ Wildcard subscriber error: {e}") |
| 275 | |
| 276 | async def _broadcast_completion(self, channel: str, message: Dict, |
| 277 | result: Any): |
| 278 | """Broadcast completion to interested listeners""" |
| 279 | completion_event = { |
| 280 | 'type': 'completion', |
| 281 | 'channel': channel, |
| 282 | 'message_id': message['id'], |
| 283 | 'message_type': message['message_type'], |
| 284 | 'result': result, |
| 285 | 'timestamp': datetime.now().isoformat() |
| 286 | } |
| 287 | |
| 288 | await self._trigger_subscribers(channel, completion_event) |
| 289 | |
| 290 | def get_channel_stats(self) -> Dict[str, Any]: |
| 291 | """Get statistics for all channels""" |
| 292 | stats = self.conn.execute(""" |
| 293 | SELECT |
| 294 | c.name, |
| 295 | c.message_count, |
| 296 | c.active_subscribers, |
| 297 | COUNT(m.id) FILTER (WHERE m.status = 'pending') as pending, |
| 298 | COUNT(m.id) FILTER (WHERE m.status = 'processing') as processing, |
| 299 | COUNT(m.id) FILTER (WHERE m.status = 'completed') as completed, |
| 300 | COUNT(m.id) FILTER (WHERE m.status = 'failed') as failed |
| 301 | FROM channels c |
| 302 | LEFT JOIN messages m ON c.name = m.channel |
| 303 | GROUP BY c.name, c.message_count, c.active_subscribers |
| 304 | """).fetchall() |
| 305 | |
| 306 | return { |
| 307 | row[0]: { |
| 308 | 'total_messages': row[1], |
| 309 | 'active_subscribers': row[2], |
| 310 | 'pending': row[3] or 0, |
| 311 | 'processing': row[4] or 0, |
| 312 | 'completed': row[5] or 0, |
| 313 | 'failed': row[6] or 0 |
| 314 | } |
| 315 | for row in stats |
| 316 | } |
| 317 | |
| 318 | async def cleanup_old_messages(self, max_age_seconds: int = 3600): |
| 319 | """Clean up old processed messages""" |
| 320 | deleted = self.conn.execute(""" |
| 321 | DELETE FROM messages |
| 322 | WHERE status IN ('completed', 'failed') |
| 323 | AND (created_at + INTERVAL '1 second' * ?) < CURRENT_TIMESTAMP |
| 324 | RETURNING COUNT(*) |
| 325 | """, [max_age_seconds]).fetchone() |
| 326 | |
| 327 | if deleted and deleted[0] > 0: |
| 328 | print(f"🧹 Cleaned up {deleted[0]} old messages") |
| 329 | |
| 330 | |
| 331 | # Example worker processors for different channels |
| 332 | async def process_order(message: Dict) -> Dict: |
| 333 | """Process order messages""" |
| 334 | payload = message['payload'] |
| 335 | # Simulate order processing |
| 336 | await asyncio.sleep(0.01) |
| 337 | return { |
| 338 | 'order_id': payload.get('order_id'), |
| 339 | 'status': 'processed', |
| 340 | 'total': payload.get('total', 0) * 1.1 # Add tax |
| 341 | } |
| 342 | |
| 343 | async def process_shipping(message: Dict) -> Dict: |
| 344 | """Process shipping messages""" |
| 345 | payload = message['payload'] |
| 346 | # Simulate shipping calculation |
| 347 | await asyncio.sleep(0.01) |
| 348 | return { |
| 349 | 'tracking_number': f"TRACK{uuid.uuid4().hex[:8].upper()}", |
| 350 | 'carrier': 'UPS', |
| 351 | 'estimated_days': 3 |
| 352 | } |
| 353 | |
| 354 | async def process_fraud(message: Dict) -> Dict: |
| 355 | """Process fraud detection messages""" |
| 356 | payload = message['payload'] |
| 357 | # Simulate fraud check |
| 358 | await asyncio.sleep(0.02) |
| 359 | risk_score = 25 if payload.get('amount', 0) < 1000 else 75 |
| 360 | return { |
| 361 | 'risk_score': risk_score, |
| 362 | 'action': 'approve' if risk_score < 50 else 'review' |
| 363 | } |
| 364 | |
| 365 | |
| 366 | # Demo |
| 367 | async def demo_channels(): |
| 368 | """Demonstrate the channel system""" |
| 369 | |
| 370 | channels = DBBasicChannels() |
| 371 | |
| 372 | print("\n🚀 DBBasic Channels Demo\n") |
| 373 | |
| 374 | # Start workers watching different channels |
| 375 | print("Starting channel watchers...") |
| 376 | |
| 377 | # Order processor |
| 378 | asyncio.create_task( |
| 379 | channels.watch_channel(Channel.ORDERS.value, "OrderWorker-1", process_order) |
| 380 | ) |
| 381 | |
| 382 | # Shipping processor |
| 383 | asyncio.create_task( |
| 384 | channels.watch_channel(Channel.SHIPPING.value, "ShippingWorker-1", process_shipping) |
| 385 | ) |
| 386 | |
| 387 | # Fraud processor |
| 388 | asyncio.create_task( |
| 389 | channels.watch_channel(Channel.FRAUD.value, "FraudWorker-1", process_fraud) |
| 390 | ) |
| 391 | |
| 392 | # Monitor all channels |
| 393 | async def monitor_callback(message): |
| 394 | print(f"📊 Monitor: {message.get('channel')} - {message.get('message_type')}") |
| 395 | |
| 396 | await channels.subscribe("*", monitor_callback, "GlobalMonitor") |
| 397 | |
| 398 | # Give workers time to start |
| 399 | await asyncio.sleep(1) |
| 400 | |
| 401 | # Publish some messages |
| 402 | print("\nPublishing messages to channels...") |
| 403 | |
| 404 | # Publish order |
| 405 | await channels.publish( |
| 406 | Channel.ORDERS.value, |
| 407 | "new_order", |
| 408 | {"order_id": "ORD123", "total": 150.00, "customer_id": "CUST456"} |
| 409 | ) |
| 410 | |
| 411 | # Publish shipping request |
| 412 | await channels.publish( |
| 413 | Channel.SHIPPING.value, |
| 414 | "calculate_shipping", |
| 415 | {"weight": 2.5, "destination": "90210"} |
| 416 | ) |
| 417 | |
| 418 | # Publish fraud check |
| 419 | await channels.publish( |
| 420 | Channel.FRAUD.value, |
| 421 | "check_transaction", |
| 422 | {"transaction_id": "TXN789", "amount": 2500.00} |
| 423 | ) |
| 424 | |
| 425 | # Wait for processing |
| 426 | await asyncio.sleep(2) |
| 427 | |
| 428 | # Show stats |
| 429 | print("\n📊 Channel Statistics:") |
| 430 | stats = channels.get_channel_stats() |
| 431 | for channel, data in stats.items(): |
| 432 | if data['total_messages'] > 0: |
| 433 | print(f" {channel}: {data}") |
| 434 | |
| 435 | # Keep running for a bit to show real-time processing |
| 436 | await asyncio.sleep(5) |
| 437 | |
| 438 | |
| 439 | if __name__ == "__main__": |
| 440 | asyncio.run(demo_channels()) |