Model-config web apps: describe the app in configuration instead of writing code. Built on Polars and DuckDB.
realtime_monitor.py · 11.7 KB · 309 lines
| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | DBBasic Real-time Monitor - Live service activity at 402M rows/sec |
| 4 | Shows actual service calls, database operations, and system metrics |
| 5 | """ |
| 6 | |
| 7 | |
| 8 | from realtime_monitor_presentation import get_realtime_monitor_ui |
| 9 | from fastapi import FastAPI, WebSocket, WebSocketDisconnect |
| 10 | from fastapi.responses import HTMLResponse, JSONResponse |
| 11 | from fastapi.staticfiles import StaticFiles |
| 12 | import asyncio |
| 13 | import json |
| 14 | import random |
| 15 | from datetime import datetime |
| 16 | from typing import Dict, Any, List |
| 17 | import psutil |
| 18 | import time |
| 19 | from collections import deque |
| 20 | import os |
| 21 | import sys |
| 22 | |
| 23 | from presentation_layer import PresentationLayer |
| 24 | from bootstrap_components import ExtendedBootstrapRenderer |
| 25 | from dbbasic_unified_ui import get_master_layout, SERVICES |
| 26 | |
| 27 | # Initialize presentation layer |
| 28 | PresentationLayer.add_renderer('bootstrap', ExtendedBootstrapRenderer()) |
| 29 | |
| 30 | |
| 31 | # Import our services |
| 32 | sys.path.append(os.path.dirname(os.path.abspath(__file__))) |
| 33 | from services.calculate_shipping import calculate_shipping |
| 34 | from services.calculate_discount import calculate_discount |
| 35 | from services.calculate_order_total import calculate_order_total |
| 36 | from services.send_notification import send_notification |
| 37 | from services.detect_fraud import detect_fraud |
| 38 | from services.segment_customer import segment_customer |
| 39 | |
| 40 | app = FastAPI(title="DBBasic Realtime Monitor") |
| 41 | |
| 42 | # Store for active connections |
| 43 | active_connections: List[WebSocket] = [] |
| 44 | |
| 45 | # Store for events |
| 46 | events: List[Dict[str, Any]] = [] |
| 47 | |
| 48 | # Metrics storage |
| 49 | metrics = { |
| 50 | "operations_per_second": deque(maxlen=60), # Last 60 seconds |
| 51 | "rows_per_second": 402000000, # Our benchmark |
| 52 | "active_connections": 0, |
| 53 | "total_operations": 0, |
| 54 | "service_calls": deque(maxlen=100), # Last 100 service calls |
| 55 | "active_tables": { |
| 56 | "customers": 0, |
| 57 | "orders": 0, |
| 58 | "products": 0, |
| 59 | "inventory": 0, |
| 60 | "analytics": 0 |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | # Service registry |
| 65 | services = { |
| 66 | "calculate_shipping": calculate_shipping, |
| 67 | "calculate_discount": calculate_discount, |
| 68 | "calculate_order_total": calculate_order_total, |
| 69 | "send_notification": send_notification, |
| 70 | "detect_fraud": detect_fraud, |
| 71 | "segment_customer": segment_customer |
| 72 | } |
| 73 | |
| 74 | async def broadcast_message(message: dict): |
| 75 | """Broadcast message to all connected clients""" |
| 76 | for connection in active_connections: |
| 77 | try: |
| 78 | await connection.send_json(message) |
| 79 | except: |
| 80 | pass |
| 81 | |
| 82 | async def simulate_service_activity(): |
| 83 | """Generate realistic service activity""" |
| 84 | while True: |
| 85 | try: |
| 86 | # Pick a random service |
| 87 | service_name = random.choice(list(services.keys())) |
| 88 | service_func = services[service_name] |
| 89 | |
| 90 | # Generate appropriate test data for each service |
| 91 | if service_name == "calculate_shipping": |
| 92 | test_data = { |
| 93 | "weight": random.uniform(0.5, 20), |
| 94 | "shipping_speed": random.choice(["standard", "express", "overnight"]), |
| 95 | "is_fragile": random.choice([True, False]), |
| 96 | "order_total": random.uniform(10, 500) |
| 97 | } |
| 98 | elif service_name == "calculate_discount": |
| 99 | test_data = { |
| 100 | "customer_tier": random.choice(["bronze", "silver", "gold", "platinum"]), |
| 101 | "order_total": random.uniform(50, 1000), |
| 102 | "items_count": random.randint(1, 10) |
| 103 | } |
| 104 | elif service_name == "calculate_order_total": |
| 105 | test_data = { |
| 106 | "items": [{"price": random.uniform(10, 100), "quantity": random.randint(1, 5)} |
| 107 | for _ in range(random.randint(1, 5))], |
| 108 | "customer_location": { |
| 109 | "state": random.choice(["CA", "TX", "NY", "FL"]), |
| 110 | "city": random.choice(["San Francisco", "Austin", "New York", "Miami"]) |
| 111 | }, |
| 112 | "apply_discounts": True |
| 113 | } |
| 114 | elif service_name == "send_notification": |
| 115 | test_data = { |
| 116 | "notification_type": random.choice(["email", "sms", "push"]), |
| 117 | "recipient": {"email": f"user{random.randint(1,1000)}@example.com"}, |
| 118 | "subject": "Order Update", |
| 119 | "message": "Your order status has been updated" |
| 120 | } |
| 121 | elif service_name == "detect_fraud": |
| 122 | test_data = { |
| 123 | "order_data": {"total": random.uniform(50, 5000)}, |
| 124 | "customer_data": { |
| 125 | "account_age_days": random.randint(0, 365), |
| 126 | "previous_orders": random.randint(0, 50) |
| 127 | }, |
| 128 | "payment_data": {"method": random.choice(["card", "paypal", "prepaid_card"])} |
| 129 | } |
| 130 | elif service_name == "segment_customer": |
| 131 | test_data = { |
| 132 | "customer_id": f"CUST{random.randint(1000, 9999)}", |
| 133 | "purchase_history": { |
| 134 | "total_orders": random.randint(0, 100), |
| 135 | "lifetime_value": random.uniform(0, 10000), |
| 136 | "days_since_last_order": random.randint(0, 365) |
| 137 | }, |
| 138 | "engagement_metrics": { |
| 139 | "email_open_rate": random.uniform(0, 0.5), |
| 140 | "app_usage_days_per_month": random.randint(0, 30) |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | # Call the actual service |
| 145 | start_time = time.time() |
| 146 | result = await service_func(test_data) |
| 147 | execution_time = (time.time() - start_time) * 1000 # ms |
| 148 | |
| 149 | # Create activity event |
| 150 | event = { |
| 151 | "type": "service_call", |
| 152 | "timestamp": datetime.now().isoformat(), |
| 153 | "service": service_name, |
| 154 | "execution_time_ms": round(execution_time, 3), |
| 155 | "success": result.get("success", False), |
| 156 | "data_preview": str(result.get("data", {}))[:100] + "..." if result.get("data") else "No data" |
| 157 | } |
| 158 | |
| 159 | # Store and broadcast |
| 160 | metrics["service_calls"].append(event) |
| 161 | metrics["total_operations"] += 1 |
| 162 | |
| 163 | # Update table activity (simulate based on service) |
| 164 | table_updates = { |
| 165 | "calculate_shipping": ["orders"], |
| 166 | "calculate_discount": ["customers", "orders"], |
| 167 | "calculate_order_total": ["orders", "products"], |
| 168 | "send_notification": ["customers"], |
| 169 | "detect_fraud": ["orders", "customers"], |
| 170 | "segment_customer": ["customers", "analytics"] |
| 171 | } |
| 172 | |
| 173 | for table in table_updates.get(service_name, []): |
| 174 | metrics["active_tables"][table] += 1 |
| 175 | |
| 176 | await broadcast_message(event) |
| 177 | |
| 178 | except Exception as e: |
| 179 | print(f"Error in service activity simulation: {e}") |
| 180 | |
| 181 | await asyncio.sleep(random.uniform(0.1, 0.5)) # Variable rate |
| 182 | |
| 183 | async def calculate_metrics(): |
| 184 | """Calculate and broadcast system metrics""" |
| 185 | while True: |
| 186 | try: |
| 187 | # Get system metrics |
| 188 | cpu_percent = psutil.cpu_percent(interval=0.1) |
| 189 | memory = psutil.virtual_memory() |
| 190 | disk_io = psutil.disk_io_counters() |
| 191 | net_io = psutil.net_io_counters() |
| 192 | |
| 193 | # Calculate operations per second |
| 194 | current_ops = len(metrics["service_calls"]) |
| 195 | metrics["operations_per_second"].append(current_ops) |
| 196 | |
| 197 | # Prepare metrics message |
| 198 | metrics_msg = { |
| 199 | "type": "metrics", |
| 200 | "timestamp": datetime.now().isoformat(), |
| 201 | "data": { |
| 202 | "operations_per_second": current_ops * 10, # Multiply for display |
| 203 | "rows_per_second": metrics["rows_per_second"], |
| 204 | "active_connections": len(active_connections), |
| 205 | "total_operations": metrics["total_operations"], |
| 206 | "cpu_percent": cpu_percent, |
| 207 | "memory_used_gb": round(memory.used / (1024**3), 2), |
| 208 | "memory_total_gb": round(memory.total / (1024**3), 2), |
| 209 | "disk_read_mb": round(disk_io.read_bytes / (1024**2), 2), |
| 210 | "disk_write_mb": round(disk_io.write_bytes / (1024**2), 2), |
| 211 | "network_sent_mb": round(net_io.bytes_sent / (1024**2), 2), |
| 212 | "network_recv_mb": round(net_io.bytes_recv / (1024**2), 2), |
| 213 | "active_tables": metrics["active_tables"] |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | await broadcast_message(metrics_msg) |
| 218 | |
| 219 | except Exception as e: |
| 220 | print(f"Error calculating metrics: {e}") |
| 221 | |
| 222 | await asyncio.sleep(1) |
| 223 | |
| 224 | @app.on_event("startup") |
| 225 | async def startup_event(): |
| 226 | """Start background tasks""" |
| 227 | asyncio.create_task(simulate_service_activity()) |
| 228 | asyncio.create_task(calculate_metrics()) |
| 229 | |
| 230 | @app.websocket("/ws") |
| 231 | async def websocket_endpoint(websocket: WebSocket): |
| 232 | """WebSocket endpoint for real-time updates""" |
| 233 | await websocket.accept() |
| 234 | active_connections.append(websocket) |
| 235 | |
| 236 | try: |
| 237 | # Send initial data |
| 238 | await websocket.send_json({ |
| 239 | "type": "connection", |
| 240 | "status": "connected", |
| 241 | "timestamp": datetime.now().isoformat() |
| 242 | }) |
| 243 | |
| 244 | # Keep connection alive and handle incoming events |
| 245 | while True: |
| 246 | try: |
| 247 | # Wait for incoming data with timeout |
| 248 | data = await asyncio.wait_for(websocket.receive_text(), timeout=30.0) |
| 249 | print(f"Received event: {data[:100]}...") |
| 250 | |
| 251 | # Parse incoming event data |
| 252 | try: |
| 253 | event_data = json.loads(data) |
| 254 | |
| 255 | # Store event for display (keep last 100 events) |
| 256 | events.append(event_data) |
| 257 | if len(events) > 100: |
| 258 | events.pop(0) |
| 259 | |
| 260 | # Broadcast to all browser connections (excluding sender) |
| 261 | for connection in active_connections[:]: |
| 262 | if connection != websocket: # Don't send back to sender |
| 263 | try: |
| 264 | await connection.send_json(event_data) |
| 265 | except: |
| 266 | # Remove broken connections |
| 267 | if connection in active_connections: |
| 268 | active_connections.remove(connection) |
| 269 | |
| 270 | except json.JSONDecodeError: |
| 271 | print(f"Invalid JSON received: {data}") |
| 272 | |
| 273 | except asyncio.TimeoutError: |
| 274 | # Send ping to keep connection alive |
| 275 | try: |
| 276 | await websocket.send_json({"type": "ping"}) |
| 277 | except: |
| 278 | # Connection is broken |
| 279 | break |
| 280 | |
| 281 | except WebSocketDisconnect: |
| 282 | print("connection closed") |
| 283 | if websocket in active_connections: |
| 284 | active_connections.remove(websocket) |
| 285 | |
| 286 | @app.get("/") |
| 287 | async def dashboard(): |
| 288 | """Serve real-time monitor dashboard using presentation layer""" |
| 289 | from realtime_monitor_presentation import get_realtime_monitor_ui |
| 290 | |
| 291 | ui_data = get_realtime_monitor_ui() |
| 292 | html_content = PresentationLayer.render(ui_data, 'bootstrap') |
| 293 | return HTMLResponse(content=html_content) |
| 294 | |
| 295 | @app.get("/api/metrics") |
| 296 | async def get_metrics(): |
| 297 | """Get current metrics as JSON""" |
| 298 | return JSONResponse({ |
| 299 | "operations_per_second": len(metrics["operations_per_second"]), |
| 300 | "rows_per_second": metrics["rows_per_second"], |
| 301 | "active_connections": len(active_connections), |
| 302 | "total_operations": metrics["total_operations"], |
| 303 | "recent_calls": list(metrics["service_calls"])[-10:], |
| 304 | "active_tables": metrics["active_tables"] |
| 305 | }) |
| 306 | |
| 307 | if __name__ == "__main__": |
| 308 | import uvicorn |
| 309 | uvicorn.run(app, host="0.0.0.0", port=8004) |