askrobots / dbbasic Public

Model-config web apps: describe the app in configuration instead of writing code. Built on Polars and DuckDB.

0 downloads

dbbasic/realtime_monitor.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Add DBBasic crate engine logo and update README 0d628db 1 year, 2 months ago History
realtime_monitor.py · 11.7 KB · 309 lines
1#!/usr/bin/env python3
2"""
3DBBasic Real-time Monitor - Live service activity at 402M rows/sec
4Shows actual service calls, database operations, and system metrics
5"""
6
7
8from realtime_monitor_presentation import get_realtime_monitor_ui
9from fastapi import FastAPI, WebSocket, WebSocketDisconnect
10from fastapi.responses import HTMLResponse, JSONResponse
11from fastapi.staticfiles import StaticFiles
12import asyncio
13import json
14import random
15from datetime import datetime
16from typing import Dict, Any, List
17import psutil
18import time
19from collections import deque
20import os
21import sys
22
23from presentation_layer import PresentationLayer
24from bootstrap_components import ExtendedBootstrapRenderer
25from dbbasic_unified_ui import get_master_layout, SERVICES
26
27# Initialize presentation layer
28PresentationLayer.add_renderer('bootstrap', ExtendedBootstrapRenderer())
29
30
31# Import our services
32sys.path.append(os.path.dirname(os.path.abspath(__file__)))
33from services.calculate_shipping import calculate_shipping
34from services.calculate_discount import calculate_discount
35from services.calculate_order_total import calculate_order_total
36from services.send_notification import send_notification
37from services.detect_fraud import detect_fraud
38from services.segment_customer import segment_customer
39
40app = FastAPI(title="DBBasic Realtime Monitor")
41
42# Store for active connections
43active_connections: List[WebSocket] = []
44
45# Store for events
46events: List[Dict[str, Any]] = []
47
48# Metrics storage
49metrics = {
50 "operations_per_second": deque(maxlen=60), # Last 60 seconds
51 "rows_per_second": 402000000, # Our benchmark
52 "active_connections": 0,
53 "total_operations": 0,
54 "service_calls": deque(maxlen=100), # Last 100 service calls
55 "active_tables": {
56 "customers": 0,
57 "orders": 0,
58 "products": 0,
59 "inventory": 0,
60 "analytics": 0
61 }
62}
63
64# Service registry
65services = {
66 "calculate_shipping": calculate_shipping,
67 "calculate_discount": calculate_discount,
68 "calculate_order_total": calculate_order_total,
69 "send_notification": send_notification,
70 "detect_fraud": detect_fraud,
71 "segment_customer": segment_customer
72}
73
74async def broadcast_message(message: dict):
75 """Broadcast message to all connected clients"""
76 for connection in active_connections:
77 try:
78 await connection.send_json(message)
79 except:
80 pass
81
82async def simulate_service_activity():
83 """Generate realistic service activity"""
84 while True:
85 try:
86 # Pick a random service
87 service_name = random.choice(list(services.keys()))
88 service_func = services[service_name]
89
90 # Generate appropriate test data for each service
91 if service_name == "calculate_shipping":
92 test_data = {
93 "weight": random.uniform(0.5, 20),
94 "shipping_speed": random.choice(["standard", "express", "overnight"]),
95 "is_fragile": random.choice([True, False]),
96 "order_total": random.uniform(10, 500)
97 }
98 elif service_name == "calculate_discount":
99 test_data = {
100 "customer_tier": random.choice(["bronze", "silver", "gold", "platinum"]),
101 "order_total": random.uniform(50, 1000),
102 "items_count": random.randint(1, 10)
103 }
104 elif service_name == "calculate_order_total":
105 test_data = {
106 "items": [{"price": random.uniform(10, 100), "quantity": random.randint(1, 5)}
107 for _ in range(random.randint(1, 5))],
108 "customer_location": {
109 "state": random.choice(["CA", "TX", "NY", "FL"]),
110 "city": random.choice(["San Francisco", "Austin", "New York", "Miami"])
111 },
112 "apply_discounts": True
113 }
114 elif service_name == "send_notification":
115 test_data = {
116 "notification_type": random.choice(["email", "sms", "push"]),
117 "recipient": {"email": f"user{random.randint(1,1000)}@example.com"},
118 "subject": "Order Update",
119 "message": "Your order status has been updated"
120 }
121 elif service_name == "detect_fraud":
122 test_data = {
123 "order_data": {"total": random.uniform(50, 5000)},
124 "customer_data": {
125 "account_age_days": random.randint(0, 365),
126 "previous_orders": random.randint(0, 50)
127 },
128 "payment_data": {"method": random.choice(["card", "paypal", "prepaid_card"])}
129 }
130 elif service_name == "segment_customer":
131 test_data = {
132 "customer_id": f"CUST{random.randint(1000, 9999)}",
133 "purchase_history": {
134 "total_orders": random.randint(0, 100),
135 "lifetime_value": random.uniform(0, 10000),
136 "days_since_last_order": random.randint(0, 365)
137 },
138 "engagement_metrics": {
139 "email_open_rate": random.uniform(0, 0.5),
140 "app_usage_days_per_month": random.randint(0, 30)
141 }
142 }
143
144 # Call the actual service
145 start_time = time.time()
146 result = await service_func(test_data)
147 execution_time = (time.time() - start_time) * 1000 # ms
148
149 # Create activity event
150 event = {
151 "type": "service_call",
152 "timestamp": datetime.now().isoformat(),
153 "service": service_name,
154 "execution_time_ms": round(execution_time, 3),
155 "success": result.get("success", False),
156 "data_preview": str(result.get("data", {}))[:100] + "..." if result.get("data") else "No data"
157 }
158
159 # Store and broadcast
160 metrics["service_calls"].append(event)
161 metrics["total_operations"] += 1
162
163 # Update table activity (simulate based on service)
164 table_updates = {
165 "calculate_shipping": ["orders"],
166 "calculate_discount": ["customers", "orders"],
167 "calculate_order_total": ["orders", "products"],
168 "send_notification": ["customers"],
169 "detect_fraud": ["orders", "customers"],
170 "segment_customer": ["customers", "analytics"]
171 }
172
173 for table in table_updates.get(service_name, []):
174 metrics["active_tables"][table] += 1
175
176 await broadcast_message(event)
177
178 except Exception as e:
179 print(f"Error in service activity simulation: {e}")
180
181 await asyncio.sleep(random.uniform(0.1, 0.5)) # Variable rate
182
183async def calculate_metrics():
184 """Calculate and broadcast system metrics"""
185 while True:
186 try:
187 # Get system metrics
188 cpu_percent = psutil.cpu_percent(interval=0.1)
189 memory = psutil.virtual_memory()
190 disk_io = psutil.disk_io_counters()
191 net_io = psutil.net_io_counters()
192
193 # Calculate operations per second
194 current_ops = len(metrics["service_calls"])
195 metrics["operations_per_second"].append(current_ops)
196
197 # Prepare metrics message
198 metrics_msg = {
199 "type": "metrics",
200 "timestamp": datetime.now().isoformat(),
201 "data": {
202 "operations_per_second": current_ops * 10, # Multiply for display
203 "rows_per_second": metrics["rows_per_second"],
204 "active_connections": len(active_connections),
205 "total_operations": metrics["total_operations"],
206 "cpu_percent": cpu_percent,
207 "memory_used_gb": round(memory.used / (1024**3), 2),
208 "memory_total_gb": round(memory.total / (1024**3), 2),
209 "disk_read_mb": round(disk_io.read_bytes / (1024**2), 2),
210 "disk_write_mb": round(disk_io.write_bytes / (1024**2), 2),
211 "network_sent_mb": round(net_io.bytes_sent / (1024**2), 2),
212 "network_recv_mb": round(net_io.bytes_recv / (1024**2), 2),
213 "active_tables": metrics["active_tables"]
214 }
215 }
216
217 await broadcast_message(metrics_msg)
218
219 except Exception as e:
220 print(f"Error calculating metrics: {e}")
221
222 await asyncio.sleep(1)
223
224@app.on_event("startup")
225async def startup_event():
226 """Start background tasks"""
227 asyncio.create_task(simulate_service_activity())
228 asyncio.create_task(calculate_metrics())
229
230@app.websocket("/ws")
231async def websocket_endpoint(websocket: WebSocket):
232 """WebSocket endpoint for real-time updates"""
233 await websocket.accept()
234 active_connections.append(websocket)
235
236 try:
237 # Send initial data
238 await websocket.send_json({
239 "type": "connection",
240 "status": "connected",
241 "timestamp": datetime.now().isoformat()
242 })
243
244 # Keep connection alive and handle incoming events
245 while True:
246 try:
247 # Wait for incoming data with timeout
248 data = await asyncio.wait_for(websocket.receive_text(), timeout=30.0)
249 print(f"Received event: {data[:100]}...")
250
251 # Parse incoming event data
252 try:
253 event_data = json.loads(data)
254
255 # Store event for display (keep last 100 events)
256 events.append(event_data)
257 if len(events) > 100:
258 events.pop(0)
259
260 # Broadcast to all browser connections (excluding sender)
261 for connection in active_connections[:]:
262 if connection != websocket: # Don't send back to sender
263 try:
264 await connection.send_json(event_data)
265 except:
266 # Remove broken connections
267 if connection in active_connections:
268 active_connections.remove(connection)
269
270 except json.JSONDecodeError:
271 print(f"Invalid JSON received: {data}")
272
273 except asyncio.TimeoutError:
274 # Send ping to keep connection alive
275 try:
276 await websocket.send_json({"type": "ping"})
277 except:
278 # Connection is broken
279 break
280
281 except WebSocketDisconnect:
282 print("connection closed")
283 if websocket in active_connections:
284 active_connections.remove(websocket)
285
286@app.get("/")
287async def dashboard():
288 """Serve real-time monitor dashboard using presentation layer"""
289 from realtime_monitor_presentation import get_realtime_monitor_ui
290
291 ui_data = get_realtime_monitor_ui()
292 html_content = PresentationLayer.render(ui_data, 'bootstrap')
293 return HTMLResponse(content=html_content)
294
295@app.get("/api/metrics")
296async def get_metrics():
297 """Get current metrics as JSON"""
298 return JSONResponse({
299 "operations_per_second": len(metrics["operations_per_second"]),
300 "rows_per_second": metrics["rows_per_second"],
301 "active_connections": len(active_connections),
302 "total_operations": metrics["total_operations"],
303 "recent_calls": list(metrics["service_calls"])[-10:],
304 "active_tables": metrics["active_tables"]
305 })
306
307if __name__ == "__main__":
308 import uvicorn
309 uvicorn.run(app, host="0.0.0.0", port=8004)