Python functions as REST objects, with clustering, load balancing and state replication; no containers or orchestration.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-primitive.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Q Consolidate duplicate QUICKSTARTs on the top-level file 503f830 7 months ago History
object_runtime.py · 15.0 KB · 520 lines
1"""
2Object Runtime
3
4Integrates all core primitives into a unified runtime for Object Primitives.
5
6An Object is:
7- Code (executable endpoint)
8- Logs (self-logging)
9- Versions (automatic versioning)
10- State (persistent state)
11- Network-accessible (can be called remotely)
12
13The runtime:
14- Loads objects from files
15- Injects logger and state manager
16- Executes methods
17- Logs all operations
18- Versions code changes
19- Provides introspection
20"""
21
22from pathlib import Path
23from typing import Any, Dict, List, Optional
24from datetime import datetime
25import importlib.util
26import sys
27import json
28import os
29import time
30import threading
31
32try:
33 import requests
34except ImportError:
35 requests = None # Replication will be disabled
36
37from src.object_primitive.core.endpoint_loader import load_endpoint, execute_endpoint
38from src.object_primitive.core.self_logger import SelfLogger
39from src.object_primitive.core.version_manager import VersionManager
40
41
42class ObjectRuntime:
43 """
44 Runtime for Object Primitives.
45
46 Provides integrated execution environment for objects.
47 """
48
49 def __init__(self, base_dir: Path | str):
50 """
51 Initialize runtime.
52
53 Args:
54 base_dir: Base directory for logs, versions, state
55 """
56 self.base_dir = Path(base_dir)
57 self.base_dir.mkdir(parents=True, exist_ok=True)
58
59 # Loaded objects cache
60 self._objects: Dict[str, 'ObjectPrimitive'] = {}
61
62 def load_object(self, path: str | Path) -> 'ObjectPrimitive':
63 """
64 Load an object from file.
65
66 Args:
67 path: Path to object file (.py)
68
69 Returns:
70 ObjectPrimitive instance
71 """
72 path = Path(path)
73 object_id = path.stem # Filename without extension
74
75 # Check cache
76 if object_id in self._objects:
77 return self._objects[object_id]
78
79 # Create object
80 obj = ObjectPrimitive(
81 object_id=object_id,
82 source_path=path,
83 runtime=self,
84 )
85
86 # Cache it
87 self._objects[object_id] = obj
88
89 return obj
90
91
92class StateManager:
93 """
94 Simple state manager for object state.
95
96 Stores state in TSV file for persistence.
97 Automatically replicates state to other stations for high availability.
98 """
99
100 def __init__(self, object_id: str, base_dir: Path, enable_replication: bool = True):
101 """
102 Initialize state manager.
103
104 Args:
105 object_id: ID of the object
106 base_dir: Base directory for state storage
107 enable_replication: Whether to enable state replication (default: True)
108 """
109 self.object_id = object_id
110 self.base_dir = Path(base_dir)
111 self.enable_replication = enable_replication
112
113 # State directory
114 self.state_dir = self.base_dir / 'state' / object_id
115 self.state_dir.mkdir(parents=True, exist_ok=True)
116
117 # State file (simple JSON-like format in TSV)
118 self.state_file = self.state_dir / 'state.tsv'
119
120 # Load state
121 self._state = self._load_state()
122
123 def get(self, key: str, default: Any = None) -> Any:
124 """Get state value"""
125 return self._state.get(key, default)
126
127 def set(self, key: str, value: Any) -> None:
128 """
129 Set state value and replicate to other stations
130
131 Replication is async (fire-and-forget) and includes timestamp
132 for last-write-wins conflict resolution.
133 """
134 self._state[key] = value
135 timestamp = time.time()
136 self._save_state_with_timestamp(timestamp)
137
138 # Replicate async to other stations
139 if self.enable_replication and requests:
140 self._replicate_async(key, value, timestamp)
141
142 def delete(self, key: str) -> None:
143 """Delete state value"""
144 if key in self._state:
145 del self._state[key]
146 timestamp = time.time()
147 self._save_state_with_timestamp(timestamp)
148
149 def get_all(self) -> Dict[str, Any]:
150 """Get all state"""
151 return self._state.copy()
152
153 def _get_replica_stations(self) -> List[dict]:
154 """
155 Get list of active stations for replication
156
157 Returns list of stations excluding ourselves
158 """
159 if not requests:
160 return []
161
162 local_station = os.environ.get('STATION_ID', 'unknown')
163 registry_file = Path('data/cluster/stations.tsv')
164
165 if not registry_file.exists():
166 return []
167
168 stations = []
169 current_time = time.time()
170 timeout = 30 # seconds
171
172 with open(registry_file, 'r') as f:
173 for line in f:
174 if line.strip():
175 parts = line.strip().split('\t')
176 if len(parts) >= 4:
177 sid, host, port, last_heartbeat = parts[:4]
178
179 # Skip ourselves
180 if sid == local_station:
181 continue
182
183 # Check if active
184 age = current_time - float(last_heartbeat)
185 if age < timeout:
186 stations.append({
187 'station_id': sid,
188 'host': host,
189 'port': int(port),
190 'url': f'http://{host}:{port}'
191 })
192
193 return stations
194
195 def _replicate_async(self, key: str, value: Any, timestamp: float) -> None:
196 """
197 Replicate state to other stations asynchronously
198
199 Uses threading to avoid blocking on network I/O
200 """
201 def replicate_to_station(station: dict):
202 """Send replication request (runs in background thread)"""
203 try:
204 url = f"{station['url']}/cluster/replicate"
205 payload = {
206 'object_id': self.object_id,
207 'key': key,
208 'value': str(value),
209 'timestamp': timestamp,
210 'source_station': os.environ.get('STATION_ID', 'unknown')
211 }
212
213 # Fire and forget (timeout quickly, don't wait for response)
214 requests.post(url, json=payload, timeout=0.5)
215 except:
216 # Silently ignore replication failures
217 # (replicas will catch up on next write)
218 pass
219
220 # Get active stations for replication
221 stations = self._get_replica_stations()
222
223 # Start background threads for each station
224 for station in stations:
225 thread = threading.Thread(
226 target=replicate_to_station,
227 args=(station,),
228 daemon=True # Don't wait for threads to finish
229 )
230 thread.start()
231
232 def _load_state(self) -> Dict[str, Any]:
233 """
234 Load state from file
235
236 Supports both old format (key\tvalue) and new format (key\tvalue\ttimestamp)
237 """
238 if not self.state_file.exists():
239 return {}
240
241 state = {}
242 with open(self.state_file, 'r') as f:
243 for line in f:
244 if line.strip():
245 parts = line.strip().split('\t')
246
247 # Skip header line if present
248 if parts[0] == 'key':
249 continue
250
251 if len(parts) >= 2:
252 key = parts[0]
253 value = parts[1]
254
255 # Try to convert to int/float
256 try:
257 value = int(value)
258 except ValueError:
259 try:
260 value = float(value)
261 except ValueError:
262 pass # Keep as string
263
264 state[key] = value
265
266 return state
267
268 def _save_state(self) -> None:
269 """Save state to file (backward compatible, no timestamp)"""
270 self._save_state_with_timestamp(time.time())
271
272 def _save_state_with_timestamp(self, timestamp: float) -> None:
273 """
274 Save state to file with timestamp
275
276 Format: key \t value \t timestamp
277 Timestamp used for last-write-wins conflict resolution
278 """
279 with open(self.state_file, 'w') as f:
280 for key, value in sorted(self._state.items()):
281 f.write(f'{key}\t{value}\t{timestamp}\n')
282
283
284class ObjectPrimitive:
285 """
286 An Object Primitive.
287
288 Combines:
289 - Code (executable)
290 - Logs (self-logging)
291 - Versions (code history)
292 - State (persistent state)
293 """
294
295 def __init__(
296 self,
297 object_id: str,
298 source_path: Path,
299 runtime: ObjectRuntime,
300 ):
301 """
302 Initialize object.
303
304 Args:
305 object_id: Unique ID for this object
306 source_path: Path to source file
307 runtime: Parent runtime
308 """
309 self.object_id = object_id
310 self.source_path = Path(source_path)
311 self.runtime = runtime
312
313 # Initialize primitives
314 self.logger = SelfLogger(
315 object_id=object_id,
316 base_dir=runtime.base_dir,
317 )
318
319 self.version_manager = VersionManager(
320 base_dir=runtime.base_dir,
321 )
322
323 self.state_manager = StateManager(
324 object_id=object_id,
325 base_dir=runtime.base_dir,
326 )
327
328 # Load the endpoint
329 self._load_endpoint()
330
331 # Save initial version
332 self._save_initial_version()
333
334 def _load_endpoint(self, reload: bool = False) -> None:
335 """Load endpoint from file"""
336 self.endpoint = load_endpoint(self.source_path, reload=reload)
337
338 # Inject logger, state manager, and runtime into module
339 if hasattr(self.endpoint, '__dict__'):
340 self.endpoint._logger = self.logger
341 self.endpoint._state_manager = self.state_manager
342 self.endpoint._runtime = self.runtime # Enable object composition
343
344 def _save_initial_version(self) -> None:
345 """Save initial version of code"""
346 # Check if any versions exist
347 history = self.version_manager.get_history(self.object_id)
348 if not history:
349 # Save v1
350 source_code = self.source_path.read_text()
351 self.version_manager.save_version(
352 object_id=self.object_id,
353 content=source_code,
354 author='system',
355 message='Initial version',
356 )
357
358 def execute(self, method: str, request: Dict[str, Any]) -> Dict[str, Any]:
359 """
360 Execute a method on this object.
361
362 Args:
363 method: HTTP method (GET, POST, PUT, DELETE)
364 request: Request data
365
366 Returns:
367 Response data
368 """
369 # Log the execution
370 self.logger.info(
371 f'Executing {method}',
372 method=method,
373 user_id=request.get('user_id'),
374 request_id=request.get('request_id'),
375 )
376
377 try:
378 # Execute
379 result = execute_endpoint(self.endpoint, method, request)
380
381 # Log success
382 self.logger.debug(
383 f'{method} completed successfully',
384 method=method,
385 status='success',
386 )
387
388 return result
389
390 except Exception as e:
391 # Log error
392 self.logger.error(
393 f'{method} failed: {str(e)}',
394 method=method,
395 status='error',
396 error=str(e),
397 )
398 raise
399
400 def update_code(self, new_code: str, author: str, message: str) -> int:
401 """
402 Update object's code.
403
404 Args:
405 new_code: New source code
406 author: Who is making the change
407 message: Commit message
408
409 Returns:
410 New version ID
411 """
412 # Save current version
413 version_id = self.version_manager.save_version(
414 object_id=self.object_id,
415 content=new_code,
416 author=author,
417 message=message,
418 )
419
420 # Write new code to file
421 self.source_path.write_text(new_code)
422
423 # Reload endpoint (force reload to bypass cache)
424 self._load_endpoint(reload=True)
425
426 # Log the update
427 self.logger.warning(
428 'Code updated',
429 author=author,
430 commit_message=message,
431 version=version_id,
432 )
433
434 return version_id
435
436 def rollback_to_version(
437 self,
438 version_id: int,
439 author: str,
440 message: str,
441 ) -> int:
442 """
443 Rollback to a previous version.
444
445 Args:
446 version_id: Version to rollback to
447 author: Who is performing the rollback
448 message: Rollback message
449
450 Returns:
451 New version ID (rollback creates new version)
452 """
453 # Rollback via version manager
454 new_version_id = self.version_manager.rollback(
455 object_id=self.object_id,
456 to_version=version_id,
457 author=author,
458 message=message,
459 )
460
461 # Get the rolled-back code
462 version = self.version_manager.get_version(self.object_id, new_version_id)
463 code = version['content']
464
465 # Write to file
466 self.source_path.write_text(code)
467
468 # Reload endpoint (force reload to bypass cache)
469 self._load_endpoint(reload=True)
470
471 # Log the rollback
472 self.logger.critical(
473 f'Rolled back to version {version_id}',
474 author=author,
475 commit_message=message,
476 from_version=version_id,
477 to_version=new_version_id,
478 )
479
480 return new_version_id
481
482 def get_logs(
483 self,
484 level: Optional[str] = None,
485 limit: Optional[int] = None,
486 **filters,
487 ) -> List[Dict[str, Any]]:
488 """Get object's logs"""
489 return self.logger.get_logs(level=level, limit=limit, **filters)
490
491 def get_state(self) -> Dict[str, Any]:
492 """Get object's current state"""
493 return self.state_manager.get_all()
494
495 def get_metadata(self) -> Dict[str, Any]:
496 """Get object metadata"""
497 from src.object_primitive.core.endpoint_loader import get_endpoint_metadata
498
499 metadata = get_endpoint_metadata(self.endpoint)
500
501 # Add runtime metadata
502 metadata['object_id'] = self.object_id
503 metadata['source_path'] = str(self.source_path)
504 metadata['log_count'] = len(self.get_logs())
505 metadata['version_count'] = len(self.get_version_history())
506 metadata['state_keys'] = list(self.get_state().keys())
507
508 return metadata
509
510 def get_source_code(self) -> str:
511 """Get object's source code"""
512 return self.source_path.read_text()
513
514 def get_version_history(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
515 """Get version history"""
516 return self.version_manager.get_history(self.object_id, limit=limit)
517
518 def get_version(self, version_id: int) -> Optional[Dict[str, Any]]:
519 """Get specific version"""
520 return self.version_manager.get_version(self.object_id, version_id=version_id)