askrobots / dbbasic-object-primitive
Public
Python functions as REST objects, with clustering, load balancing and state replication; no containers or orchestration.
migrate.py · 11.2 KB · 357 lines
| 1 | """ |
| 2 | Object Migration API |
| 3 | |
| 4 | POST /cluster/migrate - Migrate object from one station to another |
| 5 | |
| 6 | Migrates an object by copying: |
| 7 | - Code file (examples/path/to/object.py) |
| 8 | - State directory (data/object_id/) |
| 9 | - state.tsv |
| 10 | - logs.tsv |
| 11 | - versions/ |
| 12 | |
| 13 | Example: |
| 14 | POST /cluster/migrate |
| 15 | { |
| 16 | "object_id": "calculator", |
| 17 | "from_station": "station2", |
| 18 | "to_station": "station3" |
| 19 | } |
| 20 | |
| 21 | Response: |
| 22 | { |
| 23 | "status": "ok", |
| 24 | "message": "Object migrated successfully", |
| 25 | "object_id": "calculator", |
| 26 | "from_station": "station2", |
| 27 | "to_station": "station3", |
| 28 | "files_copied": { |
| 29 | "code": "examples/calculator.py", |
| 30 | "state": ["state.tsv", "logs.tsv"], |
| 31 | "versions": 5 |
| 32 | } |
| 33 | } |
| 34 | """ |
| 35 | import json |
| 36 | import os |
| 37 | import time |
| 38 | import base64 |
| 39 | import requests |
| 40 | from pathlib import Path |
| 41 | from dbbasic_web.responses import json as json_response, json_error |
| 42 | |
| 43 | |
| 44 | def POST(request): |
| 45 | """ |
| 46 | Migrate object from one station to another |
| 47 | |
| 48 | Request body: |
| 49 | { |
| 50 | "object_id": str, # Object to migrate |
| 51 | "from_station": str, # Source station ID |
| 52 | "to_station": str, # Destination station ID |
| 53 | "copy_only": bool # If true, don't delete from source (default: false) |
| 54 | } |
| 55 | |
| 56 | Response: |
| 57 | { |
| 58 | "status": "ok", |
| 59 | "message": "Object migrated successfully", |
| 60 | "object_id": str, |
| 61 | "from_station": str, |
| 62 | "to_station": str, |
| 63 | "files_copied": { |
| 64 | "code": str, # Code file path |
| 65 | "state": [str], # List of state files |
| 66 | "versions": int # Number of versions copied |
| 67 | }, |
| 68 | "duration_seconds": float |
| 69 | } |
| 70 | |
| 71 | Errors: |
| 72 | 400 - Missing required fields |
| 73 | 404 - Object not found |
| 74 | 503 - Station not found or offline |
| 75 | 500 - Migration failed |
| 76 | """ |
| 77 | start_time = time.time() |
| 78 | |
| 79 | # Parse request body |
| 80 | try: |
| 81 | if request.body: |
| 82 | data = json.loads(request.body.decode('utf-8')) |
| 83 | else: |
| 84 | return json_error('Missing request body', status=400) |
| 85 | except json.JSONDecodeError as e: |
| 86 | return json_error(f'Invalid JSON: {e}', status=400) |
| 87 | |
| 88 | # Validate required fields |
| 89 | object_id = data.get('object_id') |
| 90 | from_station = data.get('from_station') |
| 91 | to_station = data.get('to_station') |
| 92 | copy_only = data.get('copy_only', False) |
| 93 | |
| 94 | if not object_id: |
| 95 | return json_error('object_id is required', status=400) |
| 96 | if not from_station: |
| 97 | return json_error('from_station is required', status=400) |
| 98 | if not to_station: |
| 99 | return json_error('to_station is required', status=400) |
| 100 | |
| 101 | # Get local station ID |
| 102 | local_station = os.environ.get('STATION_ID', 'unknown') |
| 103 | |
| 104 | # TODO: If this is not the master, forward to master |
| 105 | # For now, assume this runs on master (station1) |
| 106 | |
| 107 | # Step 1: Collect files from source station |
| 108 | try: |
| 109 | source_files = collect_object_files(object_id, from_station, local_station) |
| 110 | except FileNotFoundError as e: |
| 111 | return json_error(f'Object not found: {e}', status=404) |
| 112 | except Exception as e: |
| 113 | return json_error(f'Failed to collect files from source: {e}', status=500) |
| 114 | |
| 115 | # Step 2: Send files to destination station |
| 116 | try: |
| 117 | result = send_files_to_station(object_id, source_files, to_station, local_station) |
| 118 | except requests.RequestException as e: |
| 119 | return json_error(f'Failed to send files to destination: {e}', status=503) |
| 120 | except Exception as e: |
| 121 | return json_error(f'Failed to send files: {e}', status=500) |
| 122 | |
| 123 | # Step 3: (Optional) Delete from source if not copy_only |
| 124 | if not copy_only: |
| 125 | # TODO: Implement source deletion |
| 126 | pass |
| 127 | |
| 128 | duration = time.time() - start_time |
| 129 | |
| 130 | return json_response(json.dumps({ |
| 131 | 'status': 'ok', |
| 132 | 'message': 'Object migrated successfully' if not copy_only else 'Object copied successfully', |
| 133 | 'object_id': object_id, |
| 134 | 'from_station': from_station, |
| 135 | 'to_station': to_station, |
| 136 | 'files_copied': result['files_copied'], |
| 137 | 'duration_seconds': round(duration, 3) |
| 138 | })) |
| 139 | |
| 140 | |
| 141 | def collect_object_files(object_id: str, source_station: str, local_station: str) -> dict: |
| 142 | """ |
| 143 | Collect all files for an object from source station |
| 144 | |
| 145 | Args: |
| 146 | object_id: Object ID to collect |
| 147 | source_station: Station where object currently lives |
| 148 | local_station: This station's ID |
| 149 | |
| 150 | Returns: |
| 151 | { |
| 152 | 'code_file': str, # Path to code file |
| 153 | 'code_content': bytes, # Code file content |
| 154 | 'state_files': { # State files (path -> content) |
| 155 | 'state.tsv': bytes, |
| 156 | 'logs.tsv': bytes, |
| 157 | ... |
| 158 | }, |
| 159 | 'version_files': { # Version files (path -> content) |
| 160 | 'v1.txt': bytes, |
| 161 | 'v2.txt': bytes, |
| 162 | 'metadata.tsv': bytes |
| 163 | } |
| 164 | } |
| 165 | """ |
| 166 | if source_station == local_station: |
| 167 | # Local collection - read files directly |
| 168 | return collect_local_files(object_id) |
| 169 | else: |
| 170 | # Remote collection - request files via HTTP |
| 171 | return collect_remote_files(object_id, source_station) |
| 172 | |
| 173 | |
| 174 | def collect_local_files(object_id: str) -> dict: |
| 175 | """Collect files from local filesystem""" |
| 176 | import importlib.util |
| 177 | |
| 178 | # Load the [id].py module |
| 179 | module_path = Path(__file__).parent.parent / 'objects' / '[id].py' |
| 180 | spec = importlib.util.spec_from_file_location('objects_id_api', module_path) |
| 181 | objects_api = importlib.util.module_from_spec(spec) |
| 182 | spec.loader.exec_module(objects_api) |
| 183 | |
| 184 | # Find object code file |
| 185 | obj_file = objects_api.find_object_file(object_id) |
| 186 | if not obj_file: |
| 187 | raise FileNotFoundError(f'Object not found: {object_id}') |
| 188 | |
| 189 | # Read code file |
| 190 | with open(obj_file, 'rb') as f: |
| 191 | code_content = f.read() |
| 192 | |
| 193 | # Collect state files |
| 194 | state_dir = Path('data') / object_id |
| 195 | state_files = {} |
| 196 | |
| 197 | if state_dir.exists(): |
| 198 | for file_path in state_dir.iterdir(): |
| 199 | if file_path.is_file(): |
| 200 | with open(file_path, 'rb') as f: |
| 201 | rel_path = file_path.name |
| 202 | state_files[rel_path] = f.read() |
| 203 | |
| 204 | # Collect version files |
| 205 | version_dir = Path('data/versions') / object_id |
| 206 | version_files = {} |
| 207 | |
| 208 | if version_dir.exists(): |
| 209 | for file_path in version_dir.rglob('*'): |
| 210 | if file_path.is_file(): |
| 211 | with open(file_path, 'rb') as f: |
| 212 | rel_path = str(file_path.relative_to(version_dir)) |
| 213 | version_files[rel_path] = f.read() |
| 214 | |
| 215 | return { |
| 216 | 'code_file': str(obj_file), |
| 217 | 'code_content': code_content, |
| 218 | 'state_files': state_files, |
| 219 | 'version_files': version_files |
| 220 | } |
| 221 | |
| 222 | |
| 223 | def collect_remote_files(object_id: str, source_station: str) -> dict: |
| 224 | """Collect files from remote station via HTTP""" |
| 225 | import importlib.util |
| 226 | |
| 227 | # Load the [id].py module |
| 228 | module_path = Path(__file__).parent.parent / 'objects' / '[id].py' |
| 229 | spec = importlib.util.spec_from_file_location('objects_id_api', module_path) |
| 230 | objects_api = importlib.util.module_from_spec(spec) |
| 231 | spec.loader.exec_module(objects_api) |
| 232 | |
| 233 | # Look up source station |
| 234 | station_info = objects_api.get_station_info(source_station) |
| 235 | |
| 236 | if not station_info: |
| 237 | raise ValueError(f'Station not found or offline: {source_station}') |
| 238 | |
| 239 | # Request files from source station |
| 240 | # TODO: Create /cluster/export endpoint on source station |
| 241 | # For now, raise NotImplementedError |
| 242 | raise NotImplementedError('Remote file collection not yet implemented') |
| 243 | |
| 244 | |
| 245 | def send_files_to_station(object_id: str, files: dict, dest_station: str, local_station: str) -> dict: |
| 246 | """ |
| 247 | Send files to destination station |
| 248 | |
| 249 | Args: |
| 250 | object_id: Object ID |
| 251 | files: Files dict from collect_object_files() |
| 252 | dest_station: Destination station ID |
| 253 | local_station: This station's ID |
| 254 | |
| 255 | Returns: |
| 256 | { |
| 257 | 'files_copied': { |
| 258 | 'code': str, |
| 259 | 'state': [str], |
| 260 | 'versions': int |
| 261 | } |
| 262 | } |
| 263 | """ |
| 264 | if dest_station == local_station: |
| 265 | # Local write - write files directly |
| 266 | return write_local_files(object_id, files) |
| 267 | else: |
| 268 | # Remote write - send files via HTTP |
| 269 | return send_remote_files(object_id, files, dest_station) |
| 270 | |
| 271 | |
| 272 | def write_local_files(object_id: str, files: dict) -> dict: |
| 273 | """Write files to local filesystem""" |
| 274 | # Determine where to write code file |
| 275 | # For now, write to examples/ directory |
| 276 | code_file = Path(files['code_file']) |
| 277 | |
| 278 | # Create parent directory if needed |
| 279 | code_file.parent.mkdir(parents=True, exist_ok=True) |
| 280 | |
| 281 | # Write code file |
| 282 | with open(code_file, 'wb') as f: |
| 283 | f.write(files['code_content']) |
| 284 | |
| 285 | # Write state files |
| 286 | state_dir = Path('data') / object_id |
| 287 | state_dir.mkdir(parents=True, exist_ok=True) |
| 288 | |
| 289 | state_files_written = [] |
| 290 | for filename, content in files['state_files'].items(): |
| 291 | file_path = state_dir / filename |
| 292 | with open(file_path, 'wb') as f: |
| 293 | f.write(content) |
| 294 | state_files_written.append(filename) |
| 295 | |
| 296 | # Write version files |
| 297 | version_dir = Path('data/versions') / object_id |
| 298 | version_dir.mkdir(parents=True, exist_ok=True) |
| 299 | |
| 300 | version_count = 0 |
| 301 | for rel_path, content in files['version_files'].items(): |
| 302 | file_path = version_dir / rel_path |
| 303 | file_path.parent.mkdir(parents=True, exist_ok=True) |
| 304 | with open(file_path, 'wb') as f: |
| 305 | f.write(content) |
| 306 | if rel_path.endswith('.txt'): # Count version files, not metadata |
| 307 | version_count += 1 |
| 308 | |
| 309 | return { |
| 310 | 'files_copied': { |
| 311 | 'code': str(code_file), |
| 312 | 'state': state_files_written, |
| 313 | 'versions': version_count |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | |
| 318 | def send_remote_files(object_id: str, files: dict, dest_station: str) -> dict: |
| 319 | """Send files to remote station via HTTP""" |
| 320 | import importlib.util |
| 321 | |
| 322 | # Load the [id].py module |
| 323 | module_path = Path(__file__).parent.parent / 'objects' / '[id].py' |
| 324 | spec = importlib.util.spec_from_file_location('objects_id_api', module_path) |
| 325 | objects_api = importlib.util.module_from_spec(spec) |
| 326 | spec.loader.exec_module(objects_api) |
| 327 | |
| 328 | # Look up destination station |
| 329 | station_info = objects_api.get_station_info(dest_station) |
| 330 | |
| 331 | if not station_info: |
| 332 | raise ValueError(f'Station not found or offline: {dest_station}') |
| 333 | |
| 334 | # Prepare payload (encode binary files as base64) |
| 335 | payload = { |
| 336 | 'object_id': object_id, |
| 337 | 'code_file': files['code_file'], |
| 338 | 'code_content': base64.b64encode(files['code_content']).decode('utf-8'), |
| 339 | 'state_files': { |
| 340 | name: base64.b64encode(content).decode('utf-8') |
| 341 | for name, content in files['state_files'].items() |
| 342 | }, |
| 343 | 'version_files': { |
| 344 | name: base64.b64encode(content).decode('utf-8') |
| 345 | for name, content in files['version_files'].items() |
| 346 | } |
| 347 | } |
| 348 | |
| 349 | # Send to destination station |
| 350 | url = f"{station_info['url']}/cluster/import" |
| 351 | response = requests.post(url, json=payload, timeout=60) |
| 352 | |
| 353 | if response.status_code != 200: |
| 354 | raise ValueError(f'Destination returned error: {response.status_code}') |
| 355 | |
| 356 | result = response.json() |
| 357 | return result |