Python functions as REST objects, with clustering, load balancing and state replication; no containers or orchestration.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-primitive.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Q Consolidate duplicate QUICKSTARTs on the top-level file 503f830 7 months ago History
migrate.py · 11.2 KB · 357 lines
1"""
2Object Migration API
3
4POST /cluster/migrate - Migrate object from one station to another
5
6Migrates an object by copying:
7- Code file (examples/path/to/object.py)
8- State directory (data/object_id/)
9 - state.tsv
10 - logs.tsv
11 - versions/
12
13Example:
14 POST /cluster/migrate
15 {
16 "object_id": "calculator",
17 "from_station": "station2",
18 "to_station": "station3"
19 }
20
21 Response:
22 {
23 "status": "ok",
24 "message": "Object migrated successfully",
25 "object_id": "calculator",
26 "from_station": "station2",
27 "to_station": "station3",
28 "files_copied": {
29 "code": "examples/calculator.py",
30 "state": ["state.tsv", "logs.tsv"],
31 "versions": 5
32 }
33 }
34"""
35import json
36import os
37import time
38import base64
39import requests
40from pathlib import Path
41from dbbasic_web.responses import json as json_response, json_error
42
43
44def POST(request):
45 """
46 Migrate object from one station to another
47
48 Request body:
49 {
50 "object_id": str, # Object to migrate
51 "from_station": str, # Source station ID
52 "to_station": str, # Destination station ID
53 "copy_only": bool # If true, don't delete from source (default: false)
54 }
55
56 Response:
57 {
58 "status": "ok",
59 "message": "Object migrated successfully",
60 "object_id": str,
61 "from_station": str,
62 "to_station": str,
63 "files_copied": {
64 "code": str, # Code file path
65 "state": [str], # List of state files
66 "versions": int # Number of versions copied
67 },
68 "duration_seconds": float
69 }
70
71 Errors:
72 400 - Missing required fields
73 404 - Object not found
74 503 - Station not found or offline
75 500 - Migration failed
76 """
77 start_time = time.time()
78
79 # Parse request body
80 try:
81 if request.body:
82 data = json.loads(request.body.decode('utf-8'))
83 else:
84 return json_error('Missing request body', status=400)
85 except json.JSONDecodeError as e:
86 return json_error(f'Invalid JSON: {e}', status=400)
87
88 # Validate required fields
89 object_id = data.get('object_id')
90 from_station = data.get('from_station')
91 to_station = data.get('to_station')
92 copy_only = data.get('copy_only', False)
93
94 if not object_id:
95 return json_error('object_id is required', status=400)
96 if not from_station:
97 return json_error('from_station is required', status=400)
98 if not to_station:
99 return json_error('to_station is required', status=400)
100
101 # Get local station ID
102 local_station = os.environ.get('STATION_ID', 'unknown')
103
104 # TODO: If this is not the master, forward to master
105 # For now, assume this runs on master (station1)
106
107 # Step 1: Collect files from source station
108 try:
109 source_files = collect_object_files(object_id, from_station, local_station)
110 except FileNotFoundError as e:
111 return json_error(f'Object not found: {e}', status=404)
112 except Exception as e:
113 return json_error(f'Failed to collect files from source: {e}', status=500)
114
115 # Step 2: Send files to destination station
116 try:
117 result = send_files_to_station(object_id, source_files, to_station, local_station)
118 except requests.RequestException as e:
119 return json_error(f'Failed to send files to destination: {e}', status=503)
120 except Exception as e:
121 return json_error(f'Failed to send files: {e}', status=500)
122
123 # Step 3: (Optional) Delete from source if not copy_only
124 if not copy_only:
125 # TODO: Implement source deletion
126 pass
127
128 duration = time.time() - start_time
129
130 return json_response(json.dumps({
131 'status': 'ok',
132 'message': 'Object migrated successfully' if not copy_only else 'Object copied successfully',
133 'object_id': object_id,
134 'from_station': from_station,
135 'to_station': to_station,
136 'files_copied': result['files_copied'],
137 'duration_seconds': round(duration, 3)
138 }))
139
140
141def collect_object_files(object_id: str, source_station: str, local_station: str) -> dict:
142 """
143 Collect all files for an object from source station
144
145 Args:
146 object_id: Object ID to collect
147 source_station: Station where object currently lives
148 local_station: This station's ID
149
150 Returns:
151 {
152 'code_file': str, # Path to code file
153 'code_content': bytes, # Code file content
154 'state_files': { # State files (path -> content)
155 'state.tsv': bytes,
156 'logs.tsv': bytes,
157 ...
158 },
159 'version_files': { # Version files (path -> content)
160 'v1.txt': bytes,
161 'v2.txt': bytes,
162 'metadata.tsv': bytes
163 }
164 }
165 """
166 if source_station == local_station:
167 # Local collection - read files directly
168 return collect_local_files(object_id)
169 else:
170 # Remote collection - request files via HTTP
171 return collect_remote_files(object_id, source_station)
172
173
174def collect_local_files(object_id: str) -> dict:
175 """Collect files from local filesystem"""
176 import importlib.util
177
178 # Load the [id].py module
179 module_path = Path(__file__).parent.parent / 'objects' / '[id].py'
180 spec = importlib.util.spec_from_file_location('objects_id_api', module_path)
181 objects_api = importlib.util.module_from_spec(spec)
182 spec.loader.exec_module(objects_api)
183
184 # Find object code file
185 obj_file = objects_api.find_object_file(object_id)
186 if not obj_file:
187 raise FileNotFoundError(f'Object not found: {object_id}')
188
189 # Read code file
190 with open(obj_file, 'rb') as f:
191 code_content = f.read()
192
193 # Collect state files
194 state_dir = Path('data') / object_id
195 state_files = {}
196
197 if state_dir.exists():
198 for file_path in state_dir.iterdir():
199 if file_path.is_file():
200 with open(file_path, 'rb') as f:
201 rel_path = file_path.name
202 state_files[rel_path] = f.read()
203
204 # Collect version files
205 version_dir = Path('data/versions') / object_id
206 version_files = {}
207
208 if version_dir.exists():
209 for file_path in version_dir.rglob('*'):
210 if file_path.is_file():
211 with open(file_path, 'rb') as f:
212 rel_path = str(file_path.relative_to(version_dir))
213 version_files[rel_path] = f.read()
214
215 return {
216 'code_file': str(obj_file),
217 'code_content': code_content,
218 'state_files': state_files,
219 'version_files': version_files
220 }
221
222
223def collect_remote_files(object_id: str, source_station: str) -> dict:
224 """Collect files from remote station via HTTP"""
225 import importlib.util
226
227 # Load the [id].py module
228 module_path = Path(__file__).parent.parent / 'objects' / '[id].py'
229 spec = importlib.util.spec_from_file_location('objects_id_api', module_path)
230 objects_api = importlib.util.module_from_spec(spec)
231 spec.loader.exec_module(objects_api)
232
233 # Look up source station
234 station_info = objects_api.get_station_info(source_station)
235
236 if not station_info:
237 raise ValueError(f'Station not found or offline: {source_station}')
238
239 # Request files from source station
240 # TODO: Create /cluster/export endpoint on source station
241 # For now, raise NotImplementedError
242 raise NotImplementedError('Remote file collection not yet implemented')
243
244
245def send_files_to_station(object_id: str, files: dict, dest_station: str, local_station: str) -> dict:
246 """
247 Send files to destination station
248
249 Args:
250 object_id: Object ID
251 files: Files dict from collect_object_files()
252 dest_station: Destination station ID
253 local_station: This station's ID
254
255 Returns:
256 {
257 'files_copied': {
258 'code': str,
259 'state': [str],
260 'versions': int
261 }
262 }
263 """
264 if dest_station == local_station:
265 # Local write - write files directly
266 return write_local_files(object_id, files)
267 else:
268 # Remote write - send files via HTTP
269 return send_remote_files(object_id, files, dest_station)
270
271
272def write_local_files(object_id: str, files: dict) -> dict:
273 """Write files to local filesystem"""
274 # Determine where to write code file
275 # For now, write to examples/ directory
276 code_file = Path(files['code_file'])
277
278 # Create parent directory if needed
279 code_file.parent.mkdir(parents=True, exist_ok=True)
280
281 # Write code file
282 with open(code_file, 'wb') as f:
283 f.write(files['code_content'])
284
285 # Write state files
286 state_dir = Path('data') / object_id
287 state_dir.mkdir(parents=True, exist_ok=True)
288
289 state_files_written = []
290 for filename, content in files['state_files'].items():
291 file_path = state_dir / filename
292 with open(file_path, 'wb') as f:
293 f.write(content)
294 state_files_written.append(filename)
295
296 # Write version files
297 version_dir = Path('data/versions') / object_id
298 version_dir.mkdir(parents=True, exist_ok=True)
299
300 version_count = 0
301 for rel_path, content in files['version_files'].items():
302 file_path = version_dir / rel_path
303 file_path.parent.mkdir(parents=True, exist_ok=True)
304 with open(file_path, 'wb') as f:
305 f.write(content)
306 if rel_path.endswith('.txt'): # Count version files, not metadata
307 version_count += 1
308
309 return {
310 'files_copied': {
311 'code': str(code_file),
312 'state': state_files_written,
313 'versions': version_count
314 }
315 }
316
317
318def send_remote_files(object_id: str, files: dict, dest_station: str) -> dict:
319 """Send files to remote station via HTTP"""
320 import importlib.util
321
322 # Load the [id].py module
323 module_path = Path(__file__).parent.parent / 'objects' / '[id].py'
324 spec = importlib.util.spec_from_file_location('objects_id_api', module_path)
325 objects_api = importlib.util.module_from_spec(spec)
326 spec.loader.exec_module(objects_api)
327
328 # Look up destination station
329 station_info = objects_api.get_station_info(dest_station)
330
331 if not station_info:
332 raise ValueError(f'Station not found or offline: {dest_station}')
333
334 # Prepare payload (encode binary files as base64)
335 payload = {
336 'object_id': object_id,
337 'code_file': files['code_file'],
338 'code_content': base64.b64encode(files['code_content']).decode('utf-8'),
339 'state_files': {
340 name: base64.b64encode(content).decode('utf-8')
341 for name, content in files['state_files'].items()
342 },
343 'version_files': {
344 name: base64.b64encode(content).decode('utf-8')
345 for name, content in files['version_files'].items()
346 }
347 }
348
349 # Send to destination station
350 url = f"{station_info['url']}/cluster/import"
351 response = requests.post(url, json=payload, timeout=60)
352
353 if response.status_code != 200:
354 raise ValueError(f'Destination returned error: {response.status_code}')
355
356 result = response.json()
357 return result