Python functions as REST objects, with clustering, load balancing and state replication; no containers or orchestration.

0 downloads

dbbasic-object-primitive/tests/integration/test_cluster_integration.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-primitive.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Q Consolidate duplicate QUICKSTARTs on the top-level file 503f830 7 months ago History
test_cluster_integration.py · 16.0 KB · 443 lines
1"""
2Integration tests for 3-station cluster.
3
4Runs a full 3-station cluster locally on high ports (9001-9003)
5to test distributed features without requiring SSH or remote machines.
6
7Tests:
8- State replication across all 3 stations
9- Log replication consistency
10- Load balancing
11- Failover scenarios
12"""
13
14import pytest
15import subprocess
16import time
17import tempfile
18import shutil
19import os
20import requests
21from pathlib import Path
22
23
24class LocalCluster:
25 """Manages a 3-station cluster running locally on different ports."""
26
27 def __init__(self):
28 self.base_dir = None
29 self.stations = []
30 self.server_processes = []
31 self.heartbeat_processes = []
32
33 def setup(self):
34 """Start a 3-station cluster locally."""
35 # Create temp directory for cluster
36 self.base_dir = Path(tempfile.mkdtemp(prefix="test_cluster_"))
37 print(f"Test cluster base: {self.base_dir}")
38
39 # Define stations
40 self.stations = [
41 {"id": "station1", "port": 9001, "is_master": True},
42 {"id": "station2", "port": 9002, "is_master": False},
43 {"id": "station3", "port": 9003, "is_master": False},
44 ]
45
46 # Get source directory (project root)
47 source_dir = Path(__file__).parent.parent.parent
48
49 # Create station directories with necessary files
50 for station in self.stations:
51 station_dir = self.base_dir / station["id"]
52 station_dir.mkdir()
53
54 # Create data directories
55 (station_dir / "data" / "cluster").mkdir(parents=True)
56 (station_dir / "data" / "state").mkdir(parents=True)
57 (station_dir / "data" / "logs").mkdir(parents=True)
58 (station_dir / "data" / "versions").mkdir(parents=True)
59
60 # Copy entire src, api, examples directories (preserve structure)
61 for dir_name in ["src", "api", "examples"]:
62 src_path = source_dir / dir_name
63 if src_path.exists():
64 shutil.copytree(src_path, station_dir / dir_name)
65
66 # Copy scripts
67 for script in ["run_server.py", "cluster_heartbeat_daemon.py"]:
68 src_script = source_dir / script
69 if src_script.exists():
70 shutil.copy(src_script, station_dir / script)
71
72 station["dir"] = station_dir
73
74 # Initialize cluster registry on master (station1)
75 self._init_registry()
76
77 # Start all servers
78 for station in self.stations:
79 self._start_server(station)
80
81 # Wait for servers to start
82 time.sleep(3)
83
84 # Verify all servers are running
85 for station in self.stations:
86 url = f"http://localhost:{station['port']}/cluster/info"
87 resp = requests.get(url, timeout=5)
88 assert resp.status_code == 200, f"{station['id']} not responding"
89 print(f"✓ {station['id']} running on port {station['port']}")
90
91 # Start heartbeat daemons (workers only, master doesn't send heartbeats)
92 for station in self.stations:
93 if not station["is_master"]:
94 self._start_heartbeat(station)
95
96 # Wait for heartbeats to register
97 time.sleep(3)
98
99 print("✓ Test cluster ready")
100
101 def _init_registry(self):
102 """Initialize cluster registry on master station."""
103 registry_path = self.stations[0]["dir"] / "data" / "cluster" / "stations.tsv"
104
105 with open(registry_path, "w") as f:
106 # Write header
107 f.write("station_id\taddress\tport\tlast_heartbeat\tis_active\n")
108
109 # Write all stations
110 for station in self.stations:
111 # Master starts as active, workers will activate via heartbeat
112 is_active = "True" if station["is_master"] else "False"
113 f.write(f"{station['id']}\tlocalhost\t{station['port']}\t0\t{is_active}\n")
114
115 def _start_server(self, station):
116 """Start server process for a station."""
117 env = os.environ.copy()
118 env["STATION_ID"] = station["id"]
119 env["SERVER_PORT"] = str(station["port"]) # For SelfLogger to query correct port
120
121 # Workers need to know where the master is (for log replication)
122 if not station["is_master"]:
123 master_station = self.stations[0] # station1 is always master
124 env["MASTER_HOST"] = "localhost"
125 env["MASTER_PORT"] = str(master_station["port"])
126
127 # Add packages directory to PYTHONPATH so server can import dbbasic packages
128 source_dir = Path(__file__).parent.parent.parent
129 packages_dir = source_dir / "packages"
130 if "PYTHONPATH" in env:
131 env["PYTHONPATH"] = f"{packages_dir}:{env['PYTHONPATH']}"
132 else:
133 env["PYTHONPATH"] = str(packages_dir)
134
135 cmd = [
136 "python",
137 "run_server.py",
138 "--port", str(station["port"])
139 ]
140
141 log_file = open(station["dir"] / "server.log", "w")
142
143 proc = subprocess.Popen(
144 cmd,
145 cwd=station["dir"], # Run from station directory
146 env=env,
147 stdout=log_file,
148 stderr=subprocess.STDOUT,
149 start_new_session=True # Detach from parent
150 )
151
152 self.server_processes.append((proc, log_file))
153 station["server_pid"] = proc.pid
154 print(f"Started {station['id']} server (PID {proc.pid})")
155
156 def _start_heartbeat(self, station):
157 """Start heartbeat daemon for a worker station."""
158 env = os.environ.copy()
159 env["STATION_ID"] = station["id"]
160
161 # Add packages directory to PYTHONPATH
162 source_dir = Path(__file__).parent.parent.parent
163 packages_dir = source_dir / "packages"
164 if "PYTHONPATH" in env:
165 env["PYTHONPATH"] = f"{packages_dir}:{env['PYTHONPATH']}"
166 else:
167 env["PYTHONPATH"] = str(packages_dir)
168
169 master_station = self.stations[0] # station1 is master
170 master_addr = "localhost"
171
172 cmd = [
173 "python",
174 "cluster_heartbeat_daemon.py",
175 master_addr,
176 "--master-port", str(master_station["port"]),
177 "--station-port", str(station["port"])
178 ]
179
180 log_file = open(station["dir"] / "heartbeat.log", "w")
181
182 proc = subprocess.Popen(
183 cmd,
184 cwd=station["dir"],
185 env=env,
186 stdout=log_file,
187 stderr=subprocess.STDOUT,
188 start_new_session=True
189 )
190
191 self.heartbeat_processes.append((proc, log_file))
192 station["heartbeat_pid"] = proc.pid
193 print(f"Started {station['id']} heartbeat (PID {proc.pid})")
194
195 def teardown(self):
196 """Stop all processes and cleanup."""
197 print("Tearing down test cluster...")
198
199 # Kill all processes
200 for proc, log_file in self.server_processes + self.heartbeat_processes:
201 try:
202 proc.terminate()
203 proc.wait(timeout=5)
204 except subprocess.TimeoutExpired:
205 proc.kill()
206 finally:
207 log_file.close()
208
209 # Clean up temp directory
210 if self.base_dir and self.base_dir.exists():
211 shutil.rmtree(self.base_dir)
212 print(f"✓ Cleaned up {self.base_dir}")
213
214 def get_url(self, station_id, path):
215 """Get full URL for a path on a station."""
216 station = next(s for s in self.stations if s["id"] == station_id)
217 return f"http://localhost:{station['port']}{path}"
218
219 def get_master_url(self, path):
220 """Get full URL for a path on the master station."""
221 return self.get_url("station1", path)
222
223
224@pytest.fixture(scope="module")
225def cluster():
226 """Pytest fixture that provides a running 3-station cluster."""
227 c = LocalCluster()
228 c.setup()
229 yield c
230 c.teardown()
231
232
233class TestClusterBasics:
234 """Test basic cluster operations."""
235
236 def test_all_stations_running(self, cluster):
237 """Verify all 3 stations are running."""
238 for station in cluster.stations:
239 url = cluster.get_url(station["id"], "/cluster/info")
240 resp = requests.get(url, timeout=5)
241 assert resp.status_code == 200
242 data = resp.json()
243 assert data["station_id"] == station["id"]
244
245 def test_registry_has_all_stations(self, cluster):
246 """Verify cluster registry knows about all stations."""
247 url = cluster.get_master_url("/cluster/stations")
248 resp = requests.get(url, timeout=5)
249
250 # Print error details if request failed
251 if resp.status_code != 200:
252 print(f"\nError: {resp.status_code}")
253 print(f"Response: {resp.text}")
254
255 assert resp.status_code == 200
256
257 data = resp.json()
258 assert data["status"] == "ok"
259 assert len(data["stations"]) == 3
260
261 station_ids = [s["station_id"] for s in data["stations"]]
262 assert "station1" in station_ids
263 assert "station2" in station_ids
264 assert "station3" in station_ids
265
266 def test_workers_are_active(self, cluster):
267 """Verify worker stations registered via heartbeat."""
268 # Give heartbeats time to register
269 time.sleep(5)
270
271 url = cluster.get_master_url("/cluster/stations")
272 resp = requests.get(url, timeout=5)
273 data = resp.json()
274
275 for station in data["stations"]:
276 if station["station_id"] in ["station2", "station3"]:
277 assert station["is_active"], f"{station['station_id']} should be active"
278
279
280class TestStateReplication:
281 """Test state replication across stations."""
282
283 def test_state_replicates_to_all_stations(self, cluster):
284 """Write state on station1, verify it appears on all stations."""
285 # Create a test object on station1
286 test_value = int(time.time())
287
288 # Write to counter object on station1
289 url = cluster.get_master_url("/objects/basics_counter")
290 resp = requests.post(
291 url,
292 json={"value": test_value},
293 headers={"Content-Type": "application/json"},
294 timeout=5
295 )
296
297 # Debug: print response details
298 print(f"\nPOST {url}")
299 print(f"Status: {resp.status_code}")
300 print(f"Response: {resp.text[:500]}")
301
302 assert resp.status_code == 200
303
304 # Wait for replication
305 time.sleep(2)
306
307 # Debug: Check what's in the data directory
308 print(f"\nChecking data directories on station1...")
309 station1_data = cluster.stations[0]["dir"] / "data"
310 if station1_data.exists():
311 import os
312 for root, dirs, files in os.walk(station1_data):
313 rel_path = os.path.relpath(root, station1_data)
314 print(f" {rel_path}/ - dirs: {dirs}, files: {files}")
315 else:
316 print(f" data/ directory doesn't exist!")
317
318 # Read state file on all 3 stations
319 # NOTE: StateManager uses filename instead of full path ID, so "counter" not "basics_counter"
320 for station in cluster.stations:
321 state_file = station["dir"] / "data" / "state" / "counter" / "state.tsv"
322
323 # State file should exist
324 assert state_file.exists(), f"State file missing on {station['id']}"
325
326 # Read state value
327 with open(state_file) as f:
328 lines = f.readlines()
329 assert len(lines) >= 1 # Just the data line (no header)
330
331 # Parse TSV (format: count\tVALUE\tTIMESTAMP)
332 data_line = lines[0].strip().split("\t")
333 actual_value = int(data_line[1])
334
335 assert actual_value == test_value, \
336 f"State mismatch on {station['id']}: expected {test_value}, got {actual_value}"
337
338 print(f"✓ State {test_value} replicated to all 3 stations")
339
340
341class TestLogReplication:
342 """Test log replication across stations."""
343
344 def test_logs_replicate_to_all_stations(self, cluster):
345 """Write logs on station2, verify they appear on all stations."""
346 # Get current log count on all stations
347 # NOTE: StateManager uses filename instead of full path ID, so "counter" not "basics_counter"
348 initial_counts = {}
349 for station in cluster.stations:
350 log_file = station["dir"] / "data" / "logs" / "counter" / "log.tsv"
351 if log_file.exists():
352 with open(log_file) as f:
353 initial_counts[station["id"]] = len(f.readlines())
354 else:
355 initial_counts[station["id"]] = 0
356
357 # Trigger some operations that generate logs (via station2)
358 url = cluster.get_url("station2", "/objects/basics_counter")
359 for i in range(5):
360 resp = requests.post(
361 url,
362 json={"value": 1000 + i},
363 headers={"Content-Type": "application/json"},
364 timeout=5
365 )
366 assert resp.status_code == 200
367
368 # Wait for log replication
369 time.sleep(3)
370
371 # First check if logs were created on station2 (where requests were made)
372 station2_log_file = cluster.stations[1]["dir"] / "data" / "logs" / "counter" / "log.tsv"
373 print(f"\nChecking if logs were created on station2...")
374 print(f"Log file path: {station2_log_file}")
375 print(f"Exists: {station2_log_file.exists()}")
376
377 if station2_log_file.exists():
378 with open(station2_log_file) as f:
379 lines = f.readlines()
380 print(f"Station2 has {len(lines)} log lines")
381 if len(lines) > 0:
382 print(f"First line: {lines[0][:100]}")
383 else:
384 # Log file doesn't exist on station2 - logging isn't working at all!
385 # Let's check what directories do exist
386 station2_data = cluster.stations[1]["dir"] / "data"
387 print(f"\nContents of {station2_data}:")
388 import os
389 for root, dirs, files in os.walk(station2_data):
390 rel_path = os.path.relpath(root, station2_data)
391 print(f" {rel_path}/ - dirs: {dirs}, files: {files}")
392
393 # Require logs to exist on station2 first (origin station)
394 assert station2_log_file.exists(), "Logs not created on origin station (station2)"
395
396 with open(station2_log_file) as f:
397 station2_count = len(f.readlines())
398 assert station2_count > initial_counts["station2"], \
399 f"No new logs on station2 (initial: {initial_counts['station2']}, final: {station2_count})"
400
401 # Now check all stations have new logs
402 for station in cluster.stations:
403 log_file = station["dir"] / "data" / "logs" / "counter" / "log.tsv"
404 assert log_file.exists(), f"Log file missing on {station['id']}"
405
406 with open(log_file) as f:
407 final_count = len(f.readlines())
408
409 # Should have at least some of the 5 new logs
410 # (may not be all due to fire-and-forget nature)
411 new_logs = final_count - initial_counts[station["id"]]
412 assert new_logs > 0, \
413 f"{station['id']} has no new logs (initial: {initial_counts[station['id']]}, final: {final_count})"
414
415 print("✓ Logs replicated to all stations")
416
417
418class TestLoadBalancing:
419 """Test load balancing across stations."""
420
421 def test_stations_report_load_metrics(self, cluster):
422 """Verify stations report CPU and memory metrics."""
423 # Wait for heartbeats with metrics
424 time.sleep(5)
425
426 url = cluster.get_master_url("/cluster/stations")
427 resp = requests.get(url, timeout=5)
428 data = resp.json()
429
430 for station in data["stations"]:
431 if station["is_active"]:
432 # Workers send metrics, master might not have them
433 if station["station_id"] != "station1":
434 assert "metrics" in station, \
435 f"{station['station_id']} missing metrics dict"
436 metrics = station["metrics"]
437 assert "cpu_percent" in metrics or "memory_percent" in metrics, \
438 f"{station['station_id']} missing load metrics in metrics dict"
439
440
441if __name__ == "__main__":
442 # Allow running tests directly
443 pytest.main([__file__, "-v", "-s"])