Python functions as REST objects, with clustering, load balancing and state replication; no containers or orchestration.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-primitive.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Q Consolidate duplicate QUICKSTARTs on the top-level file 503f830 7 months ago History
test_cluster.py · 13.8 KB · 425 lines
1"""
2Tests for cluster functionality (Phase 7.1)
3
4These tests verify:
5- Station registry
6- Heartbeat system
7- Station info
8- Health monitoring
9- Failure detection
10
11Can run on single machine (doesn't require multi-node setup)
12"""
13import json
14import os
15import time
16import pytest
17from pathlib import Path
18import shutil
19
20
21@pytest.fixture
22def cluster_data_dir(tmp_path):
23 """Create temporary cluster data directory"""
24 data_dir = tmp_path / "data" / "cluster"
25 data_dir.mkdir(parents=True)
26 return data_dir
27
28
29@pytest.fixture
30def mock_registry_file(cluster_data_dir):
31 """Create mock station registry file"""
32 registry_file = cluster_data_dir / "stations.tsv"
33 current_time = time.time()
34
35 # Write 3 mock stations
36 with open(registry_file, 'w') as f:
37 f.write(f"station1\tlocalhost\t8001\t{current_time}\n")
38 f.write(f"station2\t192.0.2.2\t8001\t{current_time}\n")
39 f.write(f"station3\t192.0.2.3\t8001\t{current_time - 100}\n") # Old heartbeat
40
41 return registry_file
42
43
44class TestClusterRegistry:
45 """Test cluster station registry"""
46
47 def test_registry_format(self, mock_registry_file):
48 """Test registry file format (TSV)"""
49 with open(mock_registry_file, 'r') as f:
50 lines = f.readlines()
51
52 assert len(lines) == 3
53
54 # Each line should have 4 fields
55 for line in lines:
56 parts = line.strip().split('\t')
57 assert len(parts) == 4
58 station_id, host, port, timestamp = parts
59 assert station_id.startswith('station')
60 assert host
61 assert port.isdigit()
62 assert float(timestamp) > 0
63
64 def test_active_station_detection(self, mock_registry_file):
65 """Test detecting active vs inactive stations"""
66 current_time = time.time()
67 timeout = 30 # seconds
68
69 with open(mock_registry_file, 'r') as f:
70 for line in f:
71 parts = line.strip().split('\t')
72 station_id, host, port, last_heartbeat = parts
73
74 age = current_time - float(last_heartbeat)
75 is_active = age < timeout
76
77 if station_id == 'station3':
78 # Old heartbeat (100 seconds ago)
79 assert not is_active
80 else:
81 # Recent heartbeat
82 assert is_active
83
84 def test_add_station_to_registry(self, cluster_data_dir):
85 """Test adding a new station to registry"""
86 registry_file = cluster_data_dir / "stations.tsv"
87 current_time = time.time()
88
89 # Write initial station
90 with open(registry_file, 'w') as f:
91 f.write(f"station1\tlocalhost\t8001\t{current_time}\n")
92
93 # Read existing
94 existing = {}
95 with open(registry_file, 'r') as f:
96 for line in f:
97 parts = line.strip().split('\t')
98 if len(parts) == 4:
99 sid = parts[0]
100 existing[sid] = parts
101
102 # Add new station
103 existing['station2'] = ['station2', '192.0.2.2', '8001', str(current_time)]
104
105 # Write back
106 with open(registry_file, 'w') as f:
107 for sid, parts in sorted(existing.items()):
108 f.write('\t'.join(parts) + '\n')
109
110 # Verify
111 with open(registry_file, 'r') as f:
112 lines = f.readlines()
113
114 assert len(lines) == 2
115 assert 'station1' in lines[0]
116 assert 'station2' in lines[1]
117
118 def test_update_existing_station(self, mock_registry_file, cluster_data_dir):
119 """Test updating existing station (heartbeat)"""
120 registry_file = mock_registry_file
121 new_time = time.time()
122
123 # Read existing
124 existing = {}
125 with open(registry_file, 'r') as f:
126 for line in f:
127 parts = line.strip().split('\t')
128 if len(parts) == 4:
129 sid = parts[0]
130 existing[sid] = parts
131
132 # Update station2 heartbeat
133 old_timestamp = existing['station2'][3]
134 existing['station2'][3] = str(new_time)
135
136 # Write back
137 with open(registry_file, 'w') as f:
138 for sid, parts in sorted(existing.items()):
139 f.write('\t'.join(parts) + '\n')
140
141 # Verify timestamp changed
142 with open(registry_file, 'r') as f:
143 for line in f:
144 if 'station2' in line:
145 parts = line.strip().split('\t')
146 assert parts[3] != old_timestamp
147 assert float(parts[3]) == pytest.approx(new_time, rel=1.0)
148
149
150class TestStationInfo:
151 """Test station information"""
152
153 def test_station_id_from_env(self):
154 """Test reading STATION_ID from environment"""
155 os.environ['STATION_ID'] = 'station1'
156 station_id = os.environ.get('STATION_ID', 'unknown')
157 assert station_id == 'station1'
158
159 os.environ['STATION_ID'] = 'station2'
160 station_id = os.environ.get('STATION_ID', 'unknown')
161 assert station_id == 'station2'
162
163 def test_master_detection(self):
164 """Test detecting master vs worker"""
165 os.environ['STATION_ID'] = 'station1'
166 station_id = os.environ.get('STATION_ID', 'unknown')
167 is_master = station_id == 'station1'
168 assert is_master is True
169
170 os.environ['STATION_ID'] = 'station2'
171 station_id = os.environ.get('STATION_ID', 'unknown')
172 is_master = station_id == 'station1'
173 assert is_master is False
174
175
176class TestHeartbeat:
177 """Test heartbeat system"""
178
179 def test_heartbeat_updates_timestamp(self, cluster_data_dir):
180 """Test heartbeat updates station timestamp"""
181 registry_file = cluster_data_dir / "stations.tsv"
182
183 # Initial heartbeat
184 time1 = time.time()
185 with open(registry_file, 'w') as f:
186 f.write(f"station2\t192.0.2.2\t8001\t{time1}\n")
187
188 # Wait a bit
189 time.sleep(0.1)
190
191 # Second heartbeat
192 time2 = time.time()
193 existing = {}
194 with open(registry_file, 'r') as f:
195 for line in f:
196 parts = line.strip().split('\t')
197 if len(parts) == 4:
198 sid = parts[0]
199 existing[sid] = parts
200
201 existing['station2'][3] = str(time2)
202
203 with open(registry_file, 'w') as f:
204 for sid, parts in sorted(existing.items()):
205 f.write('\t'.join(parts) + '\n')
206
207 # Verify timestamp increased
208 with open(registry_file, 'r') as f:
209 for line in f:
210 parts = line.strip().split('\t')
211 timestamp = float(parts[3])
212 assert timestamp > time1
213
214 def test_heartbeat_timeout_detection(self):
215 """Test detecting stale heartbeats"""
216 current_time = time.time()
217 timeout = 30 # seconds
218
219 # Recent heartbeat
220 recent = current_time - 10
221 assert (current_time - recent) < timeout
222
223 # Stale heartbeat
224 stale = current_time - 60
225 assert (current_time - stale) >= timeout
226
227 def test_multiple_station_heartbeats(self, cluster_data_dir):
228 """Test multiple stations sending heartbeats"""
229 registry_file = cluster_data_dir / "stations.tsv"
230 current_time = time.time()
231
232 # Multiple stations register
233 stations = {
234 'station1': ['station1', 'localhost', '8001', str(current_time)],
235 'station2': ['station2', '192.0.2.2', '8001', str(current_time)],
236 'station3': ['station3', '192.0.2.3', '8001', str(current_time)],
237 }
238
239 with open(registry_file, 'w') as f:
240 for sid, parts in sorted(stations.items()):
241 f.write('\t'.join(parts) + '\n')
242
243 # Verify all stations in registry
244 with open(registry_file, 'r') as f:
245 lines = f.readlines()
246
247 assert len(lines) == 3
248 assert all('station' in line for line in lines)
249
250
251class TestClusterHealth:
252 """Test cluster health monitoring"""
253
254 def test_count_active_stations(self, mock_registry_file):
255 """Test counting active vs inactive stations"""
256 current_time = time.time()
257 timeout = 30
258
259 active_count = 0
260 total_count = 0
261
262 with open(mock_registry_file, 'r') as f:
263 for line in f:
264 total_count += 1
265 parts = line.strip().split('\t')
266 timestamp = float(parts[3])
267 if (current_time - timestamp) < timeout:
268 active_count += 1
269
270 assert total_count == 3
271 assert active_count == 2 # station1 and station2 are active
272
273 def test_get_station_list(self, mock_registry_file):
274 """Test getting list of all stations"""
275 stations = []
276
277 with open(mock_registry_file, 'r') as f:
278 for line in f:
279 parts = line.strip().split('\t')
280 station_id, host, port, timestamp = parts
281 stations.append({
282 'station_id': station_id,
283 'host': host,
284 'port': int(port),
285 'last_heartbeat': float(timestamp)
286 })
287
288 assert len(stations) == 3
289 assert all(s['station_id'].startswith('station') for s in stations)
290 assert all(s['port'] == 8001 for s in stations)
291
292 def test_station_url_construction(self, mock_registry_file):
293 """Test constructing station URLs"""
294 with open(mock_registry_file, 'r') as f:
295 for line in f:
296 parts = line.strip().split('\t')
297 station_id, host, port, _ = parts
298 url = f'http://{host}:{port}'
299
300 assert url.startswith('http://')
301 assert ':8001' in url
302
303
304class TestFailureDetection:
305 """Test detecting and handling station failures"""
306
307 def test_detect_failed_station(self):
308 """Test detecting when station fails (no heartbeat)"""
309 current_time = time.time()
310 timeout = 30
311
312 # Simulate station that hasn't sent heartbeat in 2 minutes
313 last_heartbeat = current_time - 120
314 age = current_time - last_heartbeat
315
316 is_failed = age >= timeout
317 assert is_failed is True
318
319 def test_mark_station_inactive(self, mock_registry_file):
320 """Test marking station as inactive"""
321 current_time = time.time()
322 timeout = 30
323
324 stations = []
325 with open(mock_registry_file, 'r') as f:
326 for line in f:
327 parts = line.strip().split('\t')
328 station_id, host, port, timestamp = parts
329 age = current_time - float(timestamp)
330
331 stations.append({
332 'station_id': station_id,
333 'is_active': age < timeout
334 })
335
336 # station3 should be inactive (100 second old heartbeat)
337 station3 = next(s for s in stations if s['station_id'] == 'station3')
338 assert station3['is_active'] is False
339
340 # station1 and station2 should be active
341 station1 = next(s for s in stations if s['station_id'] == 'station1')
342 station2 = next(s for s in stations if s['station_id'] == 'station2')
343 assert station1['is_active'] is True
344 assert station2['is_active'] is True
345
346
347class TestRegistryPersistence:
348 """Test registry persistence across restarts"""
349
350 def test_registry_survives_restart(self, cluster_data_dir):
351 """Test registry data persists to disk"""
352 registry_file = cluster_data_dir / "stations.tsv"
353 current_time = time.time()
354
355 # Write registry
356 with open(registry_file, 'w') as f:
357 f.write(f"station1\tlocalhost\t8001\t{current_time}\n")
358
359 # Simulate server restart - read back
360 with open(registry_file, 'r') as f:
361 line = f.readline()
362 parts = line.strip().split('\t')
363
364 assert parts[0] == 'station1'
365 assert parts[1] == 'localhost'
366 assert parts[2] == '8001'
367 assert float(parts[3]) == pytest.approx(current_time, rel=1.0)
368
369 def test_registry_format_human_readable(self, mock_registry_file):
370 """Test registry is human-readable TSV"""
371 with open(mock_registry_file, 'r') as f:
372 content = f.read()
373
374 # Should be readable text
375 assert '\t' in content # TSV format
376 assert '\n' in content # Multiple lines
377 assert 'station' in content # Contains station IDs
378
379 # Should NOT be binary
380 assert content.isprintable() or '\n' in content or '\t' in content
381
382
383class TestConcurrentHeartbeats:
384 """Test handling multiple concurrent heartbeats"""
385
386 def test_concurrent_updates_dont_corrupt(self, cluster_data_dir):
387 """Test multiple stations updating registry simultaneously"""
388 registry_file = cluster_data_dir / "stations.tsv"
389
390 # Simulate 3 stations updating at same time
391 for i in range(1, 4):
392 current_time = time.time()
393
394 # Read existing
395 existing = {}
396 if registry_file.exists():
397 with open(registry_file, 'r') as f:
398 for line in f:
399 parts = line.strip().split('\t')
400 if len(parts) == 4:
401 sid = parts[0]
402 existing[sid] = parts
403
404 # Update this station
405 station_id = f'station{i}'
406 existing[station_id] = [station_id, f'192.0.2.{i}', '8001', str(current_time)]
407
408 # Write back
409 with open(registry_file, 'w') as f:
410 for sid, parts in sorted(existing.items()):
411 f.write('\t'.join(parts) + '\n')
412
413 # Verify all 3 stations in registry
414 with open(registry_file, 'r') as f:
415 lines = f.readlines()
416
417 assert len(lines) == 3
418 station_ids = [line.split('\t')[0] for line in lines]
419 assert 'station1' in station_ids
420 assert 'station2' in station_ids
421 assert 'station3' in station_ids
422
423
424if __name__ == '__main__':
425 pytest.main([__file__, '-v'])