askrobots / dbbasic-object-primitive
Public
Python functions as REST objects, with clustering, load balancing and state replication; no containers or orchestration.
test_cross_station_routing.py · 17.2 KB · 497 lines
| 1 | """ |
| 2 | Tests for cross-station object routing (Phase 7.2) |
| 3 | |
| 4 | Tests the object_id@station_id routing syntax that allows |
| 5 | calling objects on remote stations in the cluster. |
| 6 | """ |
| 7 | import json |
| 8 | import os |
| 9 | import time |
| 10 | import pytest |
| 11 | from pathlib import Path |
| 12 | from unittest.mock import Mock, patch, MagicMock |
| 13 | import requests |
| 14 | import importlib.util |
| 15 | # import sys |
| 16 | |
| 17 | |
| 18 | # Helper function to load the [id].py module |
| 19 | def load_objects_id_module(): |
| 20 | """Load the api/objects/[id].py module""" |
| 21 | module_path = Path(__file__).parent.parent / 'api' / 'objects' / '[id].py' |
| 22 | spec = importlib.util.spec_from_file_location('objects_id_api', module_path) |
| 23 | module = importlib.util.module_from_spec(spec) |
| 24 | spec.loader.exec_module(module) |
| 25 | return module |
| 26 | |
| 27 | |
| 28 | class TestRoutingSyntaxParsing: |
| 29 | """Test parsing of object_id@station_id syntax""" |
| 30 | |
| 31 | def test_parse_local_object(self): |
| 32 | """Test parsing object ID without station routing""" |
| 33 | obj_api = load_objects_id_module() |
| 34 | |
| 35 | object_id, station_id = obj_api.parse_object_routing('calculator') |
| 36 | |
| 37 | assert object_id == 'calculator' |
| 38 | assert station_id is None |
| 39 | |
| 40 | def test_parse_remote_object(self): |
| 41 | """Test parsing object ID with station routing""" |
| 42 | obj_api = load_objects_id_module() |
| 43 | |
| 44 | object_id, station_id = obj_api.parse_object_routing('calculator@station2') |
| 45 | |
| 46 | assert object_id == 'calculator' |
| 47 | assert station_id == 'station2' |
| 48 | |
| 49 | def test_parse_object_with_underscores(self): |
| 50 | """Test parsing complex object IDs with underscores""" |
| 51 | obj_api = load_objects_id_module() |
| 52 | |
| 53 | object_id, station_id = obj_api.parse_object_routing('tutorial_04_calculator@station3') |
| 54 | |
| 55 | assert object_id == 'tutorial_04_calculator' |
| 56 | assert station_id == 'station3' |
| 57 | |
| 58 | def test_parse_multiple_at_signs(self): |
| 59 | """Test parsing with multiple @ signs (only first is used)""" |
| 60 | obj_api = load_objects_id_module() |
| 61 | |
| 62 | # Only split on first @ |
| 63 | object_id, station_id = obj_api.parse_object_routing('obj@station2@invalid') |
| 64 | |
| 65 | assert object_id == 'obj' |
| 66 | assert station_id == 'station2@invalid' |
| 67 | |
| 68 | |
| 69 | class TestStationLookup: |
| 70 | """Test looking up stations in cluster registry""" |
| 71 | |
| 72 | @pytest.fixture |
| 73 | def mock_registry_file(self, tmp_path): |
| 74 | """Create a mock cluster registry file""" |
| 75 | registry_dir = tmp_path / 'data' / 'cluster' |
| 76 | registry_dir.mkdir(parents=True, exist_ok=True) |
| 77 | registry_file = registry_dir / 'stations.tsv' |
| 78 | |
| 79 | current_time = time.time() |
| 80 | |
| 81 | # Write registry with 3 stations |
| 82 | # Using RFC 5737 TEST-NET-1 addresses (192.0.2.0/24) |
| 83 | # station1: active (master) |
| 84 | # station2: active (worker) |
| 85 | # station3: inactive (old heartbeat) |
| 86 | with open(registry_file, 'w') as f: |
| 87 | f.write(f'station1\t192.0.2.1\t8001\t{current_time}\n') |
| 88 | f.write(f'station2\t192.0.2.2\t8001\t{current_time - 10}\n') |
| 89 | f.write(f'station3\t192.0.2.3\t8001\t{current_time - 100}\n') |
| 90 | |
| 91 | return registry_file |
| 92 | |
| 93 | def test_lookup_active_station(self, mock_registry_file): |
| 94 | """Test looking up an active station""" |
| 95 | # import sys |
| 96 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 97 | obj_api = load_objects_id_module() |
| 98 | |
| 99 | # Temporarily change data path |
| 100 | original_cwd = os.getcwd() |
| 101 | os.chdir(mock_registry_file.parent.parent.parent) |
| 102 | |
| 103 | try: |
| 104 | station_info = obj_api.get_station_info('station2') |
| 105 | |
| 106 | assert station_info is not None |
| 107 | assert station_info['station_id'] == 'station2' |
| 108 | assert station_info['host'] == '192.0.2.2' |
| 109 | assert station_info['port'] == 8001 |
| 110 | assert station_info['is_active'] is True |
| 111 | assert station_info['url'] == 'http://192.0.2.2:8001' |
| 112 | finally: |
| 113 | os.chdir(original_cwd) |
| 114 | |
| 115 | def test_lookup_inactive_station(self, mock_registry_file): |
| 116 | """Test looking up an inactive station (old heartbeat)""" |
| 117 | # import sys |
| 118 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 119 | obj_api = load_objects_id_module() |
| 120 | |
| 121 | original_cwd = os.getcwd() |
| 122 | os.chdir(mock_registry_file.parent.parent.parent) |
| 123 | |
| 124 | try: |
| 125 | station_info = obj_api.get_station_info('station3') |
| 126 | |
| 127 | # Should return None because station3 has old heartbeat (100 seconds ago) |
| 128 | assert station_info is None |
| 129 | finally: |
| 130 | os.chdir(original_cwd) |
| 131 | |
| 132 | def test_lookup_nonexistent_station(self, mock_registry_file): |
| 133 | """Test looking up a station that doesn't exist""" |
| 134 | # import sys |
| 135 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 136 | obj_api = load_objects_id_module() |
| 137 | |
| 138 | original_cwd = os.getcwd() |
| 139 | os.chdir(mock_registry_file.parent.parent.parent) |
| 140 | |
| 141 | try: |
| 142 | station_info = obj_api.get_station_info('station99') |
| 143 | |
| 144 | assert station_info is None |
| 145 | finally: |
| 146 | os.chdir(original_cwd) |
| 147 | |
| 148 | |
| 149 | class TestRequestForwarding: |
| 150 | """Test forwarding HTTP requests to remote stations""" |
| 151 | |
| 152 | def test_forward_get_request(self): |
| 153 | """Test forwarding a GET request""" |
| 154 | # import sys |
| 155 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 156 | obj_api = load_objects_id_module() |
| 157 | |
| 158 | station_info = { |
| 159 | 'station_id': 'station2', |
| 160 | 'host': '192.0.2.2', |
| 161 | 'port': 8001, |
| 162 | 'url': 'http://192.0.2.2:8001', |
| 163 | 'is_active': True |
| 164 | } |
| 165 | |
| 166 | # Mock the requests.get call |
| 167 | with patch('requests.get') as mock_get: |
| 168 | mock_response = Mock() |
| 169 | mock_response.json.return_value = { |
| 170 | 'status': 'ok', |
| 171 | 'result': 8.0, |
| 172 | 'object_id': 'calculator' |
| 173 | } |
| 174 | mock_get.return_value = mock_response |
| 175 | |
| 176 | result = obj_api.forward_request( |
| 177 | station_info=station_info, |
| 178 | object_id='calculator', |
| 179 | method='GET', |
| 180 | query_params={'operation': 'add', 'a': '5', 'b': '3'} |
| 181 | ) |
| 182 | |
| 183 | # Verify request was made correctly |
| 184 | mock_get.assert_called_once_with( |
| 185 | 'http://192.0.2.2:8001/objects/calculator', |
| 186 | params={'operation': 'add', 'a': '5', 'b': '3'}, |
| 187 | timeout=30 |
| 188 | ) |
| 189 | |
| 190 | # Verify routing metadata added |
| 191 | assert result['_routed_to'] == 'station2' |
| 192 | assert '_routed_from' in result |
| 193 | |
| 194 | def test_forward_post_request(self): |
| 195 | """Test forwarding a POST request with body data""" |
| 196 | # import sys |
| 197 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 198 | obj_api = load_objects_id_module() |
| 199 | |
| 200 | station_info = { |
| 201 | 'station_id': 'station3', |
| 202 | 'host': '192.0.2.3', |
| 203 | 'port': 8001, |
| 204 | 'url': 'http://192.0.2.3:8001', |
| 205 | 'is_active': True |
| 206 | } |
| 207 | |
| 208 | with patch('requests.post') as mock_post: |
| 209 | mock_response = Mock() |
| 210 | mock_response.json.return_value = { |
| 211 | 'status': 'ok', |
| 212 | 'count': 5 |
| 213 | } |
| 214 | mock_post.return_value = mock_response |
| 215 | |
| 216 | body_data = {'action': 'increment', 'amount': 5} |
| 217 | |
| 218 | result = obj_api.forward_request( |
| 219 | station_info=station_info, |
| 220 | object_id='counter', |
| 221 | method='POST', |
| 222 | body_data=body_data |
| 223 | ) |
| 224 | |
| 225 | # Verify request was made correctly |
| 226 | mock_post.assert_called_once() |
| 227 | call_args = mock_post.call_args |
| 228 | assert call_args[1]['json'] == body_data |
| 229 | assert call_args[1]['headers']['Content-Type'] == 'application/json' |
| 230 | |
| 231 | def test_forward_request_timeout(self): |
| 232 | """Test handling timeout when forwarding request""" |
| 233 | # import sys |
| 234 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 235 | obj_api = load_objects_id_module() |
| 236 | |
| 237 | station_info = { |
| 238 | 'station_id': 'station2', |
| 239 | 'host': '192.0.2.2', |
| 240 | 'port': 8001, |
| 241 | 'url': 'http://192.0.2.2:8001', |
| 242 | 'is_active': True |
| 243 | } |
| 244 | |
| 245 | with patch('requests.get') as mock_get: |
| 246 | mock_get.side_effect = requests.Timeout('Connection timeout') |
| 247 | |
| 248 | with pytest.raises(requests.Timeout): |
| 249 | obj_api.forward_request( |
| 250 | station_info=station_info, |
| 251 | object_id='calculator', |
| 252 | method='GET' |
| 253 | ) |
| 254 | |
| 255 | def test_forward_request_network_error(self): |
| 256 | """Test handling network errors when forwarding request""" |
| 257 | # import sys |
| 258 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 259 | obj_api = load_objects_id_module() |
| 260 | |
| 261 | station_info = { |
| 262 | 'station_id': 'station2', |
| 263 | 'host': '192.0.2.2', |
| 264 | 'port': 8001, |
| 265 | 'url': 'http://192.0.2.2:8001', |
| 266 | 'is_active': True |
| 267 | } |
| 268 | |
| 269 | with patch('requests.get') as mock_get: |
| 270 | mock_get.side_effect = requests.ConnectionError('Connection refused') |
| 271 | |
| 272 | with pytest.raises(requests.ConnectionError): |
| 273 | obj_api.forward_request( |
| 274 | station_info=station_info, |
| 275 | object_id='calculator', |
| 276 | method='GET' |
| 277 | ) |
| 278 | |
| 279 | |
| 280 | class TestEndToEndRouting: |
| 281 | """Test complete routing flow from request to response""" |
| 282 | |
| 283 | @pytest.fixture |
| 284 | def setup_test_env(self, tmp_path, monkeypatch): |
| 285 | """Set up test environment with registry""" |
| 286 | # Create registry |
| 287 | registry_dir = tmp_path / 'data' / 'cluster' |
| 288 | registry_dir.mkdir(parents=True, exist_ok=True) |
| 289 | registry_file = registry_dir / 'stations.tsv' |
| 290 | |
| 291 | current_time = time.time() |
| 292 | with open(registry_file, 'w') as f: |
| 293 | f.write(f'station1\t192.0.2.1\t8001\t{current_time}\n') |
| 294 | f.write(f'station2\t192.0.2.2\t8001\t{current_time}\n') |
| 295 | |
| 296 | # Change to test directory |
| 297 | original_cwd = os.getcwd() |
| 298 | os.chdir(tmp_path) |
| 299 | |
| 300 | # Set station ID |
| 301 | monkeypatch.setenv('STATION_ID', 'station1') |
| 302 | |
| 303 | yield tmp_path |
| 304 | |
| 305 | # Restore |
| 306 | os.chdir(original_cwd) |
| 307 | |
| 308 | def test_route_to_remote_station(self, setup_test_env): |
| 309 | """Test routing object@station2 from station1""" |
| 310 | # import sys |
| 311 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 312 | obj_api = load_objects_id_module() |
| 313 | |
| 314 | # Mock HTTP request |
| 315 | mock_request = Mock() |
| 316 | mock_request.GET = {'operation': 'add', 'a': '5', 'b': '3'} |
| 317 | |
| 318 | # Mock the actual HTTP call to station2 |
| 319 | with patch('requests.get') as mock_get: |
| 320 | mock_response = Mock() |
| 321 | mock_response.json.return_value = { |
| 322 | 'status': 'ok', |
| 323 | 'result': 8.0 |
| 324 | } |
| 325 | mock_get.return_value = mock_response |
| 326 | |
| 327 | # Call the GET handler with calculator@station2 |
| 328 | response = obj_api.GET(mock_request, 'calculator@station2') |
| 329 | |
| 330 | # Verify it made the remote call |
| 331 | mock_get.assert_called_once() |
| 332 | |
| 333 | # Verify response contains routing metadata |
| 334 | # Response is a tuple (status_code, headers, body) |
| 335 | status_code, headers, body = response; body = body[0] if isinstance(body, list) else body |
| 336 | response_data = json.loads(body) |
| 337 | assert response_data['_routed_to'] == 'station2' |
| 338 | assert response_data['_routed_from'] == 'station1' |
| 339 | |
| 340 | def test_route_to_local_station(self, setup_test_env): |
| 341 | """Test routing object@station1 (local) from station1""" |
| 342 | # import sys |
| 343 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 344 | obj_api = load_objects_id_module() |
| 345 | |
| 346 | mock_request = Mock() |
| 347 | mock_request.GET = {} |
| 348 | |
| 349 | # When routing to self, should execute locally |
| 350 | # Mock find_object_file to return None (object not found) |
| 351 | with patch.object(obj_api, 'find_object_file', return_value=None): |
| 352 | response = obj_api.GET(mock_request, 'nonexistent@station1') |
| 353 | |
| 354 | # Should get 404 (local execution attempted) |
| 355 | status_code, headers, body = response; body = body[0] if isinstance(body, list) else body |
| 356 | response_data = json.loads(body) |
| 357 | # Error response has 'error' key |
| 358 | assert 'error' in response_data |
| 359 | assert 'not found' in response_data['error'].lower() |
| 360 | |
| 361 | def test_route_to_offline_station(self, setup_test_env): |
| 362 | """Test routing to a station that's offline""" |
| 363 | # import sys |
| 364 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 365 | obj_api = load_objects_id_module() |
| 366 | |
| 367 | # Add offline station to registry |
| 368 | registry_file = setup_test_env / 'data' / 'cluster' / 'stations.tsv' |
| 369 | with open(registry_file, 'a') as f: |
| 370 | f.write(f'station3\t192.0.2.3\t8001\t{time.time() - 100}\n') |
| 371 | |
| 372 | mock_request = Mock() |
| 373 | mock_request.GET = {} |
| 374 | |
| 375 | # Route to offline station |
| 376 | response = obj_api.GET(mock_request, 'calculator@station3') |
| 377 | |
| 378 | # Should get 503 Service Unavailable |
| 379 | status_code, headers, body = response; body = body[0] if isinstance(body, list) else body |
| 380 | response_data = json.loads(body) |
| 381 | assert 'error' in response_data |
| 382 | assert 'offline' in response_data['error'].lower() |
| 383 | |
| 384 | def test_route_to_nonexistent_station(self, setup_test_env): |
| 385 | """Test routing to a station that doesn't exist""" |
| 386 | # import sys |
| 387 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 388 | obj_api = load_objects_id_module() |
| 389 | |
| 390 | mock_request = Mock() |
| 391 | mock_request.GET = {} |
| 392 | |
| 393 | # Route to nonexistent station |
| 394 | response = obj_api.GET(mock_request, 'calculator@station99') |
| 395 | |
| 396 | # Should get 503 Service Unavailable |
| 397 | status_code, headers, body = response; body = body[0] if isinstance(body, list) else body |
| 398 | response_data = json.loads(body) |
| 399 | assert 'error' in response_data |
| 400 | assert 'not found' in response_data['error'].lower() |
| 401 | |
| 402 | |
| 403 | class TestAllHTTPMethods: |
| 404 | """Test routing works for all HTTP methods""" |
| 405 | |
| 406 | @pytest.fixture |
| 407 | def setup_test_env(self, tmp_path, monkeypatch): |
| 408 | """Set up test environment""" |
| 409 | registry_dir = tmp_path / 'data' / 'cluster' |
| 410 | registry_dir.mkdir(parents=True, exist_ok=True) |
| 411 | registry_file = registry_dir / 'stations.tsv' |
| 412 | |
| 413 | current_time = time.time() |
| 414 | with open(registry_file, 'w') as f: |
| 415 | f.write(f'station1\t192.0.2.1\t8001\t{current_time}\n') |
| 416 | f.write(f'station2\t192.0.2.2\t8001\t{current_time}\n') |
| 417 | |
| 418 | original_cwd = os.getcwd() |
| 419 | os.chdir(tmp_path) |
| 420 | monkeypatch.setenv('STATION_ID', 'station1') |
| 421 | |
| 422 | yield tmp_path |
| 423 | |
| 424 | os.chdir(original_cwd) |
| 425 | |
| 426 | def test_post_routing(self, setup_test_env): |
| 427 | """Test POST request routing""" |
| 428 | # import sys |
| 429 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 430 | obj_api = load_objects_id_module() |
| 431 | |
| 432 | mock_request = Mock() |
| 433 | mock_request.GET = {} |
| 434 | mock_request.POST = {} |
| 435 | mock_request.body = json.dumps({'action': 'increment'}).encode('utf-8') |
| 436 | |
| 437 | with patch('requests.post') as mock_post: |
| 438 | mock_response = Mock() |
| 439 | mock_response.json.return_value = {'status': 'ok'} |
| 440 | mock_post.return_value = mock_response |
| 441 | |
| 442 | response = obj_api.POST(mock_request, 'counter@station2') |
| 443 | |
| 444 | mock_post.assert_called_once() |
| 445 | status_code, headers, body = response; body = body[0] if isinstance(body, list) else body |
| 446 | response_data = json.loads(body) |
| 447 | assert response_data['_routed_to'] == 'station2' |
| 448 | |
| 449 | def test_put_routing(self, setup_test_env): |
| 450 | """Test PUT request routing""" |
| 451 | # import sys |
| 452 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 453 | obj_api = load_objects_id_module() |
| 454 | |
| 455 | mock_request = Mock() |
| 456 | mock_request.GET = {} |
| 457 | mock_request.POST = {} |
| 458 | mock_request.body = json.dumps({'value': 42}).encode('utf-8') |
| 459 | |
| 460 | with patch('requests.put') as mock_put: |
| 461 | mock_response = Mock() |
| 462 | mock_response.json.return_value = {'status': 'ok'} |
| 463 | mock_put.return_value = mock_response |
| 464 | |
| 465 | response = obj_api.PUT(mock_request, 'config@station2') |
| 466 | |
| 467 | mock_put.assert_called_once() |
| 468 | status_code, headers, body = response; body = body[0] if isinstance(body, list) else body |
| 469 | response_data = json.loads(body) |
| 470 | assert response_data['_routed_to'] == 'station2' |
| 471 | |
| 472 | def test_delete_routing(self, setup_test_env): |
| 473 | """Test DELETE request routing""" |
| 474 | # import sys |
| 475 | # sys.path.insert(0, str(Path(__file__).parent.parent)) |
| 476 | obj_api = load_objects_id_module() |
| 477 | |
| 478 | mock_request = Mock() |
| 479 | mock_request.GET = {} |
| 480 | mock_request.POST = {} |
| 481 | mock_request.body = json.dumps({'confirm': True}).encode('utf-8') |
| 482 | |
| 483 | with patch('requests.delete') as mock_delete: |
| 484 | mock_response = Mock() |
| 485 | mock_response.json.return_value = {'status': 'ok'} |
| 486 | mock_delete.return_value = mock_response |
| 487 | |
| 488 | response = obj_api.DELETE(mock_request, 'temp_data@station2') |
| 489 | |
| 490 | mock_delete.assert_called_once() |
| 491 | status_code, headers, body = response; body = body[0] if isinstance(body, list) else body |
| 492 | response_data = json.loads(body) |
| 493 | assert response_data['_routed_to'] == 'station2' |
| 494 | |
| 495 | |
| 496 | if __name__ == '__main__': |
| 497 | pytest.main([__file__, '-v']) |