askrobots / dbbasic-object-primitive
Public
Python functions as REST objects, with clustering, load balancing and state replication; no containers or orchestration.
cluster.py · 7.9 KB · 240 lines
| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | Cluster Management - PID-based process control for distributed cluster |
| 4 | |
| 5 | Usage: |
| 6 | ./cluster.py start # Start all cluster processes |
| 7 | ./cluster.py stop # Stop all cluster processes |
| 8 | ./cluster.py restart # Restart all cluster processes |
| 9 | ./cluster.py status # Show status of all processes |
| 10 | ./cluster.py deploy # Deploy code to all stations |
| 11 | """ |
| 12 | import sys |
| 13 | import subprocess |
| 14 | import time |
| 15 | from pathlib import Path |
| 16 | from process_manager import ProcessManager |
| 17 | from cluster_config import get_config |
| 18 | |
| 19 | |
| 20 | class ClusterManager: |
| 21 | """Manage cluster processes with PID files""" |
| 22 | |
| 23 | def __init__(self): |
| 24 | self.base_dir = Path(__file__).parent |
| 25 | self.config = get_config() |
| 26 | self.master = self.config.get_master() |
| 27 | self.workers = self.config.get_workers() |
| 28 | |
| 29 | def start(self): |
| 30 | """Start all cluster processes""" |
| 31 | print("=== Starting Cluster ===\n") |
| 32 | |
| 33 | # Start master server |
| 34 | print(f"Starting master ({self.master['station_id']})...") |
| 35 | server_pm = ProcessManager(f"server_{self.master['station_id']}") |
| 36 | |
| 37 | if server_pm.is_running(): |
| 38 | print(f"✓ {self.master['station_id']} already running (PID {server_pm.get_pid()})") |
| 39 | else: |
| 40 | subprocess.Popen( |
| 41 | ['python', 'run_server.py', '--port', str(self.master['port'])], |
| 42 | env={**dict(subprocess.os.environ), 'STATION_ID': self.master['station_id']}, |
| 43 | stdout=open('server.log', 'w'), |
| 44 | stderr=subprocess.STDOUT, |
| 45 | cwd=self.base_dir, |
| 46 | ) |
| 47 | time.sleep(2) |
| 48 | |
| 49 | # Get PID and write PID file |
| 50 | result = subprocess.run( |
| 51 | ['pgrep', '-f', 'python run_server.py'], |
| 52 | capture_output=True, |
| 53 | text=True |
| 54 | ) |
| 55 | if result.stdout.strip(): |
| 56 | pid = int(result.stdout.strip().split()[0]) |
| 57 | server_pm.write_pid(pid) |
| 58 | print(f"✓ {self.master['station_id']} started (PID {pid})") |
| 59 | else: |
| 60 | print(f"✗ {self.master['station_id']} failed to start") |
| 61 | |
| 62 | # Start worker stations |
| 63 | for worker in self.workers: |
| 64 | self._start_worker(worker) |
| 65 | |
| 66 | print("\n=== Cluster Started ===") |
| 67 | time.sleep(2) |
| 68 | self.status() |
| 69 | |
| 70 | def _start_worker(self, worker: dict): |
| 71 | """Start worker station server and heartbeat daemon""" |
| 72 | station_id = worker['station_id'] |
| 73 | host = worker['host'] |
| 74 | port = worker['port'] |
| 75 | ssh_target = self.config.get_ssh_target(station_id) |
| 76 | |
| 77 | print(f"\nStarting {station_id} ({host})...") |
| 78 | |
| 79 | # Start server on worker |
| 80 | server_cmd = f"cd ~/multiplexing && STATION_ID={station_id} nohup .venv/bin/python run_server.py --port {port} > server.log 2>&1 < /dev/null & echo $!" |
| 81 | result = subprocess.run( |
| 82 | ['ssh', ssh_target, server_cmd], |
| 83 | capture_output=True, |
| 84 | text=True |
| 85 | ) |
| 86 | |
| 87 | if result.returncode == 0 and result.stdout.strip(): |
| 88 | server_pid = result.stdout.strip() |
| 89 | print(f" Server: PID {server_pid}") |
| 90 | else: |
| 91 | print(f" Server: Failed to start") |
| 92 | |
| 93 | time.sleep(2) |
| 94 | |
| 95 | # Start heartbeat daemon on worker |
| 96 | master_host = self.master['host'] |
| 97 | hb_cmd = f"cd ~/multiplexing && STATION_ID={station_id} nohup .venv/bin/python cluster_heartbeat_daemon.py {master_host} > heartbeat.log 2>&1 < /dev/null & echo $!" |
| 98 | result = subprocess.run( |
| 99 | ['ssh', ssh_target, hb_cmd], |
| 100 | capture_output=True, |
| 101 | text=True |
| 102 | ) |
| 103 | |
| 104 | if result.returncode == 0 and result.stdout.strip(): |
| 105 | hb_pid = result.stdout.strip() |
| 106 | print(f" Heartbeat: PID {hb_pid}") |
| 107 | print(f"✓ {station_id} started") |
| 108 | else: |
| 109 | print(f" Heartbeat: Failed to start") |
| 110 | |
| 111 | def stop(self): |
| 112 | """Stop all cluster processes""" |
| 113 | print("=== Stopping Cluster ===\n") |
| 114 | |
| 115 | # Stop workers first |
| 116 | for worker in self.workers: |
| 117 | self._stop_worker(worker) |
| 118 | |
| 119 | # Stop master |
| 120 | print(f"\nStopping master ({self.master['station_id']})...") |
| 121 | server_pm = ProcessManager(f"server_{self.master['station_id']}") |
| 122 | server_pm.stop() |
| 123 | |
| 124 | print("\n=== Cluster Stopped ===") |
| 125 | |
| 126 | def _stop_worker(self, worker: dict): |
| 127 | """Stop worker station processes""" |
| 128 | station_id = worker['station_id'] |
| 129 | host = worker['host'] |
| 130 | ssh_target = self.config.get_ssh_target(station_id) |
| 131 | |
| 132 | print(f"Stopping {station_id} ({host})...") |
| 133 | |
| 134 | # Kill all Python processes (servers and heartbeat daemons) |
| 135 | cmd = "pkill -f 'python.*run_server.py' && pkill -f 'python.*cluster_heartbeat_daemon.py'" |
| 136 | subprocess.run(['ssh', ssh_target, cmd], capture_output=True) |
| 137 | |
| 138 | print(f"✓ {station_id} stopped") |
| 139 | |
| 140 | def restart(self): |
| 141 | """Restart all cluster processes""" |
| 142 | self.stop() |
| 143 | time.sleep(2) |
| 144 | self.start() |
| 145 | |
| 146 | def status(self): |
| 147 | """Show status of all cluster processes""" |
| 148 | print("=== Cluster Status ===\n") |
| 149 | |
| 150 | # Check master |
| 151 | server_pm = ProcessManager(f"server_{self.master['station_id']}") |
| 152 | status = server_pm.status() |
| 153 | |
| 154 | if status['running']: |
| 155 | print(f"{self.master['station_id']} (master): RUNNING (PID {status['pid']})") |
| 156 | else: |
| 157 | print(f"{self.master['station_id']} (master): NOT RUNNING") |
| 158 | |
| 159 | # Check workers via curl |
| 160 | for worker in self.workers: |
| 161 | station_id = worker['station_id'] |
| 162 | url = self.config.get_url(station_id, '/cluster/stations') |
| 163 | |
| 164 | try: |
| 165 | result = subprocess.run( |
| 166 | ['curl', '-s', '--max-time', '2', url], |
| 167 | capture_output=True, |
| 168 | text=True, |
| 169 | timeout=3 |
| 170 | ) |
| 171 | |
| 172 | if result.returncode == 0 and 'station_id' in result.stdout: |
| 173 | print(f"{station_id}: RUNNING") |
| 174 | else: |
| 175 | print(f"{station_id}: NOT RESPONDING") |
| 176 | except subprocess.TimeoutExpired: |
| 177 | print(f"{station_id}: TIMEOUT") |
| 178 | |
| 179 | def deploy(self): |
| 180 | """Deploy code to all stations""" |
| 181 | print("=== Deploying to All Stations ===\n") |
| 182 | |
| 183 | for worker in self.workers: |
| 184 | station_id = worker['station_id'] |
| 185 | host = worker['host'] |
| 186 | ssh_target = self.config.get_ssh_target(station_id) |
| 187 | |
| 188 | print(f"Deploying to {station_id} ({host})...") |
| 189 | |
| 190 | cmd = [ |
| 191 | 'rsync', '-av', '--delete', |
| 192 | '--exclude', '.git', |
| 193 | '--exclude', '.venv', |
| 194 | '--exclude', '__pycache__', |
| 195 | '--exclude', '*.pyc', |
| 196 | '--exclude', 'pids/', |
| 197 | '--exclude', '*.pid', |
| 198 | '--exclude', '_to_move/', |
| 199 | '--exclude', 'cluster.tsv', |
| 200 | f'{self.base_dir}/', |
| 201 | f'{ssh_target}:~/multiplexing/' |
| 202 | ] |
| 203 | |
| 204 | result = subprocess.run(cmd, capture_output=True, text=True) |
| 205 | |
| 206 | if result.returncode == 0: |
| 207 | print(f"✓ {station_id} deployed") |
| 208 | else: |
| 209 | print(f"✗ {station_id} deployment failed") |
| 210 | print(result.stderr) |
| 211 | |
| 212 | print("\n=== Deployment Complete ===") |
| 213 | |
| 214 | |
| 215 | def main(): |
| 216 | if len(sys.argv) < 2: |
| 217 | print("Usage: ./cluster.py {start|stop|restart|status|deploy}") |
| 218 | sys.exit(1) |
| 219 | |
| 220 | command = sys.argv[1] |
| 221 | manager = ClusterManager() |
| 222 | |
| 223 | if command == 'start': |
| 224 | manager.start() |
| 225 | elif command == 'stop': |
| 226 | manager.stop() |
| 227 | elif command == 'restart': |
| 228 | manager.restart() |
| 229 | elif command == 'status': |
| 230 | manager.status() |
| 231 | elif command == 'deploy': |
| 232 | manager.deploy() |
| 233 | else: |
| 234 | print(f"Unknown command: {command}") |
| 235 | print("Usage: ./cluster.py {start|stop|restart|status|deploy}") |
| 236 | sys.exit(1) |
| 237 | |
| 238 | |
| 239 | if __name__ == '__main__': |
| 240 | main() |