Python functions as REST objects, with clustering, load balancing and state replication; no containers or orchestration.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-primitive.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-primitive.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Q Consolidate duplicate QUICKSTARTs on the top-level file 503f830 7 months ago History
cluster.py · 7.9 KB · 240 lines
1#!/usr/bin/env python3
2"""
3Cluster Management - PID-based process control for distributed cluster
4
5Usage:
6 ./cluster.py start # Start all cluster processes
7 ./cluster.py stop # Stop all cluster processes
8 ./cluster.py restart # Restart all cluster processes
9 ./cluster.py status # Show status of all processes
10 ./cluster.py deploy # Deploy code to all stations
11"""
12import sys
13import subprocess
14import time
15from pathlib import Path
16from process_manager import ProcessManager
17from cluster_config import get_config
18
19
20class ClusterManager:
21 """Manage cluster processes with PID files"""
22
23 def __init__(self):
24 self.base_dir = Path(__file__).parent
25 self.config = get_config()
26 self.master = self.config.get_master()
27 self.workers = self.config.get_workers()
28
29 def start(self):
30 """Start all cluster processes"""
31 print("=== Starting Cluster ===\n")
32
33 # Start master server
34 print(f"Starting master ({self.master['station_id']})...")
35 server_pm = ProcessManager(f"server_{self.master['station_id']}")
36
37 if server_pm.is_running():
38 print(f"✓ {self.master['station_id']} already running (PID {server_pm.get_pid()})")
39 else:
40 subprocess.Popen(
41 ['python', 'run_server.py', '--port', str(self.master['port'])],
42 env={**dict(subprocess.os.environ), 'STATION_ID': self.master['station_id']},
43 stdout=open('server.log', 'w'),
44 stderr=subprocess.STDOUT,
45 cwd=self.base_dir,
46 )
47 time.sleep(2)
48
49 # Get PID and write PID file
50 result = subprocess.run(
51 ['pgrep', '-f', 'python run_server.py'],
52 capture_output=True,
53 text=True
54 )
55 if result.stdout.strip():
56 pid = int(result.stdout.strip().split()[0])
57 server_pm.write_pid(pid)
58 print(f"✓ {self.master['station_id']} started (PID {pid})")
59 else:
60 print(f"✗ {self.master['station_id']} failed to start")
61
62 # Start worker stations
63 for worker in self.workers:
64 self._start_worker(worker)
65
66 print("\n=== Cluster Started ===")
67 time.sleep(2)
68 self.status()
69
70 def _start_worker(self, worker: dict):
71 """Start worker station server and heartbeat daemon"""
72 station_id = worker['station_id']
73 host = worker['host']
74 port = worker['port']
75 ssh_target = self.config.get_ssh_target(station_id)
76
77 print(f"\nStarting {station_id} ({host})...")
78
79 # Start server on worker
80 server_cmd = f"cd ~/multiplexing && STATION_ID={station_id} nohup .venv/bin/python run_server.py --port {port} > server.log 2>&1 < /dev/null & echo $!"
81 result = subprocess.run(
82 ['ssh', ssh_target, server_cmd],
83 capture_output=True,
84 text=True
85 )
86
87 if result.returncode == 0 and result.stdout.strip():
88 server_pid = result.stdout.strip()
89 print(f" Server: PID {server_pid}")
90 else:
91 print(f" Server: Failed to start")
92
93 time.sleep(2)
94
95 # Start heartbeat daemon on worker
96 master_host = self.master['host']
97 hb_cmd = f"cd ~/multiplexing && STATION_ID={station_id} nohup .venv/bin/python cluster_heartbeat_daemon.py {master_host} > heartbeat.log 2>&1 < /dev/null & echo $!"
98 result = subprocess.run(
99 ['ssh', ssh_target, hb_cmd],
100 capture_output=True,
101 text=True
102 )
103
104 if result.returncode == 0 and result.stdout.strip():
105 hb_pid = result.stdout.strip()
106 print(f" Heartbeat: PID {hb_pid}")
107 print(f"✓ {station_id} started")
108 else:
109 print(f" Heartbeat: Failed to start")
110
111 def stop(self):
112 """Stop all cluster processes"""
113 print("=== Stopping Cluster ===\n")
114
115 # Stop workers first
116 for worker in self.workers:
117 self._stop_worker(worker)
118
119 # Stop master
120 print(f"\nStopping master ({self.master['station_id']})...")
121 server_pm = ProcessManager(f"server_{self.master['station_id']}")
122 server_pm.stop()
123
124 print("\n=== Cluster Stopped ===")
125
126 def _stop_worker(self, worker: dict):
127 """Stop worker station processes"""
128 station_id = worker['station_id']
129 host = worker['host']
130 ssh_target = self.config.get_ssh_target(station_id)
131
132 print(f"Stopping {station_id} ({host})...")
133
134 # Kill all Python processes (servers and heartbeat daemons)
135 cmd = "pkill -f 'python.*run_server.py' && pkill -f 'python.*cluster_heartbeat_daemon.py'"
136 subprocess.run(['ssh', ssh_target, cmd], capture_output=True)
137
138 print(f"✓ {station_id} stopped")
139
140 def restart(self):
141 """Restart all cluster processes"""
142 self.stop()
143 time.sleep(2)
144 self.start()
145
146 def status(self):
147 """Show status of all cluster processes"""
148 print("=== Cluster Status ===\n")
149
150 # Check master
151 server_pm = ProcessManager(f"server_{self.master['station_id']}")
152 status = server_pm.status()
153
154 if status['running']:
155 print(f"{self.master['station_id']} (master): RUNNING (PID {status['pid']})")
156 else:
157 print(f"{self.master['station_id']} (master): NOT RUNNING")
158
159 # Check workers via curl
160 for worker in self.workers:
161 station_id = worker['station_id']
162 url = self.config.get_url(station_id, '/cluster/stations')
163
164 try:
165 result = subprocess.run(
166 ['curl', '-s', '--max-time', '2', url],
167 capture_output=True,
168 text=True,
169 timeout=3
170 )
171
172 if result.returncode == 0 and 'station_id' in result.stdout:
173 print(f"{station_id}: RUNNING")
174 else:
175 print(f"{station_id}: NOT RESPONDING")
176 except subprocess.TimeoutExpired:
177 print(f"{station_id}: TIMEOUT")
178
179 def deploy(self):
180 """Deploy code to all stations"""
181 print("=== Deploying to All Stations ===\n")
182
183 for worker in self.workers:
184 station_id = worker['station_id']
185 host = worker['host']
186 ssh_target = self.config.get_ssh_target(station_id)
187
188 print(f"Deploying to {station_id} ({host})...")
189
190 cmd = [
191 'rsync', '-av', '--delete',
192 '--exclude', '.git',
193 '--exclude', '.venv',
194 '--exclude', '__pycache__',
195 '--exclude', '*.pyc',
196 '--exclude', 'pids/',
197 '--exclude', '*.pid',
198 '--exclude', '_to_move/',
199 '--exclude', 'cluster.tsv',
200 f'{self.base_dir}/',
201 f'{ssh_target}:~/multiplexing/'
202 ]
203
204 result = subprocess.run(cmd, capture_output=True, text=True)
205
206 if result.returncode == 0:
207 print(f"✓ {station_id} deployed")
208 else:
209 print(f"✗ {station_id} deployment failed")
210 print(result.stderr)
211
212 print("\n=== Deployment Complete ===")
213
214
215def main():
216 if len(sys.argv) < 2:
217 print("Usage: ./cluster.py {start|stop|restart|status|deploy}")
218 sys.exit(1)
219
220 command = sys.argv[1]
221 manager = ClusterManager()
222
223 if command == 'start':
224 manager.start()
225 elif command == 'stop':
226 manager.stop()
227 elif command == 'restart':
228 manager.restart()
229 elif command == 'status':
230 manager.status()
231 elif command == 'deploy':
232 manager.deploy()
233 else:
234 print(f"Unknown command: {command}")
235 print("Usage: ./cluster.py {start|stop|restart|status|deploy}")
236 sys.exit(1)
237
238
239if __name__ == '__main__':
240 main()