Two-way flow control for Unix pipes: a downstream command tells the upstream one how much it needs, like a SQL optimizer.

0 downloads

dbbasic-pipe/dbbasic_pipe/coordinator.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-pipe.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-pipe.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-pipe.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Add PyPI and license badges to README 52c82fe 1 year, 1 month ago History
coordinator.py · 3.6 KB · 122 lines
1#!/usr/bin/env python3
2"""
3Pipe coordinator - routes control messages between pipeline commands.
4
5Usage:
6 pipe-coordinator <socket-path>
7
8Example:
9 pipe-coordinator /tmp/pipe-ctrl-1234.sock
10"""
11
12import socket
13import os
14import sys
15import json
16import select
17
18
19def main():
20 if len(sys.argv) < 2:
21 print("Usage: pipe-coordinator <socket-path>", file=sys.stderr)
22 sys.exit(1)
23
24 sock_path = sys.argv[1]
25
26 # Remove old socket if exists
27 try:
28 os.unlink(sock_path)
29 except FileNotFoundError:
30 pass
31
32 # Create Unix domain socket
33 server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
34 server.bind(sock_path)
35 server.listen(10)
36 server.setblocking(False)
37
38 clients = {} # pid -> socket
39 commands = {} # pid -> command_info
40
41 print(f"[coordinator] Listening on {sock_path}", file=sys.stderr)
42
43 sockets = [server]
44
45 try:
46 while True:
47 readable, _, _ = select.select(sockets, [], [], 1.0)
48
49 for s in readable:
50 if s is server:
51 # New connection
52 client, _ = server.accept()
53 client.setblocking(False)
54 sockets.append(client)
55 else:
56 # Data from client
57 try:
58 data = s.recv(4096).decode()
59 if not data:
60 # Connection closed
61 sockets.remove(s)
62 # Remove from clients
63 for pid, sock in list(clients.items()):
64 if sock == s:
65 del clients[pid]
66 if pid in commands:
67 del commands[pid]
68 s.close()
69 continue
70
71 for line in data.strip().split('\n'):
72 if line:
73 msg = json.loads(line)
74 handle_message(msg, s, clients, commands)
75
76 except json.JSONDecodeError as e:
77 print(f"[coordinator] JSON error: {e}", file=sys.stderr)
78 except Exception as e:
79 print(f"[coordinator] Error: {e}", file=sys.stderr)
80 sockets.remove(s)
81 s.close()
82
83 except KeyboardInterrupt:
84 print("\n[coordinator] Shutting down", file=sys.stderr)
85 finally:
86 server.close()
87 try:
88 os.unlink(sock_path)
89 except:
90 pass
91
92
93def handle_message(msg, sender, clients, commands):
94 """Route messages between pipeline commands."""
95 msg_type = msg.get('type')
96
97 if msg_type == 'register':
98 pid = msg['pid']
99 clients[pid] = sender
100 commands[pid] = msg
101 print(f"[coordinator] Registered {msg.get('command', 'unknown')} (pid {pid})", file=sys.stderr)
102
103 elif msg_type == 'backpressure':
104 # Broadcast to all OTHER clients (upstream)
105 count = msg.get('count', 0)
106 print(f"[coordinator] Broadcasting backpressure: {count}", file=sys.stderr)
107 for pid, sock in clients.items():
108 if sock != sender:
109 try:
110 sock.send(json.dumps(msg).encode() + b'\n')
111 except:
112 pass
113
114 elif msg_type == 'complete':
115 print(f"[coordinator] Command {msg.get('pid')} completed", file=sys.stderr)
116
117 elif msg_type == 'error':
118 print(f"[coordinator] Error from {msg.get('pid')}: {msg.get('message')}", file=sys.stderr)
119
120
121if __name__ == '__main__':
122 main()