Two-way flow control for Unix pipes: a downstream command tells the upstream one how much it needs, like a SQL optimizer.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-pipe.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-pipe.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-pipe.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Add PyPI and license badges to README 52c82fe 1 year, 1 month ago History
pcat.py · 3.8 KB · 128 lines
1#!/usr/bin/env python3
2"""
3Smart cat - reads files and respects backpressure signals.
4
5Usage:
6 pcat <file>
7
8Works standalone or with PIPE_CONTROL_SOCKET coordination.
9
10Examples:
11 pcat data.json
12 pcat data.json | pfilter 'age > 18'
13"""
14
15import os
16import sys
17import json
18import socket
19import select
20
21
22def main():
23 if len(sys.argv) < 2:
24 print("Usage: pcat <file>", file=sys.stderr)
25 sys.exit(1)
26
27 filename = sys.argv[1]
28
29 if not os.path.exists(filename):
30 print(f"Error: file not found: {filename}", file=sys.stderr)
31 sys.exit(1)
32
33 # Check for coordination socket
34 ctrl_socket = os.getenv('PIPE_CONTROL_SOCKET')
35 sock = None
36 limit = -1 # -1 = unlimited
37
38 if ctrl_socket and os.path.exists(ctrl_socket):
39 try:
40 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
41 sock.connect(ctrl_socket)
42 sock.setblocking(False)
43
44 # Register
45 msg = {
46 'type': 'register',
47 'pid': os.getpid(),
48 'command': 'pcat',
49 'file': filename
50 }
51 sock.send(json.dumps(msg).encode() + b'\n')
52
53 print(f"[pcat] Registered, reading: {filename}", file=sys.stderr)
54
55 # Wait a moment for backpressure signals
56 import time
57 time.sleep(0.1)
58
59 # Check for backpressure
60 readable, _, _ = select.select([sock], [], [], 0)
61 if readable:
62 try:
63 data = sock.recv(4096).decode()
64 for msg_line in data.strip().split('\n'):
65 if msg_line:
66 msg = json.loads(msg_line)
67 if msg['type'] == 'backpressure':
68 limit = msg['count']
69 print(f"[pcat] Received backpressure: {limit}", file=sys.stderr)
70 except:
71 pass
72
73 except Exception as e:
74 print(f"[pcat] Coordination failed: {e}", file=sys.stderr)
75 sock = None
76
77 # Read and output file
78 count = 0
79 try:
80 with open(filename, 'r') as f:
81 for line in f:
82 # Check for more control messages
83 if sock:
84 readable, _, _ = select.select([sock], [], [], 0)
85 if readable:
86 try:
87 data = sock.recv(4096).decode()
88 for msg_line in data.strip().split('\n'):
89 if msg_line:
90 msg = json.loads(msg_line)
91 if msg['type'] == 'backpressure':
92 if limit == -1: # First time
93 limit = msg['count']
94 print(f"[pcat] Received backpressure: {limit}", file=sys.stderr)
95 except:
96 pass
97
98 print(line, end='')
99 sys.stdout.flush()
100 count += 1
101
102 # Stop if we hit the limit
103 if limit > 0 and count >= limit * 2: # 2x buffer for filtering
104 print(f"[pcat] Reached soft limit, stopping at {count} lines", file=sys.stderr)
105 break
106
107 except BrokenPipeError:
108 pass
109 except KeyboardInterrupt:
110 pass
111 finally:
112 if sock:
113 try:
114 msg = {
115 'type': 'complete',
116 'pid': os.getpid(),
117 'lines_read': count
118 }
119 sock.send(json.dumps(msg).encode() + b'\n')
120 sock.close()
121 except:
122 pass
123
124 print(f"[pcat] Read {count} lines", file=sys.stderr)
125
126
127if __name__ == '__main__':
128 main()