askrobots / dbbasic-pipe
Public
Two-way flow control for Unix pipes: a downstream command tells the upstream one how much it needs, like a SQL optimizer.
pcat.py · 3.8 KB · 128 lines
| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | Smart cat - reads files and respects backpressure signals. |
| 4 | |
| 5 | Usage: |
| 6 | pcat <file> |
| 7 | |
| 8 | Works standalone or with PIPE_CONTROL_SOCKET coordination. |
| 9 | |
| 10 | Examples: |
| 11 | pcat data.json |
| 12 | pcat data.json | pfilter 'age > 18' |
| 13 | """ |
| 14 | |
| 15 | import os |
| 16 | import sys |
| 17 | import json |
| 18 | import socket |
| 19 | import select |
| 20 | |
| 21 | |
| 22 | def main(): |
| 23 | if len(sys.argv) < 2: |
| 24 | print("Usage: pcat <file>", file=sys.stderr) |
| 25 | sys.exit(1) |
| 26 | |
| 27 | filename = sys.argv[1] |
| 28 | |
| 29 | if not os.path.exists(filename): |
| 30 | print(f"Error: file not found: {filename}", file=sys.stderr) |
| 31 | sys.exit(1) |
| 32 | |
| 33 | # Check for coordination socket |
| 34 | ctrl_socket = os.getenv('PIPE_CONTROL_SOCKET') |
| 35 | sock = None |
| 36 | limit = -1 # -1 = unlimited |
| 37 | |
| 38 | if ctrl_socket and os.path.exists(ctrl_socket): |
| 39 | try: |
| 40 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
| 41 | sock.connect(ctrl_socket) |
| 42 | sock.setblocking(False) |
| 43 | |
| 44 | # Register |
| 45 | msg = { |
| 46 | 'type': 'register', |
| 47 | 'pid': os.getpid(), |
| 48 | 'command': 'pcat', |
| 49 | 'file': filename |
| 50 | } |
| 51 | sock.send(json.dumps(msg).encode() + b'\n') |
| 52 | |
| 53 | print(f"[pcat] Registered, reading: {filename}", file=sys.stderr) |
| 54 | |
| 55 | # Wait a moment for backpressure signals |
| 56 | import time |
| 57 | time.sleep(0.1) |
| 58 | |
| 59 | # Check for backpressure |
| 60 | readable, _, _ = select.select([sock], [], [], 0) |
| 61 | if readable: |
| 62 | try: |
| 63 | data = sock.recv(4096).decode() |
| 64 | for msg_line in data.strip().split('\n'): |
| 65 | if msg_line: |
| 66 | msg = json.loads(msg_line) |
| 67 | if msg['type'] == 'backpressure': |
| 68 | limit = msg['count'] |
| 69 | print(f"[pcat] Received backpressure: {limit}", file=sys.stderr) |
| 70 | except: |
| 71 | pass |
| 72 | |
| 73 | except Exception as e: |
| 74 | print(f"[pcat] Coordination failed: {e}", file=sys.stderr) |
| 75 | sock = None |
| 76 | |
| 77 | # Read and output file |
| 78 | count = 0 |
| 79 | try: |
| 80 | with open(filename, 'r') as f: |
| 81 | for line in f: |
| 82 | # Check for more control messages |
| 83 | if sock: |
| 84 | readable, _, _ = select.select([sock], [], [], 0) |
| 85 | if readable: |
| 86 | try: |
| 87 | data = sock.recv(4096).decode() |
| 88 | for msg_line in data.strip().split('\n'): |
| 89 | if msg_line: |
| 90 | msg = json.loads(msg_line) |
| 91 | if msg['type'] == 'backpressure': |
| 92 | if limit == -1: # First time |
| 93 | limit = msg['count'] |
| 94 | print(f"[pcat] Received backpressure: {limit}", file=sys.stderr) |
| 95 | except: |
| 96 | pass |
| 97 | |
| 98 | print(line, end='') |
| 99 | sys.stdout.flush() |
| 100 | count += 1 |
| 101 | |
| 102 | # Stop if we hit the limit |
| 103 | if limit > 0 and count >= limit * 2: # 2x buffer for filtering |
| 104 | print(f"[pcat] Reached soft limit, stopping at {count} lines", file=sys.stderr) |
| 105 | break |
| 106 | |
| 107 | except BrokenPipeError: |
| 108 | pass |
| 109 | except KeyboardInterrupt: |
| 110 | pass |
| 111 | finally: |
| 112 | if sock: |
| 113 | try: |
| 114 | msg = { |
| 115 | 'type': 'complete', |
| 116 | 'pid': os.getpid(), |
| 117 | 'lines_read': count |
| 118 | } |
| 119 | sock.send(json.dumps(msg).encode() + b'\n') |
| 120 | sock.close() |
| 121 | except: |
| 122 | pass |
| 123 | |
| 124 | print(f"[pcat] Read {count} lines", file=sys.stderr) |
| 125 | |
| 126 | |
| 127 | if __name__ == '__main__': |
| 128 | main() |