Two-way flow control for Unix pipes: a downstream command tells the upstream one how much it needs, like a SQL optimizer.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-pipe.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-pipe.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-pipe.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Add PyPI and license badges to README 52c82fe 1 year, 1 month ago History
pfilter.py · 4.1 KB · 143 lines
1#!/usr/bin/env python3
2"""
3Filter command - filters JSON lines by simple expressions.
4
5Usage:
6 pfilter '<expression>'
7
8Works standalone or with PIPE_CONTROL_SOCKET coordination.
9
10Examples:
11 cat data.json | pfilter 'age > 18'
12 cat data.json | pfilter 'name == "Alice"'
13 cat data.json | pfilter 'status == "active" and score > 50'
14"""
15
16import os
17import sys
18import json
19import socket
20import select
21
22
23def main():
24 if len(sys.argv) < 2:
25 print("Usage: pfilter '<expression>'", file=sys.stderr)
26 sys.exit(1)
27
28 predicate = sys.argv[1]
29
30 # Check for coordination socket
31 ctrl_socket = os.getenv('PIPE_CONTROL_SOCKET')
32 sock = None
33 remaining = -1 # -1 = unlimited
34
35 if ctrl_socket and os.path.exists(ctrl_socket):
36 try:
37 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
38 sock.connect(ctrl_socket)
39 sock.setblocking(False)
40
41 # Register
42 msg = {
43 'type': 'register',
44 'pid': os.getpid(),
45 'command': 'pfilter',
46 'predicate': predicate
47 }
48 sock.send(json.dumps(msg).encode() + b'\n')
49
50 print(f"[pfilter] Registered with predicate: {predicate}", file=sys.stderr)
51
52 except Exception as e:
53 print(f"[pfilter] Coordination failed: {e}", file=sys.stderr)
54 sock = None
55
56 # Process data
57 processed = 0
58 output = 0
59
60 try:
61 for line in sys.stdin:
62 # Check for control messages
63 if sock:
64 readable, _, _ = select.select([sock], [], [], 0)
65 if readable:
66 try:
67 data = sock.recv(4096).decode()
68 for msg_line in data.strip().split('\n'):
69 if msg_line:
70 msg = json.loads(msg_line)
71 if msg['type'] == 'backpressure':
72 remaining = msg['count']
73 print(f"[pfilter] Received backpressure: {remaining}", file=sys.stderr)
74 except:
75 pass
76
77 processed += 1
78 line = line.strip()
79 if not line:
80 continue
81
82 # Parse and evaluate
83 try:
84 record = json.loads(line)
85 if eval_predicate(record, predicate):
86 print(json.dumps(record))
87 sys.stdout.flush()
88 output += 1
89
90 if remaining > 0:
91 remaining -= 1
92 if remaining == 0:
93 print(f"[pfilter] Reached limit, stopping", file=sys.stderr)
94 break
95
96 except json.JSONDecodeError:
97 # Not JSON, try as plain text with simple eval
98 try:
99 if eval(predicate, {"line": line, "text": line}):
100 print(line)
101 output += 1
102 if remaining > 0:
103 remaining -= 1
104 if remaining == 0:
105 break
106 except:
107 # Can't evaluate, skip
108 pass
109
110 except BrokenPipeError:
111 pass
112 except KeyboardInterrupt:
113 pass
114 finally:
115 if sock:
116 try:
117 msg = {
118 'type': 'complete',
119 'pid': os.getpid(),
120 'processed': processed,
121 'output': output
122 }
123 sock.send(json.dumps(msg).encode() + b'\n')
124 sock.close()
125 except:
126 pass
127
128 print(f"[pfilter] Processed {processed}, output {output}", file=sys.stderr)
129
130
131def eval_predicate(record, predicate):
132 """Safely evaluate predicate against record."""
133 try:
134 # Create a safe namespace with record fields
135 namespace = dict(record)
136 # Also allow direct field access
137 return eval(predicate, {"__builtins__": {}}, namespace)
138 except:
139 return False
140
141
142if __name__ == '__main__':
143 main()