askrobots / dbbasic-pipe
Public
Two-way flow control for Unix pipes: a downstream command tells the upstream one how much it needs, like a SQL optimizer.
pfilter.py · 4.1 KB · 143 lines
| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | Filter command - filters JSON lines by simple expressions. |
| 4 | |
| 5 | Usage: |
| 6 | pfilter '<expression>' |
| 7 | |
| 8 | Works standalone or with PIPE_CONTROL_SOCKET coordination. |
| 9 | |
| 10 | Examples: |
| 11 | cat data.json | pfilter 'age > 18' |
| 12 | cat data.json | pfilter 'name == "Alice"' |
| 13 | cat data.json | pfilter 'status == "active" and score > 50' |
| 14 | """ |
| 15 | |
| 16 | import os |
| 17 | import sys |
| 18 | import json |
| 19 | import socket |
| 20 | import select |
| 21 | |
| 22 | |
| 23 | def main(): |
| 24 | if len(sys.argv) < 2: |
| 25 | print("Usage: pfilter '<expression>'", file=sys.stderr) |
| 26 | sys.exit(1) |
| 27 | |
| 28 | predicate = sys.argv[1] |
| 29 | |
| 30 | # Check for coordination socket |
| 31 | ctrl_socket = os.getenv('PIPE_CONTROL_SOCKET') |
| 32 | sock = None |
| 33 | remaining = -1 # -1 = unlimited |
| 34 | |
| 35 | if ctrl_socket and os.path.exists(ctrl_socket): |
| 36 | try: |
| 37 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
| 38 | sock.connect(ctrl_socket) |
| 39 | sock.setblocking(False) |
| 40 | |
| 41 | # Register |
| 42 | msg = { |
| 43 | 'type': 'register', |
| 44 | 'pid': os.getpid(), |
| 45 | 'command': 'pfilter', |
| 46 | 'predicate': predicate |
| 47 | } |
| 48 | sock.send(json.dumps(msg).encode() + b'\n') |
| 49 | |
| 50 | print(f"[pfilter] Registered with predicate: {predicate}", file=sys.stderr) |
| 51 | |
| 52 | except Exception as e: |
| 53 | print(f"[pfilter] Coordination failed: {e}", file=sys.stderr) |
| 54 | sock = None |
| 55 | |
| 56 | # Process data |
| 57 | processed = 0 |
| 58 | output = 0 |
| 59 | |
| 60 | try: |
| 61 | for line in sys.stdin: |
| 62 | # Check for control messages |
| 63 | if sock: |
| 64 | readable, _, _ = select.select([sock], [], [], 0) |
| 65 | if readable: |
| 66 | try: |
| 67 | data = sock.recv(4096).decode() |
| 68 | for msg_line in data.strip().split('\n'): |
| 69 | if msg_line: |
| 70 | msg = json.loads(msg_line) |
| 71 | if msg['type'] == 'backpressure': |
| 72 | remaining = msg['count'] |
| 73 | print(f"[pfilter] Received backpressure: {remaining}", file=sys.stderr) |
| 74 | except: |
| 75 | pass |
| 76 | |
| 77 | processed += 1 |
| 78 | line = line.strip() |
| 79 | if not line: |
| 80 | continue |
| 81 | |
| 82 | # Parse and evaluate |
| 83 | try: |
| 84 | record = json.loads(line) |
| 85 | if eval_predicate(record, predicate): |
| 86 | print(json.dumps(record)) |
| 87 | sys.stdout.flush() |
| 88 | output += 1 |
| 89 | |
| 90 | if remaining > 0: |
| 91 | remaining -= 1 |
| 92 | if remaining == 0: |
| 93 | print(f"[pfilter] Reached limit, stopping", file=sys.stderr) |
| 94 | break |
| 95 | |
| 96 | except json.JSONDecodeError: |
| 97 | # Not JSON, try as plain text with simple eval |
| 98 | try: |
| 99 | if eval(predicate, {"line": line, "text": line}): |
| 100 | print(line) |
| 101 | output += 1 |
| 102 | if remaining > 0: |
| 103 | remaining -= 1 |
| 104 | if remaining == 0: |
| 105 | break |
| 106 | except: |
| 107 | # Can't evaluate, skip |
| 108 | pass |
| 109 | |
| 110 | except BrokenPipeError: |
| 111 | pass |
| 112 | except KeyboardInterrupt: |
| 113 | pass |
| 114 | finally: |
| 115 | if sock: |
| 116 | try: |
| 117 | msg = { |
| 118 | 'type': 'complete', |
| 119 | 'pid': os.getpid(), |
| 120 | 'processed': processed, |
| 121 | 'output': output |
| 122 | } |
| 123 | sock.send(json.dumps(msg).encode() + b'\n') |
| 124 | sock.close() |
| 125 | except: |
| 126 | pass |
| 127 | |
| 128 | print(f"[pfilter] Processed {processed}, output {output}", file=sys.stderr) |
| 129 | |
| 130 | |
| 131 | def eval_predicate(record, predicate): |
| 132 | """Safely evaluate predicate against record.""" |
| 133 | try: |
| 134 | # Create a safe namespace with record fields |
| 135 | namespace = dict(record) |
| 136 | # Also allow direct field access |
| 137 | return eval(predicate, {"__builtins__": {}}, namespace) |
| 138 | except: |
| 139 | return False |
| 140 | |
| 141 | |
| 142 | if __name__ == '__main__': |
| 143 | main() |