askrobots / dbbasic-tsv
Public
A database in TSV text files: data you can read, fast inserts, and an optional Rust speed-up.
tsv_accelerated.py · 5.4 KB · 168 lines
| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | TSV database with optional Rust acceleration |
| 4 | Falls back to pure Python if Rust module not available |
| 5 | """ |
| 6 | |
| 7 | import time |
| 8 | from pathlib import Path |
| 9 | from typing import Dict, List, Optional, Any |
| 10 | |
| 11 | # Try to import Rust accelerated functions |
| 12 | try: |
| 13 | from dbbasic_rust import ( |
| 14 | parse_tsv_line, |
| 15 | parse_tsv_batch, |
| 16 | search_index, |
| 17 | filter_records, |
| 18 | count_matching, |
| 19 | write_tsv_batch |
| 20 | ) |
| 21 | RUST_AVAILABLE = True |
| 22 | print("🚀 Rust acceleration enabled!") |
| 23 | except ImportError: |
| 24 | RUST_AVAILABLE = False |
| 25 | print("🐍 Using pure Python implementation") |
| 26 | |
| 27 | from .tsv import TSV as PurePythonTSV |
| 28 | |
| 29 | |
| 30 | class AcceleratedTSV(PurePythonTSV): |
| 31 | """TSV database with optional Rust acceleration""" |
| 32 | |
| 33 | def __init__(self, *args, **kwargs): |
| 34 | super().__init__(*args, **kwargs) |
| 35 | self.rust_enabled = RUST_AVAILABLE |
| 36 | |
| 37 | def insert_many(self, records: List[Dict[str, Any]]) -> int: |
| 38 | """Batch insert with Rust acceleration if available""" |
| 39 | if RUST_AVAILABLE and len(records) > 100: |
| 40 | # Use Rust for large batches |
| 41 | start = time.perf_counter() |
| 42 | |
| 43 | # Convert records to the format Rust expects |
| 44 | rust_records = [ |
| 45 | {col: str(rec.get(col, "")) for col in self.columns} |
| 46 | for rec in records |
| 47 | ] |
| 48 | |
| 49 | # Write using Rust (much faster for large batches) |
| 50 | write_tsv_batch(str(self.data_file), self.columns, rust_records) |
| 51 | |
| 52 | # Update index |
| 53 | for i, record in enumerate(rust_records): |
| 54 | row_num = self.row_count + i |
| 55 | if "id" in record: |
| 56 | self.index[record["id"]].append(row_num) |
| 57 | |
| 58 | self.row_count += len(records) |
| 59 | self._save_index() |
| 60 | |
| 61 | elapsed = time.perf_counter() - start |
| 62 | rate = len(records) / elapsed |
| 63 | print(f" Rust insert: {len(records)} records in {elapsed:.3f}s ({rate:,.0f} records/sec)") |
| 64 | |
| 65 | return len(records) |
| 66 | else: |
| 67 | # Fall back to Python for small batches |
| 68 | return super().insert_many(records) |
| 69 | |
| 70 | def query(self, **conditions) -> List[Dict[str, Any]]: |
| 71 | """Query with Rust acceleration if available""" |
| 72 | if not conditions: |
| 73 | return list(self.all()) |
| 74 | |
| 75 | # Use index for primary key lookups |
| 76 | if "id" in conditions and len(conditions) == 1: |
| 77 | row_nums = self.index.get(conditions["id"], []) |
| 78 | if not row_nums: |
| 79 | return [] |
| 80 | |
| 81 | # Load specific rows |
| 82 | return self._load_rows(row_nums) |
| 83 | |
| 84 | # For other queries, use Rust if available for filtering |
| 85 | if RUST_AVAILABLE and self.row_count > 1000: |
| 86 | # Load all records |
| 87 | all_records = list(self.all()) |
| 88 | |
| 89 | # Use Rust for filtering (much faster) |
| 90 | filtered = filter_records(all_records, conditions) |
| 91 | return filtered |
| 92 | else: |
| 93 | # Fall back to Python |
| 94 | return super().query(**conditions) |
| 95 | |
| 96 | def count(self, **conditions) -> int: |
| 97 | """Count with Rust acceleration if available""" |
| 98 | if not conditions: |
| 99 | return self.row_count |
| 100 | |
| 101 | if RUST_AVAILABLE and self.row_count > 1000: |
| 102 | # Use Rust for counting |
| 103 | all_records = list(self.all()) |
| 104 | return count_matching(all_records, conditions) |
| 105 | else: |
| 106 | return super().count(**conditions) |
| 107 | |
| 108 | |
| 109 | def benchmark_rust_vs_python(): |
| 110 | """Compare Rust-accelerated vs pure Python performance""" |
| 111 | print("\n" + "=" * 60) |
| 112 | print("Rust vs Python TSV Performance") |
| 113 | print("=" * 60) |
| 114 | |
| 115 | # Test with pure Python |
| 116 | print("\n📊 Pure Python TSV:") |
| 117 | py_tsv = PurePythonTSV("bench_python", ["id", "name", "value"]) |
| 118 | |
| 119 | # Insert test |
| 120 | records = [ |
| 121 | {"id": str(i), "name": f"test_{i}", "value": str(i * 10)} |
| 122 | for i in range(10000) |
| 123 | ] |
| 124 | |
| 125 | start = time.perf_counter() |
| 126 | py_tsv.insert_many(records) |
| 127 | py_elapsed = time.perf_counter() - start |
| 128 | print(f" Insert 10K: {py_elapsed:.3f}s ({10000/py_elapsed:,.0f} records/sec)") |
| 129 | |
| 130 | # Query test |
| 131 | start = time.perf_counter() |
| 132 | for i in range(100): |
| 133 | py_tsv.query_one(id=str(i * 100)) |
| 134 | py_query_elapsed = time.perf_counter() - start |
| 135 | print(f" 100 queries: {py_query_elapsed:.3f}s ({100/py_query_elapsed:,.0f} queries/sec)") |
| 136 | |
| 137 | py_tsv.drop() |
| 138 | |
| 139 | if RUST_AVAILABLE: |
| 140 | # Test with Rust acceleration |
| 141 | print("\n🚀 Rust-Accelerated TSV:") |
| 142 | rust_tsv = AcceleratedTSV("bench_rust", ["id", "name", "value"]) |
| 143 | |
| 144 | start = time.perf_counter() |
| 145 | rust_tsv.insert_many(records) |
| 146 | rust_elapsed = time.perf_counter() - start |
| 147 | print(f" Insert 10K: {rust_elapsed:.3f}s ({10000/rust_elapsed:,.0f} records/sec)") |
| 148 | |
| 149 | # Query test |
| 150 | start = time.perf_counter() |
| 151 | for i in range(100): |
| 152 | rust_tsv.query_one(id=str(i * 100)) |
| 153 | rust_query_elapsed = time.perf_counter() - start |
| 154 | print(f" 100 queries: {rust_query_elapsed:.3f}s ({100/rust_query_elapsed:,.0f} queries/sec)") |
| 155 | |
| 156 | # Show speedup |
| 157 | print("\n📈 Speedup:") |
| 158 | print(f" Insert: {py_elapsed/rust_elapsed:.1f}x faster") |
| 159 | print(f" Query: {py_query_elapsed/rust_query_elapsed:.1f}x faster") |
| 160 | |
| 161 | rust_tsv.drop() |
| 162 | else: |
| 163 | print("\n⚠️ Install Rust module for acceleration:") |
| 164 | print(" cd rust && maturin develop --release") |
| 165 | |
| 166 | |
| 167 | if __name__ == "__main__": |
| 168 | benchmark_rust_vs_python() |