A database in TSV text files: data you can read, fast inserts, and an optional Rust speed-up.

0 downloads

dbbasic-tsv/dbbasic/tsv_accelerated.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-tsv.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-tsv.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-tsv.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Add launch announcement for dbbasic-tsv v1.0.1 1c9b0b8 1 year, 1 month ago History
tsv_accelerated.py · 5.4 KB · 168 lines
1#!/usr/bin/env python3
2"""
3TSV database with optional Rust acceleration
4Falls back to pure Python if Rust module not available
5"""
6
7import time
8from pathlib import Path
9from typing import Dict, List, Optional, Any
10
11# Try to import Rust accelerated functions
12try:
13 from dbbasic_rust import (
14 parse_tsv_line,
15 parse_tsv_batch,
16 search_index,
17 filter_records,
18 count_matching,
19 write_tsv_batch
20 )
21 RUST_AVAILABLE = True
22 print("🚀 Rust acceleration enabled!")
23except ImportError:
24 RUST_AVAILABLE = False
25 print("🐍 Using pure Python implementation")
26
27from .tsv import TSV as PurePythonTSV
28
29
30class AcceleratedTSV(PurePythonTSV):
31 """TSV database with optional Rust acceleration"""
32
33 def __init__(self, *args, **kwargs):
34 super().__init__(*args, **kwargs)
35 self.rust_enabled = RUST_AVAILABLE
36
37 def insert_many(self, records: List[Dict[str, Any]]) -> int:
38 """Batch insert with Rust acceleration if available"""
39 if RUST_AVAILABLE and len(records) > 100:
40 # Use Rust for large batches
41 start = time.perf_counter()
42
43 # Convert records to the format Rust expects
44 rust_records = [
45 {col: str(rec.get(col, "")) for col in self.columns}
46 for rec in records
47 ]
48
49 # Write using Rust (much faster for large batches)
50 write_tsv_batch(str(self.data_file), self.columns, rust_records)
51
52 # Update index
53 for i, record in enumerate(rust_records):
54 row_num = self.row_count + i
55 if "id" in record:
56 self.index[record["id"]].append(row_num)
57
58 self.row_count += len(records)
59 self._save_index()
60
61 elapsed = time.perf_counter() - start
62 rate = len(records) / elapsed
63 print(f" Rust insert: {len(records)} records in {elapsed:.3f}s ({rate:,.0f} records/sec)")
64
65 return len(records)
66 else:
67 # Fall back to Python for small batches
68 return super().insert_many(records)
69
70 def query(self, **conditions) -> List[Dict[str, Any]]:
71 """Query with Rust acceleration if available"""
72 if not conditions:
73 return list(self.all())
74
75 # Use index for primary key lookups
76 if "id" in conditions and len(conditions) == 1:
77 row_nums = self.index.get(conditions["id"], [])
78 if not row_nums:
79 return []
80
81 # Load specific rows
82 return self._load_rows(row_nums)
83
84 # For other queries, use Rust if available for filtering
85 if RUST_AVAILABLE and self.row_count > 1000:
86 # Load all records
87 all_records = list(self.all())
88
89 # Use Rust for filtering (much faster)
90 filtered = filter_records(all_records, conditions)
91 return filtered
92 else:
93 # Fall back to Python
94 return super().query(**conditions)
95
96 def count(self, **conditions) -> int:
97 """Count with Rust acceleration if available"""
98 if not conditions:
99 return self.row_count
100
101 if RUST_AVAILABLE and self.row_count > 1000:
102 # Use Rust for counting
103 all_records = list(self.all())
104 return count_matching(all_records, conditions)
105 else:
106 return super().count(**conditions)
107
108
109def benchmark_rust_vs_python():
110 """Compare Rust-accelerated vs pure Python performance"""
111 print("\n" + "=" * 60)
112 print("Rust vs Python TSV Performance")
113 print("=" * 60)
114
115 # Test with pure Python
116 print("\n📊 Pure Python TSV:")
117 py_tsv = PurePythonTSV("bench_python", ["id", "name", "value"])
118
119 # Insert test
120 records = [
121 {"id": str(i), "name": f"test_{i}", "value": str(i * 10)}
122 for i in range(10000)
123 ]
124
125 start = time.perf_counter()
126 py_tsv.insert_many(records)
127 py_elapsed = time.perf_counter() - start
128 print(f" Insert 10K: {py_elapsed:.3f}s ({10000/py_elapsed:,.0f} records/sec)")
129
130 # Query test
131 start = time.perf_counter()
132 for i in range(100):
133 py_tsv.query_one(id=str(i * 100))
134 py_query_elapsed = time.perf_counter() - start
135 print(f" 100 queries: {py_query_elapsed:.3f}s ({100/py_query_elapsed:,.0f} queries/sec)")
136
137 py_tsv.drop()
138
139 if RUST_AVAILABLE:
140 # Test with Rust acceleration
141 print("\n🚀 Rust-Accelerated TSV:")
142 rust_tsv = AcceleratedTSV("bench_rust", ["id", "name", "value"])
143
144 start = time.perf_counter()
145 rust_tsv.insert_many(records)
146 rust_elapsed = time.perf_counter() - start
147 print(f" Insert 10K: {rust_elapsed:.3f}s ({10000/rust_elapsed:,.0f} records/sec)")
148
149 # Query test
150 start = time.perf_counter()
151 for i in range(100):
152 rust_tsv.query_one(id=str(i * 100))
153 rust_query_elapsed = time.perf_counter() - start
154 print(f" 100 queries: {rust_query_elapsed:.3f}s ({100/rust_query_elapsed:,.0f} queries/sec)")
155
156 # Show speedup
157 print("\n📈 Speedup:")
158 print(f" Insert: {py_elapsed/rust_elapsed:.1f}x faster")
159 print(f" Query: {py_query_elapsed/rust_query_elapsed:.1f}x faster")
160
161 rust_tsv.drop()
162 else:
163 print("\n⚠️ Install Rust module for acceleration:")
164 print(" cd rust && maturin develop --release")
165
166
167if __name__ == "__main__":
168 benchmark_rust_vs_python()