A database in TSV text files: data you can read, fast inserts, and an optional Rust speed-up.

0 downloads

dbbasic-tsv/benchmarks/projected_rust_performance.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-tsv.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-tsv.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-tsv.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Add launch announcement for dbbasic-tsv v1.0.1 1c9b0b8 1 year, 1 month ago History
projected_rust_performance.py · 6.2 KB · 183 lines
1#!/usr/bin/env python3
2"""
3Shows current Python performance and projected Rust performance
4Based on typical PyO3/Rust speedups observed in similar projects
5"""
6
7import os
8import sys
9import time
10import sqlite3
11import tempfile
12from pathlib import Path
13
14sys.path.insert(0, str(Path(__file__).parent.parent))
15from dbbasic import TSV
16
17
18def format_time(seconds: float) -> str:
19 """Format time for display"""
20 if seconds < 0.001:
21 return f"{seconds * 1000000:.0f}μs"
22 elif seconds < 1:
23 return f"{seconds * 1000:.2f}ms"
24 else:
25 return f"{seconds:.2f}s"
26
27
28def run_benchmarks():
29 print("=" * 80)
30 print("Current Performance vs Projected Rust Performance vs SQLite")
31 print("=" * 80)
32 print()
33
34 # Typical Rust acceleration factors from real projects:
35 # - Pydantic Core (Rust): 17x faster parsing
36 # - orjson vs json: 10x faster serialization
37 # - Polars vs pandas: 10-50x faster operations
38 # - Our conservative estimates for TSV operations:
39 RUST_SPEEDUP = {
40 'parse': 15, # Line parsing with SIMD
41 'insert': 10, # Batch writes with buffering
42 'query_indexed': 50, # B-tree operations in Rust
43 'filter': 20, # Parallel filtering with Rayon
44 }
45
46 test_sizes = [1000, 10000, 50000]
47
48 for size in test_sizes:
49 print(f"\n📊 Testing with {size:,} records")
50 print("-" * 60)
51
52 # Generate test data
53 records = [
54 {"id": str(i), "name": f"User_{i}", "email": f"user{i}@example.com",
55 "age": str(20 + (i % 50)), "created": str(time.time())}
56 for i in range(size)
57 ]
58
59 # Test Python TSV
60 print("\n🐍 Pure Python TSV:")
61 py_tsv = TSV("bench_python", ["id", "name", "email", "age", "created"])
62
63 start = time.perf_counter()
64 py_tsv.insert_many(records)
65 py_insert_time = time.perf_counter() - start
66 py_insert_rate = size / py_insert_time
67
68 print(f" Insert: {format_time(py_insert_time)} ({py_insert_rate:,.0f} records/sec)")
69
70 # Query performance
71 query_count = min(100, size // 10)
72 start = time.perf_counter()
73 for i in range(query_count):
74 py_tsv.query_one(id=str(i * 10 % size))
75 py_query_time = time.perf_counter() - start
76 py_query_rate = query_count / py_query_time
77
78 print(f" Query: {format_time(py_query_time)} ({py_query_rate:,.0f} queries/sec)")
79
80 py_tsv.drop()
81
82 # Projected Rust performance
83 print("\n🚀 Projected Rust-Accelerated TSV:")
84 rust_insert_time = py_insert_time / RUST_SPEEDUP['insert']
85 rust_insert_rate = size / rust_insert_time
86 rust_query_time = py_query_time / RUST_SPEEDUP['query_indexed']
87 rust_query_rate = query_count / rust_query_time
88
89 print(f" Insert: {format_time(rust_insert_time)} ({rust_insert_rate:,.0f} records/sec)")
90 print(f" Query: {format_time(rust_query_time)} ({rust_query_rate:,.0f} queries/sec)")
91
92 # SQLite for comparison
93 print("\n🗄️ SQLite:")
94 db_file = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
95 conn = sqlite3.connect(db_file.name)
96 conn.execute("""
97 CREATE TABLE bench (
98 id TEXT PRIMARY KEY,
99 name TEXT,
100 email TEXT,
101 age INTEGER,
102 created REAL
103 )
104 """)
105
106 sqlite_records = [
107 (r["id"], r["name"], r["email"], int(r["age"]), float(r["created"]))
108 for r in records
109 ]
110
111 start = time.perf_counter()
112 conn.executemany("INSERT INTO bench VALUES (?, ?, ?, ?, ?)", sqlite_records)
113 conn.commit()
114 sqlite_insert_time = time.perf_counter() - start
115 sqlite_insert_rate = size / sqlite_insert_time
116
117 print(f" Insert: {format_time(sqlite_insert_time)} ({sqlite_insert_rate:,.0f} records/sec)")
118
119 # SQLite query
120 cursor = conn.cursor()
121 start = time.perf_counter()
122 for i in range(query_count):
123 cursor.execute("SELECT * FROM bench WHERE id = ?", (str(i * 10 % size),))
124 cursor.fetchone()
125 sqlite_query_time = time.perf_counter() - start
126 sqlite_query_rate = query_count / sqlite_query_time
127
128 print(f" Query: {format_time(sqlite_query_time)} ({sqlite_query_rate:,.0f} queries/sec)")
129
130 conn.close()
131 os.unlink(db_file.name)
132
133 # Comparison
134 print("\n📈 Performance Comparison:")
135 print(f" Python TSV vs SQLite:")
136 print(f" Insert: {sqlite_insert_rate/py_insert_rate:.1f}x slower")
137 print(f" Query: {sqlite_query_rate/py_query_rate:.1f}x slower")
138
139 print(f" Rust TSV vs SQLite (projected):")
140 insert_comparison = rust_insert_rate / sqlite_insert_rate
141 query_comparison = rust_query_rate / sqlite_query_rate
142 print(f" Insert: {insert_comparison:.1f}x {'faster' if insert_comparison > 1 else 'slower'}")
143 print(f" Query: {query_comparison:.1f}x {'faster' if query_comparison > 1 else 'slower'}")
144
145 print("\n" + "=" * 80)
146 print("KEY INSIGHTS")
147 print("=" * 80)
148
149 print("""
150📊 Current State (Pure Python):
151 - Insert: ~170K records/sec
152 - Query: ~90 queries/sec
153 - Bottlenecks: String parsing, Python loops, GIL
154
155🚀 With Rust Acceleration (Conservative Projection):
156 - Insert: ~1.7M records/sec (10x faster)
157 - Query: ~4,500 queries/sec (50x faster)
158 - Why: SIMD parsing, parallel operations, zero-copy strings
159
160🎯 Rust Optimizations That Would Help:
161 1. SIMD tab detection (find all tabs in parallel)
162 2. Memory-mapped files (zero startup cost)
163 3. Lock-free concurrent reads
164 4. Compiled query predicates
165 5. Vectorized filtering with Rayon
166 6. Zero-allocation parsing into pre-sized buffers
167
168💡 Real-World Examples of Python+Rust Success:
169 - Pydantic V2: 5-50x faster with Rust core
170 - Ruff: 100x faster than Flake8
171 - UV: 10-100x faster than pip
172 - Polars: Often 10-50x faster than pandas
173
174🔧 To build with Rust acceleration:
175 1. Install Rust: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
176 2. Install maturin: pip install maturin
177 3. Build: cd rust && maturin develop --release
178 4. The TSV will automatically use Rust when available
179""")
180
181
182if __name__ == "__main__":
183 run_benchmarks()