A database in TSV text files: data you can read, fast inserts, and an optional Rust speed-up.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-tsv.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-tsv.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-tsv.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Add launch announcement for dbbasic-tsv v1.0.1 1c9b0b8 1 year, 1 month ago History
lib.rs · 8.6 KB · 297 lines
1use pyo3::prelude::*;
2use rayon::prelude::*;
3use std::collections::HashMap;
4use ahash::{AHashMap, AHashSet};
5use std::fs::{File, OpenOptions};
6use std::io::{BufReader, BufWriter, BufRead, Write};
7use memchr::memchr_iter;
8
9/// Ultra-fast TSV line parser using memchr for tab finding
10#[pyfunction]
11fn parse_tsv_line_fast(line: &str, columns: Vec<String>) -> PyResult<HashMap<String, String>> {
12 let mut result = HashMap::with_capacity(columns.len());
13 let bytes = line.as_bytes();
14 let mut last_pos = 0;
15 let mut col_idx = 0;
16
17 // Use memchr to find tabs - much faster than split()
18 for tab_pos in memchr_iter(b'\t', bytes) {
19 if col_idx < columns.len() {
20 let value = std::str::from_utf8(&bytes[last_pos..tab_pos])
21 .unwrap_or("")
22 .to_string();
23 result.insert(columns[col_idx].clone(), value);
24 col_idx += 1;
25 }
26 last_pos = tab_pos + 1;
27 }
28
29 // Handle last field
30 if col_idx < columns.len() && last_pos < bytes.len() {
31 let value = std::str::from_utf8(&bytes[last_pos..])
32 .unwrap_or("")
33 .to_string();
34 result.insert(columns[col_idx].clone(), value);
35 }
36
37 // Fill remaining columns with empty strings
38 for i in (col_idx + 1)..columns.len() {
39 result.insert(columns[i].clone(), String::new());
40 }
41
42 Ok(result)
43}
44
45/// Batch parse TSV lines in parallel with pre-allocation
46#[pyfunction]
47fn parse_tsv_batch_fast(lines: Vec<String>, columns: Vec<String>) -> PyResult<Vec<HashMap<String, String>>> {
48 let column_refs: Vec<&str> = columns.iter().map(|s| s.as_str()).collect();
49
50 let results: Vec<HashMap<String, String>> = lines
51 .par_iter()
52 .map(|line| {
53 let mut record = HashMap::with_capacity(columns.len());
54 let bytes = line.as_bytes();
55 let mut last_pos = 0;
56 let mut col_idx = 0;
57
58 for tab_pos in memchr_iter(b'\t', bytes) {
59 if col_idx < column_refs.len() {
60 let value = std::str::from_utf8(&bytes[last_pos..tab_pos])
61 .unwrap_or("")
62 .to_string();
63 record.insert(column_refs[col_idx].to_string(), value);
64 col_idx += 1;
65 }
66 last_pos = tab_pos + 1;
67 }
68
69 if col_idx < column_refs.len() && last_pos < bytes.len() {
70 let value = std::str::from_utf8(&bytes[last_pos..])
71 .unwrap_or("")
72 .to_string();
73 record.insert(column_refs[col_idx].to_string(), value);
74 }
75
76 record
77 })
78 .collect();
79
80 Ok(results)
81}
82
83/// Build an index from records using AHash (fastest hash map)
84#[pyfunction]
85fn build_index(
86 records: Vec<HashMap<String, String>>,
87 index_column: String
88) -> PyResult<HashMap<String, Vec<usize>>> {
89 let mut index: AHashMap<String, Vec<usize>> = AHashMap::new();
90
91 for (row_num, record) in records.iter().enumerate() {
92 if let Some(key) = record.get(&index_column) {
93 index.entry(key.clone())
94 .or_insert_with(Vec::new)
95 .push(row_num);
96 }
97 }
98
99 // Convert back to standard HashMap for Python
100 Ok(index.into_iter().collect())
101}
102
103/// Ultra-fast filtering using parallel processing
104#[pyfunction]
105fn filter_records_fast(
106 records: Vec<HashMap<String, String>>,
107 conditions: HashMap<String, String>
108) -> PyResult<Vec<HashMap<String, String>>> {
109 if conditions.is_empty() {
110 return Ok(records);
111 }
112
113 // Pre-compute condition checks for speed
114 let condition_vec: Vec<(&str, &str)> = conditions
115 .iter()
116 .map(|(k, v)| (k.as_str(), v.as_str()))
117 .collect();
118
119 let results: Vec<HashMap<String, String>> = records
120 .into_par_iter()
121 .filter(|record| {
122 condition_vec.iter().all(|(key, value)| {
123 record.get(*key).map_or(false, |v| v == *value)
124 })
125 })
126 .collect();
127
128 Ok(results)
129}
130
131/// Optimized TSV file reader with buffering
132#[pyfunction]
133fn read_tsv_file(
134 file_path: String,
135 columns: Vec<String>,
136 limit: Option<usize>
137) -> PyResult<Vec<HashMap<String, String>>> {
138 let file = File::open(file_path)?;
139 let reader = BufReader::with_capacity(65536, file);
140 let mut records = Vec::new();
141 let mut lines_read = 0;
142
143 for line_result in reader.lines().skip(1) { // Skip header
144 if let Some(max) = limit {
145 if lines_read >= max {
146 break;
147 }
148 }
149
150 let line = line_result?;
151 let record = parse_tsv_line_fast(&line, columns.clone())?;
152 records.push(record);
153 lines_read += 1;
154 }
155
156 Ok(records)
157}
158
159/// Batch write with large buffer for maximum speed
160#[pyfunction]
161fn write_tsv_batch_fast(
162 file_path: String,
163 columns: Vec<String>,
164 records: Vec<HashMap<String, String>>,
165 append: bool
166) -> PyResult<usize> {
167 let file = if append {
168 OpenOptions::new()
169 .create(true)
170 .append(true)
171 .open(&file_path)?
172 } else {
173 File::create(&file_path)?
174 };
175
176 // Use 256KB buffer for fast writes
177 let mut writer = BufWriter::with_capacity(262144, file);
178
179 // If not appending, write header
180 if !append {
181 writeln!(writer, "{}", columns.join("\t"))?;
182 }
183
184 // Pre-allocate string buffer for each row
185 let mut row_buffer = String::with_capacity(1024);
186
187 for record in &records {
188 row_buffer.clear();
189 for (i, col) in columns.iter().enumerate() {
190 if i > 0 {
191 row_buffer.push('\t');
192 }
193 row_buffer.push_str(record.get(col).unwrap_or(&String::new()));
194 }
195 writeln!(writer, "{}", row_buffer)?;
196 }
197
198 writer.flush()?;
199 Ok(records.len())
200}
201
202/// Count matching records without materializing results
203#[pyfunction]
204fn count_matching_fast(
205 records: Vec<HashMap<String, String>>,
206 conditions: HashMap<String, String>
207) -> PyResult<usize> {
208 if conditions.is_empty() {
209 return Ok(records.len());
210 }
211
212 let condition_vec: Vec<(&str, &str)> = conditions
213 .iter()
214 .map(|(k, v)| (k.as_str(), v.as_str()))
215 .collect();
216
217 let count = records
218 .par_iter()
219 .filter(|record| {
220 condition_vec.iter().all(|(key, value)| {
221 record.get(*key).map_or(false, |v| v == *value)
222 })
223 })
224 .count();
225
226 Ok(count)
227}
228
229/// Find unique values in a column (useful for analytics)
230#[pyfunction]
231fn unique_values(
232 records: Vec<HashMap<String, String>>,
233 column: String
234) -> PyResult<Vec<String>> {
235 let unique_values: Vec<String> = records
236 .par_iter()
237 .filter_map(|record| record.get(&column).cloned())
238 .collect();
239
240 // Deduplicate using AHashSet
241 let unique: AHashSet<String> = unique_values.into_iter().collect();
242
243 Ok(unique.into_iter().collect())
244}
245
246/// Group by a column and count occurrences
247#[pyfunction]
248fn group_by_count(
249 records: Vec<HashMap<String, String>>,
250 column: String
251) -> PyResult<HashMap<String, usize>> {
252 let mut counts: AHashMap<String, usize> = AHashMap::new();
253
254 for record in records {
255 if let Some(value) = record.get(&column) {
256 *counts.entry(value.clone()).or_insert(0) += 1;
257 }
258 }
259
260 Ok(counts.into_iter().collect())
261}
262
263/// Aggregate sum of numeric column grouped by another column
264#[pyfunction]
265fn group_by_sum(
266 records: Vec<HashMap<String, String>>,
267 group_column: String,
268 sum_column: String
269) -> PyResult<HashMap<String, f64>> {
270 let mut sums: AHashMap<String, f64> = AHashMap::new();
271
272 for record in records {
273 if let (Some(group), Some(value_str)) = (record.get(&group_column), record.get(&sum_column)) {
274 if let Ok(value) = value_str.parse::<f64>() {
275 *sums.entry(group.clone()).or_insert(0.0) += value;
276 }
277 }
278 }
279
280 Ok(sums.into_iter().collect())
281}
282
283/// Python module definition
284#[pymodule]
285fn dbbasic_rust(_py: Python, m: &PyModule) -> PyResult<()> {
286 m.add_function(wrap_pyfunction!(parse_tsv_line_fast, m)?)?;
287 m.add_function(wrap_pyfunction!(parse_tsv_batch_fast, m)?)?;
288 m.add_function(wrap_pyfunction!(build_index, m)?)?;
289 m.add_function(wrap_pyfunction!(filter_records_fast, m)?)?;
290 m.add_function(wrap_pyfunction!(read_tsv_file, m)?)?;
291 m.add_function(wrap_pyfunction!(write_tsv_batch_fast, m)?)?;
292 m.add_function(wrap_pyfunction!(count_matching_fast, m)?)?;
293 m.add_function(wrap_pyfunction!(unique_values, m)?)?;
294 m.add_function(wrap_pyfunction!(group_by_count, m)?)?;
295 m.add_function(wrap_pyfunction!(group_by_sum, m)?)?;
296 Ok(())
297}