Structured logging to TSV files with JSON context: grep-able, compressible and searchable.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-logs.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-logs.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-logs.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Add PyPI badges and links to README cea7e20 1 year, 1 month ago History
__init__.py · 7.7 KB · 242 lines
1"""
2dbbasic-logs: Structured logging with TSV storage and compression
3
4Philosophy: "Log everything. Query anything. Compress the rest."
5
6Usage:
7 from dbbasic_logs import log
8
9 log.info("User logged in", user_id=42)
10 log.error("Payment failed", order_id=123, error="Timeout")
11 log.exception("Unexpected error", context="processing_payment")
12"""
13
14import os
15import time
16import json
17import traceback
18import subprocess
19from datetime import datetime, timedelta
20
21__version__ = "1.0.0"
22
23LOG_DIR = os.getenv('LOG_DIR', 'data/logs')
24
25
26class DBBasicLogger:
27 """Structured logger with TSV storage and compression support"""
28
29 def __init__(self, log_dir=None):
30 """Initialize logger with directory structure
31
32 Args:
33 log_dir: Base directory for logs (default: data/logs)
34 """
35 self.log_dir = log_dir or LOG_DIR
36 self._ensure_directories()
37
38 def _ensure_directories(self):
39 """Create log directory structure"""
40 for log_type in ['app', 'errors', 'access']:
41 os.makedirs(f'{self.log_dir}/{log_type}', exist_ok=True)
42
43 def _write(self, log_type, level, message, context):
44 """Write log entry to appropriate file
45
46 Args:
47 log_type: Log category (app, errors, access)
48 level: Log level (INFO, WARNING, ERROR, DEBUG)
49 message: Log message
50 context: Additional context dictionary
51 """
52 today = time.strftime('%Y-%m-%d')
53 log_file = f'{self.log_dir}/{log_type}/{today}.tsv'
54
55 timestamp = int(time.time())
56 context_json = json.dumps(context) if context else '{}'
57
58 with open(log_file, 'a') as f:
59 f.write(f'{timestamp}\t{level}\t{message}\t{context_json}\n')
60
61 def info(self, message, **context):
62 """Log informational message
63
64 Args:
65 message: Log message
66 **context: Additional context as keyword arguments
67
68 Example:
69 log.info("User logged in", user_id=42, ip="192.168.1.1")
70 """
71 self._write('app', 'INFO', message, context)
72
73 def warning(self, message, **context):
74 """Log warning message
75
76 Args:
77 message: Warning message
78 **context: Additional context as keyword arguments
79 """
80 self._write('app', 'WARNING', message, context)
81
82 def error(self, message, **context):
83 """Log error message
84
85 Args:
86 message: Error message
87 **context: Additional context as keyword arguments
88
89 Example:
90 log.error("Payment failed", order_id=123, error="Timeout")
91 """
92 self._write('app', 'ERROR', message, context)
93
94 def debug(self, message, **context):
95 """Log debug message
96
97 Args:
98 message: Debug message
99 **context: Additional context as keyword arguments
100 """
101 self._write('app', 'DEBUG', message, context)
102
103 def exception(self, message, **context):
104 """Log exception with automatic stack trace capture
105
106 Should be called from within an exception handler.
107
108 Args:
109 message: Error description
110 **context: Additional context as keyword arguments
111
112 Example:
113 try:
114 process_payment(order)
115 except Exception as e:
116 log.exception("Payment failed", order_id=order.id)
117 raise
118 """
119 context['trace'] = traceback.format_exc()
120 self._write('errors', 'ERROR', message, context)
121
122 def access(self, method, path, status, duration, **context):
123 """Log HTTP access
124
125 Args:
126 method: HTTP method (GET, POST, etc.)
127 path: Request path
128 status: Response status code
129 duration: Request duration in seconds
130 **context: Additional context (ip, user_id, etc.)
131
132 Example:
133 log.access("GET", "/api/users", 200, 0.05, ip=request.remote_addr)
134 """
135 msg = f'{method} {path} {status}'
136 context['duration'] = duration
137 self._write('access', 'INFO', msg, context)
138
139 def search(self, pattern, log_type='app', days=7):
140 """Search logs using grep/zgrep
141
142 Searches both compressed (.tsv.gz) and uncompressed (.tsv) log files.
143
144 Args:
145 pattern: Regex pattern to search for
146 log_type: Log category to search ('app', 'errors', 'access', 'all')
147 days: Number of days back to search
148
149 Returns:
150 List of matching log entries as dictionaries
151
152 Example:
153 errors = log.search("ERROR", log_type='errors', days=7)
154 user_logs = log.search("user_id.*42", log_type='all', days=30)
155 """
156 results = []
157 log_types = ['app', 'errors', 'access'] if log_type == 'all' else [log_type]
158
159 for lt in log_types:
160 for i in range(days):
161 date = (datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d')
162
163 # Try uncompressed first
164 log_file = f'{self.log_dir}/{lt}/{date}.tsv'
165 if os.path.exists(log_file):
166 cmd = ['grep', pattern, log_file]
167 else:
168 # Try compressed
169 log_file_gz = f'{log_file}.gz'
170 if os.path.exists(log_file_gz):
171 cmd = ['zgrep', pattern, log_file_gz]
172 else:
173 continue
174
175 try:
176 output = subprocess.check_output(cmd, text=True, stderr=subprocess.DEVNULL)
177 for line in output.strip().split('\n'):
178 if line:
179 parts = line.split('\t')
180 if len(parts) >= 3:
181 results.append({
182 'timestamp': int(parts[0]),
183 'level': parts[1],
184 'message': parts[2],
185 'context': json.loads(parts[3]) if len(parts) > 3 else {},
186 'log_type': lt
187 })
188 except subprocess.CalledProcessError:
189 pass # No matches
190 except (ValueError, json.JSONDecodeError):
191 pass # Skip malformed lines
192
193 return results
194
195 def tail(self, log_type='app', lines=100):
196 """Get recent log entries
197
198 Args:
199 log_type: Log category to tail
200 lines: Number of recent lines to retrieve
201
202 Returns:
203 List of recent log entries as dictionaries
204
205 Example:
206 recent = log.tail('app', lines=100)
207 recent_errors = log.tail('errors', lines=50)
208 """
209 today = time.strftime('%Y-%m-%d')
210 log_file = f'{self.log_dir}/{log_type}/{today}.tsv'
211
212 if not os.path.exists(log_file):
213 return []
214
215 # Read last N lines
216 with open(log_file) as f:
217 all_lines = f.readlines()
218 recent = all_lines[-lines:] if len(all_lines) > lines else all_lines
219
220 results = []
221 for line in recent:
222 parts = line.strip().split('\t')
223 if len(parts) >= 3:
224 try:
225 results.append({
226 'timestamp': int(parts[0]),
227 'level': parts[1],
228 'message': parts[2],
229 'context': json.loads(parts[3]) if len(parts) > 3 else {}
230 })
231 except (ValueError, json.JSONDecodeError):
232 pass # Skip malformed lines
233
234 return results
235
236
237# Global logger instance
238log = DBBasicLogger()
239
240
241# Export public API
242__all__ = ['log', 'DBBasicLogger', '__version__']