askrobots / dbbasic-logs
Public
Structured logging to TSV files with JSON context: grep-able, compressible and searchable.
__init__.py · 7.7 KB · 242 lines
| 1 | """ |
| 2 | dbbasic-logs: Structured logging with TSV storage and compression |
| 3 | |
| 4 | Philosophy: "Log everything. Query anything. Compress the rest." |
| 5 | |
| 6 | Usage: |
| 7 | from dbbasic_logs import log |
| 8 | |
| 9 | log.info("User logged in", user_id=42) |
| 10 | log.error("Payment failed", order_id=123, error="Timeout") |
| 11 | log.exception("Unexpected error", context="processing_payment") |
| 12 | """ |
| 13 | |
| 14 | import os |
| 15 | import time |
| 16 | import json |
| 17 | import traceback |
| 18 | import subprocess |
| 19 | from datetime import datetime, timedelta |
| 20 | |
| 21 | __version__ = "1.0.0" |
| 22 | |
| 23 | LOG_DIR = os.getenv('LOG_DIR', 'data/logs') |
| 24 | |
| 25 | |
| 26 | class DBBasicLogger: |
| 27 | """Structured logger with TSV storage and compression support""" |
| 28 | |
| 29 | def __init__(self, log_dir=None): |
| 30 | """Initialize logger with directory structure |
| 31 | |
| 32 | Args: |
| 33 | log_dir: Base directory for logs (default: data/logs) |
| 34 | """ |
| 35 | self.log_dir = log_dir or LOG_DIR |
| 36 | self._ensure_directories() |
| 37 | |
| 38 | def _ensure_directories(self): |
| 39 | """Create log directory structure""" |
| 40 | for log_type in ['app', 'errors', 'access']: |
| 41 | os.makedirs(f'{self.log_dir}/{log_type}', exist_ok=True) |
| 42 | |
| 43 | def _write(self, log_type, level, message, context): |
| 44 | """Write log entry to appropriate file |
| 45 | |
| 46 | Args: |
| 47 | log_type: Log category (app, errors, access) |
| 48 | level: Log level (INFO, WARNING, ERROR, DEBUG) |
| 49 | message: Log message |
| 50 | context: Additional context dictionary |
| 51 | """ |
| 52 | today = time.strftime('%Y-%m-%d') |
| 53 | log_file = f'{self.log_dir}/{log_type}/{today}.tsv' |
| 54 | |
| 55 | timestamp = int(time.time()) |
| 56 | context_json = json.dumps(context) if context else '{}' |
| 57 | |
| 58 | with open(log_file, 'a') as f: |
| 59 | f.write(f'{timestamp}\t{level}\t{message}\t{context_json}\n') |
| 60 | |
| 61 | def info(self, message, **context): |
| 62 | """Log informational message |
| 63 | |
| 64 | Args: |
| 65 | message: Log message |
| 66 | **context: Additional context as keyword arguments |
| 67 | |
| 68 | Example: |
| 69 | log.info("User logged in", user_id=42, ip="192.168.1.1") |
| 70 | """ |
| 71 | self._write('app', 'INFO', message, context) |
| 72 | |
| 73 | def warning(self, message, **context): |
| 74 | """Log warning message |
| 75 | |
| 76 | Args: |
| 77 | message: Warning message |
| 78 | **context: Additional context as keyword arguments |
| 79 | """ |
| 80 | self._write('app', 'WARNING', message, context) |
| 81 | |
| 82 | def error(self, message, **context): |
| 83 | """Log error message |
| 84 | |
| 85 | Args: |
| 86 | message: Error message |
| 87 | **context: Additional context as keyword arguments |
| 88 | |
| 89 | Example: |
| 90 | log.error("Payment failed", order_id=123, error="Timeout") |
| 91 | """ |
| 92 | self._write('app', 'ERROR', message, context) |
| 93 | |
| 94 | def debug(self, message, **context): |
| 95 | """Log debug message |
| 96 | |
| 97 | Args: |
| 98 | message: Debug message |
| 99 | **context: Additional context as keyword arguments |
| 100 | """ |
| 101 | self._write('app', 'DEBUG', message, context) |
| 102 | |
| 103 | def exception(self, message, **context): |
| 104 | """Log exception with automatic stack trace capture |
| 105 | |
| 106 | Should be called from within an exception handler. |
| 107 | |
| 108 | Args: |
| 109 | message: Error description |
| 110 | **context: Additional context as keyword arguments |
| 111 | |
| 112 | Example: |
| 113 | try: |
| 114 | process_payment(order) |
| 115 | except Exception as e: |
| 116 | log.exception("Payment failed", order_id=order.id) |
| 117 | raise |
| 118 | """ |
| 119 | context['trace'] = traceback.format_exc() |
| 120 | self._write('errors', 'ERROR', message, context) |
| 121 | |
| 122 | def access(self, method, path, status, duration, **context): |
| 123 | """Log HTTP access |
| 124 | |
| 125 | Args: |
| 126 | method: HTTP method (GET, POST, etc.) |
| 127 | path: Request path |
| 128 | status: Response status code |
| 129 | duration: Request duration in seconds |
| 130 | **context: Additional context (ip, user_id, etc.) |
| 131 | |
| 132 | Example: |
| 133 | log.access("GET", "/api/users", 200, 0.05, ip=request.remote_addr) |
| 134 | """ |
| 135 | msg = f'{method} {path} {status}' |
| 136 | context['duration'] = duration |
| 137 | self._write('access', 'INFO', msg, context) |
| 138 | |
| 139 | def search(self, pattern, log_type='app', days=7): |
| 140 | """Search logs using grep/zgrep |
| 141 | |
| 142 | Searches both compressed (.tsv.gz) and uncompressed (.tsv) log files. |
| 143 | |
| 144 | Args: |
| 145 | pattern: Regex pattern to search for |
| 146 | log_type: Log category to search ('app', 'errors', 'access', 'all') |
| 147 | days: Number of days back to search |
| 148 | |
| 149 | Returns: |
| 150 | List of matching log entries as dictionaries |
| 151 | |
| 152 | Example: |
| 153 | errors = log.search("ERROR", log_type='errors', days=7) |
| 154 | user_logs = log.search("user_id.*42", log_type='all', days=30) |
| 155 | """ |
| 156 | results = [] |
| 157 | log_types = ['app', 'errors', 'access'] if log_type == 'all' else [log_type] |
| 158 | |
| 159 | for lt in log_types: |
| 160 | for i in range(days): |
| 161 | date = (datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d') |
| 162 | |
| 163 | # Try uncompressed first |
| 164 | log_file = f'{self.log_dir}/{lt}/{date}.tsv' |
| 165 | if os.path.exists(log_file): |
| 166 | cmd = ['grep', pattern, log_file] |
| 167 | else: |
| 168 | # Try compressed |
| 169 | log_file_gz = f'{log_file}.gz' |
| 170 | if os.path.exists(log_file_gz): |
| 171 | cmd = ['zgrep', pattern, log_file_gz] |
| 172 | else: |
| 173 | continue |
| 174 | |
| 175 | try: |
| 176 | output = subprocess.check_output(cmd, text=True, stderr=subprocess.DEVNULL) |
| 177 | for line in output.strip().split('\n'): |
| 178 | if line: |
| 179 | parts = line.split('\t') |
| 180 | if len(parts) >= 3: |
| 181 | results.append({ |
| 182 | 'timestamp': int(parts[0]), |
| 183 | 'level': parts[1], |
| 184 | 'message': parts[2], |
| 185 | 'context': json.loads(parts[3]) if len(parts) > 3 else {}, |
| 186 | 'log_type': lt |
| 187 | }) |
| 188 | except subprocess.CalledProcessError: |
| 189 | pass # No matches |
| 190 | except (ValueError, json.JSONDecodeError): |
| 191 | pass # Skip malformed lines |
| 192 | |
| 193 | return results |
| 194 | |
| 195 | def tail(self, log_type='app', lines=100): |
| 196 | """Get recent log entries |
| 197 | |
| 198 | Args: |
| 199 | log_type: Log category to tail |
| 200 | lines: Number of recent lines to retrieve |
| 201 | |
| 202 | Returns: |
| 203 | List of recent log entries as dictionaries |
| 204 | |
| 205 | Example: |
| 206 | recent = log.tail('app', lines=100) |
| 207 | recent_errors = log.tail('errors', lines=50) |
| 208 | """ |
| 209 | today = time.strftime('%Y-%m-%d') |
| 210 | log_file = f'{self.log_dir}/{log_type}/{today}.tsv' |
| 211 | |
| 212 | if not os.path.exists(log_file): |
| 213 | return [] |
| 214 | |
| 215 | # Read last N lines |
| 216 | with open(log_file) as f: |
| 217 | all_lines = f.readlines() |
| 218 | recent = all_lines[-lines:] if len(all_lines) > lines else all_lines |
| 219 | |
| 220 | results = [] |
| 221 | for line in recent: |
| 222 | parts = line.strip().split('\t') |
| 223 | if len(parts) >= 3: |
| 224 | try: |
| 225 | results.append({ |
| 226 | 'timestamp': int(parts[0]), |
| 227 | 'level': parts[1], |
| 228 | 'message': parts[2], |
| 229 | 'context': json.loads(parts[3]) if len(parts) > 3 else {} |
| 230 | }) |
| 231 | except (ValueError, json.JSONDecodeError): |
| 232 | pass # Skip malformed lines |
| 233 | |
| 234 | return results |
| 235 | |
| 236 | |
| 237 | # Global logger instance |
| 238 | log = DBBasicLogger() |
| 239 | |
| 240 | |
| 241 | # Export public API |
| 242 | __all__ = ['log', 'DBBasicLogger', '__version__'] |