askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_logs.py · 12.7 KB · 445 lines
| 1 | """Object log storage. |
| 2 | |
| 3 | The working prototype stores object logs in TSV files: |
| 4 | |
| 5 | data/logs/{object_id}/log.tsv |
| 6 | |
| 7 | Rows are written with a header. The default fields are ``entry_id``, |
| 8 | ``timestamp``, ``level``, and ``message``; object code may add extra columns. |
| 9 | The active log stays plain TSV. Rotated logs are gzip-compressed by default. |
| 10 | """ |
| 11 | |
| 12 | from __future__ import annotations |
| 13 | |
| 14 | import csv |
| 15 | import gzip |
| 16 | import os |
| 17 | import shutil |
| 18 | from collections.abc import Iterable |
| 19 | from contextlib import contextmanager |
| 20 | from datetime import datetime, timezone |
| 21 | from pathlib import Path |
| 22 | from typing import Any |
| 23 | from uuid import uuid4 |
| 24 | |
| 25 | import object_correlation |
| 26 | from object_namespace import validate_object_id |
| 27 | from object_versions import DEFAULT_DATA_DIR, InvalidObjectIdError |
| 28 | |
| 29 | |
| 30 | LOG_FILE = "log.tsv" |
| 31 | LOG_MAX_BYTES_ENV = "DBBASIC_LOG_MAX_BYTES" |
| 32 | LOG_COMPRESS_ROTATED_ENV = "DBBASIC_LOG_COMPRESS_ROTATED" |
| 33 | LOG_KEEP_ROTATED_ENV = "DBBASIC_LOG_KEEP_ROTATED" |
| 34 | DEFAULT_LOG_MAX_BYTES = 10 * 1024 * 1024 |
| 35 | DEFAULT_LOG_KEEP_ROTATED = 32 |
| 36 | FALSE_VALUES = {"0", "false", "no", "off"} |
| 37 | DEFAULT_LOG_FIELDS = [ |
| 38 | "entry_id", |
| 39 | "correlation_id", |
| 40 | "timestamp", |
| 41 | "level", |
| 42 | "message", |
| 43 | "method", |
| 44 | "status", |
| 45 | "duration_ms", |
| 46 | "error_type", |
| 47 | "error", |
| 48 | ] |
| 49 | |
| 50 | |
| 51 | class ObjectLogger: |
| 52 | """Runtime logger injected into object modules as ``_logger``.""" |
| 53 | |
| 54 | def __init__( |
| 55 | self, |
| 56 | object_id: str, |
| 57 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 58 | *, |
| 59 | max_log_bytes: int | None = None, |
| 60 | compress_rotated: bool | None = None, |
| 61 | keep_rotated: int | None = None, |
| 62 | ): |
| 63 | self.object_id = object_id |
| 64 | self.base_dir = Path(base_dir) |
| 65 | self.max_log_bytes = max_log_bytes |
| 66 | self.compress_rotated = compress_rotated |
| 67 | self.keep_rotated = keep_rotated |
| 68 | |
| 69 | def log(self, level: str, message: str, **fields: Any) -> dict[str, Any]: |
| 70 | """Append one object-owned log entry.""" |
| 71 | return append_object_log( |
| 72 | self.object_id, |
| 73 | str(level).upper(), |
| 74 | str(message), |
| 75 | base_dir=self.base_dir, |
| 76 | max_log_bytes=self.max_log_bytes, |
| 77 | compress_rotated=self.compress_rotated, |
| 78 | keep_rotated=self.keep_rotated, |
| 79 | **fields, |
| 80 | ) |
| 81 | |
| 82 | def debug(self, message: str, **fields: Any) -> dict[str, Any]: |
| 83 | """Log a DEBUG message.""" |
| 84 | return self.log("DEBUG", message, **fields) |
| 85 | |
| 86 | def info(self, message: str, **fields: Any) -> dict[str, Any]: |
| 87 | """Log an INFO message.""" |
| 88 | return self.log("INFO", message, **fields) |
| 89 | |
| 90 | def warning(self, message: str, **fields: Any) -> dict[str, Any]: |
| 91 | """Log a WARNING message.""" |
| 92 | return self.log("WARNING", message, **fields) |
| 93 | |
| 94 | def error(self, message: str, **fields: Any) -> dict[str, Any]: |
| 95 | """Log an ERROR message.""" |
| 96 | return self.log("ERROR", message, **fields) |
| 97 | |
| 98 | def critical(self, message: str, **fields: Any) -> dict[str, Any]: |
| 99 | """Log a CRITICAL message.""" |
| 100 | return self.log("CRITICAL", message, **fields) |
| 101 | |
| 102 | def get_logs( |
| 103 | self, |
| 104 | *, |
| 105 | level: str | Iterable[str] | None = None, |
| 106 | limit: int | None = None, |
| 107 | offset: int = 0, |
| 108 | **filters: str, |
| 109 | ) -> list[dict[str, Any]]: |
| 110 | """Read logs for this object.""" |
| 111 | return get_object_logs( |
| 112 | self.object_id, |
| 113 | base_dir=self.base_dir, |
| 114 | level=level, |
| 115 | limit=limit, |
| 116 | offset=offset, |
| 117 | **filters, |
| 118 | ) |
| 119 | |
| 120 | |
| 121 | def append_object_log( |
| 122 | object_id: str, |
| 123 | level: str, |
| 124 | message: str, |
| 125 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 126 | *, |
| 127 | max_log_bytes: int | None = None, |
| 128 | compress_rotated: bool | None = None, |
| 129 | keep_rotated: int | None = None, |
| 130 | **fields: Any, |
| 131 | ) -> dict[str, Any]: |
| 132 | """Append one entry to ``data/logs/{object_id}/log.tsv``.""" |
| 133 | log_dir = object_log_dir(object_id, base_dir) |
| 134 | log_dir.mkdir(parents=True, exist_ok=True) |
| 135 | |
| 136 | if fields.get("correlation_id") is None: |
| 137 | fields["correlation_id"] = object_correlation.current_correlation_id() |
| 138 | |
| 139 | entry = { |
| 140 | "entry_id": uuid4().hex[:16], |
| 141 | "timestamp": _utc_timestamp(), |
| 142 | "level": level, |
| 143 | "message": message, |
| 144 | **fields, |
| 145 | } |
| 146 | entry = {key: value for key, value in entry.items() if value is not None} |
| 147 | |
| 148 | _append_log_entry( |
| 149 | log_dir / LOG_FILE, |
| 150 | entry, |
| 151 | max_log_bytes=max_log_bytes, |
| 152 | compress_rotated=compress_rotated, |
| 153 | keep_rotated=keep_rotated, |
| 154 | ) |
| 155 | return entry |
| 156 | |
| 157 | |
| 158 | def get_object_logs( |
| 159 | object_id: str, |
| 160 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 161 | *, |
| 162 | level: str | Iterable[str] | None = None, |
| 163 | limit: int | None = None, |
| 164 | offset: int = 0, |
| 165 | **filters: str, |
| 166 | ) -> list[dict[str, Any]]: |
| 167 | """Return object logs from ``data/logs/{object_id}``.""" |
| 168 | log_dir = object_log_dir(object_id, base_dir) |
| 169 | log_files = _log_files(log_dir) |
| 170 | if not log_files: |
| 171 | return [] |
| 172 | |
| 173 | entries: list[dict[str, Any]] = [] |
| 174 | for log_file in log_files: |
| 175 | entries.extend(_read_log_file(log_file)) |
| 176 | |
| 177 | entries = _filter_entries(entries, level=level, filters=filters) |
| 178 | |
| 179 | if offset > 0: |
| 180 | entries = entries[offset:] |
| 181 | |
| 182 | if limit is not None: |
| 183 | entries = entries[:limit] |
| 184 | |
| 185 | return entries |
| 186 | |
| 187 | |
| 188 | def object_log_dir(object_id: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> Path: |
| 189 | """Return a validated object log directory path.""" |
| 190 | if not validate_object_id(object_id): |
| 191 | raise InvalidObjectIdError(f"Invalid object ID: {object_id}") |
| 192 | |
| 193 | logs_root = Path(base_dir) / "logs" |
| 194 | log_dir = logs_root / object_id |
| 195 | resolved_root = logs_root.resolve(strict=False) |
| 196 | resolved_dir = log_dir.resolve(strict=False) |
| 197 | |
| 198 | try: |
| 199 | resolved_dir.relative_to(resolved_root) |
| 200 | except ValueError as exc: |
| 201 | raise InvalidObjectIdError(f"Object ID escapes logs directory: {object_id}") from exc |
| 202 | |
| 203 | return log_dir |
| 204 | |
| 205 | |
| 206 | def _log_files(log_dir: Path) -> list[Path]: |
| 207 | current_log = log_dir / LOG_FILE |
| 208 | files = [] |
| 209 | if current_log.exists() and current_log.is_file(): |
| 210 | files.append(current_log) |
| 211 | |
| 212 | files.extend(_rotated_log_files(log_dir)) |
| 213 | return files |
| 214 | |
| 215 | |
| 216 | def _read_log_file(log_file: Path) -> list[dict[str, Any]]: |
| 217 | entries: list[dict[str, Any]] = [] |
| 218 | with _open_log_file(log_file) as f: |
| 219 | reader = csv.DictReader(f, delimiter="\t") |
| 220 | for row in reader: |
| 221 | entry = _clean_entry(row) |
| 222 | if entry: |
| 223 | entries.append(entry) |
| 224 | return entries |
| 225 | |
| 226 | |
| 227 | def _open_log_file(log_file: Path): |
| 228 | if log_file.suffix == ".gz": |
| 229 | return gzip.open(log_file, "rt", newline="") |
| 230 | |
| 231 | return log_file.open("r", newline="") |
| 232 | |
| 233 | |
| 234 | def _clean_entry(row: dict[str | None, Any]) -> dict[str, Any]: |
| 235 | return {key: value for key, value in row.items() if key is not None} |
| 236 | |
| 237 | |
| 238 | def _append_log_entry( |
| 239 | log_file: Path, |
| 240 | entry: dict[str, Any], |
| 241 | *, |
| 242 | max_log_bytes: int | None = None, |
| 243 | compress_rotated: bool | None = None, |
| 244 | keep_rotated: int | None = None, |
| 245 | ) -> None: |
| 246 | with _log_file_lock(log_file): |
| 247 | _rotate_if_needed( |
| 248 | log_file, |
| 249 | max_log_bytes=_resolve_max_log_bytes(max_log_bytes), |
| 250 | compress_rotated=_resolve_compress_rotated(compress_rotated), |
| 251 | keep_rotated=_resolve_keep_rotated(keep_rotated), |
| 252 | ) |
| 253 | |
| 254 | fieldnames = _append_fieldnames(log_file, entry) |
| 255 | is_new_file = not log_file.exists() or log_file.stat().st_size == 0 |
| 256 | |
| 257 | with log_file.open("a", newline="") as f: |
| 258 | writer = csv.DictWriter(f, fieldnames=fieldnames, delimiter="\t") |
| 259 | if is_new_file: |
| 260 | writer.writeheader() |
| 261 | writer.writerow(entry) |
| 262 | |
| 263 | |
| 264 | @contextmanager |
| 265 | def _log_file_lock(log_file: Path): |
| 266 | """Use a best-effort advisory lock for rotation and appends.""" |
| 267 | lock_path = log_file.with_name(f".{log_file.name}.lock") |
| 268 | lock_path.parent.mkdir(parents=True, exist_ok=True) |
| 269 | |
| 270 | with lock_path.open("a") as lock_file: |
| 271 | try: |
| 272 | import fcntl |
| 273 | except ImportError: |
| 274 | fcntl = None |
| 275 | |
| 276 | if fcntl is not None: |
| 277 | fcntl.flock(lock_file, fcntl.LOCK_EX) |
| 278 | |
| 279 | try: |
| 280 | yield |
| 281 | finally: |
| 282 | if fcntl is not None: |
| 283 | fcntl.flock(lock_file, fcntl.LOCK_UN) |
| 284 | |
| 285 | |
| 286 | def _rotate_if_needed( |
| 287 | log_file: Path, |
| 288 | *, |
| 289 | max_log_bytes: int | None, |
| 290 | compress_rotated: bool, |
| 291 | keep_rotated: int | None, |
| 292 | ) -> None: |
| 293 | if max_log_bytes is None or not log_file.exists(): |
| 294 | return |
| 295 | |
| 296 | if log_file.stat().st_size < max_log_bytes: |
| 297 | return |
| 298 | |
| 299 | rotated_path = _next_rotated_log_path(log_file.parent, compress_rotated=compress_rotated) |
| 300 | |
| 301 | if compress_rotated: |
| 302 | temp_path = rotated_path.with_name(f".{rotated_path.name}.tmp") |
| 303 | with log_file.open("rb") as source, gzip.open(temp_path, "wb", compresslevel=6) as target: |
| 304 | shutil.copyfileobj(source, target) |
| 305 | temp_path.replace(rotated_path) |
| 306 | log_file.unlink() |
| 307 | _cleanup_rotated_logs(log_file.parent, keep_rotated=keep_rotated) |
| 308 | return |
| 309 | |
| 310 | log_file.replace(rotated_path) |
| 311 | _cleanup_rotated_logs(log_file.parent, keep_rotated=keep_rotated) |
| 312 | |
| 313 | |
| 314 | def _next_rotated_log_path(log_dir: Path, *, compress_rotated: bool) -> Path: |
| 315 | timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S-%f") |
| 316 | suffix = ".tsv.gz" if compress_rotated else ".tsv" |
| 317 | candidate = log_dir / f"log-{timestamp}{suffix}" |
| 318 | index = 1 |
| 319 | |
| 320 | while candidate.exists(): |
| 321 | candidate = log_dir / f"log-{timestamp}-{index}{suffix}" |
| 322 | index += 1 |
| 323 | |
| 324 | return candidate |
| 325 | |
| 326 | |
| 327 | def _resolve_max_log_bytes(value: int | None) -> int | None: |
| 328 | if value is not None: |
| 329 | return value if value > 0 else None |
| 330 | |
| 331 | raw_value = os.environ.get(LOG_MAX_BYTES_ENV) |
| 332 | if raw_value is None: |
| 333 | return DEFAULT_LOG_MAX_BYTES |
| 334 | |
| 335 | try: |
| 336 | parsed = int(raw_value) |
| 337 | except ValueError: |
| 338 | return DEFAULT_LOG_MAX_BYTES |
| 339 | |
| 340 | return parsed if parsed > 0 else None |
| 341 | |
| 342 | |
| 343 | def _resolve_compress_rotated(value: bool | None) -> bool: |
| 344 | if value is not None: |
| 345 | return value |
| 346 | |
| 347 | raw_value = os.environ.get(LOG_COMPRESS_ROTATED_ENV) |
| 348 | if raw_value is None: |
| 349 | return True |
| 350 | |
| 351 | return raw_value.strip().lower() not in FALSE_VALUES |
| 352 | |
| 353 | |
| 354 | def _resolve_keep_rotated(value: int | None) -> int | None: |
| 355 | if value is not None: |
| 356 | return value if value > 0 else None |
| 357 | |
| 358 | raw_value = os.environ.get(LOG_KEEP_ROTATED_ENV) |
| 359 | if raw_value is None: |
| 360 | return DEFAULT_LOG_KEEP_ROTATED |
| 361 | |
| 362 | try: |
| 363 | parsed = int(raw_value) |
| 364 | except ValueError: |
| 365 | return DEFAULT_LOG_KEEP_ROTATED |
| 366 | |
| 367 | return parsed if parsed > 0 else None |
| 368 | |
| 369 | |
| 370 | def _rotated_log_files(log_dir: Path) -> list[Path]: |
| 371 | return sorted( |
| 372 | path |
| 373 | for pattern in ("log-*.tsv", "log-*.tsv.gz") |
| 374 | for path in log_dir.glob(pattern) |
| 375 | if path.is_file() |
| 376 | ) |
| 377 | |
| 378 | |
| 379 | def _cleanup_rotated_logs(log_dir: Path, *, keep_rotated: int | None) -> None: |
| 380 | if keep_rotated is None: |
| 381 | return |
| 382 | |
| 383 | rotated_files = _rotated_log_files(log_dir) |
| 384 | stale_files = rotated_files[:-keep_rotated] |
| 385 | for stale_file in stale_files: |
| 386 | stale_file.unlink(missing_ok=True) |
| 387 | |
| 388 | |
| 389 | def _append_fieldnames(log_file: Path, entry: dict[str, Any]) -> list[str]: |
| 390 | existing = _existing_fieldnames(log_file) |
| 391 | fieldnames = _merged_fieldnames(existing, entry.keys()) |
| 392 | |
| 393 | if existing and fieldnames != existing: |
| 394 | _rewrite_log_file(log_file, fieldnames) |
| 395 | |
| 396 | return fieldnames |
| 397 | |
| 398 | |
| 399 | def _existing_fieldnames(log_file: Path) -> list[str]: |
| 400 | if not log_file.exists() or log_file.stat().st_size == 0: |
| 401 | return [] |
| 402 | |
| 403 | with log_file.open("r", newline="") as f: |
| 404 | reader = csv.DictReader(f, delimiter="\t") |
| 405 | return list(reader.fieldnames or []) |
| 406 | |
| 407 | |
| 408 | def _merged_fieldnames(existing: list[str], entry_keys: Iterable[str]) -> list[str]: |
| 409 | fieldnames: list[str] = [] |
| 410 | for field in [*existing, *DEFAULT_LOG_FIELDS, *entry_keys]: |
| 411 | if field not in fieldnames: |
| 412 | fieldnames.append(field) |
| 413 | return fieldnames |
| 414 | |
| 415 | |
| 416 | def _rewrite_log_file(log_file: Path, fieldnames: list[str]) -> None: |
| 417 | entries = _read_log_file(log_file) |
| 418 | temp_path = log_file.with_name(f".{log_file.name}.tmp") |
| 419 | |
| 420 | with temp_path.open("w", newline="") as f: |
| 421 | writer = csv.DictWriter(f, fieldnames=fieldnames, delimiter="\t") |
| 422 | writer.writeheader() |
| 423 | writer.writerows(entries) |
| 424 | |
| 425 | temp_path.replace(log_file) |
| 426 | |
| 427 | |
| 428 | def _filter_entries( |
| 429 | entries: list[dict[str, Any]], |
| 430 | *, |
| 431 | level: str | Iterable[str] | None, |
| 432 | filters: dict[str, str], |
| 433 | ) -> list[dict[str, Any]]: |
| 434 | if level is not None: |
| 435 | levels = {level} if isinstance(level, str) else set(level) |
| 436 | entries = [entry for entry in entries if entry.get("level") in levels] |
| 437 | |
| 438 | for key, value in filters.items(): |
| 439 | entries = [entry for entry in entries if entry.get(key) == value] |
| 440 | |
| 441 | return entries |
| 442 | |
| 443 | |
| 444 | def _utc_timestamp() -> str: |
| 445 | return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") |