Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_logs.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_logs.py · 12.7 KB · 445 lines
1"""Object log storage.
2
3The working prototype stores object logs in TSV files:
4
5data/logs/{object_id}/log.tsv
6
7Rows are written with a header. The default fields are ``entry_id``,
8``timestamp``, ``level``, and ``message``; object code may add extra columns.
9The active log stays plain TSV. Rotated logs are gzip-compressed by default.
10"""
11
12from __future__ import annotations
13
14import csv
15import gzip
16import os
17import shutil
18from collections.abc import Iterable
19from contextlib import contextmanager
20from datetime import datetime, timezone
21from pathlib import Path
22from typing import Any
23from uuid import uuid4
24
25import object_correlation
26from object_namespace import validate_object_id
27from object_versions import DEFAULT_DATA_DIR, InvalidObjectIdError
28
29
30LOG_FILE = "log.tsv"
31LOG_MAX_BYTES_ENV = "DBBASIC_LOG_MAX_BYTES"
32LOG_COMPRESS_ROTATED_ENV = "DBBASIC_LOG_COMPRESS_ROTATED"
33LOG_KEEP_ROTATED_ENV = "DBBASIC_LOG_KEEP_ROTATED"
34DEFAULT_LOG_MAX_BYTES = 10 * 1024 * 1024
35DEFAULT_LOG_KEEP_ROTATED = 32
36FALSE_VALUES = {"0", "false", "no", "off"}
37DEFAULT_LOG_FIELDS = [
38 "entry_id",
39 "correlation_id",
40 "timestamp",
41 "level",
42 "message",
43 "method",
44 "status",
45 "duration_ms",
46 "error_type",
47 "error",
48]
49
50
51class ObjectLogger:
52 """Runtime logger injected into object modules as ``_logger``."""
53
54 def __init__(
55 self,
56 object_id: str,
57 base_dir: Path | str = DEFAULT_DATA_DIR,
58 *,
59 max_log_bytes: int | None = None,
60 compress_rotated: bool | None = None,
61 keep_rotated: int | None = None,
62 ):
63 self.object_id = object_id
64 self.base_dir = Path(base_dir)
65 self.max_log_bytes = max_log_bytes
66 self.compress_rotated = compress_rotated
67 self.keep_rotated = keep_rotated
68
69 def log(self, level: str, message: str, **fields: Any) -> dict[str, Any]:
70 """Append one object-owned log entry."""
71 return append_object_log(
72 self.object_id,
73 str(level).upper(),
74 str(message),
75 base_dir=self.base_dir,
76 max_log_bytes=self.max_log_bytes,
77 compress_rotated=self.compress_rotated,
78 keep_rotated=self.keep_rotated,
79 **fields,
80 )
81
82 def debug(self, message: str, **fields: Any) -> dict[str, Any]:
83 """Log a DEBUG message."""
84 return self.log("DEBUG", message, **fields)
85
86 def info(self, message: str, **fields: Any) -> dict[str, Any]:
87 """Log an INFO message."""
88 return self.log("INFO", message, **fields)
89
90 def warning(self, message: str, **fields: Any) -> dict[str, Any]:
91 """Log a WARNING message."""
92 return self.log("WARNING", message, **fields)
93
94 def error(self, message: str, **fields: Any) -> dict[str, Any]:
95 """Log an ERROR message."""
96 return self.log("ERROR", message, **fields)
97
98 def critical(self, message: str, **fields: Any) -> dict[str, Any]:
99 """Log a CRITICAL message."""
100 return self.log("CRITICAL", message, **fields)
101
102 def get_logs(
103 self,
104 *,
105 level: str | Iterable[str] | None = None,
106 limit: int | None = None,
107 offset: int = 0,
108 **filters: str,
109 ) -> list[dict[str, Any]]:
110 """Read logs for this object."""
111 return get_object_logs(
112 self.object_id,
113 base_dir=self.base_dir,
114 level=level,
115 limit=limit,
116 offset=offset,
117 **filters,
118 )
119
120
121def append_object_log(
122 object_id: str,
123 level: str,
124 message: str,
125 base_dir: Path | str = DEFAULT_DATA_DIR,
126 *,
127 max_log_bytes: int | None = None,
128 compress_rotated: bool | None = None,
129 keep_rotated: int | None = None,
130 **fields: Any,
131) -> dict[str, Any]:
132 """Append one entry to ``data/logs/{object_id}/log.tsv``."""
133 log_dir = object_log_dir(object_id, base_dir)
134 log_dir.mkdir(parents=True, exist_ok=True)
135
136 if fields.get("correlation_id") is None:
137 fields["correlation_id"] = object_correlation.current_correlation_id()
138
139 entry = {
140 "entry_id": uuid4().hex[:16],
141 "timestamp": _utc_timestamp(),
142 "level": level,
143 "message": message,
144 **fields,
145 }
146 entry = {key: value for key, value in entry.items() if value is not None}
147
148 _append_log_entry(
149 log_dir / LOG_FILE,
150 entry,
151 max_log_bytes=max_log_bytes,
152 compress_rotated=compress_rotated,
153 keep_rotated=keep_rotated,
154 )
155 return entry
156
157
158def get_object_logs(
159 object_id: str,
160 base_dir: Path | str = DEFAULT_DATA_DIR,
161 *,
162 level: str | Iterable[str] | None = None,
163 limit: int | None = None,
164 offset: int = 0,
165 **filters: str,
166) -> list[dict[str, Any]]:
167 """Return object logs from ``data/logs/{object_id}``."""
168 log_dir = object_log_dir(object_id, base_dir)
169 log_files = _log_files(log_dir)
170 if not log_files:
171 return []
172
173 entries: list[dict[str, Any]] = []
174 for log_file in log_files:
175 entries.extend(_read_log_file(log_file))
176
177 entries = _filter_entries(entries, level=level, filters=filters)
178
179 if offset > 0:
180 entries = entries[offset:]
181
182 if limit is not None:
183 entries = entries[:limit]
184
185 return entries
186
187
188def object_log_dir(object_id: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> Path:
189 """Return a validated object log directory path."""
190 if not validate_object_id(object_id):
191 raise InvalidObjectIdError(f"Invalid object ID: {object_id}")
192
193 logs_root = Path(base_dir) / "logs"
194 log_dir = logs_root / object_id
195 resolved_root = logs_root.resolve(strict=False)
196 resolved_dir = log_dir.resolve(strict=False)
197
198 try:
199 resolved_dir.relative_to(resolved_root)
200 except ValueError as exc:
201 raise InvalidObjectIdError(f"Object ID escapes logs directory: {object_id}") from exc
202
203 return log_dir
204
205
206def _log_files(log_dir: Path) -> list[Path]:
207 current_log = log_dir / LOG_FILE
208 files = []
209 if current_log.exists() and current_log.is_file():
210 files.append(current_log)
211
212 files.extend(_rotated_log_files(log_dir))
213 return files
214
215
216def _read_log_file(log_file: Path) -> list[dict[str, Any]]:
217 entries: list[dict[str, Any]] = []
218 with _open_log_file(log_file) as f:
219 reader = csv.DictReader(f, delimiter="\t")
220 for row in reader:
221 entry = _clean_entry(row)
222 if entry:
223 entries.append(entry)
224 return entries
225
226
227def _open_log_file(log_file: Path):
228 if log_file.suffix == ".gz":
229 return gzip.open(log_file, "rt", newline="")
230
231 return log_file.open("r", newline="")
232
233
234def _clean_entry(row: dict[str | None, Any]) -> dict[str, Any]:
235 return {key: value for key, value in row.items() if key is not None}
236
237
238def _append_log_entry(
239 log_file: Path,
240 entry: dict[str, Any],
241 *,
242 max_log_bytes: int | None = None,
243 compress_rotated: bool | None = None,
244 keep_rotated: int | None = None,
245) -> None:
246 with _log_file_lock(log_file):
247 _rotate_if_needed(
248 log_file,
249 max_log_bytes=_resolve_max_log_bytes(max_log_bytes),
250 compress_rotated=_resolve_compress_rotated(compress_rotated),
251 keep_rotated=_resolve_keep_rotated(keep_rotated),
252 )
253
254 fieldnames = _append_fieldnames(log_file, entry)
255 is_new_file = not log_file.exists() or log_file.stat().st_size == 0
256
257 with log_file.open("a", newline="") as f:
258 writer = csv.DictWriter(f, fieldnames=fieldnames, delimiter="\t")
259 if is_new_file:
260 writer.writeheader()
261 writer.writerow(entry)
262
263
264@contextmanager
265def _log_file_lock(log_file: Path):
266 """Use a best-effort advisory lock for rotation and appends."""
267 lock_path = log_file.with_name(f".{log_file.name}.lock")
268 lock_path.parent.mkdir(parents=True, exist_ok=True)
269
270 with lock_path.open("a") as lock_file:
271 try:
272 import fcntl
273 except ImportError:
274 fcntl = None
275
276 if fcntl is not None:
277 fcntl.flock(lock_file, fcntl.LOCK_EX)
278
279 try:
280 yield
281 finally:
282 if fcntl is not None:
283 fcntl.flock(lock_file, fcntl.LOCK_UN)
284
285
286def _rotate_if_needed(
287 log_file: Path,
288 *,
289 max_log_bytes: int | None,
290 compress_rotated: bool,
291 keep_rotated: int | None,
292) -> None:
293 if max_log_bytes is None or not log_file.exists():
294 return
295
296 if log_file.stat().st_size < max_log_bytes:
297 return
298
299 rotated_path = _next_rotated_log_path(log_file.parent, compress_rotated=compress_rotated)
300
301 if compress_rotated:
302 temp_path = rotated_path.with_name(f".{rotated_path.name}.tmp")
303 with log_file.open("rb") as source, gzip.open(temp_path, "wb", compresslevel=6) as target:
304 shutil.copyfileobj(source, target)
305 temp_path.replace(rotated_path)
306 log_file.unlink()
307 _cleanup_rotated_logs(log_file.parent, keep_rotated=keep_rotated)
308 return
309
310 log_file.replace(rotated_path)
311 _cleanup_rotated_logs(log_file.parent, keep_rotated=keep_rotated)
312
313
314def _next_rotated_log_path(log_dir: Path, *, compress_rotated: bool) -> Path:
315 timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S-%f")
316 suffix = ".tsv.gz" if compress_rotated else ".tsv"
317 candidate = log_dir / f"log-{timestamp}{suffix}"
318 index = 1
319
320 while candidate.exists():
321 candidate = log_dir / f"log-{timestamp}-{index}{suffix}"
322 index += 1
323
324 return candidate
325
326
327def _resolve_max_log_bytes(value: int | None) -> int | None:
328 if value is not None:
329 return value if value > 0 else None
330
331 raw_value = os.environ.get(LOG_MAX_BYTES_ENV)
332 if raw_value is None:
333 return DEFAULT_LOG_MAX_BYTES
334
335 try:
336 parsed = int(raw_value)
337 except ValueError:
338 return DEFAULT_LOG_MAX_BYTES
339
340 return parsed if parsed > 0 else None
341
342
343def _resolve_compress_rotated(value: bool | None) -> bool:
344 if value is not None:
345 return value
346
347 raw_value = os.environ.get(LOG_COMPRESS_ROTATED_ENV)
348 if raw_value is None:
349 return True
350
351 return raw_value.strip().lower() not in FALSE_VALUES
352
353
354def _resolve_keep_rotated(value: int | None) -> int | None:
355 if value is not None:
356 return value if value > 0 else None
357
358 raw_value = os.environ.get(LOG_KEEP_ROTATED_ENV)
359 if raw_value is None:
360 return DEFAULT_LOG_KEEP_ROTATED
361
362 try:
363 parsed = int(raw_value)
364 except ValueError:
365 return DEFAULT_LOG_KEEP_ROTATED
366
367 return parsed if parsed > 0 else None
368
369
370def _rotated_log_files(log_dir: Path) -> list[Path]:
371 return sorted(
372 path
373 for pattern in ("log-*.tsv", "log-*.tsv.gz")
374 for path in log_dir.glob(pattern)
375 if path.is_file()
376 )
377
378
379def _cleanup_rotated_logs(log_dir: Path, *, keep_rotated: int | None) -> None:
380 if keep_rotated is None:
381 return
382
383 rotated_files = _rotated_log_files(log_dir)
384 stale_files = rotated_files[:-keep_rotated]
385 for stale_file in stale_files:
386 stale_file.unlink(missing_ok=True)
387
388
389def _append_fieldnames(log_file: Path, entry: dict[str, Any]) -> list[str]:
390 existing = _existing_fieldnames(log_file)
391 fieldnames = _merged_fieldnames(existing, entry.keys())
392
393 if existing and fieldnames != existing:
394 _rewrite_log_file(log_file, fieldnames)
395
396 return fieldnames
397
398
399def _existing_fieldnames(log_file: Path) -> list[str]:
400 if not log_file.exists() or log_file.stat().st_size == 0:
401 return []
402
403 with log_file.open("r", newline="") as f:
404 reader = csv.DictReader(f, delimiter="\t")
405 return list(reader.fieldnames or [])
406
407
408def _merged_fieldnames(existing: list[str], entry_keys: Iterable[str]) -> list[str]:
409 fieldnames: list[str] = []
410 for field in [*existing, *DEFAULT_LOG_FIELDS, *entry_keys]:
411 if field not in fieldnames:
412 fieldnames.append(field)
413 return fieldnames
414
415
416def _rewrite_log_file(log_file: Path, fieldnames: list[str]) -> None:
417 entries = _read_log_file(log_file)
418 temp_path = log_file.with_name(f".{log_file.name}.tmp")
419
420 with temp_path.open("w", newline="") as f:
421 writer = csv.DictWriter(f, fieldnames=fieldnames, delimiter="\t")
422 writer.writeheader()
423 writer.writerows(entries)
424
425 temp_path.replace(log_file)
426
427
428def _filter_entries(
429 entries: list[dict[str, Any]],
430 *,
431 level: str | Iterable[str] | None,
432 filters: dict[str, str],
433) -> list[dict[str, Any]]:
434 if level is not None:
435 levels = {level} if isinstance(level, str) else set(level)
436 entries = [entry for entry in entries if entry.get("level") in levels]
437
438 for key, value in filters.items():
439 entries = [entry for entry in entries if entry.get(key) == value]
440
441 return entries
442
443
444def _utc_timestamp() -> str:
445 return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")