askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_rollups.py · 32.0 KB · 771 lines
| 1 | """Rollup block: declared aggregations materialized as ordinary collections. |
| 2 | |
| 3 | Implements ``plan/vocabulary/14-rollup-spec.md``. A ``rollup_definitions`` |
| 4 | record declares one source collection, an optional flat-equality filter, a |
| 5 | ``group_by``, an optional ``time_bucket`` (day/week/month truncation), and |
| 6 | a list of ``metrics`` (count/sum/min/max/avg). ``compute_rollup`` reads the |
| 7 | source outside row-level permissions (daemon posture, same as |
| 8 | ``object_records.compact_collection``), computes the full current answer, |
| 9 | and writes it into ``target_collection`` -- a schema-derived, ordinary |
| 10 | collection that renders through the existing generators with zero page |
| 11 | code, the stated payoff (14's Surfaces section). |
| 12 | |
| 13 | **Storage is a full rewrite by effect, not by mechanism.** 14's Storage |
| 14 | section is explicit that a SCHEDULED recompute should behave like classic |
| 15 | storage's atomic replace-the-whole-file write -- "the correct operation, |
| 16 | not a workaround." There is no public bulk "replace this entire |
| 17 | collection" primitive in ``object_records`` to call (the module-private |
| 18 | one, ``_write_collection_records``, is reserved for that module's own |
| 19 | internal callers), so this reconciles the target to the newly computed row |
| 20 | set through the ordinary per-row record API instead: create a row for |
| 21 | every new id, update every row that already existed, delete every row |
| 22 | whose group no longer exists. The net effect is identical to a full |
| 23 | rewrite -- a full recompute always produces every row's current answer and |
| 24 | always drops stale groups -- it is just expressed as N create/update/delete |
| 25 | calls rather than one file swap. |
| 26 | |
| 27 | **Scheduled only in v1.** 14's Events section describes an INCREMENTAL |
| 28 | create-fast-path (a HANDLES subscriber on the source collection, updating |
| 29 | one target row additively) alongside a reconciling scheduled recompute |
| 30 | that "self-heals any drift the fast path couldn't handle correctly." This |
| 31 | module implements only the reconciling recompute side. A definition |
| 32 | declaring ``refresh_mode: incremental`` is accepted and still recomputes |
| 33 | correctly on its own ``refresh_interval_seconds`` -- 14 is explicit that |
| 34 | "scheduled-only is the v1 default and the recommendation" for most |
| 35 | sources, and that incremental's own interval is required precisely |
| 36 | because it "is not incremental instead of scheduled, it is an additive |
| 37 | create-fast-path on top of a slower scheduled full recompute." Without the |
| 38 | fast path, an incremental definition simply behaves as scheduled-only -- |
| 39 | the same graceful-degradation shape 14's own Degradation section |
| 40 | describes for ``DBBASIC_ENABLE_EVENT_HANDLERS`` being unset ("nothing |
| 41 | breaks; freshness just tracks the scheduled interval instead of |
| 42 | near-real-time"). The HANDLES-subscriber fast path is not built here. |
| 43 | |
| 44 | **Writing computed fields.** The derived target schema marks every metric |
| 45 | field (and ``computed_at``) ``"computed": true`` per 14's field contract |
| 46 | ("a target row's numbers come only from the rollup pass, never from a |
| 47 | form POST"). ``object_records.create_collection_record`` / |
| 48 | ``update_collection_record`` reject a write to a genuinely computed field |
| 49 | from ANY caller by default -- there was no existing escape hatch for a |
| 50 | system pass to write its own derived values, so this block adds one |
| 51 | (``allow_computed_submission``, see object_records.py), used only here. |
| 52 | """ |
| 53 | from __future__ import annotations |
| 54 | |
| 55 | import hashlib |
| 56 | import json |
| 57 | from dataclasses import dataclass |
| 58 | from datetime import date, datetime, timedelta, timezone |
| 59 | from typing import Any, Mapping |
| 60 | |
| 61 | import object_collections |
| 62 | import object_feature_flags |
| 63 | import object_records |
| 64 | import object_schemas |
| 65 | |
| 66 | ROLLUP_DEFINITIONS_COLLECTION = "rollup_definitions" |
| 67 | FEATURE_FLAGS_COLLECTION = "feature_flags" |
| 68 | ROLLUP_ENABLED_FLAG = "rollup_enabled" |
| 69 | ROLLUP_ACTOR = "daemon:rollup" |
| 70 | |
| 71 | _TIME_BUCKET_GRANULARITIES = frozenset({"none", "day", "week", "month"}) |
| 72 | _METRIC_OPS = frozenset({"count", "sum", "min", "max", "avg"}) |
| 73 | _REFRESH_MODES = frozenset({"scheduled", "incremental"}) |
| 74 | _RESERVED_TARGET_FIELD_NAMES = frozenset({"id", "bucket_start", "computed_at"}) |
| 75 | |
| 76 | |
| 77 | class DefinitionError(ValueError): |
| 78 | """Raised when a rollup_definitions record cannot be computed as-is. |
| 79 | |
| 80 | Covers both malformed JSON sub-fields and violations of 14's own |
| 81 | doctrine (a filter value that isn't a flat scalar, a metric field that |
| 82 | isn't numeric, and so on). The daemon pass catches this per-definition |
| 83 | (process_rollups, object_daemon.py), same isolation posture as |
| 84 | process_compactions: one bad definition is logged and skipped, never |
| 85 | allowed to stop the rest of the pass. |
| 86 | """ |
| 87 | |
| 88 | |
| 89 | @dataclass(frozen=True) |
| 90 | class Metric: |
| 91 | op: str |
| 92 | field: str | None |
| 93 | as_name: str |
| 94 | source_type: str | None # "integer" | "number" | None (count has none) |
| 95 | target_type: str # "integer" | "number" |
| 96 | |
| 97 | |
| 98 | @dataclass(frozen=True) |
| 99 | class RollupConfig: |
| 100 | definition_id: str |
| 101 | name: str |
| 102 | source_collection: str |
| 103 | target_collection: str |
| 104 | filter: dict[str, Any] |
| 105 | group_by: tuple[str, ...] |
| 106 | time_bucket_field: str | None |
| 107 | time_bucket_granularity: str |
| 108 | time_bucket_source_type: str | None |
| 109 | metrics: tuple[Metric, ...] |
| 110 | min_group_size: int | None |
| 111 | refresh_mode: str |
| 112 | refresh_interval_seconds: int |
| 113 | enabled: bool |
| 114 | |
| 115 | |
| 116 | # --- Feature flag / due gate ------------------------------------------------- |
| 117 | |
| 118 | def rollup_pass_enabled(*, base_dir: Any) -> bool: |
| 119 | """Block-wide kill switch, ``<block>_enabled`` convention (00-doctrine- |
| 120 | and-contract.md), a ``feature_flags`` row named ``rollup_enabled``. |
| 121 | |
| 122 | Default ON -- same reasoning 12-notify-spec.md gives for its own |
| 123 | block-wide flag defaulting on: this is the brownout kill switch, not |
| 124 | an adoption gate, and with zero ``rollup_definitions`` rows installed |
| 125 | it does nothing regardless. A missing row, a missing/unreadable |
| 126 | ``feature_flags`` collection, and a blank value all resolve to "on"; |
| 127 | only an explicit off/false/0/no value turns the pass off (14's |
| 128 | Degradation: "existing target collections keep serving their |
| 129 | last-computed data"). |
| 130 | """ |
| 131 | return object_feature_flags.kill_switch_on(ROLLUP_ENABLED_FLAG, base_dir=base_dir) |
| 132 | |
| 133 | |
| 134 | def is_definition_enabled(record: Mapping[str, str]) -> bool: |
| 135 | """Per-definition kill switch (14's ``enabled`` field), independent of |
| 136 | the block-wide ``rollup_enabled`` flag -- "lets an operator pause one |
| 137 | noisy or broken definition without disabling every rollup on the |
| 138 | instance." Blank/missing defaults to enabled, same as any schema |
| 139 | boolean field with ``"default": "true"``. |
| 140 | """ |
| 141 | return _truthy(record.get("enabled"), default=True) |
| 142 | |
| 143 | |
| 144 | def is_definition_due(record: Mapping[str, str], *, now: datetime | None = None) -> bool: |
| 145 | """SCHEDULED-mode due gate: ``last_computed_at`` + ``refresh_interval_ |
| 146 | seconds``, both read straight off the definition record -- 14's |
| 147 | Storage section is explicit that ``rollup_definitions`` needs no |
| 148 | separate marker file the way ``process_compactions`` does, "because it |
| 149 | has no natural per-collection record to hold state on, a rollup |
| 150 | definition already is that record." |
| 151 | |
| 152 | A never-computed definition (``last_computed_at`` blank) is always |
| 153 | due. A malformed ``refresh_interval_seconds`` is also treated as due, |
| 154 | so the definition's real problem surfaces as a logged |
| 155 | ``compute_rollup`` failure rather than the daemon silently never |
| 156 | attempting it. |
| 157 | """ |
| 158 | now = now or datetime.now(timezone.utc) |
| 159 | last_computed_raw = (record.get("last_computed_at") or "").strip() |
| 160 | if not last_computed_raw: |
| 161 | return True |
| 162 | interval = _positive_int(record.get("refresh_interval_seconds")) |
| 163 | if interval is None: |
| 164 | return True |
| 165 | last_computed = _parse_iso(last_computed_raw) |
| 166 | if last_computed is None: |
| 167 | return True |
| 168 | return (now - last_computed).total_seconds() >= interval |
| 169 | |
| 170 | |
| 171 | # --- Definition parsing ------------------------------------------------- |
| 172 | |
| 173 | def parse_definition( |
| 174 | record: Mapping[str, str], *, base_dir: Any, roots: Any = None |
| 175 | ) -> RollupConfig: |
| 176 | """Validate and normalize one ``rollup_definitions`` row. |
| 177 | |
| 178 | Raises ``DefinitionError`` for anything malformed, and lets |
| 179 | ``object_schemas.SchemaNotFoundError`` / |
| 180 | ``object_collections.CollectionNotFoundError`` propagate unchanged |
| 181 | when ``source_collection`` doesn't exist -- 14's Degradation section |
| 182 | treats a missing source as its own failure mode ("the recompute for |
| 183 | that definition fails and is logged; the target collection keeps |
| 184 | serving its last successful computation"), distinct from a malformed |
| 185 | definition. |
| 186 | """ |
| 187 | definition_id = (record.get("id") or "").strip() |
| 188 | if not definition_id: |
| 189 | raise DefinitionError("rollup definition is missing an id") |
| 190 | |
| 191 | name = (record.get("name") or "").strip() or definition_id |
| 192 | |
| 193 | source_collection = (record.get("source_collection") or "").strip() |
| 194 | if not source_collection or not object_collections.validate_collection_name(source_collection): |
| 195 | raise DefinitionError(f"{definition_id}: source_collection is required and must be a valid collection name") |
| 196 | |
| 197 | target_collection = (record.get("target_collection") or "").strip() |
| 198 | if not target_collection or not object_collections.validate_collection_name(target_collection): |
| 199 | raise DefinitionError(f"{definition_id}: target_collection is required and must be a valid collection name") |
| 200 | |
| 201 | # Raises SchemaNotFoundError when source_collection doesn't exist -- |
| 202 | # deliberately not caught here, see docstring. |
| 203 | source_schema = object_schemas.get_schema(source_collection, base_dir=base_dir, roots=roots) |
| 204 | source_fields = {field["name"]: field for field in source_schema.get("fields", [])} |
| 205 | |
| 206 | filter_ = _parse_filter(record, definition_id=definition_id) |
| 207 | group_by = _parse_group_by(record, definition_id=definition_id) |
| 208 | |
| 209 | time_bucket_field, time_bucket_granularity, time_bucket_source_type = _parse_time_bucket( |
| 210 | record, definition_id=definition_id, source_collection=source_collection, source_fields=source_fields, |
| 211 | ) |
| 212 | |
| 213 | metrics = _parse_metrics( |
| 214 | record, |
| 215 | definition_id=definition_id, |
| 216 | source_collection=source_collection, |
| 217 | source_fields=source_fields, |
| 218 | group_by=group_by, |
| 219 | ) |
| 220 | |
| 221 | min_group_size = _parse_min_group_size(record, definition_id=definition_id) |
| 222 | |
| 223 | refresh_mode = (record.get("refresh_mode") or "scheduled").strip().lower() |
| 224 | if refresh_mode not in _REFRESH_MODES: |
| 225 | raise DefinitionError(f"{definition_id}: refresh_mode must be one of {sorted(_REFRESH_MODES)}") |
| 226 | |
| 227 | refresh_interval_seconds = _positive_int(record.get("refresh_interval_seconds")) |
| 228 | if refresh_interval_seconds is None: |
| 229 | raise DefinitionError(f"{definition_id}: refresh_interval_seconds is required and must be a positive integer") |
| 230 | |
| 231 | enabled = _truthy(record.get("enabled"), default=True) |
| 232 | |
| 233 | return RollupConfig( |
| 234 | definition_id=definition_id, |
| 235 | name=name, |
| 236 | source_collection=source_collection, |
| 237 | target_collection=target_collection, |
| 238 | filter=filter_, |
| 239 | group_by=group_by, |
| 240 | time_bucket_field=time_bucket_field, |
| 241 | time_bucket_granularity=time_bucket_granularity, |
| 242 | time_bucket_source_type=time_bucket_source_type, |
| 243 | metrics=metrics, |
| 244 | min_group_size=min_group_size, |
| 245 | refresh_mode=refresh_mode, |
| 246 | refresh_interval_seconds=refresh_interval_seconds, |
| 247 | enabled=enabled, |
| 248 | ) |
| 249 | |
| 250 | |
| 251 | def _parse_filter(record: Mapping[str, str], *, definition_id: str) -> dict[str, Any]: |
| 252 | raw = _json_field(record, "filter", {}, definition_id=definition_id) |
| 253 | if raw is None: |
| 254 | return {} |
| 255 | if not isinstance(raw, dict): |
| 256 | raise DefinitionError(f"{definition_id}: filter must be a flat JSON object") |
| 257 | normalized: dict[str, Any] = {} |
| 258 | for key, value in raw.items(): |
| 259 | if isinstance(value, (dict, list)): |
| 260 | raise DefinitionError( |
| 261 | f"{definition_id}: filter.{key} must be a scalar -- rollup filters are flat " |
| 262 | "equality only, ANDed (00-doctrine-and-contract.md); escalate to a custom " |
| 263 | "object for OR/join/subquery/computed conditions" |
| 264 | ) |
| 265 | if value is None: |
| 266 | raise DefinitionError(f"{definition_id}: filter.{key} must not be null") |
| 267 | if isinstance(value, bool): |
| 268 | value = "true" if value else "false" |
| 269 | normalized[key] = value |
| 270 | return normalized |
| 271 | |
| 272 | |
| 273 | def _parse_group_by(record: Mapping[str, str], *, definition_id: str) -> tuple[str, ...]: |
| 274 | raw = _json_field(record, "group_by", [], definition_id=definition_id) |
| 275 | if raw is None: |
| 276 | raw = [] |
| 277 | if not isinstance(raw, list) or not all(isinstance(item, str) and item for item in raw): |
| 278 | raise DefinitionError(f"{definition_id}: group_by must be a JSON list of field names") |
| 279 | return tuple(raw) |
| 280 | |
| 281 | |
| 282 | def _parse_time_bucket( |
| 283 | record: Mapping[str, str], |
| 284 | *, |
| 285 | definition_id: str, |
| 286 | source_collection: str, |
| 287 | source_fields: dict[str, dict[str, Any]], |
| 288 | ) -> tuple[str | None, str, str | None]: |
| 289 | raw = _json_field(record, "time_bucket", None, definition_id=definition_id) |
| 290 | if not raw: |
| 291 | return None, "none", None |
| 292 | if not isinstance(raw, dict): |
| 293 | raise DefinitionError(f"{definition_id}: time_bucket must be a JSON object") |
| 294 | |
| 295 | field_name = (raw.get("field") or "").strip() |
| 296 | if not field_name: |
| 297 | raise DefinitionError(f"{definition_id}: time_bucket.field is required when time_bucket is set") |
| 298 | |
| 299 | granularity = (raw.get("granularity") or "none").strip().lower() |
| 300 | if granularity not in _TIME_BUCKET_GRANULARITIES: |
| 301 | raise DefinitionError( |
| 302 | f"{definition_id}: time_bucket.granularity must be one of {sorted(_TIME_BUCKET_GRANULARITIES)}" |
| 303 | ) |
| 304 | |
| 305 | source_field = source_fields.get(field_name) |
| 306 | source_type = (source_field or {}).get("type") |
| 307 | if source_type not in ("date", "datetime"): |
| 308 | raise DefinitionError( |
| 309 | f"{definition_id}: time_bucket.field '{field_name}' must be a date/datetime field on {source_collection}" |
| 310 | ) |
| 311 | return field_name, granularity, source_type |
| 312 | |
| 313 | |
| 314 | def _parse_metrics( |
| 315 | record: Mapping[str, str], |
| 316 | *, |
| 317 | definition_id: str, |
| 318 | source_collection: str, |
| 319 | source_fields: dict[str, dict[str, Any]], |
| 320 | group_by: tuple[str, ...], |
| 321 | ) -> tuple[Metric, ...]: |
| 322 | raw = _json_field(record, "metrics", [], definition_id=definition_id) |
| 323 | if raw is None: |
| 324 | raw = [] |
| 325 | if not isinstance(raw, list): |
| 326 | raise DefinitionError(f"{definition_id}: metrics must be a JSON list") |
| 327 | |
| 328 | reserved_names = set(_RESERVED_TARGET_FIELD_NAMES) | set(group_by) |
| 329 | metrics: list[Metric] = [] |
| 330 | for entry in raw: |
| 331 | if not isinstance(entry, dict): |
| 332 | raise DefinitionError(f"{definition_id}: each metrics entry must be a JSON object") |
| 333 | |
| 334 | op = str(entry.get("op") or "").strip().lower() |
| 335 | if op not in _METRIC_OPS: |
| 336 | raise DefinitionError(f"{definition_id}: metric op must be one of {sorted(_METRIC_OPS)}") |
| 337 | |
| 338 | field_name = entry.get("field") |
| 339 | if op == "count": |
| 340 | as_name = str(entry.get("as") or "count").strip() |
| 341 | metric = Metric(op=op, field=None, as_name=as_name, source_type=None, target_type="integer") |
| 342 | else: |
| 343 | if not isinstance(field_name, str) or not field_name: |
| 344 | raise DefinitionError(f"{definition_id}: metric op '{op}' requires a field") |
| 345 | source_field = source_fields.get(field_name) |
| 346 | source_type = (source_field or {}).get("type") |
| 347 | if source_type not in ("integer", "number"): |
| 348 | raise DefinitionError( |
| 349 | f"{definition_id}: metric field '{field_name}' must be numeric " |
| 350 | f"(integer/number) on {source_collection}" |
| 351 | ) |
| 352 | as_name = str(entry.get("as") or f"{field_name}_{op}").strip() |
| 353 | target_type = "number" if op == "avg" else source_type |
| 354 | metric = Metric(op=op, field=field_name, as_name=as_name, source_type=source_type, target_type=target_type) |
| 355 | |
| 356 | if not metric.as_name: |
| 357 | raise DefinitionError(f"{definition_id}: metric 'as' name cannot be blank") |
| 358 | if metric.as_name in reserved_names: |
| 359 | raise DefinitionError( |
| 360 | f"{definition_id}: metric name '{metric.as_name}' collides with id/bucket_start/" |
| 361 | "computed_at or a group_by field" |
| 362 | ) |
| 363 | reserved_names.add(metric.as_name) |
| 364 | metrics.append(metric) |
| 365 | |
| 366 | return tuple(metrics) |
| 367 | |
| 368 | |
| 369 | def _parse_min_group_size(record: Mapping[str, str], *, definition_id: str) -> int | None: |
| 370 | raw = (record.get("min_group_size") or "").strip() |
| 371 | if not raw: |
| 372 | return None |
| 373 | try: |
| 374 | value = int(raw) |
| 375 | except ValueError as exc: |
| 376 | raise DefinitionError(f"{definition_id}: min_group_size must be an integer") from exc |
| 377 | if value < 1: |
| 378 | raise DefinitionError(f"{definition_id}: min_group_size must be >= 1") |
| 379 | return value |
| 380 | |
| 381 | |
| 382 | def _json_field(record: Mapping[str, str], name: str, default: Any, *, definition_id: str) -> Any: |
| 383 | raw = record.get(name) |
| 384 | if raw is None: |
| 385 | return default |
| 386 | raw = raw.strip() |
| 387 | if not raw: |
| 388 | return default |
| 389 | try: |
| 390 | return json.loads(raw) |
| 391 | except json.JSONDecodeError as exc: |
| 392 | raise DefinitionError(f"{definition_id}: {name} is not valid JSON: {exc}") from exc |
| 393 | |
| 394 | |
| 395 | def _truthy(value: Any, *, default: bool) -> bool: |
| 396 | if value is None or value == "": |
| 397 | return default |
| 398 | return str(value).strip().lower() in {"true", "1", "yes", "on"} |
| 399 | |
| 400 | |
| 401 | def _positive_int(value: Any) -> int | None: |
| 402 | try: |
| 403 | parsed = int(str(value).strip()) |
| 404 | except (TypeError, ValueError): |
| 405 | return None |
| 406 | return parsed if parsed > 0 else None |
| 407 | |
| 408 | |
| 409 | def _parse_iso(value: str) -> datetime | None: |
| 410 | try: |
| 411 | parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) |
| 412 | except ValueError: |
| 413 | return None |
| 414 | if parsed.tzinfo is None: |
| 415 | parsed = parsed.replace(tzinfo=timezone.utc) |
| 416 | return parsed |
| 417 | |
| 418 | |
| 419 | # --- Derived target schema ------------------------------------------------- |
| 420 | |
| 421 | def derive_target_schema(config: RollupConfig) -> dict[str, Any]: |
| 422 | """Return the target collection's schema, generated from ``config``. |
| 423 | |
| 424 | Shape rules per 14's "Derived target schema" section: one plain |
| 425 | ``text`` field per ``group_by`` entry (a source enum/relation field |
| 426 | still becomes plain text on the target -- the target stores the |
| 427 | value, not a live pointer); ``bucket_start`` only when a time bucket |
| 428 | is declared, typed ``date`` for day/week/month or ``datetime`` when |
| 429 | the source field is ``datetime`` and granularity is ``none``; one |
| 430 | ``"computed": true`` field per metric, named by ``as``; and an |
| 431 | unconditional ``"computed": true`` ``computed_at``, present regardless |
| 432 | of refresh mode. ``views.list_mode: "table"`` plus ``list_fields`` |
| 433 | covering group-by + bucket + metrics are set automatically -- the |
| 434 | payoff named in 14's doctrine, a rollup target renders with zero page |
| 435 | code. |
| 436 | """ |
| 437 | fields: list[dict[str, Any]] = [{"name": "id"}] |
| 438 | for name in config.group_by: |
| 439 | fields.append({"name": name, "type": "text"}) |
| 440 | |
| 441 | if config.time_bucket_field: |
| 442 | bucket_type = "date" |
| 443 | if config.time_bucket_granularity == "none" and config.time_bucket_source_type == "datetime": |
| 444 | bucket_type = "datetime" |
| 445 | fields.append({"name": "bucket_start", "type": bucket_type}) |
| 446 | |
| 447 | for metric in config.metrics: |
| 448 | fields.append({"name": metric.as_name, "type": metric.target_type, "computed": True}) |
| 449 | |
| 450 | fields.append({"name": "computed_at", "type": "datetime", "computed": True}) |
| 451 | |
| 452 | list_fields = list(config.group_by) |
| 453 | if config.time_bucket_field: |
| 454 | list_fields.append("bucket_start") |
| 455 | list_fields.extend(metric.as_name for metric in config.metrics) |
| 456 | |
| 457 | return { |
| 458 | "name": config.target_collection, |
| 459 | "title": config.name, |
| 460 | "description": ( |
| 461 | f"Rollup target for definition '{config.definition_id}' -- generated, " |
| 462 | "never hand-edited. See plan/vocabulary/14-rollup-spec.md." |
| 463 | ), |
| 464 | "storage": object_schemas.STORAGE_CLASSIC, |
| 465 | "fields": fields, |
| 466 | "views": {"list_mode": "table", "list_fields": list_fields}, |
| 467 | } |
| 468 | |
| 469 | |
| 470 | def _next_schema_version(target_collection: str, *, base_dir: Any) -> int: |
| 471 | """One more than the target's current schema version, or 1 when there |
| 472 | isn't one yet -- 14's Upgrade Posture calls the departure from Rule 1 |
| 473 | a "regenerate": "bump the target schema, force an immediate full |
| 474 | recompute." Every recompute already regenerates and replaces the |
| 475 | schema (see module docstring); this just makes that regeneration a |
| 476 | literal version bump rather than a silent same-number overwrite. |
| 477 | """ |
| 478 | try: |
| 479 | existing = object_schemas.get_schema(target_collection, base_dir=base_dir) |
| 480 | except object_schemas.SchemaNotFoundError: |
| 481 | return 1 |
| 482 | try: |
| 483 | return int(existing.get("version") or 0) + 1 |
| 484 | except (TypeError, ValueError): |
| 485 | return 1 |
| 486 | |
| 487 | |
| 488 | # --- Time bucketing ------------------------------------------------- |
| 489 | |
| 490 | def _truncate_bucket(raw_value: str, granularity: str) -> str | None: |
| 491 | """Truncate one source field value to its bucket start. |
| 492 | |
| 493 | No timezone conversion -- 14 is explicit that bucketing "truncates |
| 494 | the field's value to the bucket start ... in the field's own timezone |
| 495 | convention," matching docs/schema-forms.md's date/datetime types, |
| 496 | which are stored and compared as given. Returns None for a value that |
| 497 | doesn't parse as a date/datetime (caller drops that row from grouping |
| 498 | rather than crashing the whole definition over one bad row). |
| 499 | """ |
| 500 | if granularity == "none": |
| 501 | return raw_value |
| 502 | |
| 503 | parsed = _parse_date_or_datetime(raw_value) |
| 504 | if parsed is None: |
| 505 | return None |
| 506 | |
| 507 | if granularity == "day": |
| 508 | bucket = parsed |
| 509 | elif granularity == "week": |
| 510 | bucket = parsed - timedelta(days=parsed.weekday()) # ISO week starts Monday |
| 511 | elif granularity == "month": |
| 512 | bucket = parsed.replace(day=1) |
| 513 | else: # pragma: no cover -- parse_definition already validated this |
| 514 | raise DefinitionError(f"unknown time_bucket granularity: {granularity}") |
| 515 | return bucket.isoformat() |
| 516 | |
| 517 | |
| 518 | def _parse_date_or_datetime(raw_value: str) -> date | None: |
| 519 | text = (raw_value or "").strip() |
| 520 | if not text: |
| 521 | return None |
| 522 | try: |
| 523 | if "T" in text or " " in text: |
| 524 | return datetime.fromisoformat(text.replace("Z", "+00:00")).date() |
| 525 | return date.fromisoformat(text) |
| 526 | except ValueError: |
| 527 | return None |
| 528 | |
| 529 | |
| 530 | # --- Grouping and metrics ------------------------------------------------- |
| 531 | |
| 532 | def _group_rows( |
| 533 | rows: list[dict[str, str]], config: RollupConfig |
| 534 | ) -> dict[tuple[str | None, tuple[str, ...]], list[dict[str, str]]]: |
| 535 | """Group already-filtered source rows by (bucket_start, group_key). |
| 536 | |
| 537 | A row whose time_bucket field is blank, or doesn't parse as a |
| 538 | date/datetime, is dropped from grouping entirely (it can't be |
| 539 | honestly bucketed) rather than raising and failing the whole |
| 540 | definition over one bad row -- consistent with this pass's isolation |
| 541 | posture elsewhere. |
| 542 | """ |
| 543 | groups: dict[tuple[str | None, tuple[str, ...]], list[dict[str, str]]] = {} |
| 544 | for row in rows: |
| 545 | if config.time_bucket_field: |
| 546 | raw_bucket_value = row.get(config.time_bucket_field) |
| 547 | if not raw_bucket_value: |
| 548 | continue |
| 549 | bucket = _truncate_bucket(raw_bucket_value, config.time_bucket_granularity) |
| 550 | if bucket is None: |
| 551 | continue |
| 552 | else: |
| 553 | bucket = None |
| 554 | group_key = tuple(row.get(name, "") for name in config.group_by) |
| 555 | groups.setdefault((bucket, group_key), []).append(row) |
| 556 | return groups |
| 557 | |
| 558 | |
| 559 | def _compute_metric(metric: Metric, rows: list[dict[str, str]]) -> str: |
| 560 | """Compute one metric's value over one group's rows. |
| 561 | |
| 562 | Integer-exact per 14's float-accumulation note: an integer-typed |
| 563 | source field is summed/averaged with Python's arbitrary-precision |
| 564 | ints, never floats, so cents arithmetic never accumulates rounding |
| 565 | error. ``avg`` is always sum/count from that group's own exact sum |
| 566 | and count, computed fresh from the full row set every recompute |
| 567 | (never a running average) -- v1's scheduled-only posture means the |
| 568 | complete row list is always available, so there is no need to persist |
| 569 | an intermediate sum/count pair the way an incremental fast path |
| 570 | eventually would (see module docstring). |
| 571 | """ |
| 572 | if metric.op == "count": |
| 573 | return str(len(rows)) |
| 574 | |
| 575 | present = [row.get(metric.field) for row in rows if row.get(metric.field) not in (None, "")] |
| 576 | |
| 577 | if metric.op in ("min", "max"): |
| 578 | if not present: |
| 579 | return "" |
| 580 | numbered = [(_parse_metric_number(value, metric.source_type), value) for value in present] |
| 581 | chosen = min(numbered) if metric.op == "min" else max(numbered) |
| 582 | return chosen[1] |
| 583 | |
| 584 | if metric.op == "sum": |
| 585 | if metric.source_type == "integer": |
| 586 | return str(sum(int(value) for value in present)) |
| 587 | return repr(sum(float(value) for value in present)) |
| 588 | |
| 589 | if metric.op == "avg": |
| 590 | if not present: |
| 591 | return "" |
| 592 | if metric.source_type == "integer": |
| 593 | total: float = sum(int(value) for value in present) |
| 594 | else: |
| 595 | total = sum(float(value) for value in present) |
| 596 | return repr(total / len(present)) |
| 597 | |
| 598 | raise DefinitionError(f"unknown metric op: {metric.op}") # pragma: no cover -- pre-validated |
| 599 | |
| 600 | |
| 601 | def _parse_metric_number(value: str, source_type: str | None) -> float: |
| 602 | return float(int(value)) if source_type == "integer" else float(value) |
| 603 | |
| 604 | |
| 605 | def _target_row_id(definition_id: str, key: tuple[str | None, tuple[str, ...]]) -> str: |
| 606 | """A deterministic target row id from (definition, group key, bucket). |
| 607 | |
| 608 | 14's Storage section requires this so "a recompute or an incremental |
| 609 | upsert always lands on the same row rather than accumulating |
| 610 | duplicates." Hashed (not concatenated) because group key values are |
| 611 | arbitrary source data that may contain characters unsafe in a record |
| 612 | id (``object_records._RECORD_ID_RE``); the hash is stable across |
| 613 | recomputes since it is a pure function of the definition id, the |
| 614 | bucket, and the group key values, in the same field order every time. |
| 615 | """ |
| 616 | bucket, group_key = key |
| 617 | canonical = json.dumps([definition_id, bucket, list(group_key)], separators=(",", ":")) |
| 618 | digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:32] |
| 619 | return f"rlp_{digest}" |
| 620 | |
| 621 | |
| 622 | def _build_target_row( |
| 623 | config: RollupConfig, |
| 624 | key: tuple[str | None, tuple[str, ...]], |
| 625 | rows: list[dict[str, str]], |
| 626 | computed_at: str, |
| 627 | ) -> dict[str, str]: |
| 628 | bucket, group_key = key |
| 629 | row: dict[str, str] = {"id": _target_row_id(config.definition_id, key)} |
| 630 | for name, value in zip(config.group_by, group_key, strict=True): |
| 631 | row[name] = value |
| 632 | if config.time_bucket_field: |
| 633 | row["bucket_start"] = bucket or "" |
| 634 | for metric in config.metrics: |
| 635 | row[metric.as_name] = _compute_metric(metric, rows) |
| 636 | row["computed_at"] = computed_at |
| 637 | return row |
| 638 | |
| 639 | |
| 640 | # --- Compute ------------------------------------------------- |
| 641 | |
| 642 | def compute_rollup( |
| 643 | definition: Mapping[str, str], *, base_dir: Any, roots: Any = None |
| 644 | ) -> dict[str, Any]: |
| 645 | """Full recompute for one ``rollup_definitions`` record. |
| 646 | |
| 647 | Reads ``source_collection`` outside the permission engine's row |
| 648 | filters -- daemon posture, same as ``object_records.compact_collection`` |
| 649 | (14's Permissions Posture: "it must see every row, across every |
| 650 | owner, to aggregate correctly into one admin-controlled target"). |
| 651 | Regenerates and replaces the target's derived schema, then reconciles |
| 652 | the target collection's rows to exactly the newly computed set (see |
| 653 | module docstring for why "reconcile via the record API" is how this |
| 654 | codebase expresses "full rewrite" without a bulk-replace primitive). |
| 655 | |
| 656 | Raises ``DefinitionError`` for a malformed definition, and lets |
| 657 | ``object_schemas.SchemaNotFoundError`` / ``object_collections. |
| 658 | CollectionNotFoundError`` propagate when ``source_collection`` is |
| 659 | missing -- the caller (``object_daemon.process_rollups``) isolates |
| 660 | each definition's failure in its own try/except, one bad definition |
| 661 | never stopping the rest of the pass (14's Degradation section). |
| 662 | """ |
| 663 | config = parse_definition(definition, base_dir=base_dir, roots=roots) |
| 664 | |
| 665 | source_records = object_records.read_collection_records( |
| 666 | config.source_collection, base_dir=base_dir, roots=roots |
| 667 | ) |
| 668 | filtered = ( |
| 669 | object_records.filter_records(source_records, config.filter) |
| 670 | if config.filter |
| 671 | else list(source_records) |
| 672 | ) |
| 673 | |
| 674 | groups = _group_rows(filtered, config) |
| 675 | |
| 676 | computed_at = _now_iso() |
| 677 | target_rows: list[dict[str, str]] = [] |
| 678 | suppressed = 0 |
| 679 | for key, rows in groups.items(): |
| 680 | if config.min_group_size is not None and len(rows) < config.min_group_size: |
| 681 | suppressed += 1 |
| 682 | continue |
| 683 | target_rows.append(_build_target_row(config, key, rows, computed_at)) |
| 684 | |
| 685 | schema_payload = derive_target_schema(config) |
| 686 | schema_payload["version"] = _next_schema_version(config.target_collection, base_dir=base_dir) |
| 687 | object_schemas.replace_schema(config.target_collection, schema_payload, base_dir=base_dir) |
| 688 | |
| 689 | reconciled = _reconcile_target_rows( |
| 690 | config.target_collection, target_rows, base_dir=base_dir, roots=roots |
| 691 | ) |
| 692 | |
| 693 | return { |
| 694 | "definition_id": config.definition_id, |
| 695 | "target_collection": config.target_collection, |
| 696 | "groups": len(target_rows), |
| 697 | "suppressed": suppressed, |
| 698 | "created": reconciled["created"], |
| 699 | "updated": reconciled["updated"], |
| 700 | "deleted": reconciled["deleted"], |
| 701 | "computed_at": computed_at, |
| 702 | } |
| 703 | |
| 704 | |
| 705 | def _reconcile_target_rows( |
| 706 | collection: str, new_rows: list[dict[str, str]], *, base_dir: Any, roots: Any = None |
| 707 | ) -> dict[str, int]: |
| 708 | """Sync ``collection`` to exactly ``new_rows`` by id: create the new |
| 709 | ones, update the ones that already existed, delete whatever's left -- |
| 710 | a full recompute's atomic rewrite naturally drops any row whose group |
| 711 | no longer exists (14: "no explicit 'delete stale rows' step is |
| 712 | needed, the rewrite IS that step"); this is that step, done per-row |
| 713 | through the public record API. |
| 714 | |
| 715 | Every row is written unconditionally, including one whose metric |
| 716 | values are unchanged since the last pass: ``computed_at`` always |
| 717 | advances, honestly reflecting "just reconfirmed as of now" rather |
| 718 | than silently reusing a stale timestamp because nothing else moved. |
| 719 | |
| 720 | None of these writes append to the change log (``record_changes=False``), |
| 721 | and that follows directly from the paragraph above. A recompute rewrites |
| 722 | every row every pass by design, so logging it produces one audit entry |
| 723 | per row per pass forever -- describing nothing a person did, and nothing |
| 724 | that could not be recomputed from the source collection and the |
| 725 | definition, both of which are still on disk. Left on, it produced a |
| 726 | 944MB change log for a collection holding 1818 rows, and the activity |
| 727 | feed, which reads every collection's log, stopped finishing. The log is |
| 728 | for who did what; a derived value has no who. |
| 729 | """ |
| 730 | existing_ids: set[str] = set() |
| 731 | if object_records.collection_has_records(collection, base_dir=base_dir): |
| 732 | existing_ids = { |
| 733 | row["id"] |
| 734 | for row in object_records.read_collection_records(collection, base_dir=base_dir, roots=roots) |
| 735 | } |
| 736 | |
| 737 | created = updated = 0 |
| 738 | seen_ids: set[str] = set() |
| 739 | for row in new_rows: |
| 740 | seen_ids.add(row["id"]) |
| 741 | if row["id"] in existing_ids: |
| 742 | object_records.update_collection_record( |
| 743 | collection, row["id"], row, |
| 744 | base_dir=base_dir, roots=roots, |
| 745 | actor=ROLLUP_ACTOR, allow_computed_submission=True, |
| 746 | # A recompute is not history. See below. |
| 747 | record_changes=False, |
| 748 | ) |
| 749 | updated += 1 |
| 750 | else: |
| 751 | object_records.create_collection_record( |
| 752 | collection, row, |
| 753 | base_dir=base_dir, roots=roots, |
| 754 | actor=ROLLUP_ACTOR, allow_computed_submission=True, |
| 755 | record_changes=False, |
| 756 | ) |
| 757 | created += 1 |
| 758 | |
| 759 | deleted = 0 |
| 760 | for stale_id in existing_ids - seen_ids: |
| 761 | object_records.delete_collection_record( |
| 762 | collection, stale_id, base_dir=base_dir, roots=roots, |
| 763 | actor=ROLLUP_ACTOR, record_changes=False, |
| 764 | ) |
| 765 | deleted += 1 |
| 766 | |
| 767 | return {"created": created, "updated": updated, "deleted": deleted} |
| 768 | |
| 769 | |
| 770 | def _now_iso() -> str: |
| 771 | return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") |