Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_rollups.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_rollups.py · 32.0 KB · 771 lines
1"""Rollup block: declared aggregations materialized as ordinary collections.
2
3Implements ``plan/vocabulary/14-rollup-spec.md``. A ``rollup_definitions``
4record declares one source collection, an optional flat-equality filter, a
5``group_by``, an optional ``time_bucket`` (day/week/month truncation), and
6a list of ``metrics`` (count/sum/min/max/avg). ``compute_rollup`` reads the
7source outside row-level permissions (daemon posture, same as
8``object_records.compact_collection``), computes the full current answer,
9and writes it into ``target_collection`` -- a schema-derived, ordinary
10collection that renders through the existing generators with zero page
11code, the stated payoff (14's Surfaces section).
12
13**Storage is a full rewrite by effect, not by mechanism.** 14's Storage
14section is explicit that a SCHEDULED recompute should behave like classic
15storage's atomic replace-the-whole-file write -- "the correct operation,
16not a workaround." There is no public bulk "replace this entire
17collection" primitive in ``object_records`` to call (the module-private
18one, ``_write_collection_records``, is reserved for that module's own
19internal callers), so this reconciles the target to the newly computed row
20set through the ordinary per-row record API instead: create a row for
21every new id, update every row that already existed, delete every row
22whose group no longer exists. The net effect is identical to a full
23rewrite -- a full recompute always produces every row's current answer and
24always drops stale groups -- it is just expressed as N create/update/delete
25calls rather than one file swap.
26
27**Scheduled only in v1.** 14's Events section describes an INCREMENTAL
28create-fast-path (a HANDLES subscriber on the source collection, updating
29one target row additively) alongside a reconciling scheduled recompute
30that "self-heals any drift the fast path couldn't handle correctly." This
31module implements only the reconciling recompute side. A definition
32declaring ``refresh_mode: incremental`` is accepted and still recomputes
33correctly on its own ``refresh_interval_seconds`` -- 14 is explicit that
34"scheduled-only is the v1 default and the recommendation" for most
35sources, and that incremental's own interval is required precisely
36because it "is not incremental instead of scheduled, it is an additive
37create-fast-path on top of a slower scheduled full recompute." Without the
38fast path, an incremental definition simply behaves as scheduled-only --
39the same graceful-degradation shape 14's own Degradation section
40describes for ``DBBASIC_ENABLE_EVENT_HANDLERS`` being unset ("nothing
41breaks; freshness just tracks the scheduled interval instead of
42near-real-time"). The HANDLES-subscriber fast path is not built here.
43
44**Writing computed fields.** The derived target schema marks every metric
45field (and ``computed_at``) ``"computed": true`` per 14's field contract
46("a target row's numbers come only from the rollup pass, never from a
47form POST"). ``object_records.create_collection_record`` /
48``update_collection_record`` reject a write to a genuinely computed field
49from ANY caller by default -- there was no existing escape hatch for a
50system pass to write its own derived values, so this block adds one
51(``allow_computed_submission``, see object_records.py), used only here.
52"""
53from __future__ import annotations
54
55import hashlib
56import json
57from dataclasses import dataclass
58from datetime import date, datetime, timedelta, timezone
59from typing import Any, Mapping
60
61import object_collections
62import object_feature_flags
63import object_records
64import object_schemas
65
66ROLLUP_DEFINITIONS_COLLECTION = "rollup_definitions"
67FEATURE_FLAGS_COLLECTION = "feature_flags"
68ROLLUP_ENABLED_FLAG = "rollup_enabled"
69ROLLUP_ACTOR = "daemon:rollup"
70
71_TIME_BUCKET_GRANULARITIES = frozenset({"none", "day", "week", "month"})
72_METRIC_OPS = frozenset({"count", "sum", "min", "max", "avg"})
73_REFRESH_MODES = frozenset({"scheduled", "incremental"})
74_RESERVED_TARGET_FIELD_NAMES = frozenset({"id", "bucket_start", "computed_at"})
75
76
77class DefinitionError(ValueError):
78 """Raised when a rollup_definitions record cannot be computed as-is.
79
80 Covers both malformed JSON sub-fields and violations of 14's own
81 doctrine (a filter value that isn't a flat scalar, a metric field that
82 isn't numeric, and so on). The daemon pass catches this per-definition
83 (process_rollups, object_daemon.py), same isolation posture as
84 process_compactions: one bad definition is logged and skipped, never
85 allowed to stop the rest of the pass.
86 """
87
88
89@dataclass(frozen=True)
90class Metric:
91 op: str
92 field: str | None
93 as_name: str
94 source_type: str | None # "integer" | "number" | None (count has none)
95 target_type: str # "integer" | "number"
96
97
98@dataclass(frozen=True)
99class RollupConfig:
100 definition_id: str
101 name: str
102 source_collection: str
103 target_collection: str
104 filter: dict[str, Any]
105 group_by: tuple[str, ...]
106 time_bucket_field: str | None
107 time_bucket_granularity: str
108 time_bucket_source_type: str | None
109 metrics: tuple[Metric, ...]
110 min_group_size: int | None
111 refresh_mode: str
112 refresh_interval_seconds: int
113 enabled: bool
114
115
116# --- Feature flag / due gate -------------------------------------------------
117
118def rollup_pass_enabled(*, base_dir: Any) -> bool:
119 """Block-wide kill switch, ``<block>_enabled`` convention (00-doctrine-
120 and-contract.md), a ``feature_flags`` row named ``rollup_enabled``.
121
122 Default ON -- same reasoning 12-notify-spec.md gives for its own
123 block-wide flag defaulting on: this is the brownout kill switch, not
124 an adoption gate, and with zero ``rollup_definitions`` rows installed
125 it does nothing regardless. A missing row, a missing/unreadable
126 ``feature_flags`` collection, and a blank value all resolve to "on";
127 only an explicit off/false/0/no value turns the pass off (14's
128 Degradation: "existing target collections keep serving their
129 last-computed data").
130 """
131 return object_feature_flags.kill_switch_on(ROLLUP_ENABLED_FLAG, base_dir=base_dir)
132
133
134def is_definition_enabled(record: Mapping[str, str]) -> bool:
135 """Per-definition kill switch (14's ``enabled`` field), independent of
136 the block-wide ``rollup_enabled`` flag -- "lets an operator pause one
137 noisy or broken definition without disabling every rollup on the
138 instance." Blank/missing defaults to enabled, same as any schema
139 boolean field with ``"default": "true"``.
140 """
141 return _truthy(record.get("enabled"), default=True)
142
143
144def is_definition_due(record: Mapping[str, str], *, now: datetime | None = None) -> bool:
145 """SCHEDULED-mode due gate: ``last_computed_at`` + ``refresh_interval_
146 seconds``, both read straight off the definition record -- 14's
147 Storage section is explicit that ``rollup_definitions`` needs no
148 separate marker file the way ``process_compactions`` does, "because it
149 has no natural per-collection record to hold state on, a rollup
150 definition already is that record."
151
152 A never-computed definition (``last_computed_at`` blank) is always
153 due. A malformed ``refresh_interval_seconds`` is also treated as due,
154 so the definition's real problem surfaces as a logged
155 ``compute_rollup`` failure rather than the daemon silently never
156 attempting it.
157 """
158 now = now or datetime.now(timezone.utc)
159 last_computed_raw = (record.get("last_computed_at") or "").strip()
160 if not last_computed_raw:
161 return True
162 interval = _positive_int(record.get("refresh_interval_seconds"))
163 if interval is None:
164 return True
165 last_computed = _parse_iso(last_computed_raw)
166 if last_computed is None:
167 return True
168 return (now - last_computed).total_seconds() >= interval
169
170
171# --- Definition parsing -------------------------------------------------
172
173def parse_definition(
174 record: Mapping[str, str], *, base_dir: Any, roots: Any = None
175) -> RollupConfig:
176 """Validate and normalize one ``rollup_definitions`` row.
177
178 Raises ``DefinitionError`` for anything malformed, and lets
179 ``object_schemas.SchemaNotFoundError`` /
180 ``object_collections.CollectionNotFoundError`` propagate unchanged
181 when ``source_collection`` doesn't exist -- 14's Degradation section
182 treats a missing source as its own failure mode ("the recompute for
183 that definition fails and is logged; the target collection keeps
184 serving its last successful computation"), distinct from a malformed
185 definition.
186 """
187 definition_id = (record.get("id") or "").strip()
188 if not definition_id:
189 raise DefinitionError("rollup definition is missing an id")
190
191 name = (record.get("name") or "").strip() or definition_id
192
193 source_collection = (record.get("source_collection") or "").strip()
194 if not source_collection or not object_collections.validate_collection_name(source_collection):
195 raise DefinitionError(f"{definition_id}: source_collection is required and must be a valid collection name")
196
197 target_collection = (record.get("target_collection") or "").strip()
198 if not target_collection or not object_collections.validate_collection_name(target_collection):
199 raise DefinitionError(f"{definition_id}: target_collection is required and must be a valid collection name")
200
201 # Raises SchemaNotFoundError when source_collection doesn't exist --
202 # deliberately not caught here, see docstring.
203 source_schema = object_schemas.get_schema(source_collection, base_dir=base_dir, roots=roots)
204 source_fields = {field["name"]: field for field in source_schema.get("fields", [])}
205
206 filter_ = _parse_filter(record, definition_id=definition_id)
207 group_by = _parse_group_by(record, definition_id=definition_id)
208
209 time_bucket_field, time_bucket_granularity, time_bucket_source_type = _parse_time_bucket(
210 record, definition_id=definition_id, source_collection=source_collection, source_fields=source_fields,
211 )
212
213 metrics = _parse_metrics(
214 record,
215 definition_id=definition_id,
216 source_collection=source_collection,
217 source_fields=source_fields,
218 group_by=group_by,
219 )
220
221 min_group_size = _parse_min_group_size(record, definition_id=definition_id)
222
223 refresh_mode = (record.get("refresh_mode") or "scheduled").strip().lower()
224 if refresh_mode not in _REFRESH_MODES:
225 raise DefinitionError(f"{definition_id}: refresh_mode must be one of {sorted(_REFRESH_MODES)}")
226
227 refresh_interval_seconds = _positive_int(record.get("refresh_interval_seconds"))
228 if refresh_interval_seconds is None:
229 raise DefinitionError(f"{definition_id}: refresh_interval_seconds is required and must be a positive integer")
230
231 enabled = _truthy(record.get("enabled"), default=True)
232
233 return RollupConfig(
234 definition_id=definition_id,
235 name=name,
236 source_collection=source_collection,
237 target_collection=target_collection,
238 filter=filter_,
239 group_by=group_by,
240 time_bucket_field=time_bucket_field,
241 time_bucket_granularity=time_bucket_granularity,
242 time_bucket_source_type=time_bucket_source_type,
243 metrics=metrics,
244 min_group_size=min_group_size,
245 refresh_mode=refresh_mode,
246 refresh_interval_seconds=refresh_interval_seconds,
247 enabled=enabled,
248 )
249
250
251def _parse_filter(record: Mapping[str, str], *, definition_id: str) -> dict[str, Any]:
252 raw = _json_field(record, "filter", {}, definition_id=definition_id)
253 if raw is None:
254 return {}
255 if not isinstance(raw, dict):
256 raise DefinitionError(f"{definition_id}: filter must be a flat JSON object")
257 normalized: dict[str, Any] = {}
258 for key, value in raw.items():
259 if isinstance(value, (dict, list)):
260 raise DefinitionError(
261 f"{definition_id}: filter.{key} must be a scalar -- rollup filters are flat "
262 "equality only, ANDed (00-doctrine-and-contract.md); escalate to a custom "
263 "object for OR/join/subquery/computed conditions"
264 )
265 if value is None:
266 raise DefinitionError(f"{definition_id}: filter.{key} must not be null")
267 if isinstance(value, bool):
268 value = "true" if value else "false"
269 normalized[key] = value
270 return normalized
271
272
273def _parse_group_by(record: Mapping[str, str], *, definition_id: str) -> tuple[str, ...]:
274 raw = _json_field(record, "group_by", [], definition_id=definition_id)
275 if raw is None:
276 raw = []
277 if not isinstance(raw, list) or not all(isinstance(item, str) and item for item in raw):
278 raise DefinitionError(f"{definition_id}: group_by must be a JSON list of field names")
279 return tuple(raw)
280
281
282def _parse_time_bucket(
283 record: Mapping[str, str],
284 *,
285 definition_id: str,
286 source_collection: str,
287 source_fields: dict[str, dict[str, Any]],
288) -> tuple[str | None, str, str | None]:
289 raw = _json_field(record, "time_bucket", None, definition_id=definition_id)
290 if not raw:
291 return None, "none", None
292 if not isinstance(raw, dict):
293 raise DefinitionError(f"{definition_id}: time_bucket must be a JSON object")
294
295 field_name = (raw.get("field") or "").strip()
296 if not field_name:
297 raise DefinitionError(f"{definition_id}: time_bucket.field is required when time_bucket is set")
298
299 granularity = (raw.get("granularity") or "none").strip().lower()
300 if granularity not in _TIME_BUCKET_GRANULARITIES:
301 raise DefinitionError(
302 f"{definition_id}: time_bucket.granularity must be one of {sorted(_TIME_BUCKET_GRANULARITIES)}"
303 )
304
305 source_field = source_fields.get(field_name)
306 source_type = (source_field or {}).get("type")
307 if source_type not in ("date", "datetime"):
308 raise DefinitionError(
309 f"{definition_id}: time_bucket.field '{field_name}' must be a date/datetime field on {source_collection}"
310 )
311 return field_name, granularity, source_type
312
313
314def _parse_metrics(
315 record: Mapping[str, str],
316 *,
317 definition_id: str,
318 source_collection: str,
319 source_fields: dict[str, dict[str, Any]],
320 group_by: tuple[str, ...],
321) -> tuple[Metric, ...]:
322 raw = _json_field(record, "metrics", [], definition_id=definition_id)
323 if raw is None:
324 raw = []
325 if not isinstance(raw, list):
326 raise DefinitionError(f"{definition_id}: metrics must be a JSON list")
327
328 reserved_names = set(_RESERVED_TARGET_FIELD_NAMES) | set(group_by)
329 metrics: list[Metric] = []
330 for entry in raw:
331 if not isinstance(entry, dict):
332 raise DefinitionError(f"{definition_id}: each metrics entry must be a JSON object")
333
334 op = str(entry.get("op") or "").strip().lower()
335 if op not in _METRIC_OPS:
336 raise DefinitionError(f"{definition_id}: metric op must be one of {sorted(_METRIC_OPS)}")
337
338 field_name = entry.get("field")
339 if op == "count":
340 as_name = str(entry.get("as") or "count").strip()
341 metric = Metric(op=op, field=None, as_name=as_name, source_type=None, target_type="integer")
342 else:
343 if not isinstance(field_name, str) or not field_name:
344 raise DefinitionError(f"{definition_id}: metric op '{op}' requires a field")
345 source_field = source_fields.get(field_name)
346 source_type = (source_field or {}).get("type")
347 if source_type not in ("integer", "number"):
348 raise DefinitionError(
349 f"{definition_id}: metric field '{field_name}' must be numeric "
350 f"(integer/number) on {source_collection}"
351 )
352 as_name = str(entry.get("as") or f"{field_name}_{op}").strip()
353 target_type = "number" if op == "avg" else source_type
354 metric = Metric(op=op, field=field_name, as_name=as_name, source_type=source_type, target_type=target_type)
355
356 if not metric.as_name:
357 raise DefinitionError(f"{definition_id}: metric 'as' name cannot be blank")
358 if metric.as_name in reserved_names:
359 raise DefinitionError(
360 f"{definition_id}: metric name '{metric.as_name}' collides with id/bucket_start/"
361 "computed_at or a group_by field"
362 )
363 reserved_names.add(metric.as_name)
364 metrics.append(metric)
365
366 return tuple(metrics)
367
368
369def _parse_min_group_size(record: Mapping[str, str], *, definition_id: str) -> int | None:
370 raw = (record.get("min_group_size") or "").strip()
371 if not raw:
372 return None
373 try:
374 value = int(raw)
375 except ValueError as exc:
376 raise DefinitionError(f"{definition_id}: min_group_size must be an integer") from exc
377 if value < 1:
378 raise DefinitionError(f"{definition_id}: min_group_size must be >= 1")
379 return value
380
381
382def _json_field(record: Mapping[str, str], name: str, default: Any, *, definition_id: str) -> Any:
383 raw = record.get(name)
384 if raw is None:
385 return default
386 raw = raw.strip()
387 if not raw:
388 return default
389 try:
390 return json.loads(raw)
391 except json.JSONDecodeError as exc:
392 raise DefinitionError(f"{definition_id}: {name} is not valid JSON: {exc}") from exc
393
394
395def _truthy(value: Any, *, default: bool) -> bool:
396 if value is None or value == "":
397 return default
398 return str(value).strip().lower() in {"true", "1", "yes", "on"}
399
400
401def _positive_int(value: Any) -> int | None:
402 try:
403 parsed = int(str(value).strip())
404 except (TypeError, ValueError):
405 return None
406 return parsed if parsed > 0 else None
407
408
409def _parse_iso(value: str) -> datetime | None:
410 try:
411 parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
412 except ValueError:
413 return None
414 if parsed.tzinfo is None:
415 parsed = parsed.replace(tzinfo=timezone.utc)
416 return parsed
417
418
419# --- Derived target schema -------------------------------------------------
420
421def derive_target_schema(config: RollupConfig) -> dict[str, Any]:
422 """Return the target collection's schema, generated from ``config``.
423
424 Shape rules per 14's "Derived target schema" section: one plain
425 ``text`` field per ``group_by`` entry (a source enum/relation field
426 still becomes plain text on the target -- the target stores the
427 value, not a live pointer); ``bucket_start`` only when a time bucket
428 is declared, typed ``date`` for day/week/month or ``datetime`` when
429 the source field is ``datetime`` and granularity is ``none``; one
430 ``"computed": true`` field per metric, named by ``as``; and an
431 unconditional ``"computed": true`` ``computed_at``, present regardless
432 of refresh mode. ``views.list_mode: "table"`` plus ``list_fields``
433 covering group-by + bucket + metrics are set automatically -- the
434 payoff named in 14's doctrine, a rollup target renders with zero page
435 code.
436 """
437 fields: list[dict[str, Any]] = [{"name": "id"}]
438 for name in config.group_by:
439 fields.append({"name": name, "type": "text"})
440
441 if config.time_bucket_field:
442 bucket_type = "date"
443 if config.time_bucket_granularity == "none" and config.time_bucket_source_type == "datetime":
444 bucket_type = "datetime"
445 fields.append({"name": "bucket_start", "type": bucket_type})
446
447 for metric in config.metrics:
448 fields.append({"name": metric.as_name, "type": metric.target_type, "computed": True})
449
450 fields.append({"name": "computed_at", "type": "datetime", "computed": True})
451
452 list_fields = list(config.group_by)
453 if config.time_bucket_field:
454 list_fields.append("bucket_start")
455 list_fields.extend(metric.as_name for metric in config.metrics)
456
457 return {
458 "name": config.target_collection,
459 "title": config.name,
460 "description": (
461 f"Rollup target for definition '{config.definition_id}' -- generated, "
462 "never hand-edited. See plan/vocabulary/14-rollup-spec.md."
463 ),
464 "storage": object_schemas.STORAGE_CLASSIC,
465 "fields": fields,
466 "views": {"list_mode": "table", "list_fields": list_fields},
467 }
468
469
470def _next_schema_version(target_collection: str, *, base_dir: Any) -> int:
471 """One more than the target's current schema version, or 1 when there
472 isn't one yet -- 14's Upgrade Posture calls the departure from Rule 1
473 a "regenerate": "bump the target schema, force an immediate full
474 recompute." Every recompute already regenerates and replaces the
475 schema (see module docstring); this just makes that regeneration a
476 literal version bump rather than a silent same-number overwrite.
477 """
478 try:
479 existing = object_schemas.get_schema(target_collection, base_dir=base_dir)
480 except object_schemas.SchemaNotFoundError:
481 return 1
482 try:
483 return int(existing.get("version") or 0) + 1
484 except (TypeError, ValueError):
485 return 1
486
487
488# --- Time bucketing -------------------------------------------------
489
490def _truncate_bucket(raw_value: str, granularity: str) -> str | None:
491 """Truncate one source field value to its bucket start.
492
493 No timezone conversion -- 14 is explicit that bucketing "truncates
494 the field's value to the bucket start ... in the field's own timezone
495 convention," matching docs/schema-forms.md's date/datetime types,
496 which are stored and compared as given. Returns None for a value that
497 doesn't parse as a date/datetime (caller drops that row from grouping
498 rather than crashing the whole definition over one bad row).
499 """
500 if granularity == "none":
501 return raw_value
502
503 parsed = _parse_date_or_datetime(raw_value)
504 if parsed is None:
505 return None
506
507 if granularity == "day":
508 bucket = parsed
509 elif granularity == "week":
510 bucket = parsed - timedelta(days=parsed.weekday()) # ISO week starts Monday
511 elif granularity == "month":
512 bucket = parsed.replace(day=1)
513 else: # pragma: no cover -- parse_definition already validated this
514 raise DefinitionError(f"unknown time_bucket granularity: {granularity}")
515 return bucket.isoformat()
516
517
518def _parse_date_or_datetime(raw_value: str) -> date | None:
519 text = (raw_value or "").strip()
520 if not text:
521 return None
522 try:
523 if "T" in text or " " in text:
524 return datetime.fromisoformat(text.replace("Z", "+00:00")).date()
525 return date.fromisoformat(text)
526 except ValueError:
527 return None
528
529
530# --- Grouping and metrics -------------------------------------------------
531
532def _group_rows(
533 rows: list[dict[str, str]], config: RollupConfig
534) -> dict[tuple[str | None, tuple[str, ...]], list[dict[str, str]]]:
535 """Group already-filtered source rows by (bucket_start, group_key).
536
537 A row whose time_bucket field is blank, or doesn't parse as a
538 date/datetime, is dropped from grouping entirely (it can't be
539 honestly bucketed) rather than raising and failing the whole
540 definition over one bad row -- consistent with this pass's isolation
541 posture elsewhere.
542 """
543 groups: dict[tuple[str | None, tuple[str, ...]], list[dict[str, str]]] = {}
544 for row in rows:
545 if config.time_bucket_field:
546 raw_bucket_value = row.get(config.time_bucket_field)
547 if not raw_bucket_value:
548 continue
549 bucket = _truncate_bucket(raw_bucket_value, config.time_bucket_granularity)
550 if bucket is None:
551 continue
552 else:
553 bucket = None
554 group_key = tuple(row.get(name, "") for name in config.group_by)
555 groups.setdefault((bucket, group_key), []).append(row)
556 return groups
557
558
559def _compute_metric(metric: Metric, rows: list[dict[str, str]]) -> str:
560 """Compute one metric's value over one group's rows.
561
562 Integer-exact per 14's float-accumulation note: an integer-typed
563 source field is summed/averaged with Python's arbitrary-precision
564 ints, never floats, so cents arithmetic never accumulates rounding
565 error. ``avg`` is always sum/count from that group's own exact sum
566 and count, computed fresh from the full row set every recompute
567 (never a running average) -- v1's scheduled-only posture means the
568 complete row list is always available, so there is no need to persist
569 an intermediate sum/count pair the way an incremental fast path
570 eventually would (see module docstring).
571 """
572 if metric.op == "count":
573 return str(len(rows))
574
575 present = [row.get(metric.field) for row in rows if row.get(metric.field) not in (None, "")]
576
577 if metric.op in ("min", "max"):
578 if not present:
579 return ""
580 numbered = [(_parse_metric_number(value, metric.source_type), value) for value in present]
581 chosen = min(numbered) if metric.op == "min" else max(numbered)
582 return chosen[1]
583
584 if metric.op == "sum":
585 if metric.source_type == "integer":
586 return str(sum(int(value) for value in present))
587 return repr(sum(float(value) for value in present))
588
589 if metric.op == "avg":
590 if not present:
591 return ""
592 if metric.source_type == "integer":
593 total: float = sum(int(value) for value in present)
594 else:
595 total = sum(float(value) for value in present)
596 return repr(total / len(present))
597
598 raise DefinitionError(f"unknown metric op: {metric.op}") # pragma: no cover -- pre-validated
599
600
601def _parse_metric_number(value: str, source_type: str | None) -> float:
602 return float(int(value)) if source_type == "integer" else float(value)
603
604
605def _target_row_id(definition_id: str, key: tuple[str | None, tuple[str, ...]]) -> str:
606 """A deterministic target row id from (definition, group key, bucket).
607
608 14's Storage section requires this so "a recompute or an incremental
609 upsert always lands on the same row rather than accumulating
610 duplicates." Hashed (not concatenated) because group key values are
611 arbitrary source data that may contain characters unsafe in a record
612 id (``object_records._RECORD_ID_RE``); the hash is stable across
613 recomputes since it is a pure function of the definition id, the
614 bucket, and the group key values, in the same field order every time.
615 """
616 bucket, group_key = key
617 canonical = json.dumps([definition_id, bucket, list(group_key)], separators=(",", ":"))
618 digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:32]
619 return f"rlp_{digest}"
620
621
622def _build_target_row(
623 config: RollupConfig,
624 key: tuple[str | None, tuple[str, ...]],
625 rows: list[dict[str, str]],
626 computed_at: str,
627) -> dict[str, str]:
628 bucket, group_key = key
629 row: dict[str, str] = {"id": _target_row_id(config.definition_id, key)}
630 for name, value in zip(config.group_by, group_key, strict=True):
631 row[name] = value
632 if config.time_bucket_field:
633 row["bucket_start"] = bucket or ""
634 for metric in config.metrics:
635 row[metric.as_name] = _compute_metric(metric, rows)
636 row["computed_at"] = computed_at
637 return row
638
639
640# --- Compute -------------------------------------------------
641
642def compute_rollup(
643 definition: Mapping[str, str], *, base_dir: Any, roots: Any = None
644) -> dict[str, Any]:
645 """Full recompute for one ``rollup_definitions`` record.
646
647 Reads ``source_collection`` outside the permission engine's row
648 filters -- daemon posture, same as ``object_records.compact_collection``
649 (14's Permissions Posture: "it must see every row, across every
650 owner, to aggregate correctly into one admin-controlled target").
651 Regenerates and replaces the target's derived schema, then reconciles
652 the target collection's rows to exactly the newly computed set (see
653 module docstring for why "reconcile via the record API" is how this
654 codebase expresses "full rewrite" without a bulk-replace primitive).
655
656 Raises ``DefinitionError`` for a malformed definition, and lets
657 ``object_schemas.SchemaNotFoundError`` / ``object_collections.
658 CollectionNotFoundError`` propagate when ``source_collection`` is
659 missing -- the caller (``object_daemon.process_rollups``) isolates
660 each definition's failure in its own try/except, one bad definition
661 never stopping the rest of the pass (14's Degradation section).
662 """
663 config = parse_definition(definition, base_dir=base_dir, roots=roots)
664
665 source_records = object_records.read_collection_records(
666 config.source_collection, base_dir=base_dir, roots=roots
667 )
668 filtered = (
669 object_records.filter_records(source_records, config.filter)
670 if config.filter
671 else list(source_records)
672 )
673
674 groups = _group_rows(filtered, config)
675
676 computed_at = _now_iso()
677 target_rows: list[dict[str, str]] = []
678 suppressed = 0
679 for key, rows in groups.items():
680 if config.min_group_size is not None and len(rows) < config.min_group_size:
681 suppressed += 1
682 continue
683 target_rows.append(_build_target_row(config, key, rows, computed_at))
684
685 schema_payload = derive_target_schema(config)
686 schema_payload["version"] = _next_schema_version(config.target_collection, base_dir=base_dir)
687 object_schemas.replace_schema(config.target_collection, schema_payload, base_dir=base_dir)
688
689 reconciled = _reconcile_target_rows(
690 config.target_collection, target_rows, base_dir=base_dir, roots=roots
691 )
692
693 return {
694 "definition_id": config.definition_id,
695 "target_collection": config.target_collection,
696 "groups": len(target_rows),
697 "suppressed": suppressed,
698 "created": reconciled["created"],
699 "updated": reconciled["updated"],
700 "deleted": reconciled["deleted"],
701 "computed_at": computed_at,
702 }
703
704
705def _reconcile_target_rows(
706 collection: str, new_rows: list[dict[str, str]], *, base_dir: Any, roots: Any = None
707) -> dict[str, int]:
708 """Sync ``collection`` to exactly ``new_rows`` by id: create the new
709 ones, update the ones that already existed, delete whatever's left --
710 a full recompute's atomic rewrite naturally drops any row whose group
711 no longer exists (14: "no explicit 'delete stale rows' step is
712 needed, the rewrite IS that step"); this is that step, done per-row
713 through the public record API.
714
715 Every row is written unconditionally, including one whose metric
716 values are unchanged since the last pass: ``computed_at`` always
717 advances, honestly reflecting "just reconfirmed as of now" rather
718 than silently reusing a stale timestamp because nothing else moved.
719
720 None of these writes append to the change log (``record_changes=False``),
721 and that follows directly from the paragraph above. A recompute rewrites
722 every row every pass by design, so logging it produces one audit entry
723 per row per pass forever -- describing nothing a person did, and nothing
724 that could not be recomputed from the source collection and the
725 definition, both of which are still on disk. Left on, it produced a
726 944MB change log for a collection holding 1818 rows, and the activity
727 feed, which reads every collection's log, stopped finishing. The log is
728 for who did what; a derived value has no who.
729 """
730 existing_ids: set[str] = set()
731 if object_records.collection_has_records(collection, base_dir=base_dir):
732 existing_ids = {
733 row["id"]
734 for row in object_records.read_collection_records(collection, base_dir=base_dir, roots=roots)
735 }
736
737 created = updated = 0
738 seen_ids: set[str] = set()
739 for row in new_rows:
740 seen_ids.add(row["id"])
741 if row["id"] in existing_ids:
742 object_records.update_collection_record(
743 collection, row["id"], row,
744 base_dir=base_dir, roots=roots,
745 actor=ROLLUP_ACTOR, allow_computed_submission=True,
746 # A recompute is not history. See below.
747 record_changes=False,
748 )
749 updated += 1
750 else:
751 object_records.create_collection_record(
752 collection, row,
753 base_dir=base_dir, roots=roots,
754 actor=ROLLUP_ACTOR, allow_computed_submission=True,
755 record_changes=False,
756 )
757 created += 1
758
759 deleted = 0
760 for stale_id in existing_ids - seen_ids:
761 object_records.delete_collection_record(
762 collection, stale_id, base_dir=base_dir, roots=roots,
763 actor=ROLLUP_ACTOR, record_changes=False,
764 )
765 deleted += 1
766
767 return {"created": created, "updated": updated, "deleted": deleted}
768
769
770def _now_iso() -> str:
771 return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")