askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_collections.py · 8.0 KB · 259 lines
| 1 | """Read-only collection summaries for object sources. |
| 2 | |
| 3 | Collections are a server-side view over object source files and permission |
| 4 | policy. They let tools such as Scroll show business/app groupings without |
| 5 | turning collection membership into a separate database table too early. |
| 6 | """ |
| 7 | |
| 8 | from __future__ import annotations |
| 9 | |
| 10 | from collections import Counter, defaultdict |
| 11 | from dataclasses import dataclass |
| 12 | from pathlib import Path |
| 13 | from typing import Any, Iterable |
| 14 | |
| 15 | import object_files |
| 16 | import object_logs |
| 17 | import object_permission_store |
| 18 | import object_permissions |
| 19 | import object_state |
| 20 | from object_namespace import ( |
| 21 | ObjectSource, |
| 22 | iter_object_sources, |
| 23 | parse_user_object_id, |
| 24 | validate_object_id, |
| 25 | ) |
| 26 | from object_versions import DEFAULT_DATA_DIR |
| 27 | |
| 28 | COLLECTIONS_DIR = "collections" |
| 29 | RECORDS_FILE = "records.tsv" |
| 30 | |
| 31 | |
| 32 | class InvalidCollectionNameError(ValueError): |
| 33 | """Raised when a collection name is not safe for routes or storage.""" |
| 34 | |
| 35 | |
| 36 | class CollectionNotFoundError(LookupError): |
| 37 | """Raised when a collection has no objects or permission rules.""" |
| 38 | |
| 39 | |
| 40 | @dataclass(frozen=True) |
| 41 | class CollectionObjectSummary: |
| 42 | object_id: str |
| 43 | path: str |
| 44 | owner: str |
| 45 | kind: str |
| 46 | state_count: int |
| 47 | has_logs: bool |
| 48 | file_count: int |
| 49 | |
| 50 | def to_dict(self) -> dict[str, Any]: |
| 51 | return { |
| 52 | "object_id": self.object_id, |
| 53 | "path": self.path, |
| 54 | "owner": self.owner, |
| 55 | "kind": self.kind, |
| 56 | "state_count": self.state_count, |
| 57 | "has_logs": self.has_logs, |
| 58 | "file_count": self.file_count, |
| 59 | } |
| 60 | |
| 61 | |
| 62 | def validate_collection_name(collection: str) -> bool: |
| 63 | """Return True when a collection name is safe to use in a route.""" |
| 64 | return validate_object_id(collection) |
| 65 | |
| 66 | |
| 67 | def collection_for_object_id(object_id: str) -> str | None: |
| 68 | """Infer the permission/UI collection for an object id.""" |
| 69 | parsed = parse_user_object_id(object_id) |
| 70 | if parsed is not None: |
| 71 | _, name = parsed |
| 72 | return name.split("_", 1)[0] if name else None |
| 73 | |
| 74 | return object_id.split("_", 1)[0] if object_id else None |
| 75 | |
| 76 | |
| 77 | def list_collections( |
| 78 | *, |
| 79 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 80 | roots: Iterable[Path] | None = None, |
| 81 | ) -> list[dict[str, Any]]: |
| 82 | """Return summaries for all known source and permission collections.""" |
| 83 | sources = iter_object_sources(roots) |
| 84 | policy = object_permission_store.load_policy(base_dir) |
| 85 | summaries = _summaries_by_collection(sources, policy, base_dir=base_dir) |
| 86 | return [summaries[name] for name in sorted(summaries)] |
| 87 | |
| 88 | |
| 89 | def get_collection( |
| 90 | collection: str, |
| 91 | *, |
| 92 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 93 | roots: Iterable[Path] | None = None, |
| 94 | ) -> dict[str, Any]: |
| 95 | """Return one collection summary including object details.""" |
| 96 | if not validate_collection_name(collection): |
| 97 | raise InvalidCollectionNameError(f"Invalid collection name: {collection}") |
| 98 | |
| 99 | sources = iter_object_sources(roots) |
| 100 | policy = object_permission_store.load_policy(base_dir) |
| 101 | summaries = _summaries_by_collection( |
| 102 | sources, |
| 103 | policy, |
| 104 | base_dir=base_dir, |
| 105 | include_objects=True, |
| 106 | ) |
| 107 | try: |
| 108 | return summaries[collection] |
| 109 | except KeyError as exc: |
| 110 | raise CollectionNotFoundError(f"Collection not found: {collection}") from exc |
| 111 | |
| 112 | |
| 113 | def _summaries_by_collection( |
| 114 | sources: list[ObjectSource], |
| 115 | policy: object_permissions.PermissionPolicy, |
| 116 | *, |
| 117 | base_dir: Path | str, |
| 118 | include_objects: bool = False, |
| 119 | ) -> dict[str, dict[str, Any]]: |
| 120 | object_groups: dict[str, list[CollectionObjectSummary]] = defaultdict(list) |
| 121 | for source in sources: |
| 122 | collection = collection_for_object_id(source.object_id) |
| 123 | if collection is None: |
| 124 | continue |
| 125 | object_groups[collection].append(_object_summary(source, base_dir=base_dir)) |
| 126 | |
| 127 | policy_collections = { |
| 128 | rule.collection |
| 129 | for rule in policy.rules |
| 130 | if rule.collection is not None and validate_collection_name(rule.collection) |
| 131 | } |
| 132 | record_collections = set(_iter_record_collections(base_dir)) |
| 133 | schema_collections = set(_iter_schema_collections(base_dir)) |
| 134 | |
| 135 | collection_names = ( |
| 136 | set(object_groups) | policy_collections | record_collections | schema_collections |
| 137 | ) |
| 138 | summaries: dict[str, dict[str, Any]] = {} |
| 139 | for collection in collection_names: |
| 140 | objects = sorted(object_groups.get(collection, []), key=lambda item: item.object_id) |
| 141 | summaries[collection] = _collection_summary( |
| 142 | collection, |
| 143 | objects, |
| 144 | policy, |
| 145 | has_records=collection in record_collections, |
| 146 | has_schema=collection in schema_collections, |
| 147 | include_objects=include_objects, |
| 148 | ) |
| 149 | |
| 150 | return summaries |
| 151 | |
| 152 | |
| 153 | def _object_summary( |
| 154 | source: ObjectSource, |
| 155 | *, |
| 156 | base_dir: Path | str, |
| 157 | ) -> CollectionObjectSummary: |
| 158 | state = object_state.get_object_state(source.object_id, base_dir=base_dir) |
| 159 | files = object_files.list_object_files(source.object_id, base_dir=base_dir) |
| 160 | |
| 161 | return CollectionObjectSummary( |
| 162 | object_id=source.object_id, |
| 163 | path=source.relative_path.as_posix(), |
| 164 | owner=_object_owner(source.object_id), |
| 165 | kind=source.kind, |
| 166 | state_count=len(state), |
| 167 | has_logs=_has_logs(source.object_id, base_dir=base_dir), |
| 168 | file_count=len(files), |
| 169 | ) |
| 170 | |
| 171 | |
| 172 | def _collection_summary( |
| 173 | collection: str, |
| 174 | objects: list[CollectionObjectSummary], |
| 175 | policy: object_permissions.PermissionPolicy, |
| 176 | *, |
| 177 | has_records: bool, |
| 178 | has_schema: bool = False, |
| 179 | include_objects: bool, |
| 180 | ) -> dict[str, Any]: |
| 181 | owners = sorted({item.owner for item in objects}) |
| 182 | kinds = Counter(item.kind for item in objects) |
| 183 | summary = { |
| 184 | "name": collection, |
| 185 | "object_count": len(objects), |
| 186 | "file_count": sum(item.file_count for item in objects), |
| 187 | "state_object_count": sum(1 for item in objects if item.state_count > 0), |
| 188 | "log_object_count": sum(1 for item in objects if item.has_logs), |
| 189 | "has_records": has_records, |
| 190 | "has_schema": has_schema, |
| 191 | "owners": owners, |
| 192 | "kinds": dict(sorted(kinds.items())), |
| 193 | "permission": _permission_summary(collection, policy), |
| 194 | } |
| 195 | if include_objects: |
| 196 | summary["objects"] = [item.to_dict() for item in objects] |
| 197 | return summary |
| 198 | |
| 199 | |
| 200 | def _permission_summary( |
| 201 | collection: str, |
| 202 | policy: object_permissions.PermissionPolicy, |
| 203 | ) -> dict[str, Any]: |
| 204 | rules = [rule for rule in policy.rules if rule.collection == collection] |
| 205 | actions = sorted({action for rule in rules for action in rule.actions}) |
| 206 | principals = sorted({rule.principal for rule in rules}) |
| 207 | return { |
| 208 | "access_mode": policy.access_mode, |
| 209 | "rule_count": len(rules), |
| 210 | "allow_count": sum(1 for rule in rules if rule.effect == "allow"), |
| 211 | "deny_count": sum(1 for rule in rules if rule.effect == "deny"), |
| 212 | "actions": actions, |
| 213 | "principals": principals, |
| 214 | } |
| 215 | |
| 216 | |
| 217 | def _object_owner(object_id: str) -> str: |
| 218 | parsed = parse_user_object_id(object_id) |
| 219 | if parsed is None: |
| 220 | return "system" |
| 221 | user_id, _ = parsed |
| 222 | return str(user_id) |
| 223 | |
| 224 | |
| 225 | def _has_logs(object_id: str, *, base_dir: Path | str) -> bool: |
| 226 | log_dir = object_logs.object_log_dir(object_id, base_dir) |
| 227 | if not log_dir.exists() or not log_dir.is_dir(): |
| 228 | return False |
| 229 | |
| 230 | if (log_dir / object_logs.LOG_FILE).is_file(): |
| 231 | return True |
| 232 | |
| 233 | return any(path.is_file() for path in log_dir.glob("log-*.tsv*")) |
| 234 | |
| 235 | |
| 236 | def _iter_schema_collections(base_dir: Path | str) -> list[str]: |
| 237 | root = Path(base_dir) / "schemas" |
| 238 | if not root.exists() or not root.is_dir(): |
| 239 | return [] |
| 240 | |
| 241 | names = [] |
| 242 | for path in sorted(root.glob("*.json"), key=lambda item: item.as_posix()): |
| 243 | name = path.stem |
| 244 | if validate_collection_name(name): |
| 245 | names.append(name) |
| 246 | return names |
| 247 | |
| 248 | |
| 249 | def _iter_record_collections(base_dir: Path | str) -> list[str]: |
| 250 | root = Path(base_dir) / COLLECTIONS_DIR |
| 251 | if not root.exists() or not root.is_dir(): |
| 252 | return [] |
| 253 | |
| 254 | names = [] |
| 255 | for path in sorted(root.glob(f"*/{RECORDS_FILE}"), key=lambda item: item.as_posix()): |
| 256 | name = path.parent.name |
| 257 | if validate_collection_name(name): |
| 258 | names.append(name) |
| 259 | return names |