lorebase-store 0.2.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. lorebase_store/__init__.py +1 -0
  2. lorebase_store/_migrations/env.py +106 -0
  3. lorebase_store/_migrations/script.py.mako +27 -0
  4. lorebase_store/_migrations/versions/0001_initial_schema.py +1071 -0
  5. lorebase_store/_migrations/versions/0002_merge_attempt_details.py +74 -0
  6. lorebase_store/_migrations/versions/0003_webhooks.py +127 -0
  7. lorebase_store/_migrations/versions/0004_defer_revision_references.py +38 -0
  8. lorebase_store/_migrations/versions/0005_security_definer_reads.py +396 -0
  9. lorebase_store/_migrations/versions/0006_api_key_usage.py +48 -0
  10. lorebase_store/_migrations/versions/0007_document_extraction.py +154 -0
  11. lorebase_store/_migrations/versions/0008_definer_argument_checks.py +178 -0
  12. lorebase_store/_migrations/versions/0009_api_key_networks.py +33 -0
  13. lorebase_store/_migrations/versions/0010_overlay_branches.py +582 -0
  14. lorebase_store/_migrations/versions/0011_branch_scoped_search_index.py +229 -0
  15. lorebase_store/_migrations/versions/0012_overlay_neighborhood_lookups.py +185 -0
  16. lorebase_store/_migrations/versions/0013_web_sessions.py +72 -0
  17. lorebase_store/_migrations/versions/0014_embedding_jobs.py +70 -0
  18. lorebase_store/_migrations/versions/0015_append_only_audit_log.py +83 -0
  19. lorebase_store/_migrations/versions/0016_sealed_webhook_keys.py +55 -0
  20. lorebase_store/_migrations/versions/0017_background_branch_copies.py +77 -0
  21. lorebase_store/_migrations/versions/0018_search_lexicon.py +477 -0
  22. lorebase_store/_migrations/versions/0019_vault_bindings.py +56 -0
  23. lorebase_store/_migrations/versions/0020_outbox_consumers.py +150 -0
  24. lorebase_store/_migrations/versions/0021_schema_suggestions.py +114 -0
  25. lorebase_store/_migrations/versions/0022_read_audit_and_shared_rate_limits.py +133 -0
  26. lorebase_store/_migrations/versions/0023_agent_trace.py +96 -0
  27. lorebase_store/_migrations/versions/0024_embedding_template_keys.py +205 -0
  28. lorebase_store/_migrations/versions/0025_snapshot_refs.py +35 -0
  29. lorebase_store/_migrations/versions/0026_plain_reverse_link_index.py +112 -0
  30. lorebase_store/_migrations/versions/0027_vault_rotation_and_tombstones.py +145 -0
  31. lorebase_store/_migrations/versions/0028_operation_deferred_events.py +71 -0
  32. lorebase_store/_migrations/versions/0029_agent_trace_sessions.py +108 -0
  33. lorebase_store/_migrations/versions/0030_path_fanout.py +31 -0
  34. lorebase_store/_migrations/versions/0031_branch_builds.py +91 -0
  35. lorebase_store/_migrations/versions/0032_lineage_revisions.py +35 -0
  36. lorebase_store/_migrations/versions/0033_vault_moved_tokens.py +30 -0
  37. lorebase_store/_migrations/versions/0034_inference_endpoint_modes.py +73 -0
  38. lorebase_store/_migrations/versions/0035_search_truncation.py +104 -0
  39. lorebase_store/_migrations/versions/0036_lineage_maps.py +325 -0
  40. lorebase_store/_migrations/versions/0037_bm25_search.py +346 -0
  41. lorebase_store/auth/__init__.py +52 -0
  42. lorebase_store/auth/api_keys.py +161 -0
  43. lorebase_store/auth/audit.py +382 -0
  44. lorebase_store/auth/checkpoints.py +185 -0
  45. lorebase_store/auth/grants.py +161 -0
  46. lorebase_store/auth/principals.py +158 -0
  47. lorebase_store/auth/scope.py +70 -0
  48. lorebase_store/auth/sessions.py +153 -0
  49. lorebase_store/auth/unit.py +53 -0
  50. lorebase_store/auth/usage.py +70 -0
  51. lorebase_store/blobs.py +73 -0
  52. lorebase_store/bm25.py +346 -0
  53. lorebase_store/bulk.py +147 -0
  54. lorebase_store/bundle.py +1830 -0
  55. lorebase_store/changefeed.py +261 -0
  56. lorebase_store/commit/__init__.py +42 -0
  57. lorebase_store/commit/changes.py +67 -0
  58. lorebase_store/commit/errors.py +60 -0
  59. lorebase_store/commit/migrations.py +335 -0
  60. lorebase_store/commit/pipeline.py +754 -0
  61. lorebase_store/commit/schema.py +123 -0
  62. lorebase_store/db.py +58 -0
  63. lorebase_store/definer.py +50 -0
  64. lorebase_store/digests.py +99 -0
  65. lorebase_store/embeddings.py +444 -0
  66. lorebase_store/endpoint_modes.py +59 -0
  67. lorebase_store/extraction/__init__.py +61 -0
  68. lorebase_store/extraction/items.py +187 -0
  69. lorebase_store/extraction/materialize.py +364 -0
  70. lorebase_store/extraction/proposal.py +65 -0
  71. lorebase_store/extraction/review.py +168 -0
  72. lorebase_store/gc.py +970 -0
  73. lorebase_store/indexes.py +1033 -0
  74. lorebase_store/jobs.py +651 -0
  75. lorebase_store/keyring.py +72 -0
  76. lorebase_store/layout.py +335 -0
  77. lorebase_store/materialize/__init__.py +103 -0
  78. lorebase_store/materialize/apply.py +436 -0
  79. lorebase_store/materialize/background.py +1184 -0
  80. lorebase_store/materialize/branch.py +268 -0
  81. lorebase_store/materialize/build.py +536 -0
  82. lorebase_store/materialize/derived.py +540 -0
  83. lorebase_store/materialize/lineage_backfill.py +502 -0
  84. lorebase_store/materialize/links.py +266 -0
  85. lorebase_store/materialize/overlay.py +292 -0
  86. lorebase_store/materialize/paths.py +315 -0
  87. lorebase_store/materialize/rebuild.py +326 -0
  88. lorebase_store/materialize/rows.py +153 -0
  89. lorebase_store/materialize/sensitivity.py +66 -0
  90. lorebase_store/materialize/snapshot.py +205 -0
  91. lorebase_store/materialize/stale.py +444 -0
  92. lorebase_store/materialize/verify.py +321 -0
  93. lorebase_store/merging/__init__.py +39 -0
  94. lorebase_store/merging/attempts.py +334 -0
  95. lorebase_store/merging/lineage.py +123 -0
  96. lorebase_store/merging/service.py +652 -0
  97. lorebase_store/migrations.py +133 -0
  98. lorebase_store/models/__init__.py +136 -0
  99. lorebase_store/models/base.py +64 -0
  100. lorebase_store/models/enums.py +132 -0
  101. lorebase_store/models/governance.py +268 -0
  102. lorebase_store/models/ingestion.py +288 -0
  103. lorebase_store/models/materialized.py +277 -0
  104. lorebase_store/models/proposals.py +195 -0
  105. lorebase_store/models/queues.py +87 -0
  106. lorebase_store/models/tracing.py +116 -0
  107. lorebase_store/models/versioning.py +222 -0
  108. lorebase_store/models/webhooks.py +96 -0
  109. lorebase_store/nodestore.py +384 -0
  110. lorebase_store/objectstore.py +162 -0
  111. lorebase_store/outbox_events.py +166 -0
  112. lorebase_store/profiling/__init__.py +58 -0
  113. lorebase_store/profiling/delimited.py +109 -0
  114. lorebase_store/profiling/postgres.py +211 -0
  115. lorebase_store/profiling/record_files.py +127 -0
  116. lorebase_store/profiling/records.py +92 -0
  117. lorebase_store/profiling/sql_databases.py +298 -0
  118. lorebase_store/ratelimit.py +129 -0
  119. lorebase_store/reads.py +1491 -0
  120. lorebase_store/replicas.py +434 -0
  121. lorebase_store/repositories/__init__.py +152 -0
  122. lorebase_store/repositories/changesets.py +59 -0
  123. lorebase_store/repositories/deferred_events.py +142 -0
  124. lorebase_store/repositories/documents.py +305 -0
  125. lorebase_store/repositories/domains.py +97 -0
  126. lorebase_store/repositories/events.py +205 -0
  127. lorebase_store/repositories/index_view.py +489 -0
  128. lorebase_store/repositories/invocations.py +68 -0
  129. lorebase_store/repositories/materializations.py +276 -0
  130. lorebase_store/repositories/materialized.py +1129 -0
  131. lorebase_store/repositories/merges.py +417 -0
  132. lorebase_store/repositories/proposal_items.py +199 -0
  133. lorebase_store/repositories/proposals.py +184 -0
  134. lorebase_store/repositories/provenance.py +77 -0
  135. lorebase_store/repositories/refs.py +452 -0
  136. lorebase_store/repositories/revisions.py +273 -0
  137. lorebase_store/repositories/scope.py +33 -0
  138. lorebase_store/repositories/source_runs.py +123 -0
  139. lorebase_store/repositories/suggestions.py +638 -0
  140. lorebase_store/runtime.py +87 -0
  141. lorebase_store/schema_cache.py +50 -0
  142. lorebase_store/settings.py +879 -0
  143. lorebase_store/storage.py +1478 -0
  144. lorebase_store/telemetry.py +142 -0
  145. lorebase_store/testing.py +108 -0
  146. lorebase_store/tracing.py +505 -0
  147. lorebase_store/traverse.py +578 -0
  148. lorebase_store/vault/__init__.py +50 -0
  149. lorebase_store/vault/keys.py +216 -0
  150. lorebase_store/vault/quotes.py +133 -0
  151. lorebase_store/vault/rotation.py +209 -0
  152. lorebase_store/vault/settings.py +152 -0
  153. lorebase_store/vault/store.py +987 -0
  154. lorebase_store/vault/transfer.py +558 -0
  155. lorebase_store/webhooks/__init__.py +24 -0
  156. lorebase_store/webhooks/delivery.py +637 -0
  157. lorebase_store/webhooks/keys.py +126 -0
  158. lorebase_store/webhooks/repository.py +142 -0
  159. lorebase_store/webhooks/settings.py +138 -0
  160. lorebase_store/webhooks/signing.py +50 -0
  161. lorebase_store-0.2.3.dist-info/METADATA +13 -0
  162. lorebase_store-0.2.3.dist-info/RECORD +163 -0
  163. lorebase_store-0.2.3.dist-info/WHEEL +4 -0
@@ -0,0 +1,754 @@
1
+ """The commit pipeline (design 6.6): every write goes through one transaction.
2
+
3
+ 1. take the branch advisory lock; read head `H` and the ref version;
4
+ 2. validate the changes against the materialized head of `H` (bulk-prefetched `IndexView`);
5
+ schema changes add their data migrations; values of vault Attributes are validated, then
6
+ replaced by vault references (`lorebase_store.vault`, under the Domain's vault lock held
7
+ shared to the end of the transaction) before the core rules run;
8
+ 3. canonicalize and hash new objects; insert them, new vault rows and vault lineage, with
9
+ `ON CONFLICT DO NOTHING`;
10
+ 4. apply the changes to the tree by path copying; insert new nodes;
11
+ 5. insert the Revision with parent `H`, its changeset and provenance entries;
12
+ 6. apply the changeset to the materialized head; when the schema changed, per-Attribute
13
+ indexes that would reject the new rows are dropped first and missing ones created after
14
+ (`lorebase_store.indexes`);
15
+ 7. compare-and-swap the ref from `H` to the new Revision;
16
+ 8. write the audit entry and the outbox event;
17
+ 9. commit.
18
+
19
+ Steps 2 to 4 run synchronously inside `AsyncSession.run_sync` (the core validation and tree
20
+ code are synchronous); their reads go through prefetching layers, so a batch costs a few
21
+ queries per tree level and per kind of question, not one per element.
22
+
23
+ `commit` runs in the caller's transaction when one is open (the ref then moves when the
24
+ caller commits) and in its own transaction otherwise. After a failure the transaction must be
25
+ rolled back; nothing of the commit is visible.
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ from collections.abc import Callable, Iterable, Mapping, Sequence
31
+ from dataclasses import dataclass, field
32
+ from datetime import UTC, datetime
33
+ from time import perf_counter
34
+ from typing import Any
35
+
36
+ from sqlalchemy.ext.asyncio import AsyncSession
37
+ from sqlalchemy.orm import Session
38
+
39
+ from lorebase_core.canonical import ObjectType, hex
40
+ from lorebase_core.expr.sensitivity import lowered_tags, schema_tags, with_tags
41
+ from lorebase_core.model import (
42
+ Domain,
43
+ IdPrefix,
44
+ ProvenanceEntry,
45
+ Schema,
46
+ Sensitivity,
47
+ new_id,
48
+ validate_schema,
49
+ )
50
+ from lorebase_core.revision import (
51
+ Author,
52
+ ChangesetEntry,
53
+ Origin,
54
+ Revision,
55
+ changeset_between,
56
+ changeset_stats,
57
+ )
58
+ from lorebase_core.tree import (
59
+ DEFAULT_LEAF_SIZE,
60
+ PARTITION_SCHEMA,
61
+ InMemoryNodeStore,
62
+ Tree,
63
+ TreeChange,
64
+ TreeParams,
65
+ kind_partition,
66
+ link_key,
67
+ relation_partition,
68
+ tree_apply,
69
+ )
70
+ from lorebase_core.validate import (
71
+ DataChange,
72
+ Issue,
73
+ LinkDelete,
74
+ LinkUpsert,
75
+ RecordDelete,
76
+ RecordUpsert,
77
+ evaluate_changeset,
78
+ )
79
+ from lorebase_store.auth.audit import AuditRepository, NewAuditEntry
80
+ from lorebase_store.commit.changes import (
81
+ Change,
82
+ CommitResult,
83
+ SchemaChange,
84
+ SchemaElementModel,
85
+ )
86
+ from lorebase_store.commit.errors import (
87
+ BranchLockTimeoutError,
88
+ BranchMaterializingError,
89
+ BranchNotFoundError,
90
+ BranchNotMaterializedError,
91
+ SensitivityLoweredError,
92
+ StaleHeadError,
93
+ ValidationFailedError,
94
+ )
95
+ from lorebase_store.commit.migrations import MigrationPlan, plan_migrations
96
+ from lorebase_store.commit.schema import (
97
+ apply_schema_changes,
98
+ element_key,
99
+ load_compiled,
100
+ object_type_of,
101
+ )
102
+ from lorebase_store.indexes import BranchIndexes
103
+ from lorebase_store.materialize.apply import MaterializedChange, apply_changeset
104
+ from lorebase_store.materialize.rows import (
105
+ ELEMENT_PROVENANCE_KEY,
106
+ PartitionType,
107
+ changed_attributes,
108
+ parse_partition,
109
+ split_link_key,
110
+ )
111
+ from lorebase_store.materialize.sensitivity import default_branch_schema
112
+ from lorebase_store.materialize.stale import queue_after_tagging
113
+ from lorebase_store.models.enums import Materialization, RefKind
114
+ from lorebase_store.nodestore import NodeRepository, PgNodeStore
115
+ from lorebase_store.objectstore import EncodedObject, PgObjectStore, encode_object
116
+ from lorebase_store.repositories.changesets import ChangesetRepository
117
+ from lorebase_store.repositories.domains import DomainRepository, DomainRow
118
+ from lorebase_store.repositories.events import OutboxRepository
119
+ from lorebase_store.repositories.index_view import MaterializedIndexView
120
+ from lorebase_store.repositories.materialized import MaterializedRepository
121
+ from lorebase_store.repositories.provenance import ProvenanceRepository, ProvenanceRow
122
+ from lorebase_store.repositories.refs import LockTimeoutError, RefMovedError, RefRepository
123
+ from lorebase_store.repositories.revisions import RevisionRepository, RevisionRow
124
+ from lorebase_store.repositories.scope import scope_to_domain
125
+ from lorebase_store.runtime import StoreRuntime, default_runtime
126
+ from lorebase_store.storage import constraints_only
127
+ from lorebase_store.telemetry import COMMIT_DURATION
128
+ from lorebase_store.vault.store import (
129
+ Lineage,
130
+ Move,
131
+ VaultLockTimeoutError,
132
+ VaultRow,
133
+ insert_lineage,
134
+ insert_rows,
135
+ mark_moved,
136
+ tokenize,
137
+ )
138
+
139
+ COMMIT_ACTION = "revision.commit"
140
+ COMMITTED_EVENT = "revision.committed"
141
+ DOMAIN_ORIGIN = "domain.create"
142
+
143
+ Clock = Callable[[], datetime]
144
+ ProvenanceSpec = str | Mapping[str, str] | None
145
+
146
+
147
+ def utc_now() -> datetime:
148
+ return datetime.now(UTC)
149
+
150
+
151
+ @dataclass
152
+ class _Prepared:
153
+ """Outcome of steps 2 to 4."""
154
+
155
+ root: bytes
156
+ changeset: list[ChangesetEntry]
157
+ bodies: dict[tuple[str, str], Mapping[str, Any] | None]
158
+ objects: list[EncodedObject]
159
+ nodes: dict[bytes, bytes]
160
+ schema: Schema
161
+ schema_hash: bytes | None
162
+ refresh_kinds: list[str] = field(default_factory=list)
163
+ provenance: list[ProvenanceEntry] = field(default_factory=list)
164
+ derived: list[DataChange] = field(default_factory=list)
165
+ vault_rows: list[VaultRow] = field(default_factory=list)
166
+ vault_lineage: list[Lineage] = field(default_factory=list)
167
+ vault_moved: list[Move] = field(default_factory=list)
168
+
169
+
170
+ @dataclass(frozen=True, slots=True)
171
+ class _Element:
172
+ """One element written to the tree: partition, key, encoded object or `None`."""
173
+
174
+ partition: str
175
+ key: str
176
+ encoded: EncodedObject | None
177
+ provenance: ProvenanceSpec = None
178
+
179
+
180
+ async def commit(
181
+ session: AsyncSession,
182
+ domain_id: str,
183
+ branch: str,
184
+ changes: Iterable[DataChange | Change],
185
+ *,
186
+ author: Author,
187
+ message: str,
188
+ origin: Origin,
189
+ schema_changes: Sequence[SchemaChange] = (),
190
+ provenance: Sequence[ProvenanceEntry] = (),
191
+ expected_head: bytes | None = None,
192
+ runtime: StoreRuntime | None = None,
193
+ clock: Clock = utc_now,
194
+ allowed_lowering: frozenset[Sensitivity] = frozenset(),
195
+ ) -> CommitResult:
196
+ """Commit data and schema changes to `branch` as one Revision.
197
+
198
+ Schema changes that remove sensitivity tags (or sensitive Attributes, Kinds, Relations)
199
+ raise `SensitivityLoweredError` for every tag not in `allowed_lowering` (the tags the
200
+ caller is cleared for), compared under the branch lock with the head actually changed.
201
+ Schemas and data are validated with the tags of the default branch's schema added
202
+ (`lorebase_core.expr.sensitivity.with_tags`).
203
+
204
+ `provenance` lists entries referenced by `Change.provenance`; changes without one are
205
+ attributed to an entry built from `origin` and `author`. `expected_head`, when given,
206
+ must be the current head (optimistic concurrency for API callers).
207
+
208
+ Raises `BranchNotFoundError`, `BranchNotMaterializedError`, `BranchLockTimeoutError`,
209
+ `StaleHeadError`, `SchemaChangeError` or `ValidationFailedError`.
210
+ """
211
+ request = _Request(
212
+ domain_id=domain_id,
213
+ branch=branch,
214
+ changes=[c if isinstance(c, Change) else Change(c) for c in changes],
215
+ author=author,
216
+ message=message,
217
+ origin=origin,
218
+ schema_changes=list(schema_changes),
219
+ provenance=list(provenance),
220
+ expected_head=expected_head,
221
+ runtime=runtime or default_runtime(),
222
+ clock=clock,
223
+ allowed_lowering=allowed_lowering,
224
+ )
225
+ began = perf_counter()
226
+ if session.in_transaction():
227
+ result = await _commit(session, request)
228
+ else:
229
+ async with session.begin():
230
+ result = await _commit(session, request)
231
+ COMMIT_DURATION.record(perf_counter() - began, {"origin": origin.type})
232
+ return result
233
+
234
+
235
+ async def create_domain(
236
+ session: AsyncSession,
237
+ domain: Domain,
238
+ *,
239
+ author: Author,
240
+ leaf_size: int = DEFAULT_LEAF_SIZE,
241
+ message: str = "Create Domain",
242
+ origin: Origin | None = None,
243
+ runtime: StoreRuntime | None = None,
244
+ clock: Clock = utc_now,
245
+ ) -> CommitResult:
246
+ """Create a Domain: its row, a root Revision holding the Domain element and the default
247
+ branch (fully materialized) pointing at it. `leaf_size` is fixed for the Domain's life."""
248
+ if session.in_transaction():
249
+ return await _create_domain(
250
+ session, domain, author, leaf_size, message, origin, runtime, clock
251
+ )
252
+ async with session.begin():
253
+ return await _create_domain(
254
+ session, domain, author, leaf_size, message, origin, runtime, clock
255
+ )
256
+
257
+
258
+ @dataclass
259
+ class _Request:
260
+ domain_id: str
261
+ branch: str
262
+ changes: list[Change]
263
+ author: Author
264
+ message: str
265
+ origin: Origin
266
+ schema_changes: list[SchemaChange]
267
+ provenance: list[ProvenanceEntry]
268
+ expected_head: bytes | None
269
+ runtime: StoreRuntime
270
+ clock: Clock
271
+ allowed_lowering: frozenset[Sensitivity] = frozenset()
272
+
273
+
274
+ async def _commit(session: AsyncSession, request: _Request) -> CommitResult:
275
+ settings = request.runtime.settings
276
+ domain_id, branch = request.domain_id, request.branch
277
+ await scope_to_domain(session, domain_id)
278
+ domain = await DomainRepository(session).require(domain_id)
279
+ refs = RefRepository(session, domain_id)
280
+ try:
281
+ await refs.lock(branch, timeout_ms=settings.commit.lock_timeout_ms)
282
+ except LockTimeoutError as error:
283
+ raise BranchLockTimeoutError(str(error)) from error
284
+ ref = await refs.get(branch)
285
+ if ref is None or ref.kind is not RefKind.BRANCH:
286
+ raise BranchNotFoundError(f"no branch {branch!r} in Domain {domain_id!r}")
287
+ if request.expected_head is not None and request.expected_head != ref.head:
288
+ raise StaleHeadError(branch, request.expected_head, ref.head)
289
+ if ref.materialization is Materialization.BUILDING:
290
+ raise BranchMaterializingError(
291
+ f"branch {branch!r} is being built from its tree in the background; retry when "
292
+ "its materialization is done"
293
+ )
294
+ revisions = RevisionRepository(session, domain_id)
295
+ head = await revisions.require(ref.head)
296
+ materialized = MaterializedRepository(session, domain_id, branch, writes=settings.writes)
297
+ compiled = await materialized.schema()
298
+ if compiled is None:
299
+ raise BranchNotMaterializedError(f"branch {branch!r} has no materialized head")
300
+ created_at = request.clock()
301
+ own = ProvenanceRow(
302
+ id=new_id(IdPrefix.PROVENANCE),
303
+ origin_type=request.origin.type,
304
+ origin_ref=request.origin.model_dump(),
305
+ actor=request.author.principal,
306
+ at=created_at,
307
+ )
308
+ old_schema = load_compiled(compiled.compiled)
309
+ reference: Schema | None = old_schema
310
+ if branch != domain.default_branch:
311
+ reference = await default_branch_schema(session, domain_id, runtime=request.runtime)
312
+ reference_tags = schema_tags(reference)
313
+
314
+ def prepare(sync: Session) -> _Prepared:
315
+ return _prepare(
316
+ sync, request, domain, head.root, old_schema, own.id, created_at, reference_tags
317
+ )
318
+
319
+ try:
320
+ prepared = await session.run_sync(prepare)
321
+ except VaultLockTimeoutError as error:
322
+ raise BranchLockTimeoutError(str(error)) from error
323
+ if not prepared.changeset:
324
+ return CommitResult(
325
+ revision_id=head.id,
326
+ parents=head.parents,
327
+ root=head.root,
328
+ generation=head.generation,
329
+ ref_version=ref.version,
330
+ created_at=head.created_at,
331
+ changeset=(),
332
+ no_op=True,
333
+ )
334
+ revision = Revision(
335
+ parents=(head.id,),
336
+ root=prepared.root,
337
+ domain=domain_id,
338
+ author=request.author,
339
+ message=request.message,
340
+ origin=request.origin,
341
+ created_at=created_at,
342
+ stats=changeset_stats(prepared.changeset),
343
+ )
344
+ provenance_rows = [
345
+ own,
346
+ *(ProvenanceRow.from_entry(e) for e in request.provenance),
347
+ *(ProvenanceRow.from_entry(e) for e in prepared.provenance),
348
+ ]
349
+ stored = await _store_revision(session, request.runtime, revision, prepared, provenance_rows)
350
+ indexes = None
351
+ if prepared.schema_hash is not None:
352
+ indexes = BranchIndexes(
353
+ session,
354
+ domain_id,
355
+ branch,
356
+ settings=settings.indexes,
357
+ constraints_only=constraints_only(settings),
358
+ maintenance=request.runtime.index_maintenance,
359
+ )
360
+ await indexes.retire(prepared.schema)
361
+ await apply_changeset(
362
+ session,
363
+ domain_id,
364
+ branch,
365
+ revision_id=stored.id,
366
+ at=stored.created_at,
367
+ changes=_materialized_changes(prepared),
368
+ schema=prepared.schema,
369
+ schema_hash=prepared.schema_hash,
370
+ refresh_kinds=prepared.refresh_kinds,
371
+ settings=settings,
372
+ expression_limits=request.runtime.schema.expression_limits,
373
+ runtime=request.runtime,
374
+ )
375
+ if indexes is not None:
376
+ await indexes.ensure(prepared.schema)
377
+ if prepared.schema_hash is not None and branch == domain.default_branch:
378
+ await queue_after_tagging(
379
+ session, domain_id, branch, old_schema, prepared.schema, runtime=request.runtime
380
+ )
381
+ try:
382
+ version = await refs.compare_and_swap(
383
+ branch,
384
+ expected_head=ref.head,
385
+ expected_version=ref.version,
386
+ new_head=stored.id,
387
+ actor=request.author.principal,
388
+ at=created_at,
389
+ reason=COMMIT_ACTION,
390
+ )
391
+ except RefMovedError as error:
392
+ current = await refs.get(branch)
393
+ raise StaleHeadError(branch, ref.head, None if current is None else current.head) from error
394
+ await _record_events(session, domain_id, branch, stored, prepared.changeset, request.message)
395
+ return CommitResult(
396
+ revision_id=stored.id,
397
+ parents=stored.parents,
398
+ root=stored.root,
399
+ generation=stored.generation,
400
+ ref_version=version,
401
+ created_at=stored.created_at,
402
+ changeset=tuple(prepared.changeset),
403
+ derived_changes=tuple(prepared.derived),
404
+ )
405
+
406
+
407
+ def _prepare(
408
+ sync: Session,
409
+ request: _Request,
410
+ domain: DomainRow,
411
+ head_root: bytes,
412
+ old_schema: Schema,
413
+ own_provenance: str,
414
+ created_at: datetime,
415
+ reference_tags: Mapping[str, frozenset[Sensitivity]],
416
+ ) -> _Prepared:
417
+ settings = request.runtime.settings
418
+ view = MaterializedIndexView(
419
+ sync, request.domain_id, request.branch, batch=settings.commit.prefetch_batch
420
+ )
421
+ schema = old_schema
422
+ elements: list[_Element] = []
423
+ issues: list[Issue] = []
424
+ plan = MigrationPlan()
425
+ if request.schema_changes:
426
+ schema, upserted, removed = apply_schema_changes(
427
+ old_schema, request.schema_changes, request.domain_id
428
+ )
429
+ refused = lowered_tags(old_schema, schema) - request.allowed_lowering
430
+ if refused:
431
+ raise SensitivityLoweredError(t.value for t in refused)
432
+ issues.extend(
433
+ i
434
+ for i in validate_schema(
435
+ with_tags(schema, reference_tags), request.runtime.schema.rules()
436
+ )
437
+ if i.is_error
438
+ )
439
+ elements.extend(_schema_element(e) for e in upserted)
440
+ elements.extend(_Element(PARTITION_SCHEMA, key, None) for key in removed)
441
+ plan = plan_migrations(
442
+ old_schema,
443
+ schema,
444
+ records_of=view.records_of_kind,
445
+ links_of=view.links_of_relation,
446
+ skip=[c.change.identity for c in request.changes],
447
+ sensitive=lambda a: a.is_sensitive or a.id in reference_tags,
448
+ actor=request.author.principal,
449
+ at=created_at,
450
+ )
451
+ issues.extend(i for i in plan.issues if i.is_error)
452
+ validated = [*plan.changes, *request.changes]
453
+ data = [c.change for c in validated]
454
+ view.prefetch_for(schema, data)
455
+ vaulted = tokenize(
456
+ sync,
457
+ request.domain_id,
458
+ schema,
459
+ data,
460
+ request.runtime.vault,
461
+ view,
462
+ erased_values=request.runtime.vault_settings.erased_values,
463
+ tombstones=request.origin.type in request.runtime.vault_settings.tombstone_origins,
464
+ lock_timeout_ms=request.runtime.vault_settings.lock_timeout_ms,
465
+ )
466
+ issues.extend(vaulted.issues)
467
+ if issues:
468
+ raise ValidationFailedError(issues)
469
+ data = vaulted.changes
470
+ validated = [Change(d, c.provenance) for d, c in zip(data, validated, strict=True)]
471
+ report = evaluate_changeset(with_tags(schema, reference_tags), view, data)
472
+ issues.extend(i for i in report.issues if i.is_error)
473
+ if issues:
474
+ raise ValidationFailedError(issues)
475
+ elements.extend(_data_element(c.change, c.provenance) for c in validated)
476
+ elements.extend(_data_element(d, None) for d in [*report.derived_changes, *plan.removals])
477
+
478
+ store = PgNodeStore(
479
+ sync,
480
+ request.domain_id,
481
+ cache=request.runtime.node_cache,
482
+ prefetch_batch=settings.nodes.prefetch_batch,
483
+ )
484
+ tree_changes = [
485
+ TreeChange(e.partition, e.key, None if e.encoded is None else e.encoded.hash)
486
+ for e in elements
487
+ ]
488
+ store.prefetch_paths(head_root, [(c.partition, c.key) for c in tree_changes])
489
+ root = tree_apply(store, head_root, tree_changes, TreeParams(domain.leaf_size))
490
+ by_key = {(e.partition, e.key): e for e in elements}
491
+ changeset = [
492
+ _attributed(entry, by_key[(entry.partition, entry.key)], view, own_provenance)
493
+ for entry in changeset_between(store, head_root, root)
494
+ ]
495
+ written = {entry.new_hash for entry in changeset}
496
+ schema_hash = None
497
+ if any(entry.partition == PARTITION_SCHEMA for entry in changeset):
498
+ schema_hash = Tree.load(store, root).partitions[PARTITION_SCHEMA].root
499
+ return _Prepared(
500
+ root=root,
501
+ changeset=changeset,
502
+ bodies={key: None if e.encoded is None else e.encoded.body for key, e in by_key.items()},
503
+ objects=[e.encoded for e in elements if e.encoded and e.encoded.hash in written],
504
+ nodes=store.take_pending(),
505
+ schema=schema,
506
+ schema_hash=schema_hash,
507
+ refresh_kinds=plan.refresh_kinds,
508
+ provenance=plan.provenance,
509
+ derived=list(report.derived_changes),
510
+ vault_rows=vaulted.rows,
511
+ vault_lineage=vaulted.lineage,
512
+ vault_moved=vaulted.moved,
513
+ )
514
+
515
+
516
+ def _schema_element(element: SchemaElementModel) -> _Element:
517
+ return _Element(
518
+ PARTITION_SCHEMA,
519
+ element_key(element),
520
+ encode_object(object_type_of(element), element.content()),
521
+ )
522
+
523
+
524
+ def _data_element(change: DataChange, provenance: ProvenanceSpec) -> _Element:
525
+ if isinstance(change, RecordUpsert):
526
+ record = change.record
527
+ encoded = encode_object(ObjectType.RECORD, record.content())
528
+ return _Element(kind_partition(record.kind), record.key, encoded, provenance)
529
+ if isinstance(change, RecordDelete):
530
+ return _Element(kind_partition(change.kind), change.key, None, provenance)
531
+ if isinstance(change, LinkUpsert):
532
+ link = change.link
533
+ encoded = encode_object(ObjectType.LINK, link.content())
534
+ key = link_key(link.from_key, link.to_key)
535
+ return _Element(relation_partition(link.relation), key, encoded, provenance)
536
+ assert isinstance(change, LinkDelete)
537
+ key = link_key(change.from_key, change.to_key)
538
+ return _Element(relation_partition(change.relation), key, None, provenance)
539
+
540
+
541
+ def _attributed(
542
+ entry: ChangesetEntry, element: _Element, view: MaterializedIndexView, own: str
543
+ ) -> ChangesetEntry:
544
+ """The changeset entry with its provenance map (changed Attribute -> entry id)."""
545
+ spec = element.provenance
546
+
547
+ def resolve(attribute: str) -> str:
548
+ if isinstance(spec, str):
549
+ return spec
550
+ if spec is not None:
551
+ return spec.get(attribute, own)
552
+ return own
553
+
554
+ attributes: list[str] = []
555
+ parsed = parse_partition(entry.partition)
556
+ if element.encoded is not None and parsed.type is not PartitionType.SCHEMA:
557
+ new_values: Mapping[str, Any] = element.encoded.body.get("values", {})
558
+ old_values: Mapping[str, Any] | None = None
559
+ if entry.old_hash is not None:
560
+ if parsed.type is PartitionType.KIND:
561
+ record = view.record_state(parsed.element_id, entry.key)
562
+ old_values = None if record is None else record.values
563
+ else:
564
+ from_key, to_key = split_link_key(entry.key)
565
+ link = view.link_state(parsed.element_id, from_key, to_key)
566
+ old_values = None if link is None else link.values
567
+ attributes = changed_attributes(old_values, new_values)
568
+ provenance = {a: resolve(a) for a in attributes} or {
569
+ ELEMENT_PROVENANCE_KEY: resolve(ELEMENT_PROVENANCE_KEY)
570
+ }
571
+ return ChangesetEntry(entry.partition, entry.key, entry.old_hash, entry.new_hash, provenance)
572
+
573
+
574
+ def _materialized_changes(prepared: _Prepared) -> list[MaterializedChange]:
575
+ return [
576
+ MaterializedChange(entry, prepared.bodies[(entry.partition, entry.key)])
577
+ for entry in prepared.changeset
578
+ ]
579
+
580
+
581
+ async def _store_revision(
582
+ session: AsyncSession,
583
+ runtime: StoreRuntime,
584
+ revision: Revision,
585
+ prepared: _Prepared,
586
+ provenance: Sequence[ProvenanceRow],
587
+ ) -> RevisionRow:
588
+ """Steps 3 to 5: objects, nodes, Revision, changeset and provenance."""
589
+ settings = runtime.settings
590
+ domain_id = revision.domain
591
+ await PgObjectStore(
592
+ session, domain_id, writes=settings.writes, batch=settings.nodes.prefetch_batch
593
+ ).put_many(prepared.objects)
594
+ await insert_rows(session, domain_id, prepared.vault_rows, at=revision.created_at)
595
+ await insert_lineage(session, domain_id, prepared.vault_lineage, at=revision.created_at)
596
+ await mark_moved(session, domain_id, prepared.vault_moved)
597
+ await NodeRepository(
598
+ session,
599
+ domain_id,
600
+ writes=settings.writes,
601
+ cache=runtime.node_cache,
602
+ batch=settings.nodes.prefetch_batch,
603
+ ).put_many(prepared.nodes)
604
+ stored = await RevisionRepository(session, domain_id).insert(revision)
605
+ await ProvenanceRepository(session, domain_id, writes=settings.writes).insert_many(provenance)
606
+ await ChangesetRepository(session, domain_id, writes=settings.writes).insert(
607
+ stored.id, prepared.changeset
608
+ )
609
+ return stored
610
+
611
+
612
+ async def _record_events(
613
+ session: AsyncSession,
614
+ domain_id: str,
615
+ branch: str,
616
+ revision: RevisionRow,
617
+ changeset: Sequence[ChangesetEntry],
618
+ message: str,
619
+ ) -> None:
620
+ """Step 8: audit entry and outbox event."""
621
+ stats = changeset_stats(changeset)
622
+ details: dict[str, Any] = {
623
+ "branch": branch,
624
+ "revision": hex(revision.id),
625
+ "parents": [hex(p) for p in revision.parents],
626
+ "changes": stats.changes,
627
+ "partitions_changed": stats.partitions_changed,
628
+ "message": message,
629
+ }
630
+ await AuditRepository(session).append(
631
+ NewAuditEntry(
632
+ domain_id=domain_id,
633
+ at=revision.created_at,
634
+ actor=revision.author,
635
+ action=COMMIT_ACTION,
636
+ target=f"{branch}@{hex(revision.id)}",
637
+ details=details,
638
+ )
639
+ )
640
+ await OutboxRepository(session).enqueue(
641
+ domain_id, COMMITTED_EVENT, {"domain": domain_id, **details}
642
+ )
643
+
644
+
645
+ async def _create_domain(
646
+ session: AsyncSession,
647
+ domain: Domain,
648
+ author: Author,
649
+ leaf_size: int,
650
+ message: str,
651
+ origin: Origin | None,
652
+ runtime: StoreRuntime | None,
653
+ clock: Clock,
654
+ ) -> CommitResult:
655
+ runtime = runtime or default_runtime()
656
+ origin = origin or Origin(type=DOMAIN_ORIGIN, ref=domain.id)
657
+ created_at = clock()
658
+ params = TreeParams(leaf_size)
659
+ await scope_to_domain(session, domain.id)
660
+ await DomainRepository(session).create(
661
+ DomainRow(
662
+ id=domain.id,
663
+ display_name=domain.display_name or domain.id,
664
+ description=domain.description or None,
665
+ default_branch=domain.default_branch,
666
+ leaf_size=params.leaf_size,
667
+ settings=dict(domain.settings),
668
+ created_at=created_at,
669
+ created_by=author.principal,
670
+ )
671
+ )
672
+ element = _schema_element(domain)
673
+ assert element.encoded is not None
674
+ tree_change = TreeChange(element.partition, element.key, element.encoded.hash)
675
+ settings = runtime.settings
676
+
677
+ def build(sync: Session) -> tuple[bytes, dict[bytes, bytes], list[ChangesetEntry]]:
678
+ store = PgNodeStore(sync, domain.id, cache=runtime.node_cache)
679
+ root = tree_apply(store, None, [tree_change], params)
680
+ entries = changeset_between(store, None, root)
681
+ return root, store.take_pending(), entries
682
+
683
+ root, nodes, entries = await session.run_sync(build)
684
+ own = ProvenanceRow(
685
+ id=new_id(IdPrefix.PROVENANCE),
686
+ origin_type=origin.type,
687
+ origin_ref=origin.model_dump(),
688
+ actor=author.principal,
689
+ at=created_at,
690
+ )
691
+ changeset = [
692
+ ChangesetEntry(e.partition, e.key, e.old_hash, e.new_hash, {ELEMENT_PROVENANCE_KEY: own.id})
693
+ for e in entries
694
+ ]
695
+ schema = Schema(domain=domain)
696
+ prepared = _Prepared(
697
+ root=root,
698
+ changeset=changeset,
699
+ bodies={(element.partition, element.key): element.encoded.body},
700
+ objects=[element.encoded],
701
+ nodes=nodes,
702
+ schema=schema,
703
+ schema_hash=_root_schema_hash(nodes, root),
704
+ )
705
+ revision = Revision(
706
+ root=root,
707
+ domain=domain.id,
708
+ author=author,
709
+ message=message,
710
+ origin=origin,
711
+ created_at=created_at,
712
+ stats=changeset_stats(changeset),
713
+ )
714
+ stored = await _store_revision(session, runtime, revision, prepared, [own])
715
+ await RefRepository(session, domain.id).create(
716
+ domain.default_branch,
717
+ stored.id,
718
+ created_at=created_at,
719
+ created_by=author.principal,
720
+ kind=RefKind.BRANCH,
721
+ materialization=Materialization.FULL,
722
+ protected=True,
723
+ reason=DOMAIN_ORIGIN,
724
+ )
725
+ await apply_changeset(
726
+ session,
727
+ domain.id,
728
+ domain.default_branch,
729
+ revision_id=stored.id,
730
+ at=stored.created_at,
731
+ changes=_materialized_changes(prepared),
732
+ schema=schema,
733
+ schema_hash=prepared.schema_hash,
734
+ settings=settings,
735
+ runtime=runtime,
736
+ )
737
+ await _record_events(session, domain.id, domain.default_branch, stored, changeset, message)
738
+ return CommitResult(
739
+ revision_id=stored.id,
740
+ parents=(),
741
+ root=root,
742
+ generation=stored.generation,
743
+ ref_version=1,
744
+ created_at=stored.created_at,
745
+ changeset=tuple(changeset),
746
+ )
747
+
748
+
749
+ def _root_schema_hash(nodes: Mapping[bytes, bytes], root: bytes) -> bytes | None:
750
+ """Schema partition hash of a tree whose root node is among `nodes`."""
751
+ store = InMemoryNodeStore()
752
+ store.put(root, nodes[root])
753
+ ref = Tree.load(store, root).partitions.get(PARTITION_SCHEMA)
754
+ return None if ref is None else ref.root