lorebase-store 0.2.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. lorebase_store/__init__.py +1 -0
  2. lorebase_store/_migrations/env.py +106 -0
  3. lorebase_store/_migrations/script.py.mako +27 -0
  4. lorebase_store/_migrations/versions/0001_initial_schema.py +1071 -0
  5. lorebase_store/_migrations/versions/0002_merge_attempt_details.py +74 -0
  6. lorebase_store/_migrations/versions/0003_webhooks.py +127 -0
  7. lorebase_store/_migrations/versions/0004_defer_revision_references.py +38 -0
  8. lorebase_store/_migrations/versions/0005_security_definer_reads.py +396 -0
  9. lorebase_store/_migrations/versions/0006_api_key_usage.py +48 -0
  10. lorebase_store/_migrations/versions/0007_document_extraction.py +154 -0
  11. lorebase_store/_migrations/versions/0008_definer_argument_checks.py +178 -0
  12. lorebase_store/_migrations/versions/0009_api_key_networks.py +33 -0
  13. lorebase_store/_migrations/versions/0010_overlay_branches.py +582 -0
  14. lorebase_store/_migrations/versions/0011_branch_scoped_search_index.py +229 -0
  15. lorebase_store/_migrations/versions/0012_overlay_neighborhood_lookups.py +185 -0
  16. lorebase_store/_migrations/versions/0013_web_sessions.py +72 -0
  17. lorebase_store/_migrations/versions/0014_embedding_jobs.py +70 -0
  18. lorebase_store/_migrations/versions/0015_append_only_audit_log.py +83 -0
  19. lorebase_store/_migrations/versions/0016_sealed_webhook_keys.py +55 -0
  20. lorebase_store/_migrations/versions/0017_background_branch_copies.py +77 -0
  21. lorebase_store/_migrations/versions/0018_search_lexicon.py +477 -0
  22. lorebase_store/_migrations/versions/0019_vault_bindings.py +56 -0
  23. lorebase_store/_migrations/versions/0020_outbox_consumers.py +150 -0
  24. lorebase_store/_migrations/versions/0021_schema_suggestions.py +114 -0
  25. lorebase_store/_migrations/versions/0022_read_audit_and_shared_rate_limits.py +133 -0
  26. lorebase_store/_migrations/versions/0023_agent_trace.py +96 -0
  27. lorebase_store/_migrations/versions/0024_embedding_template_keys.py +205 -0
  28. lorebase_store/_migrations/versions/0025_snapshot_refs.py +35 -0
  29. lorebase_store/_migrations/versions/0026_plain_reverse_link_index.py +112 -0
  30. lorebase_store/_migrations/versions/0027_vault_rotation_and_tombstones.py +145 -0
  31. lorebase_store/_migrations/versions/0028_operation_deferred_events.py +71 -0
  32. lorebase_store/_migrations/versions/0029_agent_trace_sessions.py +108 -0
  33. lorebase_store/_migrations/versions/0030_path_fanout.py +31 -0
  34. lorebase_store/_migrations/versions/0031_branch_builds.py +91 -0
  35. lorebase_store/_migrations/versions/0032_lineage_revisions.py +35 -0
  36. lorebase_store/_migrations/versions/0033_vault_moved_tokens.py +30 -0
  37. lorebase_store/_migrations/versions/0034_inference_endpoint_modes.py +73 -0
  38. lorebase_store/_migrations/versions/0035_search_truncation.py +104 -0
  39. lorebase_store/_migrations/versions/0036_lineage_maps.py +325 -0
  40. lorebase_store/_migrations/versions/0037_bm25_search.py +346 -0
  41. lorebase_store/auth/__init__.py +52 -0
  42. lorebase_store/auth/api_keys.py +161 -0
  43. lorebase_store/auth/audit.py +382 -0
  44. lorebase_store/auth/checkpoints.py +185 -0
  45. lorebase_store/auth/grants.py +161 -0
  46. lorebase_store/auth/principals.py +158 -0
  47. lorebase_store/auth/scope.py +70 -0
  48. lorebase_store/auth/sessions.py +153 -0
  49. lorebase_store/auth/unit.py +53 -0
  50. lorebase_store/auth/usage.py +70 -0
  51. lorebase_store/blobs.py +73 -0
  52. lorebase_store/bm25.py +346 -0
  53. lorebase_store/bulk.py +147 -0
  54. lorebase_store/bundle.py +1830 -0
  55. lorebase_store/changefeed.py +261 -0
  56. lorebase_store/commit/__init__.py +42 -0
  57. lorebase_store/commit/changes.py +67 -0
  58. lorebase_store/commit/errors.py +60 -0
  59. lorebase_store/commit/migrations.py +335 -0
  60. lorebase_store/commit/pipeline.py +754 -0
  61. lorebase_store/commit/schema.py +123 -0
  62. lorebase_store/db.py +58 -0
  63. lorebase_store/definer.py +50 -0
  64. lorebase_store/digests.py +99 -0
  65. lorebase_store/embeddings.py +444 -0
  66. lorebase_store/endpoint_modes.py +59 -0
  67. lorebase_store/extraction/__init__.py +61 -0
  68. lorebase_store/extraction/items.py +187 -0
  69. lorebase_store/extraction/materialize.py +364 -0
  70. lorebase_store/extraction/proposal.py +65 -0
  71. lorebase_store/extraction/review.py +168 -0
  72. lorebase_store/gc.py +970 -0
  73. lorebase_store/indexes.py +1033 -0
  74. lorebase_store/jobs.py +651 -0
  75. lorebase_store/keyring.py +72 -0
  76. lorebase_store/layout.py +335 -0
  77. lorebase_store/materialize/__init__.py +103 -0
  78. lorebase_store/materialize/apply.py +436 -0
  79. lorebase_store/materialize/background.py +1184 -0
  80. lorebase_store/materialize/branch.py +268 -0
  81. lorebase_store/materialize/build.py +536 -0
  82. lorebase_store/materialize/derived.py +540 -0
  83. lorebase_store/materialize/lineage_backfill.py +502 -0
  84. lorebase_store/materialize/links.py +266 -0
  85. lorebase_store/materialize/overlay.py +292 -0
  86. lorebase_store/materialize/paths.py +315 -0
  87. lorebase_store/materialize/rebuild.py +326 -0
  88. lorebase_store/materialize/rows.py +153 -0
  89. lorebase_store/materialize/sensitivity.py +66 -0
  90. lorebase_store/materialize/snapshot.py +205 -0
  91. lorebase_store/materialize/stale.py +444 -0
  92. lorebase_store/materialize/verify.py +321 -0
  93. lorebase_store/merging/__init__.py +39 -0
  94. lorebase_store/merging/attempts.py +334 -0
  95. lorebase_store/merging/lineage.py +123 -0
  96. lorebase_store/merging/service.py +652 -0
  97. lorebase_store/migrations.py +133 -0
  98. lorebase_store/models/__init__.py +136 -0
  99. lorebase_store/models/base.py +64 -0
  100. lorebase_store/models/enums.py +132 -0
  101. lorebase_store/models/governance.py +268 -0
  102. lorebase_store/models/ingestion.py +288 -0
  103. lorebase_store/models/materialized.py +277 -0
  104. lorebase_store/models/proposals.py +195 -0
  105. lorebase_store/models/queues.py +87 -0
  106. lorebase_store/models/tracing.py +116 -0
  107. lorebase_store/models/versioning.py +222 -0
  108. lorebase_store/models/webhooks.py +96 -0
  109. lorebase_store/nodestore.py +384 -0
  110. lorebase_store/objectstore.py +162 -0
  111. lorebase_store/outbox_events.py +166 -0
  112. lorebase_store/profiling/__init__.py +58 -0
  113. lorebase_store/profiling/delimited.py +109 -0
  114. lorebase_store/profiling/postgres.py +211 -0
  115. lorebase_store/profiling/record_files.py +127 -0
  116. lorebase_store/profiling/records.py +92 -0
  117. lorebase_store/profiling/sql_databases.py +298 -0
  118. lorebase_store/ratelimit.py +129 -0
  119. lorebase_store/reads.py +1491 -0
  120. lorebase_store/replicas.py +434 -0
  121. lorebase_store/repositories/__init__.py +152 -0
  122. lorebase_store/repositories/changesets.py +59 -0
  123. lorebase_store/repositories/deferred_events.py +142 -0
  124. lorebase_store/repositories/documents.py +305 -0
  125. lorebase_store/repositories/domains.py +97 -0
  126. lorebase_store/repositories/events.py +205 -0
  127. lorebase_store/repositories/index_view.py +489 -0
  128. lorebase_store/repositories/invocations.py +68 -0
  129. lorebase_store/repositories/materializations.py +276 -0
  130. lorebase_store/repositories/materialized.py +1129 -0
  131. lorebase_store/repositories/merges.py +417 -0
  132. lorebase_store/repositories/proposal_items.py +199 -0
  133. lorebase_store/repositories/proposals.py +184 -0
  134. lorebase_store/repositories/provenance.py +77 -0
  135. lorebase_store/repositories/refs.py +452 -0
  136. lorebase_store/repositories/revisions.py +273 -0
  137. lorebase_store/repositories/scope.py +33 -0
  138. lorebase_store/repositories/source_runs.py +123 -0
  139. lorebase_store/repositories/suggestions.py +638 -0
  140. lorebase_store/runtime.py +87 -0
  141. lorebase_store/schema_cache.py +50 -0
  142. lorebase_store/settings.py +879 -0
  143. lorebase_store/storage.py +1478 -0
  144. lorebase_store/telemetry.py +142 -0
  145. lorebase_store/testing.py +108 -0
  146. lorebase_store/tracing.py +505 -0
  147. lorebase_store/traverse.py +578 -0
  148. lorebase_store/vault/__init__.py +50 -0
  149. lorebase_store/vault/keys.py +216 -0
  150. lorebase_store/vault/quotes.py +133 -0
  151. lorebase_store/vault/rotation.py +209 -0
  152. lorebase_store/vault/settings.py +152 -0
  153. lorebase_store/vault/store.py +987 -0
  154. lorebase_store/vault/transfer.py +558 -0
  155. lorebase_store/webhooks/__init__.py +24 -0
  156. lorebase_store/webhooks/delivery.py +637 -0
  157. lorebase_store/webhooks/keys.py +126 -0
  158. lorebase_store/webhooks/repository.py +142 -0
  159. lorebase_store/webhooks/settings.py +138 -0
  160. lorebase_store/webhooks/signing.py +50 -0
  161. lorebase_store-0.2.3.dist-info/METADATA +13 -0
  162. lorebase_store-0.2.3.dist-info/RECORD +163 -0
  163. lorebase_store-0.2.3.dist-info/WHEEL +4 -0
@@ -0,0 +1,222 @@
1
+ """Versioning core (design 7.1): Domains, objects, tree nodes, Revisions, changesets, refs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from datetime import datetime
6
+ from typing import Any
7
+
8
+ from sqlalchemy import (
9
+ BigInteger,
10
+ CheckConstraint,
11
+ ForeignKeyConstraint,
12
+ Index,
13
+ Integer,
14
+ SmallInteger,
15
+ text,
16
+ )
17
+ from sqlalchemy.dialects.postgresql import JSONB
18
+ from sqlalchemy.orm import Mapped, mapped_column
19
+
20
+ from lorebase_store.models.base import (
21
+ H32,
22
+ HASH_PARTITIONED,
23
+ Base,
24
+ DomainKey,
25
+ domain_fk,
26
+ h32_check,
27
+ )
28
+ from lorebase_store.models.enums import MATERIALIZATION, REF_KIND, Materialization, RefKind
29
+
30
+
31
+ class Domain(Base):
32
+ """A Domain: the isolation boundary for schema, data, grants and history."""
33
+
34
+ __tablename__ = "domains"
35
+
36
+ id: Mapped[str] = mapped_column(primary_key=True)
37
+ display_name: Mapped[str]
38
+ description: Mapped[str | None]
39
+ default_branch: Mapped[str] = mapped_column(server_default=text("'main'"))
40
+ leaf_size: Mapped[int] = mapped_column(Integer)
41
+ settings: Mapped[dict[str, Any]] = mapped_column(server_default=text("'{}'::jsonb"))
42
+ created_at: Mapped[datetime]
43
+ created_by: Mapped[str]
44
+
45
+
46
+ class StoredObject(Base):
47
+ """Content-addressed schema element, Record or Link, stored once per Domain."""
48
+
49
+ __tablename__ = "objects"
50
+ __table_args__ = (h32_check("hash"), domain_fk(), HASH_PARTITIONED)
51
+
52
+ domain_id: Mapped[DomainKey]
53
+ hash: Mapped[H32] = mapped_column(primary_key=True)
54
+ object_type: Mapped[int] = mapped_column(SmallInteger)
55
+ body: Mapped[dict[str, Any]]
56
+ size: Mapped[int] = mapped_column(Integer)
57
+
58
+
59
+ class TreeNode(Base):
60
+ """Binary HAMT node; `body` uses lz4 TOAST compression (set by migration DDL)."""
61
+
62
+ __tablename__ = "tree_nodes"
63
+ __table_args__ = (h32_check("hash"), domain_fk(), HASH_PARTITIONED)
64
+
65
+ domain_id: Mapped[DomainKey]
66
+ hash: Mapped[H32] = mapped_column(primary_key=True)
67
+ body: Mapped[bytes]
68
+
69
+
70
+ class Revision(Base):
71
+ """Immutable Revision with its generation number (design 6.4)."""
72
+
73
+ __tablename__ = "revisions"
74
+ __table_args__ = (h32_check("id"), h32_check("root"), domain_fk())
75
+
76
+ domain_id: Mapped[DomainKey]
77
+ id: Mapped[H32] = mapped_column(primary_key=True)
78
+ root: Mapped[H32]
79
+ generation: Mapped[int] = mapped_column(BigInteger)
80
+ author: Mapped[str]
81
+ on_behalf_of: Mapped[str | None]
82
+ message: Mapped[str]
83
+ origin: Mapped[dict[str, Any]]
84
+ stats: Mapped[dict[str, Any]]
85
+ created_at: Mapped[datetime]
86
+
87
+
88
+ class RevisionParent(Base):
89
+ """Ordered parent edge of a Revision; position 0 is the first parent."""
90
+
91
+ __tablename__ = "revision_parents"
92
+ __table_args__ = (
93
+ ForeignKeyConstraint(
94
+ ["domain_id", "revision_id"],
95
+ ["revisions.domain_id", "revisions.id"],
96
+ ondelete="CASCADE",
97
+ ),
98
+ ForeignKeyConstraint(
99
+ ["domain_id", "parent_id"],
100
+ ["revisions.domain_id", "revisions.id"],
101
+ deferrable=True,
102
+ initially="DEFERRED",
103
+ ),
104
+ Index(None, "domain_id", "parent_id"),
105
+ )
106
+
107
+ domain_id: Mapped[DomainKey]
108
+ revision_id: Mapped[H32] = mapped_column(primary_key=True)
109
+ position: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
110
+ parent_id: Mapped[H32]
111
+
112
+
113
+ class ChangesetEntry(Base):
114
+ """One element change of a Revision against its first parent (design 6.3)."""
115
+
116
+ __tablename__ = "changesets"
117
+ __table_args__ = (
118
+ ForeignKeyConstraint(
119
+ ["domain_id", "revision_id"],
120
+ ["revisions.domain_id", "revisions.id"],
121
+ ondelete="CASCADE",
122
+ ),
123
+ )
124
+
125
+ domain_id: Mapped[DomainKey]
126
+ revision_id: Mapped[H32] = mapped_column(primary_key=True)
127
+ partition: Mapped[str] = mapped_column(primary_key=True)
128
+ element_key: Mapped[str] = mapped_column(primary_key=True)
129
+ old_hash: Mapped[H32 | None]
130
+ new_hash: Mapped[H32 | None]
131
+ provenance: Mapped[dict[str, Any] | None]
132
+
133
+
134
+ class Provenance(Base):
135
+ """Where a value came from: origin, actor and time."""
136
+
137
+ __tablename__ = "provenance"
138
+ __table_args__ = (domain_fk(),)
139
+
140
+ domain_id: Mapped[DomainKey]
141
+ id: Mapped[str] = mapped_column(primary_key=True)
142
+ origin_type: Mapped[str]
143
+ origin_ref: Mapped[dict[str, Any]] = mapped_column(JSONB)
144
+ actor: Mapped[str]
145
+ at: Mapped[datetime]
146
+
147
+
148
+ class Ref(Base):
149
+ """Branch or tag head; moved by compare-and-swap on `(head, version)`.
150
+
151
+ An overlay branch (design 6.11) names its base branch and the base head it was forked
152
+ from (`base_revision`, with its generation): it stores only the rows it changed and reads
153
+ the base rows as of that point. The three columns are set exactly for overlays.
154
+
155
+ `deleted_at` marks a branch whose deletion is in progress (revision 0017): it is hidden
156
+ from every read and write while a `store.branch_delete` job removes its rows in batches,
157
+ and its name stays taken until the job deletes the ref.
158
+ """
159
+
160
+ __tablename__ = "refs"
161
+ __table_args__ = (
162
+ ForeignKeyConstraint(
163
+ ["domain_id", "head"],
164
+ ["revisions.domain_id", "revisions.id"],
165
+ deferrable=True,
166
+ initially="DEFERRED",
167
+ ),
168
+ ForeignKeyConstraint(
169
+ ["domain_id", "base_branch"],
170
+ ["refs.domain_id", "refs.name"],
171
+ deferrable=True,
172
+ initially="DEFERRED",
173
+ ),
174
+ ForeignKeyConstraint(
175
+ ["domain_id", "base_revision"],
176
+ ["revisions.domain_id", "revisions.id"],
177
+ deferrable=True,
178
+ initially="DEFERRED",
179
+ ),
180
+ CheckConstraint(
181
+ "(materialization = 'overlay') = (base_branch IS NOT NULL "
182
+ "AND base_revision IS NOT NULL AND base_generation IS NOT NULL)",
183
+ name="overlay_base",
184
+ ),
185
+ Index(
186
+ "ix_refs_domain_id_base_branch",
187
+ "domain_id",
188
+ "base_branch",
189
+ postgresql_where=text("base_branch IS NOT NULL"),
190
+ ),
191
+ domain_fk(),
192
+ )
193
+
194
+ domain_id: Mapped[DomainKey]
195
+ name: Mapped[str] = mapped_column(primary_key=True)
196
+ kind: Mapped[RefKind] = mapped_column(REF_KIND)
197
+ head: Mapped[H32]
198
+ protected: Mapped[bool] = mapped_column(server_default=text("false"))
199
+ materialization: Mapped[Materialization] = mapped_column(MATERIALIZATION)
200
+ version: Mapped[int] = mapped_column(BigInteger)
201
+ created_at: Mapped[datetime]
202
+ created_by: Mapped[str]
203
+ base_branch: Mapped[str | None]
204
+ base_revision: Mapped[H32 | None]
205
+ base_generation: Mapped[int | None] = mapped_column(BigInteger)
206
+ deleted_at: Mapped[datetime | None]
207
+
208
+
209
+ class RefLogEntry(Base):
210
+ """Every head movement of a ref, kept after the ref is deleted."""
211
+
212
+ __tablename__ = "ref_log"
213
+ __table_args__ = (domain_fk(),)
214
+
215
+ domain_id: Mapped[DomainKey]
216
+ ref_name: Mapped[str] = mapped_column(primary_key=True)
217
+ seq: Mapped[int] = mapped_column(BigInteger, primary_key=True)
218
+ old_head: Mapped[H32 | None]
219
+ new_head: Mapped[H32]
220
+ reason: Mapped[str]
221
+ actor: Mapped[str]
222
+ at: Mapped[datetime]
@@ -0,0 +1,96 @@
1
+ """Webhooks of a Domain and their deliveries (design 10.5).
2
+
3
+ A webhook receives the outbox events of its Domain (optionally narrowed to event types and
4
+ branches) as HMAC-signed POST requests. Only a SHA-256 digest of its secret is kept; the
5
+ digest is the signing key (`lorebase_store.webhooks.signing`), stored sealed in `sealed_key`
6
+ when `webhooks.encryption_key` is set, else usable in `secret_hash` (`webhooks.keys`).
7
+ Deliveries are fanned out from the outbox and retried with backoff until delivered or dead.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from datetime import datetime
13
+ from typing import Any
14
+
15
+ from sqlalchemy import (
16
+ BigInteger,
17
+ CheckConstraint,
18
+ ForeignKeyConstraint,
19
+ Identity,
20
+ Index,
21
+ Integer,
22
+ UniqueConstraint,
23
+ text,
24
+ )
25
+ from sqlalchemy.dialects.postgresql import JSONB
26
+ from sqlalchemy.orm import Mapped, mapped_column
27
+
28
+ from lorebase_store.models.base import H32, Base, DomainKey, domain_fk, h32_check
29
+
30
+ DELIVERY_STATUSES = ("pending", "delivered", "dead")
31
+
32
+
33
+ class Webhook(Base):
34
+ """An HTTP endpoint notified of the events of a Domain."""
35
+
36
+ __tablename__ = "webhooks"
37
+ __table_args__ = (
38
+ domain_fk(),
39
+ h32_check("secret_hash"),
40
+ CheckConstraint("(secret_hash IS NULL) <> (sealed_key IS NULL)", name="one_signing_key"),
41
+ )
42
+
43
+ domain_id: Mapped[DomainKey]
44
+ id: Mapped[str] = mapped_column(primary_key=True)
45
+ url: Mapped[str]
46
+ description: Mapped[str | None]
47
+ event_types: Mapped[Any] = mapped_column(JSONB)
48
+ """Event types delivered (a JSON array of strings)."""
49
+ branches: Mapped[Any | None] = mapped_column(JSONB)
50
+ """Branch names whose events are delivered (a JSON array); null for every branch."""
51
+ clearances: Mapped[Any] = mapped_column(JSONB, server_default=text("'[]'::jsonb"))
52
+ """Sensitivity tags whose values the webhook receives in clear (a JSON array); values of
53
+ other sensitive Attributes reach it masked."""
54
+ secret_hash: Mapped[H32 | None]
55
+ """The signing key, stored usable (no `webhooks.encryption_key` when it was stored)."""
56
+ sealed_key: Mapped[bytes | None]
57
+ """The signing key sealed under `webhooks.encryption_key` (nonce and AES-GCM ciphertext)."""
58
+ created_by: Mapped[str]
59
+ created_at: Mapped[datetime] = mapped_column(server_default=text("now()"))
60
+
61
+
62
+ class WebhookDelivery(Base):
63
+ """One outbox event to deliver to one webhook, with its attempts."""
64
+
65
+ __tablename__ = "webhook_deliveries"
66
+ __table_args__ = (
67
+ ForeignKeyConstraint(
68
+ ["domain_id", "webhook_id"],
69
+ ["webhooks.domain_id", "webhooks.id"],
70
+ ondelete="CASCADE",
71
+ ),
72
+ UniqueConstraint("domain_id", "webhook_id", "event_id"),
73
+ CheckConstraint(
74
+ "status IN (" + ", ".join(f"'{s}'" for s in DELIVERY_STATUSES) + ")", name="status"
75
+ ),
76
+ Index(
77
+ "ix_webhook_deliveries_due",
78
+ "next_attempt_at",
79
+ postgresql_where=text("status = 'pending'"),
80
+ ),
81
+ )
82
+
83
+ domain_id: Mapped[DomainKey]
84
+ id: Mapped[int] = mapped_column(BigInteger, Identity(always=True), primary_key=True)
85
+ webhook_id: Mapped[str]
86
+ event_id: Mapped[int] = mapped_column(BigInteger)
87
+ event_type: Mapped[str]
88
+ payload: Mapped[dict[str, Any]]
89
+ status: Mapped[str] = mapped_column(server_default=text("'pending'"))
90
+ attempts: Mapped[int] = mapped_column(Integer, server_default=text("0"))
91
+ next_attempt_at: Mapped[datetime] = mapped_column(server_default=text("now()"))
92
+ last_status: Mapped[int | None] = mapped_column(Integer)
93
+ last_error: Mapped[str | None]
94
+ delivered_at: Mapped[datetime | None]
95
+ created_at: Mapped[datetime] = mapped_column(server_default=text("now()"))
96
+ updated_at: Mapped[datetime] = mapped_column(server_default=text("now()"))
@@ -0,0 +1,384 @@
1
+ """Tree nodes in Postgres (`tree_nodes`), scoped to one Domain.
2
+
3
+ Two access paths share one process-wide cache:
4
+
5
+ - `PgNodeStore` implements the synchronous core `NodeStore` protocol over a synchronous
6
+ `Session`. It is meant to run inside `AsyncSession.run_sync`, where SQLAlchemy drives the
7
+ async driver from synchronous code. Callers prefetch the nodes a tree operation will read
8
+ (`prefetch_paths` for a batch of keys) so that the operation reads from memory; a miss falls
9
+ back to a single-node query. Writes are buffered and handed to the async side in bulk.
10
+ - `NodeRepository` is the async side: batched reads, bulk writes with `ON CONFLICT DO NOTHING`,
11
+ and streaming scans of whole partitions.
12
+
13
+ Nodes are immutable and content-addressed, so cached bodies never go stale. The cache only
14
+ holds bodies read from the database; `has()` never answers from it, because a node written by
15
+ a transaction that later rolled back may still be cached.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import threading
21
+ from collections import OrderedDict, defaultdict
22
+ from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
23
+ from dataclasses import dataclass
24
+
25
+ from sqlalchemy import Table, select
26
+ from sqlalchemy.ext.asyncio import AsyncSession
27
+ from sqlalchemy.orm import Session
28
+
29
+ from lorebase_core.canonical import node_hash
30
+ from lorebase_core.tree import (
31
+ BranchNode,
32
+ LeafNode,
33
+ MissingNodeError,
34
+ NodeType,
35
+ Tree,
36
+ decode_node,
37
+ key_hash,
38
+ )
39
+ from lorebase_store.bulk import bulk_write, chunks
40
+ from lorebase_store.models import metadata
41
+ from lorebase_store.settings import NodeSettings, WriteSettings
42
+
43
+ TREE_NODES: Table = metadata.tables["tree_nodes"]
44
+
45
+ _NODE_TYPE_OFFSET = 1
46
+ """Byte offset of the node type in the node header (after the format version)."""
47
+
48
+
49
+ def node_type(body: bytes) -> NodeType:
50
+ """Type of an encoded node, read from its header without decoding the body."""
51
+ return NodeType(body[_NODE_TYPE_OFFSET])
52
+
53
+
54
+ class NodeCache:
55
+ """Thread-safe LRU of node bodies keyed by `(domain_id, hash)`, bounded in bytes."""
56
+
57
+ def __init__(self, max_bytes: int) -> None:
58
+ if max_bytes < 0:
59
+ raise ValueError("max_bytes must not be negative")
60
+ self._max_bytes = max_bytes
61
+ self._bytes = 0
62
+ self._entries: OrderedDict[tuple[str, bytes], bytes] = OrderedDict()
63
+ self._lock = threading.Lock()
64
+
65
+ @classmethod
66
+ def from_settings(cls, settings: NodeSettings) -> NodeCache:
67
+ return cls(settings.cache_bytes)
68
+
69
+ @property
70
+ def size_bytes(self) -> int:
71
+ return self._bytes
72
+
73
+ def __len__(self) -> int:
74
+ return len(self._entries)
75
+
76
+ def get(self, domain_id: str, digest: bytes) -> bytes | None:
77
+ key = (domain_id, digest)
78
+ with self._lock:
79
+ body = self._entries.get(key)
80
+ if body is not None:
81
+ self._entries.move_to_end(key)
82
+ return body
83
+
84
+ def put(self, domain_id: str, digest: bytes, body: bytes) -> None:
85
+ if len(body) > self._max_bytes:
86
+ return
87
+ key = (domain_id, digest)
88
+ with self._lock:
89
+ if key in self._entries:
90
+ self._entries.move_to_end(key)
91
+ return
92
+ self._entries[key] = body
93
+ self._bytes += len(body)
94
+ while self._bytes > self._max_bytes:
95
+ _, evicted = self._entries.popitem(last=False)
96
+ self._bytes -= len(evicted)
97
+
98
+
99
+ @dataclass
100
+ class NodeReadStats:
101
+ """Counters of a `PgNodeStore`, used to check that reads are batched."""
102
+
103
+ queries: int = 0
104
+ fetched: int = 0
105
+ misses: int = 0
106
+
107
+
108
+ class PgNodeStore:
109
+ """Synchronous `NodeStore` for one Domain over a synchronous `Session` (see module doc)."""
110
+
111
+ def __init__(
112
+ self,
113
+ session: Session,
114
+ domain_id: str,
115
+ *,
116
+ cache: NodeCache | None = None,
117
+ prefetch_batch: int = 1000,
118
+ ) -> None:
119
+ self._session = session
120
+ self._domain = domain_id
121
+ self._cache = cache
122
+ self._batch = prefetch_batch
123
+ self._local: dict[bytes, bytes] = {}
124
+ self._pending: dict[bytes, bytes] = {}
125
+ self.stats = NodeReadStats()
126
+
127
+ @property
128
+ def domain_id(self) -> str:
129
+ return self._domain
130
+
131
+ # NodeStore protocol ---------------------------------------------------------------
132
+
133
+ def get(self, node_hash: bytes) -> bytes | None:
134
+ body = self._known(node_hash)
135
+ if body is None:
136
+ self.stats.misses += 1
137
+ self.prefetch([node_hash])
138
+ body = self._local.get(node_hash)
139
+ return body
140
+
141
+ def put(self, node_hash: bytes, body: bytes) -> None:
142
+ _verify(node_hash, body)
143
+ self._pending.setdefault(node_hash, body)
144
+
145
+ def has(self, node_hash: bytes) -> bool:
146
+ if node_hash in self._pending or node_hash in self._local:
147
+ return True
148
+ self.prefetch([node_hash], use_cache=False)
149
+ return node_hash in self._local
150
+
151
+ # Batched reads ------------------------------------------------------------------
152
+
153
+ def prefetch(self, hashes: Iterable[bytes], *, use_cache: bool = True) -> None:
154
+ """Load the given nodes in batched queries; already known nodes are skipped."""
155
+ missing: list[bytes] = []
156
+ for digest in dict.fromkeys(hashes):
157
+ if digest in self._pending or digest in self._local:
158
+ continue
159
+ cached = self._cache.get(self._domain, digest) if use_cache and self._cache else None
160
+ if cached is not None:
161
+ self._local[digest] = cached
162
+ else:
163
+ missing.append(digest)
164
+ for chunk in chunks(missing, self._batch):
165
+ rows = self._session.execute(
166
+ select(TREE_NODES.c.hash, TREE_NODES.c.body).where(
167
+ TREE_NODES.c.domain_id == self._domain, TREE_NODES.c.hash.in_(chunk)
168
+ )
169
+ )
170
+ self.stats.queries += 1
171
+ for digest, body in rows:
172
+ self._local[digest] = body
173
+ self.stats.fetched += 1
174
+ if self._cache is not None:
175
+ self._cache.put(self._domain, digest, body)
176
+
177
+ def prefetch_paths(self, root: bytes | None, keys: Iterable[tuple[str, str]]) -> None:
178
+ """Load the nodes on the paths from `root` to the given `(partition, key)` entries.
179
+
180
+ One query per tree level, so a batch update over the prefetched paths reads no node
181
+ individually (except when a branch collapses into a leaf and siblings are read).
182
+ """
183
+ if root is None:
184
+ return
185
+ self.prefetch([root])
186
+ if self._known(root) is None:
187
+ raise MissingNodeError(root.hex())
188
+ partitions = Tree.load(self, root).partitions
189
+ wanted: dict[str, list[bytes]] = defaultdict(list)
190
+ for partition, key in keys:
191
+ if partition in partitions:
192
+ wanted[partition].append(key_hash(key))
193
+ frontier = [(partitions[p].root, 0, hashes) for p, hashes in wanted.items()]
194
+ while frontier:
195
+ self.prefetch(digest for digest, _, _ in frontier)
196
+ following: list[tuple[bytes, int, list[bytes]]] = []
197
+ for digest, depth, hashes in frontier:
198
+ body = self._known(digest)
199
+ if body is None:
200
+ raise MissingNodeError(digest.hex())
201
+ if node_type(body) is not NodeType.BRANCH:
202
+ continue
203
+ node = decode_node(body)
204
+ assert isinstance(node, BranchNode)
205
+ children = dict(node.children)
206
+ groups: dict[int, list[bytes]] = defaultdict(list)
207
+ for placement in hashes:
208
+ groups[placement[depth]].append(placement)
209
+ for index, group in groups.items():
210
+ child = children.get(index)
211
+ if child is not None:
212
+ following.append((child, depth + 1, group))
213
+ frontier = following
214
+
215
+ # Buffered writes ------------------------------------------------------------------
216
+
217
+ @property
218
+ def pending(self) -> Mapping[bytes, bytes]:
219
+ """Nodes written through this store and not yet persisted."""
220
+ return self._pending
221
+
222
+ def take_pending(self) -> dict[bytes, bytes]:
223
+ """Return and forget the buffered writes, to persist with `NodeRepository.put_many`."""
224
+ pending, self._pending = self._pending, {}
225
+ self._local.update(pending)
226
+ return pending
227
+
228
+ def _known(self, digest: bytes) -> bytes | None:
229
+ body = self._pending.get(digest) or self._local.get(digest)
230
+ if body is None and self._cache is not None:
231
+ body = self._cache.get(self._domain, digest)
232
+ if body is not None:
233
+ self._local[digest] = body
234
+ return body
235
+
236
+
237
+ class NodeRepository:
238
+ """Async batched reads and bulk writes of one Domain's tree nodes."""
239
+
240
+ def __init__(
241
+ self,
242
+ session: AsyncSession,
243
+ domain_id: str,
244
+ *,
245
+ writes: WriteSettings,
246
+ cache: NodeCache | None = None,
247
+ batch: int = 1000,
248
+ ) -> None:
249
+ self._session = session
250
+ self._domain = domain_id
251
+ self._writes = writes
252
+ self._cache = cache
253
+ self._batch = batch
254
+
255
+ async def get_many(
256
+ self, hashes: Iterable[bytes], *, use_cache: bool = True
257
+ ) -> dict[bytes, bytes]:
258
+ """Bodies of the given nodes that exist."""
259
+ found: dict[bytes, bytes] = {}
260
+ missing: list[bytes] = []
261
+ for digest in dict.fromkeys(hashes):
262
+ cached = self._cache.get(self._domain, digest) if use_cache and self._cache else None
263
+ if cached is None:
264
+ missing.append(digest)
265
+ else:
266
+ found[digest] = cached
267
+ for chunk in chunks(missing, self._batch):
268
+ rows = await self._session.execute(
269
+ select(TREE_NODES.c.hash, TREE_NODES.c.body).where(
270
+ TREE_NODES.c.domain_id == self._domain, TREE_NODES.c.hash.in_(chunk)
271
+ )
272
+ )
273
+ for digest, body in rows:
274
+ found[digest] = body
275
+ if use_cache and self._cache is not None:
276
+ self._cache.put(self._domain, digest, body)
277
+ return found
278
+
279
+ async def put_many(self, nodes: Mapping[bytes, bytes]) -> None:
280
+ """Persist nodes; nodes already stored are left unchanged."""
281
+ rows = [
282
+ {"domain_id": self._domain, "hash": digest, "body": body}
283
+ for digest, body in nodes.items()
284
+ ]
285
+ await bulk_write(self._session, TREE_NODES, rows, writes=self._writes)
286
+
287
+ async def load_tree(self, root: bytes) -> Tree:
288
+ """The partition map of the tree rooted at `root`."""
289
+ body = (await self.get_many([root])).get(root)
290
+ if body is None:
291
+ raise MissingNodeError(root.hex())
292
+ return Tree.load(_SingleNode(root, body), root)
293
+
294
+ async def scan(self, node: bytes) -> AsyncIterator[list[tuple[str, bytes]]]:
295
+ """Stream the `(element_key, object_hash)` entries of a partition subtree.
296
+
297
+ Reads one tree level per round in batched queries and yields the entries of each
298
+ batch of leaves; memory stays bounded by one batch plus the branch frontier. Scanned
299
+ nodes do not enter the cache.
300
+ """
301
+ frontier: list[bytes] = [node]
302
+ while frontier:
303
+ following: list[bytes] = []
304
+ for chunk in chunks(frontier, self._batch):
305
+ bodies = await self.get_many(chunk, use_cache=False)
306
+ entries: list[tuple[str, bytes]] = []
307
+ for digest in chunk:
308
+ body = bodies.get(digest)
309
+ if body is None:
310
+ raise MissingNodeError(digest.hex())
311
+ decoded = decode_node(body)
312
+ if isinstance(decoded, LeafNode):
313
+ entries.extend(decoded.entries)
314
+ elif isinstance(decoded, BranchNode):
315
+ following.extend(child for _, child in decoded.children)
316
+ if entries:
317
+ yield entries
318
+ frontier = following
319
+
320
+ async def scan_from(
321
+ self, node: bytes, *, after: Sequence[int] | None, limit: int
322
+ ) -> tuple[list[tuple[str, bytes]], list[int] | None]:
323
+ """Entries of a partition subtree, whole leaves at a time, in a fixed depth-first
324
+ order: those of the leaves after the leaf at path `after` (child bytes from `node`;
325
+ `None` starts at the first leaf), until at least `limit` entries. Returns them with
326
+ the path of the last leaf read, or `None` when no leaf is left.
327
+
328
+ The order depends on the subtree only, which never changes, so a caller can resume
329
+ from the returned path in a later transaction. Each branch node's children are read
330
+ in one query; scanned nodes do not enter the cache.
331
+ """
332
+ entries: list[tuple[str, bytes]] = []
333
+ bodies = await self.get_many([node], use_cache=False)
334
+ stack: list[tuple[bytes, list[int], list[int] | None]] = [
335
+ (node, [], None if after is None else list(after))
336
+ ]
337
+ while stack:
338
+ digest, path, bound = stack.pop()
339
+ body = bodies.pop(digest, None)
340
+ if body is None:
341
+ raise MissingNodeError(digest.hex())
342
+ decoded = decode_node(body)
343
+ if isinstance(decoded, LeafNode):
344
+ if bound is not None:
345
+ continue # the leaf at `after` itself: already read
346
+ entries.extend(decoded.entries)
347
+ if len(entries) >= limit:
348
+ return entries, path
349
+ continue
350
+ if not isinstance(decoded, BranchNode):
351
+ continue
352
+ wanted: list[tuple[bytes, list[int], list[int] | None]] = []
353
+ for byte, child in decoded.children:
354
+ if bound is not None and bound and byte < bound[0]:
355
+ continue
356
+ inner = bound[1:] if bound is not None and bound and byte == bound[0] else None
357
+ wanted.append((child, [*path, byte], inner))
358
+ missing = [child for child, _, _ in wanted if child not in bodies]
359
+ if missing:
360
+ bodies.update(await self.get_many(missing, use_cache=False))
361
+ stack.extend(reversed(wanted))
362
+ return entries, None
363
+
364
+
365
+ class _SingleNode:
366
+ """Read-only `NodeStore` over one known node."""
367
+
368
+ def __init__(self, digest: bytes, body: bytes) -> None:
369
+ self._digest = digest
370
+ self._body = body
371
+
372
+ def get(self, node_hash: bytes) -> bytes | None:
373
+ return self._body if node_hash == self._digest else None
374
+
375
+ def put(self, node_hash: bytes, body: bytes) -> None:
376
+ raise TypeError("read-only node store")
377
+
378
+ def has(self, node_hash: bytes) -> bool:
379
+ return node_hash == self._digest
380
+
381
+
382
+ def _verify(expected: bytes, body: bytes) -> None:
383
+ if node_hash(body) != expected:
384
+ raise ValueError("node hash does not match node body")