lorebase-store 0.2.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. lorebase_store/__init__.py +1 -0
  2. lorebase_store/_migrations/env.py +106 -0
  3. lorebase_store/_migrations/script.py.mako +27 -0
  4. lorebase_store/_migrations/versions/0001_initial_schema.py +1071 -0
  5. lorebase_store/_migrations/versions/0002_merge_attempt_details.py +74 -0
  6. lorebase_store/_migrations/versions/0003_webhooks.py +127 -0
  7. lorebase_store/_migrations/versions/0004_defer_revision_references.py +38 -0
  8. lorebase_store/_migrations/versions/0005_security_definer_reads.py +396 -0
  9. lorebase_store/_migrations/versions/0006_api_key_usage.py +48 -0
  10. lorebase_store/_migrations/versions/0007_document_extraction.py +154 -0
  11. lorebase_store/_migrations/versions/0008_definer_argument_checks.py +178 -0
  12. lorebase_store/_migrations/versions/0009_api_key_networks.py +33 -0
  13. lorebase_store/_migrations/versions/0010_overlay_branches.py +582 -0
  14. lorebase_store/_migrations/versions/0011_branch_scoped_search_index.py +229 -0
  15. lorebase_store/_migrations/versions/0012_overlay_neighborhood_lookups.py +185 -0
  16. lorebase_store/_migrations/versions/0013_web_sessions.py +72 -0
  17. lorebase_store/_migrations/versions/0014_embedding_jobs.py +70 -0
  18. lorebase_store/_migrations/versions/0015_append_only_audit_log.py +83 -0
  19. lorebase_store/_migrations/versions/0016_sealed_webhook_keys.py +55 -0
  20. lorebase_store/_migrations/versions/0017_background_branch_copies.py +77 -0
  21. lorebase_store/_migrations/versions/0018_search_lexicon.py +477 -0
  22. lorebase_store/_migrations/versions/0019_vault_bindings.py +56 -0
  23. lorebase_store/_migrations/versions/0020_outbox_consumers.py +150 -0
  24. lorebase_store/_migrations/versions/0021_schema_suggestions.py +114 -0
  25. lorebase_store/_migrations/versions/0022_read_audit_and_shared_rate_limits.py +133 -0
  26. lorebase_store/_migrations/versions/0023_agent_trace.py +96 -0
  27. lorebase_store/_migrations/versions/0024_embedding_template_keys.py +205 -0
  28. lorebase_store/_migrations/versions/0025_snapshot_refs.py +35 -0
  29. lorebase_store/_migrations/versions/0026_plain_reverse_link_index.py +112 -0
  30. lorebase_store/_migrations/versions/0027_vault_rotation_and_tombstones.py +145 -0
  31. lorebase_store/_migrations/versions/0028_operation_deferred_events.py +71 -0
  32. lorebase_store/_migrations/versions/0029_agent_trace_sessions.py +108 -0
  33. lorebase_store/_migrations/versions/0030_path_fanout.py +31 -0
  34. lorebase_store/_migrations/versions/0031_branch_builds.py +91 -0
  35. lorebase_store/_migrations/versions/0032_lineage_revisions.py +35 -0
  36. lorebase_store/_migrations/versions/0033_vault_moved_tokens.py +30 -0
  37. lorebase_store/_migrations/versions/0034_inference_endpoint_modes.py +73 -0
  38. lorebase_store/_migrations/versions/0035_search_truncation.py +104 -0
  39. lorebase_store/_migrations/versions/0036_lineage_maps.py +325 -0
  40. lorebase_store/_migrations/versions/0037_bm25_search.py +346 -0
  41. lorebase_store/auth/__init__.py +52 -0
  42. lorebase_store/auth/api_keys.py +161 -0
  43. lorebase_store/auth/audit.py +382 -0
  44. lorebase_store/auth/checkpoints.py +185 -0
  45. lorebase_store/auth/grants.py +161 -0
  46. lorebase_store/auth/principals.py +158 -0
  47. lorebase_store/auth/scope.py +70 -0
  48. lorebase_store/auth/sessions.py +153 -0
  49. lorebase_store/auth/unit.py +53 -0
  50. lorebase_store/auth/usage.py +70 -0
  51. lorebase_store/blobs.py +73 -0
  52. lorebase_store/bm25.py +346 -0
  53. lorebase_store/bulk.py +147 -0
  54. lorebase_store/bundle.py +1830 -0
  55. lorebase_store/changefeed.py +261 -0
  56. lorebase_store/commit/__init__.py +42 -0
  57. lorebase_store/commit/changes.py +67 -0
  58. lorebase_store/commit/errors.py +60 -0
  59. lorebase_store/commit/migrations.py +335 -0
  60. lorebase_store/commit/pipeline.py +754 -0
  61. lorebase_store/commit/schema.py +123 -0
  62. lorebase_store/db.py +58 -0
  63. lorebase_store/definer.py +50 -0
  64. lorebase_store/digests.py +99 -0
  65. lorebase_store/embeddings.py +444 -0
  66. lorebase_store/endpoint_modes.py +59 -0
  67. lorebase_store/extraction/__init__.py +61 -0
  68. lorebase_store/extraction/items.py +187 -0
  69. lorebase_store/extraction/materialize.py +364 -0
  70. lorebase_store/extraction/proposal.py +65 -0
  71. lorebase_store/extraction/review.py +168 -0
  72. lorebase_store/gc.py +970 -0
  73. lorebase_store/indexes.py +1033 -0
  74. lorebase_store/jobs.py +651 -0
  75. lorebase_store/keyring.py +72 -0
  76. lorebase_store/layout.py +335 -0
  77. lorebase_store/materialize/__init__.py +103 -0
  78. lorebase_store/materialize/apply.py +436 -0
  79. lorebase_store/materialize/background.py +1184 -0
  80. lorebase_store/materialize/branch.py +268 -0
  81. lorebase_store/materialize/build.py +536 -0
  82. lorebase_store/materialize/derived.py +540 -0
  83. lorebase_store/materialize/lineage_backfill.py +502 -0
  84. lorebase_store/materialize/links.py +266 -0
  85. lorebase_store/materialize/overlay.py +292 -0
  86. lorebase_store/materialize/paths.py +315 -0
  87. lorebase_store/materialize/rebuild.py +326 -0
  88. lorebase_store/materialize/rows.py +153 -0
  89. lorebase_store/materialize/sensitivity.py +66 -0
  90. lorebase_store/materialize/snapshot.py +205 -0
  91. lorebase_store/materialize/stale.py +444 -0
  92. lorebase_store/materialize/verify.py +321 -0
  93. lorebase_store/merging/__init__.py +39 -0
  94. lorebase_store/merging/attempts.py +334 -0
  95. lorebase_store/merging/lineage.py +123 -0
  96. lorebase_store/merging/service.py +652 -0
  97. lorebase_store/migrations.py +133 -0
  98. lorebase_store/models/__init__.py +136 -0
  99. lorebase_store/models/base.py +64 -0
  100. lorebase_store/models/enums.py +132 -0
  101. lorebase_store/models/governance.py +268 -0
  102. lorebase_store/models/ingestion.py +288 -0
  103. lorebase_store/models/materialized.py +277 -0
  104. lorebase_store/models/proposals.py +195 -0
  105. lorebase_store/models/queues.py +87 -0
  106. lorebase_store/models/tracing.py +116 -0
  107. lorebase_store/models/versioning.py +222 -0
  108. lorebase_store/models/webhooks.py +96 -0
  109. lorebase_store/nodestore.py +384 -0
  110. lorebase_store/objectstore.py +162 -0
  111. lorebase_store/outbox_events.py +166 -0
  112. lorebase_store/profiling/__init__.py +58 -0
  113. lorebase_store/profiling/delimited.py +109 -0
  114. lorebase_store/profiling/postgres.py +211 -0
  115. lorebase_store/profiling/record_files.py +127 -0
  116. lorebase_store/profiling/records.py +92 -0
  117. lorebase_store/profiling/sql_databases.py +298 -0
  118. lorebase_store/ratelimit.py +129 -0
  119. lorebase_store/reads.py +1491 -0
  120. lorebase_store/replicas.py +434 -0
  121. lorebase_store/repositories/__init__.py +152 -0
  122. lorebase_store/repositories/changesets.py +59 -0
  123. lorebase_store/repositories/deferred_events.py +142 -0
  124. lorebase_store/repositories/documents.py +305 -0
  125. lorebase_store/repositories/domains.py +97 -0
  126. lorebase_store/repositories/events.py +205 -0
  127. lorebase_store/repositories/index_view.py +489 -0
  128. lorebase_store/repositories/invocations.py +68 -0
  129. lorebase_store/repositories/materializations.py +276 -0
  130. lorebase_store/repositories/materialized.py +1129 -0
  131. lorebase_store/repositories/merges.py +417 -0
  132. lorebase_store/repositories/proposal_items.py +199 -0
  133. lorebase_store/repositories/proposals.py +184 -0
  134. lorebase_store/repositories/provenance.py +77 -0
  135. lorebase_store/repositories/refs.py +452 -0
  136. lorebase_store/repositories/revisions.py +273 -0
  137. lorebase_store/repositories/scope.py +33 -0
  138. lorebase_store/repositories/source_runs.py +123 -0
  139. lorebase_store/repositories/suggestions.py +638 -0
  140. lorebase_store/runtime.py +87 -0
  141. lorebase_store/schema_cache.py +50 -0
  142. lorebase_store/settings.py +879 -0
  143. lorebase_store/storage.py +1478 -0
  144. lorebase_store/telemetry.py +142 -0
  145. lorebase_store/testing.py +108 -0
  146. lorebase_store/tracing.py +505 -0
  147. lorebase_store/traverse.py +578 -0
  148. lorebase_store/vault/__init__.py +50 -0
  149. lorebase_store/vault/keys.py +216 -0
  150. lorebase_store/vault/quotes.py +133 -0
  151. lorebase_store/vault/rotation.py +209 -0
  152. lorebase_store/vault/settings.py +152 -0
  153. lorebase_store/vault/store.py +987 -0
  154. lorebase_store/vault/transfer.py +558 -0
  155. lorebase_store/webhooks/__init__.py +24 -0
  156. lorebase_store/webhooks/delivery.py +637 -0
  157. lorebase_store/webhooks/keys.py +126 -0
  158. lorebase_store/webhooks/repository.py +142 -0
  159. lorebase_store/webhooks/settings.py +138 -0
  160. lorebase_store/webhooks/signing.py +50 -0
  161. lorebase_store-0.2.3.dist-info/METADATA +13 -0
  162. lorebase_store-0.2.3.dist-info/RECORD +163 -0
  163. lorebase_store-0.2.3.dist-info/WHEEL +4 -0
lorebase_store/bm25.py ADDED
@@ -0,0 +1,346 @@
1
+ """Optional BM25 ranking of lexical search through ParadeDB `pg_search` (docs/search.md).
2
+
3
+ Default search ranks matches of the GIN index with `ts_rank_cd`; very common terms are cut to
4
+ a bounded sample (`lorebase_store.reads.CommonTerms`). With the `pg_search` extension (a
5
+ ParadeDB image, AGPL-3.0, see docs/deployment.md), `store.storage.bm25_index: true` builds a
6
+ BM25 index on `mat_records` and Keeper's `search.backend: bm25` ranks lexical search by BM25
7
+ with a top-k scan of that index, so common terms cost about as much as rare ones.
8
+
9
+ - The index (`INDEX`) holds, per row, `lb_search_terms(search)` (revision 0037): the lexemes
10
+ of the existing search vector, tokenized on whitespace without folding case (field `terms`).
11
+ It matches what full-text search matches, with the same configuration, Attributes and
12
+ exclusions. Domain,
13
+ branch, Kind and key are literal fields, `revision_generation` and `deleted` fast fields,
14
+ so the scope of a search is part of the index query.
15
+ - `translate` turns the tsquery of a query (its text form) into a BM25 query over `terms`:
16
+ words are `term` queries, `&` / `|` / `!` boolean clauses. What has no exact equivalent
17
+ becomes a superset: a phrase matches all its words, the rows with the words in sequence
18
+ ranked first (an optional phrase clause: the index text can put a lexeme between two words
19
+ the vector has side by side when they share a position), a negated phrase matches anything;
20
+ the definer function then ranks the superset's first candidates and rechecks them against
21
+ the search vector.
22
+ - `status` says whether the database can serve BM25 search (extension and valid index);
23
+ `ready` caches the answer per process for `STATUS_TTL_S`.
24
+
25
+ BM25 statistics (document frequencies, lengths) are those of the whole index, across Domains
26
+ and branches: the order of hits of several terms depends on other Domains' text, so the
27
+ definer functions return scores relative to the best hit (docs/search.md#bm25-ranking).
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import json
33
+ import re
34
+ import time
35
+ from collections.abc import Sequence
36
+ from dataclasses import dataclass
37
+ from typing import Any, Literal
38
+
39
+ from sqlalchemy import text
40
+ from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, AsyncSession
41
+
42
+ SearchBackend = Literal["postgres", "bm25"]
43
+
44
+ EXTENSION = "pg_search"
45
+ INDEX = "ix_mat_records_bm25"
46
+ FIELD = "terms"
47
+ STATUS_TTL_S = 60.0
48
+ """How long a process trusts the availability it found; `storage apply` builds or drops the
49
+ index while Keeper runs."""
50
+ MAX_DEPTH = 32
51
+ """Nesting the definer functions accept (`lb_bm25_query_valid`); deeper queries use the
52
+ default ranking."""
53
+ MAX_CLAUSES = 256
54
+ """Clauses of one boolean query the definer functions accept."""
55
+
56
+ COLUMNS = (
57
+ "(key::pdb.literal)",
58
+ "(domain_id::pdb.literal)",
59
+ "(branch::pdb.literal)",
60
+ "(kind_id::pdb.literal)",
61
+ "revision_generation",
62
+ "deleted",
63
+ f"(lb_search_terms(search)::pdb.whitespace('alias={FIELD}', 'lowercase=false'))",
64
+ )
65
+ """Indexed columns; `key` first, the field `pdb.score` is called with."""
66
+
67
+
68
+ def index_sql(table: str, name: str, *, concurrently: bool = False, only: bool = False) -> str:
69
+ """The BM25 index of `table` (a partition of `mat_records`, or the table `ONLY`)."""
70
+ how = "CONCURRENTLY " if concurrently else ""
71
+ target = f"ONLY {table}" if only else table
72
+ return f"CREATE INDEX {how}IF NOT EXISTS {name} ON {target} USING bm25 ({', '.join(COLUMNS)})"
73
+
74
+
75
+ def partition_index(partition: str) -> str:
76
+ """Name of the BM25 index of one partition of `mat_records`."""
77
+ return f"{partition}_bm25_idx"
78
+
79
+
80
+ # Availability -------------------------------------------------------------------------------------
81
+
82
+
83
+ @dataclass(frozen=True, slots=True)
84
+ class Bm25Status:
85
+ """Whether the database can serve BM25 search, and why not."""
86
+
87
+ ready: bool
88
+ reason: str | None = None
89
+
90
+
91
+ _STATUS = text(
92
+ "SELECT (SELECT extversion FROM pg_extension WHERE extname = :extension), "
93
+ "(SELECT x.indisvalid FROM pg_index x WHERE x.indexrelid = to_regclass(:index)), "
94
+ "EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = :extension), "
95
+ "EXISTS (SELECT 1 FROM pg_settings WHERE name LIKE 'paradedb.%')"
96
+ )
97
+ """The last column: the library is loaded in this backend (its settings exist), which it is
98
+ in every backend when preloaded. `shared_preload_libraries` itself is hidden from roles
99
+ without `pg_read_all_settings`."""
100
+
101
+
102
+ async def status(connection: AsyncConnection | AsyncSession) -> Bm25Status:
103
+ """Whether this server can serve BM25 search: `pg_search` installed and preloaded, the
104
+ BM25 index valid (catalog and settings reads only). A standby answers for itself."""
105
+ row = (await connection.execute(_STATUS, {"extension": EXTENSION, "index": INDEX})).one()
106
+ version, valid, available, preloaded = row
107
+ if version is None:
108
+ if not available:
109
+ return Bm25Status(False, "the pg_search extension is not available on this server")
110
+ return Bm25Status(
111
+ False, "the pg_search extension is not installed (storage apply with bm25_index)"
112
+ )
113
+ if not preloaded:
114
+ return Bm25Status(False, "pg_search is not in shared_preload_libraries of this server")
115
+ if valid is None:
116
+ return Bm25Status(
117
+ False, f"the index {INDEX} does not exist (storage apply with bm25_index)"
118
+ )
119
+ if not valid:
120
+ return Bm25Status(False, f"the index {INDEX} is not valid yet (run storage apply)")
121
+ return Bm25Status(True)
122
+
123
+
124
+ _cached: dict[str, tuple[float, bool]] = {}
125
+
126
+
127
+ async def ready(session: AsyncSession) -> bool:
128
+ """`status(...).ready` of the server the session reads from (the primary or a replica),
129
+ looked up again after `STATUS_TTL_S`."""
130
+ bind = session.bind
131
+ engine = bind if bind is None or isinstance(bind, AsyncEngine) else bind.engine
132
+ server = "" if engine is None else engine.url.render_as_string(hide_password=True)
133
+ now = time.monotonic()
134
+ cached = _cached.get(server)
135
+ if cached is not None and now - cached[0] < STATUS_TTL_S:
136
+ return cached[1]
137
+ found = (await status(session)).ready
138
+ _cached[server] = (now, found)
139
+ return found
140
+
141
+
142
+ def forget() -> None:
143
+ """Drop the cached availability (tests, and after `storage apply` in the same process)."""
144
+ _cached.clear()
145
+
146
+
147
+ # Translation --------------------------------------------------------------------------------------
148
+
149
+ Json = dict[str, Any]
150
+
151
+ _ALL: Json = {"all": None}
152
+ _TOKENS = re.compile(r"'((?:[^']|'')*)'(:[*A-D]+)?|<(\d+)>|<->|[&|!()]")
153
+ _ESCAPES = re.compile(r"''|\\\\")
154
+
155
+
156
+ @dataclass(frozen=True, slots=True)
157
+ class Bm25Query:
158
+ """A BM25 query (JSON) and whether it matches exactly the rows of the tsquery it comes
159
+ from; when not, it matches a superset of them."""
160
+
161
+ match: Json
162
+ exact: bool
163
+
164
+ def json(self) -> str:
165
+ return json.dumps(self.match, separators=(",", ":"), ensure_ascii=False)
166
+
167
+
168
+ @dataclass(frozen=True, slots=True)
169
+ class _Node:
170
+ """A parsed tsquery node: `term`, `and`, `or`, `not`, `phrase` (lexemes in order) or
171
+ `all` (anything: the superset of what cannot be expressed)."""
172
+
173
+ kind: str
174
+ value: str = ""
175
+ children: tuple[_Node, ...] = ()
176
+
177
+
178
+ _EVERYTHING = _Node("all")
179
+
180
+
181
+ def _term(token: re.Match[str]) -> _Node:
182
+ lexeme = _ESCAPES.sub(lambda m: m.group(0)[0], token.group(1))
183
+ if token.group(2): # a prefix (`:*`) or weight restriction: no index equivalent
184
+ return _EVERYTHING
185
+ return _Node("term", lexeme)
186
+
187
+
188
+ class _Parser:
189
+ """Recursive descent over tsquery text, binding as Postgres prints it: `!`, then `<->`
190
+ and `<N>`, then `&`, then `|`. Each method returns the node and whether it is exact."""
191
+
192
+ def __init__(self, tsquery: str) -> None:
193
+ self.tokens: list[str | re.Match[str]] = [
194
+ m if m.group(1) is not None or m.group(3) is not None else m.group(0)
195
+ for m in _TOKENS.finditer(tsquery)
196
+ ]
197
+ self.position = 0
198
+
199
+ def peek(self) -> str | re.Match[str] | None:
200
+ return self.tokens[self.position] if self.position < len(self.tokens) else None
201
+
202
+ def take(self) -> str | re.Match[str]:
203
+ self.position += 1
204
+ return self.tokens[self.position - 1]
205
+
206
+ def alternatives(self) -> tuple[_Node, bool]:
207
+ nodes = [self.conjunction()]
208
+ while self.peek() == "|":
209
+ self.take()
210
+ nodes.append(self.conjunction())
211
+ exact = all(e for _, e in nodes)
212
+ if any(n.kind == "all" for n, _ in nodes):
213
+ return _EVERYTHING, False
214
+ return _flat("or", [n for n, _ in nodes]), exact
215
+
216
+ def conjunction(self) -> tuple[_Node, bool]:
217
+ nodes = [self.phrase()]
218
+ while self.peek() == "&":
219
+ self.take()
220
+ nodes.append(self.phrase())
221
+ kept = [n for n, _ in nodes if n.kind != "all"]
222
+ exact = all(e for _, e in nodes)
223
+ if not kept:
224
+ return _EVERYTHING, False
225
+ return _flat("and", kept), exact
226
+
227
+ def phrase(self) -> tuple[_Node, bool]:
228
+ first = self.operand()
229
+ parts = [first]
230
+ adjacent = True
231
+ while (isinstance(token := self.peek(), re.Match) and token.group(3) is not None) or (
232
+ token == "<->"
233
+ ):
234
+ operator = self.take()
235
+ if isinstance(operator, re.Match) and int(operator.group(3)) != 1:
236
+ adjacent = False # <N>: a distance the index cannot match exactly
237
+ parts.append(self.operand())
238
+ if len(parts) == 1:
239
+ return first
240
+ # Phrases never match exactly: the index text has no gaps where the vector has them
241
+ # (stop words), so the search vector is checked again.
242
+ if adjacent and all(n.kind == "term" and e for n, e in parts):
243
+ return _Node("phrase", children=tuple(n for n, _ in parts)), False
244
+ # Otherwise every operand must occur somewhere: their positive parts, joined by AND.
245
+ kept = [n for n in (_present(n) for n, _ in parts) if n.kind != "all"]
246
+ return (_flat("and", kept) if kept else _EVERYTHING), False
247
+
248
+ def operand(self) -> tuple[_Node, bool]:
249
+ token = self.take()
250
+ if token == "!":
251
+ inner, exact = self.operand()
252
+ if not exact or inner.kind == "all":
253
+ return _EVERYTHING, False # NOT of a superset: no bound
254
+ return _Node("not", children=(inner,)), True
255
+ if token == "(":
256
+ found = self.alternatives()
257
+ self.take() # ")"
258
+ return found
259
+ assert isinstance(token, re.Match)
260
+ node = _term(token)
261
+ return node, node.kind == "term"
262
+
263
+
264
+ def _present(node: _Node) -> _Node:
265
+ """A superset of the rows where `node` matches at some position, as an operand of a
266
+ phrase: a negation there excludes a word at one position only, not from the row."""
267
+ if node.kind == "not":
268
+ return _EVERYTHING
269
+ if node.kind in ("and", "or"):
270
+ children = [_present(c) for c in node.children]
271
+ if node.kind == "or" and any(c.kind == "all" for c in children):
272
+ return _EVERYTHING
273
+ kept = [c for c in children if c.kind != "all"]
274
+ return _flat(node.kind, kept) if kept else _EVERYTHING
275
+ return node
276
+
277
+
278
+ def _flat(kind: str, nodes: Sequence[_Node]) -> _Node:
279
+ if len(nodes) == 1:
280
+ return nodes[0]
281
+ children: list[_Node] = []
282
+ for node in nodes:
283
+ children.extend(node.children if node.kind == kind else (node,))
284
+ return _Node(kind, children=tuple(children))
285
+
286
+
287
+ def _json(node: _Node) -> Json:
288
+ if node.kind == "all":
289
+ return dict(_ALL)
290
+ if node.kind == "term":
291
+ return {"term": {"field": FIELD, "value": node.value}}
292
+ if node.kind == "phrase":
293
+ words = [c.value for c in node.children]
294
+ return {
295
+ "boolean": {
296
+ "must": [_json(c) for c in node.children],
297
+ "should": [{"phrase": {"field": FIELD, "phrases": words, "slop": 0}}],
298
+ }
299
+ }
300
+ if node.kind == "not":
301
+ return {"boolean": {"must": [dict(_ALL)], "must_not": [_json(node.children[0])]}}
302
+ if node.kind == "or":
303
+ return {"boolean": {"should": [_json(c) for c in node.children]}}
304
+ positive = [_json(c) for c in node.children if c.kind != "not"]
305
+ negative = [_json(c.children[0]) for c in node.children if c.kind == "not"]
306
+ clauses: Json = {"must": positive or [dict(_ALL)]}
307
+ if negative:
308
+ clauses["must_not"] = negative
309
+ return {"boolean": clauses}
310
+
311
+
312
+ def _fits(query: Json, depth: int = 0) -> bool:
313
+ """Whether the definer functions accept `query` (`lb_bm25_query_valid`)."""
314
+ if depth > MAX_DEPTH:
315
+ return False
316
+ clauses = query.get("boolean")
317
+ if clauses is None:
318
+ phrase = query.get("phrase")
319
+ return phrase is None or len(phrase["phrases"]) <= 64
320
+ return all(
321
+ len(items) <= MAX_CLAUSES and all(_fits(item, depth + 1) for item in items)
322
+ for items in clauses.values()
323
+ )
324
+
325
+
326
+ def translate(tsquery: str) -> Bm25Query | None:
327
+ """The BM25 query of a tsquery (its text form); `None` when it has no lexeme (it matches
328
+ nothing) or does not fit the limits of the definer functions."""
329
+ parser = _Parser(tsquery)
330
+ if not parser.tokens:
331
+ return None
332
+ node, exact = parser.alternatives()
333
+ query = _json(node)
334
+ if not _fits(query):
335
+ return None
336
+ return Bm25Query(query, exact)
337
+
338
+
339
+ def corrected_only(written: Bm25Query, corrected: Bm25Query) -> Bm25Query:
340
+ """The rows that match the corrected query but not the query as written: exact when both
341
+ are (written excluded by the index), else the corrected query, rechecked."""
342
+ if written.exact and corrected.exact:
343
+ return Bm25Query(
344
+ {"boolean": {"must": [corrected.match], "must_not": [written.match]}}, True
345
+ )
346
+ return Bm25Query(corrected.match, False)
lorebase_store/bulk.py ADDED
@@ -0,0 +1,147 @@
1
+ """Bulk writes: multi-row INSERT for ordinary batches, COPY through a staging table for large ones.
2
+
3
+ Both paths end in one set-based statement per batch with the same conflict handling, so the
4
+ caller does not need to know which one ran.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ from collections.abc import Iterator, Mapping, Sequence
11
+ from enum import StrEnum
12
+ from typing import Any
13
+
14
+ from sqlalchemy import (
15
+ ColumnElement,
16
+ Table,
17
+ Text,
18
+ bindparam,
19
+ column,
20
+ func,
21
+ select,
22
+ table,
23
+ text,
24
+ tuple_,
25
+ )
26
+ from sqlalchemy.dialects.postgresql import ARRAY, JSONB
27
+ from sqlalchemy.dialects.postgresql import insert as pg_insert
28
+ from sqlalchemy.ext.asyncio import AsyncSession
29
+
30
+ from lorebase_store.settings import WriteSettings
31
+
32
+ STAGE_PREFIX = "lb_stage_"
33
+ """Prefix of the temporary staging tables used by the COPY path (dropped at session end)."""
34
+
35
+
36
+ class OnConflict(StrEnum):
37
+ """What a bulk write does with rows whose primary key already exists."""
38
+
39
+ IGNORE = "ignore"
40
+ UPDATE = "update"
41
+ ERROR = "error"
42
+
43
+
44
+ def chunks[T](items: Sequence[T], size: int) -> Iterator[Sequence[T]]:
45
+ """Consecutive slices of at most `size` items."""
46
+ if size < 1:
47
+ raise ValueError("chunk size must be positive")
48
+ for start in range(0, len(items), size):
49
+ yield items[start : start + size]
50
+
51
+
52
+ def pairs_in(
53
+ first: ColumnElement[Any], second: ColumnElement[Any], pairs: Sequence[tuple[str, str]]
54
+ ) -> ColumnElement[bool]:
55
+ """`(first, second) IN pairs` as a semi-join on two unnested text arrays.
56
+
57
+ Postgres plans a row-value `IN` list as one bitmap index scan per pair, and its planning
58
+ time grows with the list (about half a millisecond per pair); the semi-join plans once
59
+ and probes the index once per pair.
60
+ """
61
+ keys = (
62
+ func.unnest(
63
+ bindparam(None, [a for a, _ in pairs], type_=ARRAY(Text)),
64
+ bindparam(None, [b for _, b in pairs], type_=ARRAY(Text)),
65
+ )
66
+ .table_valued("first", "second")
67
+ .render_derived()
68
+ )
69
+ return tuple_(first, second).in_(select(keys.c.first, keys.c.second))
70
+
71
+
72
+ def uses_copy(rows: int, writes: WriteSettings) -> bool:
73
+ """Whether a batch of `rows` rows takes the COPY path."""
74
+ return writes.copy_threshold_rows > 0 and rows >= writes.copy_threshold_rows
75
+
76
+
77
+ async def bulk_write(
78
+ session: AsyncSession,
79
+ target: Table,
80
+ rows: Sequence[Mapping[str, Any]],
81
+ *,
82
+ writes: WriteSettings,
83
+ on_conflict: OnConflict = OnConflict.IGNORE,
84
+ update_columns: Sequence[str] = (),
85
+ ) -> None:
86
+ """Insert `rows` (all with the same keys) into `target`.
87
+
88
+ `OnConflict.UPDATE` overwrites `update_columns` of existing rows; `IGNORE` keeps them.
89
+ """
90
+ if not rows:
91
+ return
92
+ columns = list(rows[0])
93
+ if on_conflict is OnConflict.UPDATE and not update_columns:
94
+ raise ValueError("an updating bulk write needs update_columns")
95
+ if uses_copy(len(rows), writes):
96
+ await _copy_write(session, target, columns, rows, on_conflict, update_columns)
97
+ return
98
+ for chunk in chunks(rows, writes.insert_batch_rows):
99
+ statement = _with_conflict(pg_insert(target), target, on_conflict, update_columns)
100
+ await session.execute(statement, list(chunk))
101
+
102
+
103
+ def _with_conflict(
104
+ statement: Any, target: Table, on_conflict: OnConflict, update_columns: Sequence[str]
105
+ ) -> Any:
106
+ if on_conflict is OnConflict.IGNORE:
107
+ return statement.on_conflict_do_nothing()
108
+ if on_conflict is OnConflict.UPDATE:
109
+ keys = [c.name for c in target.primary_key.columns]
110
+ return statement.on_conflict_do_update(
111
+ index_elements=keys,
112
+ set_={name: statement.excluded[name] for name in update_columns},
113
+ )
114
+ return statement
115
+
116
+
117
+ async def _copy_write(
118
+ session: AsyncSession,
119
+ target: Table,
120
+ columns: list[str],
121
+ rows: Sequence[Mapping[str, Any]],
122
+ on_conflict: OnConflict,
123
+ update_columns: Sequence[str],
124
+ ) -> None:
125
+ stage = STAGE_PREFIX + target.name
126
+ await session.execute(
127
+ text(
128
+ f"CREATE TEMP TABLE IF NOT EXISTS {stage} "
129
+ f"(LIKE {target.name} INCLUDING DEFAULTS) ON COMMIT DELETE ROWS"
130
+ )
131
+ )
132
+ json_columns = {c.name for c in target.columns if isinstance(c.type, JSONB)}
133
+ records = [
134
+ tuple(
135
+ json.dumps(row[name]) if name in json_columns and row[name] is not None else row[name]
136
+ for name in columns
137
+ )
138
+ for row in rows
139
+ ]
140
+ connection = await session.connection()
141
+ raw = await connection.get_raw_connection()
142
+ driver: Any = raw.driver_connection
143
+ await driver.copy_records_to_table(stage, records=records, columns=columns)
144
+ staged = table(stage, *(column(name) for name in columns))
145
+ statement = pg_insert(target).from_select(columns, select(*staged.c))
146
+ await session.execute(_with_conflict(statement, target, on_conflict, update_columns))
147
+ await session.execute(text(f"TRUNCATE {stage}"))