lorebase-store 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lorebase_store/__init__.py +1 -0
- lorebase_store/_migrations/env.py +106 -0
- lorebase_store/_migrations/script.py.mako +27 -0
- lorebase_store/_migrations/versions/0001_initial_schema.py +1071 -0
- lorebase_store/_migrations/versions/0002_merge_attempt_details.py +74 -0
- lorebase_store/_migrations/versions/0003_webhooks.py +127 -0
- lorebase_store/_migrations/versions/0004_defer_revision_references.py +38 -0
- lorebase_store/_migrations/versions/0005_security_definer_reads.py +396 -0
- lorebase_store/_migrations/versions/0006_api_key_usage.py +48 -0
- lorebase_store/_migrations/versions/0007_document_extraction.py +154 -0
- lorebase_store/_migrations/versions/0008_definer_argument_checks.py +178 -0
- lorebase_store/_migrations/versions/0009_api_key_networks.py +33 -0
- lorebase_store/_migrations/versions/0010_overlay_branches.py +582 -0
- lorebase_store/_migrations/versions/0011_branch_scoped_search_index.py +229 -0
- lorebase_store/_migrations/versions/0012_overlay_neighborhood_lookups.py +185 -0
- lorebase_store/_migrations/versions/0013_web_sessions.py +72 -0
- lorebase_store/_migrations/versions/0014_embedding_jobs.py +70 -0
- lorebase_store/_migrations/versions/0015_append_only_audit_log.py +83 -0
- lorebase_store/_migrations/versions/0016_sealed_webhook_keys.py +55 -0
- lorebase_store/_migrations/versions/0017_background_branch_copies.py +77 -0
- lorebase_store/_migrations/versions/0018_search_lexicon.py +477 -0
- lorebase_store/_migrations/versions/0019_vault_bindings.py +56 -0
- lorebase_store/_migrations/versions/0020_outbox_consumers.py +150 -0
- lorebase_store/_migrations/versions/0021_schema_suggestions.py +114 -0
- lorebase_store/_migrations/versions/0022_read_audit_and_shared_rate_limits.py +133 -0
- lorebase_store/_migrations/versions/0023_agent_trace.py +96 -0
- lorebase_store/_migrations/versions/0024_embedding_template_keys.py +205 -0
- lorebase_store/_migrations/versions/0025_snapshot_refs.py +35 -0
- lorebase_store/_migrations/versions/0026_plain_reverse_link_index.py +112 -0
- lorebase_store/_migrations/versions/0027_vault_rotation_and_tombstones.py +145 -0
- lorebase_store/_migrations/versions/0028_operation_deferred_events.py +71 -0
- lorebase_store/_migrations/versions/0029_agent_trace_sessions.py +108 -0
- lorebase_store/_migrations/versions/0030_path_fanout.py +31 -0
- lorebase_store/_migrations/versions/0031_branch_builds.py +91 -0
- lorebase_store/_migrations/versions/0032_lineage_revisions.py +35 -0
- lorebase_store/_migrations/versions/0033_vault_moved_tokens.py +30 -0
- lorebase_store/_migrations/versions/0034_inference_endpoint_modes.py +73 -0
- lorebase_store/_migrations/versions/0035_search_truncation.py +104 -0
- lorebase_store/_migrations/versions/0036_lineage_maps.py +325 -0
- lorebase_store/_migrations/versions/0037_bm25_search.py +346 -0
- lorebase_store/auth/__init__.py +52 -0
- lorebase_store/auth/api_keys.py +161 -0
- lorebase_store/auth/audit.py +382 -0
- lorebase_store/auth/checkpoints.py +185 -0
- lorebase_store/auth/grants.py +161 -0
- lorebase_store/auth/principals.py +158 -0
- lorebase_store/auth/scope.py +70 -0
- lorebase_store/auth/sessions.py +153 -0
- lorebase_store/auth/unit.py +53 -0
- lorebase_store/auth/usage.py +70 -0
- lorebase_store/blobs.py +73 -0
- lorebase_store/bm25.py +346 -0
- lorebase_store/bulk.py +147 -0
- lorebase_store/bundle.py +1830 -0
- lorebase_store/changefeed.py +261 -0
- lorebase_store/commit/__init__.py +42 -0
- lorebase_store/commit/changes.py +67 -0
- lorebase_store/commit/errors.py +60 -0
- lorebase_store/commit/migrations.py +335 -0
- lorebase_store/commit/pipeline.py +754 -0
- lorebase_store/commit/schema.py +123 -0
- lorebase_store/db.py +58 -0
- lorebase_store/definer.py +50 -0
- lorebase_store/digests.py +99 -0
- lorebase_store/embeddings.py +444 -0
- lorebase_store/endpoint_modes.py +59 -0
- lorebase_store/extraction/__init__.py +61 -0
- lorebase_store/extraction/items.py +187 -0
- lorebase_store/extraction/materialize.py +364 -0
- lorebase_store/extraction/proposal.py +65 -0
- lorebase_store/extraction/review.py +168 -0
- lorebase_store/gc.py +970 -0
- lorebase_store/indexes.py +1033 -0
- lorebase_store/jobs.py +651 -0
- lorebase_store/keyring.py +72 -0
- lorebase_store/layout.py +335 -0
- lorebase_store/materialize/__init__.py +103 -0
- lorebase_store/materialize/apply.py +436 -0
- lorebase_store/materialize/background.py +1184 -0
- lorebase_store/materialize/branch.py +268 -0
- lorebase_store/materialize/build.py +536 -0
- lorebase_store/materialize/derived.py +540 -0
- lorebase_store/materialize/lineage_backfill.py +502 -0
- lorebase_store/materialize/links.py +266 -0
- lorebase_store/materialize/overlay.py +292 -0
- lorebase_store/materialize/paths.py +315 -0
- lorebase_store/materialize/rebuild.py +326 -0
- lorebase_store/materialize/rows.py +153 -0
- lorebase_store/materialize/sensitivity.py +66 -0
- lorebase_store/materialize/snapshot.py +205 -0
- lorebase_store/materialize/stale.py +444 -0
- lorebase_store/materialize/verify.py +321 -0
- lorebase_store/merging/__init__.py +39 -0
- lorebase_store/merging/attempts.py +334 -0
- lorebase_store/merging/lineage.py +123 -0
- lorebase_store/merging/service.py +652 -0
- lorebase_store/migrations.py +133 -0
- lorebase_store/models/__init__.py +136 -0
- lorebase_store/models/base.py +64 -0
- lorebase_store/models/enums.py +132 -0
- lorebase_store/models/governance.py +268 -0
- lorebase_store/models/ingestion.py +288 -0
- lorebase_store/models/materialized.py +277 -0
- lorebase_store/models/proposals.py +195 -0
- lorebase_store/models/queues.py +87 -0
- lorebase_store/models/tracing.py +116 -0
- lorebase_store/models/versioning.py +222 -0
- lorebase_store/models/webhooks.py +96 -0
- lorebase_store/nodestore.py +384 -0
- lorebase_store/objectstore.py +162 -0
- lorebase_store/outbox_events.py +166 -0
- lorebase_store/profiling/__init__.py +58 -0
- lorebase_store/profiling/delimited.py +109 -0
- lorebase_store/profiling/postgres.py +211 -0
- lorebase_store/profiling/record_files.py +127 -0
- lorebase_store/profiling/records.py +92 -0
- lorebase_store/profiling/sql_databases.py +298 -0
- lorebase_store/ratelimit.py +129 -0
- lorebase_store/reads.py +1491 -0
- lorebase_store/replicas.py +434 -0
- lorebase_store/repositories/__init__.py +152 -0
- lorebase_store/repositories/changesets.py +59 -0
- lorebase_store/repositories/deferred_events.py +142 -0
- lorebase_store/repositories/documents.py +305 -0
- lorebase_store/repositories/domains.py +97 -0
- lorebase_store/repositories/events.py +205 -0
- lorebase_store/repositories/index_view.py +489 -0
- lorebase_store/repositories/invocations.py +68 -0
- lorebase_store/repositories/materializations.py +276 -0
- lorebase_store/repositories/materialized.py +1129 -0
- lorebase_store/repositories/merges.py +417 -0
- lorebase_store/repositories/proposal_items.py +199 -0
- lorebase_store/repositories/proposals.py +184 -0
- lorebase_store/repositories/provenance.py +77 -0
- lorebase_store/repositories/refs.py +452 -0
- lorebase_store/repositories/revisions.py +273 -0
- lorebase_store/repositories/scope.py +33 -0
- lorebase_store/repositories/source_runs.py +123 -0
- lorebase_store/repositories/suggestions.py +638 -0
- lorebase_store/runtime.py +87 -0
- lorebase_store/schema_cache.py +50 -0
- lorebase_store/settings.py +879 -0
- lorebase_store/storage.py +1478 -0
- lorebase_store/telemetry.py +142 -0
- lorebase_store/testing.py +108 -0
- lorebase_store/tracing.py +505 -0
- lorebase_store/traverse.py +578 -0
- lorebase_store/vault/__init__.py +50 -0
- lorebase_store/vault/keys.py +216 -0
- lorebase_store/vault/quotes.py +133 -0
- lorebase_store/vault/rotation.py +209 -0
- lorebase_store/vault/settings.py +152 -0
- lorebase_store/vault/store.py +987 -0
- lorebase_store/vault/transfer.py +558 -0
- lorebase_store/webhooks/__init__.py +24 -0
- lorebase_store/webhooks/delivery.py +637 -0
- lorebase_store/webhooks/keys.py +126 -0
- lorebase_store/webhooks/repository.py +142 -0
- lorebase_store/webhooks/settings.py +138 -0
- lorebase_store/webhooks/signing.py +50 -0
- lorebase_store-0.2.3.dist-info/METADATA +13 -0
- lorebase_store-0.2.3.dist-info/RECORD +163 -0
- lorebase_store-0.2.3.dist-info/WHEEL +4 -0
lorebase_store/bm25.py
ADDED
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
"""Optional BM25 ranking of lexical search through ParadeDB `pg_search` (docs/search.md).
|
|
2
|
+
|
|
3
|
+
Default search ranks matches of the GIN index with `ts_rank_cd`; very common terms are cut to
|
|
4
|
+
a bounded sample (`lorebase_store.reads.CommonTerms`). With the `pg_search` extension (a
|
|
5
|
+
ParadeDB image, AGPL-3.0, see docs/deployment.md), `store.storage.bm25_index: true` builds a
|
|
6
|
+
BM25 index on `mat_records` and Keeper's `search.backend: bm25` ranks lexical search by BM25
|
|
7
|
+
with a top-k scan of that index, so common terms cost about as much as rare ones.
|
|
8
|
+
|
|
9
|
+
- The index (`INDEX`) holds, per row, `lb_search_terms(search)` (revision 0037): the lexemes
|
|
10
|
+
of the existing search vector, tokenized on whitespace without folding case (field `terms`).
|
|
11
|
+
It matches what full-text search matches, with the same configuration, Attributes and
|
|
12
|
+
exclusions. Domain,
|
|
13
|
+
branch, Kind and key are literal fields, `revision_generation` and `deleted` fast fields,
|
|
14
|
+
so the scope of a search is part of the index query.
|
|
15
|
+
- `translate` turns the tsquery of a query (its text form) into a BM25 query over `terms`:
|
|
16
|
+
words are `term` queries, `&` / `|` / `!` boolean clauses. What has no exact equivalent
|
|
17
|
+
becomes a superset: a phrase matches all its words, the rows with the words in sequence
|
|
18
|
+
ranked first (an optional phrase clause: the index text can put a lexeme between two words
|
|
19
|
+
the vector has side by side when they share a position), a negated phrase matches anything;
|
|
20
|
+
the definer function then ranks the superset's first candidates and rechecks them against
|
|
21
|
+
the search vector.
|
|
22
|
+
- `status` says whether the database can serve BM25 search (extension and valid index);
|
|
23
|
+
`ready` caches the answer per process for `STATUS_TTL_S`.
|
|
24
|
+
|
|
25
|
+
BM25 statistics (document frequencies, lengths) are those of the whole index, across Domains
|
|
26
|
+
and branches: the order of hits of several terms depends on other Domains' text, so the
|
|
27
|
+
definer functions return scores relative to the best hit (docs/search.md#bm25-ranking).
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import json
|
|
33
|
+
import re
|
|
34
|
+
import time
|
|
35
|
+
from collections.abc import Sequence
|
|
36
|
+
from dataclasses import dataclass
|
|
37
|
+
from typing import Any, Literal
|
|
38
|
+
|
|
39
|
+
from sqlalchemy import text
|
|
40
|
+
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, AsyncSession
|
|
41
|
+
|
|
42
|
+
SearchBackend = Literal["postgres", "bm25"]
|
|
43
|
+
|
|
44
|
+
EXTENSION = "pg_search"
|
|
45
|
+
INDEX = "ix_mat_records_bm25"
|
|
46
|
+
FIELD = "terms"
|
|
47
|
+
STATUS_TTL_S = 60.0
|
|
48
|
+
"""How long a process trusts the availability it found; `storage apply` builds or drops the
|
|
49
|
+
index while Keeper runs."""
|
|
50
|
+
MAX_DEPTH = 32
|
|
51
|
+
"""Nesting the definer functions accept (`lb_bm25_query_valid`); deeper queries use the
|
|
52
|
+
default ranking."""
|
|
53
|
+
MAX_CLAUSES = 256
|
|
54
|
+
"""Clauses of one boolean query the definer functions accept."""
|
|
55
|
+
|
|
56
|
+
COLUMNS = (
|
|
57
|
+
"(key::pdb.literal)",
|
|
58
|
+
"(domain_id::pdb.literal)",
|
|
59
|
+
"(branch::pdb.literal)",
|
|
60
|
+
"(kind_id::pdb.literal)",
|
|
61
|
+
"revision_generation",
|
|
62
|
+
"deleted",
|
|
63
|
+
f"(lb_search_terms(search)::pdb.whitespace('alias={FIELD}', 'lowercase=false'))",
|
|
64
|
+
)
|
|
65
|
+
"""Indexed columns; `key` first, the field `pdb.score` is called with."""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def index_sql(table: str, name: str, *, concurrently: bool = False, only: bool = False) -> str:
|
|
69
|
+
"""The BM25 index of `table` (a partition of `mat_records`, or the table `ONLY`)."""
|
|
70
|
+
how = "CONCURRENTLY " if concurrently else ""
|
|
71
|
+
target = f"ONLY {table}" if only else table
|
|
72
|
+
return f"CREATE INDEX {how}IF NOT EXISTS {name} ON {target} USING bm25 ({', '.join(COLUMNS)})"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def partition_index(partition: str) -> str:
|
|
76
|
+
"""Name of the BM25 index of one partition of `mat_records`."""
|
|
77
|
+
return f"{partition}_bm25_idx"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# Availability -------------------------------------------------------------------------------------
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True, slots=True)
|
|
84
|
+
class Bm25Status:
|
|
85
|
+
"""Whether the database can serve BM25 search, and why not."""
|
|
86
|
+
|
|
87
|
+
ready: bool
|
|
88
|
+
reason: str | None = None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
_STATUS = text(
|
|
92
|
+
"SELECT (SELECT extversion FROM pg_extension WHERE extname = :extension), "
|
|
93
|
+
"(SELECT x.indisvalid FROM pg_index x WHERE x.indexrelid = to_regclass(:index)), "
|
|
94
|
+
"EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = :extension), "
|
|
95
|
+
"EXISTS (SELECT 1 FROM pg_settings WHERE name LIKE 'paradedb.%')"
|
|
96
|
+
)
|
|
97
|
+
"""The last column: the library is loaded in this backend (its settings exist), which it is
|
|
98
|
+
in every backend when preloaded. `shared_preload_libraries` itself is hidden from roles
|
|
99
|
+
without `pg_read_all_settings`."""
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
async def status(connection: AsyncConnection | AsyncSession) -> Bm25Status:
|
|
103
|
+
"""Whether this server can serve BM25 search: `pg_search` installed and preloaded, the
|
|
104
|
+
BM25 index valid (catalog and settings reads only). A standby answers for itself."""
|
|
105
|
+
row = (await connection.execute(_STATUS, {"extension": EXTENSION, "index": INDEX})).one()
|
|
106
|
+
version, valid, available, preloaded = row
|
|
107
|
+
if version is None:
|
|
108
|
+
if not available:
|
|
109
|
+
return Bm25Status(False, "the pg_search extension is not available on this server")
|
|
110
|
+
return Bm25Status(
|
|
111
|
+
False, "the pg_search extension is not installed (storage apply with bm25_index)"
|
|
112
|
+
)
|
|
113
|
+
if not preloaded:
|
|
114
|
+
return Bm25Status(False, "pg_search is not in shared_preload_libraries of this server")
|
|
115
|
+
if valid is None:
|
|
116
|
+
return Bm25Status(
|
|
117
|
+
False, f"the index {INDEX} does not exist (storage apply with bm25_index)"
|
|
118
|
+
)
|
|
119
|
+
if not valid:
|
|
120
|
+
return Bm25Status(False, f"the index {INDEX} is not valid yet (run storage apply)")
|
|
121
|
+
return Bm25Status(True)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
_cached: dict[str, tuple[float, bool]] = {}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
async def ready(session: AsyncSession) -> bool:
|
|
128
|
+
"""`status(...).ready` of the server the session reads from (the primary or a replica),
|
|
129
|
+
looked up again after `STATUS_TTL_S`."""
|
|
130
|
+
bind = session.bind
|
|
131
|
+
engine = bind if bind is None or isinstance(bind, AsyncEngine) else bind.engine
|
|
132
|
+
server = "" if engine is None else engine.url.render_as_string(hide_password=True)
|
|
133
|
+
now = time.monotonic()
|
|
134
|
+
cached = _cached.get(server)
|
|
135
|
+
if cached is not None and now - cached[0] < STATUS_TTL_S:
|
|
136
|
+
return cached[1]
|
|
137
|
+
found = (await status(session)).ready
|
|
138
|
+
_cached[server] = (now, found)
|
|
139
|
+
return found
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def forget() -> None:
|
|
143
|
+
"""Drop the cached availability (tests, and after `storage apply` in the same process)."""
|
|
144
|
+
_cached.clear()
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# Translation --------------------------------------------------------------------------------------
|
|
148
|
+
|
|
149
|
+
Json = dict[str, Any]
|
|
150
|
+
|
|
151
|
+
_ALL: Json = {"all": None}
|
|
152
|
+
_TOKENS = re.compile(r"'((?:[^']|'')*)'(:[*A-D]+)?|<(\d+)>|<->|[&|!()]")
|
|
153
|
+
_ESCAPES = re.compile(r"''|\\\\")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@dataclass(frozen=True, slots=True)
|
|
157
|
+
class Bm25Query:
|
|
158
|
+
"""A BM25 query (JSON) and whether it matches exactly the rows of the tsquery it comes
|
|
159
|
+
from; when not, it matches a superset of them."""
|
|
160
|
+
|
|
161
|
+
match: Json
|
|
162
|
+
exact: bool
|
|
163
|
+
|
|
164
|
+
def json(self) -> str:
|
|
165
|
+
return json.dumps(self.match, separators=(",", ":"), ensure_ascii=False)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@dataclass(frozen=True, slots=True)
|
|
169
|
+
class _Node:
|
|
170
|
+
"""A parsed tsquery node: `term`, `and`, `or`, `not`, `phrase` (lexemes in order) or
|
|
171
|
+
`all` (anything: the superset of what cannot be expressed)."""
|
|
172
|
+
|
|
173
|
+
kind: str
|
|
174
|
+
value: str = ""
|
|
175
|
+
children: tuple[_Node, ...] = ()
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
_EVERYTHING = _Node("all")
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _term(token: re.Match[str]) -> _Node:
|
|
182
|
+
lexeme = _ESCAPES.sub(lambda m: m.group(0)[0], token.group(1))
|
|
183
|
+
if token.group(2): # a prefix (`:*`) or weight restriction: no index equivalent
|
|
184
|
+
return _EVERYTHING
|
|
185
|
+
return _Node("term", lexeme)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
class _Parser:
|
|
189
|
+
"""Recursive descent over tsquery text, binding as Postgres prints it: `!`, then `<->`
|
|
190
|
+
and `<N>`, then `&`, then `|`. Each method returns the node and whether it is exact."""
|
|
191
|
+
|
|
192
|
+
def __init__(self, tsquery: str) -> None:
|
|
193
|
+
self.tokens: list[str | re.Match[str]] = [
|
|
194
|
+
m if m.group(1) is not None or m.group(3) is not None else m.group(0)
|
|
195
|
+
for m in _TOKENS.finditer(tsquery)
|
|
196
|
+
]
|
|
197
|
+
self.position = 0
|
|
198
|
+
|
|
199
|
+
def peek(self) -> str | re.Match[str] | None:
|
|
200
|
+
return self.tokens[self.position] if self.position < len(self.tokens) else None
|
|
201
|
+
|
|
202
|
+
def take(self) -> str | re.Match[str]:
|
|
203
|
+
self.position += 1
|
|
204
|
+
return self.tokens[self.position - 1]
|
|
205
|
+
|
|
206
|
+
def alternatives(self) -> tuple[_Node, bool]:
|
|
207
|
+
nodes = [self.conjunction()]
|
|
208
|
+
while self.peek() == "|":
|
|
209
|
+
self.take()
|
|
210
|
+
nodes.append(self.conjunction())
|
|
211
|
+
exact = all(e for _, e in nodes)
|
|
212
|
+
if any(n.kind == "all" for n, _ in nodes):
|
|
213
|
+
return _EVERYTHING, False
|
|
214
|
+
return _flat("or", [n for n, _ in nodes]), exact
|
|
215
|
+
|
|
216
|
+
def conjunction(self) -> tuple[_Node, bool]:
|
|
217
|
+
nodes = [self.phrase()]
|
|
218
|
+
while self.peek() == "&":
|
|
219
|
+
self.take()
|
|
220
|
+
nodes.append(self.phrase())
|
|
221
|
+
kept = [n for n, _ in nodes if n.kind != "all"]
|
|
222
|
+
exact = all(e for _, e in nodes)
|
|
223
|
+
if not kept:
|
|
224
|
+
return _EVERYTHING, False
|
|
225
|
+
return _flat("and", kept), exact
|
|
226
|
+
|
|
227
|
+
def phrase(self) -> tuple[_Node, bool]:
|
|
228
|
+
first = self.operand()
|
|
229
|
+
parts = [first]
|
|
230
|
+
adjacent = True
|
|
231
|
+
while (isinstance(token := self.peek(), re.Match) and token.group(3) is not None) or (
|
|
232
|
+
token == "<->"
|
|
233
|
+
):
|
|
234
|
+
operator = self.take()
|
|
235
|
+
if isinstance(operator, re.Match) and int(operator.group(3)) != 1:
|
|
236
|
+
adjacent = False # <N>: a distance the index cannot match exactly
|
|
237
|
+
parts.append(self.operand())
|
|
238
|
+
if len(parts) == 1:
|
|
239
|
+
return first
|
|
240
|
+
# Phrases never match exactly: the index text has no gaps where the vector has them
|
|
241
|
+
# (stop words), so the search vector is checked again.
|
|
242
|
+
if adjacent and all(n.kind == "term" and e for n, e in parts):
|
|
243
|
+
return _Node("phrase", children=tuple(n for n, _ in parts)), False
|
|
244
|
+
# Otherwise every operand must occur somewhere: their positive parts, joined by AND.
|
|
245
|
+
kept = [n for n in (_present(n) for n, _ in parts) if n.kind != "all"]
|
|
246
|
+
return (_flat("and", kept) if kept else _EVERYTHING), False
|
|
247
|
+
|
|
248
|
+
def operand(self) -> tuple[_Node, bool]:
|
|
249
|
+
token = self.take()
|
|
250
|
+
if token == "!":
|
|
251
|
+
inner, exact = self.operand()
|
|
252
|
+
if not exact or inner.kind == "all":
|
|
253
|
+
return _EVERYTHING, False # NOT of a superset: no bound
|
|
254
|
+
return _Node("not", children=(inner,)), True
|
|
255
|
+
if token == "(":
|
|
256
|
+
found = self.alternatives()
|
|
257
|
+
self.take() # ")"
|
|
258
|
+
return found
|
|
259
|
+
assert isinstance(token, re.Match)
|
|
260
|
+
node = _term(token)
|
|
261
|
+
return node, node.kind == "term"
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _present(node: _Node) -> _Node:
|
|
265
|
+
"""A superset of the rows where `node` matches at some position, as an operand of a
|
|
266
|
+
phrase: a negation there excludes a word at one position only, not from the row."""
|
|
267
|
+
if node.kind == "not":
|
|
268
|
+
return _EVERYTHING
|
|
269
|
+
if node.kind in ("and", "or"):
|
|
270
|
+
children = [_present(c) for c in node.children]
|
|
271
|
+
if node.kind == "or" and any(c.kind == "all" for c in children):
|
|
272
|
+
return _EVERYTHING
|
|
273
|
+
kept = [c for c in children if c.kind != "all"]
|
|
274
|
+
return _flat(node.kind, kept) if kept else _EVERYTHING
|
|
275
|
+
return node
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _flat(kind: str, nodes: Sequence[_Node]) -> _Node:
|
|
279
|
+
if len(nodes) == 1:
|
|
280
|
+
return nodes[0]
|
|
281
|
+
children: list[_Node] = []
|
|
282
|
+
for node in nodes:
|
|
283
|
+
children.extend(node.children if node.kind == kind else (node,))
|
|
284
|
+
return _Node(kind, children=tuple(children))
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _json(node: _Node) -> Json:
|
|
288
|
+
if node.kind == "all":
|
|
289
|
+
return dict(_ALL)
|
|
290
|
+
if node.kind == "term":
|
|
291
|
+
return {"term": {"field": FIELD, "value": node.value}}
|
|
292
|
+
if node.kind == "phrase":
|
|
293
|
+
words = [c.value for c in node.children]
|
|
294
|
+
return {
|
|
295
|
+
"boolean": {
|
|
296
|
+
"must": [_json(c) for c in node.children],
|
|
297
|
+
"should": [{"phrase": {"field": FIELD, "phrases": words, "slop": 0}}],
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
if node.kind == "not":
|
|
301
|
+
return {"boolean": {"must": [dict(_ALL)], "must_not": [_json(node.children[0])]}}
|
|
302
|
+
if node.kind == "or":
|
|
303
|
+
return {"boolean": {"should": [_json(c) for c in node.children]}}
|
|
304
|
+
positive = [_json(c) for c in node.children if c.kind != "not"]
|
|
305
|
+
negative = [_json(c.children[0]) for c in node.children if c.kind == "not"]
|
|
306
|
+
clauses: Json = {"must": positive or [dict(_ALL)]}
|
|
307
|
+
if negative:
|
|
308
|
+
clauses["must_not"] = negative
|
|
309
|
+
return {"boolean": clauses}
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _fits(query: Json, depth: int = 0) -> bool:
|
|
313
|
+
"""Whether the definer functions accept `query` (`lb_bm25_query_valid`)."""
|
|
314
|
+
if depth > MAX_DEPTH:
|
|
315
|
+
return False
|
|
316
|
+
clauses = query.get("boolean")
|
|
317
|
+
if clauses is None:
|
|
318
|
+
phrase = query.get("phrase")
|
|
319
|
+
return phrase is None or len(phrase["phrases"]) <= 64
|
|
320
|
+
return all(
|
|
321
|
+
len(items) <= MAX_CLAUSES and all(_fits(item, depth + 1) for item in items)
|
|
322
|
+
for items in clauses.values()
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def translate(tsquery: str) -> Bm25Query | None:
|
|
327
|
+
"""The BM25 query of a tsquery (its text form); `None` when it has no lexeme (it matches
|
|
328
|
+
nothing) or does not fit the limits of the definer functions."""
|
|
329
|
+
parser = _Parser(tsquery)
|
|
330
|
+
if not parser.tokens:
|
|
331
|
+
return None
|
|
332
|
+
node, exact = parser.alternatives()
|
|
333
|
+
query = _json(node)
|
|
334
|
+
if not _fits(query):
|
|
335
|
+
return None
|
|
336
|
+
return Bm25Query(query, exact)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def corrected_only(written: Bm25Query, corrected: Bm25Query) -> Bm25Query:
|
|
340
|
+
"""The rows that match the corrected query but not the query as written: exact when both
|
|
341
|
+
are (written excluded by the index), else the corrected query, rechecked."""
|
|
342
|
+
if written.exact and corrected.exact:
|
|
343
|
+
return Bm25Query(
|
|
344
|
+
{"boolean": {"must": [corrected.match], "must_not": [written.match]}}, True
|
|
345
|
+
)
|
|
346
|
+
return Bm25Query(corrected.match, False)
|
lorebase_store/bulk.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Bulk writes: multi-row INSERT for ordinary batches, COPY through a staging table for large ones.
|
|
2
|
+
|
|
3
|
+
Both paths end in one set-based statement per batch with the same conflict handling, so the
|
|
4
|
+
caller does not need to know which one ran.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
11
|
+
from enum import StrEnum
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from sqlalchemy import (
|
|
15
|
+
ColumnElement,
|
|
16
|
+
Table,
|
|
17
|
+
Text,
|
|
18
|
+
bindparam,
|
|
19
|
+
column,
|
|
20
|
+
func,
|
|
21
|
+
select,
|
|
22
|
+
table,
|
|
23
|
+
text,
|
|
24
|
+
tuple_,
|
|
25
|
+
)
|
|
26
|
+
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
|
|
27
|
+
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
28
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
29
|
+
|
|
30
|
+
from lorebase_store.settings import WriteSettings
|
|
31
|
+
|
|
32
|
+
STAGE_PREFIX = "lb_stage_"
|
|
33
|
+
"""Prefix of the temporary staging tables used by the COPY path (dropped at session end)."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class OnConflict(StrEnum):
|
|
37
|
+
"""What a bulk write does with rows whose primary key already exists."""
|
|
38
|
+
|
|
39
|
+
IGNORE = "ignore"
|
|
40
|
+
UPDATE = "update"
|
|
41
|
+
ERROR = "error"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def chunks[T](items: Sequence[T], size: int) -> Iterator[Sequence[T]]:
|
|
45
|
+
"""Consecutive slices of at most `size` items."""
|
|
46
|
+
if size < 1:
|
|
47
|
+
raise ValueError("chunk size must be positive")
|
|
48
|
+
for start in range(0, len(items), size):
|
|
49
|
+
yield items[start : start + size]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def pairs_in(
|
|
53
|
+
first: ColumnElement[Any], second: ColumnElement[Any], pairs: Sequence[tuple[str, str]]
|
|
54
|
+
) -> ColumnElement[bool]:
|
|
55
|
+
"""`(first, second) IN pairs` as a semi-join on two unnested text arrays.
|
|
56
|
+
|
|
57
|
+
Postgres plans a row-value `IN` list as one bitmap index scan per pair, and its planning
|
|
58
|
+
time grows with the list (about half a millisecond per pair); the semi-join plans once
|
|
59
|
+
and probes the index once per pair.
|
|
60
|
+
"""
|
|
61
|
+
keys = (
|
|
62
|
+
func.unnest(
|
|
63
|
+
bindparam(None, [a for a, _ in pairs], type_=ARRAY(Text)),
|
|
64
|
+
bindparam(None, [b for _, b in pairs], type_=ARRAY(Text)),
|
|
65
|
+
)
|
|
66
|
+
.table_valued("first", "second")
|
|
67
|
+
.render_derived()
|
|
68
|
+
)
|
|
69
|
+
return tuple_(first, second).in_(select(keys.c.first, keys.c.second))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def uses_copy(rows: int, writes: WriteSettings) -> bool:
|
|
73
|
+
"""Whether a batch of `rows` rows takes the COPY path."""
|
|
74
|
+
return writes.copy_threshold_rows > 0 and rows >= writes.copy_threshold_rows
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
async def bulk_write(
|
|
78
|
+
session: AsyncSession,
|
|
79
|
+
target: Table,
|
|
80
|
+
rows: Sequence[Mapping[str, Any]],
|
|
81
|
+
*,
|
|
82
|
+
writes: WriteSettings,
|
|
83
|
+
on_conflict: OnConflict = OnConflict.IGNORE,
|
|
84
|
+
update_columns: Sequence[str] = (),
|
|
85
|
+
) -> None:
|
|
86
|
+
"""Insert `rows` (all with the same keys) into `target`.
|
|
87
|
+
|
|
88
|
+
`OnConflict.UPDATE` overwrites `update_columns` of existing rows; `IGNORE` keeps them.
|
|
89
|
+
"""
|
|
90
|
+
if not rows:
|
|
91
|
+
return
|
|
92
|
+
columns = list(rows[0])
|
|
93
|
+
if on_conflict is OnConflict.UPDATE and not update_columns:
|
|
94
|
+
raise ValueError("an updating bulk write needs update_columns")
|
|
95
|
+
if uses_copy(len(rows), writes):
|
|
96
|
+
await _copy_write(session, target, columns, rows, on_conflict, update_columns)
|
|
97
|
+
return
|
|
98
|
+
for chunk in chunks(rows, writes.insert_batch_rows):
|
|
99
|
+
statement = _with_conflict(pg_insert(target), target, on_conflict, update_columns)
|
|
100
|
+
await session.execute(statement, list(chunk))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _with_conflict(
|
|
104
|
+
statement: Any, target: Table, on_conflict: OnConflict, update_columns: Sequence[str]
|
|
105
|
+
) -> Any:
|
|
106
|
+
if on_conflict is OnConflict.IGNORE:
|
|
107
|
+
return statement.on_conflict_do_nothing()
|
|
108
|
+
if on_conflict is OnConflict.UPDATE:
|
|
109
|
+
keys = [c.name for c in target.primary_key.columns]
|
|
110
|
+
return statement.on_conflict_do_update(
|
|
111
|
+
index_elements=keys,
|
|
112
|
+
set_={name: statement.excluded[name] for name in update_columns},
|
|
113
|
+
)
|
|
114
|
+
return statement
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
async def _copy_write(
|
|
118
|
+
session: AsyncSession,
|
|
119
|
+
target: Table,
|
|
120
|
+
columns: list[str],
|
|
121
|
+
rows: Sequence[Mapping[str, Any]],
|
|
122
|
+
on_conflict: OnConflict,
|
|
123
|
+
update_columns: Sequence[str],
|
|
124
|
+
) -> None:
|
|
125
|
+
stage = STAGE_PREFIX + target.name
|
|
126
|
+
await session.execute(
|
|
127
|
+
text(
|
|
128
|
+
f"CREATE TEMP TABLE IF NOT EXISTS {stage} "
|
|
129
|
+
f"(LIKE {target.name} INCLUDING DEFAULTS) ON COMMIT DELETE ROWS"
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
json_columns = {c.name for c in target.columns if isinstance(c.type, JSONB)}
|
|
133
|
+
records = [
|
|
134
|
+
tuple(
|
|
135
|
+
json.dumps(row[name]) if name in json_columns and row[name] is not None else row[name]
|
|
136
|
+
for name in columns
|
|
137
|
+
)
|
|
138
|
+
for row in rows
|
|
139
|
+
]
|
|
140
|
+
connection = await session.connection()
|
|
141
|
+
raw = await connection.get_raw_connection()
|
|
142
|
+
driver: Any = raw.driver_connection
|
|
143
|
+
await driver.copy_records_to_table(stage, records=records, columns=columns)
|
|
144
|
+
staged = table(stage, *(column(name) for name in columns))
|
|
145
|
+
statement = pg_insert(target).from_select(columns, select(*staged.c))
|
|
146
|
+
await session.execute(_with_conflict(statement, target, on_conflict, update_columns))
|
|
147
|
+
await session.execute(text(f"TRUNCATE {stage}"))
|