strands-mongodb 0.1.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strands_mongodb/__init__.py +76 -0
- strands_mongodb/_client.py +508 -0
- strands_mongodb/_extras.py +39 -0
- strands_mongodb/_filter.py +221 -0
- strands_mongodb/_indexes.py +1314 -0
- strands_mongodb/_pipeline.py +408 -0
- strands_mongodb/_telemetry.py +254 -0
- strands_mongodb/memory_store.py +1282 -0
- strands_mongodb/py.typed +1 -0
- strands_mongodb/storage.py +693 -0
- strands_mongodb/types.py +236 -0
- strands_mongodb/vector_search.py +216 -0
- strands_mongodb-0.1.0.dev1.dist-info/METADATA +1032 -0
- strands_mongodb-0.1.0.dev1.dist-info/RECORD +17 -0
- strands_mongodb-0.1.0.dev1.dist-info/WHEEL +4 -0
- strands_mongodb-0.1.0.dev1.dist-info/licenses/LICENSE +175 -0
- strands_mongodb-0.1.0.dev1.dist-info/licenses/NOTICE +10 -0
|
@@ -0,0 +1,1314 @@
|
|
|
1
|
+
"""Index specifications, idempotent ensure and verify, and search-index readiness polling.
|
|
2
|
+
|
|
3
|
+
Written against ``AsyncCollection`` rather than calling ``pymongo_search_utils``' index helpers, which are
|
|
4
|
+
synchronous and poll with a blocking ``time.sleep``.
|
|
5
|
+
|
|
6
|
+
Each kind of index has its own idempotency test:
|
|
7
|
+
|
|
8
|
+
* **Ordinary** indexes (TTL included) match on the **key list**, then every option this package asked for
|
|
9
|
+
is compared. Not on the name: the server rejects a second index over the same keys (code 85) whatever
|
|
10
|
+
it is called. Not on the keys alone either: the dedup index exists for its ``unique`` flag, so a
|
|
11
|
+
non-unique index over the same fields must not count.
|
|
12
|
+
* **Atlas Search** indexes match by name, are compared on ``latestDefinition``, then polled. A build is
|
|
13
|
+
asynchronous, and a search over an unfinished index returns zero documents rather than an error.
|
|
14
|
+
|
|
15
|
+
Nothing here drops or alters an existing index. A conflict is reported with the commands that resolve it,
|
|
16
|
+
because the index may be serving queries this package knows nothing about.
|
|
17
|
+
|
|
18
|
+
Private module, but the specs are a promise to operators: the README's index commands are rendered
|
|
19
|
+
from them, and a test keeps the two identical.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import asyncio
|
|
25
|
+
import json
|
|
26
|
+
import logging
|
|
27
|
+
import time
|
|
28
|
+
from collections.abc import Mapping, Sequence
|
|
29
|
+
from dataclasses import dataclass, field
|
|
30
|
+
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple
|
|
31
|
+
|
|
32
|
+
from strands_mongodb import _telemetry
|
|
33
|
+
from strands_mongodb._extras import missing_extra
|
|
34
|
+
from strands_mongodb._filter import BAKED_IN_FILTER_FIELDS
|
|
35
|
+
|
|
36
|
+
# Both surfaces reach this module and everything it needs is in the narrower `storage` extra.
|
|
37
|
+
try:
|
|
38
|
+
from pymongo import IndexModel
|
|
39
|
+
from pymongo.errors import OperationFailure, PyMongoError
|
|
40
|
+
from pymongo.operations import SearchIndexModel
|
|
41
|
+
from strands.types.exceptions import StorageError
|
|
42
|
+
except ImportError as error: # pragma: no cover - exercised in a subprocess
|
|
43
|
+
missing_extra("storage", error, module=__name__)
|
|
44
|
+
|
|
45
|
+
if TYPE_CHECKING:
|
|
46
|
+
from collections.abc import Iterable
|
|
47
|
+
|
|
48
|
+
from opentelemetry.util.types import AttributeValue
|
|
49
|
+
from pymongo.asynchronous.collection import AsyncCollection
|
|
50
|
+
|
|
51
|
+
from strands_mongodb.types import MongoDBConfig, TextEmbedder
|
|
52
|
+
|
|
53
|
+
__all__ = [
|
|
54
|
+
"DEFAULT_EMBEDDING_MODEL",
|
|
55
|
+
"IndexSpec",
|
|
56
|
+
"MAX_NUM_CANDIDATES",
|
|
57
|
+
"POLL_INTERVAL_SECONDS",
|
|
58
|
+
"READY_TIMEOUT_SECONDS",
|
|
59
|
+
"SEARCH_UNSUPPORTED_CODES",
|
|
60
|
+
"SearchIndexSpec",
|
|
61
|
+
"SearchSupport",
|
|
62
|
+
"VectorMode",
|
|
63
|
+
"check_ttl",
|
|
64
|
+
"declared_filter_fields",
|
|
65
|
+
"drop_index_command",
|
|
66
|
+
"embed",
|
|
67
|
+
"ensure",
|
|
68
|
+
"memory_search_specs",
|
|
69
|
+
"memory_specs",
|
|
70
|
+
"RANK_FUSION_MIN_MAJOR",
|
|
71
|
+
"rank_fusion_support",
|
|
72
|
+
"readiness",
|
|
73
|
+
"search_support",
|
|
74
|
+
"storage_specs",
|
|
75
|
+
"text_index_definition",
|
|
76
|
+
"vector_index_definition",
|
|
77
|
+
"vector_mode",
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
logger = logging.getLogger(__name__)
|
|
81
|
+
|
|
82
|
+
CONTENT_FIELD: Final[str] = "content"
|
|
83
|
+
"""Document field holding the text a memory is about, indexed for vector and full-text search."""
|
|
84
|
+
|
|
85
|
+
VECTOR_FIELD: Final[str] = "embedding"
|
|
86
|
+
"""Document field holding a caller-supplied embedding, on the manual-vector path only."""
|
|
87
|
+
|
|
88
|
+
EXPIRES_FIELD: Final[str] = "expires_at"
|
|
89
|
+
"""Document field the TTL indexes watch. Absent on documents with no expiry, for which the index is inert."""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def check_ttl(ttl_seconds: int | None) -> None:
|
|
93
|
+
"""Reject a negative ``ttl_seconds``.
|
|
94
|
+
|
|
95
|
+
A negative lifetime stamps an ``expires_at`` in the past, so the TTL monitor deletes every write on its
|
|
96
|
+
next sweep. ``0`` is allowed: it asks for expiry at the next sweep.
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
ttl_seconds: The configured lifetime, or ``None`` for no expiry.
|
|
100
|
+
|
|
101
|
+
Raises:
|
|
102
|
+
ValueError: If ``ttl_seconds`` is below ``0``.
|
|
103
|
+
"""
|
|
104
|
+
if ttl_seconds is not None and ttl_seconds < 0:
|
|
105
|
+
raise ValueError(
|
|
106
|
+
f"strands_mongodb: 'ttl_seconds' is {ttl_seconds}, and a lifetime cannot be negative. A negative "
|
|
107
|
+
f"value stamps an expiry in the past, so every write would be deleted on the next TTL sweep. Use "
|
|
108
|
+
f"0 to expire at the next sweep, or None for no expiry."
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
DEFAULT_EMBEDDING_MODEL: Final[str] = "voyage-4-lite"
|
|
113
|
+
"""Atlas embedding model used when the config names none. Matches the community ``strands-mongodb-store``."""
|
|
114
|
+
|
|
115
|
+
DEFAULT_VECTOR_INDEX_NAME: Final[str] = "vector_index"
|
|
116
|
+
"""Default Atlas Vector Search index name."""
|
|
117
|
+
|
|
118
|
+
DEFAULT_TEXT_INDEX_NAME: Final[str] = "text_index"
|
|
119
|
+
"""Default Atlas Search index name."""
|
|
120
|
+
|
|
121
|
+
SIMILARITIES: Final[tuple[str, ...]] = ("cosine", "dotProduct", "euclidean")
|
|
122
|
+
"""Distance metrics an Atlas vector index accepts. Mirrors ``types.VectorSimilarity`` at runtime."""
|
|
123
|
+
|
|
124
|
+
MAX_DIMENSIONS: Final[int] = 8192
|
|
125
|
+
"""Upper bound Atlas enforces on ``numDimensions``. Measured: outside ``[1..8192]`` is code 8."""
|
|
126
|
+
|
|
127
|
+
MAX_NUM_CANDIDATES: Final[int] = 10000
|
|
128
|
+
"""Upper bound the server enforces on ``numCandidates``.
|
|
129
|
+
|
|
130
|
+
Outside ``[1..10000]`` the server returns code 8, *"numCandidates" must be within bounds [1..10000]*
|
|
131
|
+
(measured on mongot 8.3.8). An integration test re-reads it from a real deployment.
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
READY_TIMEOUT_SECONDS: Final[float] = 300.0
|
|
135
|
+
"""Default seconds to wait for search-index builds to report ``READY``.
|
|
136
|
+
|
|
137
|
+
Builds on cloud Atlas have taken about 105 seconds idle and over 120 seconds on a busy cluster. A longer
|
|
138
|
+
timeout delays only a failure. Override it with ``MongoDBConfig["index_ready_timeout"]``.
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
POLL_INTERVAL_SECONDS: Final[float] = 0.5
|
|
142
|
+
"""Delay between readiness reads, in seconds."""
|
|
143
|
+
|
|
144
|
+
_READY_STATUS: Final[str] = "READY"
|
|
145
|
+
_FAILED_STATUS: Final[str] = "FAILED"
|
|
146
|
+
_NAMESPACE_EXISTS: Final[int] = 48
|
|
147
|
+
|
|
148
|
+
RANK_FUSION_MIN_MAJOR: Final[int] = 8
|
|
149
|
+
"""Major version of ``mongod`` from which ``$rankFusion`` (with ``$vectorSearch`` inputs) exists: 8.0, not 8.1."""
|
|
150
|
+
|
|
151
|
+
SEARCH_UNSUPPORTED_CODES: Final[frozenset[int]] = frozenset({31082, 115, 6047401})
|
|
152
|
+
"""Server error codes from ``list_search_indexes()`` that mean "this deployment has no Atlas Search".
|
|
153
|
+
|
|
154
|
+
* **31082** on ``mongo:8`` (8.3.8).
|
|
155
|
+
* **115** (``CommandNotSupported``) on enterprise ``mongod`` 7.0.12 with no mongot configured.
|
|
156
|
+
* **6047401** on ``mongo:6`` (6.0.28) and ``mongo:7`` (7.0.40).
|
|
157
|
+
|
|
158
|
+
The number is the stable part: the message differs between deployments, and so does ``codeName`` (31082 is
|
|
159
|
+
``SearchNotEnabled`` on 8.3.8 and ``Location31082`` on 7.0.12).
|
|
160
|
+
|
|
161
|
+
Code 59 (``CommandNotFound``) means the same at ``create_search_index`` on 6.0 and 7.0, but is left out on
|
|
162
|
+
purpose: that call only runs after :func:`search_support` has cleared the deployment, so it is unreachable.
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
_COLLECTION_ABSENT: Final[int] = 26
|
|
166
|
+
"""Server error code for a namespace that does not exist. Not an error when reading indexes."""
|
|
167
|
+
|
|
168
|
+
SIMPLE_LOCALE: Final[str] = "simple"
|
|
169
|
+
"""The one collation locale that compares strings byte by byte. See :func:`require_binary_comparison`."""
|
|
170
|
+
|
|
171
|
+
Readiness = Literal["ready", "building", "failed"]
|
|
172
|
+
"""The three states of a search index build, each with a different remedy."""
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class SearchSupport(NamedTuple):
|
|
176
|
+
"""What :func:`search_support` found.
|
|
177
|
+
|
|
178
|
+
Attributes:
|
|
179
|
+
available: Whether the deployment runs Atlas Search.
|
|
180
|
+
code: The server error code that said it does not, so messages can name the code this server
|
|
181
|
+
sent. Set only when ``available`` is ``False``, otherwise ``None``.
|
|
182
|
+
"""
|
|
183
|
+
|
|
184
|
+
available: bool
|
|
185
|
+
code: int | None = None
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def readiness(status: str) -> Readiness:
|
|
189
|
+
"""Map a search index's reported ``status`` onto the state whose remedy the caller needs.
|
|
190
|
+
|
|
191
|
+
Any other status (a new full-text index reports ``PENDING``, for one) reads as building, which costs
|
|
192
|
+
a poll. Reading it as ready would search an unfinished index, which returns nothing and no error.
|
|
193
|
+
|
|
194
|
+
Args:
|
|
195
|
+
status: The ``status`` field as the server reported it.
|
|
196
|
+
|
|
197
|
+
Returns:
|
|
198
|
+
``"ready"`` only for ``READY``, ``"failed"`` only for ``FAILED``, ``"building"`` otherwise.
|
|
199
|
+
"""
|
|
200
|
+
if status == _READY_STATUS:
|
|
201
|
+
return "ready"
|
|
202
|
+
if status == _FAILED_STATUS:
|
|
203
|
+
return "failed"
|
|
204
|
+
return "building"
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
@dataclass(frozen=True)
|
|
208
|
+
class VectorMode:
|
|
209
|
+
"""Which of the two vector paths a config selected, resolved once and passed around.
|
|
210
|
+
|
|
211
|
+
Attributes:
|
|
212
|
+
model: Atlas embedding model, on the AutoEmbeddings path. ``None`` on the manual path.
|
|
213
|
+
dimensions: Vector length, on the manual path. ``None`` on the AutoEmbeddings path.
|
|
214
|
+
similarity: Distance metric, on the manual path. ``None`` on the AutoEmbeddings path.
|
|
215
|
+
"""
|
|
216
|
+
|
|
217
|
+
model: str | None = None
|
|
218
|
+
dimensions: int | None = None
|
|
219
|
+
similarity: str | None = None
|
|
220
|
+
|
|
221
|
+
@property
|
|
222
|
+
def auto_embed(self) -> bool:
|
|
223
|
+
"""Whether the deployment embeds the text, rather than the caller.
|
|
224
|
+
|
|
225
|
+
Returns:
|
|
226
|
+
``True`` when a model is set.
|
|
227
|
+
"""
|
|
228
|
+
return self.model is not None
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def vector_mode(config: MongoDBConfig) -> VectorMode:
|
|
232
|
+
"""Resolve the vector path from the config, rejecting the combinations that mean two things.
|
|
233
|
+
|
|
234
|
+
Manual values are checked here because Atlas rejects them only at index creation, with a code 8.
|
|
235
|
+
|
|
236
|
+
Args:
|
|
237
|
+
config: The connection configuration. Only the vector-related keys are read.
|
|
238
|
+
|
|
239
|
+
Returns:
|
|
240
|
+
The resolved mode, with exactly one path populated. An empty config resolves to AutoEmbeddings
|
|
241
|
+
with :data:`DEFAULT_EMBEDDING_MODEL`.
|
|
242
|
+
|
|
243
|
+
Raises:
|
|
244
|
+
ValueError: If both paths are configured, if only one of the manual pair is set, or if a manual
|
|
245
|
+
value is outside what Atlas accepts. The message names the conflicting keys.
|
|
246
|
+
"""
|
|
247
|
+
model = config.get("embedding_model")
|
|
248
|
+
dimensions = config.get("embedding_dimensions")
|
|
249
|
+
similarity = config.get("similarity")
|
|
250
|
+
|
|
251
|
+
if model is not None and (dimensions is not None or similarity is not None):
|
|
252
|
+
also = ", ".join(
|
|
253
|
+
name
|
|
254
|
+
for name, value in (
|
|
255
|
+
("embedding_dimensions", dimensions),
|
|
256
|
+
("similarity", similarity),
|
|
257
|
+
)
|
|
258
|
+
if value is not None
|
|
259
|
+
)
|
|
260
|
+
raise ValueError(
|
|
261
|
+
f"strands_mongodb: 'embedding_model' selects the Atlas AutoEmbeddings path, where the "
|
|
262
|
+
f"deployment embeds the text, and {also} selects the manual path, where you supply the "
|
|
263
|
+
f"vectors. They index different fields with different contents, so set one or the other. "
|
|
264
|
+
f"Remove 'embedding_model' to embed client-side, or remove {also} to let Atlas do it."
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
if dimensions is None and similarity is None:
|
|
268
|
+
return VectorMode(model=model or DEFAULT_EMBEDDING_MODEL)
|
|
269
|
+
|
|
270
|
+
if dimensions is None:
|
|
271
|
+
raise ValueError(
|
|
272
|
+
"strands_mongodb: 'similarity' is set but 'embedding_dimensions' is not, and a vector "
|
|
273
|
+
"index cannot be created without the vector length. Set 'embedding_dimensions' to the "
|
|
274
|
+
"length your embedding model produces, or remove 'similarity' to use Atlas "
|
|
275
|
+
"AutoEmbeddings."
|
|
276
|
+
)
|
|
277
|
+
if similarity is None:
|
|
278
|
+
raise ValueError(
|
|
279
|
+
"strands_mongodb: 'embedding_dimensions' is set but 'similarity' is not. There is no "
|
|
280
|
+
"default: the right metric is a property of your embeddings, and the wrong one returns "
|
|
281
|
+
f"plausible neighbours that are the wrong neighbours. Set one of {', '.join(SIMILARITIES)}."
|
|
282
|
+
)
|
|
283
|
+
if not 1 <= dimensions <= MAX_DIMENSIONS:
|
|
284
|
+
raise ValueError(
|
|
285
|
+
f"strands_mongodb: 'embedding_dimensions' is {dimensions}, and Atlas requires a value "
|
|
286
|
+
f"within bounds [1..{MAX_DIMENSIONS}]. Set it to the length your embedding model produces."
|
|
287
|
+
)
|
|
288
|
+
if similarity not in SIMILARITIES:
|
|
289
|
+
raise ValueError(
|
|
290
|
+
f"strands_mongodb: 'similarity' is {similarity!r}, which Atlas does not accept. Choose "
|
|
291
|
+
f"one of {', '.join(SIMILARITIES)}."
|
|
292
|
+
)
|
|
293
|
+
return VectorMode(dimensions=dimensions, similarity=similarity)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def declared_filter_fields(extra: Iterable[str]) -> tuple[str, ...]:
|
|
297
|
+
"""Return every path the vector index declares filterable, in index order.
|
|
298
|
+
|
|
299
|
+
Shared by the index definition and ``_filter``'s check of filter paths, so the two cannot drift.
|
|
300
|
+
|
|
301
|
+
Args:
|
|
302
|
+
extra: The config's ``filter_fields``. Duplicates of the baked-in fields are dropped, since Atlas
|
|
303
|
+
rejects a definition with two ``filter`` entries on one path.
|
|
304
|
+
|
|
305
|
+
Returns:
|
|
306
|
+
``scope`` and ``store_name`` first, then the caller's fields in the order given.
|
|
307
|
+
"""
|
|
308
|
+
fields = list(BAKED_IN_FILTER_FIELDS)
|
|
309
|
+
for path in extra:
|
|
310
|
+
if path not in fields:
|
|
311
|
+
fields.append(path)
|
|
312
|
+
return tuple(fields)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def vector_index_definition(
|
|
316
|
+
mode: VectorMode,
|
|
317
|
+
*,
|
|
318
|
+
text_field: str = CONTENT_FIELD,
|
|
319
|
+
vector_field: str = VECTOR_FIELD,
|
|
320
|
+
filter_fields: Iterable[str] = (),
|
|
321
|
+
) -> dict[str, Any]:
|
|
322
|
+
"""Build the Atlas Vector Search index definition for the ``memory`` collection.
|
|
323
|
+
|
|
324
|
+
AutoEmbeddings uses ``type: "autoEmbed"`` over the text field (``numDimensions: -1`` is code 8).
|
|
325
|
+
|
|
326
|
+
Args:
|
|
327
|
+
mode: The resolved path, from :func:`vector_mode`.
|
|
328
|
+
text_field: Field the deployment embeds, on the AutoEmbeddings path.
|
|
329
|
+
vector_field: Field holding the caller's vectors, on the manual path.
|
|
330
|
+
filter_fields: Extra paths to declare filterable, beyond ``scope`` and ``store_name``.
|
|
331
|
+
|
|
332
|
+
Returns:
|
|
333
|
+
A definition ready for ``SearchIndexModel``, with the vector field first and every filter path
|
|
334
|
+
after it.
|
|
335
|
+
"""
|
|
336
|
+
if mode.auto_embed:
|
|
337
|
+
first: dict[str, Any] = {
|
|
338
|
+
"type": "autoEmbed",
|
|
339
|
+
"path": text_field,
|
|
340
|
+
"model": mode.model,
|
|
341
|
+
"modality": "text",
|
|
342
|
+
}
|
|
343
|
+
else:
|
|
344
|
+
first = {
|
|
345
|
+
"type": "vector",
|
|
346
|
+
"path": vector_field,
|
|
347
|
+
"numDimensions": mode.dimensions,
|
|
348
|
+
"similarity": mode.similarity,
|
|
349
|
+
}
|
|
350
|
+
return {
|
|
351
|
+
"fields": [
|
|
352
|
+
first,
|
|
353
|
+
*({"type": "filter", "path": path} for path in declared_filter_fields(filter_fields)),
|
|
354
|
+
]
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def text_index_definition(*, text_field: str = CONTENT_FIELD) -> dict[str, Any]:
|
|
359
|
+
"""Build the Atlas Search full-text index definition for the ``memory`` collection.
|
|
360
|
+
|
|
361
|
+
``dynamic: False`` indexes only the text field, not metadata no query reads.
|
|
362
|
+
|
|
363
|
+
Args:
|
|
364
|
+
text_field: The field to map as a string.
|
|
365
|
+
|
|
366
|
+
Returns:
|
|
367
|
+
A definition ready for ``SearchIndexModel``.
|
|
368
|
+
"""
|
|
369
|
+
return {"mappings": {"dynamic": False, "fields": {text_field: [{"type": "string"}]}}}
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
@dataclass(frozen=True)
|
|
373
|
+
class IndexSpec:
|
|
374
|
+
"""One ordinary index: what to create, and what makes an existing index count as it.
|
|
375
|
+
|
|
376
|
+
Attributes:
|
|
377
|
+
name: Name used when this package creates the index. Never used to find one, since the server
|
|
378
|
+
does not treat the name as an identity.
|
|
379
|
+
keys: The key list, in order, used to match existing indexes. It can match several: the
|
|
380
|
+
server's identity for an index is its keys and its collation, and every match is examined.
|
|
381
|
+
options: Every option this package asks for, and so every option compared against an existing
|
|
382
|
+
index. Options set by someone else (``background``, ``hidden``) are not a mismatch.
|
|
383
|
+
date_only: Whether every key names a field that holds only BSON dates. A collation cannot change
|
|
384
|
+
how such an index compares, so the collation checks skip it.
|
|
385
|
+
"""
|
|
386
|
+
|
|
387
|
+
name: str
|
|
388
|
+
keys: tuple[tuple[str, int], ...]
|
|
389
|
+
options: Mapping[str, Any] = field(default_factory=dict)
|
|
390
|
+
date_only: bool = False
|
|
391
|
+
|
|
392
|
+
def to_model(self) -> IndexModel:
|
|
393
|
+
"""Build the driver object that creates this index.
|
|
394
|
+
|
|
395
|
+
Returns:
|
|
396
|
+
An ``IndexModel`` carrying the keys, the name and every option.
|
|
397
|
+
"""
|
|
398
|
+
return IndexModel(list(self.keys), name=self.name, **dict(self.options))
|
|
399
|
+
|
|
400
|
+
def creation_command(self, collection_name: str) -> str:
|
|
401
|
+
"""Render the raw ``createIndexes`` command (what pymongo sends) for an operator to run.
|
|
402
|
+
|
|
403
|
+
Args:
|
|
404
|
+
collection_name: Collection the index belongs on.
|
|
405
|
+
|
|
406
|
+
Returns:
|
|
407
|
+
A single-line ``db.runCommand({...})`` whose argument is valid JSON.
|
|
408
|
+
"""
|
|
409
|
+
command = {
|
|
410
|
+
"createIndexes": collection_name,
|
|
411
|
+
"indexes": [{"key": dict(self.keys), "name": self.name, **dict(self.options)}],
|
|
412
|
+
}
|
|
413
|
+
return f"db.runCommand({json.dumps(command)})"
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def drop_index_command(collection_name: str, index_name: str) -> str:
|
|
417
|
+
"""Render the command that removes a conflicting ordinary index, for a mismatch message.
|
|
418
|
+
|
|
419
|
+
Rendered, never run: the index may be serving queries this package knows nothing about.
|
|
420
|
+
|
|
421
|
+
Args:
|
|
422
|
+
collection_name: Collection the index sits on.
|
|
423
|
+
index_name: Name the deployment reported for it.
|
|
424
|
+
|
|
425
|
+
Returns:
|
|
426
|
+
A single-line ``db.runCommand({...})`` whose argument is valid JSON.
|
|
427
|
+
"""
|
|
428
|
+
# Encoded, not interpolated: the name is whatever the deployment reports, and an operator pastes this.
|
|
429
|
+
return f"db.runCommand({json.dumps({'dropIndexes': collection_name, 'index': index_name})})"
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
@dataclass(frozen=True)
|
|
433
|
+
class SearchIndexSpec:
|
|
434
|
+
"""One Atlas Search index: ``vectorSearch`` or ``search``.
|
|
435
|
+
|
|
436
|
+
Attributes:
|
|
437
|
+
name: The index name, which for a search index is its identity. ``createSearchIndexes`` rejects a
|
|
438
|
+
different definition under an existing name with code 68.
|
|
439
|
+
definition: The definition. An existing index must contain it (see ``_definition_gap``).
|
|
440
|
+
type: ``"vectorSearch"`` or ``"search"``.
|
|
441
|
+
"""
|
|
442
|
+
|
|
443
|
+
name: str
|
|
444
|
+
definition: Mapping[str, Any]
|
|
445
|
+
type: str
|
|
446
|
+
|
|
447
|
+
def to_model(self) -> SearchIndexModel:
|
|
448
|
+
"""Build the driver object that creates this index.
|
|
449
|
+
|
|
450
|
+
Returns:
|
|
451
|
+
A ``SearchIndexModel`` carrying the definition, the name and the type.
|
|
452
|
+
"""
|
|
453
|
+
return SearchIndexModel(definition=dict(self.definition), name=self.name, type=self.type)
|
|
454
|
+
|
|
455
|
+
def creation_command(self, collection_name: str) -> str:
|
|
456
|
+
"""Render the ``createSearchIndexes`` command that creates this index, for an operator to run.
|
|
457
|
+
|
|
458
|
+
Args:
|
|
459
|
+
collection_name: Collection the index belongs on.
|
|
460
|
+
|
|
461
|
+
Returns:
|
|
462
|
+
A single-line ``db.runCommand({...})`` whose argument is valid JSON.
|
|
463
|
+
"""
|
|
464
|
+
command = {
|
|
465
|
+
"createSearchIndexes": collection_name,
|
|
466
|
+
"indexes": [{"name": self.name, "type": self.type, "definition": dict(self.definition)}],
|
|
467
|
+
}
|
|
468
|
+
return f"db.runCommand({json.dumps(command)})"
|
|
469
|
+
|
|
470
|
+
def drop_command(self, collection_name: str) -> str:
|
|
471
|
+
"""Render the command that removes this search index, for a mismatch or failure message.
|
|
472
|
+
|
|
473
|
+
Args:
|
|
474
|
+
collection_name: Collection the index sits on.
|
|
475
|
+
|
|
476
|
+
Returns:
|
|
477
|
+
A single-line ``db.runCommand({...})`` whose argument is valid JSON.
|
|
478
|
+
"""
|
|
479
|
+
return f"db.runCommand({json.dumps({'dropSearchIndex': collection_name, 'name': self.name})})"
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def storage_specs() -> tuple[IndexSpec, ...]:
|
|
483
|
+
"""Return the ordinary indexes the ``storage`` collection needs.
|
|
484
|
+
|
|
485
|
+
Keys are stored in ``_id``, whose index the server creates, so only the TTL index remains.
|
|
486
|
+
|
|
487
|
+
Returns:
|
|
488
|
+
One spec, the same for every store.
|
|
489
|
+
"""
|
|
490
|
+
return (
|
|
491
|
+
IndexSpec(
|
|
492
|
+
name="strands_expires_at_ttl",
|
|
493
|
+
keys=((EXPIRES_FIELD, 1),),
|
|
494
|
+
options={"expireAfterSeconds": 0},
|
|
495
|
+
# This package only ever writes `expires_at` as a BSON date, so a collation cannot change what
|
|
496
|
+
# this index compares.
|
|
497
|
+
date_only=True,
|
|
498
|
+
),
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def memory_specs() -> tuple[IndexSpec, ...]:
|
|
503
|
+
"""Return the ordinary indexes the ``memory`` collection needs.
|
|
504
|
+
|
|
505
|
+
The TTL index exists whether or not ``ttl_seconds`` is set, so a store that sets one later needs no index
|
|
506
|
+
change: its memories expire from the first write after that. Memories written before carry no
|
|
507
|
+
``expires_at`` and never expire, because the index is inert for documents without the field.
|
|
508
|
+
|
|
509
|
+
Returns:
|
|
510
|
+
Three specs: the unique partial dedup index, the recency index, and the TTL index.
|
|
511
|
+
"""
|
|
512
|
+
return (
|
|
513
|
+
IndexSpec(
|
|
514
|
+
name="strands_scope_store_content_hash_unique",
|
|
515
|
+
keys=(("scope", 1), ("store_name", 1), ("content_hash", 1)),
|
|
516
|
+
options={
|
|
517
|
+
"unique": True,
|
|
518
|
+
# A unique index treats a missing field as `null`, so documents without a hash would collide.
|
|
519
|
+
"partialFilterExpression": {"content_hash": {"$exists": True}},
|
|
520
|
+
},
|
|
521
|
+
),
|
|
522
|
+
IndexSpec(
|
|
523
|
+
name="strands_scope_store_created_at",
|
|
524
|
+
keys=(("scope", 1), ("store_name", 1), ("created_at", -1)),
|
|
525
|
+
),
|
|
526
|
+
IndexSpec(
|
|
527
|
+
name="strands_expires_at_ttl",
|
|
528
|
+
keys=((EXPIRES_FIELD, 1),),
|
|
529
|
+
options={"expireAfterSeconds": 0},
|
|
530
|
+
# This package only ever writes `expires_at` as a BSON date, so a collation cannot change what
|
|
531
|
+
# this index compares.
|
|
532
|
+
date_only=True,
|
|
533
|
+
),
|
|
534
|
+
)
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def memory_search_specs(config: MongoDBConfig) -> tuple[SearchIndexSpec, ...]:
|
|
538
|
+
"""Return the Atlas Search indexes the ``memory`` collection needs.
|
|
539
|
+
|
|
540
|
+
Args:
|
|
541
|
+
config: The connection configuration.
|
|
542
|
+
|
|
543
|
+
Returns:
|
|
544
|
+
The vector index first, then the full-text index.
|
|
545
|
+
|
|
546
|
+
Raises:
|
|
547
|
+
ValueError: From :func:`vector_mode`, if the vector configuration means two things.
|
|
548
|
+
"""
|
|
549
|
+
mode = vector_mode(config)
|
|
550
|
+
return (
|
|
551
|
+
SearchIndexSpec(
|
|
552
|
+
name=config.get("vector_index_name") or DEFAULT_VECTOR_INDEX_NAME,
|
|
553
|
+
definition=vector_index_definition(mode, filter_fields=config.get("filter_fields") or ()),
|
|
554
|
+
type="vectorSearch",
|
|
555
|
+
),
|
|
556
|
+
SearchIndexSpec(
|
|
557
|
+
name=config.get("text_index_name") or DEFAULT_TEXT_INDEX_NAME,
|
|
558
|
+
definition=text_index_definition(),
|
|
559
|
+
type="search",
|
|
560
|
+
),
|
|
561
|
+
)
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
async def search_support(collection: AsyncCollection[Any]) -> SearchSupport:
|
|
565
|
+
"""Report whether this deployment can run Atlas Search at all, and on what evidence.
|
|
566
|
+
|
|
567
|
+
Called before :func:`ensure` gets search specs, so a deployment without mongot gets its ordinary
|
|
568
|
+
indexes and a warning. Probing first keeps errors from ``createSearchIndexes`` on a deployment that
|
|
569
|
+
has mongot from being mistaken for "no search".
|
|
570
|
+
|
|
571
|
+
The probe calls ``list_search_indexes`` because ``buildInfo`` cannot tell: ``mongo:8`` and an Atlas
|
|
572
|
+
CLI local deployment both report empty ``modules`` and only the second has search, while enterprise
|
|
573
|
+
7.0.12 reports ``modules: ['enterprise']`` and has none. The result is always truthy; test
|
|
574
|
+
``.available``.
|
|
575
|
+
|
|
576
|
+
Args:
|
|
577
|
+
collection: The collection whose deployment to probe. Need not exist.
|
|
578
|
+
|
|
579
|
+
Returns:
|
|
580
|
+
:class:`SearchSupport`. ``available`` is ``True`` if the deployment answers the call, ``False``
|
|
581
|
+
if it answers with one of :data:`SEARCH_UNSUPPORTED_CODES`, in which case ``code`` carries the
|
|
582
|
+
one it used.
|
|
583
|
+
|
|
584
|
+
Raises:
|
|
585
|
+
StorageError: If the call fails for any other reason, including authentication. A bad password
|
|
586
|
+
must not read as "no search".
|
|
587
|
+
"""
|
|
588
|
+
with _telemetry.span("strands_mongodb.indexes.probe", _probe_attributes(collection, "search")):
|
|
589
|
+
return await _search_support(collection)
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
async def _search_support(collection: AsyncCollection[Any]) -> SearchSupport:
|
|
593
|
+
"""The body of :func:`search_support`, inside its probe span."""
|
|
594
|
+
try:
|
|
595
|
+
cursor = await collection.list_search_indexes()
|
|
596
|
+
async for _ in cursor:
|
|
597
|
+
break
|
|
598
|
+
except OperationFailure as error:
|
|
599
|
+
if error.code in SEARCH_UNSUPPORTED_CODES:
|
|
600
|
+
return SearchSupport(available=False, code=error.code)
|
|
601
|
+
# Code 26 (absent collection) still means the call worked, so search is available.
|
|
602
|
+
if error.code != _COLLECTION_ABSENT:
|
|
603
|
+
raise StorageError(
|
|
604
|
+
f"strands_mongodb: probing Atlas Search support on "
|
|
605
|
+
f"{collection.database.name}.{collection.name} failed: {error}"
|
|
606
|
+
) from error
|
|
607
|
+
except PyMongoError as error:
|
|
608
|
+
raise StorageError(
|
|
609
|
+
f"strands_mongodb: probing Atlas Search support on "
|
|
610
|
+
f"{collection.database.name}.{collection.name} failed: {error}"
|
|
611
|
+
) from error
|
|
612
|
+
return SearchSupport(available=True)
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
class CollationReport(NamedTuple):
|
|
616
|
+
"""What ``listCollections`` says about one collection's string comparison.
|
|
617
|
+
|
|
618
|
+
An absent collection and an uncollated one both have no collation, but only the second is verified.
|
|
619
|
+
|
|
620
|
+
Attributes:
|
|
621
|
+
exists: Whether the server reported the collection at all.
|
|
622
|
+
collation: The collation document it reported, or ``None`` if there was none (or no collection).
|
|
623
|
+
"""
|
|
624
|
+
|
|
625
|
+
exists: bool
|
|
626
|
+
collation: dict[str, Any] | None
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
async def read_collation(collection: AsyncCollection[Any]) -> CollationReport:
|
|
630
|
+
"""Report whether the collection exists and what default collation it carries.
|
|
631
|
+
|
|
632
|
+
Uses ``listCollections`` filtered to this name, so a shared database's other collections are not
|
|
633
|
+
listed. The server (8.0.32) reports a collation under ``options.collation`` and does not store a
|
|
634
|
+
simple one: a collection created with ``{locale: "simple"}`` reports empty ``options``.
|
|
635
|
+
|
|
636
|
+
Args:
|
|
637
|
+
collection: The collection to ask about. Need not exist.
|
|
638
|
+
|
|
639
|
+
Returns:
|
|
640
|
+
A :class:`CollationReport`: ``(False, None)`` for an absent collection, ``(True, None)`` for one
|
|
641
|
+
without a collation.
|
|
642
|
+
|
|
643
|
+
Raises:
|
|
644
|
+
StorageError: If the call fails. An absent collection is not a failure.
|
|
645
|
+
"""
|
|
646
|
+
with _telemetry.span("strands_mongodb.indexes.probe", _probe_attributes(collection, "collation")):
|
|
647
|
+
return await _read_collation(collection)
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
async def _read_collation(collection: AsyncCollection[Any]) -> CollationReport:
|
|
651
|
+
"""The body of :func:`read_collation`, inside its probe span."""
|
|
652
|
+
try:
|
|
653
|
+
cursor = await collection.database.list_collections(filter={"name": collection.name})
|
|
654
|
+
async for document in cursor:
|
|
655
|
+
options = document.get("options") or {}
|
|
656
|
+
collation = options.get("collation")
|
|
657
|
+
return CollationReport(exists=True, collation=dict(collation) if collation is not None else None)
|
|
658
|
+
except PyMongoError as error:
|
|
659
|
+
raise StorageError(
|
|
660
|
+
f"strands_mongodb: reading the collation of {collection.database.name}.{collection.name} failed: {error}"
|
|
661
|
+
) from error
|
|
662
|
+
return CollationReport(exists=False, collation=None)
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
async def require_binary_comparison(collection: AsyncCollection[Any]) -> bool:
|
|
666
|
+
"""Refuse a collection whose default collation is not binary, and say whether it verified one.
|
|
667
|
+
|
|
668
|
+
An isolation check. Keys live in ``_id``, and key equality, prefix ranges and ``_id`` uniqueness all
|
|
669
|
+
use the collection's default collation. Under a case-folding one, a namespace can read and delete
|
|
670
|
+
another's documents, and keys differing only in case collide. A per-query ``{"locale": "simple"}``
|
|
671
|
+
is no fix: such queries cannot use the ``_id`` index, and ``_id`` uniqueness would stay folded.
|
|
672
|
+
|
|
673
|
+
An absent collection passes, since this package creates collections implicitly without a collation,
|
|
674
|
+
but it is not verified: someone may create it a moment later with any collation.
|
|
675
|
+
|
|
676
|
+
Args:
|
|
677
|
+
collection: The collection about to be operated on.
|
|
678
|
+
|
|
679
|
+
Returns:
|
|
680
|
+
``True`` if the collection exists and compares strings byte by byte, an answer worth caching.
|
|
681
|
+
``False`` if it does not exist yet, which must not be cached.
|
|
682
|
+
|
|
683
|
+
Raises:
|
|
684
|
+
StorageError: If the collection has a collation whose locale is not :data:`SIMPLE_LOCALE`,
|
|
685
|
+
naming the locale found and the remedy. Also if reading the collation fails, with the driver's
|
|
686
|
+
error chained.
|
|
687
|
+
"""
|
|
688
|
+
report = await read_collation(collection)
|
|
689
|
+
collation = report.collation
|
|
690
|
+
if collation is None or collation.get("locale") == SIMPLE_LOCALE:
|
|
691
|
+
return report.exists
|
|
692
|
+
namespace = f"{collection.database.name}.{collection.name}"
|
|
693
|
+
raise StorageError(
|
|
694
|
+
f"strands_mongodb: the collection {namespace} was created with the collation "
|
|
695
|
+
f"{collation.get('locale')!r}, and this package cannot store keys in a collection whose string "
|
|
696
|
+
f"comparison is not binary. Keys live in '_id': a read is an equality match on it, a prefix "
|
|
697
|
+
f"listing is a range over it, and a namespace is part of it, so a collation that folds case or "
|
|
698
|
+
f"accents lets one namespace read and delete another's documents, and makes two keys differing "
|
|
699
|
+
f"only in case collide on the unique '_id' index. A collation cannot be changed in place. Point "
|
|
700
|
+
f"this store at a collection created without one -- the 'collection' argument -- or recreate "
|
|
701
|
+
f"this one with db.createCollection({collection.name!r}) and no collation, or with "
|
|
702
|
+
f"{{collation: {{locale: {SIMPLE_LOCALE!r}}}}}, which is binary comparison stated explicitly. "
|
|
703
|
+
f"Reported: {collation!r}."
|
|
704
|
+
)
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
async def rank_fusion_support(collection: AsyncCollection[Any]) -> bool:
|
|
708
|
+
"""Report whether this deployment can run ``$rankFusion``.
|
|
709
|
+
|
|
710
|
+
Unlike :func:`search_support`, this reads ``buildInfo``: ``$rankFusion`` is a ``mongod`` stage, so
|
|
711
|
+
``mongod``'s version decides it. ``versionArray`` avoids parsing suffixes such as ``-rc1``.
|
|
712
|
+
|
|
713
|
+
Every failure reads as ``False``. The answer only picks between two working search tiers, so a wrong
|
|
714
|
+
``False`` costs ranking quality while a wrong ``True`` makes every search fail.
|
|
715
|
+
|
|
716
|
+
Args:
|
|
717
|
+
collection: Any collection on the deployment to probe. Need not exist, since ``buildInfo`` is a
|
|
718
|
+
database command.
|
|
719
|
+
|
|
720
|
+
Returns:
|
|
721
|
+
``True`` if the server reports major version :data:`RANK_FUSION_MIN_MAJOR` or later.
|
|
722
|
+
"""
|
|
723
|
+
with _telemetry.span("strands_mongodb.indexes.probe", _probe_attributes(collection, "rank_fusion")):
|
|
724
|
+
return await _rank_fusion_support(collection)
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
async def _rank_fusion_support(collection: AsyncCollection[Any]) -> bool:
|
|
728
|
+
"""The body of :func:`rank_fusion_support`, inside its probe span."""
|
|
729
|
+
try:
|
|
730
|
+
build_info = await collection.database.command("buildInfo")
|
|
731
|
+
except PyMongoError:
|
|
732
|
+
return False
|
|
733
|
+
version = build_info.get("versionArray") if isinstance(build_info, Mapping) else None
|
|
734
|
+
if not isinstance(version, Sequence) or isinstance(version, str) or not version:
|
|
735
|
+
return False
|
|
736
|
+
major = version[0]
|
|
737
|
+
# `bool` is a subclass of `int`; a boolean here is not a known version.
|
|
738
|
+
if not isinstance(major, int) or isinstance(major, bool):
|
|
739
|
+
return False
|
|
740
|
+
return major >= RANK_FUSION_MIN_MAJOR
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
async def ensure(
|
|
744
|
+
collection: AsyncCollection[Any],
|
|
745
|
+
*,
|
|
746
|
+
ordinary: Sequence[IndexSpec] = (),
|
|
747
|
+
search: Sequence[SearchIndexSpec] = (),
|
|
748
|
+
create: bool = True,
|
|
749
|
+
timeout: float = READY_TIMEOUT_SECONDS,
|
|
750
|
+
interval: float = POLL_INTERVAL_SECONDS,
|
|
751
|
+
) -> None:
|
|
752
|
+
"""Bring a collection's indexes to the declared state, or explain what is stopping that.
|
|
753
|
+
|
|
754
|
+
Safe to call on every ``initialize()``: when everything is in place it costs one read for the
|
|
755
|
+
ordinary indexes plus one per search index, and no writes. ``create=False`` verifies without
|
|
756
|
+
creating (for credentials with no DDL rights) and still waits for a build someone else started.
|
|
757
|
+
Nothing here drops or alters an existing index; each conflict is reported with the commands that
|
|
758
|
+
resolve it.
|
|
759
|
+
|
|
760
|
+
Args:
|
|
761
|
+
collection: The collection to index. Its ``database`` is used to create the collection before a
|
|
762
|
+
search index is created.
|
|
763
|
+
ordinary: Ordinary and TTL indexes, from :func:`storage_specs` or :func:`memory_specs`.
|
|
764
|
+
search: Atlas Search indexes, from :func:`memory_search_specs`.
|
|
765
|
+
create: Whether missing indexes may be created.
|
|
766
|
+
timeout: Seconds to wait for search builds, in total across all of them.
|
|
767
|
+
interval: Seconds between readiness reads.
|
|
768
|
+
|
|
769
|
+
Raises:
|
|
770
|
+
StorageError: If an existing index conflicts with a declared one, if a search build has failed,
|
|
771
|
+
if a build does not finish inside ``timeout``, if something is missing and ``create`` is
|
|
772
|
+
``False``, or if a driver call fails. Driver errors are chained, so the server's code and
|
|
773
|
+
message survive.
|
|
774
|
+
"""
|
|
775
|
+
attributes = _namespace_attributes(collection)
|
|
776
|
+
attributes["strands_mongodb.indexes.mode"] = "ensure" if create else "verify"
|
|
777
|
+
# A list the helpers append to, so the span still gets the count when one of them raises part way.
|
|
778
|
+
created: list[int] = []
|
|
779
|
+
with _telemetry.span("strands_mongodb.indexes.ensure", attributes):
|
|
780
|
+
try:
|
|
781
|
+
if ordinary:
|
|
782
|
+
await _ensure_ordinary(collection, tuple(ordinary), create=create, created=created)
|
|
783
|
+
if search:
|
|
784
|
+
await _ensure_search(
|
|
785
|
+
collection, tuple(search), create=create, timeout=timeout, interval=interval, created=created
|
|
786
|
+
)
|
|
787
|
+
finally:
|
|
788
|
+
_telemetry.annotate("strands_mongodb.indexes.created", sum(created))
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
def _namespace_attributes(collection: AsyncCollection[Any]) -> dict[_telemetry.Attribute, AttributeValue]:
|
|
792
|
+
"""The database and collection, which every index span carries."""
|
|
793
|
+
return {"strands_mongodb.database": collection.database.name, "strands_mongodb.collection": collection.name}
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _probe_attributes(
|
|
797
|
+
collection: AsyncCollection[Any], probe: Literal["search", "rank_fusion", "collation"]
|
|
798
|
+
) -> dict[_telemetry.Attribute, AttributeValue]:
|
|
799
|
+
"""A probe span's attributes: the namespace and what it probed."""
|
|
800
|
+
attributes = _namespace_attributes(collection)
|
|
801
|
+
attributes["strands_mongodb.indexes.probe"] = probe
|
|
802
|
+
return attributes
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
async def _ensure_ordinary(
|
|
806
|
+
collection: AsyncCollection[Any], specs: tuple[IndexSpec, ...], *, create: bool, created: list[int]
|
|
807
|
+
) -> None:
|
|
808
|
+
"""Create or verify the ordinary indexes, matching on the key list."""
|
|
809
|
+
namespace = f"{collection.database.name}.{collection.name}"
|
|
810
|
+
present = await _guard(collection.index_information(), what=f"reading indexes on {namespace}")
|
|
811
|
+
# Several indexes can share a key list (identity is keys plus collation), so keep all of them.
|
|
812
|
+
by_keys: dict[tuple[tuple[str, int], ...], list[tuple[str, Mapping[str, Any]]]] = {}
|
|
813
|
+
for name, found in present.items():
|
|
814
|
+
by_keys.setdefault(tuple(tuple(pair) for pair in found["key"]), []).append((name, found))
|
|
815
|
+
|
|
816
|
+
missing: list[IndexSpec] = []
|
|
817
|
+
for spec in specs:
|
|
818
|
+
matches = by_keys.get(spec.keys)
|
|
819
|
+
if not matches:
|
|
820
|
+
missing.append(spec)
|
|
821
|
+
continue
|
|
822
|
+
_check_matches(collection.name, namespace, spec, matches)
|
|
823
|
+
|
|
824
|
+
if not missing:
|
|
825
|
+
return
|
|
826
|
+
if not create:
|
|
827
|
+
raise StorageError(_missing_ordinary_message(collection.name, namespace, missing))
|
|
828
|
+
|
|
829
|
+
logger.info(
|
|
830
|
+
"strands_mongodb: creating %d index(es) on %s: %s",
|
|
831
|
+
len(missing),
|
|
832
|
+
namespace,
|
|
833
|
+
", ".join(spec.name for spec in missing),
|
|
834
|
+
extra=_telemetry.event_extra("creating_indexes", namespace=namespace),
|
|
835
|
+
)
|
|
836
|
+
# One call for all of them. The server tolerates concurrent identical creates (10 at once: no
|
|
837
|
+
# failures, one index).
|
|
838
|
+
await _guard(
|
|
839
|
+
collection.create_indexes([spec.to_model() for spec in missing]),
|
|
840
|
+
what=f"creating indexes on {namespace}",
|
|
841
|
+
)
|
|
842
|
+
created.append(len(missing))
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
def _folding_locale(found: Mapping[str, Any]) -> str | None:
|
|
846
|
+
"""Return the locale of a reported collation that does not compare strings byte for byte.
|
|
847
|
+
|
|
848
|
+
Args:
|
|
849
|
+
found: The entry ``index_information`` reported for an index.
|
|
850
|
+
|
|
851
|
+
Returns:
|
|
852
|
+
The locale, or ``None`` when the index reports no collation or ``simple``. The server does not
|
|
853
|
+
store a simple collation, so an index created with one reports nothing.
|
|
854
|
+
"""
|
|
855
|
+
collation = found.get("collation")
|
|
856
|
+
if not isinstance(collation, Mapping):
|
|
857
|
+
return None
|
|
858
|
+
locale = collation.get("locale")
|
|
859
|
+
if locale == SIMPLE_LOCALE:
|
|
860
|
+
return None
|
|
861
|
+
return str(locale)
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
def _check_matches(
|
|
865
|
+
collection_name: str,
|
|
866
|
+
namespace: str,
|
|
867
|
+
spec: IndexSpec,
|
|
868
|
+
matches: Sequence[tuple[str, Mapping[str, Any]]],
|
|
869
|
+
) -> None:
|
|
870
|
+
"""Check every existing index over a declared index's keys, not just one.
|
|
871
|
+
|
|
872
|
+
Two checks. First, a unique index under a folding collation is refused wherever it appears: the
|
|
873
|
+
server enforces its folded uniqueness even beside a correct index, so records from different scopes
|
|
874
|
+
can collide. Second, at least one match must be usable (the right options and a binary collation).
|
|
875
|
+
Once one is, the others are someone else's business, so a non-unique folding index beside a usable
|
|
876
|
+
one is accepted. ``matches`` is every existing index over the spec's keys, in server order.
|
|
877
|
+
|
|
878
|
+
Raises:
|
|
879
|
+
StorageError: If any match enforces uniqueness under a folding collation, or if no match is
|
|
880
|
+
usable. In the second case the first match's failure is reported.
|
|
881
|
+
"""
|
|
882
|
+
if not spec.date_only:
|
|
883
|
+
for found_name, found in matches:
|
|
884
|
+
locale = _folding_locale(found)
|
|
885
|
+
if locale is not None and found.get("unique"):
|
|
886
|
+
raise StorageError(_folded_constraint_message(collection_name, namespace, spec, found_name, locale))
|
|
887
|
+
|
|
888
|
+
first: StorageError | None = None
|
|
889
|
+
for found_name, found in matches:
|
|
890
|
+
try:
|
|
891
|
+
_check_collation(collection_name, namespace, spec, found_name, found)
|
|
892
|
+
_check_options(collection_name, namespace, spec, found_name, found)
|
|
893
|
+
except StorageError as refusal:
|
|
894
|
+
first = first or refusal
|
|
895
|
+
continue
|
|
896
|
+
return
|
|
897
|
+
# The caller never passes an empty `matches`, so reaching here means every match failed.
|
|
898
|
+
assert first is not None
|
|
899
|
+
raise first
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
def _folded_constraint_message(
|
|
903
|
+
collection_name: str,
|
|
904
|
+
namespace: str,
|
|
905
|
+
spec: IndexSpec,
|
|
906
|
+
found_name: str,
|
|
907
|
+
locale: str,
|
|
908
|
+
) -> str:
|
|
909
|
+
"""Build the refusal for a unique index that enforces uniqueness under a folding collation.
|
|
910
|
+
|
|
911
|
+
Separate from ``_check_collation``'s message because here the declared index may already be present
|
|
912
|
+
and correct, so creating it does not help. The message ends with the drop command and the command
|
|
913
|
+
that creates the declared index.
|
|
914
|
+
"""
|
|
915
|
+
return (
|
|
916
|
+
f"strands_mongodb: the index {found_name!r} on {namespace} covers the keys {list(spec.keys)} "
|
|
917
|
+
f"that {spec.name!r} needs, it is unique, and it was created with the collation {locale!r}. The "
|
|
918
|
+
f"server enforces that uniqueness on the folded values, so two records differing only in the case "
|
|
919
|
+
f"of those keys collide as one -- and it goes on enforcing it whether or not a correctly collated "
|
|
920
|
+
f"index sits beside it, because an index's identity here is its keys and its collation together "
|
|
921
|
+
f"and the server keeps both indexes. This package queries and deduplicates with byte-for-byte "
|
|
922
|
+
f"comparison, so it cannot see that rule and cannot report it. Dropping this index is not this "
|
|
923
|
+
f"package's decision to make: it may be serving queries nothing here knows about. To resolve it, "
|
|
924
|
+
f"drop it, and create the declared index if it is not already there:\n"
|
|
925
|
+
f"{drop_index_command(collection_name, found_name)}\n"
|
|
926
|
+
f"{spec.creation_command(collection_name)}\n"
|
|
927
|
+
)
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
def _check_collation(
|
|
931
|
+
collection_name: str,
|
|
932
|
+
namespace: str,
|
|
933
|
+
spec: IndexSpec,
|
|
934
|
+
found_name: str,
|
|
935
|
+
found: Mapping[str, Any],
|
|
936
|
+
) -> None:
|
|
937
|
+
"""Refuse a matched index whose collation does not compare strings byte for byte.
|
|
938
|
+
|
|
939
|
+
An index carries its own collation, so a collection that passes ``require_binary_comparison`` can
|
|
940
|
+
still hold a folding unique index. The dedup rule would then be the folded one: records whose keys
|
|
941
|
+
differ only in case collide, so a memory can be reported as a duplicate and never stored.
|
|
942
|
+
|
|
943
|
+
Not an :attr:`IndexSpec.options` entry: the server does not store a simple collation, so a wanted
|
|
944
|
+
``{"locale": "simple"}`` would be absent on every correct index.
|
|
945
|
+
|
|
946
|
+
Raises:
|
|
947
|
+
StorageError: If the index reports a collation whose locale is not ``simple``. The message holds
|
|
948
|
+
the drop and recreate commands; nothing is dropped here.
|
|
949
|
+
"""
|
|
950
|
+
if spec.date_only:
|
|
951
|
+
return
|
|
952
|
+
locale = _folding_locale(found)
|
|
953
|
+
if locale is None:
|
|
954
|
+
return
|
|
955
|
+
raise StorageError(
|
|
956
|
+
f"strands_mongodb: the index {found_name!r} on {namespace} covers the keys {list(spec.keys)} "
|
|
957
|
+
f"that {spec.name!r} needs, but it was created with the collation {locale!r} and this package "
|
|
958
|
+
f"needs one that compares strings byte for byte. An index that folds case makes the rule it "
|
|
959
|
+
f"enforces the folded rule, which for {spec.name!r} means two records differing only in case "
|
|
960
|
+
f"collide as one, and a query that compares exactly cannot use it at all. The server would let a "
|
|
961
|
+
f"second index be created over these keys under a different collation, but that does not help: "
|
|
962
|
+
f"both would exist and both would be enforced. Dropping this one is not this package's decision "
|
|
963
|
+
f"to make -- it may be serving queries nothing here knows about. To resolve it, drop the existing "
|
|
964
|
+
f"index and let the next call recreate it:\n"
|
|
965
|
+
f"{drop_index_command(collection_name, found_name)}\n"
|
|
966
|
+
f"{spec.creation_command(collection_name)}\n"
|
|
967
|
+
)
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def _check_options(
|
|
971
|
+
collection_name: str,
|
|
972
|
+
namespace: str,
|
|
973
|
+
spec: IndexSpec,
|
|
974
|
+
found_name: str,
|
|
975
|
+
found: Mapping[str, Any],
|
|
976
|
+
) -> None:
|
|
977
|
+
"""Compare an existing index against the options a spec asked for, raising on the first difference.
|
|
978
|
+
|
|
979
|
+
Only options this package sets are compared. ``v``, ``background``, ``hidden`` and anything else the
|
|
980
|
+
server or another tool added are ignored.
|
|
981
|
+
"""
|
|
982
|
+
for option, wanted in spec.options.items():
|
|
983
|
+
if option in found and found[option] == wanted:
|
|
984
|
+
continue
|
|
985
|
+
actual = f"{found[option]!r}" if option in found else "absent"
|
|
986
|
+
raise StorageError(
|
|
987
|
+
f"strands_mongodb: the index {found_name!r} on {namespace} covers the keys "
|
|
988
|
+
f"{list(spec.keys)} that {spec.name!r} needs, but its {option} is {actual} and this "
|
|
989
|
+
f"package needs {wanted!r}. Creating a second index over the same keys under the same "
|
|
990
|
+
f"collation is rejected by the server whatever it is named, and dropping this one is not "
|
|
991
|
+
f"this package's decision to make: it may be serving queries nothing here knows about. To "
|
|
992
|
+
f"resolve it, drop the "
|
|
993
|
+
f"existing index and let the next call recreate it:\n"
|
|
994
|
+
f"{drop_index_command(collection_name, found_name)}\n"
|
|
995
|
+
f"{spec.creation_command(collection_name)}\n"
|
|
996
|
+
)
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def _missing_ordinary_message(collection_name: str, namespace: str, missing: Sequence[IndexSpec]) -> str:
|
|
1000
|
+
"""Build the verify-only message, listing every missing index's creation command on its own line."""
|
|
1001
|
+
commands = "\n".join(spec.creation_command(collection_name) for spec in missing)
|
|
1002
|
+
return (
|
|
1003
|
+
f"strands_mongodb: {len(missing)} required index(es) are absent from {namespace} and "
|
|
1004
|
+
f"ensure_indexes is False, so nothing was created. Run these against the database as somebody "
|
|
1005
|
+
f"holding the rights to, then call initialize() again:\n"
|
|
1006
|
+
f"{commands}\n"
|
|
1007
|
+
f"Absent: {', '.join(spec.name for spec in missing)}."
|
|
1008
|
+
)
|
|
1009
|
+
|
|
1010
|
+
|
|
1011
|
+
class SearchIndexNotReady(StorageError):
|
|
1012
|
+
"""A search index exists but did not report ``READY`` within the timeout."""
|
|
1013
|
+
|
|
1014
|
+
|
|
1015
|
+
async def _ensure_search(
|
|
1016
|
+
collection: AsyncCollection[Any],
|
|
1017
|
+
specs: tuple[SearchIndexSpec, ...],
|
|
1018
|
+
*,
|
|
1019
|
+
create: bool,
|
|
1020
|
+
timeout: float,
|
|
1021
|
+
interval: float,
|
|
1022
|
+
created: list[int],
|
|
1023
|
+
) -> None:
|
|
1024
|
+
"""Create or verify the Atlas Search indexes, then wait for anything still building."""
|
|
1025
|
+
namespace = f"{collection.database.name}.{collection.name}"
|
|
1026
|
+
found = {spec.name: await _read_search_index(collection, spec.name, namespace) for spec in specs}
|
|
1027
|
+
|
|
1028
|
+
missing: list[SearchIndexSpec] = []
|
|
1029
|
+
waiting: list[SearchIndexSpec] = []
|
|
1030
|
+
for spec in specs:
|
|
1031
|
+
document = found[spec.name]
|
|
1032
|
+
if document is None:
|
|
1033
|
+
missing.append(spec)
|
|
1034
|
+
continue
|
|
1035
|
+
state = readiness(str(document.get("status", "")))
|
|
1036
|
+
if state == "failed":
|
|
1037
|
+
raise StorageError(_failed_message(collection.name, namespace, spec, document))
|
|
1038
|
+
_check_definition(collection.name, namespace, spec, document)
|
|
1039
|
+
if state != "ready":
|
|
1040
|
+
waiting.append(spec)
|
|
1041
|
+
|
|
1042
|
+
if missing:
|
|
1043
|
+
if not create:
|
|
1044
|
+
raise StorageError(_missing_search_message(collection.name, namespace, missing))
|
|
1045
|
+
# `createSearchIndexes` on a missing collection fails with code 26, unlike `createIndexes`, which
|
|
1046
|
+
# creates it implicitly.
|
|
1047
|
+
await _create_collection(collection, namespace)
|
|
1048
|
+
for spec in missing:
|
|
1049
|
+
logger.info(
|
|
1050
|
+
"strands_mongodb: creating %s index %r on %s",
|
|
1051
|
+
spec.type,
|
|
1052
|
+
spec.name,
|
|
1053
|
+
namespace,
|
|
1054
|
+
extra=_telemetry.event_extra("creating_search_index", namespace=namespace),
|
|
1055
|
+
)
|
|
1056
|
+
await _guard(
|
|
1057
|
+
collection.create_search_index(spec.to_model()),
|
|
1058
|
+
what=f"creating search index {spec.name!r} on {namespace}",
|
|
1059
|
+
)
|
|
1060
|
+
created.append(1)
|
|
1061
|
+
waiting.extend(missing)
|
|
1062
|
+
|
|
1063
|
+
if not waiting:
|
|
1064
|
+
return
|
|
1065
|
+
# Raised after the wait span closes: a timeout is the wait's outcome (`timed_out`, status unset), and
|
|
1066
|
+
# the error belongs to the `ensure` span.
|
|
1067
|
+
with _telemetry.span("strands_mongodb.indexes.wait", _namespace_attributes(collection)):
|
|
1068
|
+
stuck = await _wait_until_ready(collection, waiting, namespace, timeout=timeout, interval=interval)
|
|
1069
|
+
_telemetry.annotate("strands_mongodb.indexes.outcome", "ready" if stuck is None else "timed_out")
|
|
1070
|
+
if stuck is not None:
|
|
1071
|
+
raise SearchIndexNotReady(
|
|
1072
|
+
f"strands_mongodb: waited {timeout}s for {len(stuck)} search index(es) on "
|
|
1073
|
+
f"{namespace} to report READY and they did not: {', '.join(stuck)}. BUILDING at this point means "
|
|
1074
|
+
f"a build slower than the timeout. A store with a named search_type can call initialize() "
|
|
1075
|
+
f"again to keep waiting; under 'auto' the store answers from its other tiers and tries this one "
|
|
1076
|
+
f"again later. Any other status is worth reading directly with "
|
|
1077
|
+
f"db.{collection.name}.getSearchIndexes()."
|
|
1078
|
+
)
|
|
1079
|
+
|
|
1080
|
+
|
|
1081
|
+
async def _read_search_index(collection: AsyncCollection[Any], name: str, namespace: str) -> dict[str, Any] | None:
|
|
1082
|
+
"""Read one search index by name, or ``None`` if it is not there.
|
|
1083
|
+
|
|
1084
|
+
``list_search_indexes`` is a coroutine that returns an async cursor, so it is awaited before it is
|
|
1085
|
+
iterated.
|
|
1086
|
+
"""
|
|
1087
|
+
try:
|
|
1088
|
+
cursor = await collection.list_search_indexes(name)
|
|
1089
|
+
async for document in cursor:
|
|
1090
|
+
return dict(document)
|
|
1091
|
+
except OperationFailure as error:
|
|
1092
|
+
# Code 26: an absent collection has no indexes. `$listSearchIndexes` is not documented to hide
|
|
1093
|
+
# this the way `list_indexes` does, so it is handled here.
|
|
1094
|
+
if error.code != _COLLECTION_ABSENT:
|
|
1095
|
+
raise StorageError(
|
|
1096
|
+
f"strands_mongodb: reading search index {name!r} on {namespace} failed: {error}"
|
|
1097
|
+
) from error
|
|
1098
|
+
except PyMongoError as error:
|
|
1099
|
+
raise StorageError(f"strands_mongodb: reading search index {name!r} on {namespace} failed: {error}") from error
|
|
1100
|
+
return None
|
|
1101
|
+
|
|
1102
|
+
|
|
1103
|
+
def _check_definition(collection_name: str, namespace: str, spec: SearchIndexSpec, document: Mapping[str, Any]) -> None:
|
|
1104
|
+
"""Check that an existing search index's definition covers the declared one.
|
|
1105
|
+
|
|
1106
|
+
The server would reject a different definition under the same name with code 68, but only on
|
|
1107
|
+
creation. This catches the mismatch on the warm path too, where nothing is created.
|
|
1108
|
+
"""
|
|
1109
|
+
definition = document.get("latestDefinition")
|
|
1110
|
+
if definition is None:
|
|
1111
|
+
return
|
|
1112
|
+
gap = _definition_gap(spec.definition, definition)
|
|
1113
|
+
if gap is None:
|
|
1114
|
+
return
|
|
1115
|
+
raise StorageError(
|
|
1116
|
+
f"strands_mongodb: the {spec.type} index {spec.name!r} on {namespace} already exists and its "
|
|
1117
|
+
f"definition does not cover the one this package needs: {gap}. Reported: {definition!r}. A "
|
|
1118
|
+
f"search index cannot be redefined in place, and dropping it discards a build somebody may be "
|
|
1119
|
+
f"querying, so the remedy is left here rather than taken:\n"
|
|
1120
|
+
f"{spec.drop_command(collection_name)}\n"
|
|
1121
|
+
f"{spec.creation_command(collection_name)}\n"
|
|
1122
|
+
)
|
|
1123
|
+
|
|
1124
|
+
|
|
1125
|
+
def _definition_gap(wanted: Any, found: Any, path: str = "definition") -> str | None:
|
|
1126
|
+
"""Describe the first place ``found`` fails to cover ``wanted``, or ``None`` if it covers it.
|
|
1127
|
+
|
|
1128
|
+
Containment, not equality, because mongot rewrites a ``search`` index's ``latestDefinition``: it
|
|
1129
|
+
collapses the one-element list in ``mappings.fields.<field>`` into the object it held and fills in
|
|
1130
|
+
``indexOptions``, ``store`` and ``norms``. Equality would fail on a text index this package just
|
|
1131
|
+
created. (A ``vectorSearch`` definition comes back unchanged.)
|
|
1132
|
+
|
|
1133
|
+
The rules:
|
|
1134
|
+
|
|
1135
|
+
* A declared object is covered when every key it declares is present and covered. Added keys are
|
|
1136
|
+
ignored.
|
|
1137
|
+
* A declared one-element list is covered by an object reported in its place.
|
|
1138
|
+
* A declared list is covered when each of its entries is covered by some reported entry, in any order.
|
|
1139
|
+
|
|
1140
|
+
Args:
|
|
1141
|
+
wanted: The declared definition, or a fragment of one.
|
|
1142
|
+
found: What the server reported in the same position.
|
|
1143
|
+
path: Dotted position within the definition, used only to say where a difference is.
|
|
1144
|
+
|
|
1145
|
+
Returns:
|
|
1146
|
+
A sentence naming the position and both values, or ``None`` when nothing is missing.
|
|
1147
|
+
"""
|
|
1148
|
+
if isinstance(wanted, Mapping):
|
|
1149
|
+
if not isinstance(found, Mapping):
|
|
1150
|
+
return f"{path} is {found!r} where this package declared the object {dict(wanted)!r}"
|
|
1151
|
+
for key, value in wanted.items():
|
|
1152
|
+
if key not in found:
|
|
1153
|
+
return f"{path}.{key} is absent, and this package declared {value!r}"
|
|
1154
|
+
gap = _definition_gap(value, found[key], f"{path}.{key}")
|
|
1155
|
+
if gap is not None:
|
|
1156
|
+
return gap
|
|
1157
|
+
return None
|
|
1158
|
+
|
|
1159
|
+
if isinstance(wanted, list):
|
|
1160
|
+
if len(wanted) == 1 and isinstance(found, Mapping):
|
|
1161
|
+
return _definition_gap(wanted[0], found, path)
|
|
1162
|
+
if not isinstance(found, list):
|
|
1163
|
+
return f"{path} is {found!r} where this package declared the list {wanted!r}"
|
|
1164
|
+
for entry in wanted:
|
|
1165
|
+
if not any(_definition_gap(entry, candidate, path) is None for candidate in found):
|
|
1166
|
+
return f"{path} declares {entry!r} and nothing reported at {path} covers it"
|
|
1167
|
+
return None
|
|
1168
|
+
|
|
1169
|
+
if wanted != found:
|
|
1170
|
+
return f"{path} is {found!r} where this package declared {wanted!r}"
|
|
1171
|
+
return None
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
def _failed_message(collection_name: str, namespace: str, spec: SearchIndexSpec, document: Mapping[str, Any]) -> str:
|
|
1175
|
+
"""Build the message for a search index build that failed.
|
|
1176
|
+
|
|
1177
|
+
Reported rather than recreated: issuing the same definition over a failed build is accepted but does
|
|
1178
|
+
not restart it. ``statusDetail`` is rendered whole, since its shape varies by host and index.
|
|
1179
|
+
"""
|
|
1180
|
+
detail = json.dumps(document.get("statusDetail", []), default=str)
|
|
1181
|
+
return (
|
|
1182
|
+
f"strands_mongodb: the {spec.type} index {spec.name!r} on {namespace} reports status FAILED, "
|
|
1183
|
+
f"so searches against it return nothing rather than an error. The server's detail: {detail}. "
|
|
1184
|
+
f"A failed build does not restart when the same definition is issued again, so it has to be "
|
|
1185
|
+
f"dropped and recreated:\n"
|
|
1186
|
+
f"{spec.drop_command(collection_name)}\n"
|
|
1187
|
+
f"{spec.creation_command(collection_name)}\n"
|
|
1188
|
+
)
|
|
1189
|
+
|
|
1190
|
+
|
|
1191
|
+
def _missing_search_message(collection_name: str, namespace: str, missing: Sequence[SearchIndexSpec]) -> str:
|
|
1192
|
+
"""Build the verify-only message for absent search indexes."""
|
|
1193
|
+
commands = "\n".join(spec.creation_command(collection_name) for spec in missing)
|
|
1194
|
+
return (
|
|
1195
|
+
f"strands_mongodb: {len(missing)} required search index(es) are absent from {namespace} and "
|
|
1196
|
+
f"ensure_indexes is False, so nothing was created. A search against an absent index returns "
|
|
1197
|
+
f"no results rather than an error, so this is reported instead of ignored. Run these, wait for "
|
|
1198
|
+
f"each to report status READY, then call initialize() again:\n"
|
|
1199
|
+
f"{commands}\n"
|
|
1200
|
+
f"Absent: {', '.join(spec.name for spec in missing)}."
|
|
1201
|
+
)
|
|
1202
|
+
|
|
1203
|
+
|
|
1204
|
+
async def _create_collection(collection: AsyncCollection[Any], namespace: str) -> None:
|
|
1205
|
+
"""Create the collection if it is not already there, without checking first.
|
|
1206
|
+
|
|
1207
|
+
``check_exists=False`` avoids a check-then-act race and a round trip, and means pymongo never raises
|
|
1208
|
+
``CollectionInvalid``. A lost race arrives as code 48 (namespace exists) instead, which is ignored.
|
|
1209
|
+
8.3.8 accepts the second create outright; the handler is for other server versions.
|
|
1210
|
+
"""
|
|
1211
|
+
try:
|
|
1212
|
+
await collection.database.create_collection(collection.name, check_exists=False)
|
|
1213
|
+
except OperationFailure as error:
|
|
1214
|
+
if error.code != _NAMESPACE_EXISTS:
|
|
1215
|
+
raise StorageError(f"strands_mongodb: creating the collection {namespace} failed: {error}") from error
|
|
1216
|
+
except PyMongoError as error:
|
|
1217
|
+
raise StorageError(f"strands_mongodb: creating the collection {namespace} failed: {error}") from error
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
async def _wait_until_ready(
|
|
1221
|
+
collection: AsyncCollection[Any],
|
|
1222
|
+
specs: Sequence[SearchIndexSpec],
|
|
1223
|
+
namespace: str,
|
|
1224
|
+
*,
|
|
1225
|
+
timeout: float,
|
|
1226
|
+
interval: float,
|
|
1227
|
+
) -> list[str] | None:
|
|
1228
|
+
"""Poll until every index reports READY, or until ``timeout``.
|
|
1229
|
+
|
|
1230
|
+
Sleeps with ``asyncio.sleep`` so other coroutines keep running during a build. Status changes are
|
|
1231
|
+
logged, so a long wait does not look like a hang. An index leaves ``pending`` only when it reports
|
|
1232
|
+
READY and its reported definition still covers the declared one.
|
|
1233
|
+
|
|
1234
|
+
Returns:
|
|
1235
|
+
``None`` once every index is ready, or ``"<name> is <status>"`` for each one still pending at the
|
|
1236
|
+
deadline, for the caller to raise on.
|
|
1237
|
+
|
|
1238
|
+
Raises:
|
|
1239
|
+
StorageError: If a build reports FAILED, if a ready index's definition does not cover the declared
|
|
1240
|
+
one, or if a read fails.
|
|
1241
|
+
"""
|
|
1242
|
+
deadline = time.monotonic() + timeout
|
|
1243
|
+
pending = {spec.name: spec for spec in specs}
|
|
1244
|
+
last: dict[str, str] = {}
|
|
1245
|
+
|
|
1246
|
+
while True:
|
|
1247
|
+
for name in list(pending):
|
|
1248
|
+
document = await _read_search_index(collection, name, namespace)
|
|
1249
|
+
# A vanished index reads as building. Someone else dropped it, and recreating it would race
|
|
1250
|
+
# whoever did.
|
|
1251
|
+
status = "absent" if document is None else str(document.get("status", "unreported"))
|
|
1252
|
+
if status != last.get(name):
|
|
1253
|
+
logger.info(
|
|
1254
|
+
"strands_mongodb: search index %r on %s is %s",
|
|
1255
|
+
name,
|
|
1256
|
+
namespace,
|
|
1257
|
+
status,
|
|
1258
|
+
extra=_telemetry.event_extra("search_index_status", namespace=namespace),
|
|
1259
|
+
)
|
|
1260
|
+
last[name] = status
|
|
1261
|
+
state = readiness(status)
|
|
1262
|
+
if state == "failed":
|
|
1263
|
+
# A missing document never reports FAILED, so `or {}` is only for the type checker.
|
|
1264
|
+
raise StorageError(_failed_message(collection.name, namespace, pending[name], document or {}))
|
|
1265
|
+
if state == "ready":
|
|
1266
|
+
# Recheck the definition: another actor may have dropped and recreated the index under
|
|
1267
|
+
# the same name during the wait, and an index created by this call was never checked.
|
|
1268
|
+
# Accepting a wrong definition makes searches return nothing while every index is READY.
|
|
1269
|
+
_check_definition(collection.name, namespace, pending[name], document or {})
|
|
1270
|
+
del pending[name]
|
|
1271
|
+
|
|
1272
|
+
if not pending:
|
|
1273
|
+
return None
|
|
1274
|
+
if time.monotonic() >= deadline:
|
|
1275
|
+
return [f"{name} is {last[name]}" for name in pending]
|
|
1276
|
+
await asyncio.sleep(interval)
|
|
1277
|
+
|
|
1278
|
+
|
|
1279
|
+
async def _guard(awaitable: Any, *, what: str) -> Any:
|
|
1280
|
+
"""Await something and turn a driver error into ``StorageError``.
|
|
1281
|
+
|
|
1282
|
+
``StorageError`` is what the ``Storage`` protocol documents. The driver error is chained, so the
|
|
1283
|
+
server's code and message stay reachable through ``__cause__``.
|
|
1284
|
+
"""
|
|
1285
|
+
try:
|
|
1286
|
+
return await awaitable
|
|
1287
|
+
except PyMongoError as error:
|
|
1288
|
+
raise StorageError(f"strands_mongodb: {what} failed: {error}") from error
|
|
1289
|
+
|
|
1290
|
+
|
|
1291
|
+
async def embed(embedder: TextEmbedder, text: str, *, dimensions: int, setting: str) -> list[float]:
|
|
1292
|
+
"""Embed ``text`` and check the vector's length against the index definition.
|
|
1293
|
+
|
|
1294
|
+
The server rejects a wrong length on a query but silently leaves a written one out of the index.
|
|
1295
|
+
|
|
1296
|
+
Args:
|
|
1297
|
+
embedder: What embeds the text.
|
|
1298
|
+
text: The text to embed.
|
|
1299
|
+
dimensions: The length the index declares.
|
|
1300
|
+
setting: The name of the option that set ``dimensions``, for the error message.
|
|
1301
|
+
|
|
1302
|
+
Returns:
|
|
1303
|
+
The vector, as floats.
|
|
1304
|
+
|
|
1305
|
+
Raises:
|
|
1306
|
+
ValueError: If the vector's length is not ``dimensions``.
|
|
1307
|
+
"""
|
|
1308
|
+
vector = [float(value) for value in await embedder.embed(text)]
|
|
1309
|
+
if len(vector) != dimensions:
|
|
1310
|
+
raise ValueError(
|
|
1311
|
+
f"strands_mongodb: the embedder returned a {len(vector)}-dimension vector and {setting!r} is "
|
|
1312
|
+
f"{dimensions}. Set {setting!r} to the length the embedding model produces."
|
|
1313
|
+
)
|
|
1314
|
+
return vector
|