strands-mongodb 0.1.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strands_mongodb/__init__.py +76 -0
- strands_mongodb/_client.py +508 -0
- strands_mongodb/_extras.py +39 -0
- strands_mongodb/_filter.py +221 -0
- strands_mongodb/_indexes.py +1314 -0
- strands_mongodb/_pipeline.py +408 -0
- strands_mongodb/_telemetry.py +254 -0
- strands_mongodb/memory_store.py +1282 -0
- strands_mongodb/py.typed +1 -0
- strands_mongodb/storage.py +693 -0
- strands_mongodb/types.py +236 -0
- strands_mongodb/vector_search.py +216 -0
- strands_mongodb-0.1.0.dev1.dist-info/METADATA +1032 -0
- strands_mongodb-0.1.0.dev1.dist-info/RECORD +17 -0
- strands_mongodb-0.1.0.dev1.dist-info/WHEEL +4 -0
- strands_mongodb-0.1.0.dev1.dist-info/licenses/LICENSE +175 -0
- strands_mongodb-0.1.0.dev1.dist-info/licenses/NOTICE +10 -0
|
@@ -0,0 +1,693 @@
|
|
|
1
|
+
"""``MongoDBStorage``: MongoDB behind the Strands unified ``Storage`` protocol.
|
|
2
|
+
|
|
3
|
+
Requires the ``storage`` extra::
|
|
4
|
+
|
|
5
|
+
pip install 'strands-mongodb[storage]'
|
|
6
|
+
|
|
7
|
+
Pass one instance to ``Agent(storage=...)`` and it serves all five SDK subsystems that use storage:
|
|
8
|
+
sessions, context offloading, memory persistence, context history and transcripts.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import builtins
|
|
14
|
+
import datetime
|
|
15
|
+
import functools
|
|
16
|
+
from typing import TYPE_CHECKING, Any, Final, ParamSpec, TypeVar
|
|
17
|
+
|
|
18
|
+
from ._extras import missing_extra
|
|
19
|
+
|
|
20
|
+
# Imported only to prove the `storage` extra is installed, so a missing extra fails here with a clear
|
|
21
|
+
# message instead of as a bare ModuleNotFoundError inside a method call.
|
|
22
|
+
try:
|
|
23
|
+
import pymongo # noqa: F401
|
|
24
|
+
from strands.storage import Storage # noqa: F401
|
|
25
|
+
except ImportError as error:
|
|
26
|
+
missing_extra("storage", error, module=__name__)
|
|
27
|
+
|
|
28
|
+
from pymongo.errors import PyMongoError
|
|
29
|
+
from strands.storage.search.keyword import tokenize
|
|
30
|
+
|
|
31
|
+
# Private in the SDK, but every shipped backend uses the same two normalizers: they make `a//b` and `a/b`
|
|
32
|
+
# one key everywhere. A local copy that drifted would make keys collide on `LocalFileStorage` and not here.
|
|
33
|
+
from strands.storage.storage import _NAMESPACED, StorageSearchResult, _normalize_key, _normalize_prefix
|
|
34
|
+
from strands.types.exceptions import StorageError
|
|
35
|
+
|
|
36
|
+
from . import _client, _indexes, _telemetry
|
|
37
|
+
from ._indexes import EXPIRES_FIELD
|
|
38
|
+
|
|
39
|
+
if TYPE_CHECKING:
|
|
40
|
+
from collections.abc import Callable, Coroutine
|
|
41
|
+
|
|
42
|
+
from pymongo.asynchronous.collection import AsyncCollection
|
|
43
|
+
from strands.storage.search import SearchStrategy
|
|
44
|
+
|
|
45
|
+
from .types import MongoDBConfig
|
|
46
|
+
|
|
47
|
+
__all__ = ["MAX_DOCUMENT_BYTES", "MAX_KEY_BYTES", "MongoDBStorage"]
|
|
48
|
+
|
|
49
|
+
MAX_KEY_BYTES: Final[int] = 1024
|
|
50
|
+
"""Longest key this package writes, in UTF-8 bytes, including the instance prefix.
|
|
51
|
+
|
|
52
|
+
A package limit, not a server one. MongoDB 8.3.8 accepts a 1,000,000-byte ``_id`` at the default feature
|
|
53
|
+
compatibility version, but a long key makes every ``list`` range scan pay for a long index entry, and it
|
|
54
|
+
fails outright on a deployment pinned below FCV 4.2. 1024 is that old index key limit. The longest key
|
|
55
|
+
the SDK builds for a realistic session and agent id pair is about 131 bytes.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
MAX_DOCUMENT_BYTES: Final[int] = 16 * 1024 * 1024
|
|
59
|
+
"""The BSON document size limit (16 MiB) that ``write`` checks before sending.
|
|
60
|
+
|
|
61
|
+
Defined here because ``pymongo.common.MAX_BSON_SIZE`` is not public API. A test pins it to the
|
|
62
|
+
``maxBsonObjectSize`` a real deployment reports.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
# Bytes a stored document costs besides its key and payload: the 4-byte length, the `_id` and `data`
|
|
66
|
+
# element headers, the whole `updated_at` element and the terminator, plus the `expires_at` element when
|
|
67
|
+
# a TTL is set. A test pins both numbers to `bson.encode`.
|
|
68
|
+
_DOCUMENT_OVERHEAD_BYTES: Final[int] = 46
|
|
69
|
+
_EXPIRES_OVERHEAD_BYTES: Final[int] = 20
|
|
70
|
+
|
|
71
|
+
_P = ParamSpec("_P")
|
|
72
|
+
_R = TypeVar("_R")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _encoded_size(key: str, data: bytes, *, expires: bool) -> int:
|
|
76
|
+
"""Size of the document ``write`` would build, without building it.
|
|
77
|
+
|
|
78
|
+
Computed arithmetically because encoding a payload near 16 MiB only to measure it doubles peak
|
|
79
|
+
memory on the call most likely to be short of it. A test checks the result matches the encoder
|
|
80
|
+
exactly, including multi-byte keys.
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
key: The ``_id`` the document will carry, prefix included.
|
|
84
|
+
data: The payload.
|
|
85
|
+
expires: Whether an ``expires_at`` element will be present.
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
The number of bytes the encoded document will occupy.
|
|
89
|
+
"""
|
|
90
|
+
return _DOCUMENT_OVERHEAD_BYTES + len(key.encode("utf-8")) + len(data) + (_EXPIRES_OVERHEAD_BYTES if expires else 0)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _successor(text: str) -> str | None:
|
|
94
|
+
"""Smallest string that sorts above every key starting with ``text``.
|
|
95
|
+
|
|
96
|
+
The upper bound of the ``_id`` range scan. Simply incrementing the last character fails on two
|
|
97
|
+
inputs, both valid keys:
|
|
98
|
+
|
|
99
|
+
* ``U+D7FF`` would increment to ``U+D800``, a lone surrogate that BSON cannot encode. Nothing sorts
|
|
100
|
+
between the two, so jumping to ``U+E000`` keeps the bound exact.
|
|
101
|
+
* ``U+10FFFF`` has no successor and ``chr`` raises. Incrementing the character to its left is still
|
|
102
|
+
an exact bound, and a prefix made only of ``U+10FFFF`` has no upper bound at all.
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
text: The prefix the range starts at. Never empty; an empty prefix is an unfiltered query.
|
|
106
|
+
|
|
107
|
+
Returns:
|
|
108
|
+
The exclusive upper bound, or ``None`` when the range has no upper bound.
|
|
109
|
+
"""
|
|
110
|
+
for index in range(len(text) - 1, -1, -1):
|
|
111
|
+
point = ord(text[index]) + 1
|
|
112
|
+
if point == 0xD800:
|
|
113
|
+
point = 0xE000
|
|
114
|
+
if point > 0x10FFFF:
|
|
115
|
+
# Carry left. Every string starting with `text` also starts with this shorter prefix.
|
|
116
|
+
continue
|
|
117
|
+
return text[:index] + chr(point)
|
|
118
|
+
return None
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _abbreviate(key: str) -> str:
|
|
122
|
+
"""Shorten a key for an error message, because a rejected key can be 1025 characters long."""
|
|
123
|
+
return key if len(key) <= 60 else f"{key[:60]}[+{len(key) - 60} more]"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _reporting(
|
|
127
|
+
verb: str,
|
|
128
|
+
) -> Callable[[Callable[_P, Coroutine[Any, Any, _R]]], Callable[_P, Coroutine[Any, Any, _R]]]:
|
|
129
|
+
"""Turn any driver error raised by the wrapped method into ``StorageError``.
|
|
130
|
+
|
|
131
|
+
It wraps the whole method rather than one awaitable because ``list`` raises while its cursor drains,
|
|
132
|
+
after ``find`` has returned. Wrapping only the call would let that ``PyMongoError`` escape.
|
|
133
|
+
|
|
134
|
+
The message uses the same sentence as ``_indexes``, and a test keeps the two in step.
|
|
135
|
+
|
|
136
|
+
The wrapped type is ``Coroutine``, not ``Awaitable``: ``Storage`` declares its methods with
|
|
137
|
+
``async def``, and widening them would make ``MongoDBStorage`` fail the protocol under mypy.
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
verb: What the method was doing, as a present participle, for the message.
|
|
141
|
+
|
|
142
|
+
Returns:
|
|
143
|
+
A decorator for an async method whose first two arguments are ``self`` and a key or prefix.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
def decorate(method: Callable[_P, Coroutine[Any, Any, _R]]) -> Callable[_P, Coroutine[Any, Any, _R]]:
|
|
147
|
+
@functools.wraps(method)
|
|
148
|
+
async def reporting(*args: _P.args, **kwargs: _P.kwargs) -> _R:
|
|
149
|
+
try:
|
|
150
|
+
return await method(*args, **kwargs)
|
|
151
|
+
except PyMongoError as error:
|
|
152
|
+
# Through a local because mypy rejects indexing `args` of a `ParamSpec` directly.
|
|
153
|
+
positional: tuple[Any, ...] = args
|
|
154
|
+
storage: MongoDBStorage = positional[0]
|
|
155
|
+
subject = positional[1] if len(positional) > 1 else ""
|
|
156
|
+
raise StorageError(
|
|
157
|
+
f"strands_mongodb: {verb} {_abbreviate(str(subject))!r} in {storage._target} failed: {error}"
|
|
158
|
+
) from error
|
|
159
|
+
|
|
160
|
+
return reporting
|
|
161
|
+
|
|
162
|
+
return decorate
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
# `Storage` is satisfied structurally, not inherited, like the SDK's own backends.
|
|
166
|
+
# `for_sandbox` is absent on purpose: the SDK probes it with `hasattr` and falls back, so a stub that
|
|
167
|
+
# raised would break context offloading instead of degrading it.
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class MongoDBStorage:
|
|
171
|
+
"""A ``strands.storage.Storage`` backed by a MongoDB collection.
|
|
172
|
+
|
|
173
|
+
Each key is stored as one document whose ``_id`` is the key (with this instance's prefix), so every
|
|
174
|
+
operation is a primary-key lookup or a range scan over the ``_id`` index.
|
|
175
|
+
|
|
176
|
+
The connection settings arrive as a ``MongoDBConfig``; build it once per process and reuse it.
|
|
177
|
+
Instances given the same config object share one connection pool. Per-instance placement
|
|
178
|
+
(collection, prefix, TTL) is passed as keyword arguments.
|
|
179
|
+
|
|
180
|
+
Keys are normalized the way the SDK's own backends normalize them: runs of ``/`` collapse and leading
|
|
181
|
+
and trailing ``/`` are stripped, so ``a//b`` and ``/a/b/`` are the key ``a/b``. A key that is empty
|
|
182
|
+
after that, or that contains a backslash or a ``..`` segment, raises ``StorageError``.
|
|
183
|
+
|
|
184
|
+
Every key operation (``write``, ``read``, ``delete``, ``list``, ``delete_prefix``, and ``search`` once
|
|
185
|
+
its query has tokens) and ``ensure_indexes`` raise ``StorageError`` if the collection's default
|
|
186
|
+
collation is not binary. Keys must compare byte by byte, or one prefix could read and delete another's
|
|
187
|
+
documents.
|
|
188
|
+
|
|
189
|
+
Example:
|
|
190
|
+
```python
|
|
191
|
+
config: MongoDBConfig = {"database": "agents", "connection_string": "mongodb+srv://..."}
|
|
192
|
+
sessions = MongoDBStorage(config, collection="sessions")
|
|
193
|
+
transcripts = MongoDBStorage(config, collection="transcripts") # same pool
|
|
194
|
+
```
|
|
195
|
+
"""
|
|
196
|
+
|
|
197
|
+
# Declared without a value on purpose. The SDK reads `getattr(storage, "_namespaced", None)` to tell
|
|
198
|
+
# whether a storage is already scoped, so it must stay absent until `namespace()` sets it.
|
|
199
|
+
_namespaced: object
|
|
200
|
+
|
|
201
|
+
def __init__(
|
|
202
|
+
self,
|
|
203
|
+
config: MongoDBConfig,
|
|
204
|
+
/,
|
|
205
|
+
*,
|
|
206
|
+
collection: str = "storage",
|
|
207
|
+
prefix: str = "",
|
|
208
|
+
ttl_seconds: int | None = None,
|
|
209
|
+
search_strategy: SearchStrategy[MongoDBStorage] | None = None,
|
|
210
|
+
) -> None:
|
|
211
|
+
"""Create a storage over one collection.
|
|
212
|
+
|
|
213
|
+
Args:
|
|
214
|
+
config: Connection and deployment settings. Positional-only. Pass the same object to a
|
|
215
|
+
``MongoDBMemoryStore`` (through ``MongoDBMemoryStoreConfig.config``) to share the pool.
|
|
216
|
+
collection: Collection holding this instance's documents.
|
|
217
|
+
prefix: Prefix prepended to every key, scoping this instance within the collection.
|
|
218
|
+
Normalized to end in exactly one ``/``, or to be empty, so ``"a"``, ``"a/"`` and ``"/a"``
|
|
219
|
+
are one key space.
|
|
220
|
+
ttl_seconds: Lifetime of documents this instance writes, in seconds, counted from each write.
|
|
221
|
+
``None`` means they never expire, which is not the same as ``0`` (expire at the next TTL
|
|
222
|
+
sweep, which the server runs about once a minute). This only stamps ``expires_at``:
|
|
223
|
+
**expiry needs the TTL index, which nothing creates implicitly**. Call
|
|
224
|
+
:meth:`ensure_indexes` once at startup, or run the ``createIndexes`` command in the
|
|
225
|
+
README's "Indexes and privileges" section. Documents written before the index exists
|
|
226
|
+
expire once it does.
|
|
227
|
+
search_strategy: What :meth:`search` runs instead of the default key-only scan, such as the
|
|
228
|
+
SDK's ``KeywordSearchStrategy`` or a :class:`MongoDBVectorSearchStrategy`.
|
|
229
|
+
|
|
230
|
+
Raises:
|
|
231
|
+
KeyError: If ``config`` has no ``database``. Mypy rejects a literal that omits it; this
|
|
232
|
+
catches a config assembled at runtime.
|
|
233
|
+
ValueError: If ``config`` supplies neither ``client`` nor ``connection_string``.
|
|
234
|
+
ValueError: If ``ttl_seconds`` is negative.
|
|
235
|
+
ValueError: If the driver rejects a value in ``connection_string``, such as an out-of-range
|
|
236
|
+
port.
|
|
237
|
+
pymongo.errors.ConfigurationError: If the driver cannot parse ``connection_string``, such as
|
|
238
|
+
one with no host or a scheme other than ``mongodb://`` or ``mongodb+srv://``.
|
|
239
|
+
StorageError: If ``prefix`` contains a backslash or a ``..`` segment.
|
|
240
|
+
"""
|
|
241
|
+
# Read before `resolve`, which registers a pool in the process-wide registry. Raising after that
|
|
242
|
+
# would leave a pool no instance owns and no `close()` can reach.
|
|
243
|
+
self._database_name = config["database"]
|
|
244
|
+
_indexes.check_ttl(ttl_seconds)
|
|
245
|
+
self._collection_name = collection
|
|
246
|
+
# `_normalize_prefix` keeps a trailing slash, so strip it and add exactly one back.
|
|
247
|
+
normalized = _normalize_prefix(prefix).rstrip("/")
|
|
248
|
+
self._prefix = f"{normalized}/" if normalized else ""
|
|
249
|
+
self._ttl_seconds = ttl_seconds
|
|
250
|
+
self._search_strategy = search_strategy
|
|
251
|
+
self._config = config
|
|
252
|
+
# Only passed on to `_indexes.ensure`; read so a value set for a memory store is honoured here too.
|
|
253
|
+
self._index_ready_timeout: float = config.get("index_ready_timeout", _indexes.READY_TIMEOUT_SECONDS)
|
|
254
|
+
# Called only to fail fast on a config with no client or connection string. The result is
|
|
255
|
+
# dropped: the registry is keyed by event loop, so each operation resolves its own connection.
|
|
256
|
+
_client.resolve(config)
|
|
257
|
+
|
|
258
|
+
@property
|
|
259
|
+
def prefix(self) -> str:
|
|
260
|
+
"""The normalized key prefix (ending in ``/``), or ``""`` when none was configured."""
|
|
261
|
+
return self._prefix
|
|
262
|
+
|
|
263
|
+
@_telemetry.traced("storage", "write")
|
|
264
|
+
@_reporting("writing")
|
|
265
|
+
async def write(self, key: str, data: bytes) -> None:
|
|
266
|
+
"""Store ``data`` under ``key``, replacing whatever was there.
|
|
267
|
+
|
|
268
|
+
A single upsert, so two writers racing on one key never hit a duplicate-key error. When the
|
|
269
|
+
instance has ``ttl_seconds``, the document gets an ``expires_at`` stamp.
|
|
270
|
+
|
|
271
|
+
Args:
|
|
272
|
+
key: The key to write under. Normalized, so ``a//b`` and ``a/b`` are one key.
|
|
273
|
+
data: The bytes to store, round-tripped unchanged including ``b""``.
|
|
274
|
+
|
|
275
|
+
Raises:
|
|
276
|
+
StorageError: If the key is empty, contains a backslash or a ``..`` segment, exceeds
|
|
277
|
+
:data:`MAX_KEY_BYTES` with the prefix included, or would build a document over
|
|
278
|
+
:data:`MAX_DOCUMENT_BYTES`. Also if the collection's default collation is not binary, if
|
|
279
|
+
the write fails, with the driver's error chained, or if the search strategy fails to index
|
|
280
|
+
the value (which is stored in that case).
|
|
281
|
+
"""
|
|
282
|
+
await self._require_binary_keys()
|
|
283
|
+
document_id = self._id_for(key)
|
|
284
|
+
expires = self._ttl_seconds is not None
|
|
285
|
+
size = _encoded_size(document_id, data, expires=expires)
|
|
286
|
+
if size > MAX_DOCUMENT_BYTES:
|
|
287
|
+
# Checked before sending so the caller sees the sizes, which the driver's `DocumentTooLarge`
|
|
288
|
+
# does not report.
|
|
289
|
+
raise StorageError(
|
|
290
|
+
f"strands_mongodb: the document for the key {_abbreviate(document_id)!r} would encode "
|
|
291
|
+
f"to {size} bytes, over the {MAX_DOCUMENT_BYTES}-byte BSON limit in {self._target}. "
|
|
292
|
+
f"Store the payload elsewhere and keep a reference here, or split it across keys."
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
now = datetime.datetime.now(datetime.timezone.utc)
|
|
296
|
+
document: dict[str, Any] = {"_id": document_id, "data": data, "updated_at": now}
|
|
297
|
+
if self._ttl_seconds is not None:
|
|
298
|
+
# Only with a TTL. The TTL index never expires a non-date value, so `None` would be dead bytes.
|
|
299
|
+
document[EXPIRES_FIELD] = now + datetime.timedelta(seconds=self._ttl_seconds)
|
|
300
|
+
|
|
301
|
+
await self._collection().replace_one({"_id": document_id}, document, upsert=True)
|
|
302
|
+
|
|
303
|
+
# `SearchStrategy` gained `index` in strands-agents 1.56.0; a strategy from an older release has none.
|
|
304
|
+
index = getattr(self._search_strategy, "index", None)
|
|
305
|
+
if index is None:
|
|
306
|
+
return
|
|
307
|
+
try:
|
|
308
|
+
await index(self, document_id[len(self._prefix) :], data)
|
|
309
|
+
except Exception as error:
|
|
310
|
+
raise StorageError(
|
|
311
|
+
f"strands_mongodb: wrote {_abbreviate(document_id)!r} in {self._target} but indexing it for "
|
|
312
|
+
f"search failed: {error}"
|
|
313
|
+
) from error
|
|
314
|
+
|
|
315
|
+
@_telemetry.traced("storage", "read")
|
|
316
|
+
@_reporting("reading")
|
|
317
|
+
async def read(self, key: str) -> bytes | None:
|
|
318
|
+
"""Return the bytes stored under ``key``, or ``None`` if there are none.
|
|
319
|
+
|
|
320
|
+
A missing key is not an error: the SDK's session manager reads it as "no session yet".
|
|
321
|
+
|
|
322
|
+
Args:
|
|
323
|
+
key: The key to read.
|
|
324
|
+
|
|
325
|
+
Returns:
|
|
326
|
+
The stored bytes, or ``None`` when the key is absent.
|
|
327
|
+
|
|
328
|
+
Raises:
|
|
329
|
+
StorageError: If the key is empty, contains a backslash or a ``..`` segment, or exceeds
|
|
330
|
+
:data:`MAX_KEY_BYTES`. Also if the collection's default collation is not binary, or if
|
|
331
|
+
the read fails, with the driver's error chained. An absent key is not a failure.
|
|
332
|
+
"""
|
|
333
|
+
await self._require_binary_keys()
|
|
334
|
+
document = await self._collection().find_one({"_id": self._id_for(key)})
|
|
335
|
+
_telemetry.annotate("strands_mongodb.found", document is not None)
|
|
336
|
+
if document is None:
|
|
337
|
+
return None
|
|
338
|
+
# pymongo decodes binary subtype 0 (what this package writes) to `bytes`, but subtypes 2 and 0x80
|
|
339
|
+
# to `Binary`, whose `__eq__` also compares the subtype. Data written by another client would then
|
|
340
|
+
# not equal the stored payload.
|
|
341
|
+
return bytes(document["data"])
|
|
342
|
+
|
|
343
|
+
@_telemetry.traced("storage", "delete")
|
|
344
|
+
@_reporting("deleting")
|
|
345
|
+
async def delete(self, key: str) -> None:
|
|
346
|
+
"""Remove ``key``. Deleting a key that does not exist is not an error.
|
|
347
|
+
|
|
348
|
+
Args:
|
|
349
|
+
key: The key to remove.
|
|
350
|
+
|
|
351
|
+
Raises:
|
|
352
|
+
StorageError: If the key is empty, contains a backslash or a ``..`` segment, or exceeds
|
|
353
|
+
:data:`MAX_KEY_BYTES`. Also if the collection's default collation is not binary, or if
|
|
354
|
+
the delete fails, with the driver's error chained.
|
|
355
|
+
"""
|
|
356
|
+
await self._require_binary_keys()
|
|
357
|
+
result = await self._collection().delete_one({"_id": self._id_for(key)})
|
|
358
|
+
_telemetry.annotate("strands_mongodb.deleted_count", int(result.deleted_count))
|
|
359
|
+
|
|
360
|
+
@_telemetry.traced("storage", "list")
|
|
361
|
+
@_reporting("listing")
|
|
362
|
+
async def list(self, query: str) -> builtins.list[str]:
|
|
363
|
+
"""Return every key under the prefix ``query``, in ascending order.
|
|
364
|
+
|
|
365
|
+
Only keys are fetched, never payloads. MongoDB's byte order over UTF-8 matches Python's
|
|
366
|
+
``sorted()`` for every key this class accepts.
|
|
367
|
+
|
|
368
|
+
Args:
|
|
369
|
+
query: A key prefix. ``""`` matches everything, which is how the SDK's keyword search
|
|
370
|
+
strategy reads the whole store.
|
|
371
|
+
|
|
372
|
+
Returns:
|
|
373
|
+
Full keys (not just the part after ``query``), without this instance's own prefix.
|
|
374
|
+
|
|
375
|
+
Raises:
|
|
376
|
+
StorageError: If the prefix contains a backslash or a ``..`` segment, if the collection's
|
|
377
|
+
default collation is not binary, or if the scan fails at any point, including part way
|
|
378
|
+
through draining the cursor.
|
|
379
|
+
"""
|
|
380
|
+
await self._require_binary_keys()
|
|
381
|
+
cursor = self._collection().find(self._range_for(query), {"_id": 1})
|
|
382
|
+
# Strip only the instance prefix, so the keys returned can be passed straight back to `read`.
|
|
383
|
+
offset = len(self._prefix)
|
|
384
|
+
keys = [str(document["_id"])[offset:] async for document in cursor.sort("_id", 1)]
|
|
385
|
+
_telemetry.annotate("strands_mongodb.result_count", len(keys))
|
|
386
|
+
return keys
|
|
387
|
+
|
|
388
|
+
@_telemetry.traced("storage", "search")
|
|
389
|
+
@_reporting("searching")
|
|
390
|
+
async def search(self, query: str) -> builtins.list[StorageSearchResult]:
|
|
391
|
+
"""Rank stored keys by how many of the query's tokens they contain, unless ``search_strategy`` is set.
|
|
392
|
+
|
|
393
|
+
Scoring matches the SDK's ``KeywordSearchStrategy`` (token overlap), but **only keys are
|
|
394
|
+
scored, not payloads**: a word that appears only inside stored data does not match. In
|
|
395
|
+
exchange the search is one server-side aggregation that reads no documents, where the SDK's
|
|
396
|
+
default reads every payload. ``FileMemoryStore`` keeps facts in payloads, so give it a
|
|
397
|
+
``search_strategy``.
|
|
398
|
+
|
|
399
|
+
Args:
|
|
400
|
+
query: A natural-language string. One that tokenizes to nothing returns ``[]`` without a
|
|
401
|
+
round trip.
|
|
402
|
+
|
|
403
|
+
Returns:
|
|
404
|
+
Matched keys with relevance scores, highest first, ties in ascending key order. ``data`` is
|
|
405
|
+
``None``; the SDK's consumer reads back only the keys it keeps. A ``search_strategy``'s
|
|
406
|
+
results are returned as it gives them.
|
|
407
|
+
|
|
408
|
+
Raises:
|
|
409
|
+
StorageError: If the query has tokens and the collection's default collation is not binary,
|
|
410
|
+
if the aggregation fails, including part way through draining the cursor, or if the
|
|
411
|
+
search strategy raises, with its error chained.
|
|
412
|
+
"""
|
|
413
|
+
if self._search_strategy is not None:
|
|
414
|
+
try:
|
|
415
|
+
results = await self._search_strategy.search(self, query)
|
|
416
|
+
except (StorageError, PyMongoError):
|
|
417
|
+
raise
|
|
418
|
+
except Exception as error:
|
|
419
|
+
raise StorageError(f"strands_mongodb: searching in {self._target} failed: {error}") from error
|
|
420
|
+
_telemetry.annotate("strands_mongodb.result_count", len(results))
|
|
421
|
+
return results
|
|
422
|
+
tokens = tokenize(query)
|
|
423
|
+
if not tokens:
|
|
424
|
+
# Same short circuit as `KeywordSearchStrategy`: scoring every key zero would cost a full scan.
|
|
425
|
+
return []
|
|
426
|
+
# After the short circuit, so an empty query costs no round trip at all.
|
|
427
|
+
await self._require_binary_keys()
|
|
428
|
+
cursor = await self._collection().aggregate(self._search_pipeline(tokens))
|
|
429
|
+
offset = len(self._prefix)
|
|
430
|
+
results = [
|
|
431
|
+
StorageSearchResult(key=str(document["_id"])[offset:], score=float(document["score"]))
|
|
432
|
+
async for document in cursor
|
|
433
|
+
]
|
|
434
|
+
_telemetry.annotate("strands_mongodb.result_count", len(results))
|
|
435
|
+
return results
|
|
436
|
+
|
|
437
|
+
def _search_pipeline(self, tokens: set[str]) -> builtins.list[dict[str, Any]]:
|
|
438
|
+
"""Build the pipeline ``search`` sends. Separate so a test can ``explain`` it.
|
|
439
|
+
|
|
440
|
+
Args:
|
|
441
|
+
tokens: The already-tokenized query. Sorted on the way in so the same question always
|
|
442
|
+
produces one plan-cache shape.
|
|
443
|
+
|
|
444
|
+
Returns:
|
|
445
|
+
The pipeline. The range ``$match`` is omitted when this instance has no prefix, since an empty
|
|
446
|
+
``$match`` is still a stage to run.
|
|
447
|
+
"""
|
|
448
|
+
stages: builtins.list[dict[str, Any]] = []
|
|
449
|
+
# Bounded by the instance prefix (not the query, which is prose). This makes the scan a covered
|
|
450
|
+
# IXSCAN on `_id_` instead of a COLLSCAN.
|
|
451
|
+
instance_range = self._range_for("")
|
|
452
|
+
if instance_range:
|
|
453
|
+
stages.append({"$match": instance_range})
|
|
454
|
+
# Score the key the caller wrote, without this instance's prefix, as the SDK's backends do.
|
|
455
|
+
# Cut by code point, which is how `len` counts the prefix.
|
|
456
|
+
key: Any = {"$substrCP": ["$_id", len(self._prefix), {"$strLenCP": "$_id"}]} if self._prefix else "$_id"
|
|
457
|
+
stages.append(
|
|
458
|
+
{
|
|
459
|
+
"$project": {
|
|
460
|
+
"score": {
|
|
461
|
+
"$size": {
|
|
462
|
+
"$setIntersection": [
|
|
463
|
+
{
|
|
464
|
+
"$map": {
|
|
465
|
+
# Not `\w+`: MongoDB's `\w` is ASCII-only, so `münchen` would split
|
|
466
|
+
# into `m` and `nchen` and `日本語` would vanish. The SDK tokenizer
|
|
467
|
+
# uses Python's Unicode `\w`, which is exactly this class (letters,
|
|
468
|
+
# digits and `_`).
|
|
469
|
+
"input": {
|
|
470
|
+
"$regexFindAll": {
|
|
471
|
+
"input": {"$toLower": key},
|
|
472
|
+
"regex": r"[\p{L}\p{N}_]+",
|
|
473
|
+
}
|
|
474
|
+
},
|
|
475
|
+
"in": "$$this.match",
|
|
476
|
+
}
|
|
477
|
+
},
|
|
478
|
+
sorted(tokens),
|
|
479
|
+
]
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
)
|
|
485
|
+
stages.append({"$match": {"score": {"$gt": 0}}})
|
|
486
|
+
# A blocking sort, unavoidably, since `score` is computed above. `_id` breaks ties, matching what
|
|
487
|
+
# the SDK's stable sort gives over an already-sorted listing.
|
|
488
|
+
stages.append({"$sort": {"score": -1, "_id": 1}})
|
|
489
|
+
return stages
|
|
490
|
+
|
|
491
|
+
@_telemetry.traced("storage", "delete_prefix")
|
|
492
|
+
@_reporting("deleting the prefix")
|
|
493
|
+
async def delete_prefix(self, prefix: str) -> int:
|
|
494
|
+
"""Remove every key under ``prefix`` in one call and return how many were removed.
|
|
495
|
+
|
|
496
|
+
Not part of the ``Storage`` protocol. Use it to delete a whole session in one round trip instead
|
|
497
|
+
of a ``list`` followed by one ``delete`` per key.
|
|
498
|
+
|
|
499
|
+
Args:
|
|
500
|
+
prefix: The key prefix to remove. ``""`` empties this instance's key space.
|
|
501
|
+
|
|
502
|
+
Returns:
|
|
503
|
+
The number of documents removed.
|
|
504
|
+
|
|
505
|
+
Raises:
|
|
506
|
+
StorageError: If the prefix contains a backslash or a ``..`` segment, if the collection's
|
|
507
|
+
default collation is not binary, or if the delete fails, with the driver's error chained.
|
|
508
|
+
"""
|
|
509
|
+
await self._require_binary_keys()
|
|
510
|
+
result = await self._collection().delete_many(self._range_for(prefix))
|
|
511
|
+
removed = int(result.deleted_count)
|
|
512
|
+
_telemetry.annotate("strands_mongodb.deleted_count", removed)
|
|
513
|
+
return removed
|
|
514
|
+
|
|
515
|
+
def namespace(self, prefix: str) -> MongoDBStorage:
|
|
516
|
+
"""Return a new instance scoped to ``prefix`` within the same collection.
|
|
517
|
+
|
|
518
|
+
The result is a full ``MongoDBStorage``, so ``delete_prefix`` and the indexed ``list`` keep
|
|
519
|
+
working on it.
|
|
520
|
+
|
|
521
|
+
Args:
|
|
522
|
+
prefix: Prefix to prepend to every key, appended to any prefix already in force.
|
|
523
|
+
|
|
524
|
+
Returns:
|
|
525
|
+
A new instance sharing this one's connection pool, collection, TTL and search strategy.
|
|
526
|
+
|
|
527
|
+
Raises:
|
|
528
|
+
StorageError: If ``prefix`` contains a backslash or a ``..`` segment.
|
|
529
|
+
"""
|
|
530
|
+
# A real instance rather than the SDK's `_NamespacedStorage`, which exposes only the five protocol
|
|
531
|
+
# methods and would turn `list` into a full scan with a Python-side strip.
|
|
532
|
+
scoped = MongoDBStorage(
|
|
533
|
+
self._config,
|
|
534
|
+
collection=self._collection_name,
|
|
535
|
+
# `__init__` normalizes, so a leading slash here does not produce an empty path segment.
|
|
536
|
+
prefix=f"{self._prefix}{prefix}",
|
|
537
|
+
ttl_seconds=self._ttl_seconds,
|
|
538
|
+
search_strategy=self._search_strategy,
|
|
539
|
+
)
|
|
540
|
+
# The SDK reads this sentinel to tell that a storage is already scoped. Without it, memory keys
|
|
541
|
+
# would land one level deeper here than on `LocalFileStorage`.
|
|
542
|
+
scoped._namespaced = _NAMESPACED
|
|
543
|
+
return scoped
|
|
544
|
+
|
|
545
|
+
@_telemetry.traced("storage", "ensure_indexes")
|
|
546
|
+
async def ensure_indexes(self) -> None:
|
|
547
|
+
"""Create the TTL index that makes ``ttl_seconds`` take effect, or verify it exists.
|
|
548
|
+
|
|
549
|
+
The index is ``strands_expires_at_ttl`` over ``expires_at`` with ``expireAfterSeconds: 0``.
|
|
550
|
+
Without it, documents get an ``expires_at`` stamp but are never deleted. Also creates the search
|
|
551
|
+
index of a ``search_strategy`` that has an ``index_spec()``, such as
|
|
552
|
+
:class:`MongoDBVectorSearchStrategy`'s vector index.
|
|
553
|
+
|
|
554
|
+
**Nothing in this package calls this method.** An ``Agent`` builds its storage in code the
|
|
555
|
+
operator does not see, and index creation there could run without the rights for it. Call it once
|
|
556
|
+
at startup::
|
|
557
|
+
|
|
558
|
+
storage = MongoDBStorage(config, ttl_seconds=3600)
|
|
559
|
+
await storage.ensure_indexes()
|
|
560
|
+
|
|
561
|
+
Idempotent and safe to call concurrently. With ``ensure_indexes: False`` in the config it only
|
|
562
|
+
verifies: a missing index raises with the ``createIndexes`` command that would fix it. Documents
|
|
563
|
+
written before the index exists expire once it does.
|
|
564
|
+
|
|
565
|
+
Raises:
|
|
566
|
+
StorageError: If the collection's default collation is not binary, if an index over the same
|
|
567
|
+
keys exists with different options, if the index is absent and ``ensure_indexes`` is
|
|
568
|
+
``False``, if the strategy's index build fails or does not finish in time, or if a driver
|
|
569
|
+
call fails. The server's own code and message arrive chained.
|
|
570
|
+
"""
|
|
571
|
+
# The collation does not affect a TTL index, but a collection that cannot hold this key space is
|
|
572
|
+
# not worth provisioning: it would only fail at its first write.
|
|
573
|
+
await self._require_binary_keys()
|
|
574
|
+
# No `@_reporting`: there is no key argument for its message, and `_indexes.ensure` already turns
|
|
575
|
+
# driver errors into `StorageError`.
|
|
576
|
+
index_spec = getattr(self._search_strategy, "index_spec", None)
|
|
577
|
+
await _indexes.ensure(
|
|
578
|
+
self._collection(),
|
|
579
|
+
ordinary=_indexes.storage_specs(),
|
|
580
|
+
search=(index_spec(),) if index_spec is not None else (),
|
|
581
|
+
create=self._config.get("ensure_indexes", True),
|
|
582
|
+
timeout=self._index_ready_timeout,
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
async def _require_binary_keys(self) -> None:
|
|
586
|
+
"""Refuse a collection whose collation would not compare this instance's keys byte by byte.
|
|
587
|
+
|
|
588
|
+
The reason for refusing is on :func:`_indexes.require_binary_comparison`. This method decides
|
|
589
|
+
when: before the first operation on a collection, not at construction (``__init__`` is
|
|
590
|
+
synchronous and must not touch the network) and not in ``ensure_indexes`` (which callers may
|
|
591
|
+
never call).
|
|
592
|
+
|
|
593
|
+
No lock: the check is a side-effect-free read, so two racing first operations each issue one
|
|
594
|
+
``listCollections`` and reach the same answer.
|
|
595
|
+
|
|
596
|
+
Only a verified collection is cached. A refusal is not cached, so a collection fixed while the
|
|
597
|
+
process runs starts working. An absent collection is not cached either, because
|
|
598
|
+
``listCollections`` cannot tell it apart from one with no collation, and whoever creates it may
|
|
599
|
+
give it any collation. The check repeats until this store's first write creates the collection.
|
|
600
|
+
|
|
601
|
+
Raises:
|
|
602
|
+
StorageError: If the collection's default collation is not binary, or if reading it fails.
|
|
603
|
+
"""
|
|
604
|
+
target = (self._database_name, self._collection_name)
|
|
605
|
+
connection = self._connection
|
|
606
|
+
if target in connection.binary_keys:
|
|
607
|
+
return
|
|
608
|
+
if await _indexes.require_binary_comparison(self._collection()):
|
|
609
|
+
connection.binary_keys.add(target)
|
|
610
|
+
|
|
611
|
+
@property
|
|
612
|
+
def _target(self) -> str:
|
|
613
|
+
"""The ``database.collection`` this instance writes to, for error messages.
|
|
614
|
+
|
|
615
|
+
Not ``_namespace``, which would read like the ``namespace()`` key-prefix method.
|
|
616
|
+
"""
|
|
617
|
+
return f"{self._database_name}.{self._collection_name}"
|
|
618
|
+
|
|
619
|
+
@property
|
|
620
|
+
def _connection(self) -> _client._Connection:
|
|
621
|
+
"""The connection serving the running event loop.
|
|
622
|
+
|
|
623
|
+
Resolved on each access, never stored: the SDK runs synchronous entry points through
|
|
624
|
+
``asyncio.run`` on a fresh loop in a worker thread, so one instance can be used from several
|
|
625
|
+
loops, and a client is bound to one. A lookup hit is a dict access.
|
|
626
|
+
"""
|
|
627
|
+
return _client.resolve(self._config)
|
|
628
|
+
|
|
629
|
+
def _collection(self) -> AsyncCollection[dict[str, Any]]:
|
|
630
|
+
"""Resolve the collection, through the running loop's client (see ``_connection``)."""
|
|
631
|
+
collection: AsyncCollection[dict[str, Any]] = self._connection.client[self._database_name][
|
|
632
|
+
self._collection_name
|
|
633
|
+
]
|
|
634
|
+
return collection
|
|
635
|
+
|
|
636
|
+
def _id_for(self, key: str) -> str:
|
|
637
|
+
"""Normalize ``key``, apply this instance's prefix, and check the result against the ceiling.
|
|
638
|
+
|
|
639
|
+
Args:
|
|
640
|
+
key: The caller's key.
|
|
641
|
+
|
|
642
|
+
Returns:
|
|
643
|
+
The ``_id`` to use.
|
|
644
|
+
|
|
645
|
+
Raises:
|
|
646
|
+
StorageError: If the key is empty, contains a backslash or a ``..`` segment, or encodes to
|
|
647
|
+
more than :data:`MAX_KEY_BYTES` bytes.
|
|
648
|
+
"""
|
|
649
|
+
document_id = f"{self._prefix}{_normalize_key(key)}"
|
|
650
|
+
# Measure the prefixed form, since that is what becomes the index entry.
|
|
651
|
+
encoded = len(document_id.encode("utf-8"))
|
|
652
|
+
if encoded > MAX_KEY_BYTES:
|
|
653
|
+
raise StorageError(
|
|
654
|
+
f"strands_mongodb: the key {_abbreviate(document_id)!r} encodes to {encoded} bytes, "
|
|
655
|
+
f"over the {MAX_KEY_BYTES}-byte limit this package applies to keys in {self._target}. "
|
|
656
|
+
f"The limit is counted in UTF-8 bytes rather than characters."
|
|
657
|
+
)
|
|
658
|
+
return document_id
|
|
659
|
+
|
|
660
|
+
def _range_for(self, query: str) -> dict[str, Any]:
|
|
661
|
+
"""Build the ``_id`` filter for a prefix, as a range rather than a regular expression.
|
|
662
|
+
|
|
663
|
+
The ``_id`` index serves a range. An ``{"$regex": "^prefix"}`` filter looks equivalent but scans
|
|
664
|
+
the whole collection unless the planner recognises the anchor, which no unit test would show.
|
|
665
|
+
|
|
666
|
+
Args:
|
|
667
|
+
query: The caller's prefix, before normalization.
|
|
668
|
+
|
|
669
|
+
Returns:
|
|
670
|
+
A filter document, empty when the query and this instance's prefix are both empty.
|
|
671
|
+
|
|
672
|
+
Raises:
|
|
673
|
+
StorageError: If the prefix contains a backslash or a ``..`` segment.
|
|
674
|
+
"""
|
|
675
|
+
lower = f"{self._prefix}{_normalize_prefix(query)}"
|
|
676
|
+
if not lower:
|
|
677
|
+
return {}
|
|
678
|
+
upper = _successor(lower)
|
|
679
|
+
if upper is None:
|
|
680
|
+
return {"_id": {"$gte": lower}}
|
|
681
|
+
return {"_id": {"$gte": lower, "$lt": upper}}
|
|
682
|
+
|
|
683
|
+
async def close(self) -> None:
|
|
684
|
+
"""Release the MongoDB connection, closing the client only if this package created it.
|
|
685
|
+
|
|
686
|
+
Not part of the ``Storage`` protocol. Does nothing to a client you passed in through
|
|
687
|
+
``config["client"]``. Idempotent.
|
|
688
|
+
|
|
689
|
+
**This closes the pool for every instance built from the same config**, not just this one. Call
|
|
690
|
+
it once at process shutdown, not per request. An instance used after the close opens a new pool,
|
|
691
|
+
which stays open until ``close()`` is called again.
|
|
692
|
+
"""
|
|
693
|
+
await _client.close(self._config)
|