langgraph-checkpoint-sqlite 2.0.9__tar.gz → 2.0.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/.gitignore +1 -0
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/PKG-INFO +4 -6
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/README.md +2 -4
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/langgraph/checkpoint/sqlite/__init__.py +17 -17
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/langgraph/checkpoint/sqlite/aio.py +17 -17
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/langgraph/checkpoint/sqlite/utils.py +6 -4
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/langgraph/store/sqlite/aio.py +19 -19
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/langgraph/store/sqlite/base.py +53 -34
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/pyproject.toml +3 -3
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/tests/test_aiosqlite.py +3 -12
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/tests/test_async_store.py +60 -0
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/tests/test_sqlite.py +3 -11
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/tests/test_store.py +80 -0
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/uv.lock +690 -670
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/Makefile +0 -0
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/langgraph/cache/sqlite/__init__.py +0 -0
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/langgraph/checkpoint/sqlite/py.typed +0 -0
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/langgraph/store/sqlite/__init__.py +0 -0
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/tests/__init__.py +0 -0
- {langgraph_checkpoint_sqlite-2.0.9 → langgraph_checkpoint_sqlite-2.0.11}/tests/test_ttl.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: langgraph-checkpoint-sqlite
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.11
|
|
4
4
|
Summary: Library with a SQLite implementation of LangGraph checkpoint saver.
|
|
5
5
|
Project-URL: Repository, https://www.github.com/langchain-ai/langgraph
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Requires-Python: >=3.9
|
|
8
8
|
Requires-Dist: aiosqlite>=0.20
|
|
9
|
-
Requires-Dist: langgraph-checkpoint
|
|
9
|
+
Requires-Dist: langgraph-checkpoint<3.0.0,>=2.0.21
|
|
10
10
|
Requires-Dist: sqlite-vec>=0.1.6
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
|
|
@@ -24,7 +24,7 @@ read_config = {"configurable": {"thread_id": "1"}}
|
|
|
24
24
|
|
|
25
25
|
with SqliteSaver.from_conn_string(":memory:") as checkpointer:
|
|
26
26
|
checkpoint = {
|
|
27
|
-
"v":
|
|
27
|
+
"v": 4,
|
|
28
28
|
"ts": "2024-07-31T20:14:19.804150+00:00",
|
|
29
29
|
"id": "1ef4f797-8335-6428-8001-8a1503f9b875",
|
|
30
30
|
"channel_values": {
|
|
@@ -46,7 +46,6 @@ with SqliteSaver.from_conn_string(":memory:") as checkpointer:
|
|
|
46
46
|
"start:node": 2
|
|
47
47
|
}
|
|
48
48
|
},
|
|
49
|
-
"pending_sends": [],
|
|
50
49
|
}
|
|
51
50
|
|
|
52
51
|
# store checkpoint
|
|
@@ -66,7 +65,7 @@ from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
|
|
66
65
|
|
|
67
66
|
async with AsyncSqliteSaver.from_conn_string(":memory:") as checkpointer:
|
|
68
67
|
checkpoint = {
|
|
69
|
-
"v":
|
|
68
|
+
"v": 4,
|
|
70
69
|
"ts": "2024-07-31T20:14:19.804150+00:00",
|
|
71
70
|
"id": "1ef4f797-8335-6428-8001-8a1503f9b875",
|
|
72
71
|
"channel_values": {
|
|
@@ -88,7 +87,6 @@ async with AsyncSqliteSaver.from_conn_string(":memory:") as checkpointer:
|
|
|
88
87
|
"start:node": 2
|
|
89
88
|
}
|
|
90
89
|
},
|
|
91
|
-
"pending_sends": [],
|
|
92
90
|
}
|
|
93
91
|
|
|
94
92
|
# store checkpoint
|
|
@@ -12,7 +12,7 @@ read_config = {"configurable": {"thread_id": "1"}}
|
|
|
12
12
|
|
|
13
13
|
with SqliteSaver.from_conn_string(":memory:") as checkpointer:
|
|
14
14
|
checkpoint = {
|
|
15
|
-
"v":
|
|
15
|
+
"v": 4,
|
|
16
16
|
"ts": "2024-07-31T20:14:19.804150+00:00",
|
|
17
17
|
"id": "1ef4f797-8335-6428-8001-8a1503f9b875",
|
|
18
18
|
"channel_values": {
|
|
@@ -34,7 +34,6 @@ with SqliteSaver.from_conn_string(":memory:") as checkpointer:
|
|
|
34
34
|
"start:node": 2
|
|
35
35
|
}
|
|
36
36
|
},
|
|
37
|
-
"pending_sends": [],
|
|
38
37
|
}
|
|
39
38
|
|
|
40
39
|
# store checkpoint
|
|
@@ -54,7 +53,7 @@ from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
|
|
54
53
|
|
|
55
54
|
async with AsyncSqliteSaver.from_conn_string(":memory:") as checkpointer:
|
|
56
55
|
checkpoint = {
|
|
57
|
-
"v":
|
|
56
|
+
"v": 4,
|
|
58
57
|
"ts": "2024-07-31T20:14:19.804150+00:00",
|
|
59
58
|
"id": "1ef4f797-8335-6428-8001-8a1503f9b875",
|
|
60
59
|
"channel_values": {
|
|
@@ -76,7 +75,6 @@ async with AsyncSqliteSaver.from_conn_string(":memory:") as checkpointer:
|
|
|
76
75
|
"start:node": 2
|
|
77
76
|
}
|
|
78
77
|
},
|
|
79
|
-
"pending_sends": [],
|
|
80
78
|
}
|
|
81
79
|
|
|
82
80
|
# store checkpoint
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import random
|
|
2
4
|
import sqlite3
|
|
3
5
|
import threading
|
|
4
6
|
from collections.abc import AsyncIterator, Iterator, Sequence
|
|
5
7
|
from contextlib import closing, contextmanager
|
|
6
|
-
from typing import Any,
|
|
8
|
+
from typing import Any, cast
|
|
7
9
|
|
|
8
10
|
from langchain_core.runnables import RunnableConfig
|
|
9
11
|
|
|
@@ -19,7 +21,6 @@ from langgraph.checkpoint.base import (
|
|
|
19
21
|
get_checkpoint_metadata,
|
|
20
22
|
)
|
|
21
23
|
from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer
|
|
22
|
-
from langgraph.checkpoint.serde.types import ChannelProtocol
|
|
23
24
|
from langgraph.checkpoint.sqlite.utils import search_where
|
|
24
25
|
|
|
25
26
|
_AIO_ERROR_MSG = (
|
|
@@ -28,7 +29,7 @@ _AIO_ERROR_MSG = (
|
|
|
28
29
|
"from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver\n"
|
|
29
30
|
"Note: AsyncSqliteSaver requires the aiosqlite package to use.\n"
|
|
30
31
|
"Install with:\n`pip install aiosqlite`\n"
|
|
31
|
-
"See https://langchain-ai.github.io/langgraph/reference/checkpoints
|
|
32
|
+
"See https://langchain-ai.github.io/langgraph/reference/checkpoints/#langgraph.checkpoint.sqlite.aio.AsyncSqliteSaver"
|
|
32
33
|
"for more information."
|
|
33
34
|
)
|
|
34
35
|
|
|
@@ -77,7 +78,7 @@ class SqliteSaver(BaseCheckpointSaver[str]):
|
|
|
77
78
|
self,
|
|
78
79
|
conn: sqlite3.Connection,
|
|
79
80
|
*,
|
|
80
|
-
serde:
|
|
81
|
+
serde: SerializerProtocol | None = None,
|
|
81
82
|
) -> None:
|
|
82
83
|
super().__init__(serde=serde)
|
|
83
84
|
self.jsonplus_serde = JsonPlusSerializer()
|
|
@@ -87,7 +88,7 @@ class SqliteSaver(BaseCheckpointSaver[str]):
|
|
|
87
88
|
|
|
88
89
|
@classmethod
|
|
89
90
|
@contextmanager
|
|
90
|
-
def from_conn_string(cls, conn_string: str) -> Iterator[
|
|
91
|
+
def from_conn_string(cls, conn_string: str) -> Iterator[SqliteSaver]:
|
|
91
92
|
"""Create a new SqliteSaver instance from a connection string.
|
|
92
93
|
|
|
93
94
|
Args:
|
|
@@ -179,7 +180,7 @@ class SqliteSaver(BaseCheckpointSaver[str]):
|
|
|
179
180
|
self.conn.commit()
|
|
180
181
|
cur.close()
|
|
181
182
|
|
|
182
|
-
def get_tuple(self, config: RunnableConfig) ->
|
|
183
|
+
def get_tuple(self, config: RunnableConfig) -> CheckpointTuple | None:
|
|
183
184
|
"""Get a checkpoint tuple from the database.
|
|
184
185
|
|
|
185
186
|
This method retrieves a checkpoint tuple from the SQLite database based on the
|
|
@@ -287,11 +288,11 @@ class SqliteSaver(BaseCheckpointSaver[str]):
|
|
|
287
288
|
|
|
288
289
|
def list(
|
|
289
290
|
self,
|
|
290
|
-
config:
|
|
291
|
+
config: RunnableConfig | None,
|
|
291
292
|
*,
|
|
292
|
-
filter:
|
|
293
|
-
before:
|
|
294
|
-
limit:
|
|
293
|
+
filter: dict[str, Any] | None = None,
|
|
294
|
+
before: RunnableConfig | None = None,
|
|
295
|
+
limit: int | None = None,
|
|
295
296
|
) -> Iterator[CheckpointTuple]:
|
|
296
297
|
"""List checkpoints from the database.
|
|
297
298
|
|
|
@@ -494,7 +495,7 @@ class SqliteSaver(BaseCheckpointSaver[str]):
|
|
|
494
495
|
(str(thread_id),),
|
|
495
496
|
)
|
|
496
497
|
|
|
497
|
-
async def aget_tuple(self, config: RunnableConfig) ->
|
|
498
|
+
async def aget_tuple(self, config: RunnableConfig) -> CheckpointTuple | None:
|
|
498
499
|
"""Get a checkpoint tuple from the database asynchronously.
|
|
499
500
|
|
|
500
501
|
Note:
|
|
@@ -505,11 +506,11 @@ class SqliteSaver(BaseCheckpointSaver[str]):
|
|
|
505
506
|
|
|
506
507
|
async def alist(
|
|
507
508
|
self,
|
|
508
|
-
config:
|
|
509
|
+
config: RunnableConfig | None,
|
|
509
510
|
*,
|
|
510
|
-
filter:
|
|
511
|
-
before:
|
|
512
|
-
limit:
|
|
511
|
+
filter: dict[str, Any] | None = None,
|
|
512
|
+
before: RunnableConfig | None = None,
|
|
513
|
+
limit: int | None = None,
|
|
513
514
|
) -> AsyncIterator[CheckpointTuple]:
|
|
514
515
|
"""List checkpoints from the database asynchronously.
|
|
515
516
|
|
|
@@ -535,14 +536,13 @@ class SqliteSaver(BaseCheckpointSaver[str]):
|
|
|
535
536
|
"""
|
|
536
537
|
raise NotImplementedError(_AIO_ERROR_MSG)
|
|
537
538
|
|
|
538
|
-
def get_next_version(self, current:
|
|
539
|
+
def get_next_version(self, current: str | None, channel: None) -> str:
|
|
539
540
|
"""Generate the next version ID for a channel.
|
|
540
541
|
|
|
541
542
|
This method creates a new version identifier for a channel based on its current version.
|
|
542
543
|
|
|
543
544
|
Args:
|
|
544
545
|
current (Optional[str]): The current version identifier of the channel.
|
|
545
|
-
channel (BaseChannel): The channel being versioned.
|
|
546
546
|
|
|
547
547
|
Returns:
|
|
548
548
|
str: The next version identifier, which is guaranteed to be monotonically increasing.
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import asyncio
|
|
2
4
|
import random
|
|
3
5
|
from collections.abc import AsyncIterator, Iterator, Sequence
|
|
4
6
|
from contextlib import asynccontextmanager
|
|
5
|
-
from typing import Any, Callable,
|
|
7
|
+
from typing import Any, Callable, TypeVar, cast
|
|
6
8
|
|
|
7
9
|
import aiosqlite
|
|
8
10
|
from langchain_core.runnables import RunnableConfig
|
|
@@ -19,7 +21,6 @@ from langgraph.checkpoint.base import (
|
|
|
19
21
|
get_checkpoint_metadata,
|
|
20
22
|
)
|
|
21
23
|
from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer
|
|
22
|
-
from langgraph.checkpoint.serde.types import ChannelProtocol
|
|
23
24
|
from langgraph.checkpoint.sqlite.utils import search_where
|
|
24
25
|
|
|
25
26
|
T = TypeVar("T", bound=Callable)
|
|
@@ -109,7 +110,7 @@ class AsyncSqliteSaver(BaseCheckpointSaver[str]):
|
|
|
109
110
|
self,
|
|
110
111
|
conn: aiosqlite.Connection,
|
|
111
112
|
*,
|
|
112
|
-
serde:
|
|
113
|
+
serde: SerializerProtocol | None = None,
|
|
113
114
|
):
|
|
114
115
|
super().__init__(serde=serde)
|
|
115
116
|
self.jsonplus_serde = JsonPlusSerializer()
|
|
@@ -122,7 +123,7 @@ class AsyncSqliteSaver(BaseCheckpointSaver[str]):
|
|
|
122
123
|
@asynccontextmanager
|
|
123
124
|
async def from_conn_string(
|
|
124
125
|
cls, conn_string: str
|
|
125
|
-
) -> AsyncIterator[
|
|
126
|
+
) -> AsyncIterator[AsyncSqliteSaver]:
|
|
126
127
|
"""Create a new AsyncSqliteSaver instance from a connection string.
|
|
127
128
|
|
|
128
129
|
Args:
|
|
@@ -134,7 +135,7 @@ class AsyncSqliteSaver(BaseCheckpointSaver[str]):
|
|
|
134
135
|
async with aiosqlite.connect(conn_string) as conn:
|
|
135
136
|
yield cls(conn)
|
|
136
137
|
|
|
137
|
-
def get_tuple(self, config: RunnableConfig) ->
|
|
138
|
+
def get_tuple(self, config: RunnableConfig) -> CheckpointTuple | None:
|
|
138
139
|
"""Get a checkpoint tuple from the database.
|
|
139
140
|
|
|
140
141
|
This method retrieves a checkpoint tuple from the SQLite database based on the
|
|
@@ -166,11 +167,11 @@ class AsyncSqliteSaver(BaseCheckpointSaver[str]):
|
|
|
166
167
|
|
|
167
168
|
def list(
|
|
168
169
|
self,
|
|
169
|
-
config:
|
|
170
|
+
config: RunnableConfig | None,
|
|
170
171
|
*,
|
|
171
|
-
filter:
|
|
172
|
-
before:
|
|
173
|
-
limit:
|
|
172
|
+
filter: dict[str, Any] | None = None,
|
|
173
|
+
before: RunnableConfig | None = None,
|
|
174
|
+
limit: int | None = None,
|
|
174
175
|
) -> Iterator[CheckpointTuple]:
|
|
175
176
|
"""List checkpoints from the database asynchronously.
|
|
176
177
|
|
|
@@ -202,7 +203,7 @@ class AsyncSqliteSaver(BaseCheckpointSaver[str]):
|
|
|
202
203
|
while True:
|
|
203
204
|
try:
|
|
204
205
|
yield asyncio.run_coroutine_threadsafe(
|
|
205
|
-
anext(aiter_), # noqa: F821
|
|
206
|
+
anext(aiter_), # type: ignore[arg-type] # noqa: F821
|
|
206
207
|
self.loop,
|
|
207
208
|
).result()
|
|
208
209
|
except StopAsyncIteration:
|
|
@@ -311,7 +312,7 @@ class AsyncSqliteSaver(BaseCheckpointSaver[str]):
|
|
|
311
312
|
|
|
312
313
|
self.is_setup = True
|
|
313
314
|
|
|
314
|
-
async def aget_tuple(self, config: RunnableConfig) ->
|
|
315
|
+
async def aget_tuple(self, config: RunnableConfig) -> CheckpointTuple | None:
|
|
315
316
|
"""Get a checkpoint tuple from the database asynchronously.
|
|
316
317
|
|
|
317
318
|
This method retrieves a checkpoint tuple from the SQLite database based on the
|
|
@@ -399,11 +400,11 @@ class AsyncSqliteSaver(BaseCheckpointSaver[str]):
|
|
|
399
400
|
|
|
400
401
|
async def alist(
|
|
401
402
|
self,
|
|
402
|
-
config:
|
|
403
|
+
config: RunnableConfig | None,
|
|
403
404
|
*,
|
|
404
|
-
filter:
|
|
405
|
-
before:
|
|
406
|
-
limit:
|
|
405
|
+
filter: dict[str, Any] | None = None,
|
|
406
|
+
before: RunnableConfig | None = None,
|
|
407
|
+
limit: int | None = None,
|
|
407
408
|
) -> AsyncIterator[CheckpointTuple]:
|
|
408
409
|
"""List checkpoints from the database asynchronously.
|
|
409
410
|
|
|
@@ -590,14 +591,13 @@ class AsyncSqliteSaver(BaseCheckpointSaver[str]):
|
|
|
590
591
|
)
|
|
591
592
|
await self.conn.commit()
|
|
592
593
|
|
|
593
|
-
def get_next_version(self, current:
|
|
594
|
+
def get_next_version(self, current: str | None, channel: None) -> str:
|
|
594
595
|
"""Generate the next version ID for a channel.
|
|
595
596
|
|
|
596
597
|
This method creates a new version identifier for a channel based on its current version.
|
|
597
598
|
|
|
598
599
|
Args:
|
|
599
600
|
current (Optional[str]): The current version identifier of the channel.
|
|
600
|
-
channel (BaseChannel): The channel being versioned.
|
|
601
601
|
|
|
602
602
|
Returns:
|
|
603
603
|
str: The next version identifier, which is guaranteed to be monotonically increasing.
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import json
|
|
2
4
|
from collections.abc import Sequence
|
|
3
|
-
from typing import Any
|
|
5
|
+
from typing import Any
|
|
4
6
|
|
|
5
7
|
from langchain_core.runnables import RunnableConfig
|
|
6
8
|
|
|
@@ -52,9 +54,9 @@ def _metadata_predicate(
|
|
|
52
54
|
|
|
53
55
|
|
|
54
56
|
def search_where(
|
|
55
|
-
config:
|
|
56
|
-
filter:
|
|
57
|
-
before:
|
|
57
|
+
config: RunnableConfig | None,
|
|
58
|
+
filter: dict[str, Any] | None,
|
|
59
|
+
before: RunnableConfig | None = None,
|
|
58
60
|
) -> tuple[str, Sequence[Any]]:
|
|
59
61
|
"""Return WHERE clause predicates for (a)search() given metadata filter
|
|
60
62
|
and `before` config.
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import asyncio
|
|
2
4
|
import logging
|
|
3
5
|
from collections import defaultdict
|
|
4
6
|
from collections.abc import AsyncIterator, Iterable, Sequence
|
|
5
7
|
from contextlib import asynccontextmanager
|
|
6
8
|
from types import TracebackType
|
|
7
|
-
from typing import Any, Callable,
|
|
9
|
+
from typing import Any, Callable, cast
|
|
8
10
|
|
|
9
11
|
import aiosqlite
|
|
10
12
|
import orjson
|
|
@@ -88,11 +90,10 @@ class AsyncSqliteStore(AsyncBatchedBaseStore, BaseSqliteStore):
|
|
|
88
90
|
self,
|
|
89
91
|
conn: aiosqlite.Connection,
|
|
90
92
|
*,
|
|
91
|
-
deserializer:
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
ttl: Optional[TTLConfig] = None,
|
|
93
|
+
deserializer: Callable[[bytes | str | orjson.Fragment], dict[str, Any]]
|
|
94
|
+
| None = None,
|
|
95
|
+
index: SqliteIndexConfig | None = None,
|
|
96
|
+
ttl: TTLConfig | None = None,
|
|
96
97
|
):
|
|
97
98
|
"""Initialize the async SQLite store.
|
|
98
99
|
|
|
@@ -114,7 +115,7 @@ class AsyncSqliteStore(AsyncBatchedBaseStore, BaseSqliteStore):
|
|
|
114
115
|
else:
|
|
115
116
|
self.embeddings = None
|
|
116
117
|
self.ttl_config = ttl
|
|
117
|
-
self._ttl_sweeper_task:
|
|
118
|
+
self._ttl_sweeper_task: asyncio.Task[None] | None = None
|
|
118
119
|
self._ttl_stop_event = asyncio.Event()
|
|
119
120
|
|
|
120
121
|
@classmethod
|
|
@@ -123,9 +124,9 @@ class AsyncSqliteStore(AsyncBatchedBaseStore, BaseSqliteStore):
|
|
|
123
124
|
cls,
|
|
124
125
|
conn_string: str,
|
|
125
126
|
*,
|
|
126
|
-
index:
|
|
127
|
-
ttl:
|
|
128
|
-
) -> AsyncIterator[
|
|
127
|
+
index: SqliteIndexConfig | None = None,
|
|
128
|
+
ttl: TTLConfig | None = None,
|
|
129
|
+
) -> AsyncIterator[AsyncSqliteStore]:
|
|
129
130
|
"""Create a new AsyncSqliteStore instance from a connection string.
|
|
130
131
|
|
|
131
132
|
Args:
|
|
@@ -222,10 +223,9 @@ class AsyncSqliteStore(AsyncBatchedBaseStore, BaseSqliteStore):
|
|
|
222
223
|
Yields:
|
|
223
224
|
An SQLite cursor object.
|
|
224
225
|
"""
|
|
226
|
+
if not self.is_setup:
|
|
227
|
+
await self.setup()
|
|
225
228
|
async with self.lock:
|
|
226
|
-
if not self.is_setup:
|
|
227
|
-
await self.setup()
|
|
228
|
-
|
|
229
229
|
if transaction:
|
|
230
230
|
await self.conn.execute("BEGIN")
|
|
231
231
|
|
|
@@ -253,7 +253,7 @@ class AsyncSqliteStore(AsyncBatchedBaseStore, BaseSqliteStore):
|
|
|
253
253
|
return deleted_count
|
|
254
254
|
|
|
255
255
|
async def start_ttl_sweeper(
|
|
256
|
-
self, sweep_interval_minutes:
|
|
256
|
+
self, sweep_interval_minutes: int | None = None
|
|
257
257
|
) -> asyncio.Task[None]:
|
|
258
258
|
"""Periodically delete expired store items based on TTL.
|
|
259
259
|
|
|
@@ -298,7 +298,7 @@ class AsyncSqliteStore(AsyncBatchedBaseStore, BaseSqliteStore):
|
|
|
298
298
|
self._ttl_sweeper_task = task
|
|
299
299
|
return task
|
|
300
300
|
|
|
301
|
-
async def stop_ttl_sweeper(self, timeout:
|
|
301
|
+
async def stop_ttl_sweeper(self, timeout: float | None = None) -> bool:
|
|
302
302
|
"""Stop the TTL sweeper task if it's running.
|
|
303
303
|
|
|
304
304
|
Args:
|
|
@@ -333,14 +333,14 @@ class AsyncSqliteStore(AsyncBatchedBaseStore, BaseSqliteStore):
|
|
|
333
333
|
|
|
334
334
|
return success
|
|
335
335
|
|
|
336
|
-
async def __aenter__(self) ->
|
|
336
|
+
async def __aenter__(self) -> AsyncSqliteStore:
|
|
337
337
|
return self
|
|
338
338
|
|
|
339
339
|
async def __aexit__(
|
|
340
340
|
self,
|
|
341
|
-
exc_type:
|
|
342
|
-
exc_val:
|
|
343
|
-
exc_tb:
|
|
341
|
+
exc_type: type[BaseException] | None,
|
|
342
|
+
exc_val: BaseException | None,
|
|
343
|
+
exc_tb: TracebackType | None,
|
|
344
344
|
) -> None:
|
|
345
345
|
# Ensure the TTL sweeper task is stopped when exiting the context
|
|
346
346
|
if hasattr(self, "_ttl_sweeper_task") and self._ttl_sweeper_task is not None:
|
|
@@ -1,12 +1,15 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import concurrent.futures
|
|
2
4
|
import datetime
|
|
3
5
|
import logging
|
|
6
|
+
import re
|
|
4
7
|
import sqlite3
|
|
5
8
|
import threading
|
|
6
9
|
from collections import defaultdict
|
|
7
10
|
from collections.abc import Iterable, Iterator, Sequence
|
|
8
11
|
from contextlib import contextmanager
|
|
9
|
-
from typing import Any, Callable, Literal, NamedTuple,
|
|
12
|
+
from typing import Any, Callable, Literal, NamedTuple, cast
|
|
10
13
|
|
|
11
14
|
import orjson
|
|
12
15
|
import sqlite_vec # type: ignore[import-untyped]
|
|
@@ -105,7 +108,24 @@ def _decode_ns_text(namespace: str) -> tuple[str, ...]:
|
|
|
105
108
|
return tuple(namespace.split("."))
|
|
106
109
|
|
|
107
110
|
|
|
108
|
-
def
|
|
111
|
+
def _validate_filter_key(key: str) -> None:
|
|
112
|
+
"""Validate that a filter key is safe for use in SQL queries.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
key: The filter key to validate
|
|
116
|
+
|
|
117
|
+
Raises:
|
|
118
|
+
ValueError: If the key contains invalid characters that could enable SQL injection
|
|
119
|
+
"""
|
|
120
|
+
# Allow alphanumeric characters, underscores, dots, and hyphens
|
|
121
|
+
# This covers typical JSON property names while preventing SQL injection
|
|
122
|
+
if not re.match(r"^[a-zA-Z0-9_.-]+$", key):
|
|
123
|
+
raise ValueError(
|
|
124
|
+
f"Invalid filter key: '{key}'. Filter keys must contain only alphanumeric characters, underscores, dots, and hyphens."
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _json_loads(content: bytes | str | orjson.Fragment) -> Any:
|
|
109
129
|
if isinstance(content, orjson.Fragment):
|
|
110
130
|
if hasattr(content, "buf"):
|
|
111
131
|
content = content.buf
|
|
@@ -125,9 +145,7 @@ def _row_to_item(
|
|
|
125
145
|
namespace: tuple[str, ...],
|
|
126
146
|
row: dict[str, Any],
|
|
127
147
|
*,
|
|
128
|
-
loader:
|
|
129
|
-
Callable[[Union[bytes, str, orjson.Fragment]], dict[str, Any]]
|
|
130
|
-
] = None,
|
|
148
|
+
loader: Callable[[bytes | str | orjson.Fragment], dict[str, Any]] | None = None,
|
|
131
149
|
) -> Item:
|
|
132
150
|
"""Convert a row from the database into an Item."""
|
|
133
151
|
val = row["value"]
|
|
@@ -149,9 +167,7 @@ def _row_to_search_item(
|
|
|
149
167
|
namespace: tuple[str, ...],
|
|
150
168
|
row: dict[str, Any],
|
|
151
169
|
*,
|
|
152
|
-
loader:
|
|
153
|
-
Callable[[Union[bytes, str, orjson.Fragment]], dict[str, Any]]
|
|
154
|
-
] = None,
|
|
170
|
+
loader: Callable[[bytes | str | orjson.Fragment], dict[str, Any]] | None = None,
|
|
155
171
|
) -> SearchItem:
|
|
156
172
|
"""Convert a row from the database into a SearchItem."""
|
|
157
173
|
loader = loader or _json_loads
|
|
@@ -196,8 +212,8 @@ class BaseSqliteStore:
|
|
|
196
212
|
MIGRATIONS = MIGRATIONS
|
|
197
213
|
VECTOR_MIGRATIONS = VECTOR_MIGRATIONS
|
|
198
214
|
supports_ttl = True
|
|
199
|
-
index_config:
|
|
200
|
-
ttl_config:
|
|
215
|
+
index_config: SqliteIndexConfig | None = None
|
|
216
|
+
ttl_config: TTLConfig | None = None
|
|
201
217
|
|
|
202
218
|
def _get_batch_GET_ops_queries(
|
|
203
219
|
self, get_ops: Sequence[tuple[int, GetOp]]
|
|
@@ -259,7 +275,7 @@ class BaseSqliteStore:
|
|
|
259
275
|
self, put_ops: Sequence[tuple[int, PutOp]]
|
|
260
276
|
) -> tuple[
|
|
261
277
|
list[tuple[str, Sequence]],
|
|
262
|
-
|
|
278
|
+
tuple[str, Sequence[tuple[str, str, str, str]]] | None,
|
|
263
279
|
]:
|
|
264
280
|
# Last-write wins
|
|
265
281
|
dedupped_ops: dict[tuple[tuple[str, ...], str], PutOp] = {}
|
|
@@ -288,9 +304,7 @@ class BaseSqliteStore:
|
|
|
288
304
|
params = (_namespace_to_text(namespace), *keys)
|
|
289
305
|
queries.append((query, params))
|
|
290
306
|
|
|
291
|
-
embedding_request:
|
|
292
|
-
None
|
|
293
|
-
)
|
|
307
|
+
embedding_request: tuple[str, Sequence[tuple[str, str, str, str]]] | None = None
|
|
294
308
|
if inserts:
|
|
295
309
|
values = []
|
|
296
310
|
insertion_params = []
|
|
@@ -358,7 +372,7 @@ class BaseSqliteStore:
|
|
|
358
372
|
def _prepare_batch_search_queries(
|
|
359
373
|
self, search_ops: Sequence[tuple[int, SearchOp]]
|
|
360
374
|
) -> tuple[
|
|
361
|
-
list[tuple[str, list[
|
|
375
|
+
list[tuple[str, list[None | str | list[float]]]], # queries, params
|
|
362
376
|
list[tuple[int, str]], # idx, query_text pairs to embed
|
|
363
377
|
]:
|
|
364
378
|
"""
|
|
@@ -376,6 +390,8 @@ class BaseSqliteStore:
|
|
|
376
390
|
filter_conditions = []
|
|
377
391
|
if op.filter:
|
|
378
392
|
for key, value in op.filter.items():
|
|
393
|
+
_validate_filter_key(key)
|
|
394
|
+
|
|
379
395
|
if isinstance(value, dict):
|
|
380
396
|
for op_name, val in value.items():
|
|
381
397
|
condition, filter_params_ = self._get_filter_condition(
|
|
@@ -410,11 +426,12 @@ class BaseSqliteStore:
|
|
|
410
426
|
"json_extract(value, '$." + key + "') = " + str(value)
|
|
411
427
|
)
|
|
412
428
|
else:
|
|
413
|
-
#
|
|
429
|
+
# Complex objects (list, dict, …) – compare JSON text
|
|
414
430
|
filter_conditions.append(
|
|
415
431
|
"json_extract(value, '$." + key + "') = ?"
|
|
416
432
|
)
|
|
417
|
-
|
|
433
|
+
# orjson.dumps returns bytes → decode to str so SQLite sees TEXT
|
|
434
|
+
filter_params.append(orjson.dumps(value).decode())
|
|
418
435
|
|
|
419
436
|
# Vector search branch
|
|
420
437
|
if op.query and self.index_config:
|
|
@@ -625,6 +642,8 @@ class BaseSqliteStore:
|
|
|
625
642
|
|
|
626
643
|
def _get_filter_condition(self, key: str, op: str, value: Any) -> tuple[str, list]:
|
|
627
644
|
"""Helper to generate filter conditions."""
|
|
645
|
+
_validate_filter_key(key)
|
|
646
|
+
|
|
628
647
|
# We need to properly format values for SQLite JSON extraction comparison
|
|
629
648
|
if op == "$eq":
|
|
630
649
|
if isinstance(value, str):
|
|
@@ -784,11 +803,10 @@ class SqliteStore(BaseSqliteStore, BaseStore):
|
|
|
784
803
|
self,
|
|
785
804
|
conn: sqlite3.Connection,
|
|
786
805
|
*,
|
|
787
|
-
deserializer:
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
ttl: Optional[TTLConfig] = None,
|
|
806
|
+
deserializer: Callable[[bytes | str | orjson.Fragment], dict[str, Any]]
|
|
807
|
+
| None = None,
|
|
808
|
+
index: SqliteIndexConfig | None = None,
|
|
809
|
+
ttl: TTLConfig | None = None,
|
|
792
810
|
):
|
|
793
811
|
super().__init__()
|
|
794
812
|
self._deserializer = deserializer
|
|
@@ -801,7 +819,7 @@ class SqliteStore(BaseSqliteStore, BaseStore):
|
|
|
801
819
|
else:
|
|
802
820
|
self.embeddings = None
|
|
803
821
|
self.ttl_config = ttl
|
|
804
|
-
self._ttl_sweeper_thread:
|
|
822
|
+
self._ttl_sweeper_thread: threading.Thread | None = None
|
|
805
823
|
self._ttl_stop_event = threading.Event()
|
|
806
824
|
|
|
807
825
|
def _get_batch_GET_ops_queries(
|
|
@@ -862,6 +880,8 @@ class SqliteStore(BaseSqliteStore, BaseStore):
|
|
|
862
880
|
|
|
863
881
|
def _get_filter_condition(self, key: str, op: str, value: Any) -> tuple[str, list]:
|
|
864
882
|
"""Helper to generate filter conditions."""
|
|
883
|
+
_validate_filter_key(key)
|
|
884
|
+
|
|
865
885
|
# We need to properly format values for SQLite JSON extraction comparison
|
|
866
886
|
if op == "$eq":
|
|
867
887
|
if isinstance(value, str):
|
|
@@ -955,9 +975,9 @@ class SqliteStore(BaseSqliteStore, BaseStore):
|
|
|
955
975
|
cls,
|
|
956
976
|
conn_string: str,
|
|
957
977
|
*,
|
|
958
|
-
index:
|
|
959
|
-
ttl:
|
|
960
|
-
) -> Iterator[
|
|
978
|
+
index: SqliteIndexConfig | None = None,
|
|
979
|
+
ttl: TTLConfig | None = None,
|
|
980
|
+
) -> Iterator[SqliteStore]:
|
|
961
981
|
"""Create a new SqliteStore instance from a connection string.
|
|
962
982
|
|
|
963
983
|
Args:
|
|
@@ -985,10 +1005,9 @@ class SqliteStore(BaseSqliteStore, BaseStore):
|
|
|
985
1005
|
Args:
|
|
986
1006
|
transaction (bool): whether to use transaction for the DB operations
|
|
987
1007
|
"""
|
|
1008
|
+
if not self.is_setup:
|
|
1009
|
+
self.setup()
|
|
988
1010
|
with self.lock:
|
|
989
|
-
if not self.is_setup:
|
|
990
|
-
self.setup()
|
|
991
|
-
|
|
992
1011
|
if transaction:
|
|
993
1012
|
self.conn.execute("BEGIN")
|
|
994
1013
|
|
|
@@ -1006,10 +1025,10 @@ class SqliteStore(BaseSqliteStore, BaseStore):
|
|
|
1006
1025
|
This method creates the necessary tables in the SQLite database if they don't
|
|
1007
1026
|
already exist and runs database migrations. It should be called before first use.
|
|
1008
1027
|
"""
|
|
1009
|
-
if self.is_setup:
|
|
1010
|
-
return
|
|
1011
1028
|
|
|
1012
1029
|
with self.lock:
|
|
1030
|
+
if self.is_setup:
|
|
1031
|
+
return
|
|
1013
1032
|
# Create migrations table if it doesn't exist
|
|
1014
1033
|
self.conn.executescript(
|
|
1015
1034
|
"""
|
|
@@ -1086,7 +1105,7 @@ class SqliteStore(BaseSqliteStore, BaseStore):
|
|
|
1086
1105
|
return deleted_count
|
|
1087
1106
|
|
|
1088
1107
|
def start_ttl_sweeper(
|
|
1089
|
-
self, sweep_interval_minutes:
|
|
1108
|
+
self, sweep_interval_minutes: int | None = None
|
|
1090
1109
|
) -> concurrent.futures.Future[None]:
|
|
1091
1110
|
"""Periodically delete expired store items based on TTL.
|
|
1092
1111
|
|
|
@@ -1143,7 +1162,7 @@ class SqliteStore(BaseSqliteStore, BaseStore):
|
|
|
1143
1162
|
)
|
|
1144
1163
|
return future
|
|
1145
1164
|
|
|
1146
|
-
def stop_ttl_sweeper(self, timeout:
|
|
1165
|
+
def stop_ttl_sweeper(self, timeout: float | None = None) -> bool:
|
|
1147
1166
|
"""Stop the TTL sweeper thread if it's running.
|
|
1148
1167
|
|
|
1149
1168
|
Args:
|
|
@@ -1395,7 +1414,7 @@ def _ensure_index_config(
|
|
|
1395
1414
|
) -> tuple[Any, SqliteIndexConfig]:
|
|
1396
1415
|
"""Process and validate index configuration."""
|
|
1397
1416
|
index_config = index_config.copy()
|
|
1398
|
-
tokenized: list[tuple[str,
|
|
1417
|
+
tokenized: list[tuple[str, Literal["$"] | list[str]]] = []
|
|
1399
1418
|
tot = 0
|
|
1400
1419
|
text_fields = index_config.get("text_fields") or ["$"]
|
|
1401
1420
|
if isinstance(text_fields, str):
|