langgraph-checkpoint-objectstorage 0.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langgraph_checkpoint_objectstorage/__init__.py +5 -0
- langgraph_checkpoint_objectstorage/envelope.py +53 -0
- langgraph_checkpoint_objectstorage/keys.py +49 -0
- langgraph_checkpoint_objectstorage/py.typed +0 -0
- langgraph_checkpoint_objectstorage/saver.py +478 -0
- langgraph_checkpoint_objectstorage-0.1.7.dist-info/METADATA +360 -0
- langgraph_checkpoint_objectstorage-0.1.7.dist-info/RECORD +9 -0
- langgraph_checkpoint_objectstorage-0.1.7.dist-info/WHEEL +4 -0
- langgraph_checkpoint_objectstorage-0.1.7.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import ormsgpack
|
|
6
|
+
from langgraph.checkpoint.base import Checkpoint, CheckpointMetadata
|
|
7
|
+
from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer
|
|
8
|
+
|
|
9
|
+
_serde = JsonPlusSerializer()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def pack_checkpoint(
|
|
13
|
+
checkpoint: Checkpoint,
|
|
14
|
+
metadata: CheckpointMetadata,
|
|
15
|
+
parent_checkpoint_id: str | None,
|
|
16
|
+
) -> bytes:
|
|
17
|
+
ck_type, ck_bytes = _serde.dumps_typed(checkpoint)
|
|
18
|
+
md_type, md_bytes = _serde.dumps_typed(metadata)
|
|
19
|
+
return ormsgpack.packb(
|
|
20
|
+
{
|
|
21
|
+
"checkpoint": [ck_type, ck_bytes],
|
|
22
|
+
"metadata": [md_type, md_bytes],
|
|
23
|
+
"parent_checkpoint_id": parent_checkpoint_id,
|
|
24
|
+
}
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def unpack_checkpoint(
|
|
29
|
+
data: bytes,
|
|
30
|
+
) -> tuple[Checkpoint, CheckpointMetadata, str | None]:
|
|
31
|
+
obj = ormsgpack.unpackb(data)
|
|
32
|
+
checkpoint = _serde.loads_typed(tuple(obj["checkpoint"]))
|
|
33
|
+
metadata = _serde.loads_typed(tuple(obj["metadata"]))
|
|
34
|
+
return checkpoint, metadata, obj["parent_checkpoint_id"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def pack_write(task_id: str, idx: int, channel: str, value: Any) -> bytes:
|
|
38
|
+
v_type, v_bytes = _serde.dumps_typed(value)
|
|
39
|
+
return ormsgpack.packb(
|
|
40
|
+
{
|
|
41
|
+
"task_id": task_id,
|
|
42
|
+
"idx": idx,
|
|
43
|
+
"channel": channel,
|
|
44
|
+
"type": v_type,
|
|
45
|
+
"value": v_bytes,
|
|
46
|
+
}
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def unpack_write(data: bytes) -> tuple[str, int, str, Any]:
|
|
51
|
+
obj = ormsgpack.unpackb(data)
|
|
52
|
+
value = _serde.loads_typed((obj["type"], obj["value"]))
|
|
53
|
+
return obj["task_id"], obj["idx"], obj["channel"], value
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def _thread_ns_root(root: str, thread_id: str, checkpoint_ns: str) -> str:
|
|
5
|
+
parts = [root, thread_id]
|
|
6
|
+
if checkpoint_ns:
|
|
7
|
+
parts.append(checkpoint_ns)
|
|
8
|
+
return "/".join(parts)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def checkpoints_prefix(root: str, thread_id: str, checkpoint_ns: str) -> str:
|
|
12
|
+
return f"{_thread_ns_root(root, thread_id, checkpoint_ns)}/checkpoints/"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def checkpoint_key(
|
|
16
|
+
root: str, thread_id: str, checkpoint_ns: str, checkpoint_id: str
|
|
17
|
+
) -> str:
|
|
18
|
+
return (
|
|
19
|
+
f"{checkpoints_prefix(root, thread_id, checkpoint_ns)}{checkpoint_id}.msgpack"
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def checkpoint_id_from_key(key: str) -> str:
|
|
24
|
+
filename = key.rsplit("/", 1)[-1]
|
|
25
|
+
if not filename.endswith(".msgpack"):
|
|
26
|
+
raise ValueError(f"not a checkpoint key: {key!r}")
|
|
27
|
+
return filename[: -len(".msgpack")]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def writes_prefix(
|
|
31
|
+
root: str, thread_id: str, checkpoint_ns: str, checkpoint_id: str
|
|
32
|
+
) -> str:
|
|
33
|
+
return f"{_thread_ns_root(root, thread_id, checkpoint_ns)}/writes/{checkpoint_id}/"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def write_key(
|
|
37
|
+
root: str,
|
|
38
|
+
thread_id: str,
|
|
39
|
+
checkpoint_ns: str,
|
|
40
|
+
checkpoint_id: str,
|
|
41
|
+
task_id: str,
|
|
42
|
+
idx: int,
|
|
43
|
+
) -> str:
|
|
44
|
+
prefix = writes_prefix(root, thread_id, checkpoint_ns, checkpoint_id)
|
|
45
|
+
return f"{prefix}{task_id}/{idx}.msgpack"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def thread_prefix(root: str, thread_id: str) -> str:
|
|
49
|
+
return f"{root}/{thread_id}/"
|
|
File without changes
|
|
@@ -0,0 +1,478 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import fsspec
|
|
10
|
+
from fsspec import AbstractFileSystem
|
|
11
|
+
from fsspec.asyn import AsyncFileSystem, sync as fsspec_sync
|
|
12
|
+
from langchain_core.runnables import RunnableConfig
|
|
13
|
+
from langgraph.checkpoint.base import (
|
|
14
|
+
BaseCheckpointSaver,
|
|
15
|
+
Checkpoint,
|
|
16
|
+
CheckpointMetadata,
|
|
17
|
+
CheckpointTuple,
|
|
18
|
+
ChannelVersions,
|
|
19
|
+
WRITES_IDX_MAP,
|
|
20
|
+
get_checkpoint_metadata,
|
|
21
|
+
)
|
|
22
|
+
from typeguard import typechecked
|
|
23
|
+
|
|
24
|
+
from langgraph_checkpoint_objectstorage import envelope, keys
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger("langgraph_checkpoint_objectstorage")
|
|
27
|
+
|
|
28
|
+
_LOG_LEVEL_ENV = "LANGGRAPH_CHECKPOINT_OBJECTSTORAGE_LOG_LEVEL"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _thread_ns(config: RunnableConfig) -> tuple[str, str]:
|
|
32
|
+
configurable = config["configurable"]
|
|
33
|
+
return configurable["thread_id"], configurable.get("checkpoint_ns", "")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _cfg(
|
|
37
|
+
thread_id: str, checkpoint_ns: str, checkpoint_id: str | None
|
|
38
|
+
) -> RunnableConfig:
|
|
39
|
+
return {
|
|
40
|
+
"configurable": {
|
|
41
|
+
"thread_id": thread_id,
|
|
42
|
+
"checkpoint_ns": checkpoint_ns,
|
|
43
|
+
"checkpoint_id": checkpoint_id,
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class ObjectStorageSaver(BaseCheckpointSaver):
|
|
49
|
+
"""LangGraph checkpoint saver backed by local filesystem, GCS, or S3.
|
|
50
|
+
|
|
51
|
+
One class for all three backends -- which one is used is decided by
|
|
52
|
+
the fsspec filesystem passed in (or resolved from a URI via
|
|
53
|
+
`from_conn_string`), never by subclassing. Each checkpoint and each
|
|
54
|
+
pending write is stored as its own object under `root`, keyed by
|
|
55
|
+
thread_id/checkpoint_ns so unrelated threads never collide and writes
|
|
56
|
+
never require a read-modify-write on an existing key.
|
|
57
|
+
|
|
58
|
+
Example:
|
|
59
|
+
>>> saver = ObjectStorageSaver.from_conn_string("file:///tmp/checkpoints")
|
|
60
|
+
>>> graph = builder.compile(checkpointer=saver)
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
@typechecked
|
|
64
|
+
def __init__(self, fs: AbstractFileSystem, root: str) -> None:
|
|
65
|
+
"""Wrap an existing fsspec filesystem as a checkpoint store.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
fs: Any fsspec `AbstractFileSystem` instance (`LocalFileSystem`,
|
|
69
|
+
`S3FileSystem`, `GCSFileSystem`, ...). Async-native
|
|
70
|
+
filesystems (`s3fs`, `gcsfs`) get true async I/O; others
|
|
71
|
+
run through a thread pool.
|
|
72
|
+
root: Root prefix under which every checkpoint and write is
|
|
73
|
+
stored -- a directory path for local filesystems, or a
|
|
74
|
+
"bucket/prefix" path for object storage.
|
|
75
|
+
"""
|
|
76
|
+
super().__init__()
|
|
77
|
+
self.fs = fs
|
|
78
|
+
self.root = root.rstrip("/")
|
|
79
|
+
self._is_async_native = isinstance(fs, AsyncFileSystem)
|
|
80
|
+
level_name = os.environ.get(_LOG_LEVEL_ENV)
|
|
81
|
+
if level_name:
|
|
82
|
+
logger.setLevel(level_name.upper())
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
@typechecked
|
|
86
|
+
def from_conn_string(
|
|
87
|
+
cls, conn_string: str, **storage_options: Any
|
|
88
|
+
) -> "ObjectStorageSaver":
|
|
89
|
+
"""Build a saver from an fsspec connection string.
|
|
90
|
+
|
|
91
|
+
Args:
|
|
92
|
+
conn_string: An fsspec URI, e.g. `"file:///path"`,
|
|
93
|
+
`"s3://bucket/prefix"`, or `"gcs://bucket/prefix"`.
|
|
94
|
+
**storage_options: Forwarded to the underlying fsspec
|
|
95
|
+
filesystem constructor -- useful for explicit credentials
|
|
96
|
+
or a custom S3-compatible endpoint (MinIO, etc.).
|
|
97
|
+
|
|
98
|
+
Returns:
|
|
99
|
+
A new `ObjectStorageSaver` backed by the resolved filesystem.
|
|
100
|
+
"""
|
|
101
|
+
storage_options.setdefault("skip_instance_cache", True)
|
|
102
|
+
fs, path = fsspec.core.url_to_fs(conn_string, **storage_options)
|
|
103
|
+
return cls(fs, path)
|
|
104
|
+
|
|
105
|
+
def _run_sync(self, func, *args, **kwargs):
|
|
106
|
+
if self._is_async_native:
|
|
107
|
+
return fsspec_sync(self.fs.loop, func, *args, **kwargs)
|
|
108
|
+
return asyncio.run(func(*args, **kwargs))
|
|
109
|
+
|
|
110
|
+
async def _cat(self, key: str) -> bytes:
|
|
111
|
+
try:
|
|
112
|
+
if self._is_async_native:
|
|
113
|
+
data = await self.fs._cat_file(key)
|
|
114
|
+
else:
|
|
115
|
+
data = await asyncio.to_thread(self.fs.cat_file, key)
|
|
116
|
+
except FileNotFoundError:
|
|
117
|
+
logger.debug("cat key=%s -> not found", key)
|
|
118
|
+
raise
|
|
119
|
+
logger.debug("cat key=%s -> %d bytes", key, len(data))
|
|
120
|
+
return data
|
|
121
|
+
|
|
122
|
+
async def _pipe(self, key: str, data: bytes) -> None:
|
|
123
|
+
parent = key.rsplit("/", 1)[0]
|
|
124
|
+
if self._is_async_native:
|
|
125
|
+
await self.fs._makedirs(parent, exist_ok=True)
|
|
126
|
+
await self.fs._pipe_file(key, data)
|
|
127
|
+
else:
|
|
128
|
+
await asyncio.to_thread(self.fs.makedirs, parent, exist_ok=True)
|
|
129
|
+
await asyncio.to_thread(self.fs.pipe_file, key, data)
|
|
130
|
+
logger.debug("pipe key=%s <- %d bytes", key, len(data))
|
|
131
|
+
|
|
132
|
+
async def _find(self, prefix: str) -> list[str]:
|
|
133
|
+
try:
|
|
134
|
+
if self._is_async_native:
|
|
135
|
+
found = await self.fs._find(prefix)
|
|
136
|
+
else:
|
|
137
|
+
found = await asyncio.to_thread(self.fs.find, prefix)
|
|
138
|
+
except FileNotFoundError:
|
|
139
|
+
logger.debug("find prefix=%s -> not found", prefix)
|
|
140
|
+
raise
|
|
141
|
+
logger.debug("find prefix=%s -> %d keys", prefix, len(found))
|
|
142
|
+
return found
|
|
143
|
+
|
|
144
|
+
async def _exists(self, key: str) -> bool:
|
|
145
|
+
if self._is_async_native:
|
|
146
|
+
result = await self.fs._exists(key)
|
|
147
|
+
else:
|
|
148
|
+
result = await asyncio.to_thread(self.fs.exists, key)
|
|
149
|
+
logger.debug("exists key=%s -> %s", key, result)
|
|
150
|
+
return result
|
|
151
|
+
|
|
152
|
+
async def _rm(self, prefix: str) -> None:
|
|
153
|
+
try:
|
|
154
|
+
if self._is_async_native:
|
|
155
|
+
await self.fs._rm(prefix, recursive=True)
|
|
156
|
+
else:
|
|
157
|
+
await asyncio.to_thread(self.fs.rm, prefix, recursive=True)
|
|
158
|
+
except FileNotFoundError:
|
|
159
|
+
logger.debug("rm prefix=%s -> not found", prefix)
|
|
160
|
+
raise
|
|
161
|
+
logger.debug("rm prefix=%s -> removed", prefix)
|
|
162
|
+
|
|
163
|
+
async def _read_pending_writes(
|
|
164
|
+
self, thread_id: str, checkpoint_ns: str, checkpoint_id: str
|
|
165
|
+
) -> list[tuple[str, str, Any]]:
|
|
166
|
+
prefix = keys.writes_prefix(self.root, thread_id, checkpoint_ns, checkpoint_id)
|
|
167
|
+
try:
|
|
168
|
+
write_keys = await self._find(prefix)
|
|
169
|
+
except FileNotFoundError:
|
|
170
|
+
return []
|
|
171
|
+
entries = []
|
|
172
|
+
for key in write_keys:
|
|
173
|
+
data = await self._cat(key)
|
|
174
|
+
entries.append(envelope.unpack_write(data))
|
|
175
|
+
entries.sort(key=lambda e: (e[0], e[1]))
|
|
176
|
+
return [(task_id, channel, value) for task_id, idx, channel, value in entries]
|
|
177
|
+
|
|
178
|
+
async def _put(
|
|
179
|
+
self,
|
|
180
|
+
config: RunnableConfig,
|
|
181
|
+
checkpoint: Checkpoint,
|
|
182
|
+
metadata: CheckpointMetadata,
|
|
183
|
+
new_versions: ChannelVersions,
|
|
184
|
+
) -> RunnableConfig:
|
|
185
|
+
thread_id, checkpoint_ns = _thread_ns(config)
|
|
186
|
+
checkpoint_id = checkpoint["id"]
|
|
187
|
+
parent_checkpoint_id = config["configurable"].get("checkpoint_id")
|
|
188
|
+
full_metadata = get_checkpoint_metadata(config, metadata)
|
|
189
|
+
key = keys.checkpoint_key(self.root, thread_id, checkpoint_ns, checkpoint_id)
|
|
190
|
+
data = envelope.pack_checkpoint(checkpoint, full_metadata, parent_checkpoint_id)
|
|
191
|
+
await self._pipe(key, data)
|
|
192
|
+
return _cfg(thread_id, checkpoint_ns, checkpoint_id)
|
|
193
|
+
|
|
194
|
+
async def _get_tuple(self, config: RunnableConfig) -> CheckpointTuple | None:
|
|
195
|
+
thread_id, checkpoint_ns = _thread_ns(config)
|
|
196
|
+
checkpoint_id = config["configurable"].get("checkpoint_id")
|
|
197
|
+
if checkpoint_id is None:
|
|
198
|
+
logger.debug(
|
|
199
|
+
"get_tuple thread=%s ns=%s -> no checkpoint_id, resolving latest",
|
|
200
|
+
thread_id,
|
|
201
|
+
checkpoint_ns,
|
|
202
|
+
)
|
|
203
|
+
prefix = keys.checkpoints_prefix(self.root, thread_id, checkpoint_ns)
|
|
204
|
+
try:
|
|
205
|
+
candidates = await self._find(prefix)
|
|
206
|
+
except FileNotFoundError:
|
|
207
|
+
return None
|
|
208
|
+
if not candidates:
|
|
209
|
+
return None
|
|
210
|
+
key = max(candidates)
|
|
211
|
+
checkpoint_id = keys.checkpoint_id_from_key(key)
|
|
212
|
+
else:
|
|
213
|
+
key = keys.checkpoint_key(
|
|
214
|
+
self.root, thread_id, checkpoint_ns, checkpoint_id
|
|
215
|
+
)
|
|
216
|
+
try:
|
|
217
|
+
data = await self._cat(key)
|
|
218
|
+
except FileNotFoundError:
|
|
219
|
+
return None
|
|
220
|
+
checkpoint, metadata, parent_checkpoint_id = envelope.unpack_checkpoint(data)
|
|
221
|
+
parent_config = (
|
|
222
|
+
_cfg(thread_id, checkpoint_ns, parent_checkpoint_id)
|
|
223
|
+
if parent_checkpoint_id
|
|
224
|
+
else None
|
|
225
|
+
)
|
|
226
|
+
pending_writes = await self._read_pending_writes(
|
|
227
|
+
thread_id, checkpoint_ns, checkpoint_id
|
|
228
|
+
)
|
|
229
|
+
return CheckpointTuple(
|
|
230
|
+
config=_cfg(thread_id, checkpoint_ns, checkpoint_id),
|
|
231
|
+
checkpoint=checkpoint,
|
|
232
|
+
metadata=metadata,
|
|
233
|
+
parent_config=parent_config,
|
|
234
|
+
pending_writes=pending_writes,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
async def _put_writes(
|
|
238
|
+
self,
|
|
239
|
+
config: RunnableConfig,
|
|
240
|
+
writes: Sequence[tuple[str, Any]],
|
|
241
|
+
task_id: str,
|
|
242
|
+
task_path: str = "",
|
|
243
|
+
) -> None:
|
|
244
|
+
thread_id, checkpoint_ns = _thread_ns(config)
|
|
245
|
+
checkpoint_id = config["configurable"]["checkpoint_id"]
|
|
246
|
+
overwrite = all(channel in WRITES_IDX_MAP for channel, _ in writes)
|
|
247
|
+
for idx, (channel, value) in enumerate(writes):
|
|
248
|
+
actual_idx = WRITES_IDX_MAP.get(channel, idx)
|
|
249
|
+
key = keys.write_key(
|
|
250
|
+
self.root, thread_id, checkpoint_ns, checkpoint_id, task_id, actual_idx
|
|
251
|
+
)
|
|
252
|
+
if not overwrite and await self._exists(key):
|
|
253
|
+
logger.debug(
|
|
254
|
+
"put_writes task=%s channel=%s idx=%s -> skipped, write already exists",
|
|
255
|
+
task_id,
|
|
256
|
+
channel,
|
|
257
|
+
actual_idx,
|
|
258
|
+
)
|
|
259
|
+
continue
|
|
260
|
+
data = envelope.pack_write(task_id, actual_idx, channel, value)
|
|
261
|
+
await self._pipe(key, data)
|
|
262
|
+
|
|
263
|
+
async def _list(
|
|
264
|
+
self,
|
|
265
|
+
config: RunnableConfig | None,
|
|
266
|
+
*,
|
|
267
|
+
filter: dict[str, Any] | None = None,
|
|
268
|
+
before: RunnableConfig | None = None,
|
|
269
|
+
limit: int | None = None,
|
|
270
|
+
):
|
|
271
|
+
thread_id, checkpoint_ns = _thread_ns(config)
|
|
272
|
+
prefix = keys.checkpoints_prefix(self.root, thread_id, checkpoint_ns)
|
|
273
|
+
try:
|
|
274
|
+
candidate_keys = sorted(await self._find(prefix), reverse=True)
|
|
275
|
+
except FileNotFoundError:
|
|
276
|
+
return
|
|
277
|
+
before_id = before["configurable"]["checkpoint_id"] if before else None
|
|
278
|
+
count = 0
|
|
279
|
+
for key in candidate_keys:
|
|
280
|
+
checkpoint_id = keys.checkpoint_id_from_key(key)
|
|
281
|
+
if before_id is not None and checkpoint_id >= before_id:
|
|
282
|
+
continue
|
|
283
|
+
data = await self._cat(key)
|
|
284
|
+
checkpoint, metadata, parent_checkpoint_id = envelope.unpack_checkpoint(
|
|
285
|
+
data
|
|
286
|
+
)
|
|
287
|
+
if filter and not all(metadata.get(k) == v for k, v in filter.items()):
|
|
288
|
+
continue
|
|
289
|
+
parent_config = (
|
|
290
|
+
_cfg(thread_id, checkpoint_ns, parent_checkpoint_id)
|
|
291
|
+
if parent_checkpoint_id
|
|
292
|
+
else None
|
|
293
|
+
)
|
|
294
|
+
pending_writes = await self._read_pending_writes(
|
|
295
|
+
thread_id, checkpoint_ns, checkpoint_id
|
|
296
|
+
)
|
|
297
|
+
yield CheckpointTuple(
|
|
298
|
+
config=_cfg(thread_id, checkpoint_ns, checkpoint_id),
|
|
299
|
+
checkpoint=checkpoint,
|
|
300
|
+
metadata=metadata,
|
|
301
|
+
parent_config=parent_config,
|
|
302
|
+
pending_writes=pending_writes,
|
|
303
|
+
)
|
|
304
|
+
count += 1
|
|
305
|
+
if limit is not None and count >= limit:
|
|
306
|
+
return
|
|
307
|
+
|
|
308
|
+
async def _delete_thread(self, thread_id: str) -> None:
|
|
309
|
+
prefix = keys.thread_prefix(self.root, thread_id)
|
|
310
|
+
try:
|
|
311
|
+
await self._rm(prefix)
|
|
312
|
+
except FileNotFoundError:
|
|
313
|
+
pass
|
|
314
|
+
|
|
315
|
+
async def aget_tuple(self, config: RunnableConfig) -> CheckpointTuple | None:
|
|
316
|
+
"""Async variant of `get_tuple`. See `get_tuple` for details."""
|
|
317
|
+
return await self._get_tuple(config)
|
|
318
|
+
|
|
319
|
+
async def alist(
|
|
320
|
+
self,
|
|
321
|
+
config: RunnableConfig | None,
|
|
322
|
+
*,
|
|
323
|
+
filter: dict[str, Any] | None = None,
|
|
324
|
+
before: RunnableConfig | None = None,
|
|
325
|
+
limit: int | None = None,
|
|
326
|
+
) -> AsyncIterator[CheckpointTuple]:
|
|
327
|
+
"""Async variant of `list`. See `list` for details."""
|
|
328
|
+
async for tup in self._list(config, filter=filter, before=before, limit=limit):
|
|
329
|
+
yield tup
|
|
330
|
+
|
|
331
|
+
@typechecked
|
|
332
|
+
async def aput(
|
|
333
|
+
self,
|
|
334
|
+
config: Mapping[str, Any],
|
|
335
|
+
checkpoint: Mapping[str, Any],
|
|
336
|
+
metadata: Mapping[str, Any],
|
|
337
|
+
new_versions: ChannelVersions,
|
|
338
|
+
) -> RunnableConfig:
|
|
339
|
+
"""Async variant of `put`. See `put` for details."""
|
|
340
|
+
return await self._put(config, checkpoint, metadata, new_versions)
|
|
341
|
+
|
|
342
|
+
@typechecked
|
|
343
|
+
async def aput_writes(
|
|
344
|
+
self,
|
|
345
|
+
config: Mapping[str, Any],
|
|
346
|
+
writes: Sequence[tuple[str, Any]],
|
|
347
|
+
task_id: str,
|
|
348
|
+
task_path: str = "",
|
|
349
|
+
) -> None:
|
|
350
|
+
"""Async variant of `put_writes`. See `put_writes` for details."""
|
|
351
|
+
await self._put_writes(config, writes, task_id, task_path)
|
|
352
|
+
|
|
353
|
+
@typechecked
|
|
354
|
+
async def adelete_thread(self, thread_id: str) -> None:
|
|
355
|
+
"""Async variant of `delete_thread`. See `delete_thread` for details."""
|
|
356
|
+
await self._delete_thread(thread_id)
|
|
357
|
+
|
|
358
|
+
def get_tuple(self, config: RunnableConfig) -> CheckpointTuple | None:
|
|
359
|
+
"""Fetch a checkpoint tuple for the given configuration.
|
|
360
|
+
|
|
361
|
+
If `config["configurable"]` has no `"checkpoint_id"`, returns the
|
|
362
|
+
latest checkpoint in that thread/namespace.
|
|
363
|
+
|
|
364
|
+
Args:
|
|
365
|
+
config: Must contain `configurable.thread_id`. Optionally
|
|
366
|
+
`configurable.checkpoint_ns` (default `""`) and
|
|
367
|
+
`configurable.checkpoint_id` for an exact checkpoint
|
|
368
|
+
rather than the latest.
|
|
369
|
+
|
|
370
|
+
Returns:
|
|
371
|
+
The matching `CheckpointTuple`, or `None` if no checkpoint
|
|
372
|
+
exists for that thread/namespace/id -- never raises for
|
|
373
|
+
"not found".
|
|
374
|
+
"""
|
|
375
|
+
return self._run_sync(self._get_tuple, config)
|
|
376
|
+
|
|
377
|
+
def list(
|
|
378
|
+
self,
|
|
379
|
+
config: RunnableConfig | None,
|
|
380
|
+
*,
|
|
381
|
+
filter: dict[str, Any] | None = None,
|
|
382
|
+
before: RunnableConfig | None = None,
|
|
383
|
+
limit: int | None = None,
|
|
384
|
+
) -> Iterator[CheckpointTuple]:
|
|
385
|
+
"""List checkpoints for a thread/namespace, newest first.
|
|
386
|
+
|
|
387
|
+
Args:
|
|
388
|
+
config: Must contain `configurable.thread_id` and, optionally,
|
|
389
|
+
`configurable.checkpoint_ns` (default `""`).
|
|
390
|
+
filter: Metadata key/value pairs a checkpoint must match
|
|
391
|
+
(applied client-side -- see the README's Known
|
|
392
|
+
limitations section).
|
|
393
|
+
before: Only return checkpoints older than
|
|
394
|
+
`before["configurable"]["checkpoint_id"]`.
|
|
395
|
+
limit: Maximum number of checkpoints to return.
|
|
396
|
+
|
|
397
|
+
Returns:
|
|
398
|
+
An iterator of matching `CheckpointTuple`s, newest first. This
|
|
399
|
+
sync version collects all results eagerly before yielding the
|
|
400
|
+
first one (it wraps the async implementation via
|
|
401
|
+
`asyncio.run`, which can't stream lazily) -- use `alist` from
|
|
402
|
+
async code for true streaming.
|
|
403
|
+
"""
|
|
404
|
+
|
|
405
|
+
async def _collect() -> list[CheckpointTuple]:
|
|
406
|
+
return [
|
|
407
|
+
t
|
|
408
|
+
async for t in self._list(
|
|
409
|
+
config, filter=filter, before=before, limit=limit
|
|
410
|
+
)
|
|
411
|
+
]
|
|
412
|
+
|
|
413
|
+
yield from self._run_sync(_collect)
|
|
414
|
+
|
|
415
|
+
@typechecked
|
|
416
|
+
def put(
|
|
417
|
+
self,
|
|
418
|
+
config: Mapping[str, Any],
|
|
419
|
+
checkpoint: Mapping[str, Any],
|
|
420
|
+
metadata: Mapping[str, Any],
|
|
421
|
+
new_versions: ChannelVersions,
|
|
422
|
+
) -> RunnableConfig:
|
|
423
|
+
"""Store a checkpoint as its own object.
|
|
424
|
+
|
|
425
|
+
Args:
|
|
426
|
+
config: Must contain `configurable.thread_id`. If
|
|
427
|
+
`configurable.checkpoint_id` is set, the new checkpoint's
|
|
428
|
+
parent is set to that id; otherwise it has no parent.
|
|
429
|
+
checkpoint: The checkpoint to store, as produced by LangGraph.
|
|
430
|
+
Stored opaquely (see the README's Runtime type checking
|
|
431
|
+
section) -- never hand-extracted field by field.
|
|
432
|
+
metadata: Metadata to store alongside the checkpoint.
|
|
433
|
+
new_versions: Unused by this saver -- accepted for
|
|
434
|
+
`BaseCheckpointSaver` contract compatibility.
|
|
435
|
+
|
|
436
|
+
Returns:
|
|
437
|
+
The config to use to fetch this exact checkpoint later
|
|
438
|
+
(`configurable.thread_id`/`checkpoint_ns`/`checkpoint_id`).
|
|
439
|
+
"""
|
|
440
|
+
return self._run_sync(self._put, config, checkpoint, metadata, new_versions)
|
|
441
|
+
|
|
442
|
+
@typechecked
|
|
443
|
+
def put_writes(
|
|
444
|
+
self,
|
|
445
|
+
config: Mapping[str, Any],
|
|
446
|
+
writes: Sequence[tuple[str, Any]],
|
|
447
|
+
task_id: str,
|
|
448
|
+
task_path: str = "",
|
|
449
|
+
) -> None:
|
|
450
|
+
"""Store pending writes linked to a checkpoint.
|
|
451
|
+
|
|
452
|
+
Regular channels use "first write wins" -- a duplicate
|
|
453
|
+
`(task_id, idx)` is silently ignored, so a retried task can't
|
|
454
|
+
clobber a write another task already committed. The control
|
|
455
|
+
channels (`ERROR`, `SCHEDULED`, `INTERRUPT`, `RESUME`) always
|
|
456
|
+
overwrite instead, since they must reflect the latest state.
|
|
457
|
+
|
|
458
|
+
Args:
|
|
459
|
+
config: Must contain `configurable.thread_id` and
|
|
460
|
+
`configurable.checkpoint_id` -- the checkpoint these
|
|
461
|
+
writes are pending against.
|
|
462
|
+
writes: `(channel, value)` pairs to store.
|
|
463
|
+
task_id: Identifier for the task that produced these writes.
|
|
464
|
+
task_path: Unused by this saver -- accepted for
|
|
465
|
+
`BaseCheckpointSaver` contract compatibility.
|
|
466
|
+
"""
|
|
467
|
+
self._run_sync(self._put_writes, config, writes, task_id, task_path)
|
|
468
|
+
|
|
469
|
+
@typechecked
|
|
470
|
+
def delete_thread(self, thread_id: str) -> None:
|
|
471
|
+
"""Delete every checkpoint and write for a thread, across all namespaces.
|
|
472
|
+
|
|
473
|
+
A no-op if the thread doesn't exist.
|
|
474
|
+
|
|
475
|
+
Args:
|
|
476
|
+
thread_id: The thread to delete.
|
|
477
|
+
"""
|
|
478
|
+
self._run_sync(self._delete_thread, thread_id)
|
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: langgraph-checkpoint-objectstorage
|
|
3
|
+
Version: 0.1.7
|
|
4
|
+
Summary: LangGraph checkpoint saver backed by local filesystem, GCS, or S3 via fsspec
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Requires-Dist: fsspec>=2024.10.0
|
|
9
|
+
Requires-Dist: langchain-core>=0.2.38
|
|
10
|
+
Requires-Dist: langgraph-checkpoint<5,>=4.0
|
|
11
|
+
Requires-Dist: ormsgpack>=1.12.0
|
|
12
|
+
Requires-Dist: typeguard>=4.0
|
|
13
|
+
Provides-Extra: gcs
|
|
14
|
+
Requires-Dist: gcsfs>=2024.10.0; extra == 'gcs'
|
|
15
|
+
Provides-Extra: s3
|
|
16
|
+
Requires-Dist: s3fs>=2024.10.0; extra == 's3'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# langgraph-checkpoint-objectstorage
|
|
20
|
+
|
|
21
|
+
[](https://github.com/sergiommarcial/langgraph-objectstorage-checkpoint/actions/workflows/ci.yml)
|
|
22
|
+
|
|
23
|
+
A [LangGraph](https://github.com/langchain-ai/langgraph) `BaseCheckpointSaver`
|
|
24
|
+
that persists checkpoints to local filesystem, Google Cloud Storage, or AWS
|
|
25
|
+
S3. One class, backend picked by connection string, nothing to run beyond a
|
|
26
|
+
bucket (or a directory).
|
|
27
|
+
|
|
28
|
+
## Table of contents
|
|
29
|
+
|
|
30
|
+
- [Features](#features)
|
|
31
|
+
- [Requirements](#requirements)
|
|
32
|
+
- [Install](#install)
|
|
33
|
+
- [Quickstart](#quickstart)
|
|
34
|
+
- [Examples](#examples)
|
|
35
|
+
- [Choosing a backend](#choosing-a-backend)
|
|
36
|
+
- [Architecture](#architecture)
|
|
37
|
+
- [Runtime type checking](#runtime-type-checking)
|
|
38
|
+
- [Logging](#logging)
|
|
39
|
+
- [Known limitations](#known-limitations)
|
|
40
|
+
- [Development](#development)
|
|
41
|
+
- [Contributing](#contributing)
|
|
42
|
+
- [License](#license)
|
|
43
|
+
|
|
44
|
+
## Features
|
|
45
|
+
|
|
46
|
+
- `ObjectStorageSaver.from_conn_string(...)` picks local disk, GCS, or S3
|
|
47
|
+
from the URI scheme. No per-backend subclasses.
|
|
48
|
+
- Full sync and async support: every `BaseCheckpointSaver` method, both
|
|
49
|
+
flavors (`get_tuple`/`aget_tuple`, `put`/`aput`, `list`/`alist`,
|
|
50
|
+
`put_writes`/`aput_writes`, `delete_thread`/`adelete_thread`).
|
|
51
|
+
- Tested against the official contract with
|
|
52
|
+
[`langgraph-checkpoint-conformance`](https://pypi.org/project/langgraph-checkpoint-conformance/)
|
|
53
|
+
on all three backends, not just hand-written assertions.
|
|
54
|
+
- Runtime type checking on the public API via
|
|
55
|
+
[typeguard](https://typeguard.readthedocs.io/) catches wrong-argument-type
|
|
56
|
+
mistakes at the call site.
|
|
57
|
+
- Ships `py.typed` for full static type coverage under mypy/pyright.
|
|
58
|
+
- No database or extra service required in production, just object storage.
|
|
59
|
+
|
|
60
|
+
## Requirements
|
|
61
|
+
|
|
62
|
+
Python 3.11+.
|
|
63
|
+
|
|
64
|
+
## Install
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
pip install langgraph-checkpoint-objectstorage # local filesystem only
|
|
68
|
+
pip install "langgraph-checkpoint-objectstorage[s3]" # + AWS S3
|
|
69
|
+
pip install "langgraph-checkpoint-objectstorage[gcs]" # + Google Cloud Storage
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Quickstart
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from langgraph.graph import END, START, StateGraph
|
|
76
|
+
from langgraph_checkpoint_objectstorage import ObjectStorageSaver
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def increment(state: dict) -> dict:
|
|
80
|
+
return {"count": state["count"] + 1}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
builder = StateGraph(dict)
|
|
84
|
+
builder.add_node("increment", increment)
|
|
85
|
+
builder.add_edge(START, "increment")
|
|
86
|
+
builder.add_edge("increment", END)
|
|
87
|
+
|
|
88
|
+
saver = ObjectStorageSaver.from_conn_string("file:///tmp/checkpoints")
|
|
89
|
+
graph = builder.compile(checkpointer=saver)
|
|
90
|
+
|
|
91
|
+
config = {"configurable": {"thread_id": "1"}}
|
|
92
|
+
result = graph.invoke({"count": 0}, config)
|
|
93
|
+
print(result) # {"count": 1}
|
|
94
|
+
|
|
95
|
+
# Checkpoints persisted under the thread survive process restarts --
|
|
96
|
+
# inspect or resume from the same thread_id at any later point:
|
|
97
|
+
history = list(graph.get_state_history(config))
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Examples
|
|
101
|
+
|
|
102
|
+
This same quickstart, runnable under three build tools:
|
|
103
|
+
|
|
104
|
+
- [`examples/pip`](examples/pip): venv + `pip install -r requirements.txt`
|
|
105
|
+
- [`examples/uv/filesystem`](examples/uv/filesystem): `uv run main.py`
|
|
106
|
+
- [`examples/poetry/filesystem`](examples/poetry/filesystem): `poetry install && poetry run python main.py`
|
|
107
|
+
|
|
108
|
+
Each installs the package from this repo via a local path dependency
|
|
109
|
+
(swap for a normal PyPI dependency once the package is published).
|
|
110
|
+
|
|
111
|
+
Further along, against object storage instead of local disk (real bucket
|
|
112
|
+
or a local emulator, no cloud account needed): multiple independent
|
|
113
|
+
sessions run sequentially and one is resumed later, plus the same pattern
|
|
114
|
+
run concurrently via the async API, for both S3 and GCS:
|
|
115
|
+
|
|
116
|
+
- [`examples/uv/s3`](examples/uv/s3) / [`examples/uv/s3-async`](examples/uv/s3-async)
|
|
117
|
+
- [`examples/poetry/gcs`](examples/poetry/gcs) / [`examples/poetry/gcs-async`](examples/poetry/gcs-async)
|
|
118
|
+
|
|
119
|
+
Sequential and concurrent are separate examples rather than one combined
|
|
120
|
+
script, see [Known limitations](#known-limitations) for why.
|
|
121
|
+
|
|
122
|
+
## Choosing a backend
|
|
123
|
+
|
|
124
|
+
Swap the connection string; everything else stays the same.
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
from langgraph_checkpoint_objectstorage import ObjectStorageSaver
|
|
128
|
+
|
|
129
|
+
# Local filesystem -- handy for development, or single-node deployments
|
|
130
|
+
saver = ObjectStorageSaver.from_conn_string("file:///var/lib/my-app/checkpoints")
|
|
131
|
+
|
|
132
|
+
# Google Cloud Storage
|
|
133
|
+
saver = ObjectStorageSaver.from_conn_string("gcs://my-bucket/checkpoints")
|
|
134
|
+
|
|
135
|
+
# AWS S3
|
|
136
|
+
saver = ObjectStorageSaver.from_conn_string("s3://my-bucket/checkpoints")
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
`from_conn_string` forwards extra keyword arguments to the underlying
|
|
140
|
+
[fsspec](https://filesystem-spec.readthedocs.io/) filesystem constructor.
|
|
141
|
+
Useful for explicit credentials, non-default regions, or S3-compatible
|
|
142
|
+
endpoints (MinIO, Cloudflare R2, etc.):
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
saver = ObjectStorageSaver.from_conn_string(
|
|
146
|
+
"s3://my-bucket/checkpoints",
|
|
147
|
+
key="...",
|
|
148
|
+
secret="...",
|
|
149
|
+
client_kwargs={"endpoint_url": "https://minio.internal:9000"},
|
|
150
|
+
)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Credentials otherwise follow each backend's normal resolution: AWS's usual
|
|
154
|
+
chain (env vars, `~/.aws/credentials`, instance/task role) for S3,
|
|
155
|
+
Application Default Credentials for GCS. Nothing library-specific to
|
|
156
|
+
configure beyond the connection string.
|
|
157
|
+
|
|
158
|
+
## Architecture
|
|
159
|
+
|
|
160
|
+
Business logic (key layout, filtering, ordering, idempotency) is written
|
|
161
|
+
once, as async methods. The public sync API is a thin `asyncio.run(...)`
|
|
162
|
+
wrapper around that same async core, not a second implementation, so
|
|
163
|
+
there's a single source of truth per operation instead of sync and async
|
|
164
|
+
code drifting apart. An I/O bridge picks native async calls when the
|
|
165
|
+
backend supports them (`s3fs`, `gcsfs`) and falls back to
|
|
166
|
+
`asyncio.to_thread` when it doesn't (local disk):
|
|
167
|
+
|
|
168
|
+
```mermaid
|
|
169
|
+
flowchart TD
|
|
170
|
+
App["Your application<br/>(graph.invoke / ainvoke)"]
|
|
171
|
+
|
|
172
|
+
subgraph PublicAPI["Public API — BaseCheckpointSaver contract"]
|
|
173
|
+
Sync["put / get_tuple / list /<br/>put_writes / delete_thread"]
|
|
174
|
+
Async["aput / aget_tuple / alist /<br/>aput_writes / adelete_thread"]
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
Core["Async core<br/>(business logic, written once)"]
|
|
178
|
+
Bridge["I/O bridge<br/>_cat / _pipe / _find / _exists / _rm"]
|
|
179
|
+
Native["fsspec async-native<br/>(s3fs, gcsfs)"]
|
|
180
|
+
Threaded["asyncio.to_thread<br/>(LocalFileSystem)"]
|
|
181
|
+
Backend[("Local disk / S3 / GCS")]
|
|
182
|
+
|
|
183
|
+
App --> Sync
|
|
184
|
+
App --> Async
|
|
185
|
+
Sync -->|"asyncio.run(...)<br/>thin wrapper, not a<br/>second implementation"| Core
|
|
186
|
+
Async --> Core
|
|
187
|
+
Core --> Bridge
|
|
188
|
+
Bridge -->|backend supports async| Native --> Backend
|
|
189
|
+
Bridge -->|no native async| Threaded --> Backend
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Each checkpoint and each write becomes its own object: no read-modify-write
|
|
193
|
+
on existing keys, so concurrent writers on different threads never race,
|
|
194
|
+
and a `put` or `put_writes` call is always a single write:
|
|
195
|
+
|
|
196
|
+
```mermaid
|
|
197
|
+
flowchart TD
|
|
198
|
+
Root["{root}"] --> Thread["{thread_id}/"]
|
|
199
|
+
Thread --> NS["{checkpoint_ns}/"]
|
|
200
|
+
NS --> CkptDir["checkpoints/"]
|
|
201
|
+
NS --> WriteDir["writes/"]
|
|
202
|
+
CkptDir --> Ckpt["{checkpoint_id}.msgpack<br/>checkpoint + metadata + parent_checkpoint_id"]
|
|
203
|
+
WriteDir --> WCkpt["{checkpoint_id}/"]
|
|
204
|
+
WCkpt --> WTask["{task_id}/"]
|
|
205
|
+
WTask --> WIdx["{idx}.msgpack<br/>task_id + idx + channel + value"]
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Key technical decisions this reflects:
|
|
209
|
+
|
|
210
|
+
- `checkpoint_id` is LangGraph's own [uuid6](https://github.com/langchain-ai/langgraph/blob/main/libs/checkpoint/langgraph/checkpoint/base/id.py),
|
|
211
|
+
already time-sortable, so `list()`'s newest-first ordering falls out of a
|
|
212
|
+
plain key sort, with no secondary index to keep in sync.
|
|
213
|
+
- Checkpoints and metadata are serialized through LangGraph's own
|
|
214
|
+
`JsonPlusSerializer` and treated as opaque: never hand-extracted field by
|
|
215
|
+
field, since real LangGraph objects carry fields their own TypedDicts
|
|
216
|
+
don't declare (see [Runtime type checking](#runtime-type-checking)).
|
|
217
|
+
- `put_writes`' overwrite-vs-ignore split ("first write wins" for regular
|
|
218
|
+
channels, always-replace for the control channels `ERROR`/`SCHEDULED`/
|
|
219
|
+
`INTERRUPT`/`RESUME`) mirrors the official sqlite/postgres savers exactly.
|
|
220
|
+
- `list(filter=...)` is deliberately client-side, not an oversight: object
|
|
221
|
+
storage has no query engine to push a filter into. See
|
|
222
|
+
[Known limitations](#known-limitations).
|
|
223
|
+
|
|
224
|
+
## Runtime type checking
|
|
225
|
+
|
|
226
|
+
Public methods are decorated with [typeguard](https://typeguard.readthedocs.io/)
|
|
227
|
+
and raise `typeguard.TypeCheckError` on a call with the wrong argument
|
|
228
|
+
types (e.g. a non-string `thread_id`, a `writes` argument that isn't a
|
|
229
|
+
sequence of `(channel, value)` pairs). This catches integration mistakes
|
|
230
|
+
at the call site instead of letting them corrupt stored data silently.
|
|
231
|
+
|
|
232
|
+
`config`, `checkpoint`, and `metadata` arguments are intentionally *not*
|
|
233
|
+
strictly checked against LangGraph's `RunnableConfig`/`Checkpoint`/
|
|
234
|
+
`CheckpointMetadata` TypedDicts: real LangGraph objects don't match those
|
|
235
|
+
TypedDicts exactly (a real `RunnableConfig`'s `metadata` field is a
|
|
236
|
+
`collections.ChainMap`, not a plain `dict`; real checkpoints carry fields
|
|
237
|
+
like the legacy `pending_sends` key that isn't declared at all), and strict
|
|
238
|
+
checking would reject every real invocation. Same reasoning applies to
|
|
239
|
+
`get_tuple`/`list`'s return value, which isn't runtime-checked for the same
|
|
240
|
+
reason. Other arguments (`thread_id`, `task_id`, `writes`, `limit`, ...)
|
|
241
|
+
are checked normally.
|
|
242
|
+
|
|
243
|
+
## Logging
|
|
244
|
+
|
|
245
|
+
Uses standard `logging` under the logger name
|
|
246
|
+
`langgraph_checkpoint_objectstorage`. No handlers are configured, so it
|
|
247
|
+
stays silent until your application's logging config says otherwise.
|
|
248
|
+
|
|
249
|
+
For quick debugging, set `LANGGRAPH_CHECKPOINT_OBJECTSTORAGE_LOG_LEVEL=DEBUG`
|
|
250
|
+
before constructing an `ObjectStorageSaver`. It emits one DEBUG line per
|
|
251
|
+
storage read/write/list with the key or prefix touched.
|
|
252
|
+
|
|
253
|
+
## Known limitations
|
|
254
|
+
|
|
255
|
+
- `list(filter=...)` is client-side: every checkpoint in the thread/namespace
|
|
256
|
+
is fetched and filtered in Python, since object storage has no query
|
|
257
|
+
engine to push the filter into. Fine for typical thread histories (dozens
|
|
258
|
+
to low hundreds of checkpoints); a very long-running thread's `list` calls
|
|
259
|
+
will get proportionally slower.
|
|
260
|
+
- No garbage collection or retention policy. Old checkpoints accumulate
|
|
261
|
+
until you call `delete_thread`, or you set up bucket lifecycle rules
|
|
262
|
+
yourself.
|
|
263
|
+
- Two writers on the *same* `thread_id` writing concurrently can race, at
|
|
264
|
+
the same guarantee level as the official sqlite saver (last write "wins"
|
|
265
|
+
by whichever checkpoint_id sorts last, not by wall-clock order under
|
|
266
|
+
clock skew). Concurrent writers on different threads never race.
|
|
267
|
+
- The sync `list()` eagerly collects all matching checkpoints before
|
|
268
|
+
yielding the first one (it wraps the async implementation via
|
|
269
|
+
`asyncio.run`, which can't stream lazily). Use `alist()` from async code
|
|
270
|
+
if you need true streaming.
|
|
271
|
+
- Don't mix sync and async calls on the *same* `ObjectStorageSaver`
|
|
272
|
+
instance against S3 or GCS. Sync calls run on a persistent background
|
|
273
|
+
loop the underlying filesystem maintains; async calls run on whichever
|
|
274
|
+
loop the caller provides. The aiohttp session those backends use can
|
|
275
|
+
only belong to one loop at a time, so alternating between the two on
|
|
276
|
+
one instance breaks with `RuntimeError: ... attached to a different
|
|
277
|
+
loop`. Build a separate saver instance per usage style instead (see
|
|
278
|
+
`examples/uv/s3` vs `examples/uv/s3-async`, or `examples/poetry/gcs` vs
|
|
279
|
+
`examples/poetry/gcs-async`). Local filesystem isn't affected: it has no
|
|
280
|
+
persistent session to misalign.
|
|
281
|
+
|
|
282
|
+
## Development
|
|
283
|
+
|
|
284
|
+
```bash
|
|
285
|
+
make install # sync deps into an isolated .venv (installs uv if missing)
|
|
286
|
+
make lint # black --check, pyflakes, bandit, vulture -- runs before test/build
|
|
287
|
+
make format # apply black formatting in place
|
|
288
|
+
make test # full suite -- docker-compose integration tests auto-skip if not up
|
|
289
|
+
make test-unit # tests/unit only -- no external services, fast
|
|
290
|
+
make test-integration # tests/integration -- starts docker-compose emulators first
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
`test`/`test-unit`/`test-integration`/`build` all run `lint` first, so a
|
|
294
|
+
formatting or static-analysis failure blocks the run rather than
|
|
295
|
+
surfacing only after tests pass. `bandit`/`vulture` are scoped to `src/`
|
|
296
|
+
only: `bandit` flags every pytest `assert` and the tests' intentionally
|
|
297
|
+
fake credentials, and `vulture` can't see that `ObjectStorageSaver`'s
|
|
298
|
+
public methods are called by library consumers rather than this codebase
|
|
299
|
+
(see `vulture_whitelist.py`).
|
|
300
|
+
|
|
301
|
+
Or without `make`, directly via `uv`:
|
|
302
|
+
|
|
303
|
+
```bash
|
|
304
|
+
uv sync --all-extras --group dev
|
|
305
|
+
uv run pytest
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
Test layout: `tests/unit/` exercises internal modules (key layout,
|
|
309
|
+
serialization, the saver's core logic, logging, type checking) with no
|
|
310
|
+
external service. `tests/integration/` validates the full
|
|
311
|
+
`BaseCheckpointSaver` contract via
|
|
312
|
+
[`langgraph-checkpoint-conformance`](https://pypi.org/project/langgraph-checkpoint-conformance/)
|
|
313
|
+
against real backends: local disk, an in-process moto server for S3, and
|
|
314
|
+
(via `docker-compose.yaml`) `fake-gcs-server`/`moto-server` containers for a
|
|
315
|
+
full local GCS/S3 round-trip with no cloud account needed. A gated test
|
|
316
|
+
against a real GCS bucket runs only when `GCS_TEST_BUCKET` is set.
|
|
317
|
+
|
|
318
|
+
```bash
|
|
319
|
+
make compose-up # start local S3/GCS emulators
|
|
320
|
+
make test-integration
|
|
321
|
+
make compose-down # stop them when done
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
## Contributing
|
|
325
|
+
|
|
326
|
+
Issues and PRs welcome. Before opening one, `make test` should pass
|
|
327
|
+
(`make test-integration` too, if your change touches backend I/O). CI runs
|
|
328
|
+
lint, unit tests (Python 3.11/3.12/3.13), and integration tests against
|
|
329
|
+
docker-compose emulators on every push and PR.
|
|
330
|
+
|
|
331
|
+
Add an entry under `## [Unreleased]` in [`CHANGELOG.md`](CHANGELOG.md) for
|
|
332
|
+
any user-facing change. On merge to `main`, CI moves that section into a
|
|
333
|
+
new dated version automatically (see the `release` job in
|
|
334
|
+
`.github/workflows/ci.yml`). An empty `[Unreleased]` just gets a generic
|
|
335
|
+
placeholder line instead, so it's worth taking the extra minute.
|
|
336
|
+
|
|
337
|
+
### Releasing
|
|
338
|
+
|
|
339
|
+
The `release` job only bumps `pyproject.toml` and `CHANGELOG.md` and
|
|
340
|
+
pushes that commit to `main` -- it doesn't tag or publish anything.
|
|
341
|
+
Publishing to PyPI is a manual step, since it's the one part of this
|
|
342
|
+
pipeline that isn't reversible:
|
|
343
|
+
|
|
344
|
+
```bash
|
|
345
|
+
git pull origin main
|
|
346
|
+
git tag -a vX.Y.Z -m vX.Y.Z
|
|
347
|
+
git push origin vX.Y.Z
|
|
348
|
+
```
|
|
349
|
+
|
|
350
|
+
Pushing that tag triggers `.github/workflows/publish.yml`, which builds
|
|
351
|
+
the package and publishes it to PyPI via trusted publishing (no token
|
|
352
|
+
needed). Create the GitHub release from the same tag, e.g.:
|
|
353
|
+
|
|
354
|
+
```bash
|
|
355
|
+
gh release create vX.Y.Z --title vX.Y.Z --generate-notes
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
## License
|
|
359
|
+
|
|
360
|
+
[MIT](LICENSE).
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
langgraph_checkpoint_objectstorage/__init__.py,sha256=nd0AxTrW0Dv5MQlrlF2aJwZ_SqDbZu-e8RX27gm7LIs,180
|
|
2
|
+
langgraph_checkpoint_objectstorage/envelope.py,sha256=dMnB-KKJSd5CkT_5innloqH6mW8o95wBdt7yFT3dw1Y,1546
|
|
3
|
+
langgraph_checkpoint_objectstorage/keys.py,sha256=cgsDjztuY1PkcVPK59f2aiZ01x94egAogM9QZEZLBrE,1358
|
|
4
|
+
langgraph_checkpoint_objectstorage/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
langgraph_checkpoint_objectstorage/saver.py,sha256=r9cvLpj1FDFjL7UfuYkpEMSnGw1ZkVYMlbbMiPPnH8g,18282
|
|
6
|
+
langgraph_checkpoint_objectstorage-0.1.7.dist-info/METADATA,sha256=ZJLoQLfMaXNpBXPUi4yQfxe3lLXIUaax8OhwbVopJjQ,14945
|
|
7
|
+
langgraph_checkpoint_objectstorage-0.1.7.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
8
|
+
langgraph_checkpoint_objectstorage-0.1.7.dist-info/licenses/LICENSE,sha256=gGtZrZUyidOHBs8CU2BO9VBa_qgnurOGJMu-MZTsqXM,1063
|
|
9
|
+
langgraph_checkpoint_objectstorage-0.1.7.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sergio
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|