coco-framework 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coco/__init__.py +3 -0
- coco/agents/__init__.py +9 -0
- coco/agents/_subagent_transformer.py +307 -0
- coco/agents/factory.py +2164 -0
- coco/agents/middleware/__init__.py +100 -0
- coco/agents/middleware/_execution.py +385 -0
- coco/agents/middleware/_redaction.py +454 -0
- coco/agents/middleware/_retry.py +134 -0
- coco/agents/middleware/_trace_policy.py +61 -0
- coco/agents/middleware/context_editing.py +306 -0
- coco/agents/middleware/file_search.py +433 -0
- coco/agents/middleware/human_in_the_loop.py +680 -0
- coco/agents/middleware/internal_call_transformer.py +108 -0
- coco/agents/middleware/model_call_limit.py +267 -0
- coco/agents/middleware/model_fallback.py +417 -0
- coco/agents/middleware/model_retry.py +327 -0
- coco/agents/middleware/pii.py +878 -0
- coco/agents/middleware/provider_tool_search.py +307 -0
- coco/agents/middleware/shell_tool.py +949 -0
- coco/agents/middleware/summarization.py +910 -0
- coco/agents/middleware/todo.py +357 -0
- coco/agents/middleware/tool_call_limit.py +495 -0
- coco/agents/middleware/tool_emulator.py +251 -0
- coco/agents/middleware/tool_error.py +209 -0
- coco/agents/middleware/tool_retry.py +421 -0
- coco/agents/middleware/tool_selection.py +515 -0
- coco/agents/middleware/types.py +2201 -0
- coco/agents/structured_output.py +472 -0
- coco/chat_models/__init__.py +7 -0
- coco/chat_models/base.py +1082 -0
- coco/embeddings/__init__.py +18 -0
- coco/embeddings/base.py +279 -0
- coco/mcp/__init__.py +36 -0
- coco/mcp/adapter.py +245 -0
- coco/mcp/elicitation.py +394 -0
- coco/mcp/tools.py +307 -0
- coco/messages/__init__.py +73 -0
- coco/py.typed +0 -0
- coco/rate_limiters/__init__.py +13 -0
- coco/tools/__init__.py +22 -0
- coco/tools/tool_node.py +20 -0
- coco_anthropic/__init__.py +15 -0
- coco_anthropic/_client_utils.py +81 -0
- coco_anthropic/_compat.py +228 -0
- coco_anthropic/_sdk_compat.py +124 -0
- coco_anthropic/_version.py +3 -0
- coco_anthropic/chat_models.py +3349 -0
- coco_anthropic/data/__init__.py +1 -0
- coco_anthropic/data/_profiles.py +491 -0
- coco_anthropic/data/profile_augmentations.toml +69 -0
- coco_anthropic/experimental.py +140 -0
- coco_anthropic/llms.py +428 -0
- coco_anthropic/middleware/__init__.py +25 -0
- coco_anthropic/middleware/anthropic_tools.py +1197 -0
- coco_anthropic/middleware/bash.py +84 -0
- coco_anthropic/middleware/file_search.py +360 -0
- coco_anthropic/middleware/prompt_caching.py +276 -0
- coco_anthropic/output_parsers.py +93 -0
- coco_anthropic/py.typed +0 -0
- coco_core/__init__.py +20 -0
- coco_core/_api/__init__.py +87 -0
- coco_core/_api/beta_decorator.py +266 -0
- coco_core/_api/deprecation.py +637 -0
- coco_core/_api/internal.py +23 -0
- coco_core/_api/path.py +50 -0
- coco_core/_import_utils.py +41 -0
- coco_core/_protocol.py +625 -0
- coco_core/_security/__init__.py +36 -0
- coco_core/_security/_exceptions.py +9 -0
- coco_core/_security/_policy.py +313 -0
- coco_core/_security/_ssrf_protection.py +155 -0
- coco_core/_security/_transport.py +254 -0
- coco_core/agents.py +258 -0
- coco_core/caches.py +272 -0
- coco_core/callbacks/__init__.py +132 -0
- coco_core/callbacks/base.py +1229 -0
- coco_core/callbacks/file.py +266 -0
- coco_core/callbacks/manager.py +2826 -0
- coco_core/callbacks/stdout.py +123 -0
- coco_core/callbacks/streaming_stdout.py +154 -0
- coco_core/callbacks/usage.py +124 -0
- coco_core/chat_history.py +267 -0
- coco_core/chat_loaders.py +26 -0
- coco_core/chat_sessions.py +19 -0
- coco_core/cross_encoders.py +18 -0
- coco_core/document_loaders/__init__.py +36 -0
- coco_core/document_loaders/base.py +155 -0
- coco_core/document_loaders/blob_loaders.py +38 -0
- coco_core/documents/__init__.py +55 -0
- coco_core/documents/base.py +347 -0
- coco_core/documents/compressor.py +74 -0
- coco_core/documents/transformers.py +79 -0
- coco_core/embeddings/__init__.py +31 -0
- coco_core/embeddings/embeddings.py +78 -0
- coco_core/embeddings/fake.py +129 -0
- coco_core/env.py +22 -0
- coco_core/example_selectors/__init__.py +47 -0
- coco_core/example_selectors/base.py +60 -0
- coco_core/example_selectors/length_based.py +131 -0
- coco_core/example_selectors/semantic_similarity.py +366 -0
- coco_core/exceptions.py +166 -0
- coco_core/globals.py +72 -0
- coco_core/indexing/__init__.py +53 -0
- coco_core/indexing/api.py +954 -0
- coco_core/indexing/base.py +661 -0
- coco_core/indexing/in_memory.py +104 -0
- coco_core/language_models/__init__.py +116 -0
- coco_core/language_models/_compat_bridge.py +844 -0
- coco_core/language_models/_utils.py +343 -0
- coco_core/language_models/base.py +500 -0
- coco_core/language_models/chat_model_stream.py +1525 -0
- coco_core/language_models/chat_models.py +2748 -0
- coco_core/language_models/fake.py +137 -0
- coco_core/language_models/fake_chat_models.py +396 -0
- coco_core/language_models/llms.py +1569 -0
- coco_core/language_models/model_profile.py +183 -0
- coco_core/load/__init__.py +44 -0
- coco_core/load/_validation.py +191 -0
- coco_core/load/dump.py +120 -0
- coco_core/load/load.py +828 -0
- coco_core/load/mapping.py +1097 -0
- coco_core/load/serializable.py +406 -0
- coco_core/messages/__init__.py +198 -0
- coco_core/messages/ai.py +854 -0
- coco_core/messages/base.py +518 -0
- coco_core/messages/block_translators/__init__.py +112 -0
- coco_core/messages/block_translators/anthropic.py +534 -0
- coco_core/messages/block_translators/bedrock.py +108 -0
- coco_core/messages/block_translators/bedrock_converse.py +329 -0
- coco_core/messages/block_translators/coco_v0.py +303 -0
- coco_core/messages/block_translators/google_genai.py +610 -0
- coco_core/messages/block_translators/google_vertexai.py +21 -0
- coco_core/messages/block_translators/groq.py +157 -0
- coco_core/messages/block_translators/openai.py +1089 -0
- coco_core/messages/chat.py +64 -0
- coco_core/messages/content.py +1488 -0
- coco_core/messages/function.py +62 -0
- coco_core/messages/human.py +70 -0
- coco_core/messages/modifier.py +33 -0
- coco_core/messages/system.py +70 -0
- coco_core/messages/tool.py +418 -0
- coco_core/messages/utils.py +2442 -0
- coco_core/output_parsers/__init__.py +101 -0
- coco_core/output_parsers/base.py +361 -0
- coco_core/output_parsers/format_instructions.py +16 -0
- coco_core/output_parsers/json.py +139 -0
- coco_core/output_parsers/list.py +249 -0
- coco_core/output_parsers/openai_functions.py +324 -0
- coco_core/output_parsers/openai_tools.py +383 -0
- coco_core/output_parsers/pydantic.py +141 -0
- coco_core/output_parsers/string.py +63 -0
- coco_core/output_parsers/transform.py +175 -0
- coco_core/output_parsers/xml.py +301 -0
- coco_core/outputs/__init__.py +66 -0
- coco_core/outputs/chat_generation.py +147 -0
- coco_core/outputs/chat_result.py +39 -0
- coco_core/outputs/generation.py +80 -0
- coco_core/outputs/llm_result.py +112 -0
- coco_core/outputs/run_info.py +22 -0
- coco_core/prompt_values.py +161 -0
- coco_core/prompts/__init__.py +101 -0
- coco_core/prompts/base.py +508 -0
- coco_core/prompts/chat.py +1495 -0
- coco_core/prompts/dict.py +177 -0
- coco_core/prompts/few_shot.py +483 -0
- coco_core/prompts/few_shot_with_templates.py +239 -0
- coco_core/prompts/image.py +177 -0
- coco_core/prompts/loading.py +296 -0
- coco_core/prompts/message.py +97 -0
- coco_core/prompts/prompt.py +312 -0
- coco_core/prompts/string.py +399 -0
- coco_core/prompts/structured.py +214 -0
- coco_core/py.typed +0 -0
- coco_core/rate_limiters.py +254 -0
- coco_core/retrievers.py +328 -0
- coco_core/runnables/__init__.py +136 -0
- coco_core/runnables/base.py +6725 -0
- coco_core/runnables/branch.py +457 -0
- coco_core/runnables/config.py +712 -0
- coco_core/runnables/configurable.py +710 -0
- coco_core/runnables/fallbacks.py +663 -0
- coco_core/runnables/graph.py +741 -0
- coco_core/runnables/graph_ascii.py +366 -0
- coco_core/runnables/graph_mermaid.py +503 -0
- coco_core/runnables/graph_png.py +215 -0
- coco_core/runnables/history.py +631 -0
- coco_core/runnables/passthrough.py +849 -0
- coco_core/runnables/retry.py +379 -0
- coco_core/runnables/router.py +239 -0
- coco_core/runnables/schema.py +188 -0
- coco_core/runnables/utils.py +798 -0
- coco_core/stores.py +291 -0
- coco_core/structured_query.py +203 -0
- coco_core/sys_info.py +154 -0
- coco_core/tools/__init__.py +95 -0
- coco_core/tools/base.py +1948 -0
- coco_core/tools/convert.py +497 -0
- coco_core/tools/render.py +67 -0
- coco_core/tools/retriever.py +94 -0
- coco_core/tools/simple.py +208 -0
- coco_core/tools/structured.py +322 -0
- coco_core/tracers/__init__.py +47 -0
- coco_core/tracers/_compat.py +95 -0
- coco_core/tracers/_context.py +337 -0
- coco_core/tracers/_streaming.py +53 -0
- coco_core/tracers/base.py +955 -0
- coco_core/tracers/coco.py +279 -0
- coco_core/tracers/context.py +205 -0
- coco_core/tracers/core.py +766 -0
- coco_core/tracers/event_stream.py +1101 -0
- coco_core/tracers/log_stream.py +765 -0
- coco_core/tracers/memory_stream.py +200 -0
- coco_core/tracers/root_listeners.py +130 -0
- coco_core/tracers/run_collector.py +40 -0
- coco_core/tracers/schemas.py +14 -0
- coco_core/tracers/stdout.py +205 -0
- coco_core/utils/__init__.py +111 -0
- coco_core/utils/_gateway.py +274 -0
- coco_core/utils/_merge.py +218 -0
- coco_core/utils/aiter.py +357 -0
- coco_core/utils/env.py +86 -0
- coco_core/utils/formatting.py +80 -0
- coco_core/utils/function_calling.py +869 -0
- coco_core/utils/html.py +132 -0
- coco_core/utils/image.py +15 -0
- coco_core/utils/input.py +82 -0
- coco_core/utils/interactive_env.py +12 -0
- coco_core/utils/iter.py +229 -0
- coco_core/utils/json.py +230 -0
- coco_core/utils/json_schema.py +273 -0
- coco_core/utils/mustache.py +706 -0
- coco_core/utils/pydantic.py +631 -0
- coco_core/utils/strings.py +69 -0
- coco_core/utils/usage.py +61 -0
- coco_core/utils/utils.py +528 -0
- coco_core/utils/uuid.py +57 -0
- coco_core/vectorstores/__init__.py +53 -0
- coco_core/vectorstores/base.py +1111 -0
- coco_core/vectorstores/in_memory.py +546 -0
- coco_core/vectorstores/utils.py +163 -0
- coco_core/version.py +3 -0
- coco_framework-0.1.0.dist-info/METADATA +184 -0
- coco_framework-0.1.0.dist-info/RECORD +400 -0
- coco_framework-0.1.0.dist-info/WHEEL +4 -0
- coco_framework-0.1.0.dist-info/licenses/LICENSE +24 -0
- coco_framework-0.1.0.dist-info/licenses/NOTICE +15 -0
- coco_graph/_internal/__init__.py +4 -0
- coco_graph/_internal/_cache.py +31 -0
- coco_graph/_internal/_config.py +474 -0
- coco_graph/_internal/_constants.py +140 -0
- coco_graph/_internal/_fields.py +213 -0
- coco_graph/_internal/_future.py +220 -0
- coco_graph/_internal/_pydantic.py +275 -0
- coco_graph/_internal/_queue.py +124 -0
- coco_graph/_internal/_replay.py +90 -0
- coco_graph/_internal/_retry.py +29 -0
- coco_graph/_internal/_runnable.py +969 -0
- coco_graph/_internal/_scratchpad.py +19 -0
- coco_graph/_internal/_serde.py +253 -0
- coco_graph/_internal/_timeout.py +25 -0
- coco_graph/_internal/_typing.py +54 -0
- coco_graph/cache/base/__init__.py +48 -0
- coco_graph/cache/base/py.typed +0 -0
- coco_graph/cache/memory/__init__.py +73 -0
- coco_graph/cache/redis/__init__.py +144 -0
- coco_graph/cache/sqlite/__init__.py +120 -0
- coco_graph/callbacks.py +394 -0
- coco_graph/channels/__init__.py +29 -0
- coco_graph/channels/any_value.py +72 -0
- coco_graph/channels/base.py +121 -0
- coco_graph/channels/binop.py +155 -0
- coco_graph/channels/delta.py +202 -0
- coco_graph/channels/ephemeral_value.py +79 -0
- coco_graph/channels/last_value.py +151 -0
- coco_graph/channels/named_barrier_value.py +167 -0
- coco_graph/channels/topic.py +94 -0
- coco_graph/channels/untracked_value.py +73 -0
- coco_graph/checkpoint/base/__init__.py +861 -0
- coco_graph/checkpoint/base/id.py +109 -0
- coco_graph/checkpoint/base/py.typed +0 -0
- coco_graph/checkpoint/memory/__init__.py +698 -0
- coco_graph/checkpoint/memory/py.typed +0 -0
- coco_graph/checkpoint/postgres/__init__.py +602 -0
- coco_graph/checkpoint/postgres/_ainternal.py +23 -0
- coco_graph/checkpoint/postgres/_internal.py +21 -0
- coco_graph/checkpoint/postgres/aio.py +691 -0
- coco_graph/checkpoint/postgres/base.py +666 -0
- coco_graph/checkpoint/postgres/py.typed +0 -0
- coco_graph/checkpoint/postgres/shallow.py +967 -0
- coco_graph/checkpoint/serde/__init__.py +0 -0
- coco_graph/checkpoint/serde/_msgpack.py +97 -0
- coco_graph/checkpoint/serde/base.py +64 -0
- coco_graph/checkpoint/serde/encrypted.py +80 -0
- coco_graph/checkpoint/serde/event_hooks.py +52 -0
- coco_graph/checkpoint/serde/jsonplus.py +883 -0
- coco_graph/checkpoint/serde/py.typed +0 -0
- coco_graph/checkpoint/serde/types.py +68 -0
- coco_graph/checkpoint/sqlite/__init__.py +645 -0
- coco_graph/checkpoint/sqlite/_delta.py +172 -0
- coco_graph/checkpoint/sqlite/aio.py +738 -0
- coco_graph/checkpoint/sqlite/py.typed +0 -0
- coco_graph/checkpoint/sqlite/utils.py +116 -0
- coco_graph/config.py +196 -0
- coco_graph/constants.py +64 -0
- coco_graph/errors.py +241 -0
- coco_graph/func/__init__.py +620 -0
- coco_graph/graph/__init__.py +12 -0
- coco_graph/graph/_branch.py +225 -0
- coco_graph/graph/_node.py +103 -0
- coco_graph/graph/message.py +437 -0
- coco_graph/graph/state.py +1978 -0
- coco_graph/graph/ui.py +227 -0
- coco_graph/managed/__init__.py +3 -0
- coco_graph/managed/base.py +31 -0
- coco_graph/managed/is_last_step.py +24 -0
- coco_graph/prebuilt/__init__.py +23 -0
- coco_graph/prebuilt/_tool_call_stream.py +117 -0
- coco_graph/prebuilt/_tool_call_transformer.py +165 -0
- coco_graph/prebuilt/chat_agent_executor.py +1015 -0
- coco_graph/prebuilt/interrupt.py +105 -0
- coco_graph/prebuilt/py.typed +0 -0
- coco_graph/prebuilt/tool_node.py +2030 -0
- coco_graph/prebuilt/tool_validator.py +221 -0
- coco_graph/pregel/__init__.py +3 -0
- coco_graph/pregel/_algo.py +1460 -0
- coco_graph/pregel/_call.py +298 -0
- coco_graph/pregel/_checkpoint.py +331 -0
- coco_graph/pregel/_config.py +0 -0
- coco_graph/pregel/_draw.py +294 -0
- coco_graph/pregel/_executor.py +223 -0
- coco_graph/pregel/_io.py +174 -0
- coco_graph/pregel/_log.py +3 -0
- coco_graph/pregel/_loop.py +1988 -0
- coco_graph/pregel/_messages.py +461 -0
- coco_graph/pregel/_read.py +312 -0
- coco_graph/pregel/_retry.py +854 -0
- coco_graph/pregel/_runner.py +941 -0
- coco_graph/pregel/_tools.py +268 -0
- coco_graph/pregel/_utils.py +221 -0
- coco_graph/pregel/_validate.py +120 -0
- coco_graph/pregel/_write.py +192 -0
- coco_graph/pregel/debug.py +302 -0
- coco_graph/pregel/main.py +4364 -0
- coco_graph/pregel/protocol.py +288 -0
- coco_graph/pregel/types.py +38 -0
- coco_graph/py.typed +0 -0
- coco_graph/runtime.py +350 -0
- coco_graph/store/base/__init__.py +1322 -0
- coco_graph/store/base/batch.py +371 -0
- coco_graph/store/base/embed.py +436 -0
- coco_graph/store/base/py.typed +0 -0
- coco_graph/store/memory/__init__.py +592 -0
- coco_graph/store/memory/py.typed +0 -0
- coco_graph/store/postgres/__init__.py +4 -0
- coco_graph/store/postgres/aio.py +592 -0
- coco_graph/store/postgres/base.py +1480 -0
- coco_graph/store/postgres/py.typed +0 -0
- coco_graph/store/sqlite/__init__.py +4 -0
- coco_graph/store/sqlite/aio.py +632 -0
- coco_graph/store/sqlite/base.py +1514 -0
- coco_graph/stream/__init__.py +45 -0
- coco_graph/stream/_convert.py +32 -0
- coco_graph/stream/_mux.py +523 -0
- coco_graph/stream/_types.py +330 -0
- coco_graph/stream/run_stream.py +735 -0
- coco_graph/stream/stream_channel.py +341 -0
- coco_graph/stream/transformers.py +1039 -0
- coco_graph/types.py +1079 -0
- coco_graph/typing.py +48 -0
- coco_graph/utils/__init__.py +1 -0
- coco_graph/utils/config.py +4 -0
- coco_graph/utils/runnable.py +3 -0
- coco_graph/version.py +13 -0
- coco_graph/warnings.py +69 -0
- coco_openai/__init__.py +20 -0
- coco_openai/_compat.py +45 -0
- coco_openai/_version.py +3 -0
- coco_openai/chat_models/__init__.py +6 -0
- coco_openai/chat_models/_client_utils.py +684 -0
- coco_openai/chat_models/_compat.py +525 -0
- coco_openai/chat_models/azure.py +1227 -0
- coco_openai/chat_models/base.py +5654 -0
- coco_openai/chat_models/codex.py +557 -0
- coco_openai/chatgpt_oauth.py +1063 -0
- coco_openai/data/__init__.py +1 -0
- coco_openai/data/_profiles.py +1655 -0
- coco_openai/data/profile_augmentations.toml +127 -0
- coco_openai/embeddings/__init__.py +6 -0
- coco_openai/embeddings/azure.py +240 -0
- coco_openai/embeddings/base.py +820 -0
- coco_openai/llms/__init__.py +6 -0
- coco_openai/llms/azure.py +249 -0
- coco_openai/llms/base.py +919 -0
- coco_openai/middleware/__init__.py +11 -0
- coco_openai/middleware/openai_moderation.py +484 -0
- coco_openai/output_parsers/__init__.py +9 -0
- coco_openai/output_parsers/tools.py +9 -0
- coco_openai/py.typed +0 -0
- coco_openai/tools/__init__.py +5 -0
- coco_openai/tools/custom_tool.py +111 -0
|
@@ -0,0 +1,666 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
import warnings
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from importlib.metadata import version as get_version
|
|
7
|
+
from typing import Any, TypedDict, cast
|
|
8
|
+
|
|
9
|
+
from coco_core.runnables import RunnableConfig
|
|
10
|
+
from coco_graph.checkpoint.base import (
|
|
11
|
+
WRITES_IDX_MAP,
|
|
12
|
+
BaseCheckpointSaver,
|
|
13
|
+
ChannelVersions,
|
|
14
|
+
DeltaChannelHistory,
|
|
15
|
+
PendingWrite,
|
|
16
|
+
get_checkpoint_id,
|
|
17
|
+
)
|
|
18
|
+
from coco_graph.checkpoint.serde.types import TASKS
|
|
19
|
+
from psycopg.types.json import Jsonb
|
|
20
|
+
|
|
21
|
+
# Page size for stage-1 paged scan in `get_delta_channel_history`. Internal
|
|
22
|
+
# constant — exposing this as a kwarg is left as a follow-up.
|
|
23
|
+
_DELTA_PAGE_SIZE = 1024
|
|
24
|
+
|
|
25
|
+
MetadataInput = dict[str, Any] | None
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
major, minor = get_version("trustcoco-graph").split(".")[:2]
|
|
29
|
+
if int(major) == 0 and int(minor) < 5:
|
|
30
|
+
warnings.warn(
|
|
31
|
+
"You're using incompatible versions of coco_graph and checkpoint-postgres. Please upgrade coco_graph to avoid unexpected behavior.",
|
|
32
|
+
DeprecationWarning,
|
|
33
|
+
stacklevel=2,
|
|
34
|
+
)
|
|
35
|
+
except Exception:
|
|
36
|
+
# skip version check if running from source
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
"""
|
|
40
|
+
To add a new migration, add a new string to the MIGRATIONS list.
|
|
41
|
+
The position of the migration in the list is the version number.
|
|
42
|
+
"""
|
|
43
|
+
MIGRATIONS = [
|
|
44
|
+
"""CREATE TABLE IF NOT EXISTS checkpoint_migrations (
|
|
45
|
+
v INTEGER PRIMARY KEY
|
|
46
|
+
);""",
|
|
47
|
+
"""CREATE TABLE IF NOT EXISTS checkpoints (
|
|
48
|
+
thread_id TEXT NOT NULL,
|
|
49
|
+
checkpoint_ns TEXT NOT NULL DEFAULT '',
|
|
50
|
+
checkpoint_id TEXT NOT NULL,
|
|
51
|
+
parent_checkpoint_id TEXT,
|
|
52
|
+
type TEXT,
|
|
53
|
+
checkpoint JSONB NOT NULL,
|
|
54
|
+
metadata JSONB NOT NULL DEFAULT '{}',
|
|
55
|
+
PRIMARY KEY (thread_id, checkpoint_ns, checkpoint_id)
|
|
56
|
+
);""",
|
|
57
|
+
"""CREATE TABLE IF NOT EXISTS checkpoint_blobs (
|
|
58
|
+
thread_id TEXT NOT NULL,
|
|
59
|
+
checkpoint_ns TEXT NOT NULL DEFAULT '',
|
|
60
|
+
channel TEXT NOT NULL,
|
|
61
|
+
version TEXT NOT NULL,
|
|
62
|
+
type TEXT NOT NULL,
|
|
63
|
+
blob BYTEA,
|
|
64
|
+
PRIMARY KEY (thread_id, checkpoint_ns, channel, version)
|
|
65
|
+
);""",
|
|
66
|
+
"""CREATE TABLE IF NOT EXISTS checkpoint_writes (
|
|
67
|
+
thread_id TEXT NOT NULL,
|
|
68
|
+
checkpoint_ns TEXT NOT NULL DEFAULT '',
|
|
69
|
+
checkpoint_id TEXT NOT NULL,
|
|
70
|
+
task_id TEXT NOT NULL,
|
|
71
|
+
idx INTEGER NOT NULL,
|
|
72
|
+
channel TEXT NOT NULL,
|
|
73
|
+
type TEXT,
|
|
74
|
+
blob BYTEA NOT NULL,
|
|
75
|
+
PRIMARY KEY (thread_id, checkpoint_ns, checkpoint_id, task_id, idx)
|
|
76
|
+
);""",
|
|
77
|
+
"ALTER TABLE checkpoint_blobs ALTER COLUMN blob DROP not null;",
|
|
78
|
+
# NOTE: this is a no-op migration to ensure that the versions in the migrations table are correct.
|
|
79
|
+
# This is necessary due to an empty migration previously added to the list.
|
|
80
|
+
"SELECT 1;",
|
|
81
|
+
"""
|
|
82
|
+
CREATE INDEX CONCURRENTLY IF NOT EXISTS checkpoints_thread_id_idx ON checkpoints(thread_id);
|
|
83
|
+
""",
|
|
84
|
+
"""
|
|
85
|
+
CREATE INDEX CONCURRENTLY IF NOT EXISTS checkpoint_blobs_thread_id_idx ON checkpoint_blobs(thread_id);
|
|
86
|
+
""",
|
|
87
|
+
"""
|
|
88
|
+
CREATE INDEX CONCURRENTLY IF NOT EXISTS checkpoint_writes_thread_id_idx ON checkpoint_writes(thread_id);
|
|
89
|
+
""",
|
|
90
|
+
"""ALTER TABLE checkpoint_writes ADD COLUMN IF NOT EXISTS task_path TEXT NOT NULL DEFAULT '';""",
|
|
91
|
+
]
|
|
92
|
+
|
|
93
|
+
SELECT_SQL = """
|
|
94
|
+
select
|
|
95
|
+
thread_id,
|
|
96
|
+
checkpoint,
|
|
97
|
+
checkpoint_ns,
|
|
98
|
+
checkpoint_id,
|
|
99
|
+
parent_checkpoint_id,
|
|
100
|
+
metadata,
|
|
101
|
+
(
|
|
102
|
+
select array_agg(array[bl.channel::bytea, bl.type::bytea, bl.blob])
|
|
103
|
+
from jsonb_each_text(checkpoint -> 'channel_versions')
|
|
104
|
+
inner join checkpoint_blobs bl
|
|
105
|
+
on bl.thread_id = checkpoints.thread_id
|
|
106
|
+
and bl.checkpoint_ns = checkpoints.checkpoint_ns
|
|
107
|
+
and bl.channel = jsonb_each_text.key
|
|
108
|
+
and bl.version = jsonb_each_text.value
|
|
109
|
+
) as channel_values,
|
|
110
|
+
(
|
|
111
|
+
select
|
|
112
|
+
array_agg(array[cw.task_id::text::bytea, cw.channel::bytea, cw.type::bytea, cw.blob] order by cw.task_id, cw.idx)
|
|
113
|
+
from checkpoint_writes cw
|
|
114
|
+
where cw.thread_id = checkpoints.thread_id
|
|
115
|
+
and cw.checkpoint_ns = checkpoints.checkpoint_ns
|
|
116
|
+
and cw.checkpoint_id = checkpoints.checkpoint_id
|
|
117
|
+
) as pending_writes
|
|
118
|
+
from checkpoints """
|
|
119
|
+
|
|
120
|
+
SELECT_PENDING_SENDS_SQL = f"""
|
|
121
|
+
select
|
|
122
|
+
checkpoint_id,
|
|
123
|
+
array_agg(array[type::bytea, blob] order by task_path, task_id, idx) as sends
|
|
124
|
+
from checkpoint_writes
|
|
125
|
+
where thread_id = %s
|
|
126
|
+
and checkpoint_id = any(%s)
|
|
127
|
+
and channel = '{TASKS}'
|
|
128
|
+
group by checkpoint_id
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
UPSERT_CHECKPOINT_BLOBS_SQL = """
|
|
132
|
+
INSERT INTO checkpoint_blobs (thread_id, checkpoint_ns, channel, version, type, blob)
|
|
133
|
+
VALUES (%s, %s, %s, %s, %s, %s)
|
|
134
|
+
ON CONFLICT (thread_id, checkpoint_ns, channel, version) DO NOTHING
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
UPSERT_CHECKPOINTS_SQL = """
|
|
138
|
+
INSERT INTO checkpoints (thread_id, checkpoint_ns, checkpoint_id, parent_checkpoint_id, checkpoint, metadata)
|
|
139
|
+
VALUES (%s, %s, %s, %s, %s, %s)
|
|
140
|
+
ON CONFLICT (thread_id, checkpoint_ns, checkpoint_id)
|
|
141
|
+
DO UPDATE SET
|
|
142
|
+
checkpoint = EXCLUDED.checkpoint,
|
|
143
|
+
metadata = EXCLUDED.metadata;
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
UPSERT_CHECKPOINT_WRITES_SQL = """
|
|
147
|
+
INSERT INTO checkpoint_writes (thread_id, checkpoint_ns, checkpoint_id, task_id, task_path, idx, channel, type, blob)
|
|
148
|
+
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
149
|
+
ON CONFLICT (thread_id, checkpoint_ns, checkpoint_id, task_id, idx) DO UPDATE SET
|
|
150
|
+
channel = EXCLUDED.channel,
|
|
151
|
+
type = EXCLUDED.type,
|
|
152
|
+
blob = EXCLUDED.blob;
|
|
153
|
+
"""
|
|
154
|
+
|
|
155
|
+
INSERT_CHECKPOINT_WRITES_SQL = """
|
|
156
|
+
INSERT INTO checkpoint_writes (thread_id, checkpoint_ns, checkpoint_id, task_id, task_path, idx, channel, type, blob)
|
|
157
|
+
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
158
|
+
ON CONFLICT (thread_id, checkpoint_ns, checkpoint_id, task_id, idx) DO NOTHING
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class _DeltaStage2Row(TypedDict, total=False):
|
|
163
|
+
"""One row from `_build_delta_stage2_sql` (a UNION ALL of writes and blobs)."""
|
|
164
|
+
|
|
165
|
+
_kind: str # "w" or "b"
|
|
166
|
+
checkpoint_id: str | None # "w" rows only
|
|
167
|
+
channel: str | None # set on both "w" and "b" rows
|
|
168
|
+
type: str | None
|
|
169
|
+
blob: bytes | None
|
|
170
|
+
task_id: str | None # "w" rows only
|
|
171
|
+
idx: int | None # "w" rows only
|
|
172
|
+
version: str | None # "b" rows only
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
# Multi-channel two-stage DeltaChannel reconstruction.
|
|
176
|
+
#
|
|
177
|
+
# Stage 1 scans checkpoint metadata (no blob bytes) and emits one row per
|
|
178
|
+
# checkpoint with K parallel JSONB key lookups (one column pair per
|
|
179
|
+
# requested delta channel: ver_i / hs_i). No subqueries, no aggregation.
|
|
180
|
+
# Python walks the parent chain once across all channels.
|
|
181
|
+
#
|
|
182
|
+
# Stage 2 fetches all writes and the seed blobs for ALL channels in a
|
|
183
|
+
# single roundtrip via `channel = ANY(%s)` and chain/seed-version
|
|
184
|
+
# filtering.
|
|
185
|
+
#
|
|
186
|
+
# Empirical comparison vs an alternative "ship full channel_versions /
|
|
187
|
+
# channel_values JSONB and let Python pick" form (1000 checkpoints,
|
|
188
|
+
# 8 total channels in graph, 3 delta channels requested):
|
|
189
|
+
#
|
|
190
|
+
# Postgres execution: A=0.24ms vs B=0.38ms (both negligible)
|
|
191
|
+
# End-to-end latency: A=6.83ms vs B=2.28ms (B is 3.0x faster)
|
|
192
|
+
# Wire payload: A=836KB vs B=330KB (61% smaller)
|
|
193
|
+
# Buffer hits: identical (167 blocks)
|
|
194
|
+
#
|
|
195
|
+
# B (this dynamic-columns design) wins because it avoids JSONB
|
|
196
|
+
# serialization on the wire and JSONB-to-dict deserialization in
|
|
197
|
+
# psycopg. Even at K=8 (8 delta channels = 16 dynamic columns), B
|
|
198
|
+
# still beats A end-to-end (4.2ms vs 6.8ms).
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _build_delta_stage1_sql(channels: Sequence[str], *, paged: bool) -> str:
|
|
202
|
+
"""Build stage 1 SQL with K parallel version lookups + seed probes.
|
|
203
|
+
|
|
204
|
+
For channels=["messages", "files"] (with `paged=True`) the result is::
|
|
205
|
+
|
|
206
|
+
SELECT checkpoint_id, parent_checkpoint_id,
|
|
207
|
+
checkpoint -> 'channel_versions' ->> %s AS ver_0,
|
|
208
|
+
EXISTS (SELECT 1 FROM checkpoint_blobs b0
|
|
209
|
+
WHERE b0.thread_id = checkpoints.thread_id
|
|
210
|
+
AND b0.checkpoint_ns = checkpoints.checkpoint_ns
|
|
211
|
+
AND b0.channel = %s
|
|
212
|
+
AND b0.version = checkpoint -> 'channel_versions' ->> %s
|
|
213
|
+
AND b0.type <> 'empty') AS hb_0,
|
|
214
|
+
checkpoint -> 'channel_values' -> %s AS inline_0,
|
|
215
|
+
checkpoint -> 'channel_versions' ->> %s AS ver_1,
|
|
216
|
+
EXISTS (...) AS hb_1,
|
|
217
|
+
checkpoint -> 'channel_values' -> %s AS inline_1
|
|
218
|
+
FROM checkpoints
|
|
219
|
+
WHERE thread_id = %s AND checkpoint_ns = %s
|
|
220
|
+
AND (%s::text IS NULL OR checkpoint_id < %s)
|
|
221
|
+
ORDER BY checkpoint_id DESC
|
|
222
|
+
LIMIT %s
|
|
223
|
+
|
|
224
|
+
A stored value for a channel lives in one of two places, because `put`
|
|
225
|
+
splits them:
|
|
226
|
+
|
|
227
|
+
* **blob** — non-primitive values (and `_DeltaSnapshot`) are moved to
|
|
228
|
+
`checkpoint_blobs`. `hb_i` ("has blob") probes for one. The probe hits
|
|
229
|
+
that table's primary key `(thread_id, checkpoint_ns, channel, version)`
|
|
230
|
+
exactly, so it is an index lookup per row per channel.
|
|
231
|
+
* **inline** — `None`, `str`, `int`, `float` and `bool` stay in the
|
|
232
|
+
checkpoint's own `channel_values` and get no blob row at all. `inline_i`
|
|
233
|
+
returns that value.
|
|
234
|
+
|
|
235
|
+
Testing only for a key in `channel_values` (the previous approach) missed
|
|
236
|
+
blob-stored plain values, since `put` leaves an inline marker there for
|
|
237
|
+
`_DeltaSnapshot` but not for a plain value — which is what a thread
|
|
238
|
+
migrated from a pre-delta channel type leaves behind. Probing only the
|
|
239
|
+
blobs table would conversely miss inline primitives. Both are needed, and
|
|
240
|
+
the caller treats "either present" as the seed.
|
|
241
|
+
|
|
242
|
+
`hb_i` also disambiguates the two: for a `_DeltaSnapshot`, `inline_i` is the
|
|
243
|
+
literal `true` marker rather than the value, so a blob must win over an
|
|
244
|
+
inline reading whenever one exists. That ordering is what makes a genuine
|
|
245
|
+
inline `true` (a bool channel) distinguishable from the marker.
|
|
246
|
+
|
|
247
|
+
The `type <> 'empty'` predicate mirrors the check stage 2 already applies
|
|
248
|
+
when resolving the seed blob. `put` does not currently produce `empty` rows
|
|
249
|
+
on this path — `blob_versions` is filtered to keys present in
|
|
250
|
+
`channel_values`, so `_dump_blobs`' empty branch is unreachable from it —
|
|
251
|
+
but without the predicate the two stages could disagree: stage 1 would
|
|
252
|
+
terminate the walk on a row stage 2 then discards, yielding no seed *and* a
|
|
253
|
+
truncated write chain, which is the failure this function exists to avoid.
|
|
254
|
+
|
|
255
|
+
Channel names are passed as `%s` parameters (safe from SQL injection).
|
|
256
|
+
Only the column aliases `ver_i` / `hb_i` / `inline_i` and the subquery alias
|
|
257
|
+
`b{i}` are interpolated into the SQL string (i is bounded by len(channels)
|
|
258
|
+
and uses safe identifiers).
|
|
259
|
+
|
|
260
|
+
Caller must extend params with `[ch_0 x4, ch_1 x4, ..., thread_id, ns,
|
|
261
|
+
cursor, cursor, page_size]` when `paged=True` — four per channel: the
|
|
262
|
+
version lookup, the blob's channel, the version the blob must match, and the
|
|
263
|
+
inline lookup.
|
|
264
|
+
|
|
265
|
+
When `paged=False`, the WHERE has no cursor predicate and there's no
|
|
266
|
+
LIMIT/ORDER BY — kept as a non-public helper for tests/diagnostics.
|
|
267
|
+
"""
|
|
268
|
+
cols = []
|
|
269
|
+
for i in range(len(channels)):
|
|
270
|
+
cols.append(
|
|
271
|
+
f"checkpoint -> 'channel_versions' ->> %s AS ver_{i}, "
|
|
272
|
+
f"EXISTS (SELECT 1 FROM checkpoint_blobs b{i} "
|
|
273
|
+
f"WHERE b{i}.thread_id = checkpoints.thread_id "
|
|
274
|
+
f"AND b{i}.checkpoint_ns = checkpoints.checkpoint_ns "
|
|
275
|
+
f"AND b{i}.channel = %s "
|
|
276
|
+
f"AND b{i}.version = checkpoint -> 'channel_versions' ->> %s "
|
|
277
|
+
f"AND b{i}.type <> 'empty') AS hb_{i}, "
|
|
278
|
+
f"checkpoint -> 'channel_values' -> %s AS inline_{i}"
|
|
279
|
+
)
|
|
280
|
+
sql = (
|
|
281
|
+
"SELECT checkpoint_id, parent_checkpoint_id, "
|
|
282
|
+
+ ", ".join(cols)
|
|
283
|
+
+ " FROM checkpoints WHERE thread_id = %s AND checkpoint_ns = %s"
|
|
284
|
+
)
|
|
285
|
+
if paged:
|
|
286
|
+
sql += (
|
|
287
|
+
" AND (%s::text IS NULL OR checkpoint_id < %s)"
|
|
288
|
+
" ORDER BY checkpoint_id DESC LIMIT %s"
|
|
289
|
+
)
|
|
290
|
+
return sql
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _build_delta_stage2_sql(
|
|
294
|
+
*,
|
|
295
|
+
channels_with_chain: Sequence[str],
|
|
296
|
+
channels_with_seed: Sequence[str],
|
|
297
|
+
) -> str:
|
|
298
|
+
"""Build stage 2 SQL as a per-channel UNION ALL.
|
|
299
|
+
|
|
300
|
+
For each channel with a non-empty chain, emit one branch reading
|
|
301
|
+
`checkpoint_writes` for that specific channel + chain_cids. For each
|
|
302
|
+
channel with a seed_version, emit one branch reading `checkpoint_blobs`
|
|
303
|
+
for that channel + version. This avoids the over-fetch of the prior
|
|
304
|
+
`channel = ANY(channels) AND checkpoint_id = ANY(union)` form when
|
|
305
|
+
channels have different chain depths.
|
|
306
|
+
|
|
307
|
+
The caller must pass parameters in matching order:
|
|
308
|
+
|
|
309
|
+
for ch in channels_with_chain:
|
|
310
|
+
params += [thread_id, checkpoint_ns, ch, chain_cids[ch]]
|
|
311
|
+
for ch in channels_with_seed:
|
|
312
|
+
params += [thread_id, checkpoint_ns, ch, seed_version[ch]]
|
|
313
|
+
|
|
314
|
+
Returns an empty SQL string if both channel lists are empty (caller
|
|
315
|
+
must skip executing in that case).
|
|
316
|
+
"""
|
|
317
|
+
branches: list[str] = []
|
|
318
|
+
for _ in channels_with_chain:
|
|
319
|
+
branches.append(
|
|
320
|
+
"SELECT 'w'::text AS _kind, "
|
|
321
|
+
"checkpoint_id, channel, "
|
|
322
|
+
"type, blob, task_id, idx, NULL::text AS version "
|
|
323
|
+
"FROM checkpoint_writes "
|
|
324
|
+
"WHERE thread_id = %s AND checkpoint_ns = %s AND channel = %s "
|
|
325
|
+
"AND checkpoint_id = ANY(%s)"
|
|
326
|
+
)
|
|
327
|
+
for _ in channels_with_seed:
|
|
328
|
+
branches.append(
|
|
329
|
+
"SELECT 'b'::text AS _kind, NULL::text AS checkpoint_id, channel, "
|
|
330
|
+
"type, blob, NULL::text AS task_id, NULL::int AS idx, version "
|
|
331
|
+
"FROM checkpoint_blobs "
|
|
332
|
+
"WHERE thread_id = %s AND checkpoint_ns = %s AND channel = %s "
|
|
333
|
+
"AND version = %s"
|
|
334
|
+
)
|
|
335
|
+
return " UNION ALL ".join(branches)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
# Stage 1 rows are dynamic-shape dicts: {checkpoint_id, parent_checkpoint_id,
|
|
339
|
+
# ver_0, hs_0, ver_1, hs_1, ...}. Walking is parameterized by the channel
|
|
340
|
+
# list to map indices back to channel names — no static TypedDict here.
|
|
341
|
+
# `dict[str, Any]` is the practical signature.
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
class BasePostgresSaver(BaseCheckpointSaver[str]):
|
|
345
|
+
SELECT_SQL = SELECT_SQL
|
|
346
|
+
SELECT_PENDING_SENDS_SQL = SELECT_PENDING_SENDS_SQL
|
|
347
|
+
MIGRATIONS = MIGRATIONS
|
|
348
|
+
UPSERT_CHECKPOINT_BLOBS_SQL = UPSERT_CHECKPOINT_BLOBS_SQL
|
|
349
|
+
UPSERT_CHECKPOINTS_SQL = UPSERT_CHECKPOINTS_SQL
|
|
350
|
+
UPSERT_CHECKPOINT_WRITES_SQL = UPSERT_CHECKPOINT_WRITES_SQL
|
|
351
|
+
INSERT_CHECKPOINT_WRITES_SQL = INSERT_CHECKPOINT_WRITES_SQL
|
|
352
|
+
|
|
353
|
+
supports_pipeline: bool
|
|
354
|
+
|
|
355
|
+
def _migrate_pending_sends(
|
|
356
|
+
self,
|
|
357
|
+
pending_sends: list[tuple[bytes, bytes]],
|
|
358
|
+
checkpoint: dict[str, Any],
|
|
359
|
+
channel_values: list[tuple[bytes, bytes, bytes]],
|
|
360
|
+
) -> None:
|
|
361
|
+
if not pending_sends:
|
|
362
|
+
return
|
|
363
|
+
# add to values
|
|
364
|
+
enc, blob = self.serde.dumps_typed(
|
|
365
|
+
[self.serde.loads_typed((c.decode(), b)) for c, b in pending_sends],
|
|
366
|
+
)
|
|
367
|
+
channel_values.append((TASKS.encode(), enc.encode(), blob))
|
|
368
|
+
# add to versions
|
|
369
|
+
checkpoint["channel_versions"][TASKS] = (
|
|
370
|
+
max(checkpoint["channel_versions"].values())
|
|
371
|
+
if checkpoint["channel_versions"]
|
|
372
|
+
else self.get_next_version(None, None)
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
def _load_blobs(
|
|
376
|
+
self, blob_values: list[tuple[bytes, bytes, bytes]]
|
|
377
|
+
) -> dict[str, Any]:
|
|
378
|
+
if not blob_values:
|
|
379
|
+
return {}
|
|
380
|
+
return {
|
|
381
|
+
k.decode(): self.serde.loads_typed((t.decode(), v))
|
|
382
|
+
for k, t, v in blob_values
|
|
383
|
+
if t.decode() != "empty"
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
@staticmethod
|
|
387
|
+
def _ingest_stage1_page(
|
|
388
|
+
stage1_rows: Sequence[Mapping[str, Any]],
|
|
389
|
+
channels: Sequence[str],
|
|
390
|
+
parent_of: dict[str, str | None],
|
|
391
|
+
ver_by_i_by_cid: list[dict[str, str | None]],
|
|
392
|
+
hb_by_i_by_cid: list[dict[str, bool]],
|
|
393
|
+
inline_by_i_by_cid: list[dict[str, Any]],
|
|
394
|
+
) -> str | None:
|
|
395
|
+
"""Fold one stage-1 page into the running walk-state mappings.
|
|
396
|
+
|
|
397
|
+
Returns the oldest checkpoint_id seen on this page (smallest, since
|
|
398
|
+
pages come back DESC). Caller uses it as the cursor for the next
|
|
399
|
+
page (`AND checkpoint_id < cursor`).
|
|
400
|
+
"""
|
|
401
|
+
oldest: str | None = None
|
|
402
|
+
for r in stage1_rows:
|
|
403
|
+
cid = cast(str, r["checkpoint_id"])
|
|
404
|
+
parent_of[cid] = cast("str | None", r["parent_checkpoint_id"])
|
|
405
|
+
for i in range(len(channels)):
|
|
406
|
+
ver_by_i_by_cid[i][cid] = cast("str | None", r.get(f"ver_{i}"))
|
|
407
|
+
hb_by_i_by_cid[i][cid] = bool(r.get(f"hb_{i}"))
|
|
408
|
+
inline_by_i_by_cid[i][cid] = r.get(f"inline_{i}")
|
|
409
|
+
# Rows are DESC; the last one is the smallest cid in the page.
|
|
410
|
+
oldest = cid
|
|
411
|
+
return oldest
|
|
412
|
+
|
|
413
|
+
@staticmethod
|
|
414
|
+
def _try_advance_walks(
|
|
415
|
+
target_id: str,
|
|
416
|
+
channels: Sequence[str],
|
|
417
|
+
parent_of: Mapping[str, str | None],
|
|
418
|
+
ver_by_i_by_cid: Sequence[Mapping[str, str | None]],
|
|
419
|
+
hb_by_i_by_cid: Sequence[Mapping[str, bool]],
|
|
420
|
+
inline_by_i_by_cid: Sequence[Mapping[str, Any]],
|
|
421
|
+
chain_by_ch: dict[str, list[str]],
|
|
422
|
+
seed_ver_by_ch: dict[str, str | None],
|
|
423
|
+
seed_inline_by_ch: dict[str, Any],
|
|
424
|
+
walk_cursor_by_ch: dict[str, str | None],
|
|
425
|
+
seeded: set[str],
|
|
426
|
+
) -> None:
|
|
427
|
+
"""Advance each not-yet-seeded channel's walk as far as possible.
|
|
428
|
+
|
|
429
|
+
Uses the partial `parent_of` map accumulated so far. A walk stops
|
|
430
|
+
either because:
|
|
431
|
+
(a) it found a stored value for its channel — a blob or an inline
|
|
432
|
+
primitive (channel becomes seeded),
|
|
433
|
+
(b) it reached a real root (parent_of[cid] is None — fully
|
|
434
|
+
materialized at this point), or
|
|
435
|
+
(c) the next ancestor cid isn't in `parent_of` yet (waiting for
|
|
436
|
+
a later page; the cursor stays put).
|
|
437
|
+
|
|
438
|
+
Mutates `chain_by_ch`, `seed_ver_by_ch`, `seed_inline_by_ch`,
|
|
439
|
+
`walk_cursor_by_ch`, and `seeded` in place.
|
|
440
|
+
"""
|
|
441
|
+
for i, ch in enumerate(channels):
|
|
442
|
+
if ch in seeded:
|
|
443
|
+
continue
|
|
444
|
+
# First-time entry: cursor starts at the target's parent.
|
|
445
|
+
if ch not in walk_cursor_by_ch:
|
|
446
|
+
walk_cursor_by_ch[ch] = parent_of.get(target_id)
|
|
447
|
+
cur_cid = walk_cursor_by_ch[ch]
|
|
448
|
+
ch_chain = chain_by_ch[ch]
|
|
449
|
+
hb_i = hb_by_i_by_cid[i]
|
|
450
|
+
inline_i = inline_by_i_by_cid[i]
|
|
451
|
+
ver_i = ver_by_i_by_cid[i]
|
|
452
|
+
while cur_cid is not None:
|
|
453
|
+
if cur_cid not in parent_of:
|
|
454
|
+
# Need more pages to continue this walk.
|
|
455
|
+
break
|
|
456
|
+
ch_chain.append(cur_cid)
|
|
457
|
+
has_blob = hb_i.get(cur_cid, False)
|
|
458
|
+
inline = inline_i.get(cur_cid)
|
|
459
|
+
if has_blob or inline is not None:
|
|
460
|
+
# A blob wins: for a `_DeltaSnapshot` the inline reading is
|
|
461
|
+
# the `true` marker, not the value.
|
|
462
|
+
seed_ver_by_ch[ch] = ver_i.get(cur_cid)
|
|
463
|
+
if not has_blob:
|
|
464
|
+
seed_inline_by_ch[ch] = inline
|
|
465
|
+
seeded.add(ch)
|
|
466
|
+
cur_cid = None
|
|
467
|
+
break
|
|
468
|
+
cur_cid = parent_of[cur_cid]
|
|
469
|
+
walk_cursor_by_ch[ch] = cur_cid
|
|
470
|
+
|
|
471
|
+
def _build_delta_channels_writes_history(
|
|
472
|
+
self,
|
|
473
|
+
*,
|
|
474
|
+
channels: Sequence[str],
|
|
475
|
+
chain_by_ch: Mapping[str, list[str]],
|
|
476
|
+
seed_ver_by_ch: Mapping[str, str | None],
|
|
477
|
+
seed_inline_by_ch: Mapping[str, Any],
|
|
478
|
+
stage2_rows: Sequence[_DeltaStage2Row],
|
|
479
|
+
) -> dict[str, DeltaChannelHistory]:
|
|
480
|
+
"""Demux stage 2 rows per channel; produce per-channel histories.
|
|
481
|
+
|
|
482
|
+
stage2_rows carry `channel` on every row. We build per-channel
|
|
483
|
+
`writes_by_cid` and per-channel `seed_blob` dicts, then assemble
|
|
484
|
+
a `DeltaChannelHistory` per requested channel.
|
|
485
|
+
|
|
486
|
+
A seed comes from the blobs table when the walk found one there, and
|
|
487
|
+
otherwise from `seed_inline_by_ch` — `put` keeps `None`, `str`, `int`,
|
|
488
|
+
`float` and `bool` values in the checkpoint's own `channel_values` with
|
|
489
|
+
no blob row, so those never appear in `stage2_rows`.
|
|
490
|
+
|
|
491
|
+
The `seed` key is omitted when the walk reached root without finding a
|
|
492
|
+
stored value, or when the seed blob is sentinel "empty" — in both cases
|
|
493
|
+
the consumer treats absence as "start empty".
|
|
494
|
+
"""
|
|
495
|
+
# writes_by_ch_by_cid[channel][cid] = list of (type, blob, task_id, idx)
|
|
496
|
+
writes_by_ch_by_cid: dict[str, dict[str, list[tuple[str, bytes, str, int]]]] = {
|
|
497
|
+
ch: {} for ch in channels
|
|
498
|
+
}
|
|
499
|
+
# seed_blob_by_ver[(channel, version)] = (type, blob)
|
|
500
|
+
seed_blob_by_ver: dict[tuple[str, str], tuple[str, bytes]] = {}
|
|
501
|
+
|
|
502
|
+
for r in stage2_rows:
|
|
503
|
+
ch = cast(str, r["channel"])
|
|
504
|
+
kind = r["_kind"]
|
|
505
|
+
if kind == "w":
|
|
506
|
+
cid = cast(str, r["checkpoint_id"])
|
|
507
|
+
writes_by_ch_by_cid.setdefault(ch, {}).setdefault(cid, []).append(
|
|
508
|
+
cast(
|
|
509
|
+
"tuple[str, bytes, str, int]",
|
|
510
|
+
(r["type"], r["blob"], r["task_id"], r["idx"]),
|
|
511
|
+
)
|
|
512
|
+
)
|
|
513
|
+
else: # kind == "b"
|
|
514
|
+
ver = cast(str, r["version"])
|
|
515
|
+
seed_blob_by_ver[(ch, ver)] = cast(
|
|
516
|
+
"tuple[str, bytes]", (r["type"], r["blob"])
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
# Sort writes per (channel, cid) newest-first by (task_id, idx)
|
|
520
|
+
for cid_map in writes_by_ch_by_cid.values():
|
|
521
|
+
for ws in cid_map.values():
|
|
522
|
+
ws.sort(key=lambda w: (w[2], w[3]), reverse=True)
|
|
523
|
+
|
|
524
|
+
result: dict[str, DeltaChannelHistory] = {}
|
|
525
|
+
for ch in channels:
|
|
526
|
+
chain_cids = chain_by_ch.get(ch, [])
|
|
527
|
+
seed_version = seed_ver_by_ch.get(ch)
|
|
528
|
+
|
|
529
|
+
collected: list[PendingWrite] = []
|
|
530
|
+
cid_writes = writes_by_ch_by_cid.get(ch, {})
|
|
531
|
+
for cid in chain_cids:
|
|
532
|
+
for type_tag, write_blob, task_id, _idx in cid_writes.get(cid, []):
|
|
533
|
+
val = self.serde.loads_typed((type_tag, write_blob))
|
|
534
|
+
collected.append((task_id, ch, val))
|
|
535
|
+
collected.reverse()
|
|
536
|
+
|
|
537
|
+
entry: DeltaChannelHistory = {"writes": collected}
|
|
538
|
+
if seed_version is not None:
|
|
539
|
+
blob = seed_blob_by_ver.get((ch, seed_version))
|
|
540
|
+
if blob is not None and blob[0] != "empty":
|
|
541
|
+
entry["seed"] = self.serde.loads_typed(blob)
|
|
542
|
+
elif ch in seed_inline_by_ch:
|
|
543
|
+
# Inline primitive: stored in the checkpoint, not the blobs
|
|
544
|
+
# table, so stage 2 never returned a row for it.
|
|
545
|
+
entry["seed"] = seed_inline_by_ch[ch]
|
|
546
|
+
result[ch] = entry
|
|
547
|
+
return result
|
|
548
|
+
|
|
549
|
+
def _dump_blobs(
|
|
550
|
+
self,
|
|
551
|
+
thread_id: str,
|
|
552
|
+
checkpoint_ns: str,
|
|
553
|
+
values: dict[str, Any],
|
|
554
|
+
versions: ChannelVersions,
|
|
555
|
+
) -> list[tuple[str, str, str, str, str, bytes | None]]:
|
|
556
|
+
if not versions:
|
|
557
|
+
return []
|
|
558
|
+
|
|
559
|
+
return [
|
|
560
|
+
(
|
|
561
|
+
thread_id,
|
|
562
|
+
checkpoint_ns,
|
|
563
|
+
k,
|
|
564
|
+
cast(str, ver),
|
|
565
|
+
*(
|
|
566
|
+
self.serde.dumps_typed(values[k])
|
|
567
|
+
if k in values
|
|
568
|
+
else ("empty", None)
|
|
569
|
+
),
|
|
570
|
+
)
|
|
571
|
+
for k, ver in versions.items()
|
|
572
|
+
]
|
|
573
|
+
|
|
574
|
+
def _load_writes(
|
|
575
|
+
self, writes: list[tuple[bytes, bytes, bytes, bytes]]
|
|
576
|
+
) -> list[tuple[str, str, Any]]:
|
|
577
|
+
return (
|
|
578
|
+
[
|
|
579
|
+
(
|
|
580
|
+
tid.decode(),
|
|
581
|
+
channel.decode(),
|
|
582
|
+
self.serde.loads_typed((t.decode(), v)),
|
|
583
|
+
)
|
|
584
|
+
for tid, channel, t, v in writes
|
|
585
|
+
]
|
|
586
|
+
if writes
|
|
587
|
+
else []
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
def _dump_writes(
|
|
591
|
+
self,
|
|
592
|
+
thread_id: str,
|
|
593
|
+
checkpoint_ns: str,
|
|
594
|
+
checkpoint_id: str,
|
|
595
|
+
task_id: str,
|
|
596
|
+
task_path: str,
|
|
597
|
+
writes: Sequence[tuple[str, Any]],
|
|
598
|
+
) -> list[tuple[str, str, str, str, str, int, str, str, bytes]]:
|
|
599
|
+
return [
|
|
600
|
+
(
|
|
601
|
+
thread_id,
|
|
602
|
+
checkpoint_ns,
|
|
603
|
+
checkpoint_id,
|
|
604
|
+
task_id,
|
|
605
|
+
task_path,
|
|
606
|
+
WRITES_IDX_MAP.get(channel, idx),
|
|
607
|
+
channel,
|
|
608
|
+
*self.serde.dumps_typed(value),
|
|
609
|
+
)
|
|
610
|
+
for idx, (channel, value) in enumerate(writes)
|
|
611
|
+
]
|
|
612
|
+
|
|
613
|
+
def get_next_version(self, current: str | None, channel: None) -> str:
|
|
614
|
+
if current is None:
|
|
615
|
+
current_v = 0
|
|
616
|
+
elif isinstance(current, int):
|
|
617
|
+
current_v = current
|
|
618
|
+
else:
|
|
619
|
+
current_v = int(current.split(".")[0])
|
|
620
|
+
next_v = current_v + 1
|
|
621
|
+
next_h = random.random()
|
|
622
|
+
return f"{next_v:032}.{next_h:016}"
|
|
623
|
+
|
|
624
|
+
def _search_where(
|
|
625
|
+
self,
|
|
626
|
+
config: RunnableConfig | None,
|
|
627
|
+
filter: MetadataInput,
|
|
628
|
+
before: RunnableConfig | None = None,
|
|
629
|
+
) -> tuple[str, list[Any]]:
|
|
630
|
+
"""Return WHERE clause predicates for alist() given config, filter, before.
|
|
631
|
+
|
|
632
|
+
This method returns a tuple of a string and a tuple of values. The string
|
|
633
|
+
is the parametered WHERE clause predicate (including the WHERE keyword):
|
|
634
|
+
"WHERE column1 = $1 AND column2 IS $2". The list of values contains the
|
|
635
|
+
values for each of the corresponding parameters.
|
|
636
|
+
"""
|
|
637
|
+
wheres = []
|
|
638
|
+
param_values = []
|
|
639
|
+
|
|
640
|
+
# construct predicate for config filter
|
|
641
|
+
if config:
|
|
642
|
+
wheres.append("thread_id = %s ")
|
|
643
|
+
param_values.append(config["configurable"]["thread_id"])
|
|
644
|
+
checkpoint_ns = config["configurable"].get("checkpoint_ns")
|
|
645
|
+
if checkpoint_ns is not None:
|
|
646
|
+
wheres.append("checkpoint_ns = %s")
|
|
647
|
+
param_values.append(checkpoint_ns)
|
|
648
|
+
|
|
649
|
+
if checkpoint_id := get_checkpoint_id(config):
|
|
650
|
+
wheres.append("checkpoint_id = %s ")
|
|
651
|
+
param_values.append(checkpoint_id)
|
|
652
|
+
|
|
653
|
+
# construct predicate for metadata filter
|
|
654
|
+
if filter:
|
|
655
|
+
wheres.append("metadata @> %s ")
|
|
656
|
+
param_values.append(Jsonb(filter))
|
|
657
|
+
|
|
658
|
+
# construct predicate for `before`
|
|
659
|
+
if before is not None:
|
|
660
|
+
wheres.append("checkpoint_id < %s ")
|
|
661
|
+
param_values.append(get_checkpoint_id(before))
|
|
662
|
+
|
|
663
|
+
return (
|
|
664
|
+
"WHERE " + " AND ".join(wheres) if wheres else "",
|
|
665
|
+
param_values,
|
|
666
|
+
)
|
|
File without changes
|