memoflow-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memoflow/__init__.py +288 -0
- memoflow/_codec.py +382 -0
- memoflow/_execute.py +94 -0
- memoflow/_http_effect.py +303 -0
- memoflow/_registry.py +110 -0
- memoflow/_step_key.py +91 -0
- memoflow/_version.py +37 -0
- memoflow/activity_context.py +316 -0
- memoflow/ai.py +76 -0
- memoflow/ai_budget.py +144 -0
- memoflow/ai_budget_client.py +440 -0
- memoflow/ai_invocation.py +301 -0
- memoflow/client.py +1797 -0
- memoflow/client_types.py +281 -0
- memoflow/connection.py +185 -0
- memoflow/context.py +466 -0
- memoflow/errors.py +215 -0
- memoflow/payload_codec.py +87 -0
- memoflow/py.typed +0 -0
- memoflow/retention_client.py +285 -0
- memoflow/telemetry.py +94 -0
- memoflow/transactional_step.py +177 -0
- memoflow/v1/__init__.py +0 -0
- memoflow/v1/control_resource_service_pb2.py +226 -0
- memoflow/v1/control_resource_service_pb2.pyi +2173 -0
- memoflow/v1/control_resource_service_pb2_grpc.py +1393 -0
- memoflow/v1/control_resource_service_pb2_grpc.pyi +331 -0
- memoflow/v1/types_pb2.py +115 -0
- memoflow/v1/types_pb2.pyi +1359 -0
- memoflow/v1/types_pb2_grpc.py +24 -0
- memoflow/v1/types_pb2_grpc.pyi +20 -0
- memoflow/v1/worker_service_pb2.py +127 -0
- memoflow/v1/worker_service_pb2.pyi +1343 -0
- memoflow/v1/worker_service_pb2_grpc.py +217 -0
- memoflow/v1/worker_service_pb2_grpc.pyi +139 -0
- memoflow/v1/worker_session_pb2.py +71 -0
- memoflow/v1/worker_session_pb2.pyi +760 -0
- memoflow/v1/worker_session_pb2_grpc.py +114 -0
- memoflow/v1/worker_session_pb2_grpc.pyi +123 -0
- memoflow/v1/workflow_service_pb2.py +242 -0
- memoflow/v1/workflow_service_pb2.pyi +3460 -0
- memoflow/v1/workflow_service_pb2_grpc.py +1641 -0
- memoflow/v1/workflow_service_pb2_grpc.pyi +634 -0
- memoflow/worker.py +2156 -0
- memoflow_sdk-0.1.0.dist-info/METADATA +315 -0
- memoflow_sdk-0.1.0.dist-info/RECORD +48 -0
- memoflow_sdk-0.1.0.dist-info/WHEEL +5 -0
- memoflow_sdk-0.1.0.dist-info/top_level.txt +1 -0
memoflow/__init__.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""memoflow Python SDK — durable workflows with checkpoint/memoization.
|
|
2
|
+
|
|
3
|
+
Explicit steps (P12.2): decorators + async context + WorkerSession-stream
|
|
4
|
+
worker + WorkflowService client. Step identity is cross-SDK compatible with
|
|
5
|
+
the TypeScript SDK by construction (Design Law 1; enforced by the protocol
|
|
6
|
+
conformance kit at packages/protocol-kit).
|
|
7
|
+
"""
|
|
8
|
+
from ._codec import (
|
|
9
|
+
PAYLOAD_CODEC_VERSION,
|
|
10
|
+
PAYLOAD_CONTENT_TYPE,
|
|
11
|
+
PAYLOAD_ENVELOPE_VERSION,
|
|
12
|
+
PORTABLE_UNDEFINED,
|
|
13
|
+
PortableMap,
|
|
14
|
+
StepError,
|
|
15
|
+
decode_portable_payload,
|
|
16
|
+
deserialize_step_output,
|
|
17
|
+
encode_portable_payload,
|
|
18
|
+
is_portable_envelope,
|
|
19
|
+
parse_duration,
|
|
20
|
+
serialize_step_output,
|
|
21
|
+
)
|
|
22
|
+
from ._http_effect import idempotency_key_for
|
|
23
|
+
from ._registry import activity, query, run, signal, workflow
|
|
24
|
+
from ._step_key import NonDeterminismError, StepKeyTracker, step_key_hash
|
|
25
|
+
from ._version import SDK_VERSION, __version__
|
|
26
|
+
from .activity_context import (
|
|
27
|
+
ActivityContext,
|
|
28
|
+
ActivityExecutionIdentity,
|
|
29
|
+
CancelledError,
|
|
30
|
+
DurableOutputChunk,
|
|
31
|
+
ForkEffectBlockedError,
|
|
32
|
+
ForkExecutionOverride,
|
|
33
|
+
ForkExecutionPolicy,
|
|
34
|
+
LlmStepMeta,
|
|
35
|
+
RateLimitError,
|
|
36
|
+
activity_context,
|
|
37
|
+
)
|
|
38
|
+
from .ai import AiUsageAccumulator, AiUsageAggregationMode, NormalizedAiUsage
|
|
39
|
+
from .ai_budget import (
|
|
40
|
+
AiBudgetDeniedError,
|
|
41
|
+
AiBudgetEstimate,
|
|
42
|
+
AiBudgetPriceIdentity,
|
|
43
|
+
AiBudgetReservationHandle,
|
|
44
|
+
AiBudgetReservationsUnsupportedError,
|
|
45
|
+
)
|
|
46
|
+
from .ai_budget_client import (
|
|
47
|
+
AiBudgetAccount,
|
|
48
|
+
AiBudgetClient,
|
|
49
|
+
AiBudgetLimit,
|
|
50
|
+
AiBudgetLimits,
|
|
51
|
+
AiBudgetPolicy,
|
|
52
|
+
AiBudgetReconciliation,
|
|
53
|
+
AiBudgetReconciliationPage,
|
|
54
|
+
)
|
|
55
|
+
from .ai_invocation import (
|
|
56
|
+
AiFinishReason,
|
|
57
|
+
AiInvocationHandle,
|
|
58
|
+
AiInvocationLedgerUnsupportedError,
|
|
59
|
+
AiOperation,
|
|
60
|
+
AiPayloadDescriptor,
|
|
61
|
+
AiPriceSnapshot,
|
|
62
|
+
AiProviderExtension,
|
|
63
|
+
AiRetryAndRateLimit,
|
|
64
|
+
AiToolDefinitions,
|
|
65
|
+
AiToolEffectHandle,
|
|
66
|
+
CompleteAiInvocation,
|
|
67
|
+
CompleteAiToolEffect,
|
|
68
|
+
StartAiInvocation,
|
|
69
|
+
StartAiToolEffect,
|
|
70
|
+
)
|
|
71
|
+
from .client import (
|
|
72
|
+
ResultTimeoutError,
|
|
73
|
+
WorkflowBlockedError,
|
|
74
|
+
WorkflowCancelledError,
|
|
75
|
+
WorkflowClient,
|
|
76
|
+
WorkflowFailedError,
|
|
77
|
+
WorkflowTimedOutError,
|
|
78
|
+
)
|
|
79
|
+
from .client_types import (
|
|
80
|
+
ApprovalActor,
|
|
81
|
+
ApprovalMutationResult,
|
|
82
|
+
ApprovalPolicy,
|
|
83
|
+
ApprovalWorkItem,
|
|
84
|
+
DescribeResult,
|
|
85
|
+
EvaluationScore,
|
|
86
|
+
ForkResult,
|
|
87
|
+
ListApprovalsResult,
|
|
88
|
+
ListDebouncesResult,
|
|
89
|
+
ListQueuesResult,
|
|
90
|
+
ListWorkersResult,
|
|
91
|
+
ListWorkflowEvaluationsResult,
|
|
92
|
+
ListWorkflowsResult,
|
|
93
|
+
NamespaceConfigResult,
|
|
94
|
+
OutputChunk,
|
|
95
|
+
ScheduleInfo,
|
|
96
|
+
StartDebouncedResult,
|
|
97
|
+
StartResult,
|
|
98
|
+
UpsertScheduleResult,
|
|
99
|
+
WorkflowEvaluation,
|
|
100
|
+
WorkflowEvaluationMutationResult,
|
|
101
|
+
WorkflowHistoryResult,
|
|
102
|
+
WorkflowInfo,
|
|
103
|
+
)
|
|
104
|
+
from .connection import ConnectionConfig, RpcCallOptions, TlsConfig
|
|
105
|
+
from .context import SignalTimeoutError, WorkflowContext
|
|
106
|
+
from .errors import (
|
|
107
|
+
PAYLOAD_DESERIALIZATION_ERROR,
|
|
108
|
+
PAYLOAD_SERIALIZATION_ERROR,
|
|
109
|
+
PROTOCOL_VALIDATION_ERROR,
|
|
110
|
+
MemoflowSdkError,
|
|
111
|
+
PayloadDeserializationError,
|
|
112
|
+
PayloadSerializationError,
|
|
113
|
+
ProtocolValidationError,
|
|
114
|
+
RemoteError,
|
|
115
|
+
RemoteErrorDetail,
|
|
116
|
+
RpcTransportError,
|
|
117
|
+
parse_remote_error_detail,
|
|
118
|
+
remote_error_from_grpc,
|
|
119
|
+
)
|
|
120
|
+
from .payload_codec import (
|
|
121
|
+
ENVELOPE_KEY,
|
|
122
|
+
UNVERSIONED,
|
|
123
|
+
CodecRegistry,
|
|
124
|
+
PayloadCodec,
|
|
125
|
+
PayloadVersionSkewError,
|
|
126
|
+
codecs,
|
|
127
|
+
)
|
|
128
|
+
from .retention_client import (
|
|
129
|
+
RetentionClient,
|
|
130
|
+
RetentionDeletionJob,
|
|
131
|
+
RetentionExpiryReconciliation,
|
|
132
|
+
RetentionHttpError,
|
|
133
|
+
RetentionLegalHold,
|
|
134
|
+
RetentionMutationResult,
|
|
135
|
+
RetentionPolicy,
|
|
136
|
+
RetentionReconciliationCandidate,
|
|
137
|
+
RetentionReconciliationPreview,
|
|
138
|
+
RetentionReconciliationResult,
|
|
139
|
+
RetentionScope,
|
|
140
|
+
)
|
|
141
|
+
from .telemetry import inject_current_context, with_provider_span
|
|
142
|
+
from .transactional_step import (
|
|
143
|
+
AsyncPgQueryable,
|
|
144
|
+
StaleSqlContractError,
|
|
145
|
+
TransactionalStepConfig,
|
|
146
|
+
TransactionalStepUnsupportedError,
|
|
147
|
+
fingerprint_postgres_target,
|
|
148
|
+
)
|
|
149
|
+
from .worker import ReconnectConfig, SandboxAttestation, Worker, WorkerEvictedError
|
|
150
|
+
|
|
151
|
+
__all__ = [
|
|
152
|
+
"SDK_VERSION",
|
|
153
|
+
"AiBudgetDeniedError",
|
|
154
|
+
"AiBudgetEstimate",
|
|
155
|
+
"AiBudgetPriceIdentity",
|
|
156
|
+
"AiBudgetReservationHandle",
|
|
157
|
+
"AiBudgetReservationsUnsupportedError",
|
|
158
|
+
"AiBudgetClient",
|
|
159
|
+
"AiBudgetReconciliation",
|
|
160
|
+
"AiBudgetReconciliationPage",
|
|
161
|
+
"AiBudgetAccount",
|
|
162
|
+
"AiBudgetLimit",
|
|
163
|
+
"AiBudgetLimits",
|
|
164
|
+
"AiBudgetPolicy",
|
|
165
|
+
"RetentionClient",
|
|
166
|
+
"RetentionDeletionJob",
|
|
167
|
+
"RetentionExpiryReconciliation",
|
|
168
|
+
"RetentionHttpError",
|
|
169
|
+
"RetentionLegalHold",
|
|
170
|
+
"RetentionMutationResult",
|
|
171
|
+
"RetentionPolicy",
|
|
172
|
+
"RetentionReconciliationCandidate",
|
|
173
|
+
"RetentionReconciliationPreview",
|
|
174
|
+
"RetentionReconciliationResult",
|
|
175
|
+
"RetentionScope",
|
|
176
|
+
"__version__",
|
|
177
|
+
"workflow",
|
|
178
|
+
"run",
|
|
179
|
+
"activity",
|
|
180
|
+
"signal",
|
|
181
|
+
"query",
|
|
182
|
+
"Worker",
|
|
183
|
+
"ReconnectConfig",
|
|
184
|
+
"WorkerEvictedError",
|
|
185
|
+
"WorkflowClient",
|
|
186
|
+
"ApprovalActor",
|
|
187
|
+
"ApprovalMutationResult",
|
|
188
|
+
"ApprovalPolicy",
|
|
189
|
+
"ApprovalWorkItem",
|
|
190
|
+
"EvaluationScore",
|
|
191
|
+
"ListApprovalsResult",
|
|
192
|
+
"WorkflowEvaluation",
|
|
193
|
+
"ListWorkflowEvaluationsResult",
|
|
194
|
+
"WorkflowEvaluationMutationResult",
|
|
195
|
+
"WorkflowContext",
|
|
196
|
+
"ConnectionConfig",
|
|
197
|
+
"RpcCallOptions",
|
|
198
|
+
"TlsConfig",
|
|
199
|
+
"StartResult",
|
|
200
|
+
"StartDebouncedResult",
|
|
201
|
+
"UpsertScheduleResult",
|
|
202
|
+
"ScheduleInfo",
|
|
203
|
+
"ForkResult",
|
|
204
|
+
"WorkflowInfo",
|
|
205
|
+
"ListWorkflowsResult",
|
|
206
|
+
"NamespaceConfigResult",
|
|
207
|
+
"ListWorkersResult",
|
|
208
|
+
"ListQueuesResult",
|
|
209
|
+
"ListDebouncesResult",
|
|
210
|
+
"DescribeResult",
|
|
211
|
+
"WorkflowHistoryResult",
|
|
212
|
+
"OutputChunk",
|
|
213
|
+
"ActivityContext",
|
|
214
|
+
"ActivityExecutionIdentity",
|
|
215
|
+
"DurableOutputChunk",
|
|
216
|
+
"ForkEffectBlockedError",
|
|
217
|
+
"ForkExecutionPolicy",
|
|
218
|
+
"ForkExecutionOverride",
|
|
219
|
+
"activity_context",
|
|
220
|
+
"CancelledError",
|
|
221
|
+
"RateLimitError",
|
|
222
|
+
"LlmStepMeta",
|
|
223
|
+
"AiUsageAccumulator",
|
|
224
|
+
"AiUsageAggregationMode",
|
|
225
|
+
"NormalizedAiUsage",
|
|
226
|
+
"AiOperation",
|
|
227
|
+
"AiFinishReason",
|
|
228
|
+
"AiPayloadDescriptor",
|
|
229
|
+
"AiPriceSnapshot",
|
|
230
|
+
"AiRetryAndRateLimit",
|
|
231
|
+
"AiProviderExtension",
|
|
232
|
+
"AiToolDefinitions",
|
|
233
|
+
"StartAiInvocation",
|
|
234
|
+
"CompleteAiInvocation",
|
|
235
|
+
"StartAiToolEffect",
|
|
236
|
+
"CompleteAiToolEffect",
|
|
237
|
+
"AiInvocationHandle",
|
|
238
|
+
"AiToolEffectHandle",
|
|
239
|
+
"AiInvocationLedgerUnsupportedError",
|
|
240
|
+
"AsyncPgQueryable",
|
|
241
|
+
"TransactionalStepConfig",
|
|
242
|
+
"SandboxAttestation",
|
|
243
|
+
"TransactionalStepUnsupportedError",
|
|
244
|
+
"StaleSqlContractError",
|
|
245
|
+
"fingerprint_postgres_target",
|
|
246
|
+
"inject_current_context",
|
|
247
|
+
"with_provider_span",
|
|
248
|
+
"NonDeterminismError",
|
|
249
|
+
"SignalTimeoutError",
|
|
250
|
+
"StepError",
|
|
251
|
+
"StepKeyTracker",
|
|
252
|
+
"step_key_hash",
|
|
253
|
+
"parse_duration",
|
|
254
|
+
"serialize_step_output",
|
|
255
|
+
"deserialize_step_output",
|
|
256
|
+
"PayloadCodec",
|
|
257
|
+
"CodecRegistry",
|
|
258
|
+
"PayloadVersionSkewError",
|
|
259
|
+
"ENVELOPE_KEY",
|
|
260
|
+
"UNVERSIONED",
|
|
261
|
+
"codecs",
|
|
262
|
+
"idempotency_key_for",
|
|
263
|
+
"PortableMap",
|
|
264
|
+
"PORTABLE_UNDEFINED",
|
|
265
|
+
"encode_portable_payload",
|
|
266
|
+
"decode_portable_payload",
|
|
267
|
+
"is_portable_envelope",
|
|
268
|
+
"PAYLOAD_ENVELOPE_VERSION",
|
|
269
|
+
"PAYLOAD_CODEC_VERSION",
|
|
270
|
+
"PAYLOAD_CONTENT_TYPE",
|
|
271
|
+
"WorkflowFailedError",
|
|
272
|
+
"WorkflowCancelledError",
|
|
273
|
+
"WorkflowTimedOutError",
|
|
274
|
+
"WorkflowBlockedError",
|
|
275
|
+
"ResultTimeoutError",
|
|
276
|
+
"MemoflowSdkError",
|
|
277
|
+
"PayloadSerializationError",
|
|
278
|
+
"PayloadDeserializationError",
|
|
279
|
+
"ProtocolValidationError",
|
|
280
|
+
"PAYLOAD_SERIALIZATION_ERROR",
|
|
281
|
+
"PAYLOAD_DESERIALIZATION_ERROR",
|
|
282
|
+
"PROTOCOL_VALIDATION_ERROR",
|
|
283
|
+
"RemoteError",
|
|
284
|
+
"RemoteErrorDetail",
|
|
285
|
+
"RpcTransportError",
|
|
286
|
+
"parse_remote_error_detail",
|
|
287
|
+
"remote_error_from_grpc",
|
|
288
|
+
]
|
memoflow/_codec.py
ADDED
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
"""Portable payload envelope v1 and duration parsing.
|
|
2
|
+
|
|
3
|
+
Payloads remain opaque bytes to the server. The tagged canonical body and
|
|
4
|
+
deterministic protobuf envelope make normal TypeScript/Python values
|
|
5
|
+
byte-compatible without JSON's silent type coercions. The older TypeScript
|
|
6
|
+
``$mfv`` registered-codec wrapper may still be nested inside the portable
|
|
7
|
+
body; Python intentionally does not hydrate application-specific TS codecs.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import base64
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
import re
|
|
16
|
+
import struct
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from datetime import UTC, datetime
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from google.protobuf.message import DecodeError
|
|
22
|
+
|
|
23
|
+
from .errors import (
|
|
24
|
+
MemoflowSdkError,
|
|
25
|
+
PayloadDeserializationError,
|
|
26
|
+
PayloadSerializationError,
|
|
27
|
+
ProtocolValidationError,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
_DURATION_RE = re.compile(r"^(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)$")
|
|
31
|
+
_UNIT_MS: dict[str, int] = {"ms": 1, "s": 1000, "m": 60_000, "h": 3_600_000, "d": 86_400_000}
|
|
32
|
+
_MAX_PROTOBUF_DURATION_MS = 315_576_000_000_999
|
|
33
|
+
_DURATION_FORMAT_REASON = 'must be a duration string like "5s", "1m", or "100ms"'
|
|
34
|
+
_DURATION_VALUE_REASON = (
|
|
35
|
+
"must resolve to a positive whole number of milliseconds within the protobuf Duration range"
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
PAYLOAD_MAGIC = b"MFP1"
|
|
39
|
+
PAYLOAD_ENVELOPE_VERSION = 1
|
|
40
|
+
PAYLOAD_CODEC_NAME = "memoflow-portable-json"
|
|
41
|
+
PAYLOAD_CODEC_VERSION = 1
|
|
42
|
+
PAYLOAD_CONTENT_TYPE = "application/vnd.memoflow.portable-json.v1+json"
|
|
43
|
+
MAX_PAYLOAD_BODY_BYTES = 16 * 1024 * 1024
|
|
44
|
+
MAX_PAYLOAD_METADATA_BYTES = 16 * 1024
|
|
45
|
+
MAX_PAYLOAD_METADATA_ENTRIES = 32
|
|
46
|
+
MAX_LEGACY_JSON_DEPTH = 128
|
|
47
|
+
_META_KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$")
|
|
48
|
+
_ENVELOPE_PREFIX = b"\x0a\x04MFP1"
|
|
49
|
+
_SAFE_INTEGER_MAX = 9_007_199_254_740_991
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class PortableMap:
|
|
54
|
+
entries: tuple[tuple[Any, Any], ...]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class _PortableUndefined:
|
|
58
|
+
def __repr__(self) -> str:
|
|
59
|
+
return "PORTABLE_UNDEFINED"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
PORTABLE_UNDEFINED = _PortableUndefined()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _utf8_key(value: str) -> bytes:
|
|
66
|
+
return value.encode("utf-8")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _encode_node(value: Any, seen: set[int] | None = None) -> dict[str, Any]:
|
|
70
|
+
if seen is None:
|
|
71
|
+
seen = set()
|
|
72
|
+
if value is PORTABLE_UNDEFINED:
|
|
73
|
+
return {"t": "undefined"}
|
|
74
|
+
if value is None:
|
|
75
|
+
return {"t": "null"}
|
|
76
|
+
if isinstance(value, bool):
|
|
77
|
+
return {"t": "bool", "v": value}
|
|
78
|
+
if isinstance(value, str):
|
|
79
|
+
return {"t": "string", "v": value}
|
|
80
|
+
if isinstance(value, int):
|
|
81
|
+
if abs(value) > _SAFE_INTEGER_MAX:
|
|
82
|
+
raise PayloadSerializationError("serialize", "$", "integers must be within the portable safe-integer range")
|
|
83
|
+
return {"t": "int", "v": str(value)}
|
|
84
|
+
if isinstance(value, float):
|
|
85
|
+
if not math.isfinite(value):
|
|
86
|
+
raise PayloadSerializationError("serialize", "$", "non-finite numbers are not portable")
|
|
87
|
+
return {"t": "float64", "v": struct.pack(">d", value).hex()}
|
|
88
|
+
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
89
|
+
return {"t": "bytes", "v": base64.b64encode(bytes(value)).decode("ascii")}
|
|
90
|
+
if isinstance(value, datetime):
|
|
91
|
+
if value.tzinfo is None or value.utcoffset() is None:
|
|
92
|
+
raise PayloadSerializationError("serialize", "$", "dates must include a timezone")
|
|
93
|
+
utc = value.astimezone(UTC)
|
|
94
|
+
if utc.microsecond % 1000:
|
|
95
|
+
raise PayloadSerializationError("serialize", "$", "dates must have millisecond precision")
|
|
96
|
+
return {"t": "date", "v": utc.strftime("%Y-%m-%dT%H:%M:%S.") + f"{utc.microsecond // 1000:03d}Z"}
|
|
97
|
+
if callable(value):
|
|
98
|
+
raise PayloadSerializationError("serialize", "$", "callable values are not portable")
|
|
99
|
+
identity = id(value)
|
|
100
|
+
if identity in seen:
|
|
101
|
+
raise PayloadSerializationError("serialize", "$", "cyclic values are not portable")
|
|
102
|
+
seen.add(identity)
|
|
103
|
+
try:
|
|
104
|
+
if isinstance(value, (list, tuple)):
|
|
105
|
+
return {"t": "list", "v": [_encode_node(entry, seen) for entry in value]}
|
|
106
|
+
if isinstance(value, PortableMap):
|
|
107
|
+
return {"t": "map", "v": [[_encode_node(k, seen), _encode_node(v, seen)] for k, v in value.entries]}
|
|
108
|
+
if isinstance(value, dict):
|
|
109
|
+
if any(not isinstance(key, str) for key in value):
|
|
110
|
+
raise PayloadSerializationError("serialize", "$", "object keys must be strings; use PortableMap for other keys")
|
|
111
|
+
return {
|
|
112
|
+
"t": "object",
|
|
113
|
+
"v": [[key, _encode_node(value[key], seen)] for key in sorted(value, key=_utf8_key)],
|
|
114
|
+
}
|
|
115
|
+
raise PayloadSerializationError("serialize", "$", "class instances are not portable; use plain data")
|
|
116
|
+
finally:
|
|
117
|
+
seen.remove(identity)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _decode_node(node: Any) -> Any:
|
|
121
|
+
if not isinstance(node, dict) or not isinstance(node.get("t"), str):
|
|
122
|
+
raise ValueError("invalid portable node")
|
|
123
|
+
kind = node["t"]
|
|
124
|
+
if kind == "undefined" and set(node) == {"t"}:
|
|
125
|
+
return PORTABLE_UNDEFINED
|
|
126
|
+
if kind == "null" and set(node) == {"t"}:
|
|
127
|
+
return None
|
|
128
|
+
value = node.get("v")
|
|
129
|
+
if kind == "bool" and isinstance(value, bool):
|
|
130
|
+
return value
|
|
131
|
+
if kind == "string" and isinstance(value, str):
|
|
132
|
+
return value
|
|
133
|
+
if kind == "int" and isinstance(value, str) and re.fullmatch(r"-?(0|[1-9]\d*)", value):
|
|
134
|
+
parsed_int = int(value)
|
|
135
|
+
if abs(parsed_int) <= _SAFE_INTEGER_MAX and str(parsed_int) == value:
|
|
136
|
+
return parsed_int
|
|
137
|
+
if kind == "float64" and isinstance(value, str) and re.fullmatch(r"[0-9a-f]{16}", value):
|
|
138
|
+
parsed_float = struct.unpack(">d", bytes.fromhex(value))[0]
|
|
139
|
+
if math.isfinite(parsed_float):
|
|
140
|
+
return parsed_float
|
|
141
|
+
if kind == "bytes" and isinstance(value, str):
|
|
142
|
+
decoded = base64.b64decode(value, validate=True)
|
|
143
|
+
if base64.b64encode(decoded).decode("ascii") == value:
|
|
144
|
+
return decoded
|
|
145
|
+
if kind == "date" and isinstance(value, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z", value):
|
|
146
|
+
parsed_date = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=UTC)
|
|
147
|
+
if parsed_date.strftime("%Y-%m-%dT%H:%M:%S.") + f"{parsed_date.microsecond // 1000:03d}Z" == value:
|
|
148
|
+
return parsed_date
|
|
149
|
+
if kind == "list" and isinstance(value, list):
|
|
150
|
+
return [_decode_node(entry) for entry in value]
|
|
151
|
+
if kind == "object" and isinstance(value, list):
|
|
152
|
+
result: dict[str, Any] = {}
|
|
153
|
+
previous: bytes | None = None
|
|
154
|
+
for pair in value:
|
|
155
|
+
if not isinstance(pair, list) or len(pair) != 2 or not isinstance(pair[0], str):
|
|
156
|
+
raise ValueError("invalid object entry")
|
|
157
|
+
encoded_key = pair[0].encode("utf-8")
|
|
158
|
+
if previous is not None and previous >= encoded_key:
|
|
159
|
+
raise ValueError("object keys are not canonical")
|
|
160
|
+
previous = encoded_key
|
|
161
|
+
result[pair[0]] = _decode_node(pair[1])
|
|
162
|
+
return result
|
|
163
|
+
if kind == "map" and isinstance(value, list):
|
|
164
|
+
entries = []
|
|
165
|
+
for pair in value:
|
|
166
|
+
if not isinstance(pair, list) or len(pair) != 2:
|
|
167
|
+
raise ValueError("invalid map entry")
|
|
168
|
+
entries.append((_decode_node(pair[0]), _decode_node(pair[1])))
|
|
169
|
+
return PortableMap(tuple(entries))
|
|
170
|
+
raise ValueError(f"invalid portable node type '{kind}'")
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _validated_metadata(metadata: dict[str, str]) -> list[tuple[str, str]]:
|
|
174
|
+
entries = sorted(metadata.items(), key=lambda item: item[0].encode("utf-8"))
|
|
175
|
+
if len(entries) > MAX_PAYLOAD_METADATA_ENTRIES:
|
|
176
|
+
raise PayloadSerializationError("serialize", "metadata", "too many entries")
|
|
177
|
+
total = 0
|
|
178
|
+
for key, value in entries:
|
|
179
|
+
if not isinstance(key, str) or not _META_KEY_RE.fullmatch(key) or not isinstance(value, str):
|
|
180
|
+
raise PayloadSerializationError("serialize", "metadata", "contains an invalid key or value")
|
|
181
|
+
encoded_value = value.encode("utf-8")
|
|
182
|
+
if len(encoded_value) > 1024:
|
|
183
|
+
raise PayloadSerializationError("serialize", "metadata", "value exceeds 1024 UTF-8 bytes")
|
|
184
|
+
total += len(key.encode("utf-8")) + len(encoded_value)
|
|
185
|
+
if total > MAX_PAYLOAD_METADATA_BYTES:
|
|
186
|
+
raise PayloadSerializationError("serialize", "metadata", "exceeds the byte limit")
|
|
187
|
+
return entries
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def encode_portable_payload(value: Any, metadata: dict[str, str] | None = None) -> bytes:
|
|
191
|
+
from .v1 import types_pb2
|
|
192
|
+
|
|
193
|
+
try:
|
|
194
|
+
body = json.dumps(_encode_node(value), separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
195
|
+
if len(body) > MAX_PAYLOAD_BODY_BYTES:
|
|
196
|
+
raise PayloadSerializationError("serialize", "$", "payload exceeds 16 MiB")
|
|
197
|
+
envelope = types_pb2.PayloadEnvelope(
|
|
198
|
+
magic=PAYLOAD_MAGIC,
|
|
199
|
+
envelope_version=PAYLOAD_ENVELOPE_VERSION,
|
|
200
|
+
encoding="identity",
|
|
201
|
+
content_type=PAYLOAD_CONTENT_TYPE,
|
|
202
|
+
codec_name=PAYLOAD_CODEC_NAME,
|
|
203
|
+
codec_version=PAYLOAD_CODEC_VERSION,
|
|
204
|
+
body_length=len(body),
|
|
205
|
+
body_sha256=hashlib.sha256(body).digest(),
|
|
206
|
+
body=body,
|
|
207
|
+
)
|
|
208
|
+
for key, metadata_value in _validated_metadata(metadata or {}):
|
|
209
|
+
envelope.metadata.add(key=key, value=metadata_value)
|
|
210
|
+
return envelope.SerializeToString(deterministic=True)
|
|
211
|
+
except PayloadSerializationError:
|
|
212
|
+
raise
|
|
213
|
+
except (TypeError, ValueError, UnicodeError, OverflowError, RecursionError) as exc:
|
|
214
|
+
raise PayloadSerializationError("serialize", "$", "value could not be inspected safely") from exc
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def is_portable_envelope(value: bytes) -> bool:
|
|
218
|
+
return len(value) >= len(_ENVELOPE_PREFIX) and value.startswith(_ENVELOPE_PREFIX)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def decode_portable_payload(value: bytes) -> Any:
|
|
222
|
+
from .v1 import types_pb2
|
|
223
|
+
|
|
224
|
+
try:
|
|
225
|
+
if not is_portable_envelope(value):
|
|
226
|
+
raise ValueError("payload is missing the MFP1 envelope magic")
|
|
227
|
+
envelope = types_pb2.PayloadEnvelope.FromString(value)
|
|
228
|
+
if envelope.magic != PAYLOAD_MAGIC:
|
|
229
|
+
raise ValueError("invalid payload magic")
|
|
230
|
+
if envelope.envelope_version != PAYLOAD_ENVELOPE_VERSION:
|
|
231
|
+
raise ValueError(f"unsupported envelope version {envelope.envelope_version}")
|
|
232
|
+
if (
|
|
233
|
+
envelope.encoding != "identity"
|
|
234
|
+
or envelope.content_type != PAYLOAD_CONTENT_TYPE
|
|
235
|
+
or envelope.codec_name != PAYLOAD_CODEC_NAME
|
|
236
|
+
or envelope.codec_version != PAYLOAD_CODEC_VERSION
|
|
237
|
+
):
|
|
238
|
+
raise ValueError("unsupported payload codec contract")
|
|
239
|
+
if envelope.HasField("blob"):
|
|
240
|
+
raise ValueError("blob-backed envelope requires server-side resolution")
|
|
241
|
+
if envelope.HasField("encryption"):
|
|
242
|
+
raise ValueError("encrypted envelope requires a configured decryptor")
|
|
243
|
+
if envelope.body_length != len(envelope.body) or len(envelope.body) > MAX_PAYLOAD_BODY_BYTES:
|
|
244
|
+
raise ValueError("payload body length mismatch")
|
|
245
|
+
if len(envelope.body_sha256) != 32 or not hashlib.sha256(envelope.body).digest() == envelope.body_sha256:
|
|
246
|
+
raise ValueError("payload SHA-256 mismatch")
|
|
247
|
+
metadata_entries = [(entry.key, entry.value) for entry in envelope.metadata]
|
|
248
|
+
if metadata_entries != sorted(metadata_entries, key=lambda item: item[0].encode("utf-8")):
|
|
249
|
+
raise ValueError("payload metadata keys are not canonical")
|
|
250
|
+
metadata = dict(metadata_entries)
|
|
251
|
+
if len(metadata) != len(envelope.metadata):
|
|
252
|
+
raise ValueError("duplicate payload metadata key")
|
|
253
|
+
_validated_metadata(metadata)
|
|
254
|
+
parsed = json.loads(envelope.body.decode("utf-8"), parse_constant=_reject_non_json_constant)
|
|
255
|
+
decoded = _decode_node(parsed)
|
|
256
|
+
canonical = json.dumps(_encode_node(decoded), separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
257
|
+
if canonical != envelope.body:
|
|
258
|
+
raise ValueError("payload body is not canonical")
|
|
259
|
+
return decoded
|
|
260
|
+
except DecodeError as exc:
|
|
261
|
+
raise PayloadDeserializationError("deserialize", "$", "invalid payload envelope") from exc
|
|
262
|
+
except PayloadSerializationError as exc:
|
|
263
|
+
raise PayloadDeserializationError("deserialize", "$", str(exc)) from exc
|
|
264
|
+
except (TypeError, ValueError, UnicodeError, OverflowError, RecursionError) as exc:
|
|
265
|
+
raise PayloadDeserializationError("deserialize", "$", str(exc)) from exc
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def serialize(value: Any) -> bytes:
|
|
269
|
+
return encode_portable_payload(value)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def serialize_step_output(step_name: str, value: Any) -> bytes:
|
|
273
|
+
from .payload_codec import codecs
|
|
274
|
+
|
|
275
|
+
return serialize(codecs.envelope_for(step_name, value))
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _reject_non_json_constant(value: str) -> Any:
|
|
279
|
+
raise ValueError(f"non-JSON numeric constant {value}")
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _validate_legacy_json_limits(value: bytes) -> None:
|
|
283
|
+
if len(value) > MAX_PAYLOAD_BODY_BYTES:
|
|
284
|
+
raise ValueError("legacy JSON payload exceeds 16 MiB")
|
|
285
|
+
depth = 0
|
|
286
|
+
in_string = False
|
|
287
|
+
escaped = False
|
|
288
|
+
for byte in value:
|
|
289
|
+
if in_string:
|
|
290
|
+
if escaped:
|
|
291
|
+
escaped = False
|
|
292
|
+
elif byte == 0x5C:
|
|
293
|
+
escaped = True
|
|
294
|
+
elif byte == 0x22:
|
|
295
|
+
in_string = False
|
|
296
|
+
elif byte == 0x22:
|
|
297
|
+
in_string = True
|
|
298
|
+
elif byte in (0x5B, 0x7B):
|
|
299
|
+
depth += 1
|
|
300
|
+
if depth > MAX_LEGACY_JSON_DEPTH:
|
|
301
|
+
raise ValueError("legacy JSON payload exceeds the nesting limit")
|
|
302
|
+
elif byte in (0x5D, 0x7D):
|
|
303
|
+
depth = max(0, depth - 1)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def deserialize(buf: bytes | None) -> Any:
|
|
307
|
+
if buf is None:
|
|
308
|
+
return None
|
|
309
|
+
if not isinstance(buf, bytes):
|
|
310
|
+
raise PayloadDeserializationError("deserialize", "$", "value must be bytes")
|
|
311
|
+
if len(buf) == 0:
|
|
312
|
+
return None
|
|
313
|
+
if is_portable_envelope(buf):
|
|
314
|
+
return decode_portable_payload(buf)
|
|
315
|
+
# Explicit decode-only compatibility for pre-envelope payloads.
|
|
316
|
+
try:
|
|
317
|
+
_validate_legacy_json_limits(buf)
|
|
318
|
+
return json.loads(buf.decode("utf-8"), parse_constant=_reject_non_json_constant)
|
|
319
|
+
except (TypeError, ValueError, UnicodeError, RecursionError) as exc:
|
|
320
|
+
raise PayloadDeserializationError("deserialize", "$", "bytes are not valid JSON") from exc
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def deserialize_step_output(step_name: str, buf: bytes | None) -> Any:
|
|
324
|
+
from .payload_codec import codecs
|
|
325
|
+
|
|
326
|
+
return codecs.hydrate(step_name, deserialize(buf))
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def serialize_error(error: BaseException) -> tuple[str, str]:
|
|
330
|
+
"""(error_type, error_message) — error_type is the exception class name,
|
|
331
|
+
mirroring the TS SDK's constructor-name convention. NonDeterminismError's
|
|
332
|
+
name is load-bearing (P11.2 blocked transition)."""
|
|
333
|
+
if isinstance(error, MemoflowSdkError):
|
|
334
|
+
try:
|
|
335
|
+
error_type = error.code if isinstance(error.code, str) and error.code.strip() else "MemoflowSdkError"
|
|
336
|
+
message = str(error)
|
|
337
|
+
except Exception:
|
|
338
|
+
return "MemoflowSdkError", "MemoflowSdkError value could not be inspected"
|
|
339
|
+
return error_type, message if message.strip() else error_type
|
|
340
|
+
error_type = type(error).__name__ or "Exception"
|
|
341
|
+
try:
|
|
342
|
+
message = str(error)
|
|
343
|
+
except Exception:
|
|
344
|
+
message = "Exception value could not be stringified"
|
|
345
|
+
return error_type, message if message.strip() else error_type
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
class StepError(Exception):
|
|
349
|
+
"""A memoized FAILED step replayed on recovery. `error_type` carries the
|
|
350
|
+
original error's type name."""
|
|
351
|
+
|
|
352
|
+
def __init__(self, error_type: str, message: str):
|
|
353
|
+
self.error_type = error_type
|
|
354
|
+
super().__init__(message)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def parse_duration(duration: str) -> int:
|
|
358
|
+
"""Duration string ("5s", "1m", "100ms", "2h", "1d") -> milliseconds.
|
|
359
|
+
Same grammar as the TS SDK's parseDuration."""
|
|
360
|
+
if not isinstance(duration, str):
|
|
361
|
+
raise ProtocolValidationError("parse_duration", "duration", _DURATION_FORMAT_REASON)
|
|
362
|
+
m = _DURATION_RE.match(duration)
|
|
363
|
+
if not m:
|
|
364
|
+
raise ProtocolValidationError("parse_duration", "duration", _DURATION_FORMAT_REASON)
|
|
365
|
+
|
|
366
|
+
numeric_text = m.group(1)
|
|
367
|
+
if len(numeric_text) > 64:
|
|
368
|
+
raise ProtocolValidationError("parse_duration", "duration", _DURATION_VALUE_REASON)
|
|
369
|
+
whole, separator, fraction = numeric_text.partition(".")
|
|
370
|
+
if not separator:
|
|
371
|
+
fraction = ""
|
|
372
|
+
try:
|
|
373
|
+
numerator: int = int(f"{whole}{fraction}") * _UNIT_MS[m.group(2)]
|
|
374
|
+
denominator: int = 10 ** len(fraction)
|
|
375
|
+
milliseconds: int
|
|
376
|
+
remainder: int
|
|
377
|
+
milliseconds, remainder = divmod(numerator, denominator)
|
|
378
|
+
except (ValueError, OverflowError) as exc:
|
|
379
|
+
raise ProtocolValidationError("parse_duration", "duration", _DURATION_VALUE_REASON) from exc
|
|
380
|
+
if remainder != 0 or milliseconds <= 0 or milliseconds > _MAX_PROTOBUF_DURATION_MS:
|
|
381
|
+
raise ProtocolValidationError("parse_duration", "duration", _DURATION_VALUE_REASON)
|
|
382
|
+
return milliseconds
|