memoflow-sdk 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memoflow_sdk-0.1.0/CHANGELOG.md +11 -0
- memoflow_sdk-0.1.0/MANIFEST.in +2 -0
- memoflow_sdk-0.1.0/PKG-INFO +315 -0
- memoflow_sdk-0.1.0/README.md +274 -0
- memoflow_sdk-0.1.0/memoflow/__init__.py +288 -0
- memoflow_sdk-0.1.0/memoflow/_codec.py +382 -0
- memoflow_sdk-0.1.0/memoflow/_execute.py +94 -0
- memoflow_sdk-0.1.0/memoflow/_http_effect.py +303 -0
- memoflow_sdk-0.1.0/memoflow/_registry.py +110 -0
- memoflow_sdk-0.1.0/memoflow/_step_key.py +91 -0
- memoflow_sdk-0.1.0/memoflow/_version.py +37 -0
- memoflow_sdk-0.1.0/memoflow/activity_context.py +316 -0
- memoflow_sdk-0.1.0/memoflow/ai.py +76 -0
- memoflow_sdk-0.1.0/memoflow/ai_budget.py +144 -0
- memoflow_sdk-0.1.0/memoflow/ai_budget_client.py +440 -0
- memoflow_sdk-0.1.0/memoflow/ai_invocation.py +301 -0
- memoflow_sdk-0.1.0/memoflow/client.py +1797 -0
- memoflow_sdk-0.1.0/memoflow/client_types.py +281 -0
- memoflow_sdk-0.1.0/memoflow/connection.py +185 -0
- memoflow_sdk-0.1.0/memoflow/context.py +466 -0
- memoflow_sdk-0.1.0/memoflow/errors.py +215 -0
- memoflow_sdk-0.1.0/memoflow/payload_codec.py +87 -0
- memoflow_sdk-0.1.0/memoflow/py.typed +0 -0
- memoflow_sdk-0.1.0/memoflow/retention_client.py +285 -0
- memoflow_sdk-0.1.0/memoflow/telemetry.py +94 -0
- memoflow_sdk-0.1.0/memoflow/transactional_step.py +177 -0
- memoflow_sdk-0.1.0/memoflow/v1/__init__.py +0 -0
- memoflow_sdk-0.1.0/memoflow/v1/control_resource_service_pb2.py +226 -0
- memoflow_sdk-0.1.0/memoflow/v1/control_resource_service_pb2.pyi +2173 -0
- memoflow_sdk-0.1.0/memoflow/v1/control_resource_service_pb2_grpc.py +1393 -0
- memoflow_sdk-0.1.0/memoflow/v1/control_resource_service_pb2_grpc.pyi +331 -0
- memoflow_sdk-0.1.0/memoflow/v1/types_pb2.py +115 -0
- memoflow_sdk-0.1.0/memoflow/v1/types_pb2.pyi +1359 -0
- memoflow_sdk-0.1.0/memoflow/v1/types_pb2_grpc.py +24 -0
- memoflow_sdk-0.1.0/memoflow/v1/types_pb2_grpc.pyi +20 -0
- memoflow_sdk-0.1.0/memoflow/v1/worker_service_pb2.py +127 -0
- memoflow_sdk-0.1.0/memoflow/v1/worker_service_pb2.pyi +1343 -0
- memoflow_sdk-0.1.0/memoflow/v1/worker_service_pb2_grpc.py +217 -0
- memoflow_sdk-0.1.0/memoflow/v1/worker_service_pb2_grpc.pyi +139 -0
- memoflow_sdk-0.1.0/memoflow/v1/worker_session_pb2.py +71 -0
- memoflow_sdk-0.1.0/memoflow/v1/worker_session_pb2.pyi +760 -0
- memoflow_sdk-0.1.0/memoflow/v1/worker_session_pb2_grpc.py +114 -0
- memoflow_sdk-0.1.0/memoflow/v1/worker_session_pb2_grpc.pyi +123 -0
- memoflow_sdk-0.1.0/memoflow/v1/workflow_service_pb2.py +242 -0
- memoflow_sdk-0.1.0/memoflow/v1/workflow_service_pb2.pyi +3460 -0
- memoflow_sdk-0.1.0/memoflow/v1/workflow_service_pb2_grpc.py +1641 -0
- memoflow_sdk-0.1.0/memoflow/v1/workflow_service_pb2_grpc.pyi +634 -0
- memoflow_sdk-0.1.0/memoflow/worker.py +2156 -0
- memoflow_sdk-0.1.0/memoflow_sdk.egg-info/PKG-INFO +315 -0
- memoflow_sdk-0.1.0/memoflow_sdk.egg-info/SOURCES.txt +68 -0
- memoflow_sdk-0.1.0/memoflow_sdk.egg-info/dependency_links.txt +1 -0
- memoflow_sdk-0.1.0/memoflow_sdk.egg-info/requires.txt +22 -0
- memoflow_sdk-0.1.0/memoflow_sdk.egg-info/top_level.txt +1 -0
- memoflow_sdk-0.1.0/pyproject.toml +92 -0
- memoflow_sdk-0.1.0/setup.cfg +4 -0
- memoflow_sdk-0.1.0/tests/test_ai_budget_client.py +131 -0
- memoflow_sdk-0.1.0/tests/test_ai_budget_context.py +69 -0
- memoflow_sdk-0.1.0/tests/test_ai_invocation_context.py +109 -0
- memoflow_sdk-0.1.0/tests/test_ai_usage.py +41 -0
- memoflow_sdk-0.1.0/tests/test_approval_client.py +89 -0
- memoflow_sdk-0.1.0/tests/test_connection.py +218 -0
- memoflow_sdk-0.1.0/tests/test_durable_output.py +138 -0
- memoflow_sdk-0.1.0/tests/test_evaluation_client.py +103 -0
- memoflow_sdk-0.1.0/tests/test_payload_codec_registry.py +99 -0
- memoflow_sdk-0.1.0/tests/test_payload_envelope.py +90 -0
- memoflow_sdk-0.1.0/tests/test_retention_client.py +109 -0
- memoflow_sdk-0.1.0/tests/test_sdk.py +1111 -0
- memoflow_sdk-0.1.0/tests/test_sdk_errors.py +1255 -0
- memoflow_sdk-0.1.0/tests/test_telemetry.py +191 -0
- memoflow_sdk-0.1.0/tests/test_transactional_step.py +209 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to the MemoFlow Python SDK are recorded here. The project follows
|
|
4
|
+
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
5
|
+
|
|
6
|
+
## 0.1.0 - Unreleased
|
|
7
|
+
|
|
8
|
+
- Add the typed remote client and explicit worker runtime for the v1 protocol.
|
|
9
|
+
- Add durable activity output, payload codecs, authentication, TLS, and OpenTelemetry integration.
|
|
10
|
+
- Add clean universal-wheel and source-distribution qualification.
|
|
11
|
+
- Support CPython 3.11 through 3.14 and require Python 3.11 or newer.
|
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: memoflow-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Durable workflow client and worker SDK for MemoFlow
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://memoflow.io
|
|
7
|
+
Project-URL: Documentation, https://github.com/evigasoft/memoflow/tree/main/docs
|
|
8
|
+
Project-URL: Repository, https://github.com/evigasoft/memoflow
|
|
9
|
+
Project-URL: Changelog, https://github.com/evigasoft/memoflow/blob/main/packages/memoflow-py/CHANGELOG.md
|
|
10
|
+
Project-URL: Issues, https://github.com/evigasoft/memoflow/issues
|
|
11
|
+
Keywords: durable-execution,workflows,agents,llm
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
Requires-Dist: grpcio<2,>=1.81.1
|
|
24
|
+
Requires-Dist: protobuf<7,>=6.33.5
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: build<2,>=1.3; extra == "dev"
|
|
27
|
+
Requires-Dist: grpcio-tools==1.81.1; extra == "dev"
|
|
28
|
+
Requires-Dist: mypy<3,>=1.19; extra == "dev"
|
|
29
|
+
Requires-Dist: mypy-protobuf==5.1.0; extra == "dev"
|
|
30
|
+
Requires-Dist: opentelemetry-sdk<2,>=1.28; extra == "dev"
|
|
31
|
+
Requires-Dist: packaging<27,>=24; extra == "dev"
|
|
32
|
+
Requires-Dist: pytest<10,>=9; extra == "dev"
|
|
33
|
+
Requires-Dist: ruff<1,>=0.14; extra == "dev"
|
|
34
|
+
Requires-Dist: types-protobuf==6.32.1.20260221; extra == "dev"
|
|
35
|
+
Provides-Extra: http
|
|
36
|
+
Requires-Dist: httpx>=0.27; extra == "http"
|
|
37
|
+
Provides-Extra: postgres
|
|
38
|
+
Requires-Dist: psycopg[binary]<4,>=3.2; extra == "postgres"
|
|
39
|
+
Provides-Extra: telemetry
|
|
40
|
+
Requires-Dist: opentelemetry-api<2,>=1.28; extra == "telemetry"
|
|
41
|
+
|
|
42
|
+
# memoflow-py
|
|
43
|
+
|
|
44
|
+
> **Experimental.** The Python SDK has a bounded authenticated TLS/mTLS connection layer and
|
|
45
|
+
> resumable stream reads, but still lacks full transaction, codec and embedded parity.
|
|
46
|
+
> See [the SDK strategy](../../docs/SDK-STRATEGY.md).
|
|
47
|
+
|
|
48
|
+
The MemoFlow Python SDK implements explicit-step workflows
|
|
49
|
+
with checkpoint/memoization, speaking the same wire protocol as the
|
|
50
|
+
TypeScript SDK: the ADR-0002 `WorkerSession` bidirectional stream plus the
|
|
51
|
+
`WorkflowService` client surface.
|
|
52
|
+
|
|
53
|
+
Named explicit steps use the same identity algorithm: step keys use
|
|
54
|
+
the same content-addressed algorithm (`sha256(kind + "\n" + name)[:16] +
|
|
55
|
+
":" + occurrence`). Cross-language resume is currently limited to a conservative JSON subset
|
|
56
|
+
without TypeScript payload codecs or implicit HTTP call-site identities. The
|
|
57
|
+
protocol conformance kit ([packages/protocol-kit](../protocol-kit/))
|
|
58
|
+
contains golden vectors and a live explicit-step handoff scenario.
|
|
59
|
+
|
|
60
|
+
Published releases install as `memoflow-sdk` while the Python import remains `memoflow`:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
python -m pip install memoflow-sdk
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Inside activities, `await activity_context().append_output(...)` writes durable run output with
|
|
67
|
+
per-stream ordering and a post-commit acknowledgement. Python uses the same deterministic
|
|
68
|
+
run/stream/offset idempotency key as TypeScript, so identical retry output is safe while changed
|
|
69
|
+
bytes at a committed offset fail loudly. This is separate from best-effort telemetry streaming.
|
|
70
|
+
|
|
71
|
+
## Install (development)
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
python -m pip install --require-hashes -r requirements-codegen.txt
|
|
75
|
+
python -m pip install -e ".[dev,http]"
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Protocol runtime modules and typed stubs are included in source and wheel distributions. Repository
|
|
79
|
+
contributors regenerate every language from the root with `pnpm proto:generate`; package users do
|
|
80
|
+
not need protoc. The first command installs the repository generator's exact transitive dependency
|
|
81
|
+
set; application users installing the published package do not need that requirements file.
|
|
82
|
+
|
|
83
|
+
## A workflow
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from memoflow import workflow, run, activity, signal, query, Worker, WorkflowClient
|
|
87
|
+
|
|
88
|
+
@activity
|
|
89
|
+
class OrderActivities:
|
|
90
|
+
async def chargePayment(self, order: dict) -> dict:
|
|
91
|
+
... # normal Python — no determinism constraints
|
|
92
|
+
|
|
93
|
+
@workflow(task_queue="orders")
|
|
94
|
+
class OrderWorkflow:
|
|
95
|
+
@run
|
|
96
|
+
async def execute(self, order: dict) -> dict:
|
|
97
|
+
validated = await self.ctx.activity("OrderActivities.validateOrder", [order],
|
|
98
|
+
retry={"max_attempts": 3})
|
|
99
|
+
payment = await self.ctx.activity("OrderActivities.chargePayment", [validated])
|
|
100
|
+
shipment = await self.ctx.wait_for_signal("shipmentConfirmed", timeout="7d")
|
|
101
|
+
return {"orderId": order["orderId"], "trackingId": shipment["trackingId"]}
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Run a worker and a client:
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
worker = Worker(
|
|
108
|
+
"localhost:7233",
|
|
109
|
+
"orders",
|
|
110
|
+
max_concurrent_workflows=10,
|
|
111
|
+
max_concurrent_activities=50,
|
|
112
|
+
ping_interval_ms=10_000,
|
|
113
|
+
)
|
|
114
|
+
await worker.run()
|
|
115
|
+
|
|
116
|
+
client = WorkflowClient("localhost:7233", namespace="default")
|
|
117
|
+
await client.start(workflow_id="o-1", workflow_type="OrderWorkflow",
|
|
118
|
+
task_queue="orders", input={...})
|
|
119
|
+
result = await client.get_result("o-1", timeout_ms=60_000)
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Namespaces are always explicit: the client never silently selects a tenant. Every unary method
|
|
123
|
+
accepts `call=RpcCallOptions(timeout_s=..., max_attempts=...)`. Retry is opt-in, bounded to five
|
|
124
|
+
attempts, and only follows a structured server error marked retryable; cancel and fork deliberately
|
|
125
|
+
reject retry configuration because they are not retry-safe.
|
|
126
|
+
|
|
127
|
+
For a protected remote server, share one immutable connection policy between clients and workers:
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from pathlib import Path
|
|
131
|
+
from memoflow import ConnectionConfig, TlsConfig, WorkflowClient, Worker
|
|
132
|
+
|
|
133
|
+
connection = ConnectionConfig(
|
|
134
|
+
api_key="...",
|
|
135
|
+
tls=TlsConfig(
|
|
136
|
+
root_certificates=Path("ca.pem").read_bytes(),
|
|
137
|
+
# Optional mTLS pair:
|
|
138
|
+
private_key=Path("client-key.pem").read_bytes(),
|
|
139
|
+
certificate_chain=Path("client-cert.pem").read_bytes(),
|
|
140
|
+
server_name="memoflow.example.com",
|
|
141
|
+
),
|
|
142
|
+
default_rpc_timeout_s=30,
|
|
143
|
+
connect_timeout_s=10,
|
|
144
|
+
)
|
|
145
|
+
client = WorkflowClient("memoflow.example.com:7233", namespace="production", connection=connection)
|
|
146
|
+
worker = Worker("memoflow.example.com:7233", "orders", connection=connection)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Plaintext defaults to loopback only; non-loopback development targets require an explicit
|
|
150
|
+
`allow_insecure=True`. Credentials are excluded from representations and unstructured transport
|
|
151
|
+
errors. Channels set bounded message sizes, keepalive and SDK user-agent identity. Use
|
|
152
|
+
`async with WorkflowClient(...)` or call `await client.close()`; worker session channels are owned
|
|
153
|
+
and closed by the worker.
|
|
154
|
+
|
|
155
|
+
`await worker.shutdown()` stops new task admission, lets active handlers finish
|
|
156
|
+
for up to 30 seconds, then cancels any remainder and closes the session.
|
|
157
|
+
|
|
158
|
+
Inside an activity, `activity_context()` exposes cooperative heartbeat/cancellation,
|
|
159
|
+
`record_llm_meta(...)`, bounded best-effort `stream(...)`, and acknowledged durable
|
|
160
|
+
`append_output(...)`. Raise `RateLimitError(message, retry_after)` to carry a provider Retry-After
|
|
161
|
+
hint into the server retry decision without changing attempt or non-retryable-error policy.
|
|
162
|
+
|
|
163
|
+
Postgres-tier workers may opt into atomic database activities with
|
|
164
|
+
`transactional_step=TransactionalStepConfig(connection_string=...)`. Inside the activity,
|
|
165
|
+
`await activity_context().transaction(fn)` commits `fn`'s writes and the MemoFlow COMPLETED
|
|
166
|
+
checkpoint in the same database transaction only after the session handshake proves the targets
|
|
167
|
+
match. A mismatch refuses before user code runs unless `degrade="at-least-once"` is explicit.
|
|
168
|
+
Install the lazy driver only for this feature with `pip install "memoflow-sdk[postgres]"`; embedded,
|
|
169
|
+
memory, missing-contract, and stale-contract servers fail loudly during worker startup.
|
|
170
|
+
|
|
171
|
+
AI provider calls can reserve server-authoritative capacity before any external effect:
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
from memoflow import AiBudgetEstimate, activity_context
|
|
175
|
+
|
|
176
|
+
ctx = activity_context()
|
|
177
|
+
budget = await ctx.reserve_ai_budget(AiBudgetEstimate(total_tokens=8_000, model_calls=1))
|
|
178
|
+
# Start provider work only after admission resolves, then persist its terminal invocation.
|
|
179
|
+
await budget.settle(invocation.invocation_id)
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
No policy snapshot crosses the SDK boundary, and settlement sends only reservation/invocation
|
|
183
|
+
identity; the server derives actual usage from its terminal ledger. `AiBudgetDeniedError` means the
|
|
184
|
+
provider effect must not start.
|
|
185
|
+
|
|
186
|
+
Administrative applications can use `AiBudgetClient` with typed `AiBudgetPolicy`,
|
|
187
|
+
`AiBudgetLimits`, and `AiBudgetLimit` values to create/version, read and list namespace, workflow
|
|
188
|
+
and run policies. Hierarchy overrides require an explicit reason before transport.
|
|
189
|
+
The same client exposes `list_reconciliations()` and `reconcile()` for namespace-authorized
|
|
190
|
+
operations. Every decision requires a stable UUID, timezone-aware timestamp, reason, and lowercase
|
|
191
|
+
SHA-256 evidence digest; release and terminal-invocation settlement are distinct decisions.
|
|
192
|
+
|
|
193
|
+
Install `memoflow[http]` to use the typed retention control-plane client:
|
|
194
|
+
|
|
195
|
+
```python
|
|
196
|
+
from datetime import datetime, timezone
|
|
197
|
+
from uuid import uuid4
|
|
198
|
+
from memoflow import RetentionClient
|
|
199
|
+
|
|
200
|
+
async with RetentionClient("https://control.example", api_key=api_key) as retention:
|
|
201
|
+
review = await retention.preview_reconciliation(
|
|
202
|
+
namespace="production", scope_kind="NAMESPACE", limit=1000
|
|
203
|
+
)
|
|
204
|
+
# Persist and review `review` before the explicit mutation call.
|
|
205
|
+
await retention.apply_reconciliation(
|
|
206
|
+
namespace="production", preview=review, audit_id=str(uuid4()),
|
|
207
|
+
occurred_at=datetime.now(timezone.utc), reason="approved DATA-004 review",
|
|
208
|
+
)
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
The same client manages versioned policies and legal holds and lists deletion jobs. It reports
|
|
212
|
+
`APPLIED` versus `DUPLICATE` and rejects empty, truncated, or cross-namespace review artifacts
|
|
213
|
+
before transport. Reuse the exact audit ID and timestamp when retrying an uncertain mutation.
|
|
214
|
+
|
|
215
|
+
Register schema upgrades with the process-wide `codecs` registry (or an isolated `CodecRegistry`).
|
|
216
|
+
Outputs use the same `{ "$mfv": version, "value": ... }` wrapper as TypeScript; unregistered steps
|
|
217
|
+
remain byte-identical, legacy version-zero values upgrade in order, and newer-than-code payloads fail
|
|
218
|
+
loudly. Install `memoflow[telemetry]` to connect the SDK to your application OpenTelemetry provider.
|
|
219
|
+
Without it, propagation and spans are strict no-ops. With it, starts inject W3C context and workers
|
|
220
|
+
create parented workflow/activity spans containing identifiers and attempt metadata, never payloads
|
|
221
|
+
or credentials. Exception telemetry is fail-closed: the original exception is re-raised to
|
|
222
|
+
application code, while spans receive only a synthetic `error` (or canonical numeric gRPC code)
|
|
223
|
+
and a static redacted status. Messages, causes, tracebacks, payloads, prompts, provider responses,
|
|
224
|
+
and credentials are never copied into the exported exception event.
|
|
225
|
+
|
|
226
|
+
The full port of the canonical order-workflow example lives in
|
|
227
|
+
[`examples/order_workflow/`](examples/order_workflow/).
|
|
228
|
+
|
|
229
|
+
## The context API
|
|
230
|
+
|
|
231
|
+
| method | semantics |
|
|
232
|
+
| --- | --- |
|
|
233
|
+
| `ctx.activity(type, args, retry=..., ...)` | checkpointed activity with at-least-once execution |
|
|
234
|
+
| `ctx.llm(type, args, ...)` | documentary sugar over `activity` for LLM calls |
|
|
235
|
+
| `ctx.sleep("5m")` | durable server-side timer |
|
|
236
|
+
| `ctx.wait_for_signal(name, timeout="7d")` | durable wait; timeout raises catchable `SignalTimeoutError` |
|
|
237
|
+
| `ctx.approval(name, timeout=...)` | human gate — sugar over `wait_for_signal("approval:{name}")` |
|
|
238
|
+
| `ctx.side_effect(fn)` | compute once, memoize forever |
|
|
239
|
+
|
|
240
|
+
Durable steps park on never-settling futures and register commands on the
|
|
241
|
+
context; the worker drains the event loop and suspends once with everything
|
|
242
|
+
the slice scheduled — so `asyncio.gather(...)` fan-out dispatches every
|
|
243
|
+
branch concurrently (the collect-then-suspend model, mirroring the TS SDK's
|
|
244
|
+
execute.ts).
|
|
245
|
+
|
|
246
|
+
Non-determinism is LOUD: a step-key mismatch on a memoization hit raises
|
|
247
|
+
`NonDeterminismError`, and the server parks the run as resumable
|
|
248
|
+
`BLOCKED_NONDETERMINISTIC` (surfaced to clients as `WorkflowBlockedError` —
|
|
249
|
+
see docs/DEPLOY-PLAYBOOK.md).
|
|
250
|
+
|
|
251
|
+
## Testing
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
python -m ruff check .
|
|
255
|
+
python -m mypy
|
|
256
|
+
python -m pytest tests
|
|
257
|
+
# The wire-level acceptance gate — the protocol kit driven by this SDK:
|
|
258
|
+
MEMOFLOW_KIT_DRIVER="python $(pwd)/kit_driver.py" \
|
|
259
|
+
pnpm --filter @memoflowio/memoflow-protocol-kit run test:kit
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
## Implicit HTTP steps — what is and isn't intercepted
|
|
263
|
+
|
|
264
|
+
A bare HTTP call inside a workflow body is a durable step — same checkpoint
|
|
265
|
+
schema, same memoization, same NonDeterminism detection as explicit steps
|
|
266
|
+
(using the same checkpoint semantics as explicit steps). Step identity is the CALL SITE (`module:lineno` +
|
|
267
|
+
occurrence), never the URL. Interception is scoped to workflow execution
|
|
268
|
+
slices via `contextvars`: activity bodies and plain scripts
|
|
269
|
+
are byte-identical pass-through.
|
|
270
|
+
|
|
271
|
+
| transport | intercepted? | durability of the record |
|
|
272
|
+
| --- | --- | --- |
|
|
273
|
+
| `httpx.AsyncClient` | **yes** | eager: the record is DurableAck'd before your code sees the response |
|
|
274
|
+
| `httpx.Client` (sync) | yes | batched with the slice's suspension (a sync client inside an async workflow also blocks the event loop — prefer AsyncClient) |
|
|
275
|
+
| `aiohttp`, `requests`, `urllib` | **no** — pass-through | not recorded; wrap in an activity if you need durability |
|
|
276
|
+
|
|
277
|
+
Mutating implicit calls (non-GET/HEAD/OPTIONS) carry an **auto-injected
|
|
278
|
+
`Idempotency-Key`** derived from the step's own identity (N2 — same
|
|
279
|
+
algorithm as the TS SDK; a user-supplied header always wins), so upstreams
|
|
280
|
+
can deduplicate the at-least-once window.
|
|
281
|
+
|
|
282
|
+
Activities can renew their heartbeat lease and cooperatively observe workflow
|
|
283
|
+
cancellation through the task-local context:
|
|
284
|
+
|
|
285
|
+
```python
|
|
286
|
+
from memoflow import activity_context
|
|
287
|
+
|
|
288
|
+
async def long_running_activity() -> None:
|
|
289
|
+
ctx = activity_context()
|
|
290
|
+
while more_work():
|
|
291
|
+
await do_one_chunk()
|
|
292
|
+
if ctx is not None:
|
|
293
|
+
await ctx.heartbeat()
|
|
294
|
+
ctx.throw_if_cancelled()
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
Opt-outs always exist: send the header `x-memoflow-raw: 1` on a request to
|
|
298
|
+
bypass recording (the header is stripped before the wire), or simply run the
|
|
299
|
+
code outside a workflow slice (interception activates only inside slices).
|
|
300
|
+
|
|
301
|
+
## Scope honesty
|
|
302
|
+
|
|
303
|
+
- No API-key, TLS or mTLS channel support.
|
|
304
|
+
- Snapshot queries are supported; query inputs are not. Durable cron schedules support upsert,
|
|
305
|
+
list, pause/resume through upsert, and idempotent deletion. Server-backed restart-from-step forks
|
|
306
|
+
are supported for terminal runs. Resumable output reads and debounce clients are not yet implemented.
|
|
307
|
+
- Worker workflow/activity concurrency is enforced per session; set either
|
|
308
|
+
maximum to `0` to disable that capability. Set `ping_interval_ms=0` to
|
|
309
|
+
disable periodic workflow-task lease renewal; explicit activity heartbeats
|
|
310
|
+
are still sent by `activity_context().heartbeat()`.
|
|
311
|
+
- No `ctx.transaction()` (the narrow same-Postgres atomic checkpoint path is TS-only today).
|
|
312
|
+
- No payload-codec registry: steps recorded with a registered TS
|
|
313
|
+
codec are not cross-readable from Python; unregistered steps (the
|
|
314
|
+
default) round-trip byte-compatibly.
|
|
315
|
+
- No debounced starts from Python yet.
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
# memoflow-py
|
|
2
|
+
|
|
3
|
+
> **Experimental.** The Python SDK has a bounded authenticated TLS/mTLS connection layer and
|
|
4
|
+
> resumable stream reads, but still lacks full transaction, codec and embedded parity.
|
|
5
|
+
> See [the SDK strategy](../../docs/SDK-STRATEGY.md).
|
|
6
|
+
|
|
7
|
+
The MemoFlow Python SDK implements explicit-step workflows
|
|
8
|
+
with checkpoint/memoization, speaking the same wire protocol as the
|
|
9
|
+
TypeScript SDK: the ADR-0002 `WorkerSession` bidirectional stream plus the
|
|
10
|
+
`WorkflowService` client surface.
|
|
11
|
+
|
|
12
|
+
Named explicit steps use the same identity algorithm: step keys use
|
|
13
|
+
the same content-addressed algorithm (`sha256(kind + "\n" + name)[:16] +
|
|
14
|
+
":" + occurrence`). Cross-language resume is currently limited to a conservative JSON subset
|
|
15
|
+
without TypeScript payload codecs or implicit HTTP call-site identities. The
|
|
16
|
+
protocol conformance kit ([packages/protocol-kit](../protocol-kit/))
|
|
17
|
+
contains golden vectors and a live explicit-step handoff scenario.
|
|
18
|
+
|
|
19
|
+
Published releases install as `memoflow-sdk` while the Python import remains `memoflow`:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
python -m pip install memoflow-sdk
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Inside activities, `await activity_context().append_output(...)` writes durable run output with
|
|
26
|
+
per-stream ordering and a post-commit acknowledgement. Python uses the same deterministic
|
|
27
|
+
run/stream/offset idempotency key as TypeScript, so identical retry output is safe while changed
|
|
28
|
+
bytes at a committed offset fail loudly. This is separate from best-effort telemetry streaming.
|
|
29
|
+
|
|
30
|
+
## Install (development)
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
python -m pip install --require-hashes -r requirements-codegen.txt
|
|
34
|
+
python -m pip install -e ".[dev,http]"
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Protocol runtime modules and typed stubs are included in source and wheel distributions. Repository
|
|
38
|
+
contributors regenerate every language from the root with `pnpm proto:generate`; package users do
|
|
39
|
+
not need protoc. The first command installs the repository generator's exact transitive dependency
|
|
40
|
+
set; application users installing the published package do not need that requirements file.
|
|
41
|
+
|
|
42
|
+
## A workflow
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from memoflow import workflow, run, activity, signal, query, Worker, WorkflowClient
|
|
46
|
+
|
|
47
|
+
@activity
|
|
48
|
+
class OrderActivities:
|
|
49
|
+
async def chargePayment(self, order: dict) -> dict:
|
|
50
|
+
... # normal Python — no determinism constraints
|
|
51
|
+
|
|
52
|
+
@workflow(task_queue="orders")
|
|
53
|
+
class OrderWorkflow:
|
|
54
|
+
@run
|
|
55
|
+
async def execute(self, order: dict) -> dict:
|
|
56
|
+
validated = await self.ctx.activity("OrderActivities.validateOrder", [order],
|
|
57
|
+
retry={"max_attempts": 3})
|
|
58
|
+
payment = await self.ctx.activity("OrderActivities.chargePayment", [validated])
|
|
59
|
+
shipment = await self.ctx.wait_for_signal("shipmentConfirmed", timeout="7d")
|
|
60
|
+
return {"orderId": order["orderId"], "trackingId": shipment["trackingId"]}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Run a worker and a client:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
worker = Worker(
|
|
67
|
+
"localhost:7233",
|
|
68
|
+
"orders",
|
|
69
|
+
max_concurrent_workflows=10,
|
|
70
|
+
max_concurrent_activities=50,
|
|
71
|
+
ping_interval_ms=10_000,
|
|
72
|
+
)
|
|
73
|
+
await worker.run()
|
|
74
|
+
|
|
75
|
+
client = WorkflowClient("localhost:7233", namespace="default")
|
|
76
|
+
await client.start(workflow_id="o-1", workflow_type="OrderWorkflow",
|
|
77
|
+
task_queue="orders", input={...})
|
|
78
|
+
result = await client.get_result("o-1", timeout_ms=60_000)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Namespaces are always explicit: the client never silently selects a tenant. Every unary method
|
|
82
|
+
accepts `call=RpcCallOptions(timeout_s=..., max_attempts=...)`. Retry is opt-in, bounded to five
|
|
83
|
+
attempts, and only follows a structured server error marked retryable; cancel and fork deliberately
|
|
84
|
+
reject retry configuration because they are not retry-safe.
|
|
85
|
+
|
|
86
|
+
For a protected remote server, share one immutable connection policy between clients and workers:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from pathlib import Path
|
|
90
|
+
from memoflow import ConnectionConfig, TlsConfig, WorkflowClient, Worker
|
|
91
|
+
|
|
92
|
+
connection = ConnectionConfig(
|
|
93
|
+
api_key="...",
|
|
94
|
+
tls=TlsConfig(
|
|
95
|
+
root_certificates=Path("ca.pem").read_bytes(),
|
|
96
|
+
# Optional mTLS pair:
|
|
97
|
+
private_key=Path("client-key.pem").read_bytes(),
|
|
98
|
+
certificate_chain=Path("client-cert.pem").read_bytes(),
|
|
99
|
+
server_name="memoflow.example.com",
|
|
100
|
+
),
|
|
101
|
+
default_rpc_timeout_s=30,
|
|
102
|
+
connect_timeout_s=10,
|
|
103
|
+
)
|
|
104
|
+
client = WorkflowClient("memoflow.example.com:7233", namespace="production", connection=connection)
|
|
105
|
+
worker = Worker("memoflow.example.com:7233", "orders", connection=connection)
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Plaintext defaults to loopback only; non-loopback development targets require an explicit
|
|
109
|
+
`allow_insecure=True`. Credentials are excluded from representations and unstructured transport
|
|
110
|
+
errors. Channels set bounded message sizes, keepalive and SDK user-agent identity. Use
|
|
111
|
+
`async with WorkflowClient(...)` or call `await client.close()`; worker session channels are owned
|
|
112
|
+
and closed by the worker.
|
|
113
|
+
|
|
114
|
+
`await worker.shutdown()` stops new task admission, lets active handlers finish
|
|
115
|
+
for up to 30 seconds, then cancels any remainder and closes the session.
|
|
116
|
+
|
|
117
|
+
Inside an activity, `activity_context()` exposes cooperative heartbeat/cancellation,
|
|
118
|
+
`record_llm_meta(...)`, bounded best-effort `stream(...)`, and acknowledged durable
|
|
119
|
+
`append_output(...)`. Raise `RateLimitError(message, retry_after)` to carry a provider Retry-After
|
|
120
|
+
hint into the server retry decision without changing attempt or non-retryable-error policy.
|
|
121
|
+
|
|
122
|
+
Postgres-tier workers may opt into atomic database activities with
|
|
123
|
+
`transactional_step=TransactionalStepConfig(connection_string=...)`. Inside the activity,
|
|
124
|
+
`await activity_context().transaction(fn)` commits `fn`'s writes and the MemoFlow COMPLETED
|
|
125
|
+
checkpoint in the same database transaction only after the session handshake proves the targets
|
|
126
|
+
match. A mismatch refuses before user code runs unless `degrade="at-least-once"` is explicit.
|
|
127
|
+
Install the lazy driver only for this feature with `pip install "memoflow-sdk[postgres]"`; embedded,
|
|
128
|
+
memory, missing-contract, and stale-contract servers fail loudly during worker startup.
|
|
129
|
+
|
|
130
|
+
AI provider calls can reserve server-authoritative capacity before any external effect:
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from memoflow import AiBudgetEstimate, activity_context
|
|
134
|
+
|
|
135
|
+
ctx = activity_context()
|
|
136
|
+
budget = await ctx.reserve_ai_budget(AiBudgetEstimate(total_tokens=8_000, model_calls=1))
|
|
137
|
+
# Start provider work only after admission resolves, then persist its terminal invocation.
|
|
138
|
+
await budget.settle(invocation.invocation_id)
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
No policy snapshot crosses the SDK boundary, and settlement sends only reservation/invocation
|
|
142
|
+
identity; the server derives actual usage from its terminal ledger. `AiBudgetDeniedError` means the
|
|
143
|
+
provider effect must not start.
|
|
144
|
+
|
|
145
|
+
Administrative applications can use `AiBudgetClient` with typed `AiBudgetPolicy`,
|
|
146
|
+
`AiBudgetLimits`, and `AiBudgetLimit` values to create/version, read and list namespace, workflow
|
|
147
|
+
and run policies. Hierarchy overrides require an explicit reason before transport.
|
|
148
|
+
The same client exposes `list_reconciliations()` and `reconcile()` for namespace-authorized
|
|
149
|
+
operations. Every decision requires a stable UUID, timezone-aware timestamp, reason, and lowercase
|
|
150
|
+
SHA-256 evidence digest; release and terminal-invocation settlement are distinct decisions.
|
|
151
|
+
|
|
152
|
+
Install `memoflow[http]` to use the typed retention control-plane client:
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
from datetime import datetime, timezone
|
|
156
|
+
from uuid import uuid4
|
|
157
|
+
from memoflow import RetentionClient
|
|
158
|
+
|
|
159
|
+
async with RetentionClient("https://control.example", api_key=api_key) as retention:
|
|
160
|
+
review = await retention.preview_reconciliation(
|
|
161
|
+
namespace="production", scope_kind="NAMESPACE", limit=1000
|
|
162
|
+
)
|
|
163
|
+
# Persist and review `review` before the explicit mutation call.
|
|
164
|
+
await retention.apply_reconciliation(
|
|
165
|
+
namespace="production", preview=review, audit_id=str(uuid4()),
|
|
166
|
+
occurred_at=datetime.now(timezone.utc), reason="approved DATA-004 review",
|
|
167
|
+
)
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
The same client manages versioned policies and legal holds and lists deletion jobs. It reports
|
|
171
|
+
`APPLIED` versus `DUPLICATE` and rejects empty, truncated, or cross-namespace review artifacts
|
|
172
|
+
before transport. Reuse the exact audit ID and timestamp when retrying an uncertain mutation.
|
|
173
|
+
|
|
174
|
+
Register schema upgrades with the process-wide `codecs` registry (or an isolated `CodecRegistry`).
|
|
175
|
+
Outputs use the same `{ "$mfv": version, "value": ... }` wrapper as TypeScript; unregistered steps
|
|
176
|
+
remain byte-identical, legacy version-zero values upgrade in order, and newer-than-code payloads fail
|
|
177
|
+
loudly. Install `memoflow[telemetry]` to connect the SDK to your application OpenTelemetry provider.
|
|
178
|
+
Without it, propagation and spans are strict no-ops. With it, starts inject W3C context and workers
|
|
179
|
+
create parented workflow/activity spans containing identifiers and attempt metadata, never payloads
|
|
180
|
+
or credentials. Exception telemetry is fail-closed: the original exception is re-raised to
|
|
181
|
+
application code, while spans receive only a synthetic `error` (or canonical numeric gRPC code)
|
|
182
|
+
and a static redacted status. Messages, causes, tracebacks, payloads, prompts, provider responses,
|
|
183
|
+
and credentials are never copied into the exported exception event.
|
|
184
|
+
|
|
185
|
+
The full port of the canonical order-workflow example lives in
|
|
186
|
+
[`examples/order_workflow/`](examples/order_workflow/).
|
|
187
|
+
|
|
188
|
+
## The context API
|
|
189
|
+
|
|
190
|
+
| method | semantics |
|
|
191
|
+
| --- | --- |
|
|
192
|
+
| `ctx.activity(type, args, retry=..., ...)` | checkpointed activity with at-least-once execution |
|
|
193
|
+
| `ctx.llm(type, args, ...)` | documentary sugar over `activity` for LLM calls |
|
|
194
|
+
| `ctx.sleep("5m")` | durable server-side timer |
|
|
195
|
+
| `ctx.wait_for_signal(name, timeout="7d")` | durable wait; timeout raises catchable `SignalTimeoutError` |
|
|
196
|
+
| `ctx.approval(name, timeout=...)` | human gate — sugar over `wait_for_signal("approval:{name}")` |
|
|
197
|
+
| `ctx.side_effect(fn)` | compute once, memoize forever |
|
|
198
|
+
|
|
199
|
+
Durable steps park on never-settling futures and register commands on the
|
|
200
|
+
context; the worker drains the event loop and suspends once with everything
|
|
201
|
+
the slice scheduled — so `asyncio.gather(...)` fan-out dispatches every
|
|
202
|
+
branch concurrently (the collect-then-suspend model, mirroring the TS SDK's
|
|
203
|
+
execute.ts).
|
|
204
|
+
|
|
205
|
+
Non-determinism is LOUD: a step-key mismatch on a memoization hit raises
|
|
206
|
+
`NonDeterminismError`, and the server parks the run as resumable
|
|
207
|
+
`BLOCKED_NONDETERMINISTIC` (surfaced to clients as `WorkflowBlockedError` —
|
|
208
|
+
see docs/DEPLOY-PLAYBOOK.md).
|
|
209
|
+
|
|
210
|
+
## Testing
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
python -m ruff check .
|
|
214
|
+
python -m mypy
|
|
215
|
+
python -m pytest tests
|
|
216
|
+
# The wire-level acceptance gate — the protocol kit driven by this SDK:
|
|
217
|
+
MEMOFLOW_KIT_DRIVER="python $(pwd)/kit_driver.py" \
|
|
218
|
+
pnpm --filter @memoflowio/memoflow-protocol-kit run test:kit
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
## Implicit HTTP steps — what is and isn't intercepted
|
|
222
|
+
|
|
223
|
+
A bare HTTP call inside a workflow body is a durable step — same checkpoint
|
|
224
|
+
schema, same memoization, same NonDeterminism detection as explicit steps
|
|
225
|
+
(using the same checkpoint semantics as explicit steps). Step identity is the CALL SITE (`module:lineno` +
|
|
226
|
+
occurrence), never the URL. Interception is scoped to workflow execution
|
|
227
|
+
slices via `contextvars`: activity bodies and plain scripts
|
|
228
|
+
are byte-identical pass-through.
|
|
229
|
+
|
|
230
|
+
| transport | intercepted? | durability of the record |
|
|
231
|
+
| --- | --- | --- |
|
|
232
|
+
| `httpx.AsyncClient` | **yes** | eager: the record is DurableAck'd before your code sees the response |
|
|
233
|
+
| `httpx.Client` (sync) | yes | batched with the slice's suspension (a sync client inside an async workflow also blocks the event loop — prefer AsyncClient) |
|
|
234
|
+
| `aiohttp`, `requests`, `urllib` | **no** — pass-through | not recorded; wrap in an activity if you need durability |
|
|
235
|
+
|
|
236
|
+
Mutating implicit calls (non-GET/HEAD/OPTIONS) carry an **auto-injected
|
|
237
|
+
`Idempotency-Key`** derived from the step's own identity (N2 — same
|
|
238
|
+
algorithm as the TS SDK; a user-supplied header always wins), so upstreams
|
|
239
|
+
can deduplicate the at-least-once window.
|
|
240
|
+
|
|
241
|
+
Activities can renew their heartbeat lease and cooperatively observe workflow
|
|
242
|
+
cancellation through the task-local context:
|
|
243
|
+
|
|
244
|
+
```python
|
|
245
|
+
from memoflow import activity_context
|
|
246
|
+
|
|
247
|
+
async def long_running_activity() -> None:
|
|
248
|
+
ctx = activity_context()
|
|
249
|
+
while more_work():
|
|
250
|
+
await do_one_chunk()
|
|
251
|
+
if ctx is not None:
|
|
252
|
+
await ctx.heartbeat()
|
|
253
|
+
ctx.throw_if_cancelled()
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
Opt-outs always exist: send the header `x-memoflow-raw: 1` on a request to
|
|
257
|
+
bypass recording (the header is stripped before the wire), or simply run the
|
|
258
|
+
code outside a workflow slice (interception activates only inside slices).
|
|
259
|
+
|
|
260
|
+
## Scope honesty
|
|
261
|
+
|
|
262
|
+
- No API-key, TLS or mTLS channel support.
|
|
263
|
+
- Snapshot queries are supported; query inputs are not. Durable cron schedules support upsert,
|
|
264
|
+
list, pause/resume through upsert, and idempotent deletion. Server-backed restart-from-step forks
|
|
265
|
+
are supported for terminal runs. Resumable output reads and debounce clients are not yet implemented.
|
|
266
|
+
- Worker workflow/activity concurrency is enforced per session; set either
|
|
267
|
+
maximum to `0` to disable that capability. Set `ping_interval_ms=0` to
|
|
268
|
+
disable periodic workflow-task lease renewal; explicit activity heartbeats
|
|
269
|
+
are still sent by `activity_context().heartbeat()`.
|
|
270
|
+
- No `ctx.transaction()` (the narrow same-Postgres atomic checkpoint path is TS-only today).
|
|
271
|
+
- No payload-codec registry: steps recorded with a registered TS
|
|
272
|
+
codec are not cross-readable from Python; unregistered steps (the
|
|
273
|
+
default) round-trip byte-compatibly.
|
|
274
|
+
- No debounced starts from Python yet.
|