stepcraft 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stepcraft-0.1.0/.github/workflows/ci.yml +83 -0
- stepcraft-0.1.0/.github/workflows/publish.yml +48 -0
- stepcraft-0.1.0/.gitignore +31 -0
- stepcraft-0.1.0/.python-version +1 -0
- stepcraft-0.1.0/AUDIT.md +292 -0
- stepcraft-0.1.0/CLAUDE_CODE_HANDOFF.md +209 -0
- stepcraft-0.1.0/CODE_REVIEW_CHECKLIST.md +43 -0
- stepcraft-0.1.0/LICENSE +21 -0
- stepcraft-0.1.0/PKG-INFO +475 -0
- stepcraft-0.1.0/README.md +437 -0
- stepcraft-0.1.0/pipeline.py +3 -0
- stepcraft-0.1.0/pyproject.toml +68 -0
- stepcraft-0.1.0/src/stepcraft/__init__.py +58 -0
- stepcraft-0.1.0/src/stepcraft/async_runtime.py +50 -0
- stepcraft-0.1.0/src/stepcraft/branching.py +61 -0
- stepcraft-0.1.0/src/stepcraft/config.py +26 -0
- stepcraft-0.1.0/src/stepcraft/constants.py +16 -0
- stepcraft-0.1.0/src/stepcraft/context.py +66 -0
- stepcraft-0.1.0/src/stepcraft/decorators.py +160 -0
- stepcraft-0.1.0/src/stepcraft/exceptions.py +27 -0
- stepcraft-0.1.0/src/stepcraft/fan.py +153 -0
- stepcraft-0.1.0/src/stepcraft/graph.py +247 -0
- stepcraft-0.1.0/src/stepcraft/hooks.py +28 -0
- stepcraft-0.1.0/src/stepcraft/node.py +57 -0
- stepcraft-0.1.0/src/stepcraft/pipeline.py +217 -0
- stepcraft-0.1.0/src/stepcraft/pools.py +84 -0
- stepcraft-0.1.0/src/stepcraft/py.typed +0 -0
- stepcraft-0.1.0/src/stepcraft/result.py +24 -0
- stepcraft-0.1.0/src/stepcraft/runtime.py +52 -0
- stepcraft-0.1.0/src/stepcraft/spec.py +215 -0
- stepcraft-0.1.0/src/stepcraft/step.py +445 -0
- stepcraft-0.1.0/src/stepcraft/typecheck.py +47 -0
- stepcraft-0.1.0/src/stepcraft/utils.py +59 -0
- stepcraft-0.1.0/tests/conftest.py +33 -0
- stepcraft-0.1.0/tests/fixtures/graph_diamond.yaml +15 -0
- stepcraft-0.1.0/tests/fixtures/inc_double.json +6 -0
- stepcraft-0.1.0/tests/fixtures/inc_double.yaml +3 -0
- stepcraft-0.1.0/tests/spec_fixtures.py +13 -0
- stepcraft-0.1.0/tests/test_pipeline.py +2254 -0
- stepcraft-0.1.0/uv.lock +336 -0
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main, working_barnch]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ${{ matrix.os }}
|
|
12
|
+
timeout-minutes: 15
|
|
13
|
+
strategy:
|
|
14
|
+
fail-fast: false
|
|
15
|
+
matrix:
|
|
16
|
+
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
17
|
+
python-version: ["3.12", "3.13"]
|
|
18
|
+
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
|
|
22
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
23
|
+
uses: actions/setup-python@v5
|
|
24
|
+
with:
|
|
25
|
+
python-version: ${{ matrix.python-version }}
|
|
26
|
+
|
|
27
|
+
- name: Install uv
|
|
28
|
+
uses: astral-sh/setup-uv@v5
|
|
29
|
+
|
|
30
|
+
- name: Install dependencies
|
|
31
|
+
run: |
|
|
32
|
+
uv sync --all-extras --group dev
|
|
33
|
+
|
|
34
|
+
- name: Run tests
|
|
35
|
+
run: |
|
|
36
|
+
uv run pytest tests/ -v
|
|
37
|
+
|
|
38
|
+
test-free-threading:
|
|
39
|
+
# Free-threaded (no-GIL) CPython 3.14t. Skips gracefully if the runner
|
|
40
|
+
# cannot provision the build.
|
|
41
|
+
runs-on: ubuntu-latest
|
|
42
|
+
continue-on-error: true
|
|
43
|
+
timeout-minutes: 15
|
|
44
|
+
steps:
|
|
45
|
+
- uses: actions/checkout@v4
|
|
46
|
+
|
|
47
|
+
- name: Set up free-threaded Python 3.14
|
|
48
|
+
uses: actions/setup-python@v5
|
|
49
|
+
with:
|
|
50
|
+
python-version: "3.14t"
|
|
51
|
+
|
|
52
|
+
- name: Install uv
|
|
53
|
+
uses: astral-sh/setup-uv@v5
|
|
54
|
+
|
|
55
|
+
- name: Install dependencies
|
|
56
|
+
run: |
|
|
57
|
+
uv sync --group dev
|
|
58
|
+
|
|
59
|
+
- name: Run free-threading + parallel tests
|
|
60
|
+
run: |
|
|
61
|
+
uv run pytest tests/ -v -k "free_threading or parallel_auto or parallel or thread_parallel"
|
|
62
|
+
|
|
63
|
+
test-rsloop:
|
|
64
|
+
runs-on: ubuntu-latest
|
|
65
|
+
timeout-minutes: 10
|
|
66
|
+
steps:
|
|
67
|
+
- uses: actions/checkout@v4
|
|
68
|
+
|
|
69
|
+
- name: Set up Python 3.13
|
|
70
|
+
uses: actions/setup-python@v5
|
|
71
|
+
with:
|
|
72
|
+
python-version: "3.13"
|
|
73
|
+
|
|
74
|
+
- name: Install uv
|
|
75
|
+
uses: astral-sh/setup-uv@v5
|
|
76
|
+
|
|
77
|
+
- name: Install with rsloop extra
|
|
78
|
+
run: |
|
|
79
|
+
uv sync --extra rsloop --group dev
|
|
80
|
+
|
|
81
|
+
- name: Run rsloop tests
|
|
82
|
+
run: |
|
|
83
|
+
uv run pytest tests/ -v -m rsloop
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
name: Build distribution
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
|
|
17
|
+
- name: Install uv
|
|
18
|
+
uses: astral-sh/setup-uv@v5
|
|
19
|
+
|
|
20
|
+
- name: Build wheel and sdist
|
|
21
|
+
run: uv build --out-dir dist
|
|
22
|
+
|
|
23
|
+
- name: Upload artifacts
|
|
24
|
+
uses: actions/upload-artifact@v4
|
|
25
|
+
with:
|
|
26
|
+
name: dist
|
|
27
|
+
path: dist/*
|
|
28
|
+
|
|
29
|
+
publish:
|
|
30
|
+
name: Publish to PyPI
|
|
31
|
+
needs: [build]
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
environment: pypi
|
|
34
|
+
|
|
35
|
+
steps:
|
|
36
|
+
- name: Download artifacts
|
|
37
|
+
uses: actions/download-artifact@v4
|
|
38
|
+
with:
|
|
39
|
+
name: dist
|
|
40
|
+
path: dist
|
|
41
|
+
|
|
42
|
+
- name: Install uv
|
|
43
|
+
uses: astral-sh/setup-uv@v5
|
|
44
|
+
|
|
45
|
+
- name: Publish to PyPI
|
|
46
|
+
run: uv publish dist/*
|
|
47
|
+
env:
|
|
48
|
+
UV_PUBLISH_TOKEN: ${{ secrets.UV_PUBLISH_TOKEN }}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
*.egg
|
|
6
|
+
dist/
|
|
7
|
+
build/
|
|
8
|
+
.eggs/
|
|
9
|
+
|
|
10
|
+
# Virtual environments
|
|
11
|
+
.venv/
|
|
12
|
+
venv/
|
|
13
|
+
|
|
14
|
+
# IDE
|
|
15
|
+
.idea/
|
|
16
|
+
.vscode/
|
|
17
|
+
*.swp
|
|
18
|
+
*.swo
|
|
19
|
+
|
|
20
|
+
# Testing
|
|
21
|
+
.pytest_cache/
|
|
22
|
+
.coverage
|
|
23
|
+
htmlcov/
|
|
24
|
+
|
|
25
|
+
# Compiled extensions
|
|
26
|
+
*.so
|
|
27
|
+
|
|
28
|
+
# OS
|
|
29
|
+
.DS_Store
|
|
30
|
+
Thumbs.db
|
|
31
|
+
._*
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.13
|
stepcraft-0.1.0/AUDIT.md
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
# stepcraft Audit — Bugs & Incomplete Features
|
|
2
|
+
|
|
3
|
+
Audit date: 2026-06-24. Based on code review, runtime reproduction, and test coverage analysis.
|
|
4
|
+
Updated 2026-06-25 with findings from the multi-agent code-review pass on the
|
|
5
|
+
`working_barnch` feature diff (hooks, shared context, half-open breaker, beartype).
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Open findings — code-review pass (2026-06-25)
|
|
10
|
+
|
|
11
|
+
Verified findings against the feature diff. Ordered by severity. Each item lists
|
|
12
|
+
**what** is wrong and **how** to fix it.
|
|
13
|
+
|
|
14
|
+
### CR-1. Shared context never reaches pool workers (HIGH)
|
|
15
|
+
|
|
16
|
+
**What:** `Pipeline._activate_context` sets a `contextvars.ContextVar` on the
|
|
17
|
+
calling thread only. `pool.map` / `loop.run_in_executor` do **not** copy
|
|
18
|
+
contextvars into worker threads/processes, so any step that runs off the calling
|
|
19
|
+
thread sees `get_context() == {}`:
|
|
20
|
+
- `@piped(parallel='thread'|'process')` steps,
|
|
21
|
+
- `@piped(timeout=...)` steps (run via the thread pool),
|
|
22
|
+
- sync functions executed inside an async pipeline (`run_in_executor`),
|
|
23
|
+
- auto-mapped/batched work dispatched to executors.
|
|
24
|
+
|
|
25
|
+
README presents `get_context()` as the supported way for steps to read context,
|
|
26
|
+
so this is a silent correctness gap.
|
|
27
|
+
|
|
28
|
+
**Location:** `context.py:12`, `pipeline.py` (`_activate_context`, run loops),
|
|
29
|
+
`step.py` (`_execute_parallel`, `_execute_parallel_async`, `_execute_sync`
|
|
30
|
+
timeout branch, `_execute_auto_map*`, `_execute_batched*`).
|
|
31
|
+
|
|
32
|
+
**How to fix:** capture the active context at dispatch and run workers inside it.
|
|
33
|
+
- Threads: wrap submitted callables with `contextvars.copy_context().run(fn, ...)`,
|
|
34
|
+
or snapshot `get_context()` on the calling thread and re-set it inside the
|
|
35
|
+
worker wrapper.
|
|
36
|
+
- Processes: contextvars cannot cross the pickle boundary — pass the context dict
|
|
37
|
+
explicitly to the worker and re-set it there, or document that process pools do
|
|
38
|
+
not see `get_context()`.
|
|
39
|
+
- Add tests: `parallel='thread'`, `timeout=`, and async-with-sync-func steps all
|
|
40
|
+
reading `get_context()` inside a `Pipeline(context=...)`.
|
|
41
|
+
|
|
42
|
+
### CR-2. Half-open probe slot consumed before cancel check → breaker stuck (HIGH)
|
|
43
|
+
|
|
44
|
+
**What:** In `_preflight_check`, `_check_circuit_breaker()` runs first and
|
|
45
|
+
increments `_half_open_calls`; the cancel-event check runs *after*. If the call
|
|
46
|
+
is cancelled, the probe slot is consumed but neither `_record_failure` nor
|
|
47
|
+
`_reset_circuit_breaker` runs, leaving the breaker `HALF_OPEN` with the slot used
|
|
48
|
+
**forever** — every later call raises `CircuitBreakerError` even once healthy.
|
|
49
|
+
|
|
50
|
+
**Location:** `step.py` `_preflight_check` / `_check_circuit_breaker` (~L155, L387–391).
|
|
51
|
+
|
|
52
|
+
**How to fix:** check the cancel event **before** consuming a probe slot (reorder
|
|
53
|
+
`_preflight_check` so the cancel check precedes `_check_circuit_breaker`), or
|
|
54
|
+
release the slot on cancellation (decrement `_half_open_calls` / restore state in
|
|
55
|
+
a `finally`/except path when `CancelledError` is raised before the call runs).
|
|
56
|
+
Add a regression test: half-open + cancel set → breaker still recovers later.
|
|
57
|
+
|
|
58
|
+
### CR-3. Breaker reset happens before the schema check (MEDIUM)
|
|
59
|
+
|
|
60
|
+
**What:** `_finalize_success` calls `_reset_circuit_breaker()` and *then* validates
|
|
61
|
+
`schema`. A half-open probe whose function returns a wrong-typed value first
|
|
62
|
+
closes the breaker, then raises `TypeError`, which records a failure against a
|
|
63
|
+
now-`CLOSED` breaker (count from 0) instead of re-opening immediately.
|
|
64
|
+
|
|
65
|
+
**Location:** `step.py:169–179` (`_finalize_success`).
|
|
66
|
+
|
|
67
|
+
**How to fix:** validate the schema **before** calling `_reset_circuit_breaker()`
|
|
68
|
+
so a schema failure is treated like any other probe failure. Add a test:
|
|
69
|
+
`circuit_breaker` + `schema` where the recovered call returns the wrong type →
|
|
70
|
+
breaker re-opens rather than closing.
|
|
71
|
+
|
|
72
|
+
### CR-4. Graph never activates shared context (MEDIUM)
|
|
73
|
+
|
|
74
|
+
**What:** Context activation was added to `Pipeline` only. `Graph.run` /
|
|
75
|
+
`Graph.async_run` have no `_activate_context`, so a `@piped` node that reads
|
|
76
|
+
`get_context()` gets `{}` in a DAG even when run sequentially — inconsistent with
|
|
77
|
+
`Pipeline` and with the docs.
|
|
78
|
+
|
|
79
|
+
**Location:** `graph.py:120` (`run`) and `graph.py` `async_run`.
|
|
80
|
+
|
|
81
|
+
**How to fix:** give `Graph` an optional `context=` and wrap both run paths in the
|
|
82
|
+
same activation, ideally by extracting a shared `_activate_context` helper (e.g.
|
|
83
|
+
into `context.py`) reused by `Pipeline` and `Graph`. Note this composes with CR-1
|
|
84
|
+
(workers still need propagation). Add a Graph context test.
|
|
85
|
+
|
|
86
|
+
### CR-5. `on_step` missing on half the public run surface (MEDIUM)
|
|
87
|
+
|
|
88
|
+
**What:** `on_step` was threaded through `run` / `async_run` / `run_detailed` /
|
|
89
|
+
`async_run_detailed` only. `run_async`, `map`, `async_map`, `map_async` ignore it;
|
|
90
|
+
`pipeline.run_async(seed, on_step=hook)` raises `TypeError`. README says "Pass
|
|
91
|
+
`on_step` to any run method."
|
|
92
|
+
|
|
93
|
+
**Location:** `pipeline.py` (`run_async`, `map`, `async_map`, `map_async`).
|
|
94
|
+
|
|
95
|
+
**How to fix:** forward `on_step` from `map`/`async_map` into per-item `run`/
|
|
96
|
+
`async_run`, and from `run_async`/`map_async` into the underlying coroutine; OR
|
|
97
|
+
narrow the README to list exactly the four methods that accept it. Prefer
|
|
98
|
+
forwarding for consistency. Add coverage for at least `map(on_step=...)`.
|
|
99
|
+
|
|
100
|
+
### CR-6. Spec with both `graph:` and `steps:` builds a silent linear pipeline (LOW)
|
|
101
|
+
|
|
102
|
+
**What:** `build_pipeline_from_spec` only rejects a graph spec when `steps` is
|
|
103
|
+
absent. A spec containing **both** keys silently builds a linear `Pipeline` and
|
|
104
|
+
ignores the `graph:` block (wrong execution order / fan-in inputs, no error).
|
|
105
|
+
|
|
106
|
+
**Location:** `spec.py:144` (`build_pipeline_from_spec`).
|
|
107
|
+
|
|
108
|
+
**How to fix:** raise `ValueError` when both `graph` and `steps` are present
|
|
109
|
+
(ambiguous spec). Mirror the check in `build_graph_from_spec`. Add a test for the
|
|
110
|
+
both-keys case.
|
|
111
|
+
|
|
112
|
+
### CR-7. `_half_open_calls` increment is not atomic across threads (LOW)
|
|
113
|
+
|
|
114
|
+
**What:** `_check_circuit_breaker` does a non-atomic read-check-increment on
|
|
115
|
+
`_half_open_calls`. When one `PipeStep` is invoked concurrently (parallel graph
|
|
116
|
+
levels, `parallel='thread'` fan-out), multiple threads can each read `0` and all
|
|
117
|
+
proceed, exceeding `half_open_max_calls`. Pre-existing class of issue — breaker
|
|
118
|
+
state was never lock-guarded — but the new probe quota relies on it.
|
|
119
|
+
|
|
120
|
+
**Location:** `step.py:384–391` (`_check_circuit_breaker`, `_record_failure`).
|
|
121
|
+
|
|
122
|
+
**How to fix:** guard breaker state transitions with a `threading.Lock` on the
|
|
123
|
+
step (note: a `frozen`/`object.__setattr__` dataclass needs a non-init lock
|
|
124
|
+
field). Lower priority unless shared concurrent steps are a supported pattern;
|
|
125
|
+
otherwise document that breaker state is not thread-safe.
|
|
126
|
+
|
|
127
|
+
### Intentional behavior changes (not bugs — confirm and keep)
|
|
128
|
+
|
|
129
|
+
- **Single-probe half-open semantics** (`step.py` `_check_circuit_breaker`): the
|
|
130
|
+
breaker now allows only `half_open_max_calls` (default 1) probes and re-opens on
|
|
131
|
+
the first probe failure. This is the requested Phase 2.4 feature, **not** a
|
|
132
|
+
regression — but CR-2/CR-3/CR-7 above are real defects within it.
|
|
133
|
+
- **beartype runtime type-checking** (`__init__.py`): every function/method is
|
|
134
|
+
decorated via the import hook; calls that violate annotations now raise
|
|
135
|
+
`BeartypeCallHintParamViolation`. Intended. Caveat to document: it is always-on
|
|
136
|
+
with no opt-out env var, and the numeric tower is enabled (int accepted for
|
|
137
|
+
float). Consider an opt-out hook (e.g. `PIPECRAFT_NO_BEARTYPE`) if consumers
|
|
138
|
+
need to disable runtime checks in production.
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## Confirmed bugs
|
|
143
|
+
|
|
144
|
+
### 1. Async + `batch_size` returns unawaited coroutines
|
|
145
|
+
|
|
146
|
+
When an **async** function is used with `batch_size > 1`, `_invoke_function_async` calls the sync `_execute_batched`, which invokes `self.func(batch)` without awaiting.
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
@piped(batch_size=2)
|
|
150
|
+
async def process_batch(batch):
|
|
151
|
+
return [x * 2 for x in batch]
|
|
152
|
+
|
|
153
|
+
await process_batch.async_run([1, 2, 3, 4])
|
|
154
|
+
# -> [<coroutine ...>, <coroutine ...>] # RuntimeWarning: never awaited
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
**Location:** `src/stepcraft/step.py` — `_invoke_function_async` → `_execute_batched`
|
|
158
|
+
|
|
159
|
+
**Impact:** `async_run` silently returns coroutine objects instead of results.
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
### 2. Async + `parallel='thread'` has the same problem
|
|
164
|
+
|
|
165
|
+
The parallel async path uses `run_in_executor(pool, call, item)`, which always invokes the **sync** wrapper. Async `@piped` functions therefore produce unawaited coroutines.
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
@piped(parallel='thread')
|
|
169
|
+
async def f(x):
|
|
170
|
+
return x * 2
|
|
171
|
+
|
|
172
|
+
await f.async_run([1, 2, 3])
|
|
173
|
+
# -> [<coroutine ...>, <coroutine ...>, <coroutine ...>]
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
**Location:** `src/stepcraft/step.py` — `_execute_parallel_async`
|
|
177
|
+
|
|
178
|
+
**Impact:** `parallel` + async steps are broken on the async execution path. Sync `run()` is fine (thread pool runs sync callables).
|
|
179
|
+
|
|
180
|
+
---
|
|
181
|
+
|
|
182
|
+
### 3. `_async_run_branch_value` may not await `run()` coroutines
|
|
183
|
+
|
|
184
|
+
If a branch has `.run` but not `.async_run`, and `run()` returns a coroutine, it is returned without awaiting.
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
# src/stepcraft/utils.py
|
|
188
|
+
async def _async_run_branch_value(branch, value):
|
|
189
|
+
...
|
|
190
|
+
if hasattr(branch, 'run'):
|
|
191
|
+
return branch.run(value) # coroutine not awaited
|
|
192
|
+
result = branch(value)
|
|
193
|
+
if asyncio.iscoroutine(result):
|
|
194
|
+
return await result
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
**Impact:** Uncommon, but custom steps with only sync `run()` that return coroutines will misbehave in `ConditionalStep` / `SwitchStep` async paths.
|
|
198
|
+
|
|
199
|
+
---
|
|
200
|
+
|
|
201
|
+
## Known deferred behavior (documented, still open)
|
|
202
|
+
|
|
203
|
+
### 4. Whole collections are not auto-mapped
|
|
204
|
+
|
|
205
|
+
Passing a list into a normal step sends the **entire list** as one argument. Parallel/batch only kick in when explicitly configured.
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
@piped
|
|
209
|
+
def step(x): ...
|
|
210
|
+
|
|
211
|
+
pipeline.run([1, 2, 3]) # step receives [1,2,3], not three scalars
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
**Status:** Intentionally deferred — see `CODE_REVIEW_CHECKLIST.md` line 10. Surprising if users expect implicit map behavior.
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
## Incomplete features
|
|
219
|
+
|
|
220
|
+
| Feature | Status |
|
|
221
|
+
|--------|--------|
|
|
222
|
+
| `Pipeline.from_spec("yaml")` | ✅ Implemented (`spec.py`, `stepcraft[spec]`) |
|
|
223
|
+
| `Graph.from_spec("yaml")` | ✅ Implemented — `graph:` block with `nodes`/`edges` |
|
|
224
|
+
| `FanOutStep.async_run` + `parallel=` | ✅ Honors `parallel` on the async path |
|
|
225
|
+
| `MapReduceStep.async_run` | ✅ Async mappers mapped concurrently per batch |
|
|
226
|
+
| `jit` / `vectorize` on `@piped` | ✅ Warns when numba/numpy missing |
|
|
227
|
+
| rsloop integration | ✅ Covered by the `test-rsloop` CI job |
|
|
228
|
+
| Step hooks (`on_step`) | ✅ `Pipeline` + `Graph` run paths, see `hooks.py` |
|
|
229
|
+
| Pipeline shared context | ✅ `Pipeline(steps, context=...)` + `get_context()` |
|
|
230
|
+
| Configurable pools | ✅ `configure_pools(thread_workers=, process_workers=)` |
|
|
231
|
+
| Runtime type-checking | ✅ beartype decorates every function via import hook |
|
|
232
|
+
|
|
233
|
+
---
|
|
234
|
+
|
|
235
|
+
## Design limitations / footguns
|
|
236
|
+
|
|
237
|
+
**Circuit breaker half-open is single-probe (implemented; has open defects).** After the recovery timeout the breaker moves to `HALF_OPEN` and allows up to `CircuitBreakerConfig.half_open_max_calls` (default 1) trial calls; success closes it, failure re-opens it immediately. See `step.py` `_check_circuit_breaker` / `_record_failure`. ⚠️ Three defects in this logic are open — see **CR-2** (probe slot consumed before cancel check → permanent lock), **CR-3** (reset before schema check), and **CR-7** (non-atomic probe increment) above.
|
|
238
|
+
|
|
239
|
+
**Sync timeouts leave worker threads running.** On timeout, `fut.result(timeout=...)` raises but the thread-pool worker keeps executing `_invoke_function`. `@piped(cancel_on_timeout=True)` makes a best-effort `fut.cancel()`, which only helps if the worker has not started — a running thread cannot be interrupted. Documented footgun under load.
|
|
240
|
+
|
|
241
|
+
**Process pools are global and long-lived.** `_POOLS` only shuts down via `cleanup_pools()` (Pipeline context manager or explicit call). Long-running apps can accumulate idle pools.
|
|
242
|
+
|
|
243
|
+
**`@node` mutates class metadata** via `inst.__class__.__name__ = ...` — unusual pattern; each decorated function gets its own inner class so it works, but fragile.
|
|
244
|
+
|
|
245
|
+
**`retry` / `circuit_breaker` are immutable (resolved).** Both return a **new** `PipeStep` via `PipeStep.copy()` with fresh breaker counters, so decorating the same step twice or reusing across pipelines no longer shares state.
|
|
246
|
+
|
|
247
|
+
**`parallel` + `batch_size` together** — parallel wins; batching is skipped. Documented and tested, but easy to misconfigure.
|
|
248
|
+
|
|
249
|
+
---
|
|
250
|
+
|
|
251
|
+
## Test coverage gaps
|
|
252
|
+
|
|
253
|
+
No tests for:
|
|
254
|
+
|
|
255
|
+
- async `batch_size`
|
|
256
|
+
- async `parallel='thread'` / `'process'` with async functions
|
|
257
|
+
- `FanOutStep.async_run` with `parallel=`
|
|
258
|
+
- `MapReduceStep.async_run` with async mapper/reducer
|
|
259
|
+
- half-open circuit breaker probe semantics
|
|
260
|
+
- rsloop in CI (only optional local tests)
|
|
261
|
+
|
|
262
|
+
---
|
|
263
|
+
|
|
264
|
+
## What looks solid
|
|
265
|
+
|
|
266
|
+
Previously flagged checklist items appear fixed and covered by tests:
|
|
267
|
+
|
|
268
|
+
- Circuit breaker records one failure per logical call (not per retry)
|
|
269
|
+
- Async timeout honors `timeout=0.0`
|
|
270
|
+
- Parallel forwards kwargs (sync and async)
|
|
271
|
+
- Batched does not sum booleans
|
|
272
|
+
- FanOut process parallel uses picklable helper
|
|
273
|
+
- Topo sort cached; deque used instead of `list.pop(0)`
|
|
274
|
+
- Shared thread pool for sync timeouts
|
|
275
|
+
- Branch helpers deduplicated
|
|
276
|
+
|
|
277
|
+
**89 tests pass** locally with rsloop installed (3 skipped).
|
|
278
|
+
|
|
279
|
+
---
|
|
280
|
+
|
|
281
|
+
## Suggested priority
|
|
282
|
+
|
|
283
|
+
The async/spec/parallel items from the original audit are now resolved (see
|
|
284
|
+
**Incomplete features**). Remaining work is the code-review pass:
|
|
285
|
+
|
|
286
|
+
1. **CR-1** — propagate shared context into pool workers (or document the limit). Highest user-facing impact.
|
|
287
|
+
2. **CR-2** — half-open probe consumed before cancel check → breaker permanently stuck. Correctness.
|
|
288
|
+
3. **CR-3** — move schema validation before breaker reset.
|
|
289
|
+
4. **CR-4** — activate shared context in `Graph` (extract a shared helper).
|
|
290
|
+
5. **CR-5** — forward `on_step` through `map`/`async_map`/`run_async`/`map_async` (or narrow the docs).
|
|
291
|
+
6. **CR-6** — reject specs that contain both `graph:` and `steps:`.
|
|
292
|
+
7. **CR-7** — lock breaker state (or document non-thread-safety). Lowest priority.
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
# stepcraft — Claude Code Handoff Plan
|
|
2
|
+
|
|
3
|
+
**Author:** Senior review / planning pass
|
|
4
|
+
**Date:** 2026-06-24
|
|
5
|
+
**Sub-agent:** claude-code (available after quota reset ~12:50 AM Asia/Calcutta)
|
|
6
|
+
**Branch:** `working_barnch`
|
|
7
|
+
|
|
8
|
+
This file is the **source of truth** for what is done, what was started tonight, and what claude-code should build next. Read this before any feature work.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## Already shipped (do not redo)
|
|
13
|
+
|
|
14
|
+
| Area | Status | Key commits / files |
|
|
15
|
+
|------|--------|---------------------|
|
|
16
|
+
| Async `batch_size` awaits coroutines | Done | `step._execute_batched_async` |
|
|
17
|
+
| Async `parallel` awaits coroutines | Done | `step._execute_parallel_async` |
|
|
18
|
+
| `_async_run_branch_value` awaits `run()` coroutines | Done | `utils.py` |
|
|
19
|
+
| List auto-map on default `@piped` | Done | `step._should_auto_map`, lists only (not tuples) |
|
|
20
|
+
| `Pipeline.from_spec` YAML/JSON | Done | `spec.py`, `pip install stepcraft[spec]` |
|
|
21
|
+
| FanOut / MapReduce async parity | Done | `fan.py` |
|
|
22
|
+
| jit/vectorize warnings | Done | `decorators.py` |
|
|
23
|
+
| rsloop CI job | Done | `.github/workflows/ci.yml` `test-rsloop` |
|
|
24
|
+
| Free-threaded 3.14 parallel | Done | `runtime.py`, `parallel='auto'` |
|
|
25
|
+
|
|
26
|
+
**Tests:** 114+ passed locally. Run `uv run pytest tests/ -v` before every PR.
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## Started tonight (manager pass — verify / extend)
|
|
31
|
+
|
|
32
|
+
Claude-code should **read the diff** and finish tests/docs if incomplete:
|
|
33
|
+
|
|
34
|
+
### A. `@piped(auto_map=False)` opt-out
|
|
35
|
+
- **Why:** Steps that intentionally receive a whole list must not be split.
|
|
36
|
+
- **Files:** `decorators.py`, `step.py`, `tests/test_pipeline.py`
|
|
37
|
+
- **Acceptance:**
|
|
38
|
+
- `@piped(auto_map=False)` passes `[1,2,3]` as one argument
|
|
39
|
+
- Default remains auto-map on multi-item lists
|
|
40
|
+
- `__call__` / partial binding preserves `auto_map`
|
|
41
|
+
|
|
42
|
+
### B. Immutable `retry` / `circuit_breaker`
|
|
43
|
+
- **Why:** Decorating the same `PipeStep` twice or reusing across pipelines must not share breaker counters.
|
|
44
|
+
- **Files:** `step.py` (`PipeStep.copy()`), `decorators.py`
|
|
45
|
+
- **Acceptance:**
|
|
46
|
+
- `@retry` / `@circuit_breaker` return a **new** `PipeStep` via `copy()`
|
|
47
|
+
- Fresh `_circuit_state`, `_failure_count`, `_last_failure_time`
|
|
48
|
+
- Existing tests still pass; add test that two decorated copies do not share breaker state
|
|
49
|
+
|
|
50
|
+
### C. `Pipeline.async_run_detailed()`
|
|
51
|
+
- **Why:** Parity with `run_detailed()` for async pipelines.
|
|
52
|
+
- **Files:** `pipeline.py`, `tests/test_pipeline.py`
|
|
53
|
+
- **Acceptance:**
|
|
54
|
+
- Returns `ExecutionResult` with history, `dt`, `n`
|
|
55
|
+
- Works with async `@piped` steps and sync steps (executor fallback in `async_run`)
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## Phase 1 — Claude-code (high priority, ~1 session)
|
|
60
|
+
|
|
61
|
+
### 1. Step hooks / observability
|
|
62
|
+
**Goal:** Callback after each step without wrapping functions.
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
# API sketch (finalize in hooks.py)
|
|
66
|
+
StepHook = Callable[[str, Any, Any, float], None] # name, input, output, step_dt
|
|
67
|
+
|
|
68
|
+
pipeline.run(seed, on_step=hook)
|
|
69
|
+
pipeline.run_detailed(seed, on_step=hook) # hook in addition to history
|
|
70
|
+
await pipeline.async_run(seed, on_step=hook)
|
|
71
|
+
await pipeline.async_run_detailed(seed, on_step=hook)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
**Files to add/touch:**
|
|
75
|
+
- `src/stepcraft/hooks.py` — `StepHook` type alias, `_call_hook` helper
|
|
76
|
+
- `pipeline.py` — optional `on_step` on run paths
|
|
77
|
+
- `graph.py` — optional `on_step` on `run` / `async_run` (per node)
|
|
78
|
+
- `tests/test_pipeline.py` — hook called N times, receives correct I/O
|
|
79
|
+
|
|
80
|
+
**Rules:**
|
|
81
|
+
- Hook errors must **not** swallow pipeline errors (wrap in try/except log or re-raise based on strict flag — default: log warning, continue)
|
|
82
|
+
- `on_step` is optional; no breaking changes to existing signatures (use keyword-only)
|
|
83
|
+
|
|
84
|
+
### 2. Update `AUDIT.md` + `README.md`
|
|
85
|
+
- Mark fixed bugs incomplete features as resolved
|
|
86
|
+
- Document: `auto_map`, `parallel='auto'`, `from_spec`, free-threading, `pip install stepcraft[spec]`
|
|
87
|
+
- Add short “Parallelism” section: GIL vs 3.14t vs `process`
|
|
88
|
+
|
|
89
|
+
### 3. `@piped(map=False)` alias
|
|
90
|
+
- If manager used `auto_map`, add `map: Optional[bool] = None` where `map=False` ≡ `auto_map=False` for README-friendly naming (single implementation).
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## Phase 2 — Claude-code (production hardening)
|
|
95
|
+
|
|
96
|
+
### 4. Circuit breaker single-probe half-open
|
|
97
|
+
**Location:** `step.py` `_check_circuit_breaker`, `_record_failure`, `_reset_circuit_breaker`
|
|
98
|
+
|
|
99
|
+
**Behavior:**
|
|
100
|
+
- `HALF_OPEN`: allow **one** trial call; success → `CLOSED`, failure → `OPEN`
|
|
101
|
+
- Add `CircuitBreakerConfig.half_open_max_calls: int = 1` (optional, default 1)
|
|
102
|
+
- Tests: `test_circuit_breaker_half_open_single_probe`
|
|
103
|
+
|
|
104
|
+
### 5. Configurable thread/process pools
|
|
105
|
+
**Location:** `pools.py`
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
def configure_pools(*, thread_workers: int | None = None, process_workers: int | None = None) -> None
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
- Apply on next `_get_pool` creation; document that reconfigure requires `cleanup_pools()` first
|
|
112
|
+
- Test: pool respects max_workers when set
|
|
113
|
+
|
|
114
|
+
### 6. Sync timeout cancellation note + optional flag
|
|
115
|
+
- Document that timed-out sync work keeps running in thread pool (AUDIT footgun)
|
|
116
|
+
- Optional: `cancel_on_timeout=True` on `@piped` — best-effort, document limitations
|
|
117
|
+
- **Low priority** if complex; documenting is enough for Phase 2
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
## Phase 3 — Claude-code (larger features)
|
|
122
|
+
|
|
123
|
+
### 7. Pipeline shared context
|
|
124
|
+
```python
|
|
125
|
+
Pipeline(steps, context={"request_id": "abc"})
|
|
126
|
+
# Steps access via contextvar or optional kw PIPE_CONTEXT
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
- Use `contextvars.ContextVar[dict]` set at pipeline run entry
|
|
130
|
+
- `Node.setup()` can read context
|
|
131
|
+
- Do **not** break existing step signatures
|
|
132
|
+
|
|
133
|
+
### 8. `from_spec` graph support
|
|
134
|
+
Extend `spec.py`:
|
|
135
|
+
```yaml
|
|
136
|
+
graph:
|
|
137
|
+
nodes:
|
|
138
|
+
a: { import: "mymodule:fetch" }
|
|
139
|
+
b: { import: "mymodule:parse" }
|
|
140
|
+
edges:
|
|
141
|
+
- [a, b]
|
|
142
|
+
seed_node: a
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
- `Graph.from_spec(path)` or `Pipeline.from_spec` detects `graph:` key
|
|
146
|
+
- Tests with fixture YAML
|
|
147
|
+
|
|
148
|
+
### 9. CI: `python3.14t` job
|
|
149
|
+
- Ubuntu job installing free-threaded CPython when available on setup-python
|
|
150
|
+
- Run `tests/test_pipeline.py -k "free_threading or parallel_auto"`
|
|
151
|
+
- Skip gracefully if 3.14t not on runner
|
|
152
|
+
|
|
153
|
+
---
|
|
154
|
+
|
|
155
|
+
## Phase 4 — Defer unless user asks
|
|
156
|
+
|
|
157
|
+
- Streaming / lazy iterator pipelines
|
|
158
|
+
- Checkpoint / resume for graphs
|
|
159
|
+
- Pydantic schema validation (optional dep)
|
|
160
|
+
- OpenTelemetry exporter (build on hooks)
|
|
161
|
+
|
|
162
|
+
---
|
|
163
|
+
|
|
164
|
+
## Conventions for claude-code
|
|
165
|
+
|
|
166
|
+
1. **Minimal diff** — match existing style in `step.py`, `decorators.py`
|
|
167
|
+
2. **Tests required** for every behavior change
|
|
168
|
+
3. **No new required dependencies** — optional extras only (`spec`, `rsloop`, etc.)
|
|
169
|
+
4. **Do not commit** unless user asks (manager may commit manager pass)
|
|
170
|
+
5. **Run before handoff:** `uv run pytest tests/ -q`
|
|
171
|
+
6. **Import surface:** export new public APIs from `__init__.py` + `pipeline` shim if tests import from `pipeline`
|
|
172
|
+
|
|
173
|
+
---
|
|
174
|
+
|
|
175
|
+
## Suggested claude-code prompt (copy-paste)
|
|
176
|
+
|
|
177
|
+
```
|
|
178
|
+
Read CLAUDE_CODE_HANDOFF.md fully. You are implementing stepcraft on branch working_barnch.
|
|
179
|
+
|
|
180
|
+
1. Verify items in "Started tonight" (A–C) — add missing tests, fix gaps.
|
|
181
|
+
2. Implement Phase 1 items 1–3 (hooks, docs, map= alias).
|
|
182
|
+
3. Run full test suite; fix failures.
|
|
183
|
+
4. Summarize what changed and what remains for Phase 2.
|
|
184
|
+
|
|
185
|
+
Do not commit unless I ask. Follow existing code conventions.
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
---
|
|
189
|
+
|
|
190
|
+
## Manager pass checklist
|
|
191
|
+
|
|
192
|
+
- [x] Write this handoff file
|
|
193
|
+
- [x] `auto_map=False` implementation
|
|
194
|
+
- [x] `PipeStep.copy()` + immutable decorators
|
|
195
|
+
- [x] `async_run_detailed()`
|
|
196
|
+
- [x] Tests for above
|
|
197
|
+
- [x] Commit manager pass (if user approves) — `3c7e1f4`, pushed
|
|
198
|
+
|
|
199
|
+
---
|
|
200
|
+
|
|
201
|
+
## Scheduled trigger (12:51 AM IST)
|
|
202
|
+
|
|
203
|
+
- **Set:** 2026-06-24 ~21:38 IST
|
|
204
|
+
- **Fires:** 2026-06-25 00:51:00 Asia/Kolkata
|
|
205
|
+
- **Script:** `scripts/trigger_claude_code_at_1251.sh`
|
|
206
|
+
- **Wake sentinel:** `AGENT_LOOP_WAKE_CLAUDE_CODE`
|
|
207
|
+
- **Remote:** `origin/working_barnch` @ `3c7e1f4`
|
|
208
|
+
|
|
209
|
+
To cancel: kill the background sleeper PID shown in terminal metadata.
|