stepcraft 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. stepcraft-0.1.0/.github/workflows/ci.yml +83 -0
  2. stepcraft-0.1.0/.github/workflows/publish.yml +48 -0
  3. stepcraft-0.1.0/.gitignore +31 -0
  4. stepcraft-0.1.0/.python-version +1 -0
  5. stepcraft-0.1.0/AUDIT.md +292 -0
  6. stepcraft-0.1.0/CLAUDE_CODE_HANDOFF.md +209 -0
  7. stepcraft-0.1.0/CODE_REVIEW_CHECKLIST.md +43 -0
  8. stepcraft-0.1.0/LICENSE +21 -0
  9. stepcraft-0.1.0/PKG-INFO +475 -0
  10. stepcraft-0.1.0/README.md +437 -0
  11. stepcraft-0.1.0/pipeline.py +3 -0
  12. stepcraft-0.1.0/pyproject.toml +68 -0
  13. stepcraft-0.1.0/src/stepcraft/__init__.py +58 -0
  14. stepcraft-0.1.0/src/stepcraft/async_runtime.py +50 -0
  15. stepcraft-0.1.0/src/stepcraft/branching.py +61 -0
  16. stepcraft-0.1.0/src/stepcraft/config.py +26 -0
  17. stepcraft-0.1.0/src/stepcraft/constants.py +16 -0
  18. stepcraft-0.1.0/src/stepcraft/context.py +66 -0
  19. stepcraft-0.1.0/src/stepcraft/decorators.py +160 -0
  20. stepcraft-0.1.0/src/stepcraft/exceptions.py +27 -0
  21. stepcraft-0.1.0/src/stepcraft/fan.py +153 -0
  22. stepcraft-0.1.0/src/stepcraft/graph.py +247 -0
  23. stepcraft-0.1.0/src/stepcraft/hooks.py +28 -0
  24. stepcraft-0.1.0/src/stepcraft/node.py +57 -0
  25. stepcraft-0.1.0/src/stepcraft/pipeline.py +217 -0
  26. stepcraft-0.1.0/src/stepcraft/pools.py +84 -0
  27. stepcraft-0.1.0/src/stepcraft/py.typed +0 -0
  28. stepcraft-0.1.0/src/stepcraft/result.py +24 -0
  29. stepcraft-0.1.0/src/stepcraft/runtime.py +52 -0
  30. stepcraft-0.1.0/src/stepcraft/spec.py +215 -0
  31. stepcraft-0.1.0/src/stepcraft/step.py +445 -0
  32. stepcraft-0.1.0/src/stepcraft/typecheck.py +47 -0
  33. stepcraft-0.1.0/src/stepcraft/utils.py +59 -0
  34. stepcraft-0.1.0/tests/conftest.py +33 -0
  35. stepcraft-0.1.0/tests/fixtures/graph_diamond.yaml +15 -0
  36. stepcraft-0.1.0/tests/fixtures/inc_double.json +6 -0
  37. stepcraft-0.1.0/tests/fixtures/inc_double.yaml +3 -0
  38. stepcraft-0.1.0/tests/spec_fixtures.py +13 -0
  39. stepcraft-0.1.0/tests/test_pipeline.py +2254 -0
  40. stepcraft-0.1.0/uv.lock +336 -0
@@ -0,0 +1,83 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main, working_barnch]
6
+ pull_request:
7
+ branches: [main]
8
+
9
+ jobs:
10
+ test:
11
+ runs-on: ${{ matrix.os }}
12
+ timeout-minutes: 15
13
+ strategy:
14
+ fail-fast: false
15
+ matrix:
16
+ os: [ubuntu-latest, macos-latest, windows-latest]
17
+ python-version: ["3.12", "3.13"]
18
+
19
+ steps:
20
+ - uses: actions/checkout@v4
21
+
22
+ - name: Set up Python ${{ matrix.python-version }}
23
+ uses: actions/setup-python@v5
24
+ with:
25
+ python-version: ${{ matrix.python-version }}
26
+
27
+ - name: Install uv
28
+ uses: astral-sh/setup-uv@v5
29
+
30
+ - name: Install dependencies
31
+ run: |
32
+ uv sync --all-extras --group dev
33
+
34
+ - name: Run tests
35
+ run: |
36
+ uv run pytest tests/ -v
37
+
38
+ test-free-threading:
39
+ # Free-threaded (no-GIL) CPython 3.14t. Skips gracefully if the runner
40
+ # cannot provision the build.
41
+ runs-on: ubuntu-latest
42
+ continue-on-error: true
43
+ timeout-minutes: 15
44
+ steps:
45
+ - uses: actions/checkout@v4
46
+
47
+ - name: Set up free-threaded Python 3.14
48
+ uses: actions/setup-python@v5
49
+ with:
50
+ python-version: "3.14t"
51
+
52
+ - name: Install uv
53
+ uses: astral-sh/setup-uv@v5
54
+
55
+ - name: Install dependencies
56
+ run: |
57
+ uv sync --group dev
58
+
59
+ - name: Run free-threading + parallel tests
60
+ run: |
61
+ uv run pytest tests/ -v -k "free_threading or parallel_auto or parallel or thread_parallel"
62
+
63
+ test-rsloop:
64
+ runs-on: ubuntu-latest
65
+ timeout-minutes: 10
66
+ steps:
67
+ - uses: actions/checkout@v4
68
+
69
+ - name: Set up Python 3.13
70
+ uses: actions/setup-python@v5
71
+ with:
72
+ python-version: "3.13"
73
+
74
+ - name: Install uv
75
+ uses: astral-sh/setup-uv@v5
76
+
77
+ - name: Install with rsloop extra
78
+ run: |
79
+ uv sync --extra rsloop --group dev
80
+
81
+ - name: Run rsloop tests
82
+ run: |
83
+ uv run pytest tests/ -v -m rsloop
@@ -0,0 +1,48 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+
7
+ permissions:
8
+ contents: read
9
+
10
+ jobs:
11
+ build:
12
+ name: Build distribution
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+
17
+ - name: Install uv
18
+ uses: astral-sh/setup-uv@v5
19
+
20
+ - name: Build wheel and sdist
21
+ run: uv build --out-dir dist
22
+
23
+ - name: Upload artifacts
24
+ uses: actions/upload-artifact@v4
25
+ with:
26
+ name: dist
27
+ path: dist/*
28
+
29
+ publish:
30
+ name: Publish to PyPI
31
+ needs: [build]
32
+ runs-on: ubuntu-latest
33
+ environment: pypi
34
+
35
+ steps:
36
+ - name: Download artifacts
37
+ uses: actions/download-artifact@v4
38
+ with:
39
+ name: dist
40
+ path: dist
41
+
42
+ - name: Install uv
43
+ uses: astral-sh/setup-uv@v5
44
+
45
+ - name: Publish to PyPI
46
+ run: uv publish dist/*
47
+ env:
48
+ UV_PUBLISH_TOKEN: ${{ secrets.UV_PUBLISH_TOKEN }}
@@ -0,0 +1,31 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ *.egg
6
+ dist/
7
+ build/
8
+ .eggs/
9
+
10
+ # Virtual environments
11
+ .venv/
12
+ venv/
13
+
14
+ # IDE
15
+ .idea/
16
+ .vscode/
17
+ *.swp
18
+ *.swo
19
+
20
+ # Testing
21
+ .pytest_cache/
22
+ .coverage
23
+ htmlcov/
24
+
25
+ # Compiled extensions
26
+ *.so
27
+
28
+ # OS
29
+ .DS_Store
30
+ Thumbs.db
31
+ ._*
@@ -0,0 +1 @@
1
+ 3.13
@@ -0,0 +1,292 @@
1
+ # stepcraft Audit — Bugs & Incomplete Features
2
+
3
+ Audit date: 2026-06-24. Based on code review, runtime reproduction, and test coverage analysis.
4
+ Updated 2026-06-25 with findings from the multi-agent code-review pass on the
5
+ `working_barnch` feature diff (hooks, shared context, half-open breaker, beartype).
6
+
7
+ ---
8
+
9
+ ## Open findings — code-review pass (2026-06-25)
10
+
11
+ Verified findings against the feature diff. Ordered by severity. Each item lists
12
+ **what** is wrong and **how** to fix it.
13
+
14
+ ### CR-1. Shared context never reaches pool workers (HIGH)
15
+
16
+ **What:** `Pipeline._activate_context` sets a `contextvars.ContextVar` on the
17
+ calling thread only. `pool.map` / `loop.run_in_executor` do **not** copy
18
+ contextvars into worker threads/processes, so any step that runs off the calling
19
+ thread sees `get_context() == {}`:
20
+ - `@piped(parallel='thread'|'process')` steps,
21
+ - `@piped(timeout=...)` steps (run via the thread pool),
22
+ - sync functions executed inside an async pipeline (`run_in_executor`),
23
+ - auto-mapped/batched work dispatched to executors.
24
+
25
+ README presents `get_context()` as the supported way for steps to read context,
26
+ so this is a silent correctness gap.
27
+
28
+ **Location:** `context.py:12`, `pipeline.py` (`_activate_context`, run loops),
29
+ `step.py` (`_execute_parallel`, `_execute_parallel_async`, `_execute_sync`
30
+ timeout branch, `_execute_auto_map*`, `_execute_batched*`).
31
+
32
+ **How to fix:** capture the active context at dispatch and run workers inside it.
33
+ - Threads: wrap submitted callables with `contextvars.copy_context().run(fn, ...)`,
34
+ or snapshot `get_context()` on the calling thread and re-set it inside the
35
+ worker wrapper.
36
+ - Processes: contextvars cannot cross the pickle boundary — pass the context dict
37
+ explicitly to the worker and re-set it there, or document that process pools do
38
+ not see `get_context()`.
39
+ - Add tests: `parallel='thread'`, `timeout=`, and async-with-sync-func steps all
40
+ reading `get_context()` inside a `Pipeline(context=...)`.
41
+
42
+ ### CR-2. Half-open probe slot consumed before cancel check → breaker stuck (HIGH)
43
+
44
+ **What:** In `_preflight_check`, `_check_circuit_breaker()` runs first and
45
+ increments `_half_open_calls`; the cancel-event check runs *after*. If the call
46
+ is cancelled, the probe slot is consumed but neither `_record_failure` nor
47
+ `_reset_circuit_breaker` runs, leaving the breaker `HALF_OPEN` with the slot used
48
+ **forever** — every later call raises `CircuitBreakerError` even once healthy.
49
+
50
+ **Location:** `step.py` `_preflight_check` / `_check_circuit_breaker` (~L155, L387–391).
51
+
52
+ **How to fix:** check the cancel event **before** consuming a probe slot (reorder
53
+ `_preflight_check` so the cancel check precedes `_check_circuit_breaker`), or
54
+ release the slot on cancellation (decrement `_half_open_calls` / restore state in
55
+ a `finally`/except path when `CancelledError` is raised before the call runs).
56
+ Add a regression test: half-open + cancel set → breaker still recovers later.
57
+
58
+ ### CR-3. Breaker reset happens before the schema check (MEDIUM)
59
+
60
+ **What:** `_finalize_success` calls `_reset_circuit_breaker()` and *then* validates
61
+ `schema`. A half-open probe whose function returns a wrong-typed value first
62
+ closes the breaker, then raises `TypeError`, which records a failure against a
63
+ now-`CLOSED` breaker (count from 0) instead of re-opening immediately.
64
+
65
+ **Location:** `step.py:169–179` (`_finalize_success`).
66
+
67
+ **How to fix:** validate the schema **before** calling `_reset_circuit_breaker()`
68
+ so a schema failure is treated like any other probe failure. Add a test:
69
+ `circuit_breaker` + `schema` where the recovered call returns the wrong type →
70
+ breaker re-opens rather than closing.
71
+
72
+ ### CR-4. Graph never activates shared context (MEDIUM)
73
+
74
+ **What:** Context activation was added to `Pipeline` only. `Graph.run` /
75
+ `Graph.async_run` have no `_activate_context`, so a `@piped` node that reads
76
+ `get_context()` gets `{}` in a DAG even when run sequentially — inconsistent with
77
+ `Pipeline` and with the docs.
78
+
79
+ **Location:** `graph.py:120` (`run`) and `graph.py` `async_run`.
80
+
81
+ **How to fix:** give `Graph` an optional `context=` and wrap both run paths in the
82
+ same activation, ideally by extracting a shared `_activate_context` helper (e.g.
83
+ into `context.py`) reused by `Pipeline` and `Graph`. Note this composes with CR-1
84
+ (workers still need propagation). Add a Graph context test.
85
+
86
+ ### CR-5. `on_step` missing on half the public run surface (MEDIUM)
87
+
88
+ **What:** `on_step` was threaded through `run` / `async_run` / `run_detailed` /
89
+ `async_run_detailed` only. `run_async`, `map`, `async_map`, `map_async` ignore it;
90
+ `pipeline.run_async(seed, on_step=hook)` raises `TypeError`. README says "Pass
91
+ `on_step` to any run method."
92
+
93
+ **Location:** `pipeline.py` (`run_async`, `map`, `async_map`, `map_async`).
94
+
95
+ **How to fix:** forward `on_step` from `map`/`async_map` into per-item `run`/
96
+ `async_run`, and from `run_async`/`map_async` into the underlying coroutine; OR
97
+ narrow the README to list exactly the four methods that accept it. Prefer
98
+ forwarding for consistency. Add coverage for at least `map(on_step=...)`.
99
+
100
+ ### CR-6. Spec with both `graph:` and `steps:` builds a silent linear pipeline (LOW)
101
+
102
+ **What:** `build_pipeline_from_spec` only rejects a graph spec when `steps` is
103
+ absent. A spec containing **both** keys silently builds a linear `Pipeline` and
104
+ ignores the `graph:` block (wrong execution order / fan-in inputs, no error).
105
+
106
+ **Location:** `spec.py:144` (`build_pipeline_from_spec`).
107
+
108
+ **How to fix:** raise `ValueError` when both `graph` and `steps` are present
109
+ (ambiguous spec). Mirror the check in `build_graph_from_spec`. Add a test for the
110
+ both-keys case.
111
+
112
+ ### CR-7. `_half_open_calls` increment is not atomic across threads (LOW)
113
+
114
+ **What:** `_check_circuit_breaker` does a non-atomic read-check-increment on
115
+ `_half_open_calls`. When one `PipeStep` is invoked concurrently (parallel graph
116
+ levels, `parallel='thread'` fan-out), multiple threads can each read `0` and all
117
+ proceed, exceeding `half_open_max_calls`. Pre-existing class of issue — breaker
118
+ state was never lock-guarded — but the new probe quota relies on it.
119
+
120
+ **Location:** `step.py:384–391` (`_check_circuit_breaker`, `_record_failure`).
121
+
122
+ **How to fix:** guard breaker state transitions with a `threading.Lock` on the
123
+ step (note: a `frozen`/`object.__setattr__` dataclass needs a non-init lock
124
+ field). Lower priority unless shared concurrent steps are a supported pattern;
125
+ otherwise document that breaker state is not thread-safe.
126
+
127
+ ### Intentional behavior changes (not bugs — confirm and keep)
128
+
129
+ - **Single-probe half-open semantics** (`step.py` `_check_circuit_breaker`): the
130
+ breaker now allows only `half_open_max_calls` (default 1) probes and re-opens on
131
+ the first probe failure. This is the requested Phase 2.4 feature, **not** a
132
+ regression — but CR-2/CR-3/CR-7 above are real defects within it.
133
+ - **beartype runtime type-checking** (`__init__.py`): every function/method is
134
+ decorated via the import hook; calls that violate annotations now raise
135
+ `BeartypeCallHintParamViolation`. Intended. Caveat to document: it is always-on
136
+ with no opt-out env var, and the numeric tower is enabled (int accepted for
137
+ float). Consider an opt-out hook (e.g. `PIPECRAFT_NO_BEARTYPE`) if consumers
138
+ need to disable runtime checks in production.
139
+
140
+ ---
141
+
142
+ ## Confirmed bugs
143
+
144
+ ### 1. Async + `batch_size` returns unawaited coroutines
145
+
146
+ When an **async** function is used with `batch_size > 1`, `_invoke_function_async` calls the sync `_execute_batched`, which invokes `self.func(batch)` without awaiting.
147
+
148
+ ```python
149
+ @piped(batch_size=2)
150
+ async def process_batch(batch):
151
+ return [x * 2 for x in batch]
152
+
153
+ await process_batch.async_run([1, 2, 3, 4])
154
+ # -> [<coroutine ...>, <coroutine ...>] # RuntimeWarning: never awaited
155
+ ```
156
+
157
+ **Location:** `src/stepcraft/step.py` — `_invoke_function_async` → `_execute_batched`
158
+
159
+ **Impact:** `async_run` silently returns coroutine objects instead of results.
160
+
161
+ ---
162
+
163
+ ### 2. Async + `parallel='thread'` has the same problem
164
+
165
+ The parallel async path uses `run_in_executor(pool, call, item)`, which always invokes the **sync** wrapper. Async `@piped` functions therefore produce unawaited coroutines.
166
+
167
+ ```python
168
+ @piped(parallel='thread')
169
+ async def f(x):
170
+ return x * 2
171
+
172
+ await f.async_run([1, 2, 3])
173
+ # -> [<coroutine ...>, <coroutine ...>, <coroutine ...>]
174
+ ```
175
+
176
+ **Location:** `src/stepcraft/step.py` — `_execute_parallel_async`
177
+
178
+ **Impact:** `parallel` + async steps are broken on the async execution path. Sync `run()` is fine (thread pool runs sync callables).
179
+
180
+ ---
181
+
182
+ ### 3. `_async_run_branch_value` may not await `run()` coroutines
183
+
184
+ If a branch has `.run` but not `.async_run`, and `run()` returns a coroutine, it is returned without awaiting.
185
+
186
+ ```python
187
+ # src/stepcraft/utils.py
188
+ async def _async_run_branch_value(branch, value):
189
+ ...
190
+ if hasattr(branch, 'run'):
191
+ return branch.run(value) # coroutine not awaited
192
+ result = branch(value)
193
+ if asyncio.iscoroutine(result):
194
+ return await result
195
+ ```
196
+
197
+ **Impact:** Uncommon, but custom steps with only sync `run()` that return coroutines will misbehave in `ConditionalStep` / `SwitchStep` async paths.
198
+
199
+ ---
200
+
201
+ ## Known deferred behavior (documented, still open)
202
+
203
+ ### 4. Whole collections are not auto-mapped
204
+
205
+ Passing a list into a normal step sends the **entire list** as one argument. Parallel/batch only kick in when explicitly configured.
206
+
207
+ ```python
208
+ @piped
209
+ def step(x): ...
210
+
211
+ pipeline.run([1, 2, 3]) # step receives [1,2,3], not three scalars
212
+ ```
213
+
214
+ **Status:** Intentionally deferred — see `CODE_REVIEW_CHECKLIST.md` line 10. Surprising if users expect implicit map behavior.
215
+
216
+ ---
217
+
218
+ ## Incomplete features
219
+
220
+ | Feature | Status |
221
+ |--------|--------|
222
+ | `Pipeline.from_spec("yaml")` | ✅ Implemented (`spec.py`, `stepcraft[spec]`) |
223
+ | `Graph.from_spec("yaml")` | ✅ Implemented — `graph:` block with `nodes`/`edges` |
224
+ | `FanOutStep.async_run` + `parallel=` | ✅ Honors `parallel` on the async path |
225
+ | `MapReduceStep.async_run` | ✅ Async mappers mapped concurrently per batch |
226
+ | `jit` / `vectorize` on `@piped` | ✅ Warns when numba/numpy missing |
227
+ | rsloop integration | ✅ Covered by the `test-rsloop` CI job |
228
+ | Step hooks (`on_step`) | ✅ `Pipeline` + `Graph` run paths, see `hooks.py` |
229
+ | Pipeline shared context | ✅ `Pipeline(steps, context=...)` + `get_context()` |
230
+ | Configurable pools | ✅ `configure_pools(thread_workers=, process_workers=)` |
231
+ | Runtime type-checking | ✅ beartype decorates every function via import hook |
232
+
233
+ ---
234
+
235
+ ## Design limitations / footguns
236
+
237
+ **Circuit breaker half-open is single-probe (implemented; has open defects).** After the recovery timeout the breaker moves to `HALF_OPEN` and allows up to `CircuitBreakerConfig.half_open_max_calls` (default 1) trial calls; success closes it, failure re-opens it immediately. See `step.py` `_check_circuit_breaker` / `_record_failure`. ⚠️ Three defects in this logic are open — see **CR-2** (probe slot consumed before cancel check → permanent lock), **CR-3** (reset before schema check), and **CR-7** (non-atomic probe increment) above.
238
+
239
+ **Sync timeouts leave worker threads running.** On timeout, `fut.result(timeout=...)` raises but the thread-pool worker keeps executing `_invoke_function`. `@piped(cancel_on_timeout=True)` makes a best-effort `fut.cancel()`, which only helps if the worker has not started — a running thread cannot be interrupted. Documented footgun under load.
240
+
241
+ **Process pools are global and long-lived.** `_POOLS` only shuts down via `cleanup_pools()` (Pipeline context manager or explicit call). Long-running apps can accumulate idle pools.
242
+
243
+ **`@node` mutates class metadata** via `inst.__class__.__name__ = ...` — unusual pattern; each decorated function gets its own inner class so it works, but fragile.
244
+
245
+ **`retry` / `circuit_breaker` are immutable (resolved).** Both return a **new** `PipeStep` via `PipeStep.copy()` with fresh breaker counters, so decorating the same step twice or reusing across pipelines no longer shares state.
246
+
247
+ **`parallel` + `batch_size` together** — parallel wins; batching is skipped. Documented and tested, but easy to misconfigure.
248
+
249
+ ---
250
+
251
+ ## Test coverage gaps
252
+
253
+ No tests for:
254
+
255
+ - async `batch_size`
256
+ - async `parallel='thread'` / `'process'` with async functions
257
+ - `FanOutStep.async_run` with `parallel=`
258
+ - `MapReduceStep.async_run` with async mapper/reducer
259
+ - half-open circuit breaker probe semantics
260
+ - rsloop in CI (only optional local tests)
261
+
262
+ ---
263
+
264
+ ## What looks solid
265
+
266
+ Previously flagged checklist items appear fixed and covered by tests:
267
+
268
+ - Circuit breaker records one failure per logical call (not per retry)
269
+ - Async timeout honors `timeout=0.0`
270
+ - Parallel forwards kwargs (sync and async)
271
+ - Batched does not sum booleans
272
+ - FanOut process parallel uses picklable helper
273
+ - Topo sort cached; deque used instead of `list.pop(0)`
274
+ - Shared thread pool for sync timeouts
275
+ - Branch helpers deduplicated
276
+
277
+ **89 tests pass** locally with rsloop installed (3 skipped).
278
+
279
+ ---
280
+
281
+ ## Suggested priority
282
+
283
+ The async/spec/parallel items from the original audit are now resolved (see
284
+ **Incomplete features**). Remaining work is the code-review pass:
285
+
286
+ 1. **CR-1** — propagate shared context into pool workers (or document the limit). Highest user-facing impact.
287
+ 2. **CR-2** — half-open probe consumed before cancel check → breaker permanently stuck. Correctness.
288
+ 3. **CR-3** — move schema validation before breaker reset.
289
+ 4. **CR-4** — activate shared context in `Graph` (extract a shared helper).
290
+ 5. **CR-5** — forward `on_step` through `map`/`async_map`/`run_async`/`map_async` (or narrow the docs).
291
+ 6. **CR-6** — reject specs that contain both `graph:` and `steps:`.
292
+ 7. **CR-7** — lock breaker state (or document non-thread-safety). Lowest priority.
@@ -0,0 +1,209 @@
1
+ # stepcraft — Claude Code Handoff Plan
2
+
3
+ **Author:** Senior review / planning pass
4
+ **Date:** 2026-06-24
5
+ **Sub-agent:** claude-code (available after quota reset ~12:50 AM Asia/Calcutta)
6
+ **Branch:** `working_barnch`
7
+
8
+ This file is the **source of truth** for what is done, what was started tonight, and what claude-code should build next. Read this before any feature work.
9
+
10
+ ---
11
+
12
+ ## Already shipped (do not redo)
13
+
14
+ | Area | Status | Key commits / files |
15
+ |------|--------|---------------------|
16
+ | Async `batch_size` awaits coroutines | Done | `step._execute_batched_async` |
17
+ | Async `parallel` awaits coroutines | Done | `step._execute_parallel_async` |
18
+ | `_async_run_branch_value` awaits `run()` coroutines | Done | `utils.py` |
19
+ | List auto-map on default `@piped` | Done | `step._should_auto_map`, lists only (not tuples) |
20
+ | `Pipeline.from_spec` YAML/JSON | Done | `spec.py`, `pip install stepcraft[spec]` |
21
+ | FanOut / MapReduce async parity | Done | `fan.py` |
22
+ | jit/vectorize warnings | Done | `decorators.py` |
23
+ | rsloop CI job | Done | `.github/workflows/ci.yml` `test-rsloop` |
24
+ | Free-threaded 3.14 parallel | Done | `runtime.py`, `parallel='auto'` |
25
+
26
+ **Tests:** 114+ passed locally. Run `uv run pytest tests/ -v` before every PR.
27
+
28
+ ---
29
+
30
+ ## Started tonight (manager pass — verify / extend)
31
+
32
+ Claude-code should **read the diff** and finish tests/docs if incomplete:
33
+
34
+ ### A. `@piped(auto_map=False)` opt-out
35
+ - **Why:** Steps that intentionally receive a whole list must not be split.
36
+ - **Files:** `decorators.py`, `step.py`, `tests/test_pipeline.py`
37
+ - **Acceptance:**
38
+ - `@piped(auto_map=False)` passes `[1,2,3]` as one argument
39
+ - Default remains auto-map on multi-item lists
40
+ - `__call__` / partial binding preserves `auto_map`
41
+
42
+ ### B. Immutable `retry` / `circuit_breaker`
43
+ - **Why:** Decorating the same `PipeStep` twice or reusing across pipelines must not share breaker counters.
44
+ - **Files:** `step.py` (`PipeStep.copy()`), `decorators.py`
45
+ - **Acceptance:**
46
+ - `@retry` / `@circuit_breaker` return a **new** `PipeStep` via `copy()`
47
+ - Fresh `_circuit_state`, `_failure_count`, `_last_failure_time`
48
+ - Existing tests still pass; add test that two decorated copies do not share breaker state
49
+
50
+ ### C. `Pipeline.async_run_detailed()`
51
+ - **Why:** Parity with `run_detailed()` for async pipelines.
52
+ - **Files:** `pipeline.py`, `tests/test_pipeline.py`
53
+ - **Acceptance:**
54
+ - Returns `ExecutionResult` with history, `dt`, `n`
55
+ - Works with async `@piped` steps and sync steps (executor fallback in `async_run`)
56
+
57
+ ---
58
+
59
+ ## Phase 1 — Claude-code (high priority, ~1 session)
60
+
61
+ ### 1. Step hooks / observability
62
+ **Goal:** Callback after each step without wrapping functions.
63
+
64
+ ```python
65
+ # API sketch (finalize in hooks.py)
66
+ StepHook = Callable[[str, Any, Any, float], None] # name, input, output, step_dt
67
+
68
+ pipeline.run(seed, on_step=hook)
69
+ pipeline.run_detailed(seed, on_step=hook) # hook in addition to history
70
+ await pipeline.async_run(seed, on_step=hook)
71
+ await pipeline.async_run_detailed(seed, on_step=hook)
72
+ ```
73
+
74
+ **Files to add/touch:**
75
+ - `src/stepcraft/hooks.py` — `StepHook` type alias, `_call_hook` helper
76
+ - `pipeline.py` — optional `on_step` on run paths
77
+ - `graph.py` — optional `on_step` on `run` / `async_run` (per node)
78
+ - `tests/test_pipeline.py` — hook called N times, receives correct I/O
79
+
80
+ **Rules:**
81
+ - Hook errors must **not** swallow pipeline errors (wrap in try/except log or re-raise based on strict flag — default: log warning, continue)
82
+ - `on_step` is optional; no breaking changes to existing signatures (use keyword-only)
83
+
84
+ ### 2. Update `AUDIT.md` + `README.md`
85
+ - Mark fixed bugs incomplete features as resolved
86
+ - Document: `auto_map`, `parallel='auto'`, `from_spec`, free-threading, `pip install stepcraft[spec]`
87
+ - Add short “Parallelism” section: GIL vs 3.14t vs `process`
88
+
89
+ ### 3. `@piped(map=False)` alias
90
+ - If manager used `auto_map`, add `map: Optional[bool] = None` where `map=False` ≡ `auto_map=False` for README-friendly naming (single implementation).
91
+
92
+ ---
93
+
94
+ ## Phase 2 — Claude-code (production hardening)
95
+
96
+ ### 4. Circuit breaker single-probe half-open
97
+ **Location:** `step.py` `_check_circuit_breaker`, `_record_failure`, `_reset_circuit_breaker`
98
+
99
+ **Behavior:**
100
+ - `HALF_OPEN`: allow **one** trial call; success → `CLOSED`, failure → `OPEN`
101
+ - Add `CircuitBreakerConfig.half_open_max_calls: int = 1` (optional, default 1)
102
+ - Tests: `test_circuit_breaker_half_open_single_probe`
103
+
104
+ ### 5. Configurable thread/process pools
105
+ **Location:** `pools.py`
106
+
107
+ ```python
108
+ def configure_pools(*, thread_workers: int | None = None, process_workers: int | None = None) -> None
109
+ ```
110
+
111
+ - Apply on next `_get_pool` creation; document that reconfigure requires `cleanup_pools()` first
112
+ - Test: pool respects max_workers when set
113
+
114
+ ### 6. Sync timeout cancellation note + optional flag
115
+ - Document that timed-out sync work keeps running in thread pool (AUDIT footgun)
116
+ - Optional: `cancel_on_timeout=True` on `@piped` — best-effort, document limitations
117
+ - **Low priority** if complex; documenting is enough for Phase 2
118
+
119
+ ---
120
+
121
+ ## Phase 3 — Claude-code (larger features)
122
+
123
+ ### 7. Pipeline shared context
124
+ ```python
125
+ Pipeline(steps, context={"request_id": "abc"})
126
+ # Steps access via contextvar or optional kw PIPE_CONTEXT
127
+ ```
128
+
129
+ - Use `contextvars.ContextVar[dict]` set at pipeline run entry
130
+ - `Node.setup()` can read context
131
+ - Do **not** break existing step signatures
132
+
133
+ ### 8. `from_spec` graph support
134
+ Extend `spec.py`:
135
+ ```yaml
136
+ graph:
137
+ nodes:
138
+ a: { import: "mymodule:fetch" }
139
+ b: { import: "mymodule:parse" }
140
+ edges:
141
+ - [a, b]
142
+ seed_node: a
143
+ ```
144
+
145
+ - `Graph.from_spec(path)` or `Pipeline.from_spec` detects `graph:` key
146
+ - Tests with fixture YAML
147
+
148
+ ### 9. CI: `python3.14t` job
149
+ - Ubuntu job installing free-threaded CPython when available on setup-python
150
+ - Run `tests/test_pipeline.py -k "free_threading or parallel_auto"`
151
+ - Skip gracefully if 3.14t not on runner
152
+
153
+ ---
154
+
155
+ ## Phase 4 — Defer unless user asks
156
+
157
+ - Streaming / lazy iterator pipelines
158
+ - Checkpoint / resume for graphs
159
+ - Pydantic schema validation (optional dep)
160
+ - OpenTelemetry exporter (build on hooks)
161
+
162
+ ---
163
+
164
+ ## Conventions for claude-code
165
+
166
+ 1. **Minimal diff** — match existing style in `step.py`, `decorators.py`
167
+ 2. **Tests required** for every behavior change
168
+ 3. **No new required dependencies** — optional extras only (`spec`, `rsloop`, etc.)
169
+ 4. **Do not commit** unless user asks (manager may commit manager pass)
170
+ 5. **Run before handoff:** `uv run pytest tests/ -q`
171
+ 6. **Import surface:** export new public APIs from `__init__.py` + `pipeline` shim if tests import from `pipeline`
172
+
173
+ ---
174
+
175
+ ## Suggested claude-code prompt (copy-paste)
176
+
177
+ ```
178
+ Read CLAUDE_CODE_HANDOFF.md fully. You are implementing stepcraft on branch working_barnch.
179
+
180
+ 1. Verify items in "Started tonight" (A–C) — add missing tests, fix gaps.
181
+ 2. Implement Phase 1 items 1–3 (hooks, docs, map= alias).
182
+ 3. Run full test suite; fix failures.
183
+ 4. Summarize what changed and what remains for Phase 2.
184
+
185
+ Do not commit unless I ask. Follow existing code conventions.
186
+ ```
187
+
188
+ ---
189
+
190
+ ## Manager pass checklist
191
+
192
+ - [x] Write this handoff file
193
+ - [x] `auto_map=False` implementation
194
+ - [x] `PipeStep.copy()` + immutable decorators
195
+ - [x] `async_run_detailed()`
196
+ - [x] Tests for above
197
+ - [x] Commit manager pass (if user approves) — `3c7e1f4`, pushed
198
+
199
+ ---
200
+
201
+ ## Scheduled trigger (12:51 AM IST)
202
+
203
+ - **Set:** 2026-06-24 ~21:38 IST
204
+ - **Fires:** 2026-06-25 00:51:00 Asia/Kolkata
205
+ - **Script:** `scripts/trigger_claude_code_at_1251.sh`
206
+ - **Wake sentinel:** `AGENT_LOOP_WAKE_CLAUDE_CODE`
207
+ - **Remote:** `origin/working_barnch` @ `3c7e1f4`
208
+
209
+ To cancel: kill the background sleeper PID shown in terminal metadata.