skillflow-py 1.3.0__tar.gz → 1.3.2__tar.gz

{skillflow_py-1.3.0 → skillflow_py-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skillflow-py
-Version: 1.3.0
+Version: 1.3.2
 Summary: Config-agnostic LLM pipeline graph executor — turn skills into flows
 Author: Skillflow contributors
 License-Expression: MIT

{skillflow_py-1.3.0 → skillflow_py-1.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "skillflow-py"
-version = "1.3.0"
+version = "1.3.2"
 description = "Config-agnostic LLM pipeline graph executor — turn skills into flows"
 readme = "README.md"
 license = "MIT"

{skillflow_py-1.3.0 → skillflow_py-1.3.2}/src/skillflow/core.py

@@ -932,14 +932,14 @@ class SkillFlow:
                 tmp_dir = self._workspace.get_step_tmp_dir(
                     run["project_id"], run["graph_name"], node.id
                 )
-                # Reset staging at the start of each attempt so create/edit/write
-                # rebuild from a clean baseline (the repo). Without this, a partial
-                # result from a failed attempt pollutes the next one — e.g. a
-                # re-issued edit(old→new) would fail because 'old' was already
-                # replaced last attempt. Keeps step execution idempotent on retry.
-                import shutil
-                if tmp_dir.exists():
-                    shutil.rmtree(str(tmp_dir))
+                # Staging PERSISTS across retries (do not wipe). A retry inherits
+                # the prior attempt's prompt (KV-cache reuse), so the agent issues
+                # follow-up edits against the state it already produced — staging
+                # must match that accumulated state. Wiping would both lose files
+                # a prior attempt created and break follow-up edit() calls
+                # (old_str reverted to the repo baseline → "not found"). A
+                # successful step consumes tmp via promotion (tmp→step_dir rename),
+                # so the next step still starts clean without an explicit wipe.
                 tmp_dir.mkdir(parents=True, exist_ok=True)
                 inputs_with_tools["_output_dir"] = str(tmp_dir)
                 if node.output_fixed:
@@ -2216,6 +2216,74 @@ class SkillFlow:
             )
             return None
 
+    def _claim_tool_step_in_tx(self, run_id: str, step_id: str, node) -> int | None:
+        """Atomically claim a tool step (pending→claimed) for inline execution.
+
+        Returns the claimed step-instance id, or None if the step is not
+        claimable — i.e. a concurrent advance_run() already claimed it. Mirrors
+        claim_next_step's CAS so the in-flight guard + runaway valve (which key
+        on the 'claimed' status/trace) see inline tool steps too, making
+        execution idempotent under concurrent drivers sharing this DB.
+        """
+        with self._tx() as conn:
+            row = conn.execute(
+                "SELECT version FROM skillflow_steps "
+                "WHERE run_id = ? AND step_id = ? AND status = 'pending'",
+                (run_id, step_id),
+            ).fetchone()
+            if not row:
+                existing = conn.execute(
+                    "SELECT status FROM skillflow_steps "
+                    "WHERE run_id = ? AND step_id = ? ORDER BY id DESC LIMIT 1",
+                    (run_id, step_id),
+                ).fetchone()
+                if existing and existing["status"] == "claimed":
+                    return None  # a concurrent driver owns it
+                # First run with no row, or cyclic re-entry (prev completed/
+                # failed): open a fresh pending instance to claim.
+                conn.execute(
+                    "INSERT INTO skillflow_steps (run_id, step_id, "
+                    "step_config_json, max_retries, status, created_at, updated_at) "
+                    "VALUES (?, ?, ?, ?, 'pending', datetime('now'), datetime('now'))",
+                    (run_id, step_id, self._serialize(node.config), node.max_retries),
+                )
+                row = conn.execute(
+                    "SELECT version FROM skillflow_steps "
+                    "WHERE run_id = ? AND step_id = ? AND status = 'pending'",
+                    (run_id, step_id),
+                ).fetchone()
+                if not row:
+                    return None
+            ver = row["version"]
+            claimed_at_str = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time()))
+            cur = conn.execute(
+                "UPDATE skillflow_steps SET status = 'claimed', version = version + 1, "
+                "claimed_at = ?, claimed_by = ?, updated_at = datetime('now') "
+                "WHERE run_id = ? AND step_id = ? AND version = ? AND status = 'pending'",
+                (claimed_at_str, "tool-inline", run_id, step_id, ver),
+            )
+            if cur.rowcount == 0:
+                return None  # lost the race
+            inst = conn.execute(
+                "SELECT id FROM skillflow_steps "
+                "WHERE run_id = ? AND step_id = ? AND status = 'claimed' "
+                "ORDER BY id DESC LIMIT 1",
+                (run_id, step_id),
+            ).fetchone()
+            return inst["id"] if inst else None
+
+    def _reopen_tool_step_in_tx(self, run_id: str, step_id: str) -> None:
+        """Release a claimed tool step back to pending after a crashed execution,
+        so it is retried promptly instead of stalling until its claim times out.
+        """
+        with self._tx() as conn:
+            conn.execute(
+                "UPDATE skillflow_steps SET status = 'pending', claimed_at = NULL, "
+                "claimed_by = NULL, version = version + 1, updated_at = datetime('now') "
+                "WHERE run_id = ? AND step_id = ? AND status = 'claimed'",
+                (run_id, step_id),
+            )
+
     def advance_run(self, run_id: str) -> str | None:
         # Recover stale claims before any traversal
         self.recover_stale_claims(self._stale_threshold)
@@ -2223,23 +2291,19 @@ class SkillFlow:
         resolver = self._get_resolver_for_run(run_id)
 
         # ── Tool fast-path: execute OUTSIDE any write transaction ──
-        # Long-running tools (e.g. run_tests at 10-15 s) must not hold
-        # the SQLite write lock — that blocks agent trace writes and
-        # other scheduler ticks, causing SQLITE_BUSY → crashed agents →
-        # stale claims → infinite loops.
+        # Long-running tools (e.g. run_tests) must not hold the SQLite write
+        # lock — that blocks agent trace writes and other scheduler ticks,
+        # causing SQLITE_BUSY → crashed agents → stale claims → infinite loops.
         #
-        # IDEMPOTENCY NOTE: this inline tool execution is NOT claim-guarded —
-        # two overlapping advance_run() calls would both run the tool. That is
-        # currently safe ONLY because advance_run is synchronous and the host
-        # scheduler drives it on a single event loop in a single process (the
-        # P0-1 advisory lock), so calls for one run can never overlap. The tool
-        # runs synchronously here and blocks that loop for its whole duration.
-        # If a host ever makes tool execution non-blocking (await to_thread /
-        # run_in_executor) so the loop stays responsive during a slow tool,
-        # advance_run gains a yield point mid-tool and concurrent ticks CAN then
-        # overlap → this path must become claim-guarded (CAS pending→claimed
-        # before executing; only the winner runs the tool). Land the two
-        # changes together, never one without the other.
+        # CLAIM-GUARDED (1.3.2): a CAS pending→claimed precedes execution, so
+        # only one advance_run() call runs the tool. This is REQUIRED for
+        # correctness when more than one driver advances the same run — e.g. a
+        # host CLI + a Docker container sharing this DB, or the wake-on-confirm
+        # and interval jobs overlapping. Without it, an unclaimed slow tool gets
+        # re-launched by every tick → dozens of concurrent run_tests pile up and
+        # mutually starve (the step-5 rampage). Losers return None and back off;
+        # a crashed tool reopens to pending; a dead driver's claim is reclaimed
+        # via recover_stale_claims once the node's timeout_seconds elapses.
         run_row = self._conn.execute(
             "SELECT * FROM skillflow_runs WHERE id = ?", (run_id,)
         ).fetchone()
@@ -2250,10 +2314,22 @@ class SkillFlow:
                 tool_node = resolver.get_node(current)
                 if tool_node and not self._should_delegate_tool(
                         tool_node.tool_name):
-                    # Execute tool WITHOUT holding any lock/transaction
-                    tool_result = self._execute_tool_inline(
-                        tool_node, run_id=run_id,
-                        graph_name=run_row["graph_name"])
+                    inst_id = self._claim_tool_step_in_tx(
+                        run_id, current, tool_node)
+                    if inst_id is None:
+                        return None  # another driver owns this tool step
+                    self.trace(run_id, "step", "claimed",
+                               {"tool": tool_node.tool_name, "inline": True},
+                               step_id=current, step_instance_id=inst_id)
+                    try:
+                        # Execute tool WITHOUT holding any lock/transaction
+                        tool_result = self._execute_tool_inline(
+                            tool_node, run_id=run_id,
+                            graph_name=run_row["graph_name"])
+                    except Exception:
+                        # Don't leave a crashed tool wedged in 'claimed'.
+                        self._reopen_tool_step_in_tx(run_id, current)
+                        raise
                     return self._complete_tool_step(
                         run_id, current, tool_result, run_row, resolver)
 

{skillflow_py-1.3.0 → skillflow_py-1.3.2}/src/skillflow/tools/repo_apply/impl.py

@@ -2,9 +2,18 @@
 
 import shutil
 import subprocess
+import sys
 from pathlib import Path
 
 
+def _probe(msg: str) -> None:
+    """Diagnostic log for the step-dir-contains-whole-repo investigation.
+
+    Prints to stderr with a greppable marker so it shows in `docker logs`.
+    """
+    print(f"[repo_apply.probe] {msg}", file=sys.stderr, flush=True)
+
+
 def repo_apply(source_dir: str, *, workspace_root: str = "",
                project_root: str = "",
                step_id: str = "", project_id: str = "",
@@ -15,6 +24,12 @@ def repo_apply(source_dir: str, *, workspace_root: str = "",
     src = src.resolve()
     dst = Path(project_root).resolve()
 
+    # PROBE: what is repo_apply actually reading from, and is it the repo itself?
+    _probe(f"step={step_id} project={project_id} task={task_name!r} "
+           f"source_dir_param={source_dir!r} resolved_src={src} "
+           f"project_root={dst} workspace_root={workspace_root!r} "
+           f"src==dst? {src == dst}")
+
     if not src.exists():
         return {"applied": False, "files": [],
                 "error": f"Source dir not found: {source_dir}"}
@@ -33,6 +48,14 @@ def repo_apply(source_dir: str, *, workspace_root: str = "",
         shutil.copy2(f, target)
         applied_files.append(str(rel))
 
+    # PROBE: how many files did we just apply? A handful = normal surgical edit;
+    # dozens/hundreds = the leak. Dump a sample so we can see WHAT leaked.
+    if len(applied_files) > 15:
+        _probe(f"LARGE APPLY step={step_id}: {len(applied_files)} files from "
+               f"{src} — first 15: {applied_files[:15]}")
+    else:
+        _probe(f"step={step_id}: applied {len(applied_files)} file(s): {applied_files}")
+
     if not applied_files:
         return {"applied": False, "files": [],
                 "error": "No files to apply (empty or all filtered)"}

{skillflow_py-1.3.0 → skillflow_py-1.3.2}/src/skillflow/write_tools.py

@@ -312,6 +312,26 @@ def execute_append(slot: str, fixed: dict, params: dict,
     return {"written": base_name}
 
 
+def _unique_replace(content: str, old_str: str, new_str: str, *,
+                    tool: str, name: str) -> "tuple[str | None, dict | None]":
+    """Replace the single exact occurrence of ``old_str`` in ``content``.
+
+    Shared by the slot-mode (``edit_{slot}``) and generic (``edit``) executors so
+    the uniqueness-and-replace rule — the safety-critical heart of a surgical
+    edit — lives in exactly one place. Returns ``(updated_content, None)`` on
+    success, or ``(None, error_dict)`` when ``old_str`` is absent or not unique.
+    ``tool`` and ``name`` only shape the error message (caller's tool label and
+    the file path).
+    """
+    occurrences = content.count(old_str)
+    if occurrences == 0:
+        return None, {"error": f"{tool}: 'old_str' not found in '{name}'"}
+    if occurrences > 1:
+        return None, {"error": (f"{tool}: 'old_str' matches {occurrences} times in "
+                                f"'{name}' — include more surrounding context to make it unique")}
+    return content.replace(old_str, new_str, 1), None
+
+
 def execute_edit(slot: str, fixed: dict, params: dict,
                  output_dir: str, source_dir: str = "") -> dict:
     """Execute an edit_{slot} call: surgical str-replace on the EXISTING file.
@@ -339,14 +359,10 @@ def execute_edit(slot: str, fixed: dict, params: dict,
             return {"error": f"edit_{slot}: cannot edit '{base_name}' — file does not exist"}
 
     content = src.read_text(encoding="utf-8")
-    occurrences = content.count(old_str)
-    if occurrences == 0:
-        return {"error": f"edit_{slot}: 'old_str' not found in '{base_name}'"}
-    if occurrences > 1:
-        return {"error": (f"edit_{slot}: 'old_str' matches {occurrences} times in "
-                          f"'{base_name}' — include more surrounding context to make it unique")}
-
-    updated = content.replace(old_str, new_str, 1)
+    updated, err = _unique_replace(content, old_str, new_str,
+                                   tool=f"edit_{slot}", name=base_name)
+    if err:
+        return err
     out = Path(output_dir) / base_name
     out.parent.mkdir(parents=True, exist_ok=True)
     out.write_text(updated, encoding="utf-8")
@@ -437,14 +453,10 @@ def execute_generic_edit(params: dict, output_dir: str,
                           f"(use 'create' for a new file).")}
 
     content = src.read_text(encoding="utf-8")
-    occurrences = content.count(old_str)
-    if occurrences == 0:
-        return {"error": f"edit: 'old_str' not found in '{rel}'"}
-    if occurrences > 1:
-        return {"error": (f"edit: 'old_str' matches {occurrences} times in '{rel}' "
-                          f"— include more surrounding context to make it unique")}
-
-    updated = content.replace(old_str, new_str, 1)
+    updated, err = _unique_replace(content, old_str, new_str,
+                                   tool="edit", name=rel)
+    if err:
+        return err
     staged.parent.mkdir(parents=True, exist_ok=True)
     staged.write_text(updated, encoding="utf-8")
     return {"edited": rel}

{skillflow_py-1.3.0 → skillflow_py-1.3.2}/src/skillflow_py.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skillflow-py
-Version: 1.3.0
+Version: 1.3.2
 Summary: Config-agnostic LLM pipeline graph executor — turn skills into flows
 Author: Skillflow contributors
 License-Expression: MIT

{skillflow_py-1.3.0 → skillflow_py-1.3.2}/tests/test_tool_lock_contention.py

@@ -110,3 +110,76 @@ def test_inline_tool_does_not_hold_serialisation_lock(tmp_path):
         "inline tool executed while advance_run held the serialisation lock "
         f"(self._lock) — blocks concurrent ticks: observed={observed}"
     )
+
+
+def test_inline_tool_is_claim_guarded_against_concurrent_drivers(tmp_path):
+    """Two advance_run() calls overlapping on one tool step run it ONCE.
+
+    Root cause of the step-5 rampage (2026-06-28): the inline tool fast-path was
+    not claim-guarded, so when two drivers shared the DB (a host CLI + a Docker
+    container), each tick re-launched the still-running run_tests → dozens of
+    concurrent pytest piling up. The CAS pending->claimed makes only the winner
+    execute; the loser backs off (returns None).
+    """
+    db_path = str(tmp_path / "claimguard.db")
+    runs: list[int] = []
+    in_tool = threading.Event()
+    release = threading.Event()
+
+    def slow_probe(**kwargs):
+        runs.append(1)
+        in_tool.set()             # we hold the claim now
+        release.wait(timeout=5)   # ...while a second driver also advances
+        return {"passed": True}
+
+    tools = MockToolLoader()
+    tools.register("slow_probe", slow_probe)
+    sf = SkillFlow(db_path, tool_loader=tools)
+    sf.register_agent_config("noop_agent")
+
+    graph = PipelineGraph(
+        name="claimguard",
+        begin="gen",
+        steps=[
+            StepNode(id="gen", step_type="agent", agent_config="noop_agent",
+                     transitions=[Transition(to="probe")]),
+            StepNode(id="probe", step_type="tool", tool_name="slow_probe",
+                     transitions=[Transition(to="done")]),
+            StepNode(id="done", step_type="agent", agent_config="noop_agent",
+                     transitions=[]),
+        ],
+        end_conditions=EndConditions(
+            combinator="or",
+            conditions=[EndCondition(type="node_reached", node="done",
+                                     result="completed")],
+        ),
+    )
+    sf.register_graph(graph)
+    run_id = sf.create_run("claimguard")
+    sf.start_run(run_id)
+
+    sf.advance_run(run_id)                       # resolve -> "gen"
+    claimed = sf.claim_next_step(run_id)
+    sf.confirm_step(claimed.token, StepResult(outputs={}, flags={}))
+
+    # Resolve gen -> probe. The tool is DEFERRED (current_node=probe) and only
+    # executed by the fast-path on a SUBSEQUENT advance_run — stop the moment we
+    # reach that deferred state, before the tool runs.
+    for _ in range(5):
+        sf.advance_run(run_id)
+        if sf.get_run(run_id)["current_node"] == "probe":
+            break
+    assert sf.get_run(run_id)["current_node"] == "probe"
+    assert runs == [], "tool ran during deferral — expected execution only via fast-path"
+
+    # Now two drivers race the fast-path. A claims + runs it (holding the claim
+    # on the release event); B advances concurrently and must back off.
+    a = threading.Thread(target=lambda: sf.advance_run(run_id))
+    a.start()
+    assert in_tool.wait(timeout=5), "tool never started"
+
+    sf.advance_run(run_id)        # driver B: must see the claim and back off
+    release.set()
+    a.join(timeout=5)
+
+    assert runs == [1], f"tool executed {len(runs)}x under concurrency, expected 1"