skillflow-py 1.3.2__tar.gz → 1.4.1__tar.gz

{skillflow_py-1.3.2 → skillflow_py-1.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skillflow-py
-Version: 1.3.2
+Version: 1.4.1
 Summary: Config-agnostic LLM pipeline graph executor — turn skills into flows
 Author: Skillflow contributors
 License-Expression: MIT

{skillflow_py-1.3.2 → skillflow_py-1.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "skillflow-py"
-version = "1.3.2"
+version = "1.4.1"
 description = "Config-agnostic LLM pipeline graph executor — turn skills into flows"
 readme = "README.md"
 license = "MIT"

{skillflow_py-1.3.2 → skillflow_py-1.4.1}/src/skillflow/core.py

@@ -1057,7 +1057,11 @@ class SkillFlow:
 
                 if not hook_result.get("passed", False):
                     error = hook_result.get("error", f"Lifecycle hook '{hook_name}' failed")
-                    on_failure = hook_spec.get("on_failure", "fail") if isinstance(hook_spec, dict) else "fail"
+                    # A tool-hook sequence bubbles the failing item's on_failure
+                    # in the result; fall back to the spec-level value otherwise.
+                    on_failure = hook_result.get("on_failure") or (
+                        hook_spec.get("on_failure", "fail")
+                        if isinstance(hook_spec, dict) else "fail")
                     if on_failure == "retry":
                         self._emit_lifecycle_event(token, hook_name, "retry", error)
                         self._handle_lifecycle_retry(token, error)
@@ -1255,19 +1259,64 @@ class SkillFlow:
 
         hook_spec can be:
         - A dict with 'tool' (single tool call): used for after_validate, on_deliver
-        - A list of dicts (multi-check): used for after_deliver
+        - A list of validation specs (multi-check): used for after_deliver
+        - A list of {'tool', 'params'} dicts (sequential tool hooks): used for
+          on_deliver when several repo-mutating tools must run in order (e.g.
+          repo_apply then repo_delete). Each runs via _execute_tool_hook, so
+          $STEP_DIR is resolved and project_root injected — unlike check specs,
+          which run through StepValidator and receive neither.
 
         Returns {passed: bool, error?: str}.
         """
         self._emit_lifecycle_event(token, hook_name, "started")
 
         if isinstance(hook_spec, list):
-            return self._execute_check_hook(token, node, hook_name, hook_spec)
+            # after_deliver runs validation checks against the project repo;
+            # other slots (on_deliver) run a sequence of tool hooks with full
+            # variable resolution.
+            if hook_name == "after_deliver":
+                return self._execute_check_hook(token, node, hook_name, hook_spec)
+            return self._execute_tool_hook_sequence(token, node, hook_name, hook_spec)
         elif isinstance(hook_spec, dict) and "tool" in hook_spec:
             return self._execute_tool_hook(token, node, hook_name, hook_spec)
         else:
             return {"passed": False, "error": f"Invalid hook spec for '{hook_name}'"}
 
+    def _execute_tool_hook_sequence(self, token: ClaimToken, node: StepNode,
+                                     hook_name: str, items: list) -> dict:
+        """Run a list of tool hooks in order (e.g. repo_apply then repo_delete).
+
+        Each item is a ``{'tool', 'params', 'on_failure'?}`` dict executed via
+        :meth:`_execute_tool_hook` (full variable resolution + project_root
+        injection). Per-item ``on_failure`` is honored: ``warn``/``skip`` log and
+        continue to the next item; ``retry``/``fail`` stop the sequence and
+        bubble up (carrying the item's ``on_failure``) so the caller applies the
+        right step-level outcome.
+        """
+        detail_files = None
+        for item in items:
+            if not (isinstance(item, dict) and "tool" in item):
+                return {"passed": False,
+                        "error": f"Invalid tool hook in '{hook_name}': {item!r}"}
+            res = self._execute_tool_hook(token, node, hook_name, item)
+            if not res.get("passed", True):
+                on_failure = item.get("on_failure", "fail")
+                if on_failure in ("warn", "skip"):
+                    self._emit_lifecycle_event(
+                        token, hook_name,
+                        "warned" if on_failure == "warn" else "skipped",
+                        res.get("error", ""))
+                    continue
+                # retry / fail: stop and let the caller apply the step outcome.
+                return {**res, "on_failure": on_failure}
+            files = res.get("files")
+            if isinstance(files, list):
+                detail_files = files
+        out = {"passed": True}
+        if detail_files is not None:
+            out["files"] = detail_files
+        return out
+
     def _execute_tool_hook(self, token: ClaimToken, node: StepNode,
                             hook_name: str, hook_spec: dict) -> dict:
         """Execute a tool-type lifecycle hook (single tool call)."""

{skillflow_py-1.3.2 → skillflow_py-1.4.1}/src/skillflow/read_tools.py

@@ -27,6 +27,21 @@ from pathlib import Path
 # the ``truncated`` flag signals when there is more to page through.
 _MAX_READ_LINES = 2000
 
+# Directories never worth enumerating/searching as "source" — VCS internals and
+# build/dependency caches. Mirrors dir_tree/list_tree's BLOCKED set, which the
+# generated directory read tools previously lacked: a single list_repo_root on a
+# real repo dumped the whole .git tree (~64% of files) into the agent's context.
+_BLOCKED_DIR_PARTS = {".git", "__pycache__", ".venv", "node_modules",
+                      ".mypy_cache", ".pytest_cache", ".ruff_cache"}
+# Hard cap on entries returned by a directory listing so one call can't flood
+# the context with thousands of paths.
+_MAX_LIST_ENTRIES = 1000
+
+
+def _is_blocked_path(rel) -> bool:
+    """True if any path component is a blocked VCS/build directory."""
+    return any(part in _BLOCKED_DIR_PARTS for part in Path(rel).parts)
+
 
 def _page_lines(text: str, start_line: int = 0, end_line: int | None = None) -> dict:
     """Slice file text into a line-numbered window with paging metadata.
@@ -407,19 +422,25 @@ def make_read_tool_fns(specs: list[dict], workspace_root: str,
 
             def _list_dir(_paths=dir_paths) -> str:
                 entries = []
+                truncated = False
                 for dp in _paths:
                     d = Path(dp)
                     if not d.is_dir():
                         continue
                     for f in sorted(d.rglob("*")):
-                        if f.is_file() and f.name != ".gitkeep":
-                            rel = f.relative_to(d)
-                            st = f.stat()
-                            entries.append({
-                                "name": str(rel),
-                                "size": st.st_size,
-                            })
-                return json.dumps(entries, ensure_ascii=False)
+                        if not (f.is_file() and f.name != ".gitkeep"):
+                            continue
+                        rel = f.relative_to(d)
+                        if _is_blocked_path(rel):
+                            continue  # skip .git / build / dependency caches
+                        entries.append({"name": str(rel), "size": f.stat().st_size})
+                        if len(entries) >= _MAX_LIST_ENTRIES:
+                            truncated = True
+                            break
+                    if truncated:
+                        break
+                return json.dumps({"files": entries, "truncated": truncated},
+                                  ensure_ascii=False)
 
             def _read_dir_file(name: str, start_line: int = 0,
                                end_line: int | None = None, _paths=dir_paths) -> dict:
@@ -465,6 +486,8 @@ def make_read_tool_fns(specs: list[dict], workspace_root: str,
                     for f in sorted(d.rglob(glob) if glob else d.rglob("*")):
                         if not f.is_file() or f.name == ".gitkeep":
                             continue
+                        if _is_blocked_path(f.relative_to(d)):
+                            continue  # skip .git / build / dependency caches
                         # Skip binary-looking files
                         if f.suffix in (".pyc", ".pyo", ".so", ".o", ".bin"):
                             continue

{skillflow_py-1.3.2 → skillflow_py-1.4.1}/src/skillflow/tools/repo_apply/impl.py

@@ -1,39 +1,31 @@
 """Apply files from draft directory to project repo."""
 
+import fnmatch
 import shutil
 import subprocess
-import sys
 from pathlib import Path
 
 
-def _probe(msg: str) -> None:
-    """Diagnostic log for the step-dir-contains-whole-repo investigation.
-
-    Prints to stderr with a greppable marker so it shows in `docker logs`.
-    """
-    print(f"[repo_apply.probe] {msg}", file=sys.stderr, flush=True)
-
-
 def repo_apply(source_dir: str, *, workspace_root: str = "",
                project_root: str = "",
                step_id: str = "", project_id: str = "",
-               task_name: str = "") -> dict:
+               task_name: str = "", ignore=None) -> dict:
     src = Path(source_dir)
     if not src.is_absolute():
         src = Path(workspace_root) / source_dir
     src = src.resolve()
     dst = Path(project_root).resolve()
 
-    # PROBE: what is repo_apply actually reading from, and is it the repo itself?
-    _probe(f"step={step_id} project={project_id} task={task_name!r} "
-           f"source_dir_param={source_dir!r} resolved_src={src} "
-           f"project_root={dst} workspace_root={workspace_root!r} "
-           f"src==dst? {src == dst}")
-
     if not src.exists():
         return {"applied": False, "files": [],
                 "error": f"Source dir not found: {source_dir}"}
 
+    # Caller-supplied ignore globs (fnmatch), matched against BOTH the path
+    # relative to source_dir AND the basename — so "web/js/_x.json" and "_x.json"
+    # style patterns both work. Lets a host keep control/scratch files (e.g. a
+    # deletions manifest) in the step dir without them landing in the repo.
+    ignore_globs = tuple(ignore or ())
+
     applied_files = []
     for f in sorted(src.rglob("*")):
         if not f.is_file():
@@ -43,22 +35,21 @@ def repo_apply(source_dir: str, *, workspace_root: str = "",
         if ".git/" in str(f):
             continue
         rel = f.relative_to(src)
+        if ignore_globs and any(
+            fnmatch.fnmatch(str(rel), pat) or fnmatch.fnmatch(f.name, pat)
+            for pat in ignore_globs
+        ):
+            continue
         target = dst / rel
         target.parent.mkdir(parents=True, exist_ok=True)
         shutil.copy2(f, target)
         applied_files.append(str(rel))
 
-    # PROBE: how many files did we just apply? A handful = normal surgical edit;
-    # dozens/hundreds = the leak. Dump a sample so we can see WHAT leaked.
-    if len(applied_files) > 15:
-        _probe(f"LARGE APPLY step={step_id}: {len(applied_files)} files from "
-               f"{src} — first 15: {applied_files[:15]}")
-    else:
-        _probe(f"step={step_id}: applied {len(applied_files)} file(s): {applied_files}")
-
     if not applied_files:
-        return {"applied": False, "files": [],
-                "error": "No files to apply (empty or all filtered)"}
+        # Empty source = a legitimate no-op step (the agent determined no change
+        # was needed and produced no output). Nothing to copy or commit; report
+        # success so the on_deliver hook doesn't retry/fail a clean no-op.
+        return {"applied": True, "files": [], "committed": False}
 
     # git add + commit
     r = subprocess.run(["git", "add", "-A"], cwd=dst,

{skillflow_py-1.3.2 → skillflow_py-1.4.1}/src/skillflow/tools/repo_apply/tool.yaml

@@ -8,3 +8,7 @@ parameters:
     type: string
     description: "Source directory path (absolute or relative to workspace)"
     required: true
+  ignore:
+    type: array
+    description: "Optional list of fnmatch globs (matched against the path relative to source_dir and the basename). Matching files are NOT copied to the repo — use for host control/scratch files kept in the step dir (e.g. a deletions manifest)."
+    required: false

{skillflow_py-1.3.2 → skillflow_py-1.4.1}/src/skillflow_py.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skillflow-py
-Version: 1.3.2
+Version: 1.4.1
 Summary: Config-agnostic LLM pipeline graph executor — turn skills into flows
 Author: Skillflow contributors
 License-Expression: MIT

{skillflow_py-1.3.2 → skillflow_py-1.4.1}/src/skillflow_py.egg-info/SOURCES.txt

@@ -76,6 +76,7 @@ tests/test_graph.py
 tests/test_integration.py
 tests/test_integration_configs.py
 tests/test_migration_regression.py
+tests/test_noop_floor_fix.py
 tests/test_notifications.py
 tests/test_outbox.py
 tests/test_read_tools_exec.py

{skillflow_py-1.3.2 → skillflow_py-1.4.1}/tests/test_core.py

@@ -1142,6 +1142,111 @@ def test_lifecycle_events_in_outbox(sf: SkillFlow, tmp_path: Path):
     assert step["status"] == "completed"
 
 
+def test_on_deliver_list_runs_tools_end_to_end(sf: SkillFlow, tmp_path: Path):
+    """on_deliver as a LIST runs its tool hooks (not validation checks): a
+    repo_apply in list form resolves $STEP_DIR and commits into the repo.
+
+    Regression: a list previously routed to StepValidator (check hook), which
+    neither unwraps ``params`` nor resolves ``$STEP_DIR`` — so a repo-mutating
+    tool placed there silently no-op'd.
+    """
+    import subprocess
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
+    subprocess.run(["git", "config", "user.email", "t@t"], cwd=repo, check=True)
+    subprocess.run(["git", "config", "user.name", "t"], cwd=repo, check=True)
+
+    sf._tool_loader = ToolLoader(
+        Path(__file__).parent.parent / "src" / "skillflow" / "tools")
+    sf._workspace = WorkspaceManager(
+        str(tmp_path / "ws"), projects_base=str(tmp_path / "projects"),
+        code_path_resolver=lambda pid: str(repo),
+    )
+
+    node = StepNode(
+        id="s1", step_type="agent", output_mode="content",
+        output_fixed={"out": "main.py"},
+        lifecycle={"on_deliver": [
+            {"tool": "repo_apply", "params": {"source_dir": "$STEP_DIR"}},
+        ]},
+        transitions=[Transition(to=None)],
+    )
+    g = PipelineGraph(name="ondeliver_list", begin="s1", steps=[node])
+    sf.register_graph(g)
+
+    rid = sf.create_run("ondeliver_list", {"project_id": "p1"})
+    sf.start_run(rid)
+    sf.advance_run(rid)
+    token = sf.claim_next_step(rid)
+    tmp = sf._workspace.get_step_tmp_dir("p1", "ondeliver_list", "s1")
+    (tmp / "main.py").write_text("print('hi')\n")
+    sf.confirm_step(token.token, StepResult(outputs={}, flags={}))
+
+    # Landed AND committed → the list ran the tool hook with $STEP_DIR resolved
+    # (old code would have no-op'd via the check-hook path).
+    assert (repo / "main.py").read_text() == "print('hi')\n"
+    log = subprocess.run(["git", "log", "--oneline"], cwd=repo,
+                         capture_output=True, text=True).stdout
+    assert "step: s1" in log, f"no apply commit:\n{log}"
+
+
+def test_on_deliver_list_order_and_on_failure(sf: SkillFlow, tmp_path: Path,
+                                               monkeypatch):
+    """on_deliver list = ordered tool-hook sequence honoring per-item
+    on_failure; after_deliver list still routes to validation checks."""
+    sf._tool_loader = ToolLoader(
+        Path(__file__).parent.parent / "src" / "skillflow" / "tools")
+    sf._workspace = WorkspaceManager(str(tmp_path / "ws"),
+                                     projects_base=str(tmp_path / "projects"))
+    node = StepNode(
+        id="s1", step_type="agent", output_mode="content",
+        output_fixed={"out": "r.md"}, transitions=[Transition(to=None)],
+    )
+    g = PipelineGraph(name="seq", begin="s1", steps=[node])
+    sf.register_graph(g)
+    rid = sf.create_run("seq", {"project_id": "pid"})
+    sf.start_run(rid)
+    sf.advance_run(rid)
+    token = sf.claim_next_step(rid).token   # the ClaimToken inside the ClaimedStep
+
+    calls: list = []
+
+    def fake_tool_hook(tok, nd, hook_name, spec):
+        calls.append(spec["tool"])
+        if spec["tool"] == "boom":
+            return {"passed": False, "error": "kaboom"}
+        return {"passed": True, "files": ["x"]}
+    monkeypatch.setattr(sf, "_execute_tool_hook", fake_tool_hook)
+
+    # 1) ordering + all-pass → sequence passes, surfaces files detail
+    r = sf._execute_lifecycle_hook(token, node, "on_deliver",
+        [{"tool": "repo_apply"}, {"tool": "repo_delete"}])
+    assert calls == ["repo_apply", "repo_delete"]
+    assert r["passed"] is True and r.get("files") == ["x"]
+
+    # 2) failing item with on_failure=warn → continue; sequence still passes
+    calls.clear()
+    r = sf._execute_lifecycle_hook(token, node, "on_deliver",
+        [{"tool": "boom", "on_failure": "warn"}, {"tool": "repo_delete"}])
+    assert calls == ["boom", "repo_delete"]
+    assert r["passed"] is True
+
+    # 3) failing item with default on_failure=fail → stop + bubble on_failure
+    calls.clear()
+    r = sf._execute_lifecycle_hook(token, node, "on_deliver",
+        [{"tool": "boom"}, {"tool": "repo_delete"}])
+    assert calls == ["boom"]
+    assert r["passed"] is False and r["on_failure"] == "fail"
+
+    # 4) after_deliver list is unchanged: routes to checks, not the tool
+    #    sequence, so the faked tool-hook path is never touched.
+    calls.clear()
+    sf._execute_lifecycle_hook(token, node, "after_deliver",
+        [{"tool": "syntax_lint", "files": ["*.nomatch"]}])
+    assert calls == []
+
+
 # ── Durable run trace ─────────────────────────────────────────────────
 
 def test_trace_append_and_get(sf: SkillFlow):

skillflow_py-1.4.1/tests/test_noop_floor_fix.py

@@ -0,0 +1,65 @@
+"""Regression tests for the existing-repo "no-op floor" data-corruption bug.
+
+Two skillflow-side guarantees the host (AItelier) relies on after the fix:
+
+1. ``repo_apply`` on an EMPTY source dir is a no-op SUCCESS, not an error.
+   A legitimate "no change needed" step promotes empty staging; on_deliver
+   must not retry/fail it.
+
+2. The generated directory read tools (``list_<label>`` / ``search_<label>``)
+   exclude ``.git`` and build/dependency caches and cap their output, so a
+   single ``list_repo_root`` can't dump the whole ``.git`` tree (tens of
+   thousands of tokens) into the agent's context.
+"""
+
+import json
+import subprocess
+from pathlib import Path
+
+from skillflow.read_tools import _is_blocked_path, make_read_tool_fns
+from skillflow.tool_loader import ToolLoader
+
+_REAL_TOOLS = Path(__file__).parent.parent / "src" / "skillflow" / "tools"
+
+
+def test_repo_apply_empty_source_is_noop_success(tmp_path):
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
+
+    src = tmp_path / "empty_staging"
+    src.mkdir()  # no files
+
+    repo_apply = ToolLoader(_REAL_TOOLS).load_fn("repo_apply")
+    res = repo_apply(str(src), project_root=str(repo))
+
+    assert res["applied"] is True
+    assert res["files"] == []
+    assert res.get("committed") is False
+    assert "error" not in res
+
+
+def test_is_blocked_path_excludes_vcs_and_build_dirs():
+    assert _is_blocked_path(Path(".git/objects/ab/cd"))
+    assert _is_blocked_path("node_modules/foo/bar.js")
+    assert _is_blocked_path("pkg/__pycache__/x.pyc")
+    assert not _is_blocked_path("src/main.py")
+    assert not _is_blocked_path("docs/readme.md")
+
+
+def test_list_repo_tool_excludes_git_tree(tmp_path):
+    repo = tmp_path / "repo"
+    (repo / ".git" / "objects").mkdir(parents=True)
+    (repo / ".git" / "objects" / "deadbeef").write_text("binblob")
+    (repo / "src").mkdir()
+    (repo / "src" / "main.py").write_text("x = 1\n")
+
+    specs = [{"source_type": "repository", "mode": "tool"}]
+    fns = make_read_tool_fns(specs, str(tmp_path), code_root=str(repo))
+
+    list_fn = next(fn for name, fn in fns.items() if name.startswith("list_"))
+    payload = json.loads(list_fn())
+    names = [e["name"] for e in payload["files"]]
+
+    assert "src/main.py" in names
+    assert not any(".git" in n for n in names), names

{skillflow_py-1.3.2 → skillflow_py-1.4.1}/tests/test_tools.py

@@ -144,7 +144,67 @@ class TestRepoApply:
         result = repo_apply(source_dir=str(empty),
                             workspace_root=str(tmp_path),
                             project_root=str(proj))
-        assert result["applied"] is False
+        # Empty source is a legitimate no-op step (agent determined no change
+        # needed): report success with no files, not an error that would make
+        # the on_deliver hook retry/fail a clean no-op.
+        assert result["applied"] is True
+        assert result["files"] == []
+        assert result.get("committed") is False
+
+    def test_ignore_globs_not_copied(self, tmp_path):
+        """Files matching an `ignore` glob stay out of the repo (host control
+        files kept in the step dir, e.g. a deletions manifest)."""
+        proj = tmp_path / "project"
+        proj.mkdir()
+        subprocess.run(["git", "init", "-b", "main"], cwd=proj,
+                       capture_output=True)
+        subprocess.run(["git", "config", "user.email", "test@test.com"],
+                       cwd=proj, capture_output=True)
+        subprocess.run(["git", "config", "user.name", "Test"], cwd=proj,
+                       capture_output=True)
+        (proj / ".gitkeep").write_text("")
+        subprocess.run(["git", "add", "."], cwd=proj, capture_output=True)
+        subprocess.run(["git", "commit", "-m", "init"], cwd=proj,
+                       capture_output=True)
+
+        draft = tmp_path / "draft"
+        draft.mkdir()
+        (draft / "keep.py").write_text("x = 1\n")
+        (draft / "_deletions.json").write_text('["old.js"]')
+        (draft / "sub").mkdir()
+        (draft / "sub" / "_deletions.json").write_text('["y"]')
+
+        result = repo_apply(source_dir=str(draft),
+                            workspace_root=str(tmp_path),
+                            project_root=str(proj),
+                            ignore=["_deletions.json"])
+
+        assert result["applied"] is True
+        assert "keep.py" in result["files"]
+        # basename-matched at any depth → never copied, never committed
+        assert "_deletions.json" not in result["files"]
+        assert "sub/_deletions.json" not in result["files"]
+        assert (proj / "keep.py").exists()
+        assert not (proj / "_deletions.json").exists()
+        assert not (proj / "sub" / "_deletions.json").exists()
+
+    def test_ignore_default_none_copies_everything(self, tmp_path):
+        """Backward-compat: no `ignore` arg → every file copied (unchanged)."""
+        proj = tmp_path / "project"
+        proj.mkdir()
+        subprocess.run(["git", "init", "-b", "main"], cwd=proj,
+                       capture_output=True)
+        subprocess.run(["git", "config", "user.email", "t@t"], cwd=proj,
+                       capture_output=True)
+        subprocess.run(["git", "config", "user.name", "T"], cwd=proj,
+                       capture_output=True)
+        draft = tmp_path / "draft"
+        draft.mkdir()
+        (draft / "_deletions.json").write_text("[]")
+        result = repo_apply(source_dir=str(draft),
+                            workspace_root=str(tmp_path),
+                            project_root=str(proj))
+        assert "_deletions.json" in result["files"]
 
 
 class TestLint: