py-harness-cli 0.3.0__py3-none-any.whl

harness/skillkit/refuse_finish.py

@@ -0,0 +1,295 @@
+"""Whether a run may say it is finished.
+
+A different question from whether a change is acceptable. These look
+at the project after the work, not at the draft before it: is the
+symbol actually there, did a test run, does a test call what it was
+asked to cover.
+"""
+
+from __future__ import annotations
+
+# The oracle reuses one draft rule: a half-finished rename is a
+# reason to refuse a finish as well as a draft.
+from harness.skillkit.refuse_change import refuse_rename_incomplete
+
+"""SoC / readable-name guards. Deterministic. No model."""
+import ast
+import re
+from pathlib import Path
+from harness.task import (
+    looks_like_add_feature,
+    looks_like_bugfix,
+    looks_like_design_loop,
+    covered_symbol,
+    looks_like_everyday_code,
+    looks_like_fix_smell,
+    looks_like_write_tests,
+    looks_like_app_loop,
+    looks_like_app_overflow,
+    looks_like_new_package,
+    looks_like_refactor,
+    question_symbol,
+    rename_pair,
+)
+
+
+def _a_test_uses(body: str, symbol: str) -> bool:
+    """True when some `def test_...` in `body` actually mentions `symbol`.
+
+    Asking only whether the name appears anywhere in the test files
+    accepted a file holding one import line and nothing else. On a real
+    module the run wrote exactly that, reported `done`, and `unittest
+    discover` found no tests at all. An import is not coverage.
+    """
+    if not body.strip() or not symbol:
+        return False
+    try:
+        tree = ast.parse(body)
+    except (SyntaxError, ValueError):
+        # Unparsable here means several files concatenated. Fall back to
+        # requiring the name somewhere after a test definition.
+        return bool(re.search(rf"def test_\w*[\s\S]*?\b{re.escape(symbol)}\b", body))
+    for node in ast.walk(tree):
+        if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            continue
+        if not node.name.startswith("test"):
+            continue
+        for child in ast.walk(node):
+            if isinstance(child, ast.Name) and child.id == symbol:
+                return True
+            if isinstance(child, ast.Attribute) and child.attr == symbol:
+                return True
+    return False
+
+
+def tests_call(project: Path, symbol: str) -> bool:
+    """True when some test in tests/ actually calls symbol."""
+    tests = Path(project) / "tests"
+    if not tests.is_dir() or not symbol:
+        return False
+    body = ""
+    for path in sorted(tests.glob("test_*.py")):
+        try:
+            body += "\n" + path.read_text(encoding="utf-8")
+        except OSError:
+            continue
+    return _a_test_uses(body, symbol)
+
+
+def refuse_done_oracle(task: str, project: Path, last_path: str) -> str:
+    """Refuse done when the named file or last write still has an unbound name."""
+    from harness.scan.names import undefined_in_file
+    from harness.task import named_project_file, rename_pair
+
+    if looks_like_app_overflow(task):
+        return ""
+
+    paths: list[str] = []
+    named = named_project_file(task, project)
+    if named:
+        paths.append(named)
+    if last_path and last_path not in paths:
+        paths.append(last_path)
+    if looks_like_bugfix(task):
+        for rel in paths:
+            leftover = undefined_in_file(Path(project) / rel)
+            if leftover:
+                return (
+                    f"undefined name {leftover[0]} in {rel}. "
+                    f"Action: patch Path: {rel} Find: {leftover[0]} "
+                    "Replace: the name you assigned."
+                )
+    if looks_like_add_feature(task):
+        symbol = question_symbol(task)
+        if symbol:
+            found = False
+            root = Path(project)
+            from harness.scan.project_brief import iter_text_files
+
+            for path, _size in iter_text_files(root):
+                if path.suffix != ".py":
+                    continue
+                try:
+                    body = path.read_text(encoding="utf-8")
+                except OSError:
+                    continue
+                if re.search(rf"^def {re.escape(symbol)}\b", body, re.MULTILINE):
+                    found = True
+                    break
+            if not found:
+                from harness.skillkit.target import pick_module
+
+                dest = pick_module(root, last_path, task)
+                return (
+                    f"def {symbol} is not in the project. "
+                    f"Action: patch Path: {dest} Append: def {symbol}(...)."
+                )
+    if looks_like_write_tests(task):
+        symbol = covered_symbol(task)
+        if symbol:
+            tests = Path(project) / "tests"
+            body = ""
+            if tests.is_dir():
+                for path in sorted(tests.glob("test_*.py")):
+                    try:
+                        body += path.read_text(encoding="utf-8")
+                    except OSError:
+                        continue
+            if not _a_test_uses(body, symbol):
+                test_rel = "tests/test_module.py"
+                if named:
+                    test_rel = f"tests/test_{Path(named).stem}.py"
+                return (
+                    f"no test calls {symbol}. "
+                    f"Action: patch Path: {test_rel} "
+                    f"Append: one AAA test that calls {symbol}."
+                )
+    if looks_like_fix_smell(task) and named:
+        old, new = rename_pair(task)
+        try:
+            body = (Path(project) / named).read_text(encoding="utf-8")
+        except OSError:
+            body = ""
+        if old and new and body:
+            missed = refuse_rename_incomplete(task, named, body)
+            if missed:
+                return missed
+    return ""
+
+
+def refuse_write_done(task: str, ran_tests: bool, *, wrote: bool = True) -> str:
+    """Write tasks are not finished until a passing unittest has run.
+
+    A task that has not written yet is handled by the empty-done refuse.
+    New-package still needs a run even if the model claims it is done.
+    """
+    if ran_tests:
+        return ""
+    if looks_like_new_package(task):
+        return "not done. Action: run Argv: -m unittest discover -s tests -q"
+    if not wrote:
+        return ""
+    needs_run = (
+        looks_like_add_feature(task)
+        or looks_like_everyday_code(task)
+        or looks_like_fix_smell(task)
+        or looks_like_bugfix(task)
+        or looks_like_refactor(task)
+        or looks_like_design_loop(task)
+    )
+    if not needs_run:
+        return ""
+    return "not done. Action: run Argv: -m unittest discover -s tests -q"
+
+
+def refuse_package_done(task: str, ran_tests: bool, wrote: bool = True) -> str:
+    return refuse_write_done(task, ran_tests, wrote=wrote)
+
+
+def refuse_app_done(task: str, project: Path) -> str:
+    """Refuse done while list/show/http/mocked tests are still missing."""
+    from harness.scan.app_spec import (
+        next_app_action,
+        next_overflow_action,
+        render_app_review,
+        required_gaps,
+    )
+
+    if looks_like_app_overflow(task):
+        leftover = next_overflow_action(project, task)
+        return ("not done. " + leftover.strip()) if leftover else ""
+    if not looks_like_app_loop(task):
+        return ""
+    if required_gaps(project, task):
+        return (
+            "not done. "
+            + next_app_action(project, task).strip()
+            + "\n"
+            + render_app_review(project, task)
+        )
+    return ""
+
+
+# Names a framework calls, so nobody in this project has to.
+_CALLED_BY_SOMETHING_ELSE = ("test", "_", "main")
+
+
+def _added_functions(path: Path) -> list[str]:
+    """Top-level functions this run put in the file that were not there.
+
+    The original is on disk: every write leaves a `.bak` beside the file
+    it changed. Without it there is no way to tell a function the run
+    added from one that was already there, and refusing on the second
+    would be refusing somebody else's code.
+    """
+    backup = path.with_suffix(path.suffix + ".bak")
+    try:
+        now = ast.parse(path.read_text(encoding="utf-8"))
+        before = ast.parse(backup.read_text(encoding="utf-8"))
+    except (SyntaxError, ValueError, OSError):
+        return []
+    was = {
+        n.name for n in before.body
+        if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
+    }
+    return [
+        n.name for n in now.body
+        if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
+        and n.name not in was
+        and not n.name.startswith(_CALLED_BY_SOMETHING_ELSE)
+    ]
+
+
+def _anything_calls(project: Path, name: str, skip: Path) -> bool:
+    """Does any other file in the project call this by name?"""
+    called = re.compile(rf"\b{re.escape(name)}\s*\(")
+    for path in project.rglob("*.py"):
+        if path == skip or path.name.endswith(".bak") or ".git" in path.parts:
+            continue
+        try:
+            body = path.read_text(encoding="utf-8")
+        except OSError:
+            continue
+        if called.search(body) or _a_test_uses(body, name):
+            return True
+    return False
+
+
+def refuse_unwired_addition(
+    project: Path, last_path: str, task: str = ""
+) -> str:
+    """A function added, and nobody left to call it.
+
+    Asked to add a check that refuses to send when a prompt carries a
+    token, a run wrote `reject_github_tokens_in_prompt` and left nothing
+    nothing. The file parsed and the suite stayed green, because a
+    function nobody calls cannot fail. It also cannot work.
+
+    Whether the body is any good needs a reader. Whether anything will
+    ever run it does not, so that much is checked here.
+
+    Overflow comment/pagination/config is a later typed run. The 8B
+    writes ``def comment_on`` and the unused-function guard then
+    burns the step budget. That cell is not this rule.
+    """
+    if looks_like_app_overflow(task):
+        return ""
+    if not last_path:
+        return ""
+    path = Path(project) / last_path
+    for name in _added_functions(path):
+        try:
+            here = path.read_text(encoding="utf-8")
+        except OSError:
+            return ""
+        if re.search(rf"\b{re.escape(name)}\s*\(", here.replace(f"def {name}(", "")):
+            continue
+        if _anything_calls(Path(project), name, path):
+            continue
+        return (
+            f"`{name}` was added to {last_path} and nothing calls it. A "
+            "function nobody calls cannot fail, and cannot work either. "
+            "Action: patch the place that should use it, or write a test "
+            "that calls it."
+        )
+    return ""

harness/skillkit/target.py

@@ -0,0 +1,238 @@
+"""Replace the file paths inside a skill with paths from this project.
+
+A skill is written as a block of text for the model to copy, so any file
+path inside it will be copied as well. A path that refers to this
+repository's test fixtures does not exist in the user's project, and
+writing to it creates a file the user did not ask for.
+
+Two replacements are made before the model sees a skill:
+
+* The placeholders `{{module}}`, `{{test}}`, `{{scope}}` and `{{symbol}}`
+  are filled in from the project.
+* Any `Path:` or `Scope:` that does not exist in the project is replaced
+  with one that does.
+
+A path that does exist in the project is left unchanged, which is what
+keeps the fixture-based tests working. A path ending in `__init__.py` is
+also left unchanged, because creating a package legitimately names a file
+that does not exist yet.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from harness.task import looks_like_new_package, looks_like_ops, package_noun, question_symbol
+from harness.paths import rel_posix
+from harness.scan.project_brief import iter_text_files
+
+_DEF_NAME = re.compile(r"^def\s+([A-Za-z_]\w*)", re.MULTILINE)
+# HTTP/UI adapters are larger than the domain file beside them. Size-first
+# pick put `total_lines` in `orders_controller.py` on the live 8B demo.
+_ROLE_SUFFIX = ("_controller", "_service", "_view", "_router", "_handler")
+
+_PLACEHOLDER = re.compile(r"\{\{(module|test|scope|symbol)\}\}")
+_PATH_LINE = re.compile(r"^(Path|File):\s*(\S+)\s*$", re.MULTILINE)
+_SCOPE_LINE = re.compile(r"^Scope:\s*(\S+)\s*$", re.MULTILINE)
+_SYMBOL_TOKEN = "the_symbol_from_the_task"
+# Scaffolding a package legitimately names a file that does not exist yet.
+_KEEP_NAMES = frozenset({"__init__.py"})
+# Used only in documentation and error text, never handed to the model.
+FALLBACK_MODULE = "path/to/module.py"
+DEFAULT_NEW_MODULE = "src/main.py"
+FALLBACK_TEST = "tests/test_module.py"
+
+
+@dataclass(frozen=True)
+class Target:
+    """The files in this project that a skill should refer to.
+
+    Fields:
+        module: file where new code belongs.
+        test: file where new tests belong.
+        scope: directory to stay within.
+        symbol: name from the task, used to fill in a search query.
+    """
+
+    module: str
+    test: str
+    scope: str
+    symbol: str
+
+
+def _is_test(rel: str) -> bool:
+    parts = rel.replace("\\", "/").split("/")
+    return "tests" in parts or parts[-1].startswith("test_")
+
+
+def _rels(project: Path, suffix: str = ".py") -> list[tuple[str, int]]:
+    root = project.resolve()
+    return [
+        (rel_posix(path, root), size)
+        for path, size in iter_text_files(project)
+        if path.suffix == suffix
+    ]
+
+
+def pick_module(project: Path, located_path: str = "", task: str = "") -> str:
+    """The file a new function most likely belongs in.
+
+    A project with no Python file in it still needs a real path. The
+    placeholder used to be handed straight to the model, which then created
+    a file literally at `path/to/module.py` — the same fault as shipping a
+    fixture path, arriving by a different route.
+    """
+    if looks_like_ops(task):
+        return pick_workflow(project)
+    if looks_like_new_package(task):
+        return f"pkg/{package_noun(task)}.py"
+    if located_path:
+        rel = located_path.replace("\\", "/").lstrip("./")
+        if rel.endswith(".py") and not _is_test(rel):
+            return rel
+    usable = [
+        (rel, size)
+        for rel, size in _rels(project)
+        if not _is_test(rel) and Path(rel).name != "__init__.py"
+    ]
+    if not usable:
+        return _new_module_name(task)
+    wanted = _name_tokens(question_symbol(task))
+    usable.sort(key=lambda item: _module_score(project, item[0], item[1], wanted))
+    return usable[0][0]
+
+
+def _name_tokens(symbol: str) -> frozenset[str]:
+    return frozenset(part for part in symbol.lower().split("_") if len(part) >= 3)
+
+
+def _module_score(
+    project: Path, rel: str, size: int, wanted: frozenset[str]
+) -> tuple[int, int, int, str]:
+    """Lower is better: more name overlap, not a role adapter, then smaller."""
+    stem = Path(rel).stem.lower()
+    role = int(any(stem.endswith(suffix) for suffix in _ROLE_SUFFIX))
+    overlap = 0
+    if wanted:
+        try:
+            body = (Path(project) / rel).read_text(encoding="utf-8")
+        except OSError:
+            body = ""
+        overlap = sum(
+            1 for name in _DEF_NAME.findall(body) if _name_tokens(name) & wanted
+        )
+    return (-overlap, role, size, rel)
+
+
+def pick_workflow(project: Path) -> str:
+    """Existing workflow YAML, else the path the skill creates."""
+    root = Path(project)
+    folder = root / ".github" / "workflows"
+    if folder.is_dir():
+        found = sorted(
+            rel_posix(path, root)
+            for path in folder.iterdir()
+            if path.suffix in {".yml", ".yaml"} and path.is_file()
+        )
+        if found:
+            return found[0]
+    return ".github/workflows/tests.yml"
+
+
+def _new_module_name(task: str) -> str:
+    """Where the first module of an empty project should go."""
+    symbol = question_symbol(task)
+    if symbol and symbol != _SYMBOL_TOKEN:
+        return f"src/{symbol}.py"
+    return DEFAULT_NEW_MODULE
+
+
+def pick_test(project: Path, module: str) -> str:
+    """The matching test file, else any test file, else the name to create."""
+    stem = Path(module).stem
+    tests = sorted(rel for rel, _size in _rels(project) if _is_test(rel))
+    if not tests:
+        return f"tests/test_{stem}.py" if stem else FALLBACK_TEST
+    for rel in tests:
+        if Path(rel).stem == f"test_{stem}":
+            return rel
+    return tests[0]
+
+
+def pick_scope(scope: str, module: str) -> str:
+    if scope:
+        return scope
+    parts = module.split("/")
+    return parts[0] if len(parts) > 1 else "."
+
+
+def pick_target(
+    project: Path, task: str = "", scope: str = "", located_path: str = ""
+) -> Target:
+    module = pick_module(project, located_path, task)
+    symbol = (
+        package_noun(task)
+        if looks_like_new_package(task)
+        else question_symbol(task) or _SYMBOL_TOKEN
+    )
+    return Target(
+        module=module,
+        test=pick_test(project, module),
+        scope=pick_scope(scope, module),
+        symbol=symbol,
+    )
+
+
+def _writes_a_whole_module(body: str) -> bool:
+    """True when the skill hands over a file, not a change to one.
+
+    `Action: edit` with a module body creates a file. `Action: patch` with
+    Find or Append edits one that is already there, and only that case
+    should be pointed at an existing file.
+    """
+    return bool(re.search(r"^Action:\s*edit\s*$", body, re.MULTILINE))
+
+
+def retarget(body: str, target: Target, project: Path | None = None) -> str:
+    """Fill placeholders, then repoint any path this project does not have."""
+    values = {
+        "module": target.module,
+        "test": target.test,
+        "scope": target.scope,
+        "symbol": target.symbol,
+    }
+    text = _PLACEHOLDER.sub(lambda m: values[m.group(1)], body)
+    if target.symbol != _SYMBOL_TOKEN:
+        text = text.replace(_SYMBOL_TOKEN, target.symbol)
+    if project is None:
+        return text
+    root = project.resolve()
+
+    creates_module = _writes_a_whole_module(text)
+
+    def _path(match: re.Match[str]) -> str:
+        key, rel = match.group(1), match.group(2)
+        if (root / rel).is_file() or Path(rel).name in _KEEP_NAMES:
+            return match.group(0)
+        if creates_module and not _is_test(rel):
+            # The skill writes a new module, not a change to an existing one.
+            # Repointing it onto a file that already exists made the model
+            # write the function twice: once where it was sent, once where
+            # the skill said. Keep the name, move it beside this project's
+            # own modules.
+            home = Path(target.module).parent
+            name = Path(rel).name
+            moved = (home / name).as_posix() if home != Path(".") else name
+            return f"{key}: {moved}"
+        return f"{key}: {target.test if _is_test(rel) else target.module}"
+
+    def _scope(match: re.Match[str]) -> str:
+        rel = match.group(1)
+        if (root / rel).is_dir():
+            return match.group(0)
+        return f"Scope: {target.scope}"
+
+    text = _PATH_LINE.sub(_path, text)
+    return _SCOPE_LINE.sub(_scope, text)