py-harness-cli 0.3.0__py3-none-any.whl

harness/paths.py

@@ -0,0 +1,88 @@
+"""Locations inside this repository.
+
+A module that finds the repository root by counting parent directories
+stops working when the module is moved to a different directory depth. The
+root is resolved once here and imported everywhere else.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+# Small platform trees are Python plus a few config files. Secrets stay out.
+TEXT_SUFFIXES = frozenset(
+    {".py", ".pyi", ".md", ".toml", ".yml", ".yaml", ".cfg", ".ini", ".json"}
+)
+SECRET_NAMES = frozenset(
+    {".env", ".env.local", "credentials.json", ".pypirc", "secrets.json"}
+)
+
+
+def _find_kit_skills() -> Path:
+    """Locate the skills shipped with py-harness.
+
+    A source checkout keeps them at the repository root. An installed
+    package carries a copy inside `harness/`, because the repository root
+    is not present once the package is in site-packages.
+    """
+    packaged = Path(__file__).resolve().parent / "kit_skills"
+    if packaged.is_dir():
+        return packaged
+    return REPO_ROOT / "skills"
+
+
+KIT_SKILLS_DIR = _find_kit_skills()
+EVAL_DIR = REPO_ROOT / "eval"
+
+
+def suffix_globs() -> tuple[str, ...]:
+    """rglob patterns for every writable text suffix, sorted for stable tests."""
+    return tuple(f"*{suffix}" for suffix in sorted(TEXT_SUFFIXES))
+
+
+def is_secret_name(name: str) -> bool:
+    return name.lower() in {item.lower() for item in SECRET_NAMES}
+
+
+def is_windows(*, windows: bool | None = None) -> bool:
+    """True on this OS, or the layout a caller asked to simulate."""
+    return os.name == "nt" if windows is None else windows
+
+
+def venv_python(venv: Path, *, windows: bool | None = None) -> Path:
+    """The interpreter inside a virtual environment, on any platform.
+
+    POSIX puts it at `bin/python`; Windows puts it at `Scripts/python.exe`.
+    Pass `windows` to ask for one layout regardless of the platform running.
+    """
+    on_windows = is_windows(windows=windows)
+    if on_windows:
+        return venv / "Scripts" / "python.exe"
+    return venv / "bin" / "python"
+
+
+def rel_posix(path: Path, root: Path) -> str:
+    """Path relative to root, written with forward slashes on every platform.
+
+    Windows renders a relative path as `src\\app.py`. The model is shown
+    these paths and copies them back into `Path:`, and the skills, the
+    prompts and the tests are all written with forward slashes, so the two
+    styles must not mix. Forward slashes work as input on Windows too.
+    """
+    return path.relative_to(root).as_posix()
+
+
+def as_project_rel(rel: str) -> str:
+    """Accept a path written with either separator, return one with slashes.
+
+    A model may answer with `src\\app.py` whatever platform it runs on. On
+    Linux and macOS that is a single filename containing a backslash, not a
+    path, so it is converted before use.
+    """
+    cleaned = rel.replace("\\", "/").strip()
+    while cleaned.startswith("./"):
+        cleaned = cleaned[2:]
+    return cleaned

harness/scan/__init__.py

@@ -0,0 +1,6 @@
+"""Read facts about a project without changing it.
+
+Reports how large a project is, which files it contains, what those files
+define, what instructions the project publishes, and which parts of its
+structure are hard to read.
+"""

harness/scan/app_spec.py

@@ -0,0 +1,338 @@
+"""Checklist for a greenfield GitHub PR-review CLI. Deterministic. No model.
+
+A new-package loop used to stop after one function and a test. A typed
+"design a CLI for reviewing GitHub PRs" job is not done then: list and
+show still have to exist, HTTP has to be urllib with a token from the
+environment, and the suite has to mock the network.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from harness.scan.project_scan import SKIP_DIR
+from harness.task import package_noun
+
+CLEAN_PHRASE = "app checklist clean"
+REQUIRED_KEYS = ("init", "http", "list", "show", "mocked_tests")
+OVERFLOW_KEYS = ("comment", "pagination", "config")
+# Live 8B named the GET list_prs / fetch_pulls, not only list_pulls / get_prs.
+# After #214 the remasure left mocked_tests missing when those names were used.
+_LIST_GETTER = re.compile(
+    r"^(?:list_pulls|get_prs|(?:list|get|fetch|load)_[a-z0-9_]*"
+    r"(?:pull_requests|prs|pulls))$"
+)
+_SHOW_FN = re.compile(
+    r"^(?:show_pull|show_pr|get_pr|get_pull|fetch_pr|fetch_pull)$"
+)
+
+
+@dataclass(frozen=True)
+class Gap:
+    """One missing piece of the CLI, and the Action that closes it."""
+
+    key: str
+    next_action: str
+
+
+def _skip(path: Path) -> bool:
+    return any(part in SKIP_DIR or part.startswith(".") for part in path.parts)
+
+
+def _read_tree(project: Path) -> tuple[str, str]:
+    """Concatenate impl and test sources. Missing files are empty strings."""
+    impl_parts: list[str] = []
+    test_parts: list[str] = []
+    root = Path(project)
+    if not root.is_dir():
+        return "", ""
+    for path in sorted(root.rglob("*.py")):
+        if _skip(path) or path.name.endswith(".bak"):
+            continue
+        try:
+            text = path.read_text(encoding="utf-8")
+        except OSError:
+            continue
+        rel = path.relative_to(root).as_posix()
+        if "tests" in path.parts or path.name.startswith("test_"):
+            test_parts.append(text)
+        elif rel.endswith("__init__.py"):
+            continue
+        else:
+            impl_parts.append(text)
+    return "\n".join(impl_parts), "\n".join(test_parts)
+
+
+def _top_defs(source: str) -> list[str]:
+    return re.findall(r"^def\s+([a-z_][a-z0-9_]*)\s*\(", source, re.M)
+
+
+def list_getter_name(impl: str) -> str:
+    """The list/GET function the 8B wrote, or empty."""
+    names = _top_defs(impl)
+    for preferred in ("list_pulls", "get_prs"):
+        if preferred in names:
+            return preferred
+    for name in names:
+        if _LIST_GETTER.match(name):
+            return name
+    return ""
+
+
+def _has_list_command(impl: str) -> bool:
+    return bool(
+        re.search(r'add_parser\(\s*["\']list["\']', impl) or list_getter_name(impl)
+    )
+
+
+def _has_show_command(impl: str) -> bool:
+    if re.search(r'add_parser\(\s*["\']show["\']', impl):
+        return True
+    return any(_SHOW_FN.match(name) for name in _top_defs(impl))
+
+
+def _has_comment_command(impl: str) -> bool:
+    return bool(
+        re.search(r'add_parser\(\s*["\']comment["\']', impl)
+        or re.search(r"\bdef comment_on\b", impl)
+        or re.search(r"\bdef comment\b", impl)
+    )
+
+
+def _uses_urllib(impl: str) -> bool:
+    return "urllib.request" in impl
+
+
+def _token_from_env(impl: str) -> bool:
+    if not re.search(r"\b(os\.environ|os\.getenv)\b", impl):
+        return False
+    return bool(re.search(r"TOKEN|token", impl))
+
+
+def _mocks_http(tests: str) -> bool:
+    return bool(
+        re.search(r"\b(urlopen|urllib\.request)\b", tests)
+        and re.search(r"\b(patch|MagicMock|mock)\b", tests)
+    )
+
+
+def _tests_call_list_or_show(tests: str) -> bool:
+    """8B often names the GET get_prs / list_prs and drives list/show via main()."""
+    if re.search(r"\b(test_main_list|test_main_show)\b", tests):
+        return True
+    if re.search(r"\bmain\s*\(", tests) and re.search(r"\b(list|show)\b", tests):
+        return True
+    called = re.findall(r"\b([a-z_][a-z0-9_]*)\s*\(", tests)
+    return any(_LIST_GETTER.match(name) or _SHOW_FN.match(name) for name in called)
+
+
+def _has_pagination(impl: str) -> bool:
+    """True when page= (or Link next) sits on an indented list URL.
+
+    A live 8B left ``url = f"...pulls?page="`` at module scope. That
+    NameErrors on import. Count only a line inside a function or the
+    list argparse branch.
+    """
+    for line in impl.splitlines():
+        if not line[:1].isspace():
+            continue
+        if re.search(r"\bpage=", line):
+            return True
+        if re.search(r"rel=[\"']next|\bLink\b", line):
+            return True
+    return False
+
+
+def _has_home_config(impl: str) -> bool:
+    return "Path.home()" in impl
+
+
+def app_gaps(project: Path, task: str, *, include_overflow: bool = True) -> list[Gap]:
+    """Missing pieces, in the order the 8B should write them."""
+    noun = package_noun(task)
+    module = f"pkg/{noun}.py"
+    test = f"tests/test_{noun}.py"
+    impl, tests = _read_tree(project)
+    init = Path(project) / "pkg" / "__init__.py"
+    gaps: list[Gap] = []
+    if not init.is_file():
+        gaps.append(
+            Gap(
+                "init",
+                "Next Action must be edit Path: pkg/__init__.py "
+                "(exports only). No logic.",
+            )
+        )
+    if not _uses_urllib(impl) or not _token_from_env(impl):
+        gaps.append(
+            Gap(
+                "http",
+                f"Next Action must be edit Path: {module} with urllib.request "
+                "and a token from os.environ. No curl. No inline secrets.",
+            )
+        )
+    if not _has_list_command(impl):
+        gaps.append(
+            Gap(
+                "list",
+                f"Next Action must be edit Path: {module} with argparse "
+                "subcommand list and def list_pulls(...).",
+            )
+        )
+    if not _has_show_command(impl):
+        gaps.append(
+            Gap(
+                "show",
+                f"Next Action must be edit Path: {module} with argparse "
+                "subcommand show and def show_pull(...).",
+            )
+        )
+    if not _mocks_http(tests) or not _tests_call_list_or_show(tests):
+        gaps.append(
+            Gap(
+                "mocked_tests",
+                f"Next Action must be edit Path: {test} as a unittest.TestCase. "
+                "patch urllib.request.urlopen. Call list_pulls, get_prs, or "
+                "show_pull. Do not call the network.",
+            )
+        )
+    if include_overflow:
+        if not _has_comment_command(impl):
+            gaps.append(
+                Gap(
+                    "comment",
+                    f"Next Action must be edit Path: {module} with argparse "
+                    "subcommand comment and def comment_on(...).",
+                )
+            )
+        if not _has_pagination(impl):
+            gaps.append(
+                Gap(
+                    "pagination",
+                    f"Next Action must be patch Path: {module} so the list_pulls "
+                    "URL includes page=. Do not rename list_pulls. "
+                    "Do not add weekday or another module.",
+                )
+            )
+        if not _has_home_config(impl):
+            gaps.append(
+                Gap(
+                    "config",
+                    f"Next Action must be edit Path: pkg/config.py with "
+                    "Path.home() for the config file. No hardcoded home.",
+                )
+            )
+    return gaps
+
+
+def required_gaps(project: Path, task: str) -> list[Gap]:
+    """Gaps that block done on the first run: list, show, mocked suite."""
+    return [gap for gap in app_gaps(project, task, include_overflow=False)]
+
+
+def overflow_gaps(project: Path, task: str) -> list[Gap]:
+    """comment / pagination / config — a later typed run, not --steps."""
+    return [gap for gap in app_gaps(project, task) if gap.key in OVERFLOW_KEYS]
+
+
+def requested_overflow(task: str) -> tuple[str, ...]:
+    """Which overflow keys this typed run asked for. Empty means all of them."""
+    text = (task or "").lower()
+    keys: list[str] = []
+    if "comment" in text:
+        keys.append("comment")
+    if "pagination" in text or re.search(r"\bpage=", text):
+        keys.append("pagination")
+    if "config" in text or "path.home" in text:
+        keys.append("config")
+    return tuple(keys)
+
+
+def next_overflow_action(project: Path, task: str) -> str:
+    """The next overflow Action this run asked for, or empty when that piece exists."""
+    wanted = set(requested_overflow(task)) or set(OVERFLOW_KEYS)
+    for gap in overflow_gaps(project, task):
+        if gap.key in wanted:
+            return gap.next_action + "\n"
+    return ""
+
+
+def overflow_edit_line(task: str, project: Path | None = None) -> str:
+    """Dictator Action for this overflow run. Comment-only copy burned pagination."""
+    if project is not None:
+        leftover = next_overflow_action(project, task).strip()
+        if leftover:
+            return leftover
+    wanted = requested_overflow(task)
+    key = wanted[0] if wanted else "comment"
+    noun = package_noun(task)
+    if key == "pagination":
+        return (
+            f"Next Action must be patch Path: pkg/{noun}.py so the list_pulls "
+            "URL includes page=. Do not rename list_pulls. "
+            "Do not add weekday or another module."
+        )
+    if key == "config":
+        return (
+            "Next Action must be edit Path: pkg/config.py with "
+            "Path.home() for the config file. No hardcoded home."
+        )
+    return (
+        f"Next Action must be edit Path: pkg/{noun}.py with argparse "
+        "subcommand comment and def comment_on(...)."
+    )
+
+
+def render_app_review(project: Path, task: str) -> str:
+    gaps = required_gaps(project, task)
+    extra = overflow_gaps(project, task)
+    if not gaps:
+        if extra:
+            leftover = ", ".join(gap.key for gap in extra)
+            return (
+                f"{CLEAN_PHRASE} for list and show. "
+                f"Later run can add {leftover}."
+            )
+        return f"{CLEAN_PHRASE} — list, show, comment, pagination, config, mocked tests"
+    lines = ["app checklist (deterministic, not a model opinion):"]
+    lines.extend(f"- {gap.key}: {gap.next_action}" for gap in gaps)
+    return "\n".join(lines)
+
+
+def app_is_clean(report: str) -> bool:
+    """True when the required list/show checklist is satisfied."""
+    return CLEAN_PHRASE in (report or "")
+
+
+def next_app_action(project: Path, task: str, *, required_only: bool = True) -> str:
+    """The single next Action line, or empty when that tier is clean."""
+    gaps = required_gaps(project, task) if required_only else app_gaps(project, task)
+    if not gaps:
+        return ""
+    return gaps[0].next_action + "\n"
+
+
+def http_test_nudge(task: str) -> str:
+    """AAA mock example the 8B can copy. Not the weekday write-tests skill."""
+    noun = package_noun(task)
+    return (
+        f"Next Action must be edit Path: tests/test_{noun}.py\n"
+        "```python\n"
+        "import json\n"
+        "import unittest\n"
+        "from unittest.mock import patch\n"
+        f"from pkg.{noun} import list_pulls\n\n\n"
+        f"class Test{noun.title().replace('_', '')}(unittest.TestCase):\n"
+        "    def test_list_pulls_returns_titles(self) -> None:\n"
+        '        payload = [{"title": "Fix login", "number": 1}]\n'
+        '        with patch("urllib.request.urlopen") as fake:\n'
+        "            fake.return_value.__enter__.return_value.read.return_value = (\n"
+        "                json.dumps(payload).encode()\n"
+        "            )\n"
+        '            got = list_pulls("owner", "repo")\n'
+        "        self.assertEqual(got, payload)\n"
+        "```\n"
+        "Do not call the network. Do not copy weekday or multiply.\n"
+    )

harness/scan/design.py

@@ -0,0 +1,112 @@
+"""Deterministic structure / SoC review. No model."""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+from harness.scan.project_brief import iter_text_files
+
+MAX_FINDINGS = 16
+GOD_DEFS = 4
+# Where a function stops being one thing. The architecture test in this
+# repository refuses at 80, which is the point where a function cannot
+# be read at all; this is the earlier point, where it should be split.
+# Two numbers because they answer two questions, and 40 flags 7% of the
+# functions here rather than most of them.
+LONG_DEF = 40
+CLEAN_PHRASE = "no structure findings"
+
+
+def longest_def(source: str) -> tuple[str, int] | None:
+    """The longest top-level function in the file, and its length.
+
+    One finding per file rather than one per function: a file with six
+    long functions has one problem, and sixteen findings of the same
+    shape push everything else off the report.
+    """
+    try:
+        tree = ast.parse(source)
+    except (SyntaxError, ValueError):
+        return None
+    found = [
+        (node.name, (node.end_lineno or node.lineno) - node.lineno)
+        for node in tree.body
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
+    ]
+    return max(found, key=lambda item: item[1]) if found else None
+
+
+def _defs(source: str) -> list[str]:
+    try:
+        tree = ast.parse(source)
+    except (SyntaxError, ValueError):
+        return []
+    names: list[str] = []
+    for node in tree.body:
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            names.append(node.name)
+    return names
+
+
+def render_design_review(project: Path, scope: str = "") -> str:
+    root = project.resolve()
+    python_files = [
+        path
+        for path, _size in iter_text_files(project, scope)
+        if path.suffix == ".py"
+    ]
+    findings: list[str] = []
+    rels = [path.relative_to(root).as_posix() for path in python_files]
+    stems = {Path(rel).stem for rel in rels if "test" in rel}
+    has_tests = any(rel.startswith("tests/") or "/tests/" in rel for rel in rels)
+    has_lib = any(rel.startswith(("pkg/", "src/")) for rel in rels)
+    if not has_tests:
+        findings.append("missing tests/ — add tests/test_<module>.py beside each concern")
+    if not has_lib and any(rel.startswith("scripts/") for rel in rels):
+        findings.append("no pkg/ or src/ — library code should not live only in scripts/")
+    for path, rel in zip(python_files, rels, strict=True):
+        try:
+            source = path.read_text(encoding="utf-8")
+        except OSError:
+            continue
+        names = _defs(source)
+        if rel.endswith("__init__.py") and names:
+            findings.append(
+                f"SoC: {rel} defines {', '.join(names)} — __init__.py is exports only"
+            )
+        if rel.startswith("scripts/") and any(name != "main" for name in names):
+            extra = [name for name in names if name != "main"]
+            findings.append(
+                f"SoC: {rel} has {', '.join(extra)} — move library code to pkg/<noun>.py"
+            )
+        if "test" not in rel and not rel.endswith("__init__.py") and len(names) >= GOD_DEFS:
+            findings.append(
+                f"god module: {rel} has {len(names)} top-level functions — "
+                "Action: edit Path: pkg/<new_concern>.py with one function"
+            )
+        longest = longest_def(source)
+        if longest and longest[1] > LONG_DEF and "test" not in rel:
+            name, length = longest
+            findings.append(
+                f"long function: {rel}:{name} is {length} lines — "
+                f"over {LONG_DEF}, split it into one function per thing it does"
+            )
+        if (
+            rel.startswith(("pkg/", "src/"))
+            and not rel.endswith("__init__.py")
+            and f"test_{Path(rel).stem}" not in stems
+        ):
+            findings.append(f"missing tests: no tests/test_{Path(rel).stem}.py for {rel}")
+        if len(findings) >= MAX_FINDINGS:
+            break
+    if not findings:
+        findings.append(f"{CLEAN_PHRASE} in scope — pkg/ and tests/ look split")
+    lines = ["design review (deterministic, not a model opinion):"]
+    lines.extend(f"- {item}" for item in findings[:MAX_FINDINGS])
+    return "\n".join(lines)
+
+
+def design_is_clean(report: str) -> bool:
+    """True when the last scan reported no structure findings."""
+    return CLEAN_PHRASE in (report or "")

harness/scan/existing.py

@@ -0,0 +1,131 @@
+"""What the project already has that the task is about.
+
+A run asked to check a prompt for a leaked credential and wrote a
+function that looked for a variable name rather than the shape of one,
+and called it from nowhere. The shape was already in the tree, in
+`secrets.py`, under the same words the task had used. The preamble the
+model was given ran to twelve thousand characters and named neither the
+file nor the function.
+
+This module deliberately avoids quoting the words that case turned on.
+A search that matches its own source is a search that reports itself.
+
+Nothing was wrong with the model that a search would not have fixed. So
+the search happens here, before the model starts: take the phrases out
+of the task, find the ones that are rare in this project, and say where
+they already appear. Rare is the whole trick — "add a function" matches
+everything and means nothing.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from harness.scan.project_scan import SKIP_DIR
+
+# Words that carry no subject. A phrase built only from these is not
+# worth searching for.
+FILLER = frozenset(
+    {
+        "add", "also", "and", "any", "are", "back", "call", "called", "can",
+        "change", "check", "code", "create", "does", "file", "files", "fix",
+        "for", "from", "function", "has", "have", "how", "into", "make",
+        "move", "must", "need", "new", "not", "one", "only", "out", "put",
+        "return", "returns", "run", "should", "test", "tests", "that",
+        "the", "then", "this", "to", "use", "used", "using", "when",
+        "where", "which", "with", "write", "you",
+    }
+)
+
+# A phrase in more files than this is common vocabulary, not a pointer.
+MOST_FILES = 3
+# And one in no file is not a pointer either.
+FEWEST_FILES = 1
+
+
+def phrases(task: str) -> list[str]:
+    """Two-word phrases from the task that might name something real."""
+    words = [w for w in re.findall(r"[A-Za-z][A-Za-z0-9_]+", task.lower())]
+    found = []
+    for first, second in zip(words, words[1:], strict=False):
+        if first in FILLER or second in FILLER:
+            continue
+        if len(first) < 3 or len(second) < 3:
+            continue
+        found.append(f"{first} {second}")
+    return found
+
+
+def _searchable(project: Path) -> list[Path]:
+    """Source files only. A test names every subject in the project."""
+    return [
+        path
+        for path in sorted(Path(project).rglob("*.py"))
+        if not any(part in SKIP_DIR or part.startswith(".") for part in path.parts)
+        and not path.name.endswith(".bak")
+        and not path.name.startswith("test_")
+        and "tests" not in path.parts
+    ]
+
+
+def _ranked_hits(
+    project: Path, task: str, *, skip: str = ""
+) -> list[tuple[str, list[tuple[str, int]]]]:
+    """Rare phrases from the task, each with the files they already appear in."""
+    wanted = phrases(task)
+    if not wanted:
+        return []
+    root = Path(project)
+    hits: dict[str, list[tuple[str, int]]] = {phrase: [] for phrase in wanted}
+    for path in _searchable(root):
+        rel = path.relative_to(root).as_posix()
+        if skip and rel == skip:
+            continue
+        try:
+            lines = path.read_text(encoding="utf-8").splitlines()
+        except OSError:
+            continue
+        lowered = [line.lower() for line in lines]
+        for phrase in wanted:
+            for number, line in enumerate(lowered, 1):
+                if phrase in line:
+                    hits[phrase].append((rel, number))
+                    break
+    ranked = [
+        (phrase, found)
+        for phrase, found in hits.items()
+        if FEWEST_FILES <= len({rel for rel, _ in found}) <= MOST_FILES
+    ]
+    ranked.sort(key=lambda item: (len({rel for rel, _ in item[1]}), item[0]))
+    return ranked[:2]
+
+
+def existing_files(project: Path, task: str, *, skip: str = "") -> tuple[str, ...]:
+    """Paths already_covers would name, without the prose."""
+    found: list[str] = []
+    for _phrase, hits in _ranked_hits(project, task, skip=skip):
+        for rel, _number in hits:
+            if rel not in found:
+                found.append(rel)
+    return tuple(found)
+
+
+def already_covers(project: Path, task: str, *, skip: str = "") -> str:
+    """One line naming where the task's subject already appears. "" if nowhere.
+
+    `skip` is the file the task already names, because finding the words
+    in the file being changed is not news.
+    """
+    ranked = _ranked_hits(project, task, skip=skip)
+    if not ranked:
+        return ""
+    lines = []
+    for phrase, found in ranked:
+        where = ", ".join(f"{rel}:{number}" for rel, number in found[:2])
+        lines.append(f'  "{phrase}" is already in {where}')
+    return (
+        "This project already has something for what the task names:\n"
+        + "\n".join(lines)
+        + "\nRead those before writing anything new for it."
+    )