misterdev 0.2.0__py3-none-any.whl

misterdev/agent_helpers.py

@@ -0,0 +1,194 @@
+import time
+from typing import Optional
+
+from rich.console import Console
+
+from misterdev.analyzers.project_analyzer import has_test_files
+from misterdev.config import get_setting
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+console = Console()
+
+
+class _WorktreeProjectView:
+    """A Project facade that overrides only `path` (for worktree execution).
+
+    Everything else (config, llm_client, topography, tools, env) delegates to
+    the base project, so the executor reads shared context but writes, builds,
+    and commits inside the worktree.
+    """
+
+    def __init__(self, base, path):
+        self._base = base
+        self.path = path
+
+    def __getattr__(self, name):
+        return getattr(self._base, name)
+
+
+class ProgressReporter:
+    """Lightweight wave/task progress logger for long runs."""
+
+    def __init__(self, total_tasks: int):
+        self.total = total_tasks
+        self.completed = 0
+        self.failed = 0
+        self.current_wave = 0
+        self.start_time = time.time()
+        self._task_start: Optional[float] = None
+
+    def start_wave(self, wave_num: int, task_ids: list[str]):
+        self.current_wave = wave_num
+        logger.info(f"=== Wave {wave_num} === [{', '.join(task_ids)}]")
+
+    def start_task(self, task_id: str, title: str):
+        self._task_start = time.time()
+        logger.info(
+            f"[{self.completed + self.failed}/{self.total}] Starting {task_id}: {title}"
+        )
+
+    def end_task(self, task_id: str, success: bool):
+        elapsed = time.time() - self._task_start if self._task_start else 0
+        if success:
+            self.completed += 1
+            logger.info(
+                f"[{self.completed + self.failed}/{self.total}] {task_id} DONE ({elapsed:.0f}s)"
+            )
+        else:
+            self.failed += 1
+            logger.warning(
+                f"[{self.completed + self.failed}/{self.total}] {task_id} FAILED ({elapsed:.0f}s)"
+            )
+
+    def summary(self):
+        total_time = time.time() - self.start_time
+        logger.info(
+            f"=== Complete: {self.completed} done, {self.failed} failed, {total_time:.0f}s total ==="
+        )
+
+
+def _combine_commands(*cmds: Optional[str]) -> Optional[str]:
+    """Join shell commands with ``&&`` (each parenthesised), or None if all empty.
+
+    Used to run the visible suite and the golden suite as one pass/fail check
+    so the existing single-command gate/bisect logic enforces both.
+    """
+    present = [c for c in cmds if c]
+    if not present:
+        return None
+    return " && ".join(f"({c})" for c in present)
+
+
+def _budget_exhausted(client) -> bool:
+    """True when the client reports a non-positive remaining budget.
+
+    Defensive: a non-numeric ``budget_remaining`` (e.g. a test double) is not
+    treated as exhausted.
+    """
+    remaining = getattr(client, "budget_remaining", None)
+    return (
+        isinstance(remaining, (int, float))
+        and not isinstance(remaining, bool)
+        and remaining <= 0
+    )
+
+
+def _apply_budget_ceiling(client, flag_budget: float) -> None:
+    """Set the client budget to the tighter of its config budget and the flag.
+
+    The project.yaml budget is already on the client; both it and the --budget
+    flag are ceilings, so the minimum wins and a config cap is never silently
+    overridden by the CLI default. Falls back to the flag when the current
+    budget isn't numeric (e.g. a test double).
+    """
+    current = getattr(client, "_budget", None)
+    if isinstance(current, (int, float)) and not isinstance(current, bool):
+        client._budget = min(current, flag_budget)
+    else:
+        client._budget = flag_budget
+
+
+def _warn_if_baseline_broken(assessment, report) -> None:
+    """Loudly surface a failing baseline build before any work begins.
+
+    A red baseline silently disables the integration gate (it needs a green
+    baseline to detect regressions), so the run would execute largely ungated.
+    Make that visible and record it rather than letting it pass quietly.
+    """
+    health = assessment.health
+    if health.builds:
+        return
+    head = (health.build_output or "").strip()[:600]
+    msg = (
+        "Baseline build FAILS — the integration gate disables itself without a "
+        "green baseline, so this run will be largely ungated. Fix the build first "
+        "(consider debug mode). Build error:\n" + (head or "(no output captured)")
+    )
+    logger.error(msg)
+    console.print(f"[red]Baseline build is failing.[/] {head[:200]}")
+    report.key_decisions.append(
+        "WARNING: baseline build was failing at start; gates degraded for this run"
+    )
+
+
+def _warn_if_no_test_gate(assessment, project, report) -> None:
+    """Loudly surface that a run will proceed with NO test gate while the project
+    has a test suite — the safety hole that let an ungated run rewrite existing
+    tests while reporting build OK. Detection covers most layouts; this catches
+    the residual case where a suite exists but no command was resolved.
+    """
+    if assessment.structure.test_command:
+        return
+    # Multi-target repos gate per sub-project, so an empty top-level command is
+    # not "no gate": declared targets with a build/test command (or auto-target
+    # discovery) provide the protection. Don't cry wolf in that case.
+    cfg = getattr(project, "config", {}) or {}
+    targets = cfg.get("targets") or []
+    # `targets` is an open, non-schema-validated config list, so a malformed
+    # project.yaml could hold non-dict entries; guard so this advisory warning
+    # never aborts the build with an AttributeError.
+    if any(
+        isinstance(t, dict) and (t.get("test_command") or t.get("build_command"))
+        for t in targets
+    ):
+        return
+    if get_setting(cfg, "orchestrator", "auto_targets"):
+        return
+    if not has_test_files(project.path):
+        return
+    msg = (
+        "No test command was detected, but this project HAS test files. The run "
+        "will proceed with only a build/syntax gate, so existing tests will NOT "
+        "protect against regressions and edits to them will not be caught. Set "
+        "`test_command` in project.yaml to enable the test gate."
+    )
+    logger.warning(msg)
+    console.print(f"[yellow]No test gate:[/] {msg}")
+    report.degraded_subsystems.append(
+        "No test gate: test files exist but no test command was detected"
+    )
+
+
+def _check_golden_config(config) -> None:
+    """Warn when the golden suite is half-configured (a silent integrity hole).
+
+    The two halves are independent: ``golden_paths`` protects+conceals the
+    files, ``golden_command`` enforces them as a gate. Configuring one without
+    the other silently drops a guarantee, so surface it loudly.
+    """
+    orch = config.get("orchestrator", {})
+    paths = orch.get("golden_paths") or []
+    command = orch.get("golden_command")
+    if command and not paths:
+        logger.warning(
+            "golden_command is set but golden_paths is empty: golden tests are "
+            "enforced as a gate but NOT protected from edits; the model could "
+            "weaken them. Set golden_paths to the same files."
+        )
+    if paths and not command:
+        logger.warning(
+            "golden_paths is set but golden_command is empty: golden files are "
+            "protected and hidden but never run as a gate. Set golden_command "
+            "to enforce them."
+        )

misterdev/analyzers/project_analyzer/__init__.py

@@ -0,0 +1,246 @@
+"""Project analyzer ported from /build Phase 1.
+
+Uses LLM to analyze project structure, completeness, and context,
+then merges results into a ProjectAssessment. In /build these run
+as 3 parallel Claude sub-agents; here they are 3 sequential LLM
+calls (or concurrent via threading if desired).
+"""
+
+import json
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Optional
+
+from misterdev.core.planning.assessment import ProjectAssessment
+from misterdev.core.verification.validator import run_health_check
+from misterdev.llm.client import BaseLLMClient
+from misterdev.logging_setup import setup_logger
+
+from .detection import (
+    detect_build_command,
+    detect_test_command,
+    has_test_files,
+    _file_mentions,
+    _has_node_tests,
+    _has_python_tests,
+    _json_has_test_script,
+)
+from .merge import (
+    _as_int,
+    _as_str_list,
+    _health_ground_truth,
+    _merge_completeness,
+    _merge_context,
+    _merge_debt_risk,
+    _merge_structure,
+)
+from .overview import (
+    _IGNORE_DIRS,
+    _INTENT_KEYWORDS,
+    _OVERVIEW_CODE_EXTS,
+    _get_file_listing,
+    _get_git_log,
+    _get_source_overview,
+    _leading_doc,
+    _read_config_files,
+    _read_docs,
+    _read_file_safe,
+    _walk_limited,
+)
+from .prompts import (
+    COMPLETENESS_PROMPT,
+    CONTEXT_PROMPT,
+    DEBT_RISK_PROMPT,
+    STRUCTURE_PROMPT,
+)
+
+logger = setup_logger(__name__)
+
+
+def analyze_project(
+    project_path: Path,
+    llm_client: BaseLLMClient,
+    build_command: Optional[str] = None,
+    test_command: Optional[str] = None,
+    lint_command: Optional[str] = None,
+    env_activate: Optional[str] = None,
+    parallel: bool = True,
+    build_timeout: Optional[int] = None,
+    test_timeout: Optional[int] = None,
+    lint_timeout: Optional[int] = None,
+    project_outline: Optional[str] = None,
+) -> ProjectAssessment:
+    """Run all Phase 1 analyses and merge into a ProjectAssessment.
+
+    ``project_outline``, when supplied, is the project's already-built symbol
+    outline (its TopographyEngine graph); passing it avoids parsing a second
+    throwaway symbol graph just for the source overview.
+    """
+    assessment = ProjectAssessment()
+
+    # Gather raw project info for prompts
+    file_listing = _get_file_listing(project_path)
+    config_contents = _read_config_files(project_path)
+    docs = _read_docs(project_path)
+    source_overview = _get_source_overview(project_path, outline=project_outline)
+    readme = _read_file_safe(project_path / "README.md")
+    claude_md = _read_file_safe(project_path / "CLAUDE.md")
+    git_log = _get_git_log(project_path)
+
+    # Run the health check FIRST, using reliable config + deterministic
+    # detection (not LLM-guessed commands), so the completeness analyzer is
+    # grounded in what actually builds and passes — otherwise it reads the
+    # from-scratch docs and hallucinates that implemented features are missing.
+    bc = build_command or detect_build_command(project_path)
+    tc = test_command or detect_test_command(project_path)
+    logger.info(
+        "Running health check (build + tests) to ground the analysis; "
+        "this can take a few minutes on a large project..."
+    )
+    assessment.health = run_health_check(
+        project_path,
+        bc,
+        tc,
+        lint_command,
+        env_activate=env_activate,
+        build_timeout=build_timeout,
+        test_timeout=test_timeout,
+        lint_timeout=lint_timeout,
+    )
+    assessment.structure.build_command = bc
+    assessment.structure.test_command = tc
+    health_ground = _health_ground_truth(assessment.health)
+
+    def analyze_structure():
+        prompt = STRUCTURE_PROMPT.format(
+            file_listing=file_listing,
+            config_contents=config_contents,
+        )
+        return _call_llm_json(llm_client, prompt, "project structure analyzer")
+
+    def analyze_completeness():
+        prompt = COMPLETENESS_PROMPT.format(
+            docs=docs,
+            source_overview=source_overview,
+            health_ground=health_ground,
+        )
+        return _call_llm_json(llm_client, prompt, "completeness analyzer")
+
+    def analyze_context():
+        prompt = CONTEXT_PROMPT.format(
+            readme=readme,
+            config=claude_md or config_contents,
+            git_log=git_log,
+        )
+        return _call_llm_json(llm_client, prompt, "context analyzer")
+
+    def analyze_debt_risk(current_summary: str):
+        prompt = DEBT_RISK_PROMPT.format(
+            assessment_summary=current_summary,
+            source_overview=source_overview,
+        )
+        return _call_llm_json(llm_client, prompt, "debt and risk analyzer")
+
+    # Phase 1a-1c: run analyses (parallel or sequential)
+    results = {}
+    if parallel:
+        with ThreadPoolExecutor(max_workers=4) as pool:
+            futures = {
+                pool.submit(analyze_structure): "structure",
+                pool.submit(analyze_completeness): "completeness",
+                pool.submit(analyze_context): "context",
+            }
+            for future in as_completed(futures):
+                key = futures[future]
+                try:
+                    results[key] = future.result()
+                except Exception as e:
+                    logger.error(f"Analysis failed for {key}: {e}")
+                    results[key] = {}
+
+            # Merge preliminary results to feed into debt/risk analyzer
+            _merge_structure(assessment, results.get("structure", {}))
+            _merge_completeness(assessment, results.get("completeness", {}))
+            _merge_context(assessment, results.get("context", {}))
+
+            future_debt = pool.submit(analyze_debt_risk, assessment.summary())
+            results["debt_risk"] = future_debt.result()
+    else:
+        results["structure"] = analyze_structure()
+        results["completeness"] = analyze_completeness()
+        results["context"] = analyze_context()
+        _merge_structure(assessment, results.get("structure", {}))
+        _merge_completeness(assessment, results.get("completeness", {}))
+        _merge_context(assessment, results.get("context", {}))
+        results["debt_risk"] = analyze_debt_risk(assessment.summary())
+
+    # Phase 1d: merge remaining into assessment
+    _merge_debt_risk(assessment, results.get("debt_risk", {}))
+
+    # Health already ran (before the analyzers, to ground them). Re-assert the
+    # deterministic commands in case the structure analyzer overwrote them with
+    # nulls during merge, since downstream safety gates read them.
+    if not assessment.structure.test_command:
+        assessment.structure.test_command = tc
+    if not assessment.structure.build_command:
+        assessment.structure.build_command = bc
+
+    logger.info(f"Assessment complete: {assessment.summary()}")
+    return assessment
+
+
+def _call_llm_json(llm_client: BaseLLMClient, prompt: str, role: str) -> dict:
+    """Call LLM and parse JSON response."""
+    logger.info(f"Running {role}...")
+    try:
+        response = llm_client.generate_code(
+            prompt, f"You are a {role}. Return only valid JSON."
+        )
+        text = response.strip()
+        if text.startswith("```"):
+            lines = text.split("\n")
+            lines = [ln for ln in lines if not ln.strip().startswith("```")]
+            text = "\n".join(lines)
+        return json.loads(text)
+    except json.JSONDecodeError:
+        logger.warning(f"{role} returned non-JSON response")
+        return {}
+    except Exception as e:
+        logger.error(f"{role} failed: {e}")
+        return {}
+
+
+__all__ = [
+    "STRUCTURE_PROMPT",
+    "COMPLETENESS_PROMPT",
+    "CONTEXT_PROMPT",
+    "DEBT_RISK_PROMPT",
+    "analyze_project",
+    "_call_llm_json",
+    "detect_test_command",
+    "detect_build_command",
+    "has_test_files",
+    "_has_python_tests",
+    "_has_node_tests",
+    "_file_mentions",
+    "_json_has_test_script",
+    "_health_ground_truth",
+    "_as_str_list",
+    "_as_int",
+    "_merge_structure",
+    "_merge_completeness",
+    "_merge_context",
+    "_merge_debt_risk",
+    "_IGNORE_DIRS",
+    "_OVERVIEW_CODE_EXTS",
+    "_INTENT_KEYWORDS",
+    "_walk_limited",
+    "_get_file_listing",
+    "_read_config_files",
+    "_read_docs",
+    "_leading_doc",
+    "_get_source_overview",
+    "_get_git_log",
+    "_read_file_safe",
+    "run_health_check",
+]

misterdev/analyzers/project_analyzer/detection.py

@@ -0,0 +1,146 @@
+"""Deterministic project-marker detection of build/test commands."""
+
+import json
+from pathlib import Path
+from typing import Optional
+
+
+def detect_test_command(project_path: Path) -> Optional[str]:
+    """Detect a test command from project markers, independent of the LLM.
+
+    Returns a runnable command string, or None if no recognized test setup is
+    found. Used as a fallback when the structure analyzer leaves test_command
+    null, which would otherwise leave the suite un-run and the health check
+    blind (reporting tests=none on a project with a passing suite).
+    """
+    p = project_path
+    # Python FIRST, but only on a genuine Python signal — a bare ``tests/`` dir
+    # is NOT pytest (Rust uses tests/ for integration tests, Node for its own
+    # runner), so it must not shadow the language-specific runners below.
+    if _has_python_tests(p):
+        return "uv run pytest -q" if (p / "uv.lock").exists() else "pytest -q"
+    if (p / "Cargo.toml").exists():
+        return "cargo test"
+    # Node: an explicit `test` script wins; otherwise the built-in node test
+    # runner over a discoverable *.test.* suite (the case that, when missed, left
+    # a real suite ungated and silently rewritten).
+    if (p / "package.json").exists():
+        if _json_has_test_script(p / "package.json"):
+            return "npm test"
+        if _has_node_tests(p):
+            return "node --test"
+    if (p / "Package.swift").exists():
+        return "swift test"
+    # .NET: a solution runs every test project; a bare test csproj runs itself.
+    if any(p.glob("*.sln")) or any(p.glob("*[Tt]est*.csproj")):
+        return "dotnet test"
+    # GTK/meson tests run against a configured build dir; cmake/C++ via ctest.
+    if (p / "meson.build").exists():
+        return "meson test -C build"
+    if (p / "CMakeLists.txt").exists():
+        return "ctest --test-dir build --output-on-failure"
+    if (p / "go.mod").exists():
+        return "go test ./..."
+    return None
+
+
+def _has_python_tests(p: Path) -> bool:
+    """True when ``p`` looks like a Python project with a pytest-runnable suite.
+
+    Explicit pytest config is definitive. A ``tests/`` directory only counts when
+    it holds Python test files OR the project is clearly Python (uv.lock /
+    pyproject.toml / setup.py / setup.cfg) — so a Node or Rust project that merely
+    uses ``tests/`` for its own framework is never misrouted to pytest.
+    """
+    if (
+        (p / "pytest.ini").exists()
+        or (p / "conftest.py").exists()
+        or _file_mentions(p / "pyproject.toml", "pytest")
+        or _file_mentions(p / "setup.cfg", "pytest")
+    ):
+        return True
+    is_python_project = (
+        (p / "uv.lock").exists()
+        or (p / "pyproject.toml").exists()
+        or (p / "setup.py").exists()
+        or (p / "setup.cfg").exists()
+    )
+    for d in (p / "tests", p / "test"):
+        if d.is_dir():
+            if any(d.rglob("test_*.py")) or any(d.rglob("*_test.py")):
+                return True
+            if is_python_project:
+                return True
+    return False
+
+
+def _has_node_tests(p: Path) -> bool:
+    """True when a ``test/``/``tests/`` dir holds node-runner test files.
+
+    Looks only for the unambiguous ``*.test.{js,mjs,cjs}`` naming the built-in
+    ``node --test`` runner discovers, and only inside conventional test dirs so
+    ``node_modules`` is never scanned.
+    """
+    for d in (p / "test", p / "tests"):
+        if d.is_dir():
+            for pat in ("*.test.js", "*.test.mjs", "*.test.cjs"):
+                if any(d.rglob(pat)):
+                    return True
+    return False
+
+
+def has_test_files(project_path: Path) -> bool:
+    """True when the project appears to contain a test suite of any kind.
+
+    Used to warn when a build is about to run with no test gate even though tests
+    exist — the safety hole behind a real run that rewrote an ungated suite.
+    """
+    p = project_path
+    if _has_python_tests(p) or _has_node_tests(p):
+        return True
+    for d in (p / "tests", p / "test"):
+        if d.is_dir() and any(d.rglob("*")):
+            return True
+    return False
+
+
+def detect_build_command(project_path: Path) -> Optional[str]:
+    """Detect a build/compile-check command from project markers."""
+    p = project_path
+    if (p / "Cargo.toml").exists():
+        return "cargo build"
+    if (p / "package.json").exists():
+        # Prefer a `typecheck` script: for a TS project it's the fast, deterministic
+        # gate (tsc --noEmit), whereas `build` often runs heavy bundling/wasm.
+        if _json_has_test_script(p / "package.json", key="typecheck"):
+            return "npm run typecheck"
+        if _json_has_test_script(p / "package.json", key="build"):
+            return "npm run build"
+    if (p / "Package.swift").exists():
+        return "swift build"
+    if any(p.glob("*.sln")) or any(p.glob("*.csproj")):
+        return "dotnet build"
+    if (p / "meson.build").exists():
+        return "meson compile -C build"
+    if (p / "CMakeLists.txt").exists():
+        return "cmake --build build"
+    if (p / "Makefile").exists() or (p / "makefile").exists():
+        return "make"
+    if (p / "pyproject.toml").exists() or (p / "setup.py").exists():
+        return "python -m compileall -q ."
+    return None
+
+
+def _file_mentions(path: Path, needle: str) -> bool:
+    try:
+        return needle in path.read_text(encoding="utf-8")
+    except OSError:
+        return False
+
+
+def _json_has_test_script(path: Path, key: str = "test") -> bool:
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return False
+    return bool(data.get("scripts", {}).get(key))