misterdev 0.2.0__py3-none-any.whl

misterdev/analyzers/project_analyzer/merge.py

@@ -0,0 +1,137 @@
+"""Coercion helpers and mergers that fold analyzer JSON into the assessment."""
+
+from misterdev.core.planning.assessment import (
+    FeatureInfo,
+    ProjectAssessment,
+)
+
+
+def _health_ground_truth(health) -> str:
+    """One-line verified-state preamble to anchor the completeness analyzer."""
+    build = "passes" if health.builds else "FAILS"
+    if health.test_count:
+        passing = health.test_count - health.test_failures
+        tests = f"{passing}/{health.test_count} tests passing"
+    elif health.tests_pass:
+        tests = "test suite passes"
+    else:
+        tests = "no test results"
+    return f"VERIFIED ground truth: build {build}; {tests}."
+
+
+def _as_str_list(value, default: list) -> list:
+    """Coerce an LLM-provided value to a ``list[str]``, else return ``default``.
+
+    The analyzers occasionally emit ``null`` (or a bare string) for a field the
+    schema types as a list. ``dict.get(k, default)`` only falls back when the key
+    is ABSENT, so a present ``null`` would store ``None`` on the typed field
+    (Pydantic does not validate on assignment) and later crash a ``for``/``join``
+    over it. Normalizing here keeps the trust boundary sound.
+    """
+    if isinstance(value, list):
+        return [v for v in value if isinstance(v, str)]
+    return default
+
+
+def _as_int(value, default: int) -> int:
+    """Coerce an LLM-provided value to ``int``, else return ``default``.
+
+    Tolerates a numeric string ("75") or float (75.0); rejects bools, ``null``,
+    and unparseable text so a typed ``int`` field never holds a string.
+    """
+    if isinstance(value, bool):
+        return default
+    if isinstance(value, (int, float)):
+        return int(value)
+    if isinstance(value, str):
+        try:
+            return int(value.strip())
+        except ValueError:
+            return default
+    return default
+
+
+def _merge_structure(assessment: ProjectAssessment, data: dict) -> None:
+    if not data:
+        return
+    s = assessment.structure
+    s.project_type = data.get("project_type") or s.project_type
+    s.languages = _as_str_list(data.get("languages"), s.languages)
+    s.frameworks = _as_str_list(data.get("frameworks"), s.frameworks)
+    s.build_command = data.get("build_command", s.build_command)
+    s.test_command = data.get("test_command", s.test_command)
+    s.lint_command = data.get("lint_command", s.lint_command)
+    s.package_manager = data.get("package_manager", s.package_manager)
+    s.entry_points = _as_str_list(data.get("entry_points"), s.entry_points)
+    s.directory_structure = data.get("directory_structure") or s.directory_structure
+
+
+def _merge_completeness(assessment: ProjectAssessment, data: dict) -> None:
+    if not data:
+        return
+    f = assessment.features
+    for item in data.get("existing") or []:
+        if isinstance(item, dict):
+            f.existing.append(
+                FeatureInfo(
+                    name=item.get("name", ""), description=item.get("description", "")
+                )
+            )
+    for item in data.get("incomplete") or []:
+        if isinstance(item, dict):
+            f.incomplete.append(
+                FeatureInfo(
+                    name=item.get("name", ""),
+                    description=item.get("description", ""),
+                    complexity=item.get("complexity", "medium"),
+                )
+            )
+    for item in data.get("missing") or []:
+        if isinstance(item, dict):
+            f.missing.append(
+                FeatureInfo(
+                    name=item.get("name", ""),
+                    description=item.get("description", ""),
+                    complexity=item.get("complexity", "medium"),
+                )
+            )
+    f.dead_code = _as_str_list(data.get("dead_code"), f.dead_code)
+    f.stubs = _as_str_list(data.get("stubs"), f.stubs)
+    f.broken = _as_str_list(data.get("broken"), f.broken)
+    todos = data.get("todos")
+    f.todos = todos if isinstance(todos, list) else f.todos
+
+
+def _merge_context(assessment: ProjectAssessment, data: dict) -> None:
+    if not data:
+        return
+    c = assessment.context
+    c.purpose = data.get("purpose", c.purpose)
+    c.goals = data.get("goals", c.goals)
+    c.conventions = data.get("conventions", c.conventions)
+    c.constraints = data.get("constraints", c.constraints)
+    c.recent_activity = data.get("recent_activity", c.recent_activity)
+    c.stated_requirements = data.get("stated_requirements", c.stated_requirements)
+
+
+def _merge_debt_risk(assessment: ProjectAssessment, data: dict) -> None:
+    if not data:
+        return
+
+    debt_data = data.get("tech_debt") or {}
+    if debt_data:
+        assessment.tech_debt.score = _as_int(debt_data.get("score"), 0)
+        assessment.tech_debt.description = debt_data.get("description") or ""
+        assessment.tech_debt.critical_issues = _as_str_list(
+            debt_data.get("critical_issues"), assessment.tech_debt.critical_issues
+        )
+
+    risk_data = data.get("risk") or {}
+    if risk_data:
+        assessment.risk.level = risk_data.get("level") or "low"
+        assessment.risk.factors = _as_str_list(
+            risk_data.get("factors"), assessment.risk.factors
+        )
+        assessment.risk.mitigations = _as_str_list(
+            risk_data.get("mitigations"), assessment.risk.mitigations
+        )

misterdev/analyzers/project_analyzer/overview.py

@@ -0,0 +1,312 @@
+"""Raw project gathering: file listing, config/doc reads, source overview, git log."""
+
+import re
+from pathlib import Path
+from typing import Optional
+
+from misterdev.core.gitcmd import run_git
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+
+_IGNORE_DIRS = {
+    "venv",
+    ".venv",
+    "node_modules",
+    "__pycache__",
+    ".git",
+    "target",
+    "build",
+    "dist",
+    ".tox",
+    ".mypy_cache",
+    ".ruff_cache",
+    ".pytest_cache",
+    ".eggs",
+    "vendor",
+}
+
+
+def _walk_limited(root: Path, max_depth: int = 6):
+    """Yield files under root, pruning ignored dirs and bounding depth.
+
+    Unlike Path.rglob, this prunes large directories (node_modules, target)
+    during traversal instead of descending into them and filtering afterward,
+    so scanning a monorepo does not stall on vendored trees.
+    """
+    stack = [(root, 0)]
+    while stack:
+        directory, depth = stack.pop()
+        try:
+            entries = sorted(directory.iterdir())
+        except (PermissionError, OSError):
+            continue
+        for entry in entries:
+            if entry.is_dir():
+                if entry.name in _IGNORE_DIRS or entry.name.startswith("."):
+                    continue
+                if depth < max_depth:
+                    stack.append((entry, depth + 1))
+            elif entry.is_file():
+                yield entry
+
+
+def _get_file_listing(
+    project_path: Path, max_files: int = 200, max_depth: int = 6
+) -> str:
+    """Get a truncated file listing for the project."""
+    files = []
+    for item in _walk_limited(project_path, max_depth):
+        rel = item.relative_to(project_path)
+        files.append(str(rel))
+        if len(files) >= max_files:
+            files.append(f"... ({max_files}+ files, truncated)")
+            break
+    return "\n".join(sorted(files))
+
+
+def _read_config_files(project_path: Path) -> str:
+    """Read common config files for structure analysis."""
+    config_names = [
+        "pyproject.toml",
+        "setup.py",
+        "setup.cfg",
+        "package.json",
+        "Cargo.toml",
+        "go.mod",
+        "Makefile",
+        "CMakeLists.txt",
+        "project.yaml",
+        "tsconfig.json",
+        "webpack.config.js",
+    ]
+    contents = []
+    for name in config_names:
+        text = _read_file_safe(project_path / name, max_lines=100)
+        if text:
+            contents.append(f"### {name}\n{text}")
+    return "\n\n".join(contents) if contents else "(no config files found)"
+
+
+def _read_docs(project_path: Path) -> str:
+    """Read documentation files."""
+    doc_names = ["README.md", "CLAUDE.md", "SPEC.md", "DESIGN.md", "REQUIREMENTS.md"]
+    contents = []
+    for name in doc_names:
+        text = _read_file_safe(project_path / name, max_lines=200)
+        if text:
+            contents.append(f"### {name}\n{text}")
+    return "\n\n".join(contents) if contents else "(no docs found)"
+
+
+_OVERVIEW_CODE_EXTS = {
+    ".py",
+    ".js",
+    ".jsx",
+    ".ts",
+    ".tsx",
+    ".rs",
+    ".go",
+    ".java",
+    ".c",
+    ".h",
+    ".cpp",
+    ".cc",
+    ".cxx",
+    ".hpp",
+    ".hh",
+    ".swift",
+    ".cs",
+    ".kt",
+}
+
+
+# Phrases that mark an empty/default/no-op result as deliberate rather than
+# unfinished. When a file's leading doc contains one, the sentence carrying it is
+# preserved into the overview so the completeness analyzer does not flag the code
+# as a stub.
+_INTENT_KEYWORDS = (
+    "degrade",
+    "no-op",
+    "noop",
+    "fallback",
+    "parity",
+    "by design",
+    "intentional",
+    "graceful",
+    "historical",
+    "never panic",
+)
+
+
+def _leading_doc(path: Path, max_lines: int = 40, max_chars: int = 220) -> str:
+    """Extract a source file's leading comment/doc block as one compact line.
+
+    Carries each file's stated *intent* (e.g. "degrades to empty on wasm — the
+    historical missing-model contract") into completeness analysis, so a deep
+    file is judged by its documented purpose rather than guessed from its symbol
+    names. Returns "" when the file opens with code and no leading comment.
+    """
+    try:
+        with path.open("r", encoding="utf-8", errors="replace") as fh:
+            head = [next(fh, "") for _ in range(max_lines)]
+    except OSError:
+        return ""
+
+    # `#` is a line comment in Python but a preprocessor directive in C-family
+    # files, so only treat it (and `"""` docstrings) as a comment for Python.
+    is_py = path.suffix == ".py"
+    line_prefixes = ("///", "//!", "//", "#") if is_py else ("///", "//!", "//")
+    doc: list[str] = []
+    in_block = False
+    block_end = ""
+    for raw in head:
+        line = raw.strip()
+        if not in_block:
+            if not line:
+                # Skip blank lines anywhere in the leading comment region (not just
+                # before the first comment), so an intent stated after a blank
+                # separator line is still collected; the first real CODE line below
+                # still ends the block.
+                continue
+            if not doc and line.startswith("#!"):
+                continue  # shebang
+        if in_block:
+            end = line.find(block_end)
+            seg = (line[:end] if end != -1 else line).lstrip("*").strip()
+            if seg:
+                doc.append(seg)
+            if end != -1:
+                break
+            continue
+        if is_py and (line.startswith('"""') or line.startswith("'''")):
+            quote = line[:3]
+            rest = line[3:].rstrip()
+            if rest.endswith(quote) and len(rest) >= 3:
+                doc.append(rest[:-3].strip())
+                break
+            in_block, block_end = True, quote
+            if rest.strip():
+                doc.append(rest.strip())
+            continue
+        if line.startswith("/*"):
+            rest = line[2:]
+            if "*/" in rest:
+                doc.append(rest.split("*/", 1)[0].strip())
+                break
+            in_block, block_end = True, "*/"
+            seg = rest.lstrip("*").strip()
+            if seg:
+                doc.append(seg)
+            continue
+        matched = next((p for p in line_prefixes if line.startswith(p)), None)
+        if matched is not None:
+            doc.append(line[len(matched) :].strip())
+            continue
+        break  # first line of real code ends the leading doc block
+
+    full = " ".join(d for d in doc if d).strip()
+    if not full:
+        return ""
+    summary = full[:max_chars]
+    # The strongest "this empty/no-op is intentional" signal often sits a few
+    # sentences into a module doc, past the summary cap. If the block states such
+    # intent, graft the sentence that says so onto the summary so it is never
+    # truncated away — this is exactly what stops a documented graceful-degrade
+    # backend from being misread as an unfinished stub.
+    lowered = full.lower()
+    if any(k in lowered for k in _INTENT_KEYWORDS):
+        for sentence in re.split(r"(?<=[.;])\s+", full):
+            sl = sentence.lower()
+            if any(k in sl for k in _INTENT_KEYWORDS) and sentence not in summary:
+                summary = f"{summary.rstrip()} … {sentence.strip()}"[: max_chars + 180]
+                break
+    return summary
+
+
+def _get_source_overview(
+    project_path: Path, max_chars: int = 8000, outline: Optional[str] = None
+) -> str:
+    """Whole-project structural map plus the heads of source files.
+
+    The symbol map (every file and its symbols, from tree-sitter) conveys the
+    architecture densely so planning is grounded in the entire project rather
+    than the first few files that fit ``max_chars`` of raw heads. A per-file
+    intent map (leading doc comments) then carries each file's documented purpose
+    — including deliberate graceful-degradation — so deep files are not judged by
+    their symbol names alone.
+
+    ``outline`` lets the caller pass an already-built symbol outline (the
+    project's TopographyEngine graph). When given, this reuses it instead of
+    parsing a second throwaway ``SymbolGraph`` here, so the whole-project graph is
+    built once per run rather than once for the overview AND once for the engine.
+    """
+    parts = []
+    if outline is None:
+        try:
+            from misterdev.core.context.topography import SymbolGraph
+
+            graph = SymbolGraph(project_path)
+            graph.build()
+            outline = graph.project_outline()
+        except (ImportError, OSError, ValueError) as e:
+            logger.debug(f"symbol-based overview unavailable: {e}")
+            outline = ""
+    if outline:
+        parts.append("## Project structure (files and symbols)\n" + outline)
+
+    intents = []
+    intent_total = 0
+    for item in _walk_limited(project_path):
+        if item.suffix in _OVERVIEW_CODE_EXTS:
+            doc = _leading_doc(item)
+            if doc:
+                line = f"{item.relative_to(project_path)}: {doc}"
+                intents.append(line)
+                intent_total += len(line) + 1
+                if intent_total >= 16000:
+                    break
+    if intents:
+        parts.append(
+            "## File intents (leading doc comments)\n"
+            "Documented graceful-degradation, platform-gated no-ops, and parity "
+            "shims are intentional and COMPLETE — not stubs.\n" + "\n".join(intents)
+        )
+
+    head = []
+    total = 0
+    for item in _walk_limited(project_path):
+        if item.suffix in _OVERVIEW_CODE_EXTS:
+            text = _read_file_safe(item, max_lines=30)
+            if text:
+                rel = item.relative_to(project_path)
+                chunk = f"### {rel}\n{text}\n"
+                head.append(chunk)
+                total += len(chunk)
+                if total >= max_chars:
+                    break
+    if head:
+        parts.append("## Source heads\n" + "\n".join(head))
+    return "\n\n".join(parts) if parts else "(no source files found)"
+
+
+def _get_git_log(project_path: Path, count: int = 20) -> str:
+    """Get recent git log."""
+    proc = run_git(f"git log --oneline -n {count}", project_path, timeout=10)
+    if proc is None:
+        return "(git log unavailable)"
+    return proc.stdout.strip() if proc.returncode == 0 else "(not a git repo)"
+
+
+def _read_file_safe(path: Path, max_lines: int = 500) -> str:
+    """Read a file, returning empty string on failure."""
+    try:
+        if not path.exists():
+            return ""
+        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+        if len(lines) > max_lines:
+            lines = lines[:max_lines] + [f"... ({len(lines)} lines total, truncated)"]
+        return "\n".join(lines)
+    except Exception:
+        return ""

misterdev/analyzers/project_analyzer/prompts.py

@@ -0,0 +1,83 @@
+"""Prompt templates for the Phase 1 project analyzers."""
+
+STRUCTURE_PROMPT = """Analyze the project at the path below. Return a JSON object with these exact fields:
+  project_type (string: web-api, web-app, cli, library, worker, static-site, monorepo, or unknown),
+  languages (array of strings),
+  frameworks (array of strings),
+  build_command (string or null),
+  test_command (string or null),
+  lint_command (string or null),
+  package_manager (string or null),
+  entry_points (array of file path strings),
+  directory_structure (string, tree of src dirs, max 30 lines)
+
+Project files:
+{file_listing}
+
+Key config files:
+{config_contents}
+
+Return ONLY valid JSON, no markdown fences."""
+
+COMPLETENESS_PROMPT = """Analyze this project for completeness. Return a JSON object with:
+  existing (array of objects with "name" and "description"),
+  incomplete (array of objects with "name", "description", and "complexity"),
+  missing (array of objects with "name", "description", and "complexity"),
+  dead_code (array of file path strings),
+  stubs (array of file path strings),
+  broken (array of file path strings),
+  todos (array of objects with "file", "line", "text")
+
+{health_ground}
+Treat code that builds and is covered by passing tests as IMPLEMENTED. Docs may
+describe a from-scratch plan; do NOT report already-built, tested capabilities
+as missing or incomplete. Base "missing"/"broken" on the source, not the docs.
+
+Code documented as intentional is COMPLETE, not incomplete: graceful-degradation
+and fallback paths, platform-gated no-ops (e.g. a wasm/no-filesystem backend that
+returns empty by design), and shims "retained for parity". A function returning an
+empty or default value is NOT a stub when a comment or the design states that empty
+result is the contract — check the "File intents" section before flagging. For each
+item you place in "incomplete"/"stubs"/"missing", name the specific file and the
+concrete unmet behavior; if you cannot, omit it. Never infer incompleteness from a
+symbol name or a default return alone.
+
+Project docs:
+{docs}
+
+Project source overview:
+{source_overview}
+
+Return ONLY valid JSON, no markdown fences."""
+
+CONTEXT_PROMPT = """Gather context for this project. Return a JSON object with:
+  purpose (string, 1-2 sentences),
+  goals (string),
+  conventions (string, coding conventions found),
+  constraints (string, requirements or compatibility needs),
+  recent_activity (string, summary of recent work),
+  stated_requirements (string, from spec/design docs)
+
+README:
+{readme}
+
+CLAUDE.md / config:
+{config}
+
+Recent git log:
+{git_log}
+
+Return ONLY valid JSON, no markdown fences."""
+
+
+DEBT_RISK_PROMPT = """Analyze this project for technical debt and implementation risk. Return a JSON object with:
+  tech_debt (object with "score" [0-100], "description", "critical_issues"),
+  risk (object with "level" [low, medium, high, critical], "factors", "mitigations")
+
+Project assessment so far:
+{assessment_summary}
+
+Source code overview:
+{source_overview}
+
+Return ONLY valid JSON, no markdown fences."""