misterdev 0.2.0__py3-none-any.whl

misterdev/core/context/contracts/rust_line.py

@@ -0,0 +1,225 @@
+"""Line-based heuristic public-symbol extraction for Rust source.
+
+Fallback used when the tree-sitter Rust grammar is unavailable.
+"""
+
+from typing import Dict, List
+
+from ._text import _extract_name
+
+
+def _extract_rust_symbols(lines: List[str]) -> List[Dict[str, str]]:
+    """Extract public API from Rust source.
+
+    Handles: pub fn, pub(crate) fn, pub struct (with fields), pub enum,
+    pub trait (with method signatures), pub type, pub const, impl blocks.
+    Collects multi-line signatures up to the opening brace.
+    """
+    symbols = []
+    i = 0
+    brace_depth = 0
+    while i < len(lines):
+        stripped = lines[i].strip()
+        line_opens = stripped.count("{") - stripped.count("}")
+
+        # Skip lines inside bodies (impl, struct, enum, trait blocks)
+        if brace_depth > 0:
+            brace_depth += line_opens
+            i += 1
+            continue
+
+        # Match pub, pub(crate), pub(super)
+        if not (stripped.startswith("pub ") or stripped.startswith("pub(")):
+            # Check for impl blocks (extract public methods inside)
+            if stripped.startswith("impl ") or stripped.startswith("impl<"):
+                impl_name = _extract_impl_name(stripped)
+                impl_methods = _extract_impl_methods(lines, i)
+                for m in impl_methods:
+                    m["name"] = f"{impl_name}::{m['name']}"
+                symbols.extend(impl_methods)
+                # Skip past the impl block
+                brace_depth += line_opens
+            else:
+                brace_depth += line_opens
+            i += 1
+            continue
+
+        # Strip visibility qualifier to get the declaration
+        after_vis = _strip_visibility(stripped)
+
+        if after_vis.startswith("fn "):
+            sig = _collect_signature(lines, i, "{")
+            symbols.append(
+                {
+                    "kind": "pub fn",
+                    "name": _extract_name(after_vis[3:]),
+                    "signature": sig,
+                }
+            )
+        elif after_vis.startswith("struct "):
+            name = _extract_name(after_vis[7:])
+            sig = f"pub struct {name}"
+            # Collect generic bounds
+            generic = _extract_generics(after_vis[7:])
+            if generic:
+                sig = f"pub struct {name}{generic}"
+            fields = _collect_struct_fields(lines, i)
+            if fields:
+                sig += " { " + ", ".join(fields) + " }"
+            symbols.append({"kind": "pub struct", "name": name, "signature": sig})
+        elif after_vis.startswith("enum "):
+            name = _extract_name(after_vis[5:])
+            variants = _collect_enum_variants(lines, i)
+            sig = f"pub enum {name}"
+            if variants:
+                sig += " { " + ", ".join(variants) + " }"
+            symbols.append({"kind": "pub enum", "name": name, "signature": sig})
+        elif after_vis.startswith("trait "):
+            name = _extract_name(after_vis[6:])
+            trait_methods = _collect_trait_methods(lines, i)
+            sig = f"pub trait {name}"
+            if trait_methods:
+                sig += " { " + "; ".join(trait_methods) + "; }"
+            symbols.append({"kind": "pub trait", "name": name, "signature": sig})
+        elif after_vis.startswith("type "):
+            symbols.append(
+                {
+                    "kind": "pub type",
+                    "name": _extract_name(after_vis[5:]),
+                    "signature": stripped,
+                }
+            )
+        elif after_vis.startswith("const "):
+            symbols.append(
+                {
+                    "kind": "pub const",
+                    "name": _extract_name(after_vis[6:]),
+                    "signature": stripped,
+                }
+            )
+
+        brace_depth += line_opens
+        i += 1
+
+    return symbols
+
+
+def _strip_visibility(line: str) -> str:
+    """Strip pub/pub(crate)/pub(super) prefix."""
+    if line.startswith("pub("):
+        close = line.find(")")
+        if close >= 0:
+            return line[close + 1 :].strip()
+    if line.startswith("pub "):
+        return line[4:].strip()
+    return line
+
+
+def _extract_generics(text: str) -> str:
+    """Extract <...> generic parameters."""
+    if "<" not in text:
+        return ""
+    depth = 0
+    start = text.find("<")
+    for i in range(start, len(text)):
+        if text[i] == "<":
+            depth += 1
+        elif text[i] == ">":
+            depth -= 1
+            if depth == 0:
+                return text[start : i + 1]
+    return ""
+
+
+def _extract_impl_name(line: str) -> str:
+    """Extract type name from 'impl<T> Foo<T> for Bar'."""
+    # Remove "impl" and optional generics
+    rest = line[4:].strip()
+    if rest.startswith("<"):
+        close = rest.find(">")
+        if close >= 0:
+            rest = rest[close + 1 :].strip()
+    return _extract_name(rest)
+
+
+def _extract_impl_methods(lines: List[str], start: int) -> List[Dict[str, str]]:
+    """Extract pub fn declarations from inside an impl block."""
+    methods = []
+    depth = 0
+    for i in range(start, min(start + 200, len(lines))):
+        depth += lines[i].count("{") - lines[i].count("}")
+        stripped = lines[i].strip()
+        if stripped.startswith("pub fn ") or stripped.startswith("pub(crate) fn "):
+            after_vis = _strip_visibility(stripped)
+            sig = _collect_signature(lines, i, "{")
+            methods.append(
+                {
+                    "kind": "pub fn",
+                    "name": _extract_name(after_vis[3:]),
+                    "signature": sig,
+                }
+            )
+        if depth <= 0 and i > start:
+            break
+    return methods
+
+
+def _collect_enum_variants(lines: List[str], start: int) -> List[str]:
+    """Collect variant names from a Rust enum."""
+    variants = []
+    depth = 0
+    for i in range(start, min(start + 50, len(lines))):
+        depth += lines[i].count("{") - lines[i].count("}")
+        stripped = lines[i].strip()
+        if i > start and depth >= 1 and stripped and not stripped.startswith("//"):
+            name = stripped.split("(")[0].split("{")[0].rstrip(",").strip()
+            if name and name[0].isupper():
+                variants.append(name)
+        if depth <= 0 and i > start:
+            break
+    return variants[:15]
+
+
+def _collect_trait_methods(lines: List[str], start: int) -> List[str]:
+    """Collect fn signatures from a trait definition."""
+    methods = []
+    depth = 0
+    for i in range(start, min(start + 50, len(lines))):
+        depth += lines[i].count("{") - lines[i].count("}")
+        stripped = lines[i].strip()
+        if stripped.startswith("fn "):
+            sig = stripped.rstrip(";").rstrip("{").strip()
+            methods.append(sig)
+        if depth <= 0 and i > start:
+            break
+    return methods[:10]
+
+
+def _collect_signature(lines: List[str], start: int, terminator: str) -> str:
+    """Collect a multi-line signature up to the terminator character."""
+    sig_parts = []
+    for i in range(start, min(start + 5, len(lines))):
+        sig_parts.append(lines[i].strip())
+        if terminator in lines[i]:
+            break
+    sig = " ".join(sig_parts)
+    # Trim at the terminator
+    idx = sig.find(terminator)
+    if idx >= 0:
+        sig = sig[:idx].strip()
+    return sig
+
+
+def _collect_struct_fields(lines: List[str], start: int) -> List[str]:
+    """Collect pub field names from a Rust struct."""
+    fields = []
+    brace_depth = 0
+    for i in range(start, min(start + 30, len(lines))):
+        line = lines[i].strip()
+        brace_depth += line.count("{") - line.count("}")
+        if "pub " in line and ":" in line:
+            field = line.strip().rstrip(",")
+            fields.append(field)
+        if brace_depth <= 0 and i > start:
+            break
+    return fields[:10]

misterdev/core/context/contracts/rust_tree_sitter.py

@@ -0,0 +1,141 @@
+"""Tree-sitter based public-symbol extraction for Rust source."""
+
+from typing import Dict, List, Optional
+
+from ._log import logger
+
+
+def _extract_rust_symbols_ts(content: str) -> List[Dict[str, str]]:
+    """Extract public Rust API using tree-sitter, reusing topography's parser.
+
+    Returns [] when tree-sitter or the Rust grammar is unavailable so the
+    caller can fall back to the line-based parser.
+    """
+    try:
+        from misterdev.core.context.topography import _get_ts_parsers
+
+        parser = _get_ts_parsers().get("rust")
+    except Exception as e:
+        logger.debug(f"tree-sitter rust parser unavailable: {e}")
+        return []
+    if parser is None:
+        return []
+    try:
+        tree = parser.parse(bytes(content, "utf8"))
+    except Exception as e:
+        logger.debug(f"tree-sitter parse failed; using line parser: {e}")
+        return []
+    symbols: List[Dict[str, str]] = []
+    _walk_rust_ts(tree.root_node, content, symbols, parent=None)
+    return symbols
+
+
+def _ts_is_pub(node) -> bool:
+    return any(c.type == "visibility_modifier" for c in node.children)
+
+
+def _ts_field_text(node, field: str, content: str) -> str:
+    n = node.child_by_field_name(field)
+    return content[n.start_byte : n.end_byte] if n else ""
+
+
+def _ts_decl(node, content: str) -> str:
+    """Declaration text up to the body (captures generics/where, drops body)."""
+    body = node.child_by_field_name("body")
+    end = body.start_byte if body else node.end_byte
+    return " ".join(content[node.start_byte : end].split()).strip()
+
+
+def _walk_rust_ts(
+    node, content: str, symbols: List[Dict[str, str]], parent: Optional[str]
+):
+    t = node.type
+    if t == "function_item":
+        if _ts_is_pub(node):
+            name = _ts_field_text(node, "name", content)
+            full = f"{parent}::{name}" if parent else name
+            symbols.append(
+                {"kind": "pub fn", "name": full, "signature": _ts_decl(node, content)}
+            )
+        return
+    if t == "struct_item" and _ts_is_pub(node):
+        name = _ts_field_text(node, "name", content)
+        sig = _ts_decl(node, content)
+        fields = _ts_pub_members(node, "field_declaration", content)
+        if fields:
+            sig += " { " + ", ".join(fields) + " }"
+        symbols.append({"kind": "pub struct", "name": name, "signature": sig})
+        return
+    if t == "enum_item" and _ts_is_pub(node):
+        name = _ts_field_text(node, "name", content)
+        variants = _ts_variant_names(node, content)
+        sig = _ts_decl(node, content)
+        if variants:
+            sig += " { " + ", ".join(variants) + " }"
+        symbols.append({"kind": "pub enum", "name": name, "signature": sig})
+        return
+    if t == "trait_item" and _ts_is_pub(node):
+        name = _ts_field_text(node, "name", content)
+        methods = _ts_trait_methods(node, content)
+        sig = _ts_decl(node, content)
+        if methods:
+            sig += " { " + "; ".join(methods) + "; }"
+        symbols.append({"kind": "pub trait", "name": name, "signature": sig})
+        return
+    if t == "type_item" and _ts_is_pub(node):
+        name = _ts_field_text(node, "name", content)
+        text = " ".join(content[node.start_byte : node.end_byte].split()).rstrip(";")
+        symbols.append({"kind": "pub type", "name": name, "signature": text})
+        return
+    if t == "impl_item":
+        type_node = node.child_by_field_name("type")
+        body = node.child_by_field_name("body")
+        if type_node is not None and body is not None:
+            impl_name = (
+                content[type_node.start_byte : type_node.end_byte].split("<")[0].strip()
+            )
+            for c in body.children:
+                _walk_rust_ts(c, content, symbols, parent=impl_name)
+        return
+    for c in node.children:
+        _walk_rust_ts(c, content, symbols, parent)
+
+
+def _ts_pub_members(node, child_type: str, content: str) -> List[str]:
+    body = node.child_by_field_name("body")
+    members = []
+    if body is not None:
+        for c in body.children:
+            if c.type == child_type and any(
+                g.type == "visibility_modifier" for g in c.children
+            ):
+                members.append(
+                    " ".join(content[c.start_byte : c.end_byte].split()).rstrip(",")
+                )
+    return members[:10]
+
+
+def _ts_variant_names(node, content: str) -> List[str]:
+    body = node.child_by_field_name("body")
+    names = []
+    if body is not None:
+        for c in body.children:
+            if c.type == "enum_variant":
+                n = c.child_by_field_name("name")
+                if n is not None:
+                    names.append(content[n.start_byte : n.end_byte])
+    return names[:15]
+
+
+def _ts_trait_methods(node, content: str) -> List[str]:
+    body = node.child_by_field_name("body")
+    methods = []
+    if body is not None:
+        for c in body.children:
+            if c.type in ("function_signature_item", "function_item"):
+                b = c.child_by_field_name("body")
+                end = b.start_byte if b is not None else c.end_byte
+                methods.append(
+                    " ".join(content[c.start_byte : end].split()).rstrip(";")
+                )
+    return methods[:10]

misterdev/core/context/lsp.py

@@ -0,0 +1,174 @@
+"""Optional LSP-based semantic diagnostics.
+
+A language server surfaces errors the tree-sitter syntax gate cannot see —
+undefined names, type mismatches, unresolved imports — across languages without
+per-project linter configuration. This is strictly best-effort: the work runs in
+a daemon worker thread with a hard timeout and returns ``None`` on any problem
+(multilspy missing, unsupported language, server slow/unavailable), so it can
+never block or slow a build. It is off unless ``orchestrator.lsp_diagnostics``
+is enabled, and even then a timeout silently skips the check rather than failing.
+"""
+
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from misterdev.core.execution.bounded import run_bounded
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+# Our internal language ids -> multilspy code_language values. Languages without
+# a multilspy server (c, cpp, swift) are intentionally absent -> gate skipped.
+_LANG_MAP: Dict[str, str] = {
+    "python": "python",
+    "rust": "rust",
+    "typescript": "typescript",
+    "javascript": "javascript",
+    "csharp": "csharp",
+    "kotlin": "kotlin",
+    "java": "java",
+    "go": "go",
+    "ruby": "ruby",
+}
+
+_LSP_SEVERITY_ERROR = 1  # LSP DiagnosticSeverity.Error
+
+# Per-file diagnostic settle wait bounds (seconds). The actual wait scales down
+# with file count so the total stays under the caller's hard timeout.
+_MIN_FILE_WAIT = 0.3
+_MAX_FILE_WAIT = 1.5
+
+
+def _per_file_wait(num_files: int, settle_budget: float) -> float:
+    """Settle wait per file: the budget split across files, clamped to bounds.
+
+    Keeps the single-file wait close to the original 1.5s but shrinks it as the
+    file set grows so ``wait * num_files`` cannot overrun the gate timeout (which
+    previously caused every diagnostic to be skipped on larger projects).
+    """
+    if num_files <= 0:
+        return _MIN_FILE_WAIT
+    return max(_MIN_FILE_WAIT, min(_MAX_FILE_WAIT, settle_budget / num_files))
+
+
+_LANG_EXTS: Dict[str, tuple] = {
+    "python": (".py",),
+    "rust": (".rs",),
+    "typescript": (".ts", ".tsx"),
+    "javascript": (".js", ".jsx", ".mjs", ".cjs"),
+    "csharp": (".cs",),
+    "kotlin": (".kt", ".kts"),
+    "java": (".java",),
+    "go": (".go",),
+    "ruby": (".rb",),
+}
+
+_SKIP_DIRS = frozenset(
+    {".venv", "venv", ".git", "node_modules", "__pycache__", "target", "build", "dist"}
+)
+
+
+def find_source_files(project_root: Path, language: str, cap: int = 40) -> List[str]:
+    """Project-relative source paths for ``language`` (bounded by ``cap``)."""
+    exts = _LANG_EXTS.get((language or "").lower())
+    if not exts:
+        return []
+    root = Path(project_root)
+    out: List[str] = []
+    for path in sorted(root.rglob("*")):
+        if (
+            path.suffix in exts
+            and path.is_file()
+            and not (_SKIP_DIRS & set(path.parts))
+        ):
+            out.append(str(path.relative_to(root)))
+            if len(out) >= cap:
+                break
+    return out
+
+
+def collect_diagnostics(
+    project_root: Path,
+    language: str,
+    rel_files: List[str],
+    timeout: float = 30.0,
+) -> Optional[List[dict]]:
+    """Return error-severity diagnostics for ``rel_files``, or None if skipped.
+
+    None means "no opinion" (unsupported language, multilspy unavailable, or the
+    server did not respond within ``timeout``); callers must treat it as a
+    no-op, never a pass/fail signal. A list (possibly empty) means the server
+    ran: each item is ``{"file", "line", "message"}`` for an error diagnostic.
+    """
+    code_lang = _LANG_MAP.get((language or "").lower())
+    if code_lang is None or not rel_files:
+        return None
+
+    # Bound the in-server settle time to a fraction of the hard timeout so the
+    # per-file waits can't sum past it and force a blanket SKIP (the old fixed
+    # 1.5s * N did exactly that for ~20+ files at the 30s default).
+    settle_budget = max(timeout * 0.7, _MIN_FILE_WAIT)
+
+    def _work() -> Optional[List[dict]]:
+        try:
+            return _collect(project_root, code_lang, rel_files, settle_budget)
+        except Exception as e:  # multilspy/server failures are non-fatal
+            logger.debug(f"LSP diagnostics unavailable ({language}): {e}")
+            return None
+
+    # A hung server is abandoned to its daemon thread and the gate skips (None),
+    # so the build is never blocked.
+    return run_bounded(_work, timeout, None, f"LSP diagnostics ({language})")
+
+
+def _collect(
+    project_root: Path,
+    code_lang: str,
+    rel_files: List[str],
+    settle_budget: float = 60.0,
+) -> List[dict]:
+    import asyncio
+
+    from multilspy import LanguageServer
+    from multilspy.multilspy_config import MultilspyConfig
+    from multilspy.multilspy_logger import MultilspyLogger
+
+    per_file = _per_file_wait(len(rel_files), settle_budget)
+
+    async def _main() -> List[dict]:
+        config = MultilspyConfig.from_dict({"code_language": code_lang})
+        server = LanguageServer.create(config, MultilspyLogger(), str(project_root))
+        captured: List[dict] = []
+        async with server.start_server():
+            # multilspy discards publishDiagnostics by default; override the
+            # handler to capture them (diagnostics are server-pushed, not a
+            # request/response).
+            server.server.on_notification(
+                "textDocument/publishDiagnostics", captured.append
+            )
+            for rel in rel_files:
+                try:
+                    with server.open_file(rel):
+                        await asyncio.sleep(per_file)
+                except Exception:
+                    continue
+        return _to_errors(captured)
+
+    return asyncio.run(_main())
+
+
+def _to_errors(captured: List[dict]) -> List[dict]:
+    errors: List[dict] = []
+    for params in captured:
+        uri = params.get("uri", "")
+        for diag in params.get("diagnostics", []):
+            if diag.get("severity") == _LSP_SEVERITY_ERROR:
+                line = diag.get("range", {}).get("start", {}).get("line", 0) + 1
+                errors.append(
+                    {
+                        "file": uri.replace("file://", ""),
+                        "line": line,
+                        "message": diag.get("message", ""),
+                    }
+                )
+    return errors

misterdev/core/context/scratchpad.py

@@ -0,0 +1,111 @@
+"""Scratchpad learning system ported from /build skill Phase 4.
+
+Accumulates discoveries within a build session so later tasks can
+benefit from earlier learnings. Categories match /build:
+  env_quirk, pattern, dependency, convention, workaround, pitfall
+"""
+
+import threading
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class ScratchpadEntry:
+    category: str  # env_quirk, pattern, dependency, convention, workaround, pitfall
+    discovery: str
+    task_id: str
+    files: list[str] = field(default_factory=list)
+    tags: list[str] = field(default_factory=list)
+
+    def matches(
+        self, query_files: list[str] = None, query_tags: list[str] = None
+    ) -> bool:
+        """Check if this entry is relevant to the given files or tags."""
+        if query_files:
+            for qf in query_files:
+                for ef in self.files:
+                    if qf in ef or ef in qf:
+                        return True
+        if query_tags:
+            for qt in query_tags:
+                if qt in self.tags or qt in self.category:
+                    return True
+        return False
+
+
+class Scratchpad:
+    """In-session learning accumulator.
+
+    Before each task, query for relevant entries. After each task,
+    record discoveries. This mirrors /build's scratchpad behavior
+    but as a proper Python data structure.
+    """
+
+    def __init__(self):
+        self._entries: list[ScratchpadEntry] = []
+        self._lock = threading.Lock()
+
+    def record(
+        self,
+        category: str,
+        discovery: str,
+        task_id: str,
+        files: Optional[list[str]] = None,
+        tags: Optional[list[str]] = None,
+    ) -> ScratchpadEntry:
+        """Record a discovery from a task execution."""
+        entry = ScratchpadEntry(
+            category=category,
+            discovery=discovery,
+            task_id=task_id,
+            files=files or [],
+            tags=tags or [],
+        )
+        with self._lock:
+            self._entries.append(entry)
+        return entry
+
+    def query(
+        self,
+        files: Optional[list[str]] = None,
+        tags: Optional[list[str]] = None,
+        category: Optional[str] = None,
+    ) -> list[ScratchpadEntry]:
+        """Find relevant entries for a task about to execute."""
+        results = []
+        with self._lock:
+            snapshot = list(self._entries)
+        for entry in snapshot:
+            if category and entry.category != category:
+                continue
+            if files or tags:
+                if entry.matches(query_files=files, query_tags=tags):
+                    results.append(entry)
+            elif not files and not tags:
+                # No filter, return all (optionally filtered by category)
+                results.append(entry)
+        return results
+
+    def format_context(
+        self,
+        files: Optional[list[str]] = None,
+        tags: Optional[list[str]] = None,
+        max_entries: int = 20,
+    ) -> str:
+        """Format relevant entries as context string for LLM prompts."""
+        entries = self.query(files=files, tags=tags)
+        if not entries:
+            return ""
+        entries = entries[:max_entries]
+        lines = ["## Scratchpad (learnings from previous tasks)"]
+        for e in entries:
+            lines.append(f"- [{e.category}] {e.discovery} (from {e.task_id})")
+        return "\n".join(lines)
+
+    @property
+    def entries(self) -> list[ScratchpadEntry]:
+        return list(self._entries)
+
+    def __len__(self) -> int:
+        return len(self._entries)

misterdev/core/context/topography/__init__.py

@@ -0,0 +1,43 @@
+"""Topological context mapping: a tree-sitter symbol graph over the project.
+
+Features:
+- Scope-aware symbols via tree-sitter for Python, Rust, TypeScript/TSX,
+  JavaScript/JSX, C, C++, C#, Swift, and Kotlin (best-effort).
+- Per-file outlines (a symbol table of contents) and a whole-project structural
+  map, used to navigate and window large files for focused edits.
+- Call-neighbor traversal for related-symbol context, ranked by an optional
+  semantic embedder (else lexical).
+- check_syntax(): tree-sitter parse-based syntax verification (ERROR/MISSING
+  nodes) that understands strings/comments, used by the correctness gate.
+- Lazy loading; byte-correct slicing for non-ASCII sources.
+
+This module was split into cohesive sections (parsers, syntax, nodes, cache,
+graph, engine); every name below is re-exported so the import path
+``misterdev.core.context.topography`` is unchanged.
+"""
+
+from ._log import logger
+from .nodes import SymbolNode, _symbol_to_dict, _symbol_from_dict
+from .parsers import _get_ts_parsers, _EXT_TO_LANG, _node_text
+from .syntax import _SYNTAX_CHECK_LANGS, _first_error_node, check_syntax
+from .cache import _CACHE_FORMAT_VERSION, _TopographyCache
+from .graph import _CALL_PATTERN, SymbolGraph
+from .engine import TopographyEngine
+
+__all__ = [
+    "SymbolNode",
+    "SymbolGraph",
+    "TopographyEngine",
+    "check_syntax",
+    "_get_ts_parsers",
+    "_TopographyCache",
+    "logger",
+    "_symbol_to_dict",
+    "_symbol_from_dict",
+    "_EXT_TO_LANG",
+    "_node_text",
+    "_SYNTAX_CHECK_LANGS",
+    "_first_error_node",
+    "_CACHE_FORMAT_VERSION",
+    "_CALL_PATTERN",
+]