misterdev 0.2.0__py3-none-any.whl

misterdev/core/context/topography/nodes.py

@@ -0,0 +1,55 @@
+"""Symbol data type and its cache (de)serialization."""
+
+from typing import Dict, List, Set, Any
+
+
+class SymbolNode:
+    def __init__(
+        self,
+        name: str,
+        file_path: str,
+        kind: str,
+        start_line: int,
+        end_line: int,
+        content: str,
+    ):
+        self.name = name
+        self.file_path = file_path
+        self.kind = kind  # 'function', 'class', 'method'
+        self.start_line = start_line
+        self.end_line = end_line
+        self.content = content
+        self.outgoing_calls: Set[str] = set()
+        self.incoming_calls: Set[str] = set()
+        self.imports: List[Dict[str, str]] = []  # {name: ..., module: ...}
+
+    def __repr__(self):
+        return f"Symbol({self.kind}:{self.name} in {self.file_path})"
+
+
+def _symbol_to_dict(s: "SymbolNode") -> Dict[str, Any]:
+    """Serialize the parse-derived fields of a SymbolNode for the disk cache.
+
+    Only the fields produced by parsing are stored; call neighbors (rebuilt by
+    ``_resolve_references`` from content) and imports are intentionally omitted
+    so the cache never has to track derived/global state.
+    """
+    return {
+        "name": s.name,
+        "file_path": s.file_path,
+        "kind": s.kind,
+        "start_line": s.start_line,
+        "end_line": s.end_line,
+        "content": s.content,
+    }
+
+
+def _symbol_from_dict(d: Dict[str, Any]) -> "SymbolNode":
+    return SymbolNode(
+        d["name"],
+        d["file_path"],
+        d["kind"],
+        d["start_line"],
+        d["end_line"],
+        d["content"],
+    )

misterdev/core/context/topography/parsers.py

@@ -0,0 +1,95 @@
+"""Tree-sitter parser loading, language detection, and byte-correct slicing."""
+
+from typing import Dict, Any
+
+from ._log import logger
+
+# Lazy imports for optional heavy dependencies
+_ts_parsers: Dict[str, Any] = {}
+_ts_available = None
+
+
+def _get_ts_parsers() -> Dict[str, Any]:
+    """Lazy-load tree-sitter parsers for all available language grammars."""
+    global _ts_parsers, _ts_available
+    if _ts_available is not None:
+        return _ts_parsers
+
+    try:
+        from tree_sitter import Language, Parser
+    except ImportError:
+        logger.info("tree-sitter not installed; symbol graph will use basic fallback")
+        _ts_available = False
+        return _ts_parsers
+
+    grammars = {
+        "python": "tree_sitter_python",
+        "rust": "tree_sitter_rust",
+        "c": "tree_sitter_c",
+        "cpp": "tree_sitter_cpp",
+        "swift": "tree_sitter_swift",
+        "csharp": "tree_sitter_c_sharp",
+        "javascript": "tree_sitter_javascript",
+        "kotlin": "tree_sitter_kotlin",
+    }
+    for lang_name, module_name in grammars.items():
+        try:
+            mod = __import__(module_name)
+            parser = Parser(Language(mod.language()))
+            _ts_parsers[lang_name] = parser
+            logger.debug(f"tree-sitter {lang_name} grammar loaded")
+        except ImportError:
+            logger.debug(f"tree-sitter grammar not installed: {module_name}")
+
+    # TypeScript ships two grammar entrypoints (.ts and .tsx) rather than the
+    # single language() the loop above expects, so load it separately.
+    try:
+        import tree_sitter_typescript as tsts
+
+        _ts_parsers["typescript"] = Parser(Language(tsts.language_typescript()))
+        _ts_parsers["tsx"] = Parser(Language(tsts.language_tsx()))
+        logger.debug("tree-sitter typescript grammar loaded")
+    except ImportError:
+        logger.debug("tree-sitter grammar not installed: tree_sitter_typescript")
+
+    if _ts_parsers:
+        _ts_available = True
+        logger.info(f"tree-sitter available for: {', '.join(_ts_parsers)}")
+    else:
+        _ts_available = False
+        logger.info("No tree-sitter grammars installed; symbol graph disabled")
+
+    return _ts_parsers
+
+
+_EXT_TO_LANG: Dict[str, str] = {
+    ".py": "python",
+    ".pyi": "python",
+    ".rs": "rust",
+    ".ts": "typescript",
+    ".tsx": "tsx",
+    ".c": "c",
+    ".h": "c",
+    ".cpp": "cpp",
+    ".cc": "cpp",
+    ".cxx": "cpp",
+    ".hpp": "cpp",
+    ".hh": "cpp",
+    ".swift": "swift",
+    ".cs": "csharp",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".mjs": "javascript",
+    ".cjs": "javascript",
+    ".kt": "kotlin",
+    ".kts": "kotlin",
+}
+
+
+def _node_text(src: bytes, node: Any) -> str:
+    """Decode a node's span from the UTF-8 source bytes.
+
+    Offsets from tree-sitter are byte positions, so slicing must happen on the
+    bytes and decode after, never on the str.
+    """
+    return src[node.start_byte : node.end_byte].decode("utf-8", errors="replace")

misterdev/core/context/topography/syntax.py

@@ -0,0 +1,54 @@
+"""Tree-sitter parse-based syntax verification used by the correctness gate."""
+
+from typing import Any
+
+from .parsers import _get_ts_parsers
+
+# Languages whose tree-sitter grammar is trustworthy enough to gate edits on.
+# Kotlin is excluded: its grammar emits ERROR nodes on some valid code, which
+# would false-reject correct edits. TypeScript is parsed with the TSX grammar
+# (a superset) so JSX never trips a false syntax error.
+_SYNTAX_CHECK_LANGS = {
+    "rust",
+    "c",
+    "cpp",
+    "csharp",
+    "swift",
+    "kotlin",
+    "javascript",
+    "typescript",
+    "tsx",
+}
+
+
+def _first_error_node(node: Any):
+    """Pre-order search for the first ERROR or MISSING node, or None."""
+    if node.type == "ERROR" or node.is_missing:
+        return node
+    for child in node.children:
+        found = _first_error_node(child)
+        if found is not None:
+            return found
+    return None
+
+
+def check_syntax(source: str, lang: str):
+    """Verify ``source`` parses cleanly for ``lang`` using tree-sitter.
+
+    Returns ``(ok, message)`` when the language has a trustworthy grammar, or
+    ``None`` when it does not (the caller then falls back to a lighter check).
+    Unlike brace-counting, this understands strings and comments, so braces in
+    a string literal never false-trip and real syntax errors are caught before
+    the expensive build gate.
+    """
+    parsers = _get_ts_parsers()
+    key = "tsx" if lang in ("typescript", "tsx") else lang
+    if lang not in _SYNTAX_CHECK_LANGS or key not in parsers:
+        return None
+    tree = parsers[key].parse(source.encode("utf-8"))
+    err = _first_error_node(tree.root_node)
+    if err is None:
+        return True, None
+    line = err.start_point[0] + 1
+    snippet = source.split("\n")[err.start_point[0]].strip()[:80]
+    return False, f"{lang} syntax error near line {line}: {snippet}"

misterdev/core/economics/context_budget.py

@@ -0,0 +1,175 @@
+"""Context budget management for LLM prompts.
+
+Estimates token counts and intelligently truncates context sections
+to stay within model limits. By task 15+ in a multi-task build,
+unmanaged context can exceed 100K tokens and degrade LLM quality.
+"""
+
+from typing import Dict
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+# Approximate tokens per character for English/code (conservative). Used only
+# when tiktoken is unavailable.
+CHARS_PER_TOKEN = 3.5
+
+_encoder = None
+_encoder_loaded = False
+
+
+def _get_encoder():
+    """Lazily load a tiktoken encoder; cache the result (incl. failure)."""
+    global _encoder, _encoder_loaded
+    if _encoder_loaded:
+        return _encoder
+    _encoder_loaded = True
+    try:
+        import tiktoken
+
+        # cl100k_base is a good general proxy for code/English token counts and
+        # avoids a per-model lookup that may miss newer model ids.
+        _encoder = tiktoken.get_encoding("cl100k_base")
+    except Exception as e:  # pragma: no cover - import or offline BPE fetch
+        logger.debug(f"tiktoken unavailable; using char heuristic: {e}")
+        _encoder = None
+    return _encoder
+
+
+def estimate_tokens(text: str) -> int:
+    """Count tokens with tiktoken when available, else a char heuristic.
+
+    Accurate counts keep ContextBudget from over- or under-filling the window
+    (the heuristic mis-sizes code, especially the windowed large-file context).
+    """
+    if not text:
+        return 1
+    enc = _get_encoder()
+    if enc is not None:
+        try:
+            return max(1, len(enc.encode(text, disallowed_special=())))
+        except Exception as e:  # pragma: no cover - encoder edge cases
+            # Fall back to the heuristic, but leave a trace: a systematic encoder
+            # failure would otherwise silently mis-size every prompt.
+            logger.debug(f"tiktoken encode failed; using char heuristic: {e}")
+    return max(1, int(len(text) / CHARS_PER_TOKEN))
+
+
+class ContextBudget:
+    """Allocates a token budget across context sections and truncates as needed.
+
+    Usage:
+        budget = ContextBudget(max_tokens=100000)
+        budget.set("code_context", big_code_string, priority=1)
+        budget.set("scratchpad", scratchpad_text, priority=3)
+        budget.set("interface_contracts", contracts, priority=2)
+        budget.set("error_logs", errors, priority=1)
+
+        # Get truncated versions that fit within budget
+        sections = budget.allocate()
+        code = sections["code_context"]
+    """
+
+    def __init__(self, max_tokens: int = 100000, reserved_tokens: int = 8000):
+        self.max_tokens = max_tokens
+        self.reserved_tokens = reserved_tokens  # for system prompt + LLM response
+        self.available = max_tokens - reserved_tokens
+        self._sections: Dict[str, _Section] = {}
+
+    def set(self, name: str, content: str, priority: int = 2, min_lines: int = 10):
+        """Register a context section.
+
+        priority: 1 = essential (truncate last), 2 = important, 3 = nice-to-have (truncate first)
+        min_lines: minimum lines to keep even under pressure
+        """
+        self._sections[name] = _Section(
+            name=name,
+            content=content,
+            priority=priority,
+            min_lines=min_lines,
+            tokens=estimate_tokens(content),
+        )
+
+    def allocate(self) -> Dict[str, str]:
+        """Allocate budget and return truncated sections."""
+        total = sum(s.tokens for s in self._sections.values())
+
+        if total <= self.available:
+            return {name: s.content for name, s in self._sections.items()}
+
+        logger.warning(
+            f"Context exceeds budget: {total} tokens > {self.available} available. Truncating."
+        )
+
+        # Sort by priority (highest number = truncate first)
+        by_priority = sorted(self._sections.values(), key=lambda s: -s.priority)
+
+        overflow = total - self.available
+        result = {}
+
+        for section in by_priority:
+            if overflow <= 0:
+                result[section.name] = section.content
+                continue
+
+            # How much can we trim from this section?
+            lines = section.content.splitlines()
+            if len(lines) <= section.min_lines:
+                result[section.name] = section.content
+                continue
+
+            # Binary search for the right number of lines
+            target_tokens = max(
+                estimate_tokens("\n".join(lines[: section.min_lines])),
+                section.tokens - overflow,
+            )
+
+            kept = _truncate_to_tokens(lines, target_tokens, section.name)
+            saved = section.tokens - estimate_tokens(kept)
+            overflow -= saved
+            result[section.name] = kept
+
+        return result
+
+    def summary(self) -> str:
+        total = sum(s.tokens for s in self._sections.values())
+        parts = [f"budget={self.available}"]
+        for s in sorted(self._sections.values(), key=lambda s: s.priority):
+            parts.append(f"{s.name}={s.tokens}t(p{s.priority})")
+        parts.append(f"total={total}")
+        over = total > self.available
+        parts.append("OVER" if over else "OK")
+        return " | ".join(parts)
+
+
+class _Section:
+    def __init__(
+        self, name: str, content: str, priority: int, min_lines: int, tokens: int
+    ):
+        self.name = name
+        self.content = content
+        self.priority = priority
+        self.min_lines = min_lines
+        self.tokens = tokens
+
+
+def _truncate_to_tokens(lines: list, target_tokens: int, name: str) -> str:
+    """Keep lines from the start until we hit the target token count."""
+    kept = []
+    running = 0
+    for line in lines:
+        line_tokens = estimate_tokens(line)
+        if running + line_tokens > target_tokens and len(kept) > 0:
+            break
+        kept.append(line)
+        running += line_tokens
+
+    total_lines = len(lines)
+    omitted = total_lines - len(kept)
+    if omitted > 0:
+        kept.append(
+            f"\n... ({omitted} lines omitted from {name} to fit context budget)"
+        )
+        logger.info(f"Truncated {name}: kept {len(kept)}/{total_lines} lines")
+
+    return "\n".join(kept)

misterdev/core/economics/embeddings.py

@@ -0,0 +1,232 @@
+"""Semantic relevance ranking for context selection.
+
+When a task pulls in more candidate code symbols than fit the context budget,
+the symbols were previously truncated by arbitrary order. This ranks candidates
+by cosine similarity of their embedding to the task description and keeps the
+most relevant — the lossless-relevant version of "fit more useful information
+into context".
+
+Everything here degrades gracefully: any embedding failure falls back to the
+prior arbitrary-order selection, so semantic ranking can change *which* context
+is shown but never break a build. Embeddings of unchanged code are stable, so
+vectors are cached on disk (keyed by model + text hash) and reused across runs.
+"""
+
+import hashlib
+import json
+import math
+import re
+from pathlib import Path
+from typing import Dict, List, Optional, Protocol, Set
+
+from misterdev.logging_setup import setup_logger
+from misterdev.utils.file_utils import atomic_write_json
+
+logger = setup_logger(__name__)
+
+# Splits identifiers into subword tokens: snake_case (the separators aren't
+# matched) and camelCase boundaries, plus numbers. Single-char tokens dropped.
+_TOKEN = re.compile(r"[A-Za-z][a-z0-9]*|[A-Z]+(?![a-z])|[0-9]+")
+
+
+def _tokenize(text: str) -> Set[str]:
+    return {t.lower() for t in _TOKEN.findall(text or "") if len(t) > 1}
+
+
+def _lexical_overlap(query_tokens: Set[str], text_tokens: Set[str]) -> float:
+    """Fraction of query identifier tokens present in the candidate (0..1).
+
+    Recall-of-query rather than Jaccard, so a large candidate that contains the
+    query's identifiers isn't penalized for its size — the right bias for code,
+    where exact identifier matches are strong relevance signals.
+    """
+    if not query_tokens:
+        return 0.0
+    return len(query_tokens & text_tokens) / len(query_tokens)
+
+
+_EMBEDDING_MODELS_URL = "https://openrouter.ai/api/v1/embeddings/models"
+
+
+def _fetch_embedding_models() -> list:
+    from misterdev.core.economics.free_models import _http_fetch
+
+    return _http_fetch(_EMBEDDING_MODELS_URL)
+
+
+def pick_embedding_model(
+    configured: str, prefer: Optional[List[str]] = None, fetcher=None
+) -> Optional[str]:
+    """The embedding model to use: the explicit config value, else discovered.
+
+    Mirrors free-model harvesting — discovers OpenRouter's embedding models and
+    picks the cheapest (free sorts first). Among equally-priced models, prefers
+    ones whose id matches a ``prefer`` hint (default ['code'], since code-aware
+    embeddings rank code better); ties broken deterministically by id. Returns
+    None if nothing is configured and discovery fails, so semantic retrieval
+    stays off gracefully rather than guessing a model.
+    """
+    if configured:
+        return configured
+    hints = [h.lower() for h in (prefer if prefer is not None else ["code"])]
+    try:
+        models = (fetcher or _fetch_embedding_models)()
+    except Exception as e:
+        logger.warning(f"Embedding-model discovery failed: {e}")
+        return None
+    ranked = []
+    for m in models:
+        if not isinstance(m, dict) or not m.get("id"):
+            continue
+        pricing = m.get("pricing") or {}
+        try:
+            price = float(pricing.get("prompt", "inf"))
+        except (TypeError, ValueError):
+            price = float("inf")
+        model_id = m["id"]
+        preferred = 0 if any(h in model_id.lower() for h in hints) else 1
+        ranked.append((price, preferred, model_id))
+    if not ranked:
+        return None
+    ranked.sort(key=lambda r: (r[0], r[1], r[2]))
+    logger.info(f"Auto-selected embedding model {ranked[0][2]!r}")
+    return ranked[0][2]
+
+
+def cosine_similarity(a: List[float], b: List[float]) -> float:
+    """Cosine similarity of two equal-length vectors (0.0 on degenerate input)."""
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    dot = math.fsum(x * y for x, y in zip(a, b))
+    na = math.sqrt(math.fsum(x * x for x in a))
+    nb = math.sqrt(math.fsum(y * y for y in b))
+    if na == 0.0 or nb == 0.0:
+        return 0.0
+    return dot / (na * nb)
+
+
+class Embedder(Protocol):
+    """Minimal interface a semantic ranker needs from an embedding client."""
+
+    model: str
+
+    def embed(self, texts: List[str]) -> List[List[float]]: ...
+
+
+class EmbeddingCache:
+    """Disk-backed cache of text -> vector, keyed by model and content hash."""
+
+    def __init__(self, path: Path, model: str):
+        self.path = Path(path)
+        self.model = model
+        self._vectors: Optional[Dict[str, List[float]]] = None
+
+    def _key(self, text: str) -> str:
+        h = hashlib.sha256()
+        h.update(f"{self.model}\x00".encode("utf-8"))
+        h.update(text.encode("utf-8"))
+        return h.hexdigest()
+
+    def _load(self) -> Dict[str, List[float]]:
+        if self._vectors is None:
+            self._vectors = {}
+            if self.path.exists():
+                try:
+                    data = json.loads(self.path.read_text(encoding="utf-8"))
+                    if isinstance(data, dict):
+                        self._vectors = data
+                except (json.JSONDecodeError, OSError) as e:
+                    logger.warning(f"Embedding cache unreadable, starting fresh: {e}")
+        return self._vectors
+
+    def get(self, text: str) -> Optional[List[float]]:
+        return self._load().get(self._key(text))
+
+    def put_many(self, items: Dict[str, List[float]]) -> None:
+        """Store {text: vector} and persist."""
+        store = self._load()
+        for text, vector in items.items():
+            store[self._key(text)] = vector
+        atomic_write_json(self.path, store)
+
+
+class SemanticRanker:
+    """Selects the top-k most task-relevant items via a hybrid relevance score.
+
+    Combines dense embedding cosine with a lexical identifier-overlap signal —
+    code relevance is driven heavily by exact identifier matches, which dense
+    similarity alone can miss. The embedder is optional: with none (or on an
+    embedding failure) the ranker degrades to lexical-only, which still beats the
+    arbitrary-order slice it replaces.
+    """
+
+    def __init__(
+        self,
+        embedder: Optional[Embedder] = None,
+        cache: Optional[EmbeddingCache] = None,
+        lexical_weight: float = 0.3,
+    ):
+        self.embedder = embedder
+        self.cache = cache
+        self.lexical_weight = max(0.0, min(1.0, lexical_weight))
+
+    def _embed_with_cache(self, texts: List[str]) -> Optional[List[List[float]]]:
+        """Embed texts, serving cache hits and embedding only the misses."""
+        cached: Dict[str, Optional[List[float]]] = {}
+        misses: List[str] = []
+        for text in texts:
+            vec = self.cache.get(text) if self.cache else None
+            cached[text] = vec
+            if vec is None and text not in misses:
+                misses.append(text)
+        if misses:
+            fresh = self.embedder.embed(misses)
+            if len(fresh) != len(misses):
+                raise ValueError("embedding count mismatch")
+            new = dict(zip(misses, fresh))
+            if self.cache:
+                self.cache.put_many(new)
+            for text, vec in new.items():
+                cached[text] = vec
+        return [cached[text] for text in texts]
+
+    def top_k(self, query: str, candidates: Dict[str, str], k: int) -> List[str]:
+        """Return up to k candidate ids most relevant to query.
+
+        candidates maps id -> text. Scores blend dense cosine (when an embedder
+        is available) with lexical identifier overlap; on embedding failure it
+        uses lexical alone. Never raises — ranking is an optimization under the
+        validation gates, not load-bearing for correctness.
+        """
+        ids = list(candidates)
+        if k >= len(ids):
+            return ids
+
+        query_tokens = _tokenize(query)
+        lexical = {
+            i: _lexical_overlap(query_tokens, _tokenize(candidates[i])) for i in ids
+        }
+
+        dense: Optional[Dict[str, float]] = None
+        if self.embedder is not None:
+            try:
+                vectors = self._embed_with_cache([query] + [candidates[i] for i in ids])
+                query_vec = vectors[0]
+                dense = {
+                    i: (cosine_similarity(query_vec, v) + 1.0) / 2.0
+                    for i, v in zip(ids, vectors[1:])
+                }
+            except Exception as e:
+                logger.warning(f"Dense ranking unavailable, using lexical only: {e}")
+
+        if dense is None:
+
+            def score(i: str) -> float:
+                return lexical[i]
+        else:
+            w = self.lexical_weight
+
+            def score(i: str) -> float:
+                return (1.0 - w) * dense[i] + w * lexical[i]
+
+        return sorted(ids, key=score, reverse=True)[:k]