code-review-graph-codeblackwell 2.3.6.post1__py3-none-any.whl

code_review_graph/constants.py

@@ -0,0 +1,23 @@
+"""Shared constants for code-review-graph."""
+
+from __future__ import annotations
+
+import os
+
+SECURITY_KEYWORDS: frozenset[str] = frozenset({
+    "auth", "login", "password", "token", "session", "crypt", "secret",
+    "credential", "permission", "sql", "query", "execute", "connect",
+    "socket", "request", "http", "sanitize", "validate", "encrypt",
+    "decrypt", "hash", "sign", "verify", "admin", "privilege",
+})
+
+# ---------------------------------------------------------------------------
+# Configurable limits (override via environment variables)
+# ---------------------------------------------------------------------------
+MAX_IMPACT_NODES = int(os.environ.get("CRG_MAX_IMPACT_NODES", "500"))
+MAX_IMPACT_DEPTH = int(os.environ.get("CRG_MAX_IMPACT_DEPTH", "2"))
+MAX_BFS_DEPTH = int(os.environ.get("CRG_MAX_BFS_DEPTH", "15"))
+MAX_SEARCH_RESULTS = int(os.environ.get("CRG_MAX_SEARCH_RESULTS", "20"))
+
+# BFS engine: "sql" (SQLite recursive CTE) or "networkx" (Python-side BFS)
+BFS_ENGINE = os.environ.get("CRG_BFS_ENGINE", "sql")

code_review_graph/context_savings.py

@@ -0,0 +1,317 @@
+"""Compact estimated context savings helpers.
+
+The project intentionally labels these values as estimates: the helper uses a
+conservative character-count approximation instead of model-specific tokenizers.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Iterable
+
+CHARS_PER_TOKEN = 4
+
+
+def estimate_tokens(value: Any) -> int:
+    """Estimate token count with a conservative 4 chars/token approximation."""
+    if value is None:
+        return 0
+    if isinstance(value, str):
+        text = value
+    else:
+        text = json.dumps(
+            value,
+            default=str,
+            ensure_ascii=True,
+            separators=(",", ":"),
+            sort_keys=True,
+        )
+    if not text:
+        return 0
+    return max(1, (len(text) + CHARS_PER_TOKEN - 1) // CHARS_PER_TOKEN)
+
+
+def estimate_file_tokens(repo_root: Path, files: Iterable[str]) -> int:
+    """Estimate tokens for changed files using file sizes, not file contents."""
+    total = 0
+    root = repo_root.resolve()
+    for file_name in files:
+        path = Path(file_name)
+        full_path = path if path.is_absolute() else root / path
+        try:
+            if full_path.is_file():
+                total += max(
+                    1,
+                    (full_path.stat().st_size + CHARS_PER_TOKEN - 1)
+                    // CHARS_PER_TOKEN,
+                )
+        except OSError:
+            continue
+    return total
+
+
+def estimate_context_savings(
+    *,
+    original_context: Any | None = None,
+    returned_context: Any | None = None,
+    original_tokens: int | None = None,
+    returned_tokens: int | None = None,
+) -> dict[str, int | bool] | None:
+    """Return tiny savings metadata, or None when no baseline is available."""
+    baseline = (
+        original_tokens
+        if original_tokens is not None
+        else estimate_tokens(original_context)
+    )
+    returned = (
+        returned_tokens
+        if returned_tokens is not None
+        else estimate_tokens(returned_context)
+    )
+
+    if baseline <= 0:
+        return None
+
+    saved = max(0, baseline - returned)
+    percent = round((saved / baseline) * 100) if baseline else 0
+    return {
+        "estimated": True,
+        "saved_tokens": int(saved),
+        "saved_percent": int(percent),
+    }
+
+
+def attach_context_savings(
+    result: dict[str, Any],
+    *,
+    original_context: Any | None = None,
+    original_tokens: int | None = None,
+    returned_context: Any | None = None,
+    returned_tokens: int | None = None,
+) -> dict[str, Any]:
+    """Attach compact ``context_savings`` metadata when it can be estimated."""
+    estimate = estimate_context_savings(
+        original_context=original_context,
+        returned_context=result if returned_context is None else returned_context,
+        original_tokens=original_tokens,
+        returned_tokens=returned_tokens,
+    )
+    if estimate is not None:
+        result["context_savings"] = estimate
+    return result
+
+
+def format_context_savings(estimate: dict[str, Any] | None) -> str | None:
+    """Format a one-line human summary for CLI output."""
+    if not estimate:
+        return None
+    saved = int(estimate.get("saved_tokens", 0))
+    percent = int(estimate.get("saved_percent", 0))
+    return f"Estimated context saved: ~{saved:,} tokens (~{percent}%)"
+
+
+def _fmt_compact(n: int) -> str:
+    """Compact integer formatting: 1234 -> '1.2k', 9876 -> '9.9k', 500 -> '500'."""
+    if n >= 10_000:
+        return f"{n // 1000:,}k"
+    if n >= 1000:
+        return f"{n / 1000:.1f}k"
+    return str(n)
+
+
+def _breakdown_from_response(response: dict[str, Any]) -> dict[str, int]:
+    """Pull a per-category token estimate from a detect-changes / review response.
+
+    Only fields that exist and have content are reported, so the breakdown
+    line stays meaningful instead of padding with zeros.
+    """
+    # Friendly label -> response-dict key
+    fields = [
+        ("Functions", "changed_functions"),
+        ("Flows", "affected_flows"),
+        ("Tests", "test_gaps"),
+        ("Risk", "review_priorities"),
+        ("Impact", "impacted_nodes"),
+        ("Edges", "edges"),
+        ("Source", "source_snippets"),
+        ("Imports", "imports"),
+    ]
+    out: dict[str, int] = {}
+    for label, key in fields:
+        value = response.get(key)
+        if not value:
+            continue
+        tokens = estimate_tokens(value)
+        if tokens > 0:
+            out[label] = tokens
+    return out
+
+
+def verify_with_tiktoken(
+    repo_root: "Path | str",
+    changed_files: Iterable[str],
+    response: Any,
+    encoding_name: str = "cl100k_base",
+) -> dict[str, int] | None:
+    """Calibrate the chars/4 estimate against a real model tokenizer.
+
+    Returns ``{"verified_baseline": int, "verified_returned": int,
+    "verified_saved": int, "verified_percent": int}`` or ``None`` if
+    tiktoken is not installed. Reads every changed file's content (unlike
+    the stat-only ``estimate_file_tokens``) so the numbers reflect what
+    an agent would actually consume.
+    """
+    try:
+        import tiktoken  # type: ignore[import-untyped]
+    except ImportError:
+        return None
+
+    enc = tiktoken.get_encoding(encoding_name)
+    root = Path(repo_root).resolve()
+
+    naive_real = 0
+    for f in changed_files:
+        p = root / f
+        try:
+            if p.is_file():
+                naive_real += len(enc.encode(p.read_text(errors="replace")))
+        except OSError:
+            continue
+
+    if isinstance(response, str):
+        graph_real = len(enc.encode(response))
+    else:
+        text = json.dumps(
+            response, default=str, ensure_ascii=True,
+            separators=(",", ":"), sort_keys=True,
+        )
+        graph_real = len(enc.encode(text))
+
+    saved = max(0, naive_real - graph_real)
+    pct = round(saved * 100 / naive_real) if naive_real > 0 else 0
+    return {
+        "verified_baseline": naive_real,
+        "verified_returned": graph_real,
+        "verified_saved": saved,
+        "verified_percent": pct,
+    }
+
+
+def format_context_savings_panel(
+    estimate: dict[str, Any] | None,
+    *,
+    original_tokens: int | None = None,
+    returned_tokens: int | None = None,
+    response: dict[str, Any] | None = None,
+    breakdown: dict[str, int] | None = None,
+    verified: dict[str, int] | None = None,
+    title: str = "Token Savings",
+    width: int = 64,
+) -> str | None:
+    """Format the savings estimate as a boxed multi-line CLI panel.
+
+    Example output (width=60)::
+
+        ┌──────────────── Token Savings ────────────────┐
+        │ Full context would be: 12,932 tokens          │
+        │ Graph context used:       773 tokens          │
+        │ Saved:                 12,159 tokens (~94%)   │
+        │ Breakdown: Functions 580 · Tests 120 · ...    │
+        └───────────────────────────────────────────────┘
+
+    All numbers are labelled as estimates upstream (``estimated: true`` in the
+    metadata dict) because the project uses a 4-chars-per-token approximation,
+    not model-specific tokenization.
+
+    Args:
+        estimate: The ``context_savings`` dict from a tool response.
+        original_tokens: Optional override for the naive baseline.
+        returned_tokens: Optional override for the graph response size.
+        response: When provided, breakdown is auto-derived from common keys
+            (``changed_functions``, ``affected_flows``, ``test_gaps``,
+            ``review_priorities``, ``impacted_nodes``, ``edges``,
+            ``source_snippets``, ``imports``).
+        breakdown: Explicit ``{label: tokens}`` map; takes precedence over
+            ``response``-derived breakdown when both are provided.
+        title: Title centered in the top border.
+        width: Total panel width, capped at terminal width if larger.
+
+    Returns:
+        The panel as a single ``\\n``-joined string, or ``None`` when there
+        is nothing meaningful to display.
+    """
+    if not estimate:
+        return None
+
+    saved = int(estimate.get("saved_tokens", 0))
+    percent = int(estimate.get("saved_percent", 0))
+
+    # Derive baseline + returned from saved+percent if not provided
+    if original_tokens is None:
+        if percent > 0:
+            original_tokens = int(round(saved * 100 / percent))
+        else:
+            original_tokens = saved
+    if returned_tokens is None:
+        returned_tokens = max(0, (original_tokens or 0) - saved)
+
+    if breakdown is None and response is not None:
+        breakdown = _breakdown_from_response(response)
+
+    # Top up the breakdown with an "Other" bucket so the parts sum to
+    # ``returned_tokens`` exactly. "Other" covers fields the breakdown
+    # doesn't enumerate (status, summary, risk_score, context_savings
+    # metadata, JSON envelope chars). Skip when there's no positive
+    # remainder — the breakdown already accounts for the whole response.
+    if breakdown and returned_tokens is not None:
+        labelled_sum = sum(breakdown.values())
+        remainder = returned_tokens - labelled_sum
+        if remainder > 0:
+            breakdown = dict(breakdown)  # copy before mutating
+            breakdown["Other"] = remainder
+
+    # Lines that go inside the box (without borders)
+    inner_lines: list[str] = [
+        f"Full context would be:  {original_tokens:>9,} tokens",
+        f"Graph context used:     {returned_tokens:>9,} tokens",
+        f"Saved:                  {saved:>9,} tokens (~{percent}%)",
+    ]
+    if verified:
+        vb = verified["verified_baseline"]
+        vr = verified["verified_returned"]
+        vs = verified["verified_saved"]
+        vp = verified["verified_percent"]
+        inner_lines.append(
+            f"Verified (tiktoken):    {vs:>9,} tokens (~{vp}%)  "
+            f"[{vb:,} → {vr:,}]"
+        )
+    if breakdown:
+        parts = [f"{label} {_fmt_compact(tok)}" for label, tok in breakdown.items()]
+        bd_line = "Breakdown: " + " · ".join(parts)
+        inner_lines.append(bd_line)
+
+    # Compute final width: at least wide enough for the longest inner line + padding
+    content_width = max(len(s) for s in inner_lines)
+    inner_w = max(width - 2, content_width + 2)  # +2 for one space pad each side
+    # Title bar
+    title_str = f" {title} "
+    dash_total = inner_w - len(title_str)
+    if dash_total < 4:
+        dash_total = 4
+    left_dash = dash_total // 2
+    right_dash = dash_total - left_dash
+    top = "┌" + "─" * left_dash + title_str + "─" * right_dash + "┐"
+    bottom = "└" + "─" * inner_w + "┘"
+
+    def _box_line(content: str) -> str:
+        pad = inner_w - 2 - len(content)
+        if pad < 0:
+            pad = 0
+        return f"│ {content}{' ' * pad} │"
+
+    lines = [top]
+    for s in inner_lines:
+        lines.append(_box_line(s))
+    lines.append(bottom)
+    return "\n".join(lines)

code_review_graph/custom_languages.py

@@ -0,0 +1,322 @@
+"""Config-driven custom language support ("bring your own language").
+
+Repos can teach the parser new tree-sitter languages without forking by
+dropping a ``languages.toml`` file into ``.code-review-graph/``::
+
+    [languages.erlang]
+    extensions = [".erl", ".hrl"]
+    grammar = "erlang"                        # tree_sitter_language_pack name
+    function_node_types = ["function_clause"]
+    class_node_types = ["record_decl"]
+    import_node_types = ["import_attribute"]
+    call_node_types = ["call"]
+    comment = "Erlang via the bundled tree-sitter-erlang grammar"
+
+The loader is deliberately defensive: a broken config must never crash a
+build.  Invalid entries are skipped with a ``logger.warning``, and built-in
+languages always win — custom entries can neither override built-in file
+extensions nor reuse built-in language names.  At most
+``MAX_CUSTOM_LANGUAGES`` entries are honoured per repo.
+
+See docs/CUSTOM_LANGUAGES.md for the full schema reference (answers #320).
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import sys
+import threading
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+import tree_sitter_language_pack as tslp
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    try:
+        import tomli as tomllib  # type: ignore[no-redef]
+    except ImportError:
+        tomllib = None  # type: ignore[assignment]
+
+logger = logging.getLogger(__name__)
+
+#: Location of the config file, relative to the repo root.
+CONFIG_RELATIVE_PATH = Path(".code-review-graph") / "languages.toml"
+
+#: Hard cap on the number of custom languages loaded from a single config.
+MAX_CUSTOM_LANGUAGES = 20
+
+#: Custom language names: short lowercase identifiers.  The name becomes the
+#: ``language`` field on every node parsed from matching files.
+_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]{0,31}$")
+
+#: Extensions: a leading dot followed by 1-15 safe characters (".erl",
+#: ".cls", ".4gl").  Uppercase input is normalised to lowercase because the
+#: parser lowercases file suffixes before lookup.
+_EXTENSION_RE = re.compile(r"^\.[a-z0-9_+-]{1,15}$")
+
+#: The four node-type lists recognised in each ``[languages.<name>]`` table.
+_NODE_TYPE_KEYS = (
+    "function_node_types",
+    "class_node_types",
+    "import_node_types",
+    "call_node_types",
+)
+
+
+@dataclass(frozen=True)
+class CustomLanguage:
+    """One validated ``[languages.<name>]`` entry from languages.toml."""
+
+    name: str
+    grammar: str
+    extensions: tuple[str, ...]
+    function_node_types: tuple[str, ...] = ()
+    class_node_types: tuple[str, ...] = ()
+    import_node_types: tuple[str, ...] = ()
+    call_node_types: tuple[str, ...] = ()
+    comment: str = ""
+
+
+@dataclass(frozen=True)
+class _CacheEntry:
+    mtime_ns: int
+    size: int
+    languages: dict[str, CustomLanguage] = field(default_factory=dict)
+
+
+# Config files are re-read only when their mtime/size changes.  This matters
+# because full builds construct one CodeParser per worker task, and probing
+# tree-sitter grammars on every file parse would be wasteful.
+_cache_lock = threading.Lock()
+_cache: dict[str, _CacheEntry] = {}
+
+
+def clear_cache() -> None:
+    """Drop the loader cache (used by tests)."""
+    with _cache_lock:
+        _cache.clear()
+
+
+def load_custom_languages(
+    repo_root: Path,
+    *,
+    builtin_extensions: Mapping[str, str],
+    builtin_languages: frozenset[str],
+) -> dict[str, CustomLanguage]:
+    """Load and validate ``<repo_root>/.code-review-graph/languages.toml``.
+
+    Returns a mapping of custom language name -> :class:`CustomLanguage`.
+    Always returns (possibly empty) — a broken config never raises.
+
+    Args:
+        repo_root: Repository root containing ``.code-review-graph/``.
+        builtin_extensions: The parser's built-in extension map; custom
+            entries colliding with these are skipped (built-ins win).
+        builtin_languages: All built-in language identifiers; custom names
+            shadowing these are skipped.
+    """
+    config_path = Path(repo_root) / CONFIG_RELATIVE_PATH
+    try:
+        stat = config_path.stat()
+    except OSError:
+        return {}  # No config file — the common case; not worth a log line.
+
+    cache_key = str(config_path)
+    with _cache_lock:
+        cached = _cache.get(cache_key)
+        if (
+            cached is not None
+            and cached.mtime_ns == stat.st_mtime_ns
+            and cached.size == stat.st_size
+        ):
+            return dict(cached.languages)
+
+    languages = _load_uncached(config_path, builtin_extensions, builtin_languages)
+    with _cache_lock:
+        _cache[cache_key] = _CacheEntry(stat.st_mtime_ns, stat.st_size, dict(languages))
+    return languages
+
+
+def _load_uncached(
+    config_path: Path,
+    builtin_extensions: Mapping[str, str],
+    builtin_languages: frozenset[str],
+) -> dict[str, CustomLanguage]:
+    if tomllib is None:
+        logger.warning(
+            "%s found but TOML parsing requires the 'tomli' package on "
+            "Python < 3.11 — no custom languages loaded",
+            config_path,
+        )
+        return {}
+    try:
+        raw = config_path.read_bytes()
+    except (OSError, PermissionError) as exc:
+        logger.warning("Cannot read %s: %s — no custom languages loaded", config_path, exc)
+        return {}
+    try:
+        data = tomllib.loads(raw.decode("utf-8", errors="replace"))
+    except tomllib.TOMLDecodeError as exc:
+        logger.warning("Malformed TOML in %s: %s — no custom languages loaded", config_path, exc)
+        return {}
+
+    tables = data.get("languages")
+    if tables is None:
+        return {}
+    if not isinstance(tables, dict):
+        logger.warning(
+            "%s: [languages] must be a table of tables — no custom languages loaded",
+            config_path,
+        )
+        return {}
+
+    result: dict[str, CustomLanguage] = {}
+    claimed_extensions: set[str] = set()
+    for name, table in tables.items():
+        if len(result) >= MAX_CUSTOM_LANGUAGES:
+            logger.warning(
+                "%s defines more than %d custom languages — ignoring the rest",
+                config_path, MAX_CUSTOM_LANGUAGES,
+            )
+            break
+        lang = _validate_entry(
+            name, table, builtin_extensions, builtin_languages,
+            claimed_extensions, config_path,
+        )
+        if lang is None:
+            continue
+        result[lang.name] = lang
+        claimed_extensions.update(lang.extensions)
+    return result
+
+
+def _validate_entry(
+    name: object,
+    table: object,
+    builtin_extensions: Mapping[str, str],
+    builtin_languages: frozenset[str],
+    claimed_extensions: set[str],
+    config_path: Path,
+) -> Optional[CustomLanguage]:
+    """Validate one ``[languages.<name>]`` table; None (after a warning) on
+    any problem so a bad entry can never break a build."""
+    label = name if isinstance(name, str) else repr(name)
+    if not isinstance(table, dict):
+        logger.warning("%s: [languages.%s] is not a table — skipping", config_path, label)
+        return None
+    if not isinstance(name, str) or not _NAME_RE.match(name):
+        logger.warning(
+            "%s: invalid custom language name %r (expected lowercase "
+            "letters/digits/_/-, max 32 chars) — skipping",
+            config_path, label,
+        )
+        return None
+    if name in builtin_languages:
+        logger.warning(
+            "%s: custom language %r shadows a built-in language — skipping "
+            "(built-ins cannot be overridden)",
+            config_path, name,
+        )
+        return None
+
+    grammar = table.get("grammar")
+    if not isinstance(grammar, str) or not grammar.strip():
+        logger.warning(
+            "%s: custom language %r needs a non-empty 'grammar' string — skipping",
+            config_path, name,
+        )
+        return None
+    grammar = grammar.strip()
+
+    raw_extensions = table.get("extensions")
+    if not isinstance(raw_extensions, list) or not raw_extensions:
+        logger.warning(
+            "%s: custom language %r needs a non-empty 'extensions' list — skipping",
+            config_path, name,
+        )
+        return None
+    extensions: list[str] = []
+    for ext in raw_extensions:
+        normalized = ext.strip().lower() if isinstance(ext, str) else ""
+        if not normalized.startswith("."):
+            logger.warning(
+                "%s: custom language %r: extension %r must start with a dot — skipping",
+                config_path, name, ext,
+            )
+            return None
+        if not _EXTENSION_RE.match(normalized):
+            logger.warning(
+                "%s: custom language %r: extension %r is not a valid file "
+                "extension — skipping",
+                config_path, name, ext,
+            )
+            return None
+        if normalized in builtin_extensions:
+            logger.warning(
+                "%s: custom language %r: extension %r is already handled by "
+                "the built-in %r parser — skipping (built-ins cannot be overridden)",
+                config_path, name, normalized, builtin_extensions[normalized],
+            )
+            return None
+        if normalized in claimed_extensions:
+            logger.warning(
+                "%s: custom language %r: extension %r is already claimed by "
+                "an earlier custom language — skipping",
+                config_path, name, normalized,
+            )
+            return None
+        if normalized not in extensions:
+            extensions.append(normalized)
+
+    node_types: dict[str, tuple[str, ...]] = {}
+    for key in _NODE_TYPE_KEYS:
+        value = table.get(key, [])
+        if not isinstance(value, list) or any(
+            not isinstance(item, str) or not item.strip() for item in value
+        ):
+            logger.warning(
+                "%s: custom language %r: %s must be a list of non-empty "
+                "strings — skipping",
+                config_path, name, key,
+            )
+            return None
+        node_types[key] = tuple(item.strip() for item in value)
+    if not any(node_types.values()):
+        logger.warning(
+            "%s: custom language %r defines no node types — nothing to "
+            "extract, skipping",
+            config_path, name,
+        )
+        return None
+
+    comment = table.get("comment", "")
+    if not isinstance(comment, str):
+        comment = ""
+
+    # Probe the grammar last (it is the expensive check).  Parser objects
+    # themselves are created lazily by CodeParser._get_parser.
+    try:
+        tslp.get_language(grammar)  # type: ignore[arg-type]
+    except (LookupError, ValueError, ImportError, OSError) as exc:
+        logger.warning(
+            "%s: custom language %r: grammar %r is not available in "
+            "tree_sitter_language_pack (%s) — skipping",
+            config_path, name, grammar, exc,
+        )
+        return None
+
+    return CustomLanguage(
+        name=name,
+        grammar=grammar,
+        extensions=tuple(extensions),
+        function_node_types=node_types["function_node_types"],
+        class_node_types=node_types["class_node_types"],
+        import_node_types=node_types["import_node_types"],
+        call_node_types=node_types["call_node_types"],
+        comment=comment,
+    )