codebase-explorer 2.2.0__py3-none-any.whl

cbe/model_settings.py

@@ -0,0 +1,80 @@
+"""API model resource defaults and user configuration, in one place.
+
+Exact model ids opt in; unknown models retain the historical API behavior.
+"""
+from __future__ import annotations
+
+import json
+import math
+from pathlib import Path
+from typing import Any
+
+from cbe import batching
+
+
+DEFAULT_API_CONFIG: dict[str, Any] = {
+    "base_url": "https://api.openai.com/v1",
+    "api_key_env": "OPENAI_API_KEY",
+    "api_key_file": None,
+    "extra_headers": {},
+}
+#: Provider request fields sent only when configured; ``api_completion`` reads exactly these.
+OPTIONAL_REQUEST_KEYS = ("response_format", "top_p", "reasoning_effort")
+#: Every config key the code reads. ``load_config`` rejects anything else.
+ALLOWED_API_CONFIG_KEYS = frozenset(DEFAULT_API_CONFIG) | frozenset(OPTIONAL_REQUEST_KEYS) | {
+    "timeout", "output_max_tokens", "batch_target_tokens", "batch_max_tokens",
+    "temperature", "price_per_million",
+}
+MODEL_DEFAULTS: dict[str, dict[str, Any]] = {
+    "zai/glm-5.3-flash": {
+        "timeout": 1800,
+        "output_max_tokens": 49152,
+        "reasoning_effort": "low",
+        "response_format": {"type": "json_object"},
+        "batch_target_tokens": 8000,
+        "batch_max_tokens": 10000,
+    },
+}
+
+
+class ApiConfigError(ValueError):
+    pass
+
+
+def model_defaults(model: str) -> dict[str, Any]:
+    """Defaults for a model: the exact name, else the same model under another gateway prefix."""
+    if model in MODEL_DEFAULTS:
+        return MODEL_DEFAULTS[model]
+    name = model.rsplit("/", 1)[-1]
+    return next((row for key, row in MODEL_DEFAULTS.items() if key.rsplit("/", 1)[-1] == name), {})
+
+
+def load_config(path: Path, model: str = "") -> dict[str, Any]:
+    """Merge transport defaults < model defaults < user JSON settings."""
+    value = {**DEFAULT_API_CONFIG, **model_defaults(model)}
+    if path.is_file():
+        try:
+            loaded = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            raise ApiConfigError(f"API config is not valid JSON: {exc}") from exc
+        if not isinstance(loaded, dict):
+            raise ApiConfigError("API config must be a JSON object")
+        unknown = sorted(set(loaded) - ALLOWED_API_CONFIG_KEYS)
+        if unknown:
+            raise ApiConfigError(f"unknown API config keys: {', '.join(unknown)}")
+        value.update(loaded)
+    for key in ("timeout", "output_max_tokens", "batch_target_tokens", "batch_max_tokens"):
+        if key in value and (type(value[key]) is not int or value[key] < 1):
+            raise ApiConfigError(f"{key} must be a positive integer")
+    batch_max = value.get("batch_max_tokens", batching.MAX_TOKENS)
+    target = value.get("batch_target_tokens", min(batching.TARGET_TOKENS, batch_max))
+    if target > batch_max:
+        raise ApiConfigError("batch_target_tokens must not exceed batch_max_tokens")
+    if "batch_max_tokens" in value:
+        value.setdefault("batch_target_tokens", target)
+    rates = value.get("price_per_million")
+    if rates is not None:
+        if (not isinstance(rates, list) or len(rates) != 3 or
+                any(type(rate) not in (int, float) or not math.isfinite(rate) or rate < 0 for rate in rates)):
+            raise ApiConfigError("price_per_million needs three finite nonnegative rates: input,output,cache")
+    return value

cbe/projection.py

@@ -0,0 +1,305 @@
+"""Unit-level graph evidence for detail authors, shared by both drivers.
+
+The inventory defines identity; chunks define call-site ownership. Rendering and
+budget selection live here so host files and all detail/review/repair prompts
+cannot acquire different projection rules.
+"""
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+import re
+from typing import Any
+
+from cbe.records import SourceOwnership
+from cbe.token_budget import count_text_tokens
+
+DEFAULT_GRAPH_MAX_TOKENS = 3000
+
+# Joern names observed in the O1c Flask and O1b cjson/polyglot graphs;
+# C3b's independent cfg_check.py also documents <clinit>. Match only known
+# terminal suffixes: <redefined>, <duplicate>, <const> and <init> are real
+# source methods, and a parent <lambda> must not hide a named inner function.
+_SYNTHETIC_CFG_OWNER = re.compile(
+    r"<(?:lambda|metaClassAdapter|metaClassCallHandler|fakeNew|module|body|"
+    r"main|global|clinit)>\d*$"
+)
+
+
+def graph_state(graph: Any) -> tuple[str, str]:
+    """Use the graph module's authoritative availability classification."""
+    from cbe.graph import graph_state as classify_graph
+
+    return classify_graph(graph)
+
+
+@dataclass(frozen=True)
+class EvidenceRow:
+    category: str
+    text: str
+    count: int = 1
+
+
+@dataclass(frozen=True)
+class UnitGraph:
+    """Unbounded rows in priority order, plus non-optional mapping accounting."""
+    rows: tuple[EvidenceRow, ...]
+    internal_calls: int
+    unmapped_calls: int
+    unmapped_cfg: int = 0
+    exception_exits_exported: bool = False
+
+
+def unit_graph(graph: dict, paths: list[str], inventory: dict[str, dict],
+               chunks: list[dict]) -> UnitGraph:
+    """Map call coordinates into inventory chunks' half-open byte ranges.
+
+    Frozen chunk text reconstructs source coordinates, including UTF-8 columns.
+    Missing coordinates or source bytes are counted rather than guessed by line.
+    """
+    wanted = set(paths)
+    nodes = {n["id"]: n for n in graph.get("nodes", [])}
+    ids = {s["id"] for entry in inventory.values() for s in entry["symbols"]}
+    parts: dict[str, list[dict]] = defaultdict(list)
+    for chunk in chunks:
+        parts[chunk["path"]].append(chunk)
+    sources = {path: b"".join(c.get("text", "").encode("utf-8")
+                              for c in sorted(rows, key=lambda c: c.get("start_byte", 0))).decode("utf-8")
+               for path, rows in parts.items()}
+    at = SourceOwnership(sources, chunks, ids).at
+
+    def endpoint(node: dict) -> str | None:
+        return at(node.get("path"), node.get("start_line"))
+
+    outgoing, incoming, ambiguous = [], [], []
+    dependencies: dict[str, set[str]] = defaultdict(set)
+    external_calls: Counter = Counter()
+    unresolved: dict[str, list[str]] = defaultdict(list)
+    skeleton: Counter = Counter()
+    internal = unmapped = 0
+    for edge in sorted(graph.get("edges", []), key=lambda e: str(e.get("id", ""))):
+        source = nodes.get(edge.get("source"), {})
+        target = nodes.get(edge.get("target"), {})
+        path = edge.get("path") or source.get("path")
+        target_path = target.get("path")
+        local_source, local_target = path in wanted, target_path in wanted
+        if edge.get("kind") == "import":
+            if edge.get("status") == "external" and local_source:
+                dependencies[path].add(str(target.get("name") or edge.get("target_expression")
+                                           or edge.get("expression") or "unknown"))
+            if edge.get("status") == "resolved" and path and target_path:
+                left = path.rpartition("/")[0] or "."
+                right = target_path.rpartition("/")[0] or "."
+                skeleton[left, right] += 1
+            continue
+        if edge.get("kind") != "call" or not (local_source or local_target):
+            continue
+        caller = at(path, edge.get("line"), edge.get("column"))
+        callee = endpoint(target)
+        status = edge.get("status")
+        if status == "resolved":
+            if not caller or not callee:
+                unmapped += 1
+                continue
+            if local_source and local_target:
+                internal += 1
+            elif local_source:
+                outgoing.append((caller, callee, target_path, target.get("start_line")))
+            else:
+                incoming.append((callee, caller, path, edge.get("line")))
+        elif status == "ambiguous":
+            candidates = []
+            for candidate in (edge.get("candidates") or [])[:3]:
+                node = nodes.get(candidate, {}) if isinstance(candidate, str) else candidate
+                candidates.append(endpoint(node) or str(node.get("name") or candidate))
+            ambiguous.append(EvidenceRow("ambiguous", f"? {caller or path}: "
+                f"{edge.get('expression') or edge.get('target_expression') or '?'} => "
+                + "; ".join(candidates) + f" ({len(edge.get('candidates') or [])} candidates)"))
+        elif status == "external" and local_source:
+            external_calls[path] += 1
+        elif status == "unresolved" and local_source:
+            expression = str(edge.get("target_expression") or edge.get("expression") or "?")
+            unresolved[path].append(expression.split("(", 1)[0].strip()[:80])
+
+    rows: list[EvidenceRow] = []
+    # Group by caller/callee without duplicating an inventory id for every site.
+    # Repeated sites remain counted even if their endpoint pair is the same.
+    for category, arrow, relations in (("out", "→", outgoing), ("in", "←", incoming)):
+        grouped: dict[tuple, Counter] = defaultdict(Counter)
+        for local, remote, remote_path, line in relations:
+            grouped[local][remote, remote_path, line] += 1
+        for local, destinations in sorted(grouped.items()):
+            for (remote, remote_path, line), count in sorted(destinations.items()):
+                # Each row is independently truncatable; repeated local ids are
+                # compressed by render_rows below, keeping omission counts exact.
+                rows.append(EvidenceRow(category, f"{local} {arrow} {remote} ({remote_path}:{line})"
+                                         + (f" ×{count}" if count > 1 else ""), count))
+    rows += sorted(ambiguous, key=lambda r: r.text)
+    for path in sorted(wanted):
+        names = ", ".join(sorted(dependencies[path])) or "none"
+        rows.append(EvidenceRow("dependencies", f"External imports {path}: {names}; "
+                    f"external calls={external_calls[path]}",
+                    len(dependencies[path]) + external_calls[path]))
+        if unresolved[path]:
+            names = list(dict.fromkeys(unresolved[path]))[:5]
+            rows.append(EvidenceRow("unresolved", f"Unresolved calls {path}: {len(unresolved[path])}; "
+                                    + "; ".join(names), len(unresolved[path])))
+
+    symbol_kinds = {s["id"]: s.get("kind") for entry in inventory.values() for s in entry["symbols"]}
+    cfg_nodes: dict[str, list[dict]] = defaultdict(list)
+    unmapped_owners = set()
+    for node in nodes.values():
+        if node.get("kind") == "cfg_node" and node.get("path") in wanted:
+            owner = nodes.get(node.get("owner"), {})
+            method_name = str(owner.get("name", "")).rsplit(".", 1)[-1]
+            if _SYNTHETIC_CFG_OWNER.search(method_name):
+                continue
+            uid = endpoint(owner)
+            if uid and symbol_kinds.get(uid) in {"function", "method"}:
+                cfg_nodes[uid].append(node)
+            else:
+                unmapped_owners.add(node.get("owner"))
+    for uid, members in sorted(cfg_nodes.items()):
+        controls = sum(n.get("label") == "CONTROL_STRUCTURE" for n in members)
+        returns = sum(n.get("label") == "RETURN" for n in members)
+        if controls or returns:
+            rows.append(EvidenceRow("cfg", f"CFG {uid}: control_structures={controls}, "
+                                    f"returns={returns}"))
+    # A package skeleton is deliberately small; preserve its omitted pair count.
+    for (left, right), count in sorted(skeleton.items())[:12]:
+        rows.append(EvidenceRow("packages", f"Package imports {left} → {right}: {count}"))
+    if len(skeleton) > 12:
+        rows.append(EvidenceRow("packages", f"Package imports: {len(skeleton) - 12} other package pairs",
+                                len(skeleton) - 12))
+    if not skeleton:
+        rows.append(EvidenceRow("packages", "Package imports: no resolved package links", 0))
+    # Package context survives before lower-priority per-function cfg hints.
+    rows.sort(key=lambda row: {"out": 0, "in": 1, "ambiguous": 2,
+                              "dependencies": 3, "unresolved": 3,
+                              "packages": 4, "cfg": 5}[row.category])
+    return UnitGraph(tuple(rows), internal, unmapped, len(unmapped_owners),
+                     any(n.get("label") == "THROW" for n in nodes.values()))
+
+
+def render_rows(rows: list[EvidenceRow]) -> list[str]:
+    """Group adjacent relations by local inventory id, without new identities."""
+    lines: list[str] = []
+    previous = None
+    for row in rows:
+        if row.category in {"out", "in"}:
+            arrow = " → " if row.category == "out" else " ← "
+            local, remote = row.text.split(arrow, 1)
+            key = row.category, local
+            if previous == key:
+                lines.append("  " + arrow.strip() + " " + remote)
+            else:
+                lines.append(row.text)
+            previous = key
+        else:
+            lines.append(row.text)
+            previous = None
+    return lines
+
+
+def graph_text(evidence: UnitGraph, *, budget: int | None = None) -> str:
+    """Drop a suffix in reverse priority, accounting for every omitted relation.
+
+    The notice itself participates in token accounting. Full host files use
+    budget=None and the exact same rows. No dataflow nodes are ever rendered.
+    """
+    rows = list(evidence.rows)
+    suffix: list[Counter] = [Counter() for _ in range(len(rows) + 1)]
+    for index in range(len(rows) - 1, -1, -1):
+        suffix[index] = suffix[index + 1].copy()
+        if rows[index].count:
+            suffix[index][rows[index].category] += rows[index].count
+
+    def render(keep: int) -> str:
+        omitted = suffix[keep]
+        lines = ["LOCAL GRAPH EVIDENCE (unit ids; dependencies count import names + external call sites; graph evidence is not proof of behavior).",
+                 f"Batch-internal resolved calls: {evidence.internal_calls} (source visible); "
+                 f"unmapped resolved calls: {evidence.unmapped_calls}; "
+                 f"unmapped cfg owners: {evidence.unmapped_cfg}."
+                 + ("" if evidence.exception_exits_exported else
+                    " exception exits are not exported by this graph.")]
+        lines += render_rows(rows[:keep])
+        lines.append("Omitted: " + (", ".join(f"{k}={v}" for k, v in sorted(omitted.items())) or "none")
+                     + ". Omitted evidence is unavailable in this excerpt.")
+        return "\n".join(lines)
+
+    text = render(len(rows))
+    if budget is None or count_text_tokens(text) <= budget:
+        return text
+    low, high = 0, len(rows) - 1
+    best = ""
+    while low <= high:
+        middle = (low + high) // 2
+        candidate = render(middle)
+        if count_text_tokens(candidate) <= budget:
+            best = candidate
+            low = middle + 1
+        else:
+            high = middle - 1
+    if best:
+        return best
+    text = (f"Graph: internal={evidence.internal_calls}, unmapped={evidence.unmapped_calls}, "
+            f"unmapped_cfg={evidence.unmapped_cfg}; omitted "
+            + ",".join(f"{k}={v}" for k, v in sorted(suffix[0].items()))
+            + "; dependencies unit=import names + external call sites"
+            + ("" if evidence.exception_exits_exported else
+               "; exception exits are not exported by this graph"))
+    return text if count_text_tokens(text) <= budget else ""
+
+
+def graph_prompt_lines(graph: Any, paths: list[str], *, embed: bool,
+                       inventory: dict[str, dict] | None = None,
+                       chunks: list[dict] | None = None,
+                       source_tokens: int | None = DEFAULT_GRAPH_MAX_TOKENS,
+                       max_tokens: int | None = DEFAULT_GRAPH_MAX_TOKENS,
+                       graph_file: str | None = None) -> list[str]:
+    """Status notices retain their original meanings; provided evidence is bounded."""
+    state, detail = graph_state(graph)
+    if state == "absent":
+        return ["GRAPH: not provided for this run. Do not infer imports, calls or data flow; state the "
+                "boundary in 'uncertainties' when a relation would matter.", ""]
+    if state == "unsupported":
+        return [f"GRAPH: unsupported for this run ({detail}). Do not infer relations from the graph; "
+                "state unproven boundaries in 'uncertainties'.", ""]
+    if state == "failed":
+        return [f"GRAPH: analysis failed or was unavailable ({detail}). Treat relations as unknown, not "
+                "absent, and do not invent them.", ""]
+    if state == "empty":
+        return ["GRAPH: provided and successfully empty (the analyzer ran and found no node or relation). "
+                "An empty graph is a verified result, not a missing graph; still state any unproven boundary.", ""]
+    if not any(n.get("path") in set(paths) for n in graph.get("nodes", [])):
+        return [f"GRAPH: provided for this run ({detail}) but no graph node anchors the listed file(s). This is "
+                "not an empty graph and not a missing graph: the analyzer produced no symbol node for these "
+                "paths. Do not infer either the presence or the absence of relations from it; keep unproven "
+                "boundaries in 'uncertainties'.", ""]
+    evidence = unit_graph(graph, paths, inventory or {}, chunks or [])
+    limits = [limit for limit in (max_tokens, source_tokens) if limit is not None]
+    budget = max(0, min(limits)) if limits else None
+    text = graph_text(evidence, budget=budget)
+    if text:
+        access = []
+        if budget is not None:
+            if embed:
+                access = ["Graph access: this completion is tool-free; omitted evidence cannot be inspected.", ""]
+            elif graph_file:
+                access = [f"Graph access: Read the complete evidence for this batch at {graph_file}", ""]
+        return access + [text, ""]
+    # Like absent/failed graph notices, this is status, not relation payload.
+    # A hard zero/tiny payload cap must never imply there were no relations.
+    omitted = Counter()
+    for row in evidence.rows:
+        if row.count:
+            omitted[row.category] += row.count
+    notice = (f"GRAPH STATUS (outside payload cap): budget={budget} tokens; omitted "
+              + ",".join(f"{kind}={count}" for kind, count in sorted(omitted.items()))
+              + f"; internal={evidence.internal_calls}; unmapped={evidence.unmapped_calls}; "
+              + f"unmapped_cfg={evidence.unmapped_cfg}; dependencies unit=import names + external call sites."
+              + ("" if evidence.exception_exits_exported else
+                 " exception exits are not exported by this graph."))
+    if not embed and graph_file:
+        notice += f" Read full evidence: {graph_file}"
+    return [notice, ""]