deploy-guard-engine 0.1.0__py3-none-any.whl

deploy_guard/explain/explainer.py

@@ -0,0 +1,504 @@
+"""Turn a parsed traceback into an explanation, using the project's analysis."""
+
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from deploy_guard.analysis.findings import Finding, analyze_function
+from deploy_guard.analysis.nullability import (
+    NullabilityAnalysis,
+    _witness_conditions,
+)
+from deploy_guard.analysis.paths import behavior_spec
+from deploy_guard.explain.traceback_parse import Frame, ParsedTraceback
+from deploy_guard.frontend import build_cfg
+from deploy_guard.ir.cfg import CFG, Terminator
+from deploy_guard.ir.model import FunctionDef, Module, Project
+
+
+@dataclass
+class CrashSite:
+    module: Module
+    fn: FunctionDef | None
+    lineno: int
+    source: str
+    cfg: object = None   # CFG, set during analysis
+    ctx: object = None   # AnalysisContext, set during analysis
+
+
+@dataclass
+class Explanation:
+    exc_type: str
+    exc_message: str
+    site: CrashSite | None = None
+    why: list[str] = field(default_factory=list)
+    witness: list[str] = field(default_factory=list)
+    call_chain: list[str] = field(default_factory=list)
+    linked_finding: Finding | None = None
+    fix_hint: str | None = None
+    notes: list[str] = field(default_factory=list)
+    none_var: str | None = None  # the variable found to be None, if any
+
+
+def explain_traceback(tb: ParsedTraceback, project: Project, ctx=None) -> Explanation:
+    exp = Explanation(exc_type=tb.exc_type, exc_message=tb.exc_message)
+
+    if not tb.ok:
+        exp.notes.append(
+            "Could not find an exception line. Paste the whole traceback, "
+            "including the final `SomeError: message` line."
+        )
+        return exp
+
+    if ctx is None:
+        try:
+            from deploy_guard.engine import build_context
+
+            ctx = build_context(project)
+        except Exception:  # pragma: no cover
+            ctx = None
+
+    resolved = [_resolve_frame(f, project) for f in tb.frames]
+    exp.call_chain = _format_chain(resolved)
+
+    site = None
+    for frame, module, fn in reversed(resolved):
+        if module is not None:
+            site = CrashSite(module=module, fn=fn, lineno=frame.lineno, source=frame.source)
+            break
+    exp.site = site
+
+    if site is None:
+        exp.notes.append(
+            "No frame in the traceback points into this project. Check --project, "
+            "or the crash is entirely inside a third-party library."
+        )
+        return exp
+
+    if site.fn is None or not isinstance(
+        site.fn.raw, (ast.FunctionDef, ast.AsyncFunctionDef)
+    ):
+        exp.notes.append(
+            f"Line {site.lineno} is not inside a top-level function the engine "
+            "models (module scope, or a function nested in another). Explanation "
+            "is limited to the traceback structure."
+        )
+        return exp
+
+    cfg = build_cfg(site.fn)
+    bid = _block_for_line(cfg, site.lineno)
+    if bid is not None:
+        exp.witness = _witness_conditions(cfg, bid)
+
+    site.cfg = cfg
+    site.ctx = ctx
+    handler = _HANDLERS.get(tb.exc_short, _explain_generic)
+    handler(exp, tb, site, cfg)
+
+    # Trace a None that came in as a parameter back to the caller frame.
+    _trace_none_across_frames(exp, tb, resolved, ctx)
+
+    # Link to a full-scan finding at the same spot.
+    try:
+        spec = behavior_spec(site.fn, cfg)
+        for fnd in analyze_function(site.fn, cfg, spec, ctx):
+            if abs(fnd.lineno - site.lineno) <= 1:
+                exp.linked_finding = fnd
+                break
+    except Exception:  # pragma: no cover - never let linking break explain
+        pass
+
+    return exp
+
+
+def _trace_none_across_frames(exp: Explanation, tb, resolved, ctx) -> None:
+    """If the crash variable is a parameter, name the caller frame's call site
+    and which argument it passed."""
+    site = exp.site
+    if site is None or site.fn is None:
+        return
+    var = exp.none_var
+    if var is None:
+        return
+    param_names = [p.name for p in site.fn.params]
+    if var not in param_names:
+        return
+    param_index = param_names.index(var)
+
+    # the frame directly above the crash frame in the traceback
+    project_frames = [(f, m, fn) for (f, m, fn) in resolved if m is not None]
+    if len(project_frames) < 2:
+        exp.why.append(
+            f"`{var}` is a parameter - it was already None when this function "
+            "was called. Check the call site one frame up."
+        )
+        return
+    caller_frame, _cm, caller_fn = project_frames[-2]
+    if caller_fn is None or ctx is None:
+        return
+    for site_call in ctx.callgraph.callees_of(caller_fn.qualname):
+        if site_call.callee_qualname != site.fn.qualname:
+            continue
+        call = site_call.node
+        arg_src = None
+        if param_index < len(call.args):
+            try:
+                arg_src = ast.unparse(call.args[param_index])
+            except Exception:
+                arg_src = None
+        else:
+            for kw in call.keywords:
+                if kw.arg == var:
+                    try:
+                        arg_src = ast.unparse(kw.value)
+                    except Exception:
+                        pass
+        loc = f"{caller_fn.qualname.rsplit('.', 1)[-1]}:{site_call.lineno}"
+        if arg_src:
+            exp.why.append(
+                f"`{var}` is a parameter; {loc} passes `{arg_src}` for it - "
+                "that expression is what can be None."
+            )
+        else:
+            exp.why.append(f"`{var}` is a parameter passed by {loc}.")
+        return
+
+
+# --------------------------------------------------------------------------
+# frame resolution
+# --------------------------------------------------------------------------
+
+def _resolve_frame(
+    frame: Frame, project: Project
+) -> tuple[Frame, Module | None, FunctionDef | None]:
+    fpath = Path(frame.file)
+    for m in project.modules:
+        try:
+            if m.path == fpath or m.path.resolve() == fpath.resolve():
+                return frame, m, _fn_at(m, frame.lineno)
+        except OSError:
+            pass
+
+    best: Module | None = None
+    best_score = 0
+    fparts = fpath.parts
+    for m in project.modules:
+        score = _common_suffix_len(fparts, m.path.parts)
+        if score > best_score:
+            best_score, best = score, m
+    if best is not None and best_score >= 1:
+        return frame, best, _fn_at(best, frame.lineno)
+    return frame, None, None
+
+
+def _common_suffix_len(a: tuple[str, ...], b: tuple[str, ...]) -> int:
+    n = 0
+    for x, y in zip(reversed(a), reversed(b)):
+        if x != y:
+            break
+        n += 1
+    return n
+
+
+def _fn_at(module: Module, lineno: int) -> FunctionDef | None:
+    candidates = [
+        fn
+        for fn in module.iter_functions()
+        if fn.span.lineno <= lineno <= max(fn.span.end_lineno, fn.span.lineno)
+    ]
+    if not candidates:
+        return None
+    return max(candidates, key=lambda fn: fn.span.lineno)  # innermost
+
+
+# --------------------------------------------------------------------------
+# CFG helpers
+# --------------------------------------------------------------------------
+
+def _block_for_line(cfg: CFG, lineno: int) -> int | None:
+    exact = [
+        b.id
+        for b in cfg.iter_blocks()
+        if any(s.lineno == lineno for s in b.statements)
+    ]
+    if exact:
+        return exact[0]
+    # nearest statement at or before the line
+    best_id, best_line = None, -1
+    for b in cfg.iter_blocks():
+        for s in b.statements:
+            if s.lineno <= lineno and s.lineno > best_line:
+                best_id, best_line = b.id, s.lineno
+    return best_id
+
+
+def _statements_on_line(cfg: CFG, lineno: int) -> list[ast.AST]:
+    return [
+        s.node
+        for b in cfg.iter_blocks()
+        for s in b.statements
+        if s.lineno == lineno
+    ]
+
+
+def _names_subscripted_or_attr_on_line(cfg: CFG, lineno: int) -> list[str]:
+    out: list[str] = []
+    for node in _statements_on_line(cfg, lineno):
+        for sub in ast.walk(node):
+            if isinstance(sub, (ast.Attribute, ast.Subscript)) and isinstance(
+                sub.value, ast.Name
+            ):
+                if sub.value.id not in out:
+                    out.append(sub.value.id)
+    return out
+
+
+# --------------------------------------------------------------------------
+# per-exception explanations
+# --------------------------------------------------------------------------
+
+def _explain_none(exp: Explanation, tb: ParsedTraceback, site: CrashSite, cfg: CFG) -> None:
+    attr = None
+    m = re.search(r"has no attribute '([^']+)'", tb.exc_message)
+    if m:
+        attr = m.group(1)
+
+    hits = [
+        nf
+        for nf in NullabilityAnalysis(site.fn, cfg, site.ctx).find_none_derefs()
+        if abs(nf.lineno - site.lineno) <= 1
+    ]
+    if hits:
+        nf = hits[0]
+        exp.none_var = nf.var
+        exp.why.append(
+            f"`{nf.var}` can be None at line {site.lineno} - {nf.reason}."
+        )
+        if nf.witness:
+            exp.why.append("It reaches this line when: " + " and ".join(nf.witness))
+
+        if _none_check_is_after_deref(cfg, site.lineno, nf.var):
+            exp.why.append(
+                f"This line *does* check `{nf.var} is None`, but only after "
+                f"dereferencing it - `and`/`or` evaluate left to right, so the "
+                f"attribute access runs first and raises before the check."
+            )
+            exp.fix_hint = (
+                f"reorder the check so `{nf.var} is None` comes first: "
+                f"`if {nf.var} is None or {nf.var}...:`"
+            )
+        else:
+            exp.fix_hint = (
+                f"guard it: `if {nf.var} is not None:` before line {site.lineno}, "
+                f"or give it a safe default."
+            )
+    else:
+        cands = _names_subscripted_or_attr_on_line(cfg, site.lineno)
+        if cands:
+            exp.none_var = cands[0]
+        target = (
+            f"`{cands[0]}`" if cands else ("the value" + (f".{attr}" if attr else ""))
+        )
+        exp.why.append(
+            f"{target} is None at this line. The engine could not prove where the "
+            "None came from - it most likely is the return value of a call it "
+            "cannot see through (a library, or a function that returns None on "
+            "some path)."
+        )
+        exp.why.append(
+            "Check what feeds it: arguments passed in, and the result of the last "
+            "assignment before this line."
+        )
+        if cands:
+            exp.fix_hint = f"guard `{cands[0]}` before it is used on line {site.lineno}."
+
+
+def _explain_key(exp: Explanation, tb: ParsedTraceback, site: CrashSite, cfg: CFG) -> None:
+    key = tb.exc_message.strip().strip("'\"")
+    subs = []
+    for node in _statements_on_line(cfg, site.lineno):
+        for sub in ast.walk(node):
+            if isinstance(sub, ast.Subscript):
+                try:
+                    subs.append(ast.unparse(sub))
+                except Exception:
+                    pass
+    where = f" ({', '.join(subs)})" if subs else ""
+    exp.why.append(
+        f"A dictionary/mapping is indexed with a key that is not present: "
+        f"`{key}`{where}."
+    )
+    guarded = _line_has_guard_for_key(cfg, key)
+    if guarded:
+        exp.why.append(
+            f"Other paths in this function do check for `{key}` first "
+            f"({guarded}) - this path does not."
+        )
+    exp.fix_hint = (
+        f"use `.get({key!r})` and handle None, or check `{key!r} in <mapping>` first."
+    )
+
+
+def _explain_index(exp: Explanation, tb: ParsedTraceback, site: CrashSite, cfg: CFG) -> None:
+    exp.why.append(
+        "A sequence is indexed past its length on this path - the code does not "
+        "check that the item exists before subscripting it "
+        "(e.g. `re.search(...).group(1)`, `parts[1]`, `rows[0]`)."
+    )
+    exp.fix_hint = "check the length / truthiness of the sequence before indexing."
+
+
+def _explain_zerodiv(
+    exp: Explanation, tb: ParsedTraceback, site: CrashSite, cfg: CFG
+) -> None:
+    divisors = []
+    for node in _statements_on_line(cfg, site.lineno):
+        for sub in ast.walk(node):
+            if isinstance(sub, ast.BinOp) and isinstance(
+                sub.op, (ast.Div, ast.FloorDiv, ast.Mod)
+            ):
+                try:
+                    divisors.append(ast.unparse(sub.right))
+                except Exception:
+                    pass
+    where = f" - divisor: `{'`, `'.join(divisors)}`" if divisors else ""
+    exp.why.append(f"A division/modulo on this line has a zero right-hand side{where}.")
+    exp.fix_hint = "guard the divisor: `x / d if d else <fallback>`."
+
+
+def _explain_unbound(
+    exp: Explanation, tb: ParsedTraceback, site: CrashSite, cfg: CFG
+) -> None:
+    m = re.search(r"variable '([^']+)'", tb.exc_message) or re.search(
+        r"name '([^']+)'", tb.exc_message
+    )
+    var = m.group(1) if m else "the variable"
+    exp.why.append(
+        f"`{var}` is only assigned on some branches of this function, and this "
+        "path reaches its use without assigning it."
+    )
+    if exp.witness:
+        exp.why.append("The path that skips the assignment: " + " and ".join(exp.witness))
+    exp.fix_hint = f"initialise `{var}` once at the top of the function."
+
+
+def _explain_value(
+    exp: Explanation, tb: ParsedTraceback, site: CrashSite, cfg: CFG
+) -> None:
+    if "invalid literal for int()" in tb.exc_message or "could not convert" in tb.exc_message:
+        conv = []
+        for node in _statements_on_line(cfg, site.lineno):
+            for sub in ast.walk(node):
+                if (
+                    isinstance(sub, ast.Call)
+                    and isinstance(sub.func, ast.Name)
+                    and sub.func.id in ("int", "float")
+                ):
+                    try:
+                        conv.append(ast.unparse(sub))
+                    except Exception:
+                        pass
+        where = f" ({', '.join(conv)})" if conv else ""
+        exp.why.append(
+            f"A numeric conversion{where} is given a string the code never "
+            "validates - it can contain non-numeric characters."
+        )
+        exp.fix_hint = "validate with a regex / try-except before converting."
+    else:
+        _explain_generic(exp, tb, site, cfg)
+
+
+def _explain_generic(
+    exp: Explanation, tb: ParsedTraceback, site: CrashSite, cfg: CFG
+) -> None:
+    raises = []
+    for b in cfg.iter_blocks():
+        if b.terminator == Terminator.RAISE and b.term_node is not None:
+            try:
+                raises.append((b.first_line, ast.unparse(b.term_node)))
+            except Exception:
+                pass
+    if raises:
+        exp.why.append(
+            "This function raises explicitly at: "
+            + "; ".join(f"line {ln}: {src}" for ln, src in sorted(raises) if ln)
+        )
+    exp.why.append(
+        f"No dedicated explainer for {tb.exc_type} yet - use the call chain above "
+        "and the branch conditions to trace which inputs reach line "
+        f"{site.lineno}."
+    )
+
+
+def _none_check_is_after_deref(cfg: CFG, lineno: int, var: str) -> bool:
+    """True if a statement on ``lineno`` is a BoolOp where an operand
+    dereferences ``var`` and a *later* operand tests ``var is [not] None``."""
+    for node in _statements_on_line(cfg, lineno):
+        for boolop in ast.walk(node):
+            if not isinstance(boolop, ast.BoolOp):
+                continue
+            saw_deref = False
+            for operand in boolop.values:
+                derefs_here = any(
+                    isinstance(s, (ast.Attribute, ast.Subscript))
+                    and isinstance(s.value, ast.Name)
+                    and s.value.id == var
+                    for s in ast.walk(operand)
+                )
+                is_none_test = any(
+                    isinstance(s, ast.Compare)
+                    and isinstance(s.left, ast.Name)
+                    and s.left.id == var
+                    and any(isinstance(o, (ast.Is, ast.IsNot)) for o in s.ops)
+                    for s in ast.walk(operand)
+                )
+                if derefs_here:
+                    saw_deref = True
+                elif is_none_test and saw_deref:
+                    return True
+    return False
+
+
+def _line_has_guard_for_key(cfg: CFG, key: str) -> str | None:
+    for b in cfg.iter_blocks():
+        for e in b.succ:
+            if not e.guard_src:
+                continue
+            if f"'{key}'" in e.guard_src or f'"{key}"' in e.guard_src:
+                if " in " in e.guard_src or ".get(" in e.guard_src:
+                    return e.guard_src
+    return None
+
+
+_HANDLERS = {
+    "AttributeError": _explain_none,
+    "TypeError": _explain_none,  # 'NoneType' object is not subscriptable/callable/iterable
+    "KeyError": _explain_key,
+    "IndexError": _explain_index,
+    "ZeroDivisionError": _explain_zerodiv,
+    "UnboundLocalError": _explain_unbound,
+    "NameError": _explain_unbound,
+    "ValueError": _explain_value,
+}
+
+
+# --------------------------------------------------------------------------
+# call-chain formatting
+# --------------------------------------------------------------------------
+
+def _format_chain(
+    resolved: list[tuple[Frame, Module | None, FunctionDef | None]]
+) -> list[str]:
+    out: list[str] = []
+    for frame, module, _fn in resolved:
+        loc = (
+            f"{module.dotted_name}"
+            if module is not None
+            else Path(frame.file).name
+        )
+        marker = "" if module is not None else "   (not in project)"
+        out.append(f"{loc}:{frame.lineno}  {frame.func}{marker}")
+    return out

deploy_guard/explain/render.py

@@ -0,0 +1,60 @@
+"""Render an :class:`Explanation` as terminal text."""
+
+from __future__ import annotations
+
+from deploy_guard.explain.explainer import Explanation
+
+
+def render_explanation(exp: Explanation) -> str:
+    L: list[str] = []
+    exc = exp.exc_type + (f": {exp.exc_message}" if exp.exc_message else "")
+    L.append(f"Explaining:  {exc}")
+    L.append("")
+
+    if exp.site is not None:
+        s = exp.site
+        where = s.fn.qualname if s.fn is not None else f"{s.module.dotted_name} (module scope)"
+        L.append(f"Crash site:  {s.module.path}:{s.lineno}")
+        L.append(f"             in {where}")
+        if s.source:
+            L.append(f"             {s.lineno} | {s.source}")
+        L.append("")
+
+    if exp.why:
+        L.append("Why:")
+        for line in exp.why:
+            L.append(f"  - {line}")
+        L.append("")
+
+    if exp.witness:
+        L.append("Reached when:")
+        L.append("  " + " and ".join(exp.witness))
+        L.append("")
+
+    if exp.call_chain:
+        L.append("Call chain (from the traceback):")
+        for i, frame in enumerate(exp.call_chain):
+            prefix = "  " + ("  " * i) + ("└─ " if i else "")
+            tail = "   <- raised here" if i == len(exp.call_chain) - 1 else ""
+            L.append(f"{prefix}{frame}{tail}")
+        L.append("")
+
+    if exp.linked_finding:
+        f = exp.linked_finding
+        L.append(
+            f"Matches scan finding:  {f.rule} ({f.severity}) at "
+            f"{f.file}:{f.lineno}"
+        )
+        L.append("")
+
+    if exp.fix_hint:
+        L.append(f"Fix:  {exp.fix_hint}")
+        L.append("")
+
+    if exp.notes:
+        L.append("Notes:")
+        for n in exp.notes:
+            L.append(f"  - {n}")
+        L.append("")
+
+    return "\n".join(L).rstrip() + "\n"

deploy_guard/explain/traceback_parse.py

@@ -0,0 +1,89 @@
+"""Parse a pasted CPython traceback into frames + the exception that escaped."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+_FRAME_RE = re.compile(
+    r'^\s*File "(?P<file>.+?)", line (?P<line>\d+), in (?P<func>.+?)\s*$'
+)
+# ``ExceptionType`` or ``pkg.mod.ExceptionType: message``
+_EXC_RE = re.compile(r"^(?P<type>[A-Za-z_][\w.]*)(?::[ ]?(?P<msg>.*))?$")
+
+_CHAIN_SEPARATORS = (
+    "During handling of the above exception, another exception occurred:",
+    "The above exception was the direct cause of the following exception:",
+)
+
+
+@dataclass
+class Frame:
+    file: str
+    lineno: int
+    func: str
+    source: str = ""  # the source line CPython printed under the frame, if any
+
+
+@dataclass
+class ParsedTraceback:
+    frames: list[Frame] = field(default_factory=list)
+    exc_type: str = ""       # possibly dotted, as printed
+    exc_message: str = ""
+    raw: str = ""
+
+    @property
+    def exc_short(self) -> str:
+        return self.exc_type.rsplit(".", 1)[-1]
+
+    @property
+    def ok(self) -> bool:
+        return bool(self.exc_type)
+
+
+def parse_traceback(text: str) -> ParsedTraceback:
+    # A chained traceback prints several blocks; the last one is what escaped.
+    for sep in _CHAIN_SEPARATORS:
+        if sep in text:
+            text = text.split(sep)[-1]
+
+    lines = text.splitlines()
+    frames: list[Frame] = []
+
+    i, n = 0, len(lines)
+    while i < n:
+        m = _FRAME_RE.match(lines[i])
+        if m:
+            source = ""
+            nxt = lines[i + 1] if i + 1 < n else ""
+            if (
+                nxt.strip()
+                and not _FRAME_RE.match(nxt)
+                and (nxt.startswith("    ") or nxt.startswith("\t"))
+            ):
+                source = nxt.strip()
+                i += 1
+            frames.append(
+                Frame(
+                    file=m.group("file"),
+                    lineno=int(m.group("line")),
+                    func=m.group("func"),
+                    source=source,
+                )
+            )
+        i += 1
+
+    exc_type, exc_message = "", ""
+    for ln in reversed(lines):
+        s = ln.rstrip()
+        if not s or s[0].isspace() or s.startswith("Traceback"):
+            continue
+        m = _EXC_RE.match(s)
+        if m:
+            exc_type = m.group("type")
+            exc_message = (m.group("msg") or "").strip()
+            break
+
+    return ParsedTraceback(
+        frames=frames, exc_type=exc_type, exc_message=exc_message, raw=text
+    )

deploy_guard/frontend/__init__.py

@@ -0,0 +1,10 @@
+"""Language frontends: source -> :mod:`deploy_guard.ir`.
+
+M1 ships the Python frontend.  Java (via a bundled JavaParser fat-jar) lands
+in M4 and reuses every analysis pass unchanged.
+"""
+
+from deploy_guard.frontend.python_cfg import build_cfg
+from deploy_guard.frontend.python_frontend import lower_python
+
+__all__ = ["lower_python", "build_cfg"]