deploy-guard-engine 0.1.0__py3-none-any.whl

deploy_guard/analysis/nullability.py

@@ -0,0 +1,461 @@
+"""Flow-sensitive nullability analysis.
+
+Instead of enumerating every path (which blows up on branchy code), this runs
+a classic forward data-flow fixpoint over the CFG: it propagates, for each
+local variable, whether it is ``None`` / not ``None`` / maybe ``None`` /
+unknown, and *merges* those facts at every branch join.  That is linear-ish
+in the size of the function and covers 100% of it regardless of how many
+branches it has.
+
+It then flags every attribute access or subscript on a value that can be
+``None`` at that point - the "AttributeError/TypeError in production" class of
+bug - and reconstructs an illustrative branch path to the offending line.
+"""
+
+from __future__ import annotations
+
+import ast
+from collections import deque
+from dataclasses import dataclass
+from enum import Enum
+
+from deploy_guard.ir.cfg import CFG, BasicBlock, EdgeKind
+from deploy_guard.ir.model import FunctionDef
+
+# Common stdlib calls whose result is Optional.
+_OPTIONAL_RE_METHODS = {"match", "search", "fullmatch"}
+_BUILTIN_NONNULL = {
+    "list", "dict", "set", "tuple", "str", "bytes", "bytearray", "int", "float",
+    "bool", "frozenset", "object", "complex",
+}
+
+
+class NV(str, Enum):
+    BOTTOM = "bottom"      # not yet assigned on any path reaching here
+    NULL = "null"          # definitely None
+    NOTNULL = "notnull"    # definitely not None
+    NULLABLE = "nullable"  # known to be None on at least one path
+    TOP = "top"            # unknown (e.g. result of an unresolved call)
+
+
+def join(a: NV, b: NV) -> NV:
+    if a == b:
+        return a
+    if a == NV.BOTTOM:
+        return b
+    if b == NV.BOTTOM:
+        return a
+    if NV.TOP in (a, b):
+        return NV.TOP
+    # {NULL, NOTNULL, NULLABLE} in any mix -> NULLABLE
+    return NV.NULLABLE
+
+
+@dataclass
+class _State:
+    env: dict[str, NV]
+    why: dict[str, str]
+
+    def copy(self) -> _State:
+        return _State(dict(self.env), dict(self.why))
+
+
+def _join_states(states: list[_State]) -> _State:
+    out = _State({}, {})
+    keys: set[str] = set()
+    for s in states:
+        keys |= s.env.keys()
+    for k in keys:
+        vals = [s.env.get(k, NV.BOTTOM) for s in states]
+        merged = vals[0]
+        for v in vals[1:]:
+            merged = join(merged, v)
+        out.env[k] = merged
+        if merged in (NV.NULL, NV.NULLABLE):
+            reasons = {s.why.get(k) for s in states if s.why.get(k)}
+            out.why[k] = (
+                next(iter(reasons)) if len(reasons) == 1
+                else "None on at least one branch reaching this point"
+            )
+    return out
+
+
+class NullabilityAnalysis:
+    def __init__(self, fn: FunctionDef, cfg: CFG, ctx=None) -> None:
+        self.fn = fn
+        self.cfg = cfg
+        self.ctx = ctx  # optional AnalysisContext for interprocedural nullability
+        self.file = str(fn.span.file)
+        self._deref_hits: list[tuple[int, int, str, NV, str]] = []
+        # (block_id, lineno, varname, state, reason)
+
+    # -- public -----------------------------------------------------------
+
+    def find_none_derefs(self) -> list[NullFinding]:
+        self._run()
+        out: list[NullFinding] = []
+        seen: set[tuple[str, int]] = set()
+        for block_id, lineno, var, state, reason in self._deref_hits:
+            key = (var, lineno)
+            if key in seen:
+                continue
+            seen.add(key)
+            out.append(
+                NullFinding(
+                    var=var,
+                    lineno=lineno,
+                    definite=(state == NV.NULL),
+                    reason=reason,
+                    witness=_witness_conditions(self.cfg, block_id),
+                )
+            )
+        return out
+
+    # -- fixpoint -------------------------------------------------------
+
+    def _run(self) -> None:
+        cfg = self.cfg
+        in_state: dict[int, _State] = {}
+        out_state: dict[int, _State] = {}
+        entry_state = self._initial_state()
+
+        work: deque[int] = deque([cfg.entry])
+        in_state[cfg.entry] = entry_state
+        cap = max(50, len(cfg.blocks) * 8)
+        steps = 0
+
+        while work and steps < cap:
+            steps += 1
+            bid = work.popleft()
+            block = cfg.blocks[bid]
+
+            preds = [
+                (pid, e)
+                for pid in block.pred
+                for e in cfg.blocks[pid].succ
+                if e.target == bid and pid in out_state
+            ]
+            if bid == cfg.entry:
+                cur_in = entry_state.copy()
+            elif preds:
+                cur_in = _join_states(
+                    [self._refine_for_edge(out_state[pid], e) for pid, e in preds]
+                )
+            else:
+                cur_in = in_state.get(bid, _State({}, {})).copy()
+
+            in_state[bid] = cur_in
+            new_out = self._transfer_block(block, cur_in.copy(), record=True)
+
+            if bid not in out_state or new_out.env != out_state[bid].env:
+                out_state[bid] = new_out
+                for e in block.succ:
+                    if e.target in cfg.blocks:
+                        work.append(e.target)
+
+        # Final pass: re-record derefs against the converged in-states only
+        # (the loop above may have recorded against pre-fixpoint states).
+        self._deref_hits.clear()
+        for bid, block in cfg.blocks.items():
+            if bid in in_state:
+                self._transfer_block(block, in_state[bid].copy(), record=True)
+
+    # -- initial / transfer ------------------------------------------
+
+    def _initial_state(self) -> _State:
+        st = _State({}, {})
+        for p in self.fn.params:
+            if p.name in ("self", "cls"):
+                st.env[p.name] = NV.NOTNULL
+            elif p.annotation and _ann_is_optional(p.annotation):
+                st.env[p.name] = NV.NULLABLE
+                st.why[p.name] = f"parameter '{p.name}' is declared Optional"
+            elif p.default == "None":
+                st.env[p.name] = NV.NULLABLE
+                st.why[p.name] = f"parameter '{p.name}' defaults to None"
+            elif p.annotation:
+                st.env[p.name] = NV.NOTNULL
+            else:
+                st.env[p.name] = NV.TOP
+        return st
+
+    def _transfer_block(self, block: BasicBlock, state: _State, *, record: bool) -> _State:
+        for stmt in block.statements:
+            node = stmt.node
+            if record:
+                self._scan_derefs(node, state, block.id)
+            self._apply_stmt(node, state)
+        return state
+
+    def _apply_stmt(self, node: ast.AST, state: _State) -> None:
+        if isinstance(node, ast.Assign):
+            nv, why = self._eval(node.value, state)
+            for tgt in node.targets:
+                self._bind(tgt, nv, why, state)
+        elif isinstance(node, ast.AnnAssign) and node.value is not None:
+            nv, why = self._eval(node.value, state)
+            self._bind(node.target, nv, why, state)
+        elif isinstance(node, ast.AugAssign):
+            # x += ... : x was used, so treat as not-None afterwards.
+            self._bind(node.target, NV.NOTNULL, "", state)
+        elif isinstance(node, ast.Assert):
+            self._refine(node.test, True, state)
+        elif isinstance(node, ast.NamedExpr):  # walrus at statement level (rare)
+            nv, why = self._eval(node.value, state)
+            self._bind(node.target, nv, why, state)
+        # imports, expr-statements, pass, return, raise: no local rebinding we track
+
+    def _bind(self, target: ast.AST, nv: NV, why: str, state: _State) -> None:
+        if isinstance(target, ast.Name):
+            state.env[target.id] = nv
+            if why and nv in (NV.NULL, NV.NULLABLE):
+                state.why[target.id] = why
+            else:
+                state.why.pop(target.id, None)
+        elif isinstance(target, (ast.Tuple, ast.List)):
+            for elt in target.elts:
+                inner = elt.value if isinstance(elt, ast.Starred) else elt
+                self._bind(inner, NV.TOP, "", state)
+        # Attribute / Subscript targets: not tracked (we only model locals)
+
+    # -- expression nullability ------------------------------------
+
+    def _eval(self, node: ast.AST | None, state: _State) -> tuple[NV, str]:
+        if node is None:
+            return NV.NULL, "explicit None"
+        if isinstance(node, ast.Constant):
+            return (NV.NULL, "explicit None") if node.value is None else (NV.NOTNULL, "")
+        if isinstance(node, ast.Name):
+            return state.env.get(node.id, NV.TOP), state.why.get(node.id, "")
+        if isinstance(node, ast.NamedExpr):
+            nv, why = self._eval(node.value, state)
+            self._bind(node.target, nv, why, state)
+            return nv, why
+        if isinstance(node, (ast.List, ast.Dict, ast.Set, ast.Tuple, ast.ListComp,
+                             ast.DictComp, ast.SetComp, ast.GeneratorExp, ast.JoinedStr,
+                             ast.Lambda, ast.FormattedValue)):
+            return NV.NOTNULL, ""
+        if isinstance(node, ast.IfExp):
+            a, wa = self._eval(node.body, state)
+            b, wb = self._eval(node.orelse, state)
+            merged = join(a, b)
+            if merged in (NV.NULL, NV.NULLABLE):
+                return NV.NULLABLE, "one arm of the conditional expression is None"
+            return merged, ""
+        if isinstance(node, ast.BoolOp):
+            # `a or b` / `a and b` - result is one of the operands.
+            vals = [self._eval(v, state) for v in node.values]
+            merged = vals[0][0]
+            for v, _ in vals[1:]:
+                merged = join(merged, v)
+            return (NV.NULLABLE, "value may come from a None operand") if merged in (
+                NV.NULL, NV.NULLABLE
+            ) else (NV.TOP, "")
+        if isinstance(node, ast.Call):
+            return self._eval_call(node, state)
+        if isinstance(node, ast.Await):
+            return NV.TOP, ""
+        return NV.TOP, ""
+
+    def _eval_call(self, node: ast.Call, state: _State) -> tuple[NV, str]:
+        func = node.func
+        if isinstance(func, ast.Name):
+            if func.id in _BUILTIN_NONNULL or (func.id[:1].isupper()):
+                return NV.NOTNULL, ""
+            qn = self._callee_can_be_none(func.id)
+            if qn is not None:
+                return NV.NULLABLE, f"`{qn.rsplit('.', 1)[-1]}()` can return None"
+            return NV.TOP, ""
+        if isinstance(func, ast.Attribute):
+            # dict.get(k) / os.environ.get(k) with no or None default -> Optional
+            if func.attr == "get" and len(node.args) <= 1 and not node.keywords:
+                return NV.NULLABLE, "result of .get() with no default"
+            if func.attr == "get" and len(node.args) == 2:
+                nv, why = self._eval(node.args[1], state)
+                if nv == NV.NULL:
+                    return NV.NULL, "the default passed to .get() is None"
+                return nv, why
+            if (
+                isinstance(func.value, ast.Name)
+                and func.value.id == "re"
+                and func.attr in _OPTIONAL_RE_METHODS
+            ):
+                return NV.NULLABLE, f"result of re.{func.attr}() can be None"
+            # self.method() / module.func() resolving to a project function
+            dotted = (
+                f"{func.value.id}.{func.attr}"
+                if isinstance(func.value, ast.Name)
+                else func.attr
+            )
+            qn = self._callee_can_be_none(dotted)
+            if qn is not None:
+                return NV.NULLABLE, f"`{qn.rsplit('.', 1)[-1]}()` can return None"
+        return NV.TOP, ""
+
+    def _callee_can_be_none(self, name: str) -> str | None:
+        if self.ctx is None:
+            return None
+        try:
+            return self.ctx.callee_can_be_none(name)
+        except Exception:  # pragma: no cover - never let this break analysis
+            return None
+
+    # -- guard refinement ------------------------------------------
+
+    def _refine_for_edge(self, state: _State, edge) -> _State:
+        if edge.guard is None or edge.kind not in (EdgeKind.TRUE, EdgeKind.FALSE):
+            return state
+        new = state.copy()
+        self._refine(edge.guard, not edge.negated, new)
+        return new
+
+    def _refine(self, test: ast.AST, positive: bool, state: _State) -> None:
+        if isinstance(test, ast.UnaryOp) and isinstance(test.op, ast.Not):
+            self._refine(test.operand, not positive, state)
+            return
+        if isinstance(test, ast.BoolOp) and isinstance(test.op, ast.And) and positive:
+            for v in test.values:
+                self._refine(v, True, state)
+            return
+        if isinstance(test, ast.BoolOp) and isinstance(test.op, ast.Or) and not positive:
+            for v in test.values:
+                self._refine(v, False, state)
+            return
+        if isinstance(test, ast.Compare) and len(test.ops) == 1:
+            left, op, right = test.left, test.ops[0], test.comparators[0]
+            name = _as_name(left) or _as_name(right)
+            other = right if _as_name(left) else left
+            is_none_cmp = isinstance(other, ast.Constant) and other.value is None
+            if name and is_none_cmp and isinstance(op, (ast.Is, ast.Eq)):
+                _set(state, name, NV.NULL if positive else NV.NOTNULL)
+                return
+            if name and is_none_cmp and isinstance(op, (ast.IsNot, ast.NotEq)):
+                _set(state, name, NV.NOTNULL if positive else NV.NULL)
+                return
+        if isinstance(test, ast.Call) and isinstance(test.func, ast.Name) \
+                and test.func.id == "isinstance" and test.args:
+            name = _as_name(test.args[0])
+            if name and positive:
+                _set(state, name, NV.NOTNULL)
+            return
+        name = _as_name(test)
+        if name and positive:
+            _set(state, name, NV.NOTNULL)
+
+    # -- deref scanning ------------------------------------------
+
+    def _scan_derefs(self, node: ast.AST, state: _State, block_id: int) -> None:
+        """Report `x.attr` / `x[...]` where x can be None, honouring the
+        short-circuit narrowing of `and` / `or` and the arms of `a if c else b`
+        so that `x is None or x.attr` is *not* flagged."""
+        self._scan_expr(node, state, block_id)
+
+    def _scan_expr(self, node: ast.AST, state: _State, block_id: int) -> None:
+        if isinstance(node, ast.BoolOp):
+            # `a or b`: b runs only when a was falsy; `a and b`: b when a truthy.
+            narrowed = state.copy()
+            positive = isinstance(node.op, ast.And)
+            for operand in node.values:
+                self._scan_expr(operand, narrowed, block_id)
+                self._refine(operand, positive, narrowed)
+            return
+        if isinstance(node, ast.IfExp):
+            self._scan_expr(node.test, state, block_id)
+            t = state.copy()
+            self._refine(node.test, True, t)
+            self._scan_expr(node.body, t, block_id)
+            f = state.copy()
+            self._refine(node.test, False, f)
+            self._scan_expr(node.orelse, f, block_id)
+            return
+
+        target = None
+        if isinstance(node, ast.Attribute) or isinstance(node, ast.Subscript):
+            target = node.value
+        if isinstance(target, ast.Name):
+            nv = state.env.get(target.id, NV.TOP)
+            if nv in (NV.NULL, NV.NULLABLE):
+                self._deref_hits.append(
+                    (
+                        block_id,
+                        getattr(node, "lineno", 0),
+                        target.id,
+                        nv,
+                        state.why.get(target.id, "may be None here"),
+                    )
+                )
+                # avoid a cascade of hits on the same var in the same statement
+                state.env[target.id] = NV.NOTNULL
+
+        for child in ast.iter_child_nodes(node):
+            if isinstance(child, ast.stmt) or isinstance(child, _SCOPE_NODES):
+                continue
+            self._scan_expr(child, state, block_id)
+
+
+@dataclass
+class NullFinding:
+    var: str
+    lineno: int
+    definite: bool
+    reason: str
+    witness: list[str]
+
+
+# --------------------------------------------------------------------------
+# helpers
+# --------------------------------------------------------------------------
+
+# Never descend into a nested statement body or a nested function/class/lambda
+# scope while scanning one statement's own expressions.
+_SCOPE_NODES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)
+
+
+def _as_name(node: ast.AST) -> str | None:
+    return node.id if isinstance(node, ast.Name) else None
+
+
+def _set(state: _State, name: str, nv: NV) -> None:
+    state.env[name] = nv
+    if nv == NV.NOTNULL:
+        state.why.pop(name, None)
+
+
+def _ann_is_optional(ann: str) -> bool:
+    text = ann.replace(" ", "")
+    return (
+        "Optional[" in text
+        or "|None" in text
+        or "None|" in text
+        or ("Union[" in text and "None" in text)
+    )
+
+
+def _witness_conditions(cfg: CFG, target_block: int, limit: int = 12) -> list[str]:
+    """A shortest branch path from entry to ``target_block``, as guard strings."""
+    from collections import deque as _dq
+
+    prev: dict[int, tuple[int, object]] = {}
+    q: _dq[int] = _dq([cfg.entry])
+    seen = {cfg.entry}
+    while q:
+        bid = q.popleft()
+        if bid == target_block:
+            break
+        for e in cfg.blocks[bid].succ:
+            if e.target in cfg.blocks and e.target not in seen:
+                seen.add(e.target)
+                prev[e.target] = (bid, e)
+                q.append(e.target)
+
+    if target_block not in prev and target_block != cfg.entry:
+        return []
+    chain: list[str] = []
+    cur = target_block
+    while cur in prev:
+        pid, e = prev[cur]
+        if getattr(e, "guard_src", None) and e.kind in (EdgeKind.TRUE, EdgeKind.FALSE):
+            cond = e.guard_src if not e.negated else f"not ({e.guard_src})"
+            chain.append(cond)
+        cur = pid
+    return list(reversed(chain))[:limit]

deploy_guard/analysis/paths.py

@@ -0,0 +1,219 @@
+"""Enumerate execution paths through a CFG and fold them into a behavior spec.
+
+An :class:`ExecPath` is one route from function entry to a terminal point,
+carrying the branch conditions taken to get there and what the function does
+at the end (returns a value, returns ``None`` implicitly, or raises).
+
+Bounds keep this finite:
+
+* ``unroll`` - a block may appear at most this many times on one path, so
+  loops are explored for 0 and 1 iterations by default.
+* ``limit`` - a hard cap on the number of paths; hitting it sets
+  ``PathSet.truncated``.
+
+Exception edges are ignored in M1 (explicit ``raise`` still shows up via the
+block terminator).  Exception-flow sensitivity arrives with abstract
+interpretation in M2.
+"""
+
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass, field
+
+from deploy_guard.ir.cfg import CFG, EdgeKind, Terminator
+from deploy_guard.ir.model import FunctionDef
+
+DEFAULT_LIMIT = 2000
+DEFAULT_UNROLL = 2
+
+_TERMINAL = {Terminator.RETURN, Terminator.RAISE, Terminator.IMPLICIT_RETURN}
+
+
+@dataclass
+class ExecPath:
+    block_ids: list[int]
+    conditions: list[str]
+    outcome: str                 # returns | implicit-none | raises | truncated
+    value: str | None = None  # return value / raised expr, as source text
+    lineno: int | None = None
+
+
+@dataclass
+class PathSet:
+    qualname: str
+    paths: list[ExecPath] = field(default_factory=list)
+    truncated: bool = False          # hit the hard path cap - spec is partial
+    loop_bounded: bool = False       # a loop was explored only to unroll depth
+    notes: list[str] = field(default_factory=list)
+
+    @property
+    def outcomes(self) -> set[str]:
+        return {p.outcome for p in self.paths}
+
+
+def _dedupe_consecutive(items: list[str]) -> list[str]:
+    out: list[str] = []
+    for it in items:
+        if not out or out[-1] != it:
+            out.append(it)
+    return out
+
+
+def _terminal_outcome(term: Terminator) -> str:
+    if term == Terminator.RETURN:
+        return "returns"
+    if term == Terminator.RAISE:
+        return "raises"
+    return "implicit-none"
+
+
+def _terminal_value(term: Terminator, node: ast.AST | None) -> str | None:
+    if term == Terminator.IMPLICIT_RETURN:
+        return "None"
+    if isinstance(node, ast.Return):
+        return ast.unparse(node.value) if node.value is not None else "None"
+    if isinstance(node, ast.Raise):
+        if node.exc is not None:
+            return ast.unparse(node.exc)
+        return "<re-raise>"
+    return None
+
+
+def enumerate_paths(
+    cfg: CFG,
+    *,
+    limit: int = DEFAULT_LIMIT,
+    unroll: int = DEFAULT_UNROLL,
+) -> PathSet:
+    result = PathSet(qualname=cfg.func_qualname)
+
+    def record(blocks: list[int], conds: list[str], outcome: str,
+               value: str | None, lineno: int | None) -> None:
+        result.paths.append(
+            ExecPath(
+                block_ids=blocks,
+                conditions=_dedupe_consecutive(conds),
+                outcome=outcome,
+                value=value,
+                lineno=lineno,
+            )
+        )
+
+    def visit(bid: int, trail: list[int], counts: dict[int, int], conds: list[str]) -> None:
+        if len(result.paths) >= limit:
+            result.truncated = True
+            return
+
+        block = cfg.blocks[bid]
+        trail = [*trail, bid]
+        counts = {**counts, bid: counts.get(bid, 0) + 1}
+
+        if block.terminator in _TERMINAL:
+            record(
+                trail,
+                conds,
+                _terminal_outcome(block.terminator),
+                _terminal_value(block.terminator, block.term_node),
+                getattr(block.term_node, "lineno", block.first_line),
+            )
+            return
+
+        succ = [
+            e
+            for e in block.succ
+            if e.kind != EdgeKind.EXCEPTION and e.target in cfg.blocks
+        ]
+        if not succ:
+            # A dead end with no terminator: e.g. break/continue whose loop
+            # target was outside a modelled region, or the synthetic exit.
+            outcome = "implicit-none" if bid == cfg.exit else "falls-through"
+            record(trail, conds, outcome, "None" if outcome == "implicit-none" else None,
+                   block.first_line)
+            return
+
+        for edge in succ:
+            if counts.get(edge.target, 0) >= unroll:
+                # A loop back-edge explored to its unroll depth: stop following
+                # it, but the loop-exit edge still yields a complete path, so
+                # this is normal bounding, not truncation.
+                result.loop_bounded = True
+                continue
+            next_conds = conds
+            if edge.guard_src is not None and edge.kind in (
+                EdgeKind.TRUE, EdgeKind.FALSE,
+            ):
+                cond = edge.guard_src if not edge.negated else f"not ({edge.guard_src})"
+                next_conds = [*conds, cond]
+            visit(edge.target, trail, counts, next_conds)
+
+    visit(cfg.entry, [], {}, [])
+    if result.truncated:
+        result.notes.append(
+            f"path enumeration hit the {limit}-path cap; behavior spec is partial"
+        )
+    elif result.loop_bounded:
+        result.notes.append(f"loops explored up to {unroll - 1} iteration(s)")
+    return result
+
+
+# --------------------------------------------------------------------------
+# Behavior spec: the human-facing view of the path set.
+# --------------------------------------------------------------------------
+
+@dataclass
+class BehaviorEntry:
+    when: list[str]
+    outcome: str
+    value: str | None
+    lineno: int | None
+
+
+@dataclass
+class BehaviorSpec:
+    qualname: str
+    entries: list[BehaviorEntry] = field(default_factory=list)
+    truncated: bool = False
+    notes: list[str] = field(default_factory=list)
+
+    @property
+    def returns_value(self) -> bool:
+        return any(
+            e.outcome == "returns" and (e.value or "None") != "None" for e in self.entries
+        )
+
+    @property
+    def returns_none_implicitly(self) -> bool:
+        return any(
+            e.outcome == "implicit-none"
+            or (e.outcome == "returns" and (e.value or "None") == "None")
+            for e in self.entries
+        )
+
+
+def behavior_spec(
+    fn: FunctionDef,
+    cfg: CFG,
+    *,
+    limit: int = DEFAULT_LIMIT,
+    unroll: int = DEFAULT_UNROLL,
+) -> BehaviorSpec:
+    pset = enumerate_paths(cfg, limit=limit, unroll=unroll)
+    spec = BehaviorSpec(
+        qualname=fn.qualname, truncated=pset.truncated, notes=list(pset.notes)
+    )
+    seen: set[tuple] = set()
+    for path in pset.paths:
+        key = (tuple(path.conditions), path.outcome, path.value)
+        if key in seen:
+            continue
+        seen.add(key)
+        spec.entries.append(
+            BehaviorEntry(
+                when=path.conditions,
+                outcome=path.outcome,
+                value=path.value,
+                lineno=path.lineno,
+            )
+        )
+    return spec