vigil-codeintel 0.1.0__py3-none-any.whl

vigil_forensic/gate_checks/ml_checks.py

@@ -0,0 +1,318 @@
+"""ML/NN correctness forensic checks (static AST, Python-only).
+
+Catches machine-learning / quant-trading bugs that generic linters miss and that
+are catastrophic in backtests and live trading:
+
+  ml.lookahead_negative_shift  -- .shift(-N): future data leaks into the present row
+  ml.nondeterministic_split    -- train_test_split(...) with no random_state
+  ml.scaler_fit_on_test        -- .fit()/.fit_transform() on a *_test / *_val array
+  ml.missing_random_seed       -- module uses RNG but never seeds it
+
+Pure AST over file snapshots — never executes the model. Conservative by design:
+prefers a missed case over a false alarm (these run on any Python repo, most of
+which is not ML code).
+"""
+from __future__ import annotations
+
+import ast
+import logging
+
+from vigil_forensic._shared import (
+    EvidenceReference,
+    GateCategory,
+    GateImpact,
+    GateSeverity,
+    RepairKind,
+)
+from vigil_forensic.gate_checks.common import build_check_result, build_finding
+
+_log = logging.getLogger(__name__)
+
+
+# --------------------------------------------------------------------------
+# helpers
+# --------------------------------------------------------------------------
+
+def _negative_int(node: ast.AST) -> int | None:
+    """Return the negative int value of *node* if it is a negative int literal."""
+    if (
+        isinstance(node, ast.UnaryOp)
+        and isinstance(node.op, ast.USub)
+        and isinstance(node.operand, ast.Constant)
+        and isinstance(node.operand.value, int)
+        and not isinstance(node.operand.value, bool)
+    ):
+        return -node.operand.value
+    if (
+        isinstance(node, ast.Constant)
+        and isinstance(node.value, int)
+        and not isinstance(node.value, bool)
+        and node.value < 0
+    ):
+        return node.value
+    return None
+
+
+def _shift_negative_period(call: ast.Call) -> int | None:
+    """If *call* is ``.shift(-N)`` / ``.shift(periods=-N)`` return the negative N."""
+    if call.args:
+        v = _negative_int(call.args[0])
+        if v is not None:
+            return v
+    for kw in call.keywords:
+        if kw.arg == "periods":
+            v = _negative_int(kw.value)
+            if v is not None:
+                return v
+    return None
+
+
+def _arg_name_lower(node: ast.AST) -> str:
+    """Best-effort lowercase name of an argument expression (Name or attribute)."""
+    if isinstance(node, ast.Name):
+        return node.id.lower()
+    if isinstance(node, ast.Attribute):
+        return node.attr.lower()
+    if isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name):
+        return node.value.id.lower()
+    return ""
+
+
+# --------------------------------------------------------------------------
+# check 1: look-ahead via negative shift
+# --------------------------------------------------------------------------
+
+def _check_lookahead_shift(path: str, tree: ast.AST) -> list:
+    findings = []
+    for node in ast.walk(tree):
+        if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)):
+            continue
+        if node.func.attr != "shift":
+            continue
+        neg = _shift_negative_period(node)
+        if neg is None:
+            continue
+        ln = int(getattr(node, "lineno", 0) or 0)
+        findings.append(build_finding(
+            check_id="ml.lookahead_negative_shift",
+            category=GateCategory.ML,
+            title=f"Negative .shift({neg}) leaks future data in {path}:{ln}",
+            severity=GateSeverity.HIGH,
+            impact=GateImpact.REVISE,
+            summary=(
+                f".shift({neg}) at {path}:{ln} moves a series BACKWARD, exposing future "
+                "values to the current row. This is look-ahead bias: it inflates "
+                "backtest/validation metrics and silently fails in live use."
+            ),
+            recommendation=(
+                "Features must only see past data: use a forward (positive) shift, "
+                "or if this is target construction, ensure the model never receives "
+                "the shifted future column as an input feature."
+            ),
+            evidence=[EvidenceReference(
+                kind="file", path=str(path), detail=f"line:{ln} shift({neg})",
+            )],
+            repair_kind=RepairKind.FIX_CONTRACT.value,
+            executor_action="Replace negative shift with a causal (positive) shift or isolate target alignment.",
+            proof_required="No negative .shift() feeds a feature column; backtest uses only past data.",
+            allowlist_allowed=True,
+        ))
+    return findings
+
+
+# --------------------------------------------------------------------------
+# check 2: non-deterministic train_test_split
+# --------------------------------------------------------------------------
+
+def _is_named_call(node: ast.Call, name: str) -> bool:
+    f = node.func
+    return (isinstance(f, ast.Name) and f.id == name) or (
+        isinstance(f, ast.Attribute) and f.attr == name
+    )
+
+
+def _check_nondeterministic_split(path: str, tree: ast.AST) -> list:
+    findings = []
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call) or not _is_named_call(node, "train_test_split"):
+            continue
+        has_seed = any(kw.arg == "random_state" for kw in node.keywords)
+        if has_seed:
+            continue
+        ln = int(getattr(node, "lineno", 0) or 0)
+        findings.append(build_finding(
+            check_id="ml.nondeterministic_split",
+            category=GateCategory.ML,
+            title=f"train_test_split without random_state in {path}:{ln}",
+            severity=GateSeverity.MEDIUM,
+            impact=GateImpact.REVISE,
+            summary=(
+                f"train_test_split at {path}:{ln} has no random_state. The split is "
+                "non-reproducible: every run shuffles differently, so metrics, "
+                "hyperparameter choices, and bug reports cannot be reproduced."
+            ),
+            recommendation="Pass an explicit random_state=<int> for a reproducible split.",
+            evidence=[EvidenceReference(
+                kind="file", path=str(path), detail=f"line:{ln}",
+            )],
+            repair_kind=RepairKind.FIX_CONTRACT.value,
+            executor_action="Add random_state=<int> to train_test_split.",
+            proof_required="train_test_split carries an explicit random_state.",
+            allowlist_allowed=True,
+        ))
+    return findings
+
+
+# --------------------------------------------------------------------------
+# check 3: scaler / transformer fit on test or validation data (leakage)
+# --------------------------------------------------------------------------
+
+_FIT_METHODS = frozenset({"fit", "fit_transform"})
+_LEAK_TOKENS = ("test", "val", "valid", "holdout", "oot")
+
+
+def _check_scaler_fit_on_test(path: str, tree: ast.AST) -> list:
+    findings = []
+    for node in ast.walk(tree):
+        if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)):
+            continue
+        if node.func.attr not in _FIT_METHODS or not node.args:
+            continue
+        argname = _arg_name_lower(node.args[0])
+        if not argname:
+            continue
+        # token must appear as a word-ish piece (x_test, test_x, X_val) — substring
+        # is acceptable here because these names are conventional and specific.
+        if not any(tok in argname for tok in _LEAK_TOKENS):
+            continue
+        ln = int(getattr(node, "lineno", 0) or 0)
+        findings.append(build_finding(
+            check_id="ml.scaler_fit_on_test",
+            category=GateCategory.ML,
+            title=f".{node.func.attr}() on '{argname}' (eval data) in {path}:{ln}",
+            severity=GateSeverity.HIGH,
+            impact=GateImpact.REVISE,
+            summary=(
+                f"{node.func.attr}() is called on '{argname}' at {path}:{ln}. Fitting a "
+                "scaler/transformer/model on test or validation data leaks information "
+                "from the eval set into training, producing optimistic, invalid metrics."
+            ),
+            recommendation=(
+                "Fit transforms ONLY on the training split, then .transform() (not "
+                "fit_transform) the test/validation split."
+            ),
+            evidence=[EvidenceReference(
+                kind="file", path=str(path), detail=f"line:{ln} {node.func.attr}({argname})",
+            )],
+            repair_kind=RepairKind.FIX_CONTRACT.value,
+            executor_action="Fit on train only; use transform() on eval data.",
+            proof_required="No fit/fit_transform on a *_test/*_val array.",
+            allowlist_allowed=True,
+        ))
+    return findings
+
+
+# --------------------------------------------------------------------------
+# check 4: RNG used but never seeded (non-reproducible)
+# --------------------------------------------------------------------------
+
+# Calls that *consume* randomness (attribute chains ending in these), e.g.
+# np.random.rand / torch.randn / random.shuffle.
+_RNG_CONSUMERS = frozenset({
+    "rand", "randn", "randint", "random", "choice", "shuffle", "permutation",
+    "normal", "uniform", "standard_normal", "sample", "randperm",
+})
+# Calls that *seed* an RNG.
+_SEED_CALLS = frozenset({"seed", "manual_seed", "manual_seed_all", "set_seed"})
+
+
+def _attr_chain(node: ast.AST) -> str:
+    """Return the dotted attribute chain text for a Call.func (best effort)."""
+    parts: list[str] = []
+    cur = node
+    while isinstance(cur, ast.Attribute):
+        parts.append(cur.attr)
+        cur = cur.value
+    if isinstance(cur, ast.Name):
+        parts.append(cur.id)
+    return ".".join(reversed(parts))
+
+
+def _check_missing_seed(path: str, tree: ast.AST) -> list:
+    uses_rng = False
+    has_seed = False
+    rng_line = 0
+    for node in ast.walk(tree):
+        if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)):
+            continue
+        chain = _attr_chain(node.func)
+        leaf = node.func.attr
+        if leaf in _SEED_CALLS:
+            has_seed = True
+        # consumer must be under a random/np.random/torch namespace to avoid
+        # flagging unrelated .sample()/.choice() on domain objects.
+        if leaf in _RNG_CONSUMERS and (
+            "random" in chain or chain.startswith("np.") or chain.startswith("numpy.")
+            or chain.startswith("torch") or chain.startswith("tf.")
+        ):
+            uses_rng = True
+            if not rng_line:
+                rng_line = int(getattr(node, "lineno", 0) or 0)
+    # also count random_state=/seed= kwargs anywhere as "seeded"
+    if uses_rng and not has_seed:
+        for node in ast.walk(tree):
+            if isinstance(node, ast.keyword) and node.arg in ("random_state", "seed"):
+                has_seed = True
+                break
+    if uses_rng and not has_seed:
+        return [build_finding(
+            check_id="ml.missing_random_seed",
+            category=GateCategory.ML,
+            title=f"RNG used but never seeded in {path}",
+            severity=GateSeverity.MEDIUM,
+            impact=GateImpact.REVISE,
+            summary=(
+                f"{path} consumes randomness (first use at line {rng_line}) but never "
+                "sets a seed (np.random.seed / torch.manual_seed / random.seed) and "
+                "passes no random_state=. Runs are non-reproducible — results, bugs, "
+                "and metrics cannot be replicated."
+            ),
+            recommendation="Seed all RNGs at module/entrypoint start (np.random.seed, torch.manual_seed, random.seed) or pass random_state=.",
+            evidence=[EvidenceReference(
+                kind="file", path=str(path), detail=f"first RNG use line:{rng_line}",
+            )],
+            repair_kind=RepairKind.FIX_CONTRACT.value,
+            executor_action="Seed all random number generators deterministically.",
+            proof_required="A seed is set before any RNG consumption in the module.",
+            allowlist_allowed=True,
+        )]
+    return []
+
+
+# --------------------------------------------------------------------------
+# runner
+# --------------------------------------------------------------------------
+
+def run_ml_checks(ctx) -> "object":
+    """Run all ML/NN correctness checks over the snapshot corpus (static)."""
+    findings: list = []
+    snapshots = getattr(ctx, "file_snapshots", None) or {}
+    for path, snap in snapshots.items():
+        if not str(path).endswith(".py"):
+            continue
+        content = getattr(snap, "text", None)
+        if not content:
+            continue
+        try:
+            tree = ast.parse(content)
+        except SyntaxError:
+            continue
+        findings.extend(_check_lookahead_shift(path, tree))
+        findings.extend(_check_nondeterministic_split(path, tree))
+        findings.extend(_check_scaler_fit_on_test(path, tree))
+        findings.extend(_check_missing_seed(path, tree))
+    return build_check_result(
+        check_id="ml_checks",
+        category=GateCategory.ML,
+        findings=findings,
+    )

vigil_forensic/gate_checks/performance_checks.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import ast
+
+from vigil_forensic._shared import EvidenceReference, GateCategory, GateImpact, GateSeverity, RepairKind
+from vigil_forensic.gate_models import PostExecGateContext
+from ..source_analysis import is_source_file
+from .common import build_check_result, build_finding, iter_touched_snapshots, normalize_path
+from ._ast_helpers import parse_python_source_or_emit_finding
+import logging
+_log = logging.getLogger(__name__)
+
+
+EXPENSIVE_CALL_NAMES = {
+    "read_text",
+    "read_bytes",
+    "subprocess.run",
+    "check_output",
+    "path_exists",
+    "execute",
+    "connect",
+    "os.system",
+    "shutil.copy",
+    "shutil.copytree",
+    "shutil.move",
+}
+
+# Files that ARE file-processors by design — reading files in loops is their job.
+# Flagging these generates noise without actionable signal.
+_FILE_PROCESSOR_PATH_FRAGMENTS = (
+    "gate_checks/",
+    "map_builder/",
+    "source_adapters/",
+)
+
+
+def run_performance_checks(ctx: PostExecGateContext):
+    findings = []
+    profile = ctx.repo_profile
+    for snapshot in iter_touched_snapshots(ctx):
+        if not snapshot.exists or not is_source_file(snapshot.path):
+            continue
+        if profile and not profile.is_performance_sensitive(snapshot.path):
+            continue
+        norm_path = snapshot.path.replace("\\", "/")
+        if any(frag in norm_path for frag in _FILE_PROCESSOR_PATH_FRAGMENTS):
+            continue
+        # Sprint C2 (2026-04-23): prefer TestTopology.is_test_path. Legacy
+        # basename check preserved as fallback for contexts where
+        # ProjectContext.test_topology hasn't been built (older call sites,
+        # unit tests constructing a PostExecGateContext by hand).
+        topology = getattr(getattr(ctx, "project_context", None), "test_topology", None)
+        if topology is not None:
+            if topology.is_test_path(norm_path):
+                continue
+        elif norm_path.split("/")[-1].startswith("test_"):
+            continue
+        # B4 (2026-04-23): replaces silent `except SyntaxError: continue` —
+        # meta.syntax_parse_error is now emitted on broken Python sources.
+        tree = parse_python_source_or_emit_finding(
+            snapshot.text,
+            rel_path=normalize_path(snapshot.path),
+            emit_finding=findings.append,
+            emitting_gate="performance.expensive_in_loop",
+        )
+        if tree is None:
+            continue
+        for node in ast.walk(tree):
+            if not isinstance(node, (ast.For, ast.AsyncFor, ast.While)):
+                continue
+            for child in ast.walk(node):
+                if isinstance(child, ast.Call):
+                    name = _call_name(child)
+                    parts = name.rsplit(".", 1)
+                    bare = parts[1] if len(parts) == 2 else None
+                    if name in EXPENSIVE_CALL_NAMES or (bare and bare in EXPENSIVE_CALL_NAMES):
+                        findings.append(
+                            build_finding(
+                                check_id="performance.expensive_in_loop",
+                                category=GateCategory.PERFORMANCE,
+                                title="Touched code performs expensive work inside a loop",
+                                severity=GateSeverity.HIGH,
+                                impact=GateImpact.REVISE,
+                                summary=f"{snapshot.path} calls '{name}' inside a loop, which is a likely hot-path anti-pattern.",
+                                recommendation="Batch the work, cache repeated reads, or move the expensive call out of the loop.",
+                                evidence=[EvidenceReference(kind="file", path=snapshot.path, detail=name)],
+                                repair_kind=RepairKind.REFACTOR.value,
+                                executor_action="Optimize hot code paths",
+                                proof_required="Performance acceptable",
+                                allowlist_allowed=False,
+                            )
+                        )
+    return build_check_result(check_id="performance", category=GateCategory.PERFORMANCE, findings=findings)
+
+
+def _call_name(node: ast.Call) -> str:
+    """Return a qualified call name like 'subprocess.run' or bare 'load'."""
+    func = node.func
+    if isinstance(func, ast.Attribute):
+        if isinstance(func.value, ast.Name):
+            return f"{func.value.id}.{func.attr}"
+        # self.executor.run -> just attr
+        return str(func.attr)
+    if isinstance(func, ast.Name):
+        return str(func.id)
+    return ""