fable-engine 1.3.1__py3-none-any.whl

fable_v2/coder_fleet/mock_auditor.py

@@ -0,0 +1,306 @@
+"""Mock Auditor Engine for Detecting Fake Tautological Tests and Mock Leakage.
+
+Audits test suites for trivial assertions (assert True, 1==1), excessive mocking,
+and absence of negative / error path tests.
+"""
+from __future__ import annotations
+
+import ast
+from typing import Any
+
+
+class MockAuditorEngine:
+    """Engine for auditing test assertions, tautologies, mock stubs, and negative branch coverage."""
+
+    def audit_assertions(self, test_code: str) -> dict[str, Any]:
+        """AST visitor detecting trivial assertions (assert True, assert 1 == 1, assert x is not None as lone check),
+
+        calculating assertion density and flagging tautologies.
+        """
+        try:
+            tree = ast.parse(test_code)
+        except SyntaxError:
+            return {
+                "total_assertions": 0,
+                "trivial_assertions_count": 0,
+                "tautologies": [],
+                "assertion_density": 0.0,
+                "passed_audit": False,
+                "error": "SyntaxError parsing test code",
+            }
+
+        tautologies: list[dict[str, Any]] = []
+        total_assertions = 0
+        test_functions = 0
+
+        class AssertionVisitor(ast.NodeVisitor):
+            def __init__(self) -> None:
+                nonlocal test_functions
+                super().__init__()
+
+            def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+                nonlocal test_functions
+                if node.name.startswith("test_") or node.name.endswith("_test"):
+                    test_functions += 1
+
+                    # Check for lone 'assert x is not None'
+                    assert_stmts = [s for s in node.body if isinstance(s, ast.Assert)]
+                    if len(node.body) == 1 and len(assert_stmts) == 1:
+                        target_assert = assert_stmts[0]
+                        if isinstance(target_assert.test, ast.Compare):
+                            ops = target_assert.test.ops
+                            comparators = target_assert.test.comparators
+                            if any(isinstance(op, ast.IsNot) for op in ops) and any(
+                                isinstance(c, ast.Constant) and c.value is None for c in comparators
+                            ):
+                                tautologies.append({
+                                    "line": target_assert.lineno,
+                                    "col": target_assert.col_offset,
+                                    "type": "lone_not_none_assertion",
+                                    "code_snippet": ast.unparse(target_assert) if hasattr(ast, "unparse") else "assert x is not None",
+                                    "reason": "Test function contains only a single 'assert x is not None' check without behavior verification",
+                                })
+
+                self.generic_visit(node)
+
+            def visit_Assert(self, node: ast.Assert) -> None:
+                nonlocal total_assertions
+                total_assertions += 1
+
+                # Check assert True / assert 1
+                if isinstance(node.test, ast.Constant):
+                    if node.test.value is True or node.test.value == 1:
+                        tautologies.append({
+                            "line": node.lineno,
+                            "col": node.col_offset,
+                            "type": "constant_true_assertion",
+                            "code_snippet": ast.unparse(node) if hasattr(ast, "unparse") else "assert True",
+                            "reason": f"Tautological assertion with constant truth value '{node.test.value}'",
+                        })
+
+                # Check assert 1 == 1, assert "a" == "a"
+                elif isinstance(node.test, ast.Compare):
+                    cmp = node.test
+                    if isinstance(cmp.left, ast.Constant) and len(cmp.comparators) == 1 and isinstance(cmp.comparators[0], ast.Constant):
+                        if cmp.left.value == cmp.comparators[0].value and any(isinstance(op, (ast.Eq, ast.Is)) for op in cmp.ops):
+                            tautologies.append({
+                                "line": node.lineno,
+                                "col": node.col_offset,
+                                "type": "literal_equality_tautology",
+                                "code_snippet": ast.unparse(node) if hasattr(ast, "unparse") else "assert a == a",
+                                "reason": f"Trivial assertion comparing literal {cmp.left.value!r} to itself",
+                            })
+
+                self.generic_visit(node)
+
+            def visit_Call(self, node: ast.Call) -> None:
+                nonlocal total_assertions
+                func_name = ""
+                if isinstance(node.func, ast.Attribute):
+                    func_name = node.func.attr
+                elif isinstance(node.func, ast.Name):
+                    func_name = node.func.id
+
+                if func_name.startswith("assert"):
+                    total_assertions += 1
+
+                    if func_name in ("assertTrue", "assert_true") and node.args:
+                        first_arg = node.args[0]
+                        if isinstance(first_arg, ast.Constant) and first_arg.value is True:
+                            tautologies.append({
+                                "line": node.lineno,
+                                "col": node.col_offset,
+                                "type": "assertTrue_true_call",
+                                "code_snippet": ast.unparse(node) if hasattr(ast, "unparse") else "self.assertTrue(True)",
+                                "reason": "Trivial assertion call assertTrue(True)",
+                            })
+
+                    elif func_name in ("assertEqual", "assert_equal") and len(node.args) >= 2:
+                        a1, a2 = node.args[0], node.args[1]
+                        if isinstance(a1, ast.Constant) and isinstance(a2, ast.Constant) and a1.value == a2.value:
+                            tautologies.append({
+                                "line": node.lineno,
+                                "col": node.col_offset,
+                                "type": "assertEqual_identical_call",
+                                "code_snippet": ast.unparse(node) if hasattr(ast, "unparse") else "self.assertEqual(a, a)",
+                                "reason": f"Trivial assertion call assertEqual with identical constant {a1.value!r}",
+                            })
+
+                self.generic_visit(node)
+
+        visitor = AssertionVisitor()
+        visitor.visit(tree)
+
+        trivial_count = len(tautologies)
+        effective_tests = max(1, test_functions)
+        density = round(total_assertions / effective_tests, 2)
+        passed = (trivial_count == 0) and (total_assertions > 0)
+
+        return {
+            "total_assertions": total_assertions,
+            "trivial_assertions_count": trivial_count,
+            "tautologies": tautologies,
+            "assertion_density": density,
+            "passed_audit": passed,
+        }
+
+    def detect_mock_leakage(self, test_code: str) -> dict[str, Any]:
+        """Check for excessive mock usage where core logic is stubbed out."""
+        try:
+            tree = ast.parse(test_code)
+        except SyntaxError:
+            return {
+                "mock_count": 0,
+                "mock_ratio": 0.0,
+                "flagged_mocks": [],
+                "has_excessive_mocking": False,
+            }
+
+        mock_identifiers = {"Mock", "MagicMock", "patch", "mocker", "monkeypatch", "PropertyMock"}
+        mock_count = 0
+        total_calls = 0
+        flagged: list[dict[str, Any]] = []
+
+        class MockVisitor(ast.NodeVisitor):
+            def visit_Call(self, node: ast.Call) -> None:
+                nonlocal mock_count, total_calls
+                total_calls += 1
+                name = ""
+                if isinstance(node.func, ast.Name):
+                    name = node.func.id
+                elif isinstance(node.func, ast.Attribute):
+                    name = node.func.attr
+
+                if name in mock_identifiers or "mock" in name.lower():
+                    mock_count += 1
+                    flagged.append({
+                        "line": node.lineno,
+                        "name": name,
+                        "reason": f"Invocation of mock factory or stub '{name}'",
+                    })
+                self.generic_visit(node)
+
+            def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+                nonlocal mock_count
+                for dec in node.decorator_list:
+                    dec_name = ""
+                    if isinstance(dec, ast.Name):
+                        dec_name = dec.id
+                    elif isinstance(dec, ast.Attribute):
+                        dec_name = dec.attr
+                    elif isinstance(dec, ast.Call):
+                        if isinstance(dec.func, ast.Name):
+                            dec_name = dec.func.id
+                        elif isinstance(dec.func, ast.Attribute):
+                            dec_name = dec.func.attr
+
+                    if "patch" in dec_name.lower():
+                        mock_count += 1
+                        flagged.append({
+                            "line": dec.lineno if hasattr(dec, "lineno") else node.lineno,
+                            "name": dec_name,
+                            "reason": f"Patch decorator '@{dec_name}' stubbing dependencies",
+                        })
+                self.generic_visit(node)
+
+        MockVisitor().visit(tree)
+        ratio = round(mock_count / max(1, total_calls), 2)
+        excessive = ratio > 0.5 or (mock_count >= 5 and ratio > 0.3)
+
+        return {
+            "mock_count": mock_count,
+            "mock_ratio": ratio,
+            "flagged_mocks": flagged,
+            "has_excessive_mocking": excessive,
+        }
+
+    def enforce_negative_paths(self, test_code: str) -> dict[str, Any]:
+        """Verify presence of negative test cases (assertRaises, except, pytest.raises)."""
+        try:
+            tree = ast.parse(test_code)
+        except SyntaxError:
+            return {
+                "has_negative_tests": False,
+                "negative_test_count": 0,
+                "total_test_cases": 0,
+                "negative_ratio": 0.0,
+                "details": [],
+            }
+
+        negative_details: list[dict[str, Any]] = []
+        total_test_cases = 0
+
+        class NegativePathVisitor(ast.NodeVisitor):
+            def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+                nonlocal total_test_cases
+                if node.name.startswith("test_") or node.name.endswith("_test"):
+                    total_test_cases += 1
+                    # Check if function name indicates negative test
+                    lower_name = node.name.lower()
+                    if any(w in lower_name for w in ("error", "invalid", "fail", "exception", "reject", "negative")):
+                        negative_details.append({
+                            "line": node.lineno,
+                            "function": node.name,
+                            "type": "naming_convention",
+                            "description": f"Negative test detected from name '{node.name}'",
+                        })
+                self.generic_visit(node)
+
+            def visit_With(self, node: ast.With) -> None:
+                for item in node.items:
+                    expr = item.context_expr
+                    expr_name = ""
+                    if isinstance(expr, ast.Call):
+                        if isinstance(expr.func, ast.Attribute):
+                            expr_name = expr.func.attr
+                        elif isinstance(expr.func, ast.Name):
+                            expr_name = expr.func.id
+
+                    if expr_name in ("assertRaises", "raises", "assert_raises"):
+                        negative_details.append({
+                            "line": node.lineno,
+                            "function": None,
+                            "type": "with_assertRaises",
+                            "description": f"Context manager '{expr_name}' testing error expectation",
+                        })
+                self.generic_visit(node)
+
+            def visit_Call(self, node: ast.Call) -> None:
+                func_name = ""
+                if isinstance(node.func, ast.Attribute):
+                    func_name = node.func.attr
+                elif isinstance(node.func, ast.Name):
+                    func_name = node.func.id
+
+                if func_name in ("assertRaises", "assert_raises", "assertFalse"):
+                    negative_details.append({
+                        "line": node.lineno,
+                        "function": None,
+                        "type": "negative_assert_call",
+                        "description": f"Assertion call '{func_name}' testing negative/failure path",
+                    })
+                self.generic_visit(node)
+
+            def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None:
+                # Presence of an except handler inside a test
+                negative_details.append({
+                    "line": node.lineno,
+                    "function": None,
+                    "type": "try_except_block",
+                    "description": "Explicit exception handling checking error recovery",
+                })
+                self.generic_visit(node)
+
+        NegativePathVisitor().visit(tree)
+
+        neg_count = len(negative_details)
+        effective_total = max(1, total_test_cases)
+        ratio = round(neg_count / effective_total, 2)
+
+        return {
+            "has_negative_tests": neg_count > 0,
+            "negative_test_count": neg_count,
+            "total_test_cases": total_test_cases,
+            "negative_ratio": ratio,
+            "details": negative_details,
+        }

fable_v2/coder_fleet/mutation.py

@@ -0,0 +1,216 @@
+"""Mutation Verifier Engine for AST-based Mutant Injection and Test Rigor Auditing.
+
+Generates semantic AST mutations and verifies whether test suites actually catch mutants
+or whether tests are fake / tautological.
+"""
+from __future__ import annotations
+
+import ast
+import copy
+from typing import Any
+
+from .test_harness import TestHarnessEngine
+
+
+class MutationVerifierEngine:
+    """Engine for generating AST-based code mutants and assessing test suite kill rates."""
+
+    def __init__(self, test_harness: TestHarnessEngine | None = None) -> None:
+        self._harness = test_harness or TestHarnessEngine()
+
+    def inject_mutants(self, source_code: str) -> list[dict[str, Any]]:
+        """Generate 3-5 AST-based mutants:
+
+        - inverts >/<, ==/!=
+        - flips True/False
+        - replaces numeric constants
+        - removes return statements / alters return values
+        - inverts arithmetic operators (+/-)
+        """
+        try:
+            tree = ast.parse(source_code)
+        except SyntaxError:
+            return []
+
+        # Find mutation candidates
+        candidates: list[tuple[ast.AST, str, Any]] = []
+
+        for node in ast.walk(tree):
+            # 1. Comparison operations
+            if isinstance(node, ast.Compare):
+                for op in node.ops:
+                    if isinstance(op, ast.Gt):
+                        candidates.append((node, "invert_comparison_gt_to_lt", (op, ast.Lt())))
+                    elif isinstance(op, ast.Lt):
+                        candidates.append((node, "invert_comparison_lt_to_gt", (op, ast.Gt())))
+                    elif isinstance(op, ast.GtE):
+                        candidates.append((node, "invert_comparison_gte_to_lte", (op, ast.LtE())))
+                    elif isinstance(op, ast.LtE):
+                        candidates.append((node, "invert_comparison_lte_to_gte", (op, ast.GtE())))
+                    elif isinstance(op, ast.Eq):
+                        candidates.append((node, "invert_comparison_eq_to_neq", (op, ast.NotEq())))
+                    elif isinstance(op, ast.NotEq):
+                        candidates.append((node, "invert_comparison_neq_to_eq", (op, ast.Eq())))
+
+            # 2. Boolean and numeric constants
+            elif isinstance(node, ast.Constant):
+                if node.value is True:
+                    candidates.append((node, "flip_boolean_true_to_false", False))
+                elif node.value is False:
+                    candidates.append((node, "flip_boolean_false_to_true", True))
+                elif isinstance(node.value, (int, float)) and not isinstance(node.value, bool):
+                    new_val = 1 if node.value == 0 else (node.value + 1)
+                    candidates.append((node, "mutate_numeric_constant", new_val))
+
+            # 3. Return statements
+            elif isinstance(node, ast.Return):
+                if node.value is not None:
+                    candidates.append((node, "remove_return_value", None))
+
+            # 4. Arithmetic operators
+            elif isinstance(node, ast.BinOp):
+                if isinstance(node.op, ast.Add):
+                    candidates.append((node, "invert_arithmetic_add_to_sub", ast.Sub()))
+                elif isinstance(node.op, ast.Sub):
+                    candidates.append((node, "invert_arithmetic_sub_to_add", ast.Add()))
+
+        mutants: list[dict[str, Any]] = []
+        seen_codes: set[str] = {source_code.strip()}
+
+        for target_node, mut_type, mut_arg in candidates:
+            if len(mutants) >= 5:
+                break
+
+            mutated_tree = copy.deepcopy(tree)
+
+            class Mutator(ast.NodeTransformer):
+                def __init__(self, target_line: int, target_col: int) -> None:
+                    self.target_line = target_line
+                    self.target_col = target_col
+                    self.mutated = False
+
+                def visit_Compare(self, n: ast.Compare) -> ast.AST:
+                    self.generic_visit(n)
+                    if not self.mutated and getattr(n, "lineno", -1) == self.target_line and getattr(n, "col_offset", -1) == self.target_col:
+                        if mut_type.startswith("invert_comparison"):
+                            old_op, new_op = mut_arg
+                            new_ops = []
+                            for op in n.ops:
+                                if type(op) is type(old_op) and not self.mutated:
+                                    new_ops.append(new_op)
+                                    self.mutated = True
+                                else:
+                                    new_ops.append(op)
+                            n.ops = new_ops
+                    return n
+
+                def visit_Constant(self, n: ast.Constant) -> ast.AST:
+                    if not self.mutated and getattr(n, "lineno", -1) == self.target_line and getattr(n, "col_offset", -1) == self.target_col:
+                        if mut_type in ("flip_boolean_true_to_false", "flip_boolean_false_to_true", "mutate_numeric_constant"):
+                            n.value = mut_arg
+                            self.mutated = True
+                    return n
+
+                def visit_Return(self, n: ast.Return) -> ast.AST:
+                    self.generic_visit(n)
+                    if not self.mutated and getattr(n, "lineno", -1) == self.target_line and getattr(n, "col_offset", -1) == self.target_col:
+                        if mut_type == "remove_return_value":
+                            n.value = ast.Constant(value=None)
+                            self.mutated = True
+                    return n
+
+                def visit_BinOp(self, n: ast.BinOp) -> ast.AST:
+                    self.generic_visit(n)
+                    if not self.mutated and getattr(n, "lineno", -1) == self.target_line and getattr(n, "col_offset", -1) == self.target_col:
+                        if mut_type.startswith("invert_arithmetic"):
+                            n.op = mut_arg
+                            self.mutated = True
+                    return n
+
+            line = getattr(target_node, "lineno", 1)
+            col = getattr(target_node, "col_offset", 0)
+            mutator = Mutator(line, col)
+            mutated_tree = mutator.visit(mutated_tree)
+            ast.fix_missing_locations(mutated_tree)
+
+            try:
+                mutated_code = ast.unparse(mutated_tree)
+            except Exception:
+                continue
+
+            if mutated_code.strip() not in seen_codes:
+                seen_codes.add(mutated_code.strip())
+                mutants.append({
+                    "mutant_id": f"mutant_{len(mutants) + 1}",
+                    "mutated_code": mutated_code,
+                    "mutation_type": mut_type,
+                    "line": line,
+                })
+
+        return mutants
+
+    def audit_test_strength(self, source_code: str, test_code: str, timeout_sec: float = 3.0) -> dict[str, Any]:
+        """Run test_code against each mutant using run_scratch_test.
+
+        If tests still pass against mutant, mutant survived (test is weak/fake!).
+        If tests fail, mutant killed.
+        Returns {mutants_total, mutants_killed, mutants_survived, mutation_score, is_thorough}.
+        """
+        baseline_code = f"{source_code}\n\n{test_code}"
+        baseline_run = self._harness.run_scratch_test(baseline_code, timeout_sec=timeout_sec)
+
+        if not baseline_run["success"]:
+            return {
+                "mutants_total": 0,
+                "mutants_killed": 0,
+                "mutants_survived": 0,
+                "mutation_score": 0.0,
+                "is_thorough": False,
+                "error": f"Baseline test failed on unmutated code: {baseline_run.get('stderr')}",
+                "survived_mutant_details": [],
+            }
+
+        mutants = self.inject_mutants(source_code)
+        if not mutants:
+            return {
+                "mutants_total": 0,
+                "mutants_killed": 0,
+                "mutants_survived": 0,
+                "mutation_score": 1.0,
+                "is_thorough": True,
+                "survived_mutant_details": [],
+            }
+
+        killed: list[dict[str, Any]] = []
+        survived: list[dict[str, Any]] = []
+
+        for m in mutants:
+            mutant_test = f"{m['mutated_code']}\n\n{test_code}"
+            run_res = self._harness.run_scratch_test(mutant_test, timeout_sec=timeout_sec)
+
+            if not run_res["success"]:
+                # Mutant killed! The test failed as expected on mutated code.
+                killed.append(m)
+            else:
+                # Mutant survived! The test erroneously passed despite bug injected.
+                survived.append({
+                    "mutant_id": m["mutant_id"],
+                    "mutation_type": m["mutation_type"],
+                    "line": m["line"],
+                    "reason": "Test suite passed despite mutation (weak or missing assertion)",
+                })
+
+        total = len(mutants)
+        killed_count = len(killed)
+        survived_count = len(survived)
+        score = round(killed_count / total, 4) if total > 0 else 1.0
+        is_thorough = score >= 0.75
+
+        return {
+            "mutants_total": total,
+            "mutants_killed": killed_count,
+            "mutants_survived": survived_count,
+            "mutation_score": score,
+            "is_thorough": is_thorough,
+            "survived_mutant_details": survived,
+        }