llm-code-validator 0.1.0__py3-none-any.whl

llm_code_validator/__init__.py

@@ -0,0 +1,3 @@
+"""Deterministic API-drift checker for Python source code."""
+
+__version__ = "0.1.0"

llm_code_validator/benchmark.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import argparse
+import json
+import platform
+import statistics
+import time
+import tracemalloc
+from datetime import date
+from pathlib import Path
+
+from .core import check_file, check_source, iter_python_files
+from .versioning import build_version_context
+
+
+def run_benchmark(paths: list[str]) -> dict[str, object]:
+    files = iter_python_files(paths)
+    version_context = build_version_context(paths)
+    timings: list[float] = []
+    diagnostics = 0
+    tracemalloc.start()
+    start = time.perf_counter()
+    for path in files:
+        file_start = time.perf_counter()
+        result = check_file(path, version_context)
+        timings.append(time.perf_counter() - file_start)
+        diagnostics += len(result.diagnostics)
+    total = time.perf_counter() - start
+    _, peak = tracemalloc.get_traced_memory()
+    tracemalloc.stop()
+
+    p50 = statistics.median(timings) if timings else 0.0
+    p95 = statistics.quantiles(timings, n=20)[18] if len(timings) >= 20 else (max(timings) if timings else 0.0)
+    files_per_second = len(files) / total if total else 0.0
+    return {
+        "files": len(files),
+        "diagnostics": diagnostics,
+        "total_seconds": total,
+        "p50_ms": p50 * 1000,
+        "p95_ms": p95 * 1000,
+        "files_per_second": files_per_second,
+        "peak_ram_mb": peak / (1024 * 1024),
+        "hardware": platform.machine(),
+        "os": platform.platform(),
+        "python_version": platform.python_version(),
+        "precision": None,
+        "recall": None,
+        "false_positives": None,
+        "false_negatives": None,
+    }
+
+
+def run_labeled_benchmark(dataset_path: str | Path) -> dict[str, object]:
+    dataset_file = Path(dataset_path)
+    cases = json.loads(dataset_file.read_text(encoding="utf-8"))
+    timings: list[float] = []
+    true_positives = 0
+    false_positives = 0
+    false_negatives = 0
+    false_positive_examples: list[dict[str, str]] = []
+    false_negative_examples: list[dict[str, str]] = []
+    total_expected = 0
+    total_diagnostics = 0
+
+    tracemalloc.start()
+    start = time.perf_counter()
+    for case in cases:
+        case_start = time.perf_counter()
+        result = check_source(case["code"], case.get("path") or f"{case['id']}.py")
+        timings.append(time.perf_counter() - case_start)
+        expected = {(item["library"], item["symbol"]) for item in case.get("expected_diagnostics", [])}
+        actual = {(diagnostic.library, diagnostic.symbol) for diagnostic in result.diagnostics}
+        total_expected += len(expected)
+        total_diagnostics += len(actual)
+        true_positives += len(expected & actual)
+        case_false_positives = actual - expected
+        case_false_negatives = expected - actual
+        false_positives += len(case_false_positives)
+        false_negatives += len(case_false_negatives)
+        for library, symbol in sorted(case_false_positives):
+            false_positive_examples.append(
+                {"case_id": case["id"], "library": library, "symbol": symbol, "reason": "unexpected diagnostic"}
+            )
+        for library, symbol in sorted(case_false_negatives):
+            false_negative_examples.append(
+                {"case_id": case["id"], "library": library, "symbol": symbol, "reason": "missing rule or extraction gap"}
+            )
+    total = time.perf_counter() - start
+    _, peak = tracemalloc.get_traced_memory()
+    tracemalloc.stop()
+
+    precision = true_positives / (true_positives + false_positives) if true_positives + false_positives else 1.0
+    recall = true_positives / (true_positives + false_negatives) if true_positives + false_negatives else 1.0
+    p50 = statistics.median(timings) if timings else 0.0
+    p95 = statistics.quantiles(timings, n=20)[18] if len(timings) >= 20 else (max(timings) if timings else 0.0)
+    return {
+        "dataset": str(dataset_file),
+        "benchmark_date": date.today().isoformat(),
+        "cases": len(cases),
+        "files": len(cases),
+        "diagnostics": total_diagnostics,
+        "expected_diagnostics": total_expected,
+        "true_positives": true_positives,
+        "false_positives": false_positives,
+        "false_negatives": false_negatives,
+        "false_positive_examples": false_positive_examples,
+        "false_negative_examples": false_negative_examples,
+        "precision": precision,
+        "recall": recall,
+        "total_seconds": total,
+        "p50_ms": p50 * 1000,
+        "p95_ms": p95 * 1000,
+        "files_per_second": len(cases) / total if total else 0.0,
+        "peak_ram_mb": peak / (1024 * 1024),
+        "hardware": platform.machine(),
+        "os": platform.platform(),
+        "python_version": platform.python_version(),
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m llm_code_validator.benchmark")
+    parser.add_argument("paths", nargs="*")
+    parser.add_argument("--dataset", help="Run a labeled benchmark dataset JSON file.")
+    parser.add_argument("--output", help="Write JSON benchmark output to a file.")
+    args = parser.parse_args(argv)
+    if args.dataset:
+        payload = run_labeled_benchmark(args.dataset)
+    elif args.paths:
+        payload = run_benchmark(args.paths)
+    else:
+        parser.error("provide one or more paths or --dataset")
+    output = json.dumps(payload, indent=2, sort_keys=True)
+    if args.output:
+        Path(args.output).write_text(output + "\n", encoding="utf-8")
+    print(output)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

llm_code_validator/cli.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import argparse
+import sys
+
+from .core import CheckResult, check_paths, check_stdin, staged_python_files
+from .fixes import fix_file
+from .formatting import format_github, format_json, format_text
+from .signatures import validate_signature_database
+from .versioning import build_version_context
+
+
+def check_staged(requirements: str | None = None, python_version: str | None = None) -> CheckResult:
+    files = staged_python_files()
+    return check_paths(files, requirements=requirements, python_version=python_version) if files else check_paths([])
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="llm-code-validator")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    check = subparsers.add_parser("check", help="Check Python files for known API drift.")
+    check.add_argument("paths", nargs="*", help="Files or directories to scan. Use '-' for stdin.")
+    check.add_argument("--staged", action="store_true", help="Check staged Python files from git.")
+    check.add_argument("--format", choices=["text", "json", "github"], default="text")
+    check.add_argument("--requirements", help="Requirements file used for version assumptions.")
+    check.add_argument("--python-version", help="Target Python version label for result context.")
+    check.add_argument("--show-low-confidence", action="store_true", help="Show lower-confidence diagnostics.")
+
+    fix = subparsers.add_parser("fix", help="Preview or apply deterministic safe fixes.")
+    fix.add_argument("paths", nargs="+", help="Python files to fix.")
+    fix.add_argument("--write", action="store_true", help="Write safe fixes to disk.")
+    fix.add_argument("--requirements", help="Requirements file used for version assumptions.")
+    fix.add_argument("--python-version", help="Target Python version label for result context.")
+
+    validate = subparsers.add_parser("validate-signatures", help="Validate the signature database.")
+    validate.add_argument("--path", help="Path to library_signatures.json.")
+    validate.add_argument(
+        "--require-official-evidence",
+        action="store_true",
+        help="Require diagnostic rules to use source_url or release_note instead of generic notes.",
+    )
+    return parser
+
+
+def _render(result: CheckResult, output_format: str) -> str:
+    if output_format == "json":
+        return format_json(result)
+    if output_format == "github":
+        return format_github(result)
+    return format_text(result)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        if args.command == "check":
+            if args.staged:
+                result = check_paths(
+                    staged_python_files(),
+                    requirements=args.requirements,
+                    python_version=args.python_version,
+                    show_low_confidence=args.show_low_confidence,
+                )
+            elif args.paths == ["-"]:
+                result = check_stdin(args.requirements, args.python_version, args.show_low_confidence)
+            elif args.paths:
+                result = check_paths(args.paths, args.requirements, args.python_version, args.show_low_confidence)
+            else:
+                parser.error("check requires a path, '-', or --staged")
+            output = _render(result, args.format)
+            if output:
+                print(output)
+            return 1 if result.diagnostics else 0
+        if args.command == "fix":
+            version_context = build_version_context(args.paths, args.requirements, args.python_version)
+            exit_code = 0
+            for path in args.paths:
+                result = fix_file(path, write=args.write, version_context=version_context)
+                for preview in result.previews:
+                    print(preview)
+                for skipped in result.skipped:
+                    print(skipped)
+                if result.skipped:
+                    exit_code = 1
+            return exit_code
+        if args.command == "validate-signatures":
+            errors = validate_signature_database(args.path, args.require_official_evidence)
+            if errors:
+                for error in errors:
+                    print(error, file=sys.stderr)
+                return 1
+            print("OK: signature database is valid")
+            return 0
+    except Exception as exc:
+        print(f"llm-code-validator: {exc}", file=sys.stderr)
+        return 2
+
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

llm_code_validator/core.py

@@ -0,0 +1,359 @@
+from __future__ import annotations
+
+import ast
+import subprocess
+import sys
+from pathlib import Path
+
+from .diagnostics import CheckResult, Diagnostic, Fix
+from .signatures import SignatureRule, find_rule, load_signatures
+from .versioning import VersionContext, build_version_context
+
+
+STDIN_PATH = "<stdin>"
+EXCLUDED_DIR_NAMES = {
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pycache__",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+}
+EXCLUDED_DIR_PARTS = {"Lib", "site-packages"}
+
+
+class _CallExtractor(ast.NodeVisitor):
+    def __init__(self) -> None:
+        self.aliases: dict[str, str] = {}
+        self.alias_confidence: dict[str, float] = {}
+        self.calls: list[tuple[str, str, int, int, float, set[str]]] = []
+        self.dynamic_imports: list[tuple[str, int, int]] = []
+        self.returns: dict[str, str] = {}
+
+    def visit_Import(self, node: ast.Import) -> None:
+        for alias in node.names:
+            library = alias.name.split(".")[0]
+            used_name = alias.asname or alias.name
+            self.aliases[used_name] = library
+            self.alias_confidence[used_name] = 1.0
+            self.calls.append((library, alias.name, node.lineno, node.col_offset, 1.0, set()))
+        self.generic_visit(node)
+
+    def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
+        if node.level:
+            return
+        module = node.module or ""
+        library = module.split(".")[0]
+        for alias in node.names:
+            used_name = alias.asname or alias.name
+            self.aliases[used_name] = library
+            self.alias_confidence[used_name] = 1.0
+            qualified_name = f"{module}.{alias.name}" if module else alias.name
+            self.calls.append((library, qualified_name, node.lineno, node.col_offset, 1.0, set()))
+        self.generic_visit(node)
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+        for child in ast.walk(node):
+            if isinstance(child, ast.Return):
+                root = _root_name(child.value) if child.value else None
+                if root in self.aliases:
+                    self.returns[node.name] = self.aliases[root]
+                    break
+        self.generic_visit(node)
+
+    def visit_Assign(self, node: ast.Assign) -> None:
+        if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+            target = node.targets[0].id
+            value = node.value.func if isinstance(node.value, ast.Call) else node.value
+            root = _root_name(value)
+            if root in self.aliases:
+                self.aliases[target] = self.aliases[root]
+                self.alias_confidence[target] = self.alias_confidence.get(root, 0.85)
+            elif isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name):
+                function_name = node.value.func.id
+                if function_name in self.returns:
+                    self.aliases[target] = self.returns[function_name]
+                    self.alias_confidence[target] = 0.75
+        self.generic_visit(node)
+
+    def visit_Attribute(self, node: ast.Attribute) -> None:
+        root = _root_name(node)
+        if root in self.aliases:
+            symbol = _attribute_name(node)
+            if "." in symbol:
+                _, tail = symbol.split(".", 1)
+                symbol = f"{self.aliases[root]}.{tail}"
+            self.calls.append(
+                (
+                    self.aliases[root],
+                    symbol,
+                    node.lineno,
+                    node.col_offset,
+                    self.alias_confidence.get(root, 0.75),
+                    set(),
+                )
+            )
+        self.generic_visit(node)
+
+    def visit_Call(self, node: ast.Call) -> None:
+        if isinstance(node.func, ast.Name) and node.func.id == "__import__":
+            library = _first_string_arg(node)
+            if library:
+                self.dynamic_imports.append((library.split(".")[0], node.lineno, node.col_offset))
+        elif (
+            isinstance(node.func, ast.Attribute)
+            and node.func.attr == "import_module"
+            and isinstance(node.func.value, ast.Name)
+            and node.func.value.id == "importlib"
+        ):
+            library = _first_string_arg(node)
+            if library:
+                self.dynamic_imports.append((library.split(".")[0], node.lineno, node.col_offset))
+        root = _root_name(node.func)
+        if root in self.aliases:
+            symbol = _call_symbol(node.func, self.aliases[root])
+            if symbol:
+                keywords = {keyword.arg for keyword in node.keywords if keyword.arg}
+                self.calls.append(
+                    (
+                        self.aliases[root],
+                        symbol,
+                        node.lineno,
+                        node.col_offset,
+                        self.alias_confidence.get(root, 0.75),
+                        keywords,
+                    )
+                )
+        self.generic_visit(node)
+
+
+def _root_name(node: ast.AST) -> str | None:
+    current = node
+    while isinstance(current, ast.Attribute):
+        current = current.value
+    if isinstance(current, ast.Call):
+        return _root_name(current.func)
+    if isinstance(current, ast.Name):
+        return current.id
+    return None
+
+
+def _attribute_name(node: ast.Attribute) -> str:
+    parts = [node.attr]
+    current = node.value
+    while isinstance(current, ast.Attribute):
+        parts.append(current.attr)
+        current = current.value
+    if isinstance(current, ast.Name):
+        parts.append(current.id)
+    return ".".join(reversed(parts))
+
+
+def _call_symbol(node: ast.AST, library: str) -> str | None:
+    if isinstance(node, ast.Name):
+        return node.id
+    if isinstance(node, ast.Attribute):
+        symbol = _attribute_name(node)
+        if "." in symbol:
+            _, tail = symbol.split(".", 1)
+            return f"{library}.{tail}"
+        return symbol
+    return None
+
+
+def _first_string_arg(node: ast.Call) -> str | None:
+    if not node.args:
+        return None
+    first = node.args[0]
+    if isinstance(first, ast.Constant) and isinstance(first.value, str):
+        return first.value
+    return None
+
+
+def _diagnostic(
+    path: str,
+    line: int,
+    column: int,
+    library: str,
+    rule: SignatureRule,
+    version_context: VersionContext,
+    confidence: float = 1.0,
+) -> Diagnostic:
+    assumption = version_context.assumption_for(library, rule.version_assumption)
+    return Diagnostic(
+        path=path,
+        line=line,
+        column=column + 1,
+        code="LCV001",
+        severity=rule.severity,
+        library=library,
+        symbol=rule.symbol,
+        message=rule.message,
+        evidence=rule.evidence,
+        replacement=rule.replacement,
+        confidence=confidence,
+        version_assumption=assumption,
+        fix=Fix(replacement=rule.replacement, safety=rule.fix_safety),
+    )
+
+
+def check_source(
+    source: str,
+    path: str | None = None,
+    version_context: VersionContext | None = None,
+    show_low_confidence: bool = False,
+) -> CheckResult:
+    display_path = path or STDIN_PATH
+    version_context = version_context or VersionContext(None, {}, used_defaults=True)
+    try:
+        tree = ast.parse(source)
+    except SyntaxError as exc:
+        diagnostic = Diagnostic(
+            path=display_path,
+            line=exc.lineno or 1,
+            column=exc.offset or 1,
+            code="LCV900",
+            severity="error",
+            library="python",
+            symbol="syntax",
+            message=f"Python syntax error: {exc.msg}",
+            confidence=1.0,
+        )
+        return CheckResult(checked_files=1, diagnostics=[diagnostic])
+
+    extractor = _CallExtractor()
+    extractor.visit(tree)
+    signatures = load_signatures()
+    diagnostics: list[Diagnostic] = []
+    seen: set[tuple[str, int, str]] = set()
+
+    for library, symbol, line, column, confidence, keywords in extractor.calls:
+        rule = find_rule(library, symbol, signatures, keywords)
+        if not rule:
+            continue
+        if confidence < 0.8 and not show_low_confidence:
+            continue
+        key = (library, line, rule.symbol)
+        if key in seen:
+            continue
+        seen.add(key)
+        diagnostics.append(_diagnostic(display_path, line, column, library, rule, version_context, confidence))
+
+    for library, line, column in extractor.dynamic_imports:
+        diagnostics.append(
+            Diagnostic(
+                path=display_path,
+                line=line,
+                column=column + 1,
+                code="LCV910",
+                severity="warning",
+                library=library,
+                symbol="dynamic-import",
+                message=f"Dynamic import of {library!r} may hide API usage from static checks.",
+                confidence=0.6,
+            )
+        )
+
+    warnings = []
+    if version_context.used_defaults:
+        warnings.append("No requirements file was evaluated; diagnostics use default signature version assumptions.")
+    return CheckResult(
+        checked_files=1,
+        diagnostics=diagnostics,
+        warnings=warnings,
+    )
+
+
+def check_file(
+    path: str | Path,
+    version_context: VersionContext | None = None,
+    show_low_confidence: bool = False,
+) -> CheckResult:
+    file_path = Path(path)
+    try:
+        source = file_path.read_text(encoding="utf-8")
+    except UnicodeDecodeError:
+        source = file_path.read_text(encoding="utf-8", errors="replace")
+    except OSError as exc:
+        diagnostic = Diagnostic(
+            path=str(file_path),
+            line=1,
+            column=1,
+            code="LCV901",
+            severity="error",
+            library="filesystem",
+            symbol="read",
+            message=f"Could not read file: {exc}",
+            confidence=1.0,
+        )
+        return CheckResult(checked_files=0, diagnostics=[diagnostic])
+    return check_source(source, str(file_path), version_context, show_low_confidence)
+
+
+def iter_python_files(paths: list[str]) -> list[Path]:
+    files: list[Path] = []
+    for raw_path in paths:
+        path = Path(raw_path)
+        if path.is_dir():
+            files.extend(sorted(p for p in path.rglob("*.py") if p.is_file() and not _is_excluded_python_path(p)))
+        elif path.is_file() and path.suffix == ".py":
+            files.append(path)
+    return files
+
+
+def _is_excluded_python_path(path: Path) -> bool:
+    parts = set(path.parts)
+    if parts.intersection(EXCLUDED_DIR_NAMES):
+        return True
+    return EXCLUDED_DIR_PARTS.issubset(parts)
+
+
+def merge_results(results: list[CheckResult]) -> CheckResult:
+    return CheckResult(
+        checked_files=sum(result.checked_files for result in results),
+        diagnostics=[diagnostic for result in results for diagnostic in result.diagnostics],
+        warnings=sorted({warning for result in results for warning in result.warnings}),
+    )
+
+
+def check_paths(
+    paths: list[str],
+    requirements: str | None = None,
+    python_version: str | None = None,
+    show_low_confidence: bool = False,
+) -> CheckResult:
+    files = iter_python_files(paths)
+    if not files:
+        return CheckResult(checked_files=0, warnings=["No Python files were found."])
+    version_context = build_version_context(paths, requirements, python_version)
+    return merge_results([check_file(path, version_context, show_low_confidence) for path in files])
+
+
+def staged_python_files() -> list[str]:
+    proc = subprocess.run(
+        ["git", "diff", "--cached", "--name-only", "--diff-filter=ACMR", "--", "*.py"],
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    if proc.returncode != 0:
+        raise RuntimeError(proc.stderr.strip() or "Could not read staged files from git.")
+    return [line.strip() for line in proc.stdout.splitlines() if line.strip()]
+
+
+def check_stdin(
+    requirements: str | None = None,
+    python_version: str | None = None,
+    show_low_confidence: bool = False,
+) -> CheckResult:
+    version_context = build_version_context(None, requirements, python_version)
+    return check_source(sys.stdin.read(), STDIN_PATH, version_context, show_low_confidence)

llm_code_validator/diagnostics.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Fix:
+    replacement: str | None = None
+    safety: str = "no_fix"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"replacement": self.replacement, "safety": self.safety}
+
+
+@dataclass(frozen=True)
+class Diagnostic:
+    path: str
+    line: int
+    column: int
+    code: str
+    severity: str
+    library: str
+    symbol: str
+    message: str
+    evidence: str | None = None
+    replacement: str | None = None
+    confidence: float = 1.0
+    version_assumption: str | None = None
+    fix: Fix = field(default_factory=Fix)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "path": self.path,
+            "line": self.line,
+            "column": self.column,
+            "code": self.code,
+            "severity": self.severity,
+            "library": self.library,
+            "symbol": self.symbol,
+            "message": self.message,
+            "evidence": self.evidence,
+            "replacement": self.replacement,
+            "confidence": self.confidence,
+            "version_assumption": self.version_assumption,
+            "fix": self.fix.to_dict(),
+        }
+
+
+@dataclass(frozen=True)
+class CheckResult:
+    checked_files: int
+    diagnostics: list[Diagnostic] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "checked_files": self.checked_files,
+            "diagnostics": [diagnostic.to_dict() for diagnostic in self.diagnostics],
+            "warnings": list(self.warnings),
+        }