vallm 0.1.1__py3-none-any.whl

vallm/__init__.py

@@ -0,0 +1,16 @@
+"""vallm - A complete toolkit for validating LLM-generated code."""
+
+from vallm.config import VallmSettings
+from vallm.core.proposal import Proposal
+from vallm.scoring import ValidationResult, Verdict, Issue, validate
+
+__all__ = [
+    "validate",
+    "Proposal",
+    "ValidationResult",
+    "Verdict",
+    "Issue",
+    "VallmSettings",
+]
+
+__version__ = "0.1.1"

vallm/__main__.py

@@ -0,0 +1,5 @@
+"""Allow running vallm as: python -m vallm"""
+
+from vallm.cli import app
+
+app()

vallm/config.py

@@ -0,0 +1,59 @@
+"""Configuration management via pydantic-settings."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class VallmSettings(BaseSettings):
+    """vallm configuration with layered sources: defaults → TOML → env → CLI."""
+
+    model_config = SettingsConfigDict(
+        env_prefix="VALLM_",
+        toml_file="vallm.toml",
+    )
+
+    # Scoring thresholds
+    pass_threshold: float = Field(default=0.8, ge=0.0, le=1.0)
+    review_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
+
+    # Validator toggles
+    enable_syntax: bool = True
+    enable_imports: bool = True
+    enable_complexity: bool = True
+    enable_security: bool = False
+    enable_semantic: bool = False
+
+    # Complexity limits
+    max_cyclomatic_complexity: int = 15
+    max_cognitive_complexity: int = 20
+    max_function_length: int = 100
+
+    # LLM settings (for semantic validator)
+    llm_provider: str = "ollama"
+    llm_model: str = "qwen2.5-coder:7b"
+    llm_base_url: str = "http://localhost:11434"
+    llm_temperature: float = 0.1
+
+    # Sandbox settings
+    sandbox_backend: str = "subprocess"
+    sandbox_timeout: int = 30
+    sandbox_memory_limit: str = "256m"
+
+    # Output
+    output_format: str = "rich"
+    verbose: bool = False
+
+    # Language detection
+    default_language: str = "python"
+
+    @classmethod
+    def from_toml(cls, path: Optional[Path] = None) -> VallmSettings:
+        """Load settings, optionally from a specific TOML file."""
+        if path and path.exists():
+            return cls(_toml_file=str(path))
+        return cls()

vallm/core/__init__.py

@@ -0,0 +1 @@
+"""Core modules for vallm."""

vallm/core/ast_compare.py

@@ -0,0 +1,135 @@
+"""AST comparison utilities using tree-sitter and edit distance."""
+
+from __future__ import annotations
+
+import ast
+from typing import Optional
+
+from tree_sitter_language_pack import get_parser
+
+
+def parse_code(code: str, language: str = "python"):
+    """Parse code using tree-sitter and return the tree."""
+    parser = get_parser(language)
+    return parser.parse(code.encode("utf-8"))
+
+
+def parse_python_ast(code: str) -> Optional[ast.AST]:
+    """Parse Python code using the built-in ast module. Returns None on failure."""
+    try:
+        return ast.parse(code)
+    except SyntaxError:
+        return None
+
+
+def normalize_python_ast(tree: ast.AST) -> str:
+    """Normalize a Python AST by replacing identifiers with canonical names.
+
+    This enables fingerprint-based comparison that ignores variable naming.
+    """
+    class Normalizer(ast.NodeTransformer):
+        def __init__(self):
+            self._name_map: dict[str, str] = {}
+            self._counter = 0
+
+        def _canonical(self, name: str) -> str:
+            if name not in self._name_map:
+                self._name_map[name] = f"var_{self._counter}"
+                self._counter += 1
+            return self._name_map[name]
+
+        def visit_Name(self, node):
+            node.id = self._canonical(node.id)
+            return self.generic_visit(node)
+
+        def visit_FunctionDef(self, node):
+            node.name = self._canonical(node.name)
+            return self.generic_visit(node)
+
+        def visit_arg(self, node):
+            node.arg = self._canonical(node.arg)
+            return self.generic_visit(node)
+
+    normalized = Normalizer().visit(ast.parse(ast.unparse(tree)))
+    return ast.dump(normalized)
+
+
+def python_ast_similarity(code1: str, code2: str) -> float:
+    """Compute structural similarity between two Python code snippets.
+
+    Returns a float between 0.0 and 1.0.
+    """
+    tree1 = parse_python_ast(code1)
+    tree2 = parse_python_ast(code2)
+
+    if tree1 is None or tree2 is None:
+        return 0.0
+
+    norm1 = normalize_python_ast(tree1)
+    norm2 = normalize_python_ast(tree2)
+
+    if norm1 == norm2:
+        return 1.0
+
+    # Use SequenceMatcher for similarity on AST dumps
+    from difflib import SequenceMatcher
+
+    return SequenceMatcher(None, norm1, norm2).ratio()
+
+
+def tree_sitter_node_count(code: str, language: str = "python") -> int:
+    """Count the number of nodes in a tree-sitter parse tree."""
+    tree = parse_code(code, language)
+    count = 0
+
+    def _walk(node):
+        nonlocal count
+        count += 1
+        for child in node.children:
+            _walk(child)
+
+    _walk(tree.root_node)
+    return count
+
+
+def tree_sitter_error_count(code: str, language: str = "python") -> int:
+    """Count syntax errors reported by tree-sitter."""
+    tree = parse_code(code, language)
+    errors = 0
+
+    def _walk(node):
+        nonlocal errors
+        if node.type == "ERROR" or node.is_missing:
+            errors += 1
+        for child in node.children:
+            _walk(child)
+
+    _walk(tree.root_node)
+    return errors
+
+
+def structural_diff_summary(code1: str, code2: str, language: str = "python") -> dict:
+    """Return a summary of structural differences between two code snippets."""
+    tree1 = parse_code(code1, language)
+    tree2 = parse_code(code2, language)
+
+    def _collect_types(node):
+        types = []
+        def _walk(n):
+            types.append(n.type)
+            for child in n.children:
+                _walk(child)
+        _walk(node)
+        return types
+
+    types1 = _collect_types(tree1.root_node)
+    types2 = _collect_types(tree2.root_node)
+
+    set1, set2 = set(types1), set(types2)
+    return {
+        "nodes_before": len(types1),
+        "nodes_after": len(types2),
+        "added_types": set2 - set1,
+        "removed_types": set1 - set2,
+        "common_types": set1 & set2,
+    }

vallm/core/graph_builder.py

@@ -0,0 +1,117 @@
+"""Code graph analysis: import and call graph construction."""
+
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass, field
+
+
+@dataclass
+class ImportEdge:
+    """Represents an import dependency."""
+
+    source_module: str
+    imported_name: str
+    alias: str | None = None
+    line: int = 0
+
+
+@dataclass
+class CallEdge:
+    """Represents a function call relationship."""
+
+    caller: str
+    callee: str
+    line: int = 0
+
+
+@dataclass
+class CodeGraph:
+    """A graph of code relationships (imports and calls)."""
+
+    imports: list[ImportEdge] = field(default_factory=list)
+    calls: list[CallEdge] = field(default_factory=list)
+    functions: list[str] = field(default_factory=list)
+    classes: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        """Serialize to dict for comparison."""
+        return {
+            "imports": [(e.source_module, e.imported_name) for e in self.imports],
+            "calls": [(e.caller, e.callee) for e in self.calls],
+            "functions": sorted(self.functions),
+            "classes": sorted(self.classes),
+        }
+
+
+def build_python_graph(code: str, module_name: str = "<module>") -> CodeGraph:
+    """Build an import/call graph from Python source code."""
+    try:
+        tree = ast.parse(code)
+    except SyntaxError:
+        return CodeGraph()
+
+    graph = CodeGraph()
+    current_scope = module_name
+
+    class GraphVisitor(ast.NodeVisitor):
+        def visit_Import(self, node):
+            for alias in node.names:
+                graph.imports.append(
+                    ImportEdge(
+                        source_module=alias.name,
+                        imported_name=alias.name,
+                        alias=alias.asname,
+                        line=node.lineno,
+                    )
+                )
+            self.generic_visit(node)
+
+        def visit_ImportFrom(self, node):
+            module = node.module or ""
+            for alias in node.names:
+                graph.imports.append(
+                    ImportEdge(
+                        source_module=module,
+                        imported_name=alias.name,
+                        alias=alias.asname,
+                        line=node.lineno,
+                    )
+                )
+            self.generic_visit(node)
+
+        def visit_FunctionDef(self, node):
+            nonlocal current_scope
+            graph.functions.append(node.name)
+            old_scope = current_scope
+            current_scope = node.name
+            self.generic_visit(node)
+            current_scope = old_scope
+
+        def visit_AsyncFunctionDef(self, node):
+            self.visit_FunctionDef(node)
+
+        def visit_ClassDef(self, node):
+            graph.classes.append(node.name)
+            self.generic_visit(node)
+
+        def visit_Call(self, node):
+            callee = _get_call_name(node.func)
+            if callee:
+                graph.calls.append(
+                    CallEdge(caller=current_scope, callee=callee, line=node.lineno)
+                )
+            self.generic_visit(node)
+
+    def _get_call_name(node) -> str | None:
+        if isinstance(node, ast.Name):
+            return node.id
+        if isinstance(node, ast.Attribute):
+            prefix = _get_call_name(node.value)
+            if prefix:
+                return f"{prefix}.{node.attr}"
+            return node.attr
+        return None
+
+    GraphVisitor().visit(tree)
+    return graph

vallm/core/graph_diff.py

@@ -0,0 +1,86 @@
+"""Graph comparison for detecting structural regressions."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from vallm.core.graph_builder import CodeGraph, build_python_graph
+
+
+@dataclass
+class GraphDiffResult:
+    """Result of comparing two code graphs."""
+
+    added_imports: list[tuple[str, str]] = field(default_factory=list)
+    removed_imports: list[tuple[str, str]] = field(default_factory=list)
+    added_functions: list[str] = field(default_factory=list)
+    removed_functions: list[str] = field(default_factory=list)
+    added_classes: list[str] = field(default_factory=list)
+    removed_classes: list[str] = field(default_factory=list)
+    added_calls: list[tuple[str, str]] = field(default_factory=list)
+    removed_calls: list[tuple[str, str]] = field(default_factory=list)
+
+    @property
+    def has_changes(self) -> bool:
+        return bool(
+            self.added_imports
+            or self.removed_imports
+            or self.added_functions
+            or self.removed_functions
+            or self.added_classes
+            or self.removed_classes
+            or self.added_calls
+            or self.removed_calls
+        )
+
+    @property
+    def breaking_changes(self) -> list[str]:
+        """Identify potentially breaking changes."""
+        issues = []
+        for fn in self.removed_functions:
+            issues.append(f"Removed function: {fn}")
+        for cls in self.removed_classes:
+            issues.append(f"Removed class: {cls}")
+        for mod, name in self.removed_imports:
+            issues.append(f"Removed import: {name} from {mod}")
+        return issues
+
+
+def diff_graphs(before: CodeGraph, after: CodeGraph) -> GraphDiffResult:
+    """Compare two CodeGraphs and return the diff."""
+    before_d = before.to_dict()
+    after_d = after.to_dict()
+
+    return GraphDiffResult(
+        added_imports=_diff_list(before_d["imports"], after_d["imports"], added=True),
+        removed_imports=_diff_list(before_d["imports"], after_d["imports"], added=False),
+        added_functions=_diff_list(before_d["functions"], after_d["functions"], added=True),
+        removed_functions=_diff_list(before_d["functions"], after_d["functions"], added=False),
+        added_classes=_diff_list(before_d["classes"], after_d["classes"], added=True),
+        removed_classes=_diff_list(before_d["classes"], after_d["classes"], added=False),
+        added_calls=_diff_list(
+            [(c.caller, c.callee) for c in before.calls],
+            [(c.caller, c.callee) for c in after.calls],
+            added=True,
+        ),
+        removed_calls=_diff_list(
+            [(c.caller, c.callee) for c in before.calls],
+            [(c.caller, c.callee) for c in after.calls],
+            added=False,
+        ),
+    )
+
+
+def diff_python_code(before_code: str, after_code: str) -> GraphDiffResult:
+    """Convenience function: build graphs from code strings and diff them."""
+    before_graph = build_python_graph(before_code, "before")
+    after_graph = build_python_graph(after_code, "after")
+    return diff_graphs(before_graph, after_graph)
+
+
+def _diff_list(before, after, added: bool):
+    set_before = set(before) if not isinstance(before[0] if before else "", tuple) else set(before)
+    set_after = set(after) if not isinstance(after[0] if after else "", tuple) else set(after)
+    if added:
+        return sorted(set_after - set_before)
+    return sorted(set_before - set_after)

vallm/core/proposal.py

@@ -0,0 +1,37 @@
+"""Proposal model representing code to be validated."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class Proposal:
+    """A code proposal to be validated.
+
+    Attributes:
+        code: The proposed source code string.
+        language: Programming language (e.g., 'python', 'javascript', 'c').
+        reference_code: Optional reference/existing code for comparison.
+        filename: Optional filename for context.
+        metadata: Additional metadata (e.g., prompt, model name).
+    """
+
+    code: str
+    language: str = "python"
+    reference_code: Optional[str] = None
+    filename: Optional[str] = None
+    metadata: dict = field(default_factory=dict)
+
+    @property
+    def code_bytes(self) -> bytes:
+        """Return code as bytes for tree-sitter parsing."""
+        return self.code.encode("utf-8")
+
+    @property
+    def reference_bytes(self) -> Optional[bytes]:
+        """Return reference code as bytes for tree-sitter parsing."""
+        if self.reference_code is None:
+            return None
+        return self.reference_code.encode("utf-8")

vallm/hookspecs.py

@@ -0,0 +1,33 @@
+"""pluggy hook specifications for vallm validators."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pluggy
+
+if TYPE_CHECKING:
+    from vallm.core.proposal import Proposal
+    from vallm.scoring import ValidationResult
+
+hookspec = pluggy.HookspecMarker("vallm")
+hookimpl = pluggy.HookimplMarker("vallm")
+
+
+class VallmSpec:
+    """Hook specifications that validators must implement."""
+
+    @hookspec
+    def validate_proposal(self, proposal: Proposal, context: dict) -> ValidationResult:
+        """Validate a code proposal and return a ValidationResult."""
+        ...
+
+    @hookspec
+    def get_validator_name(self) -> str:
+        """Return the name of this validator."""
+        ...
+
+    @hookspec
+    def get_validator_tier(self) -> int:
+        """Return the tier (1-4) of this validator for pipeline ordering."""
+        ...

vallm/scoring.py

@@ -0,0 +1,191 @@
+"""Weighted scoring and verdict engine."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Optional
+
+from vallm.config import VallmSettings
+from vallm.core.proposal import Proposal
+
+
+class Verdict(Enum):
+    PASS = "pass"
+    REVIEW = "review"
+    FAIL = "fail"
+
+
+class Severity(Enum):
+    ERROR = "error"
+    WARNING = "warning"
+    INFO = "info"
+
+
+@dataclass
+class Issue:
+    """A single issue found during validation."""
+
+    message: str
+    severity: Severity = Severity.WARNING
+    line: Optional[int] = None
+    column: Optional[int] = None
+    rule: Optional[str] = None
+
+    def __str__(self) -> str:
+        loc = ""
+        if self.line is not None:
+            loc = f":{self.line}"
+            if self.column is not None:
+                loc += f":{self.column}"
+        return f"[{self.severity.value}]{loc} {self.message}"
+
+
+@dataclass
+class ValidationResult:
+    """Result from a single validator."""
+
+    validator: str
+    score: float  # 0.0–1.0
+    weight: float = 1.0  # configurable importance
+    confidence: float = 1.0  # validator's self-assessed confidence
+    issues: list[Issue] = field(default_factory=list)
+    details: dict = field(default_factory=dict)
+
+    @property
+    def weighted_score(self) -> float:
+        return self.score * self.weight * self.confidence
+
+    @property
+    def has_errors(self) -> bool:
+        return any(i.severity == Severity.ERROR for i in self.issues)
+
+
+@dataclass
+class PipelineResult:
+    """Aggregated result from all validators."""
+
+    results: list[ValidationResult] = field(default_factory=list)
+    verdict: Verdict = Verdict.FAIL
+
+    @property
+    def weighted_score(self) -> float:
+        if not self.results:
+            return 0.0
+        total_weight = sum(r.weight * r.confidence for r in self.results)
+        if total_weight == 0:
+            return 0.0
+        return sum(r.weighted_score for r in self.results) / total_weight
+
+    @property
+    def all_issues(self) -> list[Issue]:
+        issues = []
+        for r in self.results:
+            issues.extend(r.issues)
+        return issues
+
+    @property
+    def error_count(self) -> int:
+        return sum(1 for i in self.all_issues if i.severity == Severity.ERROR)
+
+    @property
+    def warning_count(self) -> int:
+        return sum(1 for i in self.all_issues if i.severity == Severity.WARNING)
+
+
+def compute_verdict(
+    results: list[ValidationResult],
+    settings: Optional[VallmSettings] = None,
+) -> PipelineResult:
+    """Compute the aggregate verdict from a list of validation results."""
+    if settings is None:
+        settings = VallmSettings()
+
+    pipeline = PipelineResult(results=results)
+
+    # Hard gate: any error-severity issue → FAIL
+    if any(r.has_errors for r in results):
+        pipeline.verdict = Verdict.FAIL
+        return pipeline
+
+    score = pipeline.weighted_score
+    if score >= settings.pass_threshold:
+        pipeline.verdict = Verdict.PASS
+    elif score >= settings.review_threshold:
+        pipeline.verdict = Verdict.REVIEW
+    else:
+        pipeline.verdict = Verdict.FAIL
+
+    return pipeline
+
+
+def validate(
+    proposal: Proposal,
+    settings: Optional[VallmSettings] = None,
+    validators: Optional[list] = None,
+    context: Optional[dict] = None,
+) -> PipelineResult:
+    """Run the full validation pipeline on a proposal.
+
+    Args:
+        proposal: The code proposal to validate.
+        settings: Optional settings override.
+        validators: Optional list of validator instances. If None, uses defaults.
+        context: Optional additional context dict passed to validators.
+
+    Returns:
+        PipelineResult with verdict and all validation results.
+    """
+    if settings is None:
+        settings = VallmSettings()
+    if context is None:
+        context = {}
+
+    if validators is None:
+        validators = _get_default_validators(settings)
+
+    # Sort validators by tier for fail-fast behavior
+    validators.sort(key=lambda v: v.tier)
+
+    results = []
+    for validator in validators:
+        result = validator.validate(proposal, context)
+        results.append(result)
+
+        # Fail fast on errors in tier 1
+        if result.has_errors and validator.tier == 1:
+            return compute_verdict(results, settings)
+
+    return compute_verdict(results, settings)
+
+
+def _get_default_validators(settings: VallmSettings) -> list:
+    """Build the default validator list based on settings."""
+    validators = []
+
+    if settings.enable_syntax:
+        from vallm.validators.syntax import SyntaxValidator
+
+        validators.append(SyntaxValidator())
+
+    if settings.enable_imports:
+        from vallm.validators.imports import ImportValidator
+
+        validators.append(ImportValidator())
+
+    if settings.enable_complexity:
+        from vallm.validators.complexity import ComplexityValidator
+
+        validators.append(ComplexityValidator(settings))
+
+    if settings.enable_security:
+        from vallm.validators.security import SecurityValidator
+
+        validators.append(SecurityValidator())
+
+    if settings.enable_semantic:
+        from vallm.validators.semantic import SemanticValidator
+
+        validators.append(SemanticValidator(settings))
+
+    return validators

vallm/validators/__init__.py

@@ -0,0 +1 @@
+"""Built-in validators for vallm."""