slopaudit 1.0.0__py3-none-any.whl

slopaudit/__init__.py

@@ -0,0 +1,24 @@
+"""slopaudit: verify AI-generated code.
+
+``import slopaudit`` is cheap. The API names below load on first use.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+__version__ = "1.0.0"
+
+if TYPE_CHECKING:
+    from slopaudit.api import Config, Finding, ScanResult, Severity, scan, scan_source
+
+_API = {"Config", "Finding", "ScanResult", "Severity", "scan", "scan_source"}
+__all__ = ["Config", "Finding", "ScanResult", "Severity", "__version__", "scan", "scan_source"]
+
+
+def __getattr__(name: str) -> Any:
+    if name in _API:
+        from slopaudit import api
+
+        return getattr(api, name)
+    raise AttributeError(f"module 'slopaudit' has no attribute {name!r}")

slopaudit/__main__.py

@@ -0,0 +1,4 @@
+from slopaudit.cli import app
+
+if __name__ == "__main__":
+    app()

slopaudit/api.py

@@ -0,0 +1,103 @@
+"""The supported Python API.
+
+Everything importable from ``slopaudit`` directly (``slopaudit.scan``, ``Finding``,
+``Severity``, ``Config``) comes from here. The CLI is a thin layer over the same functions.
+
+    import slopaudit
+
+    result = slopaudit.scan(["src"], network=False)
+    for finding in result.findings:
+        print(finding.rule_id, finding.path, finding.line, finding.message)
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+
+from slopaudit.config import MODULES, Config
+from slopaudit.context import build_project
+from slopaudit.engine import ScanResult, discover, find_root, run_scan, scan_text
+from slopaudit.models import Finding, Rule, Severity
+from slopaudit.rules import all_rules
+
+__all__ = ["Config", "Finding", "ScanResult", "Severity", "scan", "scan_source", "select_rules"]
+
+
+def select_rules(
+    config: Config, modules: Iterable[str] = (), rule_ids: Iterable[str] = ()
+) -> list[Rule]:
+    """Rules to run: those in ``modules`` (or enabled in ``config``), narrowed to ``rule_ids``.
+
+    Raises ``ValueError`` for an unknown module or rule ID.
+    """
+    modules, wanted = list(modules), {r.upper() for r in rule_ids}
+    for name in modules:
+        if name not in MODULES:
+            raise ValueError(f"unknown module {name!r}; expected one of: {', '.join(MODULES)}")
+    unknown = wanted - {r.meta.id for r in all_rules()}
+    if unknown:
+        raise ValueError(f"unknown rule(s): {', '.join(sorted(unknown))}")
+    chosen = []
+    for rule in all_rules():
+        meta = rule.meta
+        if wanted and meta.id not in wanted:
+            continue
+        if modules:
+            if meta.module not in modules:
+                continue
+        elif not config.modules.get(meta.module, True):
+            continue
+        if config.rule_enabled(meta.id):
+            chosen.append(rule)
+    return chosen
+
+
+def scan(
+    paths: Iterable[str | Path] = (".",),
+    *,
+    config: Config | None = None,
+    config_path: str | Path | None = None,
+    modules: Iterable[str] = (),
+    rules: Iterable[str] = (),
+    network: bool = True,
+) -> ScanResult:
+    """Scan files or folders and return the findings.
+
+    ``config`` wins over ``config_path``; with neither, ``slopaudit.toml`` or
+    ``[tool.slopaudit]`` is read from the project root. ``network=False`` skips PyPI lookups.
+    Raises ``slopaudit.config.ConfigError`` for a bad configuration and ``FileNotFoundError``
+    for a path that does not exist.
+    """
+    targets = [Path(p) for p in paths]
+    for target in targets:
+        if not target.exists():
+            raise FileNotFoundError(target)
+    root = find_root(targets)
+    if config is None:
+        config = Config.load(root, Path(config_path) if config_path else None)
+    selected = select_rules(config, modules, rules)
+    files = discover(targets, root, config, {s for r in selected for s in r.file_types})
+    project = build_project(root, config, network=network)
+    result = run_scan(files, selected, project)
+    if project.registry is not None:
+        project.registry.save()
+    return result
+
+
+def scan_source(
+    source: str,
+    filename: str = "snippet.py",
+    *,
+    config: Config | None = None,
+    modules: Iterable[str] = (),
+    rules: Iterable[str] = (),
+) -> list[Finding]:
+    """Scan a string of source code, for example an assistant's answer before it is saved.
+
+    Offline and read-only: PyPI lookups are skipped. Project checks use the current directory.
+    """
+    config = config or Config()
+    selected = select_rules(config, modules, rules)
+    project = build_project(Path.cwd(), config, network=False)
+    return scan_text(filename, source, selected, project)

slopaudit/bench.py

@@ -0,0 +1,136 @@
+"""Benchmark runner: precision and recall per rule on a labelled corpus.
+
+Every corpus file starts with a label line, ``# expect: RULE-ID[, RULE-ID]`` or
+``# expect: none``. A rule counts as a true positive for a file when it is both expected
+and reported, a false positive when reported but not expected, and a false negative when
+expected but not reported. Scoring is per file, so line-level accuracy is not measured.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from slopaudit.config import Config
+from slopaudit.context import build_project
+from slopaudit.engine import discover, run_scan
+from slopaudit.rules import all_rules
+
+LABEL = re.compile(r"^#\s*expect:\s*(.+)$", re.MULTILINE)
+
+
+@dataclass
+class RuleScore:
+    tp: int = 0
+    fp: int = 0
+    fn: int = 0
+
+    @property
+    def precision(self) -> float | None:
+        total = self.tp + self.fp
+        return self.tp / total if total else None
+
+    @property
+    def recall(self) -> float | None:
+        total = self.tp + self.fn
+        return self.tp / total if total else None
+
+    @property
+    def f1(self) -> float | None:
+        p, r = self.precision, self.recall
+        if p is None or r is None or p + r == 0:
+            return None
+        return 2 * p * r / (p + r)
+
+
+@dataclass
+class BenchReport:
+    files: int = 0
+    scores: dict[str, RuleScore] = field(default_factory=dict)
+    mismatches: list[str] = field(default_factory=list)
+    unlabelled: list[str] = field(default_factory=list)
+
+    def overall(self) -> RuleScore:
+        total = RuleScore()
+        for score in self.scores.values():
+            total.tp += score.tp
+            total.fp += score.fp
+            total.fn += score.fn
+        return total
+
+    def failing(self, min_precision: float, min_recall: float) -> list[str]:
+        bad = []
+        for rule_id, score in sorted(self.scores.items()):
+            if score.precision is not None and score.precision < min_precision:
+                bad.append(f"{rule_id} precision {score.precision:.2f} < {min_precision:.2f}")
+            if score.recall is not None and score.recall < min_recall:
+                bad.append(f"{rule_id} recall {score.recall:.2f} < {min_recall:.2f}")
+        return bad
+
+    def to_dict(self) -> dict:
+        def num(value: float | None) -> float | None:
+            return None if value is None else round(value, 3)
+
+        return {
+            "files": self.files,
+            "rules": {
+                rule_id: {
+                    "tp": s.tp, "fp": s.fp, "fn": s.fn,
+                    "precision": num(s.precision), "recall": num(s.recall), "f1": num(s.f1),
+                }
+                for rule_id, s in sorted(self.scores.items())
+            },
+            "overall": {
+                "precision": num(self.overall().precision),
+                "recall": num(self.overall().recall),
+                "f1": num(self.overall().f1),
+            },
+            "mismatches": self.mismatches,
+            "unlabelled": self.unlabelled,
+        }
+
+
+def parse_expected(source: str) -> set[str] | None:
+    """Return the labelled rule IDs, an empty set for ``none``, or None if unlabelled."""
+    match = LABEL.search(source)
+    if not match:
+        return None
+    ids = {part.strip().upper() for part in match.group(1).split(",") if part.strip()}
+    return set() if ids <= {"NONE"} else ids
+
+
+def run_benchmark(corpus: Path) -> BenchReport:
+    corpus = corpus.resolve()
+    config = Config()
+    project = build_project(corpus, config, network=False)
+    rules = all_rules()
+    suffixes = {s for r in rules for s in r.file_types}
+    files = discover([corpus], corpus, config, suffixes)
+    result = run_scan(files, rules, project)
+
+    found: dict[str, set[str]] = {}
+    for finding in result.findings:
+        found.setdefault(finding.path, set()).add(finding.rule_id)
+
+    report = BenchReport()
+    for path in files:
+        rel = path.relative_to(corpus).as_posix()
+        expected = parse_expected(path.read_text(errors="replace"))
+        if expected is None:
+            report.unlabelled.append(rel)
+            continue
+        report.files += 1
+        reported = found.get(rel, set())
+        for rule_id in expected | reported:
+            score = report.scores.setdefault(rule_id, RuleScore())
+            if rule_id in expected and rule_id in reported:
+                score.tp += 1
+            elif rule_id in reported:
+                score.fp += 1
+                report.mismatches.append(f"false positive  {rel}  {rule_id}")
+            else:
+                score.fn += 1
+                report.mismatches.append(f"false negative  {rel}  {rule_id}")
+    report.mismatches.sort()
+    return report