commitguardian 0.1.0__py3-none-any.whl

commitguard/research/detection.py

@@ -0,0 +1,183 @@
+"""Detection accuracy on the labelled dataset.
+
+Each case runs through the production :class:`~commitguard.services.analysis.Analyzer`
+with the built-in rules and the built-in policy - the configuration a new
+installation enforces. Three views are measured:
+
+* **decision (binary)** - positive = the case must not be allowed silently
+  (expected WARN or BLOCK); predicted positive = CommitGuard did not ALLOW.
+  This is the security-relevant question: was a violation let through?
+* **decision (exact)** - the reached decision equals the expected one
+  (BLOCK vs WARN matters: a warning does not stop a merge);
+* **rules** - per rule, whether it was reported when required, and whether a
+  rule was reported that is neither required nor permitted.
+
+Every mismatch is listed with the case, what was expected and what happened.
+"""
+
+import time
+from collections import Counter
+from collections.abc import Sequence
+
+from pydantic import BaseModel, ConfigDict
+
+from commitguard.core.decision import Action
+from commitguard.git.commit import Commit
+from commitguard.policies.defaults import DEFAULT_POLICIES, default_policy_set
+from commitguard.provenance.author import Identity
+from commitguard.research.datasets import DatasetCase
+from commitguard.research.metrics import Confusion, LatencySummary
+from commitguard.services.analysis import Analyzer
+from commitguard.services.reports import CommitReport
+
+BENCHMARK_VERSION = "1.0.0"
+
+
+class CaseOutcome(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    id: str
+    case_class: str
+    category: str
+    description: str
+    expected_decision: str
+    decision: str
+    required_rules: tuple[str, ...]
+    reported_rules: tuple[str, ...]
+    missing_rules: tuple[str, ...]
+    unexpected_rules: tuple[str, ...]
+    failed_closed: bool
+    outcome: str  # true_positive | false_positive | true_negative | false_negative
+
+
+class DetectionResult(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    benchmark: str = "detection"
+    benchmark_version: str = BENCHMARK_VERSION
+    cases: int
+    positives: int
+    negatives: int
+    decision: Confusion
+    decision_rates: dict[str, float | None]
+    exact_decision_matches: int
+    exact_decision_accuracy: float | None
+    rules: dict[str, Confusion]
+    rule_rates: dict[str, dict[str, float | None]]
+    by_class: dict[str, dict[str, int]]
+    by_decision: dict[str, dict[str, int]]
+    latency: LatencySummary
+    duration_ms: float
+    mismatches: tuple[CaseOutcome, ...]
+
+
+def commit_for(case: DatasetCase) -> Commit:
+    return Commit(
+        author=Identity(name=case.author.name, email=case.author.email),
+        committer=Identity(name=case.committer.name, email=case.committer.email),
+        message=case.message,
+    )
+
+
+def _outcome(case: DatasetCase, report: CommitReport) -> CaseOutcome:
+    reported = tuple(sorted({f.finding.rule_id for f in report.findings}))
+    required = set(case.required_rules)
+    allowed = required | set(case.permitted_rules)
+    predicted_positive = report.action is not Action.ALLOW
+    if case.positive:
+        kind = "true_positive" if predicted_positive else "false_negative"
+    else:
+        kind = "false_positive" if predicted_positive else "true_negative"
+    return CaseOutcome(
+        id=case.id,
+        case_class=case.case_class,
+        category=case.category,
+        description=case.description,
+        expected_decision=case.expected_decision,
+        decision=report.action.value,
+        required_rules=case.required_rules,
+        reported_rules=reported,
+        missing_rules=tuple(sorted(required - set(reported))),
+        unexpected_rules=tuple(sorted(set(reported) - allowed)),
+        failed_closed=bool(report.failures),
+        outcome=kind,
+    )
+
+
+def run_detection(
+    cases: Sequence[DatasetCase], analyzer: Analyzer | None = None
+) -> DetectionResult:
+    if not cases:
+        raise ValueError("the dataset is empty")
+    analyzer = analyzer or Analyzer.create(default_policy_set())
+    outcomes: list[CaseOutcome] = []
+    durations: list[float] = []
+    started = time.perf_counter()
+    for case in cases:
+        commit = commit_for(case)
+        begin = time.perf_counter()
+        report = analyzer.analyze(commit)
+        durations.append(time.perf_counter() - begin)
+        outcomes.append(_outcome(case, report))
+    total = time.perf_counter() - started
+
+    counts = Counter(o.outcome for o in outcomes)
+    decision = Confusion(
+        true_positive=counts["true_positive"],
+        false_positive=counts["false_positive"],
+        true_negative=counts["true_negative"],
+        false_negative=counts["false_negative"],
+    )
+    rules: dict[str, Confusion] = {}
+    for rule in sorted(DEFAULT_POLICIES):
+        tp = fp = tn = fn = 0
+        for case, outcome in zip(cases, outcomes, strict=True):
+            expected = rule in case.required_rules
+            permitted = rule in case.permitted_rules
+            reported = rule in outcome.reported_rules
+            if expected:
+                tp, fn = (tp + 1, fn) if reported else (tp, fn + 1)
+            elif permitted:
+                continue  # either answer is acceptable for this case
+            elif reported:
+                fp += 1
+            else:
+                tn += 1
+        rules[rule] = Confusion(
+            true_positive=tp, false_positive=fp, true_negative=tn, false_negative=fn
+        )
+
+    by_class: dict[str, dict[str, int]] = {}
+    for outcome in outcomes:
+        bucket = by_class.setdefault(outcome.case_class, Counter())
+        bucket["cases"] += 1
+        bucket[outcome.outcome] += 1
+        if outcome.decision == outcome.expected_decision:
+            bucket["exact_decision"] += 1
+    by_decision: dict[str, dict[str, int]] = {}
+    for outcome in outcomes:
+        bucket = by_decision.setdefault(outcome.expected_decision, Counter())
+        bucket[outcome.decision] += 1
+
+    exact = sum(1 for o in outcomes if o.decision == o.expected_decision)
+    mismatches = tuple(
+        o
+        for o in outcomes
+        if o.decision != o.expected_decision or o.missing_rules or o.unexpected_rules
+    )
+    return DetectionResult(
+        cases=len(cases),
+        positives=sum(1 for c in cases if c.positive),
+        negatives=sum(1 for c in cases if not c.positive),
+        decision=decision,
+        decision_rates=decision.rates(),
+        exact_decision_matches=exact,
+        exact_decision_accuracy=exact / len(cases),
+        rules=rules,
+        rule_rates={rule: confusion.rates() for rule, confusion in rules.items()},
+        by_class={k: dict(v) for k, v in sorted(by_class.items())},
+        by_decision={k: dict(v) for k, v in sorted(by_decision.items())},
+        latency=LatencySummary.from_seconds(durations),
+        duration_ms=round(total * 1000, 3),
+        mismatches=mismatches,
+    )

commitguard/research/environment.py

@@ -0,0 +1,185 @@
+"""The benchmark manifest: what was measured, with what, on what.
+
+A result without its environment cannot be compared or reproduced, so every
+benchmark result embeds a :class:`BenchmarkManifest`. Values that cannot be
+determined on a platform are recorded as ``None`` - never guessed.
+"""
+
+import os
+import platform
+import re
+import shlex
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from commitguard import __version__
+from commitguard.policies.defaults import DEFAULT_POLICIES
+from commitguard.rules.loader import builtin_rules_fingerprint
+from commitguard.security.hashing import sha256_hex
+from commitguard.utils.subprocess import run_command
+
+MANIFEST_SCHEMA = 1
+
+
+class BenchmarkManifest(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    schema_version: int = MANIFEST_SCHEMA
+    commitguard_version: str
+    git_version: str | None
+    source_revision: str | None = None  # commit of the CommitGuard checkout, if run from one
+    source_dirty: bool | None = None  # uncommitted changes in that checkout
+    python_version: str
+    python_implementation: str
+    operating_system: str
+    os_release: str
+    os_version: str
+    machine: str
+    cpu_model: str | None
+    cpu_count: int | None
+    memory_bytes: int | None
+    rules_version: str
+    policy_version: str
+    configuration_version: str
+    dataset_version: str | None
+    dataset_fingerprint: str | None
+    timestamp: datetime
+    command: str
+
+
+def git_version() -> str | None:
+    try:
+        result = run_command(["git", "--version"], timeout=10)
+    except OSError:
+        return None
+    if result.returncode != 0:
+        return None
+    match = re.search(r"(\d+\.\d+(?:\.\d+)?)", result.stdout.decode("utf-8", "replace"))
+    return match.group(1) if match else None
+
+
+def source_revision() -> tuple[str | None, bool | None]:
+    """The Git commit of the CommitGuard source checkout this code runs from, if any."""
+    package = Path(__file__).resolve().parents[1]
+    root = package.parents[1] if package.parent.name == "src" else None
+    if root is None or not (root / ".git").exists():
+        return None, None
+    try:
+        head = run_command(["git", "-C", str(root), "rev-parse", "HEAD"], timeout=10)
+        status = run_command(
+            ["git", "-C", str(root), "status", "--porcelain", "--untracked-files=no"], timeout=30
+        )
+    except OSError:
+        return None, None
+    if head.returncode != 0:
+        return None, None
+    dirty = bool(status.stdout.strip()) if status.returncode == 0 else None
+    return head.stdout.decode("ascii", "replace").strip(), dirty
+
+
+def cpu_model() -> str | None:
+    """The CPU model name, from the operating system, or None if unavailable."""
+    system = platform.system()
+    try:
+        if system == "Linux":
+            text = Path("/proc/cpuinfo").read_text(encoding="utf-8", errors="replace")
+            for line in text.splitlines():
+                if line.lower().startswith("model name"):
+                    return line.split(":", 1)[1].strip()
+        elif system == "Darwin":
+            result = run_command(["sysctl", "-n", "machdep.cpu.brand_string"], timeout=10)
+            if result.returncode == 0:
+                return result.stdout.decode("utf-8", "replace").strip() or None
+        elif system == "Windows":
+            name = os.environ.get("PROCESSOR_IDENTIFIER")
+            return name.strip() if name else None
+    except OSError:
+        return None
+    return platform.processor() or None
+
+
+def memory_bytes() -> int | None:
+    """Physical memory, or None if the platform does not expose it without extra packages."""
+    system = platform.system()
+    try:
+        if system == "Linux":
+            for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines():
+                if line.startswith("MemTotal:"):
+                    return int(line.split()[1]) * 1024
+        elif system == "Darwin":
+            result = run_command(["sysctl", "-n", "hw.memsize"], timeout=10)
+            if result.returncode == 0:
+                return int(result.stdout.strip())
+        elif system == "Windows":
+            return _windows_memory()
+    except (OSError, ValueError):
+        return None
+    return None
+
+
+def _windows_memory() -> int | None:  # pragma: no cover - exercised on Windows CI
+    import ctypes
+
+    class MemoryStatus(ctypes.Structure):
+        _fields_ = [
+            ("dwLength", ctypes.c_ulong),
+            ("dwMemoryLoad", ctypes.c_ulong),
+            ("ullTotalPhys", ctypes.c_ulonglong),
+            ("ullAvailPhys", ctypes.c_ulonglong),
+            ("ullTotalPageFile", ctypes.c_ulonglong),
+            ("ullAvailPageFile", ctypes.c_ulonglong),
+            ("ullTotalVirtual", ctypes.c_ulonglong),
+            ("ullAvailVirtual", ctypes.c_ulonglong),
+            ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
+        ]
+
+    status = MemoryStatus()
+    status.dwLength = ctypes.sizeof(MemoryStatus)
+    if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):  # type: ignore[attr-defined]
+        return None
+    return int(status.ullTotalPhys)
+
+
+def default_policy_version() -> str:
+    """Fingerprint of the built-in policy set (the policy the benchmarks evaluate)."""
+    canonical = ";".join(
+        f"{policy.id}={policy.action.value}:{int(policy.enabled)}"
+        for policy in sorted(DEFAULT_POLICIES.values(), key=lambda p: p.id)
+    )
+    return sha256_hex(canonical.encode("utf-8"))
+
+
+def collect_manifest(
+    *,
+    dataset_version: str | None = None,
+    dataset_fingerprint: str | None = None,
+    configuration_version: str = "built-in defaults",
+    argv: list[str] | None = None,
+) -> BenchmarkManifest:
+    uname = platform.uname()
+    revision, dirty = source_revision()
+    return BenchmarkManifest(
+        commitguard_version=__version__,
+        git_version=git_version(),
+        source_revision=revision,
+        source_dirty=dirty,
+        python_version=platform.python_version(),
+        python_implementation=platform.python_implementation(),
+        operating_system=uname.system,
+        os_release=uname.release,
+        os_version=uname.version,
+        machine=uname.machine,
+        cpu_model=cpu_model(),
+        cpu_count=os.cpu_count(),
+        memory_bytes=memory_bytes(),
+        rules_version=builtin_rules_fingerprint(),
+        policy_version=default_policy_version(),
+        configuration_version=configuration_version,
+        dataset_version=dataset_version,
+        dataset_fingerprint=dataset_fingerprint,
+        timestamp=datetime.now(UTC),
+        command=shlex.join(["commitguard", *(sys.argv[1:] if argv is None else argv)]),
+    )

commitguard/research/gitenv.py

@@ -0,0 +1,108 @@
+"""Isolated Git workspaces for benchmarks.
+
+Benchmarks run real ``git`` processes in temporary directories with a private
+``HOME`` and global configuration, so the developer's Git settings (hooks paths,
+signing, aliases, credential helpers) can neither influence nor be modified by a
+measurement.
+"""
+
+import os
+import shutil
+import stat
+import tempfile
+import time
+from collections.abc import Callable, Iterator, Mapping, Sequence
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+
+from commitguard.utils.subprocess import CommandResult, run_command
+
+GIT_TIMEOUT = 600.0
+
+
+@dataclass(frozen=True, slots=True)
+class Workspace:
+    root: Path
+    env: Mapping[str, str]
+
+    def git(
+        self, cwd: Path, *args: str, input_bytes: bytes | None = None, check: bool = True
+    ) -> CommandResult:
+        result = run_command(
+            ["git", *args],
+            cwd=cwd,
+            env_overrides=self.env,
+            input_bytes=input_bytes,
+            timeout=GIT_TIMEOUT,
+        )
+        if check and result.returncode != 0:
+            detail = result.stderr.decode("utf-8", "replace")[-500:]
+            raise RuntimeError(f"git {' '.join(args[:2])} failed ({result.returncode}): {detail}")
+        return result
+
+    def timed_git(
+        self, cwd: Path, *args: str, input_bytes: bytes | None = None
+    ) -> tuple[float, CommandResult]:
+        started = time.perf_counter()
+        result = self.git(cwd, *args, input_bytes=input_bytes, check=False)
+        return time.perf_counter() - started, result
+
+    def init(self, name: str, *, bare: bool = False) -> Path:
+        path = self.root / name
+        self.git(self.root, "init", "--quiet", *(["--bare"] if bare else []), str(path))
+        return path
+
+
+def _on_rm_error(function: Callable[[str], object], path: str, _error: BaseException) -> None:
+    os.chmod(path, stat.S_IWRITE)
+    function(path)
+
+
+@contextmanager
+def workspace(prefix: str = "commitguard-bench-") -> Iterator[Workspace]:
+    root = Path(tempfile.mkdtemp(prefix=prefix)).resolve()
+    home = root / "home"
+    home.mkdir()
+    config = home / ".gitconfig"
+    config.write_text(
+        "[user]\n\tname = Benchmark Developer\n\temail = dev@example.com\n"
+        "[init]\n\tdefaultBranch = main\n"
+        "[commit]\n\tgpgsign = false\n"
+        "[core]\n\tautocrlf = false\n",
+        encoding="utf-8",
+    )
+    env = {
+        "HOME": str(home),
+        "USERPROFILE": str(home),
+        "XDG_CONFIG_HOME": str(home / ".config"),
+        "GIT_CONFIG_GLOBAL": str(config),
+        "GIT_CONFIG_NOSYSTEM": "1",
+        "GIT_TERMINAL_PROMPT": "0",
+        "LC_ALL": "C.UTF-8",
+    }
+    try:
+        yield Workspace(root=root, env=env)
+    finally:
+        shutil.rmtree(root, onexc=_on_rm_error)
+
+
+def fast_import_stream(messages: Sequence[str], *, branch: str = "main") -> bytes:
+    """A ``git fast-import`` stream creating one commit per message on ``branch``."""
+    parts: list[bytes] = []
+    timestamp = 1_700_000_000
+    for index, message in enumerate(messages):
+        data = message.encode("utf-8")
+        content = f"change {index}\n".encode()
+        parts.append(f"commit refs/heads/{branch}\n".encode())
+        parts.append(f"mark :{index + 1}\n".encode())
+        parts.append(f"author Ada Lovelace <ada@example.com> {timestamp + index} +0000\n".encode())
+        parts.append(
+            f"committer Ada Lovelace <ada@example.com> {timestamp + index} +0000\n".encode()
+        )
+        parts.append(f"data {len(data)}\n".encode() + data + b"\n")
+        if index:
+            parts.append(f"from :{index}\n".encode())
+        parts.append(f"M 100644 inline file.txt\ndata {len(content)}\n".encode() + content)
+        parts.append(b"\n")
+    return b"".join(parts)