commitguardian 0.1.0__py3-none-any.whl

commitguard/research/hooks.py

@@ -0,0 +1,247 @@
+"""Git operations with and without CommitGuard hooks.
+
+Two otherwise identical repositories are created: one plain, one with the
+CommitGuard hooks (``pre-commit``, ``commit-msg``, ``pre-push``) installed by
+the production installer. The same Git commands run against both, repeatedly,
+and the difference in wall-clock time is the hook overhead.
+
+The benchmark also records what happened (exit codes), because the timings are
+only meaningful if the hooks actually enforced:
+
+* a clean commit is allowed with and without hooks;
+* an AI co-authored commit is blocked by the hooks;
+* ``git commit --no-verify`` bypasses the local hooks (measured, not assumed);
+* a push of a violating commit is blocked by ``pre-push``.
+"""
+
+import sys
+from collections.abc import Callable
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from commitguard.git.hooks import install_hooks
+from commitguard.git.repository import Repository
+from commitguard.research.gitenv import Workspace, workspace
+from commitguard.research.metrics import LatencySummary
+
+BENCHMARK_VERSION = "1.0.0"
+DEFAULT_REPETITIONS = 20
+
+CLEAN = "feat: add session rotation\n\nSigned-off-by: Ada Lovelace <ada@example.com>\n"
+AI = "feat: add payment service\n\nCo-authored-by: Claude <noreply@anthropic.com>\n"
+
+
+class Scenario(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    name: str
+    description: str
+    hooks_installed: bool
+    exit_codes: dict[str, int]
+    latency: LatencySummary
+
+
+class Overhead(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    operation: str
+    without_hooks_p50_ms: float
+    with_hooks_p50_ms: float
+    overhead_p50_ms: float
+    overhead_ratio: float | None
+
+
+class Observation(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    check: str
+    expected: str
+    observed: str
+    matches: bool
+
+
+class HooksResult(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    benchmark: str = "hooks"
+    benchmark_version: str = BENCHMARK_VERSION
+    repetitions: int
+    python: str
+    scenarios: tuple[Scenario, ...]
+    overhead: tuple[Overhead, ...]
+    observations: tuple[Observation, ...]
+    notes: tuple[str, ...]
+
+
+def _measure(
+    name: str,
+    description: str,
+    hooked: bool,
+    repetitions: int,
+    run: Callable[[int], tuple[float, int]],
+) -> Scenario:
+    durations = []
+    codes: dict[str, int] = {}
+    for index in range(repetitions):
+        seconds, code = run(index)
+        durations.append(seconds)
+        codes[str(code)] = codes.get(str(code), 0) + 1
+    return Scenario(
+        name=name,
+        description=description,
+        hooks_installed=hooked,
+        exit_codes=codes,
+        latency=LatencySummary.from_seconds(durations),
+    )
+
+
+def _repository(space: Workspace, name: str, hooked: bool) -> tuple[Repository, str]:
+    remote = space.init(f"{name}-remote.git", bare=True)
+    path = space.init(name)
+    space.git(path, "remote", "add", "origin", str(remote))
+    space.git(path, "commit", "--quiet", "--allow-empty", "--no-verify", "-m", "chore: initial")
+    space.git(path, "push", "--quiet", "--no-verify", "origin", "main")
+    repository = Repository.discover(path)
+    if hooked:
+        install_hooks(repository, python=sys.executable)
+    return repository, name
+
+
+def run_hooks(repetitions: int = DEFAULT_REPETITIONS) -> HooksResult:
+    if repetitions < 1:
+        raise ValueError("repetitions must be at least 1")
+    with workspace() as space:
+        plain, _ = _repository(space, "plain", hooked=False)
+        hooked, _ = _repository(space, "hooked", hooked=True)
+        p, h = plain.root, hooked.root
+
+        def commit(path: Path, message: str, *extra: str) -> Callable[[int], tuple[float, int]]:
+            def run(_: int) -> tuple[float, int]:
+                seconds, result = space.timed_git(
+                    path, "commit", "--quiet", "--allow-empty", *extra, "-m", message
+                )
+                return seconds, result.returncode
+
+            return run
+
+        def push(
+            label: str, path: Path, message: str, *extra: str
+        ) -> Callable[[int], tuple[float, int]]:
+            def run(index: int) -> tuple[float, int]:
+                branch = f"{label}-{index}"  # unique: a reused name could be rejected by the remote
+                # Start from the remote state so earlier scenarios cannot leak commits in.
+                space.git(path, "checkout", "--quiet", "-B", branch, "origin/main")
+                space.git(path, "commit", "--quiet", "--allow-empty", "--no-verify", "-m", message)
+                seconds, result = space.timed_git(path, "push", "--quiet", *extra, "origin", branch)
+                return seconds, result.returncode
+
+            return run
+
+        scenarios = (
+            _measure(
+                "commit-clean-plain",
+                "git commit, clean message, no hooks",
+                False,
+                repetitions,
+                commit(p, CLEAN),
+            ),
+            _measure(
+                "commit-clean-hooked",
+                "git commit, clean message, CommitGuard hooks",
+                True,
+                repetitions,
+                commit(h, CLEAN),
+            ),
+            _measure(
+                "commit-ai-hooked",
+                "git commit, AI co-author, CommitGuard hooks",
+                True,
+                repetitions,
+                commit(h, AI),
+            ),
+            _measure(
+                "commit-ai-no-verify",
+                "git commit --no-verify, AI co-author, hooks installed",
+                True,
+                repetitions,
+                commit(h, AI, "--no-verify"),
+            ),
+            _measure(
+                "push-clean-plain",
+                "git push of one new clean commit, no hooks",
+                False,
+                repetitions,
+                push("push-clean-plain", p, CLEAN),
+            ),
+            _measure(
+                "push-clean-hooked",
+                "git push of one new clean commit, CommitGuard pre-push",
+                True,
+                repetitions,
+                push("push-clean-hooked", h, CLEAN),
+            ),
+            _measure(
+                "push-ai-hooked",
+                "git push of one new AI co-authored commit, pre-push",
+                True,
+                repetitions,
+                push("push-ai-hooked", h, AI),
+            ),
+            _measure(
+                "push-ai-no-verify",
+                "git push --no-verify of an AI co-authored commit",
+                True,
+                repetitions,
+                push("push-ai-no-verify", h, AI, "--no-verify"),
+            ),
+        )
+    by_name = {s.name: s for s in scenarios}
+
+    def overhead(operation: str, plain_name: str, hooked_name: str) -> Overhead:
+        base, with_hooks = by_name[plain_name].latency.p50_ms, by_name[hooked_name].latency.p50_ms
+        return Overhead(
+            operation=operation,
+            without_hooks_p50_ms=base,
+            with_hooks_p50_ms=with_hooks,
+            overhead_p50_ms=round(with_hooks - base, 3),
+            overhead_ratio=round(with_hooks / base, 2) if base > 0 else None,
+        )
+
+    def observe(check: str, scenario: str, expected_code: str) -> Observation:
+        codes = by_name[scenario].exit_codes
+        observed = ", ".join(f"exit {code} x{count}" for code, count in sorted(codes.items()))
+        matches = set(codes) == {expected_code} if expected_code == "0" else "0" not in codes
+        return Observation(
+            check=check,
+            expected=f"exit {'0' if expected_code == '0' else 'non-zero'} every time",
+            observed=observed,
+            matches=matches,
+        )
+
+    observations = (
+        observe("clean commit allowed without hooks", "commit-clean-plain", "0"),
+        observe("clean commit allowed with hooks", "commit-clean-hooked", "0"),
+        observe("AI co-authored commit blocked by commit hooks", "commit-ai-hooked", "!0"),
+        observe("--no-verify bypasses local commit hooks", "commit-ai-no-verify", "0"),
+        observe("clean push allowed by pre-push", "push-clean-hooked", "0"),
+        observe("AI co-authored push blocked by pre-push", "push-ai-hooked", "!0"),
+        observe("--no-verify bypasses the local pre-push hook", "push-ai-no-verify", "0"),
+    )
+    return HooksResult(
+        repetitions=repetitions,
+        python=sys.executable,
+        scenarios=scenarios,
+        overhead=(
+            overhead("git commit (clean)", "commit-clean-plain", "commit-clean-hooked"),
+            overhead("git push (clean, one commit)", "push-clean-plain", "push-clean-hooked"),
+        ),
+        observations=observations,
+        notes=(
+            "Each hook starts a Python interpreter; interpreter start-up is part of the "
+            "measured overhead.",
+            "Pushes go to a local bare repository, so network latency is excluded.",
+            "--no-verify results demonstrate that local hooks are advisory; server-side "
+            "enforcement is measured separately.",
+        ),
+    )

commitguard/research/metrics.py

@@ -0,0 +1,85 @@
+"""Measurement arithmetic: confusion matrices and latency statistics.
+
+CommitGuard's detection is deterministic and rule-based. Precision, recall and
+F1 are reported because they are the standard way to state how often a
+classifier is right on a labelled set - not to suggest a statistical model.
+Undefined ratios (a zero denominator) are ``None``, never a made-up 0 or 1.
+"""
+
+import math
+from collections.abc import Sequence
+
+from pydantic import BaseModel, ConfigDict
+
+
+class Confusion(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    true_positive: int
+    false_positive: int
+    true_negative: int
+    false_negative: int
+
+    @property
+    def total(self) -> int:
+        return self.true_positive + self.false_positive + self.true_negative + self.false_negative
+
+    def rates(self) -> dict[str, float | None]:
+        tp, fp, tn, fn = (
+            self.true_positive,
+            self.false_positive,
+            self.true_negative,
+            self.false_negative,
+        )
+        precision = _ratio(tp, tp + fp)
+        recall = _ratio(tp, tp + fn)
+        f1 = (
+            None
+            if precision is None or recall is None or precision + recall == 0
+            else 2 * precision * recall / (precision + recall)
+        )
+        return {
+            "precision": precision,
+            "recall": recall,
+            "f1": f1,
+            "accuracy": _ratio(tp + tn, self.total),
+            "false_positive_rate": _ratio(fp, fp + tn),
+            "false_negative_rate": _ratio(fn, fn + tp),
+        }
+
+
+def _ratio(numerator: int, denominator: int) -> float | None:
+    return None if denominator == 0 else numerator / denominator
+
+
+def percentile(sorted_values: Sequence[float], fraction: float) -> float:
+    """Nearest-rank percentile of an ascending sequence (no interpolation)."""
+    if not sorted_values:
+        raise ValueError("no values")
+    rank = max(1, math.ceil(fraction * len(sorted_values)))
+    return sorted_values[rank - 1]
+
+
+class LatencySummary(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    samples: int
+    min_ms: float
+    mean_ms: float
+    p50_ms: float
+    p95_ms: float
+    p99_ms: float
+    max_ms: float
+
+    @classmethod
+    def from_seconds(cls, durations: Sequence[float]) -> "LatencySummary":
+        values = sorted(d * 1000 for d in durations)
+        return cls(
+            samples=len(values),
+            min_ms=round(values[0], 4),
+            mean_ms=round(sum(values) / len(values), 4),
+            p50_ms=round(percentile(values, 0.50), 4),
+            p95_ms=round(percentile(values, 0.95), 4),
+            p99_ms=round(percentile(values, 0.99), 4),
+            max_ms=round(values[-1], 4),
+        )

commitguard/research/performance.py

@@ -0,0 +1,194 @@
+"""Detection performance: latency, message-size scaling, CPU time and memory.
+
+What is measured, and how:
+
+* **wall-clock** - ``time.perf_counter`` around each ``Analyzer.analyze`` call;
+* **CPU time** - ``time.process_time`` over a whole batch (user + system time of
+  this process);
+* **Python allocations** - ``tracemalloc`` peak during a *separate* pass over the
+  same commits: memory allocated by the analysis itself, excluding the
+  interpreter's baseline. Timing passes run without ``tracemalloc``, which slows
+  Python code by roughly 2-3x;
+* **peak RSS** - the process's maximum resident set size (``resource``; not
+  available on Windows, reported as ``None`` there). RSS is a process-lifetime
+  high-water mark, so it is reported once, after all scenarios.
+
+Batches reuse one analyzer, as the hooks and the GitHub App do. A warm-up pass
+runs first so imports and rule compilation are not attributed to the first
+sample; rule loading is measured separately (``startup``).
+"""
+
+import gc
+import sys
+import time
+import tracemalloc
+from collections.abc import Sequence
+
+from pydantic import BaseModel, ConfigDict
+
+from commitguard.git.commit import Commit
+from commitguard.policies.defaults import default_policy_set
+from commitguard.provenance.author import Identity
+from commitguard.research.metrics import LatencySummary
+from commitguard.services.analysis import Analyzer
+
+BENCHMARK_VERSION = "1.0.0"
+BATCH_SIZES = (1, 10, 100, 1_000, 10_000)
+MESSAGE_SIZES = (1_024, 10_240, 102_400, 1_048_576, 10_485_760)
+
+HUMAN = Identity(name="Ada Lovelace", email="ada@example.com")
+TRAILER = "Co-authored-by: Claude <noreply@anthropic.com>"
+
+
+class BatchResult(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    commits: int
+    wall_ms: float
+    cpu_ms: float
+    commits_per_second: float
+    latency: LatencySummary
+    python_peak_allocated_bytes: int
+
+
+class SizeResult(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    message_bytes: int
+    repetitions: int
+    decision: str
+    latency: LatencySummary
+    cpu_ms_per_commit: float
+    python_peak_allocated_bytes: int
+
+
+class PerformanceResult(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    benchmark: str = "performance"
+    benchmark_version: str = BENCHMARK_VERSION
+    startup_ms: float
+    batches: tuple[BatchResult, ...]
+    message_sizes: tuple[SizeResult, ...]
+    peak_rss_bytes: int | None
+    notes: tuple[str, ...]
+
+
+def _mixed_commits(count: int) -> list[Commit]:
+    """Realistic mix: 60% clean with human trailers, 35% AI co-author, 5% bot author."""
+    commits = []
+    for index in range(count):
+        kind = index % 20
+        if kind < 12:
+            message = f"fix: handle case {index}\n\nSigned-off-by: Ada Lovelace <ada@example.com>\n"
+            author = HUMAN
+        elif kind < 19:
+            message = f"feat: add feature {index}\n\n{TRAILER}\n"
+            author = HUMAN
+        else:
+            message = "chore(deps): bump a dependency\n"
+            author = Identity(
+                name="dependabot[bot]", email="49699333+dependabot[bot]@users.noreply.github.com"
+            )
+        commits.append(Commit(author=author, committer=author, message=message))
+    return commits
+
+
+def _sized_commit(size: int) -> Commit:
+    """A commit of about ``size`` bytes: wrapped prose, then an AI co-author trailer."""
+    line = "The storage layer keeps immutable versions of every published policy.\n"
+    body = (line * (size // len(line) + 1))[: max(0, size - len(TRAILER) - 40)]
+    return Commit(author=HUMAN, committer=HUMAN, message=f"docs: notes\n\n{body}\n\n{TRAILER}\n")
+
+
+def peak_rss_bytes() -> int | None:
+    try:
+        import resource
+    except ImportError:  # Windows
+        return None
+    peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+    return int(peak if sys.platform == "darwin" else peak * 1024)  # macOS: bytes, Linux: KiB
+
+
+def _allocation_peak(analyzer: Analyzer, commits: Sequence[Commit]) -> int:
+    gc.collect()
+    tracemalloc.start()
+    try:
+        for commit in commits:
+            analyzer.analyze(commit)
+        return tracemalloc.get_traced_memory()[1]
+    finally:
+        tracemalloc.stop()
+
+
+def _batch(analyzer: Analyzer, commits: Sequence[Commit]) -> BatchResult:
+    gc.collect()
+    durations = []
+    cpu_start = time.process_time()
+    wall_start = time.perf_counter()
+    for commit in commits:
+        begin = time.perf_counter()
+        analyzer.analyze(commit)
+        durations.append(time.perf_counter() - begin)
+    wall = time.perf_counter() - wall_start
+    cpu = time.process_time() - cpu_start
+    peak = _allocation_peak(analyzer, commits)
+    return BatchResult(
+        commits=len(commits),
+        wall_ms=round(wall * 1000, 3),
+        cpu_ms=round(cpu * 1000, 3),
+        commits_per_second=round(len(commits) / wall, 1) if wall > 0 else 0.0,
+        latency=LatencySummary.from_seconds(durations),
+        python_peak_allocated_bytes=peak,
+    )
+
+
+def _size(analyzer: Analyzer, size: int) -> SizeResult:
+    commit = _sized_commit(size)
+    repetitions = max(5, min(200, 50_000_000 // max(size, 1)))
+    gc.collect()
+    durations = []
+    decision = ""
+    cpu_start = time.process_time()
+    for _ in range(repetitions):
+        begin = time.perf_counter()
+        report = analyzer.analyze(commit)
+        durations.append(time.perf_counter() - begin)
+        decision = report.action.value
+    cpu = time.process_time() - cpu_start
+    peak = _allocation_peak(analyzer, [commit])
+    return SizeResult(
+        message_bytes=len(commit.message.encode("utf-8")),
+        repetitions=repetitions,
+        decision=decision,
+        latency=LatencySummary.from_seconds(durations),
+        cpu_ms_per_commit=round(cpu * 1000 / repetitions, 4),
+        python_peak_allocated_bytes=peak,
+    )
+
+
+def run_performance(
+    *,
+    batch_sizes: Sequence[int] = BATCH_SIZES,
+    message_sizes: Sequence[int] = MESSAGE_SIZES,
+) -> PerformanceResult:
+    started = time.perf_counter()
+    analyzer = Analyzer.create(default_policy_set())
+    startup = time.perf_counter() - started
+    for commit in _mixed_commits(200):  # warm-up
+        analyzer.analyze(commit)
+    batches = tuple(_batch(analyzer, _mixed_commits(size)) for size in batch_sizes)
+    sizes = tuple(_size(analyzer, size) for size in message_sizes)
+    notes = (
+        "Commits are built in memory: this isolates detection and policy evaluation from Git I/O "
+        "(see the hooks and repository benchmarks for end-to-end Git timings).",
+        "startup_ms includes loading and compiling the built-in rules (cached per process).",
+        "Allocation peaks come from a separate tracemalloc pass; timings are measured without it.",
+    )
+    return PerformanceResult(
+        startup_ms=round(startup * 1000, 3),
+        batches=batches,
+        message_sizes=sizes,
+        peak_rss_bytes=peak_rss_bytes(),
+        notes=notes,
+    )