commitguardian 0.1.0__py3-none-any.whl

commitguard/research/repository.py

@@ -0,0 +1,111 @@
+"""Scanning real repository histories of increasing size.
+
+For each size a repository is created with ``git fast-import`` (every 20th
+commit carries an AI co-author trailer, one in twenty-five is authored by an
+automation account through its message only - the mix matters less than the
+count). The measured operation is what ``commitguard scan`` and the CI and App
+range scans do: list the commits of a range with ``git rev-list``, read their
+metadata with batched ``git cat-file``, and analyse each commit.
+
+Reported per size: repository creation time (not part of scanning), listing,
+reading and analysis time separately, commits per second, and whether every
+seeded violation was found.
+"""
+
+import time
+from collections.abc import Sequence
+
+from pydantic import BaseModel, ConfigDict
+
+from commitguard.core.decision import Action
+from commitguard.git.repository import Repository
+from commitguard.policies.defaults import default_policy_set
+from commitguard.research.gitenv import fast_import_stream, workspace
+from commitguard.services.analysis import Analyzer
+
+BENCHMARK_VERSION = "1.0.0"
+HISTORY_SIZES = (100, 1_000, 10_000, 100_000)
+AI_EVERY = 20
+
+
+class HistoryResult(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    commits: int
+    create_ms: float
+    list_ms: float
+    read_ms: float
+    analyze_ms: float
+    scan_total_ms: float
+    commits_per_second: float
+    seeded_violations: int
+    blocked: int
+    all_seeded_violations_found: bool
+
+
+class RepositoryResult(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    benchmark: str = "repository"
+    benchmark_version: str = BENCHMARK_VERSION
+    histories: tuple[HistoryResult, ...]
+    notes: tuple[str, ...]
+
+
+def _messages(count: int) -> list[str]:
+    return [
+        f"feat: change {i}\n\nCo-authored-by: Claude <noreply@anthropic.com>\n"
+        if i % AI_EVERY == AI_EVERY - 1
+        else f"fix: change {i}\n\nSigned-off-by: Ada Lovelace <ada@example.com>\n"
+        for i in range(count)
+    ]
+
+
+def run_repository(sizes: Sequence[int] = HISTORY_SIZES) -> RepositoryResult:
+    analyzer = Analyzer.create(default_policy_set())
+    histories = []
+    for size in sizes:
+        if size < 2:
+            raise ValueError("history sizes must be at least 2")
+        with workspace() as space:
+            path = space.init(f"history-{size}")
+            messages = _messages(size)
+            started = time.perf_counter()
+            space.git(path, "fast-import", "--quiet", input_bytes=fast_import_stream(messages))
+            space.git(path, "checkout", "--quiet", "main")
+            create = time.perf_counter() - started
+
+            repository = Repository.discover(path)
+            first = repository.resolve_commit("main~" + str(size - 1))
+            started = time.perf_counter()
+            shas = [first, *repository.list_commits(f"{first}..main", max_count=size)]
+            listed = time.perf_counter()
+            commits = repository.read_commits(shas)
+            read = time.perf_counter()
+            blocked = sum(1 for c in commits if analyzer.analyze(c).action is Action.BLOCK)
+            analyzed = time.perf_counter()
+        seeded = sum(1 for i in range(size) if i % AI_EVERY == AI_EVERY - 1)
+        total = analyzed - started
+        histories.append(
+            HistoryResult(
+                commits=len(shas),
+                create_ms=round(create * 1000, 1),
+                list_ms=round((listed - started) * 1000, 1),
+                read_ms=round((read - listed) * 1000, 1),
+                analyze_ms=round((analyzed - read) * 1000, 1),
+                scan_total_ms=round(total * 1000, 1),
+                commits_per_second=round(len(shas) / total, 1) if total > 0 else 0.0,
+                seeded_violations=seeded,
+                blocked=blocked,
+                all_seeded_violations_found=blocked == seeded,
+            )
+        )
+    return RepositoryResult(
+        histories=tuple(histories),
+        notes=(
+            "The commitguard scan command limits a range to 1,000 commits by default (a safety "
+            "bound); this benchmark calls the same library functions without that bound to "
+            "measure scaling.",
+            "Repositories are local; fetch time from a remote is not included.",
+        ),
+    )

commitguard/research/reproduction.py

@@ -0,0 +1,297 @@
+"""Re-running the evidence CommitGuard publishes.
+
+``commitguard reproduce`` exists so that someone who is not the author can check
+the claims on their own machine. Each step reports one of:
+
+``PASS``      the step ran and its checks held;
+``FAIL``      the step ran and a check did not hold;
+``SKIPPED``   the step could not run (missing test files, pytest or credentials),
+              with the reason - never reported as a success;
+``NOT RUN``   the step was not selected.
+
+Steps that need the repository (the test suites) are skipped when CommitGuard is
+installed without it. Steps that need GitHub credentials are skipped when they
+are not configured; no step ever invents a result.
+"""
+
+import json
+import os
+import subprocess
+import sys
+import time
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+
+import commitguard
+
+Status = Literal["PASS", "FAIL", "SKIPPED", "NOT RUN"]
+AREAS = ("security", "benchmark", "integration", "github")
+#: Environment variables the GitHub step needs (the App's own credentials).
+GITHUB_ENVIRONMENT = (
+    "COMMITGUARD_GITHUB_APP_ID",
+    "COMMITGUARD_GITHUB_WEBHOOK_SECRET",
+    "COMMITGUARD_APP_DATA_DIR",
+)
+
+
+class StepResult(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    area: str
+    name: str
+    status: Status
+    detail: str
+    seconds: float = 0.0
+    command: str = ""
+
+
+class ReproductionResult(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    benchmark: str = "reproduce"
+    benchmark_version: str = "1.0.0"
+    steps: tuple[StepResult, ...]
+    passed: int
+    failed: int
+    skipped: int
+    not_run: int
+
+    @property
+    def ok(self) -> bool:
+        return self.failed == 0
+
+
+def source_checkout() -> Path | None:
+    """The repository this package was installed from, if the tests are present."""
+    root = Path(commitguard.__file__).resolve().parents[2]
+    return root if (root / "tests").is_dir() and (root / "pyproject.toml").is_file() else None
+
+
+def _pytest_available() -> bool:
+    try:
+        import pytest  # noqa: F401
+    except ImportError:
+        return False
+    return True
+
+
+def _run(arguments: Sequence[str], *, cwd: Path, environment: dict[str, str]) -> tuple[int, str]:
+    completed = subprocess.run(  # noqa: S603 - fixed argument vector, no shell
+        list(arguments),
+        cwd=cwd,
+        env={**os.environ, **environment},
+        capture_output=True,
+        timeout=3600,
+        check=False,
+    )
+    output = (completed.stdout + completed.stderr).decode("utf-8", "replace")
+    return completed.returncode, output
+
+
+def _summary_line(output: str) -> str:
+    lines = [line.strip() for line in output.splitlines() if line.strip()]
+    for line in reversed(lines):
+        if " passed" in line or " failed" in line or " error" in line:
+            return line.strip("= ").strip()
+    return lines[-1][:200] if lines else "no output"
+
+
+def _pytest_step(
+    area: str, name: str, arguments: Sequence[str], evidence_dir: Path | None
+) -> StepResult:
+    root = source_checkout()
+    command = "python -m pytest " + " ".join(arguments)
+    if root is None:
+        return StepResult(
+            area=area,
+            name=name,
+            status="SKIPPED",
+            detail=(
+                "the test suite is not installed with the package; clone the repository and "
+                "run this from the checkout"
+            ),
+            command=command,
+        )
+    if not _pytest_available():
+        return StepResult(
+            area=area,
+            name=name,
+            status="SKIPPED",
+            detail='pytest is not installed; install the development dependencies (".[dev]")',
+            command=command,
+        )
+    environment = {"COMMITGUARD_EVIDENCE_DIR": str(evidence_dir)} if evidence_dir else {}
+    started = time.perf_counter()
+    code, output = _run(
+        [sys.executable, "-m", "pytest", *arguments], cwd=root, environment=environment
+    )
+    return StepResult(
+        area=area,
+        name=name,
+        status="PASS" if code == 0 else "FAIL",
+        detail=_summary_line(output),
+        seconds=round(time.perf_counter() - started, 3),
+        command=command,
+    )
+
+
+def security_steps(evidence_dir: Path | None) -> list[StepResult]:
+    return [
+        _pytest_step(
+            "security",
+            "security regression suite (pytest -m security)",
+            ["-q", "-m", "security", "-p", "no:cacheprovider"],
+            evidence_dir,
+        )
+    ]
+
+
+def integration_steps(evidence_dir: Path | None) -> list[StepResult]:
+    return [
+        _pytest_step(
+            "integration",
+            "integration suite (tests/integration)",
+            ["-q", "tests/integration", "-p", "no:cacheprovider"],
+            evidence_dir,
+        )
+    ]
+
+
+def benchmark_steps(results_dir: Path | None = None) -> list[StepResult]:
+    """Rebuild the dataset, check its fingerprint, and re-measure detection."""
+    from commitguard.research.datasets import (
+        DATASET_VERSION,
+        build_dataset,
+        fingerprint,
+        load_dataset,
+    )
+    from commitguard.research.detection import run_detection
+
+    steps: list[StepResult] = []
+    started = time.perf_counter()
+    cases = build_dataset(version=DATASET_VERSION)
+    built = fingerprint(cases)
+    published = results_dir.parent / "datasets" / f"v{DATASET_VERSION}" if results_dir else None
+    manifest_file = published / "manifest.json" if published else None
+    if published is not None and manifest_file is not None and manifest_file.is_file():
+        manifest = json.loads(manifest_file.read_text(encoding="utf-8"))
+        # The files are written one per class, so loading returns the cases grouped by
+        # class: compare the recorded fingerprint, and the cases themselves by id.
+        # Reads the local JSONL files in benchmarks/datasets; not the Hugging Face
+        # hub function that Bandit's B615 rule looks for.
+        published_cases = load_dataset(published)  # nosec B615
+        same_cases = sorted(published_cases, key=lambda case: case.id) == sorted(
+            cases, key=lambda case: case.id
+        )
+        matches = manifest.get("fingerprint") == built and same_cases
+        steps.append(
+            StepResult(
+                area="benchmark",
+                name=f"dataset {DATASET_VERSION} matches the published files",
+                status="PASS" if matches else "FAIL",
+                detail=(
+                    f"rebuilt {built[:16]}, published {str(manifest.get('fingerprint'))[:16]}, "
+                    f"{len(cases)} cases, contents identical: {same_cases}"
+                ),
+                seconds=round(time.perf_counter() - started, 3),
+                command=(
+                    "commitguard benchmark dataset --write <dir> "
+                    f"--dataset-version {DATASET_VERSION}"
+                ),
+            )
+        )
+    else:
+        steps.append(
+            StepResult(
+                area="benchmark",
+                name=f"dataset {DATASET_VERSION} rebuilt deterministically",
+                status="PASS",
+                detail=(
+                    f"fingerprint {built[:16]} ({len(cases)} cases); no published copy to compare"
+                ),
+                seconds=round(time.perf_counter() - started, 3),
+            )
+        )
+    started = time.perf_counter()
+    result = run_detection(cases)
+    clean = result.decision.false_negative == 0 and result.decision.false_positive == 0
+    steps.append(
+        StepResult(
+            area="benchmark",
+            name="detection: no false negative and no false positive",
+            status="PASS" if clean else "FAIL",
+            detail=(
+                f"{result.decision.false_negative} false negatives, "
+                f"{result.decision.false_positive} false positives over {len(cases)} cases"
+            ),
+            seconds=round(time.perf_counter() - started, 3),
+            command=f"commitguard benchmark detection --dataset-version {DATASET_VERSION}",
+        )
+    )
+    return steps
+
+
+def github_steps() -> list[StepResult]:
+    missing = [name for name in GITHUB_ENVIRONMENT if not os.environ.get(name)]
+    command = "commitguard github validate"
+    if missing:
+        return [
+            StepResult(
+                area="github",
+                name="GitHub App configuration and permissions",
+                status="SKIPPED",
+                detail=f"GitHub credentials not configured ({', '.join(missing)} not set)",
+                command=command,
+            )
+        ]
+    started = time.perf_counter()
+    code, output = _run(
+        [sys.executable, "-P", "-m", "commitguard", "github", "validate"],
+        cwd=Path.cwd(),
+        environment={},
+    )
+    return [
+        StepResult(
+            area="github",
+            name="GitHub App configuration and permissions",
+            status="PASS" if code == 0 else "FAIL",
+            detail=_summary_line(output),
+            seconds=round(time.perf_counter() - started, 3),
+            command=command,
+        )
+    ]
+
+
+def reproduce(
+    areas: Sequence[str], *, evidence_dir: Path | None = None, results_dir: Path | None = None
+) -> ReproductionResult:
+    selected = [area for area in AREAS if area in areas]
+    steps: list[StepResult] = []
+    for area in AREAS:
+        if area not in selected:
+            steps.append(
+                StepResult(area=area, name=f"{area} steps", status="NOT RUN", detail="not selected")
+            )
+            continue
+        if area == "security":
+            steps.extend(security_steps(evidence_dir))
+        elif area == "benchmark":
+            steps.extend(benchmark_steps(results_dir))
+        elif area == "integration":
+            steps.extend(integration_steps(evidence_dir))
+        else:
+            steps.extend(github_steps())
+    counts = {
+        status: sum(1 for step in steps if step.status == status)
+        for status in ("PASS", "FAIL", "SKIPPED", "NOT RUN")
+    }
+    return ReproductionResult(
+        steps=tuple(steps),
+        passed=counts["PASS"],
+        failed=counts["FAIL"],
+        skipped=counts["SKIPPED"],
+        not_run=counts["NOT RUN"],
+    )

commitguard/research/results.py

@@ -0,0 +1,94 @@
+"""Immutable benchmark results.
+
+A result is written once, to a new file named after the benchmark, the time and
+the CommitGuard version; an existing file is never overwritten. Newer runs do
+not replace older ones - historical results stay as they were measured, so an
+improvement between versions remains visible and verifiable.
+
+Layout::
+
+    benchmarks/results/raw/<benchmark>/<UTC timestamp>_<version>.json
+    benchmarks/results/processed/index.json     (regenerated from raw/)
+"""
+
+import json
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel
+
+from commitguard.research.environment import BenchmarkManifest
+
+RESULT_SCHEMA = 1
+_SAFE = re.compile(r"[^A-Za-z0-9._-]+")
+
+
+class ResultExistsError(FileExistsError):
+    pass
+
+
+def result_document(
+    benchmark: str, manifest: BenchmarkManifest, result: BaseModel | dict[str, Any]
+) -> dict[str, Any]:
+    payload = result.model_dump(mode="json") if isinstance(result, BaseModel) else result
+    return {
+        "schema_version": RESULT_SCHEMA,
+        "benchmark": benchmark,
+        "manifest": manifest.model_dump(mode="json"),
+        "result": payload,
+    }
+
+
+def write_result(root: Path, document: dict[str, Any]) -> Path:
+    """Write ``document`` under ``root``/raw and refresh the index. Never overwrites."""
+    benchmark = _SAFE.sub("-", str(document["benchmark"]))
+    manifest = document["manifest"]
+    stamp = datetime.fromisoformat(str(manifest["timestamp"])).astimezone(UTC)
+    version = _SAFE.sub("-", str(manifest["commitguard_version"]))
+    target = root / "raw" / benchmark / f"{stamp.strftime('%Y%m%dT%H%M%S%fZ')}_{version}.json"
+    target.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        with target.open("x", encoding="utf-8", newline="\n") as handle:
+            handle.write(json.dumps(document, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
+    except FileExistsError:
+        raise ResultExistsError(f"{target} already exists; results are never overwritten") from None
+    rebuild_index(root)
+    return target
+
+
+def load_results(root: Path) -> list[tuple[Path, dict[str, Any]]]:
+    results = []
+    for path in sorted((root / "raw").glob("*/*.json")):
+        try:
+            results.append((path, json.loads(path.read_text(encoding="utf-8"))))
+        except ValueError:
+            continue
+    return results
+
+
+def latest(root: Path, benchmark: str) -> tuple[Path, dict[str, Any]] | None:
+    matching = [item for item in load_results(root) if item[1].get("benchmark") == benchmark]
+    return matching[-1] if matching else None
+
+
+def rebuild_index(root: Path) -> Path:
+    entries = []
+    for path, document in load_results(root):
+        manifest = document.get("manifest", {})
+        entries.append(
+            {
+                "benchmark": document.get("benchmark"),
+                "file": path.relative_to(root).as_posix(),
+                "timestamp": manifest.get("timestamp"),
+                "commitguard_version": manifest.get("commitguard_version"),
+                "operating_system": manifest.get("operating_system"),
+                "dataset_version": manifest.get("dataset_version"),
+                "rules_version": manifest.get("rules_version"),
+            }
+        )
+    index = root / "processed" / "index.json"
+    index.parent.mkdir(parents=True, exist_ok=True)
+    index.write_text(json.dumps({"results": entries}, indent=2) + "\n", encoding="utf-8")
+    return index

commitguard/rules/__init__.py

@@ -0,0 +1,11 @@
+"""Detection rules as data.
+
+Detectors never hard-code agent names. The pipeline is::
+
+    rules/*.yaml --(loader: safe YAML + schema)--> RuleSet
+    RuleSet --> IdentityMatcher (normalisation, deliberate matching)
+    Detector + IdentityMatcher --> Finding
+
+:mod:`commitguard.rules.models` and :mod:`commitguard.rules.matcher` are pure;
+only :mod:`commitguard.rules.loader` reads files.
+"""

commitguard/rules/data/ai-domains.yaml

@@ -0,0 +1,51 @@
+# Email domains associated with AI agent vendors.
+#
+# A vendor domain alone is NOT evidence of an AI agent: people who work at these
+# companies commit with the same domain. Matching semantics:
+#
+#   local_parts: automation (default)
+#     <automation local part>@domain  (e.g. noreply@anthropic.com) -> strong evidence
+#     any other address at the domain (jane@anthropic.com)         -> only
+#       corroborates a name alias of the same agent; never a match on its own.
+#   local_parts: any
+#     every address at the domain is strong evidence. Only for domains used
+#     exclusively for tool attribution.
+#
+# Google, Amazon, Microsoft/GitHub domains are intentionally absent: their
+# automation addresses are used by many non-AI services.
+
+schema_version: 1
+
+automation_local_parts:
+  - noreply
+  - no-reply
+  - donotreply
+  - bot
+  - agent
+  - ai
+  - assistant
+
+domains:
+  - domain: anthropic.com
+    agent: claude
+  - domain: openai.com
+    agent: openai_codex
+  - domain: cursor.com
+    agent: cursor
+  - domain: windsurf.com
+    agent: windsurf
+  - domain: codeium.com
+    agent: codeium
+  - domain: cline.bot
+    agent: cline
+  - domain: roocode.com
+    agent: roo_code
+  - domain: cognition.ai
+    agent: devin
+  - domain: devin.ai
+    agent: devin
+  - domain: all-hands.dev
+    agent: openhands
+  - domain: aider.chat
+    agent: aider
+    local_parts: any

commitguard/rules/data/ai-identities.yaml

@@ -0,0 +1,131 @@
+# AI agent identities.
+#
+# Loaded by commitguard.rules.loader and validated against
+# commitguard.rules.models.AIIdentityRules. Matching semantics:
+#
+#   names            full name equals an alias (after case/width/whitespace
+#                    normalisation). Alone: medium confidence.
+#   ambiguous_names  aliases that are also common human first names. Only
+#                    count together with this agent's email/domain evidence.
+#   name_prefixes    distinctive leading words ("Claude Opus" matches
+#                    "Claude Opus 4.5 (1M context)"). Alone: high confidence.
+#   emails           exact address. Alone: high confidence.
+#   github_logins    login in a <id>+<login>@users.noreply.github.com address,
+#                    or a "[bot]" account name. Alone: high confidence.
+#
+# There is NO substring matching: "Claude" never matches "Claudette" or
+# "Claude Dupont". Domains live in ai-domains.yaml.
+#
+# `verified: true` requires a public reference commit or vendor documentation
+# recorded in `reference`. Unverified entries are best-known identities that
+# still need confirmation - please contribute references.
+
+schema_version: 1
+
+agents:
+  - id: claude
+    display_name: Claude
+    vendor: Anthropic
+    names: ["Claude", "Claude Code", "Anthropic Claude", "claude-code"]
+    name_prefixes: ["Claude Opus", "Claude Sonnet", "Claude Haiku", "Claude Fable"]
+    emails: ["noreply@anthropic.com"]
+    verified: true
+    reference: "Claude Code default trailer: Co-Authored-By: Claude <noreply@anthropic.com>"
+
+  - id: chatgpt
+    display_name: ChatGPT
+    vendor: OpenAI
+    names: ["ChatGPT", "OpenAI ChatGPT"]
+    verified: false
+
+  - id: openai_codex
+    display_name: OpenAI Codex
+    vendor: OpenAI
+    names: ["OpenAI Codex", "Codex", "codex-cli", "Codex CLI"]
+    github_logins: ["chatgpt-codex-connector[bot]"]
+    verified: false
+
+  - id: github_copilot
+    display_name: GitHub Copilot
+    vendor: GitHub
+    names: ["GitHub Copilot", "Copilot", "Copilot Agent", "copilot-swe-agent"]
+    github_logins: ["Copilot", "copilot-swe-agent[bot]"]
+    verified: false
+
+  - id: cursor
+    display_name: Cursor
+    vendor: Anysphere
+    names: ["Cursor", "Cursor Agent", "cursoragent"]
+    emails: ["cursoragent@cursor.com"]
+    github_logins: ["cursor[bot]"]
+    verified: false
+
+  - id: gemini
+    display_name: Gemini
+    vendor: Google
+    names: ["Gemini", "Google Gemini", "Gemini Code Assist", "Gemini CLI"]
+    github_logins: ["gemini-code-assist[bot]"]
+    verified: false
+
+  - id: jules
+    display_name: Jules
+    vendor: Google
+    names: ["Google Jules"]
+    ambiguous_names: ["Jules"]
+    github_logins: ["google-labs-jules[bot]"]
+    verified: false
+
+  - id: windsurf
+    display_name: Windsurf
+    vendor: Windsurf
+    names: ["Windsurf", "Windsurf Cascade", "Cascade"]
+    verified: false
+
+  - id: codeium
+    display_name: Codeium
+    vendor: Codeium
+    names: ["Codeium"]
+    verified: false
+
+  - id: cline
+    display_name: Cline
+    vendor: Cline
+    names: ["Cline"]
+    github_logins: ["cline[bot]"]
+    verified: false
+
+  - id: roo_code
+    display_name: Roo Code
+    vendor: Roo Code
+    names: ["Roo Code", "RooCode", "Roo Cline"]
+    verified: false
+
+  - id: amazon_q
+    display_name: Amazon Q Developer
+    vendor: Amazon
+    names: ["Amazon Q", "Amazon Q Developer", "Amazon CodeWhisperer", "CodeWhisperer"]
+    github_logins: ["amazon-q-developer[bot]"]
+    verified: false
+
+  - id: devin
+    display_name: Devin
+    vendor: Cognition
+    names: ["Devin AI"]
+    ambiguous_names: ["Devin"]
+    github_logins: ["devin-ai-integration[bot]"]
+    verified: false
+
+  - id: aider
+    display_name: aider
+    vendor: Aider
+    name_prefixes: ["aider"]
+    verified: false
+    reference: "aider attributes as 'aider (<model>)'"
+
+  - id: openhands
+    display_name: OpenHands
+    vendor: All Hands AI
+    names: ["OpenHands", "OpenHands Agent"]
+    emails: ["openhands@all-hands.dev"]
+    github_logins: ["openhands-agent"]
+    verified: false