refhound 0.1.0b1__py3-none-any.whl

refhound/__init__.py

@@ -0,0 +1,5 @@
+"""RefHound - Git repository security and forensic analysis tool."""
+
+from refhound._version import __version__
+
+__all__ = ["__version__"]

refhound/__main__.py

@@ -0,0 +1,6 @@
+"""RefHound entry point: ``python -m refhound``."""
+
+from refhound.cli import main
+
+if __name__ == "__main__":
+    main()

refhound/_version.py

@@ -0,0 +1,7 @@
+"""Version information.
+
+This file is the single source of truth for the package version. The build
+backend reads it via ``[tool.hatch.version]`` and the documentation imports it.
+"""
+
+__version__ = "0.1.0b1"

refhound/analysis/__init__.py

@@ -0,0 +1 @@
+"""RefHound analysis package."""

refhound/analysis/chronology.py

@@ -0,0 +1,108 @@
+"""Temporal anomaly detection.
+
+We only describe observable patterns and never attribute intent or actor.
+"""
+
+from __future__ import annotations
+
+import itertools
+from datetime import UTC, datetime, timedelta
+
+from refhound.models.anomaly import TemporalAnomaly
+from refhound.models.commit import CommitInfo
+
+
+def detect_temporal_anomalies(
+    commits: list[CommitInfo],
+    *,
+    min_burst: int = 5,
+    future_tolerance_minutes: int = 5,
+) -> list[TemporalAnomaly]:
+    """Find unusual timestamp patterns.
+
+    Heuristics:
+    * bursts of commits within a few seconds
+    * committer date earlier than author date
+    * huge author/committer gap (>= 7 days)
+    * future timestamps (beyond tolerance)
+    * timestamps before repository era (handled by caller via first commit)
+    * timezone jumps (>= 6h offset delta between consecutive commits)
+    """
+    anomalies: list[TemporalAnomaly] = []
+    dated = [c for c in commits if c.committer_date is not None]
+    ordered = sorted(dated, key=lambda c: c.committer_date or datetime.min)
+
+    bursts: dict[int, list[CommitInfo]] = {}
+    for info in ordered:
+        if info.committer_date is None or info.author_date is None:
+            continue
+        if info.committer_date < info.author_date:
+            anomalies.append(
+                TemporalAnomaly(
+                    kind="committer_before_author",
+                    commit_sha=info.sha,
+                    description="Committer timestamp predates author timestamp.",
+                    metadata={
+                        "author": info.author_date.isoformat(),
+                        "committer": info.committer_date.isoformat(),
+                    },
+                )
+            )
+        gap = info.committer_date - info.author_date
+        if gap > timedelta(days=7):
+            anomalies.append(
+                TemporalAnomaly(
+                    kind="large_author_committer_gap",
+                    commit_sha=info.sha,
+                    description=f"Author and committer timestamps differ by {gap.days}d.",
+                    metadata={
+                        "author": info.author_date.isoformat(),
+                        "committer": info.committer_date.isoformat(),
+                    },
+                )
+            )
+        now = datetime.now(tz=UTC)
+        if info.committer_date > now + timedelta(minutes=future_tolerance_minutes):
+            anomalies.append(
+                TemporalAnomaly(
+                    kind="future_timestamp",
+                    commit_sha=info.sha,
+                    description="Committer timestamp is in the future.",
+                    metadata={"committer": info.committer_date.isoformat()},
+                )
+            )
+        bucket = int(info.committer_date.timestamp()) // 30
+        bursts.setdefault(bucket, []).append(info)
+
+    for bucket, items in bursts.items():
+        if len(items) >= min_burst:
+            window = datetime.fromtimestamp(bucket * 30, tz=UTC)
+            anomalies.append(
+                TemporalAnomaly(
+                    kind="commit_burst",
+                    commit_sha=items[0].sha,
+                    description=f"{len(items)} commits within a 30 second window starting {window.isoformat()}.",
+                    metadata={"count": str(len(items)), "window": window.isoformat()},
+                )
+            )
+
+    # Timezone jumps between consecutive commits.
+    for prev, current in itertools.pairwise(ordered):
+        if prev.committer_date is None or current.committer_date is None:
+            continue
+        prev_tz = prev.committer_date.utcoffset() or timedelta(0)
+        curr_tz = current.committer_date.utcoffset() or timedelta(0)
+        jump = abs((curr_tz - prev_tz).total_seconds())
+        if jump >= 6 * 3600:
+            anomalies.append(
+                TemporalAnomaly(
+                    kind="timezone_jump",
+                    commit_sha=current.sha,
+                    description="Sudden shift in committer timezone between consecutive commits.",
+                    metadata={
+                        "prev_offset": str(prev_tz),
+                        "current_offset": str(curr_tz),
+                    },
+                )
+            )
+    return anomalies

refhound/analysis/commit_anomalies.py

@@ -0,0 +1,130 @@
+"""Commit-level interest and anomaly heuristics.
+
+These feed the ``interesting`` command and ``explain``. Nothing here asserts
+intent; each reason is a plain observable fact.
+"""
+
+from __future__ import annotations
+
+from refhound.models.anomaly import InterestingCommit
+from refhound.models.commit import CommitInfo
+from refhound.util.paths import classify_path_category, is_interesting_path
+
+_SUSPICIOUS_NAMES = frozenset(
+    {
+        ".env",
+        ".env.production",
+        ".env.local",
+        "secrets",
+        "credentials",
+        "id_rsa",
+        "id_ed25519",
+        "kubeconfig",
+        "terraform.tfstate",
+        ".npmrc",
+        "settings.py",
+    }
+)
+
+_REVIEW_KEYWORDS = (
+    "remove secret",
+    "cleanup credentials",
+    "temporary",
+    "debug",
+    "hotfix",
+    "revert",
+    "backup",
+    "old config",
+    "production",
+)
+
+
+def _reason(score: int, text: str) -> tuple[int, str]:
+    return score, text
+
+
+def score_commit(
+    info: CommitInfo,
+    *,
+    has_secret_added: bool = False,
+    has_secret_removed: bool = False,
+    added_paths: list[str] | None = None,
+    removed_paths: list[str] | None = None,
+    unreachable: bool = False,
+    chain_id: str | None = None,
+    unusual_author: bool = False,
+    temporal_anomaly: bool = False,
+    ci_changed: bool = False,
+) -> InterestingCommit:
+    """Compute an explainable interest score for a commit."""
+    added = added_paths or []
+    removed = removed_paths or []
+    reasons: list[tuple[int, str]] = []
+    score = 0
+
+    if has_secret_added:
+        reasons.append(_reason(30, "introduces secret material"))
+        score += 30
+    if has_secret_removed:
+        reasons.append(_reason(20, "removes secret material"))
+        score += 20
+    if unreachable and chain_id:
+        reasons.append(_reason(20, f"commit is unreachable (chain {chain_id})"))
+        score += 20
+    elif unreachable:
+        reasons.append(_reason(20, "commit is unreachable from known refs"))
+        score += 20
+    if chain_id:
+        reasons.append(_reason(5, "part of a lost commit chain"))
+        score += 5
+
+    for path in added[:20]:
+        if is_interesting_path(path):
+            reasons.append(_reason(15, "adds file under an interesting path"))
+            score += 15
+            break
+    for path in added[:20]:
+        if classify_path_category(path) == "credential":
+            reasons.append(_reason(25, "adds a credential/config file"))
+            score += 25
+            break
+    for path in removed[:20]:
+        if is_interesting_path(path):
+            reasons.append(_reason(10, "removes file under an interesting path"))
+            score += 10
+            break
+
+    if unusual_author:
+        reasons.append(_reason(9, "unusual author/committer identity"))
+        score += 9
+    if temporal_anomaly:
+        reasons.append(_reason(7, "temporal timestamp anomaly"))
+        score += 7
+    if ci_changed:
+        reasons.append(_reason(12, "CI/CD configuration changed"))
+        score += 12
+
+    subject_lower = info.subject.lower()
+    for keyword in _REVIEW_KEYWORDS:
+        if keyword in subject_lower:
+            reasons.append(_reason(6, f"subject matches review keyword '{keyword}'"))
+            score += 6
+            break
+
+    if info.is_merge:
+        reasons.append(_reason(2, "merge commit"))
+        score += 2
+    if info.deleted >= 5000:
+        reasons.append(_reason(12, "unusually large deletion (heuristic)"))
+        score += 12
+
+    return InterestingCommit(
+        sha=info.sha,
+        score=min(100, score),
+        date=info.committer_date,
+        subject=info.subject,
+        author=info.author_email,
+        reasons=reasons,
+        added_files=added[:50],
+        removed_files=removed[:50],
+    )

refhound/analysis/correlations.py

@@ -0,0 +1,150 @@
+"""Correlation: combine weak signals into prioritized findings.
+
+Rather than emitting thousands of isolated rows, we consolidate evidence
+across unreachable history, secret state, and removal windows.
+"""
+
+from __future__ import annotations
+
+from refhound.models.anomaly import ChurnFinding
+from refhound.models.finding import (
+    Finding,
+    FindingCategory,
+    SecretRecord,
+    Severity,
+    SourceState,
+)
+from refhound.models.object import LostCommitChain
+
+
+def churn_to_findings(churn: list[ChurnFinding], repo: str) -> list[Finding]:
+    """Promote churn results with secret involvement to findings."""
+    findings: list[Finding] = []
+    for item in churn:
+        if not item.secret_found:
+            continue
+        severity = Severity.HIGH
+        findings.append(
+            Finding(
+                id=f"RH-{item.added_commit[:8]}-{item.removed_commit[:8]}-churn",
+                category=FindingCategory.SECRET,
+                title="Secret introduced and removed within a short window",
+                description=(
+                    f"{item.path.split('/')[-1]} was added in {item.added_commit[:8]} "
+                    f"and removed in {item.removed_commit[:8]} ({item.lifetime_seconds:.0f}s "
+                    f"if lifetime data available)."
+                ),
+                severity=severity,
+                score=0,
+                repository=repo,
+                commit_sha=item.added_commit,
+                path=item.path,
+                source_state=SourceState.HISTORICAL,
+                introduced_commit=item.added_commit,
+                removed_commit=item.removed_commit,
+                introduced_at=item.added_at,
+                removed_at=item.removed_at,
+                metadata={"lifetime_seconds": f"{int(item.lifetime_seconds or -1)}"},
+                provenance=["git-object-db"],
+            )
+        )
+    return findings
+
+
+def cluster_secrets_into_findings(
+    secrets: list[SecretRecord], repo: str, interesting_paths: set[str]
+) -> list[Finding]:
+    """Build one Finding per unique secret with occurrence aggregation."""
+    findings: list[Finding] = []
+    for secret in secrets:
+        state = (
+            SourceState.CURRENT
+            if secret.current
+            else SourceState.HISTORICAL
+            if secret.historical
+            else SourceState.UNREACHABLE
+            if secret.unreachable
+            else SourceState.HISTORICAL
+        )
+        first = secret.occurrences[0] if secret.occurrences else None
+        path = (
+            first.path if first else (secret.occurrences[-1].path if secret.occurrences else None)
+        )
+        interesting = bool(path and path in interesting_paths)
+        severity = (
+            Severity.HIGH
+            if secret.detector in {"private-key"}
+            else Severity.HIGH
+            if secret.current or secret.unreachable
+            else Severity.MEDIUM
+        )
+        finding = Finding(
+            id=f"RH-sec-{secret.fingerprint[-12:]}",
+            category=(
+                FindingCategory.PRIVATE_KEY
+                if secret.detector == "private-key"
+                else FindingCategory.CREDENTIAL
+            ),
+            title=f"{secret.detector} credential",
+            severity=severity,
+            score=0,
+            repository=repo,
+            commit_sha=secret.introduced_commit,
+            path=path,
+            detector=secret.detector,
+            source_state=state,
+            introduced_commit=secret.introduced_commit,
+            removed_commit=secret.removed_commit,
+            introduced_at=secret.first_seen,
+            removed_at=secret.last_seen,
+            first_seen=secret.first_seen,
+            last_seen=secret.last_seen,
+            occurrence_count=secret.occurrence_count,
+            secret_fingerprint=secret.fingerprint,
+            metadata={
+                "secret_prefix": secret.prefix,
+                "secret_suffix": secret.suffix,
+                "interesting_path": "yes" if interesting else "no",
+                **(
+                    {"lifetime_seconds": f"{int(secret.lifetime_seconds or 0)}"}
+                    if secret.lifetime_seconds
+                    else {}
+                ),
+            },
+            provenance=sorted({o.source_state.value for o in secret.occurrences}),
+            remediation="Rotate the credential and verify it is not in use. Review the commits listed in this finding.",
+        )
+        findings.append(finding)
+    return findings
+
+
+def merge_chain_into_finding(
+    chain: LostCommitChain, secrets_in_chain: list[SecretRecord], repo: str
+) -> Finding | None:
+    """A high-value consolidated finding for a lost chain containing secrets."""
+    if not secrets_in_chain:
+        return None
+    severity = Severity.HIGH if any(s.unreachable for s in secrets_in_chain) else Severity.MEDIUM
+    return Finding(
+        id=f"RH-chain-{chain.chain_id}",
+        category=FindingCategory.LOST_HISTORY,
+        title="Secret present in unreachable (lost) history",
+        description=(
+            f"Lost chain {chain.chain_id} ({chain.commit_count} commits, "
+            f"{chain.root[:8]}..{chain.tip[:8]}) contains secret material. "
+            "The commits are unreachable from known refs."
+        ),
+        severity=severity,
+        score=0,
+        repository=repo,
+        commit_sha=chain.tip,
+        source_state=SourceState.UNREACHABLE,
+        chain_id=chain.chain_id,
+        metadata={
+            "chain_root": chain.root,
+            "chain_tip": chain.tip,
+            "commit_count": str(chain.commit_count),
+        },
+        provenance=["git-object-db", "unreachable-analysis"],
+        remediation="Review the lost chain commits; export them to a report for manual inspection.",
+    )

refhound/analysis/data.py

@@ -0,0 +1,120 @@
+"""Shared analysis state passed between pipeline stages.
+
+This is intentionally a plain dataclass (not pydantic) — it is scratch space
+during a scan, not part of the persisted/reported surface.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from refhound.analysis.force_push_analysis import RefTransition
+from refhound.models.anomaly import (
+    ChurnFinding,
+    IdentityAnomaly,
+    InterestingCommit,
+    TemporalAnomaly,
+    TimelineRow,
+)
+from refhound.models.commit import CommitInfo, IdentitySet
+from refhound.models.diagnostic import ScanDiagnostic
+from refhound.models.finding import Finding, SecretRecord
+from refhound.models.object import BlobRecord, DanglingObject, LostCommitChain
+from refhound.models.repository import RepoRef, RepositoryInfo
+from refhound.models.statistics import ObjectStats, RepositoryStatistics, SecretStats, SeverityStats
+
+
+@dataclass(slots=True)
+class AnalysisData:
+    repo: RepositoryInfo | None = None
+    refs: list[RepoRef] = field(default_factory=list)
+    commit_graph: dict[str, CommitInfo] = field(default_factory=dict)
+    reachable_oids: set[str] = field(default_factory=set)
+    unreachable_oids: set[str] = field(default_factory=set)
+    dangling: list[DanglingObject] = field(default_factory=list)
+    all_commit_oids: list[str] = field(default_factory=list)
+    object_stats: ObjectStats = field(default_factory=ObjectStats)
+    blobs: dict[str, BlobRecord] = field(default_factory=dict)
+    tip_trees: dict[str, dict[str, str]] = field(default_factory=dict)
+    blob_commits: dict[str, set[str]] = field(default_factory=dict)
+    secrets: list[SecretRecord] = field(default_factory=list)
+    lost_chains: list[LostCommitChain] = field(default_factory=list)
+    timeline: list[TimelineRow] = field(default_factory=list)
+    interesting: dict[str, InterestingCommit] = field(default_factory=dict)
+    churn: list[ChurnFinding] = field(default_factory=list)
+    identities: list[IdentitySet] = field(default_factory=list)
+    deleted_files: list[str] = field(default_factory=list)
+    renamed_files: list[str] = field(default_factory=list)
+    merge_commit_count: int = 0
+    signed_count: int = 0
+    unsigned_count: int = 0
+    history_components: int = 0
+    findings: list[Finding] = field(default_factory=list)
+    notes: dict[str, bytes] = field(default_factory=dict)
+    fingerprint: object | None = None
+    scan_warnings: list[str] = field(default_factory=list)
+    complete: bool = True
+    diagnostics: list[ScanDiagnostic] = field(default_factory=list)
+    failed_detectors: dict[str, int] = field(default_factory=dict)
+    temporal_anomalies: list[TemporalAnomaly] = field(default_factory=list)
+    identity_anomalies: list[IdentityAnomaly] = field(default_factory=list)
+    ref_transitions: list[RefTransition] = field(default_factory=list)
+    scan_id: str = ""
+    scan_timestamp: str = ""
+    cached_statistics: RepositoryStatistics | None = None
+
+    @property
+    def statistics(self) -> RepositoryStatistics:
+        if self.cached_statistics is not None:
+            return self.cached_statistics
+        return RepositoryStatistics(
+            total_commits=len(self.commit_graph),
+            reachable_commits=len(self.reachable_oids),
+            unreachable_commits=len(self.unreachable_oids),
+            branches=sum(1 for r in self.refs if r.ref_name.startswith("refs/heads/")),
+            tags=sum(1 for r in self.refs if r.ref_name.startswith("refs/tags/")),
+            authors=len({i.email for i in self.identities}),
+            files=len(self.blobs),
+            blobs=len(self.blobs),
+            deleted_files=len(self.deleted_files),
+            renamed_files=len(self.renamed_files),
+            merge_commits=self.merge_commit_count,
+            signed_commits=self.signed_count,
+            unsigned_commits=self.unsigned_count,
+            objects=ObjectStats(
+                commits=self.object_stats.commits,
+                trees=self.object_stats.trees,
+                blobs=self.object_stats.blobs,
+                tags=self.object_stats.tags,
+                unreachable_commits=len(self.unreachable_oids),
+                lost_chains=len(self.lost_chains),
+            ),
+            history_components=self.history_components,
+        )
+
+    @property
+    def severity_stats(self) -> SeverityStats:
+        stats = SeverityStats()
+        for finding in self.findings:
+            stats.add(finding.severity.value)
+        return stats
+
+    @property
+    def secret_stats(self) -> SecretStats:
+        stats = SecretStats(unique_secrets=len(self.secrets))
+        for secret in self.secrets:
+            if secret.current:
+                stats.current += 1
+            if secret.historical:
+                stats.historical += 1
+            if secret.unreachable:
+                stats.unreachable += 1
+            if secret.detector == "private-key":
+                stats.private_keys += 1
+            elif secret.detector in {"database-uri"}:
+                stats.connection_strings += 1
+            elif secret.detector == "generic-password":
+                stats.passwords += 1
+            else:
+                stats.tokens += 1
+        return stats

refhound/analysis/deletion_analysis.py

@@ -0,0 +1,157 @@
+"""File deletion / introduction analysis.
+
+Tracks the lifecycle of files that are interesting from a security
+perspective (added, removed, reintroduced) and computes the additive/removal
+windows used by churn analysis.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+from refhound.git.command import GitRunner, validate_oid
+from refhound.git.fsck import list_tree_blobs
+from refhound.models.anomaly import ChurnFinding
+
+_HEADER_MARK = "\x1e"
+
+
+@dataclass(slots=True)
+class PathEvent:
+    commit: str
+    status: str  # A/M/D/R
+    date: str = ""
+
+
+@dataclass(slots=True)
+class ChangeRecord:
+    commit: str
+    timestamp: str
+    added: list[str]
+    removed: list[str]
+
+
+def path_history(git: GitRunner, cwd: str | Path, path: str) -> list[PathEvent]:
+    """Chronological (oldest-first) list of changes touching ``path``."""
+    out = git.run(
+        "log",
+        "--all",
+        "--reverse",
+        "--format=%H%x1e%ct%x1e",
+        "--name-status",
+        "--no-renames",
+        "--",
+        path,
+        cwd=cwd,
+        check=False,
+    )
+    if out.returncode != 0:
+        return []
+    events: list[PathEvent] = []
+    current_commit = ""
+    current_date = ""
+    for line in out.stdout.split("\n"):
+        if _HEADER_MARK in line:
+            fields = line.split(_HEADER_MARK)
+            current_commit = fields[0]
+            current_date = fields[1] if len(fields) > 1 else ""
+            continue
+        if not current_commit:
+            continue
+        status = line[0] if line else "?"
+        if status in {"A", "M", "D", "R", "C", "T"}:
+            events.append(PathEvent(commit=current_commit, status=status, date=current_date))
+    return events
+
+
+def changed_file_status_iter(git: GitRunner, cwd: str | Path) -> Iterator[ChangeRecord]:
+    """Yield per-commit added/removed path lists for all reachable commits.
+
+    Single ``git log`` invocation, no renames (renames decompose into
+    add+delete), unit-separated format. Oldest first.
+    """
+    out = git.run(
+        "log",
+        "--all",
+        "--reverse",
+        "--format=%H%x1e%ct%x1e",
+        "--name-status",
+        "--no-renames",
+        "--no-merges",
+        cwd=cwd,
+        timeout=900.0,
+    ).stdout
+    current_commit = ""
+    current_ts = ""
+    added: list[str] = []
+    removed: list[str] = []
+    for line in out.split("\n"):
+        if _HEADER_MARK in line:
+            if current_commit:
+                yield ChangeRecord(
+                    commit=current_commit, timestamp=current_ts, added=added, removed=removed
+                )
+            fields = line.split(_HEADER_MARK)
+            current_commit = fields[0]
+            current_ts = fields[1] if len(fields) > 1 else ""
+            added, removed = [], []
+            continue
+        if not current_commit or "\t" not in line:
+            continue
+        status = line[0]
+        path = line.split("\t")[-1]
+        if status == "A":
+            added.append(path)
+        elif status == "D":
+            removed.append(path)
+    if current_commit:
+        yield ChangeRecord(
+            commit=current_commit, timestamp=current_ts, added=added, removed=removed
+        )
+
+
+def interesting_lifetimes(
+    git: GitRunner,
+    cwd: str | Path,
+    interesting_paths: list[str],
+    *,
+    max_window_seconds: int = 3600,
+) -> list[ChurnFinding]:
+    """Find files that were added then removed within a short window.
+
+    Facts only: reports the observable add/remove pair. Never claims intent.
+    """
+    findings: list[ChurnFinding] = []
+    for path in interesting_paths:
+        events = path_history(git, cwd, path)
+        if len(events) < 2:
+            continue
+        additions: list[PathEvent] = [e for e in events if e.status == "A"]
+        removals: list[PathEvent] = [e for e in events if e.status == "D"]
+        if not additions or not removals:
+            continue
+        for added in additions:
+            for removed in removals:
+                added_ts = int(added.date) if added.date.isdigit() else 0
+                removed_ts = int(removed.date) if removed.date.isdigit() else 0
+                if added_ts and removed_ts and 0 <= removed_ts - added_ts <= max_window_seconds:
+                    findings.append(
+                        ChurnFinding(
+                            path=path,
+                            added_commit=added.commit,
+                            removed_commit=removed.commit,
+                            lifetime_seconds=float(removed_ts - added_ts),
+                        )
+                    )
+                    break
+    findings.sort(key=lambda f: (f.lifetime_seconds or 0, f.path))
+    return findings
+
+
+def file_state_at(git: GitRunner, cwd: str | Path, commit: str, path: str) -> str | None:
+    """Blob oid of ``path`` at ``commit``, or None if absent."""
+    commit = validate_oid(commit)
+    blobs = dict(list_tree_blobs(git, cwd, commit))
+    return blobs.get(path)