sediment-api 0.1.0__py3-none-any.whl

sediment_api/mirror_gc.py

@@ -0,0 +1,151 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Mirror garbage collection operator CLI.
+
+Thin stdout wrapper over ``sediment_derive.gc.gc_mirrors`` — the reusable
+retention logic lives there; this script only reads args, supplies the one
+legitimate wall-clock read (``now``), and prints. Dry-run by default: pass
+``--apply`` to actually delete anything, since a removed mirror is not
+undone by anything short of a fresh clone.
+
+    sediment mirror-gc --org acme-corp
+    sediment mirror-gc --org acme-corp --retention-days 30 --apply
+    sediment mirror-gc --org acme-corp --json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from dataclasses import asdict
+from datetime import UTC, datetime
+
+from sediment_core import normalize_org_id
+from sediment_derive import MirrorManager
+from sediment_derive.gc import MirrorGCPolicy, MirrorGCResult, gc_mirrors
+
+from .database import add_database_url_argument, one_shot_fact_store
+
+_DEFAULT_MIRROR_PATH = "./mirrors"
+
+
+def _result_dict(
+    result: MirrorGCResult, *, dry_run: bool, policy: MirrorGCPolicy
+) -> dict[str, object]:
+    row = asdict(result)
+    row["action"] = result.action.value
+    if result.last_push_captured_at is not None:
+        row["last_push_captured_at"] = result.last_push_captured_at.isoformat()
+    # A dry run and an applied run both report action="removed" for an
+    # eligible mirror -- without these, a --json consumer reading that
+    # value from a dry run misreads it as actually deleted.
+    row["dry_run"] = dry_run
+    row["retention_days"] = policy.retention_days
+    row["policy_version"] = policy.policy_version
+    return row
+
+
+def _print_table(results: list[MirrorGCResult], *, dry_run: bool) -> None:
+    if not results:
+        print("no mirrored repos found")
+        return
+    mode = "dry-run (pass --apply to delete)" if dry_run else "applied"
+    print(f"mode: {mode}")
+    for result in results:
+        last_push = (
+            result.last_push_captured_at.isoformat()
+            if result.last_push_captured_at is not None
+            else "never"
+        )
+        detail = f" ({result.skip_reason})" if result.skip_reason else ""
+        repository = result.repo if result.repo is not None else "name absent"
+        if result.repository_identity is not None:
+            identity = result.repository_identity
+            repository += (
+                f" [{identity.provider}:{identity.host}/{identity.repository_id}]"
+            )
+        print(f"{repository}\t{result.action.value}{detail}\tlast_push={last_push}")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """The argv contract, extracted so the generated CLI reference can
+    walk it."""
+    parser = argparse.ArgumentParser(
+        prog="mirror_gc",
+        description="Reclaim bare mirrors for repos with no recent Push activity.",
+    )
+    parser.add_argument(
+        "--org",
+        default=os.environ.get("SEDIMENT_ORG_ID"),
+        required=not os.environ.get("SEDIMENT_ORG_ID"),
+        help="org id (default: $SEDIMENT_ORG_ID)",
+    )
+    parser.add_argument(
+        "--retention-days",
+        type=int,
+        default=MirrorGCPolicy().retention_days,
+        help=f"days of no Push before a mirror is eligible for removal "
+        f"(default: {MirrorGCPolicy().retention_days})",
+    )
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="actually delete eligible mirrors (default: dry-run, print only)",
+    )
+    parser.add_argument(
+        "--json", action="store_true", help="emit JSON instead of a text report"
+    )
+    add_database_url_argument(parser)
+    parser.add_argument(
+        "--mirror-path",
+        default=os.environ.get("SEDIMENT_MIRROR_PATH", _DEFAULT_MIRROR_PATH),
+        help="base dir for git mirrors (default: $SEDIMENT_MIRROR_PATH or "
+        f"{_DEFAULT_MIRROR_PATH!r})",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    # Reject at the trust boundary, before FactStore or MirrorManager touch
+    # anything: a zero/negative retention flips the cutoff into the future
+    # and (with --apply) removes every mirror that has push history.
+    if args.retention_days < 1:
+        print(
+            f"error: --retention-days must be >= 1, got {args.retention_days}",
+            file=sys.stderr,
+        )
+        return 2
+
+    try:
+        org_id = normalize_org_id(args.org)
+    except ValueError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    policy = MirrorGCPolicy(retention_days=args.retention_days)
+    dry_run = not args.apply
+    with one_shot_fact_store(
+        args.database_url, operation="garbage-collect mirrors"
+    ) as store:
+        results = gc_mirrors(
+            store,
+            MirrorManager(args.mirror_path),
+            org_id,
+            policy,
+            now=datetime.now(UTC),
+            dry_run=dry_run,
+        )
+
+    if args.json:
+        rows = [_result_dict(r, dry_run=dry_run, policy=policy) for r in results]
+        print(json.dumps(rows, indent=2))
+    else:
+        _print_table(results, dry_run=dry_run)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sediment_api/reports/__init__.py

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Read-only operator reports — the ``sediment report`` subcommands.
+
+Each module keeps the shared script convention (AGENTS.md's scripts section):
+``--org`` (defaulting from ``SEDIMENT_ORG_ID``), ``--database-url``/
+``SEDIMENT_DATABASE_URL``, ``--json``, and empty result = "no data" + exit 0.
+None of them import ``sediment_api.config`` — a report never needs API auth
+config to read a store.
+"""

sediment_api/reports/abandonment_report.py

@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Accepted Session status from captured Session-to-commit observations.
+
+Missing observations remain unknown and count under session_commit_unobserved.
+Legacy horizon and mirror options remain accepted for command compatibility.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from dataclasses import asdict
+
+from sediment_core import normalize_org_id
+from sediment_derive import AbandonmentPolicy, MirrorManager, derive_abandonment
+
+from ..database import add_database_url_argument, one_shot_fact_store
+
+_DEFAULT_MIRROR_PATH = "./mirrors"
+
+
+def _print_table(result, policy: AbandonmentPolicy) -> None:
+    print(f"grace_horizon_days: {policy.grace_horizon_days}")
+    print(f"as_of: {result.as_of.isoformat() if result.as_of else 'none'}")
+    print(f"abandoned_sessions: {len(result.sessions)}")
+    print(f"unshipped_accepts: {sum(s.accepted_decisions for s in result.sessions)}")
+    print(
+        f"observed_committed_sessions: {sum(item.status == 'committed' for item in result.outcomes)}"
+    )
+    print(
+        f"unknown_sessions: {sum(item.status == 'attribution_unavailable' for item in result.outcomes)}"
+    )
+    print("skipped:")
+    if not result.skipped:
+        print("  none")
+    else:
+        for reason, count in sorted(result.skipped.items()):
+            print(f"  {reason}: {count}")
+    if result.sessions:
+        print("sessions:")
+        for session in result.sessions:
+            print(
+                f"  {session.session_id}  accepts={session.accepted_decisions}"
+                f"  last_decision_at={session.last_decision_at.isoformat()}"
+            )
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """The argv contract for this report, extracted so the generated CLI
+    reference can walk it; ``main`` is unchanged."""
+    parser = argparse.ArgumentParser(
+        prog="abandonment_report", description=__doc__.splitlines()[0]
+    )
+    parser.add_argument(
+        "--org",
+        default=os.environ.get("SEDIMENT_ORG_ID"),
+        required=not os.environ.get("SEDIMENT_ORG_ID"),
+        help="org id (default: $SEDIMENT_ORG_ID)",
+    )
+    parser.add_argument(
+        "--json", action="store_true", help="emit a JSON row instead of a table"
+    )
+    add_database_url_argument(parser)
+    parser.add_argument(
+        "--mirror-path",
+        default=os.environ.get("SEDIMENT_MIRROR_PATH", _DEFAULT_MIRROR_PATH),
+        help="base dir for git mirrors (default: $SEDIMENT_MIRROR_PATH or "
+        f"{_DEFAULT_MIRROR_PATH!r})",
+    )
+    parser.add_argument(
+        "--grace-horizon-days",
+        type=int,
+        default=AbandonmentPolicy().grace_horizon_days,
+        help="legacy horizon for policy compatibility; missing observations remain unknown "
+        f"(default: {AbandonmentPolicy().grace_horizon_days})",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    try:
+        org_id = normalize_org_id(args.org)
+        policy = AbandonmentPolicy(grace_horizon_days=args.grace_horizon_days)
+    except ValueError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    with one_shot_fact_store(
+        args.database_url, operation="generate abandonment report"
+    ) as store:
+        result = derive_abandonment(
+            store, MirrorManager(args.mirror_path), org_id, policy
+        )
+
+    if args.json:
+        print(
+            json.dumps(
+                {
+                    "org_id": org_id,
+                    "grace_horizon_days": policy.grace_horizon_days,
+                    "policy_version": policy.policy_version,
+                    "as_of": result.as_of.isoformat() if result.as_of else None,
+                    "abandoned_sessions": [asdict(s) for s in result.sessions],
+                    "accepted_session_outcomes": [
+                        asdict(item) for item in result.outcomes
+                    ],
+                    "skipped": dict(sorted(result.skipped.items())),
+                },
+                indent=2,
+                default=str,
+            )
+        )
+    else:
+        _print_table(result, policy)
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

sediment_api/reports/attribution_share_report.py

@@ -0,0 +1,298 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Attribution-share metric + decline-alert operator report.
+
+Derives one org's per-repo notes-attribution share for the current
+``--window-days`` window, derives the same metric again for the trailing
+``--baseline-window-days`` window immediately preceding it, and reports any
+``check_attribution_share_alerts`` verdict between them.
+
+``--target-margin`` plans ``min_cases_for_decline_verdict`` via
+``scripts/corpus_sizing.py``'s worst-case-p=0.5 planner
+(``required_labelled_examples_for_proportion_margin``) -- the same derivation
+``MIN_DRIFT_CASES`` uses for ``threshold_drift_report``, so the gate is a
+computed corpus-size floor, never a hand-picked literal.
+
+Like ``threshold_drift_report.py``, this script is deliberately
+operator-facing only: every outcome -- an alert, or none -- exits 0. Only a
+bad org id or an invalid ``--target-margin``/``--window-days`` exits 2. It is
+not wired into any CI gate.
+
+    sediment report attribution-share --org acme-corp \
+        --target-margin 0.1
+    sediment report attribution-share --org acme-corp \
+        --target-margin 0.1 --json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from dataclasses import asdict, replace
+from datetime import UTC, datetime, timedelta
+
+from sediment_core import FactStore, Push, normalize_org_id
+from sediment_derive import (
+    AttributionShareAlert,
+    AttributionSharePolicy,
+    MirrorManager,
+    RepoAttributionShare,
+    check_attribution_share_alerts,
+    derive_attribution_share,
+    read_repository_context,
+    RepositoryContext,
+    RepositoryIdentity,
+    IdentifiedRepositoryKey,
+    LegacyRepositoryKey,
+)
+from sediment_derive.precision_harness import (
+    required_labelled_examples_for_proportion_margin,
+)
+
+from ..database import add_database_url_argument, one_shot_fact_store
+
+_DEFAULT_MIRROR_PATH = "./mirrors"
+_WORST_CASE_ASSUMED_RATE = 0.5
+
+
+def _baseline_rows(
+    store: FactStore,
+    mirrors: MirrorManager,
+    org_id: str,
+    policy: AttributionSharePolicy,
+    current: list[RepoAttributionShare],
+    *,
+    repository_context: RepositoryContext | None = None,
+) -> list[RepoAttributionShare]:
+    """The trailing baseline window per repo in ``current``: the
+    ``baseline_window_days`` immediately preceding that repo's current
+    ``window_start``. Bounds are enforced here (not left to the derive
+    function's latest-push anchor) so a cadence gap before the current
+    window cannot re-anchor the baseline at a stale pre-current push --
+    matching the contiguous-bounds discipline the model-report path
+    (``derive_model_report_attribution_share``) already follows. The
+    ``_window`` reuse lives in ``derive_attribution_share`` itself; this is
+    only the bounded-selection + bounds-forcing wiring the interim CLI owns.
+
+    Both bounds stay data-driven (ADR 0001 -- never wall-clock):
+    ``current.window_start`` is ``latest_push.captured_at - window_days``
+    (derived from push facts by ``derive_attribution_share``), and the
+    baseline lower bound is arithmetic on that value, never ``now()``."""
+    if repository_context is None:
+        with store.read_snapshot() as snapshot:
+            context = read_repository_context(snapshot, org_id, as_of=datetime.now(UTC))
+            return _baseline_rows(
+                snapshot, mirrors, org_id, policy, current, repository_context=context
+            )
+
+    def row_key(row: RepoAttributionShare):
+        return (
+            IdentifiedRepositoryKey(row.org_id, row.repository_identity)
+            if row.repository_identity is not None
+            else LegacyRepositoryKey(row.org_id, row.repo)
+        )
+
+    current_window_start = {row_key(row): row.window_start for row in current}
+    if not current_window_start:
+        return []
+    baseline_start = {
+        repo: ws - timedelta(days=policy.baseline_window_days)
+        for repo, ws in current_window_start.items()
+    }
+    older_pushes: list[Push] = [
+        push
+        for push in store.read_pushes(org_id, captured_through=repository_context.as_of)
+        if (key := repository_context.resolve_fact(push).key) in current_window_start
+        and baseline_start[key] <= push.captured_at < current_window_start[key]
+    ]
+    if not older_pushes:
+        return []
+    baseline_policy = replace(policy, window_days=policy.baseline_window_days)
+    derived = derive_attribution_share(
+        store,
+        mirrors,
+        org_id,
+        baseline_policy,
+        pushes=older_pushes,
+        repository_context=repository_context,
+        as_of=repository_context.as_of,
+    )
+    return [
+        replace(
+            row,
+            window_start=baseline_start[row_key(row)],
+            window_end=current_window_start[row_key(row)],
+        )
+        for row in derived
+    ]
+
+
+def attribution_share_row_payload(row: RepoAttributionShare) -> dict[str, object]:
+    payload = asdict(row)
+    payload["window_start"] = row.window_start.isoformat()
+    payload["window_end"] = row.window_end.isoformat()
+    return payload
+
+
+def attribution_share_alert_payload(
+    alert: AttributionShareAlert,
+) -> dict[str, object]:
+    return {
+        "org_id": alert.org_id,
+        "repo": alert.repo,
+        "repository_identity": asdict(alert.repository_identity)
+        if alert.repository_identity
+        else None,
+        "kind": alert.kind.value,
+        "current": attribution_share_row_payload(alert.current),
+        "baseline": (
+            attribution_share_row_payload(alert.baseline)
+            if alert.baseline is not None
+            else None
+        ),
+        "reason": alert.reason,
+    }
+
+
+def format_repository_label(repo: str, identity: RepositoryIdentity | None) -> str:
+    """Show the repository lifetime beside its representative display label."""
+    return (
+        repo
+        if identity is None
+        else f"{repo} [{identity.provider}/{identity.host}/{identity.repository_id}]"
+    )
+
+
+def _print_row(row: RepoAttributionShare) -> None:
+    label = format_repository_label(row.repo, row.repository_identity)
+    print(
+        f"  {label}: git_notes_share={row.git_notes_share:.4f} "
+        f"ci=({row.git_notes_share_ci[0]:.4f}, {row.git_notes_share_ci[1]:.4f}) "
+        f"agent_plausible_commits={row.agent_plausible_commits} "
+        f"git_notes={row.git_notes_attributed} jaccard={row.jaccard_attributed} "
+        f"unattributed={row.unattributed}"
+    )
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """The argv contract for this report, extracted so the generated
+    CLI reference can walk it."""
+    parser = argparse.ArgumentParser(
+        prog="attribution_share_report",
+        description=(
+            "Report notes-attribution share per repo and flag a material "
+            "decline against a trailing baseline window."
+        ),
+    )
+    parser.add_argument(
+        "--org",
+        default=os.environ.get("SEDIMENT_ORG_ID"),
+        required=not os.environ.get("SEDIMENT_ORG_ID"),
+        help="org id (default: $SEDIMENT_ORG_ID)",
+    )
+    parser.add_argument(
+        "--target-margin",
+        type=float,
+        required=True,
+        help=(
+            "target CI half-width in raw proportion points for the decline "
+            "verdict's min-cases gate, e.g. 0.1 for +/-10pp (worst-case "
+            "p=0.5 planner -- see scripts/corpus_sizing.py)"
+        ),
+    )
+    parser.add_argument(
+        "--window-days",
+        type=int,
+        default=7,
+        help="current window width in days (default: 7)",
+    )
+    parser.add_argument(
+        "--baseline-window-days",
+        type=int,
+        default=28,
+        help="trailing baseline window width in days (default: 28)",
+    )
+    parser.add_argument(
+        "--json", action="store_true", help="emit JSON instead of a text report"
+    )
+    add_database_url_argument(parser)
+    parser.add_argument(
+        "--mirror-path",
+        default=os.environ.get("SEDIMENT_MIRROR_PATH", _DEFAULT_MIRROR_PATH),
+        help="base dir for git mirrors (default: $SEDIMENT_MIRROR_PATH or "
+        f"{_DEFAULT_MIRROR_PATH!r})",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    try:
+        org_id = normalize_org_id(args.org)
+        min_cases = required_labelled_examples_for_proportion_margin(
+            _WORST_CASE_ASSUMED_RATE, args.target_margin
+        )
+        policy = AttributionSharePolicy(
+            window_days=args.window_days,
+            baseline_window_days=args.baseline_window_days,
+            min_cases_for_decline_verdict=min_cases,
+        )
+    except ValueError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    with one_shot_fact_store(
+        args.database_url, operation="generate attribution-share report"
+    ) as store:
+        mirrors = MirrorManager(args.mirror_path)
+        with store.read_snapshot() as snapshot:
+            context = read_repository_context(snapshot, org_id, as_of=datetime.now(UTC))
+            current = derive_attribution_share(
+                snapshot,
+                mirrors,
+                org_id,
+                policy,
+                repository_context=context,
+                as_of=context.as_of,
+            )
+            baseline = _baseline_rows(
+                snapshot, mirrors, org_id, policy, current, repository_context=context
+            )
+            alerts = check_attribution_share_alerts(current, baseline, policy)
+
+    if args.json:
+        print(
+            json.dumps(
+                {
+                    "min_cases_for_decline_verdict": min_cases,
+                    "current": [attribution_share_row_payload(row) for row in current],
+                    "baseline": [
+                        attribution_share_row_payload(row) for row in baseline
+                    ],
+                    "alerts": [
+                        attribution_share_alert_payload(alert) for alert in alerts
+                    ],
+                },
+                indent=2,
+            )
+        )
+    else:
+        print(f"min_cases_for_decline_verdict: {min_cases}")
+        print("current:")
+        for row in current:
+            _print_row(row)
+        print("baseline:")
+        for row in baseline:
+            _print_row(row)
+        print("alerts:")
+        for alert in alerts:
+            print(
+                f"  {format_repository_label(alert.repo, alert.repository_identity)}: {alert.kind.value} -- {alert.reason}"
+            )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())