agentpack-cli 0.1.13__tar.gz → 0.1.14__tar.gz

{agentpack_cli-0.1.13 → agentpack_cli-0.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentpack-cli
-Version: 0.1.13
+Version: 0.1.14
 Summary: Token-aware context packing for AI coding agents — Claude, Cursor, Windsurf, and Codex
 License: MIT
 License-File: LICENSE

{agentpack_cli-0.1.13 → agentpack_cli-0.1.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentpack-cli"
-version = "0.1.13"
+version = "0.1.14"
 description = "Token-aware context packing for AI coding agents — Claude, Cursor, Windsurf, and Codex"
 readme = "README.md"
 requires-python = ">=3.10"

{agentpack_cli-0.1.13 → agentpack_cli-0.1.14}/src/agentpack/__init__.py

@@ -1,3 +1,3 @@
 """AgentPack — token-aware context packing for AI coding agents."""
 
-__version__ = "0.1.13"
+__version__ = "0.1.14"

{agentpack_cli-0.1.13 → agentpack_cli-0.1.14}/src/agentpack/application/pack_service.py

@@ -305,6 +305,8 @@ class PackService:
             saving_pct=saving_pct,
             selected_count=len(plan.selected),
             changed_count=len(plan.all_changed),
+            selected_paths=[sf.path for sf in plan.selected],
+            current_changed=plan.all_changed,
         )
 
         return PackResult(
@@ -331,6 +333,56 @@ def _sf_tokens(sf: SelectedFile) -> int:
     return estimate_tokens("\n".join(parts)) if parts else 50
 
 
+def _load_last_record(metrics_path: Path) -> dict[str, Any] | None:
+    """Return the most recent metrics record that has selected_paths."""
+    if not metrics_path.exists():
+        return None
+    try:
+        lines = metrics_path.read_text(encoding="utf-8").splitlines()
+        for line in reversed(lines):
+            line = line.strip()
+            if not line:
+                continue
+            rec = json.loads(line)
+            if rec.get("selected_paths"):
+                return rec
+    except Exception:
+        pass
+    return None
+
+
+def _compute_selection_accuracy(
+    root: Path,
+    metrics_path: Path,
+    current_selected: list[str],
+    current_changed: set[str],
+) -> dict[str, float]:
+    """Compare previous pack's selected_paths vs files actually changed since then.
+
+    recall    = |predicted ∩ actual_changed| / |actual_changed|
+    precision = |predicted ∩ actual_changed| / |predicted|
+    """
+    prev = _load_last_record(metrics_path)
+    if prev is None:
+        return {}
+
+    prev_selected: set[str] = set(prev["selected_paths"])
+    # actual_changed = files changed since the previous pack (current git diff)
+    actual_changed: set[str] = current_changed
+    if not actual_changed or not prev_selected:
+        return {}
+
+    hits = prev_selected & actual_changed
+    recall = len(hits) / len(actual_changed)
+    precision = len(hits) / len(prev_selected)
+    f1 = (2 * precision * recall / (precision + recall)) if (precision + recall) > 0 else 0.0
+    return {
+        "selection_recall": round(recall, 3),
+        "selection_precision": round(precision, 3),
+        "selection_f1": round(f1, 3),
+    }
+
+
 def _record_metrics(
     root: Path,
     *,
@@ -342,9 +394,12 @@ def _record_metrics(
     saving_pct: float,
     selected_count: int,
     changed_count: int,
+    selected_paths: list[str],
+    current_changed: set[str],
 ) -> None:
     metrics_path = root / ".agentpack" / "metrics.jsonl"
-    record = {
+    accuracy = _compute_selection_accuracy(root, metrics_path, selected_paths, current_changed)
+    record: dict[str, Any] = {
         "ts": datetime.now(timezone.utc).isoformat(),
         "task": task,
         "mode": mode,
@@ -353,9 +408,11 @@ def _record_metrics(
         "saving_pct": round(saving_pct, 1),
         "selected_files": selected_count,
         "changed_files": changed_count,
+        "selected_paths": selected_paths,
         "phases": {k: round(v, 3) for k, v in phase_times.items()},
         "total_s": round(sum(phase_times.values()), 3),
     }
+    record.update(accuracy)
     try:
         with metrics_path.open("a") as fh:
             fh.write(json.dumps(record) + "\n")

{agentpack_cli-0.1.13 → agentpack_cli-0.1.14}/src/agentpack/commands/stats.py

@@ -109,9 +109,49 @@ def register(app: typer.Typer) -> None:
                 top_tbl.add_row(str(i), path, mode, why)
             console.print(top_tbl)
 
+        # --- Selection accuracy (last 10 runs) ---
+        accuracy_rows = _load_accuracy_rows(metrics_path, n=10)
+        if accuracy_rows:
+            avg_recall = sum(r["selection_recall"] for r in accuracy_rows) / len(accuracy_rows)
+            avg_precision = sum(r["selection_precision"] for r in accuracy_rows) / len(accuracy_rows)
+            avg_f1 = sum(r["selection_f1"] for r in accuracy_rows) / len(accuracy_rows)
+            console.print()
+            acc_tbl = Table(title=f"Selection Accuracy (last {len(accuracy_rows)} runs)", box=box.SIMPLE, show_header=False, padding=(0, 2))
+            acc_tbl.add_column(style="dim")
+            acc_tbl.add_column(justify="right", style="bold")
+            acc_tbl.add_row("avg recall", f"{avg_recall:.1%}")
+            acc_tbl.add_row("avg precision", f"{avg_precision:.1%}")
+            acc_tbl.add_row("avg F1", f"{avg_f1:.1%}")
+            console.print(acc_tbl)
+            console.print("[dim]recall = how many changed files were in the previous pack[/]")
+
         console.print("[dim]'manual' = hand-picking 20 most relevant full files[/]")
 
 
+def _load_accuracy_rows(metrics_path: Path, n: int = 10) -> list[dict]:
+    """Return up to n most recent metrics records that have accuracy fields."""
+    if not metrics_path.exists():
+        return []
+    try:
+        lines = metrics_path.read_text(encoding="utf-8").splitlines()
+        rows = []
+        for line in reversed(lines):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rec = json.loads(line)
+            except Exception:
+                continue
+            if "selection_recall" in rec:
+                rows.append(rec)
+                if len(rows) >= n:
+                    break
+        return rows
+    except Exception:
+        return []
+
+
 def _parse_top_files(context_path: Path) -> list[tuple[str, str, str]]:
     """Parse top selected files from context.md. Returns list of (path, mode, why)."""
     results: list[tuple[str, str, str]] = []