devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/metrics/__init__.py

@@ -0,0 +1,35 @@
+"""
+devtorch_core.metrics — Metrics calibration and scoring for DevTorch.
+
+Provides:
+  - CalibrationDefaults / CalibrationStore / load_calibration  (calibration.py)
+  - CollisionEvent / compute_mcs / mcs_to_label                (mcs.py)
+  - DHSComponents / compute_dhs / dhs_to_label                 (dhs.py)
+  - ROIEstimate / estimate_context_switch_savings / estimate_rework_savings  (roi.py)
+"""
+
+from .calibration import CalibrationDefaults, CalibrationStore, load_calibration
+from .mcs import CollisionEvent, compute_mcs, mcs_to_label, estimate_rework_cost
+from .dhs import DHSComponents, compute_dhs, dhs_to_label, compute_dhs_from_gcc
+from .roi import ROIEstimate, estimate_context_switch_savings, estimate_rework_savings
+
+__all__ = [
+    # calibration
+    "CalibrationDefaults",
+    "CalibrationStore",
+    "load_calibration",
+    # MCS
+    "CollisionEvent",
+    "compute_mcs",
+    "mcs_to_label",
+    "estimate_rework_cost",
+    # DHS
+    "DHSComponents",
+    "compute_dhs",
+    "dhs_to_label",
+    "compute_dhs_from_gcc",
+    # ROI
+    "ROIEstimate",
+    "estimate_context_switch_savings",
+    "estimate_rework_savings",
+]

devtorch_core/metrics/aggregate.py

@@ -0,0 +1,215 @@
+"""
+Sprint 20 — Query-time aggregation for token savings and latency speedup.
+
+All numbers are derived from .GCC/metrics/session/*.json files at query time.
+No pre-aggregated files are produced; this keeps the numbers consistent.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from devtorch_core.projects import ProjectRegistry
+
+
+def _read_session_files(session_dir: Path) -> List[dict]:
+    """Read all valid session metric files from a directory."""
+    files: List[dict] = []
+    if not session_dir.exists():
+        return files
+    for path in session_dir.glob("*.json"):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if isinstance(data, dict):
+                files.append(data)
+        except (json.JSONDecodeError, OSError):
+            continue
+    return files
+
+
+def aggregate_tokens(gcc_dir: Path) -> Dict[str, object]:
+    """
+    Aggregate token usage across all session files in the repository.
+
+    Returns:
+        {
+            "total_tokens_used": int,
+            "total_tokens_saved": int,
+            "total_bundle_tokens": int,
+            "total_full_history_tokens": int,
+            "sessions_count": int,
+            "avg_tokens_used": float,
+        }
+    """
+    sessions = _read_session_files(gcc_dir / "metrics" / "session")
+    total_used = sum(s.get("tokens_used", 0) for s in sessions)
+    total_saved = sum(s.get("tokens_saved", 0) for s in sessions)
+    total_bundle = sum(s.get("bundle_tokens", 0) for s in sessions)
+    total_full_history = sum(s.get("full_history_tokens", 0) for s in sessions)
+    count = len(sessions)
+    return {
+        "total_tokens_used": total_used,
+        "total_tokens_saved": total_saved,
+        "total_bundle_tokens": total_bundle,
+        "total_full_history_tokens": total_full_history,
+        "sessions_count": count,
+        "avg_tokens_used": total_used / count if count else 0.0,
+    }
+
+
+def aggregate_speedup(gcc_dir: Path) -> Dict[str, object]:
+    """
+    Aggregate latency speedup across all session files.
+
+    Speedup is computed as full_history_tokens / bundle_tokens. It is a proxy
+    for LLM call latency reduction and is always available from the session
+    files. If bundle_tokens is zero, speedup is reported as 1.0 (no reduction).
+
+    Returns:
+        {
+            "latency_speedup_factor": float,
+            "weighted_avg_speedup": float,
+            "total_full_history_tokens": int,
+            "total_bundle_tokens": int,
+            "sessions_count": int,
+        }
+    """
+    sessions = _read_session_files(gcc_dir / "metrics" / "session")
+    total_full = sum(s.get("full_history_tokens", 0) for s in sessions)
+    total_bundle = sum(s.get("bundle_tokens", 0) for s in sessions)
+    count = len(sessions)
+
+    if total_bundle > 0:
+        weighted_speedup = total_full / total_bundle
+    else:
+        weighted_speedup = 1.0
+
+    simple_speedups = []
+    for s in sessions:
+        full = s.get("full_history_tokens", 0)
+        bundle = s.get("bundle_tokens", 0)
+        if bundle > 0:
+            simple_speedups.append(full / bundle)
+
+    avg_speedup = sum(simple_speedups) / len(simple_speedups) if simple_speedups else 1.0
+
+    return {
+        "latency_speedup_factor": weighted_speedup,
+        "weighted_avg_speedup": weighted_speedup,
+        "simple_avg_speedup": avg_speedup,
+        "total_full_history_tokens": total_full,
+        "total_bundle_tokens": total_bundle,
+        "sessions_count": count,
+    }
+
+
+def aggregate_by_developer(gcc_dir: Path) -> Dict[str, dict]:
+    """
+    Aggregate tokens/speedup per developer_id found in session files.
+
+    Returns:
+        {
+            developer_id: {
+                "tokens_used": int,
+                "tokens_saved": int,
+                "bundle_tokens": int,
+                "full_history_tokens": int,
+                "sessions_count": int,
+                "latency_speedup_factor": float,
+            },
+            ...
+        }
+    """
+    sessions = _read_session_files(gcc_dir / "metrics" / "session")
+    by_dev: Dict[str, List[dict]] = {}
+    for s in sessions:
+        dev_id = s.get("developer_id") or "unknown"
+        by_dev.setdefault(dev_id, []).append(s)
+
+    result: Dict[str, dict] = {}
+    for dev_id, dev_sessions in by_dev.items():
+        total_used = sum(s.get("tokens_used", 0) for s in dev_sessions)
+        total_saved = sum(s.get("tokens_saved", 0) for s in dev_sessions)
+        total_bundle = sum(s.get("bundle_tokens", 0) for s in dev_sessions)
+        total_full = sum(s.get("full_history_tokens", 0) for s in dev_sessions)
+        if total_bundle > 0:
+            speedup = total_full / total_bundle
+        else:
+            speedup = 1.0
+        result[dev_id] = {
+            "tokens_used": total_used,
+            "tokens_saved": total_saved,
+            "bundle_tokens": total_bundle,
+            "full_history_tokens": total_full,
+            "sessions_count": len(dev_sessions),
+            "latency_speedup_factor": speedup,
+        }
+    return result
+
+
+def aggregate_all_projects() -> Dict[str, object]:
+    """
+    Aggregate token/speedup metrics across all projects in the registry.
+
+    Returns:
+        {
+            "projects": [
+                {
+                    "name": str,
+                    "path": str,
+                    "tokens_used": int,
+                    "tokens_saved": int,
+                    "latency_speedup_factor": float,
+                    "sessions_count": int,
+                },
+                ...
+            ],
+            "aggregate": {
+                "tokens_used": int,
+                "tokens_saved": int,
+                "latency_speedup_factor": float,
+                "sessions_count": int,
+            },
+        }
+    """
+    registry = ProjectRegistry()
+    projects = []
+    total_used = 0
+    total_saved = 0
+    total_full = 0
+    total_bundle = 0
+    total_sessions = 0
+
+    for entry in registry.list():
+        gcc_dir = Path(entry.path) / ".GCC"
+        tokens = aggregate_tokens(gcc_dir)
+        speedup = aggregate_speedup(gcc_dir)
+        projects.append({
+            "name": entry.name,
+            "path": entry.path,
+            "tokens_used": tokens["total_tokens_used"],
+            "tokens_saved": tokens["total_tokens_saved"],
+            "latency_speedup_factor": speedup["latency_speedup_factor"],
+            "sessions_count": tokens["sessions_count"],
+        })
+        total_used += tokens["total_tokens_used"]
+        total_saved += tokens["total_tokens_saved"]
+        total_full += speedup["total_full_history_tokens"]
+        total_bundle += speedup["total_bundle_tokens"]
+        total_sessions += tokens["sessions_count"]
+
+    if total_bundle > 0:
+        aggregate_speedup_factor = total_full / total_bundle
+    else:
+        aggregate_speedup_factor = 1.0
+
+    return {
+        "projects": projects,
+        "aggregate": {
+            "tokens_used": total_used,
+            "tokens_saved": total_saved,
+            "latency_speedup_factor": aggregate_speedup_factor,
+            "sessions_count": total_sessions,
+        },
+    }

devtorch_core/metrics/calibrate.py

@@ -0,0 +1,198 @@
+"""
+Calibration store and onboarding wizard for DevTorch metrics.
+
+CalibrationStore: persists team-specific calibration values to .GCC/metrics/calibration.json.
+run_calibrate_wizard: interactive (or non-interactive for tests) onboarding flow.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+from typing import Any, Optional, Union
+
+
+_CALIBRATION_SUBDIR = "metrics"
+_CALIBRATION_FILE = "calibration.json"
+
+
+class CalibrationStore:
+    """
+    Persists calibration values to .GCC/metrics/calibration.json.
+
+    This is distinct from devtorch_core/metrics/calibration.py's CalibrationStore,
+    which lives at .GCC/calibration.json and holds DHS/MCS weights.
+    This one lives at .GCC/metrics/calibration.json and holds onboarding baselines.
+    """
+
+    def __init__(self, gcc_dir: Path) -> None:
+        self._gcc_dir = Path(gcc_dir)
+        self.calibration_path = self._gcc_dir / _CALIBRATION_SUBDIR / _CALIBRATION_FILE
+
+    def load(self) -> dict:
+        """Read calibration.json; return {} if missing or corrupt."""
+        if not self.calibration_path.exists():
+            return {}
+        try:
+            text = self.calibration_path.read_text(encoding="utf-8")
+            data = json.loads(text)
+            if not isinstance(data, dict):
+                return {}
+            return data
+        except (json.JSONDecodeError, OSError):
+            return {}
+
+    def save(self, data: dict) -> None:
+        """Write data to .GCC/metrics/calibration.json, creating dirs if needed."""
+        self.calibration_path.parent.mkdir(parents=True, exist_ok=True)
+        self.calibration_path.write_text(
+            json.dumps(data, indent=2, sort_keys=True),
+            encoding="utf-8",
+        )
+
+    def set_value(self, key: str, value: Union[float, str]) -> None:
+        """Set a single calibration key and persist."""
+        data = self.load()
+        data[key] = value
+        self.save(data)
+
+    def get_value(self, key: str, default: Any = None) -> Any:
+        """Get a single calibration key, returning default if not set."""
+        data = self.load()
+        return data.get(key, default)
+
+
+def _estimate_cold_start_min(repo_root: Path) -> float:
+    """
+    Estimate cold_start_min from git history.
+
+    Counts commits in the last 30 days and computes the average inter-commit
+    interval as a rough proxy for context switch / cold-start time.
+    Falls back to 15.0 minutes if git is unavailable or the repo has no history.
+    """
+    default = 15.0
+    try:
+        result = subprocess.run(
+            ["git", "-C", str(repo_root), "log", "--format=%ct", "--since=30.days.ago"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        if result.returncode != 0:
+            return default
+        timestamps = [int(t.strip()) for t in result.stdout.splitlines() if t.strip()]
+        if len(timestamps) < 2:
+            return default
+        # timestamps are newest-first; sort ascending
+        timestamps.sort()
+        intervals_seconds = [
+            timestamps[i + 1] - timestamps[i]
+            for i in range(len(timestamps) - 1)
+        ]
+        avg_seconds = sum(intervals_seconds) / len(intervals_seconds)
+        avg_minutes = avg_seconds / 60.0
+        # Clamp to a sane range [1, 120]
+        return max(1.0, min(120.0, avg_minutes))
+    except Exception:
+        return default
+
+
+def run_calibrate_wizard(
+    gcc_dir: Path,
+    values: Optional[dict] = None,
+) -> dict:
+    """
+    Run the calibration onboarding wizard.
+
+    If *values* is provided, the wizard runs non-interactively using those values
+    (useful for tests and automation). Missing keys fall back to defaults.
+
+    If *values* is None, the wizard prompts the user interactively.
+
+    Returns the calibration dict that was saved.
+
+    Fields:
+      cold_start_min                            — estimated from git history (or prompted)
+      avg_rework_hours                          — default 2.0
+      token_cost_usd_per_million                — required, no default (model pricing varies 5×)
+      developer_hourly_rate                     — required, no default
+      auditor_hourly_rate                       — default 39.0
+      sprint_length_days                        — default 14
+      complex_problem_delivery_time_baseline_min — default 240.0 (4 hours). Team-calibrated input.
+    """
+    repo_root = gcc_dir.parent if gcc_dir.name == ".GCC" else gcc_dir
+
+    git_estimate = _estimate_cold_start_min(repo_root)
+
+    if values is not None:
+        # Non-interactive mode
+        token_cost_raw = values.get("token_cost_usd_per_million")
+        calibration = {
+            "cold_start_min": float(values.get("cold_start_min", git_estimate)),
+            "avg_rework_hours": float(values.get("avg_rework_hours", 2.0)),
+            "developer_hourly_rate": float(values["developer_hourly_rate"]),
+            "auditor_hourly_rate": float(values.get("auditor_hourly_rate", 39.0)),
+            "sprint_length_days": int(values.get("sprint_length_days", 14)),
+            # 240 min = 4 hours. This is a team-calibrated baseline, not a universal
+            # constant; teams should adjust it via the wizard or `calibrate --set`.
+            "complex_problem_delivery_time_baseline_min": float(values.get("complex_problem_delivery_time_baseline_min", 240.0)),
+        }
+        if token_cost_raw is not None:
+            calibration["token_cost_usd_per_million"] = float(token_cost_raw)
+    else:
+        # Interactive mode
+        print("DevTorch Metrics Calibration Wizard")
+        print("=====================================")
+        print("Press Enter to accept defaults shown in [brackets].\n")
+
+        def _prompt_float(prompt: str, default: Optional[float]) -> float:
+            default_str = f" [{default}]" if default is not None else ""
+            while True:
+                raw = input(f"{prompt}{default_str}: ").strip()
+                if not raw and default is not None:
+                    return default
+                try:
+                    return float(raw)
+                except ValueError:
+                    print("  Please enter a numeric value.")
+
+        def _prompt_int(prompt: str, default: int) -> int:
+            while True:
+                raw = input(f"{prompt} [{default}]: ").strip()
+                if not raw:
+                    return default
+                try:
+                    return int(raw)
+                except ValueError:
+                    print("  Please enter an integer value.")
+
+        cold_start_min = _prompt_float(
+            f"Cold-start time (minutes, estimated from git: {git_estimate:.1f})",
+            git_estimate,
+        )
+        avg_rework_hours = _prompt_float("Average rework hours per collision", 2.0)
+        complex_problem_delivery_time_baseline_min = _prompt_float(
+            "Complex problem delivery-time baseline (minutes, team-calibrated)", 240.0
+        )
+        token_cost_usd_per_million = _prompt_float(
+            "Token cost (USD per million tokens, required)", None
+        )
+        developer_hourly_rate = _prompt_float("Developer hourly rate (USD, required)", None)
+        auditor_hourly_rate = _prompt_float("Auditor hourly rate (USD)", 39.0)
+        sprint_length_days = _prompt_int("Sprint length (days)", 14)
+
+        calibration = {
+            "cold_start_min": cold_start_min,
+            "avg_rework_hours": avg_rework_hours,
+            "complex_problem_delivery_time_baseline_min": complex_problem_delivery_time_baseline_min,
+            "developer_hourly_rate": developer_hourly_rate,
+            "auditor_hourly_rate": auditor_hourly_rate,
+            "sprint_length_days": sprint_length_days,
+        }
+        if token_cost_usd_per_million is not None:
+            calibration["token_cost_usd_per_million"] = token_cost_usd_per_million
+
+    store = CalibrationStore(gcc_dir)
+    store.save(calibration)
+    return calibration

devtorch_core/metrics/calibration.py

@@ -0,0 +1,125 @@
+"""
+Metrics calibration store.
+
+Calibration values are team-specific. The DEFAULTS below are reasonable
+starting points but MUST be adjusted with real team data for meaningful
+numbers. Do not treat defaults as validated benchmarks.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, fields
+from pathlib import Path
+from typing import Any, Optional
+
+CALIBRATION_FILE_NAME = "calibration.json"
+
+
+@dataclass
+class CalibrationDefaults:
+    """
+    Default calibration values for DevTorch metrics.
+
+    Each field is annotated with its source citation or "design choice,
+    calibrate with team data" for values with no external benchmark.
+    """
+
+    # === Context switching cost ===
+    # Source: Parnin & Rugaber (2011) Software Quality Journal
+    # DOI: 10.1007/s11219-010-9104-9
+    # Measured: 15 min for programming tasks (not general work)
+    context_switch_minutes: float = 15.0
+
+    # === Developer hourly rate ===
+    # No universal default — teams must provide this.
+    # BLS SOC 15-1252 (2023): $55-65/hr median for software developers
+    # Set to 0 to force explicit calibration.
+    developer_hourly_rate: float = 0.0
+
+    # === Rework time per collision ===
+    # No peer-reviewed benchmark. Design choice. Calibrate with team data.
+    rework_minutes_per_collision: float = 30.0
+
+    # === MCS weights ===
+    # Design choices. No external benchmark. Calibrate with team data.
+    mcs_weight_frequency: float = 0.4
+    mcs_weight_severity: float = 0.4
+    mcs_weight_resolution_time: float = 0.2
+
+    # === DHS weights ===
+    # Design choices. No external benchmark. Calibrate with team data.
+    dhs_weight_commit_health: float = 0.3
+    dhs_weight_sensitivity_health: float = 0.25
+    dhs_weight_capability_health: float = 0.25
+    dhs_weight_variance_health: float = 0.2
+
+
+_DEFAULTS = CalibrationDefaults()
+_DEFAULTS_DICT: dict[str, Any] = {f.name: getattr(_DEFAULTS, f.name) for f in fields(_DEFAULTS)}
+
+
+class CalibrationStore:
+    """
+    Calibration value store backed by `.GCC/calibration.json`.
+
+    Overrides in the JSON file take precedence over CalibrationDefaults.
+    """
+
+    def __init__(self, gcc_root: str) -> None:
+        self._gcc_root = Path(gcc_root)
+        self._path = self._gcc_root / CALIBRATION_FILE_NAME
+        self._overrides: dict[str, Any] = self.load()
+
+    def load(self) -> dict:
+        """Read calibration JSON; return empty dict on missing or corrupt file."""
+        if not self._path.exists():
+            return {}
+        try:
+            text = self._path.read_text(encoding="utf-8")
+            data = json.loads(text)
+            if not isinstance(data, dict):
+                return {}
+            return data
+        except (json.JSONDecodeError, OSError):
+            return {}
+
+    def save(self, values: dict) -> None:
+        """Write the given values dict to .GCC/calibration.json."""
+        self._gcc_root.mkdir(parents=True, exist_ok=True)
+        self._path.write_text(
+            json.dumps(values, indent=2, sort_keys=True),
+            encoding="utf-8",
+        )
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Return the calibration value for key (overrides > defaults > default arg)."""
+        if key in self._overrides:
+            return self._overrides[key]
+        if key in _DEFAULTS_DICT:
+            return _DEFAULTS_DICT[key]
+        return default
+
+    def get_all(self) -> dict:
+        """Return the full merged dict: defaults overridden by any stored values."""
+        merged = dict(_DEFAULTS_DICT)
+        merged.update(self._overrides)
+        return merged
+
+    def set(self, key: str, value: Any) -> None:
+        """Set an override value and persist it to disk."""
+        self._overrides[key] = value
+        self.save(self._overrides)
+
+    def reset(self, key: str) -> None:
+        """Remove an override, falling back to the default value."""
+        if key in self._overrides:
+            del self._overrides[key]
+            self.save(self._overrides)
+
+
+def load_calibration(gcc_root: Optional[str] = None) -> CalibrationStore:
+    """Convenience factory. If gcc_root is None, uses '.GCC' in the current directory."""
+    if gcc_root is None:
+        gcc_root = ".GCC"
+    return CalibrationStore(gcc_root)