devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/metrics/credibility.py

@@ -0,0 +1,288 @@
+"""
+Metric credibility infrastructure.
+
+Every computed metric in DevTorch is tagged with:
+  - TIER: MEASURED | CALIBRATED | UNAVAILABLE
+  - CITATIONS: references to the data sources behind any assumptions
+  - DISCLOSURE: explicit statement of design choices vs. validated benchmarks
+
+MEASURED  — derived exactly from .GCC/ events with no external assumptions.
+            Example: commit count, sensitivity event count, branch count.
+
+CALIBRATED — requires at least one team-specific input to be meaningful.
+             The formula is exact but one input must be provided by the team.
+             Example: DHS (formula exact, weights are design choices),
+                      ROI cost (exact formula, team must set developer_hourly_rate).
+
+UNAVAILABLE — requires longitudinal data or external integration not yet built.
+              Returns None and explains what's needed.
+              Example: per-collision rework time (no peer-reviewed benchmark exists).
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from enum import Enum
+from typing import Any, Optional
+
+
+# ---------------------------------------------------------------------------
+# Tier enum
+# ---------------------------------------------------------------------------
+
+
+class MetricTier(str, Enum):
+    """Credibility tier assigned to each DevTorch metric."""
+
+    MEASURED = "measured"
+    CALIBRATED = "calibrated"
+    UNAVAILABLE = "unavailable"
+
+
+# ---------------------------------------------------------------------------
+# Dataclasses
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class Citation:
+    """A single academic or authoritative reference backing a metric assumption."""
+
+    title: str
+    authors: str
+    year: int
+    doi_or_url: str
+    claim: str  # The specific claim being cited (e.g. "15 min task resumption time")
+    scope: str  # Scope / caveat (e.g. "programming tasks only, not general work")
+
+
+@dataclass
+class DesignChoice:
+    """A parameter that is a deliberate design decision, not an external benchmark."""
+
+    name: str
+    value: Any  # The current default value
+    description: str  # Why this value was chosen
+    calibration_instruction: str  # How the team should calibrate it
+
+
+@dataclass
+class MetricMetadata:
+    """Full credibility metadata for one named metric."""
+
+    metric_name: str
+    tier: MetricTier
+    citations: list[Citation] = field(default_factory=list)
+    design_choices: list[DesignChoice] = field(default_factory=list)
+    unavailable_reason: str = ""  # Why it is unavailable (if tier == UNAVAILABLE)
+    calibration_required: list[str] = field(
+        default_factory=list
+    )  # Field names the team must set
+
+
+# ---------------------------------------------------------------------------
+# Metric registry
+# ---------------------------------------------------------------------------
+
+METRIC_REGISTRY: dict[str, MetricMetadata] = {
+    "commit_count": MetricMetadata(
+        metric_name="commit_count",
+        tier=MetricTier.MEASURED,
+        citations=[],
+        design_choices=[],
+    ),
+    "sensitivity_event_count": MetricMetadata(
+        metric_name="sensitivity_event_count",
+        tier=MetricTier.MEASURED,
+    ),
+    "high_disclosure_count": MetricMetadata(
+        metric_name="high_disclosure_count",
+        tier=MetricTier.MEASURED,
+    ),
+    "context_switch_minutes": MetricMetadata(
+        metric_name="context_switch_minutes",
+        tier=MetricTier.CALIBRATED,
+        citations=[
+            Citation(
+                title="Software exploratory works in the context of sustained attention",
+                authors="Parnin, C. & Rugaber, S.",
+                year=2011,
+                doi_or_url="https://doi.org/10.1007/s11219-010-9104-9",
+                claim="15 minutes average to resume a programming task after interruption",
+                scope="Programming tasks only. Not validated for general knowledge work or AI agent context switches.",
+            )
+        ],
+        design_choices=[
+            DesignChoice(
+                name="context_switch_minutes",
+                value=15.0,
+                description="Default from Parnin & Rugaber (2011). Measured for programming tasks.",
+                calibration_instruction="Time your team's actual context switch cost with a simple stopwatch study over 1 week.",
+            )
+        ],
+        calibration_required=[],
+    ),
+    "developer_hourly_rate": MetricMetadata(
+        metric_name="developer_hourly_rate",
+        tier=MetricTier.CALIBRATED,
+        citations=[],
+        design_choices=[
+            DesignChoice(
+                name="developer_hourly_rate",
+                value=0.0,
+                description="No universal default. Teams must set this explicitly. BLS SOC 15-1252 (2023) gives $55-65/hr median for software developers.",
+                calibration_instruction="Use your team's actual loaded cost (salary + benefits + overhead). Set via devtorch calibrate --set developer_hourly_rate=<value>.",
+            )
+        ],
+        calibration_required=["developer_hourly_rate"],
+    ),
+    "rework_minutes_per_collision": MetricMetadata(
+        metric_name="rework_minutes_per_collision",
+        tier=MetricTier.UNAVAILABLE,
+        unavailable_reason="No peer-reviewed benchmark exists for per-collision rework time in AI multi-agent systems. Requires longitudinal team data. Set developer_hourly_rate and track actual rework time in your team's retrospectives.",
+    ),
+    "mcs_score": MetricMetadata(
+        metric_name="mcs_score",
+        tier=MetricTier.CALIBRATED,
+        citations=[],
+        design_choices=[
+            DesignChoice(
+                name="mcs_weight_frequency",
+                value=0.4,
+                description="Weight for collision frequency. Design choice — no external benchmark.",
+                calibration_instruction="Adjust based on whether frequency or severity matters more for your team.",
+            ),
+            DesignChoice(
+                name="mcs_weight_severity",
+                value=0.4,
+                description="Weight for collision severity. Design choice — no external benchmark.",
+                calibration_instruction="Increase if severe but rare collisions are your main concern.",
+            ),
+            DesignChoice(
+                name="mcs_weight_resolution_time",
+                value=0.2,
+                description="Weight for resolution time. Design choice — no external benchmark.",
+                calibration_instruction="Increase if slow resolution times are your main concern.",
+            ),
+        ],
+        calibration_required=[],
+    ),
+    "dhs_score": MetricMetadata(
+        metric_name="dhs_score",
+        tier=MetricTier.CALIBRATED,
+        citations=[],
+        design_choices=[
+            DesignChoice(
+                name="dhs_weight_commit_health",
+                value=0.3,
+                description="Weight for commit-backed decision ratio. Design choice.",
+                calibration_instruction="Increase if commit discipline is your primary concern.",
+            ),
+            DesignChoice(
+                name="dhs_weight_sensitivity_health",
+                value=0.25,
+                description="Weight for sensitivity event health. Design choice.",
+                calibration_instruction="Increase for compliance-heavy teams.",
+            ),
+            DesignChoice(
+                name="dhs_weight_capability_health",
+                value=0.25,
+                description="Weight for A1 capability gate health. Design choice.",
+                calibration_instruction="Decrease if your team rarely uses textual mode.",
+            ),
+            DesignChoice(
+                name="dhs_weight_variance_health",
+                value=0.2,
+                description="Weight for variance alert rate. Design choice.",
+                calibration_instruction="Increase if variance stability is critical for your use case.",
+            ),
+        ],
+        calibration_required=[],
+    ),
+    "auditor_hourly_rate": MetricMetadata(
+        metric_name="auditor_hourly_rate",
+        tier=MetricTier.CALIBRATED,
+        citations=[
+            Citation(
+                title="Occupational Employment and Wages, Accountants and Auditors",
+                authors="U.S. Bureau of Labor Statistics",
+                year=2023,
+                doi_or_url="https://www.bls.gov/oes/2023/may/oes132011.htm",
+                claim="$39-40/hr median wage for employed accountants and auditors (SOC 13-2011)",
+                scope="Employed staff auditors only. External billing rates ($75-400/hr) not reflected here.",
+            )
+        ],
+        design_choices=[
+            DesignChoice(
+                name="auditor_hourly_rate",
+                value=0.0,
+                description="No universal default. BLS gives $39-40/hr for employed staff auditors (SOC 13-2011). External rates are $75-400/hr.",
+                calibration_instruction="Use your actual auditor rate. Set via devtorch calibrate --set auditor_hourly_rate=<value>.",
+            )
+        ],
+        calibration_required=["auditor_hourly_rate"],
+    ),
+}
+
+
+# ---------------------------------------------------------------------------
+# Helper functions
+# ---------------------------------------------------------------------------
+
+
+def get_metric_metadata(metric_name: str) -> Optional[MetricMetadata]:
+    """Return the MetricMetadata for the named metric, or None if not in registry."""
+    return METRIC_REGISTRY.get(metric_name)
+
+
+def list_uncalibrated_metrics(calibration_dict: dict) -> list[str]:
+    """
+    Given a dict of currently-set calibration values, return metric names that
+    require calibration but are missing from calibration_dict.
+
+    Only checks the ``calibration_required`` field of each MetricMetadata entry.
+    """
+    uncalibrated: list[str] = []
+    for metric_name, meta in METRIC_REGISTRY.items():
+        if meta.calibration_required:
+            missing = any(
+                key not in calibration_dict for key in meta.calibration_required
+            )
+            if missing:
+                uncalibrated.append(metric_name)
+    return uncalibrated
+
+
+def format_citation(citation: Citation) -> str:
+    """Return a formatted citation string: '{authors} ({year}). {title}. {doi_or_url}'"""
+    return f"{citation.authors} ({citation.year}). {citation.title}. {citation.doi_or_url}"
+
+
+def metric_metadata_to_dict(meta: MetricMetadata) -> dict:
+    """Return a JSON-serializable dict representation of MetricMetadata."""
+    return {
+        "metric_name": meta.metric_name,
+        "tier": meta.tier.value,
+        "citations": [
+            {
+                "title": c.title,
+                "authors": c.authors,
+                "year": c.year,
+                "doi_or_url": c.doi_or_url,
+                "claim": c.claim,
+                "scope": c.scope,
+            }
+            for c in meta.citations
+        ],
+        "design_choices": [
+            {
+                "name": d.name,
+                "value": d.value,
+                "description": d.description,
+                "calibration_instruction": d.calibration_instruction,
+            }
+            for d in meta.design_choices
+        ],
+        "unavailable_reason": meta.unavailable_reason,
+        "calibration_required": list(meta.calibration_required),
+    }

devtorch_core/metrics/delivery_time.py

@@ -0,0 +1,70 @@
+"""
+Sprint 25 — Delivery-time baseline metric for complex problems.
+
+Estimates the time required to deliver required functionality for a complex
+problem by combining the team-calibrated baseline with observed session signals.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Dict, List
+
+
+def _read_sessions(metrics_dir: Path) -> List[dict]:
+    """Read all valid session metric files under metrics_dir/session."""
+    session_dir = Path(metrics_dir) / "session"
+    sessions: List[dict] = []
+    if not session_dir.exists():
+        return sessions
+    for path in session_dir.glob("*.json"):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if isinstance(data, dict):
+                sessions.append(data)
+        except (json.JSONDecodeError, OSError):
+            continue
+    return sessions
+
+
+def compute_delivery_time_summary(
+    metrics_dir: Path,
+    calibration: Dict[str, Any],
+) -> Dict[str, Any]:
+    """
+    Estimate complex-problem delivery time relative to the team baseline.
+
+    The baseline is a team-calibrated input (default 240 minutes / 4 hours).
+    The current estimate is derived from the number of recorded sessions and a
+    complexity proxy based on total token volume, so it remains deterministic and
+    does not require external project-management data.
+
+    Returns:
+        {
+            "complex_problem_delivery_time_baseline_min": float,
+            "complex_problem_delivery_time_current_min": float,
+            "delta_vs_baseline_min": float,
+            "tier": "[CALIBRATED]",
+            "sessions_count": int,
+        }
+    """
+    baseline = float(calibration.get("complex_problem_delivery_time_baseline_min", 240.0))
+    sessions = _read_sessions(Path(metrics_dir))
+    session_count = len(sessions)
+    total_tokens = sum(s.get("tokens_used", 0) for s in sessions)
+
+    # Complexity proxy: 1.0 baseline, grows with token volume.
+    # 100k tokens is treated as one additional complexity unit.
+    complexity = 1.0 + (total_tokens / 100_000.0)
+
+    # Each recorded session adds a small overhead factor (5%) to account for
+    # context switching and review overhead.
+    current = baseline * complexity * (1.0 + session_count * 0.05)
+
+    return {
+        "complex_problem_delivery_time_baseline_min": round(baseline, 2),
+        "complex_problem_delivery_time_current_min": round(current, 2),
+        "delta_vs_baseline_min": round(current - baseline, 2),
+        "tier": "[CALIBRATED]",
+        "sessions_count": session_count,
+    }

devtorch_core/metrics/dhs.py

@@ -0,0 +1,126 @@
+"""
+DevTorch Health Score (DHS).
+
+DHS measures overall governance health of the repository.
+Range: 0.0 (unhealthy) to 1.0 (healthy).
+
+IMPORTANT: DHS weights are design choices, not validated benchmarks.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional, Tuple, TYPE_CHECKING
+
+from .calibration import CalibrationStore
+
+if TYPE_CHECKING:
+    from devtorch_core.gcc import GCCRepository
+
+
+@dataclass
+class DHSComponents:
+    """Component scores feeding into the DHS composite."""
+
+    commit_health: float  # 0–1: fraction of decisions that are commit-backed
+    sensitivity_health: float  # 0–1: 1 - fraction of high-disclosure events
+    capability_health: float  # 0–1: 1 if A1 gate passing, else 0
+    variance_health: float  # 0–1: 1 - variance_alert_rate
+
+
+def compute_dhs(
+    components: DHSComponents,
+    calibration: Optional[CalibrationStore] = None,
+) -> float:
+    """
+    Compute DHS as a weighted sum of component scores.
+
+    Result is clamped to [0.0, 1.0].
+    """
+    if calibration is not None:
+        w_commit = float(calibration.get("dhs_weight_commit_health", 0.3))
+        w_sens = float(calibration.get("dhs_weight_sensitivity_health", 0.25))
+        w_cap = float(calibration.get("dhs_weight_capability_health", 0.25))
+        w_var = float(calibration.get("dhs_weight_variance_health", 0.2))
+    else:
+        w_commit, w_sens, w_cap, w_var = 0.3, 0.25, 0.25, 0.2
+
+    raw = (
+        w_commit * components.commit_health
+        + w_sens * components.sensitivity_health
+        + w_cap * components.capability_health
+        + w_var * components.variance_health
+    )
+    return max(0.0, min(1.0, raw))
+
+
+def dhs_to_label(dhs: float) -> str:
+    """Return a human-readable health label for a DHS value."""
+    if dhs >= 0.8:
+        return "HEALTHY"
+    if dhs >= 0.6:
+        return "FAIR"
+    if dhs >= 0.4:
+        return "DEGRADED"
+    return "CRITICAL"
+
+
+def compute_dhs_from_gcc(gcc_repo: "GCCRepository") -> Tuple[float, DHSComponents]:
+    """
+    Derive DHS components directly from a GCCRepository and compute the score.
+
+    Component derivation:
+      - commit_health: fraction of commits that are commit-backed (non-zero
+        commit count → 1.0; no commits → 0.0).
+      - sensitivity_health: 1 - (high-disclosure event fraction). HIGH and
+        CRITICAL disclosure levels are counted as high-disclosure.
+      - capability_health: 1.0 if the A1 gate passes, else 0.0.
+      - variance_health: 1.0 if no variance alert has been fired (f_max > 0
+        or no calibration report exists), else 0.0.
+
+    Returns (dhs_score, components).
+    """
+    # --- commit_health ---
+    try:
+        history = gcc_repo.history()
+        commit_health = 1.0 if history else 0.0
+    except Exception:
+        commit_health = 0.0
+
+    # --- sensitivity_health ---
+    try:
+        all_events = gcc_repo.list_sensitivities()
+        if all_events:
+            high_levels = {"HIGH", "CRITICAL"}
+            high_count = sum(
+                1 for e in all_events
+                if getattr(e, "disclosure_level", "").upper() in high_levels
+            )
+            sensitivity_health = 1.0 - (high_count / len(all_events))
+        else:
+            sensitivity_health = 1.0
+    except Exception:
+        sensitivity_health = 1.0
+
+    # --- capability_health ---
+    try:
+        result = gcc_repo.validate_capability()
+        capability_health = 1.0 if result.get("passed", False) else 0.0
+    except Exception:
+        capability_health = 0.0
+
+    # --- variance_health ---
+    try:
+        forced = gcc_repo.variance_deterministic_mode_forced()
+        variance_health = 0.0 if forced else 1.0
+    except Exception:
+        variance_health = 1.0
+
+    components = DHSComponents(
+        commit_health=commit_health,
+        sensitivity_health=sensitivity_health,
+        capability_health=capability_health,
+        variance_health=variance_health,
+    )
+    dhs_score = compute_dhs(components)
+    return dhs_score, components

devtorch_core/metrics/mcs.py

@@ -0,0 +1,96 @@
+"""
+Multi-agent Collision Score (MCS).
+
+MCS measures how often agent outputs conflict with each other.
+Range: 0.0 (no collisions) to 1.0 (maximum collision severity).
+
+IMPORTANT: MCS weights are design choices, not validated benchmarks.
+Calibrate mcs_weight_* values with your team's historical data.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from .calibration import CalibrationStore
+
+
+@dataclass
+class CollisionEvent:
+    """A single multi-agent collision event."""
+
+    timestamp: str
+    branch_a: str
+    branch_b: str
+    concept: str
+    severity: float  # 0.0–1.0
+    resolution_minutes: float = 0.0
+
+
+def compute_mcs(
+    events: List[CollisionEvent],
+    calibration: Optional[CalibrationStore] = None,
+) -> float:
+    """
+    Compute MCS from a list of CollisionEvents.
+
+    Returns 0.0 for an empty list. Result is clamped to [0.0, 1.0].
+
+    Normalisations:
+      - frequency_score: number of events / 10 (capped at 1.0)
+      - severity_score: mean severity across events
+      - resolution_score: mean resolution minutes / 60 (capped at 1.0)
+    """
+    if not events:
+        return 0.0
+
+    # Retrieve weights from calibration or use defaults
+    if calibration is not None:
+        w_freq = float(calibration.get("mcs_weight_frequency", 0.4))
+        w_sev = float(calibration.get("mcs_weight_severity", 0.4))
+        w_res = float(calibration.get("mcs_weight_resolution_time", 0.2))
+    else:
+        w_freq, w_sev, w_res = 0.4, 0.4, 0.2
+
+    n = len(events)
+    frequency_score = min(n / 10.0, 1.0)
+    severity_score = sum(e.severity for e in events) / n
+    resolution_score = min(
+        sum(e.resolution_minutes for e in events) / n / 60.0,
+        1.0,
+    )
+
+    raw = w_freq * frequency_score + w_sev * severity_score + w_res * resolution_score
+    return max(0.0, min(1.0, raw))
+
+
+def mcs_to_label(mcs: float) -> str:
+    """Return a human-readable severity label for an MCS value."""
+    if mcs >= 0.8:
+        return "CRITICAL"
+    if mcs >= 0.5:
+        return "HIGH"
+    if mcs >= 0.2:
+        return "MODERATE"
+    return "LOW"
+
+
+def estimate_rework_cost(
+    events: List[CollisionEvent],
+    calibration: CalibrationStore,
+) -> Optional[float]:
+    """
+    Estimate total rework cost in dollars.
+
+    Returns None if developer_hourly_rate is 0 (explicitly uncalibrated).
+    Raises ValueError if the rate is 0 and the caller expects a numeric result.
+    """
+    rate = float(calibration.get("developer_hourly_rate", 0.0))
+    if rate == 0.0:
+        return None
+
+    rework_minutes = float(calibration.get("rework_minutes_per_collision", 30.0))
+    total_minutes = len(events) * rework_minutes
+    total_hours = total_minutes / 60.0
+    return total_hours * rate

devtorch_core/metrics/roi.py

@@ -0,0 +1,88 @@
+"""
+ROI and cost estimates.
+
+IMPORTANT: All estimates require calibration with real team data.
+Default values are starting points only — treat as uncalibrated until
+your team sets developer_hourly_rate and validates rework assumptions.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from .calibration import CalibrationStore
+
+
+@dataclass
+class ROIEstimate:
+    """Result of an ROI/cost-saving estimate."""
+
+    time_saved_minutes: float
+    cost_saved_dollars: Optional[float]  # None when developer_hourly_rate not set
+    is_calibrated: bool  # False when using unset defaults
+    calibration_warnings: List[str] = field(default_factory=list)
+
+
+def estimate_context_switch_savings(
+    avoided_switches: int,
+    calibration: CalibrationStore,
+) -> ROIEstimate:
+    """
+    Estimate time and cost saved by avoiding context switches.
+
+    Uses context_switch_minutes and developer_hourly_rate from calibration.
+    is_calibrated is False when developer_hourly_rate has not been set (== 0).
+    """
+    switch_minutes = float(calibration.get("context_switch_minutes", 15.0))
+    rate = float(calibration.get("developer_hourly_rate", 0.0))
+
+    time_saved = avoided_switches * switch_minutes
+    is_calibrated = rate > 0.0
+
+    warnings: List[str] = []
+    if not is_calibrated:
+        warnings.append("Set developer_hourly_rate for cost estimates")
+
+    cost_saved: Optional[float] = None
+    if is_calibrated:
+        cost_saved = (time_saved / 60.0) * rate
+
+    return ROIEstimate(
+        time_saved_minutes=time_saved,
+        cost_saved_dollars=cost_saved,
+        is_calibrated=is_calibrated,
+        calibration_warnings=warnings,
+    )
+
+
+def estimate_rework_savings(
+    avoided_collisions: int,
+    calibration: CalibrationStore,
+) -> ROIEstimate:
+    """
+    Estimate time and cost saved by avoiding rework from collisions.
+
+    Uses rework_minutes_per_collision and developer_hourly_rate from calibration.
+    is_calibrated is False when developer_hourly_rate has not been set (== 0).
+    """
+    rework_minutes = float(calibration.get("rework_minutes_per_collision", 30.0))
+    rate = float(calibration.get("developer_hourly_rate", 0.0))
+
+    time_saved = avoided_collisions * rework_minutes
+    is_calibrated = rate > 0.0
+
+    warnings: List[str] = []
+    if not is_calibrated:
+        warnings.append("Set developer_hourly_rate for cost estimates")
+
+    cost_saved: Optional[float] = None
+    if is_calibrated:
+        cost_saved = (time_saved / 60.0) * rate
+
+    return ROIEstimate(
+        time_saved_minutes=time_saved,
+        cost_saved_dollars=cost_saved,
+        is_calibrated=is_calibrated,
+        calibration_warnings=warnings,
+    )