psystack 0.1.0__py3-none-any.whl

psystack/evaluation/metrics/offtrack.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+class OffTrackRateMetric:
+    """Fraction of steps spent off-track."""
+
+    def metric_id(self) -> str:
+        return "offtrack_rate"
+
+    def higher_is_better(self) -> bool:
+        return False
+
+    def compute(self, episodes: list[dict[str, Any]]) -> dict[str, Any]:
+        per_episode = []
+        for ep in episodes:
+            steps = ep["steps"]
+            if not steps:
+                per_episode.append(0.0)
+                continue
+            off = sum(1 for s in steps if not s["info"].get("on_track", True))
+            per_episode.append(off / len(steps))
+
+        return {
+            "primary_value": sum(per_episode) / len(per_episode) if per_episode else 0.0,
+            "unit": "fraction",
+            "per_episode": per_episode,
+            "breakdown": {},
+        }

psystack/evaluation/metrics/prediction_error.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import torch
+
+
+class WorldModelPredictionError:
+    """MSE between predicted z_{t+1} and encode_target(obs_{t+1}).
+
+    This metric re-runs episodes to capture rasters (not stored in episode data).
+    It uses a scripted straight-ahead policy for determinism.
+    """
+
+    def metric_id(self) -> str:
+        return "prediction_error"
+
+    def higher_is_better(self) -> bool:
+        return False
+
+    def compute(self, episodes: list[dict[str, Any]]) -> dict[str, Any]:
+        # prediction_error requires world_model and env passed via episode metadata
+        # If not available, return NaN
+        if not episodes or "_world_model" not in episodes[0]:
+            return {
+                "primary_value": float("nan"),
+                "unit": "mse",
+                "per_episode": [],
+                "breakdown": {"note": "world_model not available for prediction_error"},
+            }
+
+        world_model = episodes[0]["_world_model"]
+        env = episodes[0]["_env"]
+        seed = episodes[0].get("_seed", 42)
+        num_episodes = len(episodes)
+        max_eval_steps = 200  # limit steps for prediction error eval
+
+        per_episode = []
+        for ep_idx in range(num_episodes):
+            np.random.seed(seed + ep_idx)
+            torch.manual_seed(seed + ep_idx)
+
+            obs = env.reset(seed=seed + ep_idx)
+            errors = []
+
+            # Scripted policy: gentle throttle, no steering
+            action = np.array([0.0, 0.3, 0.0], dtype=np.float32)
+
+            for _ in range(max_eval_steps):
+                z_t = world_model.encode(obs)
+                z_pred = world_model.predict(z_t, action)
+
+                obs_next, _, done, _ = env.step(action)
+                z_target = world_model.encode_target(obs_next)
+
+                mse = torch.mean((z_pred - z_target) ** 2).item()
+                errors.append(mse)
+
+                obs = obs_next
+                if done:
+                    break
+
+            per_episode.append(sum(errors) / len(errors) if errors else 0.0)
+
+        return {
+            "primary_value": sum(per_episode) / len(per_episode) if per_episode else 0.0,
+            "unit": "mse",
+            "per_episode": per_episode,
+            "breakdown": {},
+        }

psystack/evaluation/metrics/progress.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+class ProgressMetric:
+    """Track progress at episode end (from env.get_progress())."""
+
+    def metric_id(self) -> str:
+        return "progress"
+
+    def higher_is_better(self) -> bool:
+        return True
+
+    def compute(self, episodes: list[dict[str, Any]]) -> dict[str, Any]:
+        per_episode = [ep["final_track_progress"] for ep in episodes]
+        return {
+            "primary_value": sum(per_episode) / len(per_episode) if per_episode else 0.0,
+            "unit": "fraction",
+            "per_episode": per_episode,
+            "breakdown": {},
+        }

psystack/evaluation/metrics/reward.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+class CumulativeRewardMetric:
+    """Sum of rewards over the episode."""
+
+    def metric_id(self) -> str:
+        return "cumulative_reward"
+
+    def higher_is_better(self) -> bool:
+        return True
+
+    def compute(self, episodes: list[dict[str, Any]]) -> dict[str, Any]:
+        per_episode = [ep["total_reward"] for ep in episodes]
+        return {
+            "primary_value": sum(per_episode) / len(per_episode) if per_episode else 0.0,
+            "unit": "reward",
+            "per_episode": per_episode,
+            "breakdown": {},
+        }

psystack/evaluation/metrics/survival.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+class SurvivalStepsMetric:
+    """Number of steps before episode termination."""
+
+    def metric_id(self) -> str:
+        return "survival_steps"
+
+    def higher_is_better(self) -> bool:
+        return True
+
+    def compute(self, episodes: list[dict[str, Any]]) -> dict[str, Any]:
+        per_episode = [float(ep["total_steps"]) for ep in episodes]
+        return {
+            "primary_value": sum(per_episode) / len(per_episode) if per_episode else 0.0,
+            "unit": "steps",
+            "per_episode": per_episode,
+            "breakdown": {},
+        }

psystack/models/__init__.py

@@ -0,0 +1,42 @@
+from .case import Case
+from .comparison import ComparisonReport, MetricComparison
+from .event import Event, EventType
+from .evidence import EvidencePack
+from .explanation import Explanation
+from .isolation import (
+    AttributionTable,
+    EffectEstimate,
+    IsolationCase,
+    IsolationPlan,
+    IsolationResultBundle,
+)
+from .manifest import RunManifest
+from .metric import MetricResult
+from .project import DiscoveredAssets, Project
+from .run import Run
+from .signal import SignalValue
+from .swap import SwapFactors, SwapTestResult, SwapTestSpec
+
+__all__ = [
+    "AttributionTable",
+    "Case",
+    "ComparisonReport",
+    "DiscoveredAssets",
+    "EffectEstimate",
+    "Event",
+    "EventType",
+    "EvidencePack",
+    "Explanation",
+    "IsolationCase",
+    "IsolationPlan",
+    "IsolationResultBundle",
+    "MetricComparison",
+    "MetricResult",
+    "Project",
+    "Run",
+    "RunManifest",
+    "SignalValue",
+    "SwapFactors",
+    "SwapTestSpec",
+    "SwapTestResult",
+]

psystack/models/case.py

@@ -0,0 +1,30 @@
+"""Case model — a complete comparison investigation."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from psystack.models.event import Event
+from psystack.models.explanation import Explanation
+from psystack.models.run import Run
+
+
+class Case(BaseModel):
+    """A fully specified comparison case between two runs."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    id: str
+    schema_version: int = 1
+    project_id: str = ""
+    track_ref: str = ""
+    episode_count: int = 5
+    eval_seeds: list[int] | None = None
+    alignment_method: str = "progress"
+    shared_env_overrides: dict[str, Any] = Field(default_factory=dict)
+    run_a: Run
+    run_b: Run | None = None
+    events: list[Event] = Field(default_factory=list)
+    explanations: list[Explanation] = Field(default_factory=list)

psystack/models/comparison.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+ComparisonStatus = Literal["regression", "improvement", "no_change"]
+
+
+class MetricComparison(BaseModel):
+    metric_id: str
+    baseline_value: float
+    candidate_value: float
+    delta: float
+    delta_badness: float
+    higher_is_better: bool
+    baseline_per_episode: list[float] = Field(default_factory=list)
+    candidate_per_episode: list[float] = Field(default_factory=list)
+    ci_low: float | None = None
+    ci_high: float | None = None
+    p_value: float | None = None
+    p_value_adj: float | None = None
+    significant: bool = False
+    status: ComparisonStatus
+
+
+class ComparisonReport(BaseModel):
+    baseline_run_dir: str
+    candidate_run_dir: str
+    metrics: list[MetricComparison]

psystack/models/episode.py

@@ -0,0 +1,82 @@
+"""Episode outcome models for Phase 2 — run-level outcomes and verdict summary."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+# ── Display name mapping ─────────────────────────────────────────────────────
+
+METRIC_DISPLAY_NAMES: dict[str, str] = {
+    "final_track_progress": "completion",
+    "off_track_rate": "off-track rate",
+    "total_reward": "total reward",
+    "fastest_lap_time": "fastest lap",
+}
+
+
+# ── EpisodeRecord ────────────────────────────────────────────────────────────
+
+class EpisodeRecord(BaseModel):
+    """Per-episode outcome record from a single run side (a or b).
+
+    Fields map directly to values in runs/{side}/episodes.json.
+    """
+
+    model_config = ConfigDict(extra="ignore")
+
+    episode_idx: int
+    final_track_progress: float = 0.0
+    total_reward: float = 0.0
+    termination: str | None = None
+    fastest_lap_time: float | None = None
+    lap_count: int = 0
+    completed: bool = False
+
+
+# ── EpisodeOutcome ───────────────────────────────────────────────────────────
+
+class EpisodeOutcome(BaseModel):
+    """Aggregate outcome for one side (a or b) across all episodes."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    side: str
+    episodes: list[EpisodeRecord]
+    mean_progress: float
+    completion_rate: float
+    mean_reward: float
+    off_track_rate: float
+    fastest_lap: float | None
+
+
+# ── OutcomeSummary ───────────────────────────────────────────────────────────
+
+class OutcomeSummary(BaseModel):
+    """Persisted summary of A vs B comparison outcomes.
+
+    Written to analysis/outcomes.json after run_analysis() completes.
+    All display strings are pre-computed so the TUI only loads and renders.
+    """
+
+    model_config = ConfigDict(extra="ignore")
+
+    verdict: Literal["regression", "improvement", "no_change", "mixed"]
+    primary_metric: str
+    primary_metric_display: str
+    baseline_value: float
+    candidate_value: float
+    delta_pct: float
+    significant: bool
+    regression_count: int
+    improvement_count: int
+    no_change_count: int
+    verdict_headline: str
+    primary_metric_line: str
+    findings_count_line: str
+    top_run: dict[str, str] | None = None  # e.g., {"side": "b", "episode_id": "ep_0002"}
+    recommended_run_ids: list[str] = Field(default_factory=list)
+    episodes_a: list[EpisodeRecord] = Field(default_factory=list)
+    episodes_b: list[EpisodeRecord] = Field(default_factory=list)

psystack/models/evaluation_result.py

@@ -0,0 +1,51 @@
+"""EvaluationResult -- typed immutable snapshot of a completed evaluation."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ConfigSnapshot(BaseModel):
+    """Frozen copy of the case definition and adapter info at time of evaluation."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    case_id: str
+    track_ref: str = ""
+    episode_count: int = 0
+    eval_seeds: list[int] | None = None
+    run_a_world_model_ref: str = ""
+    run_b_world_model_ref: str = ""
+    run_a_planner_ref: str = ""
+    run_b_planner_ref: str = ""
+    adapter_name: str = ""
+
+
+class EvaluationResult(BaseModel):
+    """Immutable snapshot of a completed evaluation.
+
+    Saved to analysis/result.json after run_analysis() completes.
+    Bundles episode records, outcome summary, events data, and a
+    config snapshot of the case definition at time of evaluation.
+    """
+
+    model_config = ConfigDict(extra="ignore")
+
+    schema_version: int = 1
+    created_at: str = Field(default_factory=lambda: datetime.now(tz=timezone.utc).isoformat())
+
+    # Config snapshot -- frozen copy of case definition at eval time
+    config: ConfigSnapshot
+
+    # Episode data per side
+    episodes_a: list[dict[str, Any]] = Field(default_factory=list)
+    episodes_b: list[dict[str, Any]] = Field(default_factory=list)
+
+    # Outcome summary (the verdict + metrics)
+    outcomes: dict[str, Any] = Field(default_factory=dict)
+
+    # Events data (optional -- event extraction can fail)
+    events: dict[str, Any] | None = None

psystack/models/event.py

@@ -0,0 +1,40 @@
+"""Event model — a detected divergence point between two runs."""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+from psystack.models.signal import SignalValue
+
+EventType = Literal[
+    "first_signal_divergence",
+    "first_action_divergence",
+    "first_risk_spike",
+    "first_boundary_collapse",
+    "terminal",
+    "max_metric_gap",
+    # Phase 3 event types
+    "first_divergence",
+    "divergence_window",
+    "risk_spike",
+    "off_track_terminal",
+    "max_gap",
+]
+
+
+class Event(BaseModel):
+    """A detected divergence event at a specific step."""
+
+    id: str
+    type: EventType
+    step: int
+    time_s: float | None = None
+    severity: Literal["info", "warning", "critical"] = "warning"
+    score: float = 0.0
+    persistence_k: int = 1
+    active_signals: list[SignalValue] = Field(default_factory=list)
+    local_window: tuple[int, int] | None = None
+    evidence_refs: list[str] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)

psystack/models/evidence.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from .comparison import ComparisonReport
+from .isolation import AttributionTable, IsolationResultBundle
+from .manifest import RunManifest
+
+
+class EvidencePack(BaseModel):
+    pack_id: str
+    created_at: str
+    summary: str
+    baseline_manifest: RunManifest
+    candidate_manifest: RunManifest
+    compare: ComparisonReport
+    isolation: IsolationResultBundle | None = None
+    attributions: list[AttributionTable] = Field(default_factory=list)

psystack/models/explanation.py

@@ -0,0 +1,23 @@
+"""Explanation model — a tier-aware attribution hypothesis for an event."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from psystack.models.signal import SignalValue
+
+
+class Explanation(BaseModel):
+    """An evidence-backed explanation for a detected event."""
+
+    id: str
+    event_id: str
+    label: str
+    confidence: float = 0.0
+    tier: Literal["tier_0", "tier_1", "tier_2", "tier_3"] = "tier_0"
+    support_basis: list[str] = Field(default_factory=list)
+    competing: list[str] = Field(default_factory=list)
+    supporting_signals: list[SignalValue] = Field(default_factory=list)
+    falsifiers: list[str] = Field(default_factory=list)

psystack/models/isolation.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from psystack.models.swap import SwapFactors, SwapTestResult
+
+
+class IsolationCase(BaseModel):
+    test_id: str
+    factors: SwapFactors
+
+
+class IsolationPlan(BaseModel):
+    design: str = "screening_v1"
+    cases: list[IsolationCase] = Field(default_factory=list)
+
+
+class IsolationResultBundle(BaseModel):
+    design: str
+    cases: list[IsolationCase] = Field(default_factory=list)
+    swap_results: list[SwapTestResult] = Field(default_factory=list)
+
+
+class EffectEstimate(BaseModel):
+    factor: str
+    effect: float
+    confidence: float | None = None
+    support_tests: list[str] = Field(default_factory=list)
+
+
+class AttributionTable(BaseModel):
+    metric_id: str
+    main_effects: list[EffectEstimate] = Field(default_factory=list)
+    interaction_effects: list[EffectEstimate] = Field(default_factory=list)
+    decision: str

psystack/models/manifest.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel
+
+if TYPE_CHECKING:
+    from psystack.models.run import Run
+
+
+class RunManifest(BaseModel):
+    run_id: str
+    description: str = ""
+    world_model_weights: str
+    planner_config: dict[str, Any]
+    env_config: dict[str, Any]
+    num_episodes: int = 20
+    seed: int = 42
+
+    def to_run(self) -> Run:
+        """Convert to a Run domain model."""
+        from psystack.models.run import Run
+
+        return Run.from_manifest(self)

psystack/models/metric.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class MetricResult(BaseModel):
+    metric_id: str
+    value: float
+    unit: str | None = None
+    higher_is_better: bool
+    per_episode: list[float] = []
+    breakdown: dict[str, Any] = {}

psystack/models/project.py

@@ -0,0 +1,25 @@
+"""Project model — represents a discovered adapter-backed repo."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class DiscoveredAssets(BaseModel):
+    """Assets discovered by adapter's detect_project."""
+
+    weights: list[dict[str, Any]] = Field(default_factory=list)
+    envs: list[str] = Field(default_factory=list)
+    scenarios: list[str] = Field(default_factory=list)
+    configs: list[str] = Field(default_factory=list)
+
+
+class Project(BaseModel):
+    """A PsyStack project rooted at a repo with a bound adapter."""
+
+    project_root: Path
+    adapter_name: str
+    discovered_assets: DiscoveredAssets = Field(default_factory=DiscoveredAssets)

psystack/models/run.py

@@ -0,0 +1,50 @@
+"""Run model — a single execution configuration."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from psystack.models.manifest import RunManifest
+
+
+class Run(BaseModel):
+    """A fully specified run configuration."""
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True)
+
+    id: str
+
+    # Component references
+    world_model_ref: str = ""
+    planner_ref: str = ""
+    seed: int = 42
+    num_episodes: int = Field(default=20, alias="horizon")
+
+    # Full configs (hydrated from refs)
+    planner_config: dict[str, Any] = Field(default_factory=dict)
+    env_config: dict[str, Any] = Field(default_factory=dict)
+
+    @classmethod
+    def from_manifest(cls, manifest: RunManifest) -> Run:
+        """Create a Run from a legacy RunManifest."""
+        return cls(
+            id=manifest.run_id,
+            world_model_ref=manifest.world_model_weights,
+            planner_config=manifest.planner_config,
+            env_config=manifest.env_config,
+            seed=manifest.seed,
+            num_episodes=manifest.num_episodes,
+        )
+
+    def to_manifest(self) -> RunManifest:
+        """Convert back to a RunManifest for pipeline compatibility."""
+        return RunManifest(
+            run_id=self.id,
+            world_model_weights=self.world_model_ref,
+            planner_config=self.planner_config,
+            env_config=self.env_config,
+            num_episodes=self.num_episodes,
+            seed=self.seed,
+        )

psystack/models/signal.py

@@ -0,0 +1,14 @@
+"""Signal value model for event detection and display."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class SignalValue(BaseModel):
+    """A single named signal measurement at a point in time."""
+
+    name: str
+    value: float
+    unit: str | None = None
+    display_format: str = ".3f"