thesean 0.1.4__tar.gz

thesean-0.1.4/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 TheSean
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

thesean-0.1.4/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: thesean
+Version: 0.1.4
+Summary: Regression investigation harness for ML pipelines
+Author: Danny Nguyen
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/theseanai/thesean
+Project-URL: Issues, https://github.com/theseanai/thesean/issues
+Keywords: regression,investigation,ml,machine-learning,debugging,pipeline
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer<1,>=0.9.0
+Requires-Dist: pydantic<3,>=2.0.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: jinja2>=3.1.0
+Requires-Dist: tomli-w>=1.0.0
+Requires-Dist: InquirerPy>=0.3.4
+Requires-Dist: rich>=13.0.0
+Requires-Dist: torch<3,>=2.0.0
+Requires-Dist: numpy<3,>=1.24.0
+Requires-Dist: scipy>=1.11.0
+Requires-Dist: statsmodels>=0.14.0
+Requires-Dist: textual>=0.80.0
+Requires-Dist: tomli>=2.0.0; python_version < "3.11"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-textual-snapshot; extra == "dev"
+Requires-Dist: hypothesis; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
+Requires-Dist: ruff>=0.4.0; extra == "dev"
+Requires-Dist: mypy>=1.8.0; extra == "dev"
+Dynamic: license-file
+
+ # TheSean
+
+ A regression investigation harness for ML pipelines, latent action systems, and world models.
+
+ TheSean runs paired A/B evaluations across ML/world model experiments, detects regressions, and provides an interactive investigation workbench for debugging episode-level divergences.
+
+ ## What it does
+
+ - Configure A/B experiment pairs with different weights, planners, or configs
+ - Run paired evaluations with live telemetry
+ - Detect regressions via bootstrap significance testing across 5 metrics
+ - Drill into individual episodes with signal timelines and event detection
+
+ ## Quickstart
+
+ Python 3.10+ required.
+
+ ### Install
+
+```bash
+pip install thesean
+```
+
+ ### Prerequisites: TheSean requires an ML repo with a compatible adapter.
+
+ The only available adapter is for [f1worldmodel](https://github.com/justinsiek/f1worldmodel).
+
+ Clone the repo and follow its README to install dependencies.
+
+ ### Run
+
+cd into the ML repo project root:
+
+Example:
+```bash
+  cd f1worldmodel
+  thesean
+```
+
+ This launches the TUI. From there:
+
+ 1. Select or create a case.
+ 2. Pick a track, configure Run A (baseline) and Run B (candidate) with different checkpoints or planner settings
+ 3. Run the evaluation
+ 4. View the verdict and drill into episodes to investigate divergences
+
+ ## Available Adapters
+
+ | Adapter | Repo | Status |
+ |---------|------|--------|
+ | `f1` | [justinsiek/f1worldmodel](https://github.com/justinsiek/f1worldmodel) | Beta |
+
+ ## Status
+
+ Beta. APIs may change.
+
+ ## License
+
+ MIT

thesean-0.1.4/README.md

@@ -0,0 +1,59 @@
+ # TheSean
+
+ A regression investigation harness for ML pipelines, latent action systems, and world models.
+
+ TheSean runs paired A/B evaluations across ML/world model experiments, detects regressions, and provides an interactive investigation workbench for debugging episode-level divergences.
+
+ ## What it does
+
+ - Configure A/B experiment pairs with different weights, planners, or configs
+ - Run paired evaluations with live telemetry
+ - Detect regressions via bootstrap significance testing across 5 metrics
+ - Drill into individual episodes with signal timelines and event detection
+
+ ## Quickstart
+
+ Python 3.10+ required.
+
+ ### Install
+
+```bash
+pip install thesean
+```
+
+ ### Prerequisites: TheSean requires an ML repo with a compatible adapter.
+
+ The only available adapter is for [f1worldmodel](https://github.com/justinsiek/f1worldmodel).
+
+ Clone the repo and follow its README to install dependencies.
+
+ ### Run
+
+cd into the ML repo project root:
+
+Example:
+```bash
+  cd f1worldmodel
+  thesean
+```
+
+ This launches the TUI. From there:
+
+ 1. Select or create a case.
+ 2. Pick a track, configure Run A (baseline) and Run B (candidate) with different checkpoints or planner settings
+ 3. Run the evaluation
+ 4. View the verdict and drill into episodes to investigate divergences
+
+ ## Available Adapters
+
+ | Adapter | Repo | Status |
+ |---------|------|--------|
+ | `f1` | [justinsiek/f1worldmodel](https://github.com/justinsiek/f1worldmodel) | Beta |
+
+ ## Status
+
+ Beta. APIs may change.
+
+ ## License
+
+ MIT

thesean-0.1.4/pyproject.toml

@@ -0,0 +1,99 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "thesean"
+version = "0.1.4"
+description = "Regression investigation harness for ML pipelines"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{name = "Danny Nguyen"}]
+readme = "README.md"
+keywords = ["regression", "investigation", "ml", "machine-learning", "debugging", "pipeline"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+
+dependencies = [
+    "typer>=0.9.0,<1",
+    "pydantic>=2.0.0,<3",
+    "pydantic-settings>=2.0.0",
+    "jinja2>=3.1.0",
+    "tomli-w>=1.0.0",
+    "InquirerPy>=0.3.4",
+    "rich>=13.0.0",
+    "torch>=2.0.0,<3",
+    "numpy>=1.24.0,<3",
+    "scipy>=1.11.0",
+    "statsmodels>=0.14.0",
+    "textual>=0.80.0",
+    "tomli>=2.0.0; python_version < '3.11'",
+]
+
+[project.urls]
+Homepage = "https://github.com/theseanai/thesean"
+Issues = "https://github.com/theseanai/thesean/issues"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-textual-snapshot",
+    "hypothesis",
+    "pytest-asyncio>=0.23.0",
+    "ruff>=0.4.0",
+    "mypy>=1.8.0",
+]
+
+[project.entry-points."thesean.adapters"]
+f1 = "thesean.adapters.f1.factory:F1AdapterFactory"
+
+[project.scripts]
+thesean = "thesean.cli.app:app"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+thesean = ["tui/styles/*.tcss", "reporting/templates/*.j2"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]
+ignore = ["B008", "B904", "SIM102", "SIM105", "SIM108", "SIM117", "UP037", "UP038"]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_configs = true
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = ["thesean.models.*", "thesean.reporting.*", "thesean.pipeline.context", "thesean.pipeline.stages.*", "thesean.core.contracts"]
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = ["thesean.pipeline.compare.*", "thesean.pipeline.isolation.attribution"]
+disallow_untyped_defs = false
+warn_return_any = false
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "unit: fast isolated tests",
+    "integration: tests spanning multiple components",
+    "smoke: CLI entry-point smoke tests",
+    "property: hypothesis property-based tests",
+    "snapshot: textual snapshot tests",
+    "tui: tests requiring Textual runtime",
+    "f1: tests that require the real F1 adapter/repo",
+    "framework: hermetic framework tests",
+]
+asyncio_default_fixture_loop_scope = "function"

thesean-0.1.4/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

thesean-0.1.4/src/thesean/__init__.py

@@ -0,0 +1,3 @@
+"""Regression thesean harness for ML pipelines."""
+
+__version__ = "0.1.4"

thesean-0.1.4/src/thesean/__main__.py

@@ -0,0 +1,5 @@
+"""Allow running as `python -m thesean`."""
+
+from thesean.cli.app import app
+
+app()

thesean-0.1.4/src/thesean/adapters/f1/controllers.py

@@ -0,0 +1,56 @@
+"""Scripted controller adapter — wraps F1 scripted policies as PlannerPlugin."""
+
+from __future__ import annotations
+
+import inspect
+from typing import Any
+
+import numpy as np
+
+
+class ScriptedControllerAdapter:
+    """Wraps a scripted controller to satisfy PlannerPlugin protocol.
+
+    This enables the existing episode runner (run_episodes) to work with
+    scripted controllers without a separate code path.
+    """
+
+    def __init__(self, controller_cls: type, track: Any | None = None) -> None:
+        self._controller_cls = controller_cls
+        self._track = track
+        self._accepts_car_state = self._check_car_state(controller_cls)
+        self._controller = self._make_controller()
+
+    @staticmethod
+    def _check_car_state(cls: type) -> bool:
+        """Check if the controller's __call__ accepts a car_state kwarg."""
+        sig = inspect.signature(cls.__call__)
+        return "car_state" in sig.parameters
+
+    def _make_controller(self) -> Any:
+        """Instantiate the controller, passing track if the constructor accepts it."""
+        sig = inspect.signature(self._controller_cls)
+        if "track" in sig.parameters and self._track is not None:
+            return self._controller_cls(self._track)
+        return self._controller_cls()
+
+    def planner_id(self) -> str:
+        return f"scripted_{self._controller_cls.__name__}"
+
+    def configure(self, config: dict[str, Any], world_model: Any = None) -> None:
+        """No-op for scripted controllers."""
+        pass
+
+    def reset(self) -> None:
+        """Re-instantiate the controller to reset all internal state.
+
+        This is safer than manually resetting step_counter/panic_counter etc
+        because it handles any future stateful controllers automatically.
+        """
+        self._controller = self._make_controller()
+
+    def act(self, obs: dict[str, Any], car_state: dict[str, Any] | None = None) -> np.ndarray:
+        """Delegate to the underlying controller's __call__."""
+        if self._accepts_car_state:
+            return self._controller(obs, car_state=car_state)  # type: ignore[no-any-return]
+        return self._controller(obs)  # type: ignore[no-any-return]

thesean-0.1.4/src/thesean/adapters/f1/degrade.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import torch
+
+
+def degrade_model(
+    src_path: str,
+    dst_path: str,
+    noise_scale: float = 0.3,
+) -> None:
+    """Load a world model checkpoint, add noise to predictor/head weights, save."""
+    state_dict = torch.load(src_path, map_location="cpu", weights_only=True)
+
+    degraded = {}
+    for key, tensor in state_dict.items():
+        if key.startswith("predictor.") or "_head." in key:
+            noise = torch.randn_like(tensor) * noise_scale * tensor.abs().mean()
+            degraded[key] = tensor + noise
+        else:
+            degraded[key] = tensor.clone()
+
+    torch.save(degraded, dst_path)
+
+
+def load_model(weights_path: str, device: str = "cpu"):
+    """Convenience: load a WorldModel with weights."""
+    from thesean.adapters.f1.world_model import F1WorldModelAdapter
+
+    adapter = F1WorldModelAdapter()
+    adapter.load(weights_path, device=device)
+    return adapter

thesean-0.1.4/src/thesean/adapters/f1/env.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+
+
+class F1EnvAdapter:
+    """Wraps env.f1_env.F1Env to satisfy EnvPlugin protocol."""
+
+    def __init__(self) -> None:
+        self._env = None
+
+    def env_id(self) -> str:
+        return "f1_env"
+
+    def configure(self, config: dict[str, Any]) -> None:
+        import os
+        import threading
+
+        # Prevent pygame/SDL2 from initializing Cocoa display when imported
+        # from a non-main thread (causes SIGABRT on macOS).
+        if threading.current_thread() is not threading.main_thread():
+            os.environ.setdefault("SDL_VIDEODRIVER", "dummy")
+
+        from configs.default import Config
+        from env.f1_env import F1Env
+
+        accepted = {k: v for k, v in config.items() if hasattr(Config, k)}
+        dropped = set(config.keys()) - set(accepted.keys())
+        if dropped:
+            import logging
+            logging.getLogger(__name__).warning(
+                "F1EnvAdapter: config keys dropped (not in F1 Config): %s", sorted(dropped)
+            )
+        cfg = Config(**accepted)
+        self._env = F1Env.from_config(cfg)
+
+    def reset(self, seed: int | None = None) -> dict[str, Any]:
+        if self._env is None:
+            raise RuntimeError("configure() must be called before reset()")
+        # F1Env.reset() does not accept a seed parameter. Global RNG seeding
+        # is handled by the runner. If F1Env adds seed support, forward it here.
+        return self._env.reset()
+
+    def step(self, action: np.ndarray) -> tuple[dict, float, bool, dict]:
+        if self._env is None:
+            raise RuntimeError("configure() must be called before step()")
+        return self._env.step(action)
+
+    def get_car_state(self) -> dict[str, Any]:
+        return self._env.get_car_state()  # type: ignore[attr-defined, no-any-return]
+
+    def get_progress(self) -> float:
+        return self._env.get_progress()  # type: ignore[attr-defined, no-any-return]

thesean-0.1.4/src/thesean/adapters/f1/factory.py

@@ -0,0 +1,186 @@
+"""F1-specific adapter factory — the single place that knows about F1."""
+
+from __future__ import annotations
+
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from thesean.adapters.f1.env import F1EnvAdapter
+from thesean.adapters.f1.planner import F1PlannerAdapter
+from thesean.adapters.f1.signals import F1SignalTranslator
+from thesean.adapters.f1.world_model import F1WorldModelAdapter
+from thesean.core.contracts import (
+    EnvPlugin,
+    MetricPlugin,
+    PanelProvider,
+    PlannerPlugin,
+    SignalTranslator,
+    WorldModelPlugin,
+)
+from thesean.evaluation.metrics import ALL_METRICS
+
+
+class F1AdapterFactory:
+    """Implements AdapterFactory using F1 adapters."""
+
+    def __init__(self) -> None:
+        self._repo: Path | None = None
+
+    def bind_repo(self, repo: Path) -> None:
+        """Prepare the F1 repo for use: validate and add to sys.path."""
+        repo = repo.expanduser().resolve()
+        if not repo.is_dir():
+            raise ValueError(f"Adapter repo path does not exist: {repo}")
+        self._repo = repo
+        repo_str = str(repo)
+        if repo_str not in sys.path:
+            sys.path.insert(0, repo_str)
+
+    def create_env(self, config: dict[str, Any]) -> EnvPlugin:
+        env = F1EnvAdapter()
+        env.configure(config)
+        return env
+
+    def create_world_model(self, weights_path: str, device: str = "cpu") -> WorldModelPlugin:
+        wm = F1WorldModelAdapter()
+        wm.load(weights_path, device=device)
+        return wm
+
+    def create_planner(self, config: dict[str, Any], world_model: WorldModelPlugin) -> PlannerPlugin:
+        planner = F1PlannerAdapter()
+        planner.configure(config, world_model)
+        return planner
+
+    def get_metrics(self) -> list[MetricPlugin]:
+        return list(ALL_METRICS)
+
+    def discover_weights(self, repo: Path) -> list[dict[str, Any]]:
+        """Scan checkpoints/*.pth, return [{name, path, size_mb, mtime}, ...] newest-first."""
+        ckpt_dir = repo / "checkpoints"
+        if not ckpt_dir.is_dir():
+            return []
+        results = []
+        for p in ckpt_dir.glob("*.pth"):
+            st = p.stat()
+            results.append({
+                "name": p.name,
+                "path": str(p.resolve()),
+                "size_mb": round(st.st_size / 1_048_576, 1),
+                "mtime": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc)
+                         .strftime("%Y-%m-%d %H:%M"),
+            })
+        return sorted(results, key=lambda w: w["mtime"], reverse=True)
+
+    def discover_envs(self, repo: Path) -> list[str]:
+        """Scan tracks/*.csv, return sorted track names (stems)."""
+        tracks_dir = repo / "tracks"
+        if not tracks_dir.is_dir():
+            return []
+        return sorted(p.stem for p in tracks_dir.glob("*.csv"))
+
+    def discover_controllers(self) -> list[dict[str, Any]]:
+        """Discover all scripted controller classes from data.controllers.
+
+        Returns list of dicts: [{"name": "ScriptedPolicy", "requires_track": True}, ...]
+        Requires bind_repo() to have been called first (sys.path must include F1 repo).
+        """
+        import importlib
+        import inspect as _inspect
+
+        mod = importlib.import_module("data.controllers")
+        controllers: list[dict[str, Any]] = []
+        for name, cls in _inspect.getmembers(mod, _inspect.isclass):
+            if name.endswith("Policy"):
+                sig = _inspect.signature(cls.__init__)
+                requires_track = "track" in sig.parameters
+                controllers.append({
+                    "name": name,
+                    "requires_track": requires_track,
+                })
+        return sorted(controllers, key=lambda c: c["name"])
+
+    def create_scripted_controller(self, controller_name: str, track: Any = None) -> Any:
+        """Create a ScriptedControllerAdapter for the named controller.
+
+        Args:
+            controller_name: Class name from data.controllers (e.g. "ScriptedPolicy")
+            track: Track object (required for track-dependent controllers)
+        """
+        import importlib
+
+        mod = importlib.import_module("data.controllers")
+        cls = getattr(mod, controller_name)
+        from thesean.adapters.f1.controllers import ScriptedControllerAdapter
+        return ScriptedControllerAdapter(cls, track=track)
+
+    def default_planner_config(self) -> dict[str, Any]:
+        return {"num_candidates": 400, "horizon": 25, "iterations": 4, "num_elites": 40}
+
+    def default_env_config(self, env_id: str, world_model: WorldModelPlugin | None = None) -> dict[str, Any]:
+        """Build full env config for given track name. Paths are absolute if repo is bound.
+
+        If world_model is provided, raster_size is inferred from checkpoint weights.
+        """
+        raster_size = world_model.raster_size if world_model is not None else 64
+        track_csv = f"tracks/{env_id}.csv"
+        if self._repo is not None:
+            track_csv = str(self._repo / track_csv)
+        return {
+            "track_csv": track_csv,
+            "max_speed": 50.0, "dt": 0.1, "max_steps": 1000,
+            "max_steer_rate": 3.5, "off_track_tolerance": 10,
+            "raster_size": raster_size, "pixels_per_meter": 3.0,
+            "progress_reward": 0.02, "off_track_penalty": 0.5,
+            "step_penalty": 0.005, "lap_bonus": 1.0,
+        }
+
+    # ── Phase 3 optional methods ──────────────────────────────────────────
+
+    def detect_project(self, repo: Path) -> dict[str, Any]:
+        """Scan F1 repo for available assets."""
+        repo = repo.expanduser().resolve()
+        assets: dict[str, Any] = {
+            "weights": self.discover_weights(repo),
+            "envs": self.discover_envs(repo),
+            "scenarios": [],
+            "configs": [],
+        }
+        # Scan for config files
+        config_dir = repo / "configs"
+        if config_dir.is_dir():
+            assets["configs"] = sorted(p.name for p in config_dir.glob("*.py"))
+        return assets
+
+    def get_signal_translator(self) -> SignalTranslator | None:
+        return F1SignalTranslator()
+
+    def get_panel_providers(self) -> list[PanelProvider]:
+        return [F1TrackPanel()]
+
+
+class F1TrackPanel:
+    """PanelProvider for F1 track visualization."""
+
+    def panel_id(self) -> str:
+        return "f1_track"
+
+    def panel_label(self) -> str:
+        return "F1 Track View"
+
+    def load_track_geometry(self, track_csv: str) -> list[tuple[float, float]]:
+        """Load (x, y) centerline points from a track CSV file."""
+        import csv
+
+        points: list[tuple[float, float]] = []
+        with open(track_csv) as f:
+            reader = csv.reader(f)
+            for row in reader:
+                if not row or row[0].strip().startswith("#"):
+                    continue
+                try:
+                    points.append((float(row[0].strip()), float(row[1].strip())))
+                except (ValueError, IndexError):
+                    continue
+        return points