rlshiftlab 0.2.0__tar.gz → 0.3.0__tar.gz

{rlshiftlab-0.2.0 → rlshiftlab-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: rlshiftlab
-Version: 0.2.0
+Version: 0.3.0
 Summary: Stress-test frozen RL policies under controlled shifts and get degradation curves with confidence intervals.
 License: MIT
 License-File: LICENSE

rlshiftlab-0.3.0/examples/lab8_qlearning_vs_sarsa.py

@@ -0,0 +1,150 @@
+"""Complete Lab 8 Experiment: Q-Learning vs SARSA in 5x5 GridWorld with RLShiftLab."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from time import perf_counter
+
+from rlshiftlab import (
+    ActionSlip,
+    GridWorldEnv,
+    Suite,
+    Transition,
+    frozen_policy,
+    plot_grid_policy,
+    plot_q_values,
+    plot_training_curves,
+    train_q_learning,
+    train_sarsa,
+)
+
+ROOT = Path(__file__).resolve().parents[1]
+OUTPUT_DIR = ROOT / "artifacts" / "lab8"
+
+
+def main() -> None:
+    started = perf_counter()
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    print("=== RL Lab 8: Q-Learning vs SARSA in 5x5 GridWorld ===")
+
+    # 1. Initialize 5x5 GridWorld matching Lab 8 specifications
+    env = GridWorldEnv(
+        rows=5,
+        cols=5,
+        start_state=20,  # S20 (bottom-left)
+        goal_state=4,    # S4 (top-right)
+        obstacles=[6, 11, 13, 17],
+        step_reward=-1.0,
+        obstacle_reward=-10.0,
+        wall_reward=-10.0,
+        goal_reward=10.0,
+        max_episode_steps=100,
+    )
+    print("\nInitial Grid World Layout:")
+    print(env.render_ascii())
+
+    # 2. Train Q-Learning (off-policy TD control)
+    print("\nTraining Q-Learning agent (500 episodes)...")
+    q_table, q_hist = train_q_learning(
+        env=env,
+        episodes=500,
+        alpha=0.5,
+        gamma=0.9,
+        epsilon=0.1,
+        seed=42,
+    )
+    print(q_hist.summary())
+
+    # 3. Train SARSA (on-policy TD control)
+    print("\nTraining SARSA agent (500 episodes)...")
+    s_table, s_hist = train_sarsa(
+        env=env,
+        episodes=500,
+        alpha=0.5,
+        gamma=0.9,
+        epsilon=0.1,
+        seed=42,
+    )
+    print(s_hist.summary())
+
+    # 4. Generate Lab 8 Visualizations for Submission
+    print("\nGenerating Lab 8 Visualizations...")
+    plot_training_curves(
+        {"Q-learning": q_hist, "SARSA": s_hist},
+        window=15,
+        title="Lab 8: Training Convergence Comparison",
+        filepath=OUTPUT_DIR / "training_curves.png",
+    )
+
+    plot_grid_policy(
+        env=env,
+        q_table=q_table,
+        title="Q-Learning Optimal Policy (Hugs Obstacles)",
+        filepath=OUTPUT_DIR / "q_policy.png",
+    )
+
+    plot_grid_policy(
+        env=env,
+        q_table=s_table,
+        title="SARSA Optimal Policy (Safer Detour)",
+        filepath=OUTPUT_DIR / "sarsa_policy.png",
+    )
+
+    plot_q_values(
+        env=env,
+        q_table=q_table,
+        title="Q-Learning State Values V(s)",
+        filepath=OUTPUT_DIR / "q_values.png",
+    )
+
+    plot_q_values(
+        env=env,
+        q_table=s_table,
+        title="SARSA State Values V(s)",
+        filepath=OUTPUT_DIR / "sarsa_values.png",
+    )
+
+    # 5. RLShiftLab Robustness Evaluation
+    print("\nRunning RLShiftLab Robustness Evaluation under Action Slips...")
+    q_policy = frozen_policy(q_table)
+    sarsa_policy = frozen_policy(s_table)
+
+    def count_obstacles(transition: Transition) -> float:
+        return float(transition.info.get("hit_obstacle", False))
+
+    suite = Suite(
+        env_fn=lambda: GridWorldEnv(
+            rows=5,
+            cols=5,
+            start_state=20,
+            goal_state=4,
+            obstacles=[6, 11, 13, 17],
+            max_episode_steps=100,
+        ),
+        shifts=[
+            ActionSlip(levels=[0.05, 0.1, 0.2, 0.3]),
+        ],
+        n_episodes=100,
+        seed=123,
+        success_fn=lambda ep: ep.terminated,
+        step_metrics={"obstacle_hits": count_obstacles},
+    )
+
+    results = suite.evaluate({"Q-learning": q_policy, "SARSA": sarsa_policy})
+    print("\nRobustness Evaluation Summary:")
+    print(results.summary())
+
+    results.to_csv(OUTPUT_DIR / "robustness_results.csv")
+    results.plot(OUTPUT_DIR / "robustness_degradation.png")
+    results.to_html(OUTPUT_DIR / "lab8_report.html")
+
+    contrast = results.compare("Q-learning", "SARSA", metric="return")
+    contrast.to_csv(OUTPUT_DIR / "comparison_return.csv")
+    contrast.plot(OUTPUT_DIR / "comparison_return.png")
+
+    print(f"\nAll artifacts generated in: {OUTPUT_DIR}")
+    print(f"Total execution time: {perf_counter() - started:.2f}s")
+
+
+if __name__ == "__main__":
+    main()

{rlshiftlab-0.2.0 → rlshiftlab-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "rlshiftlab"
-version = "0.2.0"
+version = "0.3.0"
 description = "Stress-test frozen RL policies under controlled shifts and get degradation curves with confidence intervals."
 readme = "README.md"
 requires-python = ">=3.9"

rlshiftlab-0.3.0/src/rlshiftlab/__init__.py

@@ -0,0 +1,59 @@
+"""Stress-test frozen reinforcement learning policies under controlled shifts."""
+
+__version__ = "0.3.0"
+
+from rlshiftlab.comparisons import ComparisonResults
+from rlshiftlab.envs.gridworld import (
+    ACTION_DOWN,
+    ACTION_LEFT,
+    ACTION_NAMES,
+    ACTION_RIGHT,
+    ACTION_SYMBOLS,
+    ACTION_UP,
+    DEFAULT_OBSTACLES,
+    GridWorldEnv,
+)
+from rlshiftlab.groups import GroupResults
+from rlshiftlab.results import Results
+from rlshiftlab.runner import EpisodeRecord, EpisodeResult, Suite, Transition
+from rlshiftlab.shifts import ActionSlip, ObservationNoise, StickyActions
+from rlshiftlab.tabular.algorithms import (
+    frozen_policy,
+    train_q_learning,
+    train_sarsa,
+)
+from rlshiftlab.tabular.history import TrainingHistory
+from rlshiftlab.viz.grid import (
+    plot_grid_policy,
+    plot_q_values,
+    plot_training_curves,
+)
+
+__all__ = [
+    "ACTION_DOWN",
+    "ACTION_LEFT",
+    "ACTION_NAMES",
+    "ACTION_RIGHT",
+    "ACTION_SYMBOLS",
+    "ACTION_UP",
+    "DEFAULT_OBSTACLES",
+    "ActionSlip",
+    "ComparisonResults",
+    "EpisodeRecord",
+    "EpisodeResult",
+    "GridWorldEnv",
+    "GroupResults",
+    "ObservationNoise",
+    "Results",
+    "StickyActions",
+    "Suite",
+    "TrainingHistory",
+    "Transition",
+    "__version__",
+    "frozen_policy",
+    "plot_grid_policy",
+    "plot_q_values",
+    "plot_training_curves",
+    "train_q_learning",
+    "train_sarsa",
+]

rlshiftlab-0.3.0/src/rlshiftlab/envs/__init__.py

@@ -0,0 +1,25 @@
+"""Custom environments for RLShiftLab experiments."""
+
+from __future__ import annotations
+
+from rlshiftlab.envs.gridworld import (
+    ACTION_DOWN,
+    ACTION_LEFT,
+    ACTION_NAMES,
+    ACTION_RIGHT,
+    ACTION_SYMBOLS,
+    ACTION_UP,
+    DEFAULT_OBSTACLES,
+    GridWorldEnv,
+)
+
+__all__ = [
+    "ACTION_DOWN",
+    "ACTION_LEFT",
+    "ACTION_NAMES",
+    "ACTION_RIGHT",
+    "ACTION_SYMBOLS",
+    "ACTION_UP",
+    "DEFAULT_OBSTACLES",
+    "GridWorldEnv",
+]

rlshiftlab-0.3.0/src/rlshiftlab/envs/gridworld.py

@@ -0,0 +1,233 @@
+"""Gymnasium-compliant 5x5 GridWorld environment for RL Lab 8."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any, ClassVar
+
+import gymnasium as gym
+import numpy as np
+
+ACTION_UP = 0
+ACTION_DOWN = 1
+ACTION_LEFT = 2
+ACTION_RIGHT = 3
+
+ACTION_NAMES: dict[int, str] = {
+    ACTION_UP: "UP",
+    ACTION_DOWN: "DOWN",
+    ACTION_LEFT: "LEFT",
+    ACTION_RIGHT: "RIGHT",
+}
+
+ACTION_SYMBOLS: dict[int, str] = {
+    ACTION_UP: "^",
+    ACTION_DOWN: "v",
+    ACTION_LEFT: "<",
+    ACTION_RIGHT: ">",
+}
+
+# Default obstacles matching Lab 8:
+# Row 1 Col 1 -> S6
+# Row 2 Col 1 -> S11
+# Row 2 Col 3 -> S13
+# Row 3 Col 2 -> S17
+DEFAULT_OBSTACLES: frozenset[int] = frozenset({6, 11, 13, 17})
+
+
+class GridWorldEnv(gym.Env):
+    """A 5x5 (or N x M) navigation grid with obstacles, boundary walls, and goals.
+
+    Conforms to Gymnasium 0.29+ Env interface with Discrete observation and
+    action spaces.
+    """
+
+    metadata: ClassVar[dict[str, Any]] = {"render_modes": ["ansi", "human"]}
+
+    def __init__(
+        self,
+        rows: int = 5,
+        cols: int = 5,
+        start_state: int = 20,
+        goal_state: int = 4,
+        obstacles: Sequence[int] | set[int] | frozenset[int] | None = None,
+        step_reward: float = -1.0,
+        obstacle_reward: float = -10.0,
+        wall_reward: float = -10.0,
+        goal_reward: float = 10.0,
+        obstacle_bounce: bool = True,
+        max_episode_steps: int = 100,
+        render_mode: str | None = None,
+    ) -> None:
+        super().__init__()
+        if rows < 2 or cols < 2:
+            raise ValueError(f"rows and cols must be >= 2, got {rows}x{cols}")
+        n_states = rows * cols
+        if not (0 <= start_state < n_states):
+            raise ValueError(f"start_state {start_state} must be in [0, {n_states})")
+        if not (0 <= goal_state < n_states):
+            raise ValueError(f"goal_state {goal_state} must be in [0, {n_states})")
+        if start_state == goal_state:
+            raise ValueError("start_state and goal_state must be distinct")
+
+        self.rows = rows
+        self.cols = cols
+        self.n_states = n_states
+        self.start_state = int(start_state)
+        self.goal_state = int(goal_state)
+
+        if obstacles is None:
+            self.obstacles = frozenset(DEFAULT_OBSTACLES)
+        else:
+            self.obstacles = frozenset(int(s) for s in obstacles)
+
+        if self.start_state in self.obstacles:
+            raise ValueError(f"start_state {self.start_state} cannot be an obstacle")
+        if self.goal_state in self.obstacles:
+            raise ValueError(f"goal_state {self.goal_state} cannot be an obstacle")
+
+        self.step_reward = float(step_reward)
+        self.obstacle_reward = float(obstacle_reward)
+        self.wall_reward = float(wall_reward)
+        self.goal_reward = float(goal_reward)
+        self.obstacle_bounce = bool(obstacle_bounce)
+        self.max_episode_steps = int(max_episode_steps)
+        self.render_mode = render_mode
+
+        self.observation_space = gym.spaces.Discrete(self.n_states)
+        self.action_space = gym.spaces.Discrete(4)
+
+        self._state = self.start_state
+        self._step_count = 0
+        self._np_random: np.random.Generator = np.random.default_rng()
+
+    def to_state(self, row: int, col: int) -> int:
+        """Convert 2D (row, col) coordinates to 1D state index."""
+        return row * self.cols + col
+
+    def to_coord(self, state: int) -> tuple[int, int]:
+        """Convert 1D state index to 2D (row, col) coordinates."""
+        return divmod(state, self.cols)
+
+    def reset(
+        self,
+        *,
+        seed: int | None = None,
+        options: dict[str, Any] | None = None,
+    ) -> tuple[int, dict[str, Any]]:
+        """Reset the environment to the start state."""
+        super().reset(seed=seed)
+        if seed is not None:
+            self._np_random = np.random.default_rng(seed)
+
+        self._state = self.start_state
+        self._step_count = 0
+
+        info = {
+            "row": self.to_coord(self._state)[0],
+            "col": self.to_coord(self._state)[1],
+            "hit_wall": False,
+            "hit_obstacle": False,
+            "reached_goal": False,
+        }
+        return self._state, info
+
+    def step(self, action: int) -> tuple[int, float, bool, bool, dict[str, Any]]:
+        """Execute one action transition."""
+        action = int(action)
+        if not self.action_space.contains(action):
+            raise ValueError(f"Invalid action {action!r}; must be in [0, 3]")
+
+        self._step_count += 1
+        r, c = self.to_coord(self._state)
+
+        # Compute candidate target coordinate
+        if action == ACTION_UP:
+            nr, nc = r - 1, c
+        elif action == ACTION_DOWN:
+            nr, nc = r + 1, c
+        elif action == ACTION_LEFT:
+            nr, nc = r, c - 1
+        elif action == ACTION_RIGHT:
+            nr, nc = r, c + 1
+        else:
+            nr, nc = r, c
+
+        hit_wall = False
+        hit_obstacle = False
+        reached_goal = False
+        terminated = False
+
+        # 1. Boundary wall check: outside bounds
+        if nr < 0 or nr >= self.rows or nc < 0 or nc >= self.cols:
+            hit_wall = True
+            next_state = self._state  # remains in the same state
+            reward = self.wall_reward
+        else:
+            candidate_state = self.to_state(nr, nc)
+            # 2. Obstacle check
+            if candidate_state in self.obstacles:
+                hit_obstacle = True
+                reward = self.obstacle_reward
+                next_state = self._state if self.obstacle_bounce else candidate_state
+            # 3. Goal check
+            elif candidate_state == self.goal_state:
+                reached_goal = True
+                next_state = candidate_state
+                reward = self.goal_reward
+                terminated = True
+            # 4. Standard move
+            else:
+                next_state = candidate_state
+                reward = self.step_reward
+
+        self._state = next_state
+        truncated = self._step_count >= self.max_episode_steps and not terminated
+
+        info = {
+            "row": self.to_coord(self._state)[0],
+            "col": self.to_coord(self._state)[1],
+            "hit_wall": hit_wall,
+            "hit_obstacle": hit_obstacle,
+            "reached_goal": reached_goal,
+            "step_count": self._step_count,
+        }
+
+        if self.render_mode == "human":
+            print(self.render_ascii())
+
+        return self._state, reward, terminated, truncated, info
+
+    def render_ascii(self) -> str:
+        """Render grid as a formatted ASCII string."""
+        lines = []
+        for r in range(self.rows):
+            row_chars = []
+            for c in range(self.cols):
+                s = self.to_state(r, c)
+                if s == self._state:
+                    if s == self.start_state:
+                        row_chars.append("A(S)")
+                    elif s == self.goal_state:
+                        row_chars.append("A(G)")
+                    else:
+                        row_chars.append(" A ")
+                elif s == self.goal_state:
+                    row_chars.append(" G ")
+                elif s == self.start_state:
+                    row_chars.append(" S ")
+                elif s in self.obstacles:
+                    row_chars.append(" X ")
+                else:
+                    row_chars.append(f"{s:2d} ")
+            lines.append(" ".join(row_chars))
+        return "\n".join(lines)
+
+    def render(self) -> str | None:
+        """Gymnasium render entry point."""
+        if self.render_mode == "ansi":
+            return self.render_ascii()
+        if self.render_mode == "human":
+            print(self.render_ascii())
+            return None
+        return None

rlshiftlab-0.3.0/src/rlshiftlab/tabular/__init__.py

@@ -0,0 +1,17 @@
+"""Tabular reinforcement learning algorithms and history utilities."""
+
+from __future__ import annotations
+
+from rlshiftlab.tabular.algorithms import (
+    frozen_policy,
+    train_q_learning,
+    train_sarsa,
+)
+from rlshiftlab.tabular.history import TrainingHistory
+
+__all__ = [
+    "TrainingHistory",
+    "frozen_policy",
+    "train_q_learning",
+    "train_sarsa",
+]