saferl-lite 0.1.0__py3-none-any.whl

agents/constrained_dqn.py

@@ -0,0 +1,82 @@
+# agents/constrained_dqn.py
+
+import numpy as np
+import random
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from collections import deque
+from agents.constraints import Constraint
+
+
+class DQNetwork(nn.Module):
+    def __init__(self, input_dim, output_dim):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(input_dim, 128),
+            nn.ReLU(),
+            nn.Linear(128, 128),
+            nn.ReLU(),
+            nn.Linear(128, output_dim),
+        )
+
+    def forward(self, x):
+        return self.net(x)
+
+
+class ConstrainedDQNAgent:
+    def __init__(self, state_dim, action_dim, constraint: Constraint = None):
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.q_net = DQNetwork(state_dim, action_dim).to(self.device)
+        self.target_net = DQNetwork(state_dim, action_dim).to(self.device)
+        self.target_net.load_state_dict(self.q_net.state_dict())
+
+        self.optimizer = optim.Adam(self.q_net.parameters(), lr=1e-3)
+        self.memory = deque(maxlen=10000)
+        self.gamma = 0.99
+        self.batch_size = 64
+
+        self.constraint = constraint
+        self.action_dim = action_dim
+
+    def select_action(self, state, epsilon):
+        if random.random() < epsilon:
+            return random.randint(0, self.action_dim - 1)
+        state = torch.tensor(state, dtype=torch.float32).to(self.device)
+        with torch.no_grad():
+            q_values = self.q_net(state)
+        return q_values.argmax().item()
+
+    def store_transition(self, s, a, r, s_next, done):
+        self.memory.append((s, a, r, s_next, done))
+
+    def update(self):
+        if len(self.memory) < self.batch_size:
+            return
+        batch = random.sample(self.memory, self.batch_size)
+        s, a, r, s_next, done = zip(*batch)
+
+        s = torch.tensor(s, dtype=torch.float32).to(self.device)
+        a = torch.tensor(a).to(self.device)
+        r = torch.tensor(r, dtype=torch.float32).to(self.device)
+        s_next = torch.tensor(s_next, dtype=torch.float32).to(self.device)
+        done = torch.tensor(done, dtype=torch.float32).to(self.device)
+
+        q_values = self.q_net(s).gather(1, a.unsqueeze(1)).squeeze()
+        with torch.no_grad():
+            target_q = r + self.gamma * self.target_net(s_next).max(1)[0] * (1 - done)
+
+        loss = nn.functional.mse_loss(q_values, target_q)
+        self.optimizer.zero_grad()
+        loss.backward()
+        self.optimizer.step()
+
+    def apply_constraint(self, state, action, reward):
+        if self.constraint:
+            penalty = self.constraint.compute_penalty(state, action, reward)
+            return reward - penalty, penalty
+        return reward, 0.0
+
+    def reset_constraints(self):
+        if self.constraint:
+            self.constraint.reset()

agents/constraints.py

@@ -0,0 +1,39 @@
+# agents/constraints.py
+
+from abc import ABC, abstractmethod
+
+
+class Constraint(ABC):
+    """Abstract base class for constraints in SafeRL agents."""
+
+    @abstractmethod
+    def compute_penalty(self, state, action, reward) -> float:
+        """Return penalty value for a given step."""
+        pass
+
+    def reset(self):
+        """Optional: reset internal counters for new episode."""
+        pass
+
+
+class ActionBudgetConstraint(Constraint):
+    def __init__(self, max_actions: int):
+        self.max_actions = max_actions
+        self.counter = 0
+
+    def compute_penalty(self, state, action, reward) -> float:
+        self.counter += 1
+        return 1.0 if self.counter > self.max_actions else 0.0
+
+    def reset(self):
+        self.counter = 0
+
+
+class EnergyPenaltyConstraint(Constraint):
+    def __init__(self, energy_fn, max_energy):
+        self.energy_fn = energy_fn
+        self.max_energy = max_energy
+
+    def compute_penalty(self, state, action, reward) -> float:
+        energy = self.energy_fn(state, action)
+        return max(0.0, energy - self.max_energy)

envs/wrappers.py

@@ -0,0 +1,58 @@
+# envs/wrappers.py
+
+import gymnasium as gym
+import numpy as np
+
+
+class SafeEnvWrapper(gym.Wrapper):
+    def __init__(self, env, max_force: float = None, max_energy: float = None):
+        super().__init__(env)
+        self.max_force = max_force
+        self.max_energy = max_energy
+        self.violation_log = []
+        self.episode_log = []
+
+    def reset(self, **kwargs):
+        obs, info = self.env.reset(**kwargs)
+        self.violation_log.clear()
+        self.episode_log.clear()
+        return obs, info
+
+    def step(self, action):
+        obs, reward, terminated, truncated, info = self.env.step(action)
+        done = terminated or truncated
+
+        violation = 0.0
+
+        # Constraint: limit max force (CartPole)
+        if self.max_force is not None:
+            force = self._get_force(action)
+            if abs(force) > self.max_force:
+                violation = 1.0
+                reward -= 1.0  # penalize
+
+        # Note: remove _get_energy for now (MountainCar support will come later)
+
+        self.violation_log.append(violation)
+        self.episode_log.append(
+            {
+                "obs": obs,
+                "action": action,
+                "reward": reward,
+                "violation": violation,
+            }
+        )
+
+        return obs, reward, terminated, truncated, info
+
+    def _get_force(self, action):
+        # Assumes CartPole: force is ±10
+        return 10.0 if action == 1 else -10.0
+
+    def _get_energy(self, prev_obs, action, next_obs):
+        if prev_obs is None:
+            return 0.0
+        # Simplified energy calculation: KE + PE
+        velocity = next_obs[1]
+        height = np.cos(3 * next_obs[0])  # approximates potential
+        return 0.5 * velocity**2 + 9.8 * height

explainability/saliency.py

@@ -0,0 +1,23 @@
+# explainability/saliency.py
+
+import torch
+from captum.attr import Saliency
+
+
+class SaliencyExplainer:
+    def __init__(self, model, device="cpu"):
+        self.model = model
+        self.device = device
+        self.saliency = Saliency(self.model)
+
+    def explain(self, state_tensor, target_action: int):
+        """
+        state_tensor: 1D torch tensor (state) on correct device
+        target_action: int, index of the action you want to explain
+        Returns: 1D saliency values (array) for input features
+        """
+        state_tensor = state_tensor.unsqueeze(
+            0
+        ).requires_grad_()  # Shape: [1, input_dim]
+        attr = self.saliency.attribute(state_tensor, target=target_action)
+        return attr.squeeze().detach().cpu().numpy()

explainability/shap_explainer.py

@@ -0,0 +1,31 @@
+# explainability/shap_explainer.py
+
+import shap
+import numpy as np
+import torch
+
+
+class SHAPExplainer:
+    def __init__(self, model, input_dim, device="cpu"):
+        """
+        model: a function that maps np.array -> Q-values
+        input_dim: size of observation space
+        """
+        self.input_dim = input_dim
+        self.device = device
+
+        def model_wrapper(x_np):
+            x_tensor = torch.tensor(x_np, dtype=torch.float32).to(self.device)
+            with torch.no_grad():
+                return model(x_tensor).cpu().numpy()
+
+        self.explainer = shap.Explainer(
+            model_wrapper, shap.maskers.Independent(np.zeros((1, input_dim)))
+        )
+
+    def explain(self, state):
+        """
+        state: np.array of shape (input_dim,)
+        Returns: SHAP values for each input dimension
+        """
+        return self.explainer(np.array([state]))

saferl_lite-0.1.0.dist-info/METADATA

@@ -0,0 +1,139 @@
+Metadata-Version: 2.4
+Name: saferl-lite
+Version: 0.1.0
+Summary: A lightweight, explainable, and constrained reinforcement learning toolkit.
+Home-page: https://github.com/satyamcser/saferl-lite
+Author: Satyam Mishra
+Author-email: satyam@example.com
+Project-URL: Documentation, https://satyamcser.github.io/saferl-lite/
+Project-URL: Source, https://github.com/satyamcser/saferl-lite
+Project-URL: Bug Tracker, https://github.com/satyamcser/saferl-lite/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: gym
+Requires-Dist: gymnasium
+Requires-Dist: numpy
+Requires-Dist: torch
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+Requires-Dist: pre-commit
+Requires-Dist: flake8
+Requires-Dist: pyyaml
+Requires-Dist: shap
+Requires-Dist: captum
+Requires-Dist: typer
+Requires-Dist: scikit-learn
+Requires-Dist: pandas
+Requires-Dist: pytest
+Requires-Dist: pytest-cov
+Requires-Dist: coverage
+Requires-Dist: mkdocs
+Requires-Dist: wandb
+Requires-Dist: mkdocs>=1.5
+Requires-Dist: mkdocs-material>=9.5
+Requires-Dist: mkdocstrings[python]
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# 🔐 SafeRL-Lite
+
+A **lightweight, explainable, and modular** Python library for **Constrained Reinforcement Learning (Safe RL)** with real-time **SHAP & saliency-based explainability**, custom metrics, and Gym-compatible wrappers.
+
+<p align="center">
+  <img src="https://img.shields.io/github/license/satyamcser/saferl-lite?style=flat-square">
+  <img src="https://img.shields.io/github/stars/satyamcser/saferl-lite?style=flat-square">
+  <img src="https://img.shields.io/pypi/v/saferl-lite?style=flat-square">
+  <img src="https://img.shields.io/github/actions/workflow/status/satyamcser/saferl-lite/ci.yml?branch=main&style=flat-square">
+</p>
+
+---
+
+## 🌟 Overview
+
+**SafeRL-Lite** empowers reinforcement learning agents to act under **safety constraints**, while remaining **interpretable** and **modular** for fast experimentation. It wraps standard Gym environments and DQN-based agents with:
+
+- ✅ Safety constraint logic
+- 🔍 Visual explainability (SHAP, saliency maps)
+- 📊 Violation and reward tracking
+- 🧪 Built-in testing and evaluations
+
+---
+
+## 🔧 Installation
+
+> 📦 PyPI (coming soon)
+```bash
+pip install saferl-lite
+```
+
+## 🛠️ From source:
+
+```bash
+git clone https://github.com/satyamcser/saferl-lite.git
+cd saferl-lite
+pip install -e .
+```
+
+## 🚀 Quickstart
+Train a constrained DQN agent with saliency-based explainability:
+
+```bash
+python train.py --env CartPole-v1 --constraint pole_angle --explain shap
+```
+
+🔹 This:
+
+- Adds a pole-angle constraint wrapper to the Gym env
+
+- Logs violations
+
+- Displays SHAP or saliency explanations for agent decisions
+
+## 🧠 Features
+#### ✅ Constrained RL
+- Add custom constraints via wrapper or logic class
+
+- Violation logging and reward shaping
+
+- Safe vs unsafe episode tracking
+
+#### 🔍 Explainability
+- SaliencyExplainer — gradient-based visual heatmaps
+
+- SHAPExplainer — feature contribution values per decision
+
+- Compatible with any PyTorch-based agent
+
+#### 📊 Metrics
+- Constraint violation rate
+
+- Episode reward
+
+- Cumulative safe reward
+
+- Action entropy & temporal behavior stats
+
+#### 📚 Modularity
+- Swap out agents, constraints, evaluators, or explainers
+
+- Supports Gym environments
+
+- Configurable training pipeline
+
+## 📜 Citation
+Coming soon after arXiv/preprint release.

saferl_lite-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+agents/constrained_dqn.py,sha256=dkVfgxBUEpT1gh4L2PJBXvwqGbIGFj8mYPFnacqNBgU,2814
+agents/constraints.py,sha256=en8uB2gluDI6JDsz96lM0yUnFA-0DB9m_a3ycrVky8c,1075
+envs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+envs/wrappers.py,sha256=rfk3cfsTsfD8NqUjEcJ-o7XGMmkBBHt5kfaCiE3AgAw,1749
+explainability/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+explainability/saliency.py,sha256=EpvrpkRZWqYqd3lkRIkfIbJ0pw7G_hJ8GEiVfgPo88U,767
+explainability/shap_explainer.py,sha256=Tj-fP947z8ixFdWRXHdR6D3a_wtznGN5x-DomU34xbc,883
+saferl_lite-0.1.0.dist-info/licenses/LICENSE,sha256=WRhQPkdFDzbMFEhvoaq9gSNnbsy0lhSC8tFH3stLntY,1070
+saferl_lite-0.1.0.dist-info/METADATA,sha256=k9EwE0Clqv-yIANmGdhJPemW4EhBI9kqAnw6xc74WJE,3868
+saferl_lite-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+saferl_lite-0.1.0.dist-info/top_level.txt,sha256=f1IuezLA5sRnSuKZbl-VrS_Hh9pekOW2smLrpJLuiGg,27
+saferl_lite-0.1.0.dist-info/RECORD,,

saferl_lite-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

saferl_lite-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Satyam Mishra
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

saferl_lite-0.1.0.dist-info/top_level.txt

@@ -0,0 +1,3 @@
+agents
+envs
+explainability