agilerl 2.7.0.dev0__tar.gz → 2.7.0.dev1__tar.gz

agilerl-2.7.0.dev1/.github/workflows/linux-tests.yml

@@ -0,0 +1,96 @@
+---
+name: Linux
+
+on:
+    push:
+        branches: [main, nightly]
+        paths:
+            - agilerl/**
+            - tests/**
+            - .github/workflows/**
+            - pyproject.toml
+    pull_request:
+        paths:
+            - agilerl/**
+            - tests/**
+            - .github/workflows/**
+            - pyproject.toml
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+    cancel-in-progress: true
+
+permissions:
+    contents: read
+
+jobs:
+    tests:
+        runs-on: gha-runner-scale-set
+        strategy:
+            fail-fast: false
+            max-parallel: 4
+            matrix:
+                python-version: ['3.10', '3.11', '3.12', '3.13']
+
+        container:
+            image: pytorch/pytorch:2.7.1-cuda12.6-cudnn9-devel
+            options: --user root
+
+        # Workspace (/__w) is ~1GB with little free space; root (/) has plenty. Put cache and venv on /.
+        env:
+            UV_CACHE_DIR: /tmp/uv-cache
+            UV_PROJECT_ENVIRONMENT: /tmp/agilerl-venv
+            HF_HOME: /tmp/hf-cache
+            TORCHINDUCTOR_CACHE_DIR: /tmp/inductor-cache
+
+        steps:
+            - uses: actions/checkout@v4
+            - uses: astral-sh/setup-uv@v7
+              with:
+                  enable-cache: true
+                  python-version: ${{ matrix.python-version }}
+
+            - name: Cache HuggingFace models
+              uses: actions/cache@v4
+              with:
+                  path: /tmp/hf-cache
+                  key: hf-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
+                  restore-keys: hf-${{ matrix.python-version }}-
+
+            - name: Cache torch inductor compilations
+              uses: actions/cache@v4
+              with:
+                  path: /tmp/inductor-cache
+                  key: inductor-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
+                  restore-keys: inductor-${{ matrix.python-version }}-
+
+            - name: Install dependencies
+              # swig is needed to build box2d-py from source (no pre-built wheels for py3.10+).
+              run: |
+                  uv sync --locked --all-groups --extra all
+                  echo "$UV_PROJECT_ENVIRONMENT/bin" >> $GITHUB_PATH
+
+            - name: Reset coverage data
+              run: rm -f .coverage .coverage.*
+
+            - name: Run non-LLM tests (parallel)
+              run: uv run pytest -m "not llm" --exitfirst --cov=agilerl --cov-report= --durations=0 --durations-min=1.0
+
+            - name: Run LLM tests (sequential, exclusive GPU)
+              run: uv run pytest -n0 -m "llm" --exitfirst --cov=agilerl --cov-append --cov-report= --durations=0 --durations-min=1.0
+
+            - name: Combine coverage data
+              run: |
+                  if ls .coverage.* >/dev/null 2>&1; then
+                      uv run coverage combine
+                  else
+                      echo "No parallel coverage shards found; skipping combine."
+                  fi
+
+            - name: Generate coverage report
+              run: uv run coverage xml -o coverage.xml
+
+            - name: Upload coverage reports to Codecov
+              uses: codecov/codecov-action@v3
+              env:
+                  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

{agilerl-2.7.0.dev0 → agilerl-2.7.0.dev1}/.github/workflows/macos-tests.yml

@@ -6,10 +6,20 @@ on:
         branches: [main, nightly]
         paths:
             - agilerl/**
+            - tests/**
+            - .github/workflows/**
+            - pyproject.toml
     pull_request:
         branches: [main, nightly]
         paths:
             - agilerl/**
+            - tests/**
+            - .github/workflows/**
+            - pyproject.toml
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+    cancel-in-progress: true
 
 permissions:
     contents: read

{agilerl-2.7.0.dev0 → agilerl-2.7.0.dev1}/.github/workflows/windows-tests.yml

@@ -6,10 +6,20 @@ on:
         branches: [main, nightly]
         paths:
             - agilerl/**
+            - tests/**
+            - .github/workflows/**
+            - pyproject.toml
     pull_request:
         branches: [main, nightly]
         paths:
             - agilerl/**
+            - tests/**
+            - .github/workflows/**
+            - pyproject.toml
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+    cancel-in-progress: true
 
 permissions:
     contents: read

{agilerl-2.7.0.dev0 → agilerl-2.7.0.dev1}/.gitignore

@@ -109,6 +109,8 @@ celerybeat.pid
 # Environments
 .env
 .vscode/
+.cursor/
+.claude/
 .venv*/
 venv/
 env.bak/

{agilerl-2.7.0.dev0 → agilerl-2.7.0.dev1}/.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
                 - --skip=*.css,*.js,*.map,*.scss,*.svg
                 - --ignore-words-list=magent,pres,roate
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.15.10
+      rev: v0.15.11
       hooks:
           - id: ruff
             name: Ruff Linter
@@ -47,7 +47,7 @@ repos:
 
     - repo: https://github.com/astral-sh/uv-pre-commit
       # uv version.
-      rev: 0.11.6
+      rev: 0.11.7
       hooks:
           - id: uv-lock
 

agilerl-2.7.0.dev1/CLAUDE.md

@@ -0,0 +1,139 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+AgileRL is a deep reinforcement learning library focused on RLOps - MLOps for reinforcement learning. The key innovation is evolutionary hyperparameter optimization (HPO) that automatically tunes hyperparameters during training, reducing the need for separate HPO experiments.
+
+## Development Commands
+
+```bash
+# Install in development mode
+pip install -e .
+
+# Install with LLM dependencies (for GRPO, DPO algorithms)
+pip install -e ".[all]"
+
+# Run all tests
+pytest tests
+
+# Run tests with coverage
+pytest --cov=agilerl --cov-report=xml
+
+# Run a single test file
+pytest tests/test_algorithms/test_dqn.py
+
+# Run a specific test
+pytest tests/test_algorithms/test_dqn.py::test_dqn_init -v
+
+# Skip LLM-related tests (faster)
+pytest -m "not llm"
+
+# Run tests with fail-fast
+pytest --exitfirst
+
+# Lint and format (via pre-commit)
+pre-commit run --all-files
+
+# Install pre-commit hooks
+pre-commit install
+```
+
+## Architecture
+
+### Core Algorithm Hierarchy
+
+All RL algorithms inherit from a common base in `agilerl/algorithms/core/base.py`:
+
+- **EvolvableAlgorithm**: Base metaclass providing mutation registry, checkpointing, cloning, and evolutionary features
+- **RLAlgorithm(EvolvableAlgorithm)**: Single-agent RL base with observation preprocessing and action handling
+- **MultiAgentRLAlgorithm(EvolvableAlgorithm)**: Multi-agent RL base supporting agent-specific networks
+
+### Evolvable Networks System
+
+The evolution system uses two core concepts in `agilerl/modules/`:
+
+- **EvolvableModule** (`modules/base.py`): Base class for neural network modules that support architectural mutations (add/remove layers, nodes, change activations). Uses `@mutation` decorator to mark methods that trigger network reconstruction.
+- **EvolvableNetwork** (`networks/base.py`): Encoder-head architecture combining feature extraction with task-specific heads. Mutations can target encoder or head independently.
+
+Network configurations are dataclasses in `agilerl/modules/configs.py` (MlpNetConfig, CnnNetConfig, etc.).
+
+### HPO Components (`agilerl/hpo/`)
+
+- **Mutations** (`mutation.py`): Applies architectural and hyperparameter mutations to agent populations
+- **TournamentSelection** (`tournament.py`): Selects fittest agents based on evaluation scores
+
+### Training Loop Functions (`agilerl/training/`)
+
+Pre-built training loops that handle the full evolutionary training cycle:
+- `train_off_policy`: DQN, DDPG, TD3, Rainbow
+- `train_on_policy`: PPO
+- `train_multi_agent_off_policy`: MADDPG, MATD3
+- `train_multi_agent_on_policy`: IPPO
+- `train_bandits`: NeuralUCB, NeuralTS
+- `train_offline`: CQL
+- `train_llm`: GRPO, DPO
+
+### Registry System (`agilerl/algorithms/core/registry.py`)
+
+Algorithms declare their evolvable components via `MutationRegistry`:
+- **NetworkGroup**: Groups related networks (e.g., actor/critic, eval/target)
+- **OptimizerConfig**: Maps optimizers to networks with hyperparameter bounds
+- **HyperparameterConfig**: Defines mutable RL hyperparameters (lr, batch_size, etc.)
+
+### Protocol Types (`agilerl/protocols.py`)
+
+Runtime-checkable protocols define interfaces for:
+- `EvolvableAlgorithmProtocol`: Required methods for evolutionary algorithms
+- `EvolvableModuleProtocol`: Required methods for evolvable network modules
+- `EvolvableNetworkProtocol`: Encoder-head network interface
+
+## Key Patterns
+
+### Creating Populations
+
+```python
+from agilerl.utils.utils import create_population
+
+pop = create_population(
+    algo="DQN",
+    observation_space=obs_space,
+    action_space=act_space,
+    INIT_HP=init_hp_dict,
+    population_size=6,
+    device="cuda"
+)
+```
+
+### Mutation Flow
+
+1. Tournament selection picks winners from population
+2. Clone winners to replace losers
+3. Apply mutations via `Mutations.mutation()`:
+   - Network architecture mutations (layers, nodes)
+   - Hyperparameter mutations (lr, batch_size)
+4. Rebuild optimizers for mutated networks
+
+### Multi-Agent Support
+
+Uses PettingZoo-style parallel API. Agent networks can be:
+- **Homogeneous**: Shared architecture across agents
+- **Heterogeneous**: Per-agent architectures via `ModuleDict`
+
+## Testing Conventions
+
+- Test files mirror source structure: `tests/test_<module>/test_<file>.py`
+- Common fixtures in `tests/conftest.py` (spaces, networks, configs)
+- Helper functions in `tests/helper_functions.py`
+- LLM tests marked with `@pytest.mark.llm`
+- Tests use session-scoped fixtures for spaces to avoid recreation overhead
+
+## Branch Workflow
+
+- `main`: Stable releases
+- `nightly`: Active development (PRs target this branch)
+
+## Linting
+
+Uses Ruff for linting and formatting. Configuration in `pyproject.toml` with relaxed rules for test files.

agilerl-2.7.0.dev1/DQN_LEARNING_ALGORITHM_ANALYSIS.md

@@ -0,0 +1,309 @@
+# DQN Learning Algorithm Analysis
+
+## Overview
+Detailed analysis of the DQN learning algorithm implementation, focusing on the `learn()`, `update()`, and `soft_update()` methods.
+
+## Algorithm Flow
+
+### 1. `learn()` Method (lines 338-359)
+
+```python
+def learn(self, experiences: ExperiencesType) -> float:
+    obs = experiences["obs"]
+    actions = experiences["action"]
+    rewards = experiences["reward"]
+    next_obs = experiences["next_obs"]
+    dones = experiences["done"]
+
+    obs = self.preprocess_observation(obs)
+    next_obs = self.preprocess_observation(next_obs)
+
+    loss = self.update(obs, actions, rewards, next_obs, dones)
+
+    # soft update target network
+    self.soft_update()
+    return loss.item()
+```
+
+**Analysis**: ✅ Looks correct
+- Extracts experiences correctly
+- Preprocesses observations
+- Calls `update()` to compute loss and backpropagate
+- Calls `soft_update()` after each learning step
+- Returns scalar loss value
+
+### 2. `update()` Method (lines 286-336)
+
+```python
+def update(self, obs, actions, rewards, next_obs, dones) -> torch.Tensor:
+    with torch.no_grad():
+        if self.double:  # Double Q-learning
+            q_idx = self.actor(next_obs).argmax(dim=1).unsqueeze(1)
+            q_target = (
+                self.actor_target(next_obs).gather(dim=1, index=q_idx).detach()
+            )
+        else:
+            q_target = self.actor_target(next_obs).max(axis=1)[0].unsqueeze(1)
+
+        # target, if terminal then y_j = rewards
+        y_j = rewards + self.gamma * q_target * (1 - dones)
+
+    if actions.ndim == 1:
+        actions = actions.unsqueeze(-1)
+
+    # Compute Q-values for actions taken and loss
+    q_eval = self.actor(obs).gather(1, actions.long())
+    loss: torch.Tensor = self.criterion(q_eval, y_j)
+
+    # zero gradients, perform a backward pass, and update the weights
+    self.optimizer.zero_grad()
+    if self.accelerator is not None:
+        self.accelerator.backward(loss)
+    else:
+        loss.backward()
+
+    self.optimizer.step()
+    return loss.detach()
+```
+
+## Issues Found
+
+### ⚠️ Issue 1: Inconsistent `max()` Usage (Line 316)
+
+**Problem**: Uses `axis=1` instead of `dim=1`
+
+```python
+q_target = self.actor_target(next_obs).max(axis=1)[0].unsqueeze(1)
+```
+
+**Impact**: 
+- PyTorch's `max()` accepts `axis` but it's deprecated
+- Should use `dim=1` for consistency
+- **However**: This shouldn't prevent learning, just causes deprecation warning
+
+**Comparison**:
+- Line 311: Uses `.argmax(dim=1)` ✅ (correct)
+- Line 316: Uses `.max(axis=1)` ❌ (should be `dim=1`)
+
+**Fix**:
+```python
+q_target = self.actor_target(next_obs).max(dim=1)[0].unsqueeze(1)
+```
+
+### ⚠️ Issue 2: Target Network Initialization Method
+
+**Problem**: DQN uses a complex TensorDict-based initialization via `init_hook()`, while other algorithms use simple `load_state_dict()`
+
+**DQN Approach** (lines 185-203):
+```python
+def init_hook(self) -> None:
+    param_vals: TensorDict = from_module(self.actor).detach()
+    target_params: TensorDict = param_vals.clone().lock_()
+    try:
+        target_params.to_module(self.actor_target)
+    except KeyError:
+        pass
+    finally:
+        self.param_vals = param_vals
+        self.target_params = target_params
+```
+
+**RainbowDQN/CQN Approach**:
+```python
+self.actor_target.load_state_dict(self.actor.state_dict())
+```
+
+**Potential Issues**:
+1. The `lock_()` creates a locked TensorDict that's detached from computation graph
+2. If `to_module()` fails silently (caught by `except KeyError: pass`), target network might not be initialized
+3. The locked TensorDict might interfere with `soft_update()` parameter updates
+
+**Impact**: 
+- If `to_module()` fails, target network starts with random weights instead of copying from actor
+- This would cause incorrect Q-targets and prevent learning
+- The silent exception handling makes this hard to detect
+
+**Recommendation**: Add logging or assertion to verify target network is initialized:
+```python
+def init_hook(self) -> None:
+    param_vals: TensorDict = from_module(self.actor).detach()
+    target_params: TensorDict = param_vals.clone().lock_()
+    try:
+        target_params.to_module(self.actor_target)
+    except KeyError as e:
+        # Log the error instead of silently passing
+        warnings.warn(f"Failed to initialize target network: {e}. Using load_state_dict fallback.")
+        self.actor_target.load_state_dict(self.actor.state_dict())
+    finally:
+        self.param_vals = param_vals
+        self.target_params = target_params
+```
+
+### ⚠️ Issue 3: Missing Gradient Clipping
+
+**Problem**: DQN doesn't clip gradients, while RainbowDQN does
+
+**RainbowDQN** (line 442):
+```python
+clip_grad_norm_(self.actor.parameters(), 10.0)
+```
+
+**DQN**: No gradient clipping
+
+**Impact**: 
+- Could lead to gradient explosion in some cases
+- Not necessarily a bug, but could cause instability
+
+**Recommendation**: Consider adding gradient clipping:
+```python
+from torch.nn.utils import clip_grad_norm_
+
+# After loss.backward(), before optimizer.step()
+clip_grad_norm_(self.actor.parameters(), max_norm=10.0)
+self.optimizer.step()
+```
+
+### ✅ Correct Implementations
+
+1. **Q-Learning Update Formula** (line 319): ✅ Correct
+   ```python
+   y_j = rewards + self.gamma * q_target * (1 - dones)
+   ```
+
+2. **Double Q-Learning** (lines 310-314): ✅ Correct
+   - Uses actor to select action, target to evaluate
+
+3. **Loss Computation** (line 326): ✅ Correct
+   ```python
+   q_eval = self.actor(obs).gather(1, actions.long())
+   loss = self.criterion(q_eval, y_j)
+   ```
+
+4. **Gradient Flow** (lines 329-335): ✅ Correct
+   - Zero gradients
+   - Backward pass
+   - Optimizer step
+
+5. **Soft Update** (lines 361-368): ✅ Correct formula
+   ```python
+   target_param.data.copy_(
+       self.tau * eval_param.data + (1.0 - self.tau) * target_param.data
+   )
+   ```
+
+## Potential Learning Issues
+
+### 1. Target Network Not Initialized Properly
+
+**Most Likely Issue**: If `init_hook()` fails silently, target network has random weights, causing:
+- Incorrect Q-targets
+- No learning signal
+- Random behavior
+
+**How to Verify**:
+```python
+# After initialization, check if target network matches actor
+actor_params = list(agent.actor.parameters())
+target_params = list(agent.actor_target.parameters())
+for a, t in zip(actor_params, target_params):
+    if not torch.allclose(a.data, t.data, atol=1e-6):
+        print("WARNING: Target network not initialized correctly!")
+```
+
+### 2. Tau Too Small
+
+**Config**: `TAU: 0.001` (line 18)
+
+**Impact**: 
+- Very slow target network updates
+- Target network stays close to initial values for a long time
+- Slower learning convergence
+
+**Typical Values**: 
+- DQN papers often use `tau=0.01` or `tau=0.005`
+- `tau=0.001` means only 0.1% update per step
+
+**Recommendation**: Try `tau=0.01` or `tau=0.005`
+
+### 3. Learning Rate
+
+**Config**: `LR: 0.001` (line 12)
+
+**Impact**: 
+- Might be too high for some environments
+- Could cause instability
+
+**Typical Values**: 
+- DQN often uses `lr=1e-4` to `lr=5e-4`
+- `lr=0.001` is on the higher side
+
+**Recommendation**: Try `lr=5e-4` or `lr=1e-4`
+
+### 4. Learn Step Frequency
+
+**Config**: `LEARN_STEP: 1` (line 17)
+
+**Impact**: 
+- Learning every step (with 16 parallel envs, that's 16 steps per environment step)
+- Very frequent learning might cause instability
+- Typical DQN learns every 4-5 steps
+
+**Recommendation**: Try `LEARN_STEP: 4` or `LEARN_STEP: 5`
+
+## Summary of Critical Issues
+
+1. **🔴 HIGH PRIORITY**: Target network initialization might fail silently
+   - Check if `init_hook()` actually initializes target network
+   - Add fallback to `load_state_dict()` if TensorDict method fails
+
+2. **🟡 MEDIUM PRIORITY**: Inconsistent `max()` usage
+   - Change `axis=1` to `dim=1` for consistency
+
+3. **🟡 MEDIUM PRIORITY**: Consider adding gradient clipping
+   - Prevents gradient explosion
+
+4. **🟡 MEDIUM PRIORITY**: Hyperparameter tuning
+   - `tau=0.001` might be too small
+   - `lr=0.001` might be too high
+   - `learn_step=1` might be too frequent
+
+## Recommended Fixes
+
+### Fix 1: Improve Target Network Initialization
+
+```python
+def init_hook(self) -> None:
+    """Resets module parameters for the detached and target networks."""
+    param_vals: TensorDict = from_module(self.actor).detach()
+    target_params: TensorDict = param_vals.clone().lock_()
+    
+    try:
+        target_params.to_module(self.actor_target)
+        # Verify initialization succeeded
+        actor_first_param = next(self.actor.parameters()).data
+        target_first_param = next(self.actor_target.parameters()).data
+        if not torch.allclose(actor_first_param, target_first_param, atol=1e-5):
+            raise RuntimeError("Target network initialization verification failed")
+    except (KeyError, RuntimeError) as e:
+        warnings.warn(f"TensorDict initialization failed ({e}), using load_state_dict fallback")
+        self.actor_target.load_state_dict(self.actor.state_dict())
+    finally:
+        self.param_vals = param_vals
+        self.target_params = target_params
+```
+
+### Fix 2: Fix max() Usage
+
+```python
+# Line 316
+q_target = self.actor_target(next_obs).max(dim=1)[0].unsqueeze(1)
+```
+
+### Fix 3: Add Gradient Clipping (Optional)
+
+```python
+# After line 333 (loss.backward())
+from torch.nn.utils import clip_grad_norm_
+clip_grad_norm_(self.actor.parameters(), max_norm=10.0)
+self.optimizer.step()
+```