zeroRl 0.1.0__tar.gz

zerorl-0.1.0/.github/workflows/security.yml

@@ -0,0 +1,25 @@
+name: Security & Supply Chain
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+permissions:
+  contents: read
+  security-events: write
+
+jobs:
+  security-checks:
+    name: Vulnerability Scan
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Generate SBOM
+        uses: anchore/sbom-action@v0
+        with:
+          format: spdx-json
+          output-file: zerorl.sbom.json

zerorl-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,57 @@
+name: Tests
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  cpu-tests:
+    name: CPU Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Sync dependencies
+        run: uv sync --all-extras --dev
+
+      - name: Run CPU Tests
+        run: uv run pytest tests/ -m "not gpu" --cov=zerorl
+
+  gpu-execution:
+    name: GPU Integration Tests
+    needs: cpu-tests
+    runs-on: [self-hosted, gpu, jetson]
+    environment: gpu-cluster
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Setup UV and Python
+        uses: astral-sh/setup-uv@v6
+        
+      - name: Sync dependencies
+        # uv va installer les paquets directement dans le système du Jetson
+        run: uv sync --all-extras --dev
+
+      - name: Run GPU Tests
+        run: uv run pytest tests/ -m "gpu"

zerorl-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+.opencode
+uv.lock

zerorl-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+  - repo: local
+    hooks:
+      - id: ruff
+        name: ruff
+        entry: uv run ruff check .
+        language: system
+        types: [python]
+        always_run: true
+      - id: mypy
+        name: mypy
+        entry: uv run mypy
+        language: system
+        types: [python]
+        always_run: true

zerorl-0.1.0/AGENTS.md

@@ -0,0 +1,131 @@
+# AGENTS.md
+
+## Project
+
+RL training framework built on PyTorch + Gymnasium. Provides abstract base classes (`BaseAgent`, `BaseEnv`, `BaseTrain`) and standalone PPO functions (`gae_compute`, `ppo_loss`, `ppo`). Includes 88 tests, all passing (1 xfailed documenting a known vectorized-reset bug).
+
+## Package management
+
+- Managed with **uv** (lockfile: `uv.lock`, Python version: `.python-version` → 3.11)
+- Build system: **hatchling** (declared in `pyproject.toml`)
+- Dependencies pinned in `requirements.txt`; runtime deps declared in `pyproject.toml` (`torch`, `numpy`, `gymnasium`, `tensorboard`, `wandb`)
+
+## Lint / typecheck
+
+```
+ruff check zerorl/ tests/   # linter (Pyflakes only, pyproject.toml)
+mypy zerorl/                # strict: disallow_untyped_defs, warn_unreachable (pyproject.toml)
+```
+
+No pre-commit hooks, no CI, no formatter config beyond ruff defaults.
+
+## Running tests on this machine
+
+- GPU: GTX 1050 Ti (sm_61); installed torch cu130 has no sm_61 kernels → CUDA ops fail
+- Run CPU-only: `CUDA_VISIBLE_DEVICES="" uv run python -m pytest tests/ -v`
+- Tests are marked `@pytest.mark.gpu`; the marker is informational only (no auto-deselect)
+
+## Import style
+
+- `train.py` uses **absolute imports** (`from zerorl.agent import BaseAgent`)
+- `ppo.py` uses **relative imports** (`from ...common import Buffer`, `from ...config import AlgoConfig`)
+- `algorithms/__init__.py` is empty; `algorithms/ppo/__init__.py` re-exports `ppo` and `gae_compute`
+
+## Structure
+
+```
+zerorl/
+  __init__.py              # empty package marker
+  agent.py                 # BaseAgent (ABC, nn.Module), eval_action()
+  env.py                   # BaseEnv (ABC, gym.Env) — Gymnasium v1 API wrapper; register_env() decorator
+  vector_env.py             # VectorEnv(BaseEnv) — vectorized env wrapper (env_id string or callable class)
+  train.py                 # BaseTrain — rollout/update/save training loop
+  common.py                # Buffer — dict-based pre-allocated torch buffer
+  config.py                # TrainConfig (computed fields), AlgoConfig (mutable dataclass)
+  errors.py                # EmptyBufferError
+  function.py              # linear_schedule(), get_buffer_params_model()
+  processing.py            # NormMeanStd, NormMinMax
+  algorithms/
+    __init__.py            # empty
+    ppo/
+      __init__.py          # re-exports ppo, gae_compute
+      ppo.py               # gae_compute(), ppo_loss(), ppo() — standalone functions
+tests/
+  __init__.py
+  test_agent.py              # 6 tests
+  test_buffer_integration.py # 4 tests
+  test_common.py             # 6 tests
+  test_config.py             # 15 tests
+  test_continuous_actions.py # 1 test
+  test_env.py                # 5 tests
+  test_errors.py             # 7 tests
+  test_function.py           # 4 tests
+  test_ppo.py                # 16 tests
+  test_processing.py         # 3 tests
+  test_train.py              # 13 tests
+  test_ppo_vector.py         # 1 test
+  test_env_vector.py         # 7 tests
+```
+
+## Key APIs
+
+### Buffer
+
+```python
+Buffer(step: int, data: dict[str, tuple], num_envs: int = 1, device: torch.device)
+```
+
+Constructor takes a dict mapping field names to shapes (e.g., `{"state": (4,), "actions": ()}`). Internally pre-allocates torch tensors of shape `(step, num_envs, *shape)` on the given device. `get_all()` returns a dict of sliced tensors (same keys). No `insert_returns()` method — callers insert GAE-computed returns/advantages directly.
+
+### BaseAgent
+
+Abstract methods: `forward(state) -> tuple[Tensor, Tensor]` and `build_distribution(logits) -> Distribution` (static).
+
+`get_action(state, action=None)` is a template method that calls `forward()` → `build_distribution()` → samples or evaluates → returns `dict[str, Tensor]` with keys `"action"`, `"log_prob"`, `"entropy"`, `"value"`.
+
+### PPO (standalone functions)
+
+- `gae_compute(rewards, values, last_value, dones, hyper_params) -> (returns, advantages, deltas)`
+- `ppo_loss(agent, params, buffers, states, actions, old_log_probs, advantages, returns, hyper_params) -> dict[str, Tensor]`
+- `ppo(agent, optimizer, buffer, hyper_params, scheduler, batch_size, epochs, device) -> dict[str, Tensor]`
+
+No `PPOTrainer` class — PPO is composed as standalone functions that accept an `AlgoConfig` and a `Buffer`.
+
+### Config
+
+- `AlgoConfig`: Mutable dataclass with custom `__init__` (init=False). Fields: `lr`, `gamma`, `batch_size`, `gae_lambda`, `clip_eps`, `ent_coef`, `value_coef`, `epochs`, `tau`.
+- `TrainConfig`: Mutable dataclass with `__post_init__` for computed fields (`model_path`, `num_update`, `device`). Fields: `model_name`, `model_save_path`, `timestamp`, `rollout_steps`, `num_envs`. `num_update` is computed as `timestamp // (rollout_steps * num_envs)` — overriding `rollout_steps`/`timestamp`/`num_envs` after construction does NOT recompute `num_update`; set it manually if needed.
+
+No `PPOConfig` (replaced by `AlgoConfig`). No `WandbConfig`.
+
+### BaseTrain
+
+```python
+BaseTrain(agent, env, buffer, update_weights, train_config, algo_config=None, optimizer=None, require_buffer_size=10)
+```
+
+`update_weights` is a `Callable[[BaseAgent, Buffer, Optimizer, int, dict[str,Tensor], AlgoConfig | None], dict[str, Tensor]]`. Methods: `rollout_phase(state)`, `_log_metrics(metrics, step, use_wandb, use_tb)`, `train(use_wandb, use_tb)`, `save_model()`.
+
+### Utility modules
+
+- `function.py`: `linear_schedule(step, num_update)`, `get_buffer_params_model(model) -> (params, buffers)`
+- `processing.py`: `NormMeanStd(shape, device, epsilon)`, `NormMinMax(low, high, device)` — both have `normalize(x)` and `NormMeanStd` has `update(x)`
+- `env.py`: `register_env(env_id)` decorator registers a class with Gymnasium
+- `vector_env.py`: `VectorEnv(env_spec, num_envs)` — wraps `env_spec` (string env_id or callable class) into a vectorized env
+
+## Conventions
+
+- Abstract base classes use `@abstractmethod` with concrete bodies (template method pattern) — subclasses override and optionally call `super()`
+- `Buffer` pre-allocates torch tensors and uses a slice pointer with a `size` property and bounds checking in `insert()`
+- GPU detection is automatic in `TrainConfig.device`
+- Tests use pytest, import zerorl as a package (`from zerorl.common import Buffer`)
+- `train.py` includes wandb and tensorboard logging via `_log_metrics()`
+
+## Testing
+
+```
+CUDA_VISIBLE_DEVICES="" uv run python -m pytest tests/ -v    # run all 88 tests (CPU-only on this machine)
+```
+
+- 87 pass, 1 xfailed (documents vectorized reset bug at `train.py:141-145`: all envs reset when any finishes)
+- `test_train.py::test_rollout_phase_partial_finish_preserves_survivors` is the xfail test

zerorl-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 0xDar_rius
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.