agilerl 2.7.0.dev1__tar.gz → 2.7.0.dev2__tar.gz

{agilerl-2.7.0.dev1 → agilerl-2.7.0.dev2}/.github/workflows/linux-tests.yml

@@ -73,22 +73,20 @@ jobs:
             - name: Reset coverage data
               run: rm -f .coverage .coverage.*
 
-            - name: Run non-LLM tests (parallel)
-              run: uv run pytest -m "not llm" --exitfirst --cov=agilerl --cov-report= --durations=0 --durations-min=1.0
-
-            - name: Run LLM tests (sequential, exclusive GPU)
-              run: uv run pytest -n0 -m "llm" --exitfirst --cov=agilerl --cov-append --cov-report= --durations=0 --durations-min=1.0
-
-            - name: Combine coverage data
-              run: |
-                  if ls .coverage.* >/dev/null 2>&1; then
-                      uv run coverage combine
-                  else
-                      echo "No parallel coverage shards found; skipping combine."
-                  fi
-
-            - name: Generate coverage report
-              run: uv run coverage xml -o coverage.xml
+            # Single phase: `-n auto --dist loadgroup` from pyproject gives 8
+            # workers on the gha-runner-scale-set node. `vllm`- and `gpu`-marked
+            # tests share the 4 `gputest0..gputest3` xdist groups defined in
+            # `tests/conftest.py`, which caps GPU-touching concurrency at 4
+            # workers regardless of -n; the remaining ~4 workers fan out across
+            # CPU-only tests. See the `pytest_collection_modifyitems` docstring
+            # for the GPU-memory and port-race rationale behind the 4-group cap.
+            #
+            # Single invocation = single coverage run; pytest-cov auto-combines
+            # the per-worker `.coverage.*` shards before writing `coverage.xml`,
+            # so we don't need a manual `coverage combine` step (which was
+            # tripping over corrupted shards under the old two-phase setup).
+            - name: Run tests
+              run: uv run pytest --exitfirst --cov=agilerl --cov-report=xml --durations=0 --durations-min=1.0
 
             - name: Upload coverage reports to Codecov
               uses: codecov/codecov-action@v3

{agilerl-2.7.0.dev1 → agilerl-2.7.0.dev2}/.github/workflows/macos-tests.yml

@@ -42,7 +42,6 @@ jobs:
                   python-version: ${{ matrix.python-version }}
 
             - name: Install dependencies
-              # swig is needed to build box2d-py from source (no pre-built wheels for py3.10+).
               run: |
                   uv sync --locked --all-groups --extra all
 

{agilerl-2.7.0.dev1 → agilerl-2.7.0.dev2}/.gitignore

@@ -149,3 +149,7 @@ wandb/
 
 # Cursor
 .cursor
+
+# AI assistant instructions
+CLAUDE.md
+AGENTS.md

{agilerl-2.7.0.dev1 → agilerl-2.7.0.dev2}/.pre-commit-config.yaml

@@ -1,6 +1,9 @@
 ---
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
+
+# Vendored binary test fixtures; opaque to every hook here.
+exclude: ^tests/assets/tiny_llm/
 ci:
     autoupdate_branch: nightly
     skip: [codeql-python]
@@ -32,7 +35,7 @@ repos:
                 - --skip=*.css,*.js,*.map,*.scss,*.svg
                 - --ignore-words-list=magent,pres,roate
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.15.11
+      rev: v0.15.12
       hooks:
           - id: ruff
             name: Ruff Linter
@@ -47,7 +50,7 @@ repos:
 
     - repo: https://github.com/astral-sh/uv-pre-commit
       # uv version.
-      rev: 0.11.7
+      rev: 0.11.8
       hooks:
           - id: uv-lock
 

{agilerl-2.7.0.dev1 → agilerl-2.7.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agilerl
-Version: 2.7.0.dev1
+Version: 2.7.0.dev2
 Summary: AgileRL is a deep reinforcement learning library focused on improving RL development through RLOps.
 Author-email: Nick Ustaran-Anderegg <dev@agilerl.com>
 License-Expression: Apache-2.0

{agilerl-2.7.0.dev1 → agilerl-2.7.0.dev2}/agilerl/algorithms/core/base.py

@@ -4599,6 +4599,14 @@ class LLMAlgorithm(EvolvableAlgorithm, ABC):
             llm_kwargs["dtype"] = self.vllm_config.dtype
         if self.vllm_config.quantization is not None:
             llm_kwargs["quantization"] = self.vllm_config.quantization
+        if self.vllm_config.kv_cache_memory_bytes is not None:
+            # Forwards to vLLM's ``CacheConfig.kv_cache_memory_bytes``. When set,
+            # vLLM's ``determine_available_memory`` returns early and skips the
+            # memory-profiling assertion that otherwise fails when peer
+            # processes on the same GPU release memory mid-init. This is what
+            # lets the CI run multiple vLLM processes in parallel — see the
+            # ``VLLMConfig.kv_cache_memory_bytes`` docstring for details.
+            llm_kwargs["kv_cache_memory_bytes"] = self.vllm_config.kv_cache_memory_bytes
         try:
             self.llm = LLM(**llm_kwargs)
         except ValueError as err:

{agilerl-2.7.0.dev1 → agilerl-2.7.0.dev2}/agilerl/rollouts/on_policy.py

@@ -1,18 +1,23 @@
 """Functions for collecting rollouts for on-policy algorithms."""
 
-from typing import Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 import torch
 from gymnasium import spaces
 
-from agilerl.algorithms import GRPO, LLMPPO, LLMREINFORCE, PPO
-from agilerl.llm_envs import SyncMultiTurnVecEnv
+from agilerl import HAS_LLM_DEPENDENCIES
+from agilerl.algorithms import PPO
 from agilerl.networks import StochasticActor
 from agilerl.typing import GymEnvType
 
+if TYPE_CHECKING or HAS_LLM_DEPENDENCIES:
+    from agilerl.algorithms import GRPO, LLMPPO, LLMREINFORCE
+    from agilerl.llm_envs import SyncMultiTurnVecEnv
+
 SupportedOnPolicy = PPO
-SupportedOnPolicyLLM = LLMPPO | LLMREINFORCE | GRPO
 
 
 def _collect_rollouts(
@@ -241,7 +246,7 @@ def collect_rollouts_recurrent(
 
 
 def collect_rollouts_llm(
-    agent: SupportedOnPolicyLLM,
+    agent: LLMPPO | LLMREINFORCE | GRPO,
     env: SyncMultiTurnVecEnv,
     n_steps: int,
     batch_size: int,

{agilerl-2.7.0.dev1 → agilerl-2.7.0.dev2}/agilerl/utils/algo_utils.py

@@ -1446,18 +1446,45 @@ class VLLMConfig:
     :param frequency_penalty: Penalise tokens proportionally to how often they have
         appeared so far.  Passed to ``SamplingParams``, defaults to 0.0 (disabled).
     :type frequency_penalty: float, optional
+    :param kv_cache_memory_bytes: Manually pin KV cache size in bytes instead of
+        letting vLLM auto-size from ``gpu_memory_utilization``.  When set, vLLM
+        uses this exact value for the KV cache and skips the auto-sizing path
+        in ``determine_available_memory`` — but ``gpu_memory_utilization`` is
+        **still honoured** by the upfront ``free_memory >= total_memory *
+        gpu_memory_utilization`` startup check in
+        ``vllm/v1/worker/gpu_worker.py:init_device``.  When running multiple
+        vLLM processes concurrently you must keep ``gpu_memory_utilization``
+        small enough that every worker's startup check passes.
+
+        **Required for safe parallel/colocated vLLM**: vLLM's startup
+        ``determine_available_memory`` profile run asserts that GPU free-memory
+        does not increase between the pre- and post-profile snapshots.  When
+        peer processes on the same GPU release memory mid-profile (concurrent
+        xdist workers, sibling CI containers sharing one GPU), the assertion
+        fires with ``Error in memory profiling. Initial free memory ... current
+        free memory ...``.  Setting ``kv_cache_memory_bytes`` triggers vLLM's
+        early-return path in ``determine_available_memory`` and skips that
+        assertion entirely.  CI tests set this to a small value (e.g. 32 MiB)
+        on the tiny test fixture; production deployments running a single
+        vLLM should leave it unset.  Defaults to None.
+    :type kv_cache_memory_bytes: int | None, optional
     """
 
     # Colocate mode parameters
     tensor_parallel_size: int = 1
     gpu_memory_utilization: float = 0.3
     max_num_seqs: int = 8
+    swap_space: float | None = None
+    enforce_eager: bool | None = None
     sleep_mode: bool = False
     dtype: str | None = None
     quantization: str | None = None
     stop_sequences: list[str] | None = None
     presence_penalty: float = 0.0
     frequency_penalty: float = 0.0
+    # See class docstring above. Required to avoid vLLM's memory-profiling
+    # assertion when running multiple vLLM processes on a shared GPU.
+    kv_cache_memory_bytes: int | None = None
 
     def __post_init__(self) -> None:
         if self.sleep_mode:

{agilerl-2.7.0.dev1 → agilerl-2.7.0.dev2}/agilerl/utils/torch_utils.py

@@ -123,7 +123,7 @@ def get_transformer_logs(
 
 # --------------------------------------------------------------------------- #
 # Distribution helpers (Discrete, Box, MultiDiscrete, MultiBinary)            #
-# Used by TorchDistribution in networks/distributions_experimental.py         #
+# Used by TorchDistribution in networks/distributions.py         #
 # --------------------------------------------------------------------------- #
 
 

{agilerl-2.7.0.dev1 → agilerl-2.7.0.dev2}/agilerl/utils/utils.py

@@ -3,7 +3,7 @@ import warnings
 from collections.abc import Callable
 from datetime import datetime
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import gymnasium as gym
 import matplotlib.pyplot as plt
@@ -22,8 +22,6 @@ from agilerl.algorithms import (
     DDPG,
     DQN,
     IPPO,
-    LLMPPO,
-    LLMREINFORCE,
     MADDPG,
     MATD3,
     PPO,
@@ -41,8 +39,8 @@ from agilerl.typing import BPTTSequenceType, GymSpaceType, PopulationType
 from agilerl.utils.algo_utils import CosineLRScheduleConfig, DummyOptimizer, clone_llm
 from agilerl.vector.pz_async_vec_env import AsyncPettingZooVecEnv
 
-if HAS_LLM_DEPENDENCIES:
-    from agilerl.algorithms import CISPO, DPO, GRPO, GSPO, SFT
+if HAS_LLM_DEPENDENCIES or TYPE_CHECKING:
+    from agilerl.algorithms import CISPO, DPO, GRPO, GSPO, LLMPPO, LLMREINFORCE, SFT
     from agilerl.utils.llm_utils import get_llm_accelerator, get_state_dict
 
 SupportedObservationSpace = spaces.Box | spaces.Discrete | spaces.Dict | spaces.Tuple

{agilerl-2.7.0.dev1 → agilerl-2.7.0.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agilerl"
-version = "2.7.0.dev1"
+version = "2.7.0.dev2"
 description = "AgileRL is a deep reinforcement learning library focused on improving RL development through RLOps."
 authors = [{ name = "Nick Ustaran-Anderegg", email = "dev@agilerl.com" }]
 license = "Apache-2.0"
@@ -104,15 +104,15 @@ filterwarnings = [
     "ignore::pyparsing.exceptions.PyparsingDeprecationWarning",
 ]
 markers = [
-    "llm: LLM-related tests (exclude with -m 'not llm')",
+    "vllm: tests that actually instantiate vllm.LLM (exclude with -m 'not vllm'). Pinned to a single xdist worker because vLLM's GPU memory profiling fails when peer workers free memory mid-init.",
+    "gpu: tests that use a GPU (CUDA, DeepSpeed, etc.) but do NOT initialise real vLLM (exclude with -m 'not gpu'). Safe to fan out across multiple xdist workers.",
 ]
 
 [tool.coverage.run]
 parallel = true
 source = ["agilerl"]
-concurrency = ["multiprocessing"]
+concurrency = ["multiprocessing", "thread"]
 sigterm = true
-omit = ["agilerl/networks/distributions_experimental.py"]
 
 [tool.coverage.report]
 exclude_lines = [
@@ -147,6 +147,7 @@ exclude = ["tutorials"]
 [tool.ruff]
 target-version = "py310"
 line-length = 88
+extend-exclude = ["tests/assets/tiny_llm"]
 
 [tool.ruff.lint]
 select = ["ALL"]

agilerl-2.7.0.dev2/tests/__init__.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+# Vendored tiny LLM (Qwen2 architecture) for offline testing.
+# Generated by tests/assets/build_tiny_llm_fixture.py.
+TINY_LLM_FIXTURE_PATH = str(Path(__file__).parent / "assets" / "tiny_llm")

agilerl-2.7.0.dev2/tests/assets/build_minari_fixture.py

@@ -0,0 +1,67 @@
+"""Build the Minari test fixture used by tests/test_utils/test_minari_utils.py.
+
+Downloads ``D4RL/door/human-v2`` from the Farama/HuggingFace registry once and
+saves it under ``tests/assets/minari_cache/`` so subsequent test runs can load
+the dataset offline (``HF_HUB_OFFLINE=1`` is set globally in conftest.py).
+
+The download writes ``.cache/huggingface/`` metadata alongside the dataset; we
+prune it because Minari's offline loader doesn't need it and it inflates the
+fixture size.
+
+Run when the upstream dataset is updated, when fixture files are missing, or
+when Minari format changes. The output directory should be committed to the
+repo (a few MB).
+
+Usage::
+
+    uv run python tests/assets/build_minari_fixture.py
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+
+DATASET_ID = "D4RL/door/human-v2"
+FIXTURE_DIR = Path(__file__).resolve().parent / "minari_cache"
+SIZE_BUDGET_MB = 10
+
+
+def main() -> None:
+    if FIXTURE_DIR.exists():
+        shutil.rmtree(FIXTURE_DIR)
+    FIXTURE_DIR.mkdir(parents=True)
+
+    os.environ["MINARI_DATASETS_PATH"] = str(FIXTURE_DIR)
+    # Allow network for this build only; HF_HUB_OFFLINE is on globally for tests.
+    os.environ.pop("HF_HUB_OFFLINE", None)
+    os.environ.pop("TRANSFORMERS_OFFLINE", None)
+
+    import minari
+
+    print(f"Downloading {DATASET_ID} to {FIXTURE_DIR}...")
+    minari.download_dataset(DATASET_ID)
+
+    # Prune HF download metadata — not needed for offline loading.
+    for cache_dir in FIXTURE_DIR.rglob(".cache"):
+        if cache_dir.is_dir():
+            shutil.rmtree(cache_dir)
+
+    total_bytes = sum(p.stat().st_size for p in FIXTURE_DIR.rglob("*") if p.is_file())
+    total_mb = total_bytes / (1024 * 1024)
+    print(f"Fixture size: {total_mb:.2f} MB")
+
+    if total_mb > SIZE_BUDGET_MB:
+        msg = (
+            f"Fixture exceeds {SIZE_BUDGET_MB} MB budget ({total_mb:.2f} MB). "
+            "Aborting — investigate before committing."
+        )
+        raise RuntimeError(msg)
+
+    print("Done. Commit the fixture directory:")
+    print(f"  git add {FIXTURE_DIR.relative_to(Path.cwd())}")
+
+
+if __name__ == "__main__":
+    main()

agilerl-2.7.0.dev2/tests/assets/build_tiny_llm_fixture.py

@@ -0,0 +1,129 @@
+"""Build the tiny LLM test fixture used by unit tests.
+
+Constructs a randomly-initialised Qwen2 model with the upstream
+``trl-internal-testing/tiny-Qwen2ForCausalLM-2.5`` tokenizer and saves it
+under ``tests/assets/tiny_llm/`` so subsequent test runs can load the model
+offline via ``from_pretrained(<local_path>)``.
+
+Why we don't just download the upstream model: it ships with
+``hidden_size=8`` / ``num_attention_heads=4``, giving ``head_dim=2``. vLLM
+imposes a per-backend lower bound on ``head_dim`` that the upstream model
+violates:
+
+- **GPU FlexAttention backend** (used on GPUs without FA2, i.e. compute
+  capability < 8) requires ``head_dim >= 16``.
+- **CPU backend** (`_PagedAttention` in ``vllm/v1/attention/backends/cpu_attn.py``,
+  used on macOS / Linux-without-CUDA) only accepts head sizes from a fixed
+  whitelist: ``{32, 64, 80, 96, 112, 128, 192, 256}``. A ``head_dim`` outside
+  this set fails at runtime with ``RuntimeError: Unsupported head size: N``
+  during ``paged_attention_v1``.
+
+We construct our own Qwen2 with ``head_dim=32`` (the smallest value satisfying
+both backends) so the same fixture is usable across CI matrices and local
+macOS development.
+
+Run when transformers/safetensors compatibility shifts, or when the model
+config needs adjusting. The output directory should be committed to the repo
+(<25 MB).
+
+Usage::
+
+    uv run python tests/assets/build_tiny_llm_fixture.py
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+from pathlib import Path
+
+import torch
+from transformers import AutoTokenizer, Qwen2Config, Qwen2ForCausalLM
+
+UPSTREAM_TOKENIZER_ID = "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5"
+FIXTURE_DIR = Path(__file__).resolve().parent / "tiny_llm"
+SIZE_BUDGET_MB = 25
+# transformers >=4.57.3 flags any local non-mistral tokenizer with vocab >100k
+# as a "broken Mistral regex" unless the saved transformers_version is <=4.57.2.
+# Pin the saved version so AutoTokenizer doesn't emit the false-positive warning.
+PINNED_TRANSFORMERS_VERSION = "4.57.2"
+
+# head_dim = hidden_size // num_attention_heads must satisfy BOTH backends:
+#   - vLLM GPU FlexAttention (no-FA2 GPUs):   head_dim >= 16
+#   - vLLM CPU PagedAttention (macOS, etc.):  head_dim in
+#     {32, 64, 80, 96, 112, 128, 192, 256}
+# 32 is the smallest head_dim satisfying both. To stay under the 25 MB fixture
+# budget (tokenizer alone is ~11 MB, embeddings dominate model weights at this
+# scale because tie_word_embeddings shares them with lm_head), we keep
+# hidden_size=32 and use a single head so head_dim = 32 / 1 = 32. The single-
+# head config still exercises the full Qwen2 forward/backward path; head count
+# is an implementation detail that the LLM-level tests (DPO/GRPO/SFT/REINFORCE)
+# don't probe.
+HIDDEN_SIZE = 32
+NUM_ATTENTION_HEADS = 1  # head_dim = 32
+NUM_KEY_VALUE_HEADS = 1
+NUM_HIDDEN_LAYERS = 2
+INTERMEDIATE_SIZE = 64
+SEED = 0
+
+
+def main() -> None:
+    if FIXTURE_DIR.exists():
+        shutil.rmtree(FIXTURE_DIR)
+    FIXTURE_DIR.mkdir(parents=True)
+
+    print(f"Loading tokenizer from {UPSTREAM_TOKENIZER_ID}...")
+    tokenizer = AutoTokenizer.from_pretrained(UPSTREAM_TOKENIZER_ID)
+
+    print("Constructing tiny Qwen2 model from scratch...")
+    config = Qwen2Config(
+        vocab_size=len(tokenizer),
+        hidden_size=HIDDEN_SIZE,
+        intermediate_size=INTERMEDIATE_SIZE,
+        num_hidden_layers=NUM_HIDDEN_LAYERS,
+        num_attention_heads=NUM_ATTENTION_HEADS,
+        num_key_value_heads=NUM_KEY_VALUE_HEADS,
+        max_position_embeddings=32768,
+        rope_theta=10000.0,
+        rms_norm_eps=1e-6,
+        tie_word_embeddings=True,
+        torch_dtype="float32",
+        bos_token_id=151643,
+        eos_token_id=151645,
+    )
+    torch.manual_seed(SEED)
+    model = Qwen2ForCausalLM(config)
+    # Store weights in float16 to keep the fixture under the size budget.
+    # Tests run under bf16/fp16 anyway (DeepSpeed bf16, vLLM fp16 downcast)
+    # so this matches the working dtype and avoids any precision surprise.
+    model = model.to(torch.float16)
+
+    print(f"Saving fixture to {FIXTURE_DIR}...")
+    model.save_pretrained(FIXTURE_DIR, safe_serialization=True)
+    tokenizer.save_pretrained(FIXTURE_DIR)
+    # Drop redundant slow-tokenizer files; the fast tokenizer.json is sufficient.
+    for legacy in ("vocab.json", "merges.txt"):
+        (FIXTURE_DIR / legacy).unlink(missing_ok=True)
+
+    config_path = FIXTURE_DIR / "config.json"
+    config_dict = json.loads(config_path.read_text())
+    config_dict["transformers_version"] = PINNED_TRANSFORMERS_VERSION
+    config_path.write_text(json.dumps(config_dict, indent=2) + "\n")
+
+    total_bytes = sum(p.stat().st_size for p in FIXTURE_DIR.rglob("*") if p.is_file())
+    total_mb = total_bytes / (1024 * 1024)
+    print(f"Fixture size: {total_mb:.2f} MB")
+
+    if total_mb > SIZE_BUDGET_MB:
+        msg = (
+            f"Fixture exceeds {SIZE_BUDGET_MB} MB budget ({total_mb:.2f} MB). "
+            "Aborting — investigate before committing."
+        )
+        raise RuntimeError(msg)
+
+    print("Done. Commit the fixture directory:")
+    print(f"  git add {FIXTURE_DIR.relative_to(Path.cwd())}")
+
+
+if __name__ == "__main__":
+    main()

agilerl-2.7.0.dev2/tests/assets/minari_cache/D4RL/door/human-v2/data/metadata.json

@@ -0,0 +1 @@
+{"env_spec": "{\"id\": \"AdroitHandDoor-v1\", \"entry_point\": \"gymnasium_robotics.envs.adroit_hand.adroit_door:AdroitHandDoorEnv\", \"reward_threshold\": null, \"nondeterministic\": false, \"max_episode_steps\": 200, \"order_enforce\": true, \"disable_env_checker\": false, \"kwargs\": {\"reward_type\": \"dense\"}, \"additional_wrappers\": [], \"vector_entry_point\": null}", "dataset_id": "D4RL/door/human-v2", "author": ["Rodrigo de Lazcano"], "author_email": ["rperezvicente@farama.org"], "code_permalink": "https://github.com/rodrigodelazcano/d4rl-minari-dataset-generation", "minari_version": "0.4.3", "ref_max_score": 2940.578369140625, "ref_min_score": -45.80706024169922, "num_episodes_average_score": 100, "total_episodes": 25, "total_steps": 6729, "data_format": "hdf5", "dataset_size": 3.5, "description": "25 human demonstrations provided in the [DAPG](https://github.com/aravindr93/hand_dapg) repository. The environment used to collect the dataset is [`AdroitHandDoor-v1`](https://robotics.farama.org/envs/adroit_hand/adroit_door/).", "requirements": ["gymnasium-robotics>=1.2.4"], "action_space": "{\"type\": \"Box\", \"dtype\": \"float32\", \"shape\": [28], \"low\": [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], \"high\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}", "observation_space": "{\"type\": \"Box\", \"dtype\": \"float64\", \"shape\": [39], \"low\": [-Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity, -Infinity], \"high\": [Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity]}"}

agilerl-2.7.0.dev2/tests/assets/minari_cache/D4RL/door/namespace_metadata.json

@@ -0,0 +1 @@
+{"display_name": "Door", "description": "These datasets were generated with the [`AdroitHandDoor-v1`](https://robotics.farama.org/envs/adroit_hand/adroit_door/) environment, originally hosted in the [`hand_dapg`](https://github.com/aravindr93/hand_dapg) repository. The objective of the task is to open a door with a 24-DoF robotic hand. This domain was selected to measure the effect of a narrow expert data distributions and human demonstrations on a sparse reward, high-dimensional robotic manipulation task.\n\nThere are three types of datasets, two from the original paper[1] (`human` and `expert`), and another one introduced in D4RL[2] (`cloned`).\n\n## References\n\n[1] Rajeswaran, Aravind, et al. \u2018Learning Complex Dexterous Manipulation with Deep Reinforcement Learning and Demonstrations\u2019. CoRR, vol. abs/1709.10087, 2017, http://arxiv.org/abs/1709.10087.\n\n[2] Fu, Justin, et al. \u2018D4RL: Datasets for Deep Data-Driven Reinforcement Learning\u2019. CoRR, vol. abs/2004.07219, 2020, https://arxiv.org/abs/2004.07219."}

agilerl-2.7.0.dev2/tests/assets/minari_cache/D4RL/namespace_metadata.json

@@ -0,0 +1 @@
+{"description": "The D4RL dataset group contains a reproduction of the datasets from the [D4RL benchmark](https://github.com/Farama-Foundation/D4RL)[1]. For reproducibility purposes, not all the datasets are the same as in D4RL, but they are generated with the same principles. We provide the code that reproduces each dataset on GitHub in the repository [Farama-Foundation/minari-dataset-generation-scripts](https://github.com/Farama-Foundation/minari-dataset-generation-scripts).\n\n## References\n\n[1] Fu, Justin, et al. \u2018D4RL: Datasets for Deep Data-Driven Reinforcement Learning\u2019. CoRR, vol. abs/2004.07219, 2020, https://arxiv.org/abs/2004.07219.", "namespaces": ["D4RL/door", "D4RL/kitchen", "D4RL/hammer", "D4RL/pointmaze", "D4RL/pen", "D4RL/minigrid", "D4RL/antmaze", "D4RL/relocate"], "datasets": ["D4RL/antmaze/medium-play-v1", "D4RL/pointmaze/open-dense-v2", "D4RL/pen/human-v2", "D4RL/kitchen/partial-v2", "D4RL/antmaze/umaze-diverse-v1", "D4RL/pointmaze/umaze-v2", "D4RL/pen/cloned-v2", "D4RL/door/human-v2", "D4RL/pointmaze/large-dense-v2", "D4RL/hammer/cloned-v2", "D4RL/relocate/human-v2", "D4RL/pointmaze/medium-v2", "D4RL/pointmaze/umaze-dense-v2", "D4RL/hammer/expert-v2", "D4RL/door/cloned-v2", "D4RL/pointmaze/medium-dense-v2", "D4RL/door/expert-v2", "D4RL/minigrid/fourrooms-random-v0", "D4RL/pen/expert-v2", "D4RL/relocate/expert-v2", "D4RL/antmaze/large-diverse-v1", "D4RL/antmaze/large-play-v1", "D4RL/minigrid/fourrooms-v0", "D4RL/pointmaze/large-v2", "D4RL/antmaze/medium-diverse-v1", "D4RL/antmaze/umaze-v1", "D4RL/hammer/human-v2", "D4RL/kitchen/complete-v2", "D4RL/kitchen/mixed-v2", "D4RL/pointmaze/open-v2", "D4RL/relocate/cloned-v2"]}

agilerl-2.7.0.dev2/tests/assets/tiny_llm/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

agilerl-2.7.0.dev2/tests/assets/tiny_llm/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

agilerl-2.7.0.dev2/tests/assets/tiny_llm/config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "dtype": "float16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 32,
+  "initializer_range": 0.02,
+  "intermediate_size": 64,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 1,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151665
+}

agilerl-2.7.0.dev2/tests/assets/tiny_llm/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "transformers_version": "4.57.6"
+}

agilerl-2.7.0.dev2/tests/assets/tiny_llm/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}