stable-baselines3 2.9.0a2__tar.gz → 2.9.2__tar.gz

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/NOTICE

@@ -2,7 +2,7 @@ Large portion of the code of Stable-Baselines3 (in `common/`) were ported from S
 both licensed under the MIT License:
 
 before the fork (June 2018):
-Copyright (c) 2017 OpenAI (http://openai.com)
+Copyright (c) 2017 OpenAI (https://openai.com)
 
 after the fork (June 2018):
 Copyright (c) 2018-2019 Stable-Baselines Team

{stable_baselines3-2.9.0a2/stable_baselines3.egg-info → stable_baselines3-2.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stable_baselines3
-Version: 2.9.0a2
+Version: 2.9.2
 Summary: Pytorch version of Stable Baselines, implementations of reinforcement learning algorithms.
 Home-page: https://github.com/DLR-RM/stable-baselines3
 Author: Antonin Raffin
@@ -24,14 +24,12 @@ License-File: LICENSE
 License-File: NOTICE
 Requires-Dist: gymnasium<2.0,>=0.29.1
 Requires-Dist: numpy<3.0,>=1.20
-Requires-Dist: torch<3.0,>=2.3
+Requires-Dist: torch<3.0,>=2.8
 Requires-Dist: cloudpickle
 Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
 Requires-Dist: pytest-cov; extra == "tests"
-Requires-Dist: pytest-env; extra == "tests"
-Requires-Dist: pytest-xdist; extra == "tests"
-Requires-Dist: mypy<2,>=1.9.0; extra == "tests"
+Requires-Dist: mypy<3,>=1.9.0; extra == "tests"
 Requires-Dist: ruff>=0.5.6; extra == "tests"
 Requires-Dist: black<27,>=26.1.0; extra == "tests"
 Provides-Extra: docs
@@ -116,7 +114,7 @@ vec_env = model.get_env()
 obs = vec_env.reset()
 for i in range(1000):
     action, _states = model.predict(obs, deterministic=True)
-    obs, reward, done, info = vec_env.step(action)
+    obs, rewards, dones, info = vec_env.step(action)
     vec_env.render()
     # VecEnv resets automatically
     # if done:

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/README.md

@@ -100,7 +100,7 @@ It provides a minimal number of features compared to SB3 but can be much faster
 
 ## Installation
 
-**Note:** Stable-Baselines3 supports PyTorch >= 2.3
+**Note:** Stable-Baselines3 supports PyTorch >= 2.8
 
 ### Prerequisites
 Stable Baselines3 requires Python 3.10+.
@@ -142,8 +142,8 @@ model.learn(total_timesteps=10_000)
 vec_env = model.get_env()
 obs = vec_env.reset()
 for i in range(1000):
-    action, _states = model.predict(obs, deterministic=True)
-    obs, reward, done, info = vec_env.step(action)
+    actions, _states = model.predict(obs, deterministic=True)
+    obs, rewards, dones, info = vec_env.step(actions)
     vec_env.render()
     # VecEnv resets automatically
     # if done:
@@ -182,23 +182,28 @@ All the following examples can be executed online using Google Colab notebooks:
 
 | **Name**         | **Recurrent**      | `Box`          | `Discrete`     | `MultiDiscrete` | `MultiBinary`  | **Multi Processing**              |
 | ------------------- | ------------------ | ------------------ | ------------------ | ------------------- | ------------------ | --------------------------------- |
-| ARS<sup>[1](#f1)</sup>   | :x: | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: |
 | A2C   | :x: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
-| CrossQ<sup>[1](#f1)</sup>   | :x: | :heavy_check_mark: | :x:                | :x:                 | :x:                | :heavy_check_mark: |
+| ARS<sup>[1](#f1)</sup>   | :x: | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: |
+| CrossQ<sup>[1](#f1)[2](#f2)</sup>   | :x: | :heavy_check_mark: | :x:                | :x:                 | :x:                | :heavy_check_mark: |
 | DDPG  | :x: | :heavy_check_mark: | :x:                | :x:                 | :x:                | :heavy_check_mark: |
 | DQN   | :x: | :x: | :heavy_check_mark: | :x:                 | :x:                | :heavy_check_mark: |
+| DroQ<sup>[2](#f2)</sup>  | :x: | :heavy_check_mark: | :x:                | :x:                 | :x:                | :heavy_check_mark: |
 | HER   | :x: | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: |
+| Maskable PPO<sup>[1](#f1)</sup>   | :x: | :x: | :heavy_check_mark: | :heavy_check_mark:  | :heavy_check_mark: | :heavy_check_mark: |
 | PPO   | :x: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:  | :heavy_check_mark: | :heavy_check_mark: |
 | QR-DQN<sup>[1](#f1)</sup>  | :x: | :x: | :heavy_check_mark: | :x:                 | :x:                | :heavy_check_mark: |
 | RecurrentPPO<sup>[1](#f1)</sup>   | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:  | :heavy_check_mark: | :heavy_check_mark: |
 | SAC   | :x: | :heavy_check_mark: | :x:                | :x:                 | :x:                | :heavy_check_mark: |
+| SAC-N<sup>[2](#f2)</sup> | :x: | :heavy_check_mark: | :x:                | :x:                 | :x:                | :heavy_check_mark: |
+| SimBa<sup>[2](#f2)</sup> | :x: | :heavy_check_mark: | :x:                | :x:                 | :x:                | :heavy_check_mark: |
 | TD3   | :x: | :heavy_check_mark: | :x:                | :x:                 | :x:                | :heavy_check_mark: |
 | TQC<sup>[1](#f1)</sup>   | :x: | :heavy_check_mark: | :x:                | :x:                 | :x: | :heavy_check_mark: |
 | TRPO<sup>[1](#f1)</sup>  | :x: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:  | :heavy_check_mark: | :heavy_check_mark: |
-| Maskable PPO<sup>[1](#f1)</sup>   | :x: | :x: | :heavy_check_mark: | :heavy_check_mark:  | :heavy_check_mark: | :heavy_check_mark:  |
 
 <b id="f1">1</b>: Implemented in [SB3 Contrib](https://github.com/Stable-Baselines-Team/stable-baselines3-contrib) GitHub repository.
 
+<b id="f2">2</b>: Implemented in [SBX (SB3 + Jax)](https://github.com/araffin/sbx)
+
 Actions `gymnasium.spaces`:
  * `Box`: A N-dimensional box that contains every point in the action space.
  * `Discrete`: A list of possible actions, where only one action can be used per timestep.
@@ -256,7 +261,7 @@ To cite this repository in publications:
   volume  = {22},
   number  = {268},
   pages   = {1-8},
-  url     = {http://jmlr.org/papers/v22/20-1364.html}
+  url     = {https://jmlr.org/papers/v22/20-1364.html}
 }
 ```
 
@@ -264,7 +269,7 @@ Note: If you need to refer to a specific version of SB3, you can also use the [Z
 
 ## Maintainers
 
-Stable-Baselines3 is currently maintained by [Ashley Hill](https://github.com/hill-a) (aka @hill-a), [Antonin Raffin](https://araffin.github.io/) (aka [@araffin](https://github.com/araffin)), [Maximilian Ernestus](https://github.com/ernestum) (aka @ernestum), [Adam Gleave](https://github.com/adamgleave) (@AdamGleave), [Anssi Kanervisto](https://github.com/Miffyli) (@Miffyli) and [Quentin Gallouédec](https://gallouedec.com/) (@qgallouedec).
+Stable-Baselines3 is currently maintained by [Ashley Hill](https://github.com/hill-a) (aka @hill-a), [Antonin Raffin](https://araffin.github.io/) (aka [@araffin](https://github.com/araffin)), [Maximilian Ernestus](https://github.com/ernestum) (aka @ernestum), [Adam Gleave](https://github.com/adamgleave) (@AdamGleave), [Anssi Kanervisto](https://github.com/Miffyli) (@Miffyli) and [Quentin Gallouédec](https://github.com/qgallouedec) (@qgallouedec).
 
 **Important Note: We do not provide technical support, or consulting** and do not answer personal questions via email.
 Please post your question on the [RL Discord](https://discord.com/invite/xhfNqQv), [Reddit](https://www.reddit.com/r/reinforcementlearning/), or [Stack Overflow](https://stackoverflow.com/) in that case.
@@ -279,7 +284,7 @@ If you want to contribute, please read [**CONTRIBUTING.md**](./CONTRIBUTING.md)
 
 The initial work to develop Stable Baselines3 was partially funded by the project *Reduced Complexity Models* from the *Helmholtz-Gemeinschaft Deutscher Forschungszentren*, and by the EU Horizon 2020 Research and Innovation Programme under grant number 951992 ([VeriDream](https://www.veridream.eu/)).
 
-The original version, Stable Baselines, was created in the [robotics lab U2IS](http://u2is.ensta-paristech.fr/index.php?lang=en) ([INRIA Flowers](https://flowers.inria.fr/) team) at [ENSTA ParisTech](http://www.ensta-paristech.fr/en).
+The original version, Stable Baselines, was created in the [robotics lab U2IS](http://u2is.ensta-paris.fr/?lang=en) ([INRIA Flowers](https://flowers.inria.fr/) team) at [ENSTA ParisTech](https://www.ensta.fr/en).
 
 
 Logo credits: [L.M. Tenkes](https://www.instagram.com/lucillehue/)

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/pyproject.toml

@@ -36,20 +36,23 @@ exclude = """(?x)(
 
 [tool.pytest.ini_options]
 # Deterministic ordering for tests; useful for pytest-xdist.
-env = ["PYTHONHASHSEED=0"]
+# env = ["PYTHONHASHSEED=0"]
 
 filterwarnings = [
+    # multiprocessing and fork method, use of fork may lead to deadlocks
+    "ignore::DeprecationWarning:multiprocessing.popen_fork",
     # A2C/PPO on GPU
     "ignore:You are trying to run (PPO|A2C) on the GPU",
     # Tensorboard warnings
     "ignore::DeprecationWarning:tensorboard",
+    # Torch tensorboard numpy newshape deprecation
+    "ignore:`newshape` keyword argument is deprecated:DeprecationWarning:torch.utils.tensorboard._utils",
     # Gymnasium warnings
     "ignore::UserWarning:gymnasium",
+    # Taxi-v3 and CliffWalking-v0
+    "ignore::DeprecationWarning:gymnasium.envs.registration",
     # tqdm warning about rich being experimental
     "ignore:rich is experimental",
-    # Pygame warnings about pkg_resources
-    "ignore:pkg_resources is deprecated",
-    "ignore:Deprecated call to `pkg_resources",
 ]
 markers = [
     "expensive: marks tests as expensive (deselect with '-m \"not expensive\"')",

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/setup.py

@@ -55,7 +55,7 @@ vec_env = model.get_env()
 obs = vec_env.reset()
 for i in range(1000):
     action, _states = model.predict(obs, deterministic=True)
-    obs, reward, done, info = vec_env.step(action)
+    obs, rewards, dones, info = vec_env.step(action)
     vec_env.render()
     # VecEnv resets automatically
     # if done:
@@ -81,7 +81,7 @@ setup(
     install_requires=[
         "gymnasium>=0.29.1,<2.0",
         "numpy>=1.20,<3.0",
-        "torch>=2.3,<3.0",
+        "torch>=2.8,<3.0",
         # For saving models
         "cloudpickle",
     ],
@@ -90,10 +90,8 @@ setup(
             # Run tests and coverage
             "pytest",
             "pytest-cov",
-            "pytest-env",
-            "pytest-xdist",
             # Type check
-            "mypy>=1.9.0,<2",
+            "mypy>=1.9.0,<3",
             # Lint code and sort imports (flake8 and isort replacement)
             "ruff>=0.5.6",
             # Reformat

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/common/envs/__init__.py

@@ -16,5 +16,4 @@ __all__ = [
     "IdentityEnvMultiBinary",
     "IdentityEnvMultiDiscrete",
     "SimpleMultiObsEnv",
-    "SimpleMultiObsEnv",
 ]

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/common/logger.py

@@ -428,6 +428,7 @@ class TensorBoardOutputFormat(KVWriter):
                 self.writer.add_image(key, value.image, step, dataformats=value.dataformats)
 
             if isinstance(value, HParam):
+                assert self.writer.file_writer is not None
                 # we don't use `self.writer.add_hparams` to have control over the log_dir
                 experiment, session_start_info, session_end_info = hparams(value.hparam_dict, metric_dict=value.metric_dict)
                 self.writer.file_writer.add_summary(experiment)

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/common/monitor.py

@@ -262,10 +262,10 @@ def load_results(path: str) -> "pandas.DataFrame":
         # Only empty monitor files, return empty df
         empty_df = pandas.DataFrame(columns=["r", "l", "t"])
         # Create index to have the same columns
-        empty_df.reset_index(inplace=True)
+        empty_df = empty_df.reset_index()
         return empty_df
     data_frame = pandas.concat(data_frames)
-    data_frame.sort_values("t", inplace=True)
-    data_frame.reset_index(inplace=True)
+    data_frame = data_frame.sort_values("t")
+    data_frame = data_frame.reset_index()
     data_frame["t"] -= min(header["t_start"] for header in headers)
     return data_frame

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/common/noise.py

@@ -51,7 +51,7 @@ class OrnsteinUhlenbeckActionNoise(ActionNoise):
     """
     An Ornstein Uhlenbeck action noise, this is designed to approximate Brownian motion with friction.
 
-    Based on http://math.stackexchange.com/questions/1287634/implementing-ornstein-uhlenbeck-in-matlab
+    Based on https://math.stackexchange.com/questions/1287634/implementing-ornstein-uhlenbeck-in-matlab
 
     :param mean: Mean of the noise
     :param sigma: Scale of the noise

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/common/off_policy_algorithm.py

@@ -52,6 +52,9 @@ class OffPolicyAlgorithm(BaseAlgorithm):
         at a cost of more complexity.
         See https://github.com/DLR-RM/stable-baselines3/issues/37#issuecomment-637501195
     :param n_steps: When n_step > 1, uses n-step return (with the NStepReplayBuffer) when updating the Q-value network.
+        Note: it is only used when ``replay_buffer_class`` is ``None``, and is not supported for Dict
+        observation spaces yet. When passing a custom ``replay_buffer_class``, configure it directly
+        with ``replay_buffer_kwargs={"n_steps": ..., "gamma": ...}`` instead.
     :param policy_kwargs: Additional arguments to be passed to the policy on creation
     :param stats_window_size: Window size for the rollout logging, specifying the number of episodes to average
         the reported success rate, mean episode length, and mean reward over
@@ -175,6 +178,15 @@ class OffPolicyAlgorithm(BaseAlgorithm):
         self._setup_lr_schedule()
         self.set_random_seed(self.seed)
 
+        if self.n_steps > 1 and self.replay_buffer_class is not None:
+            warnings.warn(
+                f"`n_steps={self.n_steps}` is ignored because a custom `replay_buffer_class` "
+                f"(`{self.replay_buffer_class.__name__}`) was passed: n-step returns are only set up "
+                "automatically when `replay_buffer_class` is `None`. "
+                "To use n-step returns with a custom replay buffer, configure it explicitly, e.g. "
+                f'`replay_buffer_kwargs={{"n_steps": {self.n_steps}, "gamma": {self.gamma}}}`.'
+            )
+
         if self.replay_buffer_class is None:
             if isinstance(self.observation_space, spaces.Dict):
                 self.replay_buffer_class = DictReplayBuffer

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/common/policies.py

@@ -257,7 +257,7 @@ class BaseModel(nn.Module):
                     obs_ = np.array(obs)
                 vectorized_env = vectorized_env or is_vectorized_observation(obs_, obs_space)
                 # Add batch dimension if needed
-                observation[key] = obs_.reshape((-1, *self.observation_space[key].shape))  # type: ignore[misc]
+                observation[key] = obs_.reshape((-1, *self.observation_space[key].shape))  # type: ignore[misc, arg-type]
 
         elif is_image_space(self.observation_space):
             # Handle the different cases for images
@@ -271,7 +271,7 @@ class BaseModel(nn.Module):
             # Dict obs need to be handled separately
             vectorized_env = is_vectorized_observation(observation, self.observation_space)
             # Add batch dimension if needed
-            observation = observation.reshape((-1, *self.observation_space.shape))  # type: ignore[misc]
+            observation = observation.reshape((-1, *self.observation_space.shape))  # type: ignore[misc, arg-type]
 
         obs_tensor = obs_as_tensor(observation, self.device)
         return obs_tensor, vectorized_env
@@ -367,7 +367,7 @@ class BasePolicy(BaseModel, ABC):
         with th.no_grad():
             actions = self._predict(obs_tensor, deterministic=deterministic)
         # Convert to numpy, and reshape to the original action shape
-        actions = actions.cpu().numpy().reshape((-1, *self.action_space.shape))  # type: ignore[misc, assignment]
+        actions = actions.cpu().numpy().reshape((-1, *self.action_space.shape))  # type: ignore[misc, assignment, arg-type]
 
         if isinstance(self.action_space, spaces.Box):
             if self.squash_output:

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/common/sb2_compat/rmsprop_tf_like.py

@@ -20,7 +20,7 @@ class RMSpropTFLike(Optimizer):
         - Initialize squared gradient to ones rather than zeros
 
     Proposed by G. Hinton in his
-    `course <http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_.
+    `course <https://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_.
 
     The centered version first appears in `Generating Sequences
     With Recurrent Neural Networks <https://arxiv.org/pdf/1308.0850v5.pdf>`_.

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/common/vec_env/base_vec_env.py

@@ -289,7 +289,7 @@ class VecEnv(ABC):
             self.env_method("render")
         return None
 
-    def seed(self, seed: int | None = None) -> Sequence[None | int]:
+    def seed(self, seed: int | None = None) -> Sequence[int | None]:
         """
         Sets the random seeds for all environments, based on a given seed.
         Each individual environment will still get its own seed, by incrementing the given seed.
@@ -392,7 +392,7 @@ class VecEnvWrapper(VecEnv):
     def step_wait(self) -> VecEnvStepReturn:
         pass
 
-    def seed(self, seed: int | None = None) -> Sequence[None | int]:
+    def seed(self, seed: int | None = None) -> Sequence[int | None]:
         return self.venv.seed(seed)
 
     def set_options(self, options: list[dict] | dict | None = None) -> None:

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/common/vec_env/subproc_vec_env.py

@@ -159,6 +159,12 @@ class SubprocVecEnv(VecEnv):
             remote.send(("close", None))
         for process in self.processes:
             process.join()
+        # Close the parent side of the pipes: otherwise the sockets stay open
+        # until this instance (and its ``remotes``) is garbage collected,
+        # which can exhaust the file-descriptor limit (``Too many open files``)
+        # in long-running loops (e.g. hyperparameter optimization).
+        for remote in self.remotes:
+            remote.close()
         self.closed = True
 
     def get_images(self) -> Sequence[np.ndarray | None]:

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/ddpg/ddpg.py

@@ -15,7 +15,7 @@ class DDPG(TD3):
     """
     Deep Deterministic Policy Gradient (DDPG).
 
-    Deterministic Policy Gradient: http://proceedings.mlr.press/v32/silver14.pdf
+    Deterministic Policy Gradient: https://proceedings.mlr.press/v32/silver14.pdf
     DDPG Paper: https://arxiv.org/abs/1509.02971
     Introduction to DDPG: https://spinningup.openai.com/en/latest/algorithms/ddpg.html
 
@@ -45,6 +45,9 @@ class DDPG(TD3):
         at a cost of more complexity.
         See https://github.com/DLR-RM/stable-baselines3/issues/37#issuecomment-637501195
     :param n_steps: When n_step > 1, uses n-step return (with the NStepReplayBuffer) when updating the Q-value network.
+        Note: it is only used when ``replay_buffer_class`` is ``None``, and is not supported for Dict
+        observation spaces yet. When passing a custom ``replay_buffer_class``, configure it directly
+        with ``replay_buffer_kwargs={"n_steps": ..., "gamma": ...}`` instead.
     :param policy_kwargs: additional arguments to be passed to the policy on creation. See :ref:`ddpg_policies`
     :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for
         debug messages

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/dqn/dqn.py

@@ -45,6 +45,9 @@ class DQN(OffPolicyAlgorithm):
         at a cost of more complexity.
         See https://github.com/DLR-RM/stable-baselines3/issues/37#issuecomment-637501195
     :param n_steps: When n_step > 1, uses n-step return (with the NStepReplayBuffer) when updating the Q-value network.
+        Note: it is only used when ``replay_buffer_class`` is ``None``, and is not supported for Dict
+        observation spaces yet. When passing a custom ``replay_buffer_class``, configure it directly
+        with ``replay_buffer_kwargs={"n_steps": ..., "gamma": ...}`` instead.
     :param target_update_interval: update the target network every ``target_update_interval``
         environment steps.
     :param exploration_fraction: fraction of entire training period over which the exploration rate is reduced

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/ppo/ppo.py

@@ -88,7 +88,7 @@ class PPO(OnPolicyAlgorithm):
         gamma: float = 0.99,
         gae_lambda: float = 0.95,
         clip_range: float | Schedule = 0.2,
-        clip_range_vf: None | float | Schedule = None,
+        clip_range_vf: float | Schedule | None = None,
         normalize_advantage: bool = True,
         ent_coef: float = 0.0,
         vf_coef: float = 0.5,

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/sac/sac.py

@@ -54,6 +54,9 @@ class SAC(OffPolicyAlgorithm):
         at a cost of more complexity.
         See https://github.com/DLR-RM/stable-baselines3/issues/37#issuecomment-637501195
     :param n_steps: When n_step > 1, uses n-step return (with the NStepReplayBuffer) when updating the Q-value network.
+        Note: it is only used when ``replay_buffer_class`` is ``None``, and is not supported for Dict
+        observation spaces yet. When passing a custom ``replay_buffer_class``, configure it directly
+        with ``replay_buffer_kwargs={"n_steps": ..., "gamma": ...}`` instead.
     :param ent_coef: Entropy regularization coefficient. (Equivalent to
         inverse of reward scale in the original SAC paper.)  Controlling exploration/exploitation trade-off.
         Set it to 'auto' to learn it automatically (and 'auto_0.1' for using 0.1 as initial value)

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3/td3/td3.py

@@ -49,6 +49,9 @@ class TD3(OffPolicyAlgorithm):
         at a cost of more complexity.
         See https://github.com/DLR-RM/stable-baselines3/issues/37#issuecomment-637501195
     :param n_steps: When n_step > 1, uses n-step return (with the NStepReplayBuffer) when updating the Q-value network.
+        Note: it is only used when ``replay_buffer_class`` is ``None``, and is not supported for Dict
+        observation spaces yet. When passing a custom ``replay_buffer_class``, configure it directly
+        with ``replay_buffer_kwargs={"n_steps": ..., "gamma": ...}`` instead.
     :param policy_delay: Policy and target networks will only be updated once every policy_delay steps
         per training steps. The Q values will be updated policy_delay more often (update every training step).
     :param target_policy_noise: Standard deviation of Gaussian noise added to target policy

stable_baselines3-2.9.2/stable_baselines3/version.txt

@@ -0,0 +1 @@
+2.9.2

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2/stable_baselines3.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stable_baselines3
-Version: 2.9.0a2
+Version: 2.9.2
 Summary: Pytorch version of Stable Baselines, implementations of reinforcement learning algorithms.
 Home-page: https://github.com/DLR-RM/stable-baselines3
 Author: Antonin Raffin
@@ -24,14 +24,12 @@ License-File: LICENSE
 License-File: NOTICE
 Requires-Dist: gymnasium<2.0,>=0.29.1
 Requires-Dist: numpy<3.0,>=1.20
-Requires-Dist: torch<3.0,>=2.3
+Requires-Dist: torch<3.0,>=2.8
 Requires-Dist: cloudpickle
 Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
 Requires-Dist: pytest-cov; extra == "tests"
-Requires-Dist: pytest-env; extra == "tests"
-Requires-Dist: pytest-xdist; extra == "tests"
-Requires-Dist: mypy<2,>=1.9.0; extra == "tests"
+Requires-Dist: mypy<3,>=1.9.0; extra == "tests"
 Requires-Dist: ruff>=0.5.6; extra == "tests"
 Requires-Dist: black<27,>=26.1.0; extra == "tests"
 Provides-Extra: docs
@@ -116,7 +114,7 @@ vec_env = model.get_env()
 obs = vec_env.reset()
 for i in range(1000):
     action, _states = model.predict(obs, deterministic=True)
-    obs, reward, done, info = vec_env.step(action)
+    obs, rewards, dones, info = vec_env.step(action)
     vec_env.render()
     # VecEnv resets automatically
     # if done:

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/stable_baselines3.egg-info/requires.txt

@@ -1,6 +1,6 @@
 gymnasium<2.0,>=0.29.1
 numpy<3.0,>=1.20
-torch<3.0,>=2.3
+torch<3.0,>=2.8
 cloudpickle
 
 [docs]
@@ -26,8 +26,6 @@ matplotlib
 [tests]
 pytest
 pytest-cov
-pytest-env
-pytest-xdist
-mypy<2,>=1.9.0
+mypy<3,>=1.9.0
 ruff>=0.5.6
 black<27,>=26.1.0

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/tests/test_cnn.py

@@ -133,12 +133,19 @@ def test_features_extractor_target_net(model_class, share_features_extractor):
     if model_class == DQN and share_features_extractor:
         pytest.skip()
 
-    env = FakeImageEnv(screen_height=40, screen_width=40, n_channels=1, discrete=model_class not in {SAC, TD3})
+    env = FakeImageEnv(screen_height=36, screen_width=36, n_channels=1, discrete=model_class not in {SAC, TD3})
     # Avoid memory error when using replay buffer
     # Reduce the size of the features
-    kwargs = dict(buffer_size=250, learning_starts=100, policy_kwargs=dict(features_extractor_kwargs=dict(features_dim=32)))
+    kwargs = dict(
+        buffer_size=100,
+        learning_starts=50,
+        policy_kwargs=dict(features_extractor_kwargs=dict(features_dim=8)),
+        learning_rate=1e-3,
+    )
     if model_class != DQN:
         kwargs["policy_kwargs"]["share_features_extractor"] = share_features_extractor
+    else:
+        kwargs["target_update_interval"] = 10
 
     # No delay for TD3 (changes when the actor and polyak update take place)
     if model_class == TD3:
@@ -172,7 +179,7 @@ def test_features_extractor_target_net(model_class, share_features_extractor):
     if model_class == TD3:
         params_should_match(model.actor.parameters(), model.actor_target.parameters())
 
-    model.learn(200)
+    model.learn(100)
 
     # Critic and target should differ
     params_should_differ(model.critic.parameters(), model.critic_target.parameters())

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/tests/test_logger.py

@@ -1,4 +1,3 @@
-import importlib.util
 import json
 import os
 import sys
@@ -273,7 +272,14 @@ def test_report_video_to_tensorboard(tmp_path, read_log, capsys):
 
 
 def is_moviepy_installed():
-    return importlib.util.find_spec("moviepy") is not None
+    try:
+        import moviepy
+
+        # PyTorch doesn't support moviepy >= 2.0
+        # See: https://github.com/pytorch/pytorch/issues/147317
+        return moviepy.__version__ < "2.0"
+    except (ImportError, AttributeError):
+        return False
 
 
 @pytest.mark.parametrize("unsupported_format", ["stdout", "log", "json", "csv"])

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/tests/test_n_step_replay.py

@@ -1,10 +1,14 @@
+import warnings
+
 import gymnasium as gym
 import numpy as np
 import pytest
 
 from stable_baselines3 import DQN, SAC, TD3
-from stable_baselines3.common.buffers import NStepReplayBuffer, ReplayBuffer
+from stable_baselines3.common.buffers import DictReplayBuffer, NStepReplayBuffer, ReplayBuffer
 from stable_baselines3.common.env_util import make_vec_env
+from stable_baselines3.common.envs import BitFlippingEnv
+from stable_baselines3.her import HerReplayBuffer
 
 
 @pytest.mark.parametrize("model_class", [SAC, DQN, TD3])
@@ -30,6 +34,47 @@ def test_run(model_class):
     model.learn(total_timesteps=150)
 
 
+@pytest.mark.parametrize("replay_buffer_class", [ReplayBuffer, NStepReplayBuffer])
+def test_n_steps_warns_with_custom_replay_buffer_class(replay_buffer_class):
+    """`n_steps > 1` is ignored when the buffer class is user-provided, so it must warn."""
+    with pytest.warns(UserWarning, match="n_steps=3"):
+        model = SAC("MlpPolicy", "Pendulum-v1", n_steps=3, buffer_size=200, replay_buffer_class=replay_buffer_class)
+    # The buffer is built from the user-provided class, untouched by `n_steps`
+    assert type(model.replay_buffer) is replay_buffer_class
+
+
+@pytest.mark.parametrize("replay_buffer_class", [None, DictReplayBuffer, HerReplayBuffer])
+def test_n_steps_with_dict_obs(replay_buffer_class):
+    """Dict observation spaces do not support n-step returns: assert on the default path, warn otherwise."""
+    env = BitFlippingEnv(n_bits=4, continuous=True)
+    if replay_buffer_class is None:
+        with pytest.raises(AssertionError, match="Dict observation spaces"):
+            SAC("MultiInputPolicy", env, n_steps=3, buffer_size=200)
+    else:
+        with pytest.warns(UserWarning, match="n_steps=3"):
+            SAC("MultiInputPolicy", env, n_steps=3, buffer_size=200, replay_buffer_class=replay_buffer_class)
+
+
+def test_n_steps_custom_class_no_warnings():
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        # Note: values must differ from ``NStepReplayBuffer`` defaults (n_steps=3, gamma=0.99)
+        # otherwise the assertions below would pass even if nothing was applied.
+        model = SAC(
+            "MlpPolicy",
+            "Pendulum-v1",
+            buffer_size=200,
+            replay_buffer_class=NStepReplayBuffer,
+            replay_buffer_kwargs={"n_steps": 4, "gamma": 0.95},
+        )
+        assert isinstance(model.replay_buffer, NStepReplayBuffer)
+        assert model.replay_buffer.n_steps == 4
+        assert model.replay_buffer.gamma == 0.95
+
+        # Default n-steps
+        SAC("MlpPolicy", "Pendulum-v1", buffer_size=200, replay_buffer_class=ReplayBuffer)
+
+
 def create_buffer(buffer_size=10, n_steps=3, gamma=0.99, n_envs=1):
     obs_space = gym.spaces.Box(low=0, high=1, shape=(4,), dtype=np.float32)
     act_space = gym.spaces.Box(low=0, high=1, shape=(2,), dtype=np.float32)

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/tests/test_save_load.py

@@ -891,3 +891,34 @@ def test_save_load_clip_range_portable(tmp_path, model_class):
     assert isinstance(model.clip_range, FloatSchedule)
     assert isinstance(model.clip_range.value_schedule, ConstantSchedule)
     assert model.clip_range.value_schedule.val == 0.2
+
+
+def test_load_bad_zip(tmp_path):
+    """Test that loading a non-zip file raises a ValueError."""
+    # Write a plain text file to the path
+    bad_path = tmp_path / "not_a_zip.zip"
+    bad_path.write_text("this is not a zip file")
+
+    with pytest.raises(ValueError, match="wasn't a zip-file"):
+        PPO.load(str(bad_path))
+
+
+@pytest.mark.filterwarnings("error::ResourceWarning")
+def test_open_path_directory_and_missing_parent(tmp_path):
+    """Test IsADirectoryError and FileNotFoundError handling in open_path."""
+    # Test IsADirectoryError: path already has the suffix and is a folder
+    dir_path = tmp_path / "is_a_folder.pkl"
+    dir_path.mkdir()
+    with warnings.catch_warnings(record=True) as record:
+        open_path(dir_path, "w", suffix="pkl").close()
+    assert any("is a folder" in str(warning.message) for warning in record)
+    # Verify the fallback file was created
+    assert (tmp_path / "is_a_folder.pkl_2").exists()
+
+    # Test FileNotFoundError: parent folder doesn't exist
+    missing_parent_path = tmp_path / "nonexistent_parent" / "sub" / "file"
+    with warnings.catch_warnings(record=True) as record:
+        open_path(missing_parent_path, "w", suffix="pkl").close()
+    assert any("does not exist" in str(warning.message) for warning in record)
+    # Verify the parent was created and file was written
+    assert (tmp_path / "nonexistent_parent" / "sub" / "file.pkl").exists()

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/tests/test_spaces.py

@@ -123,7 +123,8 @@ def test_sde_multi_dim():
 @pytest.mark.parametrize("model_class", [A2C, PPO, DQN])
 @pytest.mark.parametrize("env", ["Taxi-v3"])
 def test_discrete_obs_space(model_class, env):
-    env = make_vec_env(env, n_envs=2, seed=0)
+
+    env = _make_env_safe(env, n_envs=2, seed=0)
     kwargs = {}
     if model_class == DQN:
         kwargs = dict(buffer_size=1000, learning_starts=100)
@@ -173,3 +174,17 @@ def test_multidim_binary_not_supported():
     env = DummyEnv(BOX_SPACE_FLOAT32, spaces.MultiBinary([2, 3]))
     with pytest.raises(AssertionError, match=r"Multi-dimensional MultiBinary\(.*\) action space is not supported"):
         A2C("MlpPolicy", env)
+
+
+def _make_env_safe(env_id, **kwargs):
+    try:
+        return make_vec_env(env_id, **kwargs)
+    except gym.error.DeprecatedEnv:
+        # map deprecated to new (better extendability for this)
+        fallback_map = {
+            # as of gymnasium 1.3.0 Taxi-v3 is deprecated
+            "Taxi-v3": "Taxi-v4",
+        }
+        if env_id in fallback_map:
+            return make_vec_env(fallback_map[env_id], **kwargs)
+        raise

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/tests/test_utils.py

@@ -1,4 +1,5 @@
 import os
+import pickle
 import shutil
 
 import ale_py
@@ -15,6 +16,7 @@ from stable_baselines3.common.env_util import is_wrapped, make_atari_env, make_v
 from stable_baselines3.common.evaluation import evaluate_policy
 from stable_baselines3.common.monitor import Monitor
 from stable_baselines3.common.noise import OrnsteinUhlenbeckActionNoise, VectorizedActionNoise
+from stable_baselines3.common.sb2_compat.rmsprop_tf_like import RMSpropTFLike
 from stable_baselines3.common.utils import (
     ConstantSchedule,
     FloatSchedule,
@@ -628,3 +630,57 @@ def test_deprecated_schedules():
         assert np.allclose(fn(0.5), schedule(0.5))
         assert np.allclose(fn(0.5), float_schedule(0.5))
         assert np.allclose(fn(0.5), float_schedule_2(0.5))
+
+
+class TestRMSpropTFLike:
+    """Tests for the RMSpropTFLike optimizer (sb2_compat)."""
+
+    def test_invalid_params(self):
+        """Test that invalid optimizer parameters raise ValueError."""
+        param = th.nn.Parameter(th.zeros(2))
+        with pytest.raises(ValueError, match="Invalid learning rate"):
+            RMSpropTFLike([param], lr=-1.0)
+        with pytest.raises(ValueError, match="Invalid epsilon"):
+            RMSpropTFLike([param], eps=-1.0)
+        with pytest.raises(ValueError, match="Invalid momentum"):
+            RMSpropTFLike([param], momentum=-1.0)
+        with pytest.raises(ValueError, match="Invalid weight_decay"):
+            RMSpropTFLike([param], weight_decay=-1.0)
+        with pytest.raises(ValueError, match="Invalid alpha"):
+            RMSpropTFLike([param], alpha=-1.0)
+
+    def test_centered_momentum_weight_decay(self):
+        """Test centered=True, momentum>0, weight_decay>0 branches in step()."""
+        param = th.nn.Parameter(th.tensor([1.0, 2.0]))
+        opt = RMSpropTFLike(
+            [param],
+            lr=0.1,
+            momentum=0.9,
+            alpha=0.99,
+            eps=1e-8,
+            centered=True,
+            weight_decay=0.01,
+        )
+        # Compute gradient
+        loss = (param**2).sum()
+        loss.backward()
+
+        # Run step with closure (covers closure branch)
+        def closure():
+            param.grad.zero_()
+            return float((param**2).sum().detach())
+
+        opt.step(closure=closure)
+        # Check state keys exist (covers centered + momentum init)
+        state = opt.state[param]
+        assert "grad_avg" in state  # centered=True
+        assert "momentum_buffer" in state  # momentum > 0
+
+    def test_pickle_roundtrip(self):
+        """Test that optimizer survives pickle roundtrip (covers __setstate__)."""
+        param = th.nn.Parameter(th.tensor([1.0, 2.0]))
+        opt = RMSpropTFLike([param], lr=0.1, momentum=0.5, centered=True)
+        opt2 = pickle.loads(pickle.dumps(opt))
+        assert opt2.param_groups[0]["lr"] == 0.1
+        assert opt2.param_groups[0]["momentum"] == 0.5
+        assert opt2.param_groups[0]["centered"] is True

{stable_baselines3-2.9.0a2 → stable_baselines3-2.9.2}/tests/test_vec_envs.py

@@ -317,7 +317,7 @@ def check_vecenv_obs(obs, space):
         assert space.contains(value)
 
 
-@pytest.mark.parametrize("vec_env_class,space", itertools.product(VEC_ENV_CLASSES, SPACES.values()))
+@pytest.mark.parametrize("vec_env_class,space", list(itertools.product(VEC_ENV_CLASSES, SPACES.values())))
 def test_vecenv_single_space(vec_env_class, space):
     def obs_assert(obs):
         return check_vecenv_obs(obs, space)

stable_baselines3-2.9.0a2/stable_baselines3/version.txt

@@ -1 +0,0 @@
-2.9.0a2