torchrl-nightly 2025.7.20__cp310-cp310-win_amd64.whl → 2025.7.21__cp310-cp310-win_amd64.whl

torchrl/_extension.py

@@ -10,9 +10,10 @@ import warnings
 from packaging.version import parse
 
 try:
-    from .version import __version__
+    from .version import __version__, pytorch_version
 except ImportError:
     __version__ = None
+    pytorch_version = "unknown"
 
 
 def is_module_available(*modules: str) -> bool:
@@ -47,13 +48,16 @@ if _is_nightly(__version__):
         " - make sure ninja and cmake were installed\n"
         " - make sure you ran `python setup.py clean && python setup.py develop` and that no error was raised\n"
         " - make sure the version of PyTorch you are using matches the one that was present in your virtual env during "
-        "setup."
+        f"setup. This package was built with PyTorch {pytorch_version}."
     )
 
 else:
     EXTENSION_WARNING = (
         "Failed to import torchrl C++ binaries. Some modules (eg, prioritized replay buffers) may not work with your installation. "
-        "This is likely due to a discrepancy between your package version and the PyTorch version. Make sure both are compatible. "
-        "Usually, torchrl majors follow the pytorch majors within a few days around the release. "
-        "For instance, TorchRL 0.5 requires PyTorch 2.4.0, and TorchRL 0.6 requires PyTorch 2.5.0."
+        "This is likely due to a discrepancy between your package version and the PyTorch version. "
+        "TorchRL does not tightly pin PyTorch versions to give users freedom, but the trade-off is that C++ extensions like "
+        "prioritized replay buffers can only be used with the PyTorch version they were built against. "
+        f"This package was built with PyTorch {pytorch_version}. "
+        "Workarounds include: (1) upgrading/downgrading PyTorch or TorchRL to compatible versions, "
+        "or (2) making a local install using `pip install git+https://github.com/pytorch/rl.git@<version>`."
     )

torchrl/objectives/sac.py

@@ -687,6 +687,102 @@ class SACLoss(LossModule):
             )
         return self._alpha * log_prob - min_q_logprob, {"log_prob": log_prob.detach()}
 
+    @dispatch
+    def actor_loss(
+        self, tensordict: TensorDictBase
+    ) -> tuple[Tensor, dict[str, Tensor]]:
+        """Compute the actor loss for SAC.
+
+        This method computes the actor loss which encourages the policy to maximize
+        the expected Q-value while maintaining high entropy.
+
+        Args:
+            tensordict (TensorDictBase): A tensordict containing the data needed for
+                computing the actor loss. Should contain the observation and other
+                required keys for the actor network.
+
+        Returns:
+            A tuple containing:
+                - The actor loss tensor
+                - A dictionary with metadata including the log probability of actions
+        """
+        return self._actor_loss(tensordict)
+
+    @dispatch
+    def qvalue_loss(
+        self, tensordict: TensorDictBase
+    ) -> tuple[Tensor, dict[str, Tensor]]:
+        """Compute the Q-value loss for SAC.
+
+        This method computes the Q-value loss which trains the Q-networks to estimate
+        the expected return for state-action pairs.
+
+        Args:
+            tensordict (TensorDictBase): A tensordict containing the data needed for
+                computing the Q-value loss. Should contain the observation, action,
+                reward, done, and terminated keys.
+
+        Returns:
+            A tuple containing:
+                - The Q-value loss tensor
+                - A dictionary with metadata including the TD error
+        """
+        if self._version == 1:
+            return self._qvalue_v1_loss(tensordict)
+        else:
+            return self._qvalue_v2_loss(tensordict)
+
+    @dispatch
+    def value_loss(
+        self, tensordict: TensorDictBase
+    ) -> tuple[Tensor, dict[str, Tensor]]:
+        """Compute the value loss for SAC (version 1 only).
+
+        This method computes the value loss which trains the value network to estimate
+        the expected return for states. This is only used in SAC version 1.
+
+        Args:
+            tensordict (TensorDictBase): A tensordict containing the data needed for
+                computing the value loss. Should contain the observation and other
+                required keys for the value network.
+
+        Returns:
+            A tuple containing:
+                - The value loss tensor
+                - An empty dictionary (no metadata for value loss)
+
+        Raises:
+            RuntimeError: If called on SAC version 2 (which doesn't use a value network)
+        """
+        if self._version != 1:
+            raise RuntimeError(
+                "Value loss is only available in SAC version 1. "
+                "SAC version 2 doesn't use a separate value network."
+            )
+        return self._value_loss(tensordict)
+
+    def alpha_loss(self, log_prob: Tensor) -> Tensor:
+        """Compute the alpha loss for SAC.
+
+        This method computes the alpha loss which adapts the entropy coefficient
+        to maintain the target entropy level.
+
+        Args:
+            log_prob (Tensor): The log probability of actions from the actor network.
+
+        Returns:
+            The alpha loss tensor
+        """
+        return self._alpha_loss(log_prob)
+
+    @property
+    def _alpha(self):
+        if self.min_log_alpha is not None or self.max_log_alpha is not None:
+            self.log_alpha.data.clamp_(self.min_log_alpha, self.max_log_alpha)
+        with torch.no_grad():
+            alpha = self.log_alpha.exp()
+        return alpha
+
     @property
     @_cache_values
     def _cached_target_params_actor_value(self):
@@ -882,14 +978,6 @@ class SACLoss(LossModule):
             alpha_loss = torch.zeros_like(log_prob)
         return alpha_loss
 
-    @property
-    def _alpha(self):
-        if self.min_log_alpha is not None or self.max_log_alpha is not None:
-            self.log_alpha.data.clamp_(self.min_log_alpha, self.max_log_alpha)
-        with torch.no_grad():
-            alpha = self.log_alpha.exp()
-        return alpha
-
 
 class DiscreteSACLoss(LossModule):
     """Discrete SAC Loss module.
@@ -1352,6 +1440,48 @@ class DiscreteSACLoss(LossModule):
             target_value = self.value_estimator.value_estimate(tensordict).squeeze(-1)
             return target_value
 
+    @dispatch
+    def actor_loss(
+        self, tensordict: TensorDictBase
+    ) -> tuple[Tensor, dict[str, Tensor]]:
+        """Compute the actor loss for discrete SAC.
+
+        This method computes the actor loss which encourages the policy to maximize
+        the expected Q-value while maintaining high entropy for discrete actions.
+
+        Args:
+            tensordict (TensorDictBase): A tensordict containing the data needed for
+                computing the actor loss. Should contain the observation and other
+                required keys for the actor network.
+
+        Returns:
+            A tuple containing:
+                - The actor loss tensor
+                - A dictionary with metadata including the log probability of actions
+        """
+        return self._actor_loss(tensordict)
+
+    @dispatch
+    def qvalue_loss(
+        self, tensordict: TensorDictBase
+    ) -> tuple[Tensor, dict[str, Tensor]]:
+        """Compute the Q-value loss for discrete SAC.
+
+        This method computes the Q-value loss which trains the Q-networks to estimate
+        the expected return for state-action pairs in discrete action spaces.
+
+        Args:
+            tensordict (TensorDictBase): A tensordict containing the data needed for
+                computing the Q-value loss. Should contain the observation, action,
+                reward, done, and terminated keys.
+
+        Returns:
+            A tuple containing:
+                - The Q-value loss tensor
+                - A dictionary with metadata including the TD error
+        """
+        return self._value_loss(tensordict)
+
     def _value_loss(
         self, tensordict: TensorDictBase
     ) -> tuple[Tensor, dict[str, Tensor]]:
@@ -1427,6 +1557,20 @@ class DiscreteSACLoss(LossModule):
             alpha_loss = torch.zeros_like(log_prob)
         return alpha_loss
 
+    def alpha_loss(self, log_prob: Tensor) -> Tensor:
+        """Compute the alpha loss for discrete SAC.
+
+        This method computes the alpha loss which adapts the entropy coefficient
+        to maintain the target entropy level for discrete actions.
+
+        Args:
+            log_prob (Tensor): The log probability of actions from the actor network.
+
+        Returns:
+            The alpha loss tensor
+        """
+        return self._alpha_loss(log_prob)
+
     @property
     def _alpha(self):
         if self.min_log_alpha is not None or self.max_log_alpha is not None:

torchrl/version.py

@@ -1,2 +1,3 @@
-__version__ = '2025.7.20'
-git_version = '163e23f9c930b58990287206958e2e036af5b902'
+__version__ = '2025.7.21'
+git_version = '851a0410a7c72ad656ab3712395450ae482ce3f6'
+pytorch_version = '2.9.0.dev20250721+cpu'

{torchrl_nightly-2025.7.20.dist-info → torchrl_nightly-2025.7.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: torchrl-nightly
-Version: 2025.7.20
+Version: 2025.7.21
 Summary: UNKNOWN
 Home-page: https://github.com/pytorch/rl
 Author: torchrl contributors

{torchrl_nightly-2025.7.20.dist-info → torchrl_nightly-2025.7.21.dist-info}/RECORD

@@ -2,10 +2,10 @@ build_tools/__init__.py,sha256=5xRGSM4YMr794wxfVj_SQly2cyHNyhXTdXdWUZJoM2M,183
 build_tools/setup_helpers/__init__.py,sha256=l9zlK7Nm5bT7P_onQx-hZeIGzKKyCFm1PFkFApuRxnM,284
 build_tools/setup_helpers/extension.py,sha256=ihV8jz8kqOvpqzuD006XqF1oNX5ukKGlwIOJRb1Vd-o,6075
 torchrl/__init__.py,sha256=76lKYwYKmAKORhyVt2tURmYAIRTifxxO3gWsskrHAXU,3054
-torchrl/_extension.py,sha256=x6Nqj2brF3VhlEwxmNA2fYbmpxq1HHGrHMnP0YnQwdc,2412
-torchrl/_torchrl.cp310-win_amd64.pyd,sha256=1woR6IR2RqSkShhHE1uh-ar6bWpJb6HwCX8aTXZ8gmc,440832
+torchrl/_extension.py,sha256=K3QwxgCWZIG35Z72VvqshHua1rLQ639wVDREzOhptxk,2799
+torchrl/_torchrl.cp310-win_amd64.pyd,sha256=4oj3psQjrTQDlIVI4dov_VGS71Ep9QEa2H0gphXzSEw,440832
 torchrl/_utils.py,sha256=2N35rdD65U1khMi5gVIz8-nMjlZsoVq0kCiQftVRSxw,42297
-torchrl/version.py,sha256=OZbhjLHH625CpOnLlZQqBfCIDgGNww73f_9eKa8-6x0,85
+torchrl/version.py,sha256=N40bks6QsRHErJ5Ngg0vjMoDLvCjZsSEpyORZwjkSJ4,128
 torchrl/collectors/__init__.py,sha256=LzTyfxmkNGPSa5-3rS5unQK7HfT5ZEdr2NV291rAOlU,832
 torchrl/collectors/collectors.py,sha256=UbXtDMHrXVQ-cd95TBpss2SIbNFKze2HTPYjFz2cPNQ,182146
 torchrl/collectors/utils.py,sha256=aBmBLpphhfplqQjRCyn1jtWWJ-Wtc7TWvM0rOBN8SsE,11579
@@ -191,7 +191,7 @@ torchrl/objectives/iql.py,sha256=VgCjfjKu91WCMCOK78vuc4k4kg1G3hPgOmjiKpozRM0,439
 torchrl/objectives/ppo.py,sha256=n0MXf9_e9u_u7amt41bLApr7G5gX1372HmjGfPa7EyE,77666
 torchrl/objectives/redq.py,sha256=qRN5WyA6YHh7GcKX9n5GinXyETssAXJkiH0HuOx6Uss,29177
 torchrl/objectives/reinforce.py,sha256=EnUjqDSiTla3CuHg9rspQlvecd-VXZrPZxg4rGECZ8w,22861
-torchrl/objectives/sac.py,sha256=gKOgCU399miKgpgu7Bmzs1bkIF8JTm_lybHn8V4wDuk,65478
+torchrl/objectives/sac.py,sha256=wKpfdm2y8Udp100PVp4bC0ljkdPwPQQNlYg5ZqvVO1M,70861
 torchrl/objectives/td3.py,sha256=Rq2q5gXo3AMuHm2OjRZvpfvKsAl1lIK5ALh2_sZM1ZE,23743
 torchrl/objectives/td3_bc.py,sha256=1pjB8mjCT2CLvQzjnqwAfZoc7yhjMB9UQjuJ5wZfTUY,26558
 torchrl/objectives/utils.py,sha256=M397jw5eA7IkXfw0_gOBXbxPTQNnQdC5N7-dj91scsY,25461
@@ -223,8 +223,8 @@ torchrl/trainers/helpers/losses.py,sha256=HwrovwbMOhY-5-hlOz-YHclKnoJhMijVjDNuAT
 torchrl/trainers/helpers/models.py,sha256=VujBq9H92sEzpCtU1iTrJQNlwvyOO-Rho4bzsMonX6s,22465
 torchrl/trainers/helpers/replay_buffer.py,sha256=RaZqXnHimmadiibvDBcLbtIhpPaVMTPhYMOBvX4v3CA,2060
 torchrl/trainers/helpers/trainers.py,sha256=hB1FtHtP-S0PBQ4LF6WPy37caaLpacyaLThj1BNl5Ho,12372
-torchrl_nightly-2025.7.20.dist-info/LICENSE,sha256=PGO-oZsq4EzhE1-WQS2xGiEF3UCVb9YawfQ09cIMV_8,1119
-torchrl_nightly-2025.7.20.dist-info/METADATA,sha256=Y6Bqw_PYQFrHGyyJWSheGXeD7benjQDUlvVqXWRL7ZU,44000
-torchrl_nightly-2025.7.20.dist-info/WHEEL,sha256=NVXpD7b4Gxps0cd2ds5rr5TG8W4ApEwx_i5J99qMZ5E,102
-torchrl_nightly-2025.7.20.dist-info/top_level.txt,sha256=JeTJ1jV7QJwLcUS1nr21aPn_wb-XlAZ9c-z_EH472JA,20
-torchrl_nightly-2025.7.20.dist-info/RECORD,,
+torchrl_nightly-2025.7.21.dist-info/LICENSE,sha256=PGO-oZsq4EzhE1-WQS2xGiEF3UCVb9YawfQ09cIMV_8,1119
+torchrl_nightly-2025.7.21.dist-info/METADATA,sha256=QAXDqpa4pe97bCPhDHquyjTx7K8y6bXLF2fnt2sAr3o,44000
+torchrl_nightly-2025.7.21.dist-info/WHEEL,sha256=NVXpD7b4Gxps0cd2ds5rr5TG8W4ApEwx_i5J99qMZ5E,102
+torchrl_nightly-2025.7.21.dist-info/top_level.txt,sha256=JeTJ1jV7QJwLcUS1nr21aPn_wb-XlAZ9c-z_EH472JA,20
+torchrl_nightly-2025.7.21.dist-info/RECORD,,