returnn 1.20231207.151059__tar.gz → 1.20231208.100713__tar.gz

{returnn-1.20231207.151059/returnn.egg-info → returnn-1.20231208.100713}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231207.151059
+Version: 1.20231208.100713
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20231208.100713/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20231208.100713'
+long_version = '1.20231208.100713+git.18ad52e'

{returnn-1.20231207.151059 → returnn-1.20231208.100713}/returnn/__main__.py

@@ -336,6 +336,10 @@ def init_backend_engine():
             returnn.tf.distributed.init_distributed_tf(config)
 
     elif BackendEngine.is_torch_selected():
+        if config.value("PYTORCH_CUDA_ALLOC_CONF", None):
+            # Set this very early.
+            os.environ["PYTORCH_CUDA_ALLOC_CONF"] = config.value("PYTORCH_CUDA_ALLOC_CONF", "")
+
         print("PyTorch:", util.describe_torch_version(), file=log.v3)
 
         if config.typed_value("torch_distributed") is not None:

{returnn-1.20231207.151059 → returnn-1.20231208.100713}/returnn/torch/updater.py

@@ -117,13 +117,13 @@ class Updater(object):
         self._optimizer_opts = None
         self.optimizer = None  # type: typing.Optional[torch.optim.Optimizer]
 
+        self._grad_clip = self.config.float("gradient_clip", 0.0)
         self._grad_clip_global_norm = self.config.float("gradient_clip_global_norm", 0.0)
         self._grad_noise = self.config.float("gradient_noise", 0.0)
 
         # Check other options we have in TF updater, which we might support here later as well,
         # but currently do not support.
         for opt_name in [
-            "gradient_clip",
             "gradient_clip_norm",
             "gradient_clip_avg_norm",
             "global_norm_tag",
@@ -183,6 +183,8 @@ class Updater(object):
         """
         if self._grad_noise:
             gradient_noise_(self.network.parameters(), self._grad_noise)
+        if self._grad_clip:
+            torch.nn.utils.clip_grad_value_(self.network.parameters(), self._grad_clip)
         if self._grad_clip_global_norm:
             torch.nn.utils.clip_grad_norm_(self.network.parameters(), self._grad_clip_global_norm)
 

{returnn-1.20231207.151059 → returnn-1.20231208.100713/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231207.151059
+Version: 1.20231208.100713
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20231207.151059/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20231207.151059'
-long_version = '1.20231207.151059+git.4419fbe'