returnn 1.20240919.2417__tar.gz → 1.20240919.134356__tar.gz

{returnn-1.20240919.2417 → returnn-1.20240919.134356}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240919.2417
+Version: 1.20240919.134356
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240919.134356/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240919.134356'
+long_version = '1.20240919.134356+git.d63d58d'

{returnn-1.20240919.2417 → returnn-1.20240919.134356}/returnn/torch/engine.py

@@ -519,9 +519,16 @@ class Engine(EngineBase):
 
             if self._torch_distributed_ctx and self._torch_distributed_ctx.rank() != 0:
                 # We need to make sure the data loader iterator was created for proper synchronization.
-                # However, now we only want to do evaluation on rank 0 for simplicity.
+                # However, we only want to do evaluation on rank 0 for simplicity.
                 iter(data_loader)
+                # We wait here until rank 0 is done w/ the eval.
+                # Regularly synchronizing should fix potential timeout issues, like
+                # https://github.com/rwth-i6/returnn/issues/1621.
+                _has_data = torch.tensor([True], device="cpu", dtype=torch.int8)
+                while _has_data:
+                    _has_data = torch.distributed.broadcast(_has_data, src=0)
                 continue
+
             print(f"Evaluating dataset {dataset_name!r}", file=log.v3)
 
             accumulated_losses_dict = NumbersDict()
@@ -530,6 +537,10 @@ class Engine(EngineBase):
 
             with torch.no_grad():
                 for extern_data_raw in data_loader:
+                    if self._torch_distributed_ctx and step_idx % 100 == 0:
+                        _has_data = torch.tensor([True], device="cpu", dtype=torch.int8)
+                        torch.distributed.broadcast(_has_data, src=0)
+
                     extern_data = extern_data_util.raw_dict_to_extern_data(
                         extern_data_raw, extern_data_template=self.extern_data, device=self._device
                     )
@@ -586,6 +597,11 @@ class Engine(EngineBase):
                 )
             ]
 
+            if self._torch_distributed_ctx:
+                assert self._torch_distributed_ctx.rank() == 0
+                _has_data = torch.tensor([False], device="cpu", dtype=torch.int8)
+                torch.distributed.broadcast(_has_data, src=0)
+
         if not self._torch_distributed_ctx or self._torch_distributed_ctx.rank() == 0:
             print(" ".join(eval_dump_str) if eval_dump_str else "(No evaluations.)", file=log.v1)
 
@@ -1052,6 +1068,9 @@ class Engine(EngineBase):
             else:
                 # Keep it as ndarray.
                 raw = x.raw_tensor[batch_idx : batch_idx + 1].reshape(())
+            if any(d is not d_ for d, d_ in zip(x.dims[1:], y.dims)):  # replaced any dims?
+                # Cut off any padding.
+                raw = raw[tuple(slice(None, dim.get_dim_value()) for dim in y.dims)]
             # Convert it to Numpy array.
             # Note that users might also want to get the PyTorch tensors instead.
             # They might even want to get the whole batched tensor.

{returnn-1.20240919.2417 → returnn-1.20240919.134356}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240919.2417
+Version: 1.20240919.134356
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240919.2417/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240919.002417'
-long_version = '1.20240919.002417+git.bdf1d5c'