returnn 1.20240109.202330__tar.gz → 1.20240110.175314__tar.gz

{returnn-1.20240109.202330/returnn.egg-info → returnn-1.20240110.175314}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240109.202330
+Version: 1.20240110.175314
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240110.175314/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240110.175314'
+long_version = '1.20240110.175314+git.9b5cff8'

{returnn-1.20240109.202330 → returnn-1.20240110.175314}/returnn/torch/engine.py

@@ -466,6 +466,13 @@ class Engine(EngineBase):
         if self.config.bool_or_other("cleanup_old_models", None):
             self.cleanup_old_models()
 
+        # Maybe sync train/dev/eval scores for the epoch.
+        if self._torch_distributed_ctx:
+            ls = [self.learning_rate_control.epoch_data[self.epoch]]
+            torch.distributed.broadcast_object_list(ls, src=0, device=torch.device("cpu"))
+            assert isinstance(ls[0], dict)
+            self.learning_rate_control.epoch_data[self.epoch] = ls[0]
+
     def _do_save(self):
         if self._device == "meta":
             return False

{returnn-1.20240109.202330 → returnn-1.20240110.175314}/returnn/util/train_proc_manager.py

@@ -27,6 +27,12 @@ def maybe_start_train_proc_manager(*, config: Config):
     if config.value("start_epoch", "auto") != "auto":
         return
 
+    # Not supported currently in distributed mode.
+    if config.bool("use_horovod", False):
+        return
+    if config.typed_value("torch_distributed") is not None:
+        return
+
     main_proc_manager(config=config)
 
 
@@ -36,22 +42,35 @@ def main_proc_manager(*, config: Config):
     """
     os.environ["__RETURNN_PROC_MANAGED"] = "1"
 
-    print("RETURNN starting up, version %s, train proc manager" % util.describe_returnn_version())
+    print("RETURNN train proc manager starting up, version %s" % util.describe_returnn_version())
 
     # Setup BackendEngine. This is that other utility functions know the right model file extension.
     # _select_rf_backend to avoid importing Torch/TF/anything here.
     util.BackendEngine.select_engine(config=config, _select_rf_backend=False)
 
+    final_epoch = EngineBase.config_get_final_epoch(config)
+
     total_start_time = time.time()
     num_starts = 0
-    last_model_epoch = -1
+    last_model_epoch = None
+    return_code = None
     while True:
         models = EngineBase.get_existing_models(config, for_training=True)
         cur_model_epoch = max(models) if models else 0
-        print("Most recent trained model:", models.get(cur_model_epoch))
+        print("Most recent trained model epoch:", cur_model_epoch, "file:", models.get(cur_model_epoch))
+
+        if cur_model_epoch == final_epoch:
+            if return_code is None:
+                return_code = 0
+                print("Already finished, nothing to do")
+            elif return_code == 0:
+                print("Finished training")
+            else:
+                print("Finished training, but got some error?")
+            break
 
         if last_model_epoch is not None:
-            print("Most recent trained model before RETURNN run:", last_model_epoch)
+            print("Most recent trained model epoch before RETURNN run:", last_model_epoch)
             assert last_model_epoch <= cur_model_epoch
             print(f"-> trained successfully {cur_model_epoch - last_model_epoch} epoch(s)")
             if cur_model_epoch == last_model_epoch:
@@ -65,12 +84,14 @@ def main_proc_manager(*, config: Config):
         last_model_epoch = cur_model_epoch
         num_starts += 1
         start_time = time.time()
-        proc = subprocess.Popen(sys.argv, executable=sys.executable)
+        args = [sys.executable] + sys.argv
+        print(f"Run {args}")
+        proc = subprocess.Popen(args)
         proc.wait()
         print("RETURNN runtime:", util.hms(time.time() - start_time))
         print("RETURNN return code:", proc.returncode)
-        if proc.returncode == 0:
-            break
+        return_code = proc.returncode
 
     print("Total RETURNN num starts:", num_starts)
     print("Total RETURNN runtime:", util.hms(time.time() - total_start_time))
+    sys.exit(return_code if return_code is not None else -1)

{returnn-1.20240109.202330 → returnn-1.20240110.175314/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240109.202330
+Version: 1.20240110.175314
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240109.202330/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240109.202330'
-long_version = '1.20240109.202330+git.6f4ce8e'