returnn 1.20231026.144554__tar.gz → 1.20231027.214341__tar.gz

{returnn-1.20231026.144554/returnn.egg-info → returnn-1.20231027.214341}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231026.144554
+Version: 1.20231027.214341
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20231027.214341/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20231027.214341'
+long_version = '1.20231027.214341+git.353afc8'

{returnn-1.20231026.144554 → returnn-1.20231027.214341}/returnn/engine/base.py

@@ -66,9 +66,12 @@ class EngineBase:
         return num_epochs
 
     @classmethod
-    def get_existing_models(cls, config):
+    def get_existing_models(cls, config: Config, *, for_training: Optional[bool] = None):
         """
-        :param returnn.config.Config config:
+        :param config:
+        :param for_training: if True, will only return models which are suitable for resuming training.
+            E.g. in case of PyTorch, it means that the optimizer state should be present.
+            By default, will be True if the task is "train".
         :return: dict epoch -> model filename
         :rtype: dict[int,str]
         """
@@ -77,6 +80,8 @@ class EngineBase:
             return {}
         # Automatically search the filesystem for existing models.
         file_list = {}
+        if for_training is None:
+            for_training = config.value("task", "train") == "train"
         for epoch in range(1, cls.config_get_final_epoch(config) + 1):
             for is_pretrain in [False, True] if util.BackendEngine.is_tensorflow_selected() else [False]:
                 fn = cls.epoch_model_filename(model_filename, epoch, is_pretrain=is_pretrain)
@@ -89,7 +94,7 @@ class EngineBase:
                         break
                 elif util.BackendEngine.is_torch_selected():
                     if os.path.exists(fn + ".pt"):
-                        if config.value("task", "train") == "train":
+                        if for_training:
                             # In case of training, we only want to consider the model if the optimizer state exists.
                             if not os.path.exists(fn + ".opt.pt"):
                                 continue
@@ -285,7 +290,7 @@ class EngineBase:
         from itertools import count
 
         opts = CollectionReadCheckCovered(self.config.get_of_type("cleanup_old_models", dict, {}))
-        existing_models = self.get_existing_models(config=self.config)
+        existing_models = self.get_existing_models(config=self.config, for_training=False)
         if self.learning_rate_control is not None:
             lr_control = self.learning_rate_control
         else:

{returnn-1.20231026.144554 → returnn-1.20231027.214341}/returnn/util/watch_memory.py

@@ -48,6 +48,7 @@ def _watch_memory_thread_main():
                 if old_mem_info:
                     print(prefix, f"proc {_format_proc(p)} exited, old:", _format_mem_info(old_mem_info))
                     mem_per_pid.pop(p.pid, None)
+                    procs.remove(p)
                     change = True
                 continue
             proc_prefix = "main" if p == cur_proc else "sub"
@@ -77,7 +78,11 @@ def _format_proc(proc: psutil.Process) -> str:
     try:
         proc_name = proc.name()
     except psutil.NoSuchProcess:  # race condition
-        proc_name = "(exited)"
+        proc_name = getattr(proc, "_name", None)
+        if not proc_name:
+            proc_name = "<unknown-dead>"
+    if not proc_name:
+        proc_name = "<noname>"
     return "%s(%s)" % (proc_name, proc.pid)
 
 

{returnn-1.20231026.144554 → returnn-1.20231027.214341/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231026.144554
+Version: 1.20231027.214341
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20231026.144554/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20231026.144554'
-long_version = '1.20231026.144554+git.d62891f'