returnn 1.20231024.171034__tar.gz → 1.20231025.122452__tar.gz

{returnn-1.20231024.171034/returnn.egg-info → returnn-1.20231025.122452}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231024.171034
+Version: 1.20231025.122452
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20231025.122452/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20231025.122452'
+long_version = '1.20231025.122452+git.597dfbd'

{returnn-1.20231024.171034 → returnn-1.20231025.122452}/returnn/datasets/multi_proc.py

@@ -8,6 +8,7 @@ import sys
 from .basic import init_dataset, DatasetSeq
 from .cached2 import CachedDataset2
 from returnn.config import Config, get_global_config, set_global_config
+from returnn.util.basic import try_run
 import multiprocessing as mp
 
 # noinspection PyProtectedMember
@@ -163,19 +164,23 @@ class MultiProcDataset(CachedDataset2):
 
     def __del__(self):
         if self._seq_order_proc:
+            # noinspection PyBroadException
             try:
                 self._seq_order_proc_parent_conn.send(("exit", {}))
-            except BrokenPipeError:
+                self._seq_order_proc.join()
+            except Exception:
                 pass
-            self._seq_order_proc.join()
         if self._worker_procs:
+            got_exception = False
             for worker_parent_conn in self._worker_parent_conns:
+                # noinspection PyBroadException
                 try:
                     worker_parent_conn.send(("exit", {}))
-                except BrokenPipeError:
-                    pass
-            for worker_proc in self._worker_procs:
-                worker_proc.join()
+                except Exception:
+                    got_exception = True
+            if not got_exception:
+                for worker_proc in self._worker_procs:
+                    try_run(worker_proc.join)
 
     @staticmethod
     def _seq_order_proc_loop(

{returnn-1.20231024.171034 → returnn-1.20231025.122452}/returnn/engine/base.py

@@ -76,8 +76,8 @@ class EngineBase:
         # Automatically search the filesystem for existing models.
         file_list = {}
         for epoch in range(1, cls.config_get_final_epoch(config) + 1):
-            for is_pretrain in [False, True]:
-                fn = cls.epoch_model_filename(model_filename, epoch, is_pretrain)
+            for is_pretrain in [False, True] if util.BackendEngine.is_tensorflow_selected() else [False]:
+                fn = cls.epoch_model_filename(model_filename, epoch, is_pretrain=is_pretrain)
                 if os.path.exists(fn):
                     file_list[epoch] = fn
                     break
@@ -87,6 +87,10 @@ class EngineBase:
                         break
                 elif util.BackendEngine.is_torch_selected():
                     if os.path.exists(fn + ".pt"):
+                        if config.value("task", "train") == "train":
+                            # In case of training, we only want to consider the model if the optimizer state exists.
+                            if not os.path.exists(fn + ".opt.pt"):
+                                continue
                         file_list[epoch] = fn
                         break
         return file_list
@@ -197,12 +201,11 @@ class EngineBase:
         return start_epoch
 
     @classmethod
-    def epoch_model_filename(cls, model_filename, epoch, is_pretrain):
+    def epoch_model_filename(cls, model_filename: str, epoch: int, *, is_pretrain: bool = False) -> str:
         """
-        :type model_filename: str
-        :type epoch: int
-        :type is_pretrain: bool
-        :rtype: str
+        :param model_filename:
+        :param epoch:
+        :param is_pretrain:
         """
         if sys.platform == "win32" and model_filename.startswith("/tmp/"):
             import tempfile
@@ -218,7 +221,7 @@ class EngineBase:
         """
         if not epoch:
             epoch = self.epoch
-        return self.epoch_model_filename(self.model_filename, epoch, self.is_pretrain_epoch(epoch=epoch))
+        return self.epoch_model_filename(self.model_filename, epoch, is_pretrain=self.is_pretrain_epoch(epoch=epoch))
 
     def get_epoch_str(self):
         """

{returnn-1.20231024.171034 → returnn-1.20231025.122452}/returnn/sprint/interface.py

@@ -740,7 +740,10 @@ def _init_base(configfile=None, target_mode=None, epoch=None, sprint_opts=None):
 
     if target_mode and target_mode == "forward" and epoch:
         model_filename = config.value("model", "")
-        fns = [EngineBase.epoch_model_filename(model_filename, epoch, is_pretrain) for is_pretrain in [False, True]]
+        fns = [
+            EngineBase.epoch_model_filename(model_filename, epoch, is_pretrain=is_pretrain)
+            for is_pretrain in [False, True]
+        ]
         fn_postfix = ""
         if BackendEngine.is_tensorflow_selected():
             fn_postfix += ".meta"

{returnn-1.20231024.171034 → returnn-1.20231025.122452}/returnn/tf/engine.py

@@ -1730,7 +1730,9 @@ class Engine(EngineBase):
         print("start", self.get_epoch_str(), "with learning rate", self.learning_rate, "...", file=log.v4)
 
         if self.epoch == 1 and self.save_epoch1_initial_model:
-            epoch0_model_filename = self.epoch_model_filename(self.model_filename, 0, self.is_pretrain_epoch())
+            epoch0_model_filename = self.epoch_model_filename(
+                self.model_filename, 0, is_pretrain=self.is_pretrain_epoch()
+            )
             print("save initial epoch1 model", epoch0_model_filename, file=log.v4)
             self.save_model(epoch0_model_filename)
 

{returnn-1.20231024.171034 → returnn-1.20231025.122452}/returnn/torch/engine.py

@@ -671,9 +671,15 @@ class Engine(EngineBase):
             os.makedirs(directory, exist_ok=True)
 
         print("Save model under %s" % (filename,), file=log.v4)
+        # First write to a temp-file, to be sure that writing happens without errors,
+        # and only afterward rename to the target file.
+        tmp_filename = filename + ".tmp_write"
+        if os.path.exists(tmp_filename):
+            os.unlink(tmp_filename)
         torch.save(
-            {"model": self._pt_model.state_dict(), "epoch": self.epoch, "step": self.global_train_step}, filename
+            {"model": self._pt_model.state_dict(), "epoch": self.epoch, "step": self.global_train_step}, tmp_filename
         )
+        os.rename(tmp_filename, filename)
 
     def _load_optimizer(self):
         """
@@ -693,10 +699,6 @@ class Engine(EngineBase):
         if not self._do_save():
             return
         filename = self.get_epoch_model_filename() + ".opt" + util.get_model_filename_postfix()
-        directory = os.path.dirname(filename)
-        if directory and not os.path.exists(directory):
-            os.makedirs(directory, exist_ok=True)
-
         self._updater.save_optimizer(filename)
 
         # keep only the last two optimizer states (two in case one file gets corrupted)

{returnn-1.20231024.171034 → returnn-1.20231025.122452}/returnn/torch/updater.py

@@ -5,6 +5,7 @@ and model param update logic in general.
 
 from __future__ import annotations
 
+import os
 import gc
 import torch
 import typing
@@ -171,8 +172,18 @@ class Updater(object):
 
         :param str filename: File in which to save the optimizer state.
         """
+        directory = os.path.dirname(filename)
+        if directory and not os.path.exists(directory):
+            os.makedirs(directory, exist_ok=True)
+
         print("Save optimizer under %s" % filename, file=log.v4)
-        torch.save(self.optimizer.state_dict(), filename)
+        # First write to a temp-file, to be sure that writing happens without errors,
+        # and only afterward rename to the target file.
+        tmp_filename = filename + ".tmp_write"
+        if os.path.exists(tmp_filename):
+            os.unlink(tmp_filename)
+        torch.save(self.optimizer.state_dict(), tmp_filename)
+        os.rename(tmp_filename, filename)
 
     def get_optimizer(self):
         """

{returnn-1.20231024.171034 → returnn-1.20231025.122452/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231024.171034
+Version: 1.20231025.122452
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20231024.171034/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20231024.171034'
-long_version = '1.20231024.171034+git.bf567b9'