returnn 1.20240112.5115__tar.gz → 1.20240114.223020__tar.gz

{returnn-1.20240112.5115/returnn.egg-info → returnn-1.20240114.223020}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240112.5115
+Version: 1.20240114.223020
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240114.223020/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240114.223020'
+long_version = '1.20240114.223020+git.d122112'

{returnn-1.20240112.5115 → returnn-1.20240114.223020}/returnn/engine/base.py

@@ -103,9 +103,21 @@ class EngineBase:
         return file_list
 
     @classmethod
-    def get_epoch_model(cls, config):
+    def get_start_epoch_no_existing_model(cls, config: Config) -> int:
+        """
+        :return: start epoch if no model exists
+        """
+        start_epoch_mode = config.value("start_epoch", "auto")
+        if start_epoch_mode == "auto":
+            return 1
+        else:
+            start_epoch = int(start_epoch_mode)
+            assert start_epoch >= 1
+            return start_epoch
+
+    @classmethod
+    def get_epoch_model(cls, config: Config):
         """
-        :type config: returnn.config.Config
         :return: (epoch, model_filename). epoch is the epoch of the model filename.
         :rtype: (int|None, str|None)
         """
@@ -393,7 +405,7 @@ class EngineBase:
             count_bytes += self.delete_model(existing_models[epoch])
         print("Deleted %s." % human_bytes_size(count_bytes), file=log.v2)
 
-    def _is_dataset_evaluated(self, name: str) -> bool:
+    def _is_dataset_evaluated(self, name: str, *, epoch: Optional[int] = None) -> bool:
         """
         Check via self.learning_rate_control.
 
@@ -401,7 +413,9 @@ class EngineBase:
         :return: whether there is an entry for the score in the learning rate file
         """
         assert self.learning_rate_control.filename  # otherwise we would not have stored it
-        error_dict = self.learning_rate_control.get_epoch_error_dict(self.epoch)
+        if epoch is None:
+            epoch = self.epoch
+        error_dict = self.learning_rate_control.get_epoch_error_dict(epoch)
         if not error_dict:
             return False
         return any([k.startswith("%s_score" % name) or k.startswith("%s_loss" % name) for k in error_dict.keys()])

{returnn-1.20240112.5115 → returnn-1.20240114.223020}/returnn/tf/engine.py

@@ -2132,7 +2132,7 @@ class Engine(EngineBase):
             for name, dataset in self.get_eval_datasets().items():
                 if not self._is_dataset_evaluated(name=name):
                     # This can happen when we have a previous model but did not test it yet.
-                    print("Last epoch model not yet evaluated on dev. Doing that now.", file=log.v4)
+                    print(f"Last epoch model not yet evaluated on {name}. Doing that now.", file=log.v4)
                     self.eval_model(skip_already_evaluated=True)
                     break
 

{returnn-1.20240112.5115 → returnn-1.20240114.223020}/returnn/torch/engine.py

@@ -217,6 +217,8 @@ class Engine(EngineBase):
         assert self._pt_model is not None, "Model not initialized, call init_train_from_config()."
         assert self.train_dataset is not None, "Train dataset missing, call init_train_from_config() with train_data."
 
+        self._check_missing_eval()
+
         if self._start_epoch > self._final_epoch:
             print(f"Already trained until final epoch {self._final_epoch}, nothing to do.", file=log.v3)
             return
@@ -224,10 +226,7 @@ class Engine(EngineBase):
         print(
             f"Starting training at epoch {self._start_epoch}, global train step {self.global_train_step}", file=log.v3
         )
-        start_epoch = self._start_epoch
-        self.epoch = start_epoch - 1
-        self._check_epoch_missing_eval()
-
+        self.epoch = self._start_epoch - 1
         while self.epoch + 1 <= self._final_epoch:
             self.epoch += 1
             self._epoch_mp_shared.value = self.epoch
@@ -473,13 +472,6 @@ class Engine(EngineBase):
         if self.config.bool_or_other("cleanup_old_models", None):
             self.cleanup_old_models()
 
-        # Maybe sync train/dev/eval scores for the epoch.
-        if self._torch_distributed_ctx:
-            ls = [self.learning_rate_control.epoch_data[self.epoch]]
-            torch.distributed.broadcast_object_list(ls, src=0, device=torch.device("cpu"))
-            assert isinstance(ls[0], self.learning_rate_control.EpochData)
-            self.learning_rate_control.epoch_data[self.epoch] = ls[0]
-
     def _do_save(self):
         if self._device == "meta":
             return False
@@ -575,6 +567,13 @@ class Engine(EngineBase):
 
         self._maybe_report_dev_memory_stats()
 
+        # Maybe sync train/dev/eval scores for the epoch.
+        if self._torch_distributed_ctx:
+            ls = [self.learning_rate_control.epoch_data[self.epoch]]
+            torch.distributed.broadcast_object_list(ls, src=0, device=torch.device("cpu"))
+            assert isinstance(ls[0], self.learning_rate_control.EpochData)
+            self.learning_rate_control.epoch_data[self.epoch] = ls[0]
+
     def _create_data_loader(self, dataset: Dataset) -> DataLoader:
         """
         :param dataset: RETURNN dataset
@@ -608,6 +607,7 @@ class Engine(EngineBase):
         if loader_opts.get("num_workers"):
             loader_opts.setdefault("persistent_workers", True)
             loader_opts.setdefault("worker_init_fn", _data_loader_worker_init_func)
+            loader_opts.setdefault("multiprocessing_context", "spawn")
 
         return DataLoader(
             batches_dataset,
@@ -1029,20 +1029,32 @@ class Engine(EngineBase):
         assert count_bytes > 0
         return count_bytes
 
-    def _check_epoch_missing_eval(self):
+    def _check_missing_eval(self):
         """
         Checks if there are outstanding tasks (eval_model) for the epoch,
         and executes them.
         """
-        if not self.epoch:
+        if not self.learning_rate_control.filename:
+            return
+        if self._start_epoch == 1:
             return
-        if self.learning_rate_control.filename:
-            for name, dataset in self.eval_datasets.items():
-                if not self._is_dataset_evaluated(name=name):
-                    # This can happen when we have a previous model but did not test it yet.
-                    print(f"Last epoch model not yet evaluated on {name}. Doing that now.", file=log.v3)
-                    self.eval_model(skip_already_evaluated=True)
-                    break
+
+        # If the learning rate (scores) file is somehow corrupt,
+        # check that and stop, otherwise we would potentially corrupt the data even more.
+        for epoch in range(self.get_start_epoch_no_existing_model(self.config), self._start_epoch - 1):
+            for name in ["train"] + list(self.eval_datasets.keys()):
+                if not self._is_dataset_evaluated(name=name, epoch=epoch):
+                    raise Exception(f"Scores of epoch {epoch} for {name} are missing.")
+
+        self.epoch = self._start_epoch - 1
+        if not self._is_dataset_evaluated(name="train"):
+            raise Exception(f"Scores of last train epoch {self.epoch} are missing.")
+        for name in self.eval_datasets.keys():
+            if not self._is_dataset_evaluated(name=name):
+                # This can happen when we have a previous model but did not test it yet.
+                print(f"Last epoch model not yet evaluated on {name}. Doing that now.", file=log.v3)
+                self.eval_model(skip_already_evaluated=True)
+                break
 
 
 def _to_raw(n: Union[int, float, Tensor]):

{returnn-1.20240112.5115 → returnn-1.20240114.223020/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240112.5115
+Version: 1.20240114.223020
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240112.5115/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240112.005115'
-long_version = '1.20240112.005115+git.6d28299'