returnn 1.20240307.132239__tar.gz → 1.20240309.105533__tar.gz

{returnn-1.20240307.132239/returnn.egg-info → returnn-1.20240309.105533}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240307.132239
+Version: 1.20240309.105533
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240309.105533/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240309.105533'
+long_version = '1.20240309.105533+git.7fea1f6'

{returnn-1.20240307.132239 → returnn-1.20240309.105533}/returnn/datasets/audio.py

@@ -3,7 +3,7 @@ Datasets dealing with audio
 """
 
 from __future__ import annotations
-from typing import Optional, Any, List, Dict
+from typing import Optional, Any, List, Tuple, Dict
 import numpy
 import typing
 
@@ -181,18 +181,15 @@ class OggZipDataset(CachedDataset2):
             return self._zip_files[zip_index].read(filename)
         return open("%s/%s" % (self.paths[0], filename), "rb").read()
 
-    def _collect_data_part(self, zip_index):
+    def _collect_data_part(self, zip_index) -> Tuple[List[Dict[str, Any]], Optional[Dict[str, Any]]]:
         """
         collect all the entries of a single zip-file or txt file
         :param int zip_index: index of the zip-file in self._zip_files, unused when loading without zip
-        :return: data entries
-        :rtype: list[dict[str]]
+        :return: data entries, example_entry
         """
         from returnn.util.literal_py_to_pickle import literal_eval
 
-        data = literal_eval(
-            self._read("%s.txt" % self._names[zip_index], zip_index)
-        )  # type: typing.List[typing.Dict[str]]
+        data: List[Dict[str, Any]] = literal_eval(self._read("%s.txt" % self._names[zip_index], zip_index))
         assert data and isinstance(data, list)
         first_entry = data[0]
         assert isinstance(first_entry, dict)
@@ -209,9 +206,12 @@ class OggZipDataset(CachedDataset2):
         # add index to data list
         for entry in data:
             entry["_zip_file_index"] = zip_index
+        example_entry = None
+        if data:
+            example_entry = data[0]  # before filtering
         if self.segments:
             data[:] = [entry for entry in data if self._get_tag_from_info_dict(entry) in self.segments]
-        return data
+        return data, example_entry
 
     def _lazy_init(self):
         """
@@ -230,13 +230,19 @@ class OggZipDataset(CachedDataset2):
             self._zip_files = [zipfile.ZipFile(path) for path in self.paths]
 
         data = []
+        example_entry = None
         if self._use_zip_files:
             for zip_index in range(len(self._zip_files)):
-                zip_data = self._collect_data_part(zip_index)
+                zip_data, example_entry = self._collect_data_part(zip_index)
                 data += zip_data
         else:
             # collect data from a txt file
-            data = self._collect_data_part(0)
+            data, example_entry = self._collect_data_part(0)
+
+        assert len(data) > 0, (
+            f"{self}: no data found? files {self._zip_files or self._path},"
+            f" filter {self.segments}, example {example_entry}"
+        )
 
         fixed_random_subset = self._fixed_random_subset
         if fixed_random_subset:
@@ -357,18 +363,16 @@ class OggZipDataset(CachedDataset2):
         """
         return True
 
-    def get_corpus_seq_idx(self, seq_idx):
+    def get_corpus_seq_idx(self, seq_idx: int) -> int:
         """
-        :param int seq_idx:
-        :rtype: int
+        :param seq_idx:
         """
         return self._get_ref_seq_idx(seq_idx)
 
     @staticmethod
-    def _get_tag_from_info_dict(info):
+    def _get_tag_from_info_dict(info: Dict[str, Any]) -> str:
         """
-        :param dict[str] info:
-        :rtype: str
+        :param info:
         """
         return info.get("seq_name", info.get("file", ""))
 

{returnn-1.20240307.132239 → returnn-1.20240309.105533}/returnn/datasets/util/vocabulary.py

@@ -216,6 +216,8 @@ class Vocabulary(object):
         if isinstance(label, str):
             return self.label_to_id(label, default=default)
         if isinstance(label, int):
+            if label == -1 and allow_none:
+                return None
             if self.is_id_valid(label):
                 return label
             if default is KeyError:

{returnn-1.20240307.132239 → returnn-1.20240309.105533}/returnn/engine/base.py

@@ -130,9 +130,9 @@ class EngineBase:
 
         load_model_epoch_filename = util.get_checkpoint_filepattern(config.value("load", ""))
         if load_model_epoch_filename:
-            assert os.path.exists(
-                load_model_epoch_filename + util.get_model_filename_postfix()
-            ), "load option %r, file %r does not exist" % (
+            assert os.path.exists(load_model_epoch_filename + util.get_model_filename_postfix()) or os.path.exists(
+                load_model_epoch_filename
+            ), "config option load=%r, file %r does not exist" % (
                 config.value("load", ""),
                 load_model_epoch_filename + util.get_model_filename_postfix(),
             )

{returnn-1.20240307.132239 → returnn-1.20240309.105533}/returnn/torch/engine.py

@@ -666,11 +666,13 @@ class Engine(EngineBase):
         checkpoint_state = None
         if model_epoch_filename:
             filename = model_epoch_filename + util.get_model_filename_postfix()
+            if not os.path.exists(filename) and os.path.exists(model_epoch_filename):
+                filename = model_epoch_filename
             print("Load model %s" % (filename,), file=log.v4)
             checkpoint_state = torch.load(filename, map_location=self._device)
             if epoch is None:
-                epoch = checkpoint_state["epoch"]
-            step = checkpoint_state["step"]
+                epoch = checkpoint_state.get("epoch", self._start_epoch or 1)
+            step = checkpoint_state.get("step", 1)
             print(f"  epoch {epoch}, global train step {step}", file=log.v4)
             # The checkpoint was saved when the step was already increased (but not the epoch yet).
             # Restore the last step.
@@ -690,9 +692,10 @@ class Engine(EngineBase):
         missing_keys = set()
         unexpected_keys = set()
         if checkpoint_state is not None:
-            loaded_state_keys.update(checkpoint_state["model"].keys())
+            model_state = checkpoint_state.get("model", checkpoint_state)
+            loaded_state_keys.update(model_state.keys())
             missing_keys_main_ckpt, unexpected_keys_main_ckpt = self._pt_model.load_state_dict(
-                checkpoint_state["model"], strict=False
+                model_state, strict=False
             )
             missing_keys.update(missing_keys_main_ckpt)
             unexpected_keys.update(unexpected_keys_main_ckpt)
@@ -702,6 +705,10 @@ class Engine(EngineBase):
                     + ", ".join(map(repr, sorted(unexpected_keys_main_ckpt))),
                     file=log.v4,
                 )
+            del model_state
+        # https://github.com/rwth-i6/returnn/issues/1345
+        del checkpoint_state
+        gc.collect()
 
         preload_from_files = self.config.typed_value("preload_from_files", {})
         if preload_from_files:
@@ -788,9 +795,6 @@ class Engine(EngineBase):
                 )
             )
 
-        # https://github.com/rwth-i6/returnn/issues/1345
-        del checkpoint_state
-        gc.collect()
         self._pt_model.to(self._device)
 
         if model_epoch_filename and is_training:

{returnn-1.20240307.132239 → returnn-1.20240309.105533/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240307.132239
+Version: 1.20240309.105533
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240307.132239/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240307.132239'
-long_version = '1.20240307.132239+git.c1ac801'