returnn 1.20230811.3923__tar.gz → 1.20230811.150625__tar.gz

{returnn-1.20230811.3923/returnn.egg-info → returnn-1.20230811.150625}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20230811.3923
+Version: 1.20230811.150625
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20230811.150625/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20230811.150625'
+long_version = '1.20230811.150625+git.6d464e0'

{returnn-1.20230811.3923 → returnn-1.20230811.150625}/returnn/frontend/signal.py

@@ -126,7 +126,11 @@ def mel_filterbank(
     backend = x._raw_backend
     cache_key = (f_min, f_max, sampling_rate, fft_length, out_dim.dimension, backend, x.dtype, x.device)
     if cache_key in _mel_filter_bank_matrix_cache:
-        filter_bank_matrix = _mel_filter_bank_matrix_cache[cache_key]
+        filter_bank_matrix_: Tensor = _mel_filter_bank_matrix_cache[cache_key]
+        # The in_dim might be temporarily created, so replace it by the new one.
+        # Even allow for temporary out_dim.
+        filter_bank_matrix = Tensor("mel-filter-bank", dims=(in_dim, out_dim), dtype=x.dtype)
+        filter_bank_matrix.raw_tensor = filter_bank_matrix_.raw_tensor
     else:
         filter_bank_matrix_np = _mel_filter_bank_matrix_np(
             f_min=f_min, f_max=f_max, sampling_rate=sampling_rate, fft_size=fft_length, nr_of_filters=out_dim.dimension
@@ -139,7 +143,9 @@ def mel_filterbank(
                 # Very simple cache management. No LRU logic or anything like that.
                 _mel_filter_bank_matrix_cache.clear()
             _mel_filter_bank_matrix_cache[cache_key] = filter_bank_matrix
-    return rf.matmul(x, filter_bank_matrix, reduce=in_dim)
+    out = rf.matmul(x, filter_bank_matrix, reduce=in_dim)
+    out.feature_dim = out_dim
+    return out
 
 
 # Used for eager frameworks. See mel_filterbank.

{returnn-1.20230811.3923 → returnn-1.20230811.150625}/returnn/torch/engine.py

@@ -3,7 +3,7 @@ Main engine for PyTorch
 """
 
 from __future__ import annotations
-from typing import Optional, Union, Callable, Dict, List
+from typing import Optional, Any, Union, Callable, Dict, List
 from contextlib import nullcontext
 
 import gc
@@ -236,8 +236,6 @@ class Engine(EngineBase):
         """
         print("start", self.get_epoch_str(), "with learning rate", self.learning_rate, "...", file=log.v4)
 
-        self._pt_model.train()
-
         accumulated_losses_dict = NumbersDict()
         accumulated_inv_norm_factors_dict = NumbersDict()
         step_idx = 0
@@ -246,11 +244,15 @@ class Engine(EngineBase):
         data_iter = iter(self._train_dataloader)
         elapsed_computation_time = 0
 
+        self._pt_model.train()
+
         while True:
             with torch.no_grad():
                 extern_data_raw = next(data_iter, None)
-            _has_data = torch.tensor([extern_data_raw is not None], dtype=torch.int8)
 
+            step_begin_time = time.time()
+
+            _has_data = torch.tensor([extern_data_raw is not None], dtype=torch.int8)
             if self._use_torch_distributed:
                 # use all reduce to check if all workers have data, if at least one worker does not have data,
                 # all workers finish this epoch
@@ -262,8 +264,6 @@ class Engine(EngineBase):
             if step_idx % self._accum_grad_multiple_step == 0:
                 self._updater.get_optimizer().zero_grad()
 
-            step_begin_time = time.time()
-
             extern_data = _raw_dict_to_extern_data(
                 extern_data_raw, extern_data_template=self.extern_data, device=self._device
             )
@@ -653,6 +653,9 @@ class Engine(EngineBase):
         assert isinstance(dataset, Dataset)
         assert isinstance(callback, ForwardCallbackIface)
 
+        epoch_start_time = time.time()
+        elapsed_computation_time = 0.0
+
         self._pt_model.eval()
 
         if dataset.supports_seq_order_sorting():
@@ -716,7 +719,9 @@ class Engine(EngineBase):
         with torch.no_grad():
             callback.init(model=self._orig_model)
 
+            step_idx = 0
             for extern_data_raw in data_loader:
+                step_begin_time = time.time()
                 extern_data = _raw_dict_to_extern_data(
                     extern_data_raw, extern_data_template=self.extern_data, device=self._device
                 )
@@ -732,8 +737,20 @@ class Engine(EngineBase):
                         model_outputs_per_batch.data[k] = _get_tensor_wo_batch_numpy(v)
                     callback.process_seq(seq_tag=seq_tag, outputs=model_outputs_per_batch)
 
+                elapsed_computation_time += time.time() - step_begin_time
+                _print_process(f"ep {self.epoch} {dataset.name} forward", step=step_idx, eval_info=None)
+                step_idx += 1
+
             callback.finish()
 
+        elapsed = time.time() - epoch_start_time
+        elapsed_computation_percentage = elapsed_computation_time / elapsed
+        print(
+            "Forward %i steps, %s elapsed (%.1f%% computing time)"
+            % (step_idx, hms(elapsed), (elapsed_computation_percentage * 100.0)),
+            file=log.v3,
+        )
+
     @staticmethod
     def delete_model(filename):
         """
@@ -851,13 +868,13 @@ def _get_dyn_dims_from_extern_data(extern_data: TensorDict) -> List[Dim]:
     return res
 
 
-def _print_process(report_prefix, step, eval_info):
+def _print_process(report_prefix: str, step: int, eval_info: Optional[Dict[str, Any]] = None):
     """
     Similar but simplified from TF engine _print_process.
 
-    :param str report_prefix:
-    :param int step:
-    :param dict[str] eval_info: via :func:`_collect_eval_info`
+    :param report_prefix:
+    :param step:
+    :param eval_info:
     :return: nothing, will be printed to log
     """
     if log.verbose[5]:

{returnn-1.20230811.3923 → returnn-1.20230811.150625/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20230811.3923
+Version: 1.20230811.150625
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20230811.150625/tests/test_rf_signal.py

@@ -0,0 +1,59 @@
+"""
+RETURNN frontend (returnn.frontend) tests
+"""
+
+from __future__ import annotations
+from typing import Tuple
+import _setup_test_env  # noqa
+import returnn.frontend as rf
+from returnn.tensor import Tensor, Dim, TensorDict, batch_dim
+from rf_utils import run_model
+
+
+def test_stft():
+    time_dim = Dim(Tensor("time", [batch_dim], dtype="int32"))
+    extern_data = TensorDict(
+        {
+            "data": Tensor("data", [batch_dim, time_dim], dtype="float32"),
+        }
+    )
+
+    class _Net(rf.Module):
+        def __call__(self, x: rf.Tensor, *, in_spatial_dim: Dim) -> Tuple[Tensor, Dim, Dim]:
+            return rf.stft(x, in_spatial_dim=in_spatial_dim, frame_step=3, frame_length=5, fft_length=6)
+
+    # noinspection PyShadowingNames
+    def _forward_step(*, model: _Net, extern_data: TensorDict):
+        out, out_spatial_dim, out_dim = model(extern_data["data"], in_spatial_dim=time_dim)
+        out.mark_as_default_output(shape=(batch_dim, out_spatial_dim, out_dim))
+
+    run_model(extern_data, lambda *, epoch, step: _Net(), _forward_step)
+
+
+def test_mel_filterbank():
+    time_dim = Dim(Tensor("time", [batch_dim], dtype="int32"))
+    feat_dim = Dim(10, name="mel")
+    extern_data = TensorDict(
+        {
+            "data": Tensor("data", [batch_dim, time_dim], dtype="float32"),
+        }
+    )
+
+    class _Net(rf.Module):
+        def __call__(self, source: rf.Tensor, *, in_spatial_dim: Dim) -> Tuple[Tensor, Dim]:
+            # log mel filterbank features
+            source, in_spatial_dim, in_dim_ = rf.stft(
+                source, in_spatial_dim=in_spatial_dim, frame_step=3, frame_length=5, fft_length=6
+            )
+            source = rf.abs(source) ** 2.0
+            source = rf.mel_filterbank(source, in_dim=in_dim_, out_dim=feat_dim, sampling_rate=16)
+            return source, in_spatial_dim
+
+    # noinspection PyShadowingNames
+    def _forward_step(*, model: _Net, extern_data: TensorDict):
+        out, out_spatial_dim = model(extern_data["data"], in_spatial_dim=time_dim)
+        out.mark_as_default_output(shape=(batch_dim, out_spatial_dim, feat_dim))
+
+    run_model(extern_data, lambda *, epoch, step: _Net(), _forward_step)
+    # Run again to specifically test the caching logic.
+    run_model(extern_data, lambda *, epoch, step: _Net(), _forward_step)

returnn-1.20230811.3923/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20230811.003923'
-long_version = '1.20230811.003923+git.b44d4de'

returnn-1.20230811.3923/tests/test_rf_signal.py

@@ -1,30 +0,0 @@
-"""
-RETURNN frontend (returnn.frontend) tests
-"""
-
-from __future__ import annotations
-from typing import Tuple
-import _setup_test_env  # noqa
-import returnn.frontend as rf
-from returnn.tensor import Tensor, Dim, TensorDict, batch_dim
-from rf_utils import run_model
-
-
-def test_stft():
-    time_dim = Dim(Tensor("time", [batch_dim], dtype="int32"))
-    extern_data = TensorDict(
-        {
-            "data": Tensor("data", [batch_dim, time_dim], dtype="float32"),
-        }
-    )
-
-    class _Net(rf.Module):
-        def __call__(self, x: rf.Tensor, *, in_spatial_dim: Dim) -> Tuple[Tensor, Dim, Dim]:
-            return rf.stft(x, in_spatial_dim=in_spatial_dim, frame_step=3, frame_length=5, fft_length=6)
-
-    # noinspection PyShadowingNames
-    def _forward_step(*, model: _Net, extern_data: TensorDict):
-        out, out_spatial_dim, out_dim = model(extern_data["data"], in_spatial_dim=time_dim)
-        out.mark_as_default_output(shape=(batch_dim, out_spatial_dim, out_dim))
-
-    run_model(extern_data, lambda *, epoch, step: _Net(), _forward_step)