returnn 1.20250206.144022__py3-none-any.whl → 1.20250206.151011__py3-none-any.whl

returnn/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250206.144022
+Version: 1.20250206.151011
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn/_setup_info_generated.py

@@ -1,2 +1,2 @@
-version = '1.20250206.144022'
-long_version = '1.20250206.144022+git.550e757'
+version = '1.20250206.151011'
+long_version = '1.20250206.151011+git.6fa4b38'

returnn/torch/engine.py

@@ -3,7 +3,7 @@ Main engine for PyTorch
 """
 
 from __future__ import annotations
-from typing import Optional, Any, Union, Callable, Dict, Set
+from typing import Optional, Any, Union, Callable, Dict, Set, Tuple
 from contextlib import nullcontext, ExitStack, contextmanager
 
 import gc
@@ -371,6 +371,7 @@ class Engine(EngineBase):
         total_data_size_packed = NumbersDict()
         total_data_size_padded = NumbersDict()
 
+        report_prefix = f"ep {self.epoch} train"
         try:
             while True:
                 with torch.no_grad():
@@ -398,21 +399,13 @@ class Engine(EngineBase):
                     {k: int(util.prod(extern_data_raw[k].shape[:2])) for k in keys_w_seq_len},
                 )
 
-                num_seqs_ = (
-                    int(extern_data_raw["num_seqs"]) if extern_data_raw.get("num_seqs", None) is not None else -1
+                num_seqs, last_seq_idx = _get_num_seqs_last_seq_idx(
+                    report_prefix=report_prefix,
+                    extern_data_raw=extern_data_raw,
+                    step_idx=step_idx,
+                    prev_num_seqs=num_seqs,
+                    prev_last_seq_idx=last_seq_idx,
                 )
-                # Note: The batches might have been shuffled,
-                # thus we cannot really assert that the seq_idx is always increasing.
-                last_seq_idx = max(int(extern_data_raw["seq_idx"].max()), last_seq_idx)
-                if step_idx == 0:
-                    if num_seqs_ >= 0:
-                        print(f"Epoch {self.epoch} num_seqs: {num_seqs_}", file=log.v5)
-                        num_seqs = num_seqs_
-                elif num_seqs_ >= 0:
-                    assert num_seqs_ == num_seqs
-                del num_seqs_
-                if num_seqs is not None:
-                    assert last_seq_idx < num_seqs
                 epoch_continuous = (self.epoch - 1 + (last_seq_idx + 1) / num_seqs) if num_seqs is not None else None
 
                 # clear the gradients when every gradient accumulation loop starts
@@ -485,7 +478,7 @@ class Engine(EngineBase):
                 accumulated_inv_norm_factors_dict += inv_norm_factors_dict
                 eval_info = self._maybe_extend_losses_info(losses_dict / inv_norm_factors_dict)
                 _print_process(
-                    f"ep {self.epoch} train",
+                    report_prefix,
                     step=step_idx,
                     eval_info=dict(eval_info),
                     step_duration=step_duration,
@@ -1276,6 +1269,8 @@ class Engine(EngineBase):
             new_dim.dyn_size_ext = _get_tensor_wo_batch_numpy(dim.dyn_size_ext)
             return new_dim
 
+        num_seqs = None
+        last_seq_idx = 0
         report_prefix = f"ep {self.epoch} {dataset.name} forward"
         with torch.no_grad():
             callback.init(model=self._orig_model)
@@ -1283,6 +1278,15 @@ class Engine(EngineBase):
             step_idx = 0
             for extern_data_raw in data_loader:
                 step_begin_time = time.monotonic()
+
+                num_seqs, last_seq_idx = _get_num_seqs_last_seq_idx(
+                    report_prefix=report_prefix,
+                    extern_data_raw=extern_data_raw,
+                    step_idx=step_idx,
+                    prev_num_seqs=num_seqs,
+                    prev_last_seq_idx=last_seq_idx,
+                )
+
                 if self._forward_step_expected_outputs:
                     # Also resets any dyn dims, which might have been set in the prev step.
                     self._forward_step_expected_outputs.reset_content()
@@ -1319,11 +1323,19 @@ class Engine(EngineBase):
                         model_outputs_per_batch.data[k] = _get_tensor_wo_batch_numpy(v)
                     callback.process_seq(seq_tag=seq_tag, outputs=model_outputs_per_batch)
 
-                elapsed_computation_time += time.monotonic() - step_begin_time
+                step_end_time = time.monotonic()
+                step_duration = step_end_time - step_begin_time
+                elapsed_computation_time += step_duration
+
                 _print_process(
                     report_prefix,
                     step=step_idx,
                     eval_info=None,
+                    step_duration=step_duration,
+                    start_elapsed=step_end_time - epoch_start_time,
+                    seq_idx=last_seq_idx,
+                    num_seqs=num_seqs,
+                    batch_size_info=_get_batch_size_info(extern_data) if self._log_batch_size else None,
                     log_memory_usage_device=self._device if self._log_memory_usage else None,
                 )
                 step_idx += 1
@@ -1601,3 +1613,27 @@ def _get_total_grad_norm(model: torch.nn.Module, p: float) -> float:
             p=p,
         ).item()
     )
+
+
+def _get_num_seqs_last_seq_idx(
+    *,
+    report_prefix: str,
+    extern_data_raw: Dict[str, Any],
+    step_idx: int,
+    prev_num_seqs: Optional[int],
+    prev_last_seq_idx: int,
+) -> Tuple[Optional[int], int]:
+    num_seqs = prev_num_seqs
+    num_seqs_ = int(extern_data_raw["num_seqs"]) if extern_data_raw.get("num_seqs", None) is not None else -1
+    # Note: The batches might have been shuffled,
+    # thus we cannot really assert that the seq_idx is always increasing.
+    last_seq_idx = max(int(extern_data_raw["seq_idx"].max()), prev_last_seq_idx)
+    if step_idx == 0:
+        if num_seqs_ >= 0:
+            print(f"{report_prefix} num_seqs: {num_seqs_}", file=log.v5)
+            num_seqs = num_seqs_
+    elif num_seqs_ >= 0:
+        assert num_seqs_ == num_seqs
+    if num_seqs is not None:
+        assert last_seq_idx < num_seqs
+    return num_seqs, last_seq_idx

{returnn-1.20250206.144022.dist-info → returnn-1.20250206.151011.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250206.144022
+Version: 1.20250206.151011
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250206.144022.dist-info → returnn-1.20250206.151011.dist-info}/RECORD

@@ -1,9 +1,9 @@
-returnn/PKG-INFO,sha256=vBdT0ayV-Q8OjPdp1xlJt0CiopUZKNWKwKuadpiKHDk,5215
+returnn/PKG-INFO,sha256=BbQPkoVha1AYEcED8txzZyyyDiJt3J29FBKlYy1rTYc,5215
 returnn/__init__.py,sha256=biBtRsM0WZ406vShaeH-9WFoqJ8XwTbn6g0EeFJ7l8E,1012
 returnn/__main__.py,sha256=qBFbuB1yN3adgVM5pXt2-Yq9vorjRNchNPL8kDKx44M,31752
 returnn/__old_mod_loader__.py,sha256=nvsNY-xELdS_IPNkv66Q9Rmvg4dbGW0-EBRDcCmctos,7654
 returnn/__setup__.py,sha256=22kQn2fh11iPM0hLb2Fy5sLmoU1JGvmDxXRYuRgQkwU,4659
-returnn/_setup_info_generated.py,sha256=uAkEz6DVwoN42Nh2WLNsoE4lJ0BtlRznPPlXMWKxJQo,77
+returnn/_setup_info_generated.py,sha256=9T1yfQUP7ASjffpzcwvOLEGNWMdiwS4EmjqY_bI2EdY,77
 returnn/config.py,sha256=3tmKhB6FnQZaNdtcYsiB61JnEY--iZ2qmJ4yq0b6tE0,29140
 returnn/forward_iface.py,sha256=A_OJiaXsX4MlXQRzST86ylyxSUZbC402PQL1REcqHjM,911
 returnn/learning_rate_control.py,sha256=ZvWryAn_tv9DhV8sh1LV3eE34Yltl3On3mYZAG4hR9s,34684
@@ -207,7 +207,7 @@ returnn/tf/util/open_fst.py,sha256=sZRDw4TbxvhGqpGdUJWy1ebvlZm4_RPhygpRw9uLAOQ,1
 returnn/torch/README.md,sha256=jzJ2FpOHW02vxN69yKaV97C9LI-hmvjBglKfdZXIDdc,85
 returnn/torch/__init__.py,sha256=MHEUyNHB20Vy89uKAqZoj6FxJKF1Gq3HW-i6ra1pNcI,24
 returnn/torch/distributed.py,sha256=i13cUVjI7GxpO0TAresrNyCM0ZBAaf-cXNr09Fmg_2k,6266
-returnn/torch/engine.py,sha256=Zd3ePKFSi5fkvV1FxaYn0QGgu5cag_ocKPwFmKglf3I,75095
+returnn/torch/engine.py,sha256=neM-AL7XQLpZ3V1K4ziqVmij19ey1k2MpLCaFXATOpg,76301
 returnn/torch/updater.py,sha256=GqtBvZpElPVMm0lq84JPl4NVLFFETZAzAbR0rTomSao,28249
 returnn/torch/data/__init__.py,sha256=6cLNEi8KoGI12PF6akN7mI_mtjlx-0hcQAfMYoExwik,132
 returnn/torch/data/extern_data.py,sha256=_uT_9_gd5HIh1IoRsrebVG-nufSnb7fgC5jyU05GxJg,7580
@@ -253,8 +253,8 @@ returnn/util/sig_proc.py,sha256=Tjz0VOAVyqu2qDCF5HZ1JjALjcFsHcNkcd96WgZeKfE,7265
 returnn/util/task_system.py,sha256=y4sMVXQ25Qd2z0rx03uOlXlkE-jbCYC1Sjfn-XlraVU,26003
 returnn/util/train_proc_manager.py,sha256=Pjht28k6uz6BNQ47uW6Gf880iyq5q4wx7P_K2tmoAM8,3266
 returnn/util/watch_memory.py,sha256=BR5P2kvBN6UI81cE0_1WAA6Hd1SByLbBaiDxvLhPOew,4213
-returnn-1.20250206.144022.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
-returnn-1.20250206.144022.dist-info/METADATA,sha256=vBdT0ayV-Q8OjPdp1xlJt0CiopUZKNWKwKuadpiKHDk,5215
-returnn-1.20250206.144022.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-returnn-1.20250206.144022.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
-returnn-1.20250206.144022.dist-info/RECORD,,
+returnn-1.20250206.151011.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
+returnn-1.20250206.151011.dist-info/METADATA,sha256=BbQPkoVha1AYEcED8txzZyyyDiJt3J29FBKlYy1rTYc,5215
+returnn-1.20250206.151011.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+returnn-1.20250206.151011.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
+returnn-1.20250206.151011.dist-info/RECORD,,