returnn 1.20230811.13935__tar.gz → 1.20230811.150625__tar.gz

{returnn-1.20230811.13935/returnn.egg-info → returnn-1.20230811.150625}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20230811.13935
+Version: 1.20230811.150625
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20230811.150625/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20230811.150625'
+long_version = '1.20230811.150625+git.6d464e0'

{returnn-1.20230811.13935 → returnn-1.20230811.150625}/returnn/torch/engine.py

@@ -3,7 +3,7 @@ Main engine for PyTorch
 """
 
 from __future__ import annotations
-from typing import Optional, Union, Callable, Dict, List
+from typing import Optional, Any, Union, Callable, Dict, List
 from contextlib import nullcontext
 
 import gc
@@ -236,8 +236,6 @@ class Engine(EngineBase):
         """
         print("start", self.get_epoch_str(), "with learning rate", self.learning_rate, "...", file=log.v4)
 
-        self._pt_model.train()
-
         accumulated_losses_dict = NumbersDict()
         accumulated_inv_norm_factors_dict = NumbersDict()
         step_idx = 0
@@ -246,11 +244,15 @@ class Engine(EngineBase):
         data_iter = iter(self._train_dataloader)
         elapsed_computation_time = 0
 
+        self._pt_model.train()
+
         while True:
             with torch.no_grad():
                 extern_data_raw = next(data_iter, None)
-            _has_data = torch.tensor([extern_data_raw is not None], dtype=torch.int8)
 
+            step_begin_time = time.time()
+
+            _has_data = torch.tensor([extern_data_raw is not None], dtype=torch.int8)
             if self._use_torch_distributed:
                 # use all reduce to check if all workers have data, if at least one worker does not have data,
                 # all workers finish this epoch
@@ -262,8 +264,6 @@ class Engine(EngineBase):
             if step_idx % self._accum_grad_multiple_step == 0:
                 self._updater.get_optimizer().zero_grad()
 
-            step_begin_time = time.time()
-
             extern_data = _raw_dict_to_extern_data(
                 extern_data_raw, extern_data_template=self.extern_data, device=self._device
             )
@@ -653,6 +653,9 @@ class Engine(EngineBase):
         assert isinstance(dataset, Dataset)
         assert isinstance(callback, ForwardCallbackIface)
 
+        epoch_start_time = time.time()
+        elapsed_computation_time = 0.0
+
         self._pt_model.eval()
 
         if dataset.supports_seq_order_sorting():
@@ -716,7 +719,9 @@ class Engine(EngineBase):
         with torch.no_grad():
             callback.init(model=self._orig_model)
 
+            step_idx = 0
             for extern_data_raw in data_loader:
+                step_begin_time = time.time()
                 extern_data = _raw_dict_to_extern_data(
                     extern_data_raw, extern_data_template=self.extern_data, device=self._device
                 )
@@ -732,8 +737,20 @@ class Engine(EngineBase):
                         model_outputs_per_batch.data[k] = _get_tensor_wo_batch_numpy(v)
                     callback.process_seq(seq_tag=seq_tag, outputs=model_outputs_per_batch)
 
+                elapsed_computation_time += time.time() - step_begin_time
+                _print_process(f"ep {self.epoch} {dataset.name} forward", step=step_idx, eval_info=None)
+                step_idx += 1
+
             callback.finish()
 
+        elapsed = time.time() - epoch_start_time
+        elapsed_computation_percentage = elapsed_computation_time / elapsed
+        print(
+            "Forward %i steps, %s elapsed (%.1f%% computing time)"
+            % (step_idx, hms(elapsed), (elapsed_computation_percentage * 100.0)),
+            file=log.v3,
+        )
+
     @staticmethod
     def delete_model(filename):
         """
@@ -851,13 +868,13 @@ def _get_dyn_dims_from_extern_data(extern_data: TensorDict) -> List[Dim]:
     return res
 
 
-def _print_process(report_prefix, step, eval_info):
+def _print_process(report_prefix: str, step: int, eval_info: Optional[Dict[str, Any]] = None):
     """
     Similar but simplified from TF engine _print_process.
 
-    :param str report_prefix:
-    :param int step:
-    :param dict[str] eval_info: via :func:`_collect_eval_info`
+    :param report_prefix:
+    :param step:
+    :param eval_info:
     :return: nothing, will be printed to log
     """
     if log.verbose[5]:

{returnn-1.20230811.13935 → returnn-1.20230811.150625/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20230811.13935
+Version: 1.20230811.150625
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20230811.13935/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20230811.013935'
-long_version = '1.20230811.013935+git.31c1bbe'