returnn 1.20240722.185624__tar.gz → 1.20240723.142327__tar.gz

{returnn-1.20240722.185624 → returnn-1.20240723.142327}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240722.185624
+Version: 1.20240723.142327
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240723.142327/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240723.142327'
+long_version = '1.20240723.142327+git.f1f1c76'

{returnn-1.20240722.185624 → returnn-1.20240723.142327}/returnn/datasets/lm.py

@@ -7,21 +7,23 @@ and some related helpers.
 
 from __future__ import annotations
 
+import typing
 import os
 import sys
-from .basic import DatasetSeq
-from .cached2 import CachedDataset2
+import time
+import re
 import gzip
 import xml.etree.ElementTree as ElementTree
-from returnn.util.basic import parse_orthography, parse_orthography_into_symbols, load_json, BackendEngine, unicode
-from returnn.util.literal_py_to_pickle import literal_eval
-from returnn.log import log
 import numpy
-import time
-import re
-import typing
 from random import Random
 
+from returnn.util.basic import parse_orthography, parse_orthography_into_symbols, load_json, unicode
+from returnn.util.literal_py_to_pickle import literal_eval
+from returnn.log import log
+
+from .basic import DatasetSeq
+from .cached2 import CachedDataset2
+
 
 class LmDataset(CachedDataset2):
     """
@@ -121,9 +123,11 @@ class LmDataset(CachedDataset2):
             # In this case, sentences end with self.word_end_symbol followed by the self.seq_end_symbol.
             self.parse_orth_opts.setdefault(
                 "postfix",
-                [self.word_end_symbol, self.seq_end_symbol]
-                if self.seq_end_symbol is not None
-                else [self.word_end_symbol],
+                (
+                    [self.word_end_symbol, self.seq_end_symbol]
+                    if self.seq_end_symbol is not None
+                    else [self.word_end_symbol]
+                ),
             )
         else:
             self.parse_orth_opts.setdefault("postfix", [self.seq_end_symbol] if self.seq_end_symbol is not None else [])
@@ -191,20 +195,17 @@ class LmDataset(CachedDataset2):
             self.orth_replace_map[" "] = [word_end_symbol]  # Replace all spaces by word_end_symbol.
 
         num_labels = len(self.labels["data"])
-        use_uint_types = False
-        if BackendEngine.is_tensorflow_selected():
-            use_uint_types = True
         if num_labels <= 2**7:
             self.dtype = "int8"
-        elif num_labels <= 2**8 and use_uint_types:
+        elif num_labels <= 2**8:
             self.dtype = "uint8"
         elif num_labels <= 2**31:
             self.dtype = "int32"
-        elif num_labels <= 2**32 and use_uint_types:
+        elif num_labels <= 2**32:
             self.dtype = "uint32"
-        elif num_labels <= 2**61:
+        elif num_labels <= 2**63:
             self.dtype = "int64"
-        elif num_labels <= 2**62 and use_uint_types:
+        elif num_labels <= 2**64:
             self.dtype = "uint64"
         else:
             raise Exception("cannot handle so much labels: %i" % num_labels)

{returnn-1.20240722.185624 → returnn-1.20240723.142327}/returnn/torch/engine.py

@@ -185,9 +185,9 @@ class Engine(EngineBase):
             for dataset_name, dataset_opts in config.typed_value("eval_datasets", {}).items():
                 self.eval_datasets[dataset_name] = init_dataset(dataset_opts, default_kwargs={"name": dataset_name})
 
-        self._train_dataloader = self._create_data_loader(train_data) if train_data else None
+        self._train_dataloader = self._create_data_loader(train_data, train=True) if train_data else None
         for dataset_name, dataset in self.eval_datasets.items():
-            self._eval_dataloaders[dataset_name] = self._create_data_loader(dataset)
+            self._eval_dataloaders[dataset_name] = self._create_data_loader(dataset, train=False)
 
         self._start_epoch = self.get_train_start_epoch(self.config)
         self._final_epoch = self.config_get_final_epoch(self.config)
@@ -588,7 +588,7 @@ class Engine(EngineBase):
             assert isinstance(ls[0], self.learning_rate_control.EpochData)
             self.learning_rate_control.epoch_data[self.epoch] = ls[0]
 
-    def _create_data_loader(self, dataset: Dataset) -> DataLoader:
+    def _create_data_loader(self, dataset: Dataset, *, train: bool = False) -> DataLoader:
         """
         :param dataset: RETURNN dataset
         :return: PyTorch data loader created from given RETURNN dataset
@@ -611,8 +611,9 @@ class Engine(EngineBase):
                 wrapped_dataset, chunking, min_chunk_size=min_chunk_size
             )
 
-        assert self.config.typed_value("batch_size") is not None, "batch_size not defined in config"
-        batch_size = self.config.typed_value("batch_size", 1)
+        batch_size = self.config.typed_value("batch_size", -1)
+        batch_size = self.config.typed_value(f"batch_size_{'train' if train else 'dev'}", batch_size)
+        assert batch_size != -1, f"batch_size or batch_size_{'train' if train else 'dev'} not defined in config"
         max_seqs = self.config.int("max_seqs", -1)
         batches_dataset = data_pipeline.BatchingIterDataPipe(wrapped_dataset, batch_size=batch_size, max_seqs=max_seqs)
 

{returnn-1.20240722.185624 → returnn-1.20240723.142327}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240722.185624
+Version: 1.20240723.142327
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240722.185624/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240722.185624'
-long_version = '1.20240722.185624+git.6884874'