returnn 1.20240203.124331__tar.gz → 1.20240205.110004__tar.gz

{returnn-1.20240203.124331/returnn.egg-info → returnn-1.20240205.110004}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240203.124331
+Version: 1.20240205.110004
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240205.110004/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240205.110004'
+long_version = '1.20240205.110004+git.1e0a03d'

{returnn-1.20240203.124331 → returnn-1.20240205.110004}/returnn/datasets/basic.py

@@ -705,19 +705,28 @@ class Dataset(object):
         else:
             return self.get_targets(key, seq_idx)
 
-    def get_input_data(self, sorted_seq_idx):
+    def get_input_data(self, sorted_seq_idx: int) -> numpy.ndarray:
         """
-        :type sorted_seq_idx: int
-        :rtype: numpy.ndarray
+        DEPRECATED: Some older classes still use this deprecated API,
+        but any new dataset should just implement :func:`get_data`,
+        and users also should just use :func:`get_data`.
+
+        This default implementation assumes that there is a "data" data key,
+        which is not necessarily true in all cases.
+
+        :param sorted_seq_idx:
         :returns features: format 2d (time,feature) (float)
         """
-        raise NotImplementedError
+        return self.get_data(sorted_seq_idx, "data")
 
-    def get_targets(self, target, sorted_seq_idx):
+    def get_targets(self, target: str, sorted_seq_idx: int) -> numpy.ndarray:
         """
-        :param str target: data key
-        :type sorted_seq_idx: int
-        :rtype: numpy.ndarray
+        DEPRECATED: Some older classes still use this deprecated API,
+        but any new dataset should just implement :func:`get_data`,
+        and users also should just use :func:`get_data`.
+
+        :param target: data key
+        :param sorted_seq_idx:
         :returns targets: format 1d (time) (int: idx of output-feature)
         """
         # For new-style subclasses, which just provide get_data.

{returnn-1.20240203.124331 → returnn-1.20240205.110004}/returnn/datasets/generating.py

@@ -58,17 +58,24 @@ class GeneratingDataset(Dataset):
 
     def init_seq_order(self, epoch=None, seq_list=None, seq_order=None):
         """
-        :type epoch: int|None
-        :param list[str]|None seq_list: predefined order via tags, doesn't make sense here
-        :param list[int]|None seq_order: predefined order via indices, doesn't make sense here
+        :param int|None epoch:
+        :param list[str]|None seq_list: predefined order via tags
+        :param list[int]|None seq_order: predefined order via indices
+
         This is called when we start a new epoch, or at initialization.
         """
         super(GeneratingDataset, self).init_seq_order(epoch=epoch)
-        assert seq_list is None and seq_order is None, (
-            "predefined order doesn't make sense for %s" % self.__class__.__name__
-        )
         self.random.seed(self._get_random_seed_for_epoch(epoch=epoch))
-        if self._total_num_seqs == float("inf"):
+        if seq_list is not None:
+            assert seq_order is None
+            seq_order = []
+            for tag in seq_list:
+                assert tag.startswith("seq-") and tag[len("seq-") :].isdigit()
+                seq_order.append(int(tag[len("seq-") :]))
+        if seq_order is not None:
+            self._seq_order = seq_order
+            self._num_seqs = len(self._seq_order)
+        elif self._total_num_seqs == float("inf"):
             assert self.seq_ordering == "default"
             self._seq_order = None
             self._num_seqs = self._total_num_seqs
@@ -144,7 +151,7 @@ class GeneratingDataset(Dataset):
 
     def _make_seq(self, seq_idx: int) -> DatasetSeq:
         seq = self.get_corpus_seq(self.get_corpus_seq_idx(seq_idx))
-        seq.seq_idx = seq_idx
+        seq.seq_idx = seq_idx  # we expect current seq order seq idx, not corpus seq idx
         return seq
 
     def have_get_corpus_seq(self) -> bool:

{returnn-1.20240203.124331 → returnn-1.20240205.110004}/returnn/datasets/meta.py

@@ -829,9 +829,8 @@ class ConcatDataset(CachedDataset2):
         dataset = self.datasets[dataset_idx]
         dataset_seq_idx = seq_idx + self.dataset_seq_idx_offsets[dataset_idx]
         seq_tag = dataset.get_tag(dataset_seq_idx)
-        features = dataset.get_input_data(dataset_seq_idx)
-        targets = {k: dataset.get_targets(k, dataset_seq_idx) for k in dataset.get_target_list()}
-        return DatasetSeq(seq_idx=seq_idx, seq_tag=seq_tag, features=features, targets=targets)
+        features = {k: dataset.get_data(dataset_seq_idx, k) for k in dataset.get_data_keys()}
+        return DatasetSeq(seq_idx=seq_idx, seq_tag=seq_tag, features=features)
 
     @property
     def num_seqs(self):

{returnn-1.20240203.124331 → returnn-1.20240205.110004}/returnn/torch/frontend/bridge.py

@@ -82,9 +82,9 @@ class PTModuleAsRFModule(rf.Module):
             )
             setattr(self, name, rf_param)
 
-        for name, rf_mod in pt_module.named_children():
-            pt_mod = rf_module_to_pt_module(rf_mod)
-            setattr(self, name, pt_mod)
+        for name, pt_mod in pt_module.named_children():
+            rf_mod = pt_module_to_rf_module(pt_mod)
+            setattr(self, name, rf_mod)
 
     @property
     def pt_module(self) -> rf.Module:

{returnn-1.20240203.124331 → returnn-1.20240205.110004/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240203.124331
+Version: 1.20240205.110004
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20240203.124331 → returnn-1.20240205.110004}/tests/DummySprintExec.py

@@ -91,9 +91,9 @@ def main(argv):
             kwargs = {"features": features}
             if target_mode == "target-generic":
                 if "orth" in dataset.get_target_list():
-                    kwargs["orthography"] = dataset.get_targets("orth", seq_idx)
+                    kwargs["orthography"] = dataset.get_data(seq_idx, "orth")
                 if "classes" in dataset.get_target_list():
-                    kwargs["alignment"] = dataset.get_targets("classes", seq_idx)
+                    kwargs["alignment"] = dataset.get_data(seq_idx, "classes")
                 print("DummySprintExec seq_idx %i feedInputAndTarget(**%r)" % (seq_idx, kwargs))
                 sprint_api.feedInputAndTarget(**kwargs)
             else:

{returnn-1.20240203.124331 → returnn-1.20240205.110004}/tools/collect-orth-symbols.py

@@ -13,6 +13,7 @@ import _setup_returnn_env  # noqa
 import returnn.__main__ as rnn
 from returnn.log import log
 from returnn.config import Config
+from returnn.datasets import Dataset
 import argparse
 from returnn.util.basic import hms, human_size, parse_orthography, parse_orthography_into_symbols, unicode
 import gzip
@@ -34,10 +35,11 @@ def found_sub_seq(sub_seq, seq):
     return False
 
 
-def iter_dataset(dataset, options, callback):
+def iter_dataset(dataset: Dataset, options, callback):
     """
-    :type dataset: Dataset.Dataset
+    :param dataset:
     """
+    options  # unused  # noqa
     dataset.init_seq_order(epoch=1)
     assert "orth" in dataset.get_target_list()
 
@@ -46,7 +48,7 @@ def iter_dataset(dataset, options, callback):
         dataset.load_seqs(seq_idx, seq_idx)
 
         frame_len = dataset.get_seq_length(seq_idx)["data"]
-        orth = dataset.get_targets("orth", seq_idx)
+        orth = dataset.get_data(seq_idx, "orth")
         callback(frame_len=frame_len, orth=orth)
 
         seq_idx += 1

{returnn-1.20240203.124331 → returnn-1.20240205.110004}/tools/collect-words.py

@@ -14,15 +14,16 @@ from returnn.log import log
 from returnn.config import Config
 import argparse
 from returnn.util.basic import human_size, parse_orthography
+from returnn.datasets import Dataset
 from returnn.datasets.lm import Lexicon
 import gzip
 from xml.etree import ElementTree
 import time
 
 
-def iter_dataset(dataset, callback):
+def iter_dataset(dataset: Dataset, callback):
     """
-    :param Dataset.Dataset dataset:
+    :param dataset:
     :param (*)->None callback:
     """
     dataset.init_seq_order(epoch=1)
@@ -32,7 +33,7 @@ def iter_dataset(dataset, callback):
     while dataset.is_less_than_num_seqs(seq_idx):
         dataset.load_seqs(seq_idx, seq_idx)
 
-        orth = dataset.get_targets("orth", seq_idx)
+        orth = dataset.get_data(seq_idx, "orth")
         callback(orth=orth)
 
         seq_idx += 1

{returnn-1.20240203.124331 → returnn-1.20240205.110004}/tools/dump-dataset.py

@@ -66,7 +66,7 @@ def dump_dataset(options):
                 dataset.load_seqs(seq_idx, seq_idx + 1)
                 if seq_idx % 10000 == 0:
                     if default_target:
-                        targets = dataset.get_targets(default_target, seq_idx)
+                        targets = dataset.get_data(seq_idx, default_target)
                         postfix = " (targets = %r...)" % (targets[:10],)
                     else:
                         postfix = ""
@@ -161,7 +161,7 @@ def dump_dataset(options):
             elif options.type == "plot":
                 plot(data)
             for target in dataset.get_target_list():
-                targets = dataset.get_targets(target, seq_idx)
+                targets = dataset.get_data(seq_idx, target)
                 if options.type == "numpy":
                     numpy.savetxt(
                         "%s%i.targets.%s%s" % (options.dump_prefix, seq_idx, target, options.dump_postfix),

{returnn-1.20240203.124331 → returnn-1.20240205.110004}/tools/extract_state_tying_from_dataset.py

@@ -16,7 +16,7 @@ from collections import defaultdict
 import typing
 
 import _setup_returnn_env  # noqa
-from returnn.datasets import init_dataset
+from returnn.datasets import init_dataset, Dataset
 from returnn.datasets.lm import Lexicon, AllophoneState
 from returnn.log import log
 from returnn.util.basic import uniq
@@ -71,16 +71,16 @@ def iter_bliss_orth(filename):
         yield get_segment_name(tree + [elem]), orth
 
 
-def iter_dataset_targets(dataset):
+def iter_dataset_targets(dataset: Dataset):
     """
-    :type dataset: Dataset.Dataset
+    :param dataset:
     """
     dataset.init_seq_order(epoch=1)
     seq_idx = 0
     while dataset.is_less_than_num_seqs(seq_idx):
         dataset.load_seqs(seq_idx, seq_idx + 1)
         segment_name = dataset.get_tag(seq_idx)
-        targets = dataset.get_targets("classes", seq_idx)
+        targets = dataset.get_data(seq_idx, "classes")
         assert targets.ndim == 1  # sparse
         targets = targets.astype("int32")
         yield segment_name, targets

returnn-1.20240203.124331/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240203.124331'
-long_version = '1.20240203.124331+git.db8a51b'