returnn 1.20250113.193158__tar.gz → 1.20250115.110555__tar.gz

{returnn-1.20250113.193158/returnn.egg-info → returnn-1.20250115.110555}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250113.193158
+Version: 1.20250115.110555
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250115.110555/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20250115.110555'
+long_version = '1.20250115.110555+git.641f627'

{returnn-1.20250113.193158 → returnn-1.20250115.110555}/returnn/datasets/basic.py

@@ -1084,20 +1084,21 @@ class Dataset:
         # in advance can handle this somehow.
         return n < self.num_seqs
 
-    def can_serialize_data(self, key):
+    def can_serialize_data(self, key: str) -> bool:
         """
-        :param str key: e.g. "classes"
-        :rtype: bool
+        :param key: e.g. "classes"
+        :return: whether :func:`serialize_data` is implemented for this key
         """
-        return key in self.labels
+        labels = self.labels.get(key)
+        return labels and len(labels) > 1
 
-    def serialize_data(self, key, data):
+    def serialize_data(self, key: str, data: numpy.ndarray) -> str:
         """
         In case you have a :class:`Vocabulary`, just use :func:`Vocabulary.get_seq_labels`.
 
-        :param str key: e.g. "classes". self.labels[key] should be set
+        :param key: e.g. "classes". self.labels[key] should be set
         :param numpy.ndarray data: 0D or 1D
-        :rtype: str
+        :return: serialized data
         """
         vocab = Vocabulary.create_vocab_from_labels(self.labels[key])
         if data.ndim == 0:

{returnn-1.20250113.193158 → returnn-1.20250115.110555}/returnn/datasets/postprocessing.py

@@ -138,11 +138,13 @@ class PostprocessingDataset(CachedDataset2):
         self._in_tensor_dict_template = TensorDict(
             {name: self._make_tensor_template_from_input(name) for name in self._dataset.get_data_keys()}
         )
+        self.labels = {}
         if self._map_outputs is not None:
             self._out_tensor_dict_template = TensorDict()
             self._out_tensor_dict_template.update(self._map_outputs, auto_convert=True)
         else:
             self._out_tensor_dict_template = self._in_tensor_dict_template.copy_template()
+            self.labels = self._dataset.labels.copy()
         # update only after _out_tensor_dict_template has been created from _in_tensor_dict_template
         self._in_tensor_dict_template.update({"seq_tag": {"dims": (), "dtype": "string"}}, auto_convert=True)
         self.num_outputs = {
@@ -152,8 +154,9 @@ class PostprocessingDataset(CachedDataset2):
         self._default_input = "data" if "data" in self.num_outputs else next(iter(self.num_outputs.keys()))
         self.num_inputs = self.num_outputs[self._default_input][0]
 
-        self.labels = {}
         for k, t in self._out_tensor_dict_template.data.items():
+            if self.labels.get(k):
+                continue
             if t.vocab:
                 self.labels[k] = t.vocab.labels
             elif t.sparse_dim:  # sparse_dim but not vocab
@@ -307,7 +310,8 @@ class PostprocessingDataset(CachedDataset2):
         sparse_dim = None
         if self._dataset.is_data_sparse(data_key):
             sparse_dim = Dim(dimension=self._dataset.get_data_dim(data_key), name=f"{data_key}_sparse")
-            if data_key in self._dataset.labels:
+            labels = self._dataset.labels.get(data_key)
+            if labels and len(labels) == sparse_dim.dimension:
                 sparse_dim.vocab = Vocabulary.create_vocab_from_labels(self._dataset.labels[data_key])
         return Tensor(data_key, dims=dims, dtype=dtype, sparse_dim=sparse_dim)
 

{returnn-1.20250113.193158 → returnn-1.20250115.110555}/returnn/datasets/util/vocabulary.py

@@ -180,7 +180,9 @@ class Vocabulary:
         kwargs.setdefault("unknown_label", None)
         if len(labels) < 1000 and all([len(label) == 1 for label in labels]):
             # are these actually ordered raw bytes? -> assume utf8
-            if all([ord(label) <= 255 and ord(label) == idx for idx, label in enumerate(labels)]):
+            if len(labels) == 256 and all(
+                [ord(label) <= 255 and ord(label) == idx for idx, label in enumerate(labels)]
+            ):
                 return Utf8ByteTargets()
             return CharacterTargets(vocab_file=None, labels=labels, **kwargs)
         return Vocabulary(vocab_file=None, labels=labels, **kwargs)

{returnn-1.20250113.193158 → returnn-1.20250115.110555/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250113.193158
+Version: 1.20250115.110555
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250113.193158 → returnn-1.20250115.110555}/tests/test_GeneratingDataset.py

@@ -1,17 +1,14 @@
-# -*- coding: utf8 -*-
+"""
+Tests for returnn.datasets.generating.
+"""
 
 from __future__ import annotations
 
 import _setup_test_env  # noqa
-import unittest
-from returnn.datasets.generating import *
-from returnn.datasets.basic import DatasetSeq
-from returnn.util.basic import PY3, unicode
 import os
 import unittest
-
+from returnn.datasets.generating import *
 from returnn.util import better_exchook
-from returnn.log import log
 
 
 my_dir = os.path.dirname(os.path.realpath(__file__))
@@ -86,19 +83,15 @@ def test_StaticDataset_utf8():
     s = "wër"
     print("some unicode str:", s, "repr:", repr(s), "type:", type(s), "len:", len(s))
     assert len(s) == 3
-    if PY3:
-        assert isinstance(s, str)
-        s_byte_list = list(s.encode("utf8"))
-    else:
-        assert isinstance(s, unicode)
-        s_byte_list = list(map(ord, s.encode("utf8")))
+    assert isinstance(s, str)
+    s_byte_list = list(s.encode("utf8"))
     print("utf8 byte list:", s_byte_list)
     assert len(s_byte_list) == 4 > 3
     raw = numpy.array(s_byte_list, dtype="uint8")
     assert raw.tolist() == [119, 195, 171, 114]
     data = StaticDataset([{"data": raw}], output_dim={"data": (255, 1)})
     if "data" not in data.labels:
-        data.labels["data"] = [chr(i) for i in range(255)]  # like in SprintDataset
+        data.labels["data"] = [chr(i) for i in range(256)]  # like in SprintDataset
     data.init_seq_order(epoch=1)
     data.load_seqs(0, 1)
     raw_ = data.get_data(seq_idx=0, key="data")

returnn-1.20250113.193158/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20250113.193158'
-long_version = '1.20250113.193158+git.777b035'