returnn 1.20231108.124950__tar.gz → 1.20231109.122538__tar.gz

{returnn-1.20231108.124950 → returnn-1.20231109.122538}/LICENSE

@@ -159,4 +159,10 @@ Contains code from CPython:
  - Copyright 2001..2021 Python Software Foundation
  - py-to-pickle.cpp
 
+Contains code from PyTorch:
+ - https://github.com/pytorch/pytorch/
+ - BSD 3-Clause License
+ - Copyright 2016..2023 various developers
+ - Various code snippets
+
 Various code snippets from StackOverflow, which are under Creative Commons / Public Domain.

{returnn-1.20231108.124950/returnn.egg-info → returnn-1.20231109.122538}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231108.124950
+Version: 1.20231109.122538
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20231109.122538/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20231109.122538'
+long_version = '1.20231109.122538+git.e5a5d90'

{returnn-1.20231108.124950 → returnn-1.20231109.122538}/returnn/datasets/sprint.py

@@ -23,8 +23,9 @@ import returnn.util.task_system as task_system
 from returnn.datasets.basic import Dataset, DatasetSeq
 from .cached2 import CachedDataset2
 from returnn.log import log
-from returnn.util.task_system import Unpickler, numpy_copy_and_set_unused
-from returnn.util.basic import eval_shell_str, interrupt_main, unicode, PY3, BytesIO, close_all_fds_except
+from returnn.util.task_system import numpy_copy_and_set_unused
+from returnn.util.basic import eval_shell_str, interrupt_main, unicode, PY3, close_all_fds_except
+import returnn.util.basic as util
 
 
 class SprintDatasetBase(Dataset):
@@ -913,31 +914,9 @@ class ExternSprintDataset(SprintDatasetBase):
         :return: (data_type, args)
         :rtype: (str, object)
         """
-        import struct
-
-        size_raw = self.pipe_c2p[0].read(4)
-        if len(size_raw) < 4:
-            raise EOFError
-        (size,) = struct.unpack("<i", size_raw)
-        assert size > 0, "%s: We expect to get some non-empty package. Invalid Python mod in Sprint?" % (self,)
-        stream = BytesIO()
-        read_size = 0
-        while read_size < size:
-            data_raw = self.pipe_c2p[0].read(size - read_size)
-            if len(data_raw) == 0:
-                raise EOFError("%s: expected to read %i bytes but got EOF after %i bytes" % (self, size, read_size))
-            read_size += len(data_raw)
-            stream.write(data_raw)
-        stream.seek(0)
-        try:
-            if PY3:
-                # encoding is for converting Python2 strings to Python3.
-                # Cannot use utf8 because Numpy will also encode the data as strings and there we need it as bytes.
-                data_type, args = Unpickler(stream, encoding="bytes").load()
-            else:
-                data_type, args = Unpickler(stream).load()
-        except EOFError:
-            raise Exception("%s: parse error of %i bytes (%r)" % (self, size, stream.getvalue()))
+        # encoding is for converting Python2 strings to Python3.
+        # Cannot use utf8 because Numpy will also encode the data as strings and there we need it as bytes.
+        data_type, args = util.read_pickled_object(self.pipe_c2p[0], encoding="bytes")
         return data_type, args
 
     def _join_child(self, wait=True, expected_exit_status=None):

{returnn-1.20231108.124950 → returnn-1.20231109.122538}/returnn/frontend/container.py

@@ -3,6 +3,7 @@ container functions
 """
 
 from __future__ import annotations
+import operator
 import returnn.frontend as rf
 from returnn.tensor import Tensor
 from typing import Optional, TypeVar, Generic, Iterable, Iterator, Union, Tuple, Dict, Callable
@@ -21,12 +22,9 @@ class ModuleList(rf.Module, Generic[__ModT]):
     Module list, getting passed an Iterable of Modules and creates a list of Modules in that order
     """
 
-    def __init__(self, *modules: Union[__ModT, Iterable[__ModT], Dict[str, __ModT], ModuleList]):
+    def __init__(self, *modules: Union[__ModT, Iterable[__ModT], ModuleList]):
         super().__init__()
-        if len(modules) == 1 and isinstance(modules[0], dict):
-            for key, module in modules[0].items():
-                setattr(self, key, _convert_to_module(module))
-        elif len(modules) == 1 and isinstance(modules[0], ModuleList):
+        if len(modules) == 1 and isinstance(modules[0], ModuleList):
             for key, module in modules[0]._get_modules().items():
                 setattr(self, key, _convert_to_module(module))
         elif len(modules) == 1 and _is_iterable(modules[0]):
@@ -37,7 +35,29 @@ class ModuleList(rf.Module, Generic[__ModT]):
                 setattr(self, str(idx), _convert_to_module(module))
 
     def _get_modules(self) -> Dict[str, __ModT]:
-        return {key: value for (key, value) in vars(self).items() if isinstance(value, rf.Module)}
+        # Note: Insertion order is relevant here. We use it in __getitem__ for slicing, etc.
+        res = {}
+        i = 0
+        while True:
+            try:
+                res[str(i)] = getattr(self, str(i))
+                i += 1
+            except AttributeError:
+                break
+        return res
+
+    def _get_abs_index(self, idx: int) -> int:
+        """Get the absolute index for the list of modules"""
+        idx = operator.index(idx)
+        if not (-len(self) <= idx < len(self)):
+            raise IndexError("index {} is out of range".format(idx))
+        if idx < 0:
+            idx += len(self)
+        return idx
+
+    def _get_abs_string_index(self, idx: int) -> str:
+        """Get the absolute index for the list of modules"""
+        return str(self._get_abs_index(idx))
 
     def append(self, module: __ModT) -> ModuleList[__ModT]:
         """
@@ -68,14 +88,33 @@ class ModuleList(rf.Module, Generic[__ModT]):
         from builtins import slice
 
         if isinstance(idx, slice):
-            return self.__class__(dict(list(self._get_modules().items())[idx]))
+            return self.__class__(list(self._get_modules().values())[idx])
         else:
-            return list(self._get_modules().values())[idx]
+            key = self._get_abs_string_index(idx)
+            if not hasattr(self, key):
+                raise IndexError("index {} is out of range".format(idx))
+            return getattr(self, key)
 
     def __setitem__(self, idx: int, module: __ModT) -> None:
-        key = list(self._get_modules().keys())[idx]
+        key = self._get_abs_string_index(idx)
+        if not hasattr(self, key):
+            raise IndexError("index {} is out of range".format(idx))
         return setattr(self, key, _convert_to_module(module))
 
+    def __delitem__(self, idx: Union[int, slice]) -> None:
+        # To preserve numbering, we reconstruct the list of modules after deletion.
+        modules = list(self._get_modules().values())
+        old_len = len(modules)
+        del modules[idx]
+        if isinstance(idx, slice):
+            min_idx = self._get_abs_index(idx.start)
+        else:
+            min_idx = self._get_abs_index(idx)
+        for i in range(min_idx, len(modules)):
+            setattr(self, str(i), modules[i])
+        for i in range(len(modules), old_len):
+            delattr(self, str(i))
+
     __call__ = rf.Module.__call__  # stays abstract
 
 
@@ -108,7 +147,7 @@ def _convert_to_module(obj: _ModT) -> rf.Module:
     elif callable(obj):
         return rf.Functional(obj)
     else:
-        raise TypeError(f"did not expect {obj!r}")
+        raise TypeError(f"Expected rf.Module or callable, did not expect {obj!r} ({type(obj)})")
 
 
 def _is_iterable(obj) -> bool:

{returnn-1.20231108.124950 → returnn-1.20231109.122538}/returnn/sprint/control.py

@@ -15,15 +15,14 @@ from __future__ import annotations
 import os
 import numpy
 import typing
-import io
-import struct
 from threading import Condition
 
 import returnn.__main__ as rnn
 import returnn.util.task_system as task_system
 import returnn.util.debug as debug
-from returnn.util.task_system import Pickler, Unpickler, numpy_set_unused
+from returnn.util.task_system import numpy_set_unused
 from returnn.util.basic import to_bool, long
+import returnn.util.basic as util
 
 InitTypes = set()
 Verbose = False  # disables all per-segment log messages
@@ -434,31 +433,10 @@ class PythonControl:
         return loss, error_signal
 
     def _send(self, data):
-        stream = io.BytesIO()
-        Pickler(stream).dump(data)
-        raw_data = stream.getvalue()
-        assert len(raw_data) > 0
-        self.pipe_c2p.write(struct.pack("<i", len(raw_data)))
-        self.pipe_c2p.write(raw_data)
-        self.pipe_c2p.flush()
+        util.write_pickled_object(self.pipe_c2p, data)
 
     def _read(self):
-        p = self.pipe_p2c
-        size_raw = p.read(4)
-        if len(size_raw) < 4:
-            raise EOFError
-        (size,) = struct.unpack("<i", size_raw)
-        assert size > 0, "%s: We expect to get some non-empty package. Invalid Python mod in Sprint?" % (self,)
-        stream = io.BytesIO()
-        read_size = 0
-        while read_size < size:
-            data_raw = p.read(size - read_size)
-            if len(data_raw) == 0:
-                raise EOFError("%s: expected to read %i bytes but got EOF after %i bytes" % (self, size, read_size))
-            read_size += len(data_raw)
-            stream.write(data_raw)
-        stream.seek(0)
-        return Unpickler(stream).load()
+        return util.read_pickled_object(self.pipe_p2c)
 
     def close(self):
         """

{returnn-1.20231108.124950 → returnn-1.20231109.122538}/returnn/sprint/error_signals.py

@@ -14,12 +14,11 @@ import os
 import atexit
 import signal
 import typing
-import struct
-import io
 from threading import RLock, Thread
 import returnn.util.task_system as task_system
-from returnn.util.task_system import Pickler, Unpickler, numpy_set_unused
+from returnn.util.task_system import numpy_set_unused
 from returnn.util.basic import eval_shell_str, make_hashable, close_all_fds_except
+import returnn.util.basic as util
 from returnn.log import log
 
 
@@ -219,32 +218,12 @@ class SprintSubprocessInstance:
     def _send(self, v):
         assert os.getpid() == self.parent_pid
         p = self.pipe_p2c[1]  # see _start_child
-        stream = io.BytesIO()
-        Pickler(stream).dump(v)
-        raw_data = stream.getvalue()
-        assert len(raw_data) > 0
-        p.write(struct.pack("<i", len(raw_data)))
-        p.write(raw_data)
-        p.flush()
+        util.write_pickled_object(p, v)
 
     def _read(self):
         assert os.getpid() == self.parent_pid
         p = self.pipe_c2p[0]  # see _start_child
-        size_raw = p.read(4)
-        if len(size_raw) < 4:
-            raise EOFError
-        (size,) = struct.unpack("<i", size_raw)
-        assert size > 0, "%s: We expect to get some non-empty package. Invalid Python mod in Sprint?" % (self,)
-        stream = io.BytesIO()
-        read_size = 0
-        while read_size < size:
-            data_raw = p.read(size - read_size)
-            if len(data_raw) == 0:
-                raise EOFError("%s: expected to read %i bytes but got EOF after %i bytes" % (self, size, read_size))
-            read_size += len(data_raw)
-            stream.write(data_raw)
-        stream.seek(0)
-        return Unpickler(stream).load()
+        return util.read_pickled_object(p)
 
     def _poll(self):
         assert os.getpid() == self.parent_pid

{returnn-1.20231108.124950 → returnn-1.20231109.122538}/returnn/sprint/extern_interface.py

@@ -11,8 +11,8 @@ import os
 import typing
 from returnn.util import better_exchook
 import returnn.util.task_system as task_system
-from returnn.util.task_system import Pickler
-from returnn.util.basic import to_bool, unicode, BytesIO
+from returnn.util.basic import to_bool, unicode
+import returnn.util.basic as util
 
 # Start Sprint PythonSegmentOrder interface. {
 # We use the PythonSegmentOrder just to get an estimate (upper limit) about the number of sequences.
@@ -347,15 +347,7 @@ class ExternSprintDatasetSource:
         :param object args:
         """
         assert data_type is not None
-        import struct
-
-        stream = BytesIO()
-        Pickler(stream).dump((data_type, args))
-        raw_data = stream.getvalue()
-        assert len(raw_data) > 0
-        self.pipe_c2p.write(struct.pack("<i", len(raw_data)))
-        self.pipe_c2p.write(raw_data)
-        self.pipe_c2p.flush()
+        util.write_pickled_object(self.pipe_c2p, (data_type, args))
 
     def add_new_data(self, segment_name, features, targets):
         """

{returnn-1.20231108.124950 → returnn-1.20231109.122538}/returnn/util/basic.py

@@ -5,7 +5,7 @@ Various generic utilities, which are shared across different backend engines.
 """
 
 from __future__ import annotations
-from typing import Optional, Generic, TypeVar, Iterable, Tuple, Dict, List, Callable
+from typing import Optional, Any, Generic, TypeVar, Iterable, Tuple, Dict, List, Callable
 
 import subprocess
 from subprocess import CalledProcessError
@@ -22,6 +22,7 @@ import numpy as np
 import re
 import time
 import contextlib
+import struct
 
 try:
     import thread
@@ -2597,6 +2598,59 @@ def deepcopy(x, stop_types=None):
     return c
 
 
+def read_bytes_to_new_buffer(p: typing.BinaryIO, size: int) -> BytesIO:
+    """
+    Read bytes from stream s into a BytesIO buffer.
+    Raises EOFError if not enough bytes are available.
+    Then read it via :func:`read_pickled_object`.
+    """
+    stream = BytesIO()
+    read_size = 0
+    while read_size < size:
+        data_raw = p.read(size - read_size)
+        if len(data_raw) == 0:
+            raise EOFError("expected to read %i bytes but got EOF after %i bytes" % (size, read_size))
+        read_size += len(data_raw)
+        stream.write(data_raw)
+    stream.seek(0)
+    return stream
+
+
+def read_pickled_object(p: typing.BinaryIO, *, encoding=None) -> Any:
+    """
+    Read pickled object from stream p,
+    after it was written via :func:`read_bytes_to_new_buffer`.
+
+    :param p:
+    :param encoding: if given, passed to Unpickler
+    """
+    from returnn.util.task_system import Unpickler
+
+    size_raw = read_bytes_to_new_buffer(p, 4).getvalue()
+    (size,) = struct.unpack("<i", size_raw)
+    assert size > 0, "read_pickled_object: We expect to get some non-empty package."
+    stream = read_bytes_to_new_buffer(p, size)
+    unpickler_kwargs = {}
+    if encoding:
+        unpickler_kwargs["encoding"] = encoding
+    return Unpickler(stream, **unpickler_kwargs).load()
+
+
+def write_pickled_object(p: typing.BinaryIO, obj: Any):
+    """
+    Writes pickled object to stream p.
+    """
+    from returnn.util.task_system import Pickler
+
+    stream = BytesIO()
+    Pickler(stream).dump(obj)
+    raw_data = stream.getvalue()
+    assert len(raw_data) > 0
+    p.write(struct.pack("<i", len(raw_data)))
+    p.write(raw_data)
+    p.flush()
+
+
 def load_txt_vector(filename):
     """
     Expect line-based text encoding in file.

{returnn-1.20231108.124950 → returnn-1.20231109.122538/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231108.124950
+Version: 1.20231109.122538
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20231108.124950 → returnn-1.20231109.122538}/tests/test_rf_container.py

@@ -76,41 +76,6 @@ def test_sequential_base_case():
     run_model(extern_data, lambda *, epoch, step: _Net(), _forward_step)
 
 
-def test_sequential_named_case():
-    time_dim = Dim(Tensor("time", [batch_dim], dtype="int32"))
-    in_dim = Dim(7, name="in")
-    extern_data = TensorDict(
-        {
-            "data": Tensor("data", [batch_dim, time_dim, in_dim], dtype="float32"),
-        }
-    )
-
-    class _Net(rf.Module):
-        def __init__(self):
-            super().__init__()
-            dims = [Dim(1, name="feat1"), Dim(2, name="feat2"), Dim(3, name="feat3")]
-            self.out_dim = dims[-1]
-            x = OrderedDict()
-            x["one"] = rf.Linear(in_dim, dims[0])
-            x["two"] = rf.Linear(dims[0], dims[1])
-            x["three"] = rf.Linear(dims[1], dims[2])
-            self.seq = rf.Sequential(x)
-
-        def __call__(self, data: Tensor) -> Tensor:
-            """
-            Forward
-            """
-            seq = self.seq(data)
-            return seq
-
-    # noinspection PyShadowingNames
-    def _forward_step(*, model: _Net, extern_data: TensorDict):
-        out = model(extern_data["data"])
-        out.mark_as_default_output(shape=(batch_dim, time_dim, model.out_dim))
-
-    run_model(extern_data, lambda *, epoch, step: _Net(), _forward_step)
-
-
 def test_parameter_list():
     time_dim = Dim(Tensor("time", [batch_dim], dtype="int32"))
     in_dim = Dim(7, name="in")

returnn-1.20231108.124950/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20231108.124950'
-long_version = '1.20231108.124950+git.a3d1094'