returnn 1.20240626.125143__tar.gz → 1.20240626.175748__tar.gz

{returnn-1.20240626.125143 → returnn-1.20240626.175748}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240626.125143
+Version: 1.20240626.175748
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240626.175748/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240626.175748'
+long_version = '1.20240626.175748+git.80f52ec'

{returnn-1.20240626.125143 → returnn-1.20240626.175748}/returnn/datasets/distrib_files.py

@@ -159,6 +159,7 @@ class DistributeFilesDataset(CachedDataset2):
         self._file_sizes: Optional[Dict[str, int]] = None  # key -> size. for equal distribution across sub epochs
         self._data_keys: Optional[List[str]] = None
         self._num_seqs: Optional[int] = None
+        self.distrib_shard_files = distrib_shard_files
         self._shard_index, self._num_shards = _get_rank_and_size() if distrib_shard_files else (0, 1)
 
         self._workers: Dict[int, _WorkerProcParent] = {}  # epoch -> worker

{returnn-1.20240626.125143 → returnn-1.20240626.175748}/returnn/frontend/encoder/conformer.py

@@ -33,6 +33,8 @@ class ConformerPositionwiseFeedForward(rf.Module):
         self.out_dim = out_dim
         self.dropout = dropout
         self.dropout_broadcast = rf.dropout_broadcast_default()
+        if not callable(activation):
+            raise TypeError(f"{self}: unexpected activation type {activation!r}")
         self.activation = activation
 
         self.linear_ff = rf.Linear(out_dim, ff_dim)
@@ -67,6 +69,8 @@ class ConformerConvBlock(rf.Module):
             out_dim, out_dim, filter_size=kernel_size, groups=out_dim.dimension, padding="same"
         )
         self.positionwise_conv2 = rf.Linear(out_dim, out_dim)
+        if not callable(norm):
+            raise TypeError(f"{self}: unexpected norm type {norm!r}")
         self.norm = norm
 
     def __call__(self, inp: Tensor, *, spatial_dim: Dim) -> Tensor:
@@ -179,10 +183,10 @@ class ConformerEncoderLayer(rf.Module):
         ff_activation: Callable[[Tensor], Tensor] = rf.swish,
         dropout: float = 0.1,
         conv_kernel_size: int = 32,
-        conv_norm: Union[rf.BatchNorm, type, Any] = NotSpecified,
+        conv_norm: Union[rf.BatchNorm, type, Dict[str, Any], Any] = NotSpecified,
         conv_norm_opts: Optional[Dict[str, Any]] = None,
         num_heads: int = 4,
-        self_att: Optional[Union[rf.RelPosSelfAttention, rf.Module, type, Any]] = None,
+        self_att: Optional[Union[rf.RelPosSelfAttention, rf.Module, type, Dict[str, Any], Any]] = None,
         self_att_opts: Optional[Dict[str, Any]] = None,
         att_dropout: float = 0.1,
     ):
@@ -234,7 +238,7 @@ class ConformerEncoderLayer(rf.Module):
         self.conv_block = ConformerConvBlock(out_dim=out_dim, kernel_size=conv_kernel_size, norm=conv_norm)
         self.conv_layer_norm = rf.LayerNorm(out_dim)
 
-        if self_att is None or isinstance(self_att, type):
+        if self_att is None or isinstance(self_att, (dict, type)):
             self_att_opts_ = dict(
                 in_dim=out_dim,
                 proj_dim=out_dim,
@@ -247,9 +251,16 @@ class ConformerEncoderLayer(rf.Module):
                 self_att_opts_.update(self_att_opts)
             if self_att is None:
                 self.self_att = rf.RelPosSelfAttention(**self_att_opts_)
-            else:
+            elif isinstance(self_att, type):
                 self.self_att = self_att(**self_att_opts_)
+            elif isinstance(self_att, dict):
+                self_att_opts_ = {k: v for (k, v) in self_att_opts_.items() if k not in self_att}
+                self.self_att = rf.build_from_dict(self_att, **self_att_opts_)
+            else:
+                raise TypeError(f"{self}: invalid type: self_att {self_att!r}")
         else:
+            if not callable(self_att):
+                raise TypeError(f"{self}: invalid non-callable: self_att {self_att!r}")
             self.self_att = self_att
         self.self_att_layer_norm = rf.LayerNorm(out_dim)
 
@@ -299,10 +310,10 @@ class ConformerEncoder(ISeqDownsamplingEncoder):
         ff_activation: Callable[[Tensor], Tensor] = rf.swish,
         dropout: float = 0.1,
         conv_kernel_size: int = 32,
-        conv_norm: Union[rf.BatchNorm, type, Any] = NotSpecified,
+        conv_norm: Union[rf.BatchNorm, type, Dict[str, Any], Any] = NotSpecified,
         num_heads: int = 4,
         att_dropout: float = 0.1,
-        encoder_layer: Optional[Union[ConformerEncoderLayer, rf.Module, type, Any]] = None,
+        encoder_layer: Optional[Union[ConformerEncoderLayer, rf.Module, type, Dict[str, Any], Any]] = None,
         encoder_layer_opts: Optional[Dict[str, Any]] = None,
         sequential=rf.Sequential,
     ):
@@ -337,7 +348,7 @@ class ConformerEncoder(ISeqDownsamplingEncoder):
         )
         self.input_dropout = input_dropout
 
-        if not encoder_layer or isinstance(encoder_layer, type):
+        if not encoder_layer or isinstance(encoder_layer, (dict, type)):
             encoder_layer_opts_ = dict(
                 out_dim=out_dim,
                 ff_dim=ff_dim,
@@ -354,8 +365,21 @@ class ConformerEncoder(ISeqDownsamplingEncoder):
                 encoder_layer = ConformerEncoderLayer(**encoder_layer_opts_)
             elif isinstance(encoder_layer, type):
                 encoder_layer = encoder_layer(**encoder_layer_opts_)
+            elif isinstance(encoder_layer, dict):
+                # Note: Reuse all the encoder_layer_opts_.
+                # If this does not make sense for the specific encoder_layer class here,
+                # we would suggest to use a different ConformerEncoder class.
+                # (The alternative, to not reuse encoder_layer_opts_ here,
+                #  would probably be more confusing, as those options are all ignored then.
+                #  It's also not clear what args to pass then and what not.)
+                # (Maybe we should do a ConformerEncoderV2 if this is confusing here...)
+                encoder_layer_opts_ = {k: v for (k, v) in encoder_layer_opts_.items() if k not in encoder_layer}
+                encoder_layer = rf.build_from_dict(encoder_layer, **encoder_layer_opts_)
             else:
                 raise TypeError(f"unexpected encoder_layer {encoder_layer!r}")
+        else:
+            if not callable(encoder_layer):
+                raise TypeError(f"{self}: invalid non-callable encoder_layer {encoder_layer!r}")
 
         self.layers = sequential(_copy.deepcopy(encoder_layer) for _ in range(num_layers))
 

{returnn-1.20240626.125143 → returnn-1.20240626.175748}/returnn/frontend/math_.py

@@ -58,6 +58,7 @@ __all__ = [
     "floor",
     "round",
     "relu",
+    "relu_square",
     "elu",
     "selu",
     "silu",
@@ -493,6 +494,13 @@ def relu(a: Tensor) -> Tensor:
     return a._raw_backend.activation(a, "relu")
 
 
+def relu_square(x: Tensor) -> Tensor:
+    """
+    :return: ``relu(x) ** 2``
+    """
+    return square(relu(x))
+
+
 def elu(a: Tensor) -> Tensor:
     """elu"""
     # noinspection PyProtectedMember

{returnn-1.20240626.125143 → returnn-1.20240626.175748}/returnn/util/basic.py

@@ -3979,6 +3979,22 @@ def close_all_fds_except(except_fds):
             os.closerange(except_fds[i] + 1, except_fds[i + 1])
 
 
+def is_valid_fd(fd: int) -> bool:
+    """
+    :return: whether the file descriptor (fd) is still open and valid
+    """
+    # https://stackoverflow.com/questions/12340695/how-to-check-if-a-given-file-descriptor-stored-in-a-variable-is-still-valid
+    import fcntl
+    import errno
+
+    try:
+        fcntl.fcntl(fd, fcntl.F_GETFD)
+        return True  # no matter what fcntl returned, it's a valid fd
+    except OSError as exc:
+        assert exc.errno == errno.EBADF
+        return False
+
+
 class Stats:
     """
     Collects mean and variance, running average.

{returnn-1.20240626.125143 → returnn-1.20240626.175748}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240626.125143
+Version: 1.20240626.175748
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240626.125143/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240626.125143'
-long_version = '1.20240626.125143+git.855b4cd'