returnn 1.20231211.111626__tar.gz → 1.20231212.184511__tar.gz

{returnn-1.20231211.111626/returnn.egg-info → returnn-1.20231212.184511}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231211.111626
+Version: 1.20231212.184511
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20231212.184511/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20231212.184511'
+long_version = '1.20231212.184511+git.01b9e12'

{returnn-1.20231211.111626 → returnn-1.20231212.184511}/returnn/frontend/_backend.py

@@ -931,6 +931,17 @@ class Backend(Generic[T]):
         """where"""
         raise NotImplementedError
 
+    @staticmethod
+    def clip_by_value(
+        x: Tensor,
+        clip_value_min: Union[Tensor, rf.RawTensorTypes],
+        clip_value_max: Union[Tensor, rf.RawTensorTypes],
+        *,
+        allow_broadcast_all_sources: bool = False,
+    ) -> Tensor:
+        """clip by value"""
+        raise NotImplementedError
+
     @staticmethod
     def matmul(a: Tensor[T], b: Tensor[T], *, reduce: Union[Dim, Sequence[Dim]], use_mask: bool = True) -> Tensor[T]:
         """

{returnn-1.20231211.111626 → returnn-1.20231212.184511}/returnn/frontend/array_.py

@@ -23,6 +23,7 @@ __all__ = [
     "split",
     "expand_dim",
     "squeeze",
+    "window",
     "concat",
     "concat_features",
     "pad",
@@ -261,6 +262,88 @@ def squeeze(source: Tensor, axis: Dim) -> Tensor:
     return source._raw_backend.squeeze(source, axis=axis)
 
 
+def window(
+    source: Tensor,
+    *,
+    spatial_dim: Dim,
+    window_dim: Dim,
+    window_right: Optional[Union[Dim, int]] = None,
+    window_left: Optional[Union[Dim, int]] = None,
+    padding: str = "same",
+    pad_value: Optional[Union[int, float]] = None,
+    stride: int = 1,
+) -> Tuple[Tensor, Dim]:
+    """
+    Follows the same idea as RETURNN tf_util.windowed,
+    using clever padding and reshaping.
+
+    :param source:
+    :param spatial_dim:
+    :param window_dim:
+    :param window_left:
+    :param window_right:
+    :param padding: "same" or "valid"
+    :param pad_value:
+    :param stride:
+    :return: out, out_spatial_dim
+    """
+    assert window_dim.dimension is not None
+    if padding == "same":
+        out_spatial_dim = spatial_dim
+        if window_right is not None:
+            if isinstance(window_right, int):
+                window_right = Dim(window_right, name="window_right")
+            assert isinstance(window_right, Dim)
+        if window_left is not None:
+            if isinstance(window_left, int):
+                window_left = Dim(window_left, name="window_left")
+            assert isinstance(window_left, Dim)
+        if window_right is None:
+            if window_left is None:
+                window_right = window_dim // 2
+                window_left = window_dim.ceildiv_right(2) - 1
+            else:
+                window_right = window_dim - window_left - 1
+        if window_left is None:
+            window_left = window_dim - window_right - 1
+        source, (in_spatial_dim,) = rf.pad(
+            source,
+            axes=[spatial_dim],
+            padding=[(window_left, window_right)],
+            value=pad_value,
+        )
+        # shape[0] == n_time + window - 1
+    elif padding == "valid":
+        in_spatial_dim = spatial_dim
+        out_spatial_dim = spatial_dim - window_dim + 1
+    else:
+        raise ValueError(f"invalid padding {padding!r}")
+
+    if stride > 1:
+        start_times, out_spatial_dim = rf.range_over_dim_strided(out_spatial_dim, stride=stride)  # (n_out_time,)
+        win_range = rf.range_over_dim(window_dim)  # (window,)
+        indices = rf.combine_bc(start_times, "+", win_range)  # (n_out_time,window)
+        final = rf.gather(source, indices=indices, axis=in_spatial_dim)  # (n_out_time,window,...)
+        return final, out_spatial_dim
+
+    tiled_dimshuffle = rf.expand_dim(source, dim=window_dim)  # (window,n_time+window-1,...)
+    # We want to shift every dim*time block by one to the left.
+    # To do this, we interpret that we have one more time frame (i.e. n_time+window).
+    # We have to do some dimshuffling so that we get the right layout, then we can flatten,
+    # add some padding, and then dimshuffle it back.
+    # Then we can take out the first n_time frames.
+    tiled_flat, flat_dim = rf.merge_dims(tiled_dimshuffle, dims=(window_dim, in_spatial_dim))
+    rem = window_dim
+    tiled_flat_pad_right, (flat_dim_ext,) = rf.pad(tiled_flat, axes=[flat_dim], padding=[(0, rem)], value=pad_value)
+    # add time frame, (window,n_time+window,...)
+    out_time_ext = out_spatial_dim + window_dim
+    tiled_reshape_shift = rf.reshape(tiled_flat_pad_right, in_dims=[flat_dim_ext], out_dims=[window_dim, out_time_ext])
+    final, _ = rf.slice(tiled_reshape_shift, axis=out_time_ext, size=out_spatial_dim)  # (window,n_time,...)
+    if stride > 1:
+        final, out_spatial_dim = rf.slice(final, axis=out_spatial_dim, step=stride)
+    return final, out_spatial_dim
+
+
 def concat(
     *sources: Tuple[Tensor, Dim],
     allow_broadcast: bool = False,

{returnn-1.20231211.111626 → returnn-1.20231212.184511}/returnn/frontend/dims.py

@@ -13,7 +13,8 @@ T = TypeVar("T")
 
 __all__ = [
     "range_over_dim",
-    "range_over_dims",
+    "range_over_dim_strided",
+    "range_over_merged_dims",
     "replace_dim",
     "dim_match_priority_when_needed",
     "num_elements_of_shape",
@@ -34,7 +35,30 @@ def range_over_dim(dim: Dim, *, dtype: Optional[str] = None, device: Optional[st
     return backend.range_over_dim(dim, dtype=dtype, device=device)
 
 
-def range_over_dims(dims: Sequence[Dim], *, dtype: Optional[str] = None, device: Optional[str] = None) -> Tensor[T]:
+def range_over_dim_strided(
+    dim: Dim,
+    *,
+    stride: Union[int, Tensor],
+    out_dim: Optional[Dim] = None,
+    dtype: Optional[str] = None,
+    device: Optional[str] = None,
+) -> Tuple[Tensor[T], Dim]:
+    """
+    :param dim:
+    :param stride:
+    :param out_dim:
+    :param dtype:
+    :param device,
+    :return: tensor with shape [dim], out_dim
+    """
+    if out_dim is None:
+        out_dim = dim.ceildiv_right(stride)
+    return rf.range_over_dim(out_dim, dtype=dtype, device=device) * stride, out_dim
+
+
+def range_over_merged_dims(
+    dims: Sequence[Dim], *, dtype: Optional[str] = None, device: Optional[str] = None
+) -> Tensor[T]:
     """
     This is if you want to index into a merged dim.
     Related: :func:`rf.merge_dims`.

{returnn-1.20231211.111626 → returnn-1.20231212.184511}/returnn/frontend/math_.py

@@ -38,6 +38,7 @@ __all__ = [
     "opt_logical_or",
     "maximum",
     "minimum",
+    "clip_by_value",
     "identity",
     "exp",
     "expm1",
@@ -357,6 +358,20 @@ def minimum(a: Tensor, b: Union[Tensor, _RawTensorTypes], *other_tensors) -> Ten
     return res
 
 
+def clip_by_value(
+    x: Tensor,
+    clip_value_min: Union[Tensor, _RawTensorTypes],
+    clip_value_max: Union[Tensor, _RawTensorTypes],
+    *,
+    allow_broadcast_all_sources: bool = False,
+) -> Tensor:
+    """clip by value"""
+    # noinspection PyProtectedMember
+    return x._raw_backend.clip_by_value(
+        x, clip_value_min, clip_value_max, allow_broadcast_all_sources=allow_broadcast_all_sources
+    )
+
+
 def identity(x: Tensor) -> Tensor:
     """
     Identity function. Just to have one canonical. Does nothing, returns the input.

{returnn-1.20231211.111626 → returnn-1.20231212.184511}/returnn/tf/frontend_layers/_backend.py

@@ -704,6 +704,27 @@ class ReturnnLayersBackend(Backend[Layer]):
             {"class": "switch", "condition": cond, "true_from": true_, "false_from": false_}, name="where"
         )
 
+    @staticmethod
+    def clip_by_value(
+        x: Tensor,
+        clip_value_min: Union[Tensor, rf.RawTensorTypes],
+        clip_value_max: Union[Tensor, rf.RawTensorTypes],
+        *,
+        allow_broadcast_all_sources: bool = False,
+    ) -> Tensor:
+        """clip by value"""
+        clip_value_min = rf.convert_to_tensor(clip_value_min, _backend=ReturnnLayersBackend)
+        clip_value_max = rf.convert_to_tensor(clip_value_max, _backend=ReturnnLayersBackend)
+        return rfl.make_layer(
+            {
+                "class": "eval",
+                "eval": "tf.clip_by_value(source(0), source(1), source(2))",
+                "from": [x, clip_value_min, clip_value_max],
+                "allow_broadcast_all_sources": allow_broadcast_all_sources,
+            },
+            name="clip_by_value",
+        )
+
     @staticmethod
     def matmul(a: Tensor, b: Tensor, *, reduce: Union[Dim, Sequence[Dim]], use_mask: bool = True) -> Tensor:
         """matmul"""

{returnn-1.20231211.111626 → returnn-1.20231212.184511}/returnn/torch/frontend/_backend.py

@@ -1045,13 +1045,39 @@ class TorchBackend(Backend[torch.Tensor]):
             [true_, false_, cond], allow_broadcast_all_sources=allow_broadcast_all_sources, name="where"
         )
         out.dtype = true_.dtype
-        out.sparse_dim = true_.sparse_dim
+        out.sparse_dim = true_.sparse_dim or false_.sparse_dim
+        out.feature_dim = true_.feature_dim or false_.feature_dim
         cond_bc_raw = cond.copy_compatible_to_dims_raw(out.dims)
         true_bc_raw = true_.copy_compatible_to_dims_raw(out.dims)
         false_bc_raw = false_.copy_compatible_to_dims_raw(out.dims)
         out.raw_tensor = torch.where(cond_bc_raw, true_bc_raw, false_bc_raw)
         return out
 
+    @staticmethod
+    def clip_by_value(
+        x: Tensor,
+        clip_value_min: Union[Tensor, rf.RawTensorTypes],
+        clip_value_max: Union[Tensor, rf.RawTensorTypes],
+        *,
+        allow_broadcast_all_sources: bool = False,
+    ) -> Tensor:
+        """clip by value"""
+        clip_value_min = rf.convert_to_tensor(clip_value_min, _backend=TorchBackend, device=x.device)
+        clip_value_max = rf.convert_to_tensor(clip_value_max, _backend=TorchBackend, device=x.device)
+        out = Tensor.get_common_data(
+            [x, clip_value_min, clip_value_max],
+            allow_broadcast_all_sources=allow_broadcast_all_sources,
+            name="clip_by_value",
+        )
+        out.dtype = x.dtype
+        out.sparse_dim = x.sparse_dim
+        out.feature_dim = x.feature_dim
+        x_bc_raw = x.copy_compatible_to_dims_raw(out.dims)
+        min_bc_raw = clip_value_min.copy_compatible_to_dims_raw(out.dims)
+        max_bc_raw = clip_value_max.copy_compatible_to_dims_raw(out.dims)
+        out.raw_tensor = torch.clamp(x_bc_raw, min_bc_raw, max_bc_raw)
+        return out
+
     @staticmethod
     def matmul(a: _TT, b: _TT, *, reduce: Union[Dim, Sequence[Dim]], use_mask: bool = True) -> _TT:
         """

{returnn-1.20231211.111626 → returnn-1.20231212.184511/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231211.111626
+Version: 1.20231212.184511
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20231211.111626/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20231211.111626'
-long_version = '1.20231211.111626+git.cccc69d'