returnn 1.20240906.11340__tar.gz → 1.20240906.142620__tar.gz

{returnn-1.20240906.11340 → returnn-1.20240906.142620}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240906.11340
+Version: 1.20240906.142620
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240906.142620/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240906.142620'
+long_version = '1.20240906.142620+git.94e8e1d'

{returnn-1.20240906.11340 → returnn-1.20240906.142620}/returnn/tf/layers/base.py

@@ -1259,12 +1259,13 @@ class LayerBase(object):
                 getter.__qualname__ += f"(base_var_scope.custom_getter={base_var_scope.custom_getter})"
 
             param = getter(**getter_kwargs)
+            param_ = param
 
             # Only apply this if we get a variable. Otherwise, maybe variational noise was already applied
             # (by some parent var scope), and we don't want to apply it twice.
-            if param_variational_noise and param.dtype.is_floating and isinstance(param, tf.Variable):
+            if param_variational_noise and param.dtype.is_floating and isinstance(param_, tf.Variable):
                 with default_control_flow_ctx():  # make independent from loop/cond
-                    with reuse_name_scope_of_tensor(param, postfix="_variational_noise", add_tensor_name=True):
+                    with reuse_name_scope_of_tensor(param_, postfix="_variational_noise", add_tensor_name=True):
 
                         def _apply_var_noise():
                             rnd_state = tf_util.StatelessRandomSeed.create(shape=tf_util.get_shape(param))
@@ -1280,11 +1281,11 @@ class LayerBase(object):
             if (
                 param_dropout
                 and param.dtype.is_floating
-                and isinstance(param, tf.Variable)
+                and isinstance(param_, tf.Variable)
                 and param.shape.ndims >= param_dropout_min_ndim
             ):
                 with default_control_flow_ctx():  # make independent from loop/cond
-                    with reuse_name_scope_of_tensor(param, postfix="_weight_dropout", add_tensor_name=True):
+                    with reuse_name_scope_of_tensor(param_, postfix="_weight_dropout", add_tensor_name=True):
                         param = self.network.cond_on_train(
                             fn_train=lambda: tf_util.dropout(
                                 param,

{returnn-1.20240906.11340 → returnn-1.20240906.142620}/returnn/tf/layers/basic.py

@@ -2139,6 +2139,7 @@ class LinearLayer(_ConcatInputLayer):
                     name="W", shape=weights_shape, dtype=tf.float32, initializer=fwd_weights_initializer
                 )
             )
+            self.weights = weights
             weights_ = weights
             if in_split_info:
                 tf_util.set_param_axes_split_info(
@@ -2160,6 +2161,7 @@ class LinearLayer(_ConcatInputLayer):
             else:
                 assert not bias_init
                 b = None
+            self.bias = b
 
         with tf.name_scope("linear"):
             from returnn.tf.util.basic import dot, to_int32_64, is_gpu_available_in_session, move_axis

{returnn-1.20240906.11340 → returnn-1.20240906.142620}/returnn/tf/network.py

@@ -3,7 +3,7 @@ Defines the :class:`TFNetwork` and :class:`ExternData`.
 """
 
 from __future__ import annotations
-from typing import Optional, List, Tuple
+from typing import Optional, Any, List, Tuple, Dict
 import tensorflow as tf
 import sys
 import re
@@ -650,8 +650,8 @@ class TFNetwork(object):
         self.subnets = {}  # type: typing.Dict[str,Subnetwork]
         self._selected_train_layers = None
         self._construction_stack = _NetworkConstructionStack()
-        self.layers_desc = {}  # type: typing.Dict[str,typing.Dict[str]]
-        self.layers = {}  # type: typing.Dict[str,LayerBase]
+        self.layers_desc: Dict[str, Dict[str, Any]] = {}
+        self.layers: Dict[str, LayerBase] = {}
         self.losses_dict = {}  # type: typing.Dict[str,LossHolder]
         self.total_loss = None  # type: typing.Optional[tf.Tensor]
         self.total_constraints = None  # type: typing.Optional[tf.Tensor]
@@ -663,16 +663,16 @@ class TFNetwork(object):
         self.extra_vars_to_save = []  # type: typing.List[tf.Variable]
         self.recurrent = False
         self._assigner_cache = {}  # type: typing.Dict[tf.Variable,tf_util.VariableAssigner]
-        self.concat_sources_dropout_cache = (
-            {}
-        )  # type: typing.Dict[typing.Tuple[typing.Tuple[LayerBase,...],Dim,float,typing.Optional[typing.Tuple[typing.Optional[int],...]]],Data]  # nopep8
+        self.concat_sources_dropout_cache: Dict[
+            Tuple[Tuple[LayerBase, ...], Dim, float, Optional[Tuple[Optional[int], ...]]], Data
+        ] = {}
         self._merge_all_summaries = None  # type: typing.Optional[tf.Tensor]
         self._graph_reset_callbacks = []  # type: typing.List[typing.Callable]
         self._run_opts = {}  # type: typing.Dict[str, typing.Any]
         self._run_finished_callbacks = []  # type: typing.List[typing.Callable]
         self._map_search_beam_to_search_choices = (
             {}
-        )  # type: typing.Dict[tf_util.SearchBeam,"returnn.tf.layers.base.SearchChoices"]  # nopep8
+        )  # type: typing.Dict[tf_util.SearchBeam,"returnn.tf.layers.base.SearchChoices"]
 
     def __repr__(self):
         s = "TFNetwork %r" % self.name
@@ -786,18 +786,17 @@ class TFNetwork(object):
             return self.extra_parent_net.get_absolute_name_prefix() + my_prefix
         return ""
 
-    def construct_from_dict(self, net_dict, get_layer=None):
+    def construct_from_dict(self, net_dict: Dict[str, Dict[str, Any]], get_layer=None):
         """
-        :param dict[str,dict[str]] net_dict:
+        :param net_dict:
         :param GetLayer|((str)->LayerBase)|None get_layer:
         """
         self.layers_desc.update(net_dict)
 
-        def ignore_layer(name_, layer_desc_):
+        def ignore_layer(name_: str, layer_desc_: Dict[str, Any]) -> bool:
             """
-            :param str name_:
-            :param dict layer_desc_:
-            :rtype: bool
+            :param name_:
+            :param layer_desc_:
             """
             assert isinstance(name_, str)
             if name_.startswith("#"):
@@ -1028,14 +1027,21 @@ class TFNetwork(object):
         """
         return self.get_config().bool("flat_net_construction", False)
 
-    def construct_layer(self, net_dict, name, get_layer=None, add_layer=None, check_existing=True):
+    def construct_layer(
+        self,
+        net_dict: Dict[str, Dict[str, Any]],
+        name: str,
+        get_layer=None,
+        add_layer=None,
+        check_existing: bool = True,
+    ) -> LayerBase:
         """
         This triggers the construction of the layer `name` if it is not constructed yet.
         Every construction trigger corresponds to ``add_layer`` call (which by default does the actual construction).
         This can recursively also get/construct other layers (via ``get_layer``).
 
-        :param dict[str,dict[str]] net_dict:
-        :param str name: layer name
+        :param net_dict:
+        :param name: layer name
         :param GetLayer|((str)->LayerBase)|None get_layer: optional, for source layers, for transform_config_dict.
           By default, this wraps self.construct_layer().
           I.e. the name might be misleading, as this should return an existing layer,
@@ -1050,8 +1056,8 @@ class TFNetwork(object):
             But otherwise, this is not an option!
 
         :param ((str, LayerBase, dict) -> LayerBase) | None add_layer: by default self.add_layer
-        :param bool check_existing: check self.get_layer. (self.layers will be checked in any case)
-        :rtype: LayerBase
+        :param check_existing: check self.get_layer. (self.layers will be checked in any case)
+        :return: layer
         """
         if name in self.layers:
             return self.layers[name]

{returnn-1.20240906.11340 → returnn-1.20240906.142620}/returnn/tf/util/basic.py

@@ -4,7 +4,7 @@ Lots of random utility functions for TensorFlow.
 
 from __future__ import annotations
 
-from typing import Optional, Union, Sequence, List
+from typing import Optional, Union, Any, Sequence, List, Dict
 import typing
 import contextlib
 from dataclasses import dataclass
@@ -1658,14 +1658,19 @@ class GammatoneFilterbankInitializer(init_ops.Initializer):
         return tf.convert_to_tensor(fbank, dtype=dtype)
 
 
-def get_initializer(s, seed=None, eval_local_ns=None, dtype=tf.float32):
+def get_initializer(
+    s,
+    seed: Optional[Union[int, tf.Tensor]] = None,
+    eval_local_ns: Optional[Dict[str, Any]] = None,
+    dtype: Union[tf.DType, str] = tf.float32,
+):
     """
     :param str|dict[str]|float|numpy.ndarray s: e.g. "glorot_uniform" or "truncated_normal" or "orthogonal",
       or config dict with "class",
       or string to be `eval`ed if it contains "(". constant if a float is given.
-    :param int|tf.Tensor seed: used in case the initializer has no explicit seed specified.
-    :param dict[str]|None eval_local_ns:
-    :param tf.DType|str dtype:
+    :param seed: used in case the initializer has no explicit seed specified.
+    :param eval_local_ns:
+    :param dtype:
     :return: (function (shape) -> tf.Tensor) | tf.Initializer
     :rtype: ((tuple[int]) -> tf.Tensor) | tf.Initializer
     """

{returnn-1.20240906.11340 → returnn-1.20240906.142620}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240906.11340
+Version: 1.20240906.142620
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20240906.11340 → returnn-1.20240906.142620}/tests/test_TFNetworkLayer.py

@@ -10992,6 +10992,88 @@ def test_param_weight_dropout_and_reuse_params():
         session.run(opt_op, feed_dict=make_feed_dict(network.extern_data))
 
 
+def test_param_weight_dropout_and_variational_noise():
+    from returnn.tensor import Dim, batch_dim
+    from returnn.tf.util.basic import print_graph_output, find_ops_with_tensor_input
+    from returnn.tf.util.gradient_checkpoint import prepare_gradient_checkpointing
+
+    time_dim = Dim(None, name="time")
+    feature_dim = Dim(7, name="feature")
+    classes_dim = Dim(13, name="classes")
+
+    config = Config(
+        {
+            "param_dropout": 0.1,
+            "param_variational_noise": 0.075,
+            "extern_data": {
+                "data": {
+                    "dim_tags": [batch_dim, time_dim, feature_dim],
+                    "time_dim_axis": 1,
+                    "feature_dim": feature_dim,
+                    "dtype": "float32",
+                },
+                "classes": {"dim_tags": [batch_dim, time_dim], "sparse_dim": classes_dim, "dtype": "int32"},
+            },
+        }
+    )
+    with make_scope() as session:
+        network = TFNetwork(config=config, train_flag=True)
+        # Do subnetwork by intention, to test when we have multiple variable scopes.
+        network.construct_from_dict(
+            {
+                "output": {
+                    "class": "linear",
+                    "out_dim": classes_dim,
+                    "activation": "softmax",
+                    "from": "data",
+                    "loss": "ce",
+                    "target": "classes",
+                }
+            }
+        )
+        loss = network.get_total_loss()
+
+        prepare_gradient_checkpointing()
+        opt = tf_compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
+        opt_op = opt.minimize(loss)
+        print("optimizer:")
+        print_graph_output(opt_op)
+
+        tf_log_dir = tempfile.mkdtemp()
+        print("TF log dir:", tf_log_dir)
+        writer = tf_compat.v1.summary.FileWriter(logdir=tf_log_dir, graph=session.graph, session=session)
+        params = network.get_params_list()
+        print("params:", params)
+        assert len(params) == 2  # weights and bias
+        for param in params:
+            print("param:", param)
+            ops = find_ops_with_tensor_input(param, fetches=opt_op)
+            print("param graph:")
+            print_graph_output(ops)
+            # There can be multiple ops due to gradient checkpointing.
+            assert (
+                1 <= len(ops)
+                and all("_variational_noise/" in op.name or "/ResourceApply" in op.name for op in ops)
+                and any("_variational_noise/" in op.name for op in ops)
+            ), f"ops: {ops}"
+
+        layer = network.layers["output"]
+        assert isinstance(layer, LinearLayer)
+        print("weights:", layer.weights)
+        assert layer.weights.name.startswith("output/W_weight_dropout/")
+
+        network.initialize_params(session=session)
+
+        run_metadata = tf_compat.v1.RunMetadata()
+        run_options = tf_compat.v1.RunOptions(trace_level=tf_compat.v1.RunOptions.FULL_TRACE)
+        session.run(
+            opt_op, feed_dict=make_feed_dict(network.extern_data), options=run_options, run_metadata=run_metadata
+        )
+        writer.add_run_metadata(run_metadata, tag="step_0")
+        writer.close()
+        print("TF log dir:", tf_log_dir)
+
+
 def test_LinearLayer_simple_train():
     config = Config()
     n_in, n_out = 7, 3

returnn-1.20240906.11340/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240906.011340'
-long_version = '1.20240906.011340+git.1eb8103'