returnn 1.20240906.11340__tar.gz → 1.20240906.140550__tar.gz

{returnn-1.20240906.11340 → returnn-1.20240906.140550}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240906.11340
+Version: 1.20240906.140550
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240906.140550/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240906.140550'
+long_version = '1.20240906.140550+git.fd95009'

{returnn-1.20240906.11340 → returnn-1.20240906.140550}/returnn/tf/layers/base.py

@@ -1259,12 +1259,13 @@ class LayerBase(object):
                 getter.__qualname__ += f"(base_var_scope.custom_getter={base_var_scope.custom_getter})"
 
             param = getter(**getter_kwargs)
+            param_ = param
 
             # Only apply this if we get a variable. Otherwise, maybe variational noise was already applied
             # (by some parent var scope), and we don't want to apply it twice.
-            if param_variational_noise and param.dtype.is_floating and isinstance(param, tf.Variable):
+            if param_variational_noise and param.dtype.is_floating and isinstance(param_, tf.Variable):
                 with default_control_flow_ctx():  # make independent from loop/cond
-                    with reuse_name_scope_of_tensor(param, postfix="_variational_noise", add_tensor_name=True):
+                    with reuse_name_scope_of_tensor(param_, postfix="_variational_noise", add_tensor_name=True):
 
                         def _apply_var_noise():
                             rnd_state = tf_util.StatelessRandomSeed.create(shape=tf_util.get_shape(param))
@@ -1280,11 +1281,11 @@ class LayerBase(object):
             if (
                 param_dropout
                 and param.dtype.is_floating
-                and isinstance(param, tf.Variable)
+                and isinstance(param_, tf.Variable)
                 and param.shape.ndims >= param_dropout_min_ndim
             ):
                 with default_control_flow_ctx():  # make independent from loop/cond
-                    with reuse_name_scope_of_tensor(param, postfix="_weight_dropout", add_tensor_name=True):
+                    with reuse_name_scope_of_tensor(param_, postfix="_weight_dropout", add_tensor_name=True):
                         param = self.network.cond_on_train(
                             fn_train=lambda: tf_util.dropout(
                                 param,

{returnn-1.20240906.11340 → returnn-1.20240906.140550}/returnn/tf/layers/basic.py

@@ -2139,6 +2139,7 @@ class LinearLayer(_ConcatInputLayer):
                     name="W", shape=weights_shape, dtype=tf.float32, initializer=fwd_weights_initializer
                 )
             )
+            self.weights = weights
             weights_ = weights
             if in_split_info:
                 tf_util.set_param_axes_split_info(
@@ -2160,6 +2161,7 @@ class LinearLayer(_ConcatInputLayer):
             else:
                 assert not bias_init
                 b = None
+            self.bias = b
 
         with tf.name_scope("linear"):
             from returnn.tf.util.basic import dot, to_int32_64, is_gpu_available_in_session, move_axis

{returnn-1.20240906.11340 → returnn-1.20240906.140550}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240906.11340
+Version: 1.20240906.140550
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20240906.11340 → returnn-1.20240906.140550}/tests/test_TFNetworkLayer.py

@@ -10992,6 +10992,88 @@ def test_param_weight_dropout_and_reuse_params():
         session.run(opt_op, feed_dict=make_feed_dict(network.extern_data))
 
 
+def test_param_weight_dropout_and_variational_noise():
+    from returnn.tensor import Dim, batch_dim
+    from returnn.tf.util.basic import print_graph_output, find_ops_with_tensor_input
+    from returnn.tf.util.gradient_checkpoint import prepare_gradient_checkpointing
+
+    time_dim = Dim(None, name="time")
+    feature_dim = Dim(7, name="feature")
+    classes_dim = Dim(13, name="classes")
+
+    config = Config(
+        {
+            "param_dropout": 0.1,
+            "param_variational_noise": 0.075,
+            "extern_data": {
+                "data": {
+                    "dim_tags": [batch_dim, time_dim, feature_dim],
+                    "time_dim_axis": 1,
+                    "feature_dim": feature_dim,
+                    "dtype": "float32",
+                },
+                "classes": {"dim_tags": [batch_dim, time_dim], "sparse_dim": classes_dim, "dtype": "int32"},
+            },
+        }
+    )
+    with make_scope() as session:
+        network = TFNetwork(config=config, train_flag=True)
+        # Do subnetwork by intention, to test when we have multiple variable scopes.
+        network.construct_from_dict(
+            {
+                "output": {
+                    "class": "linear",
+                    "out_dim": classes_dim,
+                    "activation": "softmax",
+                    "from": "data",
+                    "loss": "ce",
+                    "target": "classes",
+                }
+            }
+        )
+        loss = network.get_total_loss()
+
+        prepare_gradient_checkpointing()
+        opt = tf_compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
+        opt_op = opt.minimize(loss)
+        print("optimizer:")
+        print_graph_output(opt_op)
+
+        tf_log_dir = tempfile.mkdtemp()
+        print("TF log dir:", tf_log_dir)
+        writer = tf_compat.v1.summary.FileWriter(logdir=tf_log_dir, graph=session.graph, session=session)
+        params = network.get_params_list()
+        print("params:", params)
+        assert len(params) == 2  # weights and bias
+        for param in params:
+            print("param:", param)
+            ops = find_ops_with_tensor_input(param, fetches=opt_op)
+            print("param graph:")
+            print_graph_output(ops)
+            # There can be multiple ops due to gradient checkpointing.
+            assert (
+                1 <= len(ops)
+                and all("_variational_noise/" in op.name or "/ResourceApply" in op.name for op in ops)
+                and any("_variational_noise/" in op.name for op in ops)
+            ), f"ops: {ops}"
+
+        layer = network.layers["output"]
+        assert isinstance(layer, LinearLayer)
+        print("weights:", layer.weights)
+        assert layer.weights.name.startswith("output/W_weight_dropout/")
+
+        network.initialize_params(session=session)
+
+        run_metadata = tf_compat.v1.RunMetadata()
+        run_options = tf_compat.v1.RunOptions(trace_level=tf_compat.v1.RunOptions.FULL_TRACE)
+        session.run(
+            opt_op, feed_dict=make_feed_dict(network.extern_data), options=run_options, run_metadata=run_metadata
+        )
+        writer.add_run_metadata(run_metadata, tag="step_0")
+        writer.close()
+        print("TF log dir:", tf_log_dir)
+
+
 def test_LinearLayer_simple_train():
     config = Config()
     n_in, n_out = 7, 3

returnn-1.20240906.11340/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240906.011340'
-long_version = '1.20240906.011340+git.1eb8103'