tf-keras-nightly 2.18.0.dev2024071010__tar.gz → 2.18.0.dev2024071309__tar.gz

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tf_keras-nightly
-Version: 2.18.0.dev2024071010
+Version: 2.18.0.dev2024071309
 Summary: Deep learning for humans.
 Home-page: https://keras.io/
 Download-URL: https://github.com/keras-team/tf-keras/tags

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/setup.py

@@ -38,7 +38,7 @@ with open(os.path.abspath(__file__)) as f:
         )
 
 # pin version to that of tensorflow or tf_nightly.
-version = "2.18.0.dev2024071010".lower()
+version = "2.18.0.dev2024071309".lower()
 major_version, minor_version, *_ = version.split(".")
 next_minor_version = int(minor_version) + 1
 if "nightly" in "tf_keras-nightly":
@@ -61,7 +61,7 @@ setuptools.setup(
     name="tf_keras-nightly",
     # Version strings with `-` characters are semver compatible,
     # but incompatible with pip. For pip, we will remove all `-`` characters.
-    version="2.18.0.dev2024071010",
+    version="2.18.0.dev2024071309",
     description="Deep learning for humans.",
     long_description=DESCRIPTION,
     url="https://keras.io/",

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/tf_keras/__init__.py

@@ -27,4 +27,4 @@ from tf_keras.src.engine.sequential import Sequential
 from tf_keras.src.engine.training import Model
 
 
-__version__ = "2.18.0.dev2024071010"
+__version__ = "2.18.0.dev2024071309"

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/tf_keras/src/callbacks.py

@@ -1903,6 +1903,23 @@ class BackupAndRestore(Callback):
                 "only supports empty strategy, "
                 "MirroredStrategy, MultiWorkerMirroredStrategy and TPUStrategy."
             )
+
+        # Re-initialize the optimizer.
+        if self.model.built:
+            if (
+                self.model.optimizer is not None
+                and callable(getattr(self.model.optimizer, "build", None))
+                and not getattr(self.model.optimizer, "_built", False)
+            ):
+                self.model.optimizer.build(self.model.trainable_variables)
+        else:
+            logging.warning(
+                "To use the BackupAndRestore callback, "
+                "you model must be built before you call `fit()`. "
+                f"Model {self.model} is unbuilt. You can build it "
+                "beforehand by calling it on a batch of data."
+            )
+
         self.model._training_state = worker_training_state.WorkerTrainingState(
             self.model,
             self.backup_dir,

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/tf_keras/src/layers/rnn/cudnn_gru.py

@@ -172,6 +172,10 @@ class CuDNNGRU(_CuDNNRNN):
             shape=self._vector_shape,
         )
 
+        batch_dim = tf.shape(inputs)[1]
+        max_sequence_length = tf.shape(inputs)[0]
+        sequence_lengths = tf.fill([batch_dim], max_sequence_length)
+
         args = {
             "input": inputs,
             "input_h": input_h,
@@ -179,9 +183,10 @@ class CuDNNGRU(_CuDNNRNN):
             "params": params,
             "is_training": True,
             "rnn_mode": "gru",
+            "sequence_lengths": sequence_lengths,
         }
 
-        outputs, h, _, _, _ = tf.raw_ops.CudnnRNNV2(**args)
+        outputs, h, _, _, _ = tf.raw_ops.CudnnRNNV3(**args)
 
         if self.stateful or self.return_state:
             h = h[0]

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/tf_keras/src/layers/rnn/cudnn_lstm.py

@@ -204,15 +204,20 @@ class CuDNNLSTM(_CuDNNRNN):
             shape=self._vector_shape,
         )
 
+        batch_dim = tf.shape(inputs)[1]
+        max_sequence_length = tf.shape(inputs)[0]
+        sequence_lengths = tf.fill([batch_dim], max_sequence_length)
+
         args = {
             "input": inputs,
             "input_h": input_h,
             "input_c": input_c,
             "params": params,
             "is_training": True,
+            "sequence_lengths": sequence_lengths,
         }
 
-        outputs, h, c, _, _ = tf.raw_ops.CudnnRNNV2(**args)
+        outputs, h, c, _, _ = tf.raw_ops.CudnnRNNV3(**args)
 
         if self.stateful or self.return_state:
             h = h[0]

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/tf_keras/src/layers/rnn/gru.py

@@ -1034,11 +1034,13 @@ def gpu_gru(
             mask, time_major
         )
 
-    if not time_major and sequence_lengths is None:
-        inputs = tf.transpose(inputs, perm=(1, 0, 2))
-        seq_axis, batch_axis = (0, 1)
-    else:
-        seq_axis, batch_axis = (0, 1) if time_major else (1, 0)
+    seq_axis, batch_axis = (0, 1) if time_major else (1, 0)
+
+    if sequence_lengths is None:
+        max_sequence_length = tf.shape(inputs)[seq_axis]
+        batch_size = tf.shape(inputs)[batch_axis]
+        sequence_lengths = tf.fill([batch_size], max_sequence_length)
+
     # For init_h, cuDNN expects one more dim of num_layers before or after batch
     # dim for time major or batch major inputs respectively
     init_h = tf.expand_dims(init_h, axis=seq_axis)
@@ -1069,49 +1071,36 @@ def gpu_gru(
         transpose_weights=True,
     )
 
-    if sequence_lengths is not None:
-        if go_backwards:
-            # Three reversals are required. E.g.,
-            # normal input = [1, 2, 3, 0, 0]  # where 0 need to be masked
-            # reversed_input_to_cudnn = [3, 2, 1, 0, 0]
-            # output_from_cudnn = [6, 5, 4, 0, 0]
-            # expected_output = [0, 0, 6, 5 ,4]
-            inputs = tf.reverse_sequence(
-                inputs,
-                sequence_lengths,
-                seq_axis=seq_axis,
-                batch_axis=batch_axis,
-            )
-        outputs, h, _, _, _ = tf.raw_ops.CudnnRNNV3(
-            input=inputs,
-            input_h=init_h,
-            input_c=0,
-            params=params,
-            is_training=True,
-            rnn_mode="gru",
-            sequence_lengths=sequence_lengths,
-            time_major=time_major,
+    if go_backwards:
+        # Three reversals are required. E.g.,
+        # normal input = [1, 2, 3, 0, 0]  # where 0 need to be masked
+        # reversed_input_to_cudnn = [3, 2, 1, 0, 0]
+        # output_from_cudnn = [6, 5, 4, 0, 0]
+        # expected_output = [0, 0, 6, 5 ,4]
+        inputs = tf.reverse_sequence(
+            inputs,
+            sequence_lengths,
+            seq_axis=seq_axis,
+            batch_axis=batch_axis,
         )
-        if go_backwards:
-            outputs = tf.reverse_sequence(
-                outputs,
-                sequence_lengths,
-                seq_axis=seq_axis,
-                batch_axis=batch_axis,
-            )
-            outputs = tf.reverse(outputs, axis=[seq_axis])
-    else:
-        if go_backwards:
-            # Reverse axis 0 since the input is already convert to time major.
-            inputs = tf.reverse(inputs, axis=[0])
-        outputs, h, _, _ = tf.raw_ops.CudnnRNN(
-            input=inputs,
-            input_h=init_h,
-            input_c=0,
-            params=params,
-            is_training=True,
-            rnn_mode="gru",
+    outputs, h, _, _, _ = tf.raw_ops.CudnnRNNV3(
+        input=inputs,
+        input_h=init_h,
+        input_c=0,
+        params=params,
+        is_training=True,
+        rnn_mode="gru",
+        sequence_lengths=sequence_lengths,
+        time_major=time_major,
+    )
+    if go_backwards:
+        outputs = tf.reverse_sequence(
+            outputs,
+            sequence_lengths,
+            seq_axis=seq_axis,
+            batch_axis=batch_axis,
         )
+        outputs = tf.reverse(outputs, axis=[seq_axis])
 
     last_output = outputs[-1]
     if not time_major and sequence_lengths is None and return_sequences:

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/tf_keras/src/layers/rnn/lstm.py

@@ -1063,11 +1063,13 @@ def gpu_lstm(
             mask, time_major
         )
 
-    if not time_major and sequence_lengths is None:
-        inputs = tf.transpose(inputs, perm=(1, 0, 2))
-        seq_axis, batch_axis = (0, 1)
-    else:
-        seq_axis, batch_axis = (0, 1) if time_major else (1, 0)
+    seq_axis, batch_axis = (0, 1) if time_major else (1, 0)
+
+    if sequence_lengths is None:
+        max_sequence_length = tf.shape(inputs)[seq_axis]
+        batch_size = tf.shape(inputs)[batch_axis]
+        sequence_lengths = tf.fill([batch_size], max_sequence_length)
+
     # For init_h and init_c, cuDNN expects one more dim of num_layers before or
     # after batch dim for time major or batch major inputs respectively
     init_h = tf.expand_dims(init_h, axis=seq_axis)
@@ -1099,52 +1101,36 @@ def gpu_lstm(
         transpose_weights=True,
     )
 
-    if sequence_lengths is not None:
-        if go_backwards:
-            # Three reversals are required. E.g.,
-            # normal input = [1, 2, 3, 0, 0]  # where 0 need to be masked
-            # reversed_input_to_cudnn = [3, 2, 1, 0, 0]
-            # output_from_cudnn = [6, 5, 4, 0, 0]
-            # expected_output = [0, 0, 6, 5 ,4]
-            inputs = tf.reverse_sequence(
-                inputs,
-                sequence_lengths,
-                seq_axis=seq_axis,
-                batch_axis=batch_axis,
-            )
-        outputs, h, c, _, _ = tf.raw_ops.CudnnRNNV3(
-            input=inputs,
-            input_h=init_h,
-            input_c=init_c,
-            params=params,
-            is_training=True,
-            rnn_mode="lstm",
-            sequence_lengths=sequence_lengths,
-            time_major=time_major,
+    if go_backwards:
+        # Three reversals are required. E.g.,
+        # normal input = [1, 2, 3, 0, 0]  # where 0 need to be masked
+        # reversed_input_to_cudnn = [3, 2, 1, 0, 0]
+        # output_from_cudnn = [6, 5, 4, 0, 0]
+        # expected_output = [0, 0, 6, 5 ,4]
+        inputs = tf.reverse_sequence(
+            inputs,
+            sequence_lengths,
+            seq_axis=seq_axis,
+            batch_axis=batch_axis,
         )
-        if go_backwards:
-            outputs = tf.reverse_sequence(
-                outputs,
-                sequence_lengths,
-                seq_axis=seq_axis,
-                batch_axis=batch_axis,
-            )
-            outputs = tf.reverse(outputs, axis=[seq_axis])
-    else:
-        # # Fill the array with shape [batch] with value of max timesteps.
-        # sequence_length = array_ops.fill([array_ops.shape(inputs)[1]],
-        #                                  array_ops.shape(inputs)[0])
-        if go_backwards:
-            # Reverse axis 0 since the input is already convert to time major.
-            inputs = tf.reverse(inputs, axis=[0])
-        outputs, h, c, _ = tf.raw_ops.CudnnRNN(
-            input=inputs,
-            input_h=init_h,
-            input_c=init_c,
-            params=params,
-            is_training=True,
-            rnn_mode="lstm",
+    outputs, h, c, _, _ = tf.raw_ops.CudnnRNNV3(
+        input=inputs,
+        input_h=init_h,
+        input_c=init_c,
+        params=params,
+        is_training=True,
+        rnn_mode="lstm",
+        sequence_lengths=sequence_lengths,
+        time_major=time_major,
+    )
+    if go_backwards:
+        outputs = tf.reverse_sequence(
+            outputs,
+            sequence_lengths,
+            seq_axis=seq_axis,
+            batch_axis=batch_axis,
         )
+        outputs = tf.reverse(outputs, axis=[seq_axis])
 
     last_output = outputs[-1]
     if not time_major and sequence_lengths is None and return_sequences:

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/tf_keras/src/mixed_precision/autocast_variable.py

@@ -124,20 +124,21 @@ class AutoCastVariable(tf.Variable, tf.__internal__.types.Tensor):
     def _should_cast(self):
         """Returns True if this variable should be casted when accessed."""
         autocast_dtype = getattr(_autocast_dtype, "dtype", None)
-        return autocast_dtype is not None and self.dtype != autocast_dtype
+        return autocast_dtype is not None and self.true_dtype != autocast_dtype
 
     @property
     def dtype(self):
-        """The dtype of the underlying variable, before any casts are done."""
-        return self._variable.dtype
+        """The dtype when the value is accessed, that is after casting."""
+        return self._cast_dtype
 
     @property
     def true_dtype(self):
-        """Deprecated alias of `dtype`."""
+        """The dtype of the underlying variable, before any casts are done."""
         return self._variable.dtype
 
     @property
     def _cast_dtype(self):
+        """The dtype after casting."""
         dtype = getattr(_autocast_dtype, "dtype", None)
         return dtype or self._variable.dtype
 
@@ -202,7 +203,8 @@ class AutoCastVariable(tf.Variable, tf.__internal__.types.Tensor):
         if tf.executing_eagerly() and not self._in_graph_mode:
             repr_str = (
                 "<AutoCastVariable '{v.name}' shape={v.shape} "
-                "dtype={v.dtype.name} dtype_to_cast_to={v._cast_dtype.name}, "
+                "dtype={v.true_dtype.name} "
+                "dtype_to_cast_to={v._cast_dtype.name}, "
                 "numpy={np_repr}>"
             )
             return repr_str.format(
@@ -211,7 +213,8 @@ class AutoCastVariable(tf.Variable, tf.__internal__.types.Tensor):
         else:
             repr_str = (
                 "<AutoCastVariable '{v.name}' shape={v.shape} "
-                "dtype={v.dtype.name} dtype_to_cast_to={v._cast_dtype.name}>"
+                "dtype={v.true_dtype.name} "
+                "dtype_to_cast_to={v._cast_dtype.name}>"
             )
             return repr_str.format(v=self)
 
@@ -261,6 +264,9 @@ class AutoCastVariable(tf.Variable, tf.__internal__.types.Tensor):
     def _apply_assign_update(
         self, update_fn, value, use_locking=None, name=None, read_value=True
     ):
+        # In auto cast scope, we cast back to the actual variable dtype.
+        if self._should_cast():
+            value = tf.cast(value, self.true_dtype)
         # TODO(b/146181571): This logic can be simplified once
         # DistributedVariable.assign returns a DistributedVariable. Currently
         # for MirroredStrategy, it returns a Mirrored value.

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/tf_keras/src/optimizers/legacy/optimizer_v2.py

@@ -1033,6 +1033,13 @@ class OptimizerV2(tf.__internal__.tracking.Trackable):
         slot_dict = self._slots.setdefault(var_key, {})
         weight = slot_dict.get(slot_name, None)
         if weight is None:
+            # Under a mixed precision policy, variables report their "cast"
+            # dtype. However, we want to use the original dtype for slots.
+            if hasattr(var, "true_dtype"):
+                dtype = var.true_dtype
+            else:
+                dtype = var.dtype
+
             if isinstance(initializer, str) or callable(initializer):
                 initializer = initializers.get(initializer)
                 if isinstance(
@@ -1043,7 +1050,7 @@ class OptimizerV2(tf.__internal__.tracking.Trackable):
                 else:
                     slot_shape = var.shape
                 initial_value = functools.partial(
-                    initializer, shape=slot_shape, dtype=var.dtype
+                    initializer, shape=slot_shape, dtype=dtype
                 )
             else:
                 initial_value = initializer
@@ -1064,7 +1071,7 @@ class OptimizerV2(tf.__internal__.tracking.Trackable):
                 with strategy.extended.colocate_vars_with(var):
                     weight = tf.Variable(
                         name=f"{var._shared_name}/{slot_name}",
-                        dtype=var.dtype,
+                        dtype=dtype,
                         trainable=False,
                         initial_value=initial_value,
                     )

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/tf_keras/src/optimizers/optimizer.py

@@ -498,26 +498,28 @@ class _BaseOptimizer(tf.__internal__.tracking.AutoTrackable):
         Returns:
           An optimizer variable.
         """
+        # Under a mixed precision policy, variables report their "cast"
+        # dtype. However, we want to use the original dtype for slots.
+        if hasattr(model_variable, "true_dtype"):
+            dtype = model_variable.true_dtype
+        else:
+            dtype = model_variable.dtype
         if initial_value is None:
             if shape is None:
                 if model_variable.shape.rank is None:
                     # When the rank is None, we cannot get a concrete
                     # `model_variable.shape`, we use dynamic shape.
-                    initial_value = tf.zeros_like(
-                        model_variable, dtype=model_variable.dtype
-                    )
+                    initial_value = tf.zeros_like(model_variable, dtype=dtype)
                 else:
                     # We cannot always use `zeros_like`, because some cases
                     # the shape exists while values don't.
-                    initial_value = tf.zeros(
-                        model_variable.shape, dtype=model_variable.dtype
-                    )
+                    initial_value = tf.zeros(model_variable.shape, dtype=dtype)
             else:
-                initial_value = tf.zeros(shape, dtype=model_variable.dtype)
+                initial_value = tf.zeros(shape, dtype=dtype)
         variable = tf.Variable(
             initial_value=initial_value,
             name=f"{variable_name}/{model_variable._shared_name}",
-            dtype=model_variable.dtype,
+            dtype=dtype,
             trainable=False,
         )
         # If model_variable is a shard of a ShardedVariable, we should add a
@@ -1188,10 +1190,17 @@ class Optimizer(_BaseOptimizer):
                         self._mesh, rank=initial_value.shape.rank
                     ),
                 )
+            # Under a mixed precision policy, variables report their "cast"
+            # dtype. However, we want to use the original dtype for optimizer
+            # variables.
+            if hasattr(model_variable, "true_dtype"):
+                dtype = model_variable.true_dtype
+            else:
+                dtype = model_variable.dtype
             variable = tf.experimental.dtensor.DVariable(
                 initial_value=initial_value,
                 name=f"{variable_name}/{model_variable._shared_name}",
-                dtype=model_variable.dtype,
+                dtype=dtype,
                 trainable=False,
             )
             self._variables.append(variable)

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/tf_keras/src/utils/steps_per_execution_tuning.py

@@ -229,7 +229,7 @@ class StepsPerExecutionTuner:
 
         if current_spe >= spe_limit:
             new_spe = current_spe
-        elif current_spe == 0:
+        elif current_spe <= 0:
             new_spe = self.init_spe
 
         self._steps_per_execution.assign(np.round(new_spe))

{tf_keras_nightly-2.18.0.dev2024071010 → tf_keras_nightly-2.18.0.dev2024071309}/tf_keras_nightly.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tf_keras-nightly
-Version: 2.18.0.dev2024071010
+Version: 2.18.0.dev2024071309
 Summary: Deep learning for humans.
 Home-page: https://keras.io/
 Download-URL: https://github.com/keras-team/tf-keras/tags