keras-nightly 3.12.0.dev2025081903__py3-none-any.whl → 3.12.0.dev2025082003__py3-none-any.whl

keras/src/utils/audio_dataset_utils.py

@@ -411,7 +411,7 @@ def paths_and_labels_to_dataset(
     """Constructs a fixed-size dataset of audio and labels."""
     path_ds = tf.data.Dataset.from_tensor_slices(file_paths)
     if label_mode:
-        label_ds = dataset_utils.labels_to_dataset(
+        label_ds = dataset_utils.labels_to_dataset_tf(
             labels, label_mode, num_classes
         )
         ds = tf.data.Dataset.zip((path_ds, label_ds))

keras/src/utils/dataset_utils.py

@@ -8,7 +8,9 @@ import numpy as np
 
 from keras.src import tree
 from keras.src.api_export import keras_export
+from keras.src.utils import file_utils
 from keras.src.utils import io_utils
+from keras.src.utils.module_utils import grain
 from keras.src.utils.module_utils import tensorflow as tf
 
 
@@ -299,6 +301,17 @@ def is_torch_dataset(dataset):
     return False
 
 
+def is_grain_dataset(dataset):
+    if hasattr(dataset, "__class__"):
+        for parent in dataset.__class__.__mro__:
+            if parent.__name__ in (
+                "MapDataset",
+                "IterDataset",
+            ) and str(parent.__module__).startswith("grain._src.python"):
+                return True
+    return False
+
+
 def _rescale_dataset_split_sizes(left_size, right_size, total_length):
     """Rescale the dataset split sizes.
 
@@ -476,6 +489,10 @@ def _get_type_spec(dataset):
         from torch.utils.data import Dataset as TorchDataset
 
         return TorchDataset
+    elif is_grain_dataset(dataset):
+        from grain import MapDataset
+
+        return MapDataset
     else:
         return None
 
@@ -525,10 +542,17 @@ def index_directory(
         - class_names: names of the classes corresponding to these labels, in
         order.
     """
+    if file_utils.is_remote_path(directory):
+        os_module = tf.io.gfile
+        path_module = tf.io.gfile
+    else:
+        os_module = os
+        path_module = os.path
+
     if labels == "inferred":
         subdirs = []
-        for subdir in sorted(tf.io.gfile.listdir(directory)):
-            if tf.io.gfile.isdir(tf.io.gfile.join(directory, subdir)):
+        for subdir in sorted(os_module.listdir(directory)):
+            if path_module.isdir(path_module.join(directory, subdir)):
                 if not subdir.startswith("."):
                     if subdir.endswith("/"):
                         subdir = subdir[:-1]
@@ -566,7 +590,7 @@ def index_directory(
     results = []
     filenames = []
 
-    for dirpath in (tf.io.gfile.join(directory, subdir) for subdir in subdirs):
+    for dirpath in (path_module.join(directory, subdir) for subdir in subdirs):
         results.append(
             pool.apply_async(
                 index_subdirectory,
@@ -608,7 +632,7 @@ def index_directory(
             )
     pool.close()
     pool.join()
-    file_paths = [tf.io.gfile.join(directory, fname) for fname in filenames]
+    file_paths = [path_module.join(directory, fname) for fname in filenames]
 
     if shuffle:
         # Shuffle globally to erase macro-structure
@@ -623,8 +647,10 @@ def index_directory(
 
 
 def iter_valid_files(directory, follow_links, formats):
+    io_module = tf.io.gfile if file_utils.is_remote_path(directory) else os
+
     if not follow_links:
-        walk = tf.io.gfile.walk(directory)
+        walk = io_module.walk(directory)
     else:
         walk = os.walk(directory, followlinks=follow_links)
     for root, _, files in sorted(walk, key=lambda x: x[0]):
@@ -648,14 +674,18 @@ def index_subdirectory(directory, class_indices, follow_links, formats):
             paths, and `labels` is a list of integer labels corresponding
             to these files.
     """
+    path_module = (
+        tf.io.gfile if file_utils.is_remote_path(directory) else os.path
+    )
+
     dirname = os.path.basename(directory)
     valid_files = iter_valid_files(directory, follow_links, formats)
     labels = []
     filenames = []
     for root, fname in valid_files:
         labels.append(class_indices[dirname])
-        absolute_path = tf.io.gfile.join(root, fname)
-        relative_path = tf.io.gfile.join(
+        absolute_path = path_module.join(root, fname)
+        relative_path = path_module.join(
             dirname, os.path.relpath(absolute_path, directory)
         )
         filenames.append(relative_path)
@@ -700,7 +730,7 @@ def get_training_or_validation_split(samples, labels, validation_split, subset):
     return samples, labels
 
 
-def labels_to_dataset(labels, label_mode, num_classes):
+def labels_to_dataset_tf(labels, label_mode, num_classes):
     """Create a `tf.data.Dataset` from the list/tuple of labels.
 
     Args:
@@ -730,6 +760,51 @@ def labels_to_dataset(labels, label_mode, num_classes):
     return label_ds
 
 
+def labels_to_dataset_grain(labels, label_mode, num_classes):
+    """Create a `grain.MapDataset` from the list/tuple of labels.
+
+    Args:
+        labels: list/tuple of labels to be converted into a `grain.MapDataset`.
+        label_mode: String describing the encoding of `labels`. Options are:
+        - `"binary"` indicates that the labels (there can be only 2) are encoded
+            as `float32` scalars with values 0 or 1
+            (e.g. for `binary_crossentropy`).
+        - `"categorical"` means that the labels are mapped into a categorical
+            vector.  (e.g. for `categorical_crossentropy` loss).
+        num_classes: number of classes of labels.
+
+    Returns:
+        A `grain.MapDataset` instance.
+    """
+    from keras.src import backend
+    from keras.src import ops
+
+    if label_mode not in ("binary", "categorical", "int"):
+        raise ValueError(
+            f"Invalid `label_mode`: {label_mode}. "
+            "Expected one of: 'binary', 'categorical', 'int'."
+        )
+
+    def preprocess_labels_in_cpu(label_mode, x, num_classes):
+        with backend.device_scope("cpu"):
+            if label_mode == "binary":
+                return ops.expand_dims(
+                    ops.convert_to_tensor(x, dtype="float32"), axis=-1
+                )
+            elif label_mode == "categorical":
+                return ops.one_hot(
+                    ops.convert_to_tensor(x, dtype="int32"), num_classes
+                )
+            else:
+                return ops.convert_to_tensor(x, dtype="int32")
+
+    label_ds = grain.MapDataset.source(labels)
+    label_ds = label_ds.map(
+        lambda x: preprocess_labels_in_cpu(label_mode, x, num_classes),
+    )
+    return label_ds
+
+
 def check_validation_split_arg(validation_split, subset, shuffle, seed):
     """Raise errors in case of invalid argument values.
 

keras/src/utils/grain_utils.py

@@ -0,0 +1,33 @@
+from keras.src import backend
+from keras.src import tree
+
+
+def make_batch(values):
+    from keras.src import ops
+
+    if not values:
+        raise ValueError("Cannot batch 0 values. Please file a bug.")
+
+    with backend.device_scope("cpu"):
+        return tree.map_structure(lambda *xs: ops.stack(xs), *values)
+
+
+def make_string_batch(values):
+    from keras.src import ops
+
+    if not values:
+        raise ValueError("Cannot batch 0 values. Please file a bug.")
+
+    def batch_fn(*xs):
+        if isinstance(xs[0], str):
+            if backend.backend() == "tensorflow":
+                import tensorflow as tf
+
+                xs = [tf.convert_to_tensor(x, dtype=tf.string) for x in xs]
+                xs = tf.stack(xs)
+            return xs
+        else:
+            return ops.stack(xs)
+
+    with backend.device_scope("cpu"):
+        return tree.map_structure(batch_fn, *values)

keras/src/utils/image_dataset_utils.py

@@ -1,11 +1,27 @@
+import io
+import pathlib
+
 import numpy as np
 
 from keras.src.api_export import keras_export
 from keras.src.backend.config import standardize_data_format
 from keras.src.utils import dataset_utils
 from keras.src.utils import image_utils
+from keras.src.utils.grain_utils import make_batch
+from keras.src.utils.module_utils import grain
 from keras.src.utils.module_utils import tensorflow as tf
 
+try:
+    from PIL import Image as pil_image
+
+    try:
+        pil_image_resampling = pil_image.Resampling
+    except AttributeError:
+        pil_image_resampling = pil_image
+except ImportError:
+    pil_image = None
+    pil_image_resampling = None
+
 ALLOWLIST_FORMATS = (".bmp", ".gif", ".jpeg", ".jpg", ".png")
 
 
@@ -32,9 +48,10 @@ def image_dataset_from_directory(
     crop_to_aspect_ratio=False,
     pad_to_aspect_ratio=False,
     data_format=None,
+    format="tf",
     verbose=True,
 ):
-    """Generates a `tf.data.Dataset` from image files in a directory.
+    """Generates a dataset from image files in a directory.
 
     If your directory structure is:
 
@@ -49,13 +66,17 @@ def image_dataset_from_directory(
     ```
 
     Then calling `image_dataset_from_directory(main_directory,
-    labels='inferred')` will return a `tf.data.Dataset` that yields batches of
+    labels='inferred')` will return a dataset that yields batches of
     images from the subdirectories `class_a` and `class_b`, together with labels
     0 and 1 (0 corresponding to `class_a` and 1 corresponding to `class_b`).
 
     Supported image formats: `.jpeg`, `.jpg`, `.png`, `.bmp`, `.gif`.
     Animated gifs are truncated to the first frame.
 
+    By default, this function will return a `tf.data.Dataset` object. You can
+    set `format="grain"` to return a `grain.IterDataset` object instead, which
+    removes the TensorFlow dependency.
+
     Args:
         directory: Directory where the data is located.
             If `labels` is `"inferred"`, it should contain
@@ -125,12 +146,19 @@ def image_dataset_from_directory(
             preserved.
         data_format: If None uses keras.config.image_data_format()
             otherwise either 'channel_last' or 'channel_first'.
+        format: The format of the return object. Defaults to `"tf"`. Available
+            options are:
+            - `"tf"`: returns a `tf.data.Dataset` object. Requires
+                TensorFlow to be installed.
+            - `"grain"`: returns a `grain.IterDataset` object. Requires
+                Grain to be installed.
         verbose: Whether to display number information on classes and
             number of files found. Defaults to `True`.
 
     Returns:
 
-    A `tf.data.Dataset` object.
+    A `tf.data.Dataset` (`format="tf"`) or `grain.IterDataset`
+    (`format="grain"`) object.
 
     - If `label_mode` is `None`, it yields `float32` tensors of shape
         `(batch_size, image_size[0], image_size[1], num_channels)`,
@@ -222,6 +250,11 @@ def image_dataset_from_directory(
             f"{supported_interpolations}. "
             f"Received: interpolation={interpolation}"
         )
+    if format not in ("tf", "grain"):
+        raise ValueError(
+            '`format` should be either "tf" or "grain". '
+            f"Received: format={format}"
+        )
 
     dataset_utils.check_validation_split_arg(
         validation_split, subset, shuffle, seed
@@ -289,6 +322,7 @@ def image_dataset_from_directory(
             shuffle=shuffle,
             shuffle_buffer_size=shuffle_buffer_size,
             seed=seed,
+            format=format,
         )
 
         val_dataset = paths_and_labels_to_dataset(
@@ -303,14 +337,23 @@ def image_dataset_from_directory(
             pad_to_aspect_ratio=pad_to_aspect_ratio,
             data_format=data_format,
             shuffle=False,
+            format=format,
         )
 
-        if batch_size is not None:
-            train_dataset = train_dataset.batch(batch_size)
-            val_dataset = val_dataset.batch(batch_size)
-
-        train_dataset = train_dataset.prefetch(tf.data.AUTOTUNE)
-        val_dataset = val_dataset.prefetch(tf.data.AUTOTUNE)
+        if format == "tf":
+            if batch_size is not None:
+                train_dataset = train_dataset.batch(batch_size)
+                val_dataset = val_dataset.batch(batch_size)
+            train_dataset = train_dataset.prefetch(tf.data.AUTOTUNE)
+            val_dataset = val_dataset.prefetch(tf.data.AUTOTUNE)
+        else:
+            train_dataset = train_dataset.to_iter_dataset()
+            val_dataset = val_dataset.to_iter_dataset()
+            if batch_size is not None:
+                train_dataset = train_dataset.batch(
+                    batch_size, batch_fn=make_batch
+                )
+                val_dataset = val_dataset.batch(batch_size, batch_fn=make_batch)
 
         # Users may need to reference `class_names`.
         train_dataset.class_names = class_names
@@ -345,12 +388,18 @@ def image_dataset_from_directory(
             shuffle=shuffle,
             shuffle_buffer_size=shuffle_buffer_size,
             seed=seed,
+            format=format,
         )
 
-        if batch_size is not None:
-            dataset = dataset.batch(batch_size)
+        if format == "tf":
+            if batch_size is not None:
+                dataset = dataset.batch(batch_size)
+            dataset = dataset.prefetch(tf.data.AUTOTUNE)
+        else:
+            dataset = dataset.to_iter_dataset()
+            if batch_size is not None:
+                dataset = dataset.batch(batch_size, batch_fn=make_batch)
 
-        dataset = dataset.prefetch(tf.data.AUTOTUNE)
         # Users may need to reference `class_names`.
         dataset.class_names = class_names
 
@@ -374,11 +423,66 @@ def paths_and_labels_to_dataset(
     shuffle=False,
     shuffle_buffer_size=None,
     seed=None,
+    format="tf",
+):
+    """Constructs a dataset of images and labels."""
+    if format == "tf":
+        return _paths_and_labels_to_dataset_tf(
+            image_paths=image_paths,
+            image_size=image_size,
+            num_channels=num_channels,
+            labels=labels,
+            label_mode=label_mode,
+            num_classes=num_classes,
+            interpolation=interpolation,
+            data_format=data_format,
+            crop_to_aspect_ratio=crop_to_aspect_ratio,
+            pad_to_aspect_ratio=pad_to_aspect_ratio,
+            shuffle=shuffle,
+            shuffle_buffer_size=shuffle_buffer_size,
+            seed=seed,
+        )
+    elif format == "grain":
+        return _paths_and_labels_to_dataset_grain(
+            image_paths=image_paths,
+            image_size=image_size,
+            num_channels=num_channels,
+            labels=labels,
+            label_mode=label_mode,
+            num_classes=num_classes,
+            interpolation=interpolation,
+            data_format=data_format,
+            crop_to_aspect_ratio=crop_to_aspect_ratio,
+            pad_to_aspect_ratio=pad_to_aspect_ratio,
+            shuffle=shuffle,
+            seed=seed,
+        )
+    else:
+        raise ValueError(
+            '`format` should be either "tf" or "grain". '
+            f"Received: format={format}"
+        )
+
+
+def _paths_and_labels_to_dataset_tf(
+    image_paths,
+    image_size,
+    num_channels,
+    labels,
+    label_mode,
+    num_classes,
+    interpolation,
+    data_format,
+    crop_to_aspect_ratio=False,
+    pad_to_aspect_ratio=False,
+    shuffle=False,
+    shuffle_buffer_size=None,
+    seed=None,
 ):
     """Constructs a dataset of images and labels."""
     path_ds = tf.data.Dataset.from_tensor_slices(image_paths)
     if label_mode:
-        label_ds = dataset_utils.labels_to_dataset(
+        label_ds = dataset_utils.labels_to_dataset_tf(
             labels, label_mode, num_classes
         )
         ds = tf.data.Dataset.zip((path_ds, label_ds))
@@ -398,17 +502,18 @@ def paths_and_labels_to_dataset(
     )
     if label_mode:
         ds = ds.map(
-            lambda x, y: (load_image(x, *args), y),
+            lambda x, y: (_load_image_tf(x, *args), y),
             num_parallel_calls=tf.data.AUTOTUNE,
         )
     else:
         ds = ds.map(
-            lambda x: load_image(x, *args), num_parallel_calls=tf.data.AUTOTUNE
+            lambda x: _load_image_tf(x, *args),
+            num_parallel_calls=tf.data.AUTOTUNE,
         )
     return ds
 
 
-def load_image(
+def _load_image_tf(
     path,
     image_size,
     num_channels,
@@ -457,3 +562,120 @@ def load_image(
     else:
         img.set_shape((num_channels, image_size[0], image_size[1]))
     return img
+
+
+def _paths_and_labels_to_dataset_grain(
+    image_paths,
+    image_size,
+    num_channels,
+    labels,
+    label_mode,
+    num_classes,
+    interpolation,
+    data_format,
+    crop_to_aspect_ratio=False,
+    pad_to_aspect_ratio=False,
+    shuffle=False,
+    seed=None,
+):
+    """Constructs a dataset of images and labels."""
+    path_ds = grain.MapDataset.source(image_paths)
+    if label_mode:
+        label_ds = dataset_utils.labels_to_dataset_grain(
+            labels, label_mode, num_classes
+        )
+        ds = grain.experimental.ZipMapDataset([path_ds, label_ds])
+    else:
+        ds = path_ds
+
+    if shuffle:
+        ds = ds.shuffle(seed=seed)
+
+    args = (
+        image_size,
+        num_channels,
+        interpolation,
+        data_format,
+        crop_to_aspect_ratio,
+        pad_to_aspect_ratio,
+    )
+    if label_mode:
+        ds = ds.map(lambda data: (_load_image_grain(data[0], *args), data[1]))
+    else:
+        ds = ds.map(lambda x: _load_image_grain(x, *args))
+
+    return ds
+
+
+def _load_image_grain(
+    path,
+    image_size,
+    num_channels,
+    interpolation,
+    data_format,
+    crop_to_aspect_ratio=False,
+    pad_to_aspect_ratio=False,
+):
+    """Load an image from a path and resize it."""
+    from keras.src import backend
+    from keras.src import ops
+
+    if pil_image is None:
+        raise ImportError(
+            "Could not import PIL.Image. The use of `load_img` requires PIL."
+        )
+    if pad_to_aspect_ratio and crop_to_aspect_ratio:
+        raise ValueError(
+            "Only one of `pad_to_aspect_ratio`, `crop_to_aspect_ratio`"
+            " can be set to `True`."
+        )
+
+    if isinstance(path, io.BytesIO):
+        img = pil_image.open(path)
+    elif isinstance(path, (pathlib.Path, bytes, str)):
+        if isinstance(path, pathlib.Path):
+            path = str(path.resolve())
+        img = pil_image.open(path)
+    else:
+        raise TypeError(
+            f"path should be path-like or io.BytesIO, not {type(path)}"
+        )
+    if num_channels == 1:
+        # if image is not already an 8-bit, 16-bit or 32-bit grayscale image
+        # convert it to an 8-bit grayscale image.
+        if img.mode not in ("L", "I;16", "I"):
+            img = img.convert("L")
+    elif num_channels == 4:
+        if img.mode != "RGBA":
+            img = img.convert("RGBA")
+    elif num_channels == 3:
+        if img.mode != "RGB":
+            img = img.convert("RGB")
+    else:
+        raise ValueError(
+            "num_channels must be 1, 3 or 4. "
+            f"Received: num_channels={num_channels}"
+        )
+
+    with backend.device_scope("cpu"):
+        img = ops.convert_to_tensor(np.array(img), dtype="float32")
+        if len(img.shape) == 2:
+            # If the image is grayscale, expand dims to add channel axis.
+            # The reason is that `ops.image.resize` expects 3D or 4D tensors.
+            img = ops.expand_dims(img, axis=-1)
+        if data_format == "channels_first":
+            img = ops.transpose(img, (2, 0, 1))
+        img = ops.image.resize(
+            img,
+            size=image_size,
+            interpolation=interpolation,
+            crop_to_aspect_ratio=crop_to_aspect_ratio,
+            pad_to_aspect_ratio=pad_to_aspect_ratio,
+            data_format=data_format,
+        )
+        if backend.backend() == "tensorflow":
+            if data_format == "channels_last":
+                img.set_shape((image_size[0], image_size[1], num_channels))
+            else:
+                img.set_shape((num_channels, image_size[0], image_size[1]))
+    return img

keras/src/utils/module_utils.py

@@ -58,3 +58,4 @@ torch_xla = LazyModule(
 optree = LazyModule("optree")
 dmtree = LazyModule("tree")
 tf2onnx = LazyModule("tf2onnx")
+grain = LazyModule("grain")

keras/src/utils/text_dataset_utils.py

@@ -2,6 +2,8 @@ import numpy as np
 
 from keras.src.api_export import keras_export
 from keras.src.utils import dataset_utils
+from keras.src.utils.grain_utils import make_string_batch
+from keras.src.utils.module_utils import grain
 from keras.src.utils.module_utils import tensorflow as tf
 
 
@@ -23,9 +25,10 @@ def text_dataset_from_directory(
     validation_split=None,
     subset=None,
     follow_links=False,
+    format="tf",
     verbose=True,
 ):
-    """Generates a `tf.data.Dataset` from text files in a directory.
+    """Generates a dataset from text files in a directory.
 
     If your directory structure is:
 
@@ -40,12 +43,16 @@ def text_dataset_from_directory(
     ```
 
     Then calling `text_dataset_from_directory(main_directory,
-    labels='inferred')` will return a `tf.data.Dataset` that yields batches of
+    labels='inferred')` will return a dataset that yields batches of
     texts from the subdirectories `class_a` and `class_b`, together with labels
     0 and 1 (0 corresponding to `class_a` and 1 corresponding to `class_b`).
 
     Only `.txt` files are supported at this time.
 
+    By default, this function will return a `tf.data.Dataset` object. You can
+    set `format="grain"` to return a `grain.IterDataset` object instead, which
+    removes the TensorFlow dependency.
+
     Args:
         directory: Directory where the data is located.
             If `labels` is `"inferred"`, it should contain
@@ -91,19 +98,34 @@ def text_dataset_from_directory(
             (the training and validation datasets respectively).
         follow_links: Whether to visits subdirectories pointed to by symlinks.
             Defaults to `False`.
+        format: The format of the return object. Defaults to `"tf"`. Available
+            options are:
+            - `"tf"`: returns a `tf.data.Dataset` object. Requires
+                TensorFlow to be installed.
+            - `"grain"`: returns a `grain.IterDataset` object. Requires
+                Grain to be installed.
         verbose: Whether to display number information on classes and
             number of files found. Defaults to `True`.
 
     Returns:
 
-    A `tf.data.Dataset` object.
+    A `tf.data.Dataset` (`format="tf"`) or `grain.IterDataset`
+    (`format="grain"`) object.
 
+    When `format="tf"`:
     - If `label_mode` is `None`, it yields `string` tensors of shape
         `(batch_size,)`, containing the contents of a batch of text files.
     - Otherwise, it yields a tuple `(texts, labels)`, where `texts`
         has shape `(batch_size,)` and `labels` follows the format described
         below.
 
+    When `format="grain"`:
+    - If `label_mode` is `None`, it yields a list of Python strings containing
+        the contents of a batch of text files.
+    - Otherwise, it yields a tuple `(texts, labels)`, where `texts`
+        is a list of Python strings and `labels` follows the format described
+        below.
+
     Rules regarding labels format:
 
     - if `label_mode` is `int`, the labels are an `int32` tensor of shape
@@ -137,6 +159,11 @@ def text_dataset_from_directory(
             '"categorical", "binary", '
             f"or None. Received: label_mode={label_mode}"
         )
+    if format not in ("tf", "grain"):
+        raise ValueError(
+            '`format` should be either "tf" or "grain". '
+            f"Received: format={format}"
+        )
     if labels is None or label_mode is None:
         labels = None
         label_mode = None
@@ -199,6 +226,7 @@ def text_dataset_from_directory(
             shuffle=shuffle,
             shuffle_buffer_size=shuffle_buffer_size,
             seed=seed,
+            format=format,
         )
         val_dataset = paths_and_labels_to_dataset(
             file_paths=file_paths_val,
@@ -207,14 +235,25 @@ def text_dataset_from_directory(
             num_classes=len(class_names) if class_names else 0,
             max_length=max_length,
             shuffle=False,
+            format=format,
         )
 
-        if batch_size is not None:
-            train_dataset = train_dataset.batch(batch_size)
-            val_dataset = val_dataset.batch(batch_size)
-
-        train_dataset = train_dataset.prefetch(tf.data.AUTOTUNE)
-        val_dataset = val_dataset.prefetch(tf.data.AUTOTUNE)
+        if format == "tf":
+            if batch_size is not None:
+                train_dataset = train_dataset.batch(batch_size)
+                val_dataset = val_dataset.batch(batch_size)
+            train_dataset = train_dataset.prefetch(tf.data.AUTOTUNE)
+            val_dataset = val_dataset.prefetch(tf.data.AUTOTUNE)
+        else:
+            train_dataset = train_dataset.to_iter_dataset()
+            val_dataset = val_dataset.to_iter_dataset()
+            if batch_size is not None:
+                train_dataset = train_dataset.batch(
+                    batch_size, batch_fn=make_string_batch
+                )
+                val_dataset = val_dataset.batch(
+                    batch_size, batch_fn=make_string_batch
+                )
 
         # Users may need to reference `class_names`.
         train_dataset.class_names = class_names
@@ -238,10 +277,17 @@ def text_dataset_from_directory(
             shuffle=shuffle,
             shuffle_buffer_size=shuffle_buffer_size,
             seed=seed,
+            format=format,
         )
-        if batch_size is not None:
-            dataset = dataset.batch(batch_size)
-        dataset = dataset.prefetch(tf.data.AUTOTUNE)
+
+        if format == "tf":
+            if batch_size is not None:
+                dataset = dataset.batch(batch_size)
+            dataset = dataset.prefetch(tf.data.AUTOTUNE)
+        else:
+            dataset = dataset.to_iter_dataset()
+            if batch_size is not None:
+                dataset = dataset.batch(batch_size, batch_fn=make_string_batch)
 
         # Users may need to reference `class_names`.
         dataset.class_names = class_names
@@ -257,11 +303,47 @@ def paths_and_labels_to_dataset(
     shuffle=False,
     shuffle_buffer_size=None,
     seed=None,
+    format="tf",
+):
+    """Constructs a dataset of text strings and labels."""
+    if format == "tf":
+        return _paths_and_labels_to_dataset_tf(
+            file_paths,
+            labels,
+            label_mode,
+            num_classes,
+            max_length,
+            shuffle,
+            shuffle_buffer_size,
+            seed,
+        )
+    elif format == "grain":
+        return _paths_and_labels_to_dataset_grain(
+            file_paths,
+            labels,
+            label_mode,
+            num_classes,
+            max_length,
+            shuffle,
+            shuffle_buffer_size,
+            seed,
+        )
+
+
+def _paths_and_labels_to_dataset_tf(
+    file_paths,
+    labels,
+    label_mode,
+    num_classes,
+    max_length,
+    shuffle=False,
+    shuffle_buffer_size=None,
+    seed=None,
 ):
     """Constructs a dataset of text strings and labels."""
     path_ds = tf.data.Dataset.from_tensor_slices(file_paths)
     if label_mode:
-        label_ds = dataset_utils.labels_to_dataset(
+        label_ds = dataset_utils.labels_to_dataset_tf(
             labels, label_mode, num_classes
         )
         ds = tf.data.Dataset.zip((path_ds, label_ds))
@@ -273,19 +355,62 @@ def paths_and_labels_to_dataset(
 
     if label_mode:
         ds = ds.map(
-            lambda x, y: (path_to_string_content(x, max_length), y),
+            lambda x, y: (_path_to_string_content_tf(x, max_length), y),
             num_parallel_calls=tf.data.AUTOTUNE,
         )
     else:
         ds = ds.map(
-            lambda x: path_to_string_content(x, max_length),
+            lambda x: _path_to_string_content_tf(x, max_length),
             num_parallel_calls=tf.data.AUTOTUNE,
         )
     return ds
 
 
-def path_to_string_content(path, max_length):
+def _path_to_string_content_tf(path, max_length):
     txt = tf.io.read_file(path)
     if max_length is not None:
         txt = tf.strings.substr(txt, 0, max_length)
     return txt
+
+
+def _paths_and_labels_to_dataset_grain(
+    file_paths,
+    labels,
+    label_mode,
+    num_classes,
+    max_length,
+    shuffle=False,
+    shuffle_buffer_size=None,
+    seed=None,
+):
+    """Constructs a dataset of text strings and labels."""
+    path_ds = grain.MapDataset.source(file_paths)
+    if label_mode:
+        label_ds = dataset_utils.labels_to_dataset_grain(
+            labels, label_mode, num_classes
+        )
+        ds = grain.experimental.ZipMapDataset([path_ds, label_ds])
+    else:
+        ds = path_ds
+
+    if shuffle:
+        ds = ds.shuffle(seed=seed)
+
+    if label_mode:
+        ds = ds.map(
+            lambda data: (
+                _path_to_string_content_grain(data[0], max_length),
+                data[1],
+            ),
+        )
+    else:
+        ds = ds.map(lambda x: _path_to_string_content_grain(x, max_length))
+    return ds
+
+
+def _path_to_string_content_grain(path, max_length):
+    with open(path, "r") as f:
+        txt = f.read()
+    if max_length is not None:
+        txt = txt[:max_length]
+    return txt

keras/src/version.py

@@ -1,7 +1,7 @@
 from keras.src.api_export import keras_export
 
 # Unique source of truth for the version number.
-__version__ = "3.12.0.dev2025081903"
+__version__ = "3.12.0.dev2025082003"
 
 
 @keras_export("keras.version")

{keras_nightly-3.12.0.dev2025081903.dist-info → keras_nightly-3.12.0.dev2025082003.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: keras-nightly
-Version: 3.12.0.dev2025081903
+Version: 3.12.0.dev2025082003
 Summary: Multi-backend Keras
 Author-email: Keras team <keras-users@googlegroups.com>
 License: Apache License 2.0

{keras_nightly-3.12.0.dev2025081903.dist-info → keras_nightly-3.12.0.dev2025082003.dist-info}/RECORD

@@ -126,7 +126,7 @@ keras/regularizers/__init__.py,sha256=542Shphw7W8h4Dyf2rmqMKUECVZ8IVBvN9g1LWhz-b
 keras/saving/__init__.py,sha256=KvL2GZxjvgFgEhvEnkvqjIR9JSNHKz-NWZacXajsjLI,1298
 keras/src/__init__.py,sha256=Gi4S7EiCMkE03PbdGNpFdaUYySWDs_FcAJ8Taz9Y1BE,684
 keras/src/api_export.py,sha256=gXOkBOnmscV013WAc75lc4Up01-Kkg9EylIAT_QWctg,1173
-keras/src/version.py,sha256=jIhkFIrF_efXcsntiwlP3rpnKtudZLu6Cq5H26XJLBA,204
+keras/src/version.py,sha256=njY7f4Il2CTFP8VpP-i-fu_W9TrUOijOX1eeYI-KMMA,204
 keras/src/activations/__init__.py,sha256=0nL3IFDB9unlrMz8ninKOWo-uCHasTUpTo1tXZb2u44,4433
 keras/src/activations/activations.py,sha256=mogPggtp4CGldI3VOPNmesRxp6EbiR1_i4KLGaVwzL8,17614
 keras/src/applications/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -549,20 +549,21 @@ keras/src/tree/optree_impl.py,sha256=ZLEbRqE6u76miYgiRQc3eRc82_-xMxIZd3wBNQHeJds
 keras/src/tree/tree_api.py,sha256=cSOp6EMOe8p0DUIbbvELrzIjABTIYX0Fw7CBfqi8pcY,14093
 keras/src/utils/__init__.py,sha256=WSmTldk6M-XV0X84XR5vryg0BTR8KsTfxNIyRaNkqq0,1423
 keras/src/utils/argument_validation.py,sha256=uRFoLNJu3L2J8CM8L7uXGqhYi7ji8whh0H8nSHuRUXg,2876
-keras/src/utils/audio_dataset_utils.py,sha256=pxg3jOHgZMFhEkuJmCjI-dcrFyv7OlHyWW-49eedKN0,15114
+keras/src/utils/audio_dataset_utils.py,sha256=oxZuYM2LEKtCIyJLYr3ifqlA-Gmr4uzQynPOm6Eh3sU,15117
 keras/src/utils/backend_utils.py,sha256=U3oXPiuuWBjXs0uFBLf5vpyBYE44Mja-5A1YFxRuzlM,5278
 keras/src/utils/code_stats.py,sha256=1h4ifpAH5Jezm8BVrKM_WyzcG9uxrUiyzP1kcS4uqlo,1442
 keras/src/utils/config.py,sha256=3VhENVcng0DeazR-5rvjSnW_sovvOw-skEP-t3xWCEY,4643
-keras/src/utils/dataset_utils.py,sha256=k-g4akwOZyXHIq2DNlN8DwWCYSMYWZIYMdQJ3PwZ1IU,28194
+keras/src/utils/dataset_utils.py,sha256=JQJvGMrt69hwJO3eP-FR9oTw7HvCdBoZb0sx_kdEJws,30775
 keras/src/utils/dtype_utils.py,sha256=wL_WaWYoDzDDmQW6EQGdpBb9O5QJ9OaEJsvY0Mir4uc,1483
 keras/src/utils/file_utils.py,sha256=kylM3-5YZl9kwkYZIfBR0FIwFPRJjZnNbmhmx2RGNrY,17368
-keras/src/utils/image_dataset_utils.py,sha256=doL8q0q4DciFnlO-IyKN1v2Emh_gP4sI2rDhgeKL5qs,16964
+keras/src/utils/grain_utils.py,sha256=Wfwv12E3UrNZjJjTEk2JVV6_YEUav35UJ6bV1UAPEIk,886
+keras/src/utils/image_dataset_utils.py,sha256=0lOzD1CiXwZOe1wW-5uvFKuIgot9PWUC9KJJA0NVuP8,24017
 keras/src/utils/image_utils.py,sha256=HUI7Zcgqvsmm8a1xwfMwr7pOhnG4lsChP8Owv-xlCTM,16703
 keras/src/utils/io_utils.py,sha256=2u6b1SEQmkxo4IRPkKBedAkKuRIQMF4CdD-B5ko0Cek,4432
 keras/src/utils/jax_layer.py,sha256=h4MVRADUAL8t8pHaTKPYt81rxUuW6BfE4MnmQp5ETM0,27189
 keras/src/utils/jax_utils.py,sha256=vY3P4S9mfWEjdirLd81ocKqeCm-UVfgQ1yTi6UHdBiM,322
 keras/src/utils/model_visualization.py,sha256=I2NWeo-g0NpCM88HXMDyAbpvF_KIZuJr5hfOLGm922U,17799
-keras/src/utils/module_utils.py,sha256=ej6YM6I9dTapJvuhZsA7lvp1wxbKzbzI8T8abjLuxLA,1948
+keras/src/utils/module_utils.py,sha256=j3YV_MkoRffRXtJSEjbbrZ3JVdau657JEb4eNKVrmHg,1976
 keras/src/utils/naming.py,sha256=bPowKBlgiVP_6XtVlNVHxrxheKuJy2c0e-oEM8ocZQY,1776
 keras/src/utils/numerical_utils.py,sha256=Uqe5nu1HXmiZuh5-MznomtDSVSO9FgFaltdDtGnN61o,7205
 keras/src/utils/progbar.py,sha256=HWvgFustRG5WqsiIayaaSiUU2jOYkioEqbQdywmBm0c,10469
@@ -570,7 +571,7 @@ keras/src/utils/python_utils.py,sha256=j8d1oA6oEnU5J0xosWU3t9wIGiblj67OStEn7KJ7j
 keras/src/utils/rng_utils.py,sha256=NVk7Aavt8A1945YpBIGW18OPebo29g2qHgWZvkgRkW8,2168
 keras/src/utils/sequence_utils.py,sha256=CveyJ5VM5KJ4pFlo6LWT9omzd_xDeMRjTgczIKekP3Y,4716
 keras/src/utils/summary_utils.py,sha256=jjbTB6NTqMniSWXPKeNY6dvpn-U37WJdwqdfl8uX5nI,15447
-keras/src/utils/text_dataset_utils.py,sha256=JUqDauTec6uRZs71SbKeVjxHx_CNqqOWkoXQ1Q7ldRs,10701
+keras/src/utils/text_dataset_utils.py,sha256=6ACAHwEhjjd5rjfzwLl7Es2qkvmSBUWs5IYQLGrHFrQ,14543
 keras/src/utils/tf_utils.py,sha256=FTunWC5kdyjsK0TyxQxiHGaYNaAyUxhMX52Zee_Rz9c,4930
 keras/src/utils/timeseries_dataset_utils.py,sha256=rVxSuqlYLpzw_dVo8Ym5HSE2jFmndS8MAv4Uewycojo,9842
 keras/src/utils/torch_utils.py,sha256=y8vRSx_R9MtC1IpRqyK6LBRETxwrlUFnsu45R4_-7Hs,6638
@@ -592,7 +593,7 @@ keras/utils/bounding_boxes/__init__.py,sha256=jtvQll4u8ZY0Z96HwNhP1nxWEG9FM3gI-6
 keras/utils/legacy/__init__.py,sha256=oSYZz6uS8UxSElRaaJYWJEoweJ4GAasZjnn7fNaOlog,342
 keras/visualization/__init__.py,sha256=UKWmiy6sps4SWlmQi9WX8_Z53cPpLlphz2zIeHdwJpQ,722
 keras/wrappers/__init__.py,sha256=QkS-O5K8qGS7C3sytF8MpmO6PasATpNVGF8qtb7Ojsw,407
-keras_nightly-3.12.0.dev2025081903.dist-info/METADATA,sha256=wXQRJnT0-m8WG318qVx3MY6zCsQHLF2RlmBwjrcMcPQ,5970
-keras_nightly-3.12.0.dev2025081903.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-keras_nightly-3.12.0.dev2025081903.dist-info/top_level.txt,sha256=ptcw_-QuGZ4ZDjMdwi_Z0clZm8QAqFdvzzFnDEOTs9o,6
-keras_nightly-3.12.0.dev2025081903.dist-info/RECORD,,
+keras_nightly-3.12.0.dev2025082003.dist-info/METADATA,sha256=LZyWd4xCofCT4FX7BEdbxKxsT4K1qqybGB16Rj2U_II,5970
+keras_nightly-3.12.0.dev2025082003.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+keras_nightly-3.12.0.dev2025082003.dist-info/top_level.txt,sha256=ptcw_-QuGZ4ZDjMdwi_Z0clZm8QAqFdvzzFnDEOTs9o,6
+keras_nightly-3.12.0.dev2025082003.dist-info/RECORD,,