keras-nightly 3.12.0.dev2025081903__tar.gz → 3.12.0.dev2025082003__tar.gz

{keras_nightly-3.12.0.dev2025081903 → keras_nightly-3.12.0.dev2025082003}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: keras-nightly
-Version: 3.12.0.dev2025081903
+Version: 3.12.0.dev2025082003
 Summary: Multi-backend Keras
 Author-email: Keras team <keras-users@googlegroups.com>
 License: Apache License 2.0

{keras_nightly-3.12.0.dev2025081903 → keras_nightly-3.12.0.dev2025082003}/keras/src/utils/audio_dataset_utils.py

@@ -411,7 +411,7 @@ def paths_and_labels_to_dataset(
     """Constructs a fixed-size dataset of audio and labels."""
     path_ds = tf.data.Dataset.from_tensor_slices(file_paths)
     if label_mode:
-        label_ds = dataset_utils.labels_to_dataset(
+        label_ds = dataset_utils.labels_to_dataset_tf(
             labels, label_mode, num_classes
         )
         ds = tf.data.Dataset.zip((path_ds, label_ds))

{keras_nightly-3.12.0.dev2025081903 → keras_nightly-3.12.0.dev2025082003}/keras/src/utils/dataset_utils.py

@@ -8,7 +8,9 @@ import numpy as np
 
 from keras.src import tree
 from keras.src.api_export import keras_export
+from keras.src.utils import file_utils
 from keras.src.utils import io_utils
+from keras.src.utils.module_utils import grain
 from keras.src.utils.module_utils import tensorflow as tf
 
 
@@ -299,6 +301,17 @@ def is_torch_dataset(dataset):
     return False
 
 
+def is_grain_dataset(dataset):
+    if hasattr(dataset, "__class__"):
+        for parent in dataset.__class__.__mro__:
+            if parent.__name__ in (
+                "MapDataset",
+                "IterDataset",
+            ) and str(parent.__module__).startswith("grain._src.python"):
+                return True
+    return False
+
+
 def _rescale_dataset_split_sizes(left_size, right_size, total_length):
     """Rescale the dataset split sizes.
 
@@ -476,6 +489,10 @@ def _get_type_spec(dataset):
         from torch.utils.data import Dataset as TorchDataset
 
         return TorchDataset
+    elif is_grain_dataset(dataset):
+        from grain import MapDataset
+
+        return MapDataset
     else:
         return None
 
@@ -525,10 +542,17 @@ def index_directory(
         - class_names: names of the classes corresponding to these labels, in
         order.
     """
+    if file_utils.is_remote_path(directory):
+        os_module = tf.io.gfile
+        path_module = tf.io.gfile
+    else:
+        os_module = os
+        path_module = os.path
+
     if labels == "inferred":
         subdirs = []
-        for subdir in sorted(tf.io.gfile.listdir(directory)):
-            if tf.io.gfile.isdir(tf.io.gfile.join(directory, subdir)):
+        for subdir in sorted(os_module.listdir(directory)):
+            if path_module.isdir(path_module.join(directory, subdir)):
                 if not subdir.startswith("."):
                     if subdir.endswith("/"):
                         subdir = subdir[:-1]
@@ -566,7 +590,7 @@ def index_directory(
     results = []
     filenames = []
 
-    for dirpath in (tf.io.gfile.join(directory, subdir) for subdir in subdirs):
+    for dirpath in (path_module.join(directory, subdir) for subdir in subdirs):
         results.append(
             pool.apply_async(
                 index_subdirectory,
@@ -608,7 +632,7 @@ def index_directory(
             )
     pool.close()
     pool.join()
-    file_paths = [tf.io.gfile.join(directory, fname) for fname in filenames]
+    file_paths = [path_module.join(directory, fname) for fname in filenames]
 
     if shuffle:
         # Shuffle globally to erase macro-structure
@@ -623,8 +647,10 @@ def index_directory(
 
 
 def iter_valid_files(directory, follow_links, formats):
+    io_module = tf.io.gfile if file_utils.is_remote_path(directory) else os
+
     if not follow_links:
-        walk = tf.io.gfile.walk(directory)
+        walk = io_module.walk(directory)
     else:
         walk = os.walk(directory, followlinks=follow_links)
     for root, _, files in sorted(walk, key=lambda x: x[0]):
@@ -648,14 +674,18 @@ def index_subdirectory(directory, class_indices, follow_links, formats):
             paths, and `labels` is a list of integer labels corresponding
             to these files.
     """
+    path_module = (
+        tf.io.gfile if file_utils.is_remote_path(directory) else os.path
+    )
+
     dirname = os.path.basename(directory)
     valid_files = iter_valid_files(directory, follow_links, formats)
     labels = []
     filenames = []
     for root, fname in valid_files:
         labels.append(class_indices[dirname])
-        absolute_path = tf.io.gfile.join(root, fname)
-        relative_path = tf.io.gfile.join(
+        absolute_path = path_module.join(root, fname)
+        relative_path = path_module.join(
             dirname, os.path.relpath(absolute_path, directory)
         )
         filenames.append(relative_path)
@@ -700,7 +730,7 @@ def get_training_or_validation_split(samples, labels, validation_split, subset):
     return samples, labels
 
 
-def labels_to_dataset(labels, label_mode, num_classes):
+def labels_to_dataset_tf(labels, label_mode, num_classes):
     """Create a `tf.data.Dataset` from the list/tuple of labels.
 
     Args:
@@ -730,6 +760,51 @@ def labels_to_dataset(labels, label_mode, num_classes):
     return label_ds
 
 
+def labels_to_dataset_grain(labels, label_mode, num_classes):
+    """Create a `grain.MapDataset` from the list/tuple of labels.
+
+    Args:
+        labels: list/tuple of labels to be converted into a `grain.MapDataset`.
+        label_mode: String describing the encoding of `labels`. Options are:
+        - `"binary"` indicates that the labels (there can be only 2) are encoded
+            as `float32` scalars with values 0 or 1
+            (e.g. for `binary_crossentropy`).
+        - `"categorical"` means that the labels are mapped into a categorical
+            vector.  (e.g. for `categorical_crossentropy` loss).
+        num_classes: number of classes of labels.
+
+    Returns:
+        A `grain.MapDataset` instance.
+    """
+    from keras.src import backend
+    from keras.src import ops
+
+    if label_mode not in ("binary", "categorical", "int"):
+        raise ValueError(
+            f"Invalid `label_mode`: {label_mode}. "
+            "Expected one of: 'binary', 'categorical', 'int'."
+        )
+
+    def preprocess_labels_in_cpu(label_mode, x, num_classes):
+        with backend.device_scope("cpu"):
+            if label_mode == "binary":
+                return ops.expand_dims(
+                    ops.convert_to_tensor(x, dtype="float32"), axis=-1
+                )
+            elif label_mode == "categorical":
+                return ops.one_hot(
+                    ops.convert_to_tensor(x, dtype="int32"), num_classes
+                )
+            else:
+                return ops.convert_to_tensor(x, dtype="int32")
+
+    label_ds = grain.MapDataset.source(labels)
+    label_ds = label_ds.map(
+        lambda x: preprocess_labels_in_cpu(label_mode, x, num_classes),
+    )
+    return label_ds
+
+
 def check_validation_split_arg(validation_split, subset, shuffle, seed):
     """Raise errors in case of invalid argument values.
 

keras_nightly-3.12.0.dev2025082003/keras/src/utils/grain_utils.py

@@ -0,0 +1,33 @@
+from keras.src import backend
+from keras.src import tree
+
+
+def make_batch(values):
+    from keras.src import ops
+
+    if not values:
+        raise ValueError("Cannot batch 0 values. Please file a bug.")
+
+    with backend.device_scope("cpu"):
+        return tree.map_structure(lambda *xs: ops.stack(xs), *values)
+
+
+def make_string_batch(values):
+    from keras.src import ops
+
+    if not values:
+        raise ValueError("Cannot batch 0 values. Please file a bug.")
+
+    def batch_fn(*xs):
+        if isinstance(xs[0], str):
+            if backend.backend() == "tensorflow":
+                import tensorflow as tf
+
+                xs = [tf.convert_to_tensor(x, dtype=tf.string) for x in xs]
+                xs = tf.stack(xs)
+            return xs
+        else:
+            return ops.stack(xs)
+
+    with backend.device_scope("cpu"):
+        return tree.map_structure(batch_fn, *values)

{keras_nightly-3.12.0.dev2025081903 → keras_nightly-3.12.0.dev2025082003}/keras/src/utils/image_dataset_utils.py

@@ -1,11 +1,27 @@
+import io
+import pathlib
+
 import numpy as np
 
 from keras.src.api_export import keras_export
 from keras.src.backend.config import standardize_data_format
 from keras.src.utils import dataset_utils
 from keras.src.utils import image_utils
+from keras.src.utils.grain_utils import make_batch
+from keras.src.utils.module_utils import grain
 from keras.src.utils.module_utils import tensorflow as tf
 
+try:
+    from PIL import Image as pil_image
+
+    try:
+        pil_image_resampling = pil_image.Resampling
+    except AttributeError:
+        pil_image_resampling = pil_image
+except ImportError:
+    pil_image = None
+    pil_image_resampling = None
+
 ALLOWLIST_FORMATS = (".bmp", ".gif", ".jpeg", ".jpg", ".png")
 
 
@@ -32,9 +48,10 @@ def image_dataset_from_directory(
     crop_to_aspect_ratio=False,
     pad_to_aspect_ratio=False,
     data_format=None,
+    format="tf",
     verbose=True,
 ):
-    """Generates a `tf.data.Dataset` from image files in a directory.
+    """Generates a dataset from image files in a directory.
 
     If your directory structure is:
 
@@ -49,13 +66,17 @@ def image_dataset_from_directory(
     ```
 
     Then calling `image_dataset_from_directory(main_directory,
-    labels='inferred')` will return a `tf.data.Dataset` that yields batches of
+    labels='inferred')` will return a dataset that yields batches of
     images from the subdirectories `class_a` and `class_b`, together with labels
     0 and 1 (0 corresponding to `class_a` and 1 corresponding to `class_b`).
 
     Supported image formats: `.jpeg`, `.jpg`, `.png`, `.bmp`, `.gif`.
     Animated gifs are truncated to the first frame.
 
+    By default, this function will return a `tf.data.Dataset` object. You can
+    set `format="grain"` to return a `grain.IterDataset` object instead, which
+    removes the TensorFlow dependency.
+
     Args:
         directory: Directory where the data is located.
             If `labels` is `"inferred"`, it should contain
@@ -125,12 +146,19 @@ def image_dataset_from_directory(
             preserved.
         data_format: If None uses keras.config.image_data_format()
             otherwise either 'channel_last' or 'channel_first'.
+        format: The format of the return object. Defaults to `"tf"`. Available
+            options are:
+            - `"tf"`: returns a `tf.data.Dataset` object. Requires
+                TensorFlow to be installed.
+            - `"grain"`: returns a `grain.IterDataset` object. Requires
+                Grain to be installed.
         verbose: Whether to display number information on classes and
             number of files found. Defaults to `True`.
 
     Returns:
 
-    A `tf.data.Dataset` object.
+    A `tf.data.Dataset` (`format="tf"`) or `grain.IterDataset`
+    (`format="grain"`) object.
 
     - If `label_mode` is `None`, it yields `float32` tensors of shape
         `(batch_size, image_size[0], image_size[1], num_channels)`,
@@ -222,6 +250,11 @@ def image_dataset_from_directory(
             f"{supported_interpolations}. "
             f"Received: interpolation={interpolation}"
         )
+    if format not in ("tf", "grain"):
+        raise ValueError(
+            '`format` should be either "tf" or "grain". '
+            f"Received: format={format}"
+        )
 
     dataset_utils.check_validation_split_arg(
         validation_split, subset, shuffle, seed
@@ -289,6 +322,7 @@ def image_dataset_from_directory(
             shuffle=shuffle,
             shuffle_buffer_size=shuffle_buffer_size,
             seed=seed,
+            format=format,
         )
 
         val_dataset = paths_and_labels_to_dataset(
@@ -303,14 +337,23 @@ def image_dataset_from_directory(
             pad_to_aspect_ratio=pad_to_aspect_ratio,
             data_format=data_format,
             shuffle=False,
+            format=format,
         )
 
-        if batch_size is not None:
-            train_dataset = train_dataset.batch(batch_size)
-            val_dataset = val_dataset.batch(batch_size)
-
-        train_dataset = train_dataset.prefetch(tf.data.AUTOTUNE)
-        val_dataset = val_dataset.prefetch(tf.data.AUTOTUNE)
+        if format == "tf":
+            if batch_size is not None:
+                train_dataset = train_dataset.batch(batch_size)
+                val_dataset = val_dataset.batch(batch_size)
+            train_dataset = train_dataset.prefetch(tf.data.AUTOTUNE)
+            val_dataset = val_dataset.prefetch(tf.data.AUTOTUNE)
+        else:
+            train_dataset = train_dataset.to_iter_dataset()
+            val_dataset = val_dataset.to_iter_dataset()
+            if batch_size is not None:
+                train_dataset = train_dataset.batch(
+                    batch_size, batch_fn=make_batch
+                )
+                val_dataset = val_dataset.batch(batch_size, batch_fn=make_batch)
 
         # Users may need to reference `class_names`.
         train_dataset.class_names = class_names
@@ -345,12 +388,18 @@ def image_dataset_from_directory(
             shuffle=shuffle,
             shuffle_buffer_size=shuffle_buffer_size,
             seed=seed,
+            format=format,
         )
 
-        if batch_size is not None:
-            dataset = dataset.batch(batch_size)
+        if format == "tf":
+            if batch_size is not None:
+                dataset = dataset.batch(batch_size)
+            dataset = dataset.prefetch(tf.data.AUTOTUNE)
+        else:
+            dataset = dataset.to_iter_dataset()
+            if batch_size is not None:
+                dataset = dataset.batch(batch_size, batch_fn=make_batch)
 
-        dataset = dataset.prefetch(tf.data.AUTOTUNE)
         # Users may need to reference `class_names`.
         dataset.class_names = class_names
 
@@ -374,11 +423,66 @@ def paths_and_labels_to_dataset(
     shuffle=False,
     shuffle_buffer_size=None,
     seed=None,
+    format="tf",
+):
+    """Constructs a dataset of images and labels."""
+    if format == "tf":
+        return _paths_and_labels_to_dataset_tf(
+            image_paths=image_paths,
+            image_size=image_size,
+            num_channels=num_channels,
+            labels=labels,
+            label_mode=label_mode,
+            num_classes=num_classes,
+            interpolation=interpolation,
+            data_format=data_format,
+            crop_to_aspect_ratio=crop_to_aspect_ratio,
+            pad_to_aspect_ratio=pad_to_aspect_ratio,
+            shuffle=shuffle,
+            shuffle_buffer_size=shuffle_buffer_size,
+            seed=seed,
+        )
+    elif format == "grain":
+        return _paths_and_labels_to_dataset_grain(
+            image_paths=image_paths,
+            image_size=image_size,
+            num_channels=num_channels,
+            labels=labels,
+            label_mode=label_mode,
+            num_classes=num_classes,
+            interpolation=interpolation,
+            data_format=data_format,
+            crop_to_aspect_ratio=crop_to_aspect_ratio,
+            pad_to_aspect_ratio=pad_to_aspect_ratio,
+            shuffle=shuffle,
+            seed=seed,
+        )
+    else:
+        raise ValueError(
+            '`format` should be either "tf" or "grain". '
+            f"Received: format={format}"
+        )
+
+
+def _paths_and_labels_to_dataset_tf(
+    image_paths,
+    image_size,
+    num_channels,
+    labels,
+    label_mode,
+    num_classes,
+    interpolation,
+    data_format,
+    crop_to_aspect_ratio=False,
+    pad_to_aspect_ratio=False,
+    shuffle=False,
+    shuffle_buffer_size=None,
+    seed=None,
 ):
     """Constructs a dataset of images and labels."""
     path_ds = tf.data.Dataset.from_tensor_slices(image_paths)
     if label_mode:
-        label_ds = dataset_utils.labels_to_dataset(
+        label_ds = dataset_utils.labels_to_dataset_tf(
             labels, label_mode, num_classes
         )
         ds = tf.data.Dataset.zip((path_ds, label_ds))
@@ -398,17 +502,18 @@ def paths_and_labels_to_dataset(
     )
     if label_mode:
         ds = ds.map(
-            lambda x, y: (load_image(x, *args), y),
+            lambda x, y: (_load_image_tf(x, *args), y),
             num_parallel_calls=tf.data.AUTOTUNE,
         )
     else:
         ds = ds.map(
-            lambda x: load_image(x, *args), num_parallel_calls=tf.data.AUTOTUNE
+            lambda x: _load_image_tf(x, *args),
+            num_parallel_calls=tf.data.AUTOTUNE,
         )
     return ds
 
 
-def load_image(
+def _load_image_tf(
     path,
     image_size,
     num_channels,
@@ -457,3 +562,120 @@ def load_image(
     else:
         img.set_shape((num_channels, image_size[0], image_size[1]))
     return img
+
+
+def _paths_and_labels_to_dataset_grain(
+    image_paths,
+    image_size,
+    num_channels,
+    labels,
+    label_mode,
+    num_classes,
+    interpolation,
+    data_format,
+    crop_to_aspect_ratio=False,
+    pad_to_aspect_ratio=False,
+    shuffle=False,
+    seed=None,
+):
+    """Constructs a dataset of images and labels."""
+    path_ds = grain.MapDataset.source(image_paths)
+    if label_mode:
+        label_ds = dataset_utils.labels_to_dataset_grain(
+            labels, label_mode, num_classes
+        )
+        ds = grain.experimental.ZipMapDataset([path_ds, label_ds])
+    else:
+        ds = path_ds
+
+    if shuffle:
+        ds = ds.shuffle(seed=seed)
+
+    args = (
+        image_size,
+        num_channels,
+        interpolation,
+        data_format,
+        crop_to_aspect_ratio,
+        pad_to_aspect_ratio,
+    )
+    if label_mode:
+        ds = ds.map(lambda data: (_load_image_grain(data[0], *args), data[1]))
+    else:
+        ds = ds.map(lambda x: _load_image_grain(x, *args))
+
+    return ds
+
+
+def _load_image_grain(
+    path,
+    image_size,
+    num_channels,
+    interpolation,
+    data_format,
+    crop_to_aspect_ratio=False,
+    pad_to_aspect_ratio=False,
+):
+    """Load an image from a path and resize it."""
+    from keras.src import backend
+    from keras.src import ops
+
+    if pil_image is None:
+        raise ImportError(
+            "Could not import PIL.Image. The use of `load_img` requires PIL."
+        )
+    if pad_to_aspect_ratio and crop_to_aspect_ratio:
+        raise ValueError(
+            "Only one of `pad_to_aspect_ratio`, `crop_to_aspect_ratio`"
+            " can be set to `True`."
+        )
+
+    if isinstance(path, io.BytesIO):
+        img = pil_image.open(path)
+    elif isinstance(path, (pathlib.Path, bytes, str)):
+        if isinstance(path, pathlib.Path):
+            path = str(path.resolve())
+        img = pil_image.open(path)
+    else:
+        raise TypeError(
+            f"path should be path-like or io.BytesIO, not {type(path)}"
+        )
+    if num_channels == 1:
+        # if image is not already an 8-bit, 16-bit or 32-bit grayscale image
+        # convert it to an 8-bit grayscale image.
+        if img.mode not in ("L", "I;16", "I"):
+            img = img.convert("L")
+    elif num_channels == 4:
+        if img.mode != "RGBA":
+            img = img.convert("RGBA")
+    elif num_channels == 3:
+        if img.mode != "RGB":
+            img = img.convert("RGB")
+    else:
+        raise ValueError(
+            "num_channels must be 1, 3 or 4. "
+            f"Received: num_channels={num_channels}"
+        )
+
+    with backend.device_scope("cpu"):
+        img = ops.convert_to_tensor(np.array(img), dtype="float32")
+        if len(img.shape) == 2:
+            # If the image is grayscale, expand dims to add channel axis.
+            # The reason is that `ops.image.resize` expects 3D or 4D tensors.
+            img = ops.expand_dims(img, axis=-1)
+        if data_format == "channels_first":
+            img = ops.transpose(img, (2, 0, 1))
+        img = ops.image.resize(
+            img,
+            size=image_size,
+            interpolation=interpolation,
+            crop_to_aspect_ratio=crop_to_aspect_ratio,
+            pad_to_aspect_ratio=pad_to_aspect_ratio,
+            data_format=data_format,
+        )
+        if backend.backend() == "tensorflow":
+            if data_format == "channels_last":
+                img.set_shape((image_size[0], image_size[1], num_channels))
+            else:
+                img.set_shape((num_channels, image_size[0], image_size[1]))
+    return img

{keras_nightly-3.12.0.dev2025081903 → keras_nightly-3.12.0.dev2025082003}/keras/src/utils/module_utils.py

@@ -58,3 +58,4 @@ torch_xla = LazyModule(
 optree = LazyModule("optree")
 dmtree = LazyModule("tree")
 tf2onnx = LazyModule("tf2onnx")
+grain = LazyModule("grain")