datasets 4.6.0__tar.gz → 4.7.0__tar.gz

{datasets-4.6.0/src/datasets.egg-info → datasets-4.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datasets
-Version: 4.6.0
+Version: 4.7.0
 Summary: HuggingFace community-driven open-source library of datasets
 Home-page: https://github.com/huggingface/datasets
 Download-URL: https://github.com/huggingface/datasets/tags
@@ -272,7 +272,7 @@ from datasets import load_dataset
 
 # Print all the available datasets
 from huggingface_hub import list_datasets
-print([dataset.id for dataset in list_datasets()])
+print([dataset.id for dataset in list_datasets(limit=20)])
 
 # Load a dataset and print the first example in the training set
 squad_dataset = load_dataset('rajpurkar/squad')

{datasets-4.6.0 → datasets-4.7.0}/README.md

@@ -84,7 +84,7 @@ from datasets import load_dataset
 
 # Print all the available datasets
 from huggingface_hub import list_datasets
-print([dataset.id for dataset in list_datasets()])
+print([dataset.id for dataset in list_datasets(limit=20)])
 
 # Load a dataset and print the first example in the training set
 squad_dataset = load_dataset('rajpurkar/squad')

{datasets-4.6.0 → datasets-4.7.0}/setup.py

@@ -233,7 +233,7 @@ EXTRAS_REQUIRE = {
 
 setup(
     name="datasets",
-    version="4.6.0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
+    version="4.7.0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
     description="HuggingFace community-driven open-source library of datasets",
     long_description=open("README.md", encoding="utf-8").read(),
     long_description_content_type="text/markdown",

{datasets-4.6.0 → datasets-4.7.0}/src/datasets/__init__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "4.6.0"
+__version__ = "4.7.0"
 
 from .arrow_dataset import Column, Dataset
 from .arrow_reader import ReadInstruction

{datasets-4.6.0 → datasets-4.7.0}/src/datasets/arrow_dataset.py

@@ -981,6 +981,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
         features: Optional[Features] = None,
         info: Optional[DatasetInfo] = None,
         split: Optional[NamedSplit] = None,
+        on_mixed_types: Optional[Literal["use_json"]] = None,
     ) -> "Dataset":
         """
         Convert `dict` to a `pyarrow.Table` to create a [`Dataset`].
@@ -1000,9 +1001,109 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
                 Dataset information, like description, citation, etc.
             split (`NamedSplit`, *optional*):
                 Name of the dataset split.
+            on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`):
+                If "use_json", use the Json() type for mixed-types fields,
+                i.e. unstructured fields that contain data without a predefined schema.
+                In this case, a field with mixed type is set to Json().
+
+                This allow loading lists with a mix of strings/integers/floats
+                for example, or dictionaries with arbitrary value types.
+
+                <Added version="4.7.0"/>
 
         Returns:
             [`Dataset`]
+
+        Examples:
+
+        Get a Dataset from a dictionary containing one list per column:
+
+        ```py
+        >>> ds = Dataset.from_dict({"text": ["hello there !", "general kenobi !"]})
+        ```
+
+        Pass features to set the column types, e.g. for an image dataset:
+
+        ```py
+        >>> features = Features({"image": Image()})
+        >>> ds = Dataset.from_dict({"image": ["path/to/image.png"]}, features=features)
+        ```
+
+        Datasets are based on Arrow which is a columnar format, and therefore they expect every example to have the same
+        type and subtypes, and dictionaries to have the same keys and values types.
+        Loading a dataset errors out when fields have mismatching types, and fills missing fields in dictionaries with None so all dictionaries have the same keys and value types.
+
+        To avoid this and allow mixed-types without errors, you can use `on_mixed_types="use_json"` or specify `features=` with a [`Json`] type:
+
+        ```py
+        >>> ds = Dataset.from_dict({"a": [0, "foo", {"subfield": "bar"}]})
+        Traceback (most recent call last):
+          ...
+          File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
+        pyarrow.lib.ArrowInvalid: Could not convert 'foo' with type str: tried to convert to int64
+
+        >>> ds = Dataset.from_dict({"a": [0, "foo", {"subfield": "bar"}]}, on_mixed_types="use_json")
+        >>> ds.features
+        {'a': Json()}
+        >>> list(ds["a"])
+        [0, "foo", {"subfield": "bar"}]
+
+        >>> features = Features({"a": Json()})
+        >>> ds = Dataset.from_dict({"a": [0, "foo", {"subfield": "bar"}]}, features=features)
+        >>> ds.features
+        {'a': Json()}
+        >>> list(ds["a"])
+        [0, "foo", {"subfield": "bar"}]
+        ```
+
+        This is also useful for lists of dictionaries with arbitrary keys and values, to avoid filling missing fields with None:
+
+        ```py
+        >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]})
+        >>> ds.features
+        {'a': List({'b': Value('int64'), 'c': Value('int64')})}
+        >>> list(ds["a"])
+        [[{'b': 0, 'c': None}, {'b': None, 'c': 0}]]  # missing fields are filled with None
+
+        >>> features = Features({"a": List(Json())})
+        >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}, features=features)
+        >>> ds.features
+        {'a': List(Json())}
+        >>> list(ds["a"])
+        [[{'b': 0}, {'c': 0}]]  # OK
+
+        >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}, on_mixed_types="use_json")
+        >>> ds.features
+        {'a': List(Json())}
+        >>> list(ds["a"])
+        [[{'b': 0}, {'c': 0}]]  # OK
+        ```
+
+        Another example with tool calling data:
+
+        ```py
+        >>> messages = [
+        ...     {"role": "user", "content": "Turn on the living room lights and play my electronic music playlist."},
+        ...     {"role": "assistant", "tool_calls": [
+        ...         {"type": "function", "function": {
+        ...             "name": "control_light",
+        ...             "arguments": {"room": "living room", "state": "on"}
+        ...         }},
+        ...         {"type": "function", "function": {
+        ...             "name": "play_music",
+        ...             "arguments": {"playlist": "electronic"}  # mixed-type here since keys ["playlist"] and ["room", "state"] are different
+        ...         }}]
+        ...     },
+        ...     {"role": "tool", "name": "control_light", "content": "The lights in the living room are now on."},
+        ...     {"role": "tool", "name": "play_music", "content": "The music is now playing."},
+        ...     {"role": "assistant", "content": "Done!"}
+        ... ]
+        >>> ds = Dataset.from_dict({"messages": [messages]}, on_mixed_types="use_json")
+        >>> ds.features
+        {'messages': List({'role': Value('string'), 'content': Value('string'), 'tool_calls': List(Json()), 'name': Value('string')})}
+        >>> ds[0]["messages"][1]["tool_calls"][0]["function"]["arguments"]
+        {"room": "living room", "state": "on"}
+        ```
         """
         if info is not None and features is not None and info.features != features:
             raise ValueError(
@@ -1020,6 +1121,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
                     features.encode_column(data, col) if features is not None else data,
                     type=features[col] if features is not None else None,
                     col=col,
+                    on_mixed_types=on_mixed_types,
                 )
             arrow_typed_mapping[col] = data
         mapping = arrow_typed_mapping
@@ -1045,6 +1147,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
         features: Optional[Features] = None,
         info: Optional[DatasetInfo] = None,
         split: Optional[NamedSplit] = None,
+        on_mixed_types: Optional[Literal["use_json"]] = None,
     ) -> "Dataset":
         """
         Convert a list of dicts to a `pyarrow.Table` to create a [`Dataset`]`.
@@ -1063,13 +1166,113 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
             features (`Features`, optional): Dataset features.
             info (`DatasetInfo`, optional): Dataset information, like description, citation, etc.
             split (`NamedSplit`, optional): Name of the dataset split.
+            on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`):
+                If "use_json", use the Json() type for mixed-types fields,
+                i.e. unstructured fields that contain data without a predefined schema.
+                In this case, a field with mixed type is set to Json().
+
+                This allow loading lists with a mix of strings/integers/floats
+                for example, or dictionaries with arbitrary value types.
+
+                <Added version="4.7.0"/>
 
         Returns:
             [`Dataset`]
+
+        Examples:
+
+        Get a Dataset from a list containing the examples:
+
+        ```py
+        >>> ds = Dataset.from_list([{"text": "hello there !"}, {"text": "general kenobi !"}]})
+        ```
+
+        Pass features to set the column types, e.g. for an image dataset:
+
+        ```py
+        >>> features = Features({"image": Image()})
+        >>> ds = Dataset.from_list([{"image": "path/to/image.png"}], features=features)
+        ```
+
+        Datasets are based on Arrow which is a columnar format, and therefore they expect every example to have the same
+        type and subtypes, and dictionaries to have the same keys and values types.
+        Loading a dataset errors out when fields have mismatching types, and fills missing fields in dictionaries with None so all dictionaries have the same keys and value types.
+
+        To avoid this and allow mixed-types without errors, you can use `on_mixed_types="use_json"` or specify `features=` with a [`Json`] type:
+
+        ```py
+        >>> ds = Dataset.from_list([{"a": 0}, {"a": "foo"}, {"a": {"subfield": "bar"}}])
+        Traceback (most recent call last):
+          ...
+          File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
+        pyarrow.lib.ArrowInvalid: Could not convert 'foo' with type str: tried to convert to int64
+
+        >>> ds = Dataset.from_list([{"a": 0}, {"a": "foo"}, {"a": {"subfield": "bar"}}], on_mixed_types="use_json")
+        >>> ds.features
+        {'a': Json()}
+        >>> list(ds["a"])
+        [0, "foo", {"subfield": "bar"}]
+
+        >>> features = Features({"a": Json()})
+        >>> ds = Dataset.from_list([{"a": 0}, {"a": "foo"}, {"a": {"subfield": "bar"}}], features=features)
+        >>> ds.features
+        {'a': Json()}
+        >>> list(ds["a"])
+        [0, "foo", {"subfield": "bar"}]
+        ```
+
+        This is also useful for lists of dictionaries with arbitrary keys and values, to avoid filling missing fields with None:
+
+        ```py
+        >>> ds = Dataset.from_list([{"a": [{"b": 0}, {"c": 0}]}])
+        >>> ds.features
+        {'a': List({'b': Value('int64'), 'c': Value('int64')})}
+        >>> list(ds["a"])
+        [[{'b': 0, 'c': None}, {'b': None, 'c': 0}]]  # missing fields are filled with None
+
+        >>> features = Features({"a": List(Json())})
+        >>> ds = Dataset.from_list([{"a": [{"b": 0}, {"c": 0}]}], features=features)
+        >>> ds.features
+        {'a': List(Json())}
+        >>> list(ds["a"])
+        [[{'b': 0}, {'c': 0}]]  # OK
+
+        >>> ds = Dataset.from_list([{"a": [{"b": 0}, {"c": 0}]}], on_mixed_types="use_json")
+        >>> ds.features
+        {'a': List(Json())}
+        >>> list(ds["a"])
+        [[{'b': 0}, {'c': 0}]]  # OK
+        ```
+
+        Another example with tool calling data:
+
+        ```py
+        >>> messages = [
+        ...     {"role": "user", "content": "Turn on the living room lights and play my electronic music playlist."},
+        ...     {"role": "assistant", "tool_calls": [
+        ...         {"type": "function", "function": {
+        ...             "name": "control_light",
+        ...             "arguments": {"room": "living room", "state": "on"}
+        ...         }},
+        ...         {"type": "function", "function": {
+        ...             "name": "play_music",
+        ...             "arguments": {"playlist": "electronic"}  # mixed-type here since keys ["playlist"] and ["room", "state"] are different
+        ...         }}]
+        ...     },
+        ...     {"role": "tool", "name": "control_light", "content": "The lights in the living room are now on."},
+        ...     {"role": "tool", "name": "play_music", "content": "The music is now playing."},
+        ...     {"role": "assistant", "content": "Done!"}
+        ... ]
+        >>> ds = Dataset.from_list([{"messages": messages}], on_mixed_types="use_json")
+        >>> ds.features
+        {'messages': List({'role': Value('string'), 'content': Value('string'), 'tool_calls': List(Json()), 'name': Value('string')})}
+        >>> ds[0]["messages"][1]["tool_calls"][0]["function"]["arguments"]
+        {"room": "living room", "state": "on"}
+        ```
         """
         # for simplicity and consistency wrt OptimizedTypedSequence we do not use InMemoryTable.from_pylist here
         mapping = {k: [r.get(k) for r in mapping] for k in mapping[0]} if mapping else {}
-        return cls.from_dict(mapping, features, info, split)
+        return cls.from_dict(mapping, features, info, split, on_mixed_types=on_mixed_types)
 
     @staticmethod
     def from_csv(
@@ -3016,6 +3219,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
         new_fingerprint: Optional[str] = None,
         desc: Optional[str] = None,
         try_original_type: Optional[bool] = True,
+        on_mixed_types: Optional[Literal["use_json"]] = "use_json",
     ) -> "Dataset":
         """
         Apply a function to all the examples in the table (individually or in batches) and update the table.
@@ -3104,6 +3308,15 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
             try_original_type (`Optional[bool]`, defaults to `True`):
                 Try to keep the types of the original columns (e.g. int32 -> int32).
                 Set to False if you want to always infer new types.
+            on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`):
+                If "use_json", use the Json() type for mixed-types fields,
+                i.e. unstructured fields that contain data without a predefined schema.
+                In this case, a field with mixed type is set to Json().
+
+                This allow loading lists with a mix of strings/integers/floats
+                for example, or dictionaries with arbitrary value types.
+
+                <Added version="4.7.0"/>
 
         Example:
 
@@ -3205,6 +3418,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
             "disable_nullable": disable_nullable,
             "fn_kwargs": fn_kwargs,
             "try_original_type": try_original_type,
+            "on_mixed_types": on_mixed_types,
         }
 
         if new_fingerprint is None:
@@ -3459,6 +3673,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
         rank: Optional[int] = None,
         offset: int = 0,
         try_original_type: Optional[bool] = True,
+        on_mixed_types: Optional[Literal["use_json"]] = "use_json",
     ) -> Iterable[tuple[Optional[int], bool, Union[int, "Dataset"]]]:
         """Apply a function to all the elements in the table (individually or in batches)
         and update the table (if function does update examples).
@@ -3503,6 +3718,15 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
             try_original_type: (`Optional[bool]`, defaults to `True`):
                 Try to keep the types of the original columns (e.g. int32 -> int32).
                 Set to False if you want to always infer new types.
+            on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`):
+                If "use_json", use the Json() type for mixed-types fields,
+                i.e. unstructured fields that contain data without a predefined schema.
+                In this case, a field with mixed type is set to Json().
+
+                This allow loading lists with a mix of strings/integers/floats
+                for example, or dictionaries with arbitrary value types.
+
+                <Added version="4.7.0"/>
         """
         if fn_kwargs is None:
             fn_kwargs = {}
@@ -3659,6 +3883,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
                     update_features=update_features,
                     fingerprint=new_fingerprint,
                     disable_nullable=disable_nullable,
+                    on_mixed_types=on_mixed_types,
                 )
             else:
                 buf_writer = None
@@ -3673,6 +3898,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
                     update_features=update_features,
                     fingerprint=new_fingerprint,
                     disable_nullable=disable_nullable,
+                    on_mixed_types=on_mixed_types,
                 )
             return buf_writer, writer, tmp_file
 
@@ -5628,9 +5854,10 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
                 )
             except (Exception, KeyboardInterrupt):
                 tmp_file.close()
-                if Path(tmp_file.name).exists():
-                    Path(tmp_file.name).unlink()
+                Path(tmp_file.name).unlink()
                 raise
+            tmp_file.close()
+            Path(tmp_file.name).unlink()
             additions.append(shard_addition)
             yield job_id, False, 1
 
@@ -5680,6 +5907,16 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
         additions: list[CommitOperationAdd] = []
 
         num_jobs = num_proc or 1
+        if num_shards <= 1:
+            logger.warning(
+                f"Setting num_proc from {num_jobs} back to 1 for the {split} split to disable multiprocessing as it only contains one shard."
+            )
+            num_jobs = 1
+        elif num_shards < num_jobs:
+            logger.warning(
+                f"Setting num_proc from {num_jobs} to {num_shards} for the {split} split as it only contains {num_shards} shards."
+            )
+            num_proc = num_shards
         kwargs_iterable = [
             {
                 "self": self.shard(num_shards=num_jobs, index=job_id, contiguous=True),

{datasets-4.6.0 → datasets-4.7.0}/src/datasets/arrow_writer.py

@@ -13,14 +13,16 @@
 # Lint as: python3
 """To write records into Parquet files."""
 
+import io
 import json
 import sys
 from collections.abc import Iterable
-from typing import Any, Optional
+from typing import Any, Literal, Optional
 
 import fsspec
 import numpy as np
 import pyarrow as pa
+import pyarrow.json as paj
 import pyarrow.parquet as pq
 from fsspec.core import url_to_fs
 
@@ -43,6 +45,16 @@ from .filesystems import is_remote_filesystem
 from .info import DatasetInfo
 from .table import array_cast, cast_array_to_feature, embed_table_storage, table_cast
 from .utils import logging
+from .utils.json import (
+    find_mixed_struct_types_field_paths,
+    get_json_field_path_from_pyarrow_json_error,
+    get_json_field_paths_from_feature,
+    insert_json_field_path,
+    json_encode_field,
+    json_encode_fields_in_json_lines,
+    set_json_types_in_feature,
+    ujson_dumps,
+)
 from .utils.py_utils import asdict, convert_file_size_to_int, first_non_null_non_empty_value
 
 
@@ -204,6 +216,7 @@ class TypedSequence:
         type: Optional[FeatureType] = None,
         try_type: Optional[FeatureType] = None,
         optimized_int_type: Optional[FeatureType] = None,
+        on_mixed_types: Optional[Literal["use_json"]] = None,
     ):
         # assert type is None or try_type is None,
         if type is not None and try_type is not None:
@@ -213,6 +226,7 @@ class TypedSequence:
         self.type = type
         self.try_type = try_type  # is ignored if it doesn't match the data
         self.optimized_int_type = optimized_int_type
+        self.on_mixed_types = on_mixed_types
         # when trying a type (is ignored if data is not compatible)
         self.trying_type = self.try_type is not None
         self.trying_int_optimization = optimized_int_type is not None and type is None and try_type is None
@@ -231,7 +245,7 @@ class TypedSequence:
             FeatureType: inferred feature type of the sequence.
         """
         if self._inferred_type is None:
-            self._inferred_type = generate_from_arrow_type(pa.array(self).type)
+            pa.array(self)
         return self._inferred_type
 
     @staticmethod
@@ -274,6 +288,12 @@ class TypedSequence:
         return data, None
 
     def __arrow_array__(self, type: Optional[pa.DataType] = None):
+        out = self._arrow_array(type=type)
+        if self._inferred_type is None:
+            self._inferred_type = generate_from_arrow_type(out.type)
+        return out
+
+    def _arrow_array(self, type: Optional[pa.DataType] = None):
         """This function is called when calling pa.array(typed_sequence)"""
 
         if type is not None:
@@ -292,6 +312,7 @@ class TypedSequence:
             get_nested_type(self.optimized_int_type) if self.optimized_int_type is not None else None
         )
         trying_cast_to_python_objects = False
+        json_field_paths = []
         try:
             # custom pyarrow types
             if isinstance(pa_type, _ArrayXDExtensionType):
@@ -305,7 +326,25 @@ class TypedSequence:
                 out = list_of_np_array_to_pyarrow_listarray(data)
             else:
                 trying_cast_to_python_objects = True
-                out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
+                examples = data
+                # find fields to json-encode
+                if self.on_mixed_types == "use_json" and type is None:
+                    json_field_paths = find_mixed_struct_types_field_paths(examples, allow_root=True)
+                elif type is not None:
+                    json_field_paths = get_json_field_paths_from_feature(type)
+                # json encode if needed
+                if json_field_paths:
+                    for json_field_path in json_field_paths:
+                        examples = [json_encode_field(examples, json_field_path) for examples in examples]
+                # to arrow array
+                out = pa.array(cast_to_python_objects(examples, only_1d_for_numpy=True))
+                # cast to json type if needed
+                if json_field_paths:
+                    pa_table = pa.Table.from_arrays([out], names=["obj"])
+                    features = Features.from_arrow_schema(pa_table.schema)
+                    feature = set_json_types_in_feature(features["obj"], json_field_paths)
+                    pa_table = table_cast(pa_table, Features({"obj": feature}).arrow_schema)
+                    out = pa_table[0]  # get the "obj" column
             # use smaller integer precisions if possible
             if self.trying_int_optimization:
                 if pa.types.is_int64(out.type):
@@ -326,6 +365,7 @@ class TypedSequence:
             return out
         except (
             TypeError,
+            pa.lib.ArrowTypeError,
             pa.lib.ArrowInvalid,
             pa.lib.ArrowNotImplementedError,
         ) as e:  # handle type errors and overflows
@@ -372,8 +412,46 @@ class TypedSequence:
                 optimized_int_pa_type_str = np.dtype(optimized_int_pa_type.to_pandas_dtype()).name
                 logger.info(f"Failed to cast a sequence to {optimized_int_pa_type_str}. Falling back to int64.")
                 return out
-            elif trying_cast_to_python_objects and "Could not convert" in str(e):
-                out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False))
+            elif trying_cast_to_python_objects and (
+                "Could not convert" in str(e) or "cannot mix struct and non-struct" in str(e) or "Expected " in str(e)
+            ):
+                try:  # third chance
+                    out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False))
+                except (pa.ArrowInvalid, pa.ArrowTypeError) as ee:
+                    # in case of mixed types, we use the JSON Lines reader in pyarrow to locate them and set them to json fields
+                    if self.on_mixed_types == "use_json" and (
+                        "Could not convert " in str(ee)
+                        or "cannot mix struct and non-struct" in str(ee)
+                        or "Expected " in str(ee)
+                    ):
+                        # we use "obj" to have valid JSON Lines since data may contain lists
+                        original_batch = "\n".join([ujson_dumps({"obj": example}) for example in data]).encode()
+                        json_field_paths = [["obj"] + json_field_path for json_field_path in json_field_paths]
+                        batch = json_encode_fields_in_json_lines(original_batch, json_field_paths)
+                        pa_table = None
+                        while True:
+                            try:  # fourth chance
+                                pa_table = paj.read_json(
+                                    io.BytesIO(batch), read_options=paj.ReadOptions(use_threads=False)
+                                )
+                                break
+                            except pa.ArrowInvalid as eee:
+                                if "JSON parse error: Column(" in str(eee) and ") changed from" in str(eee):
+                                    json_field_path = get_json_field_path_from_pyarrow_json_error(str(eee))
+                                    insert_json_field_path(json_field_paths, json_field_path)
+                                    batch = json_encode_fields_in_json_lines(original_batch, json_field_paths)
+                                else:
+                                    break
+                        if pa_table is not None:
+                            features = Features.from_arrow_schema(pa_table.schema)
+                            features = set_json_types_in_feature(features, json_field_paths)
+                            pa_table = table_cast(pa_table, features.arrow_schema)
+                            out = pa_table[0]  # get the "obj" column
+                            return out
+                        else:
+                            raise
+                    else:
+                        raise
                 if type is not None:
                     out = cast_array_to_feature(out, type, allow_primitive_to_str=True, allow_decimal_to_str=True)
                 return out
@@ -389,6 +467,7 @@ class OptimizedTypedSequence(TypedSequence):
         try_type: Optional[FeatureType] = None,
         col: Optional[str] = None,
         optimized_int_type: Optional[FeatureType] = None,
+        on_mixed_types: Optional[Literal["use_json"]] = None,
     ):
         optimized_int_type_by_col = {
             "attention_mask": Value("int8"),  # binary tensor
@@ -400,7 +479,9 @@ class OptimizedTypedSequence(TypedSequence):
         }
         if type is None and try_type is None:
             optimized_int_type = optimized_int_type_by_col.get(col, None)
-        super().__init__(data, type=type, try_type=try_type, optimized_int_type=optimized_int_type)
+        super().__init__(
+            data, type=type, try_type=try_type, optimized_int_type=optimized_int_type, on_mixed_types=on_mixed_types
+        )
 
 
 class ArrowWriter:
@@ -416,6 +497,7 @@ class ArrowWriter:
         writer_batch_size: Optional[int] = None,
         disable_nullable: bool = False,
         update_features: bool = False,
+        on_mixed_types: Optional[Literal["use_json"]] = "use_json",
         with_metadata: bool = True,
         unit: str = "examples",
         embed_local_files: bool = False,
@@ -455,6 +537,7 @@ class ArrowWriter:
             or config.DEFAULT_MAX_BATCH_SIZE
         )
         self.update_features = update_features
+        self.on_mixed_types = on_mixed_types
         self.with_metadata = with_metadata
         self.unit = unit
         self.embed_local_files = embed_local_files
@@ -654,7 +737,9 @@ class ArrowWriter:
                     if try_features is not None and col in try_features and try_original_type
                     else None
                 )
-                typed_sequence = OptimizedTypedSequence(col_values, type=col_type, try_type=col_try_type, col=col)
+                typed_sequence = OptimizedTypedSequence(
+                    col_values, type=col_type, try_type=col_try_type, col=col, on_mixed_types=self.on_mixed_types
+                )
                 arrays.append(pa.array(typed_sequence))
                 inferred_features[col] = typed_sequence.get_inferred_type()
         schema = inferred_features.arrow_schema if self.pa_writer is None else self.schema

{datasets-4.6.0 → datasets-4.7.0}/src/datasets/dataset_dict.py

@@ -11,7 +11,7 @@ import time
 from collections.abc import Sequence
 from functools import partial
 from pathlib import Path
-from typing import Callable, Optional, Union
+from typing import Callable, Literal, Optional, Union
 
 import fsspec
 import numpy as np
@@ -836,6 +836,7 @@ class DatasetDict(dict[Union[str, NamedSplit], "Dataset"]):
         num_proc: Optional[int] = None,
         desc: Optional[str] = None,
         try_original_type: Optional[bool] = True,
+        on_mixed_types: Optional[Literal["use_json"]] = None,
     ) -> "DatasetDict":
         """
         Apply a function to all the examples in the table (individually or in batches) and update the table.
@@ -920,6 +921,15 @@ class DatasetDict(dict[Union[str, NamedSplit], "Dataset"]):
             try_original_type (`Optional[bool]`, defaults to `True`):
                 Try to keep the types of the original columns (e.g. int32 -> int32).
                 Set to False if you want to always infer new types.
+            on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`):
+                If "use_json", use the Json() type for mixed-types fields,
+                i.e. unstructured fields that contain data without a predefined schema.
+                In this case, a field with mixed type is set to Json().
+
+                This allow loading lists with a mix of strings/integers/floats
+                for example, or dictionaries with arbitrary value types.
+
+                <Added version="4.7.0"/>
 
         Example:
 
@@ -969,6 +979,7 @@ class DatasetDict(dict[Union[str, NamedSplit], "Dataset"]):
                 num_proc=num_proc,
                 desc=desc,
                 try_original_type=try_original_type,
+                on_mixed_types=on_mixed_types,
             )
 
             if with_split:

{datasets-4.6.0 → datasets-4.7.0}/src/datasets/features/__init__.py

@@ -6,6 +6,7 @@ __all__ = [
     "Array5D",
     "ClassLabel",
     "Features",
+    "Json",
     "LargeList",
     "List",
     "Sequence",
@@ -18,7 +19,7 @@ __all__ = [
     "Nifti",
 ]
 from .audio import Audio
-from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, LargeList, List, Sequence, Value
+from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, Json, LargeList, List, Sequence, Value
 from .image import Image
 from .nifti import Nifti
 from .pdf import Pdf