datachain 0.30.4__tar.gz → 0.30.5__tar.gz

{datachain-0.30.4 → datachain-0.30.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.30.4
+Version: 0.30.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -22,6 +22,7 @@ Requires-Dist: tomlkit
 Requires-Dist: tqdm
 Requires-Dist: numpy<3,>=1
 Requires-Dist: pandas>=2.0.0
+Requires-Dist: ujson>=5.10.0
 Requires-Dist: packaging
 Requires-Dist: pyarrow
 Requires-Dist: typing-extensions
@@ -38,7 +39,6 @@ Requires-Dist: shtab<2,>=1.3.4
 Requires-Dist: sqlalchemy>=2
 Requires-Dist: multiprocess==0.70.16
 Requires-Dist: cloudpickle
-Requires-Dist: orjson>=3.10.5
 Requires-Dist: pydantic
 Requires-Dist: jmespath>=1.0
 Requires-Dist: datamodel-code-generator>=0.25
@@ -92,7 +92,7 @@ Requires-Dist: pytest-mock>=3.12.0; extra == "tests"
 Requires-Dist: pytest-servers[all]>=0.5.9; extra == "tests"
 Requires-Dist: pytest-benchmark[histogram]; extra == "tests"
 Requires-Dist: pytest-xdist>=3.3.1; extra == "tests"
-Requires-Dist: pytest-env>=1.1.0; extra == "tests"
+Requires-Dist: pytest-dotenv; extra == "tests"
 Requires-Dist: virtualenv; extra == "tests"
 Requires-Dist: dulwich; extra == "tests"
 Requires-Dist: hypothesis; extra == "tests"

datachain-0.30.5/examples/get_started/nested_datamodel.py

@@ -0,0 +1,70 @@
+"""Example: Nested DataModels with parallel execution.
+
+Demonstrates mapping a function that returns a nested DataModel (a DataModel
+containing other DataModels).
+
+The example keeps things minimal: we persist a tiny dataset, run a parallel map
+that returns a nested DataModel, and display the result.
+"""
+
+from typing import Optional
+
+from pydantic import Field
+
+import datachain as dc
+
+
+class Metric(dc.DataModel):
+    """Represents a single computed metric with quality metadata."""
+
+    value: Optional[float] = Field(default=None, description="Computed metric value")
+    confidence: Optional[float] = Field(
+        default=None, description="Confidence / quality score"
+    )
+    status: Optional[str] = Field(default=None, description="Processing status label")
+    metric_error: Optional[str] = Field(
+        default=None, description="Error message if metric computation failed"
+    )
+
+
+class SampleMetrics(dc.DataModel):
+    """Container for two illustrative nested metrics.
+
+    Each sub-field is its own DataModel instance to demonstrate nested schemas
+    """
+
+    metric_primary: Metric = Field(
+        default_factory=lambda: Metric(), description="Primary metric"
+    )
+    metric_secondary: Metric = Field(
+        default_factory=lambda: Metric(), description="Secondary metric"
+    )
+
+
+def generate_sample_metrics() -> SampleMetrics:
+    """Synthesize a pair of metrics.
+
+    In real scenarios you'd compute these values; here we just return constants
+    to keep the example deterministic.
+    """
+
+    return SampleMetrics(
+        metric_primary=Metric(value=50.0, confidence=0.95, status="ok"),
+    )
+
+
+def main():
+    (
+        dc.read_values(record_id=[1, 2])
+        .settings(parallel=2)  # Keep it parallel to test serialization
+        .map(metrics=generate_sample_metrics)
+        .save("nested_datamodel")
+    )
+
+    dc.read_dataset("nested_datamodel").show()
+
+    print(dc.read_dataset("nested_datamodel").to_values("metrics"))
+
+
+if __name__ == "__main__":
+    main()

{datachain-0.30.4 → datachain-0.30.5}/pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
   "tqdm",
   "numpy>=1,<3",
   "pandas>=2.0.0",
+  "ujson>=5.10.0",
   "packaging",
   "pyarrow",
   "typing-extensions",
@@ -42,7 +43,6 @@ dependencies = [
   "sqlalchemy>=2",
   "multiprocess==0.70.16",
   "cloudpickle",
-  "orjson>=3.10.5",
   "pydantic",
   "jmespath>=1.0",
   "datamodel-code-generator>=0.25",
@@ -108,7 +108,7 @@ tests = [
   "pytest-servers[all]>=0.5.9",
   "pytest-benchmark[histogram]",
   "pytest-xdist>=3.3.1",
-  "pytest-env>=1.1.0",
+  "pytest-dotenv",
   "virtualenv",
   "dulwich",
   "hypothesis",

{datachain-0.30.4 → datachain-0.30.5}/src/datachain/data_storage/warehouse.py

@@ -1,5 +1,4 @@
 import glob
-import json
 import logging
 import posixpath
 import random
@@ -11,6 +10,7 @@ from urllib.parse import urlparse
 
 import attrs
 import sqlalchemy as sa
+import ujson as json
 from sqlalchemy.sql.expression import true
 
 from datachain.client import Client
@@ -122,7 +122,7 @@ class AbstractWarehouse(ABC, Serializable):
                 if value_type is str:
                     return val
                 if value_type in (dict, list):
-                    return json.dumps(val)
+                    return json.dumps(val, ensure_ascii=False)
                 raise ValueError(
                     f"Cannot convert value {val!r} with type {value_type} to JSON"
                 )

{datachain-0.30.4 → datachain-0.30.5}/src/datachain/lib/arrow.py

@@ -2,8 +2,8 @@ from collections.abc import Sequence
 from itertools import islice
 from typing import TYPE_CHECKING, Any, Optional
 
-import orjson
 import pyarrow as pa
+import ujson as json
 from pyarrow._csv import ParseOptions
 from pyarrow.dataset import CsvFileFormat, dataset
 from tqdm.auto import tqdm
@@ -269,7 +269,7 @@ def _get_hf_schema(
 def _get_datachain_schema(schema: "pa.Schema") -> Optional[SignalSchema]:
     """Return a restored SignalSchema from parquet metadata, if any is found."""
     if schema.metadata and DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY in schema.metadata:
-        serialized_signal_schema = orjson.loads(
+        serialized_signal_schema = json.loads(
             schema.metadata[DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY]
         )
         return SignalSchema.deserialize(serialized_signal_schema)

{datachain-0.30.4 → datachain-0.30.5}/src/datachain/lib/dc/datachain.py

@@ -19,8 +19,8 @@ from typing import (
     overload,
 )
 
-import orjson
 import sqlalchemy
+import ujson as json
 from pydantic import BaseModel
 from sqlalchemy.sql.elements import ColumnElement
 from tqdm import tqdm
@@ -462,8 +462,6 @@ class DataChain:
         Returns:
             DataChain: A new DataChain instance with the new set of columns.
         """
-        import json
-
         import pyarrow as pa
 
         from datachain.lib.arrow import schema_to_output
@@ -2129,9 +2127,9 @@ class DataChain:
             fsspec_fs = client.create_fs(**fs_kwargs)
 
         _partition_cols = list(partition_cols) if partition_cols else None
-        signal_schema_metadata = orjson.dumps(
-            self._effective_signals_schema.serialize()
-        )
+        signal_schema_metadata = json.dumps(
+            self._effective_signals_schema.serialize(), ensure_ascii=False
+        ).encode("utf-8")
 
         column_names, column_chunks = self.to_columnar_data_with_names(chunk_size)
 
@@ -2278,7 +2276,11 @@ class DataChain:
                         f.write(b"\n")
                 else:
                     is_first = False
-                f.write(orjson.dumps(row_to_nested_dict(headers, row)))
+                f.write(
+                    json.dumps(
+                        row_to_nested_dict(headers, row), ensure_ascii=False
+                    ).encode("utf-8")
+                )
             if include_outer_list:
                 # This makes the file JSON instead of JSON lines.
                 f.write(b"\n]\n")

{datachain-0.30.4 → datachain-0.30.5}/src/datachain/lib/model_store.py

@@ -89,3 +89,15 @@ class ModelStore:
             and ModelStore.is_pydantic(parent_type)
             and "@" in ModelStore.get_name(parent_type)
         )
+
+    @classmethod
+    def rebuild_all(cls) -> None:
+        """Ensure pydantic schemas are (re)built for all registered models.
+
+        Uses ``force=True`` to avoid subtle cases where a deserialized class
+        (e.g. from by-value cloudpickle in workers) reports built state but
+        nested model field schemas aren't fully resolved yet.
+        """
+        for versions in cls.store.values():
+            for model in versions.values():
+                model.model_rebuild(force=True)

{datachain-0.30.4 → datachain-0.30.5}/src/datachain/query/dispatch.py

@@ -13,6 +13,7 @@ from multiprocess import get_context
 from datachain.catalog import Catalog
 from datachain.catalog.catalog import clone_catalog_with_cache
 from datachain.catalog.loader import DISTRIBUTED_IMPORT_PATH, get_udf_distributor_class
+from datachain.lib.model_store import ModelStore
 from datachain.lib.udf import _get_cache
 from datachain.query.dataset import (
     get_download_callback,
@@ -130,6 +131,8 @@ class UDFDispatcher:
 
     def _create_worker(self) -> "UDFWorker":
         udf: UDFAdapter = loads(self.udf_data)
+        # Ensure all registered DataModels have rebuilt schemas in worker processes.
+        ModelStore.rebuild_all()
         return UDFWorker(
             self.catalog,
             udf,
@@ -196,6 +199,8 @@ class UDFDispatcher:
         generated_cb: Callback = DEFAULT_CALLBACK,
     ) -> None:
         udf: UDFAdapter = loads(self.udf_data)
+        # Rebuild schemas in single process too for consistency (cheap, idempotent).
+        ModelStore.rebuild_all()
 
         if ids_only and not self.is_batching:
             input_rows = flatten(input_rows)

{datachain-0.30.4 → datachain-0.30.5}/src/datachain/sql/sqlite/base.py

@@ -8,8 +8,8 @@ from functools import cache
 from types import MappingProxyType
 from typing import Callable, Optional
 
-import orjson
 import sqlalchemy as sa
+import ujson as json
 from sqlalchemy.dialects import sqlite
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.elements import literal
@@ -182,7 +182,7 @@ def missing_vector_function(name, exc):
 
 
 def sqlite_string_split(string: str, sep: str, maxsplit: int = -1) -> str:
-    return orjson.dumps(string.split(sep, maxsplit)).decode("utf-8")
+    return json.dumps(string.split(sep, maxsplit), ensure_ascii=False)
 
 
 def sqlite_int_hash_64(x: int) -> int:
@@ -453,17 +453,17 @@ def compile_byte_hamming_distance(element, compiler, **kwargs):
 
 
 def py_json_array_length(arr):
-    return len(orjson.loads(arr))
+    return len(json.loads(arr))
 
 
 def py_json_array_contains(arr, value, is_json):
     if is_json:
-        value = orjson.loads(value)
-    return value in orjson.loads(arr)
+        value = json.loads(value)
+    return value in json.loads(arr)
 
 
 def py_json_array_get_element(val, idx):
-    arr = orjson.loads(val)
+    arr = json.loads(val)
     try:
         return arr[idx]
     except IndexError:
@@ -471,17 +471,18 @@ def py_json_array_get_element(val, idx):
 
 
 def py_json_array_slice(val, offset: int, length: Optional[int] = None):
-    arr = orjson.loads(val)
+    arr = json.loads(val)
     try:
-        return orjson.dumps(
-            list(arr[offset : offset + length] if length is not None else arr[offset:])
-        ).decode("utf-8")
+        return json.dumps(
+            list(arr[offset : offset + length] if length is not None else arr[offset:]),
+            ensure_ascii=False,
+        )
     except IndexError:
         return None
 
 
 def py_json_array_join(val, sep: str):
-    return sep.join(orjson.loads(val))
+    return sep.join(json.loads(val))
 
 
 def compile_array_get_element(element, compiler, **kwargs):

{datachain-0.30.4 → datachain-0.30.5}/src/datachain/sql/sqlite/types.py

@@ -1,6 +1,6 @@
 import sqlite3
 
-import orjson
+import ujson as json
 from sqlalchemy import types
 
 from datachain.sql.types import TypeConverter, TypeReadConverter
@@ -28,26 +28,21 @@ class Array(types.UserDefinedType):
 
 
 def adapt_array(arr):
-    return orjson.dumps(arr).decode("utf-8")
+    return json.dumps(arr, ensure_ascii=False)
 
 
 def adapt_dict(dct):
-    return orjson.dumps(dct).decode("utf-8")
+    return json.dumps(dct, ensure_ascii=False)
 
 
 def convert_array(arr):
-    return orjson.loads(arr)
+    return json.loads(arr)
 
 
 def adapt_np_array(arr):
-    def _json_serialize(obj):
-        if isinstance(obj, np.ndarray):
-            return obj.tolist()
-        return obj
-
-    return orjson.dumps(
-        arr, option=orjson.OPT_SERIALIZE_NUMPY, default=_json_serialize
-    ).decode("utf-8")
+    # Primarily needed for UDF numpy results (e.g. WDS)
+    # tolist() gives nested Python lists + native scalars; ujson.dumps handles NaN/Inf.
+    return json.dumps(arr.tolist(), ensure_ascii=False)
 
 
 def adapt_np_generic(val):
@@ -74,5 +69,5 @@ class SQLiteTypeConverter(TypeConverter):
 class SQLiteTypeReadConverter(TypeReadConverter):
     def array(self, value, item_type, dialect):
         if isinstance(value, str):
-            value = orjson.loads(value)
+            value = json.loads(value)
         return super().array(value, item_type, dialect)

{datachain-0.30.4 → datachain-0.30.5}/src/datachain/sql/types.py

@@ -16,8 +16,8 @@ from datetime import datetime
 from types import MappingProxyType
 from typing import Any, Union
 
-import orjson
 import sqlalchemy as sa
+import ujson as jsonlib
 from sqlalchemy import TypeDecorator, types
 
 from datachain.lib.data_model import StandardType
@@ -352,7 +352,7 @@ class Array(SQLType):
     def on_read_convert(self, value, dialect):
         r = read_converter(dialect).array(value, self.item_type, dialect)
         if isinstance(self.item_type, JSON):
-            r = [orjson.loads(item) if isinstance(item, str) else item for item in r]
+            r = [jsonlib.loads(item) if isinstance(item, str) else item for item in r]
         return r
 
 
@@ -466,7 +466,7 @@ class TypeReadConverter:
         if isinstance(value, str):
             if value == "":
                 return {}
-            return orjson.loads(value)
+            return jsonlib.loads(value)
         return value
 
     def datetime(self, value):

{datachain-0.30.4 → datachain-0.30.5}/src/datachain/utils.py

@@ -417,7 +417,7 @@ class JSONSerialize(json.JSONEncoder):
 
 def inside_colab() -> bool:
     try:
-        from google import colab  # noqa: F401
+        from google import colab  # type: ignore[attr-defined]  # noqa: F401
     except ImportError:
         return False
     return True

{datachain-0.30.4 → datachain-0.30.5}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.30.4
+Version: 0.30.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -22,6 +22,7 @@ Requires-Dist: tomlkit
 Requires-Dist: tqdm
 Requires-Dist: numpy<3,>=1
 Requires-Dist: pandas>=2.0.0
+Requires-Dist: ujson>=5.10.0
 Requires-Dist: packaging
 Requires-Dist: pyarrow
 Requires-Dist: typing-extensions
@@ -38,7 +39,6 @@ Requires-Dist: shtab<2,>=1.3.4
 Requires-Dist: sqlalchemy>=2
 Requires-Dist: multiprocess==0.70.16
 Requires-Dist: cloudpickle
-Requires-Dist: orjson>=3.10.5
 Requires-Dist: pydantic
 Requires-Dist: jmespath>=1.0
 Requires-Dist: datamodel-code-generator>=0.25
@@ -92,7 +92,7 @@ Requires-Dist: pytest-mock>=3.12.0; extra == "tests"
 Requires-Dist: pytest-servers[all]>=0.5.9; extra == "tests"
 Requires-Dist: pytest-benchmark[histogram]; extra == "tests"
 Requires-Dist: pytest-xdist>=3.3.1; extra == "tests"
-Requires-Dist: pytest-env>=1.1.0; extra == "tests"
+Requires-Dist: pytest-dotenv; extra == "tests"
 Requires-Dist: virtualenv; extra == "tests"
 Requires-Dist: dulwich; extra == "tests"
 Requires-Dist: hypothesis; extra == "tests"

{datachain-0.30.4 → datachain-0.30.5}/src/datachain.egg-info/SOURCES.txt

@@ -78,6 +78,7 @@ examples/computer_vision/ultralytics-pose.py
 examples/computer_vision/ultralytics-segment.py
 examples/get_started/common_sql_functions.py
 examples/get_started/json-csv-reader.py
+examples/get_started/nested_datamodel.py
 examples/get_started/torch-loader.py
 examples/get_started/udfs/parallel.py
 examples/get_started/udfs/simple.py

{datachain-0.30.4 → datachain-0.30.5}/src/datachain.egg-info/requires.txt

@@ -3,6 +3,7 @@ tomlkit
 tqdm
 numpy<3,>=1
 pandas>=2.0.0
+ujson>=5.10.0
 packaging
 pyarrow
 typing-extensions
@@ -19,7 +20,6 @@ shtab<2,>=1.3.4
 sqlalchemy>=2
 multiprocess==0.70.16
 cloudpickle
-orjson>=3.10.5
 pydantic
 jmespath>=1.0
 datamodel-code-generator>=0.25
@@ -92,7 +92,7 @@ pytest-mock>=3.12.0
 pytest-servers[all]>=0.5.9
 pytest-benchmark[histogram]
 pytest-xdist>=3.3.1
-pytest-env>=1.1.0
+pytest-dotenv
 virtualenv
 dulwich
 hypothesis

{datachain-0.30.4 → datachain-0.30.5}/tests/func/functions/test_array.py

@@ -356,26 +356,32 @@ def test_array_contains(test_session):
 
     ds = list(
         dc.read_values(
-            id=(1, 2, 3),
+            id=(1, 2, 3, 4),
             arr=(
                 Arr(i=[10, 20, 30], f=[1.0, 2.0, 3.0], s=["a", "b", "c"]),
                 Arr(i=[40, 50, 60], f=[4.0, 5.0, 6.0], s=["d", "e", "f"]),
                 Arr(i=[50], f=[5.0], s=["g"]),
+                # New row with NaN/Inf values for testing
+                Arr(i=[100], f=[float("nan"), float("inf"), float("-inf")], s=["h"]),
             ),
             ii=(
                 [20, 30, 50, 80],
                 [10],
                 [],
+                [200],
             ),
             ff=(
                 [2.0, 3.0, 5.0, 7.0],
                 [4.0],
                 [],
+                # Test array with special float values
+                [float("inf"), float("-inf"), 1.5],
             ),
             ss=(
                 ["b", "c", "e", "f"],
                 ["d"],
                 [],
+                ["i"],
             ),
             session=test_session,
         )
@@ -395,6 +401,14 @@ def test_array_contains(test_session):
             t13=func.array.contains([1, 2, 3, 4, 5], 3),
             t14=func.array.contains([1, 2, 3, 4, 5], 7),
             t15=func.array.contains([], 1),
+            # Test NaN/Inf handling with contains
+            t16=func.array.contains("arr.f", float("inf")),  # Should find inf in row 4
+            # Should find -inf in row 4
+            t17=func.array.contains("arr.f", float("-inf")),
+            # Should NOT find nan (NaN != NaN)
+            t18=func.array.contains("arr.f", float("nan")),
+            t19=func.array.contains("ff", float("inf")),  # Should find inf in row 4
+            t20=func.array.contains("ff", float("-inf")),  # Should find -inf in row 4
         )
         .order_by("id")
         .to_list(
@@ -413,11 +427,75 @@ def test_array_contains(test_session):
             "t13",
             "t14",
             "t15",
+            "t16",
+            "t17",
+            "t18",
+            "t19",
+            "t20",
         )
     )
 
     assert ds == [
-        (1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0),
-        (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0),
-        (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0),
+        # Row 1: Regular values
+        (1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0),
+        # Row 2: Regular values
+        (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0),
+        # Row 3: Regular values
+        (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0),
+        # Row 4: Contains NaN/Inf values - inf/-inf should be found, NaN should not
+        (0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1),
     ]
+
+
+def test_array_functions_with_nan_inf(test_session):
+    class ArrWithSpecial(dc.DataModel):
+        f: list[float]  # Will contain NaN and Infinity values
+
+    ds = list(
+        dc.read_values(
+            id=(1, 2, 3),
+            arr=(
+                ArrWithSpecial(f=[1.0, float("nan"), 3.0]),
+                ArrWithSpecial(f=[float("inf"), 2.0, float("-inf")]),
+                ArrWithSpecial(f=[float("nan"), float("inf")]),
+            ),
+            special_floats=(
+                [1.0, float("nan"), float("inf")],
+                [float("-inf"), 2.0],
+                [float("nan")],
+            ),
+            session=test_session,
+        )
+        .mutate(
+            # Test array.length with NaN/INF arrays
+            len1=func.array.length("arr.f"),
+            len2=func.array.length("special_floats"),
+            # Test array.slice with NaN/INF arrays
+            slice1=func.array.slice("arr.f", 0, 2),
+            slice2=func.array.slice("special_floats", 1),
+            # Test array.get_element with NaN/INF arrays
+            elem1=func.array.get_element("arr.f", 0),
+            elem2=func.array.get_element("special_floats", 0),
+        )
+        .order_by("id")
+        .to_list("len1", "len2", "slice1", "slice2", "elem1", "elem2")
+    )
+
+    # Verify lengths are correct
+    assert ds[0][0] == 3  # [1.0, nan, 3.0]
+    assert ds[0][1] == 3  # [1.0, nan, inf]
+    assert ds[1][0] == 3  # [inf, 2.0, -inf]
+    assert ds[1][1] == 2  # [-inf, 2.0]
+    assert ds[2][0] == 2  # [nan, inf]
+    assert ds[2][1] == 1  # [nan]
+
+    # Verify slices preserve NaN/INF
+    assert len(ds[0][2]) == 2  # slice of [1.0, nan, 3.0]
+    assert ds[0][2][0] == 1.0
+    assert math.isnan(ds[0][2][1])
+
+    # Verify get_element preserves NaN/INF
+    assert ds[0][4] == 1.0  # arr.f[0] for first row
+    # special_floats[0] for second row (-inf)
+    assert math.isinf(ds[1][5]) and ds[1][5] < 0
+    assert ds[1][4] == float("inf")  # arr.f[0] for second row

{datachain-0.30.4 → datachain-0.30.5}/tests/func/test_data_storage.py

@@ -135,8 +135,8 @@ def test_convert_type(cloud_test_catalog):
 
     # JSON Tests
     assert run_convert_type('{"a": 1}', JSON()) == '{"a": 1}'
-    assert run_convert_type({"a": 1}, JSON()) == '{"a": 1}'
-    assert run_convert_type([{"a": 1}], JSON()) == '[{"a": 1}]'
+    assert run_convert_type({"a": 1}, JSON()) == '{"a":1}'
+    assert run_convert_type([{"a": 1}], JSON()) == '[{"a":1}]'
     with pytest.raises(ValueError):
         run_convert_type(0.5, JSON())
 

{datachain-0.30.4 → datachain-0.30.5}/tests/unit/lib/test_datachain.py

@@ -3230,6 +3230,35 @@ def test_read_csv_nan_inf(tmp_dir, test_session):
     assert any(r for r in res if np.isneginf(r))
 
 
+def test_dicts_nan_inf(test_session):
+    metrics_data = [
+        {"accuracy": 0.95, "loss": 0.1, "precision": 0.92},
+        {"accuracy": float("nan"), "loss": float("inf"), "precision": 0.88},
+        {"accuracy": 0.87, "loss": float("-inf"), "precision": float("nan")},
+    ]
+
+    dc.read_values(
+        id=[1, 2, 3],
+        metrics=metrics_data,
+        session=test_session,
+    ).save("test_dicts_nan_inf")
+
+    res = dc.read_dataset("test_dicts_nan_inf").order_by("id").to_values("metrics")
+    assert len(res) == 3
+
+    assert res[0]["accuracy"] == 0.95
+    assert res[0]["loss"] == 0.1
+    assert res[0]["precision"] == 0.92
+
+    assert math.isnan(res[1]["accuracy"])
+    assert math.isinf(res[1]["loss"]) and res[1]["loss"] > 0
+    assert res[1]["precision"] == 0.88
+
+    assert res[2]["accuracy"] == 0.87
+    assert math.isinf(res[2]["loss"]) and res[2]["loss"] < 0
+    assert math.isnan(res[2]["precision"])
+
+
 def test_group_by_int(test_session):
     from datachain import func