datachain 0.30.4__tar.gz → 0.30.6__tar.gz

{datachain-0.30.4 → datachain-0.30.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.30.4
+Version: 0.30.6
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -22,6 +22,7 @@ Requires-Dist: tomlkit
 Requires-Dist: tqdm
 Requires-Dist: numpy<3,>=1
 Requires-Dist: pandas>=2.0.0
+Requires-Dist: ujson>=5.10.0
 Requires-Dist: packaging
 Requires-Dist: pyarrow
 Requires-Dist: typing-extensions
@@ -38,7 +39,6 @@ Requires-Dist: shtab<2,>=1.3.4
 Requires-Dist: sqlalchemy>=2
 Requires-Dist: multiprocess==0.70.16
 Requires-Dist: cloudpickle
-Requires-Dist: orjson>=3.10.5
 Requires-Dist: pydantic
 Requires-Dist: jmespath>=1.0
 Requires-Dist: datamodel-code-generator>=0.25
@@ -92,7 +92,7 @@ Requires-Dist: pytest-mock>=3.12.0; extra == "tests"
 Requires-Dist: pytest-servers[all]>=0.5.9; extra == "tests"
 Requires-Dist: pytest-benchmark[histogram]; extra == "tests"
 Requires-Dist: pytest-xdist>=3.3.1; extra == "tests"
-Requires-Dist: pytest-env>=1.1.0; extra == "tests"
+Requires-Dist: pytest-dotenv; extra == "tests"
 Requires-Dist: virtualenv; extra == "tests"
 Requires-Dist: dulwich; extra == "tests"
 Requires-Dist: hypothesis; extra == "tests"

{datachain-0.30.4 → datachain-0.30.6}/docs/guide/delta.md

@@ -80,3 +80,23 @@ Delta processing can be combined with [retry processing](./retry.md) to create a
 
 1. Processes only new or changed records (delta)
 2. Reprocesses records with errors or that are missing (retry)
+
+## Using Delta with Restricted Methods
+
+By default, delta updates cannot be combined with the following methods:
+
+1. `merge`
+2. `union`
+3. `distinct`
+4. `agg`
+5. `group_by`
+
+These methods are restricted because they may produce **unexpected results** when used with delta processing. Delta runs the chain only on a subset of rows (new and changed records), while methods like `distinct`, `agg`, or `group_by` are designed to operate on the entire dataset.
+
+Similarly, combining delta with methods like `merge` or `union` may result in duplicated rows when merging with a static dataset.
+
+If you still need to use these methods together with delta, you can override this restriction by setting the additional flag:
+
+```python
+delta_unsafe=True
+```

{datachain-0.30.4 → datachain-0.30.6}/examples/get_started/json-csv-reader.py

@@ -1,3 +1,4 @@
+import os
 from typing import Optional
 
 import datachain as dc
@@ -39,7 +40,7 @@ def main():
     uri = "gs://datachain-demo/coco2017/annotations_captions/"
 
     # Print JSON schema in Pydantic format from main COCO annotation
-    chain = dc.read_storage(uri, anon="True").filter(dc.C("file.path").glob("*.json"))
+    chain = dc.read_storage(uri, anon=True).filter(dc.C("file.path").glob("*.json"))
     file = chain.limit(1).to_values("file")[0]
     print(gen_datamodel_code(file, jmespath="@", model_name="Coco"))
 
@@ -65,11 +66,12 @@ def main():
     dynamic_csv_ds.print_schema()
     dynamic_csv_ds.show()
 
-    print(
-        "Note: script might hang at the end due to https://github.com/apache/arrow/issues/43497"
-    )
-    print("Just press Ctrl+C to exit.")
-
 
 if __name__ == "__main__":
     main()
+
+    # Force exit without cleanup to avoid hanging due to arrow issue
+    print(
+        "Note: script might warn about leaked semaphore at the end due to https://github.com/apache/arrow/issues/43497"
+    )
+    os._exit(0)

datachain-0.30.6/examples/get_started/nested_datamodel.py

@@ -0,0 +1,70 @@
+"""Example: Nested DataModels with parallel execution.
+
+Demonstrates mapping a function that returns a nested DataModel (a DataModel
+containing other DataModels).
+
+The example keeps things minimal: we persist a tiny dataset, run a parallel map
+that returns a nested DataModel, and display the result.
+"""
+
+from typing import Optional
+
+from pydantic import Field
+
+import datachain as dc
+
+
+class Metric(dc.DataModel):
+    """Represents a single computed metric with quality metadata."""
+
+    value: Optional[float] = Field(default=None, description="Computed metric value")
+    confidence: Optional[float] = Field(
+        default=None, description="Confidence / quality score"
+    )
+    status: Optional[str] = Field(default=None, description="Processing status label")
+    metric_error: Optional[str] = Field(
+        default=None, description="Error message if metric computation failed"
+    )
+
+
+class SampleMetrics(dc.DataModel):
+    """Container for two illustrative nested metrics.
+
+    Each sub-field is its own DataModel instance to demonstrate nested schemas
+    """
+
+    metric_primary: Metric = Field(
+        default_factory=lambda: Metric(), description="Primary metric"
+    )
+    metric_secondary: Metric = Field(
+        default_factory=lambda: Metric(), description="Secondary metric"
+    )
+
+
+def generate_sample_metrics() -> SampleMetrics:
+    """Synthesize a pair of metrics.
+
+    In real scenarios you'd compute these values; here we just return constants
+    to keep the example deterministic.
+    """
+
+    return SampleMetrics(
+        metric_primary=Metric(value=50.0, confidence=0.95, status="ok"),
+    )
+
+
+def main():
+    (
+        dc.read_values(record_id=[1, 2])
+        .settings(parallel=2)  # Keep it parallel to test serialization
+        .map(metrics=generate_sample_metrics)
+        .save("nested_datamodel")
+    )
+
+    dc.read_dataset("nested_datamodel").show()
+
+    print(dc.read_dataset("nested_datamodel").to_values("metrics"))
+
+
+if __name__ == "__main__":
+    main()

{datachain-0.30.4 → datachain-0.30.6}/pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
   "tqdm",
   "numpy>=1,<3",
   "pandas>=2.0.0",
+  "ujson>=5.10.0",
   "packaging",
   "pyarrow",
   "typing-extensions",
@@ -42,7 +43,6 @@ dependencies = [
   "sqlalchemy>=2",
   "multiprocess==0.70.16",
   "cloudpickle",
-  "orjson>=3.10.5",
   "pydantic",
   "jmespath>=1.0",
   "datamodel-code-generator>=0.25",
@@ -108,7 +108,7 @@ tests = [
   "pytest-servers[all]>=0.5.9",
   "pytest-benchmark[histogram]",
   "pytest-xdist>=3.3.1",
-  "pytest-env>=1.1.0",
+  "pytest-dotenv",
   "virtualenv",
   "dulwich",
   "hypothesis",

{datachain-0.30.4 → datachain-0.30.6}/src/datachain/cli/commands/datasets.py

@@ -1,30 +1,41 @@
 import sys
-from typing import TYPE_CHECKING, Optional
+from collections.abc import Iterable, Iterator
+from typing import TYPE_CHECKING, Optional, Union
 
 from tabulate import tabulate
 
-if TYPE_CHECKING:
-    from datachain.catalog import Catalog
-
+from datachain import semver
 from datachain.catalog import is_namespace_local
 from datachain.cli.utils import determine_flavors
 from datachain.config import Config
 from datachain.error import DataChainError, DatasetNotFoundError
 from datachain.studio import list_datasets as list_datasets_studio
 
+if TYPE_CHECKING:
+    from datachain.catalog import Catalog
+
+
+def group_dataset_versions(
+    datasets: Iterable[tuple[str, str]], latest_only=True
+) -> dict[str, Union[str, list[str]]]:
+    grouped: dict[str, list[tuple[int, int, int]]] = {}
 
-def group_dataset_versions(datasets, latest_only=True):
-    grouped = {}
     # Sort to ensure groupby works as expected
     # (groupby expects consecutive items with the same key)
     for name, version in sorted(datasets):
-        grouped.setdefault(name, []).append(version)
+        grouped.setdefault(name, []).append(semver.parse(version))
 
     if latest_only:
         # For each dataset name, pick the highest version.
-        return {name: max(versions) for name, versions in grouped.items()}
+        return {
+            name: semver.create(*(max(versions))) for name, versions in grouped.items()
+        }
+
     # For each dataset name, return a sorted list of unique versions.
-    return {name: sorted(set(versions)) for name, versions in grouped.items()}
+    return {
+        name: [semver.create(*v) for v in sorted(set(versions))]
+        for name, versions in grouped.items()
+    }
 
 
 def list_datasets(
@@ -35,7 +46,7 @@ def list_datasets(
     team: Optional[str] = None,
     latest_only: bool = True,
     name: Optional[str] = None,
-):
+) -> None:
     token = Config().read().get("studio", {}).get("token")
     all, local, studio = determine_flavors(studio, local, all, token)
     if name:
@@ -95,27 +106,31 @@ def list_datasets(
     print(tabulate(rows, headers="keys"))
 
 
-def list_datasets_local(catalog: "Catalog", name: Optional[str] = None):
+def list_datasets_local(
+    catalog: "Catalog", name: Optional[str] = None
+) -> Iterator[tuple[str, str]]:
     if name:
         yield from list_datasets_local_versions(catalog, name)
         return
 
     for d in catalog.ls_datasets():
         for v in d.versions:
-            yield (d.full_name, v.version)
+            yield d.full_name, v.version
 
 
-def list_datasets_local_versions(catalog: "Catalog", name: str):
+def list_datasets_local_versions(
+    catalog: "Catalog", name: str
+) -> Iterator[tuple[str, str]]:
     namespace_name, project_name, name = catalog.get_full_dataset_name(name)
 
     ds = catalog.get_dataset(
         name, namespace_name=namespace_name, project_name=project_name
     )
     for v in ds.versions:
-        yield (name, v.version)
+        yield name, v.version
 
 
-def _datasets_tabulate_row(name, both, local_version, studio_version):
+def _datasets_tabulate_row(name, both, local_version, studio_version) -> dict[str, str]:
     row = {
         "Name": name,
     }
@@ -136,7 +151,7 @@ def rm_dataset(
     force: Optional[bool] = False,
     studio: Optional[bool] = False,
     team: Optional[str] = None,
-):
+) -> None:
     namespace_name, project_name, name = catalog.get_full_dataset_name(name)
 
     if studio:
@@ -166,7 +181,7 @@ def edit_dataset(
     description: Optional[str] = None,
     attrs: Optional[list[str]] = None,
     team: Optional[str] = None,
-):
+) -> None:
     from datachain.lib.dc.utils import is_studio
 
     namespace_name, project_name, name = catalog.get_full_dataset_name(name)

{datachain-0.30.4 → datachain-0.30.6}/src/datachain/data_storage/warehouse.py

@@ -1,5 +1,4 @@
 import glob
-import json
 import logging
 import posixpath
 import random
@@ -11,6 +10,7 @@ from urllib.parse import urlparse
 
 import attrs
 import sqlalchemy as sa
+import ujson as json
 from sqlalchemy.sql.expression import true
 
 from datachain.client import Client
@@ -122,7 +122,7 @@ class AbstractWarehouse(ABC, Serializable):
                 if value_type is str:
                     return val
                 if value_type in (dict, list):
-                    return json.dumps(val)
+                    return json.dumps(val, ensure_ascii=False)
                 raise ValueError(
                     f"Cannot convert value {val!r} with type {value_type} to JSON"
                 )

{datachain-0.30.4 → datachain-0.30.6}/src/datachain/delta.py

@@ -4,7 +4,7 @@ from functools import wraps
 from typing import TYPE_CHECKING, Callable, Optional, TypeVar, Union
 
 import datachain
-from datachain.dataset import DatasetDependency
+from datachain.dataset import DatasetDependency, DatasetRecord
 from datachain.error import DatasetNotFoundError
 from datachain.project import Project
 
@@ -30,9 +30,10 @@ def delta_disabled(
 
     @wraps(method)
     def _inner(self: T, *args: "P.args", **kwargs: "P.kwargs") -> T:
-        if self.delta:
+        if self.delta and not self._delta_unsafe:
             raise NotImplementedError(
-                f"Delta update cannot be used with {method.__name__}"
+                f"Cannot use {method.__name__} with delta datasets - may cause"
+                " inconsistency. Use delta_unsafe flag to allow this operation."
             )
         return method(self, *args, **kwargs)
 
@@ -124,10 +125,19 @@ def _get_retry_chain(
     # Subtract also diff chain since some items might be picked
     # up by `delta=True` itself (e.g. records got modified AND are missing in the
     # result dataset atm)
-    return retry_chain.subtract(diff_chain, on=on) if retry_chain else None
+    on = [on] if isinstance(on, str) else on
+
+    return (
+        retry_chain.diff(
+            diff_chain, on=on, added=True, same=True, modified=False, deleted=False
+        ).distinct(*on)
+        if retry_chain
+        else None
+    )
 
 
 def _get_source_info(
+    source_ds: DatasetRecord,
     name: str,
     namespace_name: str,
     project_name: str,
@@ -154,25 +164,23 @@ def _get_source_info(
         indirect=False,
     )
 
-    dep = dependencies[0]
-    if not dep:
+    source_ds_dep = next((d for d in dependencies if d.name == source_ds.name), None)
+    if not source_ds_dep:
         # Starting dataset was removed, back off to normal dataset creation
         return None, None, None, None, None
 
-    source_ds_project = catalog.metastore.get_project(dep.project, dep.namespace)
-    source_ds_name = dep.name
-    source_ds_version = dep.version
-    source_ds_latest_version = catalog.get_dataset(
-        source_ds_name,
-        namespace_name=source_ds_project.namespace.name,
-        project_name=source_ds_project.name,
-    ).latest_version
+    # Refresh starting dataset to have new versions if they are created
+    source_ds = catalog.get_dataset(
+        source_ds.name,
+        namespace_name=source_ds.project.namespace.name,
+        project_name=source_ds.project.name,
+    )
 
     return (
-        source_ds_name,
-        source_ds_project,
-        source_ds_version,
-        source_ds_latest_version,
+        source_ds.name,
+        source_ds.project,
+        source_ds_dep.version,
+        source_ds.latest_version,
         dependencies,
     )
 
@@ -244,7 +252,14 @@ def delta_retry_update(
         source_ds_version,
         source_ds_latest_version,
         dependencies,
-    ) = _get_source_info(name, namespace_name, project_name, latest_version, catalog)
+    ) = _get_source_info(
+        dc._query.starting_step.dataset,  # type: ignore[union-attr]
+        name,
+        namespace_name,
+        project_name,
+        latest_version,
+        catalog,
+    )
 
     # If source_ds_name is None, starting dataset was removed
     if source_ds_name is None:
@@ -267,8 +282,9 @@ def delta_retry_update(
     if dependencies:
         dependencies = copy(dependencies)
         dependencies = [d for d in dependencies if d is not None]
+        source_ds_dep = next(d for d in dependencies if d.name == source_ds_name)
         # Update to latest version
-        dependencies[0].version = source_ds_latest_version  # type: ignore[union-attr]
+        source_ds_dep.version = source_ds_latest_version  # type: ignore[union-attr]
 
     # Handle retry functionality if enabled
     if delta_retry:

{datachain-0.30.4 → datachain-0.30.6}/src/datachain/lib/arrow.py

@@ -2,8 +2,8 @@ from collections.abc import Sequence
 from itertools import islice
 from typing import TYPE_CHECKING, Any, Optional
 
-import orjson
 import pyarrow as pa
+import ujson as json
 from pyarrow._csv import ParseOptions
 from pyarrow.dataset import CsvFileFormat, dataset
 from tqdm.auto import tqdm
@@ -269,7 +269,7 @@ def _get_hf_schema(
 def _get_datachain_schema(schema: "pa.Schema") -> Optional[SignalSchema]:
     """Return a restored SignalSchema from parquet metadata, if any is found."""
     if schema.metadata and DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY in schema.metadata:
-        serialized_signal_schema = orjson.loads(
+        serialized_signal_schema = json.loads(
             schema.metadata[DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY]
         )
         return SignalSchema.deserialize(serialized_signal_schema)

{datachain-0.30.4 → datachain-0.30.6}/src/datachain/lib/dc/datachain.py

@@ -19,8 +19,8 @@ from typing import (
     overload,
 )
 
-import orjson
 import sqlalchemy
+import ujson as json
 from pydantic import BaseModel
 from sqlalchemy.sql.elements import ColumnElement
 from tqdm import tqdm
@@ -193,6 +193,7 @@ class DataChain:
         self._setup: dict = setup or {}
         self._sys = _sys
         self._delta = False
+        self._delta_unsafe = False
         self._delta_on: Optional[Union[str, Sequence[str]]] = None
         self._delta_result_on: Optional[Union[str, Sequence[str]]] = None
         self._delta_compare: Optional[Union[str, Sequence[str]]] = None
@@ -216,6 +217,7 @@ class DataChain:
         right_on: Optional[Union[str, Sequence[str]]] = None,
         compare: Optional[Union[str, Sequence[str]]] = None,
         delta_retry: Optional[Union[bool, str]] = None,
+        delta_unsafe: bool = False,
     ) -> "Self":
         """Marks this chain as delta, which means special delta process will be
         called on saving dataset for optimization"""
@@ -226,6 +228,7 @@ class DataChain:
         self._delta_result_on = right_on
         self._delta_compare = compare
         self._delta_retry = delta_retry
+        self._delta_unsafe = delta_unsafe
         return self
 
     @property
@@ -238,6 +241,10 @@ class DataChain:
         """Returns True if this chain is ran in "delta" update mode"""
         return self._delta
 
+    @property
+    def delta_unsafe(self) -> bool:
+        return self._delta_unsafe
+
     @property
     def schema(self) -> dict[str, DataType]:
         """Get schema of the chain."""
@@ -328,6 +335,7 @@ class DataChain:
                 right_on=self._delta_result_on,
                 compare=self._delta_compare,
                 delta_retry=self._delta_retry,
+                delta_unsafe=self._delta_unsafe,
             )
 
         return chain
@@ -462,8 +470,6 @@ class DataChain:
         Returns:
             DataChain: A new DataChain instance with the new set of columns.
         """
-        import json
-
         import pyarrow as pa
 
         from datachain.lib.arrow import schema_to_output
@@ -2129,9 +2135,9 @@ class DataChain:
             fsspec_fs = client.create_fs(**fs_kwargs)
 
         _partition_cols = list(partition_cols) if partition_cols else None
-        signal_schema_metadata = orjson.dumps(
-            self._effective_signals_schema.serialize()
-        )
+        signal_schema_metadata = json.dumps(
+            self._effective_signals_schema.serialize(), ensure_ascii=False
+        ).encode("utf-8")
 
         column_names, column_chunks = self.to_columnar_data_with_names(chunk_size)
 
@@ -2278,7 +2284,11 @@ class DataChain:
                         f.write(b"\n")
                 else:
                     is_first = False
-                f.write(orjson.dumps(row_to_nested_dict(headers, row)))
+                f.write(
+                    json.dumps(
+                        row_to_nested_dict(headers, row), ensure_ascii=False
+                    ).encode("utf-8")
+                )
             if include_outer_list:
                 # This makes the file JSON instead of JSON lines.
                 f.write(b"\n]\n")

{datachain-0.30.4 → datachain-0.30.6}/src/datachain/lib/dc/datasets.py

@@ -40,6 +40,7 @@ def read_dataset(
     delta_result_on: Optional[Union[str, Sequence[str]]] = None,
     delta_compare: Optional[Union[str, Sequence[str]]] = None,
     delta_retry: Optional[Union[bool, str]] = None,
+    delta_unsafe: bool = False,
     update: bool = False,
 ) -> "DataChain":
     """Get data from a saved Dataset. It returns the chain itself.
@@ -80,6 +81,8 @@ def read_dataset(
         update: If True always checks for newer versions available on Studio, even if
             some version of the dataset exists locally already. If False (default), it
             will only fetch the dataset from Studio if it is not found locally.
+        delta_unsafe: Allow restricted ops in delta: merge, agg, union, group_by,
+            distinct.
 
 
     Example:
@@ -205,6 +208,7 @@ def read_dataset(
             right_on=delta_result_on,
             compare=delta_compare,
             delta_retry=delta_retry,
+            delta_unsafe=delta_unsafe,
         )
 
     return chain

{datachain-0.30.4 → datachain-0.30.6}/src/datachain/lib/dc/storage.py

@@ -43,6 +43,7 @@ def read_storage(
     delta_result_on: Optional[Union[str, Sequence[str]]] = None,
     delta_compare: Optional[Union[str, Sequence[str]]] = None,
     delta_retry: Optional[Union[bool, str]] = None,
+    delta_unsafe: bool = False,
     client_config: Optional[dict] = None,
 ) -> "DataChain":
     """Get data from storage(s) as a list of file with all file attributes.
@@ -77,6 +78,9 @@ def read_storage(
               (error mode)
             - True: Reprocess records missing from the result dataset (missing mode)
             - None: No retry processing (default)
+        delta_unsafe: Allow restricted ops in delta: merge, agg, union, group_by,
+            distinct. Caller must ensure datasets are consistent and not partially
+            updated.
 
     Returns:
         DataChain: A DataChain object containing the file information.
@@ -218,6 +222,7 @@ def read_storage(
             right_on=delta_result_on,
             compare=delta_compare,
             delta_retry=delta_retry,
+            delta_unsafe=delta_unsafe,
         )
 
     return storage_chain

{datachain-0.30.4 → datachain-0.30.6}/src/datachain/lib/model_store.py

@@ -89,3 +89,15 @@ class ModelStore:
             and ModelStore.is_pydantic(parent_type)
             and "@" in ModelStore.get_name(parent_type)
         )
+
+    @classmethod
+    def rebuild_all(cls) -> None:
+        """Ensure pydantic schemas are (re)built for all registered models.
+
+        Uses ``force=True`` to avoid subtle cases where a deserialized class
+        (e.g. from by-value cloudpickle in workers) reports built state but
+        nested model field schemas aren't fully resolved yet.
+        """
+        for versions in cls.store.values():
+            for model in versions.values():
+                model.model_rebuild(force=True)

{datachain-0.30.4 → datachain-0.30.6}/src/datachain/query/dispatch.py

@@ -13,6 +13,7 @@ from multiprocess import get_context
 from datachain.catalog import Catalog
 from datachain.catalog.catalog import clone_catalog_with_cache
 from datachain.catalog.loader import DISTRIBUTED_IMPORT_PATH, get_udf_distributor_class
+from datachain.lib.model_store import ModelStore
 from datachain.lib.udf import _get_cache
 from datachain.query.dataset import (
     get_download_callback,
@@ -130,6 +131,8 @@ class UDFDispatcher:
 
     def _create_worker(self) -> "UDFWorker":
         udf: UDFAdapter = loads(self.udf_data)
+        # Ensure all registered DataModels have rebuilt schemas in worker processes.
+        ModelStore.rebuild_all()
         return UDFWorker(
             self.catalog,
             udf,
@@ -196,6 +199,8 @@ class UDFDispatcher:
         generated_cb: Callback = DEFAULT_CALLBACK,
     ) -> None:
         udf: UDFAdapter = loads(self.udf_data)
+        # Rebuild schemas in single process too for consistency (cheap, idempotent).
+        ModelStore.rebuild_all()
 
         if ids_only and not self.is_batching:
             input_rows = flatten(input_rows)