datachain 0.34.3__tar.gz → 0.34.5__tar.gz

{datachain-0.34.3 → datachain-0.34.5}/.github/workflows/tests-studio.yml

@@ -32,7 +32,7 @@ jobs:
           POSTGRES_DB: database
           POSTGRES_HOST_AUTH_METHOD: trust
       clickhouse:
-        image: clickhouse/clickhouse-server:24.8
+        image: clickhouse/clickhouse-server:25.8
         ports:
           - 8123:8123
           - 9010:9000

{datachain-0.34.3 → datachain-0.34.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.34.3
+Version: 0.34.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.34.3 → datachain-0.34.5}/src/datachain/lib/signal_schema.py

@@ -1,6 +1,8 @@
 import copy
 import hashlib
 import json
+import logging
+import math
 import warnings
 from collections.abc import Iterator, Sequence
 from dataclasses import dataclass
@@ -23,7 +25,7 @@ from typing import (  # noqa: UP035
     get_origin,
 )
 
-from pydantic import BaseModel, Field, create_model
+from pydantic import BaseModel, Field, ValidationError, create_model
 from sqlalchemy import ColumnElement
 from typing_extensions import Literal as LiteralEx
 
@@ -43,6 +45,8 @@ if TYPE_CHECKING:
     from datachain.catalog import Catalog
 
 
+logger = logging.getLogger(__name__)
+
 NAMES_TO_TYPES = {
     "int": int,
     "str": str,
@@ -463,12 +467,32 @@ class SignalSchema:
                 objs.append(self.setup_values.get(name))
             elif (fr := ModelStore.to_pydantic(fr_type)) is not None:
                 j, pos = unflatten_to_json_pos(fr, row, pos)
-                objs.append(fr(**j))
+                try:
+                    obj = fr(**j)
+                except ValidationError as e:
+                    if self._all_values_none(j):
+                        logger.debug("Failed to create input for %s: %s", name, e)
+                        obj = None
+                    else:
+                        raise
+                objs.append(obj)
             else:
                 objs.append(row[pos])
                 pos += 1
         return objs
 
+    @staticmethod
+    def _all_values_none(value: Any) -> bool:
+        if isinstance(value, dict):
+            return all(SignalSchema._all_values_none(v) for v in value.values())
+        if isinstance(value, (list, tuple, set)):
+            return all(SignalSchema._all_values_none(v) for v in value)
+        if isinstance(value, float):
+            # NaN is used to represent NULL and NaN float values in datachain
+            # Since SQLite does not have a separate NULL type, we need to check for NaN
+            return math.isnan(value) or value is None
+        return value is None
+
     def get_file_signal(self) -> Optional[str]:
         for signal_name, signal_type in self.values.items():
             if (fr := ModelStore.to_pydantic(signal_type)) is not None and issubclass(
@@ -546,8 +570,15 @@ class SignalSchema:
                 pos += 1
             else:
                 json, pos = unflatten_to_json_pos(fr, row, pos)  # type: ignore[union-attr]
-                obj = fr(**json)
-                SignalSchema._set_file_stream(obj, catalog, cache)
+                try:
+                    obj = fr(**json)
+                    SignalSchema._set_file_stream(obj, catalog, cache)
+                except ValidationError as e:
+                    if self._all_values_none(json):
+                        logger.debug("Failed to create feature for %s: %s", fr_cls, e)
+                        obj = None
+                    else:
+                        raise
                 res.append(obj)
         return res
 

{datachain-0.34.3 → datachain-0.34.5}/src/datachain/lib/udf.py

@@ -160,9 +160,15 @@ class UDFBase(AbstractUDF):
         """
         Creates SHA hash of this UDF function. It takes into account function,
         inputs and outputs.
+
+        For function-based UDFs, hashes self._func.
+        For class-based UDFs, hashes the process method.
         """
+        # Hash user code: either _func (function-based) or process method (class-based)
+        func_to_hash = self._func if self._func else self.process
+
         parts = [
-            hash_callable(self._func),
+            hash_callable(func_to_hash),
             self.params.hash() if self.params else "",
             self.output.hash(),
         ]

{datachain-0.34.3 → datachain-0.34.5}/src/datachain/query/dataset.py

@@ -982,18 +982,26 @@ class SQLUnion(Step):
 
         columns1, columns2 = _order_columns(q1.columns, q2.columns)
 
+        union_select = sqlalchemy.select(*columns1).union_all(
+            sqlalchemy.select(*columns2)
+        )
+        union_cte = union_select.cte()
+        regenerated = self.query1.catalog.warehouse._regenerate_system_columns(
+            union_cte
+        )
+        result_columns = tuple(regenerated.selected_columns)
+
         def q(*columns):
-            names = {c.name for c in columns}
-            col1 = [c for c in columns1 if c.name in names]
-            col2 = [c for c in columns2 if c.name in names]
-            res = sqlalchemy.select(*col1).union_all(sqlalchemy.select(*col2))
+            if not columns:
+                return regenerated
 
-            subquery = res.subquery()
-            return sqlalchemy.select(*subquery.c).select_from(subquery)
+            names = {c.name for c in columns}
+            selected = [c for c in result_columns if c.name in names]
+            return regenerated.with_only_columns(*selected)
 
         return step_result(
             q,
-            columns1,
+            result_columns,
             dependencies=self.query1.dependencies | self.query2.dependencies,
         )
 
@@ -1009,7 +1017,9 @@ class SQLJoin(Step):
     rname: str
 
     def hash_inputs(self) -> str:
-        predicates = ensure_sequence(self.predicates or [])
+        predicates = (
+            ensure_sequence(self.predicates) if self.predicates is not None else []
+        )
 
         parts = [
             bytes.fromhex(self.query1.hash()),

{datachain-0.34.3 → datachain-0.34.5}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.34.3
+Version: 0.34.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.34.3 → datachain-0.34.5}/src/datachain.egg-info/SOURCES.txt

@@ -334,6 +334,8 @@ tests/func/test_storage_pattern.py
 tests/func/test_studio_datetime_parsing.py
 tests/func/test_to_database.py
 tests/func/test_toolkit.py
+tests/func/test_udf.py
+tests/func/test_union.py
 tests/func/test_video.py
 tests/func/test_warehouse.py
 tests/func/data/Big_Buck_Bunny_360_10s_1MB.mp4