datachain 0.34.3__py3-none-any.whl → 0.34.5__py3-none-any.whl

datachain/lib/signal_schema.py

@@ -1,6 +1,8 @@
 import copy
 import hashlib
 import json
+import logging
+import math
 import warnings
 from collections.abc import Iterator, Sequence
 from dataclasses import dataclass
@@ -23,7 +25,7 @@ from typing import (  # noqa: UP035
     get_origin,
 )
 
-from pydantic import BaseModel, Field, create_model
+from pydantic import BaseModel, Field, ValidationError, create_model
 from sqlalchemy import ColumnElement
 from typing_extensions import Literal as LiteralEx
 
@@ -43,6 +45,8 @@ if TYPE_CHECKING:
     from datachain.catalog import Catalog
 
 
+logger = logging.getLogger(__name__)
+
 NAMES_TO_TYPES = {
     "int": int,
     "str": str,
@@ -463,12 +467,32 @@ class SignalSchema:
                 objs.append(self.setup_values.get(name))
             elif (fr := ModelStore.to_pydantic(fr_type)) is not None:
                 j, pos = unflatten_to_json_pos(fr, row, pos)
-                objs.append(fr(**j))
+                try:
+                    obj = fr(**j)
+                except ValidationError as e:
+                    if self._all_values_none(j):
+                        logger.debug("Failed to create input for %s: %s", name, e)
+                        obj = None
+                    else:
+                        raise
+                objs.append(obj)
             else:
                 objs.append(row[pos])
                 pos += 1
         return objs
 
+    @staticmethod
+    def _all_values_none(value: Any) -> bool:
+        if isinstance(value, dict):
+            return all(SignalSchema._all_values_none(v) for v in value.values())
+        if isinstance(value, (list, tuple, set)):
+            return all(SignalSchema._all_values_none(v) for v in value)
+        if isinstance(value, float):
+            # NaN is used to represent NULL and NaN float values in datachain
+            # Since SQLite does not have a separate NULL type, we need to check for NaN
+            return math.isnan(value) or value is None
+        return value is None
+
     def get_file_signal(self) -> Optional[str]:
         for signal_name, signal_type in self.values.items():
             if (fr := ModelStore.to_pydantic(signal_type)) is not None and issubclass(
@@ -546,8 +570,15 @@ class SignalSchema:
                 pos += 1
             else:
                 json, pos = unflatten_to_json_pos(fr, row, pos)  # type: ignore[union-attr]
-                obj = fr(**json)
-                SignalSchema._set_file_stream(obj, catalog, cache)
+                try:
+                    obj = fr(**json)
+                    SignalSchema._set_file_stream(obj, catalog, cache)
+                except ValidationError as e:
+                    if self._all_values_none(json):
+                        logger.debug("Failed to create feature for %s: %s", fr_cls, e)
+                        obj = None
+                    else:
+                        raise
                 res.append(obj)
         return res
 

datachain/lib/udf.py

@@ -160,9 +160,15 @@ class UDFBase(AbstractUDF):
         """
         Creates SHA hash of this UDF function. It takes into account function,
         inputs and outputs.
+
+        For function-based UDFs, hashes self._func.
+        For class-based UDFs, hashes the process method.
         """
+        # Hash user code: either _func (function-based) or process method (class-based)
+        func_to_hash = self._func if self._func else self.process
+
         parts = [
-            hash_callable(self._func),
+            hash_callable(func_to_hash),
             self.params.hash() if self.params else "",
             self.output.hash(),
         ]

datachain/query/dataset.py

@@ -982,18 +982,26 @@ class SQLUnion(Step):
 
         columns1, columns2 = _order_columns(q1.columns, q2.columns)
 
+        union_select = sqlalchemy.select(*columns1).union_all(
+            sqlalchemy.select(*columns2)
+        )
+        union_cte = union_select.cte()
+        regenerated = self.query1.catalog.warehouse._regenerate_system_columns(
+            union_cte
+        )
+        result_columns = tuple(regenerated.selected_columns)
+
         def q(*columns):
-            names = {c.name for c in columns}
-            col1 = [c for c in columns1 if c.name in names]
-            col2 = [c for c in columns2 if c.name in names]
-            res = sqlalchemy.select(*col1).union_all(sqlalchemy.select(*col2))
+            if not columns:
+                return regenerated
 
-            subquery = res.subquery()
-            return sqlalchemy.select(*subquery.c).select_from(subquery)
+            names = {c.name for c in columns}
+            selected = [c for c in result_columns if c.name in names]
+            return regenerated.with_only_columns(*selected)
 
         return step_result(
             q,
-            columns1,
+            result_columns,
             dependencies=self.query1.dependencies | self.query2.dependencies,
         )
 
@@ -1009,7 +1017,9 @@ class SQLJoin(Step):
     rname: str
 
     def hash_inputs(self) -> str:
-        predicates = ensure_sequence(self.predicates or [])
+        predicates = (
+            ensure_sequence(self.predicates) if self.predicates is not None else []
+        )
 
         parts = [
             bytes.fromhex(self.query1.hash()),

{datachain-0.34.3.dist-info → datachain-0.34.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.34.3
+Version: 0.34.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.34.3.dist-info → datachain-0.34.5.dist-info}/RECORD

@@ -90,10 +90,10 @@ datachain/lib/namespaces.py,sha256=ZyIYUa3WMrv6R5HrSoLsmLiEbvUQDl8sBINLUmWOYG0,3
 datachain/lib/projects.py,sha256=_YeU9PPcH_pC8-sbX-47XtWSdl1ltVKnALY8azWLJkM,4112
 datachain/lib/pytorch.py,sha256=S-st2SAczYut13KMf6eSqP_OQ8otWI5TRmzhK5fN3k0,7828
 datachain/lib/settings.py,sha256=xBQEPZfgaYKhHIFLd0u5CBTYDcJS8ZHCm47x7GJErFU,7666
-datachain/lib/signal_schema.py,sha256=WDFLbzXEOhgv865TePcFpLQHxsKQHtn8kTzaQGUG_XA,39479
+datachain/lib/signal_schema.py,sha256=NsL2ISnSRN-lKRpXzB9CtsUj2tVKcoAe73TaaZKMT-0,40774
 datachain/lib/tar.py,sha256=MLcVjzIgBqRuJacCNpZ6kwSZNq1i2tLyROc8PVprHsA,999
 datachain/lib/text.py,sha256=UNHm8fhidk7wdrWqacEWaA6I9ykfYqarQ2URby7jc7M,1261
-datachain/lib/udf.py,sha256=DdUxGBo9Y7Jz6aTBKgwex7YfK1RNaGm1JUlXCqs7qnw,18122
+datachain/lib/udf.py,sha256=bU_7xj6Mz4SsajFJ6tmrZm9Ygoi1ESoOyh1Q00W2zX4,18389
 datachain/lib/udf_signature.py,sha256=Yz20iJ-WF1pijT3hvcDIKFzgWV9gFxZM73KZRx3NbPk,7560
 datachain/lib/utils.py,sha256=RLji1gHnfDXtJCnBo8BcNu1obndFpVsXJ_1Vb-FQ9Qo,4554
 datachain/lib/video.py,sha256=ddVstiMkfxyBPDsnjCKY0d_93bw-DcMqGqN60yzsZoo,6851
@@ -131,7 +131,7 @@ datachain/model/ultralytics/pose.py,sha256=pvoXrWWUSWT_UBaMwUb5MBHAY57Co2HFDPigF
 datachain/model/ultralytics/segment.py,sha256=v9_xDxd5zw_I8rXsbl7yQXgEdTs2T38zyY_Y4XGN8ok,3194
 datachain/query/__init__.py,sha256=7DhEIjAA8uZJfejruAVMZVcGFmvUpffuZJwgRqNwe-c,263
 datachain/query/batch.py,sha256=ocPeNgrJM6Y_6SYCx3O2cwlCFAhNMfoYgB99GP6A1Bg,4294
-datachain/query/dataset.py,sha256=P7pyRiWc9G3AfzxvyB2yToKW3bXoUCrfFOtFdiVbCrU,67836
+datachain/query/dataset.py,sha256=I55ubMnoWpjoc4Ntw8zbp-i-49w0I95J7hCk_OCU6IU,68110
 datachain/query/dispatch.py,sha256=pygp7xg3lUDKlYHhecKxW5fB3zOSX1fPJfZBU4dfijk,16067
 datachain/query/metrics.py,sha256=DOK5HdNVaRugYPjl8qnBONvTkwjMloLqAr7Mi3TjCO0,858
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
@@ -165,9 +165,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.34.3.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.34.3.dist-info/METADATA,sha256=l1d5np6lvB4K8ohVibIbhzlNobGtlglmBhK0VcQqV-U,13655
-datachain-0.34.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.34.3.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.34.3.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.34.3.dist-info/RECORD,,
+datachain-0.34.5.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.34.5.dist-info/METADATA,sha256=aovavI1WF3QvHC23Vcp4HxUykcEFNypIK3LlKo_wrDA,13655
+datachain-0.34.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.34.5.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.34.5.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.34.5.dist-info/RECORD,,