datachain 0.32.0__py3-none-any.whl → 0.32.1__py3-none-any.whl

datachain/__init__.py

@@ -37,7 +37,7 @@ from datachain.lib.file import (
     VideoFrame,
 )
 from datachain.lib.model_store import ModelStore
-from datachain.lib.namespaces import delete as delete_namespace
+from datachain.lib.namespaces import delete_namespace
 from datachain.lib.projects import create as create_project
 from datachain.lib.udf import Aggregator, Generator, Mapper
 from datachain.lib.utils import AbstractUDF, DataChainError

datachain/lib/convert/python_to_sql.py

@@ -1,8 +1,14 @@
 import inspect
+import sys
 from datetime import datetime
 from enum import Enum
 from typing import Annotated, Literal, Union, get_args, get_origin
 
+if sys.version_info >= (3, 10):
+    from types import UnionType
+else:
+    UnionType = None
+
 from pydantic import BaseModel
 from typing_extensions import Literal as LiteralEx
 
@@ -34,6 +40,13 @@ PYTHON_TO_SQL = {
 }
 
 
+def _is_union(orig) -> bool:
+    if orig == Union:
+        return True
+    # some code is unreachab in python<3.10
+    return UnionType is not None and orig is UnionType  # type: ignore[unreachable]
+
+
 def python_to_sql(typ):  # noqa: PLR0911
     if inspect.isclass(typ):
         if issubclass(typ, SQLType):
@@ -69,9 +82,10 @@ def python_to_sql(typ):  # noqa: PLR0911
     if inspect.isclass(orig) and issubclass(dict, orig):
         return JSON
 
-    if orig == Union:
+    if _is_union(orig):
         if len(args) == 2 and (type(None) in args):
-            return python_to_sql(args[0])
+            non_none_arg = args[0] if args[0] is not type(None) else args[1]
+            return python_to_sql(non_none_arg)
 
         if _is_union_str_literal(orig, args):
             return String
@@ -95,7 +109,7 @@ def list_of_args_to_type(args) -> SQLType:
 
 
 def _is_json_inside_union(orig, args) -> bool:
-    if orig == Union and len(args) >= 2:
+    if _is_union(orig) and len(args) >= 2:
         # List in JSON: Union[dict, list[dict]]
         args_no_nones = [arg for arg in args if arg != type(None)]  # noqa: E721
         if len(args_no_nones) == 2:
@@ -112,6 +126,6 @@ def _is_json_inside_union(orig, args) -> bool:
 
 
 def _is_union_str_literal(orig, args) -> bool:
-    if orig != Union:
+    if not _is_union(orig):
         return False
     return all(arg is str or get_origin(arg) in (Literal, LiteralEx) for arg in args)

datachain/lib/namespaces.py

@@ -77,7 +77,7 @@ def ls(session: Optional[Session] = None) -> list[Namespace]:
     return Session.get(session).catalog.metastore.list_namespaces()
 
 
-def delete(name: str, session: Optional[Session]) -> None:
+def delete_namespace(name: str, session: Optional[Session]) -> None:
     """
     Removes a namespace by name.
 
@@ -88,14 +88,13 @@ def delete(name: str, session: Optional[Session]) -> None:
             as these cannot be removed.
 
     Parameters:
-        name : The name of the namespace.
-        session : Session to use for getting project.
+        name: The name of the namespace.
+        session: Session to use for getting project.
 
     Example:
         ```py
         import datachain as dc
-        from datachain.lib.namespace import delete as delete_namespace
-        delete_namespace("dev")
+        dc.delete_namespace("dev")
         ```
     """
     session = Session.get(session)

{datachain-0.32.0.dist-info → datachain-0.32.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.32.0
+Version: 0.32.1
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -210,45 +210,33 @@ datasets that evolve over time and may occasionally have processing errors.
 .. code:: py
 
     import datachain as dc
-    from datachain import C, File
 
-    def process_file(file: File):
-        """Process a file, which may occasionally fail."""
+    def process_file(file: dc.File) -> tuple[str, str, str]:
+        """Analyze a file, may occasionally fail."""
         try:
             # Your processing logic here
             content = file.read_text()
-            result = analyze_content(content)
-            return {
-                "content": content,
-                "result": result,
-                "error": None  # No error
-            }
+            result = content.upper()
+            return content, result, ""  # No error
         except Exception as e:
             # Return an error that will trigger reprocessing next time
-            return {
-                "content": None,
-                "result": None,
-                "error": str(e)  # Error field will trigger retry
-            }
+            return "", "", str(e)  # Error field will trigger retry
 
     # Process files efficiently with delta and retry
+    # Run it many times, keep adding files, to see delta and retry in action
     chain = (
         dc.read_storage(
             "data/",
             update=True,
             delta=True,              # Process only new/changed files
             delta_on="file.path",    # Identify files by path
-            retry_on="error"         # Field that indicates errors
+            delta_retry="error",     # Process files with error again
         )
-        .map(processed_result=process_file)
-        .mutate(
-            content=C("processed_result.content"),
-            result=C("processed_result.result"),
-            error=C("processed_result.error")
-        )
-        .save(name="processed_data")
+        .map(process_file, output=("content", "result", "error"))
+        .save("processed-data")
     )
 
+
 Example: LLM based text-file evaluation
 ---------------------------------------
 

{datachain-0.32.0.dist-info → datachain-0.32.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-datachain/__init__.py,sha256=5DFB1P58c35C_WBMrhmaynsP1WwCukC-9gTJIaPy0E8,1832
+datachain/__init__.py,sha256=BRqfLPoBRRycnndaxyba-i4ZrZCJl0As2pwV9RiNBr8,1822
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=RH_jFwJcTXxhEFomaI9yL6S3Onau6NZ6FSKfKFGtrJE,9689
 datachain/cache.py,sha256=ESVRaCJXEThMIfGEFVHx6wJPOZA7FYk9V6WxjyuqUBY,3626
@@ -82,7 +82,7 @@ datachain/lib/listing.py,sha256=U-2stsTEwEsq4Y80dqGfktGzkmB5-ZntnL1_rzXlH0k,7089
 datachain/lib/listing_info.py,sha256=9ua40Hw0aiQByUw3oAEeNzMavJYfW0Uhe8YdCTK-m_g,1110
 datachain/lib/meta_formats.py,sha256=zdyg6XLk3QIsSk3I7s0Ez5kaCJSlE3uq7JiGxf7UwtU,6348
 datachain/lib/model_store.py,sha256=A0pSVQ7uaZ9RvANapzirF8Cqq9N6ysosPpMSkzdRPkU,3226
-datachain/lib/namespaces.py,sha256=ij67QHnRJhC8uquR21aD8u1Um2jfxnBX8PecuOQZpYw,3828
+datachain/lib/namespaces.py,sha256=IMfEgs680C3XpaLAiWIPhy8Wry4jT8iyRslcKV2cFOM,3768
 datachain/lib/projects.py,sha256=_YeU9PPcH_pC8-sbX-47XtWSdl1ltVKnALY8azWLJkM,4112
 datachain/lib/pytorch.py,sha256=S-st2SAczYut13KMf6eSqP_OQ8otWI5TRmzhK5fN3k0,7828
 datachain/lib/settings.py,sha256=xBQEPZfgaYKhHIFLd0u5CBTYDcJS8ZHCm47x7GJErFU,7666
@@ -97,7 +97,7 @@ datachain/lib/webdataset.py,sha256=CkW8FfGigNx6wo2EEK4KMjhEE8FamRHWGs2HZuH7jDY,7
 datachain/lib/webdataset_laion.py,sha256=xvT6m_r5y0KbOx14BUe7UC5mOgrktJq53Mh-H0EVlUE,2525
 datachain/lib/convert/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/convert/flatten.py,sha256=_5rjGFnN6t1KCX5ftL5rG7tiiNat7j0SdNqajO15KUY,1539
-datachain/lib/convert/python_to_sql.py,sha256=wg-O5FRKX3x3Wh8ZL1b9ntMlgf1zRO4djMP3t8CHJLo,3188
+datachain/lib/convert/python_to_sql.py,sha256=ACIHtiPujlG9DRChSlxifcMJCls1PvrB273w_cgR6OQ,3584
 datachain/lib/convert/sql_to_python.py,sha256=Gxc4FylWC_Pvvuawuc2MKZIiuAWI7wje8pyeN1MxRrU,670
 datachain/lib/convert/unflatten.py,sha256=ysMkstwJzPMWUlnxn-Z-tXJR3wmhjHeSN_P-sDcLS6s,2010
 datachain/lib/convert/values_to_tuples.py,sha256=j5yZMrVUH6W7b-7yUvdCTGI7JCUAYUOzHUGPoyZXAB0,4360
@@ -161,9 +161,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.32.0.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.32.0.dist-info/METADATA,sha256=gLsH5khLc-z_s0MKTt3H6qk_UXDAaWuHy2lk1yytgBw,13898
-datachain-0.32.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.32.0.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.32.0.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.32.0.dist-info/RECORD,,
+datachain-0.32.1.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.32.1.dist-info/METADATA,sha256=4T7E05Y-6wELDtaFv4De-D3dVQPk3Fm_-B3WqKkLvQQ,13607
+datachain-0.32.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.32.1.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.32.1.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.32.1.dist-info/RECORD,,