datachain 0.8.11__py3-none-any.whl → 0.8.12__py3-none-any.whl

datachain/catalog/catalog.py

@@ -38,7 +38,6 @@ from datachain.dataset import (
     DatasetDependency,
     DatasetListRecord,
     DatasetRecord,
-    DatasetStats,
     DatasetStatus,
     StorageURI,
     create_dataset_uri,
@@ -1235,17 +1234,6 @@ class Catalog:
         dataset = self.get_dataset(name)
         return self.warehouse.dataset_table_export_file_names(dataset, version)
 
-    def dataset_stats(self, name: str, version: Optional[int]) -> DatasetStats:
-        """
-        Returns tuple with dataset stats: total number of rows and total dataset size.
-        """
-        dataset = self.get_dataset(name)
-        dataset_version = dataset.get_version(version or dataset.latest_version)
-        return DatasetStats(
-            num_objects=dataset_version.num_objects,
-            size=dataset_version.size,
-        )
-
     def remove_dataset(
         self,
         name: str,
@@ -1391,19 +1379,12 @@ class Catalog:
         except DatasetNotFoundError:
             pass
 
-        stats_response = studio_client.dataset_stats(
-            remote_ds_name, remote_ds_version.version
-        )
-        if not stats_response.ok:
-            raise_remote_error(stats_response.message)
-        ds_stats = stats_response.data
-
         dataset_save_progress_bar = tqdm(
             desc=f"Saving dataset {remote_ds_uri} locally: ",
             unit=" rows",
             unit_scale=True,
             unit_divisor=1000,
-            total=ds_stats.num_objects,  # type: ignore [union-attr]
+            total=remote_ds_version.num_objects,  # type: ignore [union-attr]
             leave=False,
         )
 

datachain/cli/__init__.py

@@ -11,7 +11,6 @@ from datachain.telemetry import telemetry
 from .commands import (
     clear_cache,
     completion,
-    dataset_stats,
     du,
     edit_dataset,
     garbage_collect,
@@ -182,13 +181,6 @@ def handle_dataset_command(args, catalog):
             all=args.all,
             team=args.team,
         ),
-        "stats": lambda: dataset_stats(
-            catalog,
-            args.name,
-            args.version,
-            show_bytes=args.bytes,
-            si=args.si,
-        ),
     }
 
     handler = dataset_commands.get(args.datasets_cmd)

datachain/cli/commands/__init__.py

@@ -1,5 +1,4 @@
 from .datasets import (
-    dataset_stats,
     edit_dataset,
     list_datasets,
     list_datasets_local,
@@ -15,7 +14,6 @@ from .show import show
 __all__ = [
     "clear_cache",
     "completion",
-    "dataset_stats",
     "du",
     "edit_dataset",
     "garbage_collect",

datachain/cli/commands/datasets.py

@@ -3,8 +3,6 @@ from typing import TYPE_CHECKING, Optional
 
 from tabulate import tabulate
 
-from datachain import utils
-
 if TYPE_CHECKING:
     from datachain.catalog import Catalog
 
@@ -109,20 +107,3 @@ def edit_dataset(
 
     if (all or studio) and token:
         edit_studio_dataset(team, name, new_name, description, labels)
-
-
-def dataset_stats(
-    catalog: "Catalog",
-    name: str,
-    version: int,
-    show_bytes=False,
-    si=False,
-):
-    stats = catalog.dataset_stats(name, version)
-
-    if stats:
-        print(f"Number of objects: {stats.num_objects}")
-        if show_bytes:
-            print(f"Total objects size: {stats.size}")
-        else:
-            print(f"Total objects size: {utils.sizeof_fmt(stats.size, si=si): >7}")

datachain/cli/parser/__init__.py

@@ -307,31 +307,6 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         help="The team to delete a dataset. By default, it will use team from config",
     )
 
-    dataset_stats_parser = datasets_subparser.add_parser(
-        "stats", parents=[parent_parser], description="Show basic dataset statistics."
-    )
-    dataset_stats_parser.add_argument("name", type=str, help="Dataset name")
-    dataset_stats_parser.add_argument(
-        "--version",
-        action="store",
-        default=None,
-        type=int,
-        help="Dataset version",
-    )
-    dataset_stats_parser.add_argument(
-        "-b",
-        "--bytes",
-        default=False,
-        action="store_true",
-        help="Display size in bytes instead of human-readable size",
-    )
-    dataset_stats_parser.add_argument(
-        "--si",
-        default=False,
-        action="store_true",
-        help="Display size using powers of 1000 not 1024",
-    )
-
     parse_ls = subp.add_parser(
         "ls", parents=[parent_parser], description="List storage contents."
     )

datachain/dataset.py

@@ -150,12 +150,6 @@ class DatasetDependency:
         return hash(f"{self.type}_{self.name}_{self.version}")
 
 
-@dataclass
-class DatasetStats:
-    num_objects: Optional[int]  # None if table is missing
-    size: Optional[int]  # in bytes None if table is missing or empty
-
-
 class DatasetStatus:
     CREATED = 1
     PENDING = 2

datachain/func/conditional.py

@@ -9,7 +9,7 @@ from datachain.sql.functions import conditional
 
 from .func import ColT, Func
 
-CaseT = Union[int, float, complex, bool, str, Func]
+CaseT = Union[int, float, complex, bool, str, Func, ColumnElement]
 
 
 def greatest(*args: Union[ColT, float]) -> Func:
@@ -94,11 +94,12 @@ def case(
     """
     Returns the case function that produces case expression which has a list of
     conditions and corresponding results. Results can be python primitives like string,
-    numbers or booleans but can also be other nested function (including case function).
+    numbers or booleans but can also be other nested functions (including case function)
+    or columns.
     Result type is inferred from condition results.
 
     Args:
-        args (tuple((ColumnElement, Func), (str | int | float | complex | bool, Func))):
+        args tuple((ColumnElement | Func),(str | int | float | complex | bool, Func, ColumnElement)):
             Tuple of condition and values pair.
         else_ (str | int | float | complex | bool, Func): optional else value in case
             expression. If omitted, and no case conditions are satisfied, the result
@@ -113,13 +114,16 @@ def case(
             res=func.case((C("num") > 0, "P"), (C("num") < 0, "N"), else_="Z"),
         )
         ```
-    """
+    """  # noqa: E501
     supported_types = [int, float, complex, str, bool]
 
     def _get_type(val):
         if isinstance(val, Func):
             # nested functions
             return val.result_type
+        if isinstance(val, Column):
+            # at this point we cannot know what is the type of a column
+            return None
         return type(val)
 
     if not args:
@@ -129,13 +133,16 @@ def case(
 
     for arg in args:
         arg_type = _get_type(arg[1])
+        if arg_type is None:
+            # we couldn't figure out the type of case value
+            continue
         if type_ and arg_type != type_:
             raise DataChainParamsError(
                 f"Statement values must be of the same type, got {type_} and {arg_type}"
             )
         type_ = arg_type
 
-    if type_ not in supported_types:
+    if type_ is not None and type_ not in supported_types:
         raise DataChainParamsError(
             f"Only python literals ({supported_types}) are supported for values"
         )
@@ -151,15 +158,15 @@ def ifelse(
     """
     Returns the ifelse function that produces if expression which has a condition
     and values for true and false outcome. Results can be one of python primitives
-    like string, numbers or booleans, but can also be nested functions.
+    like string, numbers or booleans, but can also be nested functions or columns.
     Result type is inferred from the values.
 
     Args:
         condition (ColumnElement, Func):  Condition which is evaluated.
-        if_val (str | int | float | complex | bool, Func): Value for true
+        if_val (str | int | float | complex | bool, Func, ColumnElement): Value for true
             condition outcome.
-        else_val (str | int | float | complex | bool, Func): Value for false condition
-            outcome.
+        else_val (str | int | float | complex | bool, Func, ColumnElement): Value for
+            false condition outcome.
 
     Returns:
         Func: A Func object that represents the ifelse function.

datachain/func/func.py

@@ -424,10 +424,9 @@ class Func(Function):
 
 def get_db_col_type(signals_schema: "SignalSchema", col: ColT) -> "DataType":
     if isinstance(col, tuple):
-        raise DataChainParamsError(
-            "Cannot get type from tuple, please provide type hint to the function"
-        )
-
+        # we can only get tuple from case statement where the first tuple item
+        # is condition, and second one is value which type is important
+        col = col[1]
     if isinstance(col, Func):
         return col.get_result_type(signals_schema)
 
@@ -435,7 +434,7 @@ def get_db_col_type(signals_schema: "SignalSchema", col: ColT) -> "DataType":
         return sql_to_python(col)
 
     return signals_schema.get_column_type(
-        col.name if isinstance(col, ColumnElement) else col
+        col.name if isinstance(col, ColumnElement) else col  # type: ignore[arg-type]
     )
 
 

datachain/remote/studio.py

@@ -16,14 +16,12 @@ from urllib.parse import urlparse, urlunparse
 import websockets
 
 from datachain.config import Config
-from datachain.dataset import DatasetStats
 from datachain.error import DataChainError
 from datachain.utils import STUDIO_URL, retry_with_backoff
 
 T = TypeVar("T")
 LsData = Optional[list[dict[str, Any]]]
 DatasetInfoData = Optional[dict[str, Any]]
-DatasetStatsData = Optional[DatasetStats]
 DatasetRowsData = Optional[Iterable[dict[str, Any]]]
 DatasetJobVersionsData = Optional[dict[str, Any]]
 DatasetExportStatus = Optional[dict[str, Any]]
@@ -309,7 +307,7 @@ class StudioClient:
             "datachain/datasets",
             {
                 "dataset_name": name,
-                "version": version,
+                "dataset_version": version,
                 "force": force,
             },
             method="DELETE",
@@ -347,16 +345,6 @@ class StudioClient:
             method="GET",
         )
 
-    def dataset_stats(self, name: str, version: int) -> Response[DatasetStatsData]:
-        response = self._send_request(
-            "datachain/datasets/stats",
-            {"dataset_name": name, "dataset_version": version},
-            method="GET",
-        )
-        if response.ok:
-            response.data = DatasetStats(**response.data)
-        return response
-
     def export_dataset_table(
         self, name: str, version: int
     ) -> Response[DatasetExportSignedUrls]:

{datachain-0.8.11.dist-info → datachain-0.8.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.8.11
+Version: 0.8.12
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.8.11.dist-info → datachain-0.8.12.dist-info}/RECORD

@@ -3,7 +3,7 @@ datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=RH_jFwJcTXxhEFomaI9yL6S3Onau6NZ6FSKfKFGtrJE,9689
 datachain/cache.py,sha256=yQblPhOh_Mq74Ma7xT1CL1idLJ0HgrQxpGVYvRy_9Eg,3623
 datachain/config.py,sha256=g8qbNV0vW2VEKpX-dGZ9pAn0DAz6G2ZFcr7SAV3PoSM,4272
-datachain/dataset.py,sha256=8bUiEHsO_-Ziwxt7NWTSFM2s3R-EsgWaxTifWSETDv4,19155
+datachain/dataset.py,sha256=uqP6gtVFcVMVUFyB9Twr6Uk2onx-aBurbli8_VZu4-s,18993
 datachain/error.py,sha256=P1VI-etraA08ZrXHUEg1-xnOa2MkONd7vV0qA5uxBig,1314
 datachain/job.py,sha256=Jt4sNutMHJReaGsj3r3scueN5aESLGfhimAa8pUP7Is,1271
 datachain/listing.py,sha256=HNB-xeKA6aUA-HTWr--H22S6jVOxP2OVQ-3d07ISqAk,7109
@@ -16,20 +16,20 @@ datachain/studio.py,sha256=iMVDm9wfc86_G02N2p7qF4sdmKDGUkGz7kTOKc9m3Ao,9408
 datachain/telemetry.py,sha256=0A4IOPPp9VlP5pyW9eBfaTK3YhHGzHl7dQudQjUAx9A,994
 datachain/utils.py,sha256=LBeg-9n48saBTHSPk7u_j-kjJnPUAq5Oyps_peSaqlM,14128
 datachain/catalog/__init__.py,sha256=cMZzSz3VoUi-6qXSVaHYN-agxQuAcz2XSqnEPZ55crE,353
-datachain/catalog/catalog.py,sha256=gjGNnUMRXSu9Hhu63yad8pvxA_JB4ctDTnL0Wk0dZP8,59312
+datachain/catalog/catalog.py,sha256=Kg5JBfuf-e7QoiHx1wLKRq4h3KmWEMpCHpHLd-WBX9E,58611
 datachain/catalog/datasource.py,sha256=IkGMh0Ttg6Q-9DWfU_H05WUnZepbGa28HYleECi6K7I,1353
 datachain/catalog/loader.py,sha256=HA_mBC7q_My8j2WnSvIjUGuJpl6SIdg5vvy_lagxJlA,5733
-datachain/cli/__init__.py,sha256=vvxW6CekdfO2tF6VT3U7K4AOOfVZlfHVnArNJlOU7P4,8418
+datachain/cli/__init__.py,sha256=B6xw0qTcBgrICPqeWOhVXPaWJcxdKKg0Os6j2_IGAIc,8219
 datachain/cli/utils.py,sha256=wrLnAh7Wx8O_ojZE8AE4Lxn5WoxHbOj7as8NWlLAA74,3036
-datachain/cli/commands/__init__.py,sha256=uc77ggTRWrq-w1AVsH3Muy6v1ATkNsXUBPIRaOFgNus,533
-datachain/cli/commands/datasets.py,sha256=q1FkvFfeBCkuIuaA8pick0y51ZQuQK89ULUFse5xsu0,3583
+datachain/cli/commands/__init__.py,sha256=zp3bYIioO60x_X04A4-IpZqSYVnpwOa1AdERQaRlIhI,493
+datachain/cli/commands/datasets.py,sha256=k_CwJ_wYX-Jcc_Z8t9a8vX5jFHVt7qDQ0dehaA94iKs,3140
 datachain/cli/commands/du.py,sha256=9edEzDEs98K2VYk8Wf-ZMpUzALcgm9uD6YtoqbvtUGU,391
 datachain/cli/commands/index.py,sha256=eglNaIe1yyIadUHHumjtNbgIjht6kme7SS7xE3YHR88,198
 datachain/cli/commands/ls.py,sha256=Wb8hXyBwyhb62Zk6ZhNFPFrj2lJhdbRcnBQQkgL_qyw,5174
 datachain/cli/commands/misc.py,sha256=c0DmkOLwcDI2YhA8ArOuLJk6aGzSMZCiKL_E2JGibVE,600
 datachain/cli/commands/query.py,sha256=2S7hQxialt1fkbocxi6JXZI6jS5QnFrD1aOjKgZkzfI,1471
 datachain/cli/commands/show.py,sha256=RVb_7Kjd1kzqTxRKYFvmD04LaJHOtrCc4FYMyc-ZEYw,1149
-datachain/cli/parser/__init__.py,sha256=OYD2hOKxjEopTBAH8PTVmM5SROroFNPfLbBHka05KPk,15155
+datachain/cli/parser/__init__.py,sha256=mfOf3tbN4xGr4WQND1B5qMQ4LXoqEU9OhYac7wI-WBc,14393
 datachain/cli/parser/job.py,sha256=Zpi_bEsMp71YCr8xay0I93Taz8zS0_jHbxtvvTzXj6c,3197
 datachain/cli/parser/studio.py,sha256=CwmfdnsDNvDTOEbhLmjun18s4yo8zCgrtGTpF67qf8Q,2968
 datachain/cli/parser/utils.py,sha256=7ZtzGXfBAjVthnc-agz7R9rplnxqFan0LT6x2tq-6Fk,2007
@@ -54,8 +54,8 @@ datachain/func/__init__.py,sha256=qaSjakSaTsRtnU7Hcb4lJk71tbwk7M0oWmjRqXExCLA,10
 datachain/func/aggregate.py,sha256=7_IPrIwb2XSs3zG4iOr1eTvzn6kNVe2mkzvNzjusDHk,10942
 datachain/func/array.py,sha256=zHDNWuWLA7HVa9FEvQeHhVi00_xqenyleTqcLwkXWBI,5477
 datachain/func/base.py,sha256=wA0sBQAVyN9LPxoo7Ox83peS0zUVnyuKxukwAcjGLfY,534
-datachain/func/conditional.py,sha256=7f-fQfpCbirWSgN-pMn2FmDUp7l7LTqEa5L17bzBbIk,6037
-datachain/func/func.py,sha256=IfNOj5s4oyiViyAyaUJPgWAyT6TCdIIalur_U442Icg,16115
+datachain/func/conditional.py,sha256=g46zwW-i87uA45zWJnPHtHaqr6qOXSg6xLb4p9W3Gtk,6400
+datachain/func/func.py,sha256=PnwTRAiEJUus3e4NYdQ-hldqLzKS9hY0FjiyBMZhsSo,16183
 datachain/func/numeric.py,sha256=gMe1Ks0dqQKHkjcpvj7I5S-neECzQ_gltPQLNoaWOyo,5632
 datachain/func/path.py,sha256=mqN_mfkwv44z2II7DMTp_fGGw95hmTCNls_TOFNpr4k,3155
 datachain/func/random.py,sha256=pENOLj9rSmWfGCnOsUIaCsVC5486zQb66qfQvXaz9Z4,452
@@ -111,7 +111,7 @@ datachain/query/session.py,sha256=fQAtl5zRESRDfRS2d5J9KgrWauunCtrd96vP4Ns1KlE,59
 datachain/query/udf.py,sha256=GY8E9pnzPE7ZKl_jvetZpn9R2rlUtMlhoYj4UmrzFzw,594
 datachain/query/utils.py,sha256=u0A_BwG9PNs0DxoDcvSWgWLpj3ByTUv8CqH13CIuGag,1293
 datachain/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/remote/studio.py,sha256=8Ml5OMZsE2pA7fqdmVbLSyXeXcQ61mHxUWII9roAYQU,13362
+datachain/remote/studio.py,sha256=3v4ZqP06BwBMLXQ4mbcTS95oUodYgBv9A5XisL6ffWo,12915
 datachain/sql/__init__.py,sha256=6SQRdbljO3d2hx3EAVXEZrHQKv5jth0Jh98PogT59No,262
 datachain/sql/selectable.py,sha256=cTc60qVoAwqqss0Vop8Lt5Z-ROnM1XrQmL_GLjRxhXs,1765
 datachain/sql/types.py,sha256=ASSPkmM5EzdRindqj2O7WHLXq8VHAgFYedG8lYfGvVI,14045
@@ -133,9 +133,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=z3zRJNzjWrpPuRw-zgFbCOBKInyYxJew8ygrYQRQLNc,2930
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.8.11.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.8.11.dist-info/METADATA,sha256=6zSquPCm1gTPDYu2kYk0juVlOjtbTteekcERpBoD9uw,10880
-datachain-0.8.11.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-datachain-0.8.11.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.8.11.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.8.11.dist-info/RECORD,,
+datachain-0.8.12.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.8.12.dist-info/METADATA,sha256=C1vaFTVw44GIVe32CcfLthfCi5nbbqTgS7HL61iSFGg,10880
+datachain-0.8.12.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+datachain-0.8.12.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.8.12.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.8.12.dist-info/RECORD,,