datachain 0.8.11__tar.gz → 0.8.12__tar.gz

{datachain-0.8.11 → datachain-0.8.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.8.11
+Version: 0.8.12
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

datachain-0.8.12/docs/references/func.md

@@ -0,0 +1,5 @@
+# Functions
+
+Use built-in functions for data manipulation and analysis to operate on the underlying database storing the chain data. These functions are useful for operations like [`DataChain.filter`](datachain.md#datachain.lib.dc.DataChain.filter) and [`DataChain.mutate`](datachain.md#datachain.lib.dc.DataChain.mutate). Import these functions from `datachain.func`.
+
+::: datachain.func

{datachain-0.8.11 → datachain-0.8.12}/docs/references/index.md

@@ -10,5 +10,5 @@ DataChain's API is organized into several modules:
 - [DataType](./datatype.md) - Type system and schema definitions
 - [File](./file.md) - File handling and storage operations
 - [UDF](./udf.md) - User-defined functions and transformations
-- [SQL](./sql.md) - SQL query integration
+- [Functions](./func.md) - Built-in functions for data manipulation and analysis
 - [Torch](./torch.md) - PyTorch data loading utilities

{datachain-0.8.11 → datachain-0.8.12}/mkdocs.yml

@@ -73,7 +73,7 @@ nav:
           - File: references/file.md
           - UDF: references/udf.md
           - Torch: references/torch.md
-          - SQL: references/sql.md
+          - Functions: references/func.md
       - 🤝 Contributing: contributing.md
 
   - DataChain Website ↗: https://datachain.ai" target="_blank"

{datachain-0.8.11 → datachain-0.8.12}/src/datachain/catalog/catalog.py

@@ -38,7 +38,6 @@ from datachain.dataset import (
     DatasetDependency,
     DatasetListRecord,
     DatasetRecord,
-    DatasetStats,
     DatasetStatus,
     StorageURI,
     create_dataset_uri,
@@ -1235,17 +1234,6 @@ class Catalog:
         dataset = self.get_dataset(name)
         return self.warehouse.dataset_table_export_file_names(dataset, version)
 
-    def dataset_stats(self, name: str, version: Optional[int]) -> DatasetStats:
-        """
-        Returns tuple with dataset stats: total number of rows and total dataset size.
-        """
-        dataset = self.get_dataset(name)
-        dataset_version = dataset.get_version(version or dataset.latest_version)
-        return DatasetStats(
-            num_objects=dataset_version.num_objects,
-            size=dataset_version.size,
-        )
-
     def remove_dataset(
         self,
         name: str,
@@ -1391,19 +1379,12 @@ class Catalog:
         except DatasetNotFoundError:
             pass
 
-        stats_response = studio_client.dataset_stats(
-            remote_ds_name, remote_ds_version.version
-        )
-        if not stats_response.ok:
-            raise_remote_error(stats_response.message)
-        ds_stats = stats_response.data
-
         dataset_save_progress_bar = tqdm(
             desc=f"Saving dataset {remote_ds_uri} locally: ",
             unit=" rows",
             unit_scale=True,
             unit_divisor=1000,
-            total=ds_stats.num_objects,  # type: ignore [union-attr]
+            total=remote_ds_version.num_objects,  # type: ignore [union-attr]
             leave=False,
         )
 

{datachain-0.8.11 → datachain-0.8.12}/src/datachain/cli/__init__.py

@@ -11,7 +11,6 @@ from datachain.telemetry import telemetry
 from .commands import (
     clear_cache,
     completion,
-    dataset_stats,
     du,
     edit_dataset,
     garbage_collect,
@@ -182,13 +181,6 @@ def handle_dataset_command(args, catalog):
             all=args.all,
             team=args.team,
         ),
-        "stats": lambda: dataset_stats(
-            catalog,
-            args.name,
-            args.version,
-            show_bytes=args.bytes,
-            si=args.si,
-        ),
     }
 
     handler = dataset_commands.get(args.datasets_cmd)

{datachain-0.8.11 → datachain-0.8.12}/src/datachain/cli/commands/__init__.py

@@ -1,5 +1,4 @@
 from .datasets import (
-    dataset_stats,
     edit_dataset,
     list_datasets,
     list_datasets_local,
@@ -15,7 +14,6 @@ from .show import show
 __all__ = [
     "clear_cache",
     "completion",
-    "dataset_stats",
     "du",
     "edit_dataset",
     "garbage_collect",

{datachain-0.8.11 → datachain-0.8.12}/src/datachain/cli/commands/datasets.py

@@ -3,8 +3,6 @@ from typing import TYPE_CHECKING, Optional
 
 from tabulate import tabulate
 
-from datachain import utils
-
 if TYPE_CHECKING:
     from datachain.catalog import Catalog
 
@@ -109,20 +107,3 @@ def edit_dataset(
 
     if (all or studio) and token:
         edit_studio_dataset(team, name, new_name, description, labels)
-
-
-def dataset_stats(
-    catalog: "Catalog",
-    name: str,
-    version: int,
-    show_bytes=False,
-    si=False,
-):
-    stats = catalog.dataset_stats(name, version)
-
-    if stats:
-        print(f"Number of objects: {stats.num_objects}")
-        if show_bytes:
-            print(f"Total objects size: {stats.size}")
-        else:
-            print(f"Total objects size: {utils.sizeof_fmt(stats.size, si=si): >7}")

{datachain-0.8.11 → datachain-0.8.12}/src/datachain/cli/parser/__init__.py

@@ -307,31 +307,6 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         help="The team to delete a dataset. By default, it will use team from config",
     )
 
-    dataset_stats_parser = datasets_subparser.add_parser(
-        "stats", parents=[parent_parser], description="Show basic dataset statistics."
-    )
-    dataset_stats_parser.add_argument("name", type=str, help="Dataset name")
-    dataset_stats_parser.add_argument(
-        "--version",
-        action="store",
-        default=None,
-        type=int,
-        help="Dataset version",
-    )
-    dataset_stats_parser.add_argument(
-        "-b",
-        "--bytes",
-        default=False,
-        action="store_true",
-        help="Display size in bytes instead of human-readable size",
-    )
-    dataset_stats_parser.add_argument(
-        "--si",
-        default=False,
-        action="store_true",
-        help="Display size using powers of 1000 not 1024",
-    )
-
     parse_ls = subp.add_parser(
         "ls", parents=[parent_parser], description="List storage contents."
     )

{datachain-0.8.11 → datachain-0.8.12}/src/datachain/dataset.py

@@ -150,12 +150,6 @@ class DatasetDependency:
         return hash(f"{self.type}_{self.name}_{self.version}")
 
 
-@dataclass
-class DatasetStats:
-    num_objects: Optional[int]  # None if table is missing
-    size: Optional[int]  # in bytes None if table is missing or empty
-
-
 class DatasetStatus:
     CREATED = 1
     PENDING = 2

{datachain-0.8.11 → datachain-0.8.12}/src/datachain/func/conditional.py

@@ -9,7 +9,7 @@ from datachain.sql.functions import conditional
 
 from .func import ColT, Func
 
-CaseT = Union[int, float, complex, bool, str, Func]
+CaseT = Union[int, float, complex, bool, str, Func, ColumnElement]
 
 
 def greatest(*args: Union[ColT, float]) -> Func:
@@ -94,11 +94,12 @@ def case(
     """
     Returns the case function that produces case expression which has a list of
     conditions and corresponding results. Results can be python primitives like string,
-    numbers or booleans but can also be other nested function (including case function).
+    numbers or booleans but can also be other nested functions (including case function)
+    or columns.
     Result type is inferred from condition results.
 
     Args:
-        args (tuple((ColumnElement, Func), (str | int | float | complex | bool, Func))):
+        args tuple((ColumnElement | Func),(str | int | float | complex | bool, Func, ColumnElement)):
             Tuple of condition and values pair.
         else_ (str | int | float | complex | bool, Func): optional else value in case
             expression. If omitted, and no case conditions are satisfied, the result
@@ -113,13 +114,16 @@ def case(
             res=func.case((C("num") > 0, "P"), (C("num") < 0, "N"), else_="Z"),
         )
         ```
-    """
+    """  # noqa: E501
     supported_types = [int, float, complex, str, bool]
 
     def _get_type(val):
         if isinstance(val, Func):
             # nested functions
             return val.result_type
+        if isinstance(val, Column):
+            # at this point we cannot know what is the type of a column
+            return None
         return type(val)
 
     if not args:
@@ -129,13 +133,16 @@ def case(
 
     for arg in args:
         arg_type = _get_type(arg[1])
+        if arg_type is None:
+            # we couldn't figure out the type of case value
+            continue
         if type_ and arg_type != type_:
             raise DataChainParamsError(
                 f"Statement values must be of the same type, got {type_} and {arg_type}"
             )
         type_ = arg_type
 
-    if type_ not in supported_types:
+    if type_ is not None and type_ not in supported_types:
         raise DataChainParamsError(
             f"Only python literals ({supported_types}) are supported for values"
         )
@@ -151,15 +158,15 @@ def ifelse(
     """
     Returns the ifelse function that produces if expression which has a condition
     and values for true and false outcome. Results can be one of python primitives
-    like string, numbers or booleans, but can also be nested functions.
+    like string, numbers or booleans, but can also be nested functions or columns.
     Result type is inferred from the values.
 
     Args:
         condition (ColumnElement, Func):  Condition which is evaluated.
-        if_val (str | int | float | complex | bool, Func): Value for true
+        if_val (str | int | float | complex | bool, Func, ColumnElement): Value for true
             condition outcome.
-        else_val (str | int | float | complex | bool, Func): Value for false condition
-            outcome.
+        else_val (str | int | float | complex | bool, Func, ColumnElement): Value for
+            false condition outcome.
 
     Returns:
         Func: A Func object that represents the ifelse function.

{datachain-0.8.11 → datachain-0.8.12}/src/datachain/func/func.py

@@ -424,10 +424,9 @@ class Func(Function):
 
 def get_db_col_type(signals_schema: "SignalSchema", col: ColT) -> "DataType":
     if isinstance(col, tuple):
-        raise DataChainParamsError(
-            "Cannot get type from tuple, please provide type hint to the function"
-        )
-
+        # we can only get tuple from case statement where the first tuple item
+        # is condition, and second one is value which type is important
+        col = col[1]
     if isinstance(col, Func):
         return col.get_result_type(signals_schema)
 
@@ -435,7 +434,7 @@ def get_db_col_type(signals_schema: "SignalSchema", col: ColT) -> "DataType":
         return sql_to_python(col)
 
     return signals_schema.get_column_type(
-        col.name if isinstance(col, ColumnElement) else col
+        col.name if isinstance(col, ColumnElement) else col  # type: ignore[arg-type]
     )
 
 

{datachain-0.8.11 → datachain-0.8.12}/src/datachain/remote/studio.py

@@ -16,14 +16,12 @@ from urllib.parse import urlparse, urlunparse
 import websockets
 
 from datachain.config import Config
-from datachain.dataset import DatasetStats
 from datachain.error import DataChainError
 from datachain.utils import STUDIO_URL, retry_with_backoff
 
 T = TypeVar("T")
 LsData = Optional[list[dict[str, Any]]]
 DatasetInfoData = Optional[dict[str, Any]]
-DatasetStatsData = Optional[DatasetStats]
 DatasetRowsData = Optional[Iterable[dict[str, Any]]]
 DatasetJobVersionsData = Optional[dict[str, Any]]
 DatasetExportStatus = Optional[dict[str, Any]]
@@ -309,7 +307,7 @@ class StudioClient:
             "datachain/datasets",
             {
                 "dataset_name": name,
-                "version": version,
+                "dataset_version": version,
                 "force": force,
             },
             method="DELETE",
@@ -347,16 +345,6 @@ class StudioClient:
             method="GET",
         )
 
-    def dataset_stats(self, name: str, version: int) -> Response[DatasetStatsData]:
-        response = self._send_request(
-            "datachain/datasets/stats",
-            {"dataset_name": name, "dataset_version": version},
-            method="GET",
-        )
-        if response.ok:
-            response.data = DatasetStats(**response.data)
-        return response
-
     def export_dataset_table(
         self, name: str, version: int
     ) -> Response[DatasetExportSignedUrls]:

{datachain-0.8.11 → datachain-0.8.12}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.8.11
+Version: 0.8.12
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.8.11 → datachain-0.8.12}/src/datachain.egg-info/SOURCES.txt

@@ -31,8 +31,8 @@ docs/overrides/main.html
 docs/references/datachain.md
 docs/references/datatype.md
 docs/references/file.md
+docs/references/func.md
 docs/references/index.md
-docs/references/sql.md
 docs/references/torch.md
 docs/references/udf.md
 examples/computer_vision/iptc_exif_xmp_lib.py

{datachain-0.8.11 → datachain-0.8.12}/tests/func/test_catalog.py

@@ -17,7 +17,8 @@ from tests.utils import DEFAULT_TREE, skip_if_not_sqlite, tree_from_path
 def listing_stats(uri, catalog):
     list_dataset_name, _, _ = parse_listing_uri(uri, catalog.client_config)
     dataset = catalog.get_dataset(list_dataset_name)
-    return catalog.dataset_stats(dataset.name, dataset.latest_version)
+    dataset_version = dataset.get_version(dataset.latest_version)
+    return dataset_version.num_objects, dataset_version.size
 
 
 @pytest.fixture
@@ -582,23 +583,23 @@ def test_listing_stats(cloud_test_catalog):
         listing_stats(src_uri, catalog)
 
     catalog.enlist_source(src_uri)
-    stats = listing_stats(src_uri, catalog)
-    assert stats.num_objects == 7
-    assert stats.size == 36
+    num_objects, size = listing_stats(src_uri, catalog)
+    assert num_objects == 7
+    assert size == 36
 
     catalog.enlist_source(f"{src_uri}/dogs/", update=True)
-    stats = listing_stats(src_uri, catalog)
-    assert stats.num_objects == 7
-    assert stats.size == 36
+    num_objects, size = listing_stats(src_uri, catalog)
+    assert num_objects == 7
+    assert size == 36
 
-    stats = listing_stats(f"{src_uri}/dogs/", catalog)
-    assert stats.num_objects == 4
-    assert stats.size == 15
+    num_objects, size = listing_stats(f"{src_uri}/dogs/", catalog)
+    assert num_objects == 4
+    assert size == 15
 
     catalog.enlist_source(f"{src_uri}/dogs/")
-    stats = listing_stats(src_uri, catalog)
-    assert stats.num_objects == 7
-    assert stats.size == 36
+    num_objects, size = listing_stats(src_uri, catalog)
+    assert num_objects == 7
+    assert size == 36
 
 
 @pytest.mark.parametrize("cloud_type", ["s3", "azure", "gs"], indirect=True)
@@ -608,15 +609,15 @@ def test_enlist_source_handles_slash(cloud_test_catalog):
     src_path = f"{src_uri}/dogs"
 
     catalog.enlist_source(src_path)
-    stats = listing_stats(src_path, catalog)
-    assert stats.num_objects == len(DEFAULT_TREE["dogs"])
-    assert stats.size == 15
+    num_objects, size = listing_stats(src_path, catalog)
+    assert num_objects == len(DEFAULT_TREE["dogs"])
+    assert size == 15
 
     src_path = f"{src_uri}/dogs"
     catalog.enlist_source(src_path, update=True)
-    stats = listing_stats(src_path, catalog)
-    assert stats.num_objects == len(DEFAULT_TREE["dogs"])
-    assert stats.size == 15
+    num_objects, size = listing_stats(src_path, catalog)
+    assert num_objects == len(DEFAULT_TREE["dogs"])
+    assert size == 15
 
 
 @pytest.mark.parametrize("cloud_type", ["s3", "azure", "gs"], indirect=True)
@@ -626,10 +627,10 @@ def test_enlist_source_handles_glob(cloud_test_catalog):
     src_path = f"{src_uri}/dogs/*.jpg"
 
     catalog.enlist_source(src_path)
-    stats = listing_stats(src_path, catalog)
+    num_objects, size = listing_stats(src_path, catalog)
 
-    assert stats.num_objects == len(DEFAULT_TREE["dogs"])
-    assert stats.size == 15
+    assert num_objects == len(DEFAULT_TREE["dogs"])
+    assert size == 15
 
 
 @pytest.mark.parametrize("cloud_type", ["s3", "azure", "gs"], indirect=True)

{datachain-0.8.11 → datachain-0.8.12}/tests/func/test_datachain.py

@@ -20,7 +20,7 @@ from sqlalchemy import Column
 from datachain import DataModel, func
 from datachain.catalog.catalog import QUERY_SCRIPT_CANCELED_EXIT_CODE
 from datachain.data_storage.sqlite import SQLiteWarehouse
-from datachain.dataset import DatasetDependencyType, DatasetStats
+from datachain.dataset import DatasetDependencyType
 from datachain.func import path as pathfunc
 from datachain.lib.dc import C, DataChain
 from datachain.lib.file import File, ImageFile
@@ -515,8 +515,9 @@ def test_from_storage_dataset_stats(tmp_dir, test_session):
     dc = DataChain.from_storage(tmp_dir.as_uri(), session=test_session).save(
         "test-data"
     )
-    stats = test_session.catalog.dataset_stats(dc.name, dc.version)
-    assert stats == DatasetStats(num_objects=4, size=20)
+    version = test_session.catalog.get_dataset(dc.name).get_version(dc.version)
+    assert version.num_objects == 4
+    assert version.size == 20
 
 
 def test_from_storage_check_rows(tmp_dir, test_session):

{datachain-0.8.11 → datachain-0.8.12}/tests/func/test_datasets.py

@@ -845,9 +845,9 @@ def test_row_random(cloud_test_catalog):
 
 def test_dataset_stats_registered_ds(cloud_test_catalog, dogs_dataset):
     catalog = cloud_test_catalog.catalog
-    stats = catalog.dataset_stats(dogs_dataset.name, 1)
-    assert stats.num_objects == 4
-    assert stats.size == 15
+    dataset = catalog.get_dataset(dogs_dataset.name).get_version(1)
+    assert dataset.num_objects == 4
+    assert dataset.size == 15
     rows_count = catalog.warehouse.dataset_rows_count(dogs_dataset, 1)
     assert rows_count == 4
 

{datachain-0.8.11 → datachain-0.8.12}/tests/func/test_pull.py

@@ -154,14 +154,6 @@ def remote_dataset_info(requests_mock, remote_dataset):
     requests_mock.get(f"{STUDIO_URL}/api/datachain/datasets/info", json=remote_dataset)
 
 
-@pytest.fixture
-def remote_dataset_stats(requests_mock):
-    requests_mock.get(
-        f"{STUDIO_URL}/api/datachain/datasets/stats",
-        json={"num_objects": 5, "size": 1000},
-    )
-
-
 @pytest.fixture
 def dataset_export(requests_mock, remote_dataset_chunk_url):
     requests_mock.get(
@@ -194,7 +186,6 @@ def test_pull_dataset_success(
     mocker,
     cloud_test_catalog,
     remote_dataset_info,
-    remote_dataset_stats,
     dataset_export,
     dataset_export_status,
     dataset_export_data_chunk,
@@ -322,25 +313,6 @@ def test_pull_dataset_not_found_in_remote(
     assert str(exc_info.value) == "Error from server: Dataset not found"
 
 
-@pytest.mark.parametrize("cloud_type, version_aware", [("s3", False)], indirect=True)
-@skip_if_not_sqlite
-def test_pull_dataset_error_on_fetching_stats(
-    requests_mock,
-    cloud_test_catalog,
-    remote_dataset_info,
-):
-    requests_mock.get(
-        f"{STUDIO_URL}/api/datachain/datasets/stats",
-        status_code=400,
-        json={"message": "Internal error"},
-    )
-    catalog = cloud_test_catalog.catalog
-
-    with pytest.raises(DataChainError) as exc_info:
-        catalog.pull_dataset("ds://dogs@v1")
-    assert str(exc_info.value) == "Error from server: Internal error"
-
-
 @pytest.mark.parametrize("cloud_type, version_aware", [("s3", False)], indirect=True)
 @pytest.mark.parametrize("export_status", ["failed", "removed"])
 @skip_if_not_sqlite
@@ -348,7 +320,6 @@ def test_pull_dataset_exporting_dataset_failed_in_remote(
     requests_mock,
     cloud_test_catalog,
     remote_dataset_info,
-    remote_dataset_stats,
     dataset_export,
     export_status,
 ):
@@ -372,7 +343,6 @@ def test_pull_dataset_empty_parquet(
     requests_mock,
     cloud_test_catalog,
     remote_dataset_info,
-    remote_dataset_stats,
     dataset_export,
     dataset_export_status,
     remote_dataset_chunk_url,
@@ -389,7 +359,6 @@ def test_pull_dataset_empty_parquet(
 def test_pull_dataset_already_exists_locally(
     cloud_test_catalog,
     remote_dataset_info,
-    remote_dataset_stats,
     dataset_export,
     dataset_export_status,
     dataset_export_data_chunk,
@@ -416,7 +385,6 @@ def test_pull_dataset_already_exists_locally(
 def test_pull_dataset_local_name_already_exists(
     cloud_test_catalog,
     remote_dataset_info,
-    remote_dataset_stats,
     dataset_export,
     dataset_export_status,
     dataset_export_data_chunk,

{datachain-0.8.11 → datachain-0.8.12}/tests/test_cli_studio.py

@@ -294,7 +294,7 @@ def test_studio_rm_dataset(capsys, mocker):
         assert last_request.json() == {
             "dataset_name": "name",
             "team_name": "team_name",
-            "version": 1,
+            "dataset_version": 1,
             "force": True,
         }