datachain 0.14.2__tar.gz → 0.14.4__tar.gz

{datachain-0.14.2/src/datachain.egg-info → datachain-0.14.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.14.2
+Version: 0.14.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.14.2 → datachain-0.14.4}/src/datachain/catalog/loader.py

@@ -7,6 +7,7 @@ from datachain.utils import get_envs_by_prefix
 if TYPE_CHECKING:
     from datachain.catalog import Catalog
     from datachain.data_storage import AbstractMetastore, AbstractWarehouse
+    from datachain.query.udf import AbstractUDFDistributor
 
 METASTORE_SERIALIZED = "DATACHAIN__METASTORE"
 METASTORE_IMPORT_PATH = "DATACHAIN_METASTORE"
@@ -15,7 +16,6 @@ WAREHOUSE_SERIALIZED = "DATACHAIN__WAREHOUSE"
 WAREHOUSE_IMPORT_PATH = "DATACHAIN_WAREHOUSE"
 WAREHOUSE_ARG_PREFIX = "DATACHAIN_WAREHOUSE_ARG_"
 DISTRIBUTED_IMPORT_PATH = "DATACHAIN_DISTRIBUTED"
-DISTRIBUTED_ARG_PREFIX = "DATACHAIN_DISTRIBUTED_ARG_"
 
 IN_MEMORY_ERROR_MESSAGE = "In-memory is only supported on SQLite"
 
@@ -100,27 +100,22 @@ def get_warehouse(in_memory: bool = False) -> "AbstractWarehouse":
     return warehouse_class(**warehouse_args)
 
 
-def get_distributed_class(**kwargs):
+def get_udf_distributor_class() -> type["AbstractUDFDistributor"]:
     distributed_import_path = os.environ.get(DISTRIBUTED_IMPORT_PATH)
-    distributed_arg_envs = get_envs_by_prefix(DISTRIBUTED_ARG_PREFIX)
-    # Convert env variable names to keyword argument names by lowercasing them
-    distributed_args = {k.lower(): v for k, v in distributed_arg_envs.items()}
 
     if not distributed_import_path:
         raise RuntimeError(
             f"{DISTRIBUTED_IMPORT_PATH} import path is required "
             "for distributed UDF processing."
         )
-    # Distributed class paths are specified as (for example):
-    # module.classname
+    # Distributed class paths are specified as (for example): module.classname
     if "." not in distributed_import_path:
         raise RuntimeError(
             f"Invalid {DISTRIBUTED_IMPORT_PATH} import path: {distributed_import_path}"
         )
     module_name, _, class_name = distributed_import_path.rpartition(".")
     distributed = import_module(module_name)
-    distributed_class = getattr(distributed, class_name)
-    return distributed_class(**distributed_args | kwargs)
+    return getattr(distributed, class_name)
 
 
 def get_catalog(

{datachain-0.14.2 → datachain-0.14.4}/src/datachain/data_storage/warehouse.py

@@ -199,6 +199,15 @@ class AbstractWarehouse(ABC, Serializable):
     # Query Execution
     #
 
+    def query_count(self, query: sa.sql.selectable.Select) -> int:
+        """Count the number of rows in a query."""
+        count_query = sa.select(func.count(1)).select_from(query.subquery())
+        return next(self.db.execute(count_query))[0]
+
+    def table_rows_count(self, table) -> int:
+        count_query = sa.select(func.count(1)).select_from(table)
+        return next(self.db.execute(count_query))[0]
+
     def dataset_select_paginated(
         self,
         query,

{datachain-0.14.2 → datachain-0.14.4}/src/datachain/lib/dataset_info.py

@@ -12,6 +12,7 @@ from datachain.dataset import (
 )
 from datachain.job import Job
 from datachain.lib.data_model import DataModel
+from datachain.query.session import Session
 from datachain.utils import TIME_ZERO
 
 if TYPE_CHECKING:
@@ -32,6 +33,10 @@ class DatasetInfo(DataModel):
     error_message: str = Field(default="")
     error_stack: str = Field(default="")
 
+    @property
+    def is_temp(self) -> bool:
+        return Session.is_temp_dataset(self.name)
+
     @staticmethod
     def _validate_dict(
         v: Optional[Union[str, dict]],

{datachain-0.14.2 → datachain-0.14.4}/src/datachain/lib/dc/datasets.py

@@ -140,6 +140,8 @@ def datasets(
         )
     ]
 
+    datasets_values = [d for d in datasets_values if not d.is_temp]
+
     return read_values(
         session=session,
         settings=settings,

{datachain-0.14.2 → datachain-0.14.4}/src/datachain/lib/udf.py

@@ -16,7 +16,6 @@ from datachain.lib.convert.flatten import flatten
 from datachain.lib.data_model import DataValue
 from datachain.lib.file import File
 from datachain.lib.utils import AbstractUDF, DataChainError, DataChainParamsError
-from datachain.progress import CombinedDownloadCallback
 from datachain.query.batch import (
     Batch,
     BatchingStrategy,
@@ -327,8 +326,9 @@ def _prefetch_inputs(
 
     if after_prefetch is None:
         after_prefetch = noop
-        if isinstance(download_cb, CombinedDownloadCallback):
-            after_prefetch = download_cb.increment_file_count
+        if download_cb and hasattr(download_cb, "increment_file_count"):
+            increment_file_count: Callable[[], None] = download_cb.increment_file_count
+            after_prefetch = increment_file_count
 
     f = partial(_prefetch_input, download_cb=download_cb, after_prefetch=after_prefetch)
     mapper = AsyncMapper(f, prepared_inputs, workers=prefetch)

{datachain-0.14.2 → datachain-0.14.4}/src/datachain/query/dataset.py

@@ -55,10 +55,12 @@ from datachain.lib.udf import UDFAdapter, _get_cache
 from datachain.progress import CombinedDownloadCallback, TqdmCombinedDownloadCallback
 from datachain.query.schema import C, UDFParamSpec, normalize_param
 from datachain.query.session import Session
+from datachain.query.udf import UdfInfo
 from datachain.sql.functions.random import rand
 from datachain.utils import (
     batched,
     determine_processes,
+    determine_workers,
     filtered_cloudpickle_dumps,
     get_datachain_executable,
     safe_closing,
@@ -74,7 +76,6 @@ if TYPE_CHECKING:
     from datachain.data_storage import AbstractWarehouse
     from datachain.dataset import DatasetRecord
     from datachain.lib.udf import UDFAdapter, UDFResult
-    from datachain.query.udf import UdfInfo
 
     P = ParamSpec("P")
 
@@ -414,20 +415,15 @@ class UDFStep(Step, ABC):
     def populate_udf_table(self, udf_table: "Table", query: Select) -> None:
         from datachain.catalog import QUERY_SCRIPT_CANCELED_EXIT_CODE
 
-        use_partitioning = self.partition_by is not None
-        batching = self.udf.get_batching(use_partitioning)
-        workers = self.workers
-        if (
-            not workers
-            and os.environ.get("DATACHAIN_DISTRIBUTED")
-            and os.environ.get("DATACHAIN_SETTINGS_WORKERS")
-        ):
-            # Enable distributed processing by default if the module is available,
-            # and a default number of workers is provided.
-            workers = True
+        rows_total = self.catalog.warehouse.query_count(query)
+        if rows_total == 0:
+            return
 
-        processes = determine_processes(self.parallel)
+        workers = determine_workers(self.workers, rows_total=rows_total)
+        processes = determine_processes(self.parallel, rows_total=rows_total)
 
+        use_partitioning = self.partition_by is not None
+        batching = self.udf.get_batching(use_partitioning)
         udf_fields = [str(c.name) for c in query.selected_columns]
 
         prefetch = self.udf.prefetch
@@ -441,23 +437,24 @@ class UDFStep(Step, ABC):
                             "distributed processing."
                         )
 
-                    from datachain.catalog.loader import get_distributed_class
-
-                    distributor = get_distributed_class(
-                        min_task_size=self.min_task_size
-                    )
-                    distributor(
-                        self.udf,
-                        catalog,
-                        udf_table,
-                        query,
-                        workers,
-                        processes,
+                    from datachain.catalog.loader import get_udf_distributor_class
+
+                    udf_distributor_class = get_udf_distributor_class()
+                    udf_distributor = udf_distributor_class(
+                        catalog=catalog,
+                        table=udf_table,
+                        query=query,
+                        udf_data=filtered_cloudpickle_dumps(self.udf),
+                        batching=batching,
+                        workers=workers,
+                        processes=processes,
                         udf_fields=udf_fields,
+                        rows_total=rows_total,
+                        use_cache=self.cache,
                         is_generator=self.is_generator,
-                        use_partitioning=use_partitioning,
-                        cache=self.cache,
+                        min_task_size=self.min_task_size,
                     )
+                    udf_distributor()
                 elif processes:
                     # Parallel processing (faster for more CPU-heavy UDFs)
                     if catalog.in_memory:
@@ -465,19 +462,21 @@ class UDFStep(Step, ABC):
                             "In-memory databases cannot be used "
                             "with parallel processing."
                         )
-                    udf_info: UdfInfo = {
-                        "udf_data": filtered_cloudpickle_dumps(self.udf),
-                        "catalog_init": catalog.get_init_params(),
-                        "metastore_clone_params": catalog.metastore.clone_params(),
-                        "warehouse_clone_params": catalog.warehouse.clone_params(),
-                        "table": udf_table,
-                        "query": query,
-                        "udf_fields": udf_fields,
-                        "batching": batching,
-                        "processes": processes,
-                        "is_generator": self.is_generator,
-                        "cache": self.cache,
-                    }
+
+                    udf_info = UdfInfo(
+                        udf_data=filtered_cloudpickle_dumps(self.udf),
+                        catalog_init=catalog.get_init_params(),
+                        metastore_clone_params=catalog.metastore.clone_params(),
+                        warehouse_clone_params=catalog.warehouse.clone_params(),
+                        table=udf_table,
+                        query=query,
+                        udf_fields=udf_fields,
+                        batching=batching,
+                        processes=processes,
+                        is_generator=self.is_generator,
+                        cache=self.cache,
+                        rows_total=rows_total,
+                    )
 
                     # Run the UDFDispatcher in another process to avoid needing
                     # if __name__ == '__main__': in user scripts

{datachain-0.14.2 → datachain-0.14.4}/src/datachain/query/dispatch.py

@@ -4,18 +4,16 @@ from itertools import chain
 from multiprocessing import cpu_count
 from sys import stdin
 from threading import Timer
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Literal, Optional
 
-import attrs
 import multiprocess
 from cloudpickle import load, loads
 from fsspec.callbacks import DEFAULT_CALLBACK, Callback
 from multiprocess import get_context
-from sqlalchemy.sql import func
 
 from datachain.catalog import Catalog
 from datachain.catalog.catalog import clone_catalog_with_cache
-from datachain.catalog.loader import get_distributed_class
+from datachain.catalog.loader import get_udf_distributor_class
 from datachain.lib.udf import _get_cache
 from datachain.query.batch import RowsOutput, RowsOutputBatch
 from datachain.query.dataset import (
@@ -59,7 +57,9 @@ def udf_entrypoint() -> int:
     dispatch = UDFDispatcher(udf_info)
 
     query = udf_info["query"]
+    rows_total = udf_info["rows_total"]
     batching = udf_info["batching"]
+    is_generator = udf_info["is_generator"]
     n_workers = udf_info["processes"]
     if n_workers is True:
         n_workers = None  # Use default number of CPUs (cores)
@@ -67,34 +67,31 @@ def udf_entrypoint() -> int:
     wh_cls, wh_args, wh_kwargs = udf_info["warehouse_clone_params"]
     warehouse: AbstractWarehouse = wh_cls(*wh_args, **wh_kwargs)
 
-    total_rows = next(
-        warehouse.db.execute(
-            query.with_only_columns(func.count(query.c.sys__id)).order_by(None)
-        )
-    )[0]
-
     with contextlib.closing(
         batching(warehouse.dataset_select_paginated, query, ids_only=True)
     ) as udf_inputs:
         download_cb = get_download_callback()
         processed_cb = get_processed_callback()
+        generated_cb = get_generated_callback(is_generator)
         try:
             dispatch.run_udf_parallel(
                 udf_inputs,
-                total_rows=total_rows,
+                rows_total=rows_total,
                 n_workers=n_workers,
-                processed_cb=processed_cb,
                 download_cb=download_cb,
+                processed_cb=processed_cb,
+                generated_cb=generated_cb,
             )
         finally:
             download_cb.close()
             processed_cb.close()
+            generated_cb.close()
 
     return 0
 
 
 def udf_worker_entrypoint() -> int:
-    return get_distributed_class().run_worker()
+    return get_udf_distributor_class().run_worker()
 
 
 class UDFDispatcher:
@@ -134,7 +131,6 @@ class UDFDispatcher:
             self.done_queue,
             self.query,
             self.table,
-            self.is_generator,
             self.is_batching,
             self.cache,
             self.udf_fields,
@@ -158,20 +154,18 @@ class UDFDispatcher:
         for _ in range(n_workers):
             put_into_queue(task_queue, STOP_SIGNAL)
 
-    def create_input_queue(self):
-        return self.ctx.Queue()
-
     def run_udf_parallel(  # noqa: C901, PLR0912
         self,
         input_rows: Iterable[RowsOutput],
-        total_rows: int,
+        rows_total: int,
         n_workers: Optional[int] = None,
-        processed_cb: Callback = DEFAULT_CALLBACK,
         download_cb: Callback = DEFAULT_CALLBACK,
+        processed_cb: Callback = DEFAULT_CALLBACK,
+        generated_cb: Callback = DEFAULT_CALLBACK,
     ) -> None:
         n_workers = get_n_workers_from_arg(n_workers)
 
-        input_batch_size = total_rows // n_workers
+        input_batch_size = rows_total // n_workers
         if input_batch_size == 0:
             input_batch_size = 1
         elif input_batch_size > DEFAULT_BATCH_SIZE:
@@ -220,6 +214,8 @@ class UDFDispatcher:
                     download_cb.relative_update(downloaded)
                 if processed := result.get("processed"):
                     processed_cb.relative_update(processed)
+                if generated := result.get("generated"):
+                    generated_cb.relative_update(generated)
 
                 status = result["status"]
                 if status in (OK_STATUS, NOTIFY_STATUS):
@@ -266,46 +262,61 @@ class UDFDispatcher:
                 p.join()
 
 
-class WorkerCallback(Callback):
-    def __init__(self, queue: "multiprocess.Queue"):
+class DownloadCallback(Callback):
+    def __init__(self, queue: "multiprocess.Queue") -> None:
         self.queue = queue
         super().__init__()
 
     def relative_update(self, inc: int = 1) -> None:
+        # This callback is used to notify the size of the downloaded files
+        pass
+
+    def increment_file_count(self, inc: int = 1) -> None:
         put_into_queue(self.queue, {"status": NOTIFY_STATUS, "downloaded": inc})
 
 
 class ProcessedCallback(Callback):
-    def __init__(self):
-        self.processed_rows: Optional[int] = None
+    def __init__(
+        self,
+        name: Literal["processed", "generated"],
+        queue: "multiprocess.Queue",
+    ) -> None:
+        self.name = name
+        self.queue = queue
         super().__init__()
 
     def relative_update(self, inc: int = 1) -> None:
-        self.processed_rows = inc
+        put_into_queue(self.queue, {"status": NOTIFY_STATUS, self.name: inc})
 
 
-@attrs.define
 class UDFWorker:
-    catalog: "Catalog"
-    udf: "UDFAdapter"
-    task_queue: "multiprocess.Queue"
-    done_queue: "multiprocess.Queue"
-    query: "Select"
-    table: "Table"
-    is_generator: bool
-    is_batching: bool
-    cache: bool
-    udf_fields: Sequence[str]
-    cb: Callback = attrs.field()
-
-    @cb.default
-    def _default_callback(self) -> WorkerCallback:
-        return WorkerCallback(self.done_queue)
+    def __init__(
+        self,
+        catalog: "Catalog",
+        udf: "UDFAdapter",
+        task_queue: "multiprocess.Queue",
+        done_queue: "multiprocess.Queue",
+        query: "Select",
+        table: "Table",
+        is_batching: bool,
+        cache: bool,
+        udf_fields: Sequence[str],
+    ) -> None:
+        self.catalog = catalog
+        self.udf = udf
+        self.task_queue = task_queue
+        self.done_queue = done_queue
+        self.query = query
+        self.table = table
+        self.is_batching = is_batching
+        self.cache = cache
+        self.udf_fields = udf_fields
+
+        self.download_cb = DownloadCallback(self.done_queue)
+        self.processed_cb = ProcessedCallback("processed", self.done_queue)
+        self.generated_cb = ProcessedCallback("generated", self.done_queue)
 
     def run(self) -> None:
-        processed_cb = ProcessedCallback()
-        generated_cb = get_generated_callback(self.is_generator)
-
         prefetch = self.udf.prefetch
         with _get_cache(self.catalog.cache, prefetch, use_cache=self.cache) as _cache:
             catalog = clone_catalog_with_cache(self.catalog, _cache)
@@ -314,29 +325,22 @@ class UDFWorker:
                 self.get_inputs(),
                 catalog,
                 self.cache,
-                download_cb=self.cb,
-                processed_cb=processed_cb,
+                download_cb=self.download_cb,
+                processed_cb=self.processed_cb,
             )
             with safe_closing(udf_results):
                 process_udf_outputs(
                     catalog.warehouse,
                     self.table,
-                    self.notify_and_process(udf_results, processed_cb),
+                    self.notify_and_process(udf_results),
                     self.udf,
-                    cb=generated_cb,
+                    cb=self.generated_cb,
                 )
+        put_into_queue(self.done_queue, {"status": FINISHED_STATUS})
 
-        put_into_queue(
-            self.done_queue,
-            {"status": FINISHED_STATUS, "processed": processed_cb.processed_rows},
-        )
-
-    def notify_and_process(self, udf_results, processed_cb):
+    def notify_and_process(self, udf_results):
         for row in udf_results:
-            put_into_queue(
-                self.done_queue,
-                {"status": OK_STATUS, "processed": processed_cb.processed_rows},
-            )
+            put_into_queue(self.done_queue, {"status": OK_STATUS})
             yield row
 
     def get_inputs(self):

{datachain-0.14.2 → datachain-0.14.4}/src/datachain/query/session.py

@@ -100,6 +100,10 @@ class Session:
     def get_temp_prefix(self) -> str:
         return f"{self.DATASET_PREFIX}{self.name}_"
 
+    @classmethod
+    def is_temp_dataset(cls, name) -> bool:
+        return name.startswith(cls.DATASET_PREFIX)
+
     def _cleanup_temp_datasets(self) -> None:
         prefix = self.get_temp_prefix()
         try:

datachain-0.14.4/src/datachain/query/udf.py

@@ -0,0 +1,49 @@
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any, Callable, Optional, TypedDict, Union
+
+if TYPE_CHECKING:
+    from sqlalchemy import Select, Table
+
+    from datachain.catalog import Catalog
+    from datachain.query.batch import BatchingStrategy
+
+
+class UdfInfo(TypedDict):
+    udf_data: bytes
+    catalog_init: dict[str, Any]
+    metastore_clone_params: tuple[Callable[..., Any], list[Any], dict[str, Any]]
+    warehouse_clone_params: tuple[Callable[..., Any], list[Any], dict[str, Any]]
+    table: "Table"
+    query: "Select"
+    udf_fields: list[str]
+    batching: "BatchingStrategy"
+    processes: Optional[int]
+    is_generator: bool
+    cache: bool
+    rows_total: int
+
+
+class AbstractUDFDistributor(ABC):
+    @abstractmethod
+    def __init__(
+        self,
+        catalog: "Catalog",
+        table: "Table",
+        query: "Select",
+        udf_data: bytes,
+        batching: "BatchingStrategy",
+        workers: Union[bool, int],
+        processes: Union[bool, int],
+        udf_fields: list[str],
+        rows_total: int,
+        use_cache: bool,
+        is_generator: bool = False,
+        min_task_size: Optional[Union[str, int]] = None,
+    ) -> None: ...
+
+    @abstractmethod
+    def __call__(self) -> None: ...
+
+    @staticmethod
+    @abstractmethod
+    def run_worker() -> int: ...

{datachain-0.14.2 → datachain-0.14.4}/src/datachain/utils.py

@@ -286,15 +286,41 @@ def retry_with_backoff(retries=5, backoff_sec=1, errors=(Exception,)):
     return retry
 
 
-def determine_processes(parallel: Optional[Union[bool, int]]) -> Union[bool, int]:
+def determine_workers(
+    workers: Union[bool, int],
+    rows_total: Optional[int] = None,
+) -> Union[bool, int]:
+    """Determine the number of workers to use for distributed processing."""
+    if rows_total is not None and rows_total <= 1:
+        # Disable distributed processing if there is no rows or only one row.
+        return False
+    if (
+        workers is False
+        and os.environ.get("DATACHAIN_DISTRIBUTED")
+        and os.environ.get("DATACHAIN_SETTINGS_WORKERS")
+    ):
+        # Enable distributed processing by default if the module is available,
+        # and a default number of workers is provided.
+        workers = int(os.environ["DATACHAIN_SETTINGS_WORKERS"])
+    if not workers or workers <= 0:
+        return False
+    return workers
+
+
+def determine_processes(
+    parallel: Optional[Union[bool, int]] = None,
+    rows_total: Optional[int] = None,
+) -> Union[bool, int]:
+    """Determine the number of processes to use for parallel processing."""
+    if rows_total is not None and rows_total <= 1:
+        # Disable parallel processing if there is no rows or only one row.
+        return False
     if parallel is None and os.environ.get("DATACHAIN_SETTINGS_PARALLEL") is not None:
         parallel = int(os.environ["DATACHAIN_SETTINGS_PARALLEL"])
-    if parallel is None or parallel is False:
+    if parallel is None or parallel is False or parallel == 0:
         return False
     if parallel is True:
         return True
-    if parallel == 0:
-        return False
     if parallel < 0:
         return True
     return parallel

{datachain-0.14.2 → datachain-0.14.4/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.14.2
+Version: 0.14.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.14.2 → datachain-0.14.4}/tests/func/test_datachain.py

@@ -538,6 +538,17 @@ def test_show(capsys, test_session):
         assert f"{i} {first_name[i]}" in normalized_output
 
 
+def test_show_without_temp_datasets(capsys, test_session):
+    dc.read_values(
+        key=[1, 2, 3, 4], session=test_session
+    ).save()  # creates temp dataset
+    dc.datasets().show()
+    captured = capsys.readouterr()
+    normalized_output = re.sub(r"\s+", " ", captured.out)
+    print(normalized_output)
+    assert "Empty result" in normalized_output
+
+
 def test_class_method_deprecated(capsys, test_session):
     with pytest.warns(DeprecationWarning):
         dc.DataChain.from_values(key=["a", "b", "c"], session=test_session)

{datachain-0.14.2 → datachain-0.14.4}/tests/unit/lib/test_datachain.py

@@ -4,7 +4,7 @@ import math
 import os
 import re
 from collections.abc import Generator, Iterator
-from unittest.mock import ANY
+from unittest.mock import ANY, patch
 
 import numpy as np
 import pandas as pd
@@ -26,6 +26,7 @@ from datachain.lib.signal_schema import (
     SignalResolvingTypeError,
     SignalSchema,
 )
+from datachain.lib.udf import UDFAdapter
 from datachain.lib.udf_signature import UdfSignatureError
 from datachain.lib.utils import DataChainColumnError, DataChainParamsError
 from datachain.sql.types import Float, Int64, String
@@ -270,6 +271,17 @@ def test_read_record_empty_chain_without_schema(test_session):
     )
 
 
+def test_empty_chain_skip_udf_run(test_session):
+    # Test that UDF is not called for empty chain
+    with patch.object(UDFAdapter, "run") as mock_udf_run:
+        (
+            dc.read_records([], schema={"val": int}, session=test_session)
+            .map(lambda val: val * 2, params="val", output={"val2": int})
+            .exec()
+        )
+        mock_udf_run.assert_not_called()
+
+
 def test_datasets(test_session):
     ds = dc.datasets(session=test_session)
     datasets = [d for d in ds.collect("dataset") if d.name == "fibonacci"]