datachain 0.16.4__py3-none-any.whl → 0.17.0__py3-none-any.whl

datachain/data_storage/warehouse.py

@@ -11,16 +11,15 @@ from urllib.parse import urlparse
 
 import attrs
 import sqlalchemy as sa
-from sqlalchemy import Table, case, select
-from sqlalchemy.sql import func
 from sqlalchemy.sql.expression import true
-from tqdm.auto import tqdm
 
 from datachain.client import Client
 from datachain.data_storage.schema import convert_rows_custom_column_types
 from datachain.data_storage.serializer import Serializable
 from datachain.dataset import DatasetRecord, StorageURI
 from datachain.node import DirType, DirTypeGroup, Node, NodeWithPath, get_path
+from datachain.query.batch import RowsOutput
+from datachain.query.utils import get_query_id_column
 from datachain.sql.functions import path as pathfunc
 from datachain.sql.types import Int, SQLType
 from datachain.utils import sql_escape_like
@@ -31,7 +30,6 @@ if TYPE_CHECKING:
         _FromClauseArgument,
         _OnClauseArgument,
     )
-    from sqlalchemy.sql.selectable import Select
     from sqlalchemy.types import TypeEngine
 
     from datachain.data_storage import schema
@@ -178,7 +176,7 @@ class AbstractWarehouse(ABC, Serializable):
     def dataset_rows(
         self,
         dataset: DatasetRecord,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         column: str = "file",
     ):
         version = version or dataset.latest_version
@@ -199,13 +197,13 @@ class AbstractWarehouse(ABC, Serializable):
     # Query Execution
     #
 
-    def query_count(self, query: sa.sql.selectable.Select) -> int:
+    def query_count(self, query: sa.Select) -> int:
         """Count the number of rows in a query."""
-        count_query = sa.select(func.count(1)).select_from(query.subquery())
+        count_query = sa.select(sa.func.count(1)).select_from(query.subquery())
         return next(self.db.execute(count_query))[0]
 
     def table_rows_count(self, table) -> int:
-        count_query = sa.select(func.count(1)).select_from(table)
+        count_query = sa.select(sa.func.count(1)).select_from(table)
         return next(self.db.execute(count_query))[0]
 
     def dataset_select_paginated(
@@ -255,7 +253,7 @@ class AbstractWarehouse(ABC, Serializable):
         name = parsed.path if parsed.scheme == "file" else parsed.netloc
         return parsed.scheme, name
 
-    def dataset_table_name(self, dataset_name: str, version: int) -> str:
+    def dataset_table_name(self, dataset_name: str, version: str) -> str:
         prefix = self.DATASET_TABLE_PREFIX
         if Client.is_data_source_uri(dataset_name):
             # for datasets that are created for bucket listing we use different prefix
@@ -278,18 +276,18 @@ class AbstractWarehouse(ABC, Serializable):
         name: str,
         columns: Sequence["sa.Column"] = (),
         if_not_exists: bool = True,
-    ) -> Table:
+    ) -> sa.Table:
         """Creates a dataset rows table for the given dataset name and columns"""
 
     def drop_dataset_rows_table(
         self,
         dataset: DatasetRecord,
-        version: int,
+        version: str,
         if_exists: bool = True,
     ) -> None:
         """Drops a dataset rows table for the given dataset name."""
         table_name = self.dataset_table_name(dataset.name, version)
-        table = Table(table_name, self.db.metadata)
+        table = sa.Table(table_name, self.db.metadata)
         self.db.drop_table(table, if_exists=if_exists)
 
     @abstractmethod
@@ -297,8 +295,8 @@ class AbstractWarehouse(ABC, Serializable):
         self,
         src: "DatasetRecord",
         dst: "DatasetRecord",
-        src_version: int,
-        dst_version: int,
+        src_version: str,
+        dst_version: str,
     ) -> None:
         """
         Merges source dataset rows and current latest destination dataset rows
@@ -309,7 +307,7 @@ class AbstractWarehouse(ABC, Serializable):
 
     def dataset_rows_select(
         self,
-        query: sa.sql.selectable.Select,
+        query: sa.Select,
         **kwargs,
     ) -> Iterator[tuple[Any, ...]]:
         """
@@ -320,17 +318,35 @@ class AbstractWarehouse(ABC, Serializable):
             query.selected_columns, rows, self.db.dialect
         )
 
+    def dataset_rows_select_from_ids(
+        self,
+        query: sa.Select,
+        ids: Iterable[RowsOutput],
+        is_batched: bool,
+    ) -> Iterator[RowsOutput]:
+        """
+        Fetch dataset rows from database using a list of IDs.
+        """
+        if (id_col := get_query_id_column(query)) is None:
+            raise RuntimeError("sys__id column not found in query")
+
+        if is_batched:
+            for batch in ids:
+                yield list(self.dataset_rows_select(query.where(id_col.in_(batch))))
+        else:
+            yield from self.dataset_rows_select(query.where(id_col.in_(ids)))
+
     @abstractmethod
     def get_dataset_sources(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> list[StorageURI]: ...
 
     def rename_dataset_table(
         self,
         old_name: str,
         new_name: str,
-        old_version: int,
-        new_version: int,
+        old_version: str,
+        new_version: str,
     ) -> None:
         old_ds_table_name = self.dataset_table_name(old_name, old_version)
         new_ds_table_name = self.dataset_table_name(new_name, new_version)
@@ -341,12 +357,12 @@ class AbstractWarehouse(ABC, Serializable):
         """Returns total number of rows in a dataset"""
         dr = self.dataset_rows(dataset, version)
         table = dr.get_table()
-        query = select(sa.func.count(table.c.sys__id))
+        query = sa.select(sa.func.count(table.c.sys__id))
         (res,) = self.db.execute(query)
         return res[0]
 
     def dataset_stats(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> tuple[Optional[int], Optional[int]]:
         """
         Returns tuple with dataset stats: total number of rows and total dataset size.
@@ -364,7 +380,7 @@ class AbstractWarehouse(ABC, Serializable):
         ]
         if size_columns:
             expressions = (*expressions, sa.func.sum(sum(size_columns)))
-        query = select(*expressions)
+        query = sa.select(*expressions)
         ((nrows, *rest),) = self.db.execute(query)
         return nrows, rest[0] if rest else 0
 
@@ -373,17 +389,17 @@ class AbstractWarehouse(ABC, Serializable):
         """Convert File entries so they can be passed on to `insert_rows()`"""
 
     @abstractmethod
-    def insert_rows(self, table: Table, rows: Iterable[dict[str, Any]]) -> None:
+    def insert_rows(self, table: sa.Table, rows: Iterable[dict[str, Any]]) -> None:
         """Does batch inserts of any kind of rows into table"""
 
-    def insert_rows_done(self, table: Table) -> None:
+    def insert_rows_done(self, table: sa.Table) -> None:
         """
         Only needed for certain implementations
         to signal when rows inserts are complete.
         """
 
     @abstractmethod
-    def insert_dataset_rows(self, df, dataset: DatasetRecord, version: int) -> int:
+    def insert_dataset_rows(self, df, dataset: DatasetRecord, version: str) -> int:
         """Inserts dataset rows directly into dataset table"""
 
     @abstractmethod
@@ -402,7 +418,7 @@ class AbstractWarehouse(ABC, Serializable):
 
     @abstractmethod
     def dataset_table_export_file_names(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> list[str]:
         """
         Returns list of file names that will be created when user runs dataset export
@@ -413,7 +429,7 @@ class AbstractWarehouse(ABC, Serializable):
         self,
         bucket_uri: str,
         dataset: DatasetRecord,
-        version: int,
+        version: str,
         client_config=None,
     ) -> list[str]:
         """
@@ -497,7 +513,7 @@ class AbstractWarehouse(ABC, Serializable):
         ).subquery()
         path_glob = "/".join([*path_list, glob_name])
         dirpath = path_glob[: -len(glob_name)]
-        relpath = func.substr(de.c(q, "path"), len(dirpath) + 1)
+        relpath = sa.func.substr(de.c(q, "path"), len(dirpath) + 1)
 
         return self.get_nodes(
             self.expand_query(de, q, dr)
@@ -584,13 +600,13 @@ class AbstractWarehouse(ABC, Serializable):
             default = getattr(
                 attrs.fields(Node), dr.without_object(column.name)
             ).default
-            return func.coalesce(column, default).label(column.name)
+            return sa.func.coalesce(column, default).label(column.name)
 
         return sa.select(
             q.c.sys__id,
-            case((de.c(q, "is_dir") == true(), DirType.DIR), else_=DirType.FILE).label(
-                dr.col_name("dir_type")
-            ),
+            sa.case(
+                (de.c(q, "is_dir") == true(), DirType.DIR), else_=DirType.FILE
+            ).label(dr.col_name("dir_type")),
             de.c(q, "path"),
             with_default(dr.c("etag")),
             de.c(q, "version"),
@@ -665,7 +681,7 @@ class AbstractWarehouse(ABC, Serializable):
             return de.c(inner_query, f)
 
         return self.db.execute(
-            select(*(field_to_expr(f) for f in fields)).order_by(
+            sa.select(*(field_to_expr(f) for f in fields)).order_by(
                 de.c(inner_query, "source"),
                 de.c(inner_query, "path"),
                 de.c(inner_query, "version"),
@@ -687,7 +703,7 @@ class AbstractWarehouse(ABC, Serializable):
             return dr.c(f)
 
         q = (
-            select(*(field_to_expr(f) for f in fields))
+            sa.select(*(field_to_expr(f) for f in fields))
             .where(
                 dr.c("path").like(f"{sql_escape_like(dirpath)}%"),
                 ~self.instr(pathfunc.name(dr.c("path")), "/"),
@@ -722,10 +738,10 @@ class AbstractWarehouse(ABC, Serializable):
         sub_glob = posixpath.join(path, "*")
         dr = dataset_rows
         selections: list[sa.ColumnElement] = [
-            func.sum(dr.c("size")),
+            sa.func.sum(dr.c("size")),
         ]
         if count_files:
-            selections.append(func.count())
+            selections.append(sa.func.count())
         results = next(
             self.db.execute(
                 dr.select(*selections).where(
@@ -842,7 +858,7 @@ class AbstractWarehouse(ABC, Serializable):
         self,
         columns: Sequence["sa.Column"] = (),
         name: Optional[str] = None,
-    ) -> "sa.Table":
+    ) -> sa.Table:
         """
         Create a temporary table for storing custom signals generated by a UDF.
         SQLite TEMPORARY tables cannot be directly used as they are process-specific,
@@ -860,8 +876,8 @@ class AbstractWarehouse(ABC, Serializable):
     @abstractmethod
     def copy_table(
         self,
-        table: Table,
-        query: "Select",
+        table: sa.Table,
+        query: sa.Select,
         progress_cb: Optional[Callable[[int], None]] = None,
     ) -> None:
         """
@@ -875,13 +891,13 @@ class AbstractWarehouse(ABC, Serializable):
         right: "_FromClauseArgument",
         onclause: "_OnClauseArgument",
         inner: bool = True,
-    ) -> "Select":
+    ) -> sa.Select:
         """
         Join two tables together.
         """
 
     @abstractmethod
-    def create_pre_udf_table(self, query: "Select") -> "Table":
+    def create_pre_udf_table(self, query: sa.Select) -> sa.Table:
         """
         Create a temporary table from a query for use in a UDF.
         """
@@ -906,12 +922,8 @@ class AbstractWarehouse(ABC, Serializable):
         are cleaned up as soon as they are no longer needed.
         """
         to_drop = set(names)
-        with tqdm(
-            desc="Cleanup", unit=" tables", total=len(to_drop), leave=False
-        ) as pbar:
-            for name in to_drop:
-                self.db.drop_table(Table(name, self.db.metadata), if_exists=True)
-                pbar.update(1)
+        for name in to_drop:
+            self.db.drop_table(sa.Table(name, self.db.metadata), if_exists=True)
 
 
 def _random_string(length: int) -> str:

datachain/dataset.py

@@ -12,6 +12,7 @@ from typing import (
 )
 from urllib.parse import urlparse
 
+from datachain import semver
 from datachain.error import DatasetVersionNotFoundError
 from datachain.sql.types import NAME_TYPES_MAPPING, SQLType
 
@@ -25,6 +26,8 @@ DATASET_PREFIX = "ds://"
 QUERY_DATASET_PREFIX = "ds_query_"
 LISTING_PREFIX = "lst__"
 
+DEFAULT_DATASET_VERSION = "1.0.0"
+
 
 # StorageURI represents a normalised URI to a valid storage location (full bucket or
 # absolute local path).
@@ -33,12 +36,12 @@ LISTING_PREFIX = "lst__"
 StorageURI = NewType("StorageURI", str)
 
 
-def parse_dataset_uri(uri: str) -> tuple[str, Optional[int]]:
+def parse_dataset_uri(uri: str) -> tuple[str, Optional[str]]:
     """
     Parse dataser uri to extract name and version out of it (if version is defined)
     Example:
-        Input: ds://zalando@v3
-        Output: (zalando, 3)
+        Input: ds://zalando@v3.0.1
+        Output: (zalando, 3.0.1)
     """
     p = urlparse(uri)
     if p.scheme != "ds":
@@ -51,16 +54,15 @@ def parse_dataset_uri(uri: str) -> tuple[str, Optional[int]]:
         raise Exception(
             "Wrong dataset uri format, it should be: ds://<name>@v<version>"
         )
-    version = int(s[1])
-    return name, version
+    return name, s[1]
 
 
-def create_dataset_uri(name: str, version: Optional[int] = None) -> str:
+def create_dataset_uri(name: str, version: Optional[str] = None) -> str:
     """
     Creates a dataset uri based on dataset name and optionally version
     Example:
-        Input: zalando, 3
-        Output: ds//zalando@v3
+        Input: zalando, 3.0.1
+        Output: ds//zalando@v3.0.1
     """
     uri = f"{DATASET_PREFIX}{name}"
     if version:
@@ -79,7 +81,7 @@ class DatasetDependency:
     id: int
     type: str
     name: str
-    version: str  # TODO change to int
+    version: str
     created_at: datetime
     dependencies: list[Optional["DatasetDependency"]]
 
@@ -102,7 +104,7 @@ class DatasetDependency:
         dataset_id: Optional[int],
         dataset_version_id: Optional[int],
         dataset_name: Optional[str],
-        dataset_version: Optional[int],
+        dataset_version: Optional[str],
         dataset_version_created_at: Optional[datetime],
     ) -> Optional["DatasetDependency"]:
         from datachain.client import Client
@@ -124,7 +126,7 @@ class DatasetDependency:
             dependency_type,
             dependency_name,
             (
-                str(dataset_version)  # type: ignore[arg-type]
+                dataset_version  # type: ignore[arg-type]
                 if dataset_version
                 else None
             ),
@@ -163,7 +165,7 @@ class DatasetVersion:
     id: int
     uuid: str
     dataset_id: int
-    version: int
+    version: str
     status: int
     feature_schema: dict
     created_at: datetime
@@ -185,7 +187,7 @@ class DatasetVersion:
         id: int,
         uuid: str,
         dataset_id: int,
-        version: int,
+        version: str,
         status: int,
         feature_schema: Optional[str],
         created_at: datetime,
@@ -222,6 +224,10 @@ class DatasetVersion:
             job_id,
         )
 
+    @property
+    def version_value(self) -> int:
+        return semver.value(self.version)
+
     def __eq__(self, other):
         if not isinstance(other, DatasetVersion):
             return False
@@ -230,7 +236,7 @@ class DatasetVersion:
     def __lt__(self, other):
         if not isinstance(other, DatasetVersion):
             return False
-        return self.version < other.version
+        return self.version_value < other.version_value
 
     def __hash__(self):
         return hash(f"{self.dataset_id}_{self.version}")
@@ -275,7 +281,7 @@ class DatasetListVersion:
     id: int
     uuid: str
     dataset_id: int
-    version: int
+    version: str
     status: int
     created_at: datetime
     finished_at: Optional[datetime]
@@ -292,7 +298,7 @@ class DatasetListVersion:
         id: int,
         uuid: str,
         dataset_id: int,
-        version: int,
+        version: str,
         status: int,
         created_at: datetime,
         finished_at: Optional[datetime],
@@ -323,6 +329,10 @@ class DatasetListVersion:
     def __hash__(self):
         return hash(f"{self.dataset_id}_{self.version}")
 
+    @property
+    def version_value(self) -> int:
+        return semver.value(self.version)
+
 
 @dataclass
 class DatasetRecord:
@@ -371,7 +381,7 @@ class DatasetRecord:
         version_id: int,
         version_uuid: str,
         version_dataset_id: int,
-        version: int,
+        version: str,
         version_status: int,
         version_feature_schema: Optional[str],
         version_created_at: datetime,
@@ -441,7 +451,7 @@ class DatasetRecord:
             for c_name, c_type in self.schema.items()
         }
 
-    def get_schema(self, version: int) -> dict[str, Union[SQLType, type[SQLType]]]:
+    def get_schema(self, version: str) -> dict[str, Union[SQLType, type[SQLType]]]:
         return self.get_version(version).schema if version else self.schema
 
     def update(self, **kwargs):
@@ -460,20 +470,23 @@ class DatasetRecord:
             self.versions = []
 
         self.versions = list(set(self.versions + other.versions))
-        self.versions.sort(key=lambda v: v.version)
+        self.versions.sort(key=lambda v: v.version_value)
         return self
 
-    def has_version(self, version: int) -> bool:
-        return version in self.versions_values
+    def has_version(self, version: str) -> bool:
+        return version in [v.version for v in self.versions]
 
-    def is_valid_next_version(self, version: int) -> bool:
+    def is_valid_next_version(self, version: str) -> bool:
         """
         Checks if a number can be a valid next latest version for dataset.
         The only rule is that it cannot be lower than current latest version
         """
-        return not (self.latest_version and self.latest_version >= version)
+        return not (
+            self.latest_version
+            and semver.value(self.latest_version) >= semver.value(version)
+        )
 
-    def get_version(self, version: int) -> DatasetVersion:
+    def get_version(self, version: str) -> DatasetVersion:
         if not self.has_version(version):
             raise DatasetVersionNotFoundError(
                 f"Dataset {self.name} does not have version {version}"
@@ -496,15 +509,15 @@ class DatasetRecord:
                 f"Dataset {self.name} does not have version with uuid {uuid}"
             ) from None
 
-    def remove_version(self, version: int) -> None:
+    def remove_version(self, version: str) -> None:
         if not self.versions or not self.has_version(version):
             return
 
         self.versions = [v for v in self.versions if v.version != version]
 
-    def identifier(self, version: int) -> str:
+    def identifier(self, version: str) -> str:
         """
-        Get identifier in the form my-dataset@v3
+        Get identifier in the form my-dataset@v3.0.1
         """
         if not self.has_version(version):
             raise DatasetVersionNotFoundError(
@@ -512,43 +525,73 @@ class DatasetRecord:
             )
         return f"{self.name}@v{version}"
 
-    def uri(self, version: int) -> str:
+    def uri(self, version: str) -> str:
         """
-        Dataset uri example: ds://dogs@v3
+        Dataset uri example: ds://dogs@v3.0.1
         """
         identifier = self.identifier(version)
         return f"{DATASET_PREFIX}{identifier}"
 
     @property
-    def versions_values(self) -> list[int]:
+    def next_version_major(self) -> str:
         """
-        Extracts actual versions from list of DatasetVersion objects
-        in self.versions attribute
+        Returns the next auto-incremented version if the major part is being bumped.
         """
         if not self.versions:
-            return []
+            return "1.0.0"
 
-        return sorted(v.version for v in self.versions)
+        major, minor, patch = semver.parse(self.latest_version)
+        return semver.create(major + 1, 0, 0)
 
     @property
-    def next_version(self) -> int:
-        """Returns what should be next autoincrement version of dataset"""
+    def next_version_minor(self) -> str:
+        """
+        Returns the next auto-incremented version if the minor part is being bumped.
+        """
         if not self.versions:
-            return 1
-        return max(self.versions_values) + 1
+            return "1.0.0"
+
+        major, minor, patch = semver.parse(self.latest_version)
+        return semver.create(major, minor + 1, 0)
 
     @property
-    def latest_version(self) -> int:
+    def next_version_patch(self) -> str:
+        """
+        Returns the next auto-incremented version if the patch part is being bumped.
+        """
+        if not self.versions:
+            return "1.0.0"
+
+        major, minor, patch = semver.parse(self.latest_version)
+        return semver.create(major, minor, patch + 1)
+
+    @property
+    def latest_version(self) -> str:
         """Returns latest version of a dataset"""
-        return max(self.versions_values)
+        return max(self.versions).version
+
+    def latest_major_version(self, major: int) -> Optional[str]:
+        """
+        Returns latest specific major version, e.g if dataset has versions:
+            - 1.4.1
+            - 2.0.1
+            - 2.1.1
+            - 2.4.0
+        and we call `.latest_major_version(2)` it will return: "2.4.0".
+        If no major version is find with input value, None will be returned
+        """
+        versions = [v for v in self.versions if semver.parse(v.version)[0] == major]
+        if not versions:
+            return None
+        return max(versions).version
 
     @property
-    def prev_version(self) -> Optional[int]:
+    def prev_version(self) -> Optional[str]:
         """Returns previous version of a dataset"""
         if len(self.versions) == 1:
             return None
 
-        return sorted(self.versions_values)[-2]
+        return sorted(self.versions)[-2].version
 
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "DatasetRecord":
@@ -577,7 +620,7 @@ class DatasetListRecord:
         version_id: int,
         version_uuid: str,
         version_dataset_id: int,
-        version: int,
+        version: str,
         version_status: int,
         version_created_at: datetime,
         version_finished_at: Optional[datetime],
@@ -626,11 +669,11 @@ class DatasetListRecord:
             self.versions = []
 
         self.versions = list(set(self.versions + other.versions))
-        self.versions.sort(key=lambda v: v.version)
+        self.versions.sort(key=lambda v: v.version_value)
         return self
 
     def latest_version(self) -> DatasetListVersion:
-        return max(self.versions, key=lambda v: v.version)
+        return max(self.versions, key=lambda v: v.version_value)
 
     @property
     def is_bucket_listing(self) -> bool:

datachain/lib/arrow.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Optional
 
 import orjson
 import pyarrow as pa
+from pyarrow._csv import ParseOptions
 from pyarrow.dataset import CsvFileFormat, dataset
 from tqdm.auto import tqdm
 
@@ -26,6 +27,18 @@ if TYPE_CHECKING:
 DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY = b"DataChain SignalSchema"
 
 
+def fix_pyarrow_format(format, parse_options=None):
+    # Re-init invalid row handler: https://issues.apache.org/jira/browse/ARROW-17641
+    if (
+        format
+        and isinstance(format, CsvFileFormat)
+        and parse_options
+        and isinstance(parse_options, ParseOptions)
+    ):
+        format.parse_options = parse_options
+    return format
+
+
 class ArrowGenerator(Generator):
     DEFAULT_BATCH_SIZE = 2**17  # same as `pyarrow._dataset._DEFAULT_BATCH_SIZE`
 
@@ -53,6 +66,7 @@ class ArrowGenerator(Generator):
         self.output_schema = output_schema
         self.source = source
         self.nrows = nrows
+        self.parse_options = kwargs.pop("parse_options", None)
         self.kwargs = kwargs
 
     def process(self, file: File):
@@ -64,7 +78,11 @@ class ArrowGenerator(Generator):
         else:
             fs, fs_path = file.get_fs(), file.get_path()
 
-        ds = dataset(fs_path, schema=self.input_schema, filesystem=fs, **self.kwargs)
+        kwargs = self.kwargs
+        if format := kwargs.get("format"):
+            kwargs["format"] = fix_pyarrow_format(format, self.parse_options)
+
+        ds = dataset(fs_path, schema=self.input_schema, filesystem=fs, **kwargs)
 
         hf_schema = _get_hf_schema(ds.schema)
         use_datachain_schema = (
@@ -137,6 +155,10 @@ class ArrowGenerator(Generator):
 
 
 def infer_schema(chain: "DataChain", **kwargs) -> pa.Schema:
+    parse_options = kwargs.pop("parse_options", None)
+    if format := kwargs.get("format"):
+        kwargs["format"] = fix_pyarrow_format(format, parse_options)
+
     schemas = []
     for file in chain.collect("file"):
         ds = dataset(file.get_path(), filesystem=file.get_fs(), **kwargs)  # type: ignore[union-attr]