datachain 0.3.15__py3-none-any.whl → 0.3.17__py3-none-any.whl

datachain/catalog/catalog.py

@@ -68,8 +68,6 @@ from datachain.utils import (
     DataChainDir,
     batched,
     datachain_paths_join,
-    import_object,
-    parse_params_string,
 )
 
 from .datasource import DataSource
@@ -843,7 +841,7 @@ class Catalog:
         from datachain.query import DatasetQuery
 
         def _row_to_node(d: dict[str, Any]) -> Node:
-            del d["source"]
+            del d["file__source"]
             return Node.from_dict(d)
 
         enlisted_sources: list[tuple[bool, bool, Any]] = []
@@ -1148,30 +1146,28 @@ class Catalog:
         if not sources:
             raise ValueError("Sources needs to be non empty list")
 
-        from datachain.query import DatasetQuery
+        from datachain.lib.dc import DataChain
+        from datachain.query.session import Session
+
+        session = Session.get(catalog=self, client_config=client_config)
 
-        dataset_queries = []
+        chains = []
         for source in sources:
             if source.startswith(DATASET_PREFIX):
-                dq = DatasetQuery(
-                    name=source[len(DATASET_PREFIX) :],
-                    catalog=self,
-                    client_config=client_config,
+                dc = DataChain.from_dataset(
+                    source[len(DATASET_PREFIX) :], session=session
                 )
             else:
-                dq = DatasetQuery(
-                    path=source,
-                    catalog=self,
-                    client_config=client_config,
-                    recursive=recursive,
+                dc = DataChain.from_storage(
+                    source, session=session, recursive=recursive
                 )
 
-            dataset_queries.append(dq)
+            chains.append(dc)
 
         # create union of all dataset queries created from sources
-        dq = reduce(lambda ds1, ds2: ds1.union(ds2), dataset_queries)
+        dc = reduce(lambda dc1, dc2: dc1.union(dc2), chains)
         try:
-            dq.save(name)
+            dc.save(name)
         except Exception as e:  # noqa: BLE001
             try:
                 ds = self.get_dataset(name)
@@ -1731,26 +1727,6 @@ class Catalog:
             output, sources, client_config=client_config, recursive=recursive
         )
 
-    def apply_udf(
-        self,
-        udf_location: str,
-        source: str,
-        target_name: str,
-        parallel: Optional[int] = None,
-        params: Optional[str] = None,
-    ):
-        from datachain.query import DatasetQuery
-
-        if source.startswith(DATASET_PREFIX):
-            ds = DatasetQuery(name=source[len(DATASET_PREFIX) :], catalog=self)
-        else:
-            ds = DatasetQuery(path=source, catalog=self)
-        udf = import_object(udf_location)
-        if params:
-            args, kwargs = parse_params_string(params)
-            udf = udf(*args, **kwargs)
-        ds.add_signals(udf, parallel=parallel).save(target_name)
-
     def query(
         self,
         query_script: str,

datachain/cli.py

@@ -494,27 +494,6 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         help="Query parameters",
     )
 
-    apply_udf_parser = subp.add_parser(
-        "apply-udf", parents=[parent_parser], description="Apply UDF"
-    )
-    apply_udf_parser.add_argument("udf", type=str, help="UDF location")
-    apply_udf_parser.add_argument("source", type=str, help="Source storage or dataset")
-    apply_udf_parser.add_argument("target", type=str, help="Target dataset name")
-    apply_udf_parser.add_argument(
-        "--parallel",
-        nargs="?",
-        type=int,
-        const=-1,
-        default=None,
-        metavar="N",
-        help=(
-            "Use multiprocessing to run the UDF with N worker processes. "
-            "N defaults to the CPU count."
-        ),
-    )
-    apply_udf_parser.add_argument(
-        "--udf-params", type=str, default=None, help="UDF class parameters"
-    )
     subp.add_parser(
         "clear-cache", parents=[parent_parser], description="Clear the local file cache"
     )
@@ -1016,10 +995,6 @@ def main(argv: Optional[list[str]] = None) -> int:  # noqa: C901, PLR0912, PLR09
                 parallel=args.parallel,
                 params=args.param,
             )
-        elif args.command == "apply-udf":
-            catalog.apply_udf(
-                args.udf, args.source, args.target, args.parallel, args.udf_params
-            )
         elif args.command == "clear-cache":
             clear_cache(catalog)
         elif args.command == "gc":

datachain/data_storage/metastore.py

@@ -297,39 +297,6 @@ class AbstractMetastore(ABC, Serializable):
     #
     # Dataset dependencies
     #
-
-    def add_dependency(
-        self,
-        dependency: DatasetDependency,
-        source_dataset_name: str,
-        source_dataset_version: int,
-    ) -> None:
-        """Add dependency to dataset or storage."""
-        if dependency.is_dataset:
-            self.add_dataset_dependency(
-                source_dataset_name,
-                source_dataset_version,
-                dependency.dataset_name,
-                int(dependency.version),
-            )
-        else:
-            self.add_storage_dependency(
-                source_dataset_name,
-                source_dataset_version,
-                StorageURI(dependency.name),
-                dependency.version,
-            )
-
-    @abstractmethod
-    def add_storage_dependency(
-        self,
-        source_dataset_name: str,
-        source_dataset_version: int,
-        storage_uri: StorageURI,
-        storage_timestamp_str: Optional[str] = None,
-    ) -> None:
-        """Adds storage dependency to dataset."""
-
     @abstractmethod
     def add_dataset_dependency(
         self,
@@ -1268,32 +1235,6 @@ class AbstractDBMetastore(AbstractMetastore):
     #
     # Dataset dependencies
     #
-
-    def _insert_dataset_dependency(self, data: dict[str, Any]) -> None:
-        """Method for inserting dependencies."""
-        self.db.execute(self._datasets_dependencies_insert().values(**data))
-
-    def add_storage_dependency(
-        self,
-        source_dataset_name: str,
-        source_dataset_version: int,
-        storage_uri: StorageURI,
-        storage_timestamp_str: Optional[str] = None,
-    ) -> None:
-        source_dataset = self.get_dataset(source_dataset_name)
-        storage = self.get_storage(storage_uri)
-
-        self._insert_dataset_dependency(
-            {
-                "source_dataset_id": source_dataset.id,
-                "source_dataset_version_id": (
-                    source_dataset.get_version(source_dataset_version).id
-                ),
-                "bucket_id": storage.id,
-                "bucket_version": storage_timestamp_str,
-            }
-        )
-
     def add_dataset_dependency(
         self,
         source_dataset_name: str,
@@ -1305,15 +1246,15 @@ class AbstractDBMetastore(AbstractMetastore):
         source_dataset = self.get_dataset(source_dataset_name)
         dataset = self.get_dataset(dataset_name)
 
-        self._insert_dataset_dependency(
-            {
-                "source_dataset_id": source_dataset.id,
-                "source_dataset_version_id": (
+        self.db.execute(
+            self._datasets_dependencies_insert().values(
+                source_dataset_id=source_dataset.id,
+                source_dataset_version_id=(
                     source_dataset.get_version(source_dataset_version).id
                 ),
-                "dataset_id": dataset.id,
-                "dataset_version_id": dataset.get_version(dataset_version).id,
-            }
+                dataset_id=dataset.id,
+                dataset_version_id=dataset.get_version(dataset_version).id,
+            )
         )
 
     def update_dataset_dependency_source(

datachain/data_storage/sqlite.py

@@ -40,7 +40,9 @@ if TYPE_CHECKING:
     from sqlalchemy.dialects.sqlite import Insert
     from sqlalchemy.engine.base import Engine
     from sqlalchemy.schema import SchemaItem
+    from sqlalchemy.sql._typing import _FromClauseArgument, _OnClauseArgument
     from sqlalchemy.sql.elements import ColumnElement
+    from sqlalchemy.sql.selectable import Join
     from sqlalchemy.types import TypeEngine
 
     from datachain.lib.file import File
@@ -649,11 +651,14 @@ class SQLiteWarehouse(AbstractWarehouse):
         self, dataset: DatasetRecord, version: int
     ) -> list[StorageURI]:
         dr = self.dataset_rows(dataset, version)
-        query = dr.select(dr.c.source).distinct()
+        query = dr.select(dr.c.file__source).distinct()
         cur = self.db.cursor()
         cur.row_factory = sqlite3.Row  # type: ignore[assignment]
 
-        return [StorageURI(row["source"]) for row in self.db.execute(query, cursor=cur)]
+        return [
+            StorageURI(row["file__source"])
+            for row in self.db.execute(query, cursor=cur)
+        ]
 
     def merge_dataset_rows(
         self,
@@ -788,6 +793,23 @@ class SQLiteWarehouse(AbstractWarehouse):
             if progress_cb:
                 progress_cb(len(batch_ids))
 
+    def join(
+        self,
+        left: "_FromClauseArgument",
+        right: "_FromClauseArgument",
+        onclause: "_OnClauseArgument",
+        inner: bool = True,
+    ) -> "Join":
+        """
+        Join two tables together.
+        """
+        return sqlalchemy.join(
+            left,
+            right,
+            onclause,
+            isouter=not inner,
+        )
+
     def create_pre_udf_table(self, query: "Select") -> "Table":
         """
         Create a temporary table from a query for use in a UDF.

datachain/data_storage/warehouse.py

@@ -27,8 +27,12 @@ from datachain.storage import StorageURI
 from datachain.utils import sql_escape_like
 
 if TYPE_CHECKING:
-    from sqlalchemy.sql._typing import _ColumnsClauseArgument
-    from sqlalchemy.sql.selectable import Select
+    from sqlalchemy.sql._typing import (
+        _ColumnsClauseArgument,
+        _FromClauseArgument,
+        _OnClauseArgument,
+    )
+    from sqlalchemy.sql.selectable import Join, Select
     from sqlalchemy.types import TypeEngine
 
     from datachain.data_storage import AbstractIDGenerator, schema
@@ -894,6 +898,18 @@ class AbstractWarehouse(ABC, Serializable):
         Copy the results of a query into a table.
         """
 
+    @abstractmethod
+    def join(
+        self,
+        left: "_FromClauseArgument",
+        right: "_FromClauseArgument",
+        onclause: "_OnClauseArgument",
+        inner: bool = True,
+    ) -> "Join":
+        """
+        Join two tables together.
+        """
+
     @abstractmethod
     def create_pre_udf_table(self, query: "Select") -> "Table":
         """
@@ -922,32 +938,10 @@ class AbstractWarehouse(ABC, Serializable):
         are cleaned up as soon as they are no longer needed.
         """
         with tqdm(desc="Cleanup", unit=" tables") as pbar:
-            for name in names:
+            for name in set(names):
                 self.db.drop_table(Table(name, self.db.metadata), if_exists=True)
                 pbar.update(1)
 
-    def changed_query(
-        self,
-        source_query: sa.sql.selectable.Select,
-        target_query: sa.sql.selectable.Select,
-    ) -> sa.sql.selectable.Select:
-        sq = source_query.alias("source_query")
-        tq = target_query.alias("target_query")
-
-        source_target_join = sa.join(
-            sq, tq, (sq.c.source == tq.c.source) & (sq.c.path == tq.c.path)
-        )
-
-        return (
-            select(*sq.c)
-            .select_from(source_target_join)
-            .where(
-                (sq.c.last_modified > tq.c.last_modified)
-                & (sq.c.is_latest == true())
-                & (tq.c.is_latest == true())
-            )
-        )
-
 
 def _random_string(length: int) -> str:
     return "".join(

datachain/lib/dc.py

@@ -1337,8 +1337,7 @@ class DataChain(DatasetQuery):
                     other.signals_schema.resolve(*right_on).db_signals(),
                 )  # type: ignore[arg-type]
             )
-
-        return super()._subtract(other, signals)  # type: ignore[arg-type]
+        return super().subtract(other, signals)  # type: ignore[arg-type]
 
     @classmethod
     def from_values(

datachain/lib/listing.py

@@ -77,6 +77,7 @@ def parse_listing_uri(uri: str, cache, client_config) -> tuple[str, str, str]:
     """
     Parsing uri and returns listing dataset name, listing uri and listing path
     """
+    client_config = client_config or {}
     client = Client.get_client(uri, cache, **client_config)
     storage_uri, path = Client.parse_url(uri)
 

datachain/lib/tar.py

@@ -30,4 +30,5 @@ def process_tar(file: File) -> Iterator[File]:
     with file.open() as fd:
         with tarfile.open(fileobj=fd) as tar:
             for entry in tar.getmembers():
-                yield build_tar_member(file, entry)
+                if entry.isfile():
+                    yield build_tar_member(file, entry)

datachain/node.py

@@ -114,9 +114,23 @@ class Node:
         )
 
     @classmethod
-    def from_dict(cls, d: dict[str, Any]) -> "Self":
-        kw = {f.name: d[f.name] for f in attrs.fields(cls) if f.name in d}
-        return cls(**kw)
+    def from_dict(cls, d: dict[str, Any], file_prefix: str = "file") -> "Self":
+        def _dval(field_name: str):
+            return d.get(f"{file_prefix}__{field_name}")
+
+        return cls(
+            sys__id=d["sys__id"],
+            sys__rand=d["sys__rand"],
+            source=_dval("source"),
+            path=_dval("path"),
+            etag=_dval("etag"),
+            is_latest=_dval("is_latest"),
+            size=_dval("size"),
+            last_modified=_dval("last_modified"),
+            version=_dval("version"),
+            location=_dval("location"),
+            dir_type=DirType.FILE,
+        )
 
     @classmethod
     def from_dir(cls, path, **kwargs) -> "Node":

datachain/query/__init__.py

@@ -2,7 +2,6 @@ from .dataset import DatasetQuery
 from .params import param
 from .schema import C, DatasetRow, LocalFilename, Object, Stream
 from .session import Session
-from .udf import udf
 
 __all__ = [
     "C",
@@ -13,5 +12,4 @@ __all__ = [
     "Session",
     "Stream",
     "param",
-    "udf",
 ]

datachain/query/dataset.py

@@ -3,7 +3,6 @@ import inspect
 import logging
 import os
 import random
-import re
 import string
 import subprocess
 import sys
@@ -33,11 +32,9 @@ from sqlalchemy.sql.elements import ColumnClause, ColumnElement
 from sqlalchemy.sql.expression import label
 from sqlalchemy.sql.schema import TableClause
 from sqlalchemy.sql.selectable import Select
-from tqdm import tqdm
 
 from datachain.asyn import ASYNC_WORKERS, AsyncMapper, OrderedMapper
 from datachain.catalog import QUERY_SCRIPT_CANCELED_EXIT_CODE, get_catalog
-from datachain.client import Client
 from datachain.data_storage.schema import (
     PARTITION_COLUMN_ID,
     partition_col_names,
@@ -47,7 +44,6 @@ from datachain.dataset import DatasetStatus, RowDict
 from datachain.error import DatasetNotFoundError, QueryScriptCancelError
 from datachain.progress import CombinedDownloadCallback
 from datachain.sql.functions import rand
-from datachain.storage import Storage, StorageURI
 from datachain.utils import (
     batched,
     determine_processes,
@@ -78,9 +74,7 @@ INSERT_BATCH_SIZE = 10000
 
 PartitionByType = Union[ColumnElement, Sequence[ColumnElement]]
 JoinPredicateType = Union[str, ColumnClause, ColumnElement]
-# dependency can be either dataset_name + dataset_version tuple or just storage uri
-# depending what type of dependency we are adding
-DatasetDependencyType = Union[tuple[str, int], StorageURI]
+DatasetDependencyType = tuple[str, int]
 
 logger = logging.getLogger("datachain")
 
@@ -186,38 +180,6 @@ class QueryStep(StartingStep):
         )
 
 
-@frozen
-class IndexingStep(StartingStep):
-    path: str
-    catalog: "Catalog"
-    kwargs: dict[str, Any]
-    recursive: Optional[bool] = True
-
-    def apply(self):
-        self.catalog.index([self.path], **self.kwargs)
-        uri, path = Client.parse_url(self.path)
-        _partial_id, partial_path = self.catalog.metastore.get_valid_partial_id(
-            uri, path
-        )
-        dataset = self.catalog.get_dataset(Storage.dataset_name(uri, partial_path))
-        dataset_rows = self.catalog.warehouse.dataset_rows(
-            dataset, dataset.latest_version
-        )
-
-        def q(*columns):
-            col_names = [c.name for c in columns]
-            return self.catalog.warehouse.nodes_dataset_query(
-                dataset_rows,
-                column_names=col_names,
-                path=path,
-                recursive=self.recursive,
-            )
-
-        storage = self.catalog.metastore.get_storage(uri)
-
-        return step_result(q, dataset_rows.c, dependencies=[storage.uri])
-
-
 def generator_then_call(generator, func: Callable):
     """
     Yield items from generator then execute a function and yield
@@ -231,7 +193,7 @@ def generator_then_call(generator, func: Callable):
 class DatasetDiffOperation(Step):
     """
     Abstract class for operations that are calculation some kind of diff between
-    datasets queries like subtract, changed etc.
+    datasets queries like subtract etc.
     """
 
     dq: "DatasetQuery"
@@ -305,28 +267,6 @@ class Subtract(DatasetDiffOperation):
         return sq.select().except_(sq.select().where(where_clause))
 
 
-@frozen
-class Changed(DatasetDiffOperation):
-    """
-    Calculates rows that are changed in a source query compared to target query
-    Changed means it has same source + path but different last_modified
-    Example:
-        >>> ds = DatasetQuery(name="dogs_cats") # some older dataset with embeddings
-        >>> ds_updated = (
-                DatasetQuery("gs://dvcx-datalakes/dogs-and-cats")
-                .filter(C.size > 1000) # we can also filter out source query
-                .changed(ds)
-                .add_signals(calc_embeddings) # calculae embeddings only on changed rows
-                .union(ds) # union with old dataset that's missing updated rows
-                .save("dogs_cats_updated")
-            )
-
-    """
-
-    def query(self, source_query: Select, target_query: Select) -> Select:
-        return self.catalog.warehouse.changed_query(source_query, target_query)
-
-
 def adjust_outputs(
     warehouse: "AbstractWarehouse", row: dict[str, Any], udf_col_types: list[tuple]
 ) -> dict[str, Any]:
@@ -899,12 +839,36 @@ class SQLUnion(Step):
 
 @frozen
 class SQLJoin(Step):
+    catalog: "Catalog"
     query1: "DatasetQuery"
     query2: "DatasetQuery"
     predicates: Union[JoinPredicateType, tuple[JoinPredicateType, ...]]
     inner: bool
     rname: str
 
+    def get_query(self, dq: "DatasetQuery", temp_tables: list[str]) -> sa.Subquery:
+        query = dq.apply_steps().select()
+        temp_tables.extend(dq.temp_table_names)
+
+        if not any(isinstance(step, (SQLJoin, SQLUnion)) for step in dq.steps):
+            return query.subquery(dq.table.name)
+
+        warehouse = self.catalog.warehouse
+
+        columns = [
+            c if isinstance(c, Column) else Column(c.name, c.type)
+            for c in query.subquery().columns
+        ]
+        temp_table = warehouse.create_dataset_rows_table(
+            warehouse.temp_table_name(),
+            columns=columns,
+        )
+        temp_tables.append(temp_table.name)
+
+        warehouse.copy_table(temp_table, query)
+
+        return temp_table.select().subquery(dq.table.name)
+
     def validate_expression(self, exp: "ClauseElement", q1, q2):
         """
         Checking if columns used in expression actually exist in left / right
@@ -937,10 +901,8 @@ class SQLJoin(Step):
     def apply(
         self, query_generator: QueryGenerator, temp_tables: list[str]
     ) -> StepResult:
-        q1 = self.query1.apply_steps().select().subquery(self.query1.table.name)
-        temp_tables.extend(self.query1.temp_table_names)
-        q2 = self.query2.apply_steps().select().subquery(self.query2.table.name)
-        temp_tables.extend(self.query2.temp_table_names)
+        q1 = self.get_query(self.query1, temp_tables)
+        q2 = self.get_query(self.query2, temp_tables)
 
         q1_columns = list(q1.c)
         q1_column_names = {c.name for c in q1_columns}
@@ -951,7 +913,12 @@ class SQLJoin(Step):
                 continue
 
             if c.name in q1_column_names:
-                c = c.label(self.rname.format(name=c.name))
+                new_name = self.rname.format(name=c.name)
+                new_name_idx = 0
+                while new_name in q1_column_names:
+                    new_name_idx += 1
+                    new_name = self.rname.format(name=f"{c.name}_{new_name_idx}")
+                c = c.label(new_name)
             q2_columns.append(c)
 
         res_columns = q1_columns + q2_columns
@@ -979,16 +946,14 @@ class SQLJoin(Step):
         self.validate_expression(join_expression, q1, q2)
 
         def q(*columns):
-            join_query = sqlalchemy.join(
+            join_query = self.catalog.warehouse.join(
                 q1,
                 q2,
                 join_expression,
-                isouter=not self.inner,
+                inner=self.inner,
             )
-
-            res = sqlalchemy.select(*columns).select_from(join_query)
-            subquery = res.subquery()
-            return sqlalchemy.select(*subquery.c).select_from(subquery)
+            return sqlalchemy.select(*columns).select_from(join_query)
+            # return sqlalchemy.select(*subquery.c).select_from(subquery)
 
         return step_result(
             q,
@@ -1072,28 +1037,14 @@ class ResultIter:
 class DatasetQuery:
     def __init__(
         self,
-        path: str = "",
-        name: str = "",
+        name: str,
         version: Optional[int] = None,
         catalog: Optional["Catalog"] = None,
-        client_config=None,
-        recursive: Optional[bool] = True,
         session: Optional[Session] = None,
-        anon: bool = False,
-        indexing_feature_schema: Optional[dict] = None,
         indexing_column_types: Optional[dict[str, Any]] = None,
-        update: Optional[bool] = False,
         in_memory: bool = False,
     ):
-        if client_config is None:
-            client_config = {}
-
-        if anon:
-            client_config["anon"] = True
-
-        self.session = Session.get(
-            session, catalog=catalog, client_config=client_config, in_memory=in_memory
-        )
+        self.session = Session.get(session, catalog=catalog, in_memory=in_memory)
         self.catalog = catalog or self.session.catalog
         self.steps: list[Step] = []
         self._chunk_index: Optional[int] = None
@@ -1107,26 +1058,14 @@ class DatasetQuery:
         self.feature_schema: Optional[dict] = None
         self.column_types: Optional[dict[str, Any]] = None
 
-        if path:
-            kwargs = {"update": True} if update else {}
-            self.starting_step = IndexingStep(path, self.catalog, kwargs, recursive)
-            self.feature_schema = indexing_feature_schema
-            self.column_types = indexing_column_types
-        elif name:
-            self.name = name
-            ds = self.catalog.get_dataset(name)
-            self.version = version or ds.latest_version
-            self.feature_schema = ds.get_version(self.version).feature_schema
-            self.column_types = copy(ds.schema)
-            if "sys__id" in self.column_types:
-                self.column_types.pop("sys__id")
-            self.starting_step = QueryStep(self.catalog, name, self.version)
-        else:
-            raise ValueError("must provide path or name")
-
-    @staticmethod
-    def is_storage_path(path):
-        return bool(re.compile(r"^[a-zA-Z0-9]+://").match(path))
+        self.name = name
+        ds = self.catalog.get_dataset(name)
+        self.version = version or ds.latest_version
+        self.feature_schema = ds.get_version(self.version).feature_schema
+        self.column_types = copy(ds.schema)
+        if "sys__id" in self.column_types:
+            self.column_types.pop("sys__id")
+        self.starting_step = QueryStep(self.catalog, name, self.version)
 
     def __iter__(self):
         return iter(self.db_results())
@@ -1511,7 +1450,7 @@ class DatasetQuery:
             if isinstance(predicates, (str, ColumnClause, ColumnElement))
             else tuple(predicates)
         )
-        new_query.steps = [SQLJoin(left, right, predicates, inner, rname)]
+        new_query.steps = [SQLJoin(self.catalog, left, right, predicates, inner, rname)]
         return new_query
 
     @detach
@@ -1571,21 +1510,11 @@ class DatasetQuery:
         return query
 
     @detach
-    def subtract(self, dq: "DatasetQuery") -> "Self":
-        return self._subtract(dq, on=[("source", "source"), ("path", "path")])
-
-    @detach
-    def _subtract(self, dq: "DatasetQuery", on: Sequence[tuple[str, str]]) -> "Self":
+    def subtract(self, dq: "DatasetQuery", on: Sequence[tuple[str, str]]) -> "Self":
         query = self.clone()
         query.steps.append(Subtract(dq, self.catalog, on=on))
         return query
 
-    @detach
-    def changed(self, dq: "DatasetQuery") -> "Self":
-        query = self.clone()
-        query.steps.append(Changed(dq, self.catalog))
-        return query
-
     @detach
     def generate(
         self,
@@ -1616,24 +1545,13 @@ class DatasetQuery:
 
     def _add_dependencies(self, dataset: "DatasetRecord", version: int):
         for dependency in self.dependencies:
-            if isinstance(dependency, tuple):
-                # dataset dependency
-                ds_dependency_name, ds_dependency_version = dependency
-                self.catalog.metastore.add_dataset_dependency(
-                    dataset.name,
-                    version,
-                    ds_dependency_name,
-                    ds_dependency_version,
-                )
-            else:
-                # storage dependency - its name is a valid StorageURI
-                storage = self.catalog.metastore.get_storage(dependency)
-                self.catalog.metastore.add_storage_dependency(
-                    StorageURI(dataset.name),
-                    version,
-                    storage.uri,
-                    storage.timestamp_str,
-                )
+            ds_dependency_name, ds_dependency_version = dependency
+            self.catalog.metastore.add_dataset_dependency(
+                dataset.name,
+                version,
+                ds_dependency_name,
+                ds_dependency_version,
+            )
 
     def exec(self) -> "Self":
         """Execute the query."""
@@ -1687,12 +1605,7 @@ class DatasetQuery:
 
             dr = self.catalog.warehouse.dataset_rows(dataset)
 
-            with tqdm(desc="Saving", unit=" rows") as pbar:
-                self.catalog.warehouse.copy_table(
-                    dr.get_table(),
-                    query.select(),
-                    progress_cb=pbar.update,
-                )
+            self.catalog.warehouse.copy_table(dr.get_table(), query.select())
 
             self.catalog.metastore.update_dataset_status(
                 dataset, DatasetStatus.COMPLETE, version=version

datachain/query/schema.py

@@ -19,6 +19,17 @@ if TYPE_CHECKING:
 DEFAULT_DELIMITER = "__"
 
 
+def file_signals(row, signal_name="file"):
+    # TODO this is workaround until we decide what to do with these classes
+    prefix = f"{signal_name}{DEFAULT_DELIMITER}"
+    return {
+        c_name.removeprefix(prefix): c_value
+        for c_name, c_value in row.items()
+        if c_name.startswith(prefix)
+        and DEFAULT_DELIMITER not in c_name.removeprefix(prefix)
+    }
+
+
 class ColumnMeta(type):
     @staticmethod
     def to_db_name(name: str) -> str:
@@ -86,8 +97,8 @@ class Object(UDFParameter):
         cb: Callback = DEFAULT_CALLBACK,
         **kwargs,
     ) -> Any:
-        client = catalog.get_client(row["source"])
-        uid = catalog._get_row_uid(row)
+        client = catalog.get_client(row["file__source"])
+        uid = catalog._get_row_uid(file_signals(row))
         if cache:
             client.download(uid, callback=cb)
         with client.open_object(uid, use_cache=cache, cb=cb) as f:
@@ -103,8 +114,8 @@ class Object(UDFParameter):
         cb: Callback = DEFAULT_CALLBACK,
         **kwargs,
     ) -> Any:
-        client = catalog.get_client(row["source"])
-        uid = catalog._get_row_uid(row)
+        client = catalog.get_client(row["file__source"])
+        uid = catalog._get_row_uid(file_signals(row))
         if cache:
             await client._download(uid, callback=cb)
         obj = await mapper.to_thread(
@@ -129,8 +140,8 @@ class Stream(UDFParameter):
         cb: Callback = DEFAULT_CALLBACK,
         **kwargs,
     ) -> Any:
-        client = catalog.get_client(row["source"])
-        uid = catalog._get_row_uid(row)
+        client = catalog.get_client(row["file__source"])
+        uid = catalog._get_row_uid(file_signals(row))
         if cache:
             client.download(uid, callback=cb)
         return client.open_object(uid, use_cache=cache, cb=cb)
@@ -145,8 +156,8 @@ class Stream(UDFParameter):
         cb: Callback = DEFAULT_CALLBACK,
         **kwargs,
     ) -> Any:
-        client = catalog.get_client(row["source"])
-        uid = catalog._get_row_uid(row)
+        client = catalog.get_client(row["file__source"])
+        uid = catalog._get_row_uid(file_signals(row))
         if cache:
             await client._download(uid, callback=cb)
         return await mapper.to_thread(
@@ -178,8 +189,8 @@ class LocalFilename(UDFParameter):
             # If the glob pattern is specified and the row filename
             # does not match it, then return None
             return None
-        client = catalog.get_client(row["source"])
-        uid = catalog._get_row_uid(row)
+        client = catalog.get_client(row["file__source"])
+        uid = catalog._get_row_uid(file_signals(row))
         client.download(uid, callback=cb)
         return client.cache.get_path(uid)
 
@@ -197,8 +208,8 @@ class LocalFilename(UDFParameter):
             # If the glob pattern is specified and the row filename
             # does not match it, then return None
             return None
-        client = catalog.get_client(row["source"])
-        uid = catalog._get_row_uid(row)
+        client = catalog.get_client(row["file__source"])
+        uid = catalog._get_row_uid(file_signals(row))
         await client._download(uid, callback=cb)
         return client.cache.get_path(uid)
 

datachain/query/udf.py

@@ -1,8 +1,7 @@
 import typing
-from collections.abc import Iterable, Iterator, Mapping, Sequence
+from collections.abc import Iterable, Iterator, Sequence
 from dataclasses import dataclass
 from functools import WRAPPER_ASSIGNMENTS
-from inspect import isclass
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -23,11 +22,7 @@ from .batch import (
     RowsOutputBatch,
     UDFInputBatch,
 )
-from .schema import (
-    UDFParameter,
-    UDFParamSpec,
-    normalize_param,
-)
+from .schema import UDFParameter
 
 if TYPE_CHECKING:
     from datachain.catalog import Catalog
@@ -66,41 +61,6 @@ class UDFProperties:
         return self.output.keys()
 
 
-def udf(
-    params: Sequence[UDFParamSpec],
-    output: UDFOutputSpec,
-    *,
-    method: Optional[str] = None,  # only used for class-based UDFs
-    batch: int = 1,
-):
-    """
-    Decorate a function or a class to be used as a UDF.
-
-    The decorator expects both the outputs and inputs of the UDF to be specified.
-    The outputs are defined as a collection of tuples containing the signal name
-    and type.
-    Parameters are defined as a list of column objects (e.g. C.name).
-    Optionally, UDFs can be run on batches of rows to improve performance, this
-    is determined by the 'batch' parameter. When operating on batches of inputs,
-    the UDF function will be called with a single argument - a list
-    of tuples containing inputs (e.g. ((input1_a, input1_b), (input2_a, input2b))).
-    """
-    if isinstance(params, str):
-        params = (params,)
-    if not isinstance(output, Mapping):
-        raise TypeError(f"'output' must be a mapping, got {type(output).__name__}")
-
-    properties = UDFProperties([normalize_param(p) for p in params], output, batch)
-
-    def decorator(udf_base: Union[Callable, type]):
-        if isclass(udf_base):
-            return UDFClassWrapper(udf_base, properties, method=method)
-        if callable(udf_base):
-            return UDFWrapper(udf_base, properties)
-
-    return decorator
-
-
 class UDFBase:
     """A base class for implementing stateful UDFs."""
 

datachain/utils.py

@@ -1,5 +1,4 @@
 import glob
-import importlib.util
 import io
 import json
 import os
@@ -198,45 +197,6 @@ def get_envs_by_prefix(prefix: str) -> dict[str, str]:
     return variables
 
 
-def import_object(object_spec):
-    filename, identifier = object_spec.rsplit(":", 1)
-    filename = filename.strip()
-    identifier = identifier.strip()
-
-    if not identifier.isidentifier() or not filename.endswith(".py"):
-        raise ValueError(f"Invalid object spec: {object_spec}")
-
-    modname = os.path.abspath(filename)
-    if modname in sys.modules:
-        module = sys.modules[modname]
-    else:
-        # Use importlib to find and load the module from the given filename
-        spec = importlib.util.spec_from_file_location(modname, filename)
-        module = importlib.util.module_from_spec(spec)
-        sys.modules[modname] = module
-        spec.loader.exec_module(module)
-
-    return getattr(module, identifier)
-
-
-def parse_params_string(params: str):
-    """
-    Parse a string containing UDF class constructor parameters in the form
-    `a, b, key=val` into *args and **kwargs.
-    """
-    args = []
-    kwargs = {}
-    for part in params.split():
-        if "=" in part:
-            key, val = part.split("=")
-            kwargs[key] = val
-        else:
-            args.append(part)
-    if any((args, kwargs)):
-        return args, kwargs
-    return None, None
-
-
 _T_co = TypeVar("_T_co", covariant=True)
 
 

{datachain-0.3.15.dist-info → datachain-0.3.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.15
+Version: 0.3.17
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.3.15.dist-info → datachain-0.3.17.dist-info}/RECORD

@@ -2,22 +2,22 @@ datachain/__init__.py,sha256=GeyhE-5LgfJav2OKYGaieP2lBvf2Gm-ihj7thnK9zjI,800
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=Lg3Ck1PQLjQziMx9KU4atzbEnJXTE0924WMYkhgWtGU,8247
 datachain/cache.py,sha256=WP-ktH_bRn3w2g1JOOQ7rCPsZyR4OM6K1Kb7yZsSSns,4056
-datachain/cli.py,sha256=alMjnoBUBLvBSMBR51N09rA_aUEdHJwyxSRogF7VbbA,30891
+datachain/cli.py,sha256=zObcD5W8dzUJKk2RGQ1MxQLEr3jnox6bybU8WyDaIqE,29941
 datachain/cli_utils.py,sha256=jrn9ejGXjybeO1ur3fjdSiAyCHZrX0qsLLbJzN9ErPM,2418
 datachain/config.py,sha256=PfC7W5yO6HFO6-iMB4YB-0RR88LPiGmD6sS_SfVbGso,1979
 datachain/dataset.py,sha256=sHnsmKfMg2bK88gZH1izk8jlbmJDEhQpyOemdaPQVFo,14761
 datachain/error.py,sha256=OnZ8OaBtDdTZPy8XQiy29SAjqdQArQeorYbP5ju7ldc,1199
 datachain/job.py,sha256=Jt4sNutMHJReaGsj3r3scueN5aESLGfhimAa8pUP7Is,1271
 datachain/listing.py,sha256=vfjOlcb98A7xkGGKWEYON6l7lfrOqNv6kldmdVnlJn4,8178
-datachain/node.py,sha256=2pF3Y9oYzElfiUBcw2LIv7LNNt--V4E-K021zjv0b0I,4748
+datachain/node.py,sha256=-Y8O7q7NtIm_jX0HgjhjvdFwm73TrO5QBslxvFVwTJE,5208
 datachain/nodes_fetcher.py,sha256=kca19yvu11JxoVY1t4_ydp1FmchiV88GnNicNBQ9NIA,831
 datachain/nodes_thread_pool.py,sha256=ZyzBvUImIPmi4WlKC2SW2msA0UhtembbTdcs2nx29A0,3191
 datachain/progress.py,sha256=7_8FtJs770ITK9sMq-Lt4k4k18QmYl4yIG_kCoWID3o,4559
 datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/storage.py,sha256=RiSJLYdHUjnrEWkLBKPcETHpAxld_B2WxLg711t0aZI,3733
-datachain/utils.py,sha256=Z9-lPNvrrAh_VWpzVBJ7L5-Oy_Oo1V0ZW7G0MVDyPK4,13065
+datachain/utils.py,sha256=VGAcTWjGF0e2qB3Se77shhpiqGMo-ol0QAwf3MH5b7c,11857
 datachain/catalog/__init__.py,sha256=g2iAAFx_gEIrqshXlhSEbrc8qDaEH11cjU40n3CHDz4,409
-datachain/catalog/catalog.py,sha256=kPg5ILeCWSjXCj3ewUZY6kzj36HTEqajB3mJDkbs-Vo,69023
+datachain/catalog/catalog.py,sha256=IAaaSVFxtJxVlIGEbu8sHinmYyeDGY6dg7APrtOchVk,68278
 datachain/catalog/datasource.py,sha256=D-VWIVDCM10A8sQavLhRXdYSCG7F4o4ifswEF80_NAQ,1412
 datachain/catalog/loader.py,sha256=-6VelNfXUdgUnwInVyA8g86Boxv2xqhTh9xNS-Zlwig,8242
 datachain/client/__init__.py,sha256=T4wiYL9KIM0ZZ_UqIyzV8_ufzYlewmizlV4iymHNluE,86
@@ -32,28 +32,28 @@ datachain/data_storage/__init__.py,sha256=cEOJpyu1JDZtfUupYucCDNFI6e5Wmp_Oyzq6rZ
 datachain/data_storage/db_engine.py,sha256=81Ol1of9TTTzD97ORajCnP366Xz2mEJt6C-kTUCaru4,3406
 datachain/data_storage/id_generator.py,sha256=lCEoU0BM37Ai2aRpSbwo5oQT0GqZnSpYwwvizathRMQ,4292
 datachain/data_storage/job.py,sha256=w-7spowjkOa1P5fUVtJou3OltT0L48P0RYWZ9rSJ9-s,383
-datachain/data_storage/metastore.py,sha256=cHN0xmbUvChyayHHZm3Vqxr87jFqojPSlGBqhTPStlE,54519
+datachain/data_storage/metastore.py,sha256=BePe3bVxo-Zuuccok8TLRo4cMHVnAIa8hfZMadbxzqM,52649
 datachain/data_storage/schema.py,sha256=AGbjyEir5UmRZXI3m0jChZogUh5wd8csj6-YlUWaAxQ,8383
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
-datachain/data_storage/sqlite.py,sha256=yooLHQXrpoqDguGlF0SGcCiMU1T82OEc4wr1ra8eBHo,28285
-datachain/data_storage/warehouse.py,sha256=Pq6Nt3fyz1WFv6Mdtv2ZUr0_GFCNbafbtS4PdibblUg,32507
+datachain/data_storage/sqlite.py,sha256=EBKJncuzcyQfcKFm2mUjvHjHRTODsteM-k_zndunBrw,28834
+datachain/data_storage/warehouse.py,sha256=Vwhu_OfcNAoTtg1BHui80VCzlPeTUjZQL0QWziu8awY,32186
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/arrow.py,sha256=voY9KuJ2uhPxw_DS6rIjwfKjWXi84T3LFJ7kGFcDQuk,7272
 datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
 datachain/lib/data_model.py,sha256=gHIjlow84GMRDa78yLL1Ud-N18or21fnTyPEwsatpXY,2045
 datachain/lib/dataset_info.py,sha256=srPPhI2UHf6hFPBecyFEVw2SS5aPisIIMsvGgKqi7ss,2366
-datachain/lib/dc.py,sha256=HERJNR4TISbaAtSLARV72INgKPfQRItyd1l28P-GtzU,68871
+datachain/lib/dc.py,sha256=DkzuKS14kgAEax47Gi9w_XJXV3dbboW85A0YOxwNjKY,68869
 datachain/lib/file.py,sha256=elQLorLbIkusuQSVfiuC_KrGSZI8cGm-iT8fHmckJlo,13774
 datachain/lib/hf.py,sha256=cPnmLuprr0pYABH7KqA5FARQ1JGlywdDwD3yDzVAm4k,5920
 datachain/lib/image.py,sha256=AMXYwQsmarZjRbPCZY3M1jDsM2WAB_b3cTY4uOIuXNU,2675
-datachain/lib/listing.py,sha256=e4O1gs3rKJ0eGwb0hSEfD-l9U7x-f-TYqYGF7Ni-x38,3973
+datachain/lib/listing.py,sha256=NrKb7_6jwp1vEqp5TavSvx3SbLJdvuBzSEclPvbQr30,4013
 datachain/lib/listing_info.py,sha256=36NZ-tXY5Y118wurkajuWWbcE8UCjkRwZlacDtN9F3g,954
 datachain/lib/meta_formats.py,sha256=3f-0vpMTesagS9iMd3y9-u9r-7g0eqYsxmK4fVfNWlw,6635
 datachain/lib/model_store.py,sha256=DNIv8Y6Jtk1_idNLzIpsThOsdW2BMAudyUCbPUcgcxk,2515
 datachain/lib/pytorch.py,sha256=8LNyFaBrx8zws--MEsFg5g3pb8oLnaQAUlgGvtjKxX4,5960
 datachain/lib/settings.py,sha256=39thOpYJw-zPirzeNO6pmRC2vPrQvt4eBsw1xLWDFsw,2344
 datachain/lib/signal_schema.py,sha256=iqgubjCBRiUJB30miv05qFX4uU04dA_Pzi3DCUsHZGs,24177
-datachain/lib/tar.py,sha256=d7FpYyxbHCL1twRt_Oe9QoPbZa2Tn5lj7iWP0HvvRn0,999
+datachain/lib/tar.py,sha256=3WIzao6yD5fbLqXLTt9GhPGNonbFIs_fDRu-9vgLgsA,1038
 datachain/lib/text.py,sha256=UNHm8fhidk7wdrWqacEWaA6I9ykfYqarQ2URby7jc7M,1261
 datachain/lib/udf.py,sha256=nG7DDuPgZ5ZuijwvDoCq-OZMxlDM8vFNzyxMmik0Y1c,11716
 datachain/lib/udf_signature.py,sha256=gMStcEeYJka5M6cg50Z9orC6y6HzCAJ3MkFqqn1fjZg,7137
@@ -67,17 +67,16 @@ datachain/lib/convert/python_to_sql.py,sha256=40SAOdoOgikZRhn8iomCPDRoxC3RFxjJLi
 datachain/lib/convert/sql_to_python.py,sha256=lGnKzSF_tz9Y_5SSKkrIU95QEjpcDzvOxIRkEKTQag0,443
 datachain/lib/convert/unflatten.py,sha256=Ogvh_5wg2f38_At_1lN0D_e2uZOOpYEvwvB2xdq56Tw,2012
 datachain/lib/convert/values_to_tuples.py,sha256=YOdbjzHq-uj6-cV2Qq43G72eN2avMNDGl4x5t6yQMl8,3931
-datachain/query/__init__.py,sha256=tv-spkjUCYamMN9ys_90scYrZ8kJ7C7d1MTYVmxGtk4,325
+datachain/query/__init__.py,sha256=0NBOZVgIDpCcj1Ci883dQ9A0iiwe03xzmotkOCFbxYc,293
 datachain/query/batch.py,sha256=-vlpINJiertlnaoUVv1C95RatU0F6zuhpIYRufJRo1M,3660
-datachain/query/builtins.py,sha256=U6yHPF9bzxqK5iwyqCqbJxo8ggBVx9FtuXxRrQQ0SNM,2244
-datachain/query/dataset.py,sha256=9lhcgccavqypVParE4pvd_Hgg8gmoDAN6m1IkpSwXhE,58219
+datachain/query/dataset.py,sha256=27QCFhwz4hK-pqTY0hvfFqwxNIH5yxuSjWRl1ZfELd0,55004
 datachain/query/dispatch.py,sha256=GBh3EZHDp5AaXxrjOpfrpfsuy7Umnqxu-MAXcK9X3gc,12945
 datachain/query/metrics.py,sha256=r5b0ygYhokbXp8Mg3kCH8iFSRw0jxzyeBe-C-J_bKFc,938
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
 datachain/query/queue.py,sha256=waqM_KzavU8C-G95-4211Nd4GXna_u2747Chgwtgz2w,3839
-datachain/query/schema.py,sha256=ytlkA1xFAUOia25u8d6pxvxBSRl3uivLuOe2eHaw-qc,7550
+datachain/query/schema.py,sha256=A-Btmu-rR00841PNcs9CjcppfTUc176wT6-m3BOaSgo,8049
 datachain/query/session.py,sha256=UPH5Z4fzCDsvj81ji0e8GA6Mgra3bOAEpVq4htqOtis,4317
-datachain/query/udf.py,sha256=j3NhmKK5rYG5TclcM2Sr0LhS1tmYLMjzMugx9G9iFLM,8100
+datachain/query/udf.py,sha256=QRDcSgJ_zrY63dyMVD8nq4ky9Q2kaKSoM6bsDqpXOvQ,6682
 datachain/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/remote/studio.py,sha256=f5s6qSZ9uB4URGUoU_8_W1KZRRQQVSm6cgEBkBUEfuE,7226
 datachain/sql/__init__.py,sha256=A2djrbQwSMUZZEIKGnm-mnRA-NDSbiDJNpAmmwGNyIo,303
@@ -97,9 +96,9 @@ datachain/sql/sqlite/base.py,sha256=WLPHBhZbXbiqPoRV1VgDrXJqku4UuvJpBhYeQ0k5rI8,
 datachain/sql/sqlite/types.py,sha256=yzvp0sXSEoEYXs6zaYC_2YubarQoZH-MiUNXcpuEP4s,1573
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.3.15.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.3.15.dist-info/METADATA,sha256=E3jImGtRTyvMPTSqFsgwhsHsnZn_9SRVeThmrDXRuf0,17073
-datachain-0.3.15.dist-info/WHEEL,sha256=5Mi1sN9lKoFv_gxcPtisEVrJZihrm_beibeg5R6xb4I,91
-datachain-0.3.15.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.3.15.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.3.15.dist-info/RECORD,,
+datachain-0.3.17.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.3.17.dist-info/METADATA,sha256=bPMIQkvQjnflmMQhJa3BH3Mi4DpHTAI8KQ5Vd2ur8Mo,17073
+datachain-0.3.17.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+datachain-0.3.17.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.3.17.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.3.17.dist-info/RECORD,,

{datachain-0.3.15.dist-info → datachain-0.3.17.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.0.0)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

datachain/query/builtins.py

@@ -1,96 +0,0 @@
-import hashlib
-import tarfile
-from functools import partial
-
-from datachain.sql.types import String
-
-from .schema import C, DatasetRow, Object
-from .udf import udf
-
-md5 = partial(hashlib.md5, usedforsecurity=False)
-
-__all__ = ["checksum", "index_tar"]
-
-
-def load_tar(raw):
-    with tarfile.open(fileobj=raw, mode="r:") as tar:
-        return tar.getmembers()
-
-
-@udf(
-    (
-        C.source,
-        C.path,
-        C.size,
-        C.is_latest,
-        C.last_modified,
-        C.version,
-        C.etag,
-        Object(load_tar),
-    ),
-    DatasetRow.schema,
-)
-def index_tar(
-    source,
-    parent_path,
-    size,
-    is_latest,
-    last_modified,
-    version,
-    etag,
-    tar_entries,
-):
-    # generate original tar files as well, along with subobjects
-    yield DatasetRow.create(
-        source=source,
-        path=parent_path,
-        size=size,
-        is_latest=bool(is_latest),
-        last_modified=last_modified,
-        version=version,
-        etag=etag,
-    )
-
-    for info in tar_entries:
-        if info.isfile():
-            full_path = f"{parent_path}/{info.name}"
-            yield DatasetRow.create(
-                source=source,
-                path=full_path,
-                size=info.size,
-                location={
-                    "vtype": "tar",
-                    "offset": info.offset_data,
-                    "size": info.size,
-                    "parent": {
-                        "source": source,
-                        "path": parent_path,
-                        "version": version,
-                        "size": size,
-                        "etag": etag,
-                        "location": None,
-                    },
-                },
-            )
-
-
-BUFSIZE = 2**18
-
-
-def file_digest(fileobj):
-    """Calculate the digest of a file-like object."""
-    buf = bytearray(BUFSIZE)  # Reusable buffer to reduce allocations.
-    view = memoryview(buf)
-    digestobj = md5()
-    # From 3.11's hashlib.filedigest()
-    while True:
-        size = fileobj.readinto(buf)
-        if size == 0:
-            break  # EOF
-        digestobj.update(view[:size])
-    return digestobj.hexdigest()
-
-
-@udf(params=[Object(file_digest)], output={"checksum": String})
-def checksum(digest):
-    return (digest,)