datachain 0.2.18__py3-none-any.whl → 0.3.1__py3-none-any.whl

datachain/cache.py

@@ -24,8 +24,7 @@ sha256 = partial(hashlib.sha256, usedforsecurity=False)
 @attrs.frozen
 class UniqueId:
     storage: "StorageURI"
-    parent: str
-    name: str
+    path: str
     size: int
     etag: str
     version: str = ""
@@ -34,10 +33,6 @@ class UniqueId:
     location: Optional[str] = None
     last_modified: datetime = TIME_ZERO
 
-    @property
-    def path(self) -> str:
-        return f"{self.parent}/{self.name}" if self.parent else self.name
-
     def get_parsed_location(self) -> Optional[dict]:
         if not self.location:
             return None
@@ -53,10 +48,10 @@ class UniqueId:
         return loc_stack[0]
 
     def get_hash(self) -> str:
-        etag = f"{self.vtype}{self.location}" if self.vtype else self.etag
-        return sha256(
-            f"{self.storage}/{self.parent}/{self.name}/{self.version}/{etag}".encode()
-        ).hexdigest()
+        fingerprint = f"{self.storage}/{self.path}/{self.version}/{self.etag}"
+        if self.location:
+            fingerprint += f"/{self.location}"
+        return sha256(fingerprint.encode()).hexdigest()
 
 
 def try_scandir(path):

datachain/catalog/catalog.py

@@ -529,21 +529,16 @@ def find_column_to_str(  # noqa: PLR0911
     if column == "du":
         return str(
             src.listing.du(
-                {
-                    f: row[field_lookup[f]]
-                    for f in ["dir_type", "size", "parent", "name"]
-                }
+                {f: row[field_lookup[f]] for f in ["dir_type", "size", "path"]}
             )[0]
         )
     if column == "name":
-        return row[field_lookup["name"]] or ""
+        return posixpath.basename(row[field_lookup["path"]]) or ""
     if column == "owner":
         return row[field_lookup["owner_name"]] or ""
     if column == "path":
         is_dir = row[field_lookup["dir_type"]] == DirType.DIR
-        parent = row[field_lookup["parent"]]
-        name = row[field_lookup["name"]]
-        path = f"{parent}/{name}" if parent else name
+        path = row[field_lookup["path"]]
         if is_dir and path:
             full_path = path + "/"
         else:
@@ -681,7 +676,7 @@ class Catalog:
 
     def parse_url(self, uri: str, **config: Any) -> tuple[Client, str]:
         config = config or self.client_config
-        return Client.parse_url(uri, self.metastore, self.cache, **config)
+        return Client.parse_url(uri, self.cache, **config)
 
     def get_client(self, uri: StorageURI, **config: Any) -> Client:
         """
@@ -724,8 +719,7 @@ class Catalog:
         columns = [
             Column("vtype", String),
             Column("dir_type", Int),
-            Column("parent", String),
-            Column("name", String),
+            Column("path", String),
             Column("etag", String),
             Column("version", String),
             Column("is_latest", Boolean),
@@ -1623,8 +1617,7 @@ class Catalog:
         Example output:
             {
                 "source": "s3://ldb-public",
-                "parent": "animals/dogs",
-                "name": "dog.jpg",
+                "path": "animals/dogs/dog.jpg",
                 ...
             }
         """
@@ -1675,8 +1668,7 @@ class Catalog:
     def _get_row_uid(self, row: RowDict) -> UniqueId:
         return UniqueId(
             row["source"],
-            row["parent"],
-            row["name"],
+            row["path"],
             row["size"],
             row["etag"],
             row["version"],
@@ -2308,16 +2300,14 @@ class Catalog:
             if column == "du":
                 field_set.add("dir_type")
                 field_set.add("size")
-                field_set.add("parent")
-                field_set.add("name")
+                field_set.add("path")
             elif column == "name":
-                field_set.add("name")
+                field_set.add("path")
             elif column == "owner":
                 field_set.add("owner_name")
             elif column == "path":
                 field_set.add("dir_type")
-                field_set.add("parent")
-                field_set.add("name")
+                field_set.add("path")
             elif column == "size":
                 field_set.add("size")
             elif column == "type":

datachain/client/azure.py

@@ -1,4 +1,3 @@
-import posixpath
 from typing import Any
 
 from adlfs import AzureBlobFileSystem
@@ -14,16 +13,10 @@ class AzureClient(Client):
     PREFIX = "az://"
     protocol = "az"
 
-    def convert_info(self, v: dict[str, Any], parent: str) -> Entry:
+    def convert_info(self, v: dict[str, Any], path: str) -> Entry:
         version_id = v.get("version_id")
-        name = v.get("name", "").split(DELIMITER)[-1]
-        if version_id:
-            version_suffix = f"?versionid={version_id}"
-            if name.endswith(version_suffix):
-                name = name[: -len(version_suffix)]
         return Entry.from_file(
-            parent=parent,
-            name=name,
+            path=path,
             etag=v.get("etag", "").strip('"'),
             version=version_id or "",
             is_latest=version_id is None or bool(v.get("is_current_version")),
@@ -50,9 +43,9 @@ class AzureClient(Client):
                             if not self._is_valid_key(b["name"]):
                                 continue
                             info = (await self.fs._details([b]))[0]
-                            full_path = info["name"]
-                            parent = posixpath.dirname(self.rel_path(full_path))
-                            entries.append(self.convert_info(info, parent))
+                            entries.append(
+                                self.convert_info(info, self.rel_path(info["name"]))
+                            )
                         if entries:
                             await result_queue.put(entries)
                             pbar.update(len(entries))

datachain/client/fsspec.py

@@ -37,7 +37,6 @@ from datachain.storage import StorageURI
 if TYPE_CHECKING:
     from fsspec.spec import AbstractFileSystem
 
-    from datachain.data_storage import AbstractMetastore
 
 logger = logging.getLogger("datachain")
 
@@ -116,13 +115,12 @@ class Client(ABC):
     @staticmethod
     def parse_url(
         source: str,
-        metastore: "AbstractMetastore",
         cache: DataChainCache,
         **kwargs,
     ) -> tuple["Client", str]:
         cls = Client.get_implementation(source)
         storage_url, rel_path = cls.split_url(source)
-        client = cls.from_name(storage_url, metastore, cache, kwargs)
+        client = cls.from_name(storage_url, cache, kwargs)
         return client, rel_path
 
     @classmethod
@@ -136,7 +134,6 @@ class Client(ABC):
     def from_name(
         cls,
         name: str,
-        metastore: "AbstractMetastore",
         cache: DataChainCache,
         kwargs: dict[str, Any],
     ) -> "Client":
@@ -277,7 +274,7 @@ class Client(ABC):
             if info["type"] == "directory":
                 subdirs.add(subprefix)
             else:
-                files.append(self.convert_info(info, prefix))
+                files.append(self.convert_info(info, subprefix))
         if files:
             await result_queue.put(files)
         found_count = len(subdirs) + len(files)
@@ -360,12 +357,11 @@ class Client(ABC):
 
         parent_uid = UniqueId(
             parent["source"],
-            parent["parent"],
-            parent["name"],
-            parent["etag"],
+            parent["path"],
             parent["size"],
-            parent["vtype"],
-            parent["location"],
+            parent["etag"],
+            vtype=parent["vtype"],
+            location=parent["location"],
         )
         f = self.open_object(parent_uid, use_cache=use_cache)
         return FileSlice(f, offset, size, posixpath.basename(uid.path))

datachain/client/gcs.py

@@ -1,7 +1,6 @@
 import asyncio
 import json
 import os
-import posixpath
 from collections.abc import Iterable
 from datetime import datetime
 from typing import Any, Optional, cast
@@ -110,20 +109,11 @@ class GCSClient(Client):
 
     def _entry_from_dict(self, d: dict[str, Any]) -> Entry:
         info = self.fs._process_object(self.name, d)
-        full_path = info["name"]
-        subprefix = self.rel_path(full_path)
-        parent = posixpath.dirname(subprefix)
-        return self.convert_info(info, parent)
-
-    def convert_info(self, v: dict[str, Any], parent: str) -> Entry:
-        name = v.get("name", "").split(DELIMITER)[-1]
-        if "generation" in v:
-            gen = f"#{v['generation']}"
-            if name.endswith(gen):
-                name = name[: -len(gen)]
+        return self.convert_info(info, self.rel_path(info["name"]))
+
+    def convert_info(self, v: dict[str, Any], path: str) -> Entry:
         return Entry.from_file(
-            parent=parent,
-            name=name,
+            path=path,
             etag=v.get("etag", ""),
             version=v.get("generation", ""),
             is_latest=not v.get("timeDeleted"),

datachain/client/local.py

@@ -2,7 +2,7 @@ import os
 import posixpath
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import Any
 from urllib.parse import urlparse
 
 from fsspec.implementations.local import LocalFileSystem
@@ -12,9 +12,6 @@ from datachain.storage import StorageURI
 
 from .fsspec import Client
 
-if TYPE_CHECKING:
-    from datachain.data_storage import AbstractMetastore
-
 
 class FileClient(Client):
     FS_CLASS = LocalFileSystem
@@ -97,9 +94,7 @@ class FileClient(Client):
         return cls.root_dir(), uri.removeprefix(cls.root_path().as_uri())
 
     @classmethod
-    def from_name(
-        cls, name: str, metastore: "AbstractMetastore", cache, kwargs
-    ) -> "FileClient":
+    def from_name(cls, name: str, cache, kwargs) -> "FileClient":
         use_symlinks = kwargs.pop("use_symlinks", False)
         return cls(name, kwargs, cache, use_symlinks=use_symlinks)
 
@@ -140,11 +135,9 @@ class FileClient(Client):
             full_path += "/"
         return full_path
 
-    def convert_info(self, v: dict[str, Any], parent: str) -> Entry:
-        name = posixpath.basename(v["name"])
+    def convert_info(self, v: dict[str, Any], path: str) -> Entry:
         return Entry.from_file(
-            parent=parent,
-            name=name,
+            path=path,
             etag=v["mtime"].hex(),
             is_latest=True,
             last_modified=datetime.fromtimestamp(v["mtime"], timezone.utc),

datachain/client/s3.py

@@ -1,5 +1,4 @@
 import asyncio
-import posixpath
 from typing import Any, cast
 
 from botocore.exceptions import NoCredentialsError
@@ -112,10 +111,8 @@ class ClientS3(Client):
         await self._fetch_flat(start_prefix, result_queue)
 
     def _entry_from_boto(self, v, bucket, versions=False):
-        parent, name = posixpath.split(v["Key"])
         return Entry.from_file(
-            parent=parent,
-            name=name,
+            path=v["Key"],
             etag=v.get("ETag", "").strip('"'),
             version=ClientS3.clean_s3_version(v.get("VersionId", "")),
             is_latest=v.get("IsLatest", True),
@@ -145,7 +142,7 @@ class ClientS3(Client):
             if info["type"] == "directory":
                 subdirs.add(subprefix)
             else:
-                files.append(self.convert_info(info, prefix.rstrip("/")))
+                files.append(self.convert_info(info, subprefix))
                 pbar.update()
             found = True
         if not found:
@@ -159,10 +156,9 @@ class ClientS3(Client):
     def clean_s3_version(ver):
         return ver if ver != "null" else ""
 
-    def convert_info(self, v: dict[str, Any], parent: str) -> Entry:
+    def convert_info(self, v: dict[str, Any], path: str) -> Entry:
         return Entry.from_file(
-            parent=parent,
-            name=v.get("Key", "").split(DELIMITER)[-1],
+            path=path,
             etag=v.get("ETag", "").strip('"'),
             version=ClientS3.clean_s3_version(v.get("VersionId", "")),
             is_latest=v.get("IsLatest", True),

datachain/data_storage/schema.py

@@ -80,8 +80,7 @@ class DirExpansion:
             q.c.vtype,
             (q.c.dir_type == DirType.DIR).label("is_dir"),
             q.c.source,
-            q.c.parent,
-            q.c.name,
+            q.c.path,
             q.c.version,
             q.c.location,
         )
@@ -94,36 +93,29 @@ class DirExpansion:
                 q.c.vtype,
                 q.c.is_dir,
                 q.c.source,
-                q.c.parent,
-                q.c.name,
+                q.c.path,
                 q.c.version,
                 f.max(q.c.location).label("location"),
             )
             .select_from(q)
-            .group_by(
-                q.c.source, q.c.parent, q.c.name, q.c.vtype, q.c.is_dir, q.c.version
-            )
-            .order_by(
-                q.c.source, q.c.parent, q.c.name, q.c.vtype, q.c.is_dir, q.c.version
-            )
+            .group_by(q.c.source, q.c.path, q.c.vtype, q.c.is_dir, q.c.version)
+            .order_by(q.c.source, q.c.path, q.c.vtype, q.c.is_dir, q.c.version)
         )
 
     @classmethod
     def query(cls, q):
         q = cls.base_select(q).cte(recursive=True)
-        parent_parent = path.parent(q.c.parent)
-        parent_name = path.name(q.c.parent)
+        parent = path.parent(q.c.path)
         q = q.union_all(
             sa.select(
                 sa.literal(-1).label("sys__id"),
                 sa.literal("").label("vtype"),
                 true().label("is_dir"),
                 q.c.source,
-                parent_parent.label("parent"),
-                parent_name.label("name"),
+                parent.label("path"),
                 sa.literal("").label("version"),
                 null().label("location"),
-            ).where((parent_name != "") | (parent_parent != ""))
+            ).where(parent != "")
         )
         return cls.apply_group_by(q)
 

datachain/data_storage/warehouse.py

@@ -17,8 +17,9 @@ from sqlalchemy.sql.expression import true
 
 from datachain.client import Client
 from datachain.data_storage.serializer import Serializable
-from datachain.dataset import DatasetRecord, RowDict
+from datachain.dataset import DatasetRecord
 from datachain.node import DirType, DirTypeGroup, Entry, Node, NodeWithPath, get_path
+from datachain.sql.functions import path as pathfunc
 from datachain.sql.types import Int, SQLType
 from datachain.storage import StorageURI
 from datachain.utils import sql_escape_like
@@ -200,23 +201,17 @@ class AbstractWarehouse(ABC, Serializable):
     def dataset_select_paginated(
         self,
         query,
-        limit: Optional[int] = None,
-        order_by: tuple["ColumnElement[Any]", ...] = (),
         page_size: int = SELECT_BATCH_SIZE,
-    ) -> Generator[RowDict, None, None]:
+    ) -> Generator[Sequence, None, None]:
         """
         This is equivalent to `db.execute`, but for selecting rows in batches
         """
-        cols = query.selected_columns
-        cols_names = [c.name for c in cols]
+        limit = query._limit
+        paginated_query = query.limit(page_size)
 
-        if not order_by:
-            ordering = [cols.sys__id]
-        else:
-            ordering = order_by  # type: ignore[assignment]
-
-        # reset query order by and apply new order by id
-        paginated_query = query.order_by(None).order_by(*ordering).limit(page_size)
+        if not paginated_query._order_by_clauses:
+            # default order by is order by `sys__id`
+            paginated_query = paginated_query.order_by(query.selected_columns.sys__id)
 
         results = None
         offset = 0
@@ -235,7 +230,7 @@ class AbstractWarehouse(ABC, Serializable):
                 processed = False
                 for row in results:
                     processed = True
-                    yield RowDict(zip(cols_names, row))
+                    yield row
                     num_yielded += 1
 
                 if not processed:
@@ -373,9 +368,7 @@ class AbstractWarehouse(ABC, Serializable):
 
         else:
             parent = self.get_node_by_path(dr, path.lstrip("/").rstrip("/*"))
-            select_query = select_query.where(
-                (dr.c.parent == parent.path) | (self.path_expr(dr) == path)
-            )
+            select_query = select_query.where(pathfunc.parent(dr.c.path) == parent.path)
         return select_query
 
     def rename_dataset_table(
@@ -532,8 +525,8 @@ class AbstractWarehouse(ABC, Serializable):
             dr,
             parent_path,
             type="dir",
-            conds=[sa.Column("parent") == parent_path],
-            order_by=["source", "parent", "name"],
+            conds=[pathfunc.parent(sa.Column("path")) == parent_path],
+            order_by=["source", "path"],
         )
         return self.get_nodes(query)
 
@@ -556,7 +549,7 @@ class AbstractWarehouse(ABC, Serializable):
                 & ~self.instr(relpath, "/")
                 & (self.path_expr(de) != dirpath)
             )
-            .order_by(de.c.source, de.c.parent, de.c.name, de.c.version)
+            .order_by(de.c.source, de.c.path, de.c.version)
         )
 
     def _get_node_by_path_list(
@@ -572,8 +565,8 @@ class AbstractWarehouse(ABC, Serializable):
         ).subquery()
         query = self.expand_query(de, dr)
 
-        q = query.where((de.c.parent == parent) & (de.c.name == name)).order_by(
-            de.c.source, de.c.parent, de.c.name, de.c.version
+        q = query.where(de.c.path == get_path(parent, name)).order_by(
+            de.c.source, de.c.path, de.c.version
         )
         row = next(self.dataset_rows_select(q), None)
         if not row:
@@ -636,8 +629,7 @@ class AbstractWarehouse(ABC, Serializable):
             case((de.c.is_dir == true(), DirType.DIR), else_=dr.c.dir_type).label(
                 "dir_type"
             ),
-            de.c.parent,
-            de.c.name,
+            de.c.path,
             with_default(dr.c.etag),
             de.c.version,
             with_default(dr.c.is_latest),
@@ -670,7 +662,7 @@ class AbstractWarehouse(ABC, Serializable):
             .where(
                 dr.c.is_latest == true(),
                 dr.c.dir_type != DirType.DIR,
-                (dr.c.parent + "/").startswith(path),
+                dr.c.path.startswith(path),
             )
             .exists()
         )
@@ -678,8 +670,7 @@ class AbstractWarehouse(ABC, Serializable):
         if not row:
             raise FileNotFoundError(f"Unable to resolve path {path}")
         path = path.removesuffix("/")
-        parent, name = path.rsplit("/", 1) if "/" in path else ("", path)
-        return Node.from_dir(parent, name)
+        return Node.from_dir(path)
 
     def expand_path(self, dataset_rows: "DataTable", path: str) -> list[Node]:
         """Simulates Unix-like shell expansion"""
@@ -703,18 +694,21 @@ class AbstractWarehouse(ABC, Serializable):
         de = dr.dataset_dir_expansion(
             dr.select().where(dr.c.is_latest == true()).subquery()
         ).subquery()
-        where_cond = de.c.parent == parent_path
+        where_cond = pathfunc.parent(de.c.path) == parent_path
         if parent_path == "":
             # Exclude the root dir
-            where_cond = where_cond & (de.c.name != "")
+            where_cond = where_cond & (de.c.path != "")
         inner_query = self.expand_query(de, dr).where(where_cond).subquery()
+
+        def field_to_expr(f):
+            if f == "name":
+                return pathfunc.name(inner_query.c.path)
+            return getattr(inner_query.c, f)
+
         return self.db.execute(
-            sa.select(*(getattr(inner_query.c, f) for f in fields))
-            .select_from(inner_query)
-            .order_by(
+            select(*(field_to_expr(f) for f in fields)).order_by(
                 inner_query.c.source,
-                inner_query.c.parent,
-                inner_query.c.name,
+                inner_query.c.path,
                 inner_query.c.version,
             )
         )
@@ -727,21 +721,20 @@ class AbstractWarehouse(ABC, Serializable):
         """
         dr = dataset_rows
         dirpath = f"{parent_path}/"
-        relpath = func.substr(self.path_expr(dr), len(dirpath) + 1)
 
         def field_to_expr(f):
             if f == "name":
-                return relpath
+                return pathfunc.name(dr.c.path)
             return getattr(dr.c, f)
 
         q = (
             select(*(field_to_expr(f) for f in fields))
             .where(
                 self.path_expr(dr).like(f"{sql_escape_like(dirpath)}%"),
-                ~self.instr(relpath, "/"),
+                ~self.instr(pathfunc.name(dr.c.path), "/"),
                 dr.c.is_latest == true(),
             )
-            .order_by(dr.c.source, dr.c.parent, dr.c.name, dr.c.version, dr.c.etag)
+            .order_by(dr.c.source, dr.c.path, dr.c.version, dr.c.etag)
         )
         return self.db.execute(q)
 
@@ -758,7 +751,7 @@ class AbstractWarehouse(ABC, Serializable):
         if isinstance(node, dict):
             is_dir = node.get("is_dir", node["dir_type"] in DirTypeGroup.SUBOBJ_DIR)
             node_size = node["size"]
-            path = get_path(node["parent"], node["name"])
+            path = node["path"]
         else:
             is_dir = node.is_container
             node_size = node.size
@@ -790,7 +783,7 @@ class AbstractWarehouse(ABC, Serializable):
         return results[0] or 0, 0
 
     def path_expr(self, t):
-        return case((t.c.parent == "", t.c.name), else_=t.c.parent + "/" + t.c.name)
+        return t.c.path
 
     def _find_query(
         self,
@@ -947,11 +940,7 @@ class AbstractWarehouse(ABC, Serializable):
         tq = target_query.alias("target_query")
 
         source_target_join = sa.join(
-            sq,
-            tq,
-            (sq.c.source == tq.c.source)
-            & (sq.c.parent == tq.c.parent)
-            & (sq.c.name == tq.c.name),
+            sq, tq, (sq.c.source == tq.c.source) & (sq.c.path == tq.c.path)
         )
 
         return (

datachain/lib/dc.py

@@ -49,6 +49,7 @@ from datachain.query.dataset import (
     detach,
 )
 from datachain.query.schema import Column, DatasetRow
+from datachain.sql.functions import path as pathfunc
 from datachain.utils import inside_notebook
 
 if TYPE_CHECKING:
@@ -202,7 +203,7 @@ class DataChain(DatasetQuery):
 
     DEFAULT_FILE_RECORD: ClassVar[dict] = {
         "source": "",
-        "name": "",
+        "path": "",
         "vtype": "",
         "size": 0,
     }
@@ -1586,10 +1587,11 @@ class DataChain(DatasetQuery):
         use_cache: bool = True,
     ) -> None:
         """Method that exports all files from chain to some folder."""
-        if placement == "filename":
-            print("Checking if file names are unique")
-            if self.distinct(f"{signal}.name").count() != self.count():
-                raise ValueError("Files with the same name found")
+        if placement == "filename" and (
+            super().distinct(pathfunc.name(C(f"{signal}__path"))).count()
+            != self.count()
+        ):
+            raise ValueError("Files with the same name found")
 
         for file in self.collect(signal):
             file.export(output, placement, use_cache)  # type: ignore[union-attr]
@@ -1621,7 +1623,7 @@ class DataChain(DatasetQuery):
 
             Using glob to match patterns
             ```py
-            dc.filter(C("file.name").glob("*.jpg))
+            dc.filter(C("file.name").glob("*.jpg"))
             ```
 
             Using `datachain.sql.functions`