datachain 0.20.1__py3-none-any.whl → 0.20.3__py3-none-any.whl

datachain/lib/file.py

@@ -5,13 +5,14 @@ import json
 import logging
 import os
 import posixpath
+import warnings
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from contextlib import contextmanager
 from datetime import datetime
 from functools import partial
 from io import BytesIO
-from pathlib import Path, PurePosixPath
+from pathlib import Path, PurePath, PurePosixPath
 from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union
 from urllib.parse import unquote, urlparse
 from urllib.request import url2pathname
@@ -69,7 +70,7 @@ class FileExporter(NodesThreadPool):
         for task in done:
             task.result()
 
-    def do_task(self, file):
+    def do_task(self, file: "File"):
         file.export(
             self.output,
             self.placement,
@@ -274,8 +275,8 @@ class File(DataModel):
 
     @field_validator("path", mode="before")
     @classmethod
-    def validate_path(cls, path):
-        return Path(path).as_posix() if path else ""
+    def validate_path(cls, path: str) -> str:
+        return PurePath(path).as_posix() if path else ""
 
     def model_dump_custom(self):
         res = self.model_dump()
@@ -337,11 +338,11 @@ class File(DataModel):
         return cls(**{key: row[key] for key in cls._datachain_column_types})
 
     @property
-    def name(self):
+    def name(self) -> str:
         return PurePosixPath(self.path).name
 
     @property
-    def parent(self):
+    def parent(self) -> str:
         return str(PurePosixPath(self.path).parent)
 
     @contextmanager
@@ -391,7 +392,7 @@ class File(DataModel):
 
         client.upload(self.read(), destination)
 
-    def _symlink_to(self, destination: str):
+    def _symlink_to(self, destination: str) -> None:
         if self.location:
             raise OSError(errno.ENOTSUP, "Symlinking virtual file is not supported")
 
@@ -400,7 +401,7 @@ class File(DataModel):
             source = self.get_local_path()
             assert source, "File was not cached"
         elif self.source.startswith("file://"):
-            source = self.get_path()
+            source = self.get_fs_path()
         else:
             raise OSError(errno.EXDEV, "can't link across filesystems")
 
@@ -481,27 +482,62 @@ class File(DataModel):
 
     def get_file_ext(self):
         """Returns last part of file name without `.`."""
-        return PurePosixPath(self.path).suffix.strip(".")
+        return PurePosixPath(self.path).suffix.lstrip(".")
 
     def get_file_stem(self):
         """Returns file name without extension."""
         return PurePosixPath(self.path).stem
 
     def get_full_name(self):
-        """Returns name with parent directories."""
+        """
+        [DEPRECATED] Use `file.path` directly instead.
+
+        Returns name with parent directories.
+        """
+        warnings.warn(
+            "file.get_full_name() is deprecated and will be removed "
+            "in a future version. Use `file.path` directly.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         return self.path
 
-    def get_uri(self):
+    def get_path_normalized(self) -> str:
+        if not self.path:
+            raise FileError("path must not be empty", self.source, self.path)
+
+        if self.path.endswith("/"):
+            raise FileError("path must not be a directory", self.source, self.path)
+
+        normpath = os.path.normpath(self.path)
+        normpath = PurePath(normpath).as_posix()
+
+        if normpath == ".":
+            raise FileError("path must not be a directory", self.source, self.path)
+
+        if any(part == ".." for part in PurePath(normpath).parts):
+            raise FileError("path must not contain '..'", self.source, self.path)
+
+        return normpath
+
+    def get_uri(self) -> str:
         """Returns file URI."""
-        return f"{self.source}/{self.get_full_name()}"
+        return f"{self.source}/{self.get_path_normalized()}"
 
-    def get_path(self) -> str:
-        """Returns file path."""
+    def get_fs_path(self) -> str:
+        """
+        Returns file path with respect to the filescheme.
+
+        If `normalize` is True, the path is normalized to remove any redundant
+        separators and up-level references.
+
+        If the file scheme is "file", the path is converted to a local file path
+        using `url2pathname`. Otherwise, the original path with scheme is returned.
+        """
         path = unquote(self.get_uri())
-        source = urlparse(self.source)
-        if source.scheme == "file":
-            path = urlparse(path).path
-            path = url2pathname(path)
+        path_parsed = urlparse(path)
+        if path_parsed.scheme == "file":
+            path = url2pathname(path_parsed.path)
         return path
 
     def get_destination_path(
@@ -516,7 +552,7 @@ class File(DataModel):
         elif placement == "etag":
             path = f"{self.etag}{self.get_file_suffix()}"
         elif placement == "fullpath":
-            path = unquote(self.get_full_name())
+            path = unquote(self.get_path_normalized())
             source = urlparse(self.source)
             if source.scheme and source.scheme != "file":
                 path = posixpath.join(source.netloc, path)
@@ -554,8 +590,9 @@ class File(DataModel):
             ) from e
 
         try:
-            info = client.fs.info(client.get_full_path(self.path))
-            converted_info = client.info_to_file(info, self.path)
+            normalized_path = self.get_path_normalized()
+            info = client.fs.info(client.get_full_path(normalized_path))
+            converted_info = client.info_to_file(info, normalized_path)
             return type(self)(
                 path=self.path,
                 source=self.source,
@@ -566,8 +603,17 @@ class File(DataModel):
                 last_modified=converted_info.last_modified,
                 location=self.location,
             )
+        except FileError as e:
+            logger.warning(
+                "File error when resolving %s/%s: %s", self.source, self.path, str(e)
+            )
         except (FileNotFoundError, PermissionError, OSError) as e:
-            logger.warning("File system error when resolving %s: %s", self.path, str(e))
+            logger.warning(
+                "File system error when resolving %s/%s: %s",
+                self.source,
+                self.path,
+                str(e),
+            )
 
         return type(self)(
             path=self.path,
@@ -583,6 +629,8 @@ class File(DataModel):
 
 def resolve(file: File) -> File:
     """
+    [DEPRECATED] Use `file.resolve()` directly instead.
+
     Resolve a File object by checking its existence and updating its metadata.
 
     This function is a wrapper around the File.resolve() method, designed to be
@@ -598,6 +646,12 @@ def resolve(file: File) -> File:
         RuntimeError: If the file's catalog is not set or if
         the file source protocol is unsupported.
     """
+    warnings.warn(
+        "resolve() is deprecated and will be removed "
+        "in a future version. Use file.resolve() directly.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     return file.resolve()
 
 
@@ -945,7 +999,7 @@ class ArrowRow(DataModel):
             ds = dataset(path, **self.kwargs)
 
         else:
-            path = self.file.get_path()
+            path = self.file.get_fs_path()
             ds = dataset(path, filesystem=self.file.get_fs(), **self.kwargs)
 
         return ds.take([self.index]).to_reader()

datachain/lib/meta_formats.py

@@ -106,7 +106,7 @@ def read_meta(  # noqa: C901
     from datachain import read_storage
 
     if schema_from:
-        file = next(read_storage(schema_from, type="text").limit(1).collect("file"))
+        file = read_storage(schema_from, type="text").limit(1).to_values("file")[0]
         model_code = gen_datamodel_code(
             file, format=format, jmespath=jmespath, model_name=model_name
         )

datachain/lib/namespaces.py

@@ -6,39 +6,37 @@ from datachain.query import Session
 
 
 def create(
-    name: str, description: Optional[str] = None, session: Optional[Session] = None
+    name: str, descr: Optional[str] = None, session: Optional[Session] = None
 ) -> Namespace:
     """
-    Creates a new custom namespace.
-    A Namespace is an object used to organize datasets. It has name and a list of
-    Project objects underneath it. On the other hand, each Project can have multiple
-    datasets.
-    Note that creating namespaces is not allowed in the local environment, unlike
-    in Studio, where it is allowed.
-    In local environment all datasets are created under the default `local` namespace.
+    Creates a new namespace.
+
+    Namespaces organize projects, which in turn organize datasets. A default
+    namespace always exists and is used if none is specified. Multiple namespaces
+    can be created in Studio, but only the default is available in the CLI.
 
     Parameters:
-        name : The name of the namespace.
-        description : A description of the namespace.
-        session : Session to use for creating namespace.
+        name: Name of the new namespace.
+        descr: Optional description of the namespace.
+        session: Optional session to use for the operation.
 
     Example:
         ```py
-        import datachain as dc
-        namespace = dc.namespaces.create("dev", "Dev namespace")
+        from datachain.lib.namespaces import create as create_namespace
+        namespace = create_namespace("dev", "Dev namespace")
         ```
     """
     session = Session.get(session)
 
     if not session.catalog.metastore.namespace_allowed_to_create:
-        raise NamespaceCreateNotAllowedError("Creating custom namespace is not allowed")
+        raise NamespaceCreateNotAllowedError("Creating namespace is not allowed")
 
     Namespace.validate_name(name)
 
-    return session.catalog.metastore.create_namespace(name, description)
+    return session.catalog.metastore.create_namespace(name, descr)
 
 
-def get(name: str, session: Optional[Session]) -> Namespace:
+def get(name: str, session: Optional[Session] = None) -> Namespace:
     """
     Gets a namespace by name.
     If the namespace is not found, a `NamespaceNotFoundError` is raised.
@@ -66,8 +64,8 @@ def ls(session: Optional[Session] = None) -> list[Namespace]:
 
     Example:
         ```py
-        import datachain as dc
-        namespaces = dc.namespaces.ls()
+        from datachain.lib.namespaces import ls as ls_namespaces
+        namespaces = ls_namespaces()
         ```
     """
     return Session.get(session).catalog.metastore.list_namespaces()

datachain/lib/projects.py

@@ -6,81 +6,81 @@ from datachain.query import Session
 
 
 def create(
+    namespace: str,
     name: str,
-    namespace_name: str,
-    description: Optional[str] = None,
+    descr: Optional[str] = None,
     session: Optional[Session] = None,
 ) -> Project:
     """
-    Creates a new custom project.
-    A Project is an object used to organize datasets. It is created under a
-    specific namespace and has a list of datasets underneath it.
-    Note that creating projects is not allowed in the local environment, unlike
-    in Studio, where it is allowed.
-    In local environment all datasets are created under the default `local` project.
+    Creates a new project under a specified namespace.
+
+    Projects help organize datasets. A default project is always available,
+    but users can create additional ones (only in Studio, not via CLI).
+
 
     Parameters:
-        name : The name of the project.
-        namespace : The name of the namespace under which the new project is being
-            created.
-        description : A description of the project.
-        session : Session to use for creating project.
+        name: Name of the new project.
+        namespace: Namespace to create the project in. Created if it doesn't exist.
+        descr: Optional description of the project.
+        session: Optional session to use for the operation.
 
     Example:
         ```py
         import datachain as dc
-        project = dc.projects.create("my-project", "dev", "My personal project")
+        project = dc.create_project("dev", "my-project", "My personal project")
         ```
     """
     session = Session.get(session)
 
     if not session.catalog.metastore.project_allowed_to_create:
-        raise ProjectCreateNotAllowedError("Creating custom project is not allowed")
+        raise ProjectCreateNotAllowedError("Creating project is not allowed")
 
     Project.validate_name(name)
 
-    return session.catalog.metastore.create_project(name, namespace_name, description)
+    return session.catalog.metastore.create_project(namespace, name, descr)
 
 
-def get(name: str, namespace_name: str, session: Optional[Session]) -> Project:
+def get(name: str, namespace: str, session: Optional[Session]) -> Project:
     """
     Gets a project by name in some namespace.
     If the project is not found, a `ProjectNotFoundError` is raised.
 
     Parameters:
         name : The name of the project.
-        namespace_name : The name of the namespace.
+        namespace : The name of the namespace.
         session : Session to use for getting project.
 
     Example:
         ```py
         import datachain as dc
-        project  = dc.get_project("my-project", "local")
+        from datachain.lib.projects import get as get_project
+        project  = get_project("my-project", "local")
         ```
     """
-    return Session.get(session).catalog.metastore.get_project(name, namespace_name)
+    return Session.get(session).catalog.metastore.get_project(name, namespace)
 
 
 def ls(
-    namespace_name: Optional[str] = None, session: Optional[Session] = None
+    namespace: Optional[str] = None, session: Optional[Session] = None
 ) -> list[Project]:
     """
     Gets a list of projects in a specific namespace or from all namespaces.
 
     Parameters:
-        namespace_name : An optional namespace name.
+        namespace : An optional namespace name.
         session : Session to use for getting project.
 
     Example:
         ```py
         import datachain as dc
-        local_namespace_projects = dc.projects.ls("local")
-        all_projects = dc.projects.ls()
+        from datachain.lib.projects import ls as ls_projects
+        local_namespace_projects = ls_projects("local")
+        all_projects = ls_projects()
         ```
     """
     session = Session.get(session)
     namespace_id = None
-    if namespace_name:
-        namespace_id = session.catalog.metastore.get_namespace(namespace_name).id
+    if namespace:
+        namespace_id = session.catalog.metastore.get_namespace(namespace).id
 
     return session.catalog.metastore.list_projects(namespace_id)

datachain/lib/pytorch.py

@@ -130,7 +130,7 @@ class PytorchDataset(IterableDataset):
         if self.num_samples > 0:
             ds = ds.sample(self.num_samples)
         ds = ds.chunk(total_rank, total_workers)
-        yield from ds.collect()
+        yield from ds.to_iter()
 
     def _iter_with_prefetch(self) -> Generator[tuple[Any], None, None]:
         from datachain.lib.udf import _prefetch_inputs

datachain/lib/tar.py

@@ -6,12 +6,11 @@ from datachain.lib.file import File, TarVFile
 
 
 def build_tar_member(parent: File, info: tarfile.TarInfo) -> File:
-    new_parent = parent.get_full_name()
     etag_string = "-".join([parent.etag, info.name, str(info.mtime)])
     etag = hashlib.md5(etag_string.encode(), usedforsecurity=False).hexdigest()
     return File(
         source=parent.source,
-        path=f"{new_parent}/{info.name}",
+        path=f"{parent.path}/{info.name}",
         version=parent.version,
         size=info.size,
         etag=etag,

datachain/lib/udf_signature.py

@@ -16,7 +16,7 @@ class UdfSignatureError(DataChainParamsError):
 
 
 @dataclass
-class UdfSignature:
+class UdfSignature:  # noqa: PLW1641
     func: Union[Callable, UDFBase]
     params: dict[str, Union[DataType, Any]]
     output_schema: SignalSchema

datachain/lib/webdataset.py

@@ -34,29 +34,29 @@ warnings.filterwarnings(
 
 
 class WDSError(DataChainError):
-    def __init__(self, tar_stream, message: str):
-        super().__init__(f"WebDataset error '{tar_stream.get_full_name()}': {message}")
+    def __init__(self, tar_name: str, message: str):
+        super().__init__(f"WebDataset error '{tar_name}': {message}")
 
 
 class CoreFileDuplicationError(WDSError):
-    def __init__(self, tar_stream, file1: str, file2: str):
+    def __init__(self, tar_name: str, file1: str, file2: str):
         super().__init__(
-            tar_stream, f"duplication of files with core extensions: {file1}, {file2}"
+            tar_name, f"duplication of files with core extensions: {file1}, {file2}"
         )
 
 
 class CoreFileNotFoundError(WDSError):
-    def __init__(self, tar_stream, extensions, stem):
+    def __init__(self, tar_name: str, extensions: Sequence[str], stem: str):
         super().__init__(
-            tar_stream,
+            tar_name,
             f"no files with the extensions '{','.join(extensions)}'"
             f" were found for file stem {stem}",
         )
 
 
 class UnknownFileExtensionError(WDSError):
-    def __init__(self, tar_stream, name, ext):
-        super().__init__(tar_stream, f"unknown extension '{ext}' for file '{name}'")
+    def __init__(self, tar_name, name: str, ext: str):
+        super().__init__(tar_name, f"unknown extension '{ext}' for file '{name}'")
 
 
 class WDSBasic(DataModel):
@@ -113,10 +113,10 @@ class Builder:
     def __init__(
         self,
         tar_stream: File,
-        core_extensions: list[str],
+        core_extensions: Sequence[str],
         wds_class: type[WDSBasic],
-        tar,
-        encoding="utf-8",
+        tar: tarfile.TarFile,
+        encoding: str = "utf-8",
     ):
         self._core_extensions = core_extensions
         self._tar_stream = tar_stream
@@ -145,18 +145,20 @@ class Builder:
         if ext in self._core_extensions:
             if self.state.core_file is not None:
                 raise CoreFileDuplicationError(
-                    self._tar_stream, file.name, self.state.core_file.name
+                    self._tar_stream.name, file.name, self.state.core_file.name
                 )
             self.state.core_file = file
         elif ext in self.state.data:
             raise WDSError(
-                self._tar_stream,
+                self._tar_stream.name,
                 f"file with extension '.{ext}' already exists in the archive",
             )
         else:
             type_ = self._get_type(ext)
             if type_ is None:
-                raise UnknownFileExtensionError(self._tar_stream, fstream.name, ext)
+                raise UnknownFileExtensionError(
+                    self._tar_stream.name, fstream.name, ext
+                )
 
             if issubclass(type_, WDSReadableSubclass):
                 reader = type_._reader
@@ -165,7 +167,7 @@ class Builder:
 
             if reader is None:
                 raise WDSError(
-                    self._tar_stream,
+                    self._tar_stream.name,
                     f"unable to find a reader for type {type_}, extension .{ext}",
                 )
             self.state.data[ext] = reader(self, file)
@@ -173,7 +175,7 @@ class Builder:
     def produce(self):
         if self.state.core_file is None:
             raise CoreFileNotFoundError(
-                self._tar_stream, self._core_extensions, self.state.stem
+                self._tar_stream.name, self._core_extensions, self.state.stem
             )
 
         file = build_tar_member(self._tar_stream, self.state.core_file)
@@ -194,7 +196,13 @@ class Builder:
         return anno
 
 
-def get_tar_groups(stream, tar, core_extensions, spec, encoding="utf-8"):
+def get_tar_groups(
+    stream: File,
+    tar: tarfile.TarFile,
+    core_extensions: Sequence[str],
+    spec: type[WDSBasic],
+    encoding: str = "utf-8",
+) -> Iterator[WDSBasic]:
     builder = Builder(stream, core_extensions, spec, tar, encoding)
 
     for item in sorted(tar.getmembers(), key=lambda m: Path(m.name).stem):
@@ -210,9 +218,11 @@ def get_tar_groups(stream, tar, core_extensions, spec, encoding="utf-8"):
 
 
 def process_webdataset(
-    core_extensions: Sequence[str] = ("jpg", "png"), spec=WDSAllFile, encoding="utf-8"
-) -> Callable:
-    def wds_func(file: File) -> Iterator[spec]:
+    core_extensions: Sequence[str] = ("jpg", "png"),
+    spec: type[WDSBasic] = WDSAllFile,
+    encoding: str = "utf-8",
+) -> Callable[[File], Iterator]:
+    def wds_func(file: File) -> Iterator[spec]:  # type: ignore[valid-type]
         with file.open() as fd:
             with tarfile.open(fileobj=fd) as tar:
                 yield from get_tar_groups(file, tar, core_extensions, spec, encoding)

datachain/namespace.py

@@ -14,7 +14,7 @@ class Namespace:
     id: int
     uuid: str
     name: str
-    description: Optional[str]
+    descr: Optional[str]
     created_at: datetime
 
     @staticmethod
@@ -54,10 +54,10 @@ class Namespace:
         id: int,
         uuid: str,
         name: str,
-        description: Optional[str],
+        descr: Optional[str],
         created_at: datetime,
     ) -> "Namespace":
-        return cls(id, uuid, name, description, created_at)
+        return cls(id, uuid, name, descr, created_at)
 
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "Namespace":

datachain/project.py

@@ -15,7 +15,7 @@ class Project:
     id: int
     uuid: str
     name: str
-    description: Optional[str]
+    descr: Optional[str]
     created_at: datetime
     namespace: Namespace
 
@@ -52,12 +52,12 @@ class Project:
         namespace_id: int,
         namespace_uuid: str,
         namespace_name: str,
-        namespace_description: Optional[str],
+        namespace_descr: Optional[str],
         namespace_created_at: datetime,
         project_id: int,
         uuid: str,
         name: str,
-        description: Optional[str],
+        descr: Optional[str],
         created_at: datetime,
         project_namespace_id: int,
     ) -> "Project":
@@ -65,11 +65,11 @@ class Project:
             namespace_id,
             namespace_uuid,
             namespace_name,
-            namespace_description,
+            namespace_descr,
             namespace_created_at,
         )
 
-        return cls(project_id, uuid, name, description, created_at, namespace)
+        return cls(project_id, uuid, name, descr, created_at, namespace)
 
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "Project":

{datachain-0.20.1.dist-info → datachain-0.20.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.20.1
+Version: 0.20.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0