datachain 0.21.1__py3-none-any.whl → 0.23.0__py3-none-any.whl

datachain/lib/dc/datasets.py

@@ -1,11 +1,16 @@
 from collections.abc import Sequence
 from typing import TYPE_CHECKING, Optional, Union, get_origin, get_type_hints
 
-from datachain.error import DatasetVersionNotFoundError
+from datachain.error import (
+    DatasetNotFoundError,
+    DatasetVersionNotFoundError,
+    ProjectNotFoundError,
+)
 from datachain.lib.dataset_info import DatasetInfo
 from datachain.lib.file import (
     File,
 )
+from datachain.lib.projects import get as get_project
 from datachain.lib.settings import Settings
 from datachain.lib.signal_schema import SignalSchema
 from datachain.query import Session
@@ -24,12 +29,18 @@ if TYPE_CHECKING:
 
 def read_dataset(
     name: str,
+    namespace: Optional[str] = None,
+    project: Optional[str] = None,
     version: Optional[Union[str, int]] = None,
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
     fallback_to_studio: bool = True,
     delta: Optional[bool] = False,
-    delta_on: Optional[Union[str, Sequence[str]]] = None,
+    delta_on: Optional[Union[str, Sequence[str]]] = (
+        "file.path",
+        "file.etag",
+        "file.version",
+    ),
     delta_result_on: Optional[Union[str, Sequence[str]]] = None,
     delta_compare: Optional[Union[str, Sequence[str]]] = None,
     delta_retry: Optional[Union[bool, str]] = None,
@@ -38,47 +49,36 @@ def read_dataset(
     If dataset or version is not found locally, it will try to pull it from Studio.
 
     Parameters:
-        name : dataset name
+        name: The dataset name, which can be a fully qualified name including the
+            namespace and project. Alternatively, it can be a regular name, in which
+            case the explicitly defined namespace and project will be used if they are
+            set; otherwise, default values will be applied.
+        namespace : optional name of namespace in which dataset to read is created
+        project : optional name of project in which dataset to read is created
         version : dataset version
         session : Session to use for the chain.
         settings : Settings to use for the chain.
         fallback_to_studio : Try to pull dataset from Studio if not found locally.
             Default is True.
-        delta: If set to True, we optimize the creation of new dataset versions by
-            calculating the diff between the latest version of this storage and the
-            version used to create the most recent version of the resulting chain
-            dataset (the one specified in `.save()`). We then run the "diff" chain
-            using only the diff data, rather than the entire storage data, and merge
-            that diff chain with the latest version of the resulting dataset to create
-            a new version. This approach avoids applying modifications to all records
-            from storage every time, which can be an expensive operation.
-            The diff is calculated using the `DataChain.compare()` method, which
-            compares the `delta_on` fields to find matches and checks the compare
-            fields to determine if a record has changed. Note that this process only
-            considers added and modified records in storage; deleted records are not
-            removed from the new dataset version.
-            This calculation is based on the difference between the current version
-            of the source and the version used to create the dataset.
-        delta_on: A list of fields that uniquely identify rows in the source.
-            If two rows have the same values, they are considered the same (e.g., they
-            could be different versions of the same row in a versioned source).
-            This is used in the delta update to calculate the diff.
-        delta_result_on: A list of fields in the resulting dataset that correspond
-            to the `delta_on` fields from the source.
-            This is needed to identify rows that have changed in the source but are
-            already present in the current version of the resulting dataset, in order
-            to avoid including outdated versions of those rows in the new dataset.
-            We retain only the latest versions of rows to prevent duplication.
-            There is no need to define this if the `delta_on` fields are present in
-            the final dataset and have not been renamed.
-        delta_compare: A list of fields used to check if the same row has been modified
-            in the new version of the source.
-            If not defined, all fields except those defined in delta_on will be used.
-        delta_retry: Specifies retry behavior for delta processing. If a string,
-            it's the name of a field in the result dataset that indicates an error
-            when not None - records with errors will be reprocessed. If True,
-            records that exist in the source dataset but not in the result dataset
-            will be reprocessed.
+        delta: If True, only process new or changed files instead of reprocessing
+            everything. This saves time by skipping files that were already processed in
+            previous versions. The optimization is working when a new version of the
+            dataset is created.
+            Default is False.
+        delta_on: Field(s) that uniquely identify each record in the source data.
+            Used to detect which records are new or changed.
+            Default is ("file.path", "file.etag", "file.version").
+        delta_result_on: Field(s) in the result dataset that match `delta_on` fields.
+            Only needed if you rename the identifying fields during processing.
+            Default is None.
+        delta_compare: Field(s) used to detect if a record has changed.
+            If not specified, all fields except `delta_on` fields are used.
+            Default is None.
+        delta_retry: Controls retry behavior for failed records:
+            - String (field name): Reprocess records where this field is not empty
+              (error mode)
+            - True: Reprocess records missing from the result dataset (missing mode)
+            - None: No retry processing (default)
 
     Example:
         ```py
@@ -86,6 +86,11 @@ def read_dataset(
         chain = dc.read_dataset("my_cats")
         ```
 
+        ```py
+        import datachain as dc
+        chain = dc.read_dataset("dev.animals.my_cats")
+        ```
+
         ```py
         chain = dc.read_dataset("my_cats", fallback_to_studio=False)
         ```
@@ -116,6 +121,15 @@ def read_dataset(
 
     from .datachain import DataChain
 
+    session = Session.get(session)
+    catalog = session.catalog
+
+    namespace_name, project_name, name = catalog.get_full_dataset_name(
+        name,
+        project_name=project,
+        namespace_name=namespace,
+    )
+
     if version is not None:
         try:
             # for backward compatibility we still allow users to put version as integer
@@ -125,7 +139,15 @@ def read_dataset(
             # all 2.* dataset versions). If dataset doesn't have any versions where
             # major part is equal to that input, exception is thrown.
             major = int(version)
-            dataset = Session.get(session).catalog.get_dataset(name)
+            try:
+                ds_project = get_project(project_name, namespace_name, session=session)
+            except ProjectNotFoundError:
+                raise DatasetNotFoundError(
+                    f"Dataset {name} not found in namespace {namespace_name} and",
+                    f" project {project_name}",
+                ) from None
+
+            dataset = session.catalog.get_dataset(name, ds_project)
             latest_major = dataset.latest_major_version(major)
             if not latest_major:
                 raise DatasetVersionNotFoundError(
@@ -136,19 +158,22 @@ def read_dataset(
             # version is in new semver string format, continuing as normal
             pass
 
+    if settings:
+        _settings = Settings(**settings)
+    else:
+        _settings = Settings()
+
     query = DatasetQuery(
         name=name,
+        project_name=project_name,
+        namespace_name=namespace_name,
         version=version,  #  type: ignore[arg-type]
         session=session,
         indexing_column_types=File._datachain_column_types,
         fallback_to_studio=fallback_to_studio,
     )
-    telemetry.send_event_once("class", "datachain_init", name=name, version=version)
-    if settings:
-        _settings = Settings(**settings)
-    else:
-        _settings = Settings()
 
+    telemetry.send_event_once("class", "datachain_init", name=name, version=version)
     signals_schema = SignalSchema({"sys": Sys})
     if query.feature_schema:
         signals_schema |= SignalSchema.deserialize(query.feature_schema)
@@ -200,7 +225,7 @@ def datasets(
         import datachain as dc
 
         chain = dc.datasets(column="dataset")
-        for ds in chain.collect("dataset"):
+        for ds in chain.to_iter("dataset"):
             print(f"{ds.name}@v{ds.version}")
         ```
     """
@@ -251,6 +276,8 @@ def datasets(
 
 def delete_dataset(
     name: str,
+    namespace: Optional[str] = None,
+    project: Optional[str] = None,
     version: Optional[str] = None,
     force: Optional[bool] = False,
     studio: Optional[bool] = False,
@@ -261,11 +288,16 @@ def delete_dataset(
     a force flag.
 
     Args:
-        name : Dataset name
+        name: The dataset name, which can be a fully qualified name including the
+            namespace and project. Alternatively, it can be a regular name, in which
+            case the explicitly defined namespace and project will be used if they are
+            set; otherwise, default values will be applied.
+        namespace : optional name of namespace in which dataset to delete is created
+        project : optional name of project in which dataset to delete is created
         version : Optional dataset version
         force: If true, all datasets versions will be removed. Defaults to False.
-        studio: If True, removes dataset from Studio only,
-            otherwise remove from local. Defaults to False.
+        studio: If True, removes dataset from Studio only, otherwise removes local
+            dataset. Defaults to False.
         session: Optional session instance. If not provided, uses default session.
         in_memory: If True, creates an in-memory session. Defaults to False.
 
@@ -282,11 +314,32 @@ def delete_dataset(
         dc.delete_dataset("cats", version="1.0.0")
         ```
     """
+    from datachain.studio import remove_studio_dataset
 
     session = Session.get(session, in_memory=in_memory)
     catalog = session.catalog
+
+    namespace_name, project_name, name = catalog.get_full_dataset_name(
+        name,
+        project_name=project,
+        namespace_name=namespace,
+    )
+
+    if not catalog.metastore.is_local_dataset(namespace_name) and studio:
+        return remove_studio_dataset(
+            None, name, namespace_name, project_name, version=version, force=force
+        )
+
+    try:
+        ds_project = get_project(project_name, namespace_name, session=session)
+    except ProjectNotFoundError:
+        raise DatasetNotFoundError(
+            f"Dataset {name} not found in namespace {namespace_name} and project",
+            f" {project_name}",
+        ) from None
+
     if not force:
-        version = version or catalog.get_dataset(name).latest_version
+        version = version or catalog.get_dataset(name, ds_project).latest_version
     else:
         version = None
-    catalog.remove_dataset(name, version=version, force=force, studio=studio)
+    catalog.remove_dataset(name, ds_project, version=version, force=force)

datachain/lib/dc/listings.py

@@ -37,7 +37,7 @@ class ReadOnlyQueryStep(QueryStep):
             return sa.select(*columns)
 
         table_name = self.catalog.warehouse.dataset_table_name(
-            self.dataset_name, self.dataset_version
+            self.dataset, self.dataset_version
         )
         dataset_row_cls = self.catalog.warehouse.schema.dataset_row_cls
         table = dataset_row_cls.new_table(
@@ -51,7 +51,7 @@ class ReadOnlyQueryStep(QueryStep):
         )
 
         return step_result(
-            q, table.columns, dependencies=[(self.dataset_name, self.dataset_version)]
+            q, table.columns, dependencies=[(self.dataset, self.dataset_version)]
         )
 
 
@@ -142,7 +142,7 @@ def read_listing_dataset(
         _settings = Settings(prefetch=0)
     signal_schema = SignalSchema({"sys": Sys, "file": File})
 
-    query.starting_step = ReadOnlyQueryStep(query.catalog, name, version)
+    query.starting_step = ReadOnlyQueryStep(query.catalog, dataset, version)
     query.version = version
     # We already know that this is a listing dataset,
     # so we can set the listing function to True

datachain/lib/dc/records.py

@@ -68,6 +68,7 @@ def read_records(
 
     dsr = catalog.create_dataset(
         name,
+        catalog.metastore.default_project,
         columns=columns,
         feature_schema=(
             signal_schema.clone_without_sys_signals().serialize()
@@ -96,4 +97,4 @@ def read_records(
     for chunk in batched(records, INSERT_BATCH_SIZE):
         warehouse.insert_rows(table, chunk)
     warehouse.insert_rows_done(table)
-    return read_dataset(name=dsr.name, session=session, settings=settings)
+    return read_dataset(name=dsr.full_name, session=session, settings=settings)

datachain/lib/dc/storage.py

@@ -35,7 +35,11 @@ def read_storage(
     update: bool = False,
     anon: bool = False,
     delta: Optional[bool] = False,
-    delta_on: Optional[Union[str, Sequence[str]]] = None,
+    delta_on: Optional[Union[str, Sequence[str]]] = (
+        "file.path",
+        "file.etag",
+        "file.version",
+    ),
     delta_result_on: Optional[Union[str, Sequence[str]]] = None,
     delta_compare: Optional[Union[str, Sequence[str]]] = None,
     delta_retry: Optional[Union[bool, str]] = None,
@@ -54,43 +58,25 @@ def read_storage(
         update : force storage reindexing. Default is False.
         anon : If True, we will treat cloud bucket as public one
         client_config : Optional client configuration for the storage client.
-        delta: If set to True, we optimize the creation of new dataset versions by
-            calculating the diff between the latest version of this storage and the
-            version used to create the most recent version of the resulting chain
-            dataset (the one specified in `.save()`). We then run the "diff" chain
-            using only the diff data, rather than the entire storage data, and merge
-            that diff chain with the latest version of the resulting dataset to create
-            a new version. This approach avoids applying modifications to all records
-            from storage every time, which can be an expensive operation.
-            The diff is calculated using the `DataChain.compare()` method, which
-            compares the `delta_on` fields to find matches and checks the compare
-            fields to determine if a record has changed. Note that this process only
-            considers added and modified records in storage; deleted records are not
-            removed from the new dataset version.
-            This calculation is based on the difference between the current version
-            of the source and the version used to create the dataset.
-        delta_on: A list of fields that uniquely identify rows in the source.
-            If two rows have the same values, they are considered the same (e.g., they
-            could be different versions of the same row in a versioned source).
-            This is used in the delta update to calculate the diff.
-        delta_result_on: A list of fields in the resulting dataset that correspond
-            to the `delta_on` fields from the source.
-            This is needed to identify rows that have changed in the source but are
-            already present in the current version of the resulting dataset, in order
-            to avoid including outdated versions of those rows in the new dataset.
-            We retain only the latest versions of rows to prevent duplication.
-            There is no need to define this if the `delta_on` fields are present in
-            the final dataset and have not been renamed.
-        delta_compare: A list of fields used to check if the same row has been modified
-            in the new version of the source.
-            If not defined, all fields except those defined in `delta_on` will be used.
-        delta_retry: Controls which records to reprocess. Can be:
-            - A string specifying a field name: Records where this field is not None
-              will be reprocessed (error checking mode).
-            - True: Records that exist in the source dataset but not in the result
-              dataset (based on delta_on/delta_result_on fields) will be reprocessed
-              (missing records mode).
-            - False or None: No retry processing.
+        delta: If True, only process new or changed files instead of reprocessing
+            everything. This saves time by skipping files that were already processed in
+            previous versions. The optimization is working when a new version of the
+            dataset is created.
+            Default is False.
+        delta_on: Field(s) that uniquely identify each record in the source data.
+            Used to detect which records are new or changed.
+            Default is ("file.path", "file.etag", "file.version").
+        delta_result_on: Field(s) in the result dataset that match `delta_on` fields.
+            Only needed if you rename the identifying fields during processing.
+            Default is None.
+        delta_compare: Field(s) used to detect if a record has changed.
+            If not specified, all fields except `delta_on` fields are used.
+            Default is None.
+        delta_retry: Controls retry behavior for failed records:
+            - String (field name): Reprocess records where this field is not empty
+              (error mode)
+            - True: Reprocess records missing from the result dataset (missing mode)
+            - None: No retry processing (default)
 
     Returns:
         DataChain: A DataChain object containing the file information.
@@ -144,6 +130,8 @@ def read_storage(
     catalog = session.catalog
     cache = catalog.cache
     client_config = session.catalog.client_config
+    listing_namespace_name = catalog.metastore.system_namespace_name
+    listing_project_name = catalog.metastore.listing_project_name
 
     uris = uri if isinstance(uri, (list, tuple)) else [uri]
 
@@ -167,7 +155,13 @@ def read_storage(
             )
             continue
 
-        dc = read_dataset(list_ds_name, session=session, settings=settings)
+        dc = read_dataset(
+            list_ds_name,
+            namespace=listing_namespace_name,
+            project=listing_project_name,
+            session=session,
+            settings=settings,
+        )
         dc._query.update = update
         dc.signals_schema = dc.signals_schema.mutate({f"{column}": file_type})
 
@@ -182,7 +176,11 @@ def read_storage(
                         settings=settings,
                         in_memory=in_memory,
                     )
-                    .settings(prefetch=0)
+                    .settings(
+                        prefetch=0,
+                        namespace=listing_namespace_name,
+                        project=listing_project_name,
+                    )
                     .gen(
                         list_bucket(lst_uri, cache, client_config=client_config),
                         output={f"{column}": file_type},

datachain/lib/file.py

@@ -5,13 +5,14 @@ import json
 import logging
 import os
 import posixpath
+import warnings
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from contextlib import contextmanager
 from datetime import datetime
 from functools import partial
 from io import BytesIO
-from pathlib import Path, PurePosixPath
+from pathlib import Path, PurePath, PurePosixPath
 from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union
 from urllib.parse import unquote, urlparse
 from urllib.request import url2pathname
@@ -69,7 +70,7 @@ class FileExporter(NodesThreadPool):
         for task in done:
             task.result()
 
-    def do_task(self, file):
+    def do_task(self, file: "File"):
         file.export(
             self.output,
             self.placement,
@@ -274,8 +275,8 @@ class File(DataModel):
 
     @field_validator("path", mode="before")
     @classmethod
-    def validate_path(cls, path):
-        return Path(path).as_posix() if path else ""
+    def validate_path(cls, path: str) -> str:
+        return PurePath(path).as_posix() if path else ""
 
     def model_dump_custom(self):
         res = self.model_dump()
@@ -337,11 +338,11 @@ class File(DataModel):
         return cls(**{key: row[key] for key in cls._datachain_column_types})
 
     @property
-    def name(self):
+    def name(self) -> str:
         return PurePosixPath(self.path).name
 
     @property
-    def parent(self):
+    def parent(self) -> str:
         return str(PurePosixPath(self.path).parent)
 
     @contextmanager
@@ -391,7 +392,7 @@ class File(DataModel):
 
         client.upload(self.read(), destination)
 
-    def _symlink_to(self, destination: str):
+    def _symlink_to(self, destination: str) -> None:
         if self.location:
             raise OSError(errno.ENOTSUP, "Symlinking virtual file is not supported")
 
@@ -400,7 +401,7 @@ class File(DataModel):
             source = self.get_local_path()
             assert source, "File was not cached"
         elif self.source.startswith("file://"):
-            source = self.get_path()
+            source = self.get_fs_path()
         else:
             raise OSError(errno.EXDEV, "can't link across filesystems")
 
@@ -481,27 +482,62 @@ class File(DataModel):
 
     def get_file_ext(self):
         """Returns last part of file name without `.`."""
-        return PurePosixPath(self.path).suffix.strip(".")
+        return PurePosixPath(self.path).suffix.lstrip(".")
 
     def get_file_stem(self):
         """Returns file name without extension."""
         return PurePosixPath(self.path).stem
 
     def get_full_name(self):
-        """Returns name with parent directories."""
+        """
+        [DEPRECATED] Use `file.path` directly instead.
+
+        Returns name with parent directories.
+        """
+        warnings.warn(
+            "file.get_full_name() is deprecated and will be removed "
+            "in a future version. Use `file.path` directly.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         return self.path
 
-    def get_uri(self):
+    def get_path_normalized(self) -> str:
+        if not self.path:
+            raise FileError("path must not be empty", self.source, self.path)
+
+        if self.path.endswith("/"):
+            raise FileError("path must not be a directory", self.source, self.path)
+
+        normpath = os.path.normpath(self.path)
+        normpath = PurePath(normpath).as_posix()
+
+        if normpath == ".":
+            raise FileError("path must not be a directory", self.source, self.path)
+
+        if any(part == ".." for part in PurePath(normpath).parts):
+            raise FileError("path must not contain '..'", self.source, self.path)
+
+        return normpath
+
+    def get_uri(self) -> str:
         """Returns file URI."""
-        return f"{self.source}/{self.get_full_name()}"
+        return f"{self.source}/{self.get_path_normalized()}"
 
-    def get_path(self) -> str:
-        """Returns file path."""
+    def get_fs_path(self) -> str:
+        """
+        Returns file path with respect to the filescheme.
+
+        If `normalize` is True, the path is normalized to remove any redundant
+        separators and up-level references.
+
+        If the file scheme is "file", the path is converted to a local file path
+        using `url2pathname`. Otherwise, the original path with scheme is returned.
+        """
         path = unquote(self.get_uri())
-        source = urlparse(self.source)
-        if source.scheme == "file":
-            path = urlparse(path).path
-            path = url2pathname(path)
+        path_parsed = urlparse(path)
+        if path_parsed.scheme == "file":
+            path = url2pathname(path_parsed.path)
         return path
 
     def get_destination_path(
@@ -516,7 +552,7 @@ class File(DataModel):
         elif placement == "etag":
             path = f"{self.etag}{self.get_file_suffix()}"
         elif placement == "fullpath":
-            path = unquote(self.get_full_name())
+            path = unquote(self.get_path_normalized())
             source = urlparse(self.source)
             if source.scheme and source.scheme != "file":
                 path = posixpath.join(source.netloc, path)
@@ -554,8 +590,9 @@ class File(DataModel):
             ) from e
 
         try:
-            info = client.fs.info(client.get_full_path(self.path))
-            converted_info = client.info_to_file(info, self.path)
+            normalized_path = self.get_path_normalized()
+            info = client.fs.info(client.get_full_path(normalized_path))
+            converted_info = client.info_to_file(info, normalized_path)
             return type(self)(
                 path=self.path,
                 source=self.source,
@@ -566,8 +603,17 @@ class File(DataModel):
                 last_modified=converted_info.last_modified,
                 location=self.location,
             )
+        except FileError as e:
+            logger.warning(
+                "File error when resolving %s/%s: %s", self.source, self.path, str(e)
+            )
         except (FileNotFoundError, PermissionError, OSError) as e:
-            logger.warning("File system error when resolving %s: %s", self.path, str(e))
+            logger.warning(
+                "File system error when resolving %s/%s: %s",
+                self.source,
+                self.path,
+                str(e),
+            )
 
         return type(self)(
             path=self.path,
@@ -583,6 +629,8 @@ class File(DataModel):
 
 def resolve(file: File) -> File:
     """
+    [DEPRECATED] Use `file.resolve()` directly instead.
+
     Resolve a File object by checking its existence and updating its metadata.
 
     This function is a wrapper around the File.resolve() method, designed to be
@@ -598,6 +646,12 @@ def resolve(file: File) -> File:
         RuntimeError: If the file's catalog is not set or if
         the file source protocol is unsupported.
     """
+    warnings.warn(
+        "resolve() is deprecated and will be removed "
+        "in a future version. Use file.resolve() directly.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     return file.resolve()
 
 
@@ -945,7 +999,7 @@ class ArrowRow(DataModel):
             ds = dataset(path, **self.kwargs)
 
         else:
-            path = self.file.get_path()
+            path = self.file.get_fs_path()
             ds = dataset(path, filesystem=self.file.get_fs(), **self.kwargs)
 
         return ds.take([self.index]).to_reader()

datachain/lib/listing.py

@@ -123,6 +123,9 @@ def parse_listing_uri(uri: str) -> tuple[str, str, str]:
         f"{LISTING_PREFIX}{storage_uri}/{posixpath.join(lst_uri_path, '').lstrip('/')}"
     )
 
+    # we should remove dots from the name
+    ds_name = ds_name.replace(".", "_")
+
     return ds_name, lst_uri, path
 
 
@@ -195,5 +198,4 @@ def get_listing(
         list_path = f"{ds_name.strip('/').removeprefix(listing.name)}/{list_path}"
 
     ds_name = listing.name if listing else ds_name
-
     return ds_name, list_uri, list_path, bool(listing)

datachain/lib/meta_formats.py

@@ -106,7 +106,7 @@ def read_meta(  # noqa: C901
     from datachain import read_storage
 
     if schema_from:
-        file = next(read_storage(schema_from, type="text").limit(1).collect("file"))
+        file = read_storage(schema_from, type="text").limit(1).to_values("file")[0]
         model_code = gen_datamodel_code(
             file, format=format, jmespath=jmespath, model_name=model_name
         )