datachain 0.20.1__py3-none-any.whl → 0.20.3__py3-none-any.whl

datachain/__init__.py

@@ -1,4 +1,3 @@
-from datachain.lib import namespaces, projects
 from datachain.lib.data_model import DataModel, DataType, is_chain_type
 from datachain.lib.dc import (
     C,
@@ -33,6 +32,7 @@ from datachain.lib.file import (
     VideoFrame,
 )
 from datachain.lib.model_store import ModelStore
+from datachain.lib.projects import create as create_project
 from datachain.lib.udf import Aggregator, Generator, Mapper
 from datachain.lib.utils import AbstractUDF, DataChainError
 from datachain.query import metrics, param
@@ -63,14 +63,13 @@ __all__ = [
     "VideoFile",
     "VideoFragment",
     "VideoFrame",
+    "create_project",
     "datasets",
     "delete_dataset",
     "is_chain_type",
     "listings",
     "metrics",
-    "namespaces",
     "param",
-    "projects",
     "read_csv",
     "read_database",
     "read_dataset",

datachain/cache.py

@@ -39,7 +39,7 @@ def temporary_cache(
             cache.destroy()
 
 
-class Cache:
+class Cache:  # noqa: PLW1641
     def __init__(self, cache_dir: str, tmp_dir: str):
         self.odb = LocalHashFileDB(
             LocalFileSystem(),
@@ -76,9 +76,9 @@ class Cache:
     async def download(
         self, file: "File", client: "Client", callback: Optional[Callback] = None
     ) -> None:
-        from_path = f"{file.source}/{file.path}"
         from dvc_objects.fs.utils import tmp_fname
 
+        from_path = file.get_uri()
         odb_fs = self.odb.fs
         tmp_info = odb_fs.join(self.odb.tmp_dir, tmp_fname())  # type: ignore[arg-type]
         size = file.size

datachain/catalog/catalog.py

@@ -1491,13 +1491,13 @@ class Catalog:
 
         namespace = self.metastore.create_namespace(
             remote_ds.project.namespace.name,
-            description=remote_ds.project.namespace.description,
+            description=remote_ds.project.namespace.descr,
             uuid=remote_ds.project.namespace.uuid,
         )
         project = self.metastore.create_project(
-            remote_ds.project.name,
             namespace.name,
-            description=remote_ds.project.description,
+            remote_ds.project.name,
+            description=remote_ds.project.descr,
             uuid=remote_ds.project.uuid,
         )
 

datachain/cli/commands/ls.py

@@ -63,8 +63,8 @@ def ls_local(
                     print(format_ls_entry(entry))
     else:
         # Collect results in a list here to prevent interference from `tqdm` and `print`
-        listing = list(listings().collect("listing"))
-        for ls in listing:
+        listing = listings().to_list("listing")
+        for (ls,) in listing:
             print(format_ls_entry(f"{ls.uri}@v{ls.version}"))  # type: ignore[union-attr]
 
 

datachain/client/fsspec.py

@@ -207,13 +207,14 @@ class Client(ABC):
         )
 
     async def get_current_etag(self, file: "File") -> str:
+        file_path = file.get_path_normalized()
         kwargs = {}
         if self._is_version_aware():
             kwargs["version_id"] = file.version
         info = await self.fs._info(
-            self.get_full_path(file.path, file.version), **kwargs
+            self.get_full_path(file_path, file.version), **kwargs
         )
-        return self.info_to_file(info, file.path).etag
+        return self.info_to_file(info, file_path).etag
 
     def get_file_info(self, path: str, version_id: Optional[str] = None) -> "File":
         info = self.fs.info(self.get_full_path(path, version_id), version_id=version_id)
@@ -386,7 +387,8 @@ class Client(ABC):
             return open(cache_path, mode="rb")
         assert not file.location
         return FileWrapper(
-            self.fs.open(self.get_full_path(file.path, file.version)), cb
+            self.fs.open(self.get_full_path(file.get_path_normalized(), file.version)),
+            cb,
         )  # type: ignore[return-value]
 
     def upload(self, data: bytes, path: str) -> "File":

datachain/client/hf.py

@@ -21,6 +21,9 @@ def _wrap_class(sync_fs_class):
     asynchronous to False by default. This is similar to other Async FS
     we initialize. E.g. it means we don't break things in Jupyter where code
     run in async.
+
+    This also fixes write operations by ensuring they are properly forwarded
+    to the underlying filesystem without async buffering issues.
     """
     from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
 
@@ -29,6 +32,13 @@ def _wrap_class(sync_fs_class):
             sync_fs = sync_fs_class(*args, **kwargs)
             super().__init__(sync_fs, asynchronous=False)
 
+        def open(self, path, mode="rb", **kwargs):
+            # Override open to ensure write operations work correctly.
+            # It seems to be a bug in the fsspec wrapper. It avoids
+            # wrapping open() explicitly but also doesn't redirect it to
+            # sync filesystem.
+            return self.sync_fs.open(path, mode, **kwargs)
+
     GeneratedAsyncFileSystemWrapper.__name__ = f"Async{sync_fs_class.__name__}Wrapper"
     return GeneratedAsyncFileSystemWrapper
 

datachain/client/local.py

@@ -99,7 +99,7 @@ class FileClient(Client):
         )
 
     async def get_current_etag(self, file: "File") -> str:
-        info = self.fs.info(self.get_full_path(file.path))
+        info = self.fs.info(self.get_full_path(file.get_path_normalized()))
         return self.info_to_file(info, "").etag
 
     async def get_size(self, path: str, version_id: Optional[str] = None) -> int:
@@ -138,8 +138,8 @@ class FileClient(Client):
         if not self.use_symlinks:
             super().fetch_nodes(nodes, shared_progress_bar)
 
-    def do_instantiate_object(self, uid, dst):
+    def do_instantiate_object(self, file: File, dst: str) -> None:
         if self.use_symlinks:
-            os.symlink(Path(self.name, uid.path), dst)
+            os.symlink(Path(self.name, file.path), dst)
         else:
-            super().do_instantiate_object(uid, dst)
+            super().do_instantiate_object(file, dst)

datachain/data_storage/metastore.py

@@ -185,8 +185,8 @@ class AbstractMetastore(ABC, Serializable):
     @abstractmethod
     def create_project(
         self,
-        name: str,
         namespace_name: str,
+        name: str,
         description: Optional[str] = None,
         uuid: Optional[str] = None,
         ignore_if_exists: bool = True,
@@ -195,8 +195,13 @@ class AbstractMetastore(ABC, Serializable):
         """Creates new project in specific namespace"""
 
     @abstractmethod
-    def get_project(self, name: str, namespace_name: str, conn=None) -> Project:
-        """Gets a single project inside some namespace by name"""
+    def get_project(
+        self, name: str, namespace_name: str, create: bool = False, conn=None
+    ) -> Project:
+        """
+        Gets a single project inside some namespace by name.
+        It also creates project if not found and create flag is set to True.
+        """
 
     @abstractmethod
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
@@ -763,14 +768,18 @@ class AbstractDBMetastore(AbstractMetastore):
 
     def create_project(
         self,
-        name: str,
         namespace_name: str,
+        name: str,
         description: Optional[str] = None,
         uuid: Optional[str] = None,
         ignore_if_exists: bool = True,
         **kwargs,
     ) -> Project:
-        namespace = self.get_namespace(namespace_name)
+        try:
+            namespace = self.get_namespace(namespace_name)
+        except NamespaceNotFoundError:
+            namespace = self.create_namespace(namespace_name)
+
         query = self._projects_insert().values(
             namespace_id=namespace.id,
             uuid=uuid or str(uuid4()),
@@ -788,7 +797,9 @@ class AbstractDBMetastore(AbstractMetastore):
 
         return self.get_project(name, namespace.name)
 
-    def get_project(self, name: str, namespace_name: str, conn=None) -> Project:
+    def get_project(
+        self, name: str, namespace_name: str, create: bool = False, conn=None
+    ) -> Project:
         """Gets a single project inside some namespace by name"""
         n = self._namespaces
         p = self._projects
@@ -803,6 +814,8 @@ class AbstractDBMetastore(AbstractMetastore):
 
         rows = list(self.db.execute(query, conn=conn))
         if not rows:
+            if create:
+                return self.create_project(namespace_name, name)
             raise ProjectNotFoundError(
                 f"Project {name} in namespace {namespace_name} not found."
             )

datachain/data_storage/sqlite.py

@@ -469,10 +469,10 @@ class SQLiteMetastore(AbstractDBMetastore):
         Studio dataset.
         """
         system_namespace = self.create_namespace(Namespace.system(), "System namespace")
-        self.create_project(Project.listing(), system_namespace.name, "Listing project")
+        self.create_project(system_namespace.name, Project.listing(), "Listing project")
 
         local_namespace = self.create_namespace(Namespace.default(), "Local namespace")
-        self.create_project(Project.default(), local_namespace.name, "Local project")
+        self.create_project(local_namespace.name, Project.default(), "Local project")
 
     def _check_schema_version(self) -> None:
         """

datachain/dataset.py

@@ -83,10 +83,11 @@ def parse_dataset_name(name: str) -> tuple[Optional[str], Optional[str], str]:
     if not name:
         raise ValueError("Name must be defined to parse it")
     split = name.split(".")
-    if len(split) == 3:
-        return tuple(split)  # type: ignore[return-value]
+    name = split[-1]
+    project_name = split[-2] if len(split) > 1 else None
+    namespace_name = split[-3] if len(split) > 2 else None
 
-    return None, None, name
+    return namespace_name, project_name, name
 
 
 class DatasetDependencyType:

datachain/delta.py

@@ -62,7 +62,7 @@ def _get_delta_chain(
     )
 
     # Calculate diff between source versions
-    return source_dc_latest.compare(source_dc, on=on, compare=compare, deleted=False)
+    return source_dc_latest.diff(source_dc, on=on, compare=compare, deleted=False)
 
 
 def _get_retry_chain(
@@ -237,7 +237,7 @@ def delta_retry_update(
         return None, None, False
 
     latest_dataset = datachain.read_dataset(name, version=latest_version)
-    compared_chain = latest_dataset.compare(
+    compared_chain = latest_dataset.diff(
         processing_chain,
         on=right_on or on,
         added=True,

datachain/func/func.py

@@ -25,7 +25,7 @@ if TYPE_CHECKING:
 ColT = Union[str, Column, ColumnElement, "Func", tuple]
 
 
-class Func(Function):
+class Func(Function):  # noqa: PLW1641
     """Represents a function to be applied to a column in a SQL query."""
 
     def __init__(

datachain/lib/arrow.py

@@ -76,7 +76,7 @@ class ArrowGenerator(Generator):
             fs_path = file.path
             fs = ReferenceFileSystem({fs_path: [cache_path]})
         else:
-            fs, fs_path = file.get_fs(), file.get_path()
+            fs, fs_path = file.get_fs(), file.get_fs_path()
 
         kwargs = self.kwargs
         if format := kwargs.get("format"):
@@ -160,8 +160,8 @@ def infer_schema(chain: "DataChain", **kwargs) -> pa.Schema:
         kwargs["format"] = fix_pyarrow_format(format, parse_options)
 
     schemas = []
-    for file in chain.collect("file"):
-        ds = dataset(file.get_path(), filesystem=file.get_fs(), **kwargs)  # type: ignore[union-attr]
+    for (file,) in chain.to_iter("file"):
+        ds = dataset(file.get_fs_path(), filesystem=file.get_fs(), **kwargs)  # type: ignore[union-attr]
         schemas.append(ds.schema)
     if not schemas:
         raise ValueError(

datachain/lib/dataset_info.py

@@ -22,8 +22,8 @@ if TYPE_CHECKING:
 
 class DatasetInfo(DataModel):
     name: str
-    namespace_name: str
-    project_name: str
+    namespace: str
+    project: str
     uuid: str = Field(default=str(uuid4()))
     version: str = Field(default=DEFAULT_DATASET_VERSION)
     status: int = Field(default=DatasetStatus.CREATED)
@@ -93,8 +93,8 @@ class DatasetInfo(DataModel):
         return cls(
             uuid=version.uuid,
             name=dataset.name,
-            namespace_name=dataset.project.namespace.name,
-            project_name=dataset.project.name,
+            namespace=dataset.project.namespace.name,
+            project=dataset.project.name,
             version=version.version,
             status=version.status,
             created_at=version.created_at,