datachain 0.21.1__py3-none-any.whl → 0.22.0__py3-none-any.whl

datachain/cli/commands/datasets.py

@@ -8,7 +8,8 @@ if TYPE_CHECKING:
 
 from datachain.cli.utils import determine_flavors
 from datachain.config import Config
-from datachain.error import DatasetNotFoundError
+from datachain.dataset import parse_dataset_name
+from datachain.error import DataChainError, DatasetNotFoundError
 from datachain.studio import list_datasets as list_datasets_studio
 
 
@@ -101,11 +102,15 @@ def list_datasets_local(catalog: "Catalog", name: Optional[str] = None):
 
     for d in catalog.ls_datasets():
         for v in d.versions:
-            yield (d.name, v.version)
+            yield (d.full_name, v.version)
 
 
 def list_datasets_local_versions(catalog: "Catalog", name: str):
-    ds = catalog.get_dataset(name)
+    namespace_name, project_name, name = parse_dataset_name(name)
+    namespace_name = namespace_name or catalog.metastore.default_namespace_name
+    project_name = project_name or catalog.metastore.default_project_name
+    project = catalog.metastore.get_project(project_name, namespace_name)
+    ds = catalog.get_dataset(name, project)
     for v in ds.versions:
         yield (name, v.version)
 
@@ -129,25 +134,29 @@ def rm_dataset(
     name: str,
     version: Optional[str] = None,
     force: Optional[bool] = False,
-    studio: bool = False,
-    local: bool = False,
-    all: bool = True,
+    studio: Optional[bool] = False,
     team: Optional[str] = None,
 ):
-    from datachain.studio import remove_studio_dataset
-
-    token = Config().read().get("studio", {}).get("token")
-    all, local, studio = determine_flavors(studio, local, all, token)
-
-    if all or local:
+    namespace_name, project_name, name = parse_dataset_name(name)
+    namespace_name = namespace_name or catalog.metastore.default_namespace_name
+    project_name = project_name or catalog.metastore.default_project_name
+
+    if not catalog.metastore.is_local_dataset(namespace_name) and studio:
+        from datachain.studio import remove_studio_dataset
+
+        token = Config().read().get("studio", {}).get("token")
+        if not token:
+            raise DataChainError(
+                "Not logged in to Studio. Log in with 'datachain auth login'."
+            )
+        remove_studio_dataset(team, name, namespace_name, project_name, version, force)
+    else:
         try:
-            catalog.remove_dataset(name, version=version, force=force)
+            project = catalog.metastore.get_project(project_name, namespace_name)
+            catalog.remove_dataset(name, project, version=version, force=force)
         except DatasetNotFoundError:
             print("Dataset not found in local", file=sys.stderr)
 
-    if (all or studio) and token:
-        remove_studio_dataset(team, name, version, force)
-
 
 def edit_dataset(
     catalog: "Catalog",
@@ -155,21 +164,27 @@ def edit_dataset(
     new_name: Optional[str] = None,
     description: Optional[str] = None,
     attrs: Optional[list[str]] = None,
-    studio: bool = False,
-    local: bool = False,
-    all: bool = True,
     team: Optional[str] = None,
 ):
-    from datachain.studio import edit_studio_dataset
+    namespace_name, project_name, name = parse_dataset_name(name)
+    namespace_name = namespace_name or catalog.metastore.default_namespace_name
+    project_name = project_name or catalog.metastore.default_project_name
 
-    token = Config().read().get("studio", {}).get("token")
-    all, local, studio = determine_flavors(studio, local, all, token)
-
-    if all or local:
+    if catalog.metastore.is_local_dataset(namespace_name):
         try:
-            catalog.edit_dataset(name, new_name, description, attrs)
+            catalog.edit_dataset(
+                name, catalog.metastore.default_project, new_name, description, attrs
+            )
         except DatasetNotFoundError:
             print("Dataset not found in local", file=sys.stderr)
-
-    if (all or studio) and token:
-        edit_studio_dataset(team, name, new_name, description, attrs)
+    else:
+        from datachain.studio import edit_studio_dataset
+
+        token = Config().read().get("studio", {}).get("token")
+        if not token:
+            raise DataChainError(
+                "Not logged in to Studio. Log in with 'datachain auth login'."
+            )
+        edit_studio_dataset(
+            team, name, namespace_name, project_name, new_name, description, attrs
+        )

datachain/cli/commands/ls.py

@@ -63,8 +63,8 @@ def ls_local(
                     print(format_ls_entry(entry))
     else:
         # Collect results in a list here to prevent interference from `tqdm` and `print`
-        listing = list(listings().collect("listing"))
-        for ls in listing:
+        listing = listings().to_list("listing")
+        for (ls,) in listing:
             print(format_ls_entry(f"{ls.uri}@v{ls.version}"))  # type: ignore[union-attr]
 
 

datachain/cli/parser/__init__.py

@@ -221,26 +221,6 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         nargs="+",
         help="Dataset attributes",
     )
-    parse_edit_dataset.add_argument(
-        "--studio",
-        action="store_true",
-        default=False,
-        help="Edit dataset from Studio",
-    )
-    parse_edit_dataset.add_argument(
-        "-L",
-        "--local",
-        action="store_true",
-        default=False,
-        help="Edit local dataset only",
-    )
-    parse_edit_dataset.add_argument(
-        "-a",
-        "--all",
-        action="store_true",
-        default=True,
-        help="Edit both datasets from studio and local",
-    )
     parse_edit_dataset.add_argument(
         "--team",
         action="store",
@@ -315,21 +295,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--studio",
         action="store_true",
         default=False,
-        help="Remove dataset from Studio",
-    )
-    rm_dataset_parser.add_argument(
-        "-L",
-        "--local",
-        action="store_true",
-        default=False,
-        help="Remove local datasets only",
-    )
-    rm_dataset_parser.add_argument(
-        "-a",
-        "--all",
-        action="store_true",
-        default=True,
-        help="Remove both local and studio",
+        help="Remove dataset from Studio only",
     )
     rm_dataset_parser.add_argument(
         "--team",

datachain/client/fsspec.py

@@ -207,13 +207,14 @@ class Client(ABC):
         )
 
     async def get_current_etag(self, file: "File") -> str:
+        file_path = file.get_path_normalized()
         kwargs = {}
         if self._is_version_aware():
             kwargs["version_id"] = file.version
         info = await self.fs._info(
-            self.get_full_path(file.path, file.version), **kwargs
+            self.get_full_path(file_path, file.version), **kwargs
         )
-        return self.info_to_file(info, file.path).etag
+        return self.info_to_file(info, file_path).etag
 
     def get_file_info(self, path: str, version_id: Optional[str] = None) -> "File":
         info = self.fs.info(self.get_full_path(path, version_id), version_id=version_id)
@@ -386,7 +387,8 @@ class Client(ABC):
             return open(cache_path, mode="rb")
         assert not file.location
         return FileWrapper(
-            self.fs.open(self.get_full_path(file.path, file.version)), cb
+            self.fs.open(self.get_full_path(file.get_path_normalized(), file.version)),
+            cb,
         )  # type: ignore[return-value]
 
     def upload(self, data: bytes, path: str) -> "File":

datachain/client/hf.py

@@ -21,6 +21,9 @@ def _wrap_class(sync_fs_class):
     asynchronous to False by default. This is similar to other Async FS
     we initialize. E.g. it means we don't break things in Jupyter where code
     run in async.
+
+    This also fixes write operations by ensuring they are properly forwarded
+    to the underlying filesystem without async buffering issues.
     """
     from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
 
@@ -29,6 +32,13 @@ def _wrap_class(sync_fs_class):
             sync_fs = sync_fs_class(*args, **kwargs)
             super().__init__(sync_fs, asynchronous=False)
 
+        def open(self, path, mode="rb", **kwargs):
+            # Override open to ensure write operations work correctly.
+            # It seems to be a bug in the fsspec wrapper. It avoids
+            # wrapping open() explicitly but also doesn't redirect it to
+            # sync filesystem.
+            return self.sync_fs.open(path, mode, **kwargs)
+
     GeneratedAsyncFileSystemWrapper.__name__ = f"Async{sync_fs_class.__name__}Wrapper"
     return GeneratedAsyncFileSystemWrapper
 

datachain/client/local.py

@@ -99,7 +99,7 @@ class FileClient(Client):
         )
 
     async def get_current_etag(self, file: "File") -> str:
-        info = self.fs.info(self.get_full_path(file.path))
+        info = self.fs.info(self.get_full_path(file.get_path_normalized()))
         return self.info_to_file(info, "").etag
 
     async def get_size(self, path: str, version_id: Optional[str] = None) -> int:
@@ -138,8 +138,8 @@ class FileClient(Client):
         if not self.use_symlinks:
             super().fetch_nodes(nodes, shared_progress_bar)
 
-    def do_instantiate_object(self, uid, dst):
+    def do_instantiate_object(self, file: File, dst: str) -> None:
         if self.use_symlinks:
-            os.symlink(Path(self.name, uid.path), dst)
+            os.symlink(Path(self.name, file.path), dst)
         else:
-            super().do_instantiate_object(uid, dst)
+            super().do_instantiate_object(file, dst)