datachain 0.20.4__py3-none-any.whl → 0.21.0__py3-none-any.whl

datachain/cli/commands/datasets.py

@@ -8,8 +8,7 @@ if TYPE_CHECKING:
 
 from datachain.cli.utils import determine_flavors
 from datachain.config import Config
-from datachain.dataset import parse_dataset_name
-from datachain.error import DataChainError, DatasetNotFoundError
+from datachain.error import DatasetNotFoundError
 from datachain.studio import list_datasets as list_datasets_studio
 
 
@@ -102,15 +101,11 @@ def list_datasets_local(catalog: "Catalog", name: Optional[str] = None):
 
     for d in catalog.ls_datasets():
         for v in d.versions:
-            yield (d.full_name, v.version)
+            yield (d.name, v.version)
 
 
 def list_datasets_local_versions(catalog: "Catalog", name: str):
-    namespace_name, project_name, name = parse_dataset_name(name)
-    namespace_name = namespace_name or catalog.metastore.default_namespace_name
-    project_name = project_name or catalog.metastore.default_project_name
-    project = catalog.metastore.get_project(project_name, namespace_name)
-    ds = catalog.get_dataset(name, project)
+    ds = catalog.get_dataset(name)
     for v in ds.versions:
         yield (name, v.version)
 
@@ -134,29 +129,25 @@ def rm_dataset(
     name: str,
     version: Optional[str] = None,
     force: Optional[bool] = False,
-    studio: Optional[bool] = False,
+    studio: bool = False,
+    local: bool = False,
+    all: bool = True,
     team: Optional[str] = None,
 ):
-    namespace_name, project_name, name = parse_dataset_name(name)
-    namespace_name = namespace_name or catalog.metastore.default_namespace_name
-    project_name = project_name or catalog.metastore.default_project_name
-
-    if not catalog.metastore.is_local_dataset(namespace_name) and studio:
-        from datachain.studio import remove_studio_dataset
-
-        token = Config().read().get("studio", {}).get("token")
-        if not token:
-            raise DataChainError(
-                "Not logged in to Studio. Log in with 'datachain auth login'."
-            )
-        remove_studio_dataset(team, name, namespace_name, project_name, version, force)
-    else:
+    from datachain.studio import remove_studio_dataset
+
+    token = Config().read().get("studio", {}).get("token")
+    all, local, studio = determine_flavors(studio, local, all, token)
+
+    if all or local:
         try:
-            project = catalog.metastore.get_project(project_name, namespace_name)
-            catalog.remove_dataset(name, project, version=version, force=force)
+            catalog.remove_dataset(name, version=version, force=force)
         except DatasetNotFoundError:
             print("Dataset not found in local", file=sys.stderr)
 
+    if (all or studio) and token:
+        remove_studio_dataset(team, name, version, force)
+
 
 def edit_dataset(
     catalog: "Catalog",
@@ -164,27 +155,21 @@ def edit_dataset(
     new_name: Optional[str] = None,
     description: Optional[str] = None,
     attrs: Optional[list[str]] = None,
+    studio: bool = False,
+    local: bool = False,
+    all: bool = True,
     team: Optional[str] = None,
 ):
-    namespace_name, project_name, name = parse_dataset_name(name)
-    namespace_name = namespace_name or catalog.metastore.default_namespace_name
-    project_name = project_name or catalog.metastore.default_project_name
+    from datachain.studio import edit_studio_dataset
 
-    if catalog.metastore.is_local_dataset(namespace_name):
+    token = Config().read().get("studio", {}).get("token")
+    all, local, studio = determine_flavors(studio, local, all, token)
+
+    if all or local:
         try:
-            catalog.edit_dataset(
-                name, catalog.metastore.default_project, new_name, description, attrs
-            )
+            catalog.edit_dataset(name, new_name, description, attrs)
         except DatasetNotFoundError:
             print("Dataset not found in local", file=sys.stderr)
-    else:
-        from datachain.studio import edit_studio_dataset
-
-        token = Config().read().get("studio", {}).get("token")
-        if not token:
-            raise DataChainError(
-                "Not logged in to Studio. Log in with 'datachain auth login'."
-            )
-        edit_studio_dataset(
-            team, name, namespace_name, project_name, new_name, description, attrs
-        )
+
+    if (all or studio) and token:
+        edit_studio_dataset(team, name, new_name, description, attrs)

datachain/cli/commands/ls.py

@@ -63,8 +63,8 @@ def ls_local(
                     print(format_ls_entry(entry))
     else:
         # Collect results in a list here to prevent interference from `tqdm` and `print`
-        listing = listings().to_list("listing")
-        for (ls,) in listing:
+        listing = list(listings().collect("listing"))
+        for ls in listing:
             print(format_ls_entry(f"{ls.uri}@v{ls.version}"))  # type: ignore[union-attr]
 
 

datachain/cli/parser/__init__.py

@@ -221,6 +221,26 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         nargs="+",
         help="Dataset attributes",
     )
+    parse_edit_dataset.add_argument(
+        "--studio",
+        action="store_true",
+        default=False,
+        help="Edit dataset from Studio",
+    )
+    parse_edit_dataset.add_argument(
+        "-L",
+        "--local",
+        action="store_true",
+        default=False,
+        help="Edit local dataset only",
+    )
+    parse_edit_dataset.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=True,
+        help="Edit both datasets from studio and local",
+    )
     parse_edit_dataset.add_argument(
         "--team",
         action="store",
@@ -295,7 +315,21 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--studio",
         action="store_true",
         default=False,
-        help="Remove dataset from Studio only",
+        help="Remove dataset from Studio",
+    )
+    rm_dataset_parser.add_argument(
+        "-L",
+        "--local",
+        action="store_true",
+        default=False,
+        help="Remove local datasets only",
+    )
+    rm_dataset_parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=True,
+        help="Remove both local and studio",
     )
     rm_dataset_parser.add_argument(
         "--team",

datachain/client/fsspec.py

@@ -207,14 +207,13 @@ class Client(ABC):
         )
 
     async def get_current_etag(self, file: "File") -> str:
-        file_path = file.get_path_normalized()
         kwargs = {}
         if self._is_version_aware():
             kwargs["version_id"] = file.version
         info = await self.fs._info(
-            self.get_full_path(file_path, file.version), **kwargs
+            self.get_full_path(file.path, file.version), **kwargs
         )
-        return self.info_to_file(info, file_path).etag
+        return self.info_to_file(info, file.path).etag
 
     def get_file_info(self, path: str, version_id: Optional[str] = None) -> "File":
         info = self.fs.info(self.get_full_path(path, version_id), version_id=version_id)
@@ -387,8 +386,7 @@ class Client(ABC):
             return open(cache_path, mode="rb")
         assert not file.location
         return FileWrapper(
-            self.fs.open(self.get_full_path(file.get_path_normalized(), file.version)),
-            cb,
+            self.fs.open(self.get_full_path(file.path, file.version)), cb
         )  # type: ignore[return-value]
 
     def upload(self, data: bytes, path: str) -> "File":

datachain/client/hf.py

@@ -21,9 +21,6 @@ def _wrap_class(sync_fs_class):
     asynchronous to False by default. This is similar to other Async FS
     we initialize. E.g. it means we don't break things in Jupyter where code
     run in async.
-
-    This also fixes write operations by ensuring they are properly forwarded
-    to the underlying filesystem without async buffering issues.
     """
     from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
 
@@ -32,13 +29,6 @@ def _wrap_class(sync_fs_class):
             sync_fs = sync_fs_class(*args, **kwargs)
             super().__init__(sync_fs, asynchronous=False)
 
-        def open(self, path, mode="rb", **kwargs):
-            # Override open to ensure write operations work correctly.
-            # It seems to be a bug in the fsspec wrapper. It avoids
-            # wrapping open() explicitly but also doesn't redirect it to
-            # sync filesystem.
-            return self.sync_fs.open(path, mode, **kwargs)
-
     GeneratedAsyncFileSystemWrapper.__name__ = f"Async{sync_fs_class.__name__}Wrapper"
     return GeneratedAsyncFileSystemWrapper
 

datachain/client/local.py

@@ -99,7 +99,7 @@ class FileClient(Client):
         )
 
     async def get_current_etag(self, file: "File") -> str:
-        info = self.fs.info(self.get_full_path(file.get_path_normalized()))
+        info = self.fs.info(self.get_full_path(file.path))
         return self.info_to_file(info, "").etag
 
     async def get_size(self, path: str, version_id: Optional[str] = None) -> int:
@@ -138,8 +138,8 @@ class FileClient(Client):
         if not self.use_symlinks:
             super().fetch_nodes(nodes, shared_progress_bar)
 
-    def do_instantiate_object(self, file: File, dst: str) -> None:
+    def do_instantiate_object(self, uid, dst):
         if self.use_symlinks:
-            os.symlink(Path(self.name, file.path), dst)
+            os.symlink(Path(self.name, uid.path), dst)
         else:
-            super().do_instantiate_object(file, dst)
+            super().do_instantiate_object(uid, dst)