datachain 0.31.4__tar.gz → 0.32.0__tar.gz

{datachain-0.31.4 → datachain-0.32.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.31.4
+Version: 0.32.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.31.4 → datachain-0.32.0}/src/datachain/__init__.py

@@ -37,6 +37,7 @@ from datachain.lib.file import (
     VideoFrame,
 )
 from datachain.lib.model_store import ModelStore
+from datachain.lib.namespaces import delete as delete_namespace
 from datachain.lib.projects import create as create_project
 from datachain.lib.udf import Aggregator, Generator, Mapper
 from datachain.lib.utils import AbstractUDF, DataChainError
@@ -74,6 +75,7 @@ __all__ = [
     "create_project",
     "datasets",
     "delete_dataset",
+    "delete_namespace",
     "is_chain_type",
     "is_studio",
     "listings",

{datachain-0.31.4 → datachain-0.32.0}/src/datachain/data_storage/metastore.py

@@ -22,6 +22,7 @@ from sqlalchemy import (
     UniqueConstraint,
     select,
 )
+from sqlalchemy.sql import func as f
 
 from datachain.data_storage import JobQueryType, JobStatus
 from datachain.data_storage.serializer import Serializable
@@ -37,7 +38,9 @@ from datachain.dataset import (
 from datachain.error import (
     DatasetNotFoundError,
     DatasetVersionNotFoundError,
+    NamespaceDeleteNotAllowedError,
     NamespaceNotFoundError,
+    ProjectDeleteNotAllowedError,
     ProjectNotFoundError,
     TableMissingError,
 )
@@ -141,6 +144,10 @@ class AbstractMetastore(ABC, Serializable):
     def get_namespace(self, name: str, conn=None) -> Namespace:
         """Gets a single namespace by name"""
 
+    @abstractmethod
+    def remove_namespace(self, namespace_id: int, conn=None) -> None:
+        """Removes a single namespace by id"""
+
     @abstractmethod
     def list_namespaces(self, conn=None) -> list[Namespace]:
         """Gets a list of all namespaces"""
@@ -190,10 +197,30 @@ class AbstractMetastore(ABC, Serializable):
         It also creates project if not found and create flag is set to True.
         """
 
+    def is_default_project(self, project_name: str, namespace_name: str) -> bool:
+        return (
+            project_name == self.default_project_name
+            and namespace_name == self.default_namespace_name
+        )
+
+    def is_listing_project(self, project_name: str, namespace_name: str) -> bool:
+        return (
+            project_name == self.listing_project_name
+            and namespace_name == self.system_namespace_name
+        )
+
     @abstractmethod
     def get_project_by_id(self, project_id: int, conn=None) -> Project:
         """Gets a single project by id"""
 
+    @abstractmethod
+    def count_projects(self, namespace_id: Optional[int] = None) -> int:
+        """Counts projects in some namespace or in general."""
+
+    @abstractmethod
+    def remove_project(self, project_id: int, conn=None) -> None:
+        """Removes a single project by id"""
+
     @abstractmethod
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
         """Gets list of projects in some namespace or in general (in all namespaces)"""
@@ -270,6 +297,10 @@ class AbstractMetastore(ABC, Serializable):
     ) -> Iterator[DatasetListRecord]:
         """Lists all datasets in some project or in all projects."""
 
+    @abstractmethod
+    def count_datasets(self, project_id: Optional[int] = None) -> int:
+        """Counts datasets in some project or in all projects."""
+
     @abstractmethod
     def list_datasets_by_prefix(
         self, prefix: str, project_id: Optional[int] = None
@@ -735,6 +766,18 @@ class AbstractDBMetastore(AbstractMetastore):
 
         return self.get_namespace(name)
 
+    def remove_namespace(self, namespace_id: int, conn=None) -> None:
+        num_projects = self.count_projects(namespace_id)
+        if num_projects > 0:
+            raise NamespaceDeleteNotAllowedError(
+                f"Namespace cannot be removed. It contains {num_projects} project(s). "
+                "Please remove the project(s) first."
+            )
+
+        n = self._namespaces
+        with self.db.transaction():
+            self.db.execute(self._namespaces_delete().where(n.c.id == namespace_id))
+
     def get_namespace(self, name: str, conn=None) -> Namespace:
         """Gets a single namespace by name"""
         n = self._namespaces
@@ -796,18 +839,6 @@ class AbstractDBMetastore(AbstractMetastore):
 
         return self.get_project(name, namespace.name)
 
-    def _is_listing_project(self, project_name: str, namespace_name: str) -> bool:
-        return (
-            project_name == self.listing_project_name
-            and namespace_name == self.system_namespace_name
-        )
-
-    def _is_default_project(self, project_name: str, namespace_name: str) -> bool:
-        return (
-            project_name == self.default_project_name
-            and namespace_name == self.default_namespace_name
-        )
-
     def get_project(
         self, name: str, namespace_name: str, create: bool = False, conn=None
     ) -> Project:
@@ -816,7 +847,7 @@ class AbstractDBMetastore(AbstractMetastore):
         p = self._projects
         validate = True
 
-        if self._is_listing_project(name, namespace_name) or self._is_default_project(
+        if self.is_listing_project(name, namespace_name) or self.is_default_project(
             name, namespace_name
         ):
             # we are always creating default and listing projects if they don't exist
@@ -858,7 +889,31 @@ class AbstractDBMetastore(AbstractMetastore):
             raise ProjectNotFoundError(f"Project with id {project_id} not found.")
         return self.project_class.parse(*rows[0])
 
-    def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
+    def count_projects(self, namespace_id: Optional[int] = None) -> int:
+        p = self._projects
+        query = self._projects_select()
+        if namespace_id:
+            query = query.where(p.c.namespace_id == namespace_id)
+
+        query = select(f.count(1)).select_from(query.subquery())
+
+        return next(self.db.execute(query))[0]
+
+    def remove_project(self, project_id: int, conn=None) -> None:
+        num_datasets = self.count_datasets(project_id)
+        if num_datasets > 0:
+            raise ProjectDeleteNotAllowedError(
+                f"Project cannot be removed. It contains {num_datasets} dataset(s). "
+                "Please remove the dataset(s) first."
+            )
+
+        p = self._projects
+        with self.db.transaction():
+            self.db.execute(self._projects_delete().where(p.c.id == project_id))
+
+    def list_projects(
+        self, namespace_id: Optional[int] = None, conn=None
+    ) -> list[Project]:
         """
         Gets a list of projects inside some namespace, or in all namespaces
         """
@@ -1189,7 +1244,6 @@ class AbstractDBMetastore(AbstractMetastore):
     def list_datasets(
         self, project_id: Optional[int] = None
     ) -> Iterator["DatasetListRecord"]:
-        """Lists all datasets."""
         d = self._datasets
         query = self._base_list_datasets_query().order_by(
             self._datasets.c.name, self._datasets_versions.c.version
@@ -1198,6 +1252,16 @@ class AbstractDBMetastore(AbstractMetastore):
             query = query.where(d.c.project_id == project_id)
         yield from self._parse_dataset_list(self.db.execute(query))
 
+    def count_datasets(self, project_id: Optional[int] = None) -> int:
+        d = self._datasets
+        query = self._datasets_select()
+        if project_id:
+            query = query.where(d.c.project_id == project_id)
+
+        query = select(f.count(1)).select_from(query.subquery())
+
+        return next(self.db.execute(query))[0]
+
     def list_datasets_by_prefix(
         self, prefix: str, project_id: Optional[int] = None, conn=None
     ) -> Iterator["DatasetListRecord"]:

{datachain-0.31.4 → datachain-0.32.0}/src/datachain/error.py

@@ -34,6 +34,14 @@ class ProjectCreateNotAllowedError(NotAllowedError):
     pass
 
 
+class ProjectDeleteNotAllowedError(NotAllowedError):
+    pass
+
+
+class NamespaceDeleteNotAllowedError(NotAllowedError):
+    pass
+
+
 class ProjectNotFoundError(NotFoundError):
     pass
 

{datachain-0.31.4 → datachain-0.32.0}/src/datachain/lib/namespaces.py

@@ -1,7 +1,11 @@
 from typing import Optional
 
-from datachain.error import NamespaceCreateNotAllowedError
-from datachain.namespace import Namespace
+from datachain.error import (
+    NamespaceCreateNotAllowedError,
+    NamespaceDeleteNotAllowedError,
+)
+from datachain.lib.projects import delete as delete_project
+from datachain.namespace import Namespace, parse_name
 from datachain.query import Session
 
 
@@ -71,3 +75,54 @@ def ls(session: Optional[Session] = None) -> list[Namespace]:
         ```
     """
     return Session.get(session).catalog.metastore.list_namespaces()
+
+
+def delete(name: str, session: Optional[Session]) -> None:
+    """
+    Removes a namespace by name.
+
+    Raises:
+        NamespaceNotFoundError: If the namespace does not exist.
+        NamespaceDeleteNotAllowedError: If the namespace is non-empty,
+            is the default namespace, or is a system namespace,
+            as these cannot be removed.
+
+    Parameters:
+        name : The name of the namespace.
+        session : Session to use for getting project.
+
+    Example:
+        ```py
+        import datachain as dc
+        from datachain.lib.namespace import delete as delete_namespace
+        delete_namespace("dev")
+        ```
+    """
+    session = Session.get(session)
+    metastore = session.catalog.metastore
+
+    namespace_name, project_name = parse_name(name)
+
+    if project_name:
+        return delete_project(project_name, namespace_name, session)
+
+    namespace = metastore.get_namespace(name)
+
+    if name == metastore.system_namespace_name:
+        raise NamespaceDeleteNotAllowedError(
+            f"Namespace {metastore.system_namespace_name} cannot be removed"
+        )
+
+    if name == metastore.default_namespace_name:
+        raise NamespaceDeleteNotAllowedError(
+            f"Namespace {metastore.default_namespace_name} cannot be removed"
+        )
+
+    num_projects = metastore.count_projects(namespace.id)
+    if num_projects > 0:
+        raise NamespaceDeleteNotAllowedError(
+            f"Namespace cannot be removed. It contains {num_projects} project(s). "
+            "Please remove the project(s) first."
+        )
+
+    metastore.remove_namespace(namespace.id)

{datachain-0.31.4 → datachain-0.32.0}/src/datachain/lib/projects.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from datachain.error import ProjectCreateNotAllowedError
+from datachain.error import ProjectCreateNotAllowedError, ProjectDeleteNotAllowedError
 from datachain.project import Project
 from datachain.query import Session
 
@@ -86,3 +86,49 @@ def ls(
         namespace_id = session.catalog.metastore.get_namespace(namespace).id
 
     return session.catalog.metastore.list_projects(namespace_id)
+
+
+def delete(name: str, namespace: str, session: Optional[Session] = None) -> None:
+    """
+    Removes a project by name within a namespace.
+
+    Raises:
+        ProjectNotFoundError: If the project does not exist.
+        ProjectDeleteNotAllowedError: If the project is non-empty,
+            is the default project, or is a listing project,
+            as these cannot be removed.
+
+    Parameters:
+        name : The name of the project.
+        namespace : The name of the namespace.
+        session : Session to use for getting project.
+
+    Example:
+        ```py
+        import datachain as dc
+        dc.delete_project("my-project", "local")
+        ```
+    """
+    session = Session.get(session)
+    metastore = session.catalog.metastore
+
+    project = metastore.get_project(name, namespace)
+
+    if metastore.is_listing_project(name, namespace):
+        raise ProjectDeleteNotAllowedError(
+            f"Project {metastore.listing_project_name} cannot be removed"
+        )
+
+    if metastore.is_default_project(name, namespace):
+        raise ProjectDeleteNotAllowedError(
+            f"Project {metastore.default_project_name} cannot be removed"
+        )
+
+    num_datasets = metastore.count_datasets(project.id)
+    if num_datasets > 0:
+        raise ProjectDeleteNotAllowedError(
+            f"Project cannot be removed. It contains {num_datasets} dataset(s). "
+            "Please remove the dataset(s) first."
+        )
+
+    metastore.remove_project(project.id)

{datachain-0.31.4 → datachain-0.32.0}/src/datachain/namespace.py

@@ -9,6 +9,25 @@ N = TypeVar("N", bound="Namespace")
 NAMESPACE_NAME_RESERVED_CHARS = [".", "@"]
 
 
+def parse_name(name: str) -> tuple[str, Optional[str]]:
+    """
+    Parses namespace name into namespace and optional project name.
+    If both namespace and project are defined in name, they need to be split by dot
+    e.g dev.my-project
+    Valid inputs:
+        - dev.my-project
+        - dev
+    """
+    parts = name.split(".")
+    if len(parts) == 1:
+        return name, None
+    if len(parts) == 2:
+        return parts[0], parts[1]
+    raise InvalidNamespaceNameError(
+        f"Invalid namespace format: {name}. Expected 'namespace' or 'ns1.ns2'."
+    )
+
+
 @dataclass(frozen=True)
 class Namespace:
     id: int

{datachain-0.31.4 → datachain-0.32.0}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.31.4
+Version: 0.32.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.31.4 → datachain-0.32.0}/tests/unit/lib/test_namespace.py

@@ -3,11 +3,14 @@ import pytest
 from datachain.error import (
     InvalidNamespaceNameError,
     NamespaceCreateNotAllowedError,
+    NamespaceDeleteNotAllowedError,
     NamespaceNotFoundError,
 )
 from datachain.lib.namespaces import create as create_namespace
+from datachain.lib.namespaces import delete as delete_namespace
 from datachain.lib.namespaces import get as get_namespace
 from datachain.lib.namespaces import ls as ls_namespaces
+from datachain.lib.projects import create as create_project
 from tests.utils import skip_if_not_sqlite
 
 
@@ -77,3 +80,45 @@ def test_ls_namespaces_just_local(test_session):
     system_namespace_name = test_session.catalog.metastore.system_namespace_name
     namespaces = ls_namespaces(session=test_session)
     assert [n.name for n in namespaces] == [system_namespace_name]
+
+
+def test_delete_namespace(test_session, dev_namespace):
+    delete_namespace(dev_namespace.name, session=test_session)
+    with pytest.raises(NamespaceNotFoundError):
+        get_namespace(dev_namespace.name, session=test_session)
+
+
+def test_delete_namespace_not_found(test_session, dev_namespace):
+    with pytest.raises(NamespaceNotFoundError):
+        delete_namespace("missing", session=test_session)
+
+
+def test_delete_namespace_system(test_session):
+    metastore = test_session.catalog.metastore
+    with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo:
+        delete_namespace(metastore.system_namespace_name, session=test_session)
+    assert str(excinfo.value) == (
+        f"Namespace {metastore.system_namespace_name} cannot be removed"
+    )
+
+
+def test_delete_namespace_default(test_session):
+    metastore = test_session.catalog.metastore
+    metastore.create_namespace(metastore.default_namespace_name, validate=False)
+    with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo:
+        delete_namespace(metastore.default_namespace_name, session=test_session)
+    assert str(excinfo.value) == (
+        f"Namespace {metastore.default_namespace_name} cannot be removed"
+    )
+
+
+def test_delete_namespace_non_empty(test_session, dev_namespace):
+    create_project(dev_namespace.name, "my-project")
+
+    with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo:
+        delete_namespace(dev_namespace.name, session=test_session)
+
+    assert str(excinfo.value) == (
+        "Namespace cannot be removed. It contains 1 project(s)."
+        " Please remove the project(s) first."
+    )

{datachain-0.31.4 → datachain-0.32.0}/tests/unit/lib/test_project.py

@@ -4,9 +4,11 @@ import datachain as dc
 from datachain.error import (
     InvalidProjectNameError,
     ProjectCreateNotAllowedError,
+    ProjectDeleteNotAllowedError,
     ProjectNotFoundError,
 )
 from datachain.lib.namespaces import create as create_namespace
+from datachain.lib.namespaces import delete as delete_namespace
 from datachain.lib.namespaces import get as get_namespace
 from datachain.lib.projects import get as get_project
 from datachain.lib.projects import ls as ls_projects
@@ -155,3 +157,61 @@ def test_ls_projects_empty_in_namespace(test_session):
     create_namespace("ns1")
     projects = ls_projects("ns1", session=test_session)
     assert [(p.namespace.name, p.name) for p in projects] == []
+
+
+def test_delete_project(test_session, chatbot_project, dev_namespace):
+    delete_namespace(
+        f"{dev_namespace.name}.{chatbot_project.name}", session=test_session
+    )
+    with pytest.raises(ProjectNotFoundError):
+        get_project(chatbot_project.name, dev_namespace.name, session=test_session)
+
+    # namespace should not be deleted
+    get_namespace(dev_namespace.name, session=test_session)
+
+
+def test_delete_project_not_found(test_session, chatbot_project, dev_namespace):
+    with pytest.raises(ProjectNotFoundError):
+        delete_namespace(f"{dev_namespace.name}.missing", session=test_session)
+
+
+def test_delete_project_listing(test_session):
+    metastore = test_session.catalog.metastore
+    with pytest.raises(ProjectDeleteNotAllowedError) as excinfo:
+        delete_namespace(
+            f"{metastore.system_namespace_name}.{metastore.listing_project_name}",
+            session=test_session,
+        )
+    assert str(excinfo.value) == (
+        f"Project {metastore.listing_project_name} cannot be removed"
+    )
+
+
+def test_delete_project_default(test_session):
+    metastore = test_session.catalog.metastore
+    with pytest.raises(ProjectDeleteNotAllowedError) as excinfo:
+        delete_namespace(
+            f"{metastore.default_namespace_name}.{metastore.default_project_name}",
+            session=test_session,
+        )
+    assert str(excinfo.value) == (
+        f"Project {metastore.default_project_name} cannot be removed"
+    )
+
+
+def test_delete_project_non_empty(test_session, chatbot_project, dev_namespace):
+    (
+        dc.read_values(num=[1, 2, 3])
+        .settings(namespace=dev_namespace.name, project=chatbot_project.name)
+        .save("numbers")
+    )
+
+    with pytest.raises(ProjectDeleteNotAllowedError) as excinfo:
+        delete_namespace(
+            f"{dev_namespace.name}.{chatbot_project.name}", session=test_session
+        )
+
+    assert str(excinfo.value) == (
+        "Project cannot be removed. It contains 1 dataset(s)."
+        " Please remove the dataset(s) first."
+    )