datachain 0.31.4__py3-none-any.whl → 0.32.0__py3-none-any.whl

datachain/__init__.py

@@ -37,6 +37,7 @@ from datachain.lib.file import (
     VideoFrame,
 )
 from datachain.lib.model_store import ModelStore
+from datachain.lib.namespaces import delete as delete_namespace
 from datachain.lib.projects import create as create_project
 from datachain.lib.udf import Aggregator, Generator, Mapper
 from datachain.lib.utils import AbstractUDF, DataChainError
@@ -74,6 +75,7 @@ __all__ = [
     "create_project",
     "datasets",
     "delete_dataset",
+    "delete_namespace",
     "is_chain_type",
     "is_studio",
     "listings",

datachain/data_storage/metastore.py

@@ -22,6 +22,7 @@ from sqlalchemy import (
     UniqueConstraint,
     select,
 )
+from sqlalchemy.sql import func as f
 
 from datachain.data_storage import JobQueryType, JobStatus
 from datachain.data_storage.serializer import Serializable
@@ -37,7 +38,9 @@ from datachain.dataset import (
 from datachain.error import (
     DatasetNotFoundError,
     DatasetVersionNotFoundError,
+    NamespaceDeleteNotAllowedError,
     NamespaceNotFoundError,
+    ProjectDeleteNotAllowedError,
     ProjectNotFoundError,
     TableMissingError,
 )
@@ -141,6 +144,10 @@ class AbstractMetastore(ABC, Serializable):
     def get_namespace(self, name: str, conn=None) -> Namespace:
         """Gets a single namespace by name"""
 
+    @abstractmethod
+    def remove_namespace(self, namespace_id: int, conn=None) -> None:
+        """Removes a single namespace by id"""
+
     @abstractmethod
     def list_namespaces(self, conn=None) -> list[Namespace]:
         """Gets a list of all namespaces"""
@@ -190,10 +197,30 @@ class AbstractMetastore(ABC, Serializable):
         It also creates project if not found and create flag is set to True.
         """
 
+    def is_default_project(self, project_name: str, namespace_name: str) -> bool:
+        return (
+            project_name == self.default_project_name
+            and namespace_name == self.default_namespace_name
+        )
+
+    def is_listing_project(self, project_name: str, namespace_name: str) -> bool:
+        return (
+            project_name == self.listing_project_name
+            and namespace_name == self.system_namespace_name
+        )
+
     @abstractmethod
     def get_project_by_id(self, project_id: int, conn=None) -> Project:
         """Gets a single project by id"""
 
+    @abstractmethod
+    def count_projects(self, namespace_id: Optional[int] = None) -> int:
+        """Counts projects in some namespace or in general."""
+
+    @abstractmethod
+    def remove_project(self, project_id: int, conn=None) -> None:
+        """Removes a single project by id"""
+
     @abstractmethod
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
         """Gets list of projects in some namespace or in general (in all namespaces)"""
@@ -270,6 +297,10 @@ class AbstractMetastore(ABC, Serializable):
     ) -> Iterator[DatasetListRecord]:
         """Lists all datasets in some project or in all projects."""
 
+    @abstractmethod
+    def count_datasets(self, project_id: Optional[int] = None) -> int:
+        """Counts datasets in some project or in all projects."""
+
     @abstractmethod
     def list_datasets_by_prefix(
         self, prefix: str, project_id: Optional[int] = None
@@ -735,6 +766,18 @@ class AbstractDBMetastore(AbstractMetastore):
 
         return self.get_namespace(name)
 
+    def remove_namespace(self, namespace_id: int, conn=None) -> None:
+        num_projects = self.count_projects(namespace_id)
+        if num_projects > 0:
+            raise NamespaceDeleteNotAllowedError(
+                f"Namespace cannot be removed. It contains {num_projects} project(s). "
+                "Please remove the project(s) first."
+            )
+
+        n = self._namespaces
+        with self.db.transaction():
+            self.db.execute(self._namespaces_delete().where(n.c.id == namespace_id))
+
     def get_namespace(self, name: str, conn=None) -> Namespace:
         """Gets a single namespace by name"""
         n = self._namespaces
@@ -796,18 +839,6 @@ class AbstractDBMetastore(AbstractMetastore):
 
         return self.get_project(name, namespace.name)
 
-    def _is_listing_project(self, project_name: str, namespace_name: str) -> bool:
-        return (
-            project_name == self.listing_project_name
-            and namespace_name == self.system_namespace_name
-        )
-
-    def _is_default_project(self, project_name: str, namespace_name: str) -> bool:
-        return (
-            project_name == self.default_project_name
-            and namespace_name == self.default_namespace_name
-        )
-
     def get_project(
         self, name: str, namespace_name: str, create: bool = False, conn=None
     ) -> Project:
@@ -816,7 +847,7 @@ class AbstractDBMetastore(AbstractMetastore):
         p = self._projects
         validate = True
 
-        if self._is_listing_project(name, namespace_name) or self._is_default_project(
+        if self.is_listing_project(name, namespace_name) or self.is_default_project(
             name, namespace_name
         ):
             # we are always creating default and listing projects if they don't exist
@@ -858,7 +889,31 @@ class AbstractDBMetastore(AbstractMetastore):
             raise ProjectNotFoundError(f"Project with id {project_id} not found.")
         return self.project_class.parse(*rows[0])
 
-    def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
+    def count_projects(self, namespace_id: Optional[int] = None) -> int:
+        p = self._projects
+        query = self._projects_select()
+        if namespace_id:
+            query = query.where(p.c.namespace_id == namespace_id)
+
+        query = select(f.count(1)).select_from(query.subquery())
+
+        return next(self.db.execute(query))[0]
+
+    def remove_project(self, project_id: int, conn=None) -> None:
+        num_datasets = self.count_datasets(project_id)
+        if num_datasets > 0:
+            raise ProjectDeleteNotAllowedError(
+                f"Project cannot be removed. It contains {num_datasets} dataset(s). "
+                "Please remove the dataset(s) first."
+            )
+
+        p = self._projects
+        with self.db.transaction():
+            self.db.execute(self._projects_delete().where(p.c.id == project_id))
+
+    def list_projects(
+        self, namespace_id: Optional[int] = None, conn=None
+    ) -> list[Project]:
         """
         Gets a list of projects inside some namespace, or in all namespaces
         """
@@ -1189,7 +1244,6 @@ class AbstractDBMetastore(AbstractMetastore):
     def list_datasets(
         self, project_id: Optional[int] = None
     ) -> Iterator["DatasetListRecord"]:
-        """Lists all datasets."""
         d = self._datasets
         query = self._base_list_datasets_query().order_by(
             self._datasets.c.name, self._datasets_versions.c.version
@@ -1198,6 +1252,16 @@ class AbstractDBMetastore(AbstractMetastore):
             query = query.where(d.c.project_id == project_id)
         yield from self._parse_dataset_list(self.db.execute(query))
 
+    def count_datasets(self, project_id: Optional[int] = None) -> int:
+        d = self._datasets
+        query = self._datasets_select()
+        if project_id:
+            query = query.where(d.c.project_id == project_id)
+
+        query = select(f.count(1)).select_from(query.subquery())
+
+        return next(self.db.execute(query))[0]
+
     def list_datasets_by_prefix(
         self, prefix: str, project_id: Optional[int] = None, conn=None
     ) -> Iterator["DatasetListRecord"]:

datachain/error.py

@@ -34,6 +34,14 @@ class ProjectCreateNotAllowedError(NotAllowedError):
     pass
 
 
+class ProjectDeleteNotAllowedError(NotAllowedError):
+    pass
+
+
+class NamespaceDeleteNotAllowedError(NotAllowedError):
+    pass
+
+
 class ProjectNotFoundError(NotFoundError):
     pass
 

datachain/lib/namespaces.py

@@ -1,7 +1,11 @@
 from typing import Optional
 
-from datachain.error import NamespaceCreateNotAllowedError
-from datachain.namespace import Namespace
+from datachain.error import (
+    NamespaceCreateNotAllowedError,
+    NamespaceDeleteNotAllowedError,
+)
+from datachain.lib.projects import delete as delete_project
+from datachain.namespace import Namespace, parse_name
 from datachain.query import Session
 
 
@@ -71,3 +75,54 @@ def ls(session: Optional[Session] = None) -> list[Namespace]:
         ```
     """
     return Session.get(session).catalog.metastore.list_namespaces()
+
+
+def delete(name: str, session: Optional[Session]) -> None:
+    """
+    Removes a namespace by name.
+
+    Raises:
+        NamespaceNotFoundError: If the namespace does not exist.
+        NamespaceDeleteNotAllowedError: If the namespace is non-empty,
+            is the default namespace, or is a system namespace,
+            as these cannot be removed.
+
+    Parameters:
+        name : The name of the namespace.
+        session : Session to use for getting project.
+
+    Example:
+        ```py
+        import datachain as dc
+        from datachain.lib.namespace import delete as delete_namespace
+        delete_namespace("dev")
+        ```
+    """
+    session = Session.get(session)
+    metastore = session.catalog.metastore
+
+    namespace_name, project_name = parse_name(name)
+
+    if project_name:
+        return delete_project(project_name, namespace_name, session)
+
+    namespace = metastore.get_namespace(name)
+
+    if name == metastore.system_namespace_name:
+        raise NamespaceDeleteNotAllowedError(
+            f"Namespace {metastore.system_namespace_name} cannot be removed"
+        )
+
+    if name == metastore.default_namespace_name:
+        raise NamespaceDeleteNotAllowedError(
+            f"Namespace {metastore.default_namespace_name} cannot be removed"
+        )
+
+    num_projects = metastore.count_projects(namespace.id)
+    if num_projects > 0:
+        raise NamespaceDeleteNotAllowedError(
+            f"Namespace cannot be removed. It contains {num_projects} project(s). "
+            "Please remove the project(s) first."
+        )
+
+    metastore.remove_namespace(namespace.id)

datachain/lib/projects.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from datachain.error import ProjectCreateNotAllowedError
+from datachain.error import ProjectCreateNotAllowedError, ProjectDeleteNotAllowedError
 from datachain.project import Project
 from datachain.query import Session
 
@@ -86,3 +86,49 @@ def ls(
         namespace_id = session.catalog.metastore.get_namespace(namespace).id
 
     return session.catalog.metastore.list_projects(namespace_id)
+
+
+def delete(name: str, namespace: str, session: Optional[Session] = None) -> None:
+    """
+    Removes a project by name within a namespace.
+
+    Raises:
+        ProjectNotFoundError: If the project does not exist.
+        ProjectDeleteNotAllowedError: If the project is non-empty,
+            is the default project, or is a listing project,
+            as these cannot be removed.
+
+    Parameters:
+        name : The name of the project.
+        namespace : The name of the namespace.
+        session : Session to use for getting project.
+
+    Example:
+        ```py
+        import datachain as dc
+        dc.delete_project("my-project", "local")
+        ```
+    """
+    session = Session.get(session)
+    metastore = session.catalog.metastore
+
+    project = metastore.get_project(name, namespace)
+
+    if metastore.is_listing_project(name, namespace):
+        raise ProjectDeleteNotAllowedError(
+            f"Project {metastore.listing_project_name} cannot be removed"
+        )
+
+    if metastore.is_default_project(name, namespace):
+        raise ProjectDeleteNotAllowedError(
+            f"Project {metastore.default_project_name} cannot be removed"
+        )
+
+    num_datasets = metastore.count_datasets(project.id)
+    if num_datasets > 0:
+        raise ProjectDeleteNotAllowedError(
+            f"Project cannot be removed. It contains {num_datasets} dataset(s). "
+            "Please remove the dataset(s) first."
+        )
+
+    metastore.remove_project(project.id)

datachain/namespace.py

@@ -9,6 +9,25 @@ N = TypeVar("N", bound="Namespace")
 NAMESPACE_NAME_RESERVED_CHARS = [".", "@"]
 
 
+def parse_name(name: str) -> tuple[str, Optional[str]]:
+    """
+    Parses namespace name into namespace and optional project name.
+    If both namespace and project are defined in name, they need to be split by dot
+    e.g dev.my-project
+    Valid inputs:
+        - dev.my-project
+        - dev
+    """
+    parts = name.split(".")
+    if len(parts) == 1:
+        return name, None
+    if len(parts) == 2:
+        return parts[0], parts[1]
+    raise InvalidNamespaceNameError(
+        f"Invalid namespace format: {name}. Expected 'namespace' or 'ns1.ns2'."
+    )
+
+
 @dataclass(frozen=True)
 class Namespace:
     id: int

{datachain-0.31.4.dist-info → datachain-0.32.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.31.4
+Version: 0.32.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.31.4.dist-info → datachain-0.32.0.dist-info}/RECORD

@@ -1,14 +1,14 @@
-datachain/__init__.py,sha256=Ze-u6SSNsTFBRFw0lVPCdoP0kt8ybKxJIhO8jfC22Cw,1744
+datachain/__init__.py,sha256=5DFB1P58c35C_WBMrhmaynsP1WwCukC-9gTJIaPy0E8,1832
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=RH_jFwJcTXxhEFomaI9yL6S3Onau6NZ6FSKfKFGtrJE,9689
 datachain/cache.py,sha256=ESVRaCJXEThMIfGEFVHx6wJPOZA7FYk9V6WxjyuqUBY,3626
 datachain/config.py,sha256=g8qbNV0vW2VEKpX-dGZ9pAn0DAz6G2ZFcr7SAV3PoSM,4272
 datachain/dataset.py,sha256=ATGa-CBTFoZeTN2V40-zHEzfMBcdYK0WuoJ6H2yEAvo,25268
 datachain/delta.py,sha256=X5Lw6GQ8MAYNl2YIExNvl0tPIkylQEWwnCw0We7NtHM,10693
-datachain/error.py,sha256=OWwWMkzZYJrkcoEDGhJHMf7SfKvxcsOLRF94mjPf29I,1609
+datachain/error.py,sha256=comKx1JCdjsBpxabrOWaiRP0aHBspBDZl1mkKFnBSq0,1739
 datachain/job.py,sha256=x5PB6d5sqx00hePNNkirESlOVAvnmkEM5ygUgQmAhsk,1262
 datachain/listing.py,sha256=aqayl5St3D9PwdwM6nR1STkpLSw-S3U8pudO9PWi3N8,7241
-datachain/namespace.py,sha256=MozcXYxedIbamzY56YKy9r9fgSpOm2VryhWfIf6stYk,1791
+datachain/namespace.py,sha256=sgIF90KEaC_VlMFivDIJiFz8RUsTftMxW4kOUTyxo3A,2356
 datachain/node.py,sha256=KWDT0ClYXB7FYI-QOvzAa-UDkLJErUI2eWm5FBteYuU,5577
 datachain/nodes_fetcher.py,sha256=_wgaKyqEjkqdwJ_Hj6D8vUYz7hnU7g6xhm0H6ZnYxmE,1095
 datachain/nodes_thread_pool.py,sha256=mdo0s-VybuSZkRUARcUO4Tjh8KFfZr9foHqmupx2SmM,3989
@@ -49,7 +49,7 @@ datachain/client/s3.py,sha256=6DNVGLg-woPS1DVlYVX2rIlunNblsuxyOnI1rSzhW3k,7515
 datachain/data_storage/__init__.py,sha256=9Wit-oe5P46V7CJQTD0BJ5MhOa2Y9h3ddJ4VWTe-Lec,273
 datachain/data_storage/db_engine.py,sha256=n8ojCbvVMPY2e3SG8fUaaD0b9GkVfpl_Naa_6EiHfWg,3788
 datachain/data_storage/job.py,sha256=ZkeXCNUj_VCkoKYx29hqB4AcfVUielnRjY-GYUcUxt4,426
-datachain/data_storage/metastore.py,sha256=aSeTRh43hmrOhULi9YD2VlgCj8B4bjE3jqCOvnb_HQs,53851
+datachain/data_storage/metastore.py,sha256=2-FNdrhV-UoE6ztzdVea2MLpjr80Mvf3M3kbDfd3jSs,56222
 datachain/data_storage/schema.py,sha256=o3JbURKXRg3IJyIVA4QjHHkn6byRuz7avbydU2FlvNY,9897
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
 datachain/data_storage/sqlite.py,sha256=1fIeIhmB3O8oQVzP8dDKap0KUIgI0n2TdBQSyv0R8J4,30345
@@ -82,8 +82,8 @@ datachain/lib/listing.py,sha256=U-2stsTEwEsq4Y80dqGfktGzkmB5-ZntnL1_rzXlH0k,7089
 datachain/lib/listing_info.py,sha256=9ua40Hw0aiQByUw3oAEeNzMavJYfW0Uhe8YdCTK-m_g,1110
 datachain/lib/meta_formats.py,sha256=zdyg6XLk3QIsSk3I7s0Ez5kaCJSlE3uq7JiGxf7UwtU,6348
 datachain/lib/model_store.py,sha256=A0pSVQ7uaZ9RvANapzirF8Cqq9N6ysosPpMSkzdRPkU,3226
-datachain/lib/namespaces.py,sha256=I6gLC4ZzgyatFtHL85MWR4ml7-yuQOzxHE7IQNbt_ac,2107
-datachain/lib/projects.py,sha256=VJgmzHzKjmNPZD1tm0a1RNHmUQwn6WLWCLpKyc4UrSk,2605
+datachain/lib/namespaces.py,sha256=ij67QHnRJhC8uquR21aD8u1Um2jfxnBX8PecuOQZpYw,3828
+datachain/lib/projects.py,sha256=_YeU9PPcH_pC8-sbX-47XtWSdl1ltVKnALY8azWLJkM,4112
 datachain/lib/pytorch.py,sha256=S-st2SAczYut13KMf6eSqP_OQ8otWI5TRmzhK5fN3k0,7828
 datachain/lib/settings.py,sha256=xBQEPZfgaYKhHIFLd0u5CBTYDcJS8ZHCm47x7GJErFU,7666
 datachain/lib/signal_schema.py,sha256=YMMcc9gHIzBz88zfsreGa1nOoO_56HBtZlT6jf3V1WE,39224
@@ -161,9 +161,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.31.4.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.31.4.dist-info/METADATA,sha256=wqjT5wVjclsvbSjyXxABcJ46-JKCGT5t8-MJK55VApM,13898
-datachain-0.31.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.31.4.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.31.4.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.31.4.dist-info/RECORD,,
+datachain-0.32.0.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.32.0.dist-info/METADATA,sha256=gLsH5khLc-z_s0MKTt3H6qk_UXDAaWuHy2lk1yytgBw,13898
+datachain-0.32.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.32.0.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.32.0.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.32.0.dist-info/RECORD,,