datachain 0.19.1__py3-none-any.whl → 0.20.0__py3-none-any.whl

datachain/lib/projects.py

@@ -0,0 +1,86 @@
+from typing import Optional
+
+from datachain.error import ProjectCreateNotAllowedError
+from datachain.project import Project
+from datachain.query import Session
+
+
+def create(
+    name: str,
+    namespace_name: str,
+    description: Optional[str] = None,
+    session: Optional[Session] = None,
+) -> Project:
+    """
+    Creates a new custom project.
+    A Project is an object used to organize datasets. It is created under a
+    specific namespace and has a list of datasets underneath it.
+    Note that creating projects is not allowed in the local environment, unlike
+    in Studio, where it is allowed.
+    In local environment all datasets are created under the default `local` project.
+
+    Parameters:
+        name : The name of the project.
+        namespace : The name of the namespace under which the new project is being
+            created.
+        description : A description of the project.
+        session : Session to use for creating project.
+
+    Example:
+        ```py
+        import datachain as dc
+        project = dc.projects.create("my-project", "dev", "My personal project")
+        ```
+    """
+    session = Session.get(session)
+
+    if not session.catalog.metastore.project_allowed_to_create:
+        raise ProjectCreateNotAllowedError("Creating custom project is not allowed")
+
+    Project.validate_name(name)
+
+    return session.catalog.metastore.create_project(name, namespace_name, description)
+
+
+def get(name: str, namespace_name: str, session: Optional[Session]) -> Project:
+    """
+    Gets a project by name in some namespace.
+    If the project is not found, a `ProjectNotFoundError` is raised.
+
+    Parameters:
+        name : The name of the project.
+        namespace_name : The name of the namespace.
+        session : Session to use for getting project.
+
+    Example:
+        ```py
+        import datachain as dc
+        project  = dc.get_project("my-project", "local")
+        ```
+    """
+    return Session.get(session).catalog.metastore.get_project(name, namespace_name)
+
+
+def ls(
+    namespace_name: Optional[str] = None, session: Optional[Session] = None
+) -> list[Project]:
+    """
+    Gets a list of projects in a specific namespace or from all namespaces.
+
+    Parameters:
+        namespace_name : An optional namespace name.
+        session : Session to use for getting project.
+
+    Example:
+        ```py
+        import datachain as dc
+        local_namespace_projects = dc.projects.ls("local")
+        all_projects = dc.projects.ls()
+        ```
+    """
+    session = Session.get(session)
+    namespace_id = None
+    if namespace_name:
+        namespace_id = session.catalog.metastore.get_namespace(namespace_name).id
+
+    return session.catalog.metastore.list_projects(namespace_id)

datachain/lib/settings.py

@@ -14,12 +14,16 @@ class Settings:
         workers=None,
         min_task_size=None,
         prefetch=None,
+        namespace=None,
+        project=None,
     ):
         self._cache = cache
         self.parallel = parallel
         self._workers = workers
         self.min_task_size = min_task_size
         self.prefetch = prefetch
+        self.namespace = namespace
+        self.project = project
 
         if not isinstance(cache, bool) and cache is not None:
             raise SettingsError(
@@ -67,6 +71,10 @@ class Settings:
             res["workers"] = self.workers
         if self.min_task_size is not None:
             res["min_task_size"] = self.min_task_size
+        if self.namespace is not None:
+            res["namespace"] = self.namespace
+        if self.project is not None:
+            res["project"] = self.project
         return res
 
     def add(self, settings: "Settings"):
@@ -74,5 +82,7 @@ class Settings:
         self.parallel = settings.parallel or self.parallel
         self._workers = settings._workers or self._workers
         self.min_task_size = settings.min_task_size or self.min_task_size
+        self.namespace = settings.namespace or self.namespace
+        self.project = settings.project or self.project
         if settings.prefetch is not None:
             self.prefetch = settings.prefetch

datachain/listing.py

@@ -66,7 +66,9 @@ class Listing:
     @cached_property
     def dataset(self) -> "DatasetRecord":
         assert self.dataset_name
-        return self.metastore.get_dataset(self.dataset_name)
+        return self.metastore.get_dataset(
+            self.dataset_name, self.metastore.listing_project.id
+        )
 
     @cached_property
     def dataset_rows(self):

datachain/namespace.py

@@ -0,0 +1,65 @@
+import builtins
+from dataclasses import dataclass, fields
+from datetime import datetime
+from typing import Any, Optional, TypeVar
+
+from datachain.error import InvalidNamespaceNameError
+
+N = TypeVar("N", bound="Namespace")
+NAMESPACE_NAME_RESERVED_CHARS = ["."]
+
+
+@dataclass(frozen=True)
+class Namespace:
+    id: int
+    uuid: str
+    name: str
+    description: Optional[str]
+    created_at: datetime
+
+    @staticmethod
+    def validate_name(name: str) -> None:
+        """Throws exception if name is invalid, otherwise returns None"""
+        if not name:
+            raise InvalidNamespaceNameError("Namespace name cannot be empty")
+
+        for c in NAMESPACE_NAME_RESERVED_CHARS:
+            if c in name:
+                raise InvalidNamespaceNameError(
+                    f"Character {c} is reserved and not allowed in namespace name"
+                )
+
+        if name in [Namespace.default(), Namespace.system()]:
+            raise InvalidNamespaceNameError(
+                f"Namespace name {name} is reserved and cannot be used."
+            )
+
+    @staticmethod
+    def default() -> str:
+        """Name of default namespace"""
+        return "local"
+
+    @staticmethod
+    def system() -> str:
+        """Name of the system namespace"""
+        return "system"
+
+    @property
+    def is_system(self):
+        return self.name == Namespace.system()
+
+    @classmethod
+    def parse(
+        cls: builtins.type[N],
+        id: int,
+        uuid: str,
+        name: str,
+        description: Optional[str],
+        created_at: datetime,
+    ) -> "Namespace":
+        return cls(id, uuid, name, description, created_at)
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "Namespace":
+        kwargs = {f.name: d[f.name] for f in fields(cls) if f.name in d}
+        return cls(**kwargs)

datachain/project.py

@@ -0,0 +1,78 @@
+import builtins
+from dataclasses import dataclass, fields
+from datetime import datetime
+from typing import Any, Optional, TypeVar
+
+from datachain.error import InvalidProjectNameError
+from datachain.namespace import Namespace
+
+P = TypeVar("P", bound="Project")
+PROJECT_NAME_RESERVED_CHARS = ["."]
+
+
+@dataclass(frozen=True)
+class Project:
+    id: int
+    uuid: str
+    name: str
+    description: Optional[str]
+    created_at: datetime
+    namespace: Namespace
+
+    @staticmethod
+    def validate_name(name: str) -> None:
+        """Throws exception if name is invalid, otherwise returns None"""
+        if not name:
+            raise InvalidProjectNameError("Project name cannot be empty")
+
+        for c in PROJECT_NAME_RESERVED_CHARS:
+            if c in name:
+                raise InvalidProjectNameError(
+                    f"Character {c} is reserved and not allowed in project name."
+                )
+
+        if name in [Project.default(), Project.listing()]:
+            raise InvalidProjectNameError(
+                f"Project name {name} is reserved and cannot be used."
+            )
+
+    @staticmethod
+    def default() -> str:
+        """Name of default project"""
+        return "local"
+
+    @staticmethod
+    def listing() -> str:
+        """Name of listing project where all listing datasets will be saved"""
+        return "listing"
+
+    @classmethod
+    def parse(
+        cls: builtins.type[P],
+        namespace_id: int,
+        namespace_uuid: str,
+        namespace_name: str,
+        namespace_description: Optional[str],
+        namespace_created_at: datetime,
+        project_id: int,
+        uuid: str,
+        name: str,
+        description: Optional[str],
+        created_at: datetime,
+        project_namespace_id: int,
+    ) -> "Project":
+        namespace = Namespace.parse(
+            namespace_id,
+            namespace_uuid,
+            namespace_name,
+            namespace_description,
+            namespace_created_at,
+        )
+
+        return cls(project_id, uuid, name, description, created_at, namespace)
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "Project":
+        namespace = Namespace.from_dict(d.pop("namespace"))
+        kwargs = {f.name: d[f.name] for f in fields(cls) if f.name in d}
+        return cls(**kwargs, namespace=namespace)

datachain/query/dataset.py

@@ -41,12 +41,13 @@ from datachain.data_storage.schema import (
     partition_col_names,
     partition_columns,
 )
-from datachain.dataset import DATASET_PREFIX, DatasetDependency, DatasetStatus, RowDict
+from datachain.dataset import DatasetDependency, DatasetStatus, RowDict
 from datachain.error import DatasetNotFoundError, QueryScriptCancelError
 from datachain.func.base import Function
 from datachain.lib.listing import is_listing_dataset, listing_dataset_expired
 from datachain.lib.udf import UDFAdapter, _get_cache
 from datachain.progress import CombinedDownloadCallback, TqdmCombinedDownloadCallback
+from datachain.project import Project
 from datachain.query.schema import C, UDFParamSpec, normalize_param
 from datachain.query.session import Session
 from datachain.query.udf import UdfInfo
@@ -83,7 +84,7 @@ PartitionByType = Union[
     Function, ColumnElement, Sequence[Union[Function, ColumnElement]]
 ]
 JoinPredicateType = Union[str, ColumnClause, ColumnElement]
-DatasetDependencyType = tuple[str, str]
+DatasetDependencyType = tuple["DatasetRecord", str]
 
 logger = logging.getLogger("datachain")
 
@@ -169,18 +170,17 @@ class QueryStep:
     """A query that returns all rows from specific dataset version"""
 
     catalog: "Catalog"
-    dataset_name: str
+    dataset: "DatasetRecord"
     dataset_version: str
 
     def apply(self) -> "StepResult":
         def q(*columns):
             return sqlalchemy.select(*columns)
 
-        dataset = self.catalog.get_dataset(self.dataset_name)
-        dr = self.catalog.warehouse.dataset_rows(dataset, self.dataset_version)
+        dr = self.catalog.warehouse.dataset_rows(self.dataset, self.dataset_version)
 
         return step_result(
-            q, dr.columns, dependencies=[(self.dataset_name, self.dataset_version)]
+            q, dr.columns, dependencies=[(self.dataset, self.dataset_version)]
         )
 
 
@@ -1095,6 +1095,8 @@ class DatasetQuery:
         self,
         name: str,
         version: Optional[str] = None,
+        project_name: Optional[str] = None,
+        namespace_name: Optional[str] = None,
         catalog: Optional["Catalog"] = None,
         session: Optional[Session] = None,
         indexing_column_types: Optional[dict[str, Any]] = None,
@@ -1128,33 +1130,38 @@ class DatasetQuery:
         if version:
             self.version = version
 
-        if is_listing_dataset(name):
-            if version:
-                # this listing dataset should already be listed as we specify
-                # exact version
-                self._set_starting_step(self.catalog.get_dataset(name))
-            else:
-                # not setting query step yet as listing dataset might not exist at
-                # this point
-                self.list_ds_name = name
+        namespace_name = namespace_name or self.catalog.metastore.default_namespace_name
+        project_name = project_name or self.catalog.metastore.default_project_name
+
+        if is_listing_dataset(name) and not version:
+            # not setting query step yet as listing dataset might not exist at
+            # this point
+            self.list_ds_name = name
         elif fallback_to_studio and is_token_set():
             self._set_starting_step(
-                self.catalog.get_dataset_with_remote_fallback(name, version)
+                self.catalog.get_dataset_with_remote_fallback(
+                    name,
+                    namespace_name=namespace_name,
+                    project_name=project_name,
+                    version=version,
+                )
             )
         else:
-            self._set_starting_step(self.catalog.get_dataset(name))
+            project = self.catalog.metastore.get_project(project_name, namespace_name)
+            self._set_starting_step(self.catalog.get_dataset(name, project=project))
 
     def _set_starting_step(self, ds: "DatasetRecord") -> None:
         if not self.version:
             self.version = ds.latest_version
 
-        self.starting_step = QueryStep(self.catalog, ds.name, self.version)
+        self.starting_step = QueryStep(self.catalog, ds, self.version)
 
         # at this point we know our starting dataset so setting up schemas
         self.feature_schema = ds.get_version(self.version).feature_schema
         self.column_types = copy(ds.schema)
         if "sys__id" in self.column_types:
             self.column_types.pop("sys__id")
+        self.project = ds.project
 
     def __iter__(self):
         return iter(self.db_results())
@@ -1162,21 +1169,6 @@ class DatasetQuery:
     def __or__(self, other):
         return self.union(other)
 
-    def pull_dataset(self, name: str, version: Optional[str] = None) -> "DatasetRecord":
-        print("Dataset not found in local catalog, trying to get from studio")
-
-        remote_ds_uri = f"{DATASET_PREFIX}{name}"
-        if version:
-            remote_ds_uri += f"@v{version}"
-
-        self.catalog.pull_dataset(
-            remote_ds_uri=remote_ds_uri,
-            local_ds_name=name,
-            local_ds_version=version,
-        )
-
-        return self.catalog.get_dataset(name)
-
     @staticmethod
     def get_table() -> "TableClause":
         table_name = "".join(
@@ -1657,6 +1649,8 @@ class DatasetQuery:
         workers: Union[bool, int] = False,
         min_task_size: Optional[int] = None,
         partition_by: Optional[PartitionByType] = None,
+        namespace: Optional[str] = None,
+        project: Optional[str] = None,
         cache: bool = False,
     ) -> "Self":
         query = self.clone()
@@ -1676,26 +1670,36 @@ class DatasetQuery:
 
     def _add_dependencies(self, dataset: "DatasetRecord", version: str):
         dependencies: set[DatasetDependencyType] = set()
-        for dep_name, dep_version in self.dependencies:
-            if Session.is_temp_dataset(dep_name):
+        for dep_dataset, dep_dataset_version in self.dependencies:
+            if Session.is_temp_dataset(dep_dataset.name):
                 # temp dataset are created for optimization and they will be removed
                 # afterwards. Therefore, we should not put them as dependencies, but
                 # their own direct dependencies
                 for dep in self.catalog.get_dataset_dependencies(
-                    dep_name, dep_version, indirect=False
+                    dep_dataset.name,
+                    dep_dataset_version,
+                    dep_dataset.project,
+                    indirect=False,
                 ):
                     if dep:
-                        dependencies.add((dep.name, dep.version))
+                        dep_project = self.catalog.metastore.get_project(
+                            dep.project, dep.namespace
+                        )
+                        dependencies.add(
+                            (
+                                self.catalog.get_dataset(dep.name, dep_project),
+                                dep.version,
+                            )
+                        )
             else:
-                dependencies.add((dep_name, dep_version))
+                dependencies.add((dep_dataset, dep_dataset_version))
 
-        for dep_name, dep_version in dependencies:
-            # ds_dependency_name, ds_dependency_version = dependency
+        for dep_dataset, dep_dataset_version in dependencies:
             self.catalog.metastore.add_dataset_dependency(
-                dataset.name,
+                dataset,
                 version,
-                dep_name,
-                dep_version,
+                dep_dataset,
+                dep_dataset_version,
             )
 
     def exec(self) -> "Self":
@@ -1711,6 +1715,7 @@ class DatasetQuery:
         self,
         name: Optional[str] = None,
         version: Optional[str] = None,
+        project: Optional[Project] = None,
         feature_schema: Optional[dict] = None,
         dependencies: Optional[list[DatasetDependency]] = None,
         description: Optional[str] = None,
@@ -1719,8 +1724,13 @@ class DatasetQuery:
         **kwargs,
     ) -> "Self":
         """Save the query as a dataset."""
+        project = project or self.catalog.metastore.default_project
         try:
-            if name and version and self.catalog.get_dataset(name).has_version(version):
+            if (
+                name
+                and version
+                and self.catalog.get_dataset(name, project).has_version(version)
+            ):
                 raise RuntimeError(f"Dataset {name} already has version {version}")
         except DatasetNotFoundError:
             pass
@@ -1745,6 +1755,7 @@ class DatasetQuery:
 
             dataset = self.catalog.create_dataset(
                 name,
+                project,
                 version=version,
                 feature_schema=feature_schema,
                 columns=columns,
@@ -1770,11 +1781,25 @@ class DatasetQuery:
 
             if dependencies:
                 # overriding dependencies
-                self.dependencies = {(dep.name, dep.version) for dep in dependencies}
+                self.dependencies = set()
+                for dep in dependencies:
+                    dep_project = self.catalog.metastore.get_project(
+                        dep.project, dep.namespace
+                    )
+                    self.dependencies.add(
+                        (self.catalog.get_dataset(dep.name, dep_project), dep.version)
+                    )
+
             self._add_dependencies(dataset, version)  # type: ignore [arg-type]
         finally:
             self.cleanup()
-        return self.__class__(name=name, version=version, catalog=self.catalog)
+        return self.__class__(
+            name=name,
+            namespace_name=project.namespace.name,
+            project_name=project.name,
+            version=version,
+            catalog=self.catalog,
+        )
 
     @property
     def is_ordered(self) -> bool:

datachain/query/session.py

@@ -108,7 +108,7 @@ class Session:
         prefix = self.get_temp_prefix()
         try:
             for dataset in list(self.catalog.metastore.list_datasets_by_prefix(prefix)):
-                self.catalog.remove_dataset(dataset.name, force=True)
+                self.catalog.remove_dataset(dataset.name, dataset.project, force=True)
         # suppress error when metastore has been reset during testing
         except TableMissingError:
             pass