datachain 0.21.1__py3-none-any.whl → 0.23.0__py3-none-any.whl

datachain/data_storage/sqlite.py

@@ -3,7 +3,7 @@ import os
 import sqlite3
 from collections.abc import Iterable, Sequence
 from contextlib import contextmanager
-from functools import wraps
+from functools import cached_property, wraps
 from time import sleep
 from typing import (
     TYPE_CHECKING,
@@ -15,7 +15,15 @@ from typing import (
 )
 
 import sqlalchemy
-from sqlalchemy import MetaData, Table, UniqueConstraint, exists, select
+from sqlalchemy import (
+    Column,
+    Integer,
+    MetaData,
+    Table,
+    UniqueConstraint,
+    exists,
+    select,
+)
 from sqlalchemy.dialects import sqlite
 from sqlalchemy.schema import CreateIndex, CreateTable, DropTable
 from sqlalchemy.sql import func
@@ -30,7 +38,9 @@ from datachain.data_storage import AbstractDBMetastore, AbstractWarehouse
 from datachain.data_storage.db_engine import DatabaseEngine
 from datachain.data_storage.schema import DefaultSchema
 from datachain.dataset import DatasetRecord, StorageURI
-from datachain.error import DataChainError
+from datachain.error import DataChainError, OutdatedDatabaseSchemaError
+from datachain.namespace import Namespace
+from datachain.project import Project
 from datachain.sql.sqlite import create_user_defined_sql_functions, sqlite_dialect
 from datachain.sql.sqlite.base import load_usearch_extension
 from datachain.sql.types import SQLType
@@ -60,6 +70,14 @@ datachain.sql.sqlite.setup()
 quote_schema = sqlite_dialect.identifier_preparer.quote_schema
 quote = sqlite_dialect.identifier_preparer.quote
 
+# NOTE! This should be manually increased when we change our DB schema in codebase
+SCHEMA_VERSION = 1
+
+OUTDATED_SCHEMA_ERROR_MESSAGE = (
+    "You have an old version of the database schema. Please refer to the documentation"
+    " for more information."
+)
+
 
 def _get_in_memory_uri():
     return "file::memory:?cache=shared"
@@ -303,6 +321,11 @@ class SQLiteDatabaseEngine(DatabaseEngine):
         )
         return bool(next(self.execute(query))[0])
 
+    @property
+    def table_names(self) -> list[str]:
+        query = "SELECT name FROM sqlite_master WHERE type='table';"
+        return [r[0] for r in self.execute_str(query).fetchall()]
+
     def create_table(self, table: "Table", if_not_exists: bool = True) -> None:
         self.execute(CreateTable(table, if_not_exists=if_not_exists))
 
@@ -321,6 +344,8 @@ class SQLiteMetastore(AbstractDBMetastore):
     This is currently used for the local cli.
     """
 
+    META_TABLE = "meta"
+
     db: "SQLiteDatabaseEngine"
 
     def __init__(
@@ -342,7 +367,11 @@ class SQLiteMetastore(AbstractDBMetastore):
 
         self.db = db or SQLiteDatabaseEngine.from_db_file(db_file)
 
+        self._init_meta_table()
+        self._init_meta_schema_value()
+        self._check_schema_version()
         self._init_tables()
+        self._init_namespaces_projects()
 
     def __exit__(self, exc_type, exc_value, traceback) -> None:
         """Close connection upon exit from context manager."""
@@ -383,8 +412,44 @@ class SQLiteMetastore(AbstractDBMetastore):
         (db_class, db_args, db_kwargs) = db_clone_params
         return cls(uri=uri, db=db_class(*db_args, **db_kwargs))
 
+    @cached_property
+    def _meta(self) -> Table:
+        return Table(self.META_TABLE, self.db.metadata, *self._meta_columns())
+
+    def _meta_select(self, *columns) -> "Select":
+        if not columns:
+            return self._meta.select()
+        return select(*columns)
+
+    def _meta_insert(self) -> "Insert":
+        return sqlite.insert(self._meta)
+
+    def _init_meta_table(self) -> None:
+        """Initializes meta table"""
+        # NOTE! needs to be called before _init_tables()
+        table_names = self.db.table_names
+        if table_names and self.META_TABLE not in table_names:
+            # this will happen on first run
+            raise OutdatedDatabaseSchemaError(OUTDATED_SCHEMA_ERROR_MESSAGE)
+
+        self.db.create_table(self._meta, if_not_exists=True)
+        self.default_table_names.append(self._meta.name)
+
+    def _init_meta_schema_value(self) -> None:
+        """Inserts current schema version value if not present in meta table yet"""
+        stmt = (
+            self._meta_insert()
+            .values(id=1, schema_version=SCHEMA_VERSION)
+            .on_conflict_do_nothing(index_elements=["id"])
+        )
+        self.db.execute(stmt)
+
     def _init_tables(self) -> None:
         """Initialize tables."""
+        self.db.create_table(self._namespaces, if_not_exists=True)
+        self.default_table_names.append(self._namespaces.name)
+        self.db.create_table(self._projects, if_not_exists=True)
+        self.default_table_names.append(self._projects.name)
         self.db.create_table(self._datasets, if_not_exists=True)
         self.default_table_names.append(self._datasets.name)
         self.db.create_table(self._datasets_versions, if_not_exists=True)
@@ -394,10 +459,56 @@ class SQLiteMetastore(AbstractDBMetastore):
         self.db.create_table(self._jobs, if_not_exists=True)
         self.default_table_names.append(self._jobs.name)
 
+    def _init_namespaces_projects(self) -> None:
+        """
+        Creates local namespace and local project connected to it.
+        In local environment user cannot explicitly create other namespaces and
+        projects and all datasets user creates will be stored in those.
+        When pulling dataset from Studio, then other namespaces and projects will
+        be created implicitly though, to keep the same fully qualified name with
+        Studio dataset.
+        """
+        system_namespace = self.create_namespace(
+            Namespace.system(), "System namespace", validate=False
+        )
+        self.create_project(
+            system_namespace.name, Project.listing(), "Listing project", validate=False
+        )
+
+    def _check_schema_version(self) -> None:
+        """
+        Checks if current DB schema is up to date with latest DB model and schema
+        version. If not, OutdatedDatabaseSchemaError is raised.
+        """
+        schema_version = next(self.db.execute(self._meta_select()))[1]
+        if schema_version < SCHEMA_VERSION:
+            raise OutdatedDatabaseSchemaError(OUTDATED_SCHEMA_ERROR_MESSAGE)
+
+    #
+    # Dataset dependencies
+    #
+    @classmethod
+    def _meta_columns(cls) -> list["SchemaItem"]:
+        return [
+            Column("id", Integer, primary_key=True),
+            Column("schema_version", Integer, default=SCHEMA_VERSION),
+        ]
+
     @classmethod
     def _datasets_columns(cls) -> list["SchemaItem"]:
         """Datasets table columns."""
-        return [*super()._datasets_columns(), UniqueConstraint("name")]
+        return [*super()._datasets_columns(), UniqueConstraint("project_id", "name")]
+
+    @classmethod
+    def _namespaces_columns(cls) -> list["SchemaItem"]:
+        """Datasets table columns."""
+        return [*super()._namespaces_columns(), UniqueConstraint("name")]
+
+    def _namespaces_insert(self) -> "Insert":
+        return sqlite.insert(self._namespaces)
+
+    def _projects_insert(self) -> "Insert":
+        return sqlite.insert(self._projects)
 
     def _datasets_insert(self) -> "Insert":
         return sqlite.insert(self._datasets)
@@ -414,6 +525,8 @@ class SQLiteMetastore(AbstractDBMetastore):
 
     def _dataset_dependencies_select_columns(self) -> list["SchemaItem"]:
         return [
+            self._namespaces.c.name,
+            self._projects.c.name,
             self._datasets_dependencies.c.id,
             self._datasets_dependencies.c.dataset_id,
             self._datasets_dependencies.c.dataset_version_id,
@@ -429,6 +542,26 @@ class SQLiteMetastore(AbstractDBMetastore):
     def _jobs_insert(self) -> "Insert":
         return sqlite.insert(self._jobs)
 
+    @property
+    def is_studio(self) -> bool:
+        return False
+
+    #
+    # Namespaces
+    #
+
+    @property
+    def default_namespace_name(self):
+        return Namespace.default()
+
+    #
+    # Projects
+    #
+
+    @property
+    def default_project_name(self):
+        return Project.default()
+
 
 class SQLiteWarehouse(AbstractWarehouse):
     """
@@ -534,16 +667,16 @@ class SQLiteWarehouse(AbstractWarehouse):
     ) -> None:
         dst_empty = False
 
-        if not self.db.has_table(self.dataset_table_name(src.name, src_version)):
+        if not self.db.has_table(self.dataset_table_name(src, src_version)):
             # source table doesn't exist, nothing to do
             return
 
         src_dr = self.dataset_rows(src, src_version).table
 
-        if not self.db.has_table(self.dataset_table_name(dst.name, dst_version)):
+        if not self.db.has_table(self.dataset_table_name(dst, dst_version)):
             # destination table doesn't exist, create it
             self.create_dataset_rows_table(
-                self.dataset_table_name(dst.name, dst_version),
+                self.dataset_table_name(dst, dst_version),
                 columns=src_dr.columns,
             )
             dst_empty = True

datachain/data_storage/warehouse.py

@@ -182,7 +182,7 @@ class AbstractWarehouse(ABC, Serializable):
     ):
         version = version or dataset.latest_version
 
-        table_name = self.dataset_table_name(dataset.name, version)
+        table_name = self.dataset_table_name(dataset, version)
         return self.schema.dataset_row_cls(
             table_name,
             self.db,
@@ -254,12 +254,24 @@ class AbstractWarehouse(ABC, Serializable):
         name = parsed.path if parsed.scheme == "file" else parsed.netloc
         return parsed.scheme, name
 
-    def dataset_table_name(self, dataset_name: str, version: str) -> str:
+    def dataset_table_name(self, dataset: DatasetRecord, version: str) -> str:
+        return self._construct_dataset_table_name(
+            dataset.project.namespace.name,
+            dataset.project.name,
+            dataset.name,
+            version,
+        )
+
+    def _construct_dataset_table_name(
+        self, namespace: str, project: str, dataset_name: str, version: str
+    ) -> str:
         prefix = self.DATASET_TABLE_PREFIX
         if Client.is_data_source_uri(dataset_name):
             # for datasets that are created for bucket listing we use different prefix
             prefix = self.DATASET_SOURCE_TABLE_PREFIX
-        return f"{prefix}{dataset_name}_{version.replace('.', '_')}"
+        return (
+            f"{prefix}{namespace}_{project}_{dataset_name}_{version.replace('.', '_')}"
+        )
 
     def temp_table_name(self) -> str:
         return self.TMP_TABLE_NAME_PREFIX + _random_string(6)
@@ -287,7 +299,7 @@ class AbstractWarehouse(ABC, Serializable):
         if_exists: bool = True,
     ) -> None:
         """Drops a dataset rows table for the given dataset name."""
-        table_name = self.dataset_table_name(dataset.name, version)
+        table_name = self.dataset_table_name(dataset, version)
         table = sa.Table(table_name, self.db.metadata)
         self.db.drop_table(table, if_exists=if_exists)
 
@@ -344,13 +356,20 @@ class AbstractWarehouse(ABC, Serializable):
 
     def rename_dataset_table(
         self,
+        dataset: DatasetRecord,
         old_name: str,
         new_name: str,
         old_version: str,
         new_version: str,
     ) -> None:
-        old_ds_table_name = self.dataset_table_name(old_name, old_version)
-        new_ds_table_name = self.dataset_table_name(new_name, new_version)
+        namespace = dataset.project.namespace.name
+        project = dataset.project.name
+        old_ds_table_name = self._construct_dataset_table_name(
+            namespace, project, old_name, old_version
+        )
+        new_ds_table_name = self._construct_dataset_table_name(
+            namespace, project, new_name, new_version
+        )
 
         self.db.rename_table(old_ds_table_name, new_ds_table_name)
 
@@ -368,7 +387,7 @@ class AbstractWarehouse(ABC, Serializable):
         """
         Returns tuple with dataset stats: total number of rows and total dataset size.
         """
-        if not (self.db.has_table(self.dataset_table_name(dataset.name, version))):
+        if not (self.db.has_table(self.dataset_table_name(dataset, version))):
             return None, None
 
         file_signals = list(

datachain/dataset.py

@@ -13,7 +13,9 @@ from typing import (
 from urllib.parse import urlparse
 
 from datachain import semver
-from datachain.error import DatasetVersionNotFoundError
+from datachain.error import DatasetVersionNotFoundError, InvalidDatasetNameError
+from datachain.namespace import Namespace
+from datachain.project import Project
 from datachain.sql.types import NAME_TYPES_MAPPING, SQLType
 
 T = TypeVar("T", bound="DatasetRecord")
@@ -27,6 +29,8 @@ QUERY_DATASET_PREFIX = "ds_query_"
 LISTING_PREFIX = "lst__"
 
 DEFAULT_DATASET_VERSION = "1.0.0"
+DATASET_NAME_RESERVED_CHARS = ["."]
+DATASET_NAME_REPLACEMENT_CHAR = "_"
 
 
 # StorageURI represents a normalised URI to a valid storage location (full bucket or
@@ -57,20 +61,37 @@ def parse_dataset_uri(uri: str) -> tuple[str, Optional[str]]:
     return name, s[1]
 
 
-def create_dataset_uri(name: str, version: Optional[str] = None) -> str:
+def create_dataset_uri(
+    name: str, namespace: str, project: str, version: Optional[str] = None
+) -> str:
     """
-    Creates a dataset uri based on dataset name and optionally version
+    Creates a dataset uri based on namespace, project, dataset name and optionally
+    version.
     Example:
-        Input: zalando, 3.0.1
-        Output: ds//zalando@v3.0.1
+        Input: dev, clothes, zalando, 3.0.1
+        Output: ds//dev.clothes.zalando@v3.0.1
     """
-    uri = f"{DATASET_PREFIX}{name}"
+    uri = f"{DATASET_PREFIX}{namespace}.{project}.{name}"
     if version:
         uri += f"@v{version}"
 
     return uri
 
 
+def parse_dataset_name(name: str) -> tuple[Optional[str], Optional[str], str]:
+    """Parses dataset name and returns namespace, project and name"""
+    if not name:
+        raise InvalidDatasetNameError("Name must be defined to parse it")
+    split = name.split(".")
+    if len(split) > 3:
+        raise InvalidDatasetNameError(f"Invalid dataset name {name}")
+    name = split[-1]
+    project_name = split[-2] if len(split) > 1 else None
+    namespace_name = split[-3] if len(split) > 2 else None
+
+    return namespace_name, project_name, name
+
+
 class DatasetDependencyType:
     DATASET = "dataset"
     STORAGE = "storage"
@@ -78,8 +99,12 @@ class DatasetDependencyType:
 
 @dataclass
 class DatasetDependency:
+    # TODO put `DatasetRecord` instead of name + version which will
+    # simplify codebase in various places
     id: int
     type: str
+    namespace: str
+    project: str
     name: str
     version: str
     created_at: datetime
@@ -100,6 +125,8 @@ class DatasetDependency:
     @classmethod
     def parse(
         cls: builtins.type[DD],
+        namespace_name: str,
+        project_name: str,
         id: int,
         dataset_id: Optional[int],
         dataset_version_id: Optional[int],
@@ -121,6 +148,8 @@ class DatasetDependency:
                 if is_listing_dataset(dataset_name)
                 else DatasetDependencyType.DATASET
             ),
+            namespace_name,
+            project_name,
             dataset_name,
             (
                 dataset_version  # type: ignore[arg-type]
@@ -335,6 +364,7 @@ class DatasetListVersion:
 class DatasetRecord:
     id: int
     name: str
+    project: Project
     description: Optional[str]
     attrs: list[str]
     schema: dict[str, Union[SQLType, type[SQLType]]]
@@ -349,6 +379,9 @@ class DatasetRecord:
     sources: str = ""
     query_script: str = ""
 
+    def __hash__(self):
+        return hash(f"{self.id}")
+
     @staticmethod
     def parse_schema(
         ct: dict[str, Any],
@@ -358,10 +391,31 @@ class DatasetRecord:
             for c_name, c_type in ct.items()
         }
 
+    @staticmethod
+    def validate_name(name: str) -> None:
+        """Throws exception if name has reserved characters"""
+        for c in DATASET_NAME_RESERVED_CHARS:
+            if c in name:
+                raise InvalidDatasetNameError(
+                    f"Character {c} is reserved and not allowed in dataset name"
+                )
+
     @classmethod
     def parse(  # noqa: PLR0913
         cls,
-        id: int,
+        namespace_id: int,
+        namespace_uuid: str,
+        namespace_name: str,
+        namespace_description: Optional[str],
+        namespace_created_at: datetime,
+        project_id: int,
+        project_uuid: str,
+        project_name: str,
+        project_description: Optional[str],
+        project_created_at: datetime,
+        project_namespace_id: int,
+        dataset_id: int,
+        dataset_project_id: int,
         name: str,
         description: Optional[str],
         attrs: str,
@@ -400,6 +454,23 @@ class DatasetRecord:
             json.loads(version_schema) if version_schema else {}
         )
 
+        namespace = Namespace(
+            namespace_id,
+            namespace_uuid,
+            namespace_name,
+            namespace_description,
+            namespace_created_at,
+        )
+
+        project = Project(
+            project_id,
+            project_uuid,
+            project_name,
+            project_description,
+            project_created_at,
+            namespace,
+        )
+
         dataset_version = DatasetVersion.parse(
             version_id,
             version_uuid,
@@ -422,8 +493,9 @@ class DatasetRecord:
         )
 
         return cls(
-            id,
+            dataset_id,
             name,
+            project,
             description,
             attrs_lst,
             cls.parse_schema(schema_dct),  # type: ignore[arg-type]
@@ -448,6 +520,10 @@ class DatasetRecord:
             for c_name, c_type in self.schema.items()
         }
 
+    @property
+    def full_name(self) -> str:
+        return f"{self.project.namespace.name}.{self.project.name}.{self.name}"
+
     def get_schema(self, version: str) -> dict[str, Union[SQLType, type[SQLType]]]:
         return self.get_version(version).schema if version else self.schema
 
@@ -527,7 +603,10 @@ class DatasetRecord:
         Dataset uri example: ds://dogs@v3.0.1
         """
         identifier = self.identifier(version)
-        return f"{DATASET_PREFIX}{identifier}"
+        return (
+            f"{DATASET_PREFIX}{self.project.namespace.name}"
+            f".{self.project.name}.{identifier}"
+        )
 
     @property
     def next_version_major(self) -> str:
@@ -592,15 +671,17 @@ class DatasetRecord:
 
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "DatasetRecord":
+        project = Project.from_dict(d.pop("project"))
         versions = [DatasetVersion.from_dict(v) for v in d.pop("versions", [])]
         kwargs = {f.name: d[f.name] for f in fields(cls) if f.name in d}
-        return cls(**kwargs, versions=versions)
+        return cls(**kwargs, versions=versions, project=project)
 
 
 @dataclass
 class DatasetListRecord:
     id: int
     name: str
+    project: Project
     description: Optional[str]
     attrs: list[str]
     versions: list[DatasetListVersion]
@@ -609,7 +690,18 @@ class DatasetListRecord:
     @classmethod
     def parse(  # noqa: PLR0913
         cls,
-        id: int,
+        namespace_id: int,
+        namespace_uuid: str,
+        namespace_name: str,
+        namespace_description: Optional[str],
+        namespace_created_at: datetime,
+        project_id: int,
+        project_uuid: str,
+        project_name: str,
+        project_description: Optional[str],
+        project_created_at: datetime,
+        project_namespace_id: int,
+        dataset_id: int,
         name: str,
         description: Optional[str],
         attrs: str,
@@ -630,6 +722,23 @@ class DatasetListRecord:
     ) -> "DatasetListRecord":
         attrs_lst: list[str] = json.loads(attrs) if attrs else []
 
+        namespace = Namespace(
+            namespace_id,
+            namespace_uuid,
+            namespace_name,
+            namespace_description,
+            namespace_created_at,
+        )
+
+        project = Project(
+            project_id,
+            project_uuid,
+            project_name,
+            project_description,
+            project_created_at,
+            namespace,
+        )
+
         dataset_version = DatasetListVersion.parse(
             version_id,
             version_uuid,
@@ -647,14 +756,19 @@ class DatasetListRecord:
         )
 
         return cls(
-            id,
+            dataset_id,
             name,
+            project,
             description,
             attrs_lst,
             [dataset_version],
             created_at,
         )
 
+    @property
+    def full_name(self) -> str:
+        return f"{self.project.namespace.name}.{self.project.name}.{self.name}"
+
     def merge_versions(self, other: "DatasetListRecord") -> "DatasetListRecord":
         """Merge versions from another dataset"""
         if other.id != self.id:
@@ -691,9 +805,11 @@ class DatasetListRecord:
 
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "DatasetListRecord":
+        project = Project.from_dict(d.pop("project"))
         versions = [DatasetListVersion.parse(**v) for v in d.get("versions", [])]
         kwargs = {f.name: d[f.name] for f in fields(cls) if f.name in d}
         kwargs["versions"] = versions
+        kwargs["project"] = project
         return cls(**kwargs)
 
 

datachain/delta.py

@@ -56,11 +56,13 @@ def _get_delta_chain(
     compare: Optional[Union[str, Sequence[str]]] = None,
 ) -> "DataChain":
     """Get delta chain for processing changes between versions."""
-    source_dc = datachain.read_dataset(source_ds_name, source_ds_version)
-    source_dc_latest = datachain.read_dataset(source_ds_name, source_ds_latest_version)
+    source_dc = datachain.read_dataset(source_ds_name, version=source_ds_version)
+    source_dc_latest = datachain.read_dataset(
+        source_ds_name, version=source_ds_latest_version
+    )
 
     # Calculate diff between source versions
-    return source_dc_latest.compare(source_dc, on=on, compare=compare, deleted=False)
+    return source_dc_latest.diff(source_dc, on=on, compare=compare, deleted=False)
 
 
 def _get_retry_chain(
@@ -79,8 +81,10 @@ def _get_retry_chain(
     retry_chain = None
 
     # Read the latest version of the result dataset for retry logic
-    result_dataset = datachain.read_dataset(name, latest_version)
-    source_dc_latest = datachain.read_dataset(source_ds_name, source_ds_latest_version)
+    result_dataset = datachain.read_dataset(name, version=latest_version)
+    source_dc_latest = datachain.read_dataset(
+        source_ds_name, version=source_ds_latest_version
+    )
 
     # Handle error records if delta_retry is a string (column name)
     if isinstance(delta_retry, str):
@@ -232,8 +236,8 @@ def delta_retry_update(
     if processing_chain is None or (processing_chain and processing_chain.empty):
         return None, None, False
 
-    latest_dataset = datachain.read_dataset(name, latest_version)
-    compared_chain = latest_dataset.compare(
+    latest_dataset = datachain.read_dataset(name, version=latest_version)
+    compared_chain = latest_dataset.diff(
         processing_chain,
         on=right_on or on,
         added=True,

datachain/error.py

@@ -2,10 +2,42 @@ class DataChainError(RuntimeError):
     pass
 
 
+class InvalidDatasetNameError(RuntimeError):
+    pass
+
+
+class InvalidNamespaceNameError(RuntimeError):
+    pass
+
+
+class InvalidProjectNameError(RuntimeError):
+    pass
+
+
 class NotFoundError(Exception):
     pass
 
 
+class NamespaceNotFoundError(NotFoundError):
+    pass
+
+
+class NotAllowedError(Exception):
+    pass
+
+
+class NamespaceCreateNotAllowedError(NotAllowedError):
+    pass
+
+
+class ProjectCreateNotAllowedError(NotAllowedError):
+    pass
+
+
+class ProjectNotFoundError(NotFoundError):
+    pass
+
+
 class DatasetNotFoundError(NotFoundError):
     pass
 
@@ -53,3 +85,7 @@ class ClientError(RuntimeError):
 
 class TableMissingError(DataChainError):
     pass
+
+
+class OutdatedDatabaseSchemaError(DataChainError):
+    pass