datachain 0.20.3__py3-none-any.whl → 0.21.0__py3-none-any.whl

datachain/data_storage/sqlite.py

@@ -3,7 +3,7 @@ import os
 import sqlite3
 from collections.abc import Iterable, Sequence
 from contextlib import contextmanager
-from functools import cached_property, wraps
+from functools import wraps
 from time import sleep
 from typing import (
     TYPE_CHECKING,
@@ -15,15 +15,7 @@ from typing import (
 )
 
 import sqlalchemy
-from sqlalchemy import (
-    Column,
-    Integer,
-    MetaData,
-    Table,
-    UniqueConstraint,
-    exists,
-    select,
-)
+from sqlalchemy import MetaData, Table, UniqueConstraint, exists, select
 from sqlalchemy.dialects import sqlite
 from sqlalchemy.schema import CreateIndex, CreateTable, DropTable
 from sqlalchemy.sql import func
@@ -38,9 +30,7 @@ from datachain.data_storage import AbstractDBMetastore, AbstractWarehouse
 from datachain.data_storage.db_engine import DatabaseEngine
 from datachain.data_storage.schema import DefaultSchema
 from datachain.dataset import DatasetRecord, StorageURI
-from datachain.error import DataChainError, OutdatedDatabaseSchemaError
-from datachain.namespace import Namespace
-from datachain.project import Project
+from datachain.error import DataChainError
 from datachain.sql.sqlite import create_user_defined_sql_functions, sqlite_dialect
 from datachain.sql.sqlite.base import load_usearch_extension
 from datachain.sql.types import SQLType
@@ -70,14 +60,6 @@ datachain.sql.sqlite.setup()
 quote_schema = sqlite_dialect.identifier_preparer.quote_schema
 quote = sqlite_dialect.identifier_preparer.quote
 
-# NOTE! This should be manually increased when we change our DB schema in codebase
-SCHEMA_VERSION = 1
-
-OUTDATED_SCHEMA_ERROR_MESSAGE = (
-    "You have an old version of the database schema. Please refer to the documentation"
-    " for more information."
-)
-
 
 def _get_in_memory_uri():
     return "file::memory:?cache=shared"
@@ -321,11 +303,6 @@ class SQLiteDatabaseEngine(DatabaseEngine):
         )
         return bool(next(self.execute(query))[0])
 
-    @property
-    def table_names(self) -> list[str]:
-        query = "SELECT name FROM sqlite_master WHERE type='table';"
-        return [r[0] for r in self.execute_str(query).fetchall()]
-
     def create_table(self, table: "Table", if_not_exists: bool = True) -> None:
         self.execute(CreateTable(table, if_not_exists=if_not_exists))
 
@@ -344,8 +321,6 @@ class SQLiteMetastore(AbstractDBMetastore):
     This is currently used for the local cli.
     """
 
-    META_TABLE = "meta"
-
     db: "SQLiteDatabaseEngine"
 
     def __init__(
@@ -367,11 +342,7 @@ class SQLiteMetastore(AbstractDBMetastore):
 
         self.db = db or SQLiteDatabaseEngine.from_db_file(db_file)
 
-        self._init_meta_table()
-        self._init_meta_schema_value()
-        self._check_schema_version()
         self._init_tables()
-        self._init_namespaces_projects()
 
     def __exit__(self, exc_type, exc_value, traceback) -> None:
         """Close connection upon exit from context manager."""
@@ -412,44 +383,8 @@ class SQLiteMetastore(AbstractDBMetastore):
         (db_class, db_args, db_kwargs) = db_clone_params
         return cls(uri=uri, db=db_class(*db_args, **db_kwargs))
 
-    @cached_property
-    def _meta(self) -> Table:
-        return Table(self.META_TABLE, self.db.metadata, *self._meta_columns())
-
-    def _meta_select(self, *columns) -> "Select":
-        if not columns:
-            return self._meta.select()
-        return select(*columns)
-
-    def _meta_insert(self) -> "Insert":
-        return sqlite.insert(self._meta)
-
-    def _init_meta_table(self) -> None:
-        """Initializes meta table"""
-        # NOTE! needs to be called before _init_tables()
-        table_names = self.db.table_names
-        if table_names and self.META_TABLE not in table_names:
-            # this will happen on first run
-            raise OutdatedDatabaseSchemaError(OUTDATED_SCHEMA_ERROR_MESSAGE)
-
-        self.db.create_table(self._meta, if_not_exists=True)
-        self.default_table_names.append(self._meta.name)
-
-    def _init_meta_schema_value(self) -> None:
-        """Inserts current schema version value if not present in meta table yet"""
-        stmt = (
-            self._meta_insert()
-            .values(id=1, schema_version=SCHEMA_VERSION)
-            .on_conflict_do_nothing(index_elements=["id"])
-        )
-        self.db.execute(stmt)
-
     def _init_tables(self) -> None:
         """Initialize tables."""
-        self.db.create_table(self._namespaces, if_not_exists=True)
-        self.default_table_names.append(self._namespaces.name)
-        self.db.create_table(self._projects, if_not_exists=True)
-        self.default_table_names.append(self._projects.name)
         self.db.create_table(self._datasets, if_not_exists=True)
         self.default_table_names.append(self._datasets.name)
         self.db.create_table(self._datasets_versions, if_not_exists=True)
@@ -459,55 +394,10 @@ class SQLiteMetastore(AbstractDBMetastore):
         self.db.create_table(self._jobs, if_not_exists=True)
         self.default_table_names.append(self._jobs.name)
 
-    def _init_namespaces_projects(self) -> None:
-        """
-        Creates local namespace and local project connected to it.
-        In local environment user cannot explicitly create other namespaces and
-        projects and all datasets user creates will be stored in those.
-        When pulling dataset from Studio, then other namespaces and projects will
-        be created implicitly though, to keep the same fully qualified name with
-        Studio dataset.
-        """
-        system_namespace = self.create_namespace(Namespace.system(), "System namespace")
-        self.create_project(system_namespace.name, Project.listing(), "Listing project")
-
-        local_namespace = self.create_namespace(Namespace.default(), "Local namespace")
-        self.create_project(local_namespace.name, Project.default(), "Local project")
-
-    def _check_schema_version(self) -> None:
-        """
-        Checks if current DB schema is up to date with latest DB model and schema
-        version. If not, OutdatedDatabaseSchemaError is raised.
-        """
-        schema_version = next(self.db.execute(self._meta_select()))[1]
-        if schema_version < SCHEMA_VERSION:
-            raise OutdatedDatabaseSchemaError(OUTDATED_SCHEMA_ERROR_MESSAGE)
-
-    #
-    # Dataset dependencies
-    #
-    @classmethod
-    def _meta_columns(cls) -> list["SchemaItem"]:
-        return [
-            Column("id", Integer, primary_key=True),
-            Column("schema_version", Integer, default=SCHEMA_VERSION),
-        ]
-
     @classmethod
     def _datasets_columns(cls) -> list["SchemaItem"]:
         """Datasets table columns."""
-        return [*super()._datasets_columns(), UniqueConstraint("project_id", "name")]
-
-    @classmethod
-    def _namespaces_columns(cls) -> list["SchemaItem"]:
-        """Datasets table columns."""
-        return [*super()._namespaces_columns(), UniqueConstraint("name")]
-
-    def _namespaces_insert(self) -> "Insert":
-        return sqlite.insert(self._namespaces)
-
-    def _projects_insert(self) -> "Insert":
-        return sqlite.insert(self._projects)
+        return [*super()._datasets_columns(), UniqueConstraint("name")]
 
     def _datasets_insert(self) -> "Insert":
         return sqlite.insert(self._datasets)
@@ -524,8 +414,6 @@ class SQLiteMetastore(AbstractDBMetastore):
 
     def _dataset_dependencies_select_columns(self) -> list["SchemaItem"]:
         return [
-            self._namespaces.c.name,
-            self._projects.c.name,
             self._datasets_dependencies.c.id,
             self._datasets_dependencies.c.dataset_id,
             self._datasets_dependencies.c.dataset_version_id,
@@ -541,26 +429,6 @@ class SQLiteMetastore(AbstractDBMetastore):
     def _jobs_insert(self) -> "Insert":
         return sqlite.insert(self._jobs)
 
-    @property
-    def is_studio(self) -> bool:
-        return False
-
-    #
-    # Namespaces
-    #
-
-    @property
-    def default_namespace_name(self):
-        return Namespace.default()
-
-    #
-    # Projects
-    #
-
-    @property
-    def default_project_name(self):
-        return Project.default()
-
 
 class SQLiteWarehouse(AbstractWarehouse):
     """
@@ -666,16 +534,16 @@ class SQLiteWarehouse(AbstractWarehouse):
     ) -> None:
         dst_empty = False
 
-        if not self.db.has_table(self.dataset_table_name(src, src_version)):
+        if not self.db.has_table(self.dataset_table_name(src.name, src_version)):
             # source table doesn't exist, nothing to do
             return
 
         src_dr = self.dataset_rows(src, src_version).table
 
-        if not self.db.has_table(self.dataset_table_name(dst, dst_version)):
+        if not self.db.has_table(self.dataset_table_name(dst.name, dst_version)):
             # destination table doesn't exist, create it
             self.create_dataset_rows_table(
-                self.dataset_table_name(dst, dst_version),
+                self.dataset_table_name(dst.name, dst_version),
                 columns=src_dr.columns,
             )
             dst_empty = True

datachain/data_storage/warehouse.py

@@ -182,7 +182,7 @@ class AbstractWarehouse(ABC, Serializable):
     ):
         version = version or dataset.latest_version
 
-        table_name = self.dataset_table_name(dataset, version)
+        table_name = self.dataset_table_name(dataset.name, version)
         return self.schema.dataset_row_cls(
             table_name,
             self.db,
@@ -254,24 +254,12 @@ class AbstractWarehouse(ABC, Serializable):
         name = parsed.path if parsed.scheme == "file" else parsed.netloc
         return parsed.scheme, name
 
-    def dataset_table_name(self, dataset: DatasetRecord, version: str) -> str:
-        return self._construct_dataset_table_name(
-            dataset.project.namespace.name,
-            dataset.project.name,
-            dataset.name,
-            version,
-        )
-
-    def _construct_dataset_table_name(
-        self, namespace: str, project: str, dataset_name: str, version: str
-    ) -> str:
+    def dataset_table_name(self, dataset_name: str, version: str) -> str:
         prefix = self.DATASET_TABLE_PREFIX
         if Client.is_data_source_uri(dataset_name):
             # for datasets that are created for bucket listing we use different prefix
             prefix = self.DATASET_SOURCE_TABLE_PREFIX
-        return (
-            f"{prefix}{namespace}_{project}_{dataset_name}_{version.replace('.', '_')}"
-        )
+        return f"{prefix}{dataset_name}_{version.replace('.', '_')}"
 
     def temp_table_name(self) -> str:
         return self.TMP_TABLE_NAME_PREFIX + _random_string(6)
@@ -299,7 +287,7 @@ class AbstractWarehouse(ABC, Serializable):
         if_exists: bool = True,
     ) -> None:
         """Drops a dataset rows table for the given dataset name."""
-        table_name = self.dataset_table_name(dataset, version)
+        table_name = self.dataset_table_name(dataset.name, version)
         table = sa.Table(table_name, self.db.metadata)
         self.db.drop_table(table, if_exists=if_exists)
 
@@ -356,20 +344,13 @@ class AbstractWarehouse(ABC, Serializable):
 
     def rename_dataset_table(
         self,
-        dataset: DatasetRecord,
         old_name: str,
         new_name: str,
         old_version: str,
         new_version: str,
     ) -> None:
-        namespace = dataset.project.namespace.name
-        project = dataset.project.name
-        old_ds_table_name = self._construct_dataset_table_name(
-            namespace, project, old_name, old_version
-        )
-        new_ds_table_name = self._construct_dataset_table_name(
-            namespace, project, new_name, new_version
-        )
+        old_ds_table_name = self.dataset_table_name(old_name, old_version)
+        new_ds_table_name = self.dataset_table_name(new_name, new_version)
 
         self.db.rename_table(old_ds_table_name, new_ds_table_name)
 
@@ -387,7 +368,7 @@ class AbstractWarehouse(ABC, Serializable):
         """
         Returns tuple with dataset stats: total number of rows and total dataset size.
         """
-        if not (self.db.has_table(self.dataset_table_name(dataset, version))):
+        if not (self.db.has_table(self.dataset_table_name(dataset.name, version))):
             return None, None
 
         file_signals = list(

datachain/dataset.py

@@ -13,9 +13,7 @@ from typing import (
 from urllib.parse import urlparse
 
 from datachain import semver
-from datachain.error import DatasetVersionNotFoundError, InvalidDatasetNameError
-from datachain.namespace import Namespace
-from datachain.project import Project
+from datachain.error import DatasetVersionNotFoundError
 from datachain.sql.types import NAME_TYPES_MAPPING, SQLType
 
 T = TypeVar("T", bound="DatasetRecord")
@@ -29,8 +27,6 @@ QUERY_DATASET_PREFIX = "ds_query_"
 LISTING_PREFIX = "lst__"
 
 DEFAULT_DATASET_VERSION = "1.0.0"
-DATASET_NAME_RESERVED_CHARS = ["."]
-DATASET_NAME_REPLACEMENT_CHAR = "_"
 
 
 # StorageURI represents a normalised URI to a valid storage location (full bucket or
@@ -61,35 +57,20 @@ def parse_dataset_uri(uri: str) -> tuple[str, Optional[str]]:
     return name, s[1]
 
 
-def create_dataset_uri(
-    name: str, namespace: str, project: str, version: Optional[str] = None
-) -> str:
+def create_dataset_uri(name: str, version: Optional[str] = None) -> str:
     """
-    Creates a dataset uri based on namespace, project, dataset name and optionally
-    version.
+    Creates a dataset uri based on dataset name and optionally version
     Example:
-        Input: dev, clothes, zalando, 3.0.1
-        Output: ds//dev.clothes.zalando@v3.0.1
+        Input: zalando, 3.0.1
+        Output: ds//zalando@v3.0.1
     """
-    uri = f"{DATASET_PREFIX}{namespace}.{project}.{name}"
+    uri = f"{DATASET_PREFIX}{name}"
     if version:
         uri += f"@v{version}"
 
     return uri
 
 
-def parse_dataset_name(name: str) -> tuple[Optional[str], Optional[str], str]:
-    """Parses dataset name and returns namespace, project and name"""
-    if not name:
-        raise ValueError("Name must be defined to parse it")
-    split = name.split(".")
-    name = split[-1]
-    project_name = split[-2] if len(split) > 1 else None
-    namespace_name = split[-3] if len(split) > 2 else None
-
-    return namespace_name, project_name, name
-
-
 class DatasetDependencyType:
     DATASET = "dataset"
     STORAGE = "storage"
@@ -97,12 +78,8 @@ class DatasetDependencyType:
 
 @dataclass
 class DatasetDependency:
-    # TODO put `DatasetRecord` instead of name + version which will
-    # simplify codebase in various places
     id: int
     type: str
-    namespace: str
-    project: str
     name: str
     version: str
     created_at: datetime
@@ -123,8 +100,6 @@ class DatasetDependency:
     @classmethod
     def parse(
         cls: builtins.type[DD],
-        namespace_name: str,
-        project_name: str,
         id: int,
         dataset_id: Optional[int],
         dataset_version_id: Optional[int],
@@ -146,8 +121,6 @@ class DatasetDependency:
                 if is_listing_dataset(dataset_name)
                 else DatasetDependencyType.DATASET
             ),
-            namespace_name,
-            project_name,
             dataset_name,
             (
                 dataset_version  # type: ignore[arg-type]
@@ -362,7 +335,6 @@ class DatasetListVersion:
 class DatasetRecord:
     id: int
     name: str
-    project: Project
     description: Optional[str]
     attrs: list[str]
     schema: dict[str, Union[SQLType, type[SQLType]]]
@@ -377,9 +349,6 @@ class DatasetRecord:
     sources: str = ""
     query_script: str = ""
 
-    def __hash__(self):
-        return hash(f"{self.id}")
-
     @staticmethod
     def parse_schema(
         ct: dict[str, Any],
@@ -389,31 +358,10 @@ class DatasetRecord:
             for c_name, c_type in ct.items()
         }
 
-    @staticmethod
-    def validate_name(name: str) -> None:
-        """Throws exception if name has reserved characters"""
-        for c in DATASET_NAME_RESERVED_CHARS:
-            if c in name:
-                raise InvalidDatasetNameError(
-                    f"Character {c} is reserved and not allowed in dataset name"
-                )
-
     @classmethod
     def parse(  # noqa: PLR0913
         cls,
-        namespace_id: int,
-        namespace_uuid: str,
-        namespace_name: str,
-        namespace_description: Optional[str],
-        namespace_created_at: datetime,
-        project_id: int,
-        project_uuid: str,
-        project_name: str,
-        project_description: Optional[str],
-        project_created_at: datetime,
-        project_namespace_id: int,
-        dataset_id: int,
-        dataset_project_id: int,
+        id: int,
         name: str,
         description: Optional[str],
         attrs: str,
@@ -452,23 +400,6 @@ class DatasetRecord:
             json.loads(version_schema) if version_schema else {}
         )
 
-        namespace = Namespace(
-            namespace_id,
-            namespace_uuid,
-            namespace_name,
-            namespace_description,
-            namespace_created_at,
-        )
-
-        project = Project(
-            project_id,
-            project_uuid,
-            project_name,
-            project_description,
-            project_created_at,
-            namespace,
-        )
-
         dataset_version = DatasetVersion.parse(
             version_id,
             version_uuid,
@@ -491,9 +422,8 @@ class DatasetRecord:
         )
 
         return cls(
-            dataset_id,
+            id,
             name,
-            project,
             description,
             attrs_lst,
             cls.parse_schema(schema_dct),  # type: ignore[arg-type]
@@ -518,10 +448,6 @@ class DatasetRecord:
             for c_name, c_type in self.schema.items()
         }
 
-    @property
-    def full_name(self) -> str:
-        return f"{self.project.namespace.name}.{self.project.name}.{self.name}"
-
     def get_schema(self, version: str) -> dict[str, Union[SQLType, type[SQLType]]]:
         return self.get_version(version).schema if version else self.schema
 
@@ -601,10 +527,7 @@ class DatasetRecord:
         Dataset uri example: ds://dogs@v3.0.1
         """
         identifier = self.identifier(version)
-        return (
-            f"{DATASET_PREFIX}{self.project.namespace.name}"
-            f".{self.project.name}.{identifier}"
-        )
+        return f"{DATASET_PREFIX}{identifier}"
 
     @property
     def next_version_major(self) -> str:
@@ -669,17 +592,15 @@ class DatasetRecord:
 
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "DatasetRecord":
-        project = Project.from_dict(d.pop("project"))
         versions = [DatasetVersion.from_dict(v) for v in d.pop("versions", [])]
         kwargs = {f.name: d[f.name] for f in fields(cls) if f.name in d}
-        return cls(**kwargs, versions=versions, project=project)
+        return cls(**kwargs, versions=versions)
 
 
 @dataclass
 class DatasetListRecord:
     id: int
     name: str
-    project: Project
     description: Optional[str]
     attrs: list[str]
     versions: list[DatasetListVersion]
@@ -688,18 +609,7 @@ class DatasetListRecord:
     @classmethod
     def parse(  # noqa: PLR0913
         cls,
-        namespace_id: int,
-        namespace_uuid: str,
-        namespace_name: str,
-        namespace_description: Optional[str],
-        namespace_created_at: datetime,
-        project_id: int,
-        project_uuid: str,
-        project_name: str,
-        project_description: Optional[str],
-        project_created_at: datetime,
-        project_namespace_id: int,
-        dataset_id: int,
+        id: int,
         name: str,
         description: Optional[str],
         attrs: str,
@@ -720,23 +630,6 @@ class DatasetListRecord:
     ) -> "DatasetListRecord":
         attrs_lst: list[str] = json.loads(attrs) if attrs else []
 
-        namespace = Namespace(
-            namespace_id,
-            namespace_uuid,
-            namespace_name,
-            namespace_description,
-            namespace_created_at,
-        )
-
-        project = Project(
-            project_id,
-            project_uuid,
-            project_name,
-            project_description,
-            project_created_at,
-            namespace,
-        )
-
         dataset_version = DatasetListVersion.parse(
             version_id,
             version_uuid,
@@ -754,19 +647,14 @@ class DatasetListRecord:
         )
 
         return cls(
-            dataset_id,
+            id,
             name,
-            project,
             description,
             attrs_lst,
             [dataset_version],
             created_at,
         )
 
-    @property
-    def full_name(self) -> str:
-        return f"{self.project.namespace.name}.{self.project.name}.{self.name}"
-
     def merge_versions(self, other: "DatasetListRecord") -> "DatasetListRecord":
         """Merge versions from another dataset"""
         if other.id != self.id:
@@ -803,11 +691,9 @@ class DatasetListRecord:
 
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "DatasetListRecord":
-        project = Project.from_dict(d.pop("project"))
         versions = [DatasetListVersion.parse(**v) for v in d.get("versions", [])]
         kwargs = {f.name: d[f.name] for f in fields(cls) if f.name in d}
         kwargs["versions"] = versions
-        kwargs["project"] = project
         return cls(**kwargs)
 
 

datachain/delta.py

@@ -56,13 +56,11 @@ def _get_delta_chain(
     compare: Optional[Union[str, Sequence[str]]] = None,
 ) -> "DataChain":
     """Get delta chain for processing changes between versions."""
-    source_dc = datachain.read_dataset(source_ds_name, version=source_ds_version)
-    source_dc_latest = datachain.read_dataset(
-        source_ds_name, version=source_ds_latest_version
-    )
+    source_dc = datachain.read_dataset(source_ds_name, source_ds_version)
+    source_dc_latest = datachain.read_dataset(source_ds_name, source_ds_latest_version)
 
     # Calculate diff between source versions
-    return source_dc_latest.diff(source_dc, on=on, compare=compare, deleted=False)
+    return source_dc_latest.compare(source_dc, on=on, compare=compare, deleted=False)
 
 
 def _get_retry_chain(
@@ -81,10 +79,8 @@ def _get_retry_chain(
     retry_chain = None
 
     # Read the latest version of the result dataset for retry logic
-    result_dataset = datachain.read_dataset(name, version=latest_version)
-    source_dc_latest = datachain.read_dataset(
-        source_ds_name, version=source_ds_latest_version
-    )
+    result_dataset = datachain.read_dataset(name, latest_version)
+    source_dc_latest = datachain.read_dataset(source_ds_name, source_ds_latest_version)
 
     # Handle error records if delta_retry is a string (column name)
     if isinstance(delta_retry, str):
@@ -236,8 +232,8 @@ def delta_retry_update(
     if processing_chain is None or (processing_chain and processing_chain.empty):
         return None, None, False
 
-    latest_dataset = datachain.read_dataset(name, version=latest_version)
-    compared_chain = latest_dataset.diff(
+    latest_dataset = datachain.read_dataset(name, latest_version)
+    compared_chain = latest_dataset.compare(
         processing_chain,
         on=right_on or on,
         added=True,

datachain/error.py

@@ -2,42 +2,10 @@ class DataChainError(RuntimeError):
     pass
 
 
-class InvalidDatasetNameError(RuntimeError):
-    pass
-
-
-class InvalidNamespaceNameError(RuntimeError):
-    pass
-
-
-class InvalidProjectNameError(RuntimeError):
-    pass
-
-
 class NotFoundError(Exception):
     pass
 
 
-class NamespaceNotFoundError(NotFoundError):
-    pass
-
-
-class NotAllowedError(Exception):
-    pass
-
-
-class NamespaceCreateNotAllowedError(NotAllowedError):
-    pass
-
-
-class ProjectCreateNotAllowedError(NotAllowedError):
-    pass
-
-
-class ProjectNotFoundError(NotFoundError):
-    pass
-
-
 class DatasetNotFoundError(NotFoundError):
     pass
 
@@ -85,7 +53,3 @@ class ClientError(RuntimeError):
 
 class TableMissingError(DataChainError):
     pass
-
-
-class OutdatedDatabaseSchemaError(DataChainError):
-    pass