datachain 0.24.5__tar.gz → 0.25.0__tar.gz

{datachain-0.24.5 → datachain-0.25.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.24.5
+Version: 0.25.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.24.5 → datachain-0.25.0}/docs/guide/env.md

@@ -4,7 +4,7 @@ List of environment variables used to configure DataChain behavior.
 
 ### Core Configuration
 
-- `DATACHAIN_ROOT_DIR` – Specifies the root directory where DataChain will create the `.datachain` folder to store its internal data. (default: the current working directory).
+- `DATACHAIN_ROOT_DIR` – Specifies the root directory where DataChain will create the `.datachain` folder to store its internal data. (default: user home directory).
 - `DATACHAIN_SYSTEM_CONFIG_DIR` – Overrides the system-wide configuration directory (default depends on the platform).
 - `DATACHAIN_GLOBAL_CONFIG_DIR` – Overrides the user's global configuration directory (default depends on the platform).
 - `DATACHAIN_NO_ANALYTICS` – Disables telemetry.

{datachain-0.24.5 → datachain-0.25.0}/src/datachain/__init__.py

@@ -7,6 +7,7 @@ from datachain.lib.dc import (
     datasets,
     delete_dataset,
     listings,
+    move_dataset,
     read_csv,
     read_database,
     read_dataset,
@@ -69,6 +70,7 @@ __all__ = [
     "is_chain_type",
     "listings",
     "metrics",
+    "move_dataset",
     "param",
     "read_csv",
     "read_database",

{datachain-0.24.5 → datachain-0.25.0}/src/datachain/catalog/catalog.py

@@ -956,26 +956,9 @@ class Catalog:
         self, dataset: DatasetRecord, conn=None, **kwargs
     ) -> DatasetRecord:
         """Updates dataset fields."""
-        old_name = None
-        new_name = None
-        if "name" in kwargs and kwargs["name"] != dataset.name:
-            old_name = dataset.name
-            new_name = kwargs["name"]
-
-        dataset = self.metastore.update_dataset(dataset, conn=conn, **kwargs)
-
-        if old_name and new_name:
-            # updating name must result in updating dataset table names as well
-            for version in [v.version for v in dataset.versions]:
-                self.warehouse.rename_dataset_table(
-                    dataset,
-                    old_name,
-                    new_name,
-                    old_version=version,
-                    new_version=version,
-                )
-
-        return dataset
+        dataset_updated = self.metastore.update_dataset(dataset, conn=conn, **kwargs)
+        self.warehouse.rename_dataset_tables(dataset, dataset_updated)
+        return dataset_updated
 
     def remove_dataset_version(
         self, dataset: DatasetRecord, version: str, drop_rows: Optional[bool] = True
@@ -1555,12 +1538,14 @@ class Catalog:
             remote_ds.project.namespace.name,
             description=remote_ds.project.namespace.descr,
             uuid=remote_ds.project.namespace.uuid,
+            validate=False,
         )
         project = self.metastore.create_project(
             namespace.name,
             remote_ds.project.name,
             description=remote_ds.project.descr,
             uuid=remote_ds.project.uuid,
+            validate=False,
         )
 
         try:

{datachain-0.24.5 → datachain-0.25.0}/src/datachain/data_storage/metastore.py

@@ -207,6 +207,10 @@ class AbstractMetastore(ABC, Serializable):
         It also creates project if not found and create flag is set to True.
         """
 
+    @abstractmethod
+    def get_project_by_id(self, project_id: int, conn=None) -> Project:
+        """Gets a single project by id"""
+
     @abstractmethod
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
         """Gets list of projects in some namespace or in general (in all namespaces)"""
@@ -851,6 +855,24 @@ class AbstractDBMetastore(AbstractMetastore):
             )
         return self.project_class.parse(*rows[0])
 
+    def get_project_by_id(self, project_id: int, conn=None) -> Project:
+        """Gets a single project by id"""
+        n = self._namespaces
+        p = self._projects
+
+        query = self._projects_select(
+            *(getattr(n.c, f) for f in self._namespaces_fields),
+            *(getattr(p.c, f) for f in self._projects_fields),
+        )
+        query = query.select_from(n.join(p, n.c.id == p.c.namespace_id)).where(
+            p.c.id == project_id
+        )
+
+        rows = list(self.db.execute(query, conn=conn))
+        if not rows:
+            raise ProjectNotFoundError(f"Project with id {project_id} not found.")
+        return self.project_class.parse(*rows[0])
+
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
         """
         Gets a list of projects inside some namespace, or in all namespaces
@@ -1008,6 +1030,11 @@ class AbstractDBMetastore(AbstractMetastore):
                 else:
                     values[field] = json.dumps(value)
                     dataset_values[field] = DatasetRecord.parse_schema(value)
+            elif field == "project_id":
+                if not value:
+                    raise ValueError("Cannot set empty project_id for dataset")
+                dataset_values["project"] = self.get_project_by_id(value)
+                values[field] = value
             else:
                 values[field] = value
                 dataset_values[field] = value
@@ -1017,7 +1044,9 @@ class AbstractDBMetastore(AbstractMetastore):
 
         d = self._datasets
         self.db.execute(
-            self._datasets_update().where(d.c.name == dataset.name).values(values),
+            self._datasets_update()
+            .where(d.c.name == dataset.name, d.c.project_id == dataset.project.id)
+            .values(values),
             conn=conn,
         )  # type: ignore [attr-defined]
 

{datachain-0.24.5 → datachain-0.25.0}/src/datachain/data_storage/warehouse.py

@@ -356,24 +356,23 @@ class AbstractWarehouse(ABC, Serializable):
         self, dataset: DatasetRecord, version: str
     ) -> list[StorageURI]: ...
 
-    def rename_dataset_table(
-        self,
-        dataset: DatasetRecord,
-        old_name: str,
-        new_name: str,
-        old_version: str,
-        new_version: str,
+    def rename_dataset_tables(
+        self, dataset: DatasetRecord, dataset_updated: DatasetRecord
     ) -> None:
-        namespace = dataset.project.namespace.name
-        project = dataset.project.name
-        old_ds_table_name = self._construct_dataset_table_name(
-            namespace, project, old_name, old_version
-        )
-        new_ds_table_name = self._construct_dataset_table_name(
-            namespace, project, new_name, new_version
-        )
-
-        self.db.rename_table(old_ds_table_name, new_ds_table_name)
+        """
+        Renames all dataset version tables when parts of the dataset that
+        are used in constructing table name are updated.
+        If nothing important is changed, nothing will be renamed (no DB calls
+        will be made at all).
+        """
+        for version in [v.version for v in dataset_updated.versions]:
+            if not dataset.has_version(version):
+                continue
+            src = self.dataset_table_name(dataset, version)
+            dest = self.dataset_table_name(dataset_updated, version)
+            if src == dest:
+                continue
+            self.db.rename_table(src, dest)
 
     def dataset_rows_count(self, dataset: DatasetRecord, version=None) -> int:
         """Returns total number of rows in a dataset"""

{datachain-0.24.5 → datachain-0.25.0}/src/datachain/lib/dc/__init__.py

@@ -1,7 +1,7 @@
 from .csv import read_csv
 from .database import read_database
 from .datachain import C, Column, DataChain
-from .datasets import datasets, delete_dataset, read_dataset
+from .datasets import datasets, delete_dataset, move_dataset, read_dataset
 from .hf import read_hf
 from .json import read_json
 from .listings import listings
@@ -22,6 +22,7 @@ __all__ = [
     "datasets",
     "delete_dataset",
     "listings",
+    "move_dataset",
     "read_csv",
     "read_database",
     "read_dataset",

{datachain-0.24.5 → datachain-0.25.0}/src/datachain/lib/dc/datasets.py

@@ -361,3 +361,58 @@ def delete_dataset(
     else:
         version = None
     catalog.remove_dataset(name, ds_project, version=version, force=force)
+
+
+def move_dataset(
+    src: str,
+    dest: str,
+    session: Optional[Session] = None,
+    in_memory: bool = False,
+) -> None:
+    """Moves an entire dataset between namespaces and projects.
+
+    Args:
+        src: The source dataset name. This can be a fully qualified name that includes
+            the namespace and project, or a regular name. If a regular name is used,
+            default values will be applied. The source dataset will no longer exist
+            after the move.
+        dst: The destination dataset name. This can also be a fully qualified
+            name with a namespace and project, or just a regular name (default values
+            will be used in that case). The original dataset will be moved here.
+        session: An optional session instance. If not provided, the default session
+            will be used.
+        in_memory: If True, creates an in-memory session. Defaults to False.
+
+    Returns:
+        None
+
+    Examples:
+        ```python
+        import datachain as dc
+        dc.move_dataset("cats", "new_cats")
+        ```
+
+        ```python
+        import datachain as dc
+        dc.move_dataset("dev.animals.cats", "prod.animals.cats")
+        ```
+    """
+    session = Session.get(session, in_memory=in_memory)
+    catalog = session.catalog
+
+    namespace, project, name = catalog.get_full_dataset_name(src)
+    dest_namespace, dest_project, dest_name = catalog.get_full_dataset_name(dest)
+
+    dataset = catalog.get_dataset(
+        name, catalog.metastore.get_project(project, namespace)
+    )
+
+    catalog.update_dataset(
+        dataset,
+        name=dest_name,
+        project_id=catalog.metastore.get_project(
+            dest_project,
+            dest_namespace,
+            create=catalog.metastore.project_allowed_to_create,
+        ).id,
+    )

{datachain-0.24.5 → datachain-0.25.0}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.24.5
+Version: 0.25.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.24.5 → datachain-0.25.0}/tests/conftest.py

@@ -576,6 +576,12 @@ def mock_allowed_to_create_namespace(allow_create_namespace):
             yield
 
 
+@pytest.fixture
+def mock_is_local_dataset():
+    with patch.object(AbstractMetastore, "is_local_dataset", return_value=True):
+        yield
+
+
 @pytest.fixture
 def project(test_session):
     return dc.create_project("dev", "animals", "Animals project")

{datachain-0.24.5 → datachain-0.25.0}/tests/func/test_datasets.py

@@ -11,12 +11,13 @@ from datachain.dataset import DatasetDependencyType, DatasetStatus
 from datachain.error import (
     DatasetInvalidVersionError,
     DatasetNotFoundError,
+    ProjectNotFoundError,
 )
 from datachain.lib.file import File
 from datachain.lib.listing import parse_listing_uri
 from datachain.query.dataset import DatasetQuery
 from datachain.sql.types import Float32, Int, Int64
-from tests.utils import assert_row_names, dataset_dependency_asdict
+from tests.utils import assert_row_names, dataset_dependency_asdict, table_row_count
 
 FILE_SCHEMA = {
     f"file__{name}": _type if _type != Int else Int64
@@ -169,14 +170,6 @@ def test_get_dataset(cloud_test_catalog, dogs_dataset):
         catalog.get_dataset("wrong name", dogs_dataset.project)
 
 
-# Returns None if the table does not exist
-def get_table_row_count(db, table_name):
-    if not db.has_table(table_name):
-        return None
-    query = sa.select(sa.func.count()).select_from(sa.table(table_name))
-    return next(db.execute(query), (None,))[0]
-
-
 def test_create_dataset_from_sources(listed_bucket, cloud_test_catalog, project):
     dataset_name = uuid.uuid4().hex
     src_uri = cloud_test_catalog.src_uri
@@ -327,7 +320,7 @@ def test_remove_dataset(cloud_test_catalog, dogs_dataset):
         catalog.get_dataset(dogs_dataset.name, dogs_dataset.project)
 
     dataset_table_name = catalog.warehouse.dataset_table_name(dogs_dataset, "1.0.0")
-    assert get_table_row_count(catalog.warehouse.db, dataset_table_name) is None
+    assert table_row_count(catalog.warehouse.db, dataset_table_name) is None
 
     assert (
         catalog.metastore.get_direct_dataset_dependencies(dogs_dataset, "1.0.0") == []
@@ -391,14 +384,108 @@ def test_edit_dataset(cloud_test_catalog, dogs_dataset):
     old_dataset_table_name = catalog.warehouse.dataset_table_name(dogs_dataset, "1.0.0")
     new_dataset_table_name = catalog.warehouse.dataset_table_name(dataset, "1.0.0")
 
-    assert get_table_row_count(catalog.warehouse.db, old_dataset_table_name) is None
-    expected_table_row_count = get_table_row_count(
+    assert table_row_count(catalog.warehouse.db, old_dataset_table_name) is None
+    expected_table_row_count = table_row_count(
         catalog.warehouse.db, new_dataset_table_name
     )
     assert expected_table_row_count
     assert dataset.get_version("1.0.0").num_objects == expected_table_row_count
 
 
+@pytest.mark.parametrize(
+    "old_name,new_name",
+    [
+        ("old.old.numbers", "new.new.numbers"),
+        ("old.old.numbers", "new.new.numbers_new"),
+        ("old.old.numbers", "old.new.numbers"),
+        ("old.old.numbers", "old.old.numbers"),
+        ("numbers", "numbers2"),
+        ("numbers", "numbers"),
+    ],
+)
+def test_move_dataset(
+    test_session,
+    old_name,
+    new_name,
+    mock_is_local_dataset,
+):
+    catalog = test_session.catalog
+
+    # create 2 versions of dataset in old project
+    for _ in range(2):
+        (dc.read_values(num=[1, 2, 3], session=test_session).save(old_name))
+
+    dataset = dc.read_dataset(old_name).dataset
+
+    dc.move_dataset(old_name, new_name, session=test_session)
+
+    if old_name != new_name:
+        # check that old dataset doesn't exist any more
+        with pytest.raises(DatasetNotFoundError):
+            dc.read_dataset(old_name).save("wrong")
+
+    dataset_updated = dc.read_dataset(new_name).dataset
+
+    # check if dataset tables are renamed correctly as well
+    for version in [v.version for v in dataset.versions]:
+        old_table_name = catalog.warehouse.dataset_table_name(dataset, version)
+        new_table_name = catalog.warehouse.dataset_table_name(dataset_updated, version)
+        if old_name == new_name:
+            assert old_table_name == new_table_name
+        else:
+            assert table_row_count(catalog.warehouse.db, old_table_name) is None
+
+        assert table_row_count(catalog.warehouse.db, new_table_name) == 3
+
+
+def test_move_dataset_then_save_into(test_session):
+    old_name = "old.old.numbers"
+    new_name = "new.new.numbers"
+
+    # create 2 versions of dataset in old project
+    for _ in range(2):
+        dc.read_values(num=[1, 2, 3], session=test_session).save(old_name)
+
+    dc.move_dataset(old_name, new_name, session=test_session)
+    dc.read_values(num=[1, 2, 3], session=test_session).save(new_name)
+
+    ds = dc.datasets(column="dataset", session=test_session)
+    datasets = [
+        d
+        for d in ds.to_values("dataset")
+        if d.name == "numbers" and d.project == "new" and d.namespace == "new"
+    ]
+
+    assert len(datasets) == 3
+
+
+def test_move_dataset_wrong_old_project(test_session, project):
+    dc.read_values(num=[1, 2, 3], session=test_session).save("old.old.numbers")
+
+    with pytest.raises(ProjectNotFoundError):
+        dc.move_dataset("wrong.wrong.numbers", "new.new.numbers", session=test_session)
+
+
+def test_move_dataset_error_in_session_moved_dataset_removed(catalog):
+    from datachain.query.session import Session
+
+    old_name = "old.old.numbers"
+    new_name = "new.new.numbers"
+
+    with pytest.raises(DatasetNotFoundError):
+        with Session("new", catalog=catalog) as test_session:
+            dc.read_values(num=[1, 2, 3]).save("aa")
+            dc.read_values(num=[1, 2, 3], session=test_session).save(old_name)
+            dc.move_dataset(old_name, new_name, session=test_session)
+
+            # throws DatasetNotFoundError
+            dc.read_dataset("wrong", session=test_session)
+
+    ds = dc.datasets(column="dataset")
+    datasets = [d for d in ds.to_values("dataset")]  # noqa: C416
+    assert len(datasets) == 0
+
+
 def test_edit_dataset_same_name(cloud_test_catalog, dogs_dataset):
     dataset_new_name = dogs_dataset.name
     catalog = cloud_test_catalog.catalog
@@ -414,12 +501,12 @@ def test_edit_dataset_same_name(cloud_test_catalog, dogs_dataset):
     old_dataset_table_name = catalog.warehouse.dataset_table_name(dogs_dataset, "1.0.0")
     new_dataset_table_name = catalog.warehouse.dataset_table_name(dataset, "1.0.0")
 
-    expected_table_row_count = get_table_row_count(
+    expected_table_row_count = table_row_count(
         catalog.warehouse.db, old_dataset_table_name
     )
     assert expected_table_row_count
     assert dataset.get_version("1.0.0").num_objects == expected_table_row_count
-    assert expected_table_row_count == get_table_row_count(
+    assert expected_table_row_count == table_row_count(
         catalog.warehouse.db, new_dataset_table_name
     )
 

{datachain-0.24.5 → datachain-0.25.0}/tests/unit/lib/test_datachain.py

@@ -43,7 +43,13 @@ from datachain.lib.udf_signature import UdfSignatureError
 from datachain.lib.utils import DataChainColumnError, DataChainParamsError
 from datachain.sql.types import Float, Int64, String
 from datachain.utils import STUDIO_URL
-from tests.utils import ANY_VALUE, df_equal, skip_if_not_sqlite, sort_df, sorted_dicts
+from tests.utils import (
+    ANY_VALUE,
+    df_equal,
+    skip_if_not_sqlite,
+    sort_df,
+    sorted_dicts,
+)
 
 DF_DATA = {
     "first_name": ["Alice", "Bob", "Charlie", "David", "Eva"],

{datachain-0.24.5 → datachain-0.25.0}/tests/utils.py

@@ -10,6 +10,7 @@ from time import sleep, time
 from typing import Any, Callable, Optional
 
 import pytest
+import sqlalchemy as sa
 from PIL import Image
 
 import datachain as dc
@@ -231,3 +232,10 @@ def sort_df(df):
 def df_equal(df1, df2) -> bool:
     """Helper function to check if two dataframes are equal regardless of ordering"""
     return sort_df(df1).equals(sort_df(df2))
+
+
+def table_row_count(db, table_name) -> Optional[int]:
+    if not db.has_table(table_name):
+        return None
+    query = sa.select(sa.func.count()).select_from(sa.table(table_name))
+    return next(db.execute(query), (None,))[0]