datachain 0.24.5__py3-none-any.whl → 0.25.0__py3-none-any.whl

datachain/__init__.py

@@ -7,6 +7,7 @@ from datachain.lib.dc import (
     datasets,
     delete_dataset,
     listings,
+    move_dataset,
     read_csv,
     read_database,
     read_dataset,
@@ -69,6 +70,7 @@ __all__ = [
     "is_chain_type",
     "listings",
     "metrics",
+    "move_dataset",
     "param",
     "read_csv",
     "read_database",

datachain/catalog/catalog.py

@@ -956,26 +956,9 @@ class Catalog:
         self, dataset: DatasetRecord, conn=None, **kwargs
     ) -> DatasetRecord:
         """Updates dataset fields."""
-        old_name = None
-        new_name = None
-        if "name" in kwargs and kwargs["name"] != dataset.name:
-            old_name = dataset.name
-            new_name = kwargs["name"]
-
-        dataset = self.metastore.update_dataset(dataset, conn=conn, **kwargs)
-
-        if old_name and new_name:
-            # updating name must result in updating dataset table names as well
-            for version in [v.version for v in dataset.versions]:
-                self.warehouse.rename_dataset_table(
-                    dataset,
-                    old_name,
-                    new_name,
-                    old_version=version,
-                    new_version=version,
-                )
-
-        return dataset
+        dataset_updated = self.metastore.update_dataset(dataset, conn=conn, **kwargs)
+        self.warehouse.rename_dataset_tables(dataset, dataset_updated)
+        return dataset_updated
 
     def remove_dataset_version(
         self, dataset: DatasetRecord, version: str, drop_rows: Optional[bool] = True
@@ -1555,12 +1538,14 @@ class Catalog:
             remote_ds.project.namespace.name,
             description=remote_ds.project.namespace.descr,
             uuid=remote_ds.project.namespace.uuid,
+            validate=False,
         )
         project = self.metastore.create_project(
             namespace.name,
             remote_ds.project.name,
             description=remote_ds.project.descr,
             uuid=remote_ds.project.uuid,
+            validate=False,
         )
 
         try:

datachain/data_storage/metastore.py

@@ -207,6 +207,10 @@ class AbstractMetastore(ABC, Serializable):
         It also creates project if not found and create flag is set to True.
         """
 
+    @abstractmethod
+    def get_project_by_id(self, project_id: int, conn=None) -> Project:
+        """Gets a single project by id"""
+
     @abstractmethod
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
         """Gets list of projects in some namespace or in general (in all namespaces)"""
@@ -851,6 +855,24 @@ class AbstractDBMetastore(AbstractMetastore):
             )
         return self.project_class.parse(*rows[0])
 
+    def get_project_by_id(self, project_id: int, conn=None) -> Project:
+        """Gets a single project by id"""
+        n = self._namespaces
+        p = self._projects
+
+        query = self._projects_select(
+            *(getattr(n.c, f) for f in self._namespaces_fields),
+            *(getattr(p.c, f) for f in self._projects_fields),
+        )
+        query = query.select_from(n.join(p, n.c.id == p.c.namespace_id)).where(
+            p.c.id == project_id
+        )
+
+        rows = list(self.db.execute(query, conn=conn))
+        if not rows:
+            raise ProjectNotFoundError(f"Project with id {project_id} not found.")
+        return self.project_class.parse(*rows[0])
+
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
         """
         Gets a list of projects inside some namespace, or in all namespaces
@@ -1008,6 +1030,11 @@ class AbstractDBMetastore(AbstractMetastore):
                 else:
                     values[field] = json.dumps(value)
                     dataset_values[field] = DatasetRecord.parse_schema(value)
+            elif field == "project_id":
+                if not value:
+                    raise ValueError("Cannot set empty project_id for dataset")
+                dataset_values["project"] = self.get_project_by_id(value)
+                values[field] = value
             else:
                 values[field] = value
                 dataset_values[field] = value
@@ -1017,7 +1044,9 @@ class AbstractDBMetastore(AbstractMetastore):
 
         d = self._datasets
         self.db.execute(
-            self._datasets_update().where(d.c.name == dataset.name).values(values),
+            self._datasets_update()
+            .where(d.c.name == dataset.name, d.c.project_id == dataset.project.id)
+            .values(values),
             conn=conn,
         )  # type: ignore [attr-defined]
 

datachain/data_storage/warehouse.py

@@ -356,24 +356,23 @@ class AbstractWarehouse(ABC, Serializable):
         self, dataset: DatasetRecord, version: str
     ) -> list[StorageURI]: ...
 
-    def rename_dataset_table(
-        self,
-        dataset: DatasetRecord,
-        old_name: str,
-        new_name: str,
-        old_version: str,
-        new_version: str,
+    def rename_dataset_tables(
+        self, dataset: DatasetRecord, dataset_updated: DatasetRecord
     ) -> None:
-        namespace = dataset.project.namespace.name
-        project = dataset.project.name
-        old_ds_table_name = self._construct_dataset_table_name(
-            namespace, project, old_name, old_version
-        )
-        new_ds_table_name = self._construct_dataset_table_name(
-            namespace, project, new_name, new_version
-        )
-
-        self.db.rename_table(old_ds_table_name, new_ds_table_name)
+        """
+        Renames all dataset version tables when parts of the dataset that
+        are used in constructing table name are updated.
+        If nothing important is changed, nothing will be renamed (no DB calls
+        will be made at all).
+        """
+        for version in [v.version for v in dataset_updated.versions]:
+            if not dataset.has_version(version):
+                continue
+            src = self.dataset_table_name(dataset, version)
+            dest = self.dataset_table_name(dataset_updated, version)
+            if src == dest:
+                continue
+            self.db.rename_table(src, dest)
 
     def dataset_rows_count(self, dataset: DatasetRecord, version=None) -> int:
         """Returns total number of rows in a dataset"""

datachain/lib/dc/__init__.py

@@ -1,7 +1,7 @@
 from .csv import read_csv
 from .database import read_database
 from .datachain import C, Column, DataChain
-from .datasets import datasets, delete_dataset, read_dataset
+from .datasets import datasets, delete_dataset, move_dataset, read_dataset
 from .hf import read_hf
 from .json import read_json
 from .listings import listings
@@ -22,6 +22,7 @@ __all__ = [
     "datasets",
     "delete_dataset",
     "listings",
+    "move_dataset",
     "read_csv",
     "read_database",
     "read_dataset",

datachain/lib/dc/datasets.py

@@ -361,3 +361,58 @@ def delete_dataset(
     else:
         version = None
     catalog.remove_dataset(name, ds_project, version=version, force=force)
+
+
+def move_dataset(
+    src: str,
+    dest: str,
+    session: Optional[Session] = None,
+    in_memory: bool = False,
+) -> None:
+    """Moves an entire dataset between namespaces and projects.
+
+    Args:
+        src: The source dataset name. This can be a fully qualified name that includes
+            the namespace and project, or a regular name. If a regular name is used,
+            default values will be applied. The source dataset will no longer exist
+            after the move.
+        dst: The destination dataset name. This can also be a fully qualified
+            name with a namespace and project, or just a regular name (default values
+            will be used in that case). The original dataset will be moved here.
+        session: An optional session instance. If not provided, the default session
+            will be used.
+        in_memory: If True, creates an in-memory session. Defaults to False.
+
+    Returns:
+        None
+
+    Examples:
+        ```python
+        import datachain as dc
+        dc.move_dataset("cats", "new_cats")
+        ```
+
+        ```python
+        import datachain as dc
+        dc.move_dataset("dev.animals.cats", "prod.animals.cats")
+        ```
+    """
+    session = Session.get(session, in_memory=in_memory)
+    catalog = session.catalog
+
+    namespace, project, name = catalog.get_full_dataset_name(src)
+    dest_namespace, dest_project, dest_name = catalog.get_full_dataset_name(dest)
+
+    dataset = catalog.get_dataset(
+        name, catalog.metastore.get_project(project, namespace)
+    )
+
+    catalog.update_dataset(
+        dataset,
+        name=dest_name,
+        project_id=catalog.metastore.get_project(
+            dest_project,
+            dest_namespace,
+            create=catalog.metastore.project_allowed_to_create,
+        ).id,
+    )

{datachain-0.24.5.dist-info → datachain-0.25.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.24.5
+Version: 0.25.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.24.5.dist-info → datachain-0.25.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-datachain/__init__.py,sha256=gGeaUZXIGQIMCllVXCyDinLfW6oIn33vlK1bXfCAJjI,1578
+datachain/__init__.py,sha256=ofXacfzLKYzTqU1oyHz5xZi1L4skQCoJdUMC4YARenk,1616
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=RH_jFwJcTXxhEFomaI9yL6S3Onau6NZ6FSKfKFGtrJE,9689
 datachain/cache.py,sha256=ESVRaCJXEThMIfGEFVHx6wJPOZA7FYk9V6WxjyuqUBY,3626
@@ -21,7 +21,7 @@ datachain/studio.py,sha256=bLok-eJNFRHQScEyAyA_Fas52dmijd5r-73KudWxV4k,13337
 datachain/telemetry.py,sha256=0A4IOPPp9VlP5pyW9eBfaTK3YhHGzHl7dQudQjUAx9A,994
 datachain/utils.py,sha256=DNqOi-Ydb7InyWvD9m7_yailxz6-YGpZzh00biQaHNo,15305
 datachain/catalog/__init__.py,sha256=cMZzSz3VoUi-6qXSVaHYN-agxQuAcz2XSqnEPZ55crE,353
-datachain/catalog/catalog.py,sha256=3QwiljkEHWm5xNmvqT3ey_JvKS8viiJgGWwhbqWIH0M,65996
+datachain/catalog/catalog.py,sha256=QTWCXy75iWo-0MCXyfV_WbsKeZ1fpLpvL8d60rxn1ws,65528
 datachain/catalog/datasource.py,sha256=IkGMh0Ttg6Q-9DWfU_H05WUnZepbGa28HYleECi6K7I,1353
 datachain/catalog/loader.py,sha256=UXjYD6BNRoupPvkiz3-b04jepXhtLHCA4gzKFnXxOtQ,5987
 datachain/cli/__init__.py,sha256=WvBqnwjG8Wp9xGCn-4eqfoZ3n7Sj1HJemCi4MayJh_c,8221
@@ -49,11 +49,11 @@ datachain/client/s3.py,sha256=6DNVGLg-woPS1DVlYVX2rIlunNblsuxyOnI1rSzhW3k,7515
 datachain/data_storage/__init__.py,sha256=9Wit-oe5P46V7CJQTD0BJ5MhOa2Y9h3ddJ4VWTe-Lec,273
 datachain/data_storage/db_engine.py,sha256=n8ojCbvVMPY2e3SG8fUaaD0b9GkVfpl_Naa_6EiHfWg,3788
 datachain/data_storage/job.py,sha256=9r0OGwh22bHNIvLHqg8_-eJSP1YYB-BN5HOla5TdCxw,402
-datachain/data_storage/metastore.py,sha256=af7TsMHbANXmWKIu-LzQmsZpNxM6-hhzZfInWx7MQXI,52667
+datachain/data_storage/metastore.py,sha256=Qw332arvhgXB4UY0yX-Hu8Vgl3smU12l6bvxrL9Q-vo,53810
 datachain/data_storage/schema.py,sha256=o3JbURKXRg3IJyIVA4QjHHkn6byRuz7avbydU2FlvNY,9897
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
 datachain/data_storage/sqlite.py,sha256=TTQjdDXUaZSr3MEaxZjDhsVIkIJqxFNA-sD25TO3m_4,30228
-datachain/data_storage/warehouse.py,sha256=2Bp2fXfcm-acwYjDWqVzGjoIQSAR4L56GPNtPcaT2gU,32418
+datachain/data_storage/warehouse.py,sha256=nhF8yfpdJpstpXnv_sj7WFzU97JkvSeqetqJQp33cyE,32563
 datachain/diff/__init__.py,sha256=-OFZzgOplqO84iWgGY7kfe60NXaWR9JRIh9T-uJboAM,9668
 datachain/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/fs/reference.py,sha256=A8McpXF0CqbXPqanXuvpKu50YLB3a2ZXA3YAPxtBXSM,914
@@ -100,11 +100,11 @@ datachain/lib/convert/python_to_sql.py,sha256=wg-O5FRKX3x3Wh8ZL1b9ntMlgf1zRO4djM
 datachain/lib/convert/sql_to_python.py,sha256=XXCBYDQFUXJIBNWkjEP944cnCfJ8GF2Tji0DLF3A_zQ,315
 datachain/lib/convert/unflatten.py,sha256=ysMkstwJzPMWUlnxn-Z-tXJR3wmhjHeSN_P-sDcLS6s,2010
 datachain/lib/convert/values_to_tuples.py,sha256=j5yZMrVUH6W7b-7yUvdCTGI7JCUAYUOzHUGPoyZXAB0,4360
-datachain/lib/dc/__init__.py,sha256=HD0NYrdy44u6kkpvgGjJcvGz-UGTHui2azghcT8ZUg0,838
+datachain/lib/dc/__init__.py,sha256=TFci5HTvYGjBesNUxDAnXaX36PnzPEUSn5a6JxB9o0U,872
 datachain/lib/dc/csv.py,sha256=q6a9BpapGwP6nwy6c5cklxQumep2fUp9l2LAjtTJr6s,4411
 datachain/lib/dc/database.py,sha256=g5M6NjYR1T0vKte-abV-3Ejnm-HqxTIMir5cRi_SziE,6051
 datachain/lib/dc/datachain.py,sha256=_FJnpgNN_b2xz39MsgeS0NTto0hzpcFPbJlaUBLcqTs,87094
-datachain/lib/dc/datasets.py,sha256=MzM7MTn90Q-dZYuMNUzJXRW1YHOpHeFHspadRjQoI70,13297
+datachain/lib/dc/datasets.py,sha256=eBhcybEeXHcQ_7RweRCh5uJyF5Ym1EEDPmD0YWYDPHw,15097
 datachain/lib/dc/hf.py,sha256=PJl2wiLjdRsMz0SYbLT-6H8b-D5i2WjeH7li8HHOk_0,2145
 datachain/lib/dc/json.py,sha256=dNijfJ-H92vU3soyR7X1IiDrWhm6yZIGG3bSnZkPdAE,2733
 datachain/lib/dc/listings.py,sha256=V379Cb-7ZyquM0w7sWArQZkzInZy4GB7QQ1ZfowKzQY,4544
@@ -157,9 +157,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.24.5.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.24.5.dist-info/METADATA,sha256=Z2iA5yN6lZ_rNmi3YqSOj-j5jvJ-HCIpy8sVQvb3vR4,13281
-datachain-0.24.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.24.5.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.24.5.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.24.5.dist-info/RECORD,,
+datachain-0.25.0.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.25.0.dist-info/METADATA,sha256=8CTAh5kMX-1sYBuh5CXD6u_gLS1yIWwWTwF1_umz7ek,13281
+datachain-0.25.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.25.0.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.25.0.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.25.0.dist-info/RECORD,,