datachain 0.24.6__py3-none-any.whl → 0.25.1__py3-none-any.whl

datachain/__init__.py

@@ -7,6 +7,7 @@ from datachain.lib.dc import (
     datasets,
     delete_dataset,
     listings,
+    move_dataset,
     read_csv,
     read_database,
     read_dataset,
@@ -69,6 +70,7 @@ __all__ = [
     "is_chain_type",
     "listings",
     "metrics",
+    "move_dataset",
     "param",
     "read_csv",
     "read_database",

datachain/catalog/catalog.py

@@ -956,26 +956,9 @@ class Catalog:
         self, dataset: DatasetRecord, conn=None, **kwargs
     ) -> DatasetRecord:
         """Updates dataset fields."""
-        old_name = None
-        new_name = None
-        if "name" in kwargs and kwargs["name"] != dataset.name:
-            old_name = dataset.name
-            new_name = kwargs["name"]
-
-        dataset = self.metastore.update_dataset(dataset, conn=conn, **kwargs)
-
-        if old_name and new_name:
-            # updating name must result in updating dataset table names as well
-            for version in [v.version for v in dataset.versions]:
-                self.warehouse.rename_dataset_table(
-                    dataset,
-                    old_name,
-                    new_name,
-                    old_version=version,
-                    new_version=version,
-                )
-
-        return dataset
+        dataset_updated = self.metastore.update_dataset(dataset, conn=conn, **kwargs)
+        self.warehouse.rename_dataset_tables(dataset, dataset_updated)
+        return dataset_updated
 
     def remove_dataset_version(
         self, dataset: DatasetRecord, version: str, drop_rows: Optional[bool] = True

datachain/data_storage/metastore.py

@@ -207,6 +207,10 @@ class AbstractMetastore(ABC, Serializable):
         It also creates project if not found and create flag is set to True.
         """
 
+    @abstractmethod
+    def get_project_by_id(self, project_id: int, conn=None) -> Project:
+        """Gets a single project by id"""
+
     @abstractmethod
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
         """Gets list of projects in some namespace or in general (in all namespaces)"""
@@ -851,6 +855,24 @@ class AbstractDBMetastore(AbstractMetastore):
             )
         return self.project_class.parse(*rows[0])
 
+    def get_project_by_id(self, project_id: int, conn=None) -> Project:
+        """Gets a single project by id"""
+        n = self._namespaces
+        p = self._projects
+
+        query = self._projects_select(
+            *(getattr(n.c, f) for f in self._namespaces_fields),
+            *(getattr(p.c, f) for f in self._projects_fields),
+        )
+        query = query.select_from(n.join(p, n.c.id == p.c.namespace_id)).where(
+            p.c.id == project_id
+        )
+
+        rows = list(self.db.execute(query, conn=conn))
+        if not rows:
+            raise ProjectNotFoundError(f"Project with id {project_id} not found.")
+        return self.project_class.parse(*rows[0])
+
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
         """
         Gets a list of projects inside some namespace, or in all namespaces
@@ -1008,6 +1030,11 @@ class AbstractDBMetastore(AbstractMetastore):
                 else:
                     values[field] = json.dumps(value)
                     dataset_values[field] = DatasetRecord.parse_schema(value)
+            elif field == "project_id":
+                if not value:
+                    raise ValueError("Cannot set empty project_id for dataset")
+                dataset_values["project"] = self.get_project_by_id(value)
+                values[field] = value
             else:
                 values[field] = value
                 dataset_values[field] = value
@@ -1017,7 +1044,9 @@ class AbstractDBMetastore(AbstractMetastore):
 
         d = self._datasets
         self.db.execute(
-            self._datasets_update().where(d.c.name == dataset.name).values(values),
+            self._datasets_update()
+            .where(d.c.name == dataset.name, d.c.project_id == dataset.project.id)
+            .values(values),
             conn=conn,
         )  # type: ignore [attr-defined]
 

datachain/data_storage/warehouse.py

@@ -356,24 +356,23 @@ class AbstractWarehouse(ABC, Serializable):
         self, dataset: DatasetRecord, version: str
     ) -> list[StorageURI]: ...
 
-    def rename_dataset_table(
-        self,
-        dataset: DatasetRecord,
-        old_name: str,
-        new_name: str,
-        old_version: str,
-        new_version: str,
+    def rename_dataset_tables(
+        self, dataset: DatasetRecord, dataset_updated: DatasetRecord
     ) -> None:
-        namespace = dataset.project.namespace.name
-        project = dataset.project.name
-        old_ds_table_name = self._construct_dataset_table_name(
-            namespace, project, old_name, old_version
-        )
-        new_ds_table_name = self._construct_dataset_table_name(
-            namespace, project, new_name, new_version
-        )
-
-        self.db.rename_table(old_ds_table_name, new_ds_table_name)
+        """
+        Renames all dataset version tables when parts of the dataset that
+        are used in constructing table name are updated.
+        If nothing important is changed, nothing will be renamed (no DB calls
+        will be made at all).
+        """
+        for version in [v.version for v in dataset_updated.versions]:
+            if not dataset.has_version(version):
+                continue
+            src = self.dataset_table_name(dataset, version)
+            dest = self.dataset_table_name(dataset_updated, version)
+            if src == dest:
+                continue
+            self.db.rename_table(src, dest)
 
     def dataset_rows_count(self, dataset: DatasetRecord, version=None) -> int:
         """Returns total number of rows in a dataset"""

datachain/lib/arrow.py

@@ -126,7 +126,16 @@ class ArrowGenerator(Generator):
             if isinstance(kwargs.get("format"), CsvFileFormat):
                 kwargs["format"] = "csv"
             arrow_file = ArrowRow(file=file, index=index, kwargs=kwargs)
+
+            if self.output_schema and hasattr(vals[0], "source"):
+                # if we are reading parquet file written by datachain it might have
+                # source inside of it already, so we should not duplicate it, instead
+                # we are re-creating it of the self.source flag
+                vals[0].source = arrow_file  # type: ignore[attr-defined]
+
+                return vals
             return [arrow_file, *vals]
+
         return vals
 
     def _process_non_datachain_record(

datachain/lib/dc/__init__.py

@@ -1,7 +1,7 @@
 from .csv import read_csv
 from .database import read_database
 from .datachain import C, Column, DataChain
-from .datasets import datasets, delete_dataset, read_dataset
+from .datasets import datasets, delete_dataset, move_dataset, read_dataset
 from .hf import read_hf
 from .json import read_json
 from .listings import listings
@@ -22,6 +22,7 @@ __all__ = [
     "datasets",
     "delete_dataset",
     "listings",
+    "move_dataset",
     "read_csv",
     "read_database",
     "read_dataset",

datachain/lib/dc/datasets.py

@@ -361,3 +361,58 @@ def delete_dataset(
     else:
         version = None
     catalog.remove_dataset(name, ds_project, version=version, force=force)
+
+
+def move_dataset(
+    src: str,
+    dest: str,
+    session: Optional[Session] = None,
+    in_memory: bool = False,
+) -> None:
+    """Moves an entire dataset between namespaces and projects.
+
+    Args:
+        src: The source dataset name. This can be a fully qualified name that includes
+            the namespace and project, or a regular name. If a regular name is used,
+            default values will be applied. The source dataset will no longer exist
+            after the move.
+        dst: The destination dataset name. This can also be a fully qualified
+            name with a namespace and project, or just a regular name (default values
+            will be used in that case). The original dataset will be moved here.
+        session: An optional session instance. If not provided, the default session
+            will be used.
+        in_memory: If True, creates an in-memory session. Defaults to False.
+
+    Returns:
+        None
+
+    Examples:
+        ```python
+        import datachain as dc
+        dc.move_dataset("cats", "new_cats")
+        ```
+
+        ```python
+        import datachain as dc
+        dc.move_dataset("dev.animals.cats", "prod.animals.cats")
+        ```
+    """
+    session = Session.get(session, in_memory=in_memory)
+    catalog = session.catalog
+
+    namespace, project, name = catalog.get_full_dataset_name(src)
+    dest_namespace, dest_project, dest_name = catalog.get_full_dataset_name(dest)
+
+    dataset = catalog.get_dataset(
+        name, catalog.metastore.get_project(project, namespace)
+    )
+
+    catalog.update_dataset(
+        dataset,
+        name=dest_name,
+        project_id=catalog.metastore.get_project(
+            dest_project,
+            dest_namespace,
+            create=catalog.metastore.project_allowed_to_create,
+        ).id,
+    )

datachain/lib/hf.py

@@ -11,7 +11,7 @@ try:
         Image,
         IterableDataset,
         IterableDatasetDict,
-        Sequence,
+        List,
         Value,
         load_dataset,
     )
@@ -59,7 +59,6 @@ class HFImage(DataModel):
 
 
 class HFAudio(DataModel):
-    path: str
     array: list[float]
     sampling_rate: int
 
@@ -116,26 +115,24 @@ def stream_splits(ds: Union[str, HFDatasetType], *args, **kwargs):
     return {"": ds}
 
 
-def convert_feature(val: Any, feat: Any, anno: Any) -> Any:  # noqa: PLR0911
-    if isinstance(feat, (Value, Array2D, Array3D, Array4D, Array5D)):
+def convert_feature(val: Any, feat: Any, anno: Any) -> Any:
+    if isinstance(feat, (Value, Array2D, Array3D, Array4D, Array5D, List)):
         return val
     if isinstance(feat, ClassLabel):
         return HFClassLabel(string=feat.names[val], integer=val)
-    if isinstance(feat, Sequence):
-        if isinstance(feat.feature, dict):
-            sdict = {}
-            for sname in val:
-                sfeat = feat.feature[sname]
-                sanno = anno.model_fields[sname].annotation
-                sdict[sname] = [convert_feature(v, sfeat, sanno) for v in val[sname]]
-            return anno(**sdict)
-        return val
+    if isinstance(feat, dict):
+        sdict = {}
+        for sname in val:
+            sfeat = feat[sname]
+            sanno = anno.model_fields[sname].annotation
+            sdict[sname] = [convert_feature(v, sfeat, sanno) for v in val[sname]]
+        return anno(**sdict)
     if isinstance(feat, Image):
         if isinstance(val, dict):
             return HFImage(img=val["bytes"])
         return HFImage(img=image_to_bytes(val))
     if isinstance(feat, Audio):
-        return HFAudio(**val)
+        return HFAudio(array=val["array"], sampling_rate=val["sampling_rate"])
 
 
 def get_output_schema(features: Features) -> dict[str, DataType]:
@@ -151,13 +148,13 @@ def _feature_to_chain_type(name: str, val: Any) -> DataType:  # noqa: PLR0911
         return arrow_type_mapper(val.pa_type)
     if isinstance(val, ClassLabel):
         return HFClassLabel
-    if isinstance(val, Sequence):
-        if isinstance(val.feature, dict):
-            sequence_dict = {}
-            for sname, sval in val.feature.items():
-                dtype = _feature_to_chain_type(sname, sval)
-                sequence_dict[sname] = list[dtype]  # type: ignore[valid-type]
-            return dict_to_data_model(name, sequence_dict)  # type: ignore[arg-type]
+    if isinstance(val, dict):
+        sequence_dict = {}
+        for sname, sval in val.items():
+            dtype = _feature_to_chain_type(sname, sval)
+            sequence_dict[sname] = dtype  # type: ignore[valid-type]
+        return dict_to_data_model(name, sequence_dict)  # type: ignore[arg-type]
+    if isinstance(val, List):
         return list[_feature_to_chain_type(name, val.feature)]  # type: ignore[arg-type,misc,return-value]
     if isinstance(val, Array2D):
         dtype = arrow_type_mapper(string_to_arrow(val.dtype))

{datachain-0.24.6.dist-info → datachain-0.25.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.24.6
+Version: 0.25.1
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -70,7 +70,8 @@ Provides-Extra: vector
 Requires-Dist: usearch; extra == "vector"
 Provides-Extra: hf
 Requires-Dist: numba>=0.60.0; extra == "hf"
-Requires-Dist: datasets[audio,vision]>=2.21.0; extra == "hf"
+Requires-Dist: datasets[vision]>=4.0.0; extra == "hf"
+Requires-Dist: datasets[audio]>=4.0.0; (sys_platform == "linux" or sys_platform == "darwin") and extra == "hf"
 Requires-Dist: fsspec>=2024.12.0; extra == "hf"
 Provides-Extra: video
 Requires-Dist: ffmpeg-python; extra == "video"

{datachain-0.24.6.dist-info → datachain-0.25.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-datachain/__init__.py,sha256=gGeaUZXIGQIMCllVXCyDinLfW6oIn33vlK1bXfCAJjI,1578
+datachain/__init__.py,sha256=ofXacfzLKYzTqU1oyHz5xZi1L4skQCoJdUMC4YARenk,1616
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=RH_jFwJcTXxhEFomaI9yL6S3Onau6NZ6FSKfKFGtrJE,9689
 datachain/cache.py,sha256=ESVRaCJXEThMIfGEFVHx6wJPOZA7FYk9V6WxjyuqUBY,3626
@@ -21,7 +21,7 @@ datachain/studio.py,sha256=bLok-eJNFRHQScEyAyA_Fas52dmijd5r-73KudWxV4k,13337
 datachain/telemetry.py,sha256=0A4IOPPp9VlP5pyW9eBfaTK3YhHGzHl7dQudQjUAx9A,994
 datachain/utils.py,sha256=DNqOi-Ydb7InyWvD9m7_yailxz6-YGpZzh00biQaHNo,15305
 datachain/catalog/__init__.py,sha256=cMZzSz3VoUi-6qXSVaHYN-agxQuAcz2XSqnEPZ55crE,353
-datachain/catalog/catalog.py,sha256=IYSQimZYSUPZ0Ox5zsxsSJV_9SzsjHKS5jwH4s9DUJg,66052
+datachain/catalog/catalog.py,sha256=QTWCXy75iWo-0MCXyfV_WbsKeZ1fpLpvL8d60rxn1ws,65528
 datachain/catalog/datasource.py,sha256=IkGMh0Ttg6Q-9DWfU_H05WUnZepbGa28HYleECi6K7I,1353
 datachain/catalog/loader.py,sha256=UXjYD6BNRoupPvkiz3-b04jepXhtLHCA4gzKFnXxOtQ,5987
 datachain/cli/__init__.py,sha256=WvBqnwjG8Wp9xGCn-4eqfoZ3n7Sj1HJemCi4MayJh_c,8221
@@ -49,11 +49,11 @@ datachain/client/s3.py,sha256=6DNVGLg-woPS1DVlYVX2rIlunNblsuxyOnI1rSzhW3k,7515
 datachain/data_storage/__init__.py,sha256=9Wit-oe5P46V7CJQTD0BJ5MhOa2Y9h3ddJ4VWTe-Lec,273
 datachain/data_storage/db_engine.py,sha256=n8ojCbvVMPY2e3SG8fUaaD0b9GkVfpl_Naa_6EiHfWg,3788
 datachain/data_storage/job.py,sha256=9r0OGwh22bHNIvLHqg8_-eJSP1YYB-BN5HOla5TdCxw,402
-datachain/data_storage/metastore.py,sha256=af7TsMHbANXmWKIu-LzQmsZpNxM6-hhzZfInWx7MQXI,52667
+datachain/data_storage/metastore.py,sha256=Qw332arvhgXB4UY0yX-Hu8Vgl3smU12l6bvxrL9Q-vo,53810
 datachain/data_storage/schema.py,sha256=o3JbURKXRg3IJyIVA4QjHHkn6byRuz7avbydU2FlvNY,9897
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
 datachain/data_storage/sqlite.py,sha256=TTQjdDXUaZSr3MEaxZjDhsVIkIJqxFNA-sD25TO3m_4,30228
-datachain/data_storage/warehouse.py,sha256=2Bp2fXfcm-acwYjDWqVzGjoIQSAR4L56GPNtPcaT2gU,32418
+datachain/data_storage/warehouse.py,sha256=nhF8yfpdJpstpXnv_sj7WFzU97JkvSeqetqJQp33cyE,32563
 datachain/diff/__init__.py,sha256=-OFZzgOplqO84iWgGY7kfe60NXaWR9JRIh9T-uJboAM,9668
 datachain/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/fs/reference.py,sha256=A8McpXF0CqbXPqanXuvpKu50YLB3a2ZXA3YAPxtBXSM,914
@@ -70,12 +70,12 @@ datachain/func/random.py,sha256=t7jwXsI8-hy0qAdvjAntgzy-AHtTAfozlZ1CpKR-QZE,458
 datachain/func/string.py,sha256=X9u4ip97U63RCaKRhMddoze7HgPiY3LbPRn9G06UWWo,7311
 datachain/func/window.py,sha256=ImyRpc1QI8QUSPO7KdD60e_DPVo7Ja0G5kcm6BlyMcw,1584
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/lib/arrow.py,sha256=PyLXiscZ7sVEo65CAhYXmgHh1OLSH2lvbM5dAYhE8x4,10348
+datachain/lib/arrow.py,sha256=hdEQ8I1JgNmEAaXTaqaU1qvZDi5dgtes1IC69ycthz8,10753
 datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
 datachain/lib/data_model.py,sha256=ZwBXELtqROEdLL4DmxTipnwUZmhQvMz_UVDzyf7nQ9Y,2899
 datachain/lib/dataset_info.py,sha256=7w-DoKOyIVoOtWGCgciMLcP5CiAWJB3rVI-vUDF80k0,3311
 datachain/lib/file.py,sha256=gTzJXaGIyFOrw_B4yiOEs7U23n4oAQuWDI2v9KWwp2o,33889
-datachain/lib/hf.py,sha256=gjxuStZBlKtNk3-4yYSlWZDv9zBGblOdvEy_Lwap5hA,5882
+datachain/lib/hf.py,sha256=_dCoGTv7n5cBgxhCDfZI-t3hnMCXGHd6sEsxRThcizE,5754
 datachain/lib/image.py,sha256=erWvZW5M3emnbl6_fGAOPyKm-1EKbt3vOdWPfe3Oo7U,3265
 datachain/lib/listing.py,sha256=U-2stsTEwEsq4Y80dqGfktGzkmB5-ZntnL1_rzXlH0k,7089
 datachain/lib/listing_info.py,sha256=9ua40Hw0aiQByUw3oAEeNzMavJYfW0Uhe8YdCTK-m_g,1110
@@ -100,11 +100,11 @@ datachain/lib/convert/python_to_sql.py,sha256=wg-O5FRKX3x3Wh8ZL1b9ntMlgf1zRO4djM
 datachain/lib/convert/sql_to_python.py,sha256=XXCBYDQFUXJIBNWkjEP944cnCfJ8GF2Tji0DLF3A_zQ,315
 datachain/lib/convert/unflatten.py,sha256=ysMkstwJzPMWUlnxn-Z-tXJR3wmhjHeSN_P-sDcLS6s,2010
 datachain/lib/convert/values_to_tuples.py,sha256=j5yZMrVUH6W7b-7yUvdCTGI7JCUAYUOzHUGPoyZXAB0,4360
-datachain/lib/dc/__init__.py,sha256=HD0NYrdy44u6kkpvgGjJcvGz-UGTHui2azghcT8ZUg0,838
+datachain/lib/dc/__init__.py,sha256=TFci5HTvYGjBesNUxDAnXaX36PnzPEUSn5a6JxB9o0U,872
 datachain/lib/dc/csv.py,sha256=q6a9BpapGwP6nwy6c5cklxQumep2fUp9l2LAjtTJr6s,4411
 datachain/lib/dc/database.py,sha256=g5M6NjYR1T0vKte-abV-3Ejnm-HqxTIMir5cRi_SziE,6051
 datachain/lib/dc/datachain.py,sha256=_FJnpgNN_b2xz39MsgeS0NTto0hzpcFPbJlaUBLcqTs,87094
-datachain/lib/dc/datasets.py,sha256=MzM7MTn90Q-dZYuMNUzJXRW1YHOpHeFHspadRjQoI70,13297
+datachain/lib/dc/datasets.py,sha256=eBhcybEeXHcQ_7RweRCh5uJyF5Ym1EEDPmD0YWYDPHw,15097
 datachain/lib/dc/hf.py,sha256=PJl2wiLjdRsMz0SYbLT-6H8b-D5i2WjeH7li8HHOk_0,2145
 datachain/lib/dc/json.py,sha256=dNijfJ-H92vU3soyR7X1IiDrWhm6yZIGG3bSnZkPdAE,2733
 datachain/lib/dc/listings.py,sha256=V379Cb-7ZyquM0w7sWArQZkzInZy4GB7QQ1ZfowKzQY,4544
@@ -157,9 +157,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.24.6.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.24.6.dist-info/METADATA,sha256=eymzsZv5CR7ZiJlr41202PvWYx3LWvKzUeyV1hYEVO8,13281
-datachain-0.24.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.24.6.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.24.6.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.24.6.dist-info/RECORD,,
+datachain-0.25.1.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.25.1.dist-info/METADATA,sha256=NaMV5K1wxCrOI7zW8agwmNfDMMkJJgaQ2fNX2PsuHnc,13385
+datachain-0.25.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.25.1.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.25.1.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.25.1.dist-info/RECORD,,