datachain 0.36.5__py3-none-any.whl → 0.36.6__py3-none-any.whl

datachain/delta.py

@@ -1,12 +1,16 @@
+import hashlib
 from collections.abc import Sequence
 from copy import copy
 from functools import wraps
 from typing import TYPE_CHECKING, TypeVar
 
+from attrs import frozen
+
 import datachain
 from datachain.dataset import DatasetDependency, DatasetRecord
 from datachain.error import DatasetNotFoundError
 from datachain.project import Project
+from datachain.query.dataset import Step, step_result
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -14,7 +18,9 @@ if TYPE_CHECKING:
 
     from typing_extensions import ParamSpec
 
+    from datachain.catalog import Catalog
     from datachain.lib.dc import DataChain
+    from datachain.query.dataset import QueryGenerator
 
     P = ParamSpec("P")
 
@@ -43,11 +49,38 @@ def delta_disabled(
     return _inner
 
 
+@frozen
+class _RegenerateSystemColumnsStep(Step):
+    catalog: "Catalog"
+
+    def hash_inputs(self) -> str:
+        return hashlib.sha256(b"regenerate_sys_columns").hexdigest()
+
+    def apply(self, query_generator: "QueryGenerator", temp_tables: list[str]):
+        selectable = query_generator.select()
+        regenerated = self.catalog.warehouse._regenerate_system_columns(
+            selectable,
+            keep_existing_columns=True,
+            regenerate_columns=None,
+        )
+
+        def q(*columns):
+            return regenerated.with_only_columns(*columns)
+
+        return step_result(q, regenerated.selected_columns)
+
+
 def _append_steps(dc: "DataChain", other: "DataChain"):
     """Returns cloned chain with appended steps from other chain.
     Steps are all those modification methods applied like filters, mappers etc.
     """
     dc = dc.clone()
+    dc._query.steps.append(
+        _RegenerateSystemColumnsStep(
+            catalog=dc.session.catalog,
+        )
+    )
+
     dc._query.steps += other._query.steps.copy()
     dc.signals_schema = other.signals_schema
     return dc

{datachain-0.36.5.dist-info → datachain-0.36.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.36.5
+Version: 0.36.6
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.36.5.dist-info → datachain-0.36.6.dist-info}/RECORD

@@ -5,7 +5,7 @@ datachain/cache.py,sha256=Klkc7iL_KvryeZk-UNjtByTFk7URbpb60XblalqHoYI,3604
 datachain/checkpoint.py,sha256=AOMqN_2fNuEBJDAsmc-P4L7FU444eQxTU4MCgr-XEH8,1121
 datachain/config.py,sha256=KPXef6P4NAZiEbSDMUcFwuNVTul2fZBs5xrCbyRl6Tg,4193
 datachain/dataset.py,sha256=PQwgeFPmEyN8xucaU41q371VJ1EAFXdMVbeQOVeCPFQ,24995
-datachain/delta.py,sha256=RJ0KtI8M7Jlrlp8sA6Zcnv6INaOrsbju-9e91b_zxlE,10634
+datachain/delta.py,sha256=rqQAPlNFdjpVpRyVDJ2jA_2WJ6uWEgmZZBgIRatoefY,11585
 datachain/error.py,sha256=P_5KXlfVIsW4E42JJCoFhGsgvY8la-6jXBEWbHbgqKo,1846
 datachain/hash_utils.py,sha256=FHzZS8WC4Qr_e-kZeQlfl-ilZ78IXWxj-xMZOqm8Ies,4455
 datachain/job.py,sha256=Mix3Nc56nh1iER4p_nH2h2zIymIVknn_gD-RaKPiIz4,1326
@@ -165,9 +165,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=xQzzmvQRKsPteDKbpgOxd4r971BnFaK33mcOl0FuGeI,2883
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.36.5.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.36.5.dist-info/METADATA,sha256=ZgoI7ZqtJ43SPJ6svwlk44SgmJy0YAx4BiF_hwFYopU,13762
-datachain-0.36.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.36.5.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.36.5.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.36.5.dist-info/RECORD,,
+datachain-0.36.6.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.36.6.dist-info/METADATA,sha256=GALJIHTdB1xuW7KltCGtl1wusxhZHgZH6IKd_ipzMr8,13762
+datachain-0.36.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.36.6.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.36.6.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.36.6.dist-info/RECORD,,