datachain 0.37.1__tar.gz → 0.37.3__tar.gz

{datachain-0.37.1 → datachain-0.37.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.37.1
+Version: 0.37.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.37.1 → datachain-0.37.3}/src/datachain/data_storage/sqlite.py

@@ -29,7 +29,6 @@ from sqlalchemy.sql.selectable import Select
 from tqdm.auto import tqdm
 
 import datachain.sql.sqlite
-from datachain import semver
 from datachain.data_storage import AbstractDBMetastore, AbstractWarehouse
 from datachain.data_storage.db_engine import DatabaseEngine
 from datachain.data_storage.schema import DefaultSchema
@@ -692,61 +691,6 @@ class SQLiteWarehouse(AbstractWarehouse):
             for row in self.db.execute(query, cursor=cur)
         ]
 
-    def merge_dataset_rows(
-        self,
-        src: DatasetRecord,
-        dst: DatasetRecord,
-        src_version: str,
-        dst_version: str,
-    ) -> None:
-        dst_empty = False
-
-        if not self.db.has_table(self.dataset_table_name(src, src_version)):
-            # source table doesn't exist, nothing to do
-            return
-
-        src_dr = self.dataset_rows(src, src_version).table
-
-        if not self.db.has_table(self.dataset_table_name(dst, dst_version)):
-            # destination table doesn't exist, create it
-            self.create_dataset_rows_table(
-                self.dataset_table_name(dst, dst_version),
-                columns=src_dr.columns,
-            )
-            dst_empty = True
-
-        dst_dr = self.dataset_rows(dst, dst_version).table
-        merge_fields = [c.name for c in src_dr.columns if c.name != "sys__id"]
-        select_src = select(*(getattr(src_dr.columns, f) for f in merge_fields))
-
-        if dst_empty:
-            # we don't need union, but just select from source to destination
-            insert_query = sqlite.insert(dst_dr).from_select(merge_fields, select_src)
-        else:
-            dst_version_latest = None
-            # find the previous version of the destination dataset
-            dst_previous_versions = [
-                v.version
-                for v in dst.versions  # type: ignore [union-attr]
-                if semver.compare(v.version, dst_version) == -1
-            ]
-            if dst_previous_versions:
-                dst_version_latest = max(dst_previous_versions)
-
-            dst_dr_latest = self.dataset_rows(dst, dst_version_latest).table
-
-            select_dst_latest = select(
-                *(getattr(dst_dr_latest.c, f) for f in merge_fields)
-            )
-            union_query = sqlalchemy.union(select_src, select_dst_latest)
-            insert_query = (
-                sqlite.insert(dst_dr)
-                .from_select(merge_fields, union_query)
-                .prefix_with("OR IGNORE")
-            )
-
-        self.db.execute(insert_query)
-
     def prepare_entries(self, entries: "Iterable[File]") -> Iterable[dict[str, Any]]:
         return (e.model_dump() for e in entries)
 

{datachain-0.37.1 → datachain-0.37.3}/src/datachain/data_storage/warehouse.py

@@ -371,21 +371,6 @@ class AbstractWarehouse(ABC, Serializable):
         table = sa.Table(table_name, self.db.metadata)
         self.db.drop_table(table, if_exists=if_exists)
 
-    @abstractmethod
-    def merge_dataset_rows(
-        self,
-        src: "DatasetRecord",
-        dst: "DatasetRecord",
-        src_version: str,
-        dst_version: str,
-    ) -> None:
-        """
-        Merges source dataset rows and current latest destination dataset rows
-        into a new rows table created for new destination dataset version.
-        Note that table for new destination version must be created upfront.
-        Merge results should not contain duplicates.
-        """
-
     def dataset_rows_select(
         self,
         query: sa.Select,

{datachain-0.37.1 → datachain-0.37.3}/src/datachain/delta.py

@@ -1,16 +1,12 @@
-import hashlib
 from collections.abc import Sequence
 from copy import copy
 from functools import wraps
 from typing import TYPE_CHECKING, TypeVar
 
-from attrs import frozen
-
 import datachain
 from datachain.dataset import DatasetDependency, DatasetRecord
 from datachain.error import DatasetNotFoundError
 from datachain.project import Project
-from datachain.query.dataset import Step, step_result
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -18,9 +14,7 @@ if TYPE_CHECKING:
 
     from typing_extensions import ParamSpec
 
-    from datachain.catalog import Catalog
     from datachain.lib.dc import DataChain
-    from datachain.query.dataset import QueryGenerator
 
     P = ParamSpec("P")
 
@@ -49,38 +43,11 @@ def delta_disabled(
     return _inner
 
 
-@frozen
-class _RegenerateSystemColumnsStep(Step):
-    catalog: "Catalog"
-
-    def hash_inputs(self) -> str:
-        return hashlib.sha256(b"regenerate_sys_columns").hexdigest()
-
-    def apply(self, query_generator: "QueryGenerator", temp_tables: list[str]):
-        selectable = query_generator.select()
-        regenerated = self.catalog.warehouse._regenerate_system_columns(
-            selectable,
-            keep_existing_columns=True,
-            regenerate_columns=None,
-        )
-
-        def q(*columns):
-            return regenerated.with_only_columns(*columns)
-
-        return step_result(q, regenerated.selected_columns)
-
-
 def _append_steps(dc: "DataChain", other: "DataChain"):
     """Returns cloned chain with appended steps from other chain.
     Steps are all those modification methods applied like filters, mappers etc.
     """
     dc = dc.clone()
-    dc._query.steps.append(
-        _RegenerateSystemColumnsStep(
-            catalog=dc.session.catalog,
-        )
-    )
-
     dc._query.steps += other._query.steps.copy()
     dc.signals_schema = other.signals_schema
     return dc
@@ -150,7 +117,9 @@ def _get_retry_chain(
         error_records = result_dataset.filter(C(delta_retry) != "")
         error_source_records = source_dc.merge(
             error_records, on=on, right_on=right_on, inner=True
-        ).select(*list(source_dc.signals_schema.values))
+        ).select(
+            *list(source_dc.signals_schema.clone_without_sys_signals().values.keys())
+        )
         retry_chain = error_source_records
 
     # Handle missing records if delta_retry is True

{datachain-0.37.1 → datachain-0.37.3}/src/datachain/lib/dc/datachain.py

@@ -1697,14 +1697,13 @@ class DataChain:
         query.feature_schema = None
         ds = self._evolve(query=query)
 
+        # Note: merge drops sys signals from both sides, make sure to not include it
+        # in the resulting schema
         signals_schema = self.signals_schema.clone_without_sys_signals()
         right_signals_schema = right_ds.signals_schema.clone_without_sys_signals()
 
         ds.signals_schema = signals_schema.merge(right_signals_schema, rname)
 
-        if not full:
-            ds.signals_schema = SignalSchema({"sys": Sys}) | ds.signals_schema
-
         return ds
 
     @delta_disabled

{datachain-0.37.1 → datachain-0.37.3}/src/datachain/lib/dc/datasets.py

@@ -200,6 +200,10 @@ def read_dataset(
         signals_schema |= SignalSchema.deserialize(query.feature_schema)
     else:
         signals_schema |= SignalSchema.from_column_types(query.column_types or {})
+
+    if delta:
+        signals_schema = signals_schema.clone_without_sys_signals()
+
     chain = DataChain(query, _settings, signals_schema)
 
     if delta:

{datachain-0.37.1 → datachain-0.37.3}/src/datachain/lib/dc/storage.py

@@ -187,6 +187,12 @@ def read_storage(
             project=listing_project_name,
             session=session,
             settings=settings,
+            delta=delta,
+            delta_on=delta_on,
+            delta_result_on=delta_result_on,
+            delta_compare=delta_compare,
+            delta_retry=delta_retry,
+            delta_unsafe=delta_unsafe,
         )
         dc._query.update = update
         dc.signals_schema = dc.signals_schema.mutate({f"{column}": file_type})
@@ -252,13 +258,4 @@ def read_storage(
 
     assert storage_chain is not None
 
-    if delta:
-        storage_chain = storage_chain._as_delta(
-            on=delta_on,
-            right_on=delta_result_on,
-            compare=delta_compare,
-            delta_retry=delta_retry,
-            delta_unsafe=delta_unsafe,
-        )
-
     return storage_chain

{datachain-0.37.1 → datachain-0.37.3}/src/datachain/query/dataset.py

@@ -1065,7 +1065,7 @@ class SQLJoin(Step):
         q1 = self.get_query(self.query1, temp_tables)
         q2 = self.get_query(self.query2, temp_tables)
 
-        q1_columns = _drop_system_columns(q1.c) if self.full else list(q1.c)
+        q1_columns = _drop_system_columns(q1.c)
         q1_column_names = {c.name for c in q1_columns}
 
         q2_columns = []

{datachain-0.37.1 → datachain-0.37.3}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.37.1
+Version: 0.37.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.37.1 → datachain-0.37.3}/tests/func/test_delta.py

@@ -314,17 +314,66 @@ def test_delta_replay_regenerates_system_columns(test_session):
 
     build_chain(delta=False).save(result_name)
 
-    build_chain(delta=True).save(
-        result_name,
-        delta=True,
-        delta_on="measurement_id",
-    )
+    build_chain(delta=True).save(result_name)
 
     assert set(
         dc.read_dataset(result_name, session=test_session).to_values("measurement_id")
     ) == {1, 2}
 
 
+def test_storage_delta_replay_regenerates_system_columns(test_session, tmp_dir):
+    data_dir = tmp_dir / f"regen_storage_{uuid.uuid4().hex[:8]}"
+    data_dir.mkdir()
+    storage_uri = data_dir.as_uri()
+    result_name = f"regen_storage_result_{uuid.uuid4().hex[:8]}"
+
+    def write_payload(index: int) -> None:
+        (data_dir / f"item{index}.txt").write_text(f"payload-{index}")
+
+    write_payload(1)
+    write_payload(2)
+
+    def build_chain(delta: bool):
+        read_kwargs = {"session": test_session, "update": True}
+        if delta:
+            read_kwargs |= {
+                "delta": True,
+                "delta_on": ["file.path"],
+                "delta_result_on": ["file.path"],
+            }
+
+        def get_measurement_id(file: File) -> int:
+            match = re.search(r"item(\d+)\.txt$", file.path)
+            assert match
+            return int(match.group(1))
+
+        def get_num(file: File) -> int:
+            return get_measurement_id(file)
+
+        chain = dc.read_storage(storage_uri, **read_kwargs)
+        return (
+            chain.mutate(num=1)
+            .select_except("num")
+            .map(measurement_id=get_measurement_id)
+            .map(err=lambda file: "")
+            .map(num=get_num)
+            .filter(C.err == "")
+            .select_except("err")
+            .map(double=lambda num: num * 2, output=int)
+            .select_except("num")
+        )
+
+    build_chain(delta=False).save(result_name)
+
+    write_payload(3)
+
+    build_chain(delta=True).save(result_name)
+
+    assert set(
+        dc.read_dataset(result_name, session=test_session).to_values("measurement_id")
+    ) == {1, 2, 3}
+
+
 def test_delta_update_from_storage(test_session, tmp_dir, tmp_path):
     ds_name = "delta_ds"
     path = tmp_dir.as_uri()

{datachain-0.37.1 → datachain-0.37.3}/tests/unit/lib/test_datachain_merge.py

@@ -140,7 +140,7 @@ def test_merge_similar_objects(test_session):
     rname = "qq"
     ch = ch1.merge(ch2, "emp.person.name", rname=rname)
 
-    assert list(ch.signals_schema.values.keys()) == ["sys", "emp", rname + "emp"]
+    assert list(ch.signals_schema.values.keys()) == ["emp", rname + "emp"]
 
     empl = list(ch.to_list())
     assert len(empl) == 4
@@ -175,7 +175,7 @@ def test_merge_similar_objects_in_memory():
     assert ch.session.catalog.metastore.db.db_file == ":memory:"
     assert ch.session.catalog.warehouse.db.db_file == ":memory:"
 
-    assert list(ch.signals_schema.values.keys()) == ["sys", "emp", rname + "emp"]
+    assert list(ch.signals_schema.values.keys()) == ["emp", rname + "emp"]
 
     empl = list(ch.to_list())
     assert len(empl) == 4
@@ -198,7 +198,6 @@ def test_merge_values(test_session):
     ch = ch1.merge(ch2, "id")
 
     assert list(ch.signals_schema.values.keys()) == [
-        "sys",
         "id",
         "descr",
         "right_id",
@@ -339,3 +338,42 @@ def test_merge_on_expression(test_session):
         count += 1
 
     assert count == len(team) * len(team)
+
+
+def test_merge_with_drops_sys_columns(test_session):
+    left = dc.read_values(id=[1, 1], lval=[10, 20], session=test_session)
+    right = dc.read_values(id=[1, 1], rval=["a", "b"], session=test_session)
+
+    merged = left.merge(right, on="id")
+
+    assert "sys" not in merged.signals_schema.values
+
+    cols = merged.settings(sys=True).to_pandas(flatten=True).columns
+    assert all(not str(col).startswith("sys") for col in cols)
+
+    ds_name = "merge_left_dups_sys_check_sys"
+    merged.save(ds_name)
+
+    df_with_sys = (
+        dc.read_dataset(ds_name, session=test_session)
+        .settings(sys=True)
+        .to_pandas(flatten=True)
+    )
+
+    sys_cols = [c for c in df_with_sys.columns if str(c).startswith("sys")]
+    assert sys_cols
+
+    def _col(name: str) -> str:
+        for col in df_with_sys.columns:
+            if str(col) == f"sys.{name}":
+                return str(col)
+        raise AssertionError(f"Missing sys column for {name}")
+
+    sys_id_col = _col("id")
+    sys_rand_col = _col("rand")
+
+    sys_ids = list(df_with_sys[sys_id_col])
+    assert len(sys_ids) == len(set(sys_ids))
+
+    sys_rand = list(df_with_sys[sys_rand_col])
+    assert len(sys_rand) == len(set(sys_rand))