datachain 0.34.2__tar.gz → 0.34.3__tar.gz

{datachain-0.34.2 → datachain-0.34.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.34.2
+Version: 0.34.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.34.2 → datachain-0.34.3}/src/datachain/data_storage/sqlite.py

@@ -875,8 +875,17 @@ class SQLiteWarehouse(AbstractWarehouse):
                 if isinstance(c, BinaryExpression):
                     right_left_join = add_left_rows_filter(c)
 
-        union = sqlalchemy.union(left_right_join, right_left_join).subquery()
-        return sqlalchemy.select(*union.c).select_from(union)
+        # Use CTE instead of subquery to force SQLite to materialize the result
+        # This breaks deep nesting and prevents parser stack overflow.
+        union_cte = sqlalchemy.union(left_right_join, right_left_join).cte()
+
+        return self._regenerate_system_columns(union_cte)
+
+    def _system_row_number_expr(self):
+        return func.row_number().over()
+
+    def _system_random_expr(self):
+        return self._system_row_number_expr() * 1103515245 + 12345
 
     def create_pre_udf_table(self, query: "Select") -> "Table":
         """

{datachain-0.34.2 → datachain-0.34.3}/src/datachain/data_storage/warehouse.py

@@ -246,6 +246,44 @@ class AbstractWarehouse(ABC, Serializable):
                     break  # no more results
                 offset += page_size
 
+    def _regenerate_system_columns(self, selectable):
+        """Return a SELECT that regenerates sys__id and sys__rand deterministically."""
+
+        base = selectable.subquery() if hasattr(selectable, "subquery") else selectable
+
+        system_types: dict[str, sa.types.TypeEngine] = {
+            sys_col.name: sys_col.type
+            for sys_col in self.schema.dataset_row_cls.sys_columns()
+        }
+
+        result_columns = []
+        for col in base.c:
+            if col.name == "sys__id":
+                expr = self._system_row_number_expr()
+                expr = sa.cast(expr, system_types["sys__id"])
+                result_columns.append(expr.label("sys__id"))
+            elif col.name == "sys__rand":
+                expr = self._system_random_expr()
+                expr = sa.cast(expr, system_types["sys__rand"])
+                result_columns.append(expr.label("sys__rand"))
+            else:
+                result_columns.append(col)
+
+        # Wrap in subquery to materialize window functions, then wrap again in SELECT
+        # This ensures window functions are computed before INSERT...FROM SELECT
+        inner = sa.select(*result_columns).select_from(base).subquery()
+        return sa.select(*inner.c).select_from(inner)
+
+    def _system_row_number_expr(self):
+        """Return an expression that produces deterministic row numbers."""
+
+        raise NotImplementedError
+
+    def _system_random_expr(self):
+        """Return an expression that produces deterministic random values."""
+
+        raise NotImplementedError
+
     #
     # Table Name Internal Functions
     #
@@ -923,6 +961,8 @@ class AbstractWarehouse(ABC, Serializable):
         right: "_FromClauseArgument",
         onclause: "_OnClauseArgument",
         inner: bool = True,
+        full: bool = False,
+        columns=None,
     ) -> sa.Select:
         """
         Join two tables together.

{datachain-0.34.2 → datachain-0.34.3}/src/datachain/lib/dc/datachain.py

@@ -1701,7 +1701,11 @@ class DataChain:
             )
 
         query = self._query.join(
-            right_ds._query, sqlalchemy.and_(*ops), inner, full, rname + "{name}"
+            right_ds._query,
+            sqlalchemy.and_(*ops),
+            inner,
+            full,
+            rname + "{name}",
         )
         query.feature_schema = None
         ds = self._evolve(query=query)
@@ -1989,7 +1993,8 @@ class DataChain:
         results = self.results(include_hidden=include_hidden)
         if as_object:
             df = pd.DataFrame(results, columns=columns, dtype=object)
-            return df.where(pd.notna(df), None)
+            df.where(pd.notna(df), None, inplace=True)
+            return df
         return pd.DataFrame.from_records(results, columns=columns)
 
     def show(

{datachain-0.34.2 → datachain-0.34.3}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.34.2
+Version: 0.34.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.34.2 → datachain-0.34.3}/tests/func/test_datachain_merge.py

@@ -1,6 +1,7 @@
 import pytest
 
 import datachain as dc
+from datachain import File
 from datachain.sql.types import Int
 
 
@@ -99,3 +100,46 @@ def test_merge_multiple(cloud_test_catalog, inner1, inner2, inner3):
             ("dogs/dog3", 1, signal_default_value),
             ("dogs/others/dog4", 1, signal_default_value),
         ]
+
+
+def test_full_outer_join_preserves_all_rows(test_session):
+    """Test that full outer join correctly saves all rows including right-only rows.
+    This test verifies the fix for the NULL sys__id bug where right-only rows
+    from a full outer join had NULL sys__id values
+    """
+    # Create two datasets with no overlapping file paths
+    ds1 = dc.read_values(
+        id=[1, 2, 3],
+        file=[File(path=str(i)) for i in [1, 2, 3]],
+        session=test_session,
+    ).persist()
+
+    ds2 = dc.read_values(
+        id=[5, 6, 7],
+        file=[File(path=str(i)) for i in [5, 6, 7]],
+        session=test_session,
+    ).persist()
+
+    merged = ds1.merge(ds2, on="file.path", full=True)
+
+    # Use internal method to get all records including sys columns
+    records = merged._query.to_db_records()
+
+    assert len(records) == 6
+
+    # Extract sys__id and sys__rand from records
+    sys_ids = [r["sys__id"] for r in records]
+    sys_rands = [r["sys__rand"] for r in records]
+
+    # All sys__id values should be non-NULL and unique
+    assert all(sid is not None for sid in sys_ids)
+    assert len(set(sys_ids)) == 6
+    assert all(rand is not None for rand in sys_rands)
+
+    count_before = merged.count()
+
+    # Save and verify all rows are persisted
+    merged.save("test_merge")
+    count_after = dc.read_dataset("test_merge", session=test_session).count()
+
+    assert count_before == count_after == 6