loom-kernel 2.1.3__tar.gz → 2.1.4__tar.gz

{loom_kernel-2.1.3 → loom_kernel-2.1.4}/CHANGELOG.md

@@ -18,6 +18,22 @@
   so an undefined name still fails at import; the model class carries the
   evaluated `__annotations__` and no `__annotate__` function.
 
+### etl
+
+- **etl:** `IntoHistory(..., mode="snapshot")` no longer reopens a row on
+  every run when a tracked column is null. Classification joined existing
+  and incoming rows on `keys + track`, and a plain equi-join never matches
+  two nulls (SQL semantics): an entity whose tracked value was null looked
+  "changed" forever, closed its open version and opened an identical one on
+  every write, stamping `valid_to` each time even though nothing changed.
+  The same null-mismatch broke `overwrite` — the refreshed value silently
+  never landed on the open row. The Polars and Spark join now treat two
+  nulls as equal (`nulls_equal` on Polars, `eqNullSafe` on
+  Spark) for both the SNAPSHOT classification joins and the overwrite join;
+  LOG mode was already null-safe and is unaffected.
+  The `etl-polars` extra now requires `polars>=1.24`, the release that
+  named that join option `nulls_equal`.
+
 ## ♻️ Refactor
 
 ### etl

{loom_kernel-2.1.3 → loom_kernel-2.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: loom-kernel
-Version: 2.1.3
+Version: 2.1.4
 Summary: Loom Python project
 License-File: LICENSE
 Classifier: Typing :: Typed
@@ -57,7 +57,7 @@ Requires-Dist: opentelemetry-api<2.0,>=1.27; extra == 'etl-polars'
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2.0,>=1.27; extra == 'etl-polars'
 Requires-Dist: opentelemetry-exporter-otlp-proto-http<2.0,>=1.27; extra == 'etl-polars'
 Requires-Dist: opentelemetry-sdk<2.0,>=1.27; extra == 'etl-polars'
-Requires-Dist: polars<2.0,>=1.0; extra == 'etl-polars'
+Requires-Dist: polars<2.0,>=1.24; extra == 'etl-polars'
 Requires-Dist: pyarrow>=14.0; extra == 'etl-polars'
 Provides-Extra: etl-spark
 Requires-Dist: delta-spark<4.0,>=3.2; extra == 'etl-spark'

{loom_kernel-2.1.3 → loom_kernel-2.1.4}/pyproject.toml

@@ -61,7 +61,7 @@ cache = [
     "omegaconf>=2.3,<3.0",
 ]
 etl-polars = [
-    "polars>=1.0,<2.0",
+    "polars>=1.24,<2.0",
     "deltalake>=1.6.2,<2.0",
     "pyarrow>=14.0",
     "fsspec>=2024.2.0",
@@ -230,7 +230,7 @@ dev = [
     "redis>=5.0.0",
     "ruff>=0.9.0,<1.0",
     "sqlalchemy>=2.0,<3.0",
-    "polars>=1.0,<2.0",
+    "polars>=1.24,<2.0",
     "clickhouse-connect>=0.9.2,<1.0",
     "pyarrow>=14.0",
     "deltalake>=1.6.2,<2.0",

{loom_kernel-2.1.3 → loom_kernel-2.1.4}/src/loom/etl/backends/polars/_historify.py

@@ -44,10 +44,12 @@ class PolarsHistorifyBackend:
         return frame.filter(pl.col(col) != expr)
 
     def anti_join(self, left: pl.DataFrame, right: pl.DataFrame, on: list[str]) -> pl.DataFrame:
-        return left.join(right, on=on, how="anti")
+        # `on` carries the tracked columns: a null must match itself or the row
+        # reads as changed and reopens on every run.
+        return left.join(right, on=on, how="anti", nulls_equal=True)
 
     def semi_join(self, left: pl.DataFrame, right: pl.DataFrame, on: list[str]) -> pl.DataFrame:
-        return left.join(right, on=on, how="semi")
+        return left.join(right, on=on, how="semi", nulls_equal=True)
 
     def union(self, frames: list[pl.DataFrame]) -> pl.DataFrame:
         return pl.concat([self._utc_datetimes(f) for f in frames], how="diagonal_relaxed")
@@ -90,7 +92,9 @@ class PolarsHistorifyBackend:
         overwrite: tuple[str, ...],
     ) -> pl.DataFrame:
         overwrite_vals = incoming.select(join_key + list(overwrite))
-        return unchanged.drop(list(overwrite)).join(overwrite_vals, on=join_key, how="left")
+        return unchanged.drop(list(overwrite)).join(
+            overwrite_vals, on=join_key, how="left", nulls_equal=True
+        )
 
     def rewind_to(
         self,

{loom_kernel-2.1.3 → loom_kernel-2.1.4}/src/loom/etl/backends/spark/_historify.py

@@ -46,10 +46,22 @@ class SparkHistorifyBackend:
         return frame.filter(F.col(col) != expr)
 
     def anti_join(self, left: DataFrame, right: DataFrame, on: list[str]) -> DataFrame:
-        return left.join(right, on=on, how="left_anti")
+        return left.join(right, on=self._null_safe_condition(left, right, on), how="left_anti")
 
     def semi_join(self, left: DataFrame, right: DataFrame, on: list[str]) -> DataFrame:
-        return left.join(right, on=on, how="left_semi")
+        return left.join(right, on=self._null_safe_condition(left, right, on), how="left_semi")
+
+    @staticmethod
+    def _null_safe_condition(left: DataFrame, right: DataFrame, on: list[str]) -> Any:
+        # eqNullSafe: a null in a tracked column must compare equal to itself,
+        # not act as SQL NULL (never matches). Otherwise a null-valued row is
+        # forever "changed" and reopened on every run. left_semi/left_anti only
+        # project the left side, so an explicit condition (vs. `on=list`) does
+        # not introduce duplicate key columns here.
+        condition = left[on[0]].eqNullSafe(right[on[0]])
+        for col in on[1:]:
+            condition = condition & left[col].eqNullSafe(right[col])
+        return condition
 
     def union(self, frames: list[DataFrame]) -> DataFrame:
         result = frames[0]
@@ -95,8 +107,21 @@ class SparkHistorifyBackend:
         join_key: list[str],
         overwrite: tuple[str, ...],
     ) -> DataFrame:
+        # eqNullSafe join_key: same null-safety as anti_join/semi_join, so an
+        # entity whose join_key includes a null track column still matches its
+        # own overwrite row. An explicit condition (vs. `on=list`) does not
+        # dedupe the key columns, so rename incoming's copy and drop it after.
+        key_rename = {k: f"__ov_key_{k}__" for k in join_key}
         overwrite_vals = incoming.select(join_key + list(overwrite))
-        return unchanged.drop(*overwrite).join(overwrite_vals, on=join_key, how="left")
+        for old, new in key_rename.items():
+            overwrite_vals = overwrite_vals.withColumnRenamed(old, new)
+
+        condition = F.col(join_key[0]).eqNullSafe(F.col(key_rename[join_key[0]]))
+        for k in join_key[1:]:
+            condition = condition & F.col(k).eqNullSafe(F.col(key_rename[k]))
+
+        joined = unchanged.drop(*overwrite).join(overwrite_vals, condition, how="left")
+        return joined.drop(*key_rename.values())
 
     def rewind_to(
         self,

{loom_kernel-2.1.3 → loom_kernel-2.1.4}/tests/unit/etl/backends/_historify_contract.py

@@ -337,6 +337,94 @@ class HistorifyContractTests:
         rows = read_table(self._uri(root))
         assert rows[0]["valid_from"] == date(2024, 1, 1)
 
+    # ------------------------------------------------------------------
+    # Null-safe track comparison (no-op when the tracked value is null)
+    # ------------------------------------------------------------------
+
+    def test_null_tracked_value_is_stable_across_runs(
+        self,
+        writer: Any,
+        root: Path,
+        make_frame: Callable,
+        read_table: Callable,
+    ) -> None:
+        """A track column that is null in both existing and incoming rows must
+        compare equal, not "changed" — a null must not reopen the vector on
+        every run."""
+        frame = make_frame([{"player_id": 1, "team_id": None}, {"player_id": 2, "team_id": "RM"}])
+        writer.write(frame, _snapshot_spec(), _Params(run_date=date(2024, 1, 1)))
+        writer.write(frame, _snapshot_spec(), _Params(run_date=date(2024, 1, 2)))
+        rows = read_table(self._uri(root))
+        player_1 = [r for r in rows if r["player_id"] == 1]
+        assert len(player_1) == 1
+        assert player_1[0]["valid_from"] == date(2024, 1, 1)
+        assert player_1[0]["valid_to"] is None
+
+    def test_overwrite_refreshes_open_row_with_null_track(
+        self,
+        writer: Any,
+        root: Path,
+        make_frame: Callable,
+        read_table: Callable,
+    ) -> None:
+        """Overwrite must still land on the open row when join_key includes a
+        null track value — the overwrite join must be null-safe too."""
+        spec = _snapshot_spec(track=("team_id",), overwrite=("email",))
+        writer.write(
+            make_frame(
+                [
+                    {"player_id": 1, "team_id": None, "email": "old@x.com"},
+                    {"player_id": 2, "team_id": "RM", "email": "rm@x.com"},
+                ]
+            ),
+            spec,
+            _Params(run_date=date(2024, 1, 1)),
+        )
+        writer.write(
+            make_frame(
+                [
+                    {"player_id": 1, "team_id": None, "email": "new@x.com"},
+                    {"player_id": 2, "team_id": "RM", "email": "rm@x.com"},
+                ]
+            ),
+            spec,
+            _Params(run_date=date(2024, 6, 1)),
+        )
+        rows = read_table(self._uri(root))
+        player_1 = [r for r in rows if r["player_id"] == 1]
+        assert len(player_1) == 1
+        assert player_1[0]["email"] == "new@x.com"
+        assert player_1[0]["valid_from"] == date(2024, 1, 1)
+
+    def test_null_to_value_transition_closes_and_opens(
+        self,
+        writer: Any,
+        root: Path,
+        make_frame: Callable,
+        read_table: Callable,
+    ) -> None:
+        """A track column flipping null -> value is a real change: it must
+        close the old (null) vector and open a new one, same as any other
+        tracked change."""
+        spec = _snapshot_spec(track=("team_id",))
+        writer.write(
+            make_frame([{"player_id": 1, "team_id": None}, {"player_id": 2, "team_id": "RM"}]),
+            spec,
+            _Params(run_date=date(2024, 1, 1)),
+        )
+        writer.write(
+            make_frame([{"player_id": 1, "team_id": "BCA"}, {"player_id": 2, "team_id": "RM"}]),
+            spec,
+            _Params(run_date=date(2024, 6, 1)),
+        )
+        rows = read_table(self._uri(root))
+        player_1 = sorted((r for r in rows if r["player_id"] == 1), key=lambda r: r["valid_from"])
+        assert len(player_1) == 2
+        assert player_1[0]["team_id"] is None
+        assert player_1[0]["valid_to"] == date(2024, 5, 31)
+        assert player_1[1]["team_id"] == "BCA"
+        assert player_1[1]["valid_to"] is None
+
     # ------------------------------------------------------------------
     # New entity
     # ------------------------------------------------------------------

{loom_kernel-2.1.3 → loom_kernel-2.1.4}/uv.lock

@@ -2361,7 +2361,7 @@ requires-dist = [
     { name = "opentelemetry-sdk", marker = "extra == 'etl-spark'", specifier = ">=1.27,<2.0" },
     { name = "opentelemetry-sdk", marker = "extra == 'logfire'", specifier = ">=1.27,<2.0" },
     { name = "opentelemetry-sdk", marker = "extra == 'streaming'", specifier = ">=1.27,<2.0" },
-    { name = "polars", marker = "extra == 'etl-polars'", specifier = ">=1.0,<2.0" },
+    { name = "polars", marker = "extra == 'etl-polars'", specifier = ">=1.24,<2.0" },
     { name = "prefect", marker = "extra == 'prefect'", specifier = ">=3.0,<4.0" },
     { name = "prometheus-client", specifier = ">=0.20" },
     { name = "prometheus-client", marker = "extra == 'prometheus'", specifier = ">=0.20" },
@@ -2420,7 +2420,7 @@ dev = [
     { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.27,<2.0" },
     { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.27,<2.0" },
     { name = "opentelemetry-sdk", specifier = ">=1.27,<2.0" },
-    { name = "polars", specifier = ">=1.0,<2.0" },
+    { name = "polars", specifier = ">=1.24,<2.0" },
     { name = "pre-commit", specifier = ">=3.8.0" },
     { name = "prefect", specifier = ">=3.0,<4.0" },
     { name = "prometheus-client", specifier = ">=0.20" },