loom-kernel 2.21.0__tar.gz → 2.22.0__tar.gz

{loom_kernel-2.21.0 → loom_kernel-2.22.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: loom-kernel
-Version: 2.21.0
+Version: 2.22.0
 Summary: Loom Python project
 License-File: LICENSE
 Classifier: Programming Language :: Python :: 3

{loom_kernel-2.21.0 → loom_kernel-2.22.0}/src/loom/etl/__init__.py

@@ -63,6 +63,7 @@ from loom.etl.declarative import (
     FromTable,
     FromTemp,
     HistorifyDateCollisionError,
+    HistorifyDeleteMarkerError,
     HistorifyInputMode,
     HistorifyKeyConflictError,
     HistorifyRepairReport,
@@ -203,6 +204,7 @@ __all__ = [
     # SCD2 errors
     "HistorifyKeyConflictError",
     "HistorifyDateCollisionError",
+    "HistorifyDeleteMarkerError",
     "HistorifyTemporalConflictError",
     "EmptyReplaceError",
     # intermediates

{loom_kernel-2.21.0 → loom_kernel-2.22.0}/src/loom/etl/backends/_historify/_common.py

@@ -42,9 +42,20 @@ def resolve_track_cols(spec: HistorifySpec, frame_cols: list[str]) -> tuple[str,
     if spec.track is not None:
         return spec.track
     excluded = set(spec.keys) | {spec.valid_from, spec.valid_to}
+    if spec.delete_marker is not None:
+        excluded.add(spec.delete_marker)
     return tuple(c for c in frame_cols if c not in excluded)
 
 
+def log_run_cols(spec: HistorifySpec) -> list[str]:
+    """Return the columns whose change starts a new LOG run: ``track`` plus the delete marker.
+
+    Including the marker keeps a tombstone from collapsing into the run it closes.
+    """
+    marker = [spec.delete_marker] if spec.delete_marker is not None else []
+    return [*(spec.track or ()), *marker]
+
+
 def prev_period_value(eff_date: Any, spec: HistorifySpec) -> Any:
     """Return one unit before eff_date (one day or one microsecond)."""
     delta = _DATE_DELTA if spec.date_type is HistoryDateType.DATE else _TS_DELTA

{loom_kernel-2.21.0 → loom_kernel-2.22.0}/src/loom/etl/backends/_historify/_log.py

@@ -24,6 +24,9 @@ def apply_log(
     history is rebuilt from the union of existing events and incoming events.
     Incoming events win on same ``(entity key, effective date)`` conflicts, so
     a correction with different tracked values replaces the stored event.
+    With ``delete_marker`` set, every stored closure without a successor is
+    replayed as a tombstone so the rebuild keeps it closed. Removing the marker
+    later would merge those closures back into open history.
     """
     eff_col = str(spec.effective_date)
     entity_keys = list(spec.keys)
@@ -36,6 +39,12 @@ def apply_log(
         c for c in [spec.valid_to, spec.deleted_at] if c in ops.columns(existing_events)
     ]
     existing_events = ops.drop(existing_events, boundary_cols)
+    marker = spec.delete_marker
+    if marker is not None:
+        stored_tombstones = ops.stamp_col(
+            ops.closure_events(affected_keys, spec), marker, True, None
+        )
+        existing_events = ops.union([existing_events, stored_tombstones])
 
     all_events = ops.union(
         [
@@ -46,5 +55,21 @@ def apply_log(
     deduped = ops.dedup_priority(all_events, entity_keys + [eff_col], _ORIGIN_COL)
     deduped = ops.drop(deduped, [_ORIGIN_COL])
 
-    rebuilt = ops.build_log_boundaries(deduped, spec)
+    rebuilt = build_log_history(ops, deduped, spec)
     return ops.union([untouched, rebuilt])
+
+
+def build_log_history(ops: HistorifyBackend[F], events: F, spec: HistorifySpec) -> F:
+    """Build LOG history vectors from an event frame, resolving tombstones.
+
+    Tombstones take part in boundary computation as events of their own, so the
+    vector before each one closes a period earlier; the tombstone rows and the
+    ``delete_marker`` column are then dropped, leaving the entity without an
+    open vector until its next regular event.
+    """
+    marker = spec.delete_marker
+    if marker is None:
+        return ops.build_log_boundaries(events, spec)
+    flagged = ops.fill_null(events, marker, False)
+    history = ops.build_log_boundaries(flagged, spec)
+    return ops.drop(ops.filter_eq(history, marker, False, None), [marker])

{loom_kernel-2.21.0 → loom_kernel-2.22.0}/src/loom/etl/backends/_historify/_ops.py

@@ -14,6 +14,10 @@ class HistorifyBackend(Protocol[F]):
 
     def columns(self, frame: F) -> list[str]: ...
 
+    def non_boolean_dtype(self, frame: F, col: str) -> str | None:
+        """Return the dtype name of ``col`` when it is not boolean, else ``None``."""
+        ...
+
     def history_dtype(self, spec: HistorifySpec) -> Any:
         """Return the native dtype for valid_from / valid_to columns."""
         ...
@@ -40,6 +44,10 @@ class HistorifyBackend(Protocol[F]):
         """Add null column with given dtype."""
         ...
 
+    def fill_null(self, frame: F, col: str, value: Any) -> F:
+        """Replace nulls in ``col`` with a literal value."""
+        ...
+
     def rename(self, frame: F, rename_map: dict[str, str]) -> F:
         """Rename columns according to mapping."""
         ...
@@ -77,11 +85,24 @@ class HistorifyBackend(Protocol[F]):
         """Compute valid_from / valid_to from sorted event frame (LOG mode)."""
         ...
 
+    def closure_events(self, history: F, spec: HistorifySpec) -> F:
+        """Return one tombstone instant per stored vector closed without a successor.
+
+        LOG history has no gaps except those left by tombstones, so every gap is
+        read as a deletion: a vector has no successor when no later vector of its
+        entity starts within one period after its ``valid_to``. Each row carries
+        ``keys`` and the effective-date column at that next period (the
+        tombstone's instant); the caller flags it with ``delete_marker``.
+        """
+        ...
+
     def apply_delete_policy(self, deleted: F, spec: HistorifySpec, eff_date: Any) -> F: ...
 
     def ensure_soft_delete_col(self, result: F, spec: HistorifySpec) -> F: ...
 
-    def assert_unique_keys(self, frame: F, keys: list[str]) -> None: ...
+    def duplicate_keys(self, frame: F, keys: list[str]) -> str | None:
+        """Describe up to five ``keys`` combinations repeated in ``frame``, or ``None``."""
+        ...
 
     def assert_no_date_collisions(
         self, frame: F, keys: list[str], eff_col: str, spec: HistorifySpec

{loom_kernel-2.21.0 → loom_kernel-2.22.0}/src/loom/etl/backends/_historify/_transform.py

@@ -9,11 +9,13 @@ from loom.etl.backends._historify._common import (
     resolve_effective_date,
     resolve_track_cols,
 )
-from loom.etl.backends._historify._log import apply_log
+from loom.etl.backends._historify._log import apply_log, build_log_history
 from loom.etl.backends._historify._ops import HistorifyBackend
 from loom.etl.backends._historify._snapshot import apply_snapshot
 from loom.etl.declarative.target._history import (
+    HistorifyDeleteMarkerError,
     HistorifyInputMode,
+    HistorifyKeyConflictError,
     HistorifyRepairReport,
     HistorifySpec,
     HistorifyTemporalConflictError,
@@ -44,6 +46,7 @@ def scd2_transform(
         (SNAPSHOT mode with ``allow_temporal_rerun``), otherwise ``None``.
 
     Raises:
+        HistorifyDeleteMarkerError:     ``delete_marker`` column missing or not boolean.
         HistorifyKeyConflictError:      Duplicate entity state vectors.
         HistorifyDateCollisionError:    Same-date collisions in LOG mode.
         HistorifyTemporalConflictError: Future-open records, re-weave off.
@@ -52,8 +55,13 @@ def scd2_transform(
     join_key = list(spec.keys) + list(track_cols)
     eff_date = resolve_effective_date(spec, params_instance)
 
+    if spec.delete_marker is not None:
+        _assert_delete_marker(ops, frame, spec.delete_marker)
     is_log = spec.mode is HistorifyInputMode.LOG
-    ops.assert_unique_keys(frame, [*spec.keys, str(spec.effective_date)] if is_log else join_key)
+    unique_key = [*spec.keys, str(spec.effective_date)] if is_log else join_key
+    duplicates = ops.duplicate_keys(frame, unique_key)
+    if duplicates is not None:
+        raise HistorifyKeyConflictError(duplicates, mode=spec.mode)
     if is_log:
         ops.assert_no_date_collisions(frame, join_key, str(spec.effective_date), spec)
 
@@ -68,6 +76,17 @@ def scd2_transform(
     return apply_log(ops, frame, existing, spec), None
 
 
+def _assert_delete_marker(ops: HistorifyBackend[F], frame: F, marker: str) -> None:
+    """Raise unless the incoming frame carries ``marker`` as a boolean column."""
+    if marker not in ops.columns(frame):
+        raise HistorifyDeleteMarkerError(marker, "is missing from the incoming frame")
+    dtype = ops.non_boolean_dtype(frame, marker)
+    if dtype is not None:
+        raise HistorifyDeleteMarkerError(
+            marker, f"has dtype {dtype}, not boolean; cast it to Boolean (null = regular event)"
+        )
+
+
 def _first_run(
     ops: HistorifyBackend[F],
     frame: F,
@@ -84,7 +103,7 @@ def _first_run(
             dtype,
         )
         return ops.ensure_soft_delete_col(result, spec)
-    return ops.build_log_boundaries(frame, spec)
+    return build_log_history(ops, frame, spec)
 
 
 def _rewind_report_if_enabled(

{loom_kernel-2.21.0 → loom_kernel-2.22.0}/src/loom/etl/backends/_write_policy.py

@@ -12,6 +12,7 @@ from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
 from loom.core.logger import get_logger
+from loom.etl.backends._historify import HistorifyBackend, scd2_transform
 from loom.etl.declarative.target import (
     AppendSpec,
     ClientSpec,
@@ -421,9 +422,11 @@ class _WritePolicy(TargetWriter, Generic[InputFrameT, WriteFrameT, PhysicalSchem
     ) -> HistorifyRepairReport | None:
         """SCD Type 2 historify policy: read existing, validate, transform, write.
 
-        Reads existing target data, validates creation rights when absent, then
-        delegates to :meth:`_historify` with the existing frame so the backend
-        only handles the transform + write — not the read.
+        Reads existing target data, validates creation rights when absent, runs
+        :func:`~loom.etl.backends._historify.scd2_transform` with the engine's
+        :meth:`_historify_backend`, then writes the rebuilt history. The existing
+        frame is dropped before the write, so eager backends release the table
+        snapshot instead of holding it alongside the rebuilt history.
 
         Args:
             frame:           Incoming input frame.
@@ -435,18 +438,59 @@ class _WritePolicy(TargetWriter, Generic[InputFrameT, WriteFrameT, PhysicalSchem
         Returns:
             A :class:`~loom.etl.HistorifyRepairReport` when a re-weave was
             performed, or ``None`` for a normal forward-only run.
+
+        Raises:
+            HistorifyDeleteMarkerError:     ``delete_marker`` missing or not boolean.
+            HistorifyKeyConflictError:      Duplicate entity state vectors.
+            HistorifyDateCollisionError:    Same-date ties in LOG mode.
+            HistorifyTemporalConflictError: Future-open records detected.
         """
         existing = self._read_existing_data(target, frame, spec)
-        if existing is None:
+        create = existing is None
+        if create:
             _ensure_can_create_missing_table(
                 target=target,
                 schema_mode=spec.schema_mode,
                 missing_table_policy=self._missing_table_policy,
             )
         materialized = self._materialize_checked(frame, target, streaming=False)
-        return self._historify(
-            materialized, existing, target, spec=spec, params_instance=params_instance
+        result, report = scd2_transform(
+            self._historify_backend(), materialized, existing, spec, params_instance
         )
+        del existing
+        if report is not None:
+            _log.warning(
+                "historify_temporal_rerun",
+                target=target.logical_ref.ref,
+                affected_key_count=len(report.affected_keys),
+                dates_requiring_rerun=[str(d) for d in report.dates_requiring_rerun],
+                warnings=list(report.warnings),
+            )
+        self._write_history(result, target, spec=spec, create=create)
+        return report
+
+    def _write_history(
+        self,
+        result: WriteFrameT,
+        target: ResolvedTarget,
+        *,
+        spec: HistorifySpec,
+        create: bool,
+    ) -> None:
+        """Write rebuilt history: create the table, or replace its scoped content."""
+        if create:
+            self._create(
+                result,
+                target,
+                schema_mode=spec.schema_mode,
+                partition_cols=spec.partition_scope or (),
+            )
+        elif spec.partition_scope:
+            self._replace_partitions(
+                result, target, partition_cols=spec.partition_scope, schema_mode=spec.schema_mode
+            )
+        else:
+            self._replace(result, target, schema_mode=spec.schema_mode)
 
     # ========================================================================
     # Audit Hook (overrideable — default is a no-op)
@@ -664,42 +708,8 @@ class _WritePolicy(TargetWriter, Generic[InputFrameT, WriteFrameT, PhysicalSchem
         """Matched-only merge into existing table — never inserts."""
 
     @abstractmethod
-    def _historify(
-        self,
-        frame: WriteFrameT,
-        existing: WriteFrameT | None,
-        target: ResolvedTarget,
-        *,
-        spec: HistorifySpec,
-        params_instance: Any,
-    ) -> HistorifyRepairReport | None:
-        """Apply SCD Type 2 transform and write result to target.
-
-        Called after :meth:`_read_existing_data` has already fetched the current
-        target state.  The implementation must:
-
-        * Run the SCD2 algorithm (via :func:`~loom.etl.backends._historify.scd2_transform`).
-        * Write the result using the existing write hooks (``_create``,
-          ``_replace``, or ``_replace_partitions``).
-
-        Args:
-            frame:           Materialised incoming frame (write-ready).
-            existing:        Current target frame, or ``None`` for first run.
-            target:          Resolved Delta target.
-            spec:            Compiled historify spec.
-            params_instance: Runtime params; used to resolve
-                             ``spec.effective_date`` when it is a
-                             :class:`~loom.etl.ParamExpr`.
-
-        Returns:
-            A :class:`~loom.etl.HistorifyRepairReport` when re-weave was
-            triggered, or ``None`` for a normal forward-only run.
-
-        Raises:
-            HistorifyKeyConflictError:     Duplicate entity state vectors.
-            HistorifyDateCollisionError:   Same-date ties in LOG mode.
-            HistorifyTemporalConflictError: Future-open records detected.
-        """
+    def _historify_backend(self) -> HistorifyBackend[WriteFrameT]:
+        """Return the SCD2 primitives of this engine."""
 
     @abstractmethod
     def _write_file(

{loom_kernel-2.21.0 → loom_kernel-2.22.0}/src/loom/etl/backends/polars/_historify.py

@@ -6,12 +6,11 @@ from typing import Any
 
 import polars as pl
 
-from loom.etl.backends._historify._common import prev_period_value
+from loom.etl.backends._historify._common import log_run_cols, prev_period_value
 from loom.etl.backends.polars._frame import FrameT
 from loom.etl.declarative.target._history import (
     DeletePolicy,
     HistorifyDateCollisionError,
-    HistorifyKeyConflictError,
     HistorifySpec,
     HistoryDateType,
 )
@@ -21,12 +20,22 @@ def _history_boundary_dtype(spec: HistorifySpec) -> type[pl.Date] | pl.Datetime:
     return pl.Date if spec.date_type is HistoryDateType.DATE else pl.Datetime("us", "UTC")
 
 
+def _period_offset(spec: HistorifySpec) -> pl.Expr:
+    if spec.date_type is HistoryDateType.DATE:
+        return pl.duration(days=1)
+    return pl.duration(microseconds=1)
+
+
 class PolarsHistorifyBackend:
     """Polars implementation of HistorifyBackend."""
 
     def columns(self, frame: pl.DataFrame) -> list[str]:
         return list(frame.columns)
 
+    def non_boolean_dtype(self, frame: pl.DataFrame, col: str) -> str | None:
+        dtype = frame.collect_schema()[col]
+        return None if dtype == pl.Boolean else str(dtype)
+
     def history_dtype(self, spec: HistorifySpec) -> type[pl.Date] | pl.Datetime:
         return _history_boundary_dtype(spec)
 
@@ -74,6 +83,9 @@ class PolarsHistorifyBackend:
     def null_col(self, frame: pl.DataFrame, name: str, dtype: Any) -> pl.DataFrame:
         return frame.with_columns(pl.lit(None, dtype=dtype).alias(name))
 
+    def fill_null(self, frame: pl.DataFrame, col: str, value: Any) -> pl.DataFrame:
+        return frame.with_columns(pl.col(col).fill_null(value))
+
     def rename(self, frame: pl.DataFrame, rename_map: dict[str, str]) -> pl.DataFrame:
         return frame.rename(rename_map)
 
@@ -143,11 +155,7 @@ class PolarsHistorifyBackend:
         eff_col = str(spec.effective_date)
         entity_key = list(spec.keys)
         boundary_dtype = _history_boundary_dtype(spec)
-        offset = (
-            pl.duration(days=1)
-            if spec.date_type is HistoryDateType.DATE
-            else pl.duration(microseconds=1)
-        )
+        offset = _period_offset(spec)
 
         collapsed = self._collapse_same_track_runs(frame, spec, entity_key, eff_col)
         sorted_events = collapsed.sort(entity_key + [eff_col])
@@ -171,6 +179,24 @@ class PolarsHistorifyBackend:
             clamped_valid_to.cast(boundary_dtype).alias(spec.valid_to),
         ).drop(eff_col)
 
+    def closure_events(self, history: pl.DataFrame, spec: HistorifySpec) -> pl.DataFrame:
+        entity_key = list(spec.keys)
+        closed_at = pl.col(spec.valid_to) + _period_offset(spec)
+        next_from = pl.col("__next_from__")
+        # Nulls last: a zero-width vector sorts before the vector sharing its valid_from.
+        return (
+            history.sort([*entity_key, spec.valid_from, spec.valid_to], nulls_last=True)
+            .with_columns(pl.col(spec.valid_from).shift(-1).over(entity_key).alias("__next_from__"))
+            .filter(
+                pl.col(spec.valid_to).is_not_null()
+                & (next_from.is_null() | (next_from > closed_at))
+            )
+            .select(
+                *entity_key,
+                closed_at.cast(_history_boundary_dtype(spec)).alias(str(spec.effective_date)),
+            )
+        )
+
     @staticmethod
     def _collapse_same_track_runs(
         frame: FrameT,
@@ -187,9 +213,8 @@ class PolarsHistorifyBackend:
         if spec.track is None:
             return frame
 
-        track = list(spec.track)
         overwrite: list[str] = list(spec.overwrite or ())
-        track_struct = pl.struct(track)
+        track_struct = pl.struct(log_run_cols(spec))
 
         runs = (
             frame.sort(entity_key + [eff_col])
@@ -242,11 +267,11 @@ class PolarsHistorifyBackend:
         boundary_dtype = _history_boundary_dtype(spec)
         return result.with_columns(pl.lit(None, dtype=boundary_dtype).alias(spec.deleted_at))
 
-    def assert_unique_keys(self, frame: pl.DataFrame, keys: list[str]) -> None:
+    def duplicate_keys(self, frame: pl.DataFrame, keys: list[str]) -> str | None:
         if len(frame) == frame.select(keys).n_unique():
-            return
+            return None
         dupes = frame.filter(frame.select(keys).is_duplicated()).select(keys).head(5)
-        raise HistorifyKeyConflictError(str(dupes.to_dicts()))
+        return str(dupes.to_dicts())
 
     def assert_no_date_collisions(
         self,

{loom_kernel-2.21.0 → loom_kernel-2.22.0}/src/loom/etl/backends/polars/_writer.py

@@ -10,7 +10,6 @@ from deltalake import CommitProperties, DeltaTable, WriterProperties, write_delt
 from deltalake.table import TableMerger
 
 from loom.core.logger import get_logger
-from loom.etl.backends._historify._transform import scd2_transform
 from loom.etl.backends._merge import (
     SOURCE_ALIAS,
     TARGET_ALIAS,
@@ -31,7 +30,7 @@ from loom.etl.backends.polars._schema import (
 )
 from loom.etl.declarative.target import SchemaMode
 from loom.etl.declarative.target._file import FileSpec
-from loom.etl.declarative.target._history import HistorifyRepairReport, HistorifySpec
+from loom.etl.declarative.target._history import HistorifySpec
 from loom.etl.declarative.target._table import (
     AppendSpec,
     ReplacePartitionsSpec,
@@ -620,40 +619,8 @@ class PolarsTargetWriter(_WritePolicy[pl.LazyFrame, pl.DataFrame, PolarsPhysical
         except (TableNotFoundError, FileNotFoundError):
             return None
 
-    def _historify(
-        self,
-        frame: pl.DataFrame,
-        existing: pl.DataFrame | None,
-        target: ResolvedTarget,
-        *,
-        spec: HistorifySpec,
-        params_instance: Any,
-    ) -> HistorifyRepairReport | None:
-        """Run SCD Type 2 transform and write result via existing write hooks."""
-        result, report = scd2_transform(
-            PolarsHistorifyBackend(), frame, existing, spec, params_instance
-        )
-        if report is not None:
-            _log.warning(
-                "historify temporal rerun rewound future history",
-                affected_key_count=len(report.affected_keys),
-                dates_requiring_rerun=[str(d) for d in report.dates_requiring_rerun],
-                warnings=list(report.warnings),
-            )
-        if existing is None:
-            self._create(
-                result,
-                target,
-                schema_mode=spec.schema_mode,
-                partition_cols=spec.partition_scope or (),
-            )
-        elif spec.partition_scope:
-            self._replace_partitions(
-                result, target, partition_cols=spec.partition_scope, schema_mode=spec.schema_mode
-            )
-        else:
-            self._replace(result, target, schema_mode=spec.schema_mode)
-        return report
+    def _historify_backend(self) -> PolarsHistorifyBackend:
+        return PolarsHistorifyBackend()
 
     def _write_file(
         self,

{loom_kernel-2.21.0 → loom_kernel-2.22.0}/src/loom/etl/backends/spark/_historify.py

@@ -4,15 +4,15 @@ from __future__ import annotations
 
 from typing import Any
 
-from pyspark.sql import DataFrame
+from pyspark.sql import Column, DataFrame
 from pyspark.sql import functions as F
+from pyspark.sql.types import BooleanType
 from pyspark.sql.window import Window
 
-from loom.etl.backends._historify._common import prev_period_value
+from loom.etl.backends._historify._common import log_run_cols, prev_period_value
 from loom.etl.declarative.target._history import (
     DeletePolicy,
     HistorifyDateCollisionError,
-    HistorifyKeyConflictError,
     HistorifySpec,
     HistoryDateType,
 )
@@ -22,12 +22,22 @@ def _history_boundary_dtype_sql(spec: HistorifySpec) -> str:
     return "date" if spec.date_type is HistoryDateType.DATE else "timestamp"
 
 
+def _next_period(col: Column, spec: HistorifySpec) -> Column:
+    if spec.date_type is HistoryDateType.DATE:
+        return F.date_add(col, 1)
+    return col + F.expr("INTERVAL 1 MICROSECOND")
+
+
 class SparkHistorifyBackend:
     """Spark implementation of HistorifyBackend."""
 
     def columns(self, frame: DataFrame) -> list[str]:
         return list(frame.columns)
 
+    def non_boolean_dtype(self, frame: DataFrame, col: str) -> str | None:
+        dtype = frame.schema[col].dataType
+        return None if isinstance(dtype, BooleanType) else dtype.simpleString()
+
     def history_dtype(self, spec: HistorifySpec) -> str:
         return _history_boundary_dtype_sql(spec)
 
@@ -77,6 +87,9 @@ class SparkHistorifyBackend:
     def null_col(self, frame: DataFrame, name: str, dtype: Any) -> DataFrame:
         return frame.withColumn(name, F.lit(None).cast(dtype))
 
+    def fill_null(self, frame: DataFrame, col: str, value: Any) -> DataFrame:
+        return frame.withColumn(col, F.coalesce(F.col(col), F.lit(value)))
+
     def rename(self, frame: DataFrame, rename_map: dict[str, str]) -> DataFrame:
         result = frame
         for old, new in rename_map.items():
@@ -199,6 +212,25 @@ class SparkHistorifyBackend:
             .drop(eff_col)
         )
 
+    def closure_events(self, history: DataFrame, spec: HistorifySpec) -> DataFrame:
+        entity_key = list(spec.keys)
+        # Nulls last: a zero-width vector sorts before the vector sharing its valid_from.
+        window = Window.partitionBy(entity_key).orderBy(
+            F.col(spec.valid_from), F.col(spec.valid_to).asc_nulls_last()
+        )
+        closed_at = _next_period(F.col(spec.valid_to), spec)
+        next_from = F.col("__next_from__")
+        return (
+            history.withColumn("__next_from__", F.lead(spec.valid_from).over(window))
+            .filter(
+                F.col(spec.valid_to).isNotNull() & (next_from.isNull() | (next_from > closed_at))
+            )
+            .select(
+                *entity_key,
+                closed_at.cast(_history_boundary_dtype_sql(spec)).alias(str(spec.effective_date)),
+            )
+        )
+
     @staticmethod
     def _collapse_same_track_runs(
         frame: DataFrame,
@@ -215,9 +247,8 @@ class SparkHistorifyBackend:
         if spec.track is None:
             return frame
 
-        track = list(spec.track)
         overwrite = list(spec.overwrite or ())
-        track_struct = F.struct(*[F.col(c) for c in track])
+        track_struct = F.struct(*[F.col(c) for c in log_run_cols(spec)])
 
         order_w = Window.partitionBy(entity_key).orderBy(eff_col)
         prev_struct = F.lag(track_struct).over(order_w)
@@ -264,15 +295,11 @@ class SparkHistorifyBackend:
         boundary_dtype = _history_boundary_dtype_sql(spec)
         return result.withColumn(spec.deleted_at, F.lit(None).cast(boundary_dtype))
 
-    def assert_unique_keys(self, frame: DataFrame, keys: list[str]) -> None:
-        n_rows = frame.count()
-        n_unique = frame.select(keys).distinct().count()
-        if n_rows == n_unique:
-            return
+    def duplicate_keys(self, frame: DataFrame, keys: list[str]) -> str | None:
         dupes = (
             frame.groupBy(keys).count().filter(F.col("count") > 1).drop("count").limit(5).collect()
         )
-        raise HistorifyKeyConflictError(str([row.asDict() for row in dupes]))
+        return str([row.asDict() for row in dupes]) if dupes else None
 
     def assert_no_date_collisions(
         self,

{loom_kernel-2.21.0 → loom_kernel-2.22.0}/src/loom/etl/backends/spark/_writer.py

@@ -15,7 +15,6 @@ from pyspark.sql.column import Column
 from pyspark.storagelevel import StorageLevel
 
 from loom.etl.backends._format_registry import resolve_format_handler, write_options_or_default
-from loom.etl.backends._historify._transform import scd2_transform
 from loom.etl.backends._merge import (
     SOURCE_ALIAS,
     TARGET_ALIAS,
@@ -38,7 +37,7 @@ from loom.etl.declarative._write_options import (
 )
 from loom.etl.declarative.target import SchemaMode
 from loom.etl.declarative.target._file import FileSpec
-from loom.etl.declarative.target._history import HistorifyRepairReport, HistorifySpec
+from loom.etl.declarative.target._history import HistorifySpec
 from loom.etl.declarative.target._table import AppendSpec, UpdateSpec, UpsertSpec
 from loom.etl.lineage._records import LineageRecord, get_lineage_schema
 from loom.etl.storage._config import MissingTablePolicy
@@ -404,41 +403,8 @@ class SparkTargetWriter(_WritePolicy[DataFrame, DataFrame, SparkPhysicalSchema])
             return self._spark.read.format("delta").load(target.location.uri)
         return None
 
-    def _historify(
-        self,
-        frame: DataFrame,
-        existing: DataFrame | None,
-        target: ResolvedTarget,
-        *,
-        spec: HistorifySpec,
-        params_instance: Any,
-    ) -> HistorifyRepairReport | None:
-        """Run SCD Type 2 transform and write result via existing write hooks."""
-        result, report = scd2_transform(
-            SparkHistorifyBackend(), frame, existing, spec, params_instance
-        )
-        if report is not None:
-            _log.warning(
-                "historify temporal rerun rewound future history: "
-                "affected_key_count=%d dates_requiring_rerun=%s warnings=%s",
-                len(report.affected_keys),
-                [str(d) for d in report.dates_requiring_rerun],
-                list(report.warnings),
-            )
-        if existing is None:
-            self._create(
-                result,
-                target,
-                schema_mode=spec.schema_mode,
-                partition_cols=spec.partition_scope or (),
-            )
-        elif spec.partition_scope:
-            self._replace_partitions(
-                result, target, partition_cols=spec.partition_scope, schema_mode=spec.schema_mode
-            )
-        else:
-            self._replace(result, target, schema_mode=spec.schema_mode)
-        return report
+    def _historify_backend(self) -> SparkHistorifyBackend:
+        return SparkHistorifyBackend()
 
     def _write_file(
         self,