floe-python 0.8.0__py3-none-any.whl

floe/iceberg/compact.py

@@ -0,0 +1,139 @@
+"""Experimental compaction (#952): rewrite every data file, fold the position deletes, one REPLACE commit.
+
+PyIceberg 0.12 has no `RewriteFiles`, so this subclasses its private `_OverwriteFiles` and `ManifestWriterV2`
+(spike #618 S5). `tests/unit/iceberg/test_compact.py` pins the PyIceberg version they were checked against.
+"""
+
+# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportMissingTypeStubs=false, reportPrivateUsage=false
+# ^ pyarrow ships no type stubs; the private PyIceberg internals are the point of this module
+
+from collections.abc import Mapping
+
+import duckdb
+import pyarrow as pa
+from pyiceberg.io.pyarrow import _dataframe_to_data_files
+from pyiceberg.manifest import (
+    DataFileContent,
+    ManifestContent,
+    ManifestEntry,
+    ManifestFile,
+    ManifestWriter,
+    ManifestWriterV2,
+)
+from pyiceberg.table import Table
+from pyiceberg.table.snapshots import Operation, Summary
+from pyiceberg.table.update import UpdatesAndRequirements
+from pyiceberg.table.update.snapshot import CommitWindow, _OverwriteFiles
+from pyiceberg.table.update.validate import (
+    _validate_data_files_exist,
+    _validate_no_new_deletes_for_data_files,
+)
+
+from floe.errors import FloeRunError
+from floe.iceberg.catalog import run_error
+from floe.iceberg.merge import _target
+from floe.iceberg.write import CHUNK_ROWS
+
+
+class _DeleteManifestWriter(ManifestWriterV2):
+    """PyIceberg 0.12's `ManifestWriterV2` writes `content=data` manifests only."""
+
+    def content(self) -> ManifestContent:
+        return ManifestContent.DELETES
+
+    @property
+    def _meta(self) -> dict[str, str]:
+        return {**super()._meta, "content": "deletes"}
+
+
+class _RewriteFiles(_OverwriteFiles):
+    """Java `RewriteFiles`: `_deleted_data_files` holds the rewritten data files and their delete files."""
+
+    def _deleted_entries(self) -> list[ManifestEntry]:
+        return []  # marked DELETED in place by `_existing_manifests`
+
+    def _existing_manifests(self) -> list[ManifestFile]:
+        """The head's manifests; one that holds a removed file is rewritten with that entry DELETED."""
+        out: list[ManifestFile] = []
+        head = self._transaction.table_metadata.snapshot_by_name(self._target_branch)
+        for manifest in head.manifests(self._io) if head else []:
+            entries = manifest.fetch_manifest_entry(self._io, discard_deleted=True)
+            if not any(e.data_file in self._deleted_data_files for e in entries):
+                out.append(manifest)
+                continue
+            spec = self.spec(manifest.partition_spec_id)
+            writer: ManifestWriter = (
+                _DeleteManifestWriter(spec, self.schema(), self.new_manifest_output(), self._snapshot_id, self._compression)
+                if manifest.content == ManifestContent.DELETES
+                else self.new_manifest_writer(spec)
+            )  # fmt: skip
+            with writer:
+                for e in entries:
+                    writer.delete(e) if e.data_file in self._deleted_data_files else writer.existing(e)  # fmt: skip
+            out.append(writer.to_manifest_file())
+        return out
+
+    def _summary(self, snapshot_properties: Mapping[str, str] = {}) -> Summary:
+        # built as an OVERWRITE: PyIceberg 0.12's `update_snapshot_summaries` rejects REPLACE
+        summary = super()._summary(dict(snapshot_properties))
+        return Summary(Operation.REPLACE, **summary.additional_properties)
+
+    def _validate_concurrency(self) -> None:
+        """Only a removed rewritten file, or a new delete against one, conflicts; concurrent appends don't."""
+        window = self._commit_window
+        if window is None or window.is_empty() or window.head is None:
+            return
+        table, data = self._transaction._table, {f for f in self._deleted_data_files if f.content == DataFileContent.DATA}  # fmt: skip
+        _validate_data_files_exist(
+            table, window.head, self._deleted_data_files, window.base
+        )
+        _validate_no_new_deletes_for_data_files(
+            table, window.head, None, data, window.base
+        )
+
+    def _commit(self) -> UpdatesAndRequirements:
+        # first attempt: validate the commits since the snapshot the rewrite read
+        if self._commit_window is None:
+            self._commit_window = CommitWindow.resolve(self._transaction.table_metadata, self._starting_snapshot_id, self._target_branch)  # fmt: skip
+            self._validate_concurrency()
+        return super()._commit()
+
+
+def compact(table: Table, con: duckdb.DuckDBPyConnection) -> dict[str, int]:
+    """Rewrite the whole table at its current snapshot through DuckDB (deletes applied), then commit once.
+
+    Concurrent appends are kept; a concurrent MERGE on the same files makes the compaction fail.
+    """
+    base = table.current_snapshot()
+    if base is None:
+        return {"rewritten_data_files": 0, "removed_delete_files": 0, "added_data_files": 0}  # fmt: skip
+    if not table.spec().is_unpartitioned() or table.format_version != 2:
+        # ponytail: PyIceberg 0.12 streams unpartitioned v2 writes only; partitioned needs an ordered per-partition flush
+        raise FloeRunError(f"iceberg compaction supports unpartitioned format v2 tables only ({'.'.join(table.name())})")  # fmt: skip
+    with run_error("iceberg compaction plan failed"):
+        tasks = list(table.scan(snapshot_id=base.snapshot_id).plan_files())
+    data = {t.file for t in tasks}
+    deletes = {d for t in tasks for d in t.delete_files}
+    if any(d.content == DataFileContent.EQUALITY_DELETES for d in deletes):
+        raise FloeRunError("iceberg compaction does not support equality deletes")
+    source = f"{_target(table)} AT (VERSION => {base.snapshot_id})"
+    schema = table.schema().as_arrow()
+    with run_error("iceberg compaction rewrite failed"):
+        cursor = con.execute(f"SELECT * FROM {source}")
+        batches = (b.cast(schema) for b in cursor.to_arrow_reader(CHUNK_ROWS))
+        new = list(_dataframe_to_data_files(table.metadata, pa.RecordBatchReader.from_batches(schema, batches), table.io))  # fmt: skip
+        live = con.execute(f"SELECT count(*) FROM {source}").fetchone()
+    if live is None or live[0] != sum(f.record_count for f in new):
+        raise FloeRunError(f"iceberg compaction wrote {sum(f.record_count for f in new)} rows, expected {live and live[0]}; nothing was committed")  # fmt: skip
+    with run_error("iceberg compaction commit failed"):
+        table.refresh()  # the head may have moved: its manifests are the ones carried over
+        tx = table.transaction()
+        rewrite = _RewriteFiles(Operation.OVERWRITE, tx, table.io, snapshot_properties={"floe.maintenance": "compact"})  # fmt: skip
+        rewrite._starting_snapshot_id = base.snapshot_id
+        for f in data | deletes:
+            rewrite.delete_data_file(f)
+        for f in new:
+            rewrite.append_data_file(f)
+        rewrite.commit()
+        tx.commit_transaction()
+    return {"rewritten_data_files": len(data), "removed_delete_files": len(deletes), "added_data_files": len(new)}  # fmt: skip

floe/iceberg/merge.py

@@ -0,0 +1,297 @@
+"""SCD1 and SCD2 merges into Iceberg: the batch is staged as Parquet, then DuckDB `MERGE INTO` (#598).
+
+SCD1 follows `io/write/strategy/merge/scd1.rs`: match on the merge key, update every non-key,
+non-ignored column, insert unmatched rows with every column. Rust updates every matched row; here
+the update is guarded by a null-safe change predicate, so rerunning a batch writes nothing. The
+resulting rows are the same. SCD2 follows `merge/scd2.rs` (see `scd2_sql`).
+
+A bucket-partitioned target gets one MERGE per bucket in one transaction, so one catalog commit:
+a single MERGE runs out of memory there (#609).
+"""
+
+# ponytail: DuckDB only as the Iceberg MERGE writer (merge-on-read, bounded memory); switch to Polars/PyIceberg and drop duckdb per #827 once upstream supports it.
+# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportUnknownParameterType=false, reportMissingTypeStubs=false
+# ^ pyarrow ships no type stubs
+
+import datetime as dt
+import tempfile
+from collections.abc import Callable, Collection, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+import duckdb
+import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.parquet as pq
+from pyiceberg.partitioning import PartitionSpec
+from pyiceberg.schema import Schema
+from pyiceberg.table import Table
+from pyiceberg.transforms import BucketTransform
+
+from floe.config.validate import SCD2_DEFAULTS
+from floe.errors import FloeRunError
+from floe.iceberg.catalog import DUCKDB_CATALOG, run_error
+from floe.iceberg.write import IcebergWriteResult, added_data_files, write_result
+from floe.state import (
+    _backoff,  # pyright: ignore[reportPrivateUsage] -- one jittered backoff for every CAS retry
+)
+
+# (is_current, valid_from, valid_to)
+SCD2_COLUMNS = tuple(SCD2_DEFAULTS.values())
+MERGE_ATTEMPTS = 5
+
+
+def _q(name: str) -> str:
+    return '"' + name.replace('"', '""') + '"'
+
+
+@dataclass(frozen=True)
+class MergeResult:
+    write: IcebergWriteResult
+    inserted_count: int
+    # SCD1: matched rows that changed; Rust counts every matched row (D2). SCD2: closed rows.
+    updated_count: int
+    closed_count: int | None = None
+    unchanged_count: int | None = None
+
+
+def scd1_sql(
+    target: str, columns: Sequence[str], key: Sequence[str], ignore: Collection[str]
+) -> tuple[str, str]:
+    """(count, merge) statements: the source is `read_parquet($1)`, aliased `s`; the target is `t`.
+
+    The count returns (inserted, updated) as the merge will apply them. `MERGE ... RETURNING`
+    can't be aggregated in SQL, and returning every row would not stay bounded.
+    """
+    update = [c for c in columns if c not in key and c not in ignore]
+    on = " AND ".join(f"t.{_q(k)} = s.{_q(k)}" for k in key)
+    changed = " OR ".join(f"t.{_q(c)} IS DISTINCT FROM s.{_q(c)}" for c in update)
+    matched = ""
+    if update:
+        assign = ", ".join(f"{_q(c)} = s.{_q(c)}" for c in update)
+        matched = f"WHEN MATCHED AND ({changed}) THEN UPDATE SET {assign} "
+    # a matched target key is never NULL, since NULL = x never holds
+    unmatched = f"t.{_q(key[0])} IS NULL"
+    count = (
+        f"SELECT count(*) FILTER ({unmatched}), "
+        f"count(*) FILTER (NOT {unmatched} AND ({changed or 'false'})) "
+        f"FROM read_parquet($1) s LEFT JOIN {target} t ON {on}"
+    )
+    merge = (
+        f"MERGE INTO {target} t USING read_parquet($1) s ON {on} {matched}"
+        f"WHEN NOT MATCHED THEN INSERT ({', '.join(map(_q, columns))}) "
+        f"VALUES ({', '.join(f's.{_q(c)}' for c in columns)})"
+    )
+    return count, merge
+
+
+def scd2_sql(
+    target: str,
+    columns: Sequence[str],
+    key: Sequence[str],
+    compare: Sequence[str],
+    system: Sequence[str],
+    now: dt.datetime,
+) -> tuple[str, str, str]:
+    """(count, changed, merge), as `scd1_sql`; the count returns (inserted, closed, unchanged).
+
+    Rust closes changed current rows, then inserts new versions, in two Delta merges. Here it is
+    one MERGE: each changed row is in the source twice, once with its key (it matches the current
+    version and closes it) and once with a NULL key (it never matches, so it is inserted).
+    `changed` copies the changed rows of `$1` to `$2` first, so the MERGE's source never reads
+    the target the MERGE is writing.
+    """
+    is_current, valid_from, valid_to = map(_q, system)
+    on = " AND ".join(f"t.{_q(k)} = s.{_q(k)}" for k in key) + f" AND t.{is_current}"
+    changed = " OR ".join(f"t.{_q(c)} IS DISTINCT FROM s.{_q(c)}" for c in compare)
+    changed = f"({changed or 'false'})"
+    # the join keeps only current target rows, whose flag is never NULL
+    unmatched = f"t.{is_current} IS NULL"
+    count = (
+        f"SELECT count(*) FILTER ({unmatched} OR {changed}), "
+        f"count(*) FILTER (NOT {unmatched} AND {changed}), "
+        f"count(*) FILTER (NOT {unmatched} AND NOT {changed}) "
+        f"FROM read_parquet($1) s LEFT JOIN {target} t ON {on}"
+    )
+    keys = [f"__floe_key_{i}" for i in range(len(key))]
+    values = ", ".join(f"s.{_q(c)}" for c in columns)
+    copy = (
+        f"COPY (SELECT {values} FROM read_parquet($1) s JOIN {target} t ON {on} "
+        f"WHERE {changed}) TO $2 (FORMAT parquet)"
+    )
+    source = (
+        f"SELECT {', '.join(f's.{_q(k)} AS {m}' for k, m in zip(key, keys))}, {values} "
+        f"FROM read_parquet($1) s UNION ALL "
+        f"SELECT {', '.join(f'NULL AS {m}' for m in keys)}, {values} FROM read_parquet($2) s"
+    )
+    merge_on = " AND ".join(f"t.{_q(k)} = s.{m}" for k, m in zip(key, keys))
+    ts = f"TIMESTAMP '{now.isoformat(sep=' ')}'"
+    insert = ", ".join([*map(_q, columns), is_current, valid_from, valid_to])
+    merge = (
+        f"MERGE INTO {target} t USING ({source}) s ON {merge_on} AND t.{is_current} "
+        f"WHEN MATCHED AND {changed} THEN UPDATE SET {is_current} = false, {valid_to} = {ts} "
+        f"WHEN NOT MATCHED THEN INSERT ({insert}) VALUES ({values}, true, {ts}, NULL)"
+    )
+    return count, copy, merge
+
+
+def stage_batch(
+    directory: Path, reader: pa.RecordBatchReader, spec: PartitionSpec, schema: Schema
+) -> list[str]:
+    """The batch as one Parquet file, or one per bucket when `spec` has a bucket field."""
+    bucket = next(
+        (f for f in spec.fields if isinstance(f.transform, BucketTransform)), None
+    )
+    writers: dict[object, pq.ParquetWriter] = {}
+
+    def write(value: object, batch: pa.RecordBatch) -> None:
+        if value not in writers:
+            path = directory / f"batch-{len(writers)}.parquet"
+            writers[value] = pq.ParquetWriter(path, reader.schema)
+        writers[value].write_batch(batch)
+
+    try:
+        for batch in reader:
+            if bucket is None:
+                write(None, batch)
+                continue
+            column = schema.find_column_name(bucket.source_id)
+            source_type = schema.find_type(bucket.source_id)
+            buckets = bucket.transform.pyarrow_transform(source_type)(
+                batch.column(column)
+            )
+            for value in buckets.unique().to_pylist():
+                mask = (
+                    buckets.is_null() if value is None else pc.equal(buckets, value)  # pyright: ignore[reportAttributeAccessIssue] -- pyarrow.compute builds its functions at import
+                )
+                write(value, batch.filter(mask))
+        if not writers:  # an empty batch still needs a file to read
+            write(None, pa.RecordBatch.from_pylist([], reader.schema))
+    finally:
+        for writer in writers.values():
+            writer.close()
+    return [str(directory / f"batch-{i}.parquet") for i in range(len(writers))]
+
+
+def run_merge(
+    con: duckdb.DuckDBPyConnection,
+    count: str,
+    merge: str,
+    files: list[str],
+    changed: str | None = None,
+) -> tuple[int, ...]:
+    """The counts, then one MERGE per staged file, all in one transaction (one catalog commit).
+
+    With `changed` (SCD2), each file's changed rows are copied next to it before any MERGE.
+    """
+    counts = con.execute(count, [files]).fetchone()
+    assert counts is not None  # an aggregate always returns one row
+    # nothing to insert, update or close: no MERGE, so no snapshot
+    if counts[0] or counts[1]:
+        params = [
+            [f] if changed is None else [f, f"{f}.changed.parquet"] for f in files
+        ]
+        if changed is not None:
+            for p in params:
+                con.execute(changed, p)
+        con.begin()
+        try:
+            for p in params:
+                con.execute(merge, p)
+        except BaseException:
+            con.rollback()
+            raise
+        con.commit()  # a failed commit has already ended the transaction: no rollback
+    return counts
+
+
+def _is_conflict(e: FloeRunError) -> bool:
+    """The catalog's 409 on commit: DuckDB raises it from `commit()` and does not retry it."""
+    return isinstance(e.__cause__, duckdb.TransactionException) and "Conflict_409" in str(e)  # fmt: skip
+
+
+def _merge(
+    con: duckdb.DuckDBPyConnection,
+    table: Table,
+    reader: pa.RecordBatchReader,
+    statements: Callable[[], tuple[str, str, str | None]],
+) -> tuple[IcebergWriteResult, tuple[int, ...]]:
+    """`statements` gives (count, merge, changed); it runs per attempt, so SCD2 stamps each retry's own time.
+
+    On a commit conflict the whole MERGE runs again on the fresh table (#922): the counts and
+    the changed rows are recomputed, so a key both writers inserted becomes an update.
+    """
+    with tempfile.TemporaryDirectory(prefix="floe-merge-") as staging:
+        with run_error("iceberg merge staging failed"):
+            files = stage_batch(Path(staging), reader, table.spec(), table.schema())
+        for attempt in range(MERGE_ATTEMPTS):
+            if attempt:
+                _backoff(attempt - 1)
+                with run_error("rest catalog load_table failed"):
+                    table.refresh()
+            before = table.current_snapshot()
+            try:
+                with run_error("duckdb merge failed"):
+                    count, merge, changed = statements()
+                    counts = run_merge(con, count, merge, files, changed)
+            except FloeRunError as e:
+                if not _is_conflict(e):
+                    raise
+                continue
+            break
+        else:
+            raise FloeRunError(f"duckdb merge failed: conflicted {MERGE_ATTEMPTS} times in a row")  # fmt: skip
+    with run_error("rest catalog load_table failed"):
+        table.refresh()
+        added = added_data_files(table, None if before is None else before.snapshot_id)
+    return write_result(table, added), counts
+
+
+def _target(table: Table) -> str:
+    return ".".join(map(_q, (DUCKDB_CATALOG, *table.name())))
+
+
+def merge_scd1(
+    con: duckdb.DuckDBPyConnection,
+    table: Table,
+    reader: pa.RecordBatchReader,
+    merge_key: Sequence[str],
+    ignore_columns: Collection[str] = (),
+) -> MergeResult:
+    """`con` comes from `open_duckdb` on the table's catalog; the table already has the batch's columns."""
+    count, merge = scd1_sql(
+        _target(table), reader.schema.names, merge_key, ignore_columns
+    )
+    write, counts = _merge(con, table, reader, lambda: (count, merge, None))
+    return MergeResult(write, counts[0], counts[1])
+
+
+def merge_scd2(
+    con: duckdb.DuckDBPyConnection,
+    table: Table,
+    reader: pa.RecordBatchReader,
+    merge_key: Sequence[str],
+    compare_columns: Sequence[str] | None = None,
+    ignore_columns: Collection[str] = (),
+    system_columns: Sequence[str] = SCD2_COLUMNS,
+) -> MergeResult:
+    """As `merge_scd1`; the table also has the system columns (is_current, valid_from, valid_to).
+
+    Compare columns default to every non-key, non-ignored batch column, as in Rust.
+    """
+    columns = reader.schema.names
+    if compare_columns is None:
+        compare_columns = [
+            c for c in columns if c not in merge_key and c not in ignore_columns
+        ]
+
+    def statements() -> tuple[str, str, str | None]:
+        # Rust stamps both versions with the UTC time of the merge, as a naive timestamp
+        now = dt.datetime.now(dt.UTC).replace(tzinfo=None)
+        count, changed, merge = scd2_sql(
+            _target(table), columns, merge_key, compare_columns, system_columns, now
+        )
+        return count, merge, changed
+
+    write, (inserted, closed, unchanged) = _merge(con, table, reader, statements)
+    return MergeResult(write, inserted, closed, closed, unchanged)

floe/iceberg/sink.py

@@ -0,0 +1,208 @@
+"""The run's Iceberg accepted sink: catalog target, table and one commit (Rust `io/write/iceberg.rs` `write`)."""
+
+# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportUnknownParameterType=false, reportMissingTypeStubs=false
+# ^ pyarrow ships no type stubs
+
+import hashlib
+import json
+import time
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import polars as pl
+import pyarrow as pa
+from pyiceberg.catalog import Catalog
+from pyiceberg.schema import Schema, assign_fresh_schema_ids
+from pyiceberg.table import Table
+
+from floe.checks.unique import resolve_unique_keys
+from floe.config.storage import (
+    CatalogDefinition,
+    CatalogsConfig,
+    StorageDefinition,
+    StoragesConfig,
+)
+from floe.config.types import EntityConfig, WriteMode
+from floe.config.validate import SCD2_DEFAULTS
+from floe.errors import FloeConfigError, FloeRunError
+from floe.iceberg.catalog import (
+    PartitionFields,
+    ensure_table,
+    open_catalog,
+    open_duckdb,
+    run_error,
+)
+from floe.iceberg.merge import merge_scd1, merge_scd2
+from floe.iceberg.write import (
+    SMALL_FILE_THRESHOLD_BYTES,
+    IcebergWriteResult,
+    write_table,
+)
+from floe.storage.inputs import InputFile
+from floe.storage.resolve import resolve_path
+
+
+@dataclass(frozen=True)
+class IcebergTarget:
+    catalog: CatalogDefinition
+    storage: StorageDefinition | None  # the table location's storage
+    namespace: str
+    table: str
+    location: str
+    accepted_uri: str  # `sink.accepted.path` resolved, as Rust reports it
+
+
+def catalog_ident(value: str) -> str:
+    """Rust `normalize_catalog_ident`."""
+    out = "".join(c.lower() if c.isascii() and (c.isalnum() or c in "_-") else "_" for c in value)  # fmt: skip
+    return out.strip("_") or "default"
+
+
+def resolve_target(storages: StoragesConfig | None, catalogs: CatalogsConfig | None, config_dir: Path, entity: EntityConfig) -> IcebergTarget:  # fmt: skip
+    """Rust `CatalogResolver::resolve_iceberg_target`; the catalog reference is validated."""
+    sink = entity.sink.accepted
+    if sink.iceberg is None:
+        # ponytail: Rust 0.7 also wrote catalog-less filesystem tables; PyIceberg has no such catalog without sqlalchemy
+        raise FloeRunError(f"entity.name={entity.name} sink.accepted.format=iceberg requires sink.accepted.iceberg with a catalog in 0.8")  # fmt: skip
+    name = sink.iceberg.catalog or (catalogs and catalogs.default)
+    catalog = next((c for c in (catalogs.definitions if catalogs else []) if c.name == name), None)  # fmt: skip
+    if catalog is None:  # validated; kept for callers that skip validation
+        raise FloeConfigError(f"entity.name={entity.name} sink.accepted.iceberg.catalog references unknown catalog {name}")  # fmt: skip
+    namespace = catalog_ident(sink.iceberg.namespace or entity.domain or "default")
+    table = catalog_ident(sink.iceberg.table or entity.name)
+    field = f"entity.name={entity.name} sink.accepted.storage"
+    accepted_uri, storage = resolve_path(storages, config_dir, sink.storage, sink.path, field)  # fmt: skip
+    where = catalog.warehouse_storage or sink.storage
+    if sink.iceberg.location is not None:
+        location, storage = resolve_path(storages, config_dir, where, sink.iceberg.location, field)  # fmt: skip
+    elif catalog.warehouse_storage is not None or catalog.warehouse_prefix is not None:
+        parts = ((catalog.warehouse_prefix or "").strip("/"), namespace, table)
+        location, storage = resolve_path(storages, config_dir, where, "/".join(p for p in parts if p), field)  # fmt: skip
+    else:
+        location = accepted_uri
+    return IcebergTarget(catalog, storage, namespace, table, location, accepted_uri)  # fmt: skip
+
+
+def existing_table(target: IcebergTarget) -> Table | None:
+    catalog = open_catalog(target.catalog, target.storage)
+    identifier = (target.namespace, target.table)
+    with run_error("rest catalog table_exists check failed"):
+        if not catalog.table_exists(identifier):
+            return None
+    with run_error("rest catalog load_table failed"):
+        return catalog.load_table(identifier)
+
+
+def input_hash(entity: EntityConfig, files: Sequence[InputFile]) -> str:
+    """`floe.input_hash`: the entity and its claimed files with size and mtime (ADR "Retry guard"; Rust has none)."""
+    claimed = sorted([f.source_uri, f.source_size, f.source_mtime] for f in files)
+    return hashlib.sha256(json.dumps([entity.id, claimed]).encode()).hexdigest()
+
+
+_WIDER: dict[pl.DataType, pl.DataType] = {pl.UInt8(): pl.Int32(), pl.UInt16(): pl.Int32(), pl.UInt32(): pl.Int64()}  # fmt: skip
+
+
+def _reader(entity: EntityConfig, frames: list[pl.DataFrame], names: list[str]) -> pa.RecordBatchReader:  # fmt: skip
+    """Output columns in schema order; `nullable: false` columns are required, as in Rust."""
+    columns = entity.schema_.columns
+    for name, c in zip(names, columns, strict=True):
+        if c.nullable is False and any(f[name].null_count() for f in frames):
+            raise FloeRunError(f"iceberg write rejected nulls for non-nullable column {name}")  # fmt: skip
+
+    def arrow(df: pl.DataFrame) -> pa.Table:  # Iceberg has no unsigned types
+        return df.select(pl.col(n).cast(_WIDER.get(df.schema[n], df.schema[n])) for n in names).to_arrow()  # fmt: skip
+
+    schema = arrow(frames[0].head(0)).schema
+    for i, c in enumerate(columns):
+        schema = schema.set(i, schema.field(i).with_nullable(c.nullable is not False))
+
+    def batches() -> Iterator[pa.RecordBatch]:
+        for df in frames:
+            yield from arrow(df).cast(schema).to_batches()
+
+    return pa.RecordBatchReader.from_batches(schema, batches())
+
+
+def partition_fields(entity: EntityConfig) -> PartitionFields | None:
+    """`partition_spec` as `ensure_table` takes it; a bucket is PyIceberg's `bucket[N]` (D12)."""
+    spec = entity.sink.accepted.partition_spec
+    if spec is None:
+        return None
+    return [(f.column, f.transform if f.num_buckets is None else f"bucket[{f.num_buckets}]") for f in spec]  # fmt: skip
+
+
+def write_iceberg(
+    entity: EntityConfig,
+    target: IcebergTarget,
+    frames: list[pl.DataFrame],
+    names: list[str],
+    mode: WriteMode,
+    run_id: str,
+    digest: str,
+) -> tuple[IcebergWriteResult, dict[str, Any], dict[str, Any] | None]:
+    """`frames` hold the output columns `names`; returns the result, its `accepted_output` table and merge fields."""
+    reader = _reader(entity, frames, names)
+    options = entity.sink.accepted.merge
+    scd2 = options.scd2 if options else None
+    # (is_current, valid_from, valid_to), as Rust `resolve_scd2_system_columns`
+    system = [(d if (v := getattr(scd2, f, None)) is None else v).strip() for f, d in SCD2_DEFAULTS.items()]  # fmt: skip
+    table_schema = reader.schema
+    if mode == "merge_scd2":  # the table also holds the system columns (Rust `scd2.rs`)
+        for name, kind, nullable in zip(system, (pa.bool_(), pa.timestamp("us"), pa.timestamp("us")), (False, False, True)):  # fmt: skip
+            table_schema = table_schema.append(pa.field(name, kind, nullable=nullable))
+    arrow_schema = Catalog._convert_schema_if_needed(table_schema)  # pyright: ignore[reportPrivateUsage] -- PyIceberg's own pyarrow -> Iceberg conversion
+    schema: Schema = assign_fresh_schema_ids(arrow_schema)
+    evolution = entity.schema_.schema_evolution
+    catalog = open_catalog(target.catalog, target.storage)
+    output = {c.name: n for c, n in zip(entity.schema_.columns, names, strict=True)}
+    primary_key = entity.schema_.primary_key or []  # validated non-empty for merges
+    key = [output.get(k.strip(), k.strip()) for k in primary_key]
+    identifier = (target.namespace, target.table)
+    with run_error("rest catalog table_exists check failed"):
+        exists = mode in ("merge_scd1", "merge_scd2") and catalog.table_exists(identifier)  # fmt: skip
+    if exists:
+        with run_error("rest catalog load_table failed"):
+            existing = set(catalog.load_table(identifier).schema().column_names)
+        if added := [k for k in key if k not in existing]:  # Rust `merge/shared.rs`
+            raise FloeRunError(f"entity.name={entity.name} iceberg schema evolution failed: merge key columns cannot be added: {', '.join(added)}")  # fmt: skip
+    table = ensure_table(
+        catalog, target.namespace, target.table, schema, entity.name, target.location.removeprefix("local://"),
+        partition_fields(entity), evolution.mode if evolution else "strict",
+    )  # fmt: skip
+    merge: dict[str, Any] | None = None
+    if mode in ("merge_scd1", "merge_scd2"):
+        ignore = {output.get(c.strip(), c.strip()) for c in (options.ignore_columns or [])} if options else set()  # fmt: skip
+        started = time.monotonic()
+        with open_duckdb(target.catalog, target.storage) as con:
+            if mode == "merge_scd1":
+                merged = merge_scd1(con, table, reader, key, ignore)
+            else:
+                compare = options and options.compare_columns
+                compare = list(dict.fromkeys(output.get(c.strip(), c.strip()) for c in compare if c.strip())) if compare else None  # fmt: skip
+                merged = merge_scd2(con, table, reader, key, compare, ignore, system)
+        result = merged.write
+        merge = {
+            "merge_key": primary_key,  # Rust's Delta merge reports the configured key
+            "inserted_count": merged.inserted_count,
+            "updated_count": merged.updated_count,
+        }
+        if mode == "merge_scd2":  # Rust's report omits these for SCD1
+            merge |= {"closed_count": merged.closed_count, "unchanged_count": merged.unchanged_count}  # fmt: skip
+        merge["merge_elapsed_ms"] = int((time.monotonic() - started) * 1000)
+    else:
+        options = entity.sink.accepted.options
+        size = options.max_size_per_file if options else None
+        # Rust `default_small_file_threshold_bytes`
+        threshold = SMALL_FILE_THRESHOLD_BYTES if size is None else max(1, size // 2)
+        append = "overwrite" if mode == "overwrite" else "append"
+        unique = [[output.get(c, c) for c in k] for k in resolve_unique_keys(entity.schema_.unique_keys, [c.name for c in entity.schema_.columns if c.unique], entity.schema_.primary_key)]  # fmt: skip
+        result = write_table(table, reader, append, run_id, digest, threshold, unique_keys=unique)  # fmt: skip
+    ids = {"table_version": result.metadata_version, "snapshot_id": result.snapshot_id}
+    fields = {k: v for k, v in ids.items() if v is not None} | {
+        "iceberg_catalog_name": target.catalog.name,
+        "iceberg_namespace": target.namespace,
+        "iceberg_table": target.table,
+    }
+    return result, fields, merge