floe-python 0.8.3__py3-none-any.whl → 0.8.4__py3-none-any.whl

floe/checks/normalize.py

@@ -76,7 +76,7 @@ def source_names(columns: Sequence[ColumnConfig], strategy: str | None) -> list[
         name = normalize_name(source, strategy)
         if name in seen:
             raise FloeConfigError(
-                f"column source collision: {seen[name]} and {column.name} -> {name}"
+                f"column source collision: {seen[name]} and {source} -> {name}"
             )
         seen[name] = source
     return list(seen)

floe/checks/unique.py

@@ -4,12 +4,13 @@
 # pyright: reportUnknownVariableType=false, reportUnknownArgumentType=false
 
 import math
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field
 from datetime import datetime
 from decimal import Decimal
 
 import polars as pl
+import polars.selectors as cs
 import pyarrow as pa
 
 from floe.checks.rows import RowCheck
@@ -245,3 +246,52 @@ class UniqueTracker:
             for row in top.iter_rows(named=True)
         ]
         return UniqueResult(c.columns, c.duplicates, samples, c.target)
+
+
+def seed_from_history(
+    tracker: UniqueTracker,
+    history: pl.LazyFrame,
+    batch: pl.DataFrame,
+    stored: Mapping[str, str],
+) -> None:
+    """Seed `tracker` with the keys of `history` that `batch` also holds; `stored` maps runtime to history names.
+
+    A history key the batch lacks is never hit, so the flags are Rust's while memory follows the
+    batch: `history` is streamed, key columns only. The join matches -0.0 to 0.0 and NaN to any
+    NaN; the tracker then compares float bits, so those extra keys are never hit.
+    A key with a column `history` lacks (`add_columns` adds it on write) is skipped: historical
+    rows hold NULL there, which never duplicates.
+    """
+    columns = set(history.collect_schema().names())
+    for key in tracker.keys:
+        names = [stored.get(c, c) for c in key]
+        if not columns.issuperset(names):
+            continue
+        # positional names (k0, b0, ...): no stored column name can collide with a temporary
+        ids = [f"k{i}" for i in range(len(key))]
+        distinct = (
+            # by_name: a column named like a pattern (`*`, `^...$`) is selected literally
+            batch.select(cs.by_name(c).alias(k) for c, k in zip(key, ids, strict=True))
+            .drop_nulls()
+            .unique()
+        )
+        if not distinct.height:  # all-NULL keys are never checked; nothing to look up
+            continue
+        # distinct by float bits, as the tracker keys them: `unique` on a float merges -0.0 with 0.0
+        floats = [k for c, k in zip(key, ids, strict=True) if batch.schema[c].is_float()]  # fmt: skip
+        bits = {f"b{k}": pl.col(k).map_batches(_float_bits, return_dtype=pl.UInt64, is_elementwise=True) for k in floats}  # fmt: skip
+        subset = [k for k in ids if k not in floats] + list(bits)
+        scan = history.select(cs.by_name(n).alias(k) for n, k in zip(names, ids, strict=True))  # fmt: skip
+        # the history's dtypes can differ (unsigned widened, datetimes in us); a key its type
+        # cannot hold casts to NULL, which never matches
+        lookup = distinct.cast(scan.collect_schema(), strict=False).lazy()
+        hits = (
+            scan.join(lookup, on=ids, how="semi")
+            .with_columns(**bits)
+            # distinct keys only: memory follows the batch, not the history
+            .unique(subset)
+            .drop(*bits)
+            .collect(engine="streaming")
+        )
+        back = pl.Schema({k: batch.schema[c] for c, k in zip(key, ids, strict=True)})
+        tracker.seed(hits.cast(back).rename(dict(zip(ids, key, strict=True))))

floe/config/validate.py

@@ -4,7 +4,12 @@ import re
 
 from floe import __version__
 from floe.checks.cast import decimal_type
-from floe.checks.normalize import normalize_name, output_names
+from floe.checks.normalize import (
+    normalize_name,
+    output_names,
+    resolve_strategy,
+    source_names,
+)
 from floe.config.project import (
     LineageConfig,
     RootConfig,
@@ -410,6 +415,10 @@ def validate_schema(entity: EntityConfig, version: str) -> None:
             f"schema.normalize_columns.strategy={strategy} is unsupported "
             f"(allowed: {', '.join(NORMALIZE_STRATEGIES)})",
         )
+    try:
+        source_names(schema.columns, resolve_strategy(schema))
+    except FloeConfigError as err:
+        raise _fail(entity, str(err)) from err
     mismatch = schema.mismatch
     if mismatch is not None:
         for field, policy, allowed in (

floe/iceberg/catalog.py

@@ -19,7 +19,7 @@ from pyiceberg.types import NestedField
 
 from floe.config.storage import CatalogDefinition, StorageDefinition
 from floe.config.types import SchemaEvolutionMode
-from floe.errors import FloeConfigError, FloeRunError
+from floe.errors import FloeConfigError, FloeError, FloeRunError
 
 # (column, transform) pairs, as in the entity's `sink.accepted.partition_spec`;
 # a bucket is `bucket[N]` (PyIceberg's notation) with N its `num_buckets` (D12)
@@ -44,6 +44,10 @@ def transform_name(field: PartitionField) -> str:
 def run_error(context: str) -> Generator[None]:
     try:
         yield
+    except (
+        FloeError
+    ):  # already worded for the user, e.g. raised by the stream a write reads
+        raise
     except Exception as e:  # PyIceberg has no common base exception
         raise FloeRunError(f"{context}: {e}") from e
 
@@ -308,10 +312,10 @@ def ensure_table(
     partition_fields: PartitionFields | None = None,
     schema_evolution: SchemaEvolutionMode = "strict",
     create_namespace: bool = True,
-) -> Table:
+) -> tuple[Table, list[str]]:
     """Create the namespace (unless `create_namespace` is off) and table if missing; otherwise check the table matches (Rust's append path).
 
-    With `add_columns`, columns new in `schema` are added as optional (D13); partition changes stay rejected.
+    With `add_columns`, columns new in `schema` are added as optional (D13) and returned by name; partition changes stay rejected.
     """
     ident = (namespace,)
     with run_error("rest catalog namespace_exists check failed"):
@@ -348,16 +352,17 @@ def ensure_table(
                     for field in added:
                         update.add_column(field.name, field.field_type, required=False)
                 table.refresh()
-        return table
+        return table, [f.name for f in added]
     spec = (
         None
         if partition_fields is None
         else build_partition_spec(schema, partition_fields, entity)
     )
     with run_error("rest catalog create_table failed"):
-        return catalog.create_table(  # pyright: ignore[reportUnknownMemberType] -- PyIceberg's `schema` is `Schema | Unknown`
+        created = catalog.create_table(  # pyright: ignore[reportUnknownMemberType] -- PyIceberg's `schema` is `Schema | Unknown`
             identifier,
             schema,
             location=location,
             partition_spec=spec if spec is not None else PartitionSpec(),
         )
+    return created, []

floe/iceberg/sink.py

@@ -6,8 +6,9 @@
 import hashlib
 import json
 import time
-from collections.abc import Iterator, Sequence
+from collections.abc import Callable, Iterator, Sequence
 from dataclasses import dataclass
+from itertools import chain
 from pathlib import Path
 from typing import Any
 
@@ -112,23 +113,30 @@ def input_hash(entity: EntityConfig, files: Sequence[InputFile]) -> str:
 _WIDER: dict[pl.DataType, pl.DataType] = {pl.UInt8(): pl.Int32(), pl.UInt16(): pl.Int32(), pl.UInt32(): pl.Int64()}  # fmt: skip
 
 
-def _reader(entity: EntityConfig, frames: list[pl.DataFrame], names: list[str]) -> pa.RecordBatchReader:  # fmt: skip
-    """Output columns in schema order; `nullable: false` columns are required, as in Rust."""
+def _reader(entity: EntityConfig, frames: Iterator[pl.DataFrame], names: list[str]) -> pa.RecordBatchReader:  # fmt: skip
+    """Output columns in schema order, from at least one frame; `nullable: false` columns are required, as in Rust."""
     columns = entity.schema_.columns
-    for name, c in zip(names, columns, strict=True):
-        if c.nullable is False and any(f[name].null_count() for f in frames):
-            raise FloeRunError(f"iceberg write rejected nulls for non-nullable column {name}")  # fmt: skip
 
     def arrow(df: pl.DataFrame) -> pa.Table:  # Iceberg has no unsigned types
         return df.select(pl.col(n).cast(_WIDER.get(df.schema[n], df.schema[n])) for n in names).to_arrow()  # fmt: skip
 
-    schema = arrow(frames[0].head(0)).schema
+    def checked(df: pl.DataFrame) -> pl.DataFrame:
+        for name, c in zip(names, columns, strict=True):
+            if c.nullable is False and df[name].null_count():
+                raise FloeRunError(f"iceberg write rejected nulls for non-nullable column {name}")  # fmt: skip
+        return df
+
+    first = checked(
+        next(frames)
+    )  # a violation in the first chunk fails before the table is created
+    schema = arrow(first.head(0)).schema
+    frames = chain([first], frames)  # `batches` must not keep `first` alive
     for i, c in enumerate(columns):
         schema = schema.set(i, schema.field(i).with_nullable(c.nullable is not False))
 
     def batches() -> Iterator[pa.RecordBatch]:
         for df in frames:
-            yield from arrow(df).cast(schema).to_batches()
+            yield from arrow(checked(df)).cast(schema).to_batches()
 
     return pa.RecordBatchReader.from_batches(schema, batches())
 
@@ -144,13 +152,13 @@ def partition_fields(entity: EntityConfig) -> PartitionFields | None:
 def write_iceberg(
     entity: EntityConfig,
     target: IcebergTarget,
-    frames: list[pl.DataFrame],
+    frames: Iterator[pl.DataFrame],
     names: list[str],
     mode: WriteMode,
     run_id: str,
-    digest: str,
-) -> tuple[IcebergWriteResult, dict[str, Any], dict[str, Any] | None]:
-    """`frames` hold the output columns `names`; returns the result, its `accepted_output` table and merge fields."""
+    digest: Callable[[], str],
+) -> tuple[IcebergWriteResult, dict[str, Any], dict[str, Any] | None, list[str]]:
+    """`frames` (at least one) hold the output columns `names` and are read while the data files are written; returns the result, its `accepted_output` table and merge fields, and the columns `add_columns` added."""
     reader = _reader(entity, frames, names)
     options = entity.sink.accepted.merge
     scd2 = options.scd2 if options else None
@@ -175,7 +183,7 @@ def write_iceberg(
             existing = set(catalog.load_table(identifier).schema().column_names)
         if added := [k for k in key if k not in existing]:  # Rust `merge/shared.rs`
             raise FloeRunError(f"entity.name={entity.name} iceberg schema evolution failed: merge key columns cannot be added: {', '.join(added)}")  # fmt: skip
-    table = ensure_table(
+    table, added = ensure_table(
         catalog, target.namespace, target.table, schema, entity.name, target.location.removeprefix("local://"),
         partition_fields(entity), evolution.mode if evolution else "strict", target.catalog.create_database_if_missing,
     )  # fmt: skip
@@ -213,4 +221,4 @@ def write_iceberg(
         "iceberg_namespace": target.namespace,
         "iceberg_table": target.table,
     }
-    return result, fields, merge
+    return result, fields, merge, added

floe/iceberg/unique.py

@@ -1,61 +1,27 @@
 """Unique keys already in an Iceberg table, for append (Rust `seed_unique_tracker_for_append`)."""
 
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
+from pathlib import Path
 
 import polars as pl
+import polars.selectors as cs
 from pyiceberg.table import Table
 
-from floe.checks.unique import (
-    UniqueTracker,
-    _float_bits,  # pyright: ignore[reportPrivateUsage] -- the tracker's own float key
-)
 from floe.iceberg.catalog import run_error
 
 
-def seed_from_table(
-    tracker: UniqueTracker,
-    table: Table,
-    batch: pl.DataFrame,
-    stored: Mapping[str, str],
-) -> None:
-    """Seed `tracker` with the keys of `table` that `batch` also holds; `stored` maps runtime to table names.
+def cache_table_keys(
+    table: Table, keys: Sequence[Sequence[str]], stored: Mapping[str, str], path: Path
+) -> bool:
+    """Write the key columns of `table` to `path` in one streamed scan; `stored` maps runtime to table names.
 
-    A target key the batch lacks is never hit, so the flags are Rust's while memory follows the
-    batch: the table is streamed, key columns only. The join matches -0.0 to 0.0 and NaN to any
-    NaN; the tracker then compares float bits, so those extra keys are never hit.
-    A key with a column the table lacks (`add_columns` adds it on write) is skipped: historical
-    rows hold NULL there, which never duplicates.
+    A key with a column the table lacks (`add_columns` adds it on write) is left out: the tracker
+    skips it, as historical rows hold NULL there. False when no key is left, and nothing is written.
     """
-    columns = set(table.schema().column_names)
-    for key in tracker.keys:
-        names = [stored.get(c, c) for c in key]
-        if not columns.issuperset(names):
-            continue
-        # positional names (k0, b0, ...): no stored column name can collide with a temporary
-        ids = [f"k{i}" for i in range(len(key))]
-        distinct = (
-            batch.select(pl.col(c).alias(k) for c, k in zip(key, ids, strict=True))
-            .drop_nulls()
-            .unique()
-        )
-        if not distinct.height:  # all-NULL keys are never checked; nothing to look up
-            continue
-        # distinct by float bits, as the tracker keys them: `unique` on a float merges -0.0 with 0.0
-        floats = [k for c, k in zip(key, ids, strict=True) if batch.schema[c].is_float()]  # fmt: skip
-        bits = {f"b{k}": pl.col(k).map_batches(_float_bits, return_dtype=pl.UInt64, is_elementwise=True) for k in floats}  # fmt: skip
-        subset = [k for k in ids if k not in floats] + list(bits)
-        with run_error("iceberg unique seed scan failed"):
-            scan = pl.scan_iceberg(table).select(pl.col(n).alias(k) for n, k in zip(names, ids, strict=True))  # fmt: skip
-            # the table's dtypes can differ (unsigned widened, datetimes in us); a key the table type
-            # cannot hold casts to NULL, which never matches
-            lookup = distinct.cast(scan.collect_schema(), strict=False).lazy()
-            hits = (
-                scan.join(lookup, on=ids, how="semi")
-                .with_columns(**bits)
-                # distinct keys only: memory follows the batch, not the history
-                .unique(subset)
-                .drop(*bits)
-                .collect(engine="streaming")
-            )
-        back = pl.Schema({k: batch.schema[c] for c, k in zip(key, ids, strict=True)})
-        tracker.seed(hits.cast(back).rename(dict(zip(ids, key, strict=True))))
+    present = set(table.schema().column_names)
+    names = {n for k in keys if present.issuperset(ns := [stored.get(c, c) for c in k]) for n in ns}  # fmt: skip
+    if not names:
+        return False
+    with run_error("iceberg unique seed scan failed"):
+        pl.scan_iceberg(table).select(cs.by_name(sorted(names))).sink_parquet(path)
+    return True

floe/iceberg/write.py

@@ -5,7 +5,7 @@
 
 import itertools
 import uuid
-from collections.abc import Iterator, Sequence
+from collections.abc import Callable, Iterator, Sequence
 from dataclasses import dataclass, replace
 from datetime import datetime
 from pathlib import PurePosixPath
@@ -82,7 +82,7 @@ def write_table(
     reader: pa.RecordBatchReader,
     mode: WriteMode,
     run_id: str,
-    input_hash: str,
+    input_hash: str | Callable[[], str],
     small_file_threshold_bytes: int = SMALL_FILE_THRESHOLD_BYTES,
     chunk_rows: int = CHUNK_ROWS,
     unique_keys: Sequence[Sequence[str]] = (),
@@ -104,6 +104,9 @@ def write_table(
                 table.metadata, chunk, table.io, write_uuid, counter
             )
         ]
+    digest = (
+        input_hash() if callable(input_hash) else input_hash
+    )  # known once the stream has been read
     base = (
         table.metadata.current_snapshot_id
     )  # only commits after it can be a rival's copy of this batch
@@ -112,9 +115,7 @@ def write_table(
             _backoff(attempt - 1)
             with run_error("rest iceberg table refresh failed"):
                 table.refresh()
-                seen = mode == "append" and find_committed(
-                    table, input_hash, after=base
-                )
+                seen = mode == "append" and find_committed(table, digest, after=base)
             if seen:
                 return replace(write_result(table, [], small_file_threshold_bytes), already_committed=True)  # fmt: skip
             if mode == "append":
@@ -131,7 +132,7 @@ def write_table(
                 table.metadata = table.metadata.model_copy(update={"properties": {**table.properties, TableProperties.COMMIT_NUM_RETRIES: "0"}})  # fmt: skip
                 tx = table.transaction()
                 update = tx.update_snapshot(
-                    snapshot_properties={RUN_ID: run_id, INPUT_HASH: input_hash}
+                    snapshot_properties={RUN_ID: run_id, INPUT_HASH: digest}
                 )
                 with (
                     update.overwrite()

floe/report/entity.py

@@ -157,6 +157,7 @@ def build_report(
     table_root_uri: str | None = None,
     table: Mapping[str, Any] | None = None,
     merge: Mapping[str, Any] | None = None,
+    added_columns: Sequence[str] = (),
 ) -> dict[str, Any]:
     """The entity's `run.json`; `files` are the `file_report`s, in input order.
 
@@ -234,9 +235,10 @@ def build_report(
             **(merge or {}),
         },
         "schema_evolution": {
-            "enabled": False,
+            "enabled": evolution.mode == "add_columns",
             "mode": evolution.mode,
-            "applied": False,
+            "applied": bool(added_columns),
+            **({"added_columns": list(added_columns)} if added_columns else {}),
             "incompatible_changes_detected": False,
         },
     }  # fmt: skip

floe/run/entity.py

@@ -4,7 +4,7 @@ import io
 import tempfile
 import threading
 import time
-from collections.abc import Callable, Iterator
+from collections.abc import Callable, Generator, Iterable, Iterator
 from contextlib import suppress
 from dataclasses import dataclass, field, replace
 from datetime import datetime
@@ -39,7 +39,12 @@ from floe.checks.rows import (
     not_null_check,
     split_rows,
 )
-from floe.checks.unique import UniqueResult, UniqueTracker, resolve_unique_keys
+from floe.checks.unique import (
+    UniqueResult,
+    UniqueTracker,
+    resolve_unique_keys,
+    seed_from_history,
+)
 from floe.config.storage import CatalogsConfig, StorageDefinition, StoragesConfig
 from floe.config.types import EntityConfig, with_partition_columns
 from floe.errors import FloeConfigError, FloeIoError, FloeRunError
@@ -49,7 +54,7 @@ from floe.iceberg.sink import (
     resolve_target,
     write_iceberg,
 )
-from floe.iceberg.unique import seed_from_table
+from floe.iceberg.unique import cache_table_keys
 from floe.iceberg.write import find_committed
 from floe.pii import apply_pii_masking
 from floe.read.avro import read_avro
@@ -253,6 +258,9 @@ class _Plan:
     table: Table | None = (
         None  # the Iceberg target whose keys seed the tracker (append)
     )
+    history: pl.LazyFrame | None = (
+        None  # the target's key columns, cached on local disk (append, #1057)
+    )
     merge_key: str | None = (
         None  # the merge key's unique check; its duplicates are rejected even under warn (Rust `enforce_reject`)
     )
@@ -354,14 +362,29 @@ def _typed(
     if entity.policy.severity == "abort" and any(c.mask.any() for c in cast_checks):
         return typed, cast_checks  # fmt: skip  # Rust: abort-mode cast errors skip not_null and unique
     required = [not_null_check(typed, n) for n in plan.required]
-    if plan.table is not None:
-        # ponytail: one key scan of the table per chunk; a DuckDB anti-join (ADR) if large histories make it slow
-        seed_from_table(plan.tracker, plan.table, typed, plan.out_map)
+    if plan.history is not None:
+        seed_from_history(plan.tracker, plan.history, typed, plan.out_map)
     return typed, [*required, *cast_checks, *plan.tracker.check(typed)]
 
 
-def _run_file(plan: _Plan, file: InputFile, path: Path) -> tuple[dict[str, Any], list[pl.DataFrame]]:  # fmt: skip
-    """The file's report and its accepted frames (none when it aborted the run); `path` is the local copy."""
+def _unspill(path: Path) -> pl.DataFrame:
+    df = pl.read_ipc(path)
+    path.unlink()
+    return df
+
+
+def _replay(plan: _Plan, held: list[pl.DataFrame | Path]) -> Iterator[pl.DataFrame]:
+    while held:
+        item = held.pop(0)
+        yield _output(plan, item if isinstance(item, pl.DataFrame) else _unspill(item))
+
+
+def _run_file(plan: _Plan, file: InputFile, path: Path, spill: Path) -> tuple[dict[str, Any], Iterable[pl.DataFrame]]:  # fmt: skip
+    """The file's report and its accepted frames, one by one (none when it aborted the run); `path` is the local copy.
+
+    A file's rows are accepted only once it is read to the end (a late read error or an abort drops them all), so
+    every chunk after the first waits in `spill`, not in memory.
+    """
     entity, uri = plan.entity, file.source_uri
     severity = entity.policy.severity
     derived = plan.derived.get(uri, [])
@@ -372,7 +395,7 @@ def _run_file(plan: _Plan, file: InputFile, path: Path) -> tuple[dict[str, Any],
     chunks = _chunks(entity, path, file_names)
     keys = plan.tracker.snapshot()
 
-    def failed(err: FloeIoError, mismatch: MismatchOutcome | None) -> tuple[dict[str, Any], list[pl.DataFrame]]:  # fmt: skip
+    def failed(err: FloeIoError, mismatch: MismatchOutcome | None) -> tuple[dict[str, Any], Iterable[pl.DataFrame]]:  # fmt: skip
         # Rust skips the raw copy under PII: the file cannot be masked
         plan.tracker.restore(keys)  # the unreadable file leaves no keys behind
         keep = entity.pii is None and entity.sink.rejected is not None
@@ -403,7 +426,7 @@ def _run_file(plan: _Plan, file: InputFile, path: Path) -> tuple[dict[str, Any],
 
     rows, bad_total, forced_total = 0, 0, 0
     masks: dict[tuple[str, str, str], list[pl.Series]] = {}
-    accepted: list[pl.DataFrame] = []
+    accepted: list[pl.DataFrame | Path] = []
     rejected: list[pl.DataFrame] = []
     errors: list[tuple[int, str]] = []
     try:
@@ -423,7 +446,12 @@ def _run_file(plan: _Plan, file: InputFile, path: Path) -> tuple[dict[str, Any],
                 forced = forced.fill_null(False)
                 forced_total += int(forced.sum())
                 typed = typed.filter(~forced)
-            accepted.append(good if severity == "reject" else typed)
+            frame = good if severity == "reject" else typed
+            if accepted:
+                frame.write_ipc(spilled := spill / f"{len(accepted)}.ipc")
+                accepted.append(spilled)
+            else:
+                accepted.append(frame)
             if severity == "reject" and bad.height:
                 rejected.append(bad)
     except FloeIoError as err:
@@ -445,22 +473,30 @@ def _run_file(plan: _Plan, file: InputFile, path: Path) -> tuple[dict[str, Any],
         errors_path = _error_report(plan, file.source_stem, errors)
     plan.to_archive.append(file)
     out = file_report(entity, uri, rows, bad_total, merged, mismatch, rejected_path, None, errors_path, forced_total)  # fmt: skip
-    return out, [] if out["status"] == "aborted" else [_output(plan, df) for df in accepted]  # fmt: skip
+    return out, [] if out["status"] == "aborted" else _replay(plan, accepted)
+
 
+def _key_cache(plan: _Plan, client: StorageClient, base: str, temp: Path) -> pl.LazyFrame | None:  # fmt: skip
+    """Append mode: the key columns of the target's existing data, written once to local disk (#1057).
 
-def _seed_unique(plan: _Plan, client: StorageClient, base: str, columns: list[str]) -> None:  # fmt: skip
-    """Append mode: the keys already in the sink's parts count as seen (Rust `seed_unique_tracker_for_append`)."""
-    if not columns:
-        return  # no unique keys: never read the history
-    stored = {n: plan.out_map.get(n, n) for n in columns}  # the sink holds output names
-    for uri in part_uris(client, base.rstrip("/"), "parquet"):
-        part = uri.removeprefix("local://")
-        try:  # only the key columns are read
-            body = client.read(uri)
-            keys = pl.read_parquet(io.BytesIO(body.body if body else b""), columns=list(stored.values()))  # fmt: skip
-        except pl.exceptions.PolarsError as err:
-            raise FloeRunError(f"entity.name={plan.entity.name} cannot seed unique keys from {part}: {err}") from err  # fmt: skip
-        plan.tracker.seed(keys.rename({v: k for k, v in stored.items()}))
+    Each chunk then joins its keys against this cache, so the target is read once per entity and
+    memory follows the chunk, not the history (Rust `seed_unique_tracker_for_append`).
+    """
+    cache = temp / "unique-keys"
+    cache.mkdir()
+    if plan.table is not None:
+        cache_table_keys(plan.table, plan.tracker.keys, plan.out_map, cache / "table.parquet")  # fmt: skip
+    else:
+        columns = sorted({plan.out_map.get(n, n) for k in plan.tracker.keys for n in k})  # the sink holds output names  # fmt: skip
+        for i, uri in enumerate(part_uris(client, base.rstrip("/"), "parquet")):
+            part = uri.removeprefix("local://")
+            try:  # only the key columns are read
+                body = client.read(uri)
+                keys = pl.read_parquet(io.BytesIO(body.body if body else b""), columns=columns)  # fmt: skip
+            except pl.exceptions.PolarsError as err:
+                raise FloeRunError(f"entity.name={plan.entity.name} cannot seed unique keys from {part}: {err}") from err  # fmt: skip
+            keys.write_parquet(cache / f"{i}.parquet")
+    return pl.scan_parquet(cache / "*.parquet") if any(cache.iterdir()) else None
 
 
 def _report_names(
@@ -855,14 +891,7 @@ def _run_entity(
             pk[0] if len(pk) == 1 else f"[{','.join(pk)}]"
         )  # the tracker's check name
     accepted_client = storage_client(accepted)
-    if sink.write_mode == "append" and target is None:
-        _seed_unique(
-            plan,
-            accepted_client,
-            base,
-            sorted({runtime.get(c, c) for k in keys for c in k}),
-        )
-    elif sink.write_mode == "append" and target is not None:
+    if sink.write_mode == "append" and target is not None:
         plan.table = existing_table(target)  # fmt: skip  # Rust `seed_unique_tracker`, and the retry guard
 
     def source() -> ResolvedInputs:
@@ -894,52 +923,73 @@ def _run_entity(
         plan.to_archive += [f for f in pending if f.source_uri in skip]
         pending = [f for f in pending if f.source_uri not in skip]
     plan.input_hash = input_hash(entity, pending)
-    frames: list[pl.DataFrame] = []
     # #545: the snapshot's fingerprint names only the files it holds in full, never an aborted batch
     complete: list[InputFile] = []
     skipped = len(files)
     downloader = None if isinstance(client, LocalClient) else client
-    # Rust `localize_input`: a remote input is read from a local copy
-    with tempfile.TemporaryDirectory(prefix="floe-") as temp_dir:
-        for f in pending:
-            emit("file_started", entity=entity.name, input=f.source_uri)
-            timer = time.monotonic()
-            local = localize_input(f, downloader, Path(temp_dir))
-            report, kept = _run_file(plan, f, local.local_path)
-            if local.is_ephemeral:
-                local.local_path.unlink()
-            emit("file_finished", entity=entity.name, input=f.source_uri, status=report["status"],
-                 rows=report["row_count"], accepted=report["accepted_count"],
-                 rejected=report["rejected_count"], elapsed_ms=int((time.monotonic() - timer) * 1000))  # fmt: skip
-            files.append(report)
-            frames += kept
-            if report["status"] == "aborted":
-                break
-            complete.append(f)
-    files[skipped:] = sorted(files[skipped:], key=lambda r: r["input_file"] not in plan.prechecked)  # fmt: skip
+    failure: list[Exception] = []
+
+    def accepted_frames() -> Generator[pl.DataFrame]:
+        """Files are read as the sink pulls (#1056); `files` and `complete` are final once it is exhausted."""
+        try:
+            # Rust `localize_input`: a remote input is read from a local copy
+            with tempfile.TemporaryDirectory(prefix="floe-") as temp_dir:
+                if pending and sink.write_mode == "append" and plan.tracker.keys and (target is None or plan.table is not None):  # fmt: skip
+                    plan.history = _key_cache(plan, accepted_client, base, Path(temp_dir))  # fmt: skip
+                for f in pending:
+                    emit("file_started", entity=entity.name, input=f.source_uri)
+                    timer = time.monotonic()
+                    local = localize_input(f, downloader, Path(temp_dir))
+                    report, kept = _run_file(plan, f, local.local_path, Path(temp_dir))
+                    if local.is_ephemeral:
+                        local.local_path.unlink()
+                    emit("file_finished", entity=entity.name, input=f.source_uri, status=report["status"],
+                         rows=report["row_count"], accepted=report["accepted_count"],
+                         rejected=report["rejected_count"], elapsed_ms=int((time.monotonic() - timer) * 1000))  # fmt: skip
+                    files.append(report)
+                    yield from kept
+                    if report["status"] == "aborted":
+                        break
+                    complete.append(f)
+        except Exception as err:
+            # the sinks wrap what they read; this error reaches the caller as it is
+            failure.append(err)
+            raise
 
     table: dict[str, Any] | None = None
     merge: dict[str, Any] | None = None
-    if pending and (
-        frames or sink.write_mode == "overwrite"
-    ):  # Rust writes nothing when every input was skipped
-        empty = pl.DataFrame(schema=dict.fromkeys(names, pl.String)).select(
-            cast_column(n, c.column_type)[0]
-            for n, c in zip(names, columns, strict=True)
-        )
-        frames = frames or [_output(plan, empty)]
-        if target is None:
-            write = write_accepted(pl.concat(frames), accepted_client, base, sink.write_mode, sink.accepted.options)  # fmt: skip
+    added: list[str] = []
+    stream = accepted_frames()
+    try:
+        first = next(stream, None)  # Rust writes nothing when every input was skipped
+        if pending and (first is not None or sink.write_mode == "overwrite"):
+            empty = pl.DataFrame(schema=dict.fromkeys(names, pl.String)).select(
+                cast_column(n, c.column_type)[0]
+                for n, c in zip(names, columns, strict=True)
+            )
+            frames = chain([_output(plan, empty) if first is None else first], stream)
+            del first  # the sink holds the chunk it is on, not the first
+            if target is None:
+                write = write_accepted(frames, accepted_client, base, sink.write_mode, sink.accepted.options)  # fmt: skip
+            else:
+                outputs = [plan.out_map.get(n, n) for n in names]
+                result, table, merge, added = write_iceberg(entity, target, frames, outputs, sink.write_mode, emit.run_id, lambda: input_hash(entity, complete))  # fmt: skip
+                write = AcceptedWrite(result.files_written, result.file_paths, result.total_bytes_written,
+                                      result.avg_file_size_mb, result.small_files_count)  # fmt: skip
+                if added:
+                    emit("schema_evolution_applied", entity=entity.name, mode="add_columns", added_columns=added)  # fmt: skip
+                if result.already_committed:  # #920: the commit conflicted and the guard found this batch in the table
+                    done = {f.source_uri for f in complete}
+                    files[:] = [skipped_report(r["input_file"], "already_committed") if r["input_file"] in done else r for r in files]  # fmt: skip
         else:
-            outputs = [plan.out_map.get(n, n) for n in names]
-            result, table, merge = write_iceberg(entity, target, frames, outputs, sink.write_mode, emit.run_id, input_hash(entity, complete))  # fmt: skip
-            write = AcceptedWrite(result.files_written, result.file_paths, result.total_bytes_written,
-                                  result.avg_file_size_mb, result.small_files_count)  # fmt: skip
-            if result.already_committed:  # #920: the commit conflicted and the guard found this batch in the table
-                done = {f.source_uri for f in complete}
-                files[:] = [skipped_report(r["input_file"], "already_committed") if r["input_file"] in done else r for r in files]  # fmt: skip
-    else:
-        write = AcceptedWrite(0, [], 0, 0.0, 0)
+            write = AcceptedWrite(0, [], 0, 0.0, 0)
+    except Exception:
+        if not failure:
+            raise
+        raise failure[0] from failure[0].__cause__
+    finally:
+        stream.close()
+    files[skipped:] = sorted(files[skipped:], key=lambda r: r["input_file"] not in plan.prechecked)  # fmt: skip
     inc.committed = files  # #915: from here on a failure promotes the written files
     # #914: archive only once the sink committed, so a failed commit leaves the inputs in the source (Rust archives before)
     by_uri = {r["input_file"]: r for r in files}
@@ -959,5 +1009,6 @@ def _run_entity(
         table_root_uri=target.location if target and table else None,
         table=table,
         merge=merge,
+        added_columns=added,
     )
     return EntityRun(report, any(f["status"] == "aborted" for f in files))

floe/write/outputs.py

@@ -4,7 +4,7 @@ import io
 import math
 import re
 import uuid
-from collections.abc import Callable, Generator
+from collections.abc import Callable, Generator, Iterable, Iterator
 from contextlib import contextmanager, suppress
 from dataclasses import dataclass
 
@@ -59,38 +59,32 @@ def part_uris(client: StorageClient, base: str, ext: str) -> list[str]:
 
 @contextmanager
 def _parts(client: StorageClient, base: str, mode: WriteMode, ext: str) -> Generator[Callable[[bytes], str]]:  # fmt: skip
-    """Yield `put(body) -> name`. Non-overwrite modes write `part-<uuid>` directly.
+    """Yield `put(body) -> name`. Every part is staged as `_staging-<id>-<name>` (invisible to `part_uris`).
 
-    Overwrite stages every part as `_staging-<id>-part-NNNNN` (invisible to `part_uris`) and
-    only when all were written copies them over `part-NNNNN`, deletes the old parts that no new
-    part replaced, then the staged copies (#1054). A failed write leaves the old parts untouched.
+    Only when all were written are they copied over `<name>` and the staged copies deleted (#1054).
+    Overwrite names the parts `part-NNNNN` and deletes the old parts that no new part replaced, so a
+    failed write leaves them untouched. Other modes name them `part-<uuid>` and keep the old parts;
+    staging keeps a killed run from leaving visible parts for a retry to duplicate (#1056: parts are
+    written while the input is still read). A failed write deletes the staged parts it wrote.
     """
-    if mode != "overwrite":
-
-        def put(body: bytes) -> str:
-            name = f"part-{uuid.uuid4()}.{ext}"
-            client.write(f"{base}/{name}", body)
-            return name
-
-        yield put
-        return
+    overwrite = mode == "overwrite"
     stage = f"_staging-{uuid.uuid4()}-"
     names: list[str] = []
 
-    def put_staged(body: bytes) -> str:
-        name = f"part-{len(names):05}.{ext}"
+    def put(body: bytes) -> str:
+        name = f"part-{len(names):05}.{ext}" if overwrite else f"part-{uuid.uuid4()}.{ext}"  # fmt: skip
         names.append(name)  # before the write: a partial file needs cleanup too
         client.write(f"{base}/{stage}{name}", body)
         return name
 
     try:
-        yield put_staged
+        yield put
     except BaseException:
         for name in names:
             with suppress(FloeError, OSError):  # best effort: the original error wins
                 client.delete(f"{base}/{stage}{name}")
         raise
-    old = part_uris(client, base, ext)
+    old = part_uris(client, base, ext) if overwrite else []
     for name in names:
         client.copy(f"{base}/{stage}{name}", f"{base}/{name}")
     for uri in set(old) - {f"{base}/{n}" for n in names}:
@@ -99,18 +93,38 @@ def _parts(client: StorageClient, base: str, mode: WriteMode, ext: str) -> Gener
         client.delete(f"{base}/{stage}{name}")
 
 
+def _part_frames(
+    frames: Iterable[pl.DataFrame], max_size: int
+) -> Iterator[pl.DataFrame]:
+    """`frames` regrouped into parts of about `max_size` bytes in memory; at most one part and a frame are pooled."""
+    pool: list[pl.DataFrame] = []
+    size = height = 0
+    for df in frames:
+        pool.append(df)
+        size += df.estimated_size()
+        height += df.height
+        rows = max(1, max_size // max(1, math.ceil(size / height))) if height else 1
+        if height >= rows:
+            whole = pl.concat(pool)
+            full = height - height % rows
+            yield from (whole.slice(i, rows) for i in range(0, full, rows))
+            rest = whole.slice(full)
+            pool = [rest] if rest.height else []
+            size, height = rest.estimated_size(), rest.height
+    if pool:  # also the one empty part of a run that accepted no row
+        yield pl.concat(pool)
+
+
 def write_accepted(
-    df: pl.DataFrame,
+    frames: Iterable[pl.DataFrame],
     client: StorageClient,
     base: str,
     mode: WriteMode = "overwrite",
     options: SinkOptions | None = None,
 ) -> AcceptedWrite:
-    """Write `df` as `part-*.parquet` files under `base`, split so each holds about `max_size_per_file`."""
+    """Write `frames` (at least one) as `part-*.parquet` files under `base`, each holding about `max_size_per_file`."""
     base = base.rstrip("/")
     max_size = (options and options.max_size_per_file) or DEFAULT_MAX_SIZE_PER_FILE
-    avg_row = max(1, math.ceil(df.estimated_size() / df.height)) if df.height else 1
-    rows = max(1, max_size // avg_row)
     kwargs: dict[str, str | int] = {}
     if options and options.compression:
         kwargs["compression"] = options.compression
@@ -119,7 +133,7 @@ def write_accepted(
     files: list[str] = []
     sizes: list[int] = []
     with _os_errors(), _parts(client, base, mode, "parquet") as put:
-        for chunk in [df.slice(i, rows) for i in range(0, df.height, rows)] or [df]:
+        for chunk in _part_frames(frames, max_size):
             # ponytail: a part is held in memory (≤ max_size_per_file) before the put; stream multipart uploads if that is too much
             body = io.BytesIO()
             chunk.write_parquet(body, **kwargs)  # pyright: ignore[reportArgumentType]  # ponytail: kwargs loses the compression Literal; validated by the config

{floe_python-0.8.3.dist-info → floe_python-0.8.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: floe-python
-Version: 0.8.3
+Version: 0.8.4
 Summary: Floe: a data-contract runtime for file ingestion
 License-File: LICENSE
 Requires-Python: >=3.11

{floe_python-0.8.3.dist-info → floe_python-0.8.4.dist-info}/RECORD

@@ -13,9 +13,9 @@ floe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/state.py,sha256=XFHF6zVxbdGg1bkONZDLryTD4noDF9IgaimaEI-3Nlg,11104
 floe/checks/cast.py,sha256=lM40xtKEggWI8FW3jijM3T2JFQtaXs8FhsFXo3vy5oM,6788
 floe/checks/mismatch.py,sha256=2Gz-tiWKxy-iehSjAlsGzPLtYeNIfBWvi4yQPlASVYQ,5309
-floe/checks/normalize.py,sha256=INYh3NO3JkdgDjTXDi7pylr79nzrJmDuBlrIrm_8LRc,4326
+floe/checks/normalize.py,sha256=8ZfpGFauFLIrq_xHY3iIe-rtuKO88YACDVO0WQJFWCU,4321
 floe/checks/rows.py,sha256=E-YUZUvBUyOB7zat2dtbPWFM-0T1-01lYKuQU1A6wto,4156
-floe/checks/unique.py,sha256=BUu6ezLtQKj8_9_SybGND6rxbUbzlmqh_WJomTHRfGw,9782
+floe/checks/unique.py,sha256=qoZVqS_uG_mWZlV7-LdxT6_dMVQRlklEvMutO_V0qaY,12396
 floe/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/config/entity.py,sha256=CLXLXzkUbQhxenD_LwjBt0eoPreaYaxiBVEsdmnFIco,15883
 floe/config/include.py,sha256=XKYhp2QK_VUmAODAXJ35Wl4FCpQEMQm24m-WKlpI_og,5033
@@ -24,15 +24,15 @@ floe/config/profile.py,sha256=jWackztS-0H3dTMXHALTZ7LsIOmwjriwoeY4vFgARr0,16046
 floe/config/project.py,sha256=zgTOhTDlbMr1chViMEZyv0drozZdumTaIoKDL0vv9sw,14141
 floe/config/storage.py,sha256=BK3P65ukFqNgmbjCcuYmXR04OScf1yD8f19vGJq-aDI,6234
 floe/config/types.py,sha256=4eveoOlyGmNxyZpI1PSjli9lnXLPc1-jYj4fL8qHN_8,5038
-floe/config/validate.py,sha256=-cDn9CKfR1BmAATgYy1PWlTWSkpu4mjt4PMKNL7k2FE,35858
+floe/config/validate.py,sha256=4lU6yJ1qpI8AIr42SGy-xJR2brz_DBG6pNS_dkZw4zo,36065
 floe/config/vars.py,sha256=7Bq0IU0KrFeRlyA8W9-F0gDSDKfXCfFpBrX08H8Flvw,4577
 floe/config/yaml.py,sha256=HeFGhKFxYkiqQOwljYOgUD5naJyxL_ZIX6hUIS36tzU,10071
-floe/iceberg/catalog.py,sha256=QW8L6DKVr4Ug0CbuEP4wgy7Lh5-TsoOXRZWQxfM5zSQ,15115
+floe/iceberg/catalog.py,sha256=n2NlgqRgnTe3Etdyra70Ib2aYtpR7zXWrajKl7spyYg,15340
 floe/iceberg/compact.py,sha256=5O372vAIsvgRc2PGihryAsrQ0otn0F30AQu-cetRCP0,7123
 floe/iceberg/merge.py,sha256=VE1Odf2B34Gxj7ZcuhbrfbgTGM9AdiDz6_2BbcQ-kSQ,12448
-floe/iceberg/sink.py,sha256=uE0jLhhxRFOeKi7ChAzstKexy1ZrovE4EKATGTYn-Gk,11255
-floe/iceberg/unique.py,sha256=PKUFEcCkU71k66__KfUE7NTos73ZYlZbb8GJPn0yFoo,2958
-floe/iceberg/write.py,sha256=zrFeeJu-YT4Pxo9u-fRujUtFb1Epx1KFoq2qFyP1iGs,11036
+floe/iceberg/sink.py,sha256=ySEAsKYpaKnqF9t7WjOW03KwpKynNIqLxOP1kQ-0Tbg,11723
+floe/iceberg/unique.py,sha256=OmiPQthhB5jGTaXsCu9ZztNFEFUf6eT0MLU2Gnh-C2A,1095
+floe/iceberg/write.py,sha256=SRX5tb17zUeWwpg6uXKkVQ_muNJ2GqlQd0n-kmjWPVc,11141
 floe/lineage/events.py,sha256=4txd8bGUwKBZd-8Aw_ebTurhlxAYk8SVMgH8vSjC3bs,6682
 floe/lineage/observer.py,sha256=dDYfWvPpFGjdqvFqQDZ2pTjj7uDfqji2-BTID9hzg90,10075
 floe/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -43,10 +43,10 @@ floe/read/orc.py,sha256=TWgdlLBR2NF_oVZ5zEmRKNXoTkq5QfF2BOFzKyzSjhY,1553
 floe/read/parquet.py,sha256=JwfuWaHj2TxBKJeaKignq8p0Op_pUfsOdVh7dDC3wcE,1491
 floe/read/xlsx.py,sha256=dengSY8HuNh9ahyR_LR8beqXTZQubqQ1Jx8qEak5jrY,3009
 floe/read/xml.py,sha256=7m6DtoU8iExuLROavyojysqOUE_Bd_N0uwFaLAzjI_Y,8004
-floe/report/entity.py,sha256=ekuB-1JDATvxIFnBTCQpku8uXtrSQNwvryjdTidXk6I,9707
+floe/report/entity.py,sha256=FRwLtXrlJXDNXrOKscitm-NCNJWQXd0vr_JIf_cUp20,9867
 floe/report/summary.py,sha256=tb547VRkM5WGWHQ-4vZB2DUW4VTWgv9FUoym-NpuWE4,4452
 floe/run/__init__.py,sha256=6teCk4GZoX7grwtyhQUbqwbx_TSQebzViUJA19CmEl4,8107
-floe/run/entity.py,sha256=4UGhUtjYR84MOe3jjwFmLGWAaDln1z5KOHC0MOvDLpQ,45204
+floe/run/entity.py,sha256=dML8VR-UyLUNVupkKaHeFIbR06TdPP2qcJ2rAMW4GeU,47568
 floe/run/events.py,sha256=OE7zYkt0Dz2xrvgNavT1nHfZJ-97IIJ8LpHcameUJZU,2859
 floe/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/storage/archive.py,sha256=1gPImm9PDSJFCpWjg37ax4ubb0wwc6FZm38quQXhORg,1101
@@ -58,9 +58,9 @@ floe/storage/resolve.py,sha256=pFAuVytTfnSEcTAVlJjFTj7S_Vipjry1ormWZP3Egmc,2593
 floe/storage/s3.py,sha256=XcG1qJTEY3KLSMLT950mUdrAxuCA8-Gx3fpNSzyV_Xo,7508
 floe/storage/types.py,sha256=FvAEXWMdVdopRK276JyETRYGOva2-uR8pZvsWj6Xvy0,993
 floe/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-floe/write/outputs.py,sha256=mPS3W2bHgeNR_FJ3Ghb13gMgiPpk5-mpSad1bwq97Kg,6034
-floe_python-0.8.3.dist-info/METADATA,sha256=DApXCIMAHZD6G1V_b9v5T2APwgUeKUEfWYmUk10EsK4,442
-floe_python-0.8.3.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-floe_python-0.8.3.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
-floe_python-0.8.3.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
-floe_python-0.8.3.dist-info/RECORD,,
+floe/write/outputs.py,sha256=cXxGUIJnZ38XPaqEI6a83GFUsapBRdY52XQm3GlUPcI,6937
+floe_python-0.8.4.dist-info/METADATA,sha256=kGo2pIkok-Gpky0VSm-2S5ttVTHx3tDcE21ynZE-HM4,442
+floe_python-0.8.4.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+floe_python-0.8.4.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
+floe_python-0.8.4.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
+floe_python-0.8.4.dist-info/RECORD,,