floe-python 0.8.2__py3-none-any.whl → 0.8.4__py3-none-any.whl

floe/_cli.py

@@ -40,7 +40,7 @@ def _realpath(path: str) -> Path:
 def _profile(location: str | None) -> ProfileConfig | None:
     if not location:
         return None
-    return parse_profile(location if location.startswith("s3://") else _realpath(location))  # fmt: skip
+    return parse_profile(location if location.startswith(("s3://", "env:")) else _realpath(location))  # fmt: skip
 
 
 def _load(path: Path, profile: ProfileConfig | None) -> RootConfig:
@@ -83,7 +83,7 @@ def _validate(args: argparse.Namespace) -> int:
         print(f"  Severity: {e.policy.severity}")
     profile_arg = ""
     if args.profile:
-        profile_arg = f" -p {args.profile if args.profile.startswith('s3://') else os.path.realpath(args.profile)}"  # fmt: skip
+        profile_arg = f" -p {args.profile if args.profile.startswith(('s3://', 'env:')) else os.path.realpath(args.profile)}"  # fmt: skip
     print(f"Next: floe run -c {path}{profile_arg}")
     return 0
 

floe/checks/normalize.py

@@ -76,7 +76,7 @@ def source_names(columns: Sequence[ColumnConfig], strategy: str | None) -> list[
         name = normalize_name(source, strategy)
         if name in seen:
             raise FloeConfigError(
-                f"column source collision: {seen[name]} and {column.name} -> {name}"
+                f"column source collision: {seen[name]} and {source} -> {name}"
             )
         seen[name] = source
     return list(seen)

floe/checks/unique.py

@@ -4,12 +4,13 @@
 # pyright: reportUnknownVariableType=false, reportUnknownArgumentType=false
 
 import math
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field
 from datetime import datetime
 from decimal import Decimal
 
 import polars as pl
+import polars.selectors as cs
 import pyarrow as pa
 
 from floe.checks.rows import RowCheck
@@ -245,3 +246,52 @@ class UniqueTracker:
             for row in top.iter_rows(named=True)
         ]
         return UniqueResult(c.columns, c.duplicates, samples, c.target)
+
+
+def seed_from_history(
+    tracker: UniqueTracker,
+    history: pl.LazyFrame,
+    batch: pl.DataFrame,
+    stored: Mapping[str, str],
+) -> None:
+    """Seed `tracker` with the keys of `history` that `batch` also holds; `stored` maps runtime to history names.
+
+    A history key the batch lacks is never hit, so the flags are Rust's while memory follows the
+    batch: `history` is streamed, key columns only. The join matches -0.0 to 0.0 and NaN to any
+    NaN; the tracker then compares float bits, so those extra keys are never hit.
+    A key with a column `history` lacks (`add_columns` adds it on write) is skipped: historical
+    rows hold NULL there, which never duplicates.
+    """
+    columns = set(history.collect_schema().names())
+    for key in tracker.keys:
+        names = [stored.get(c, c) for c in key]
+        if not columns.issuperset(names):
+            continue
+        # positional names (k0, b0, ...): no stored column name can collide with a temporary
+        ids = [f"k{i}" for i in range(len(key))]
+        distinct = (
+            # by_name: a column named like a pattern (`*`, `^...$`) is selected literally
+            batch.select(cs.by_name(c).alias(k) for c, k in zip(key, ids, strict=True))
+            .drop_nulls()
+            .unique()
+        )
+        if not distinct.height:  # all-NULL keys are never checked; nothing to look up
+            continue
+        # distinct by float bits, as the tracker keys them: `unique` on a float merges -0.0 with 0.0
+        floats = [k for c, k in zip(key, ids, strict=True) if batch.schema[c].is_float()]  # fmt: skip
+        bits = {f"b{k}": pl.col(k).map_batches(_float_bits, return_dtype=pl.UInt64, is_elementwise=True) for k in floats}  # fmt: skip
+        subset = [k for k in ids if k not in floats] + list(bits)
+        scan = history.select(cs.by_name(n).alias(k) for n, k in zip(names, ids, strict=True))  # fmt: skip
+        # the history's dtypes can differ (unsigned widened, datetimes in us); a key its type
+        # cannot hold casts to NULL, which never matches
+        lookup = distinct.cast(scan.collect_schema(), strict=False).lazy()
+        hits = (
+            scan.join(lookup, on=ids, how="semi")
+            .with_columns(**bits)
+            # distinct keys only: memory follows the batch, not the history
+            .unique(subset)
+            .drop(*bits)
+            .collect(engine="streaming")
+        )
+        back = pl.Schema({k: batch.schema[c] for c, k in zip(key, ids, strict=True)})
+        tracker.seed(hits.cast(back).rename(dict(zip(ids, key, strict=True))))

floe/config/profile.py

@@ -1,5 +1,6 @@
 """Environment profiles, ported from `profile/parse.rs` and `profile/validate.rs`."""
 
+import os
 from collections.abc import Callable
 from dataclasses import dataclass
 from pathlib import Path
@@ -404,7 +405,20 @@ def apply_profile(config: RootConfig, profile: ProfileConfig) -> RootConfig:
     return config.model_copy(update={k: v for k, v in blocks.items() if v is not None})
 
 
+ENV_PREFIX = "env:"
+
+
+def read_env_profile(location: str) -> str:
+    """`env:NAME`: the profile YAML held in environment variable `NAME`."""
+    name = location.removeprefix(ENV_PREFIX)
+    if not (text := os.environ.get(name, "")).strip():
+        raise FloeConfigError(f"profile environment variable {name} is unset or empty")
+    return text
+
+
 def _docs(location: str | Path) -> list[object]:
+    if isinstance(location, str) and location.startswith(ENV_PREFIX):
+        return parse_yaml(read_env_profile(location))
     if isinstance(location, Path) or not location.startswith("s3://"):
         return load_yaml(Path(location))
     try:

floe/config/validate.py

@@ -4,7 +4,12 @@ import re
 
 from floe import __version__
 from floe.checks.cast import decimal_type
-from floe.checks.normalize import normalize_name, output_names
+from floe.checks.normalize import (
+    normalize_name,
+    output_names,
+    resolve_strategy,
+    source_names,
+)
 from floe.config.project import (
     LineageConfig,
     RootConfig,
@@ -410,6 +415,10 @@ def validate_schema(entity: EntityConfig, version: str) -> None:
             f"schema.normalize_columns.strategy={strategy} is unsupported "
             f"(allowed: {', '.join(NORMALIZE_STRATEGIES)})",
         )
+    try:
+        source_names(schema.columns, resolve_strategy(schema))
+    except FloeConfigError as err:
+        raise _fail(entity, str(err)) from err
     mismatch = schema.mismatch
     if mismatch is not None:
         for field, policy, allowed in (

floe/iceberg/catalog.py

@@ -2,6 +2,9 @@
 
 import os
 import re
+import shutil
+import tempfile
+import weakref
 from collections.abc import Generator, Sequence
 from contextlib import contextmanager
 from urllib.parse import urlsplit
@@ -16,7 +19,7 @@ from pyiceberg.types import NestedField
 
 from floe.config.storage import CatalogDefinition, StorageDefinition
 from floe.config.types import SchemaEvolutionMode
-from floe.errors import FloeConfigError, FloeRunError
+from floe.errors import FloeConfigError, FloeError, FloeRunError
 
 # (column, transform) pairs, as in the entity's `sink.accepted.partition_spec`;
 # a bucket is `bucket[N]` (PyIceberg's notation) with N its `num_buckets` (D12)
@@ -26,6 +29,11 @@ _TRANSFORMS = frozenset({"identity", "year", "month", "day", "hour"})
 # first id of a partition field, as in the Iceberg spec
 _PARTITION_FIELD_ID_START = 1000
 DUCKDB_CATALOG = "floe_catalog"
+# the ADR's limits (python-core-for-olf.md, "Supported limits"); DuckDB's own default is 80% of the
+# host's RAM, not the cgroup limit, so a Kubernetes pod is OOM-killed instead of spilling.
+# Size pods at the memory limit + 2 GiB. The spill bound fails a runaway merge before it fills the disk.
+DUCKDB_MEMORY_LIMIT = "4GB"
+DUCKDB_MAX_TEMP_DIRECTORY_SIZE = "50GB"
 
 
 def transform_name(field: PartitionField) -> str:
@@ -36,6 +44,10 @@ def transform_name(field: PartitionField) -> str:
 def run_error(context: str) -> Generator[None]:
     try:
         yield
+    except (
+        FloeError
+    ):  # already worded for the user, e.g. raised by the stream a write reads
+        raise
     except Exception as e:  # PyIceberg has no common base exception
         raise FloeRunError(f"{context}: {e}") from e
 
@@ -174,8 +186,10 @@ def open_duckdb(
         raise FloeConfigError(f"iceberg catalog {definition.name}: merges and compaction require a rest catalog (got {definition.type})")  # fmt: skip
     statements = _duckdb_setup(definition, storage)
     with run_error("duckdb catalog attach failed"):
-        # ponytail: DuckDB's default memory_limit and spill dir; set the ADR's limits when the run wires merges
-        con = duckdb.connect()
+        spill = tempfile.mkdtemp(prefix="floe-duckdb-")
+        con = duckdb.connect(config={"memory_limit": DUCKDB_MEMORY_LIMIT, "temp_directory": spill, "max_temp_directory_size": DUCKDB_MAX_TEMP_DIRECTORY_SIZE})  # fmt: skip
+        # the spill directory lives as long as the connection (also on `with` exit and at interpreter exit)
+        weakref.finalize(con, shutil.rmtree, spill, True)
         for statement in statements:
             con.execute(statement)
     return con
@@ -298,10 +312,10 @@ def ensure_table(
     partition_fields: PartitionFields | None = None,
     schema_evolution: SchemaEvolutionMode = "strict",
     create_namespace: bool = True,
-) -> Table:
+) -> tuple[Table, list[str]]:
     """Create the namespace (unless `create_namespace` is off) and table if missing; otherwise check the table matches (Rust's append path).
 
-    With `add_columns`, columns new in `schema` are added as optional (D13); partition changes stay rejected.
+    With `add_columns`, columns new in `schema` are added as optional (D13) and returned by name; partition changes stay rejected.
     """
     ident = (namespace,)
     with run_error("rest catalog namespace_exists check failed"):
@@ -338,16 +352,17 @@ def ensure_table(
                     for field in added:
                         update.add_column(field.name, field.field_type, required=False)
                 table.refresh()
-        return table
+        return table, [f.name for f in added]
     spec = (
         None
         if partition_fields is None
         else build_partition_spec(schema, partition_fields, entity)
     )
     with run_error("rest catalog create_table failed"):
-        return catalog.create_table(  # pyright: ignore[reportUnknownMemberType] -- PyIceberg's `schema` is `Schema | Unknown`
+        created = catalog.create_table(  # pyright: ignore[reportUnknownMemberType] -- PyIceberg's `schema` is `Schema | Unknown`
             identifier,
             schema,
             location=location,
             partition_spec=spec if spec is not None else PartitionSpec(),
         )
+    return created, []

floe/iceberg/sink.py

@@ -6,8 +6,9 @@
 import hashlib
 import json
 import time
-from collections.abc import Iterator, Sequence
+from collections.abc import Callable, Iterator, Sequence
 from dataclasses import dataclass
+from itertools import chain
 from pathlib import Path
 from typing import Any
 
@@ -112,23 +113,30 @@ def input_hash(entity: EntityConfig, files: Sequence[InputFile]) -> str:
 _WIDER: dict[pl.DataType, pl.DataType] = {pl.UInt8(): pl.Int32(), pl.UInt16(): pl.Int32(), pl.UInt32(): pl.Int64()}  # fmt: skip
 
 
-def _reader(entity: EntityConfig, frames: list[pl.DataFrame], names: list[str]) -> pa.RecordBatchReader:  # fmt: skip
-    """Output columns in schema order; `nullable: false` columns are required, as in Rust."""
+def _reader(entity: EntityConfig, frames: Iterator[pl.DataFrame], names: list[str]) -> pa.RecordBatchReader:  # fmt: skip
+    """Output columns in schema order, from at least one frame; `nullable: false` columns are required, as in Rust."""
     columns = entity.schema_.columns
-    for name, c in zip(names, columns, strict=True):
-        if c.nullable is False and any(f[name].null_count() for f in frames):
-            raise FloeRunError(f"iceberg write rejected nulls for non-nullable column {name}")  # fmt: skip
 
     def arrow(df: pl.DataFrame) -> pa.Table:  # Iceberg has no unsigned types
         return df.select(pl.col(n).cast(_WIDER.get(df.schema[n], df.schema[n])) for n in names).to_arrow()  # fmt: skip
 
-    schema = arrow(frames[0].head(0)).schema
+    def checked(df: pl.DataFrame) -> pl.DataFrame:
+        for name, c in zip(names, columns, strict=True):
+            if c.nullable is False and df[name].null_count():
+                raise FloeRunError(f"iceberg write rejected nulls for non-nullable column {name}")  # fmt: skip
+        return df
+
+    first = checked(
+        next(frames)
+    )  # a violation in the first chunk fails before the table is created
+    schema = arrow(first.head(0)).schema
+    frames = chain([first], frames)  # `batches` must not keep `first` alive
     for i, c in enumerate(columns):
         schema = schema.set(i, schema.field(i).with_nullable(c.nullable is not False))
 
     def batches() -> Iterator[pa.RecordBatch]:
         for df in frames:
-            yield from arrow(df).cast(schema).to_batches()
+            yield from arrow(checked(df)).cast(schema).to_batches()
 
     return pa.RecordBatchReader.from_batches(schema, batches())
 
@@ -144,13 +152,13 @@ def partition_fields(entity: EntityConfig) -> PartitionFields | None:
 def write_iceberg(
     entity: EntityConfig,
     target: IcebergTarget,
-    frames: list[pl.DataFrame],
+    frames: Iterator[pl.DataFrame],
     names: list[str],
     mode: WriteMode,
     run_id: str,
-    digest: str,
-) -> tuple[IcebergWriteResult, dict[str, Any], dict[str, Any] | None]:
-    """`frames` hold the output columns `names`; returns the result, its `accepted_output` table and merge fields."""
+    digest: Callable[[], str],
+) -> tuple[IcebergWriteResult, dict[str, Any], dict[str, Any] | None, list[str]]:
+    """`frames` (at least one) hold the output columns `names` and are read while the data files are written; returns the result, its `accepted_output` table and merge fields, and the columns `add_columns` added."""
     reader = _reader(entity, frames, names)
     options = entity.sink.accepted.merge
     scd2 = options.scd2 if options else None
@@ -175,7 +183,7 @@ def write_iceberg(
             existing = set(catalog.load_table(identifier).schema().column_names)
         if added := [k for k in key if k not in existing]:  # Rust `merge/shared.rs`
             raise FloeRunError(f"entity.name={entity.name} iceberg schema evolution failed: merge key columns cannot be added: {', '.join(added)}")  # fmt: skip
-    table = ensure_table(
+    table, added = ensure_table(
         catalog, target.namespace, target.table, schema, entity.name, target.location.removeprefix("local://"),
         partition_fields(entity), evolution.mode if evolution else "strict", target.catalog.create_database_if_missing,
     )  # fmt: skip
@@ -213,4 +221,4 @@ def write_iceberg(
         "iceberg_namespace": target.namespace,
         "iceberg_table": target.table,
     }
-    return result, fields, merge
+    return result, fields, merge, added

floe/iceberg/unique.py

@@ -1,61 +1,27 @@
 """Unique keys already in an Iceberg table, for append (Rust `seed_unique_tracker_for_append`)."""
 
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
+from pathlib import Path
 
 import polars as pl
+import polars.selectors as cs
 from pyiceberg.table import Table
 
-from floe.checks.unique import (
-    UniqueTracker,
-    _float_bits,  # pyright: ignore[reportPrivateUsage] -- the tracker's own float key
-)
 from floe.iceberg.catalog import run_error
 
 
-def seed_from_table(
-    tracker: UniqueTracker,
-    table: Table,
-    batch: pl.DataFrame,
-    stored: Mapping[str, str],
-) -> None:
-    """Seed `tracker` with the keys of `table` that `batch` also holds; `stored` maps runtime to table names.
+def cache_table_keys(
+    table: Table, keys: Sequence[Sequence[str]], stored: Mapping[str, str], path: Path
+) -> bool:
+    """Write the key columns of `table` to `path` in one streamed scan; `stored` maps runtime to table names.
 
-    A target key the batch lacks is never hit, so the flags are Rust's while memory follows the
-    batch: the table is streamed, key columns only. The join matches -0.0 to 0.0 and NaN to any
-    NaN; the tracker then compares float bits, so those extra keys are never hit.
-    A key with a column the table lacks (`add_columns` adds it on write) is skipped: historical
-    rows hold NULL there, which never duplicates.
+    A key with a column the table lacks (`add_columns` adds it on write) is left out: the tracker
+    skips it, as historical rows hold NULL there. False when no key is left, and nothing is written.
     """
-    columns = set(table.schema().column_names)
-    for key in tracker.keys:
-        names = [stored.get(c, c) for c in key]
-        if not columns.issuperset(names):
-            continue
-        # positional names (k0, b0, ...): no stored column name can collide with a temporary
-        ids = [f"k{i}" for i in range(len(key))]
-        distinct = (
-            batch.select(pl.col(c).alias(k) for c, k in zip(key, ids, strict=True))
-            .drop_nulls()
-            .unique()
-        )
-        if not distinct.height:  # all-NULL keys are never checked; nothing to look up
-            continue
-        # distinct by float bits, as the tracker keys them: `unique` on a float merges -0.0 with 0.0
-        floats = [k for c, k in zip(key, ids, strict=True) if batch.schema[c].is_float()]  # fmt: skip
-        bits = {f"b{k}": pl.col(k).map_batches(_float_bits, return_dtype=pl.UInt64, is_elementwise=True) for k in floats}  # fmt: skip
-        subset = [k for k in ids if k not in floats] + list(bits)
-        with run_error("iceberg unique seed scan failed"):
-            scan = pl.scan_iceberg(table).select(pl.col(n).alias(k) for n, k in zip(names, ids, strict=True))  # fmt: skip
-            # the table's dtypes can differ (unsigned widened, datetimes in us); a key the table type
-            # cannot hold casts to NULL, which never matches
-            lookup = distinct.cast(scan.collect_schema(), strict=False).lazy()
-            hits = (
-                scan.join(lookup, on=ids, how="semi")
-                .with_columns(**bits)
-                # distinct keys only: memory follows the batch, not the history
-                .unique(subset)
-                .drop(*bits)
-                .collect(engine="streaming")
-            )
-        back = pl.Schema({k: batch.schema[c] for c, k in zip(key, ids, strict=True)})
-        tracker.seed(hits.cast(back).rename(dict(zip(ids, key, strict=True))))
+    present = set(table.schema().column_names)
+    names = {n for k in keys if present.issuperset(ns := [stored.get(c, c) for c in k]) for n in ns}  # fmt: skip
+    if not names:
+        return False
+    with run_error("iceberg unique seed scan failed"):
+        pl.scan_iceberg(table).select(cs.by_name(sorted(names))).sink_parquet(path)
+    return True

floe/iceberg/write.py

@@ -5,7 +5,7 @@
 
 import itertools
 import uuid
-from collections.abc import Iterator, Sequence
+from collections.abc import Callable, Iterator, Sequence
 from dataclasses import dataclass, replace
 from datetime import datetime
 from pathlib import PurePosixPath
@@ -82,7 +82,7 @@ def write_table(
     reader: pa.RecordBatchReader,
     mode: WriteMode,
     run_id: str,
-    input_hash: str,
+    input_hash: str | Callable[[], str],
     small_file_threshold_bytes: int = SMALL_FILE_THRESHOLD_BYTES,
     chunk_rows: int = CHUNK_ROWS,
     unique_keys: Sequence[Sequence[str]] = (),
@@ -104,6 +104,9 @@ def write_table(
                 table.metadata, chunk, table.io, write_uuid, counter
             )
         ]
+    digest = (
+        input_hash() if callable(input_hash) else input_hash
+    )  # known once the stream has been read
     base = (
         table.metadata.current_snapshot_id
     )  # only commits after it can be a rival's copy of this batch
@@ -112,9 +115,7 @@ def write_table(
             _backoff(attempt - 1)
             with run_error("rest iceberg table refresh failed"):
                 table.refresh()
-                seen = mode == "append" and find_committed(
-                    table, input_hash, after=base
-                )
+                seen = mode == "append" and find_committed(table, digest, after=base)
             if seen:
                 return replace(write_result(table, [], small_file_threshold_bytes), already_committed=True)  # fmt: skip
             if mode == "append":
@@ -131,7 +132,7 @@ def write_table(
                 table.metadata = table.metadata.model_copy(update={"properties": {**table.properties, TableProperties.COMMIT_NUM_RETRIES: "0"}})  # fmt: skip
                 tx = table.transaction()
                 update = tx.update_snapshot(
-                    snapshot_properties={RUN_ID: run_id, INPUT_HASH: input_hash}
+                    snapshot_properties={RUN_ID: run_id, INPUT_HASH: digest}
                 )
                 with (
                     update.overwrite()

floe/manifest.py

@@ -17,7 +17,7 @@ from typing import Any, NamedTuple, cast
 from pydantic import BaseModel
 
 from floe import __version__
-from floe.config.profile import ProfileConfig
+from floe.config.profile import ENV_PREFIX, ProfileConfig, read_env_profile
 from floe.config.project import ReportConfig, RootConfig
 from floe.config.storage import StoragesConfig
 from floe.config.types import ArchiveTarget, EntityConfig, SinkTarget, SourceConfig
@@ -464,7 +464,10 @@ def build_manifest(
     profile_uri = profile_checksum = None
     if options.profile_arg is not None:  # recorded like `config_uri` and its checksum
         # an s3 profile is read from there by the run, so it is recorded as typed
-        if options.profile_arg.startswith("s3://"):
+        if options.profile_arg.startswith(ENV_PREFIX):  # the Job reads the variable too
+            profile_uri = options.profile_arg
+            profile_checksum = _sha256(read_env_profile(profile_uri).encode())
+        elif options.profile_arg.startswith("s3://"):
             profile_uri = options.profile_arg
             profile_checksum = _sha256(read_s3(profile_uri, "profile"))
         else:

floe/report/entity.py

@@ -157,6 +157,7 @@ def build_report(
     table_root_uri: str | None = None,
     table: Mapping[str, Any] | None = None,
     merge: Mapping[str, Any] | None = None,
+    added_columns: Sequence[str] = (),
 ) -> dict[str, Any]:
     """The entity's `run.json`; `files` are the `file_report`s, in input order.
 
@@ -234,9 +235,10 @@ def build_report(
             **(merge or {}),
         },
         "schema_evolution": {
-            "enabled": False,
+            "enabled": evolution.mode == "add_columns",
             "mode": evolution.mode,
-            "applied": False,
+            "applied": bool(added_columns),
+            **({"added_columns": list(added_columns)} if added_columns else {}),
             "incompatible_changes_detected": False,
         },
     }  # fmt: skip

floe/run/entity.py

@@ -4,7 +4,7 @@ import io
 import tempfile
 import threading
 import time
-from collections.abc import Callable, Iterator
+from collections.abc import Callable, Generator, Iterable, Iterator
 from contextlib import suppress
 from dataclasses import dataclass, field, replace
 from datetime import datetime
@@ -39,7 +39,12 @@ from floe.checks.rows import (
     not_null_check,
     split_rows,
 )
-from floe.checks.unique import UniqueResult, UniqueTracker, resolve_unique_keys
+from floe.checks.unique import (
+    UniqueResult,
+    UniqueTracker,
+    resolve_unique_keys,
+    seed_from_history,
+)
 from floe.config.storage import CatalogsConfig, StorageDefinition, StoragesConfig
 from floe.config.types import EntityConfig, with_partition_columns
 from floe.errors import FloeConfigError, FloeIoError, FloeRunError
@@ -49,7 +54,7 @@ from floe.iceberg.sink import (
     resolve_target,
     write_iceberg,
 )
-from floe.iceberg.unique import seed_from_table
+from floe.iceberg.unique import cache_table_keys
 from floe.iceberg.write import find_committed
 from floe.pii import apply_pii_masking
 from floe.read.avro import read_avro
@@ -253,6 +258,9 @@ class _Plan:
     table: Table | None = (
         None  # the Iceberg target whose keys seed the tracker (append)
     )
+    history: pl.LazyFrame | None = (
+        None  # the target's key columns, cached on local disk (append, #1057)
+    )
     merge_key: str | None = (
         None  # the merge key's unique check; its duplicates are rejected even under warn (Rust `enforce_reject`)
     )
@@ -354,14 +362,29 @@ def _typed(
     if entity.policy.severity == "abort" and any(c.mask.any() for c in cast_checks):
         return typed, cast_checks  # fmt: skip  # Rust: abort-mode cast errors skip not_null and unique
     required = [not_null_check(typed, n) for n in plan.required]
-    if plan.table is not None:
-        # ponytail: one key scan of the table per chunk; a DuckDB anti-join (ADR) if large histories make it slow
-        seed_from_table(plan.tracker, plan.table, typed, plan.out_map)
+    if plan.history is not None:
+        seed_from_history(plan.tracker, plan.history, typed, plan.out_map)
     return typed, [*required, *cast_checks, *plan.tracker.check(typed)]
 
 
-def _run_file(plan: _Plan, file: InputFile, path: Path) -> tuple[dict[str, Any], list[pl.DataFrame]]:  # fmt: skip
-    """The file's report and its accepted frames (none when it aborted the run); `path` is the local copy."""
+def _unspill(path: Path) -> pl.DataFrame:
+    df = pl.read_ipc(path)
+    path.unlink()
+    return df
+
+
+def _replay(plan: _Plan, held: list[pl.DataFrame | Path]) -> Iterator[pl.DataFrame]:
+    while held:
+        item = held.pop(0)
+        yield _output(plan, item if isinstance(item, pl.DataFrame) else _unspill(item))
+
+
+def _run_file(plan: _Plan, file: InputFile, path: Path, spill: Path) -> tuple[dict[str, Any], Iterable[pl.DataFrame]]:  # fmt: skip
+    """The file's report and its accepted frames, one by one (none when it aborted the run); `path` is the local copy.
+
+    A file's rows are accepted only once it is read to the end (a late read error or an abort drops them all), so
+    every chunk after the first waits in `spill`, not in memory.
+    """
     entity, uri = plan.entity, file.source_uri
     severity = entity.policy.severity
     derived = plan.derived.get(uri, [])
@@ -372,7 +395,7 @@ def _run_file(plan: _Plan, file: InputFile, path: Path) -> tuple[dict[str, Any],
     chunks = _chunks(entity, path, file_names)
     keys = plan.tracker.snapshot()
 
-    def failed(err: FloeIoError, mismatch: MismatchOutcome | None) -> tuple[dict[str, Any], list[pl.DataFrame]]:  # fmt: skip
+    def failed(err: FloeIoError, mismatch: MismatchOutcome | None) -> tuple[dict[str, Any], Iterable[pl.DataFrame]]:  # fmt: skip
         # Rust skips the raw copy under PII: the file cannot be masked
         plan.tracker.restore(keys)  # the unreadable file leaves no keys behind
         keep = entity.pii is None and entity.sink.rejected is not None
@@ -403,7 +426,7 @@ def _run_file(plan: _Plan, file: InputFile, path: Path) -> tuple[dict[str, Any],
 
     rows, bad_total, forced_total = 0, 0, 0
     masks: dict[tuple[str, str, str], list[pl.Series]] = {}
-    accepted: list[pl.DataFrame] = []
+    accepted: list[pl.DataFrame | Path] = []
     rejected: list[pl.DataFrame] = []
     errors: list[tuple[int, str]] = []
     try:
@@ -423,7 +446,12 @@ def _run_file(plan: _Plan, file: InputFile, path: Path) -> tuple[dict[str, Any],
                 forced = forced.fill_null(False)
                 forced_total += int(forced.sum())
                 typed = typed.filter(~forced)
-            accepted.append(good if severity == "reject" else typed)
+            frame = good if severity == "reject" else typed
+            if accepted:
+                frame.write_ipc(spilled := spill / f"{len(accepted)}.ipc")
+                accepted.append(spilled)
+            else:
+                accepted.append(frame)
             if severity == "reject" and bad.height:
                 rejected.append(bad)
     except FloeIoError as err:
@@ -445,22 +473,30 @@ def _run_file(plan: _Plan, file: InputFile, path: Path) -> tuple[dict[str, Any],
         errors_path = _error_report(plan, file.source_stem, errors)
     plan.to_archive.append(file)
     out = file_report(entity, uri, rows, bad_total, merged, mismatch, rejected_path, None, errors_path, forced_total)  # fmt: skip
-    return out, [] if out["status"] == "aborted" else [_output(plan, df) for df in accepted]  # fmt: skip
+    return out, [] if out["status"] == "aborted" else _replay(plan, accepted)
+
 
+def _key_cache(plan: _Plan, client: StorageClient, base: str, temp: Path) -> pl.LazyFrame | None:  # fmt: skip
+    """Append mode: the key columns of the target's existing data, written once to local disk (#1057).
 
-def _seed_unique(plan: _Plan, client: StorageClient, base: str, columns: list[str]) -> None:  # fmt: skip
-    """Append mode: the keys already in the sink's parts count as seen (Rust `seed_unique_tracker_for_append`)."""
-    if not columns:
-        return  # no unique keys: never read the history
-    stored = {n: plan.out_map.get(n, n) for n in columns}  # the sink holds output names
-    for uri in part_uris(client, base.rstrip("/"), "parquet"):
-        part = uri.removeprefix("local://")
-        try:  # only the key columns are read
-            body = client.read(uri)
-            keys = pl.read_parquet(io.BytesIO(body.body if body else b""), columns=list(stored.values()))  # fmt: skip
-        except pl.exceptions.PolarsError as err:
-            raise FloeRunError(f"entity.name={plan.entity.name} cannot seed unique keys from {part}: {err}") from err  # fmt: skip
-        plan.tracker.seed(keys.rename({v: k for k, v in stored.items()}))
+    Each chunk then joins its keys against this cache, so the target is read once per entity and
+    memory follows the chunk, not the history (Rust `seed_unique_tracker_for_append`).
+    """
+    cache = temp / "unique-keys"
+    cache.mkdir()
+    if plan.table is not None:
+        cache_table_keys(plan.table, plan.tracker.keys, plan.out_map, cache / "table.parquet")  # fmt: skip
+    else:
+        columns = sorted({plan.out_map.get(n, n) for k in plan.tracker.keys for n in k})  # the sink holds output names  # fmt: skip
+        for i, uri in enumerate(part_uris(client, base.rstrip("/"), "parquet")):
+            part = uri.removeprefix("local://")
+            try:  # only the key columns are read
+                body = client.read(uri)
+                keys = pl.read_parquet(io.BytesIO(body.body if body else b""), columns=columns)  # fmt: skip
+            except pl.exceptions.PolarsError as err:
+                raise FloeRunError(f"entity.name={plan.entity.name} cannot seed unique keys from {part}: {err}") from err  # fmt: skip
+            keys.write_parquet(cache / f"{i}.parquet")
+    return pl.scan_parquet(cache / "*.parquet") if any(cache.iterdir()) else None
 
 
 def _report_names(
@@ -855,14 +891,7 @@ def _run_entity(
             pk[0] if len(pk) == 1 else f"[{','.join(pk)}]"
         )  # the tracker's check name
     accepted_client = storage_client(accepted)
-    if sink.write_mode == "append" and target is None:
-        _seed_unique(
-            plan,
-            accepted_client,
-            base,
-            sorted({runtime.get(c, c) for k in keys for c in k}),
-        )
-    elif sink.write_mode == "append" and target is not None:
+    if sink.write_mode == "append" and target is not None:
         plan.table = existing_table(target)  # fmt: skip  # Rust `seed_unique_tracker`, and the retry guard
 
     def source() -> ResolvedInputs:
@@ -894,52 +923,73 @@ def _run_entity(
         plan.to_archive += [f for f in pending if f.source_uri in skip]
         pending = [f for f in pending if f.source_uri not in skip]
     plan.input_hash = input_hash(entity, pending)
-    frames: list[pl.DataFrame] = []
     # #545: the snapshot's fingerprint names only the files it holds in full, never an aborted batch
     complete: list[InputFile] = []
     skipped = len(files)
     downloader = None if isinstance(client, LocalClient) else client
-    # Rust `localize_input`: a remote input is read from a local copy
-    with tempfile.TemporaryDirectory(prefix="floe-") as temp_dir:
-        for f in pending:
-            emit("file_started", entity=entity.name, input=f.source_uri)
-            timer = time.monotonic()
-            local = localize_input(f, downloader, Path(temp_dir))
-            report, kept = _run_file(plan, f, local.local_path)
-            if local.is_ephemeral:
-                local.local_path.unlink()
-            emit("file_finished", entity=entity.name, input=f.source_uri, status=report["status"],
-                 rows=report["row_count"], accepted=report["accepted_count"],
-                 rejected=report["rejected_count"], elapsed_ms=int((time.monotonic() - timer) * 1000))  # fmt: skip
-            files.append(report)
-            frames += kept
-            if report["status"] == "aborted":
-                break
-            complete.append(f)
-    files[skipped:] = sorted(files[skipped:], key=lambda r: r["input_file"] not in plan.prechecked)  # fmt: skip
+    failure: list[Exception] = []
+
+    def accepted_frames() -> Generator[pl.DataFrame]:
+        """Files are read as the sink pulls (#1056); `files` and `complete` are final once it is exhausted."""
+        try:
+            # Rust `localize_input`: a remote input is read from a local copy
+            with tempfile.TemporaryDirectory(prefix="floe-") as temp_dir:
+                if pending and sink.write_mode == "append" and plan.tracker.keys and (target is None or plan.table is not None):  # fmt: skip
+                    plan.history = _key_cache(plan, accepted_client, base, Path(temp_dir))  # fmt: skip
+                for f in pending:
+                    emit("file_started", entity=entity.name, input=f.source_uri)
+                    timer = time.monotonic()
+                    local = localize_input(f, downloader, Path(temp_dir))
+                    report, kept = _run_file(plan, f, local.local_path, Path(temp_dir))
+                    if local.is_ephemeral:
+                        local.local_path.unlink()
+                    emit("file_finished", entity=entity.name, input=f.source_uri, status=report["status"],
+                         rows=report["row_count"], accepted=report["accepted_count"],
+                         rejected=report["rejected_count"], elapsed_ms=int((time.monotonic() - timer) * 1000))  # fmt: skip
+                    files.append(report)
+                    yield from kept
+                    if report["status"] == "aborted":
+                        break
+                    complete.append(f)
+        except Exception as err:
+            # the sinks wrap what they read; this error reaches the caller as it is
+            failure.append(err)
+            raise
 
     table: dict[str, Any] | None = None
     merge: dict[str, Any] | None = None
-    if pending and (
-        frames or sink.write_mode == "overwrite"
-    ):  # Rust writes nothing when every input was skipped
-        empty = pl.DataFrame(schema=dict.fromkeys(names, pl.String)).select(
-            cast_column(n, c.column_type)[0]
-            for n, c in zip(names, columns, strict=True)
-        )
-        frames = frames or [_output(plan, empty)]
-        if target is None:
-            write = write_accepted(pl.concat(frames), accepted_client, base, sink.write_mode, sink.accepted.options)  # fmt: skip
+    added: list[str] = []
+    stream = accepted_frames()
+    try:
+        first = next(stream, None)  # Rust writes nothing when every input was skipped
+        if pending and (first is not None or sink.write_mode == "overwrite"):
+            empty = pl.DataFrame(schema=dict.fromkeys(names, pl.String)).select(
+                cast_column(n, c.column_type)[0]
+                for n, c in zip(names, columns, strict=True)
+            )
+            frames = chain([_output(plan, empty) if first is None else first], stream)
+            del first  # the sink holds the chunk it is on, not the first
+            if target is None:
+                write = write_accepted(frames, accepted_client, base, sink.write_mode, sink.accepted.options)  # fmt: skip
+            else:
+                outputs = [plan.out_map.get(n, n) for n in names]
+                result, table, merge, added = write_iceberg(entity, target, frames, outputs, sink.write_mode, emit.run_id, lambda: input_hash(entity, complete))  # fmt: skip
+                write = AcceptedWrite(result.files_written, result.file_paths, result.total_bytes_written,
+                                      result.avg_file_size_mb, result.small_files_count)  # fmt: skip
+                if added:
+                    emit("schema_evolution_applied", entity=entity.name, mode="add_columns", added_columns=added)  # fmt: skip
+                if result.already_committed:  # #920: the commit conflicted and the guard found this batch in the table
+                    done = {f.source_uri for f in complete}
+                    files[:] = [skipped_report(r["input_file"], "already_committed") if r["input_file"] in done else r for r in files]  # fmt: skip
         else:
-            outputs = [plan.out_map.get(n, n) for n in names]
-            result, table, merge = write_iceberg(entity, target, frames, outputs, sink.write_mode, emit.run_id, input_hash(entity, complete))  # fmt: skip
-            write = AcceptedWrite(result.files_written, result.file_paths, result.total_bytes_written,
-                                  result.avg_file_size_mb, result.small_files_count)  # fmt: skip
-            if result.already_committed:  # #920: the commit conflicted and the guard found this batch in the table
-                done = {f.source_uri for f in complete}
-                files[:] = [skipped_report(r["input_file"], "already_committed") if r["input_file"] in done else r for r in files]  # fmt: skip
-    else:
-        write = AcceptedWrite(0, [], 0, 0.0, 0)
+            write = AcceptedWrite(0, [], 0, 0.0, 0)
+    except Exception:
+        if not failure:
+            raise
+        raise failure[0] from failure[0].__cause__
+    finally:
+        stream.close()
+    files[skipped:] = sorted(files[skipped:], key=lambda r: r["input_file"] not in plan.prechecked)  # fmt: skip
     inc.committed = files  # #915: from here on a failure promotes the written files
     # #914: archive only once the sink committed, so a failed commit leaves the inputs in the source (Rust archives before)
     by_uri = {r["input_file"]: r for r in files}
@@ -959,5 +1009,6 @@ def _run_entity(
         table_root_uri=target.location if target and table else None,
         table=table,
         merge=merge,
+        added_columns=added,
     )
     return EntityRun(report, any(f["status"] == "aborted" for f in files))

floe/storage/local.py

@@ -101,6 +101,7 @@ class LocalClient:
         try:
             dst.parent.mkdir(parents=True, exist_ok=True)
             mode = src.stat().st_mode & 0o777
+            dst.unlink(missing_ok=True)  # not through a symlink or shared inode
             fd = os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
             os.fchmod(fd, mode)  # #779: the open mode applies on create only
             with src.open("rb") as fin, os.fdopen(fd, "wb") as fout:

floe/write/outputs.py

@@ -4,10 +4,9 @@ import io
 import math
 import re
 import uuid
-from collections.abc import Generator, Iterator
-from contextlib import contextmanager
+from collections.abc import Callable, Generator, Iterable, Iterator
+from contextlib import contextmanager, suppress
 from dataclasses import dataclass
-from itertools import count
 
 import polars as pl
 
@@ -58,30 +57,74 @@ def part_uris(client: StorageClient, base: str, ext: str) -> list[str]:
     return sorted(u for u in uris if u.rpartition("/")[0] == base and u.endswith(f".{ext}") and _PART.fullmatch(u.rpartition("/")[2].removesuffix(f".{ext}")))  # fmt: skip
 
 
-def _part_names(client: StorageClient, base: str, mode: WriteMode, ext: str) -> Iterator[str]:  # fmt: skip
-    """Overwrite clears the old parts and numbers from 0 (`part-00000`); other modes use `part-<uuid>`."""
-    if mode != "overwrite":
-        while True:
-            yield f"part-{uuid.uuid4()}.{ext}"
-    for uri in part_uris(client, base, ext):
+@contextmanager
+def _parts(client: StorageClient, base: str, mode: WriteMode, ext: str) -> Generator[Callable[[bytes], str]]:  # fmt: skip
+    """Yield `put(body) -> name`. Every part is staged as `_staging-<id>-<name>` (invisible to `part_uris`).
+
+    Only when all were written are they copied over `<name>` and the staged copies deleted (#1054).
+    Overwrite names the parts `part-NNNNN` and deletes the old parts that no new part replaced, so a
+    failed write leaves them untouched. Other modes name them `part-<uuid>` and keep the old parts;
+    staging keeps a killed run from leaving visible parts for a retry to duplicate (#1056: parts are
+    written while the input is still read). A failed write deletes the staged parts it wrote.
+    """
+    overwrite = mode == "overwrite"
+    stage = f"_staging-{uuid.uuid4()}-"
+    names: list[str] = []
+
+    def put(body: bytes) -> str:
+        name = f"part-{len(names):05}.{ext}" if overwrite else f"part-{uuid.uuid4()}.{ext}"  # fmt: skip
+        names.append(name)  # before the write: a partial file needs cleanup too
+        client.write(f"{base}/{stage}{name}", body)
+        return name
+
+    try:
+        yield put
+    except BaseException:
+        for name in names:
+            with suppress(FloeError, OSError):  # best effort: the original error wins
+                client.delete(f"{base}/{stage}{name}")
+        raise
+    old = part_uris(client, base, ext) if overwrite else []
+    for name in names:
+        client.copy(f"{base}/{stage}{name}", f"{base}/{name}")
+    for uri in set(old) - {f"{base}/{n}" for n in names}:
         client.delete(uri)
-    for i in count():
-        yield f"part-{i:05}.{ext}"
+    for name in names:
+        client.delete(f"{base}/{stage}{name}")
+
+
+def _part_frames(
+    frames: Iterable[pl.DataFrame], max_size: int
+) -> Iterator[pl.DataFrame]:
+    """`frames` regrouped into parts of about `max_size` bytes in memory; at most one part and a frame are pooled."""
+    pool: list[pl.DataFrame] = []
+    size = height = 0
+    for df in frames:
+        pool.append(df)
+        size += df.estimated_size()
+        height += df.height
+        rows = max(1, max_size // max(1, math.ceil(size / height))) if height else 1
+        if height >= rows:
+            whole = pl.concat(pool)
+            full = height - height % rows
+            yield from (whole.slice(i, rows) for i in range(0, full, rows))
+            rest = whole.slice(full)
+            pool = [rest] if rest.height else []
+            size, height = rest.estimated_size(), rest.height
+    if pool:  # also the one empty part of a run that accepted no row
+        yield pl.concat(pool)
 
 
 def write_accepted(
-    df: pl.DataFrame,
+    frames: Iterable[pl.DataFrame],
     client: StorageClient,
     base: str,
     mode: WriteMode = "overwrite",
     options: SinkOptions | None = None,
 ) -> AcceptedWrite:
-    """Write `df` as `part-*.parquet` files under `base`, split so each holds about `max_size_per_file`."""
+    """Write `frames` (at least one) as `part-*.parquet` files under `base`, each holding about `max_size_per_file`."""
     base = base.rstrip("/")
-    names = _part_names(client, base, mode, "parquet")
     max_size = (options and options.max_size_per_file) or DEFAULT_MAX_SIZE_PER_FILE
-    avg_row = max(1, math.ceil(df.estimated_size() / df.height)) if df.height else 1
-    rows = max(1, max_size // avg_row)
     kwargs: dict[str, str | int] = {}
     if options and options.compression:
         kwargs["compression"] = options.compression
@@ -89,14 +132,12 @@ def write_accepted(
         kwargs["row_group_size"] = options.row_group_size
     files: list[str] = []
     sizes: list[int] = []
-    with _os_errors():
-        for chunk in [df.slice(i, rows) for i in range(0, df.height, rows)] or [df]:
-            name = next(names)
+    with _os_errors(), _parts(client, base, mode, "parquet") as put:
+        for chunk in _part_frames(frames, max_size):
             # ponytail: a part is held in memory (≤ max_size_per_file) before the put; stream multipart uploads if that is too much
             body = io.BytesIO()
             chunk.write_parquet(body, **kwargs)  # pyright: ignore[reportArgumentType]  # ponytail: kwargs loses the compression Literal; validated by the config
-            client.write(f"{base}/{name}", body.getvalue())
-            files.append(name)
+            files.append(put(body.getvalue()))
             sizes.append(body.tell())
     total = sum(sizes)
     return AcceptedWrite(
@@ -115,7 +156,10 @@ def write_rejected(df: pl.DataFrame, client: StorageClient, base: str, mode: Wri
     """
     with _os_errors():
         base = base.rstrip("/")
-        name = f"part-{uuid.uuid5(uuid.NAMESPACE_URL, key)}.csv" if key and mode != "overwrite" else next(_part_names(client, base, mode, "csv"))  # fmt: skip
-        uri = f"{base}/{name}"
-        client.write(uri, df.write_csv().encode())
-    return uri
+        if key and mode != "overwrite":
+            name = f"part-{uuid.uuid5(uuid.NAMESPACE_URL, key)}.csv"
+            client.write(f"{base}/{name}", df.write_csv().encode())
+        else:
+            with _parts(client, base, mode, "csv") as put:
+                name = put(df.write_csv().encode())
+    return f"{base}/{name}"

{floe_python-0.8.2.dist-info → floe_python-0.8.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: floe-python
-Version: 0.8.2
+Version: 0.8.4
 Summary: Floe: a data-contract runtime for file ingestion
 License-File: LICENSE
 Requires-Python: >=3.11

{floe_python-0.8.2.dist-info → floe_python-0.8.4.dist-info}/RECORD

@@ -1,38 +1,38 @@
 floe/__init__.py,sha256=0X9pm8uWiwoDslKZyyjsEAYd6g1mlvIWhkWgE5uhGng,13279
 floe/__main__.py,sha256=Nzz6wrGXRpJiO59qlX7WEJRTlb-XcRFswmP0TA2KjyM,57
-floe/_cli.py,sha256=6zQRRczAgARgc7PcLrWS7lEzJa6GKsbpf8Zz8GsDjFo,14081
+floe/_cli.py,sha256=hwast3uMX0iUtkIpcChSIVHYkoYi4fR4jwCwQT53lXE,14101
 floe/_cli_output.py,sha256=VLxWQr-EPtIwklo9xr2CnjMD17U-7JrMUWdTtAP3jrY,8809
 floe/add_entity.py,sha256=42b7l8o55dzgaR_DDgpW7YWdnU2weYMf4U_rf7qCY6M,21986
 floe/errors.py,sha256=hLSkgPQATBnjY1CAQqX4aB0oI0-WKoHW09bDXAgzC7k,186
 floe/glob.py,sha256=A0HdePLXLGl1T6heJbgdrNHr8pZfdidG7pQ1QsXWOl8,10579
 floe/maintain.py,sha256=CGSoZhO7p0u14o10FCKCO4FprzpVO548A71eq5FowF8,7343
-floe/manifest.py,sha256=GDaa01K9n5vxI0kRQSkM_xjjIwoeF6JN-N1Fktas8Xc,20216
+floe/manifest.py,sha256=sAWjv6yS4GokQDTnZwutyplLz1ICPBYl2qJmm9IORdM,20462
 floe/manifest_run.py,sha256=1XLInkpaCofMezMBs_5uIFoR5W7lbwCBFZeqRcnwUAw,12398
 floe/pii.py,sha256=uJjU-C4c7h45E7rEL9K3Zp1LTTLh-FNKfUV5av2QMO4,4351
 floe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/state.py,sha256=XFHF6zVxbdGg1bkONZDLryTD4noDF9IgaimaEI-3Nlg,11104
 floe/checks/cast.py,sha256=lM40xtKEggWI8FW3jijM3T2JFQtaXs8FhsFXo3vy5oM,6788
 floe/checks/mismatch.py,sha256=2Gz-tiWKxy-iehSjAlsGzPLtYeNIfBWvi4yQPlASVYQ,5309
-floe/checks/normalize.py,sha256=INYh3NO3JkdgDjTXDi7pylr79nzrJmDuBlrIrm_8LRc,4326
+floe/checks/normalize.py,sha256=8ZfpGFauFLIrq_xHY3iIe-rtuKO88YACDVO0WQJFWCU,4321
 floe/checks/rows.py,sha256=E-YUZUvBUyOB7zat2dtbPWFM-0T1-01lYKuQU1A6wto,4156
-floe/checks/unique.py,sha256=BUu6ezLtQKj8_9_SybGND6rxbUbzlmqh_WJomTHRfGw,9782
+floe/checks/unique.py,sha256=qoZVqS_uG_mWZlV7-LdxT6_dMVQRlklEvMutO_V0qaY,12396
 floe/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/config/entity.py,sha256=CLXLXzkUbQhxenD_LwjBt0eoPreaYaxiBVEsdmnFIco,15883
 floe/config/include.py,sha256=XKYhp2QK_VUmAODAXJ35Wl4FCpQEMQm24m-WKlpI_og,5033
 floe/config/parse.py,sha256=B3gOi80-M0y9Y4OyNjM4MYmM2mFHKGVpqipX1JqaLjM,5290
-floe/config/profile.py,sha256=mYbyFse2pBUFuV5TMtl0RFrfyPUnJhdklbAX5eSAM88,15564
+floe/config/profile.py,sha256=jWackztS-0H3dTMXHALTZ7LsIOmwjriwoeY4vFgARr0,16046
 floe/config/project.py,sha256=zgTOhTDlbMr1chViMEZyv0drozZdumTaIoKDL0vv9sw,14141
 floe/config/storage.py,sha256=BK3P65ukFqNgmbjCcuYmXR04OScf1yD8f19vGJq-aDI,6234
 floe/config/types.py,sha256=4eveoOlyGmNxyZpI1PSjli9lnXLPc1-jYj4fL8qHN_8,5038
-floe/config/validate.py,sha256=-cDn9CKfR1BmAATgYy1PWlTWSkpu4mjt4PMKNL7k2FE,35858
+floe/config/validate.py,sha256=4lU6yJ1qpI8AIr42SGy-xJR2brz_DBG6pNS_dkZw4zo,36065
 floe/config/vars.py,sha256=7Bq0IU0KrFeRlyA8W9-F0gDSDKfXCfFpBrX08H8Flvw,4577
 floe/config/yaml.py,sha256=HeFGhKFxYkiqQOwljYOgUD5naJyxL_ZIX6hUIS36tzU,10071
-floe/iceberg/catalog.py,sha256=TeINK-jVZJAurROEeovVNK3s4PY-czeWih1y_KWohP8,14455
+floe/iceberg/catalog.py,sha256=n2NlgqRgnTe3Etdyra70Ib2aYtpR7zXWrajKl7spyYg,15340
 floe/iceberg/compact.py,sha256=5O372vAIsvgRc2PGihryAsrQ0otn0F30AQu-cetRCP0,7123
 floe/iceberg/merge.py,sha256=VE1Odf2B34Gxj7ZcuhbrfbgTGM9AdiDz6_2BbcQ-kSQ,12448
-floe/iceberg/sink.py,sha256=uE0jLhhxRFOeKi7ChAzstKexy1ZrovE4EKATGTYn-Gk,11255
-floe/iceberg/unique.py,sha256=PKUFEcCkU71k66__KfUE7NTos73ZYlZbb8GJPn0yFoo,2958
-floe/iceberg/write.py,sha256=zrFeeJu-YT4Pxo9u-fRujUtFb1Epx1KFoq2qFyP1iGs,11036
+floe/iceberg/sink.py,sha256=ySEAsKYpaKnqF9t7WjOW03KwpKynNIqLxOP1kQ-0Tbg,11723
+floe/iceberg/unique.py,sha256=OmiPQthhB5jGTaXsCu9ZztNFEFUf6eT0MLU2Gnh-C2A,1095
+floe/iceberg/write.py,sha256=SRX5tb17zUeWwpg6uXKkVQ_muNJ2GqlQd0n-kmjWPVc,11141
 floe/lineage/events.py,sha256=4txd8bGUwKBZd-8Aw_ebTurhlxAYk8SVMgH8vSjC3bs,6682
 floe/lineage/observer.py,sha256=dDYfWvPpFGjdqvFqQDZ2pTjj7uDfqji2-BTID9hzg90,10075
 floe/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -43,24 +43,24 @@ floe/read/orc.py,sha256=TWgdlLBR2NF_oVZ5zEmRKNXoTkq5QfF2BOFzKyzSjhY,1553
 floe/read/parquet.py,sha256=JwfuWaHj2TxBKJeaKignq8p0Op_pUfsOdVh7dDC3wcE,1491
 floe/read/xlsx.py,sha256=dengSY8HuNh9ahyR_LR8beqXTZQubqQ1Jx8qEak5jrY,3009
 floe/read/xml.py,sha256=7m6DtoU8iExuLROavyojysqOUE_Bd_N0uwFaLAzjI_Y,8004
-floe/report/entity.py,sha256=ekuB-1JDATvxIFnBTCQpku8uXtrSQNwvryjdTidXk6I,9707
+floe/report/entity.py,sha256=FRwLtXrlJXDNXrOKscitm-NCNJWQXd0vr_JIf_cUp20,9867
 floe/report/summary.py,sha256=tb547VRkM5WGWHQ-4vZB2DUW4VTWgv9FUoym-NpuWE4,4452
 floe/run/__init__.py,sha256=6teCk4GZoX7grwtyhQUbqwbx_TSQebzViUJA19CmEl4,8107
-floe/run/entity.py,sha256=4UGhUtjYR84MOe3jjwFmLGWAaDln1z5KOHC0MOvDLpQ,45204
+floe/run/entity.py,sha256=dML8VR-UyLUNVupkKaHeFIbR06TdPP2qcJ2rAMW4GeU,47568
 floe/run/events.py,sha256=OE7zYkt0Dz2xrvgNavT1nHfZJ-97IIJ8LpHcameUJZU,2859
 floe/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/storage/archive.py,sha256=1gPImm9PDSJFCpWjg37ax4ubb0wwc6FZm38quQXhORg,1101
 floe/storage/inputs.py,sha256=nmgEo2ub3hPUghLUXHU1wnKzpL4aYZbHb6S0XPYvMew,10410
-floe/storage/local.py,sha256=NKjBf_-PkDnbZ3FzXWE7KZ1xY0G_o3tRy0_7nxW7FLs,6307
+floe/storage/local.py,sha256=UiGFYEY33j45CQspWzRyJBW0HC2Nk9_zG--wL-aUuJI,6388
 floe/storage/partitioning.py,sha256=MG8RJDrwgZhucTswlXC2RwCmzrYIoOE6SH_aBM11KW8,6178
 floe/storage/paths.py,sha256=Z0fry3Gp0Tu2DpUeKqNXdqYP6n_KcKzK2llGx0TGHB8,2343
 floe/storage/resolve.py,sha256=pFAuVytTfnSEcTAVlJjFTj7S_Vipjry1ormWZP3Egmc,2593
 floe/storage/s3.py,sha256=XcG1qJTEY3KLSMLT950mUdrAxuCA8-Gx3fpNSzyV_Xo,7508
 floe/storage/types.py,sha256=FvAEXWMdVdopRK276JyETRYGOva2-uR8pZvsWj6Xvy0,993
 floe/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-floe/write/outputs.py,sha256=5LgN_8PMPPOgkRVZw_KvxYXmc02-meL7iZLeMKVtufQ,4932
-floe_python-0.8.2.dist-info/METADATA,sha256=O9qNqOIcnbGRrkwUAcPGKh35ayNq0bG4YwZMpQ-z90g,442
-floe_python-0.8.2.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-floe_python-0.8.2.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
-floe_python-0.8.2.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
-floe_python-0.8.2.dist-info/RECORD,,
+floe/write/outputs.py,sha256=cXxGUIJnZ38XPaqEI6a83GFUsapBRdY52XQm3GlUPcI,6937
+floe_python-0.8.4.dist-info/METADATA,sha256=kGo2pIkok-Gpky0VSm-2S5ttVTHx3tDcE21ynZE-HM4,442
+floe_python-0.8.4.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+floe_python-0.8.4.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
+floe_python-0.8.4.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
+floe_python-0.8.4.dist-info/RECORD,,