floe-python 0.8.8__py3-none-any.whl → 0.9.0__py3-none-any.whl

floe/checks/unique.py

@@ -17,7 +17,6 @@ from floe.checks.rows import RowCheck
 
 SAMPLE_LIMIT = 5
 _SEEN = "__floe_seen"
-_DEBUG = "__floe_debug"
 _COUNT = "__floe_count"
 _BITS = "_bits"  # suffix of the bit-pattern column that keys a float column
 
@@ -56,45 +55,11 @@ def resolve_unique_keys(
     return resolved
 
 
-# `str` Debug; ponytail: non-ASCII non-printable characters (Rust `\\u{..}`) are left raw
-_STR_ESCAPES = {
-    "\\": "\\\\",
-    '"': '\\"',
-    "\n": "\\n",
-    "\r": "\\r",
-    "\t": "\\t",
-    "\0": "\\0",
-    **{chr(i): f"\\u{{{i:x}}}" for i in [*range(1, 32), 127] if chr(i) not in "\n\r\t"},
-}
-
-
 def _float_bits(s: pl.Series) -> pl.Series:
     """Rust's `to_bits()`: Polars cannot reinterpret a float, so view the Arrow buffer."""
     return pl.Series(s.cast(pl.Float64).to_arrow().view(pa.uint64()))
 
 
-def _debug(name: str, dtype: pl.DataType) -> pl.Expr:
-    """Rust's `{:?}` of the `UniqueKey` variant: the sample tie-break sorts on these strings."""
-    c = pl.col(name)
-    if dtype == pl.String:
-        escaped = c
-        for raw, esc in _STR_ESCAPES.items():
-            escaped = escaped.str.replace_all(raw, esc, literal=True)
-        return pl.format('String("{}")', escaped)
-    if dtype == pl.Boolean:
-        return pl.format("Bool({})", c.cast(pl.String))
-    if dtype.is_signed_integer():
-        return pl.format("I64({})", c.cast(pl.String))
-    if dtype.is_unsigned_integer():
-        return pl.format("U64({})", c.cast(pl.String))
-    if dtype.is_float():
-        return pl.format("F64({})", pl.col(name + _BITS).cast(pl.String))
-    if isinstance(dtype, pl.Datetime):  # chrono prints the fraction only when not zero
-        return pl.format('Other("{}")', c.dt.to_string("%Y-%m-%d %H:%M:%S%.f"))
-    # ponytail: date keys use `Other(display)`, unverified against Rust; ties may order differently
-    return pl.format('Other("{}")', c.cast(pl.String))
-
-
 def _text(value: object) -> str:
     """Rust's `to_string` of a key value."""
     if isinstance(value, bool):
@@ -221,7 +186,7 @@ class UniqueTracker:
         return RowCheck("unique", name, message, dup)
 
     def results(self) -> list[UniqueResult]:
-        """Per constraint, the top `SAMPLE_LIMIT` keys by count, ties by Rust's Debug string."""
+        """Per constraint, the top `SAMPLE_LIMIT` keys by count, ties by the key values (numbers numerically, strings by code point)."""
         return [self._result(c) for c in self._constraints]
 
     def _result(self, c: _Constraint) -> UniqueResult:
@@ -229,14 +194,9 @@ class UniqueTracker:
             return UniqueResult(c.columns, 0, [])
         parts = pl.concat(c.dup_counts)
         counts = parts.group_by(parts.columns[:-1]).agg(pl.col(_COUNT).sum())
-        debug = pl.format(
-            "CompositeKey([{}])",
-            pl.concat_str([_debug(k, counts.schema[k]) for k in c.ids], separator=", "),
-        )
-        top = (
-            counts.with_columns(debug.alias(_DEBUG))
-            .sort([_COUNT, _DEBUG], descending=[True, False])
-            .head(SAMPLE_LIMIT)
+        by = [_COUNT, *counts.columns[:-1]]
+        top = counts.sort(by, descending=[True] + [False] * (len(by) - 1)).head(
+            SAMPLE_LIMIT
         )
         samples = [  # at most SAMPLE_LIMIT rows
             UniqueSample(

floe/config/profile.py

@@ -1,5 +1,6 @@
 """Environment profiles, ported from `profile/parse.rs` and `profile/validate.rs`."""
 
+import math
 import os
 from collections.abc import Callable, Mapping
 from dataclasses import dataclass
@@ -45,7 +46,8 @@ CONTRACT = {
 @dataclass(frozen=True)
 class ProfileConfig:
     name: str
-    runner: dict[str, object]  # `type` and every field, None when unset
+    # `type` and every field, None when unset; then a kubernetes_job's pass-through keys
+    runner: dict[str, object]
     variables: dict[str, str]
     catalogs: CatalogsConfig | None
     storages: StoragesConfig | None
@@ -157,17 +159,6 @@ def _str_keyed(value: object, where: str) -> dict[str, object]:
     return cast(dict[str, object], hash)
 
 
-def _object(hash: Hash, key: str, ctx: str) -> dict[str, object] | None:
-    return _str_keyed(hash[key], f"{ctx}.{key}") if key in hash else None
-
-
-def _objects(hash: Hash, key: str, ctx: str) -> list[dict[str, object]] | None:
-    if key not in hash:
-        return None
-    where = f"{ctx}.{key}"
-    return [_str_keyed(item, f"{where}[]") for item in yaml_array(hash[key], where)]
-
-
 def _auth(hash: Hash, key: str, ctx: str) -> dict[str, str | None] | None:
     if key not in hash:
         return None
@@ -200,24 +191,41 @@ _RUNNER_FIELDS: tuple[tuple[str, Reader], ...] = (
     ("env_parameters", _map),
     ("job_scope", _job_scope),
     ("domain_resources", _domain_resources),
-    ("labels", _map),
-    ("annotations", _map),
-    ("image_pull_policy", _string),
-    ("security_context", _object),
-    ("node_selector", _map),
-    ("tolerations", _objects),
-    ("backoff_limit", _u64),
-    ("env_from", _objects),
 )
+# Legacy: the keys any runner type accepted before #1065. A non-k8s runner still accepts and ignores them,
+# so a profile copied from a kubernetes_job keeps loading; they are not validated here.
+_LEGACY_IGNORED = (
+    *("labels", "annotations", "image_pull_policy", "security_context"),
+    *("node_selector", "tolerations", "backoff_limit", "env_from"),
+)
+
+
+def _json(value: object, where: str) -> object:
+    """A pass-through value as written, once JSON can carry it: string mapping keys, finite numbers."""
+    if isinstance(value, dict):
+        hash = _str_keyed(cast(object, value), where)
+        return {k: _json(v, f"{where}.{k}") for k, v in hash.items()}
+    if isinstance(value, list):
+        return [_json(v, f"{where}[]") for v in cast(list[object], value)]
+    if isinstance(value, float) and not math.isfinite(value):
+        raise FloeConfigError(f"{where} must be a finite number")
+    return value
 
 
 def _runner(value: object) -> dict[str, object]:
     ctx = "profile.execution.runner"
     hash = yaml_hash(value, ctx)
-    validate_known_keys(hash, ctx, ("type", *(name for name, _ in _RUNNER_FIELDS)))
+    known = ("type", *(name for name, _ in _RUNNER_FIELDS))
+    # #1065: a kubernetes_job's other keys are pod options its orchestrator validates; they pass through
+    open_ = hash.get("type") == "kubernetes_job"
+    validate_known_keys(hash, ctx, hash.keys() if open_ else (*known, *_LEGACY_IGNORED))
     read: dict[str, object] = {"type": _string(hash, "type", ctx, True)}
     for name, reader in _RUNNER_FIELDS:
         read[name] = reader(hash, name, ctx)
+    if open_:
+        read |= {
+            str(k): _json(v, f"{ctx}.{k}") for k, v in hash.items() if k not in known
+        }
     return read
 
 

floe/iceberg/catalog.py

@@ -12,7 +12,11 @@ from urllib.parse import urlsplit
 
 import duckdb
 from pyiceberg.catalog import Catalog, load_catalog
-from pyiceberg.exceptions import NamespaceAlreadyExistsError
+from pyiceberg.exceptions import (
+    NamespaceAlreadyExistsError,
+    NoSuchNamespaceError,
+    NoSuchTableError,
+)
 from pyiceberg.partitioning import PartitionField, PartitionSpec
 from pyiceberg.schema import Schema
 from pyiceberg.table import Table
@@ -345,55 +349,59 @@ def ensure_table(
 
     With `add_columns`, columns new in `schema` are added as optional (D13) and returned by name; partition changes stay rejected.
     """
-    ident = (namespace,)
-    with run_error("rest catalog namespace_exists check failed"):
-        namespace_exists = catalog.namespace_exists(ident)
-    if not namespace_exists:
-        if not create_namespace:
-            raise FloeRunError(f"entity.name={entity} iceberg namespace {namespace} does not exist and create_database_if_missing is false")  # fmt: skip
-        with (
-            run_error("rest catalog namespace create failed"),
-            suppress(NamespaceAlreadyExistsError),
-        ):  # another entity or process created it first
-            catalog.create_namespace(ident)
     identifier = (namespace, name)
-    with run_error("rest catalog table_exists check failed"):
-        exists = catalog.table_exists(identifier)
-    if exists:
-        with run_error("rest catalog load_table failed"):
+    with run_error("rest catalog load_table failed"):
+        try:
             table = catalog.load_table(identifier)
-        added: list[NestedField] = []
-        if schema_evolution == "add_columns":
-            added = additive_columns(table.schema(), schema, entity)
-            # field ids of `schema` may differ from the table's once columns are added
-            check_schema = table.schema()
-        else:
-            check_schema_matches(table.schema(), schema, entity)
-            check_schema = schema
-        check_partition_spec_matches(
-            table.spec(),
+        except (
+            NoSuchTableError,
+            NoSuchNamespaceError,
+        ):  # fmt: skip -- a missing namespace raises either, per catalog
+            table = None
+    if table is None:
+        ident = (namespace,)
+        with run_error("rest catalog namespace_exists check failed"):
+            namespace_exists = catalog.namespace_exists(ident)
+        if not namespace_exists:
+            if not create_namespace:
+                raise FloeRunError(f"entity.name={entity} iceberg namespace {namespace} does not exist and create_database_if_missing is false")  # fmt: skip
+            with (
+                run_error("rest catalog namespace create failed"),
+                suppress(NamespaceAlreadyExistsError),
+            ):  # another entity or process created it first
+                catalog.create_namespace(ident)
+        spec = (
             None
             if partition_fields is None
-            else build_partition_spec(check_schema, partition_fields, entity),
-            entity,
+            else build_partition_spec(schema, partition_fields, entity)
         )
-        if added:
-            with run_error("rest catalog update_schema failed"):
-                with table.update_schema() as update:
-                    for field in added:
-                        update.add_column(field.name, field.field_type, required=False)
-                table.refresh()
-        return table, [f.name for f in added]
-    spec = (
+        with run_error("rest catalog create_table failed"):
+            created = catalog.create_table(  # pyright: ignore[reportUnknownMemberType] -- PyIceberg's `schema` is `Schema | Unknown`
+                identifier,
+                schema,
+                location=location,
+                partition_spec=spec if spec is not None else PartitionSpec(),
+            )
+        return created, []
+    added: list[NestedField] = []
+    if schema_evolution == "add_columns":
+        added = additive_columns(table.schema(), schema, entity)
+        # field ids of `schema` may differ from the table's once columns are added
+        check_schema = table.schema()
+    else:
+        check_schema_matches(table.schema(), schema, entity)
+        check_schema = schema
+    check_partition_spec_matches(
+        table.spec(),
         None
         if partition_fields is None
-        else build_partition_spec(schema, partition_fields, entity)
+        else build_partition_spec(check_schema, partition_fields, entity),
+        entity,
     )
-    with run_error("rest catalog create_table failed"):
-        created = catalog.create_table(  # pyright: ignore[reportUnknownMemberType] -- PyIceberg's `schema` is `Schema | Unknown`
-            identifier,
-            schema,
-            location=location,
-            partition_spec=spec if spec is not None else PartitionSpec(),
-        )
-    return created, []
+    if added:
+        with run_error("rest catalog update_schema failed"):
+            with table.update_schema() as update:
+                for field in added:
+                    update.add_column(field.name, field.field_type, required=False)
+            table.refresh()
+    return table, [f.name for f in added]

floe/iceberg/compact.py

@@ -71,7 +71,9 @@ class _RewriteFiles(_OverwriteFiles):
                 for e in entries:
                     writer.delete(e) if e.data_file in self._deleted_data_files else writer.existing(e)  # fmt: skip
             out.append(writer.to_manifest_file())
-        return out
+        return [
+            m for m in out if m.has_added_files() or m.has_existing_files()
+        ]  # #1136: as `_LiveOverwrite`
 
     def _summary(self, snapshot_properties: Mapping[str, str] = {}) -> Summary:
         # built as an OVERWRITE: PyIceberg 0.12's `update_snapshot_summaries` rejects REPLACE

floe/iceberg/sink.py

@@ -15,6 +15,7 @@ from typing import Any
 import polars as pl
 import pyarrow as pa
 from pyiceberg.catalog import Catalog
+from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchTableError
 from pyiceberg.schema import Schema, assign_fresh_schema_ids
 from pyiceberg.table import Table
 
@@ -39,6 +40,7 @@ from floe.iceberg.merge import merge_scd1, merge_scd2
 from floe.iceberg.write import (
     SMALL_FILE_THRESHOLD_BYTES,
     IcebergWriteResult,
+    parquet_properties,
     write_table,
 )
 from floe.storage.inputs import InputFile
@@ -176,13 +178,15 @@ def write_iceberg(
     primary_key = entity.schema_.primary_key or []  # validated non-empty for merges
     key = [output.get(k.strip(), k.strip()) for k in primary_key]
     identifier = (target.namespace, target.table)
-    with run_error("rest catalog table_exists check failed"):
-        # the configured mode: a refreshed merge writes as overwrite and keeps the key guard (#1091)
-        exists = entity.sink.write_mode in ("merge_scd1", "merge_scd2") and catalog.table_exists(identifier)  # fmt: skip
-    if exists:
+    # the configured mode: a refreshed merge writes as overwrite and keeps the key guard (#1091)
+    if entity.sink.write_mode in ("merge_scd1", "merge_scd2"):
         with run_error("rest catalog load_table failed"):
-            existing = set(catalog.load_table(identifier).schema().column_names)
-        if added := [k for k in key if k not in existing]:  # Rust `merge/shared.rs`
+            try:
+                existing = set(catalog.load_table(identifier).schema().column_names)
+            except (NoSuchTableError, NoSuchNamespaceError):
+                existing = None
+        # Rust `merge/shared.rs`
+        if existing is not None and (added := [k for k in key if k not in existing]):
             raise FloeRunError(f"entity.name={entity.name} iceberg schema evolution failed: merge key columns cannot be added: {', '.join(added)}")  # fmt: skip
     table, added = ensure_table(
         catalog, target.namespace, target.table, schema, entity.name, target.location.removeprefix("local://"),
@@ -215,7 +219,7 @@ def write_iceberg(
         threshold = SMALL_FILE_THRESHOLD_BYTES if size is None else max(1, size // 2)
         append = "overwrite" if mode == "overwrite" else "append"
         unique = [[output.get(c, c) for c in k] for k in resolve_unique_keys(entity.schema_.unique_keys, [c.name for c in entity.schema_.columns if c.unique], entity.schema_.primary_key)]  # fmt: skip
-        result = write_table(table, reader, append, run_id, digest, threshold, unique_keys=unique)  # fmt: skip
+        result = write_table(table, reader, append, run_id, digest, threshold, unique_keys=unique, parquet=parquet_properties(options and options.compression, options and options.row_group_size))  # fmt: skip
     ids = {"table_version": result.metadata_version, "snapshot_id": result.snapshot_id}
     fields = {k: v for k, v in ids.items() if v is not None} | {
         "iceberg_catalog_name": target.catalog.name,

floe/iceberg/write.py

@@ -5,7 +5,7 @@
 
 import itertools
 import uuid
-from collections.abc import Callable, Iterator, Sequence
+from collections.abc import Callable, Iterator, Mapping, Sequence
 from dataclasses import dataclass, replace
 from datetime import datetime
 from pathlib import PurePosixPath
@@ -18,9 +18,17 @@ from pyiceberg.io.pyarrow import (
     _check_pyarrow_schema_compatible,  # pyright: ignore[reportPrivateUsage] -- same check `Transaction.append` makes
     _dataframe_to_data_files,  # pyright: ignore[reportPrivateUsage] -- feeds one snapshot producer from many chunks
 )
-from pyiceberg.manifest import DataFile, DataFileContent, ManifestEntryStatus
+from pyiceberg.manifest import (
+    DataFile,
+    DataFileContent,
+    ManifestEntryStatus,
+    ManifestFile,
+)
 from pyiceberg.table import Table, TableProperties
-from pyiceberg.table.snapshots import ancestors_of
+from pyiceberg.table.snapshots import Operation, ancestors_of
+from pyiceberg.table.update.snapshot import (
+    _OverwriteFiles,  # pyright: ignore[reportPrivateUsage] -- PyIceberg 0.12 has no public hook on the manifest list
+)
 
 from floe.errors import FloeRunError
 from floe.iceberg.catalog import run_error
@@ -55,6 +63,14 @@ class IcebergWriteResult:
     )
 
 
+class _LiveOverwrite(_OverwriteFiles):
+    """PyIceberg 0.12 keeps the all-DELETED manifest of every earlier overwrite in the manifest list, one more per
+    overwrite, so the scan and the commit read more manifests each time (#1132); Java drops them, so do we."""
+
+    def _existing_manifests(self) -> list[ManifestFile]:
+        return [m for m in super()._existing_manifests() if m.has_added_files() or m.has_existing_files()]  # fmt: skip
+
+
 def _chunks(reader: pa.RecordBatchReader, chunk_rows: int) -> Iterator[pa.Table]:
     pending: list[pa.RecordBatch] = []
     rows = 0
@@ -72,6 +88,16 @@ def _chunks(reader: pa.RecordBatchReader, chunk_rows: int) -> Iterator[pa.Table]
         yield pa.Table.from_batches(pending, reader.schema)
 
 
+def parquet_properties(
+    compression: str | None, row_group_size: int | None
+) -> dict[str, str]:
+    """The sink's `compression` and `row_group_size` as PyIceberg's write properties; unset keeps the table's."""
+    props = {TableProperties.PARQUET_COMPRESSION: compression} if compression else {}
+    if row_group_size:
+        props[TableProperties.PARQUET_ROW_GROUP_LIMIT] = str(row_group_size)
+    return props
+
+
 def _metadata_version(location: str | None) -> int | None:
     prefix = PurePosixPath(location or "").name.split("-", 1)[0]
     return int(prefix) if prefix.isdigit() else None
@@ -86,22 +112,25 @@ def write_table(
     small_file_threshold_bytes: int = SMALL_FILE_THRESHOLD_BYTES,
     chunk_rows: int = CHUNK_ROWS,
     unique_keys: Sequence[Sequence[str]] = (),
+    parquet: Mapping[str, str] | None = None,
 ) -> IcebergWriteResult:
     """Data files are written chunk by chunk (partitioned tables split each chunk), then committed once.
 
     Overwrite replaces every existing file in the same snapshot. An append of no rows commits nothing.
     On a conflict an append checks `unique_keys` (table column names) against the files added since the base
     snapshot and fails closed, committing nothing, when a rival already holds a key (#549).
+    `parquet` holds `write.parquet.*` properties for these data files only; the table's own properties are not changed.
     """
     with run_error("iceberg data write failed"):
         _check_pyarrow_schema_compatible(table.schema(), reader.schema)
+        metadata = table.metadata.model_copy(update={"properties": {**table.properties, **(parquet or {})}})  # fmt: skip
         counter = itertools.count()
         write_uuid = uuid.uuid4()
         data_files: list[DataFile] = [
             data_file
             for chunk in _chunks(reader, chunk_rows)
             for data_file in _dataframe_to_data_files(
-                table.metadata, chunk, table.io, write_uuid, counter
+                metadata, chunk, table.io, write_uuid, counter
             )
         ]
     digest = (
@@ -131,13 +160,21 @@ def write_table(
                 # PyIceberg retries a 409 itself (refresh, rebuild) and would skip the guard; in memory only, never sent
                 table.metadata = table.metadata.model_copy(update={"properties": {**table.properties, TableProperties.COMMIT_NUM_RETRIES: "0"}})  # fmt: skip
                 tx = table.transaction()
-                update = tx.update_snapshot(
-                    snapshot_properties={RUN_ID: run_id, INPUT_HASH: digest}
-                )
+                props = {RUN_ID: run_id, INPUT_HASH: digest}
                 with (
-                    update.overwrite()
+                    _LiveOverwrite(
+                        operation=Operation.OVERWRITE
+                        if table.current_snapshot()
+                        else Operation.APPEND,
+                        transaction=tx,
+                        io=table.io,
+                        branch="main",
+                        snapshot_properties=props,
+                    )
                     if mode == "overwrite"
-                    else update.fast_append() as snap
+                    else tx.update_snapshot(
+                        snapshot_properties=props
+                    ).fast_append() as snap
                 ):
                     for old in replaced:
                         snap.delete_data_file(old)

floe/manifest.py

@@ -275,10 +275,6 @@ _RUNNER_FIELDS = (
     *("resources", "env", "workspace_url", "existing_cluster_id", "config_uri"),
     *("python_file_uri", "job_name", "auth", "env_parameters"),
 )
-_PYTHON_RUNNER_FIELDS = (  # carried only when set
-    *("job_scope", "domain_resources", "labels", "annotations", "image_pull_policy"),
-    *("security_context", "node_selector", "tolerations", "backoff_limit", "env_from"),
-)
 
 
 # Rust `runners_contract`: the fields each profile runner type carries into the manifest
@@ -303,9 +299,10 @@ def _runners(profile: ProfileConfig | None) -> Json:
             v = dict(sorted(cast(dict[str, str], v).items()))
         runner[k] = v
     if kind == "kubernetes_job":
-        for k in _PYTHON_RUNNER_FIELDS:
-            if (v := profile.runner.get(k)) is not None:
-                runner[k] = v
+        # `job_scope`, `domain_resources` and the pod options (#1065), only when set
+        runner |= {
+            k: v for k, v in profile.runner.items() if k not in runner and v is not None
+        }
     if kind == "databricks_job":
         runner["job_name"] = runner["job_name"] or "floe-{domain}-{env}"
     return {"default": "default", "definitions": {"default": runner}}

floe/run/__init__.py

@@ -2,6 +2,7 @@
 
 import time
 from collections.abc import Sequence
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from pathlib import Path
@@ -158,12 +159,15 @@ def run(
 
     emit("run_started", config=config, report_base=report_path)
     # Rust resolves every entity's inputs before the first entity_started (#864); run_entity reuses them
-    listed: dict[str, ResolvedInputs] = {}
-    for e in selected:
-        # D24: a full refresh may rebuild from the archive alone, so run_entity decides
-        archive = full_refresh and e.sink.archive and e.incremental_mode in ("none", "archive")  # fmt: skip
-        if not archive:
-            listed[e.id] = inputs(e)
+    # D24: a full refresh may rebuild from the archive alone, so run_entity decides
+    to_list = [e for e in selected if not (full_refresh and e.sink.archive and e.incremental_mode in ("none", "archive"))]  # fmt: skip
+    # `map` yields in config order, so the first failure in config order is raised
+    if workers > 1 and len(to_list) > 1:
+        with ThreadPoolExecutor(workers, thread_name_prefix="floe-list") as pool:
+            found = list(pool.map(inputs, to_list))
+    else:
+        found = [inputs(e) for e in to_list]
+    listed = {e.id: f for e, f in zip(to_list, found, strict=True)}
     client = storage_client(report_storage)
 
     def finish(e: EntityConfig) -> dict[str, Any]:

floe/run/entity.py

@@ -7,7 +7,7 @@ import time
 from collections.abc import Callable, Generator, Iterable, Iterator
 from contextlib import closing, suppress
 from dataclasses import dataclass, field, replace
-from datetime import datetime
+from datetime import date, datetime
 from functools import partial
 from itertools import chain
 from pathlib import Path
@@ -93,6 +93,7 @@ from floe.storage.partitioning import (
     advance,
     in_load_order,
     listing_start,
+    partition_date,
     partition_of,
     source_root,
     window,
@@ -313,7 +314,11 @@ def _with_archived(e: EntityConfig, archive: tuple[str, StorageClient], source:
             source()
         )  # nothing archived: the source's "no input files matched" stands
     if p is not None:  # D29: the archive keeps the partition sub-paths
-        old = in_load_order(p, e.name, f"{base}/{e.name}", old, archived=True)
+        # #939: D28 orders a day's batches by the source's modification time, which the copy's own does not keep
+        kept = [replace(f, source_mtime=client.source_mtime(f.source_uri) or f.source_mtime) for f in old.files]  # fmt: skip
+        old = in_load_order(
+            p, e.name, f"{base}/{e.name}", replace(old, files=kept), archived=True
+        )
     try:
         new = source()
     except NoInputsError:  # no new source file: rebuild from the archive alone
@@ -402,7 +407,8 @@ def _run_file(plan: _Plan, file: InputFile, path: Path, spill: Path) -> tuple[di
     def failed(err: FloeIoError, mismatch: MismatchOutcome | None) -> tuple[dict[str, Any], Iterable[pl.DataFrame]]:  # fmt: skip
         # Rust skips the raw copy under PII: the file cannot be masked
         plan.tracker.restore(keys)  # the unreadable file leaves no keys behind
-        keep = entity.pii is None and entity.sink.rejected is not None
+        # a file removed after listing has nothing to copy: the read error alone is reported
+        keep = entity.pii is None and entity.sink.rejected is not None and path.exists()
         raw = _copy_raw(plan, path, severity, file.source_name) if keep else None
         return _failed_file(entity, uri, len(declared) + len(derived), mismatch, err, raw), []  # fmt: skip
 
@@ -633,6 +639,7 @@ class Incremental:
         # the file reports, set once the sink step is done (#915)
         self.committed: list[dict[str, Any]] | None = None
         self.advance: Callable[[EntityState, list[str]], None] | None = None  # D30
+        self.dated: Callable[[str], date | None] | None = None  # #949: an input's partition date  # fmt: skip
         self._stop = threading.Event()
         self._thread: threading.Thread | None = None
 
@@ -643,9 +650,11 @@ class Incremental:
             return inputs, []
         self.uri, self.client = state_uri(e, self.config_dir, self.storages)
         if self.full_refresh:
-            self._hold(claim_all(self.client, self.uri, e.name, self.run_id, inputs))
+            # #949: `expired` holds the claims it replaced, which `release` restores if the refresh wrote nothing
+            claimed, self.expired = claim_all(self.client, self.uri, e.name, self.run_id, inputs)  # fmt: skip
+            self._hold(claimed)
             return inputs, []
-        got = claim_inputs(self.client, self.uri, e.name, self.run_id, inputs, e.sink.write_mode.startswith("merge"))  # fmt: skip
+        got = claim_inputs(self.client, self.uri, e.name, self.run_id, inputs, e.sink.write_mode.startswith("merge"), self.dated)  # fmt: skip
         reports = skipped_reports(e, got.already_processed, got.active_claims)
         _emit_skipped(emit, e.name, reports)
         self.expired = got.expired
@@ -715,10 +724,17 @@ class Incremental:
         return claimed
 
     def release(self) -> None:
-        """The dead runs' claims this run took over go back: the next run checks them again (#545)."""
+        """The dead runs' claims this run took over go back: the next run checks them again (#545).
+
+        #949: a full refresh puts back the claims it replaced only if it wrote nothing, as its overwrite drops their rows
+        and the retry guard would find their run's snapshot behind it.
+        """
         if claimed := self._end():
+            restore = (
+                {} if self.full_refresh and self.committed is not None else self.expired
+            )
             # D30: an older day's released file stays listed for a retry
-            release(self.client, self.uri, self.run_id, claimed, self.expired, self.advance is not None and not self.full_refresh)  # fmt: skip
+            release(self.client, self.uri, self.run_id, claimed, restore, self.advance is not None and not self.full_refresh)  # fmt: skip
 
     def finish(self, files: list[dict[str, Any]]) -> None:
         """Promote unless the entity failed (Rust commits on success, warnings and rejected).
@@ -786,12 +802,18 @@ def list_source(entity: EntityConfig, config_dir: Path, storages: StoragesConfig
     after = listing_start(p, start) if p and start else None
 
     def source() -> ResolvedInputs:
-        inputs = resolve_inputs(
-            config_dir, src.path if uri.startswith("local://") else uri, src.format, entity.name,
-            src.storage or "local", client, glob_override=src.options.glob, exclude=archive and archive[0],
-            recursive=bool(src.options.recursive) or p is not None,  # D28: partitioning lists recursively
-            after=f"{root}/{after}" if after else "",
-        )  # fmt: skip
+        try:
+            inputs = resolve_inputs(
+                config_dir, src.path if uri.startswith("local://") else uri, src.format, entity.name,
+                src.storage or "local", client, glob_override=src.options.glob, exclude=archive and archive[0],
+                recursive=bool(src.options.recursive) or p is not None,  # D28: partitioning lists recursively
+                after=f"{root}/{after}" if after else "",
+            )  # fmt: skip
+        except NoInputsError:
+            # #1139: an empty incremental source has nothing pending; a full refresh still fails (a broken extraction)
+            if entity.incremental_mode != "file" or full_refresh:
+                raise
+            return ResolvedInputs([], [], "directory")
         if not p:
             return inputs
         inputs = in_load_order(p, entity.name, root, inputs, start=start)
@@ -921,6 +943,7 @@ def _run_entity(
     if entity.source.partitioning and entity.incremental_mode == "file":
         dated = entity.source.partitioning, entity.name, source_root(source_uri)
         inc.advance = partial(advance, *dated)
+        inc.dated = partial(partition_date, *dated)
     archive = plan.archive  # a full refresh lists it too
     plan.derived = _derived(entity, source_uri, archive and archive[0], inputs.files)
     # D34 (#931): same store only (a bucket name repeats across endpoints); before the claim, so a refusal leaves nothing to release

floe/state.py

@@ -6,7 +6,7 @@ import random
 import time
 from collections.abc import Callable
 from dataclasses import asdict, dataclass
-from datetime import UTC, datetime, timedelta
+from datetime import UTC, date, datetime, timedelta
 from typing import NamedTuple
 
 from floe.errors import FloeConfigError
@@ -44,6 +44,8 @@ class EntityState:
     files: dict[str, FileState]
     claims: dict[str, Claim]
     watermark: str | None = None  # D30: the latest partition date ingested (ISO)
+    # #949: `files` holds no entry dated before it (ISO); only a full refresh lists before it
+    pruned_before: str | None = None
 
 
 class Loaded(NamedTuple):
@@ -74,6 +76,8 @@ def dump_state(state: EntityState) -> bytes:
     }
     if state.watermark:  # D30: additive, absent without partitioning
         doc["watermark"] = state.watermark
+    if state.pruned_before:  # #949: additive too
+        doc["pruned_before"] = state.pruned_before
     return json.dumps(doc, indent=2, ensure_ascii=False).encode()
 
 
@@ -89,6 +93,7 @@ def parse_state(entity: str, body: bytes) -> EntityState:
             if schema != SCHEMA_V1
             else {},
             doc.get("watermark"),
+            doc.get("pruned_before"),
         )
     except (ValueError, KeyError, TypeError, AttributeError) as e:
         raise FloeConfigError(f"entity.name={entity} invalid state file: {e}") from e
@@ -96,6 +101,13 @@ def parse_state(entity: str, body: bytes) -> EntityState:
         raise FloeConfigError(
             f"entity.name={entity} state schema mismatch: expected {SCHEMA_V1} or {SCHEMA_V2}, got {schema}"
         )
+    if state.pruned_before is not None:  # #949: compared as an ISO date string
+        try:
+            date.fromisoformat(state.pruned_before)
+        except (ValueError, TypeError) as e:
+            raise FloeConfigError(
+                f"entity.name={entity} invalid state file: pruned_before: {e}"
+            ) from e
     if state.entity != entity:
         raise FloeConfigError(
             f"entity.name={entity} state entity mismatch: expected {entity}, got {state.entity}"
@@ -138,8 +150,13 @@ def claim_inputs(
     run_id: str,
     inputs: list[InputFile],
     barrier: bool = False,
+    dated: Callable[[str], date | None] | None = None,
 ) -> ClaimOutcome:
-    """`barrier` (merge modes, #545 S7): while another run's claim is live, every input waits as `active_claim`."""
+    """`barrier` (merge modes, #545 S7): while another run's claim is live, every input waits as `active_claim`.
+
+    `dated` (#949): an input's partition date; an unclaimed one dated before `pruned_before` is dropped, as a later
+    window would not list it (a run that listed before another pruned would re-ingest it).
+    """
     if not inputs:
         return ClaimOutcome([], None, [], [], {})
     for attempt in range(CAS_RETRIES):
@@ -157,6 +174,10 @@ def claim_inputs(
         processed: list[tuple[InputFile, FileState]] = []
         acquired_at, expires_at = _now(), _now(CLAIM_TTL_SECONDS)
         for f in inputs:
+            day = dated(f.source_uri) if dated and state.pruned_before else None
+            # a claimed one is a retry: the window is held back to it
+            if day and day.isoformat() < (state.pruned_before or "") and f.source_uri not in expired and f.source_uri not in state.claims:  # fmt: skip
+                continue
             if f.source_uri in state.files:
                 processed.append((f, state.files[f.source_uri]))
             elif blocked or f.source_uri in state.claims:
@@ -186,8 +207,11 @@ def claim_inputs(
 
 def claim_all(
     client: StorageClient, uri: str, entity: str, run_id: str, inputs: list[InputFile]
-) -> Loaded:
-    """Full refresh: claim every input, dropping other claims (Rust `claim_all_entity_inputs`)."""
+) -> tuple[Loaded, dict[str, Claim]]:
+    """Full refresh: claim every input, dropping other claims (Rust `claim_all_entity_inputs`).
+
+    Also the claims it replaced, by URI, for `release` to restore if the refresh fails (#949).
+    """
     for attempt in range(CAS_RETRIES):
         if attempt:
             _backoff(attempt - 1)
@@ -197,11 +221,13 @@ def claim_all(
             f.source_uri: Claim(run_id, acquired_at, expires_at, f.source_size, f.source_mtime)
             for f in inputs
         }  # fmt: skip
-        # D30: no watermark until the refresh promotes, so a failed one leaves the next run listing everything
-        fresh = Loaded(EntityState(entity, acquired_at, loaded.state.files, claims), loaded.version)  # fmt: skip
+        # D30: no watermark until the refresh promotes; #949: `pruned_before` stays, so after a failed one the next run
+        # lists from it, never the pruned days
+        fresh = Loaded(EntityState(entity, acquired_at, loaded.state.files, claims, None, loaded.state.pruned_before), loaded.version)  # fmt: skip
         version = _persist(client, uri, fresh)
         if version is not None:
-            return Loaded(fresh.state, version)
+            replaced = {u: c for u, c in loaded.state.claims.items() if u in claims}
+            return Loaded(fresh.state, version), replaced
     raise FloeConfigError(
         f"entity.name={entity} full-refresh state write conflicted after {CAS_RETRIES} retries"
     )
@@ -225,7 +251,7 @@ def _mutate(
         )
         mutate(loaded.state, ours)
         state = loaded.state
-        if state.files or state.claims or state.watermark:
+        if state.files or state.claims or state.watermark or state.pruned_before:
             done = _persist(client, uri, loaded) is not None
         else:  # Rust deletes an emptied state
             done = client.delete_conditional(uri, loaded.version)

floe/storage/archive.py

@@ -21,7 +21,7 @@ def archive_input_file(
         f"{entity_name}/{subdir}" if subdir else entity_name, input_file.source_name, run_id, input_file.source_uri
     )  # fmt: skip
     dest_uri = join_relative(relative)
-    client.copy(input_file.source_uri, dest_uri)
+    client.copy(input_file.source_uri, dest_uri, input_file.source_mtime)
     try:
         client.delete(input_file.source_uri)
     except FloeError as e:

floe/storage/local.py

@@ -96,7 +96,8 @@ class LocalClient:
             key=lambda r: r.uri,
         )
 
-    def copy(self, src_uri: str, dst_uri: str) -> None:
+    def copy(self, src_uri: str, dst_uri: str, source_mtime: str | None = None) -> None:
+        """`source_mtime` is not kept: a local copy's nanosecond mtime already follows the archive (load) order (#939)."""
         src, dst = _path(src_uri), _path(dst_uri)
         try:
             dst.parent.mkdir(parents=True, exist_ok=True)
@@ -109,6 +110,9 @@ class LocalClient:
         except OSError as e:
             raise FloeStorageError(f"local copy failed from {src} to {dst}: {e}") from e
 
+    def source_mtime(self, uri: str) -> str | None:
+        return None
+
     def write(self, uri: str, body: bytes) -> None:
         path = _path(uri)
         try:

floe/storage/partitioning.py

@@ -76,7 +76,7 @@ def _day(when: date | datetime) -> date:
     return when.date() if isinstance(when, datetime) else when
 
 
-def _dated(p: SourcePartitioning, entity: str, root: str, uri: str) -> date | None:
+def partition_date(p: SourcePartitioning, entity: str, root: str, uri: str) -> date | None:  # fmt: skip
     """The partition date of a state entry; None for one outside the pattern (another source path)."""
     try:
         return _day(partition_of(p, entity, root, uri).when)
@@ -89,10 +89,12 @@ def window(p: SourcePartitioning, entity: str, root: str, state: EntityState | N
 
     An outstanding claim holds it back to the claim's date, so a crashed run's files stay listed until it is retried.
     """
-    if state is None or state.watermark is None:
+    if state is None or not (state.watermark or state.pruned_before):
         return None
-    start = date.fromisoformat(state.watermark) - timedelta(days=p.lookback_days)
-    return min([start, *(d for u in state.claims if (d := _dated(p, entity, root, u)))])  # fmt: skip
+    # #949: never before `pruned_before`, whatever `lookback_days` became
+    floor = date.fromisoformat(state.pruned_before or date.min.isoformat())
+    start = max(floor, date.fromisoformat(state.watermark) - timedelta(days=p.lookback_days)) if state.watermark else floor  # fmt: skip
+    return min([start, *(d for u in state.claims if (d := partition_date(p, entity, root, u)))])  # fmt: skip
 
 
 def listing_start(p: SourcePartitioning, start: date) -> str | None:
@@ -107,10 +109,18 @@ def listing_start(p: SourcePartitioning, start: date) -> str | None:
 
 
 def advance(p: SourcePartitioning, entity: str, root: str, state: EntityState, promoted: list[str]) -> None:  # fmt: skip
-    """D30, on promote: the watermark moves to the latest date promoted, never back (#949 will prune `files`).
+    """D30, on promote: the watermark moves to the latest date promoted, never back.
 
     Without one (a first run, or after a failed full refresh), every file in the state counts.
+    #949: `pruned_before` moves to the window start, never back nor past a claim's date, and `files` drops the entries
+    dated before it, in the same state write.
     """
     known = promoted if state.watermark else [*promoted, *state.files]
-    days = [d.isoformat() for u in known if (d := _dated(p, entity, root, u))]
+    days = [d.isoformat() for u in known if (d := partition_date(p, entity, root, u))]
     state.watermark = max([state.watermark or "", *days]) or None
+    if state.watermark is None:
+        return
+    start = date.fromisoformat(state.watermark) - timedelta(days=p.lookback_days)
+    held = [d for u in state.claims if (d := partition_date(p, entity, root, u))]
+    state.pruned_before = max(state.pruned_before or "", min([start, *held]).isoformat())  # fmt: skip
+    state.files = {u: f for u, f in state.files.items() if (d := partition_date(p, entity, root, u)) is None or d.isoformat() >= state.pruned_before}  # fmt: skip

floe/storage/s3.py

@@ -4,6 +4,7 @@ import os
 import shutil
 from datetime import UTC, datetime
 from pathlib import Path
+from typing import Any
 
 import boto3
 from botocore.config import Config
@@ -15,6 +16,17 @@ from floe.storage.types import ObjectRef, StoredObject
 
 _CONFLICT = ("PreconditionFailed", "ConditionalRequestConflict", "412", "409")
 _NOT_FOUND = ("NoSuchKey", "NotFound", "404")
+# #939: an archive copy's `LastModified` is its copy time, to the second; this keeps the source's, for D28 replay order
+_SOURCE_MTIME = "floe-source-mtime"
+# the headers a `MetadataDirective=REPLACE` copy would otherwise drop
+_KEPT = (
+    "ContentType",
+    "ContentEncoding",
+    "ContentDisposition",
+    "ContentLanguage",
+    "CacheControl",
+    "Expires",
+)
 
 
 def resolve_path_style_access(configured: bool | None) -> bool:
@@ -117,15 +129,33 @@ class S3Client:
         except (ClientError, BotoCoreError) as e:
             raise FloeStorageError(f"s3 put object failed: {e}") from e
 
-    def copy(self, src_uri: str, dst_uri: str) -> None:
+    def copy(self, src_uri: str, dst_uri: str, source_mtime: str | None = None) -> None:
+        """`source_mtime` goes to the copy's `x-amz-meta-floe-source-mtime`, with the source's metadata and headers."""
         (sb, sk), (db, dk) = parse_s3_uri(src_uri), parse_s3_uri(dst_uri)
         try:
+            extra: dict[str, Any] = {}
+            if source_mtime:
+                head = self._client.head_object(Bucket=sb, Key=sk)
+                meta = {**head.get("Metadata", {}), _SOURCE_MTIME: source_mtime}
+                extra = {k: head[k] for k in _KEPT if k in head} | {"Metadata": meta, "MetadataDirective": "REPLACE"}  # fmt: skip
             self._client.copy_object(
-                Bucket=db, Key=dk, CopySource={"Bucket": sb, "Key": sk}
+                Bucket=db, Key=dk, CopySource={"Bucket": sb, "Key": sk}, **extra
             )
         except (ClientError, BotoCoreError) as e:
             raise FloeStorageError(f"s3 copy object failed: {e}") from e
 
+    def source_mtime(self, uri: str) -> str | None:
+        """#939: the source modification time an archive copy keeps; None on a copy made before it was kept."""
+        bucket, key = parse_s3_uri(uri)
+        try:
+            meta = self._client.head_object(Bucket=bucket, Key=key).get("Metadata", {})
+            # a Go server (MinIO, SeaweedFS) may send the header canonicalised: `X-Amz-Meta-Floe-Source-Mtime`
+            return next(
+                (v for k, v in meta.items() if k.lower() == _SOURCE_MTIME), None
+            )
+        except (ClientError, BotoCoreError) as e:
+            raise FloeStorageError(f"s3 head object failed for {uri}: {e}") from e
+
     def delete(self, uri: str) -> None:
         bucket, key = parse_s3_uri(uri)
         try:

floe/storage/types.py

@@ -20,7 +20,10 @@ class StorageClient(Protocol):
     def resolve_uri(self, path: str) -> str: ...
     def list(self, prefix: str, start_after: str = "") -> list[ObjectRef]: ...
     def write(self, uri: str, body: bytes) -> None: ...
-    def copy(self, src_uri: str, dst_uri: str) -> None: ...
+    def copy(
+        self, src_uri: str, dst_uri: str, source_mtime: str | None = None
+    ) -> None: ...
+    def source_mtime(self, uri: str) -> str | None: ...
     def delete(self, uri: str) -> None: ...
     def read(self, uri: str) -> StoredObject | None: ...
     def write_conditional(

{floe_python-0.8.8.dist-info → floe_python-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: floe-python
-Version: 0.8.8
+Version: 0.9.0
 Summary: Floe: a data-contract runtime for file ingestion
 License-File: LICENSE
 Requires-Python: >=3.11

{floe_python-0.8.8.dist-info → floe_python-0.9.0.dist-info}/RECORD

@@ -6,33 +6,33 @@ floe/add_entity.py,sha256=42b7l8o55dzgaR_DDgpW7YWdnU2weYMf4U_rf7qCY6M,21986
 floe/errors.py,sha256=hLSkgPQATBnjY1CAQqX4aB0oI0-WKoHW09bDXAgzC7k,186
 floe/glob.py,sha256=A0HdePLXLGl1T6heJbgdrNHr8pZfdidG7pQ1QsXWOl8,10579
 floe/maintain.py,sha256=CGSoZhO7p0u14o10FCKCO4FprzpVO548A71eq5FowF8,7343
-floe/manifest.py,sha256=sAWjv6yS4GokQDTnZwutyplLz1ICPBYl2qJmm9IORdM,20462
+floe/manifest.py,sha256=TDjSOrDXoH59lcYQ_FGIGCEg2JZ0EKGMw4cVtd1kzCA,20312
 floe/manifest_run.py,sha256=8rkHX2RuxHyp02NI2NLBsU6xq6m5yebR0eoYETV_7Tw,12591
 floe/pii.py,sha256=uJjU-C4c7h45E7rEL9K3Zp1LTTLh-FNKfUV5av2QMO4,4351
 floe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-floe/state.py,sha256=XFHF6zVxbdGg1bkONZDLryTD4noDF9IgaimaEI-3Nlg,11104
+floe/state.py,sha256=H7BRVmpw2zfwPR9FiLqbHHV53hoxHNvXGTbT9jHonRU,12635
 floe/checks/cast.py,sha256=lM40xtKEggWI8FW3jijM3T2JFQtaXs8FhsFXo3vy5oM,6788
 floe/checks/mismatch.py,sha256=RNqRL1nihH6Y83RI-dxPXf0WTWZccXq6mnR4m6V0IIo,5827
 floe/checks/normalize.py,sha256=9fdRYFjWGXBAEw1Po71N5tsUWG8eekmSWIQktPduOig,4431
 floe/checks/rows.py,sha256=-yPieSus0-7ZewLv2vR8waHqQ9DGMbVvmJh1_hK4eDs,4433
-floe/checks/unique.py,sha256=qoZVqS_uG_mWZlV7-LdxT6_dMVQRlklEvMutO_V0qaY,12396
+floe/checks/unique.py,sha256=ZVckMOoGuKIzq3M84TGeZ1ks0nab9YTaI54w5Q_rjZc,10864
 floe/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/config/entity.py,sha256=CLXLXzkUbQhxenD_LwjBt0eoPreaYaxiBVEsdmnFIco,15883
 floe/config/include.py,sha256=XKYhp2QK_VUmAODAXJ35Wl4FCpQEMQm24m-WKlpI_og,5033
 floe/config/parse.py,sha256=B3gOi80-M0y9Y4OyNjM4MYmM2mFHKGVpqipX1JqaLjM,5290
-floe/config/profile.py,sha256=4imKVdZ-Ehbj71OA3K4PrShE-gOYuicMDnTD-QHIkA0,16552
+floe/config/profile.py,sha256=d_Lw8SGdO1h4of7Cl0fraAfB3W5sMYZvECguxgsdIGA,17253
 floe/config/project.py,sha256=hUsnjbNAEiiy_iVJYA1IDquZ4OY4OF7JpNaEVJ_ocCg,14221
 floe/config/storage.py,sha256=BK3P65ukFqNgmbjCcuYmXR04OScf1yD8f19vGJq-aDI,6234
 floe/config/types.py,sha256=4eveoOlyGmNxyZpI1PSjli9lnXLPc1-jYj4fL8qHN_8,5038
 floe/config/validate.py,sha256=7tyofwoTBL-BBCNxnmkXeSOybwmMNngeEcj8zviRyLs,36176
 floe/config/vars.py,sha256=7Bq0IU0KrFeRlyA8W9-F0gDSDKfXCfFpBrX08H8Flvw,4577
 floe/config/yaml.py,sha256=HeFGhKFxYkiqQOwljYOgUD5naJyxL_ZIX6hUIS36tzU,10071
-floe/iceberg/catalog.py,sha256=R0IBTaLe80k49SUuVMoAw0j7Ira2EDgjjk3XHSISIGg,16573
-floe/iceberg/compact.py,sha256=5O372vAIsvgRc2PGihryAsrQ0otn0F30AQu-cetRCP0,7123
+floe/iceberg/catalog.py,sha256=qEhMvH4NFBd4GxqBEg4jqgTNxtqWkL_RRzIlgXr2GmA,16722
+floe/iceberg/compact.py,sha256=te8QLzg-iU_Y7Z9z0BSqMf_vjuV6CzQGV_3OeutSaZg,7237
 floe/iceberg/merge.py,sha256=VE1Odf2B34Gxj7ZcuhbrfbgTGM9AdiDz6_2BbcQ-kSQ,12448
-floe/iceberg/sink.py,sha256=WyT6yrp5JQyfaqWES96NWh3V-EzgTD4Y-IjB84mZ9Sg,11842
+floe/iceberg/sink.py,sha256=vdkgLjqvWH7opOrfc-147xCMQhLnxRjUzLjHLchHlAM,12043
 floe/iceberg/unique.py,sha256=OmiPQthhB5jGTaXsCu9ZztNFEFUf6eT0MLU2Gnh-C2A,1095
-floe/iceberg/write.py,sha256=SRX5tb17zUeWwpg6uXKkVQ_muNJ2GqlQd0n-kmjWPVc,11141
+floe/iceberg/write.py,sha256=YNE8IwLV9jK3vksO0XCj36HKHqW6ZEHCu4173Xpi6hg,12854
 floe/lineage/events.py,sha256=4txd8bGUwKBZd-8Aw_ebTurhlxAYk8SVMgH8vSjC3bs,6682
 floe/lineage/observer.py,sha256=IFHjzsT65L-NkcQ8Noa_8CDVpLu4iGadRB2-K0Lj8t8,10216
 floe/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -45,24 +45,24 @@ floe/read/xlsx.py,sha256=dengSY8HuNh9ahyR_LR8beqXTZQubqQ1Jx8qEak5jrY,3009
 floe/read/xml.py,sha256=7m6DtoU8iExuLROavyojysqOUE_Bd_N0uwFaLAzjI_Y,8004
 floe/report/entity.py,sha256=FRwLtXrlJXDNXrOKscitm-NCNJWQXd0vr_JIf_cUp20,9867
 floe/report/summary.py,sha256=tb547VRkM5WGWHQ-4vZB2DUW4VTWgv9FUoym-NpuWE4,4452
-floe/run/__init__.py,sha256=leKCs7YOoPADLFzKzU8m-mdfmStbGz_VH9kAg12kzyo,8902
-floe/run/entity.py,sha256=e-IfES1ajZtIBrX9UnWDOpFHSoxjfYRtCBp63QavCFY,49653
+floe/run/__init__.py,sha256=yCLCoLbtA_Iuy7wabkFSImeuU8xrLp8Q0ZgYmw8WDRg,9229
+floe/run/entity.py,sha256=MlNTdUM8dcZxubB_0FkjfSGbSz2TMcaBBm9Q9V4P7Rk,51072
 floe/run/events.py,sha256=OE7zYkt0Dz2xrvgNavT1nHfZJ-97IIJ8LpHcameUJZU,2859
 floe/run/parallel.py,sha256=lKBW9NOpOu6KFOcEo-RQ56ukEOKHXPPoLI3o6gmlY_g,3435
 floe/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-floe/storage/archive.py,sha256=1gPImm9PDSJFCpWjg37ax4ubb0wwc6FZm38quQXhORg,1101
+floe/storage/archive.py,sha256=JQKizd5PcWWhOFr4nxnkp2OT8Tl1h-Bu28EXWWj-gMc,1126
 floe/storage/inputs.py,sha256=nmgEo2ub3hPUghLUXHU1wnKzpL4aYZbHb6S0XPYvMew,10410
-floe/storage/local.py,sha256=UiGFYEY33j45CQspWzRyJBW0HC2Nk9_zG--wL-aUuJI,6388
-floe/storage/partitioning.py,sha256=MG8RJDrwgZhucTswlXC2RwCmzrYIoOE6SH_aBM11KW8,6178
+floe/storage/local.py,sha256=uz7LrtjItFKVtnzna3b06hv5Cv3ov1l79Y2NmxUJ-pg,6617
+floe/storage/partitioning.py,sha256=kheJLDtRyiizgsCiv_8DChAv9Rc5dOxF0q8_yVZEjFE,7057
 floe/storage/paths.py,sha256=Z0fry3Gp0Tu2DpUeKqNXdqYP6n_KcKzK2llGx0TGHB8,2343
 floe/storage/prefetch.py,sha256=b-_be13xkJIw3LVndOb6SouTd1Pf8uZwG6Wy0Nk1pr8,1801
 floe/storage/resolve.py,sha256=0hGSdU_p10j9vzET25Kcnd-QF6H7uN-hzpo_SC-DV_c,3154
-floe/storage/s3.py,sha256=XcG1qJTEY3KLSMLT950mUdrAxuCA8-Gx3fpNSzyV_Xo,7508
-floe/storage/types.py,sha256=FvAEXWMdVdopRK276JyETRYGOva2-uR8pZvsWj6Xvy0,993
+floe/storage/s3.py,sha256=jG9GWCSSl1NtFus8C_vCe1wtHuKKZs1Jwz0i3jFFT1M,9074
+floe/storage/types.py,sha256=sb-XQr0f7f4JgmfuAFRjHaaGmn63T7v09AvvScXGOTg,1096
 floe/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/write/outputs.py,sha256=cXxGUIJnZ38XPaqEI6a83GFUsapBRdY52XQm3GlUPcI,6937
-floe_python-0.8.8.dist-info/METADATA,sha256=nkx-wlv0W9muP3wviq1E6TBG-fS7wPa1Szy7qjGig_E,442
-floe_python-0.8.8.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-floe_python-0.8.8.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
-floe_python-0.8.8.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
-floe_python-0.8.8.dist-info/RECORD,,
+floe_python-0.9.0.dist-info/METADATA,sha256=XrdM3vzJi9FKxGWCeybRfWpBKMYvvnZAMt-zD2AZTKA,442
+floe_python-0.9.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+floe_python-0.9.0.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
+floe_python-0.9.0.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
+floe_python-0.9.0.dist-info/RECORD,,