floe-python 0.8.4__py3-none-any.whl → 0.8.6__py3-none-any.whl

floe/checks/normalize.py

@@ -94,12 +94,12 @@ def output_column_mapping(
     targets: dict[str, str] = {}
     mapping: dict[str, str] = {}
     for column, target in zip(columns, output_names(columns, strategy), strict=True):
-        source = normalize_name(column.source or column.name, strategy)
         if target in targets:
             raise FloeConfigError(
-                f"output column name collision: {targets[target]} and {source} -> {target}"
+                f"output column name collision: {targets[target]} and {column.name} -> {target}"
             )
-        targets[target] = source
+        targets[target] = column.name
+        source = normalize_name(column.source or column.name, strategy)
         if source != target:
             mapping[source] = target
     return mapping

floe/config/validate.py

@@ -6,6 +6,7 @@ from floe import __version__
 from floe.checks.cast import decimal_type
 from floe.checks.normalize import (
     normalize_name,
+    output_column_mapping,
     output_names,
     resolve_strategy,
     source_names,
@@ -416,7 +417,9 @@ def validate_schema(entity: EntityConfig, version: str) -> None:
             f"(allowed: {', '.join(NORMALIZE_STRATEGIES)})",
         )
     try:
-        source_names(schema.columns, resolve_strategy(schema))
+        resolved = resolve_strategy(schema)
+        source_names(schema.columns, resolved)
+        output_column_mapping(schema.columns, resolved)
     except FloeConfigError as err:
         raise _fail(entity, str(err)) from err
     mismatch = schema.mismatch

floe/iceberg/sink.py

@@ -177,7 +177,8 @@ def write_iceberg(
     key = [output.get(k.strip(), k.strip()) for k in primary_key]
     identifier = (target.namespace, target.table)
     with run_error("rest catalog table_exists check failed"):
-        exists = mode in ("merge_scd1", "merge_scd2") and catalog.table_exists(identifier)  # fmt: skip
+        # the configured mode: a refreshed merge writes as overwrite and keeps the key guard (#1091)
+        exists = entity.sink.write_mode in ("merge_scd1", "merge_scd2") and catalog.table_exists(identifier)  # fmt: skip
     if exists:
         with run_error("rest catalog load_table failed"):
             existing = set(catalog.load_table(identifier).schema().column_names)

floe/run/__init__.py

@@ -72,10 +72,10 @@ def _rfc3339() -> str:
 
 
 def _refresh(entity: EntityConfig) -> EntityConfig:
-    """Full refresh: an append sink overwrites; merge sinks cannot refresh."""
+    """Full refresh: an append sink overwrites; `merge_scd1` overwrites in `run_entity`, which keeps its merge checks."""
     mode = entity.sink.write_mode
-    if mode in ("merge_scd1", "merge_scd2"):
-        raise FloeConfigError(f"entity '{entity.name}': --full-refresh is not supported with write_mode '{mode}'")  # fmt: skip
+    if mode == "merge_scd2":
+        raise FloeConfigError(f"entity '{entity.name}': --full-refresh is not supported with write_mode '{mode}': its version history cannot be rebuilt from the inputs")  # fmt: skip
     if mode != "append":
         return entity
     return entity.model_copy(update={"sink": entity.sink.model_copy(update={"write_mode": "overwrite"})})  # fmt: skip

floe/run/entity.py

@@ -848,6 +848,11 @@ def _run_entity(
     entity = with_partition_columns(
         entity
     )  # D28: the derived columns are schema columns
+    merging = entity.sink.write_mode in ("merge_scd1", "merge_scd2")
+    configured = entity  # the sink still checks a merge's key against the table
+    # #1091: a full refresh rebuilds a merge_scd1 table from every input in one commit
+    if inc.full_refresh and entity.sink.write_mode == "merge_scd1":
+        entity = entity.model_copy(update={"sink": entity.sink.model_copy(update={"write_mode": "overwrite"})})  # fmt: skip
     sink = entity.sink
     if sink.accepted.format not in ("parquet", "iceberg"):
         raise FloeRunError(f"entity.name={entity.name} unsupported sink format: {sink.accepted.format}")  # fmt: skip
@@ -885,7 +890,7 @@ def _run_entity(
     if location and entity.incremental_mode in ("none", "archive"):
         plan.archive = location[0], storage_client(location[1])
         plan.root = source_root(source_uri) if entity.source.partitioning else None
-    if sink.write_mode in ("merge_scd1", "merge_scd2") and entity.schema_.primary_key:
+    if merging and entity.schema_.primary_key:
         pk = [runtime.get(k.strip(), k.strip()) for k in entity.schema_.primary_key]
         plan.merge_key = (
             pk[0] if len(pk) == 1 else f"[{','.join(pk)}]"
@@ -928,6 +933,9 @@ def _run_entity(
     skipped = len(files)
     downloader = None if isinstance(client, LocalClient) else client
     failure: list[Exception] = []
+    # #1090: a merge keeps a key's latest version, so the newest file is checked first; the first row of a file still wins.
+    # Not under abort: a duplicate aborts there anyway, and newer files committed ahead of a pending older one would let it merge stale rows later
+    latest_first = merging and entity.policy.severity != "abort"
 
     def accepted_frames() -> Generator[pl.DataFrame]:
         """Files are read as the sink pulls (#1056); `files` and `complete` are final once it is exhausted."""
@@ -936,7 +944,7 @@ def _run_entity(
             with tempfile.TemporaryDirectory(prefix="floe-") as temp_dir:
                 if pending and sink.write_mode == "append" and plan.tracker.keys and (target is None or plan.table is not None):  # fmt: skip
                     plan.history = _key_cache(plan, accepted_client, base, Path(temp_dir))  # fmt: skip
-                for f in pending:
+                for f in pending[::-1] if latest_first else pending:
                     emit("file_started", entity=entity.name, input=f.source_uri)
                     timer = time.monotonic()
                     local = localize_input(f, downloader, Path(temp_dir))
@@ -973,7 +981,7 @@ def _run_entity(
                 write = write_accepted(frames, accepted_client, base, sink.write_mode, sink.accepted.options)  # fmt: skip
             else:
                 outputs = [plan.out_map.get(n, n) for n in names]
-                result, table, merge, added = write_iceberg(entity, target, frames, outputs, sink.write_mode, emit.run_id, lambda: input_hash(entity, complete))  # fmt: skip
+                result, table, merge, added = write_iceberg(configured, target, frames, outputs, sink.write_mode, emit.run_id, lambda: input_hash(entity, complete))  # fmt: skip
                 write = AcceptedWrite(result.files_written, result.file_paths, result.total_bytes_written,
                                       result.avg_file_size_mb, result.small_files_count)  # fmt: skip
                 if added:
@@ -989,11 +997,15 @@ def _run_entity(
         raise failure[0] from failure[0].__cause__
     finally:
         stream.close()
+    if latest_first:  # the report lists the files in file order
+        files[skipped:] = files[skipped:][::-1]
     files[skipped:] = sorted(files[skipped:], key=lambda r: r["input_file"] not in plan.prechecked)  # fmt: skip
     inc.committed = files  # #915: from here on a failure promotes the written files
     # #914: archive only once the sink committed, so a failed commit leaves the inputs in the source (Rust archives before)
     by_uri = {r["input_file"]: r for r in files}
-    for f in plan.to_archive:
+    order = {f.source_uri: i for i, f in enumerate(inputs.files)}
+    # #1090: oldest first, so a partial archive leaves the newest batch pending, never only an older one
+    for f in sorted(plan.to_archive, key=lambda f: order[f.source_uri]) if latest_first else plan.to_archive:  # fmt: skip
         by_uri[f.source_uri]["output"]["archived_path"] = _archive(plan, f)
     unique = [_report_names(r, key, runtime) for r, key in zip(plan.tracker.results(), keys, strict=True)]  # fmt: skip
     report = build_report(

{floe_python-0.8.4.dist-info → floe_python-0.8.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: floe-python
-Version: 0.8.4
+Version: 0.8.6
 Summary: Floe: a data-contract runtime for file ingestion
 License-File: LICENSE
 Requires-Python: >=3.11

{floe_python-0.8.4.dist-info → floe_python-0.8.6.dist-info}/RECORD

@@ -13,7 +13,7 @@ floe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/state.py,sha256=XFHF6zVxbdGg1bkONZDLryTD4noDF9IgaimaEI-3Nlg,11104
 floe/checks/cast.py,sha256=lM40xtKEggWI8FW3jijM3T2JFQtaXs8FhsFXo3vy5oM,6788
 floe/checks/mismatch.py,sha256=2Gz-tiWKxy-iehSjAlsGzPLtYeNIfBWvi4yQPlASVYQ,5309
-floe/checks/normalize.py,sha256=8ZfpGFauFLIrq_xHY3iIe-rtuKO88YACDVO0WQJFWCU,4321
+floe/checks/normalize.py,sha256=4g-Mw-UOtLnitavD3lPWUUYweGqGMUx_8dhq2AJU8oc,4331
 floe/checks/rows.py,sha256=E-YUZUvBUyOB7zat2dtbPWFM-0T1-01lYKuQU1A6wto,4156
 floe/checks/unique.py,sha256=qoZVqS_uG_mWZlV7-LdxT6_dMVQRlklEvMutO_V0qaY,12396
 floe/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -24,13 +24,13 @@ floe/config/profile.py,sha256=jWackztS-0H3dTMXHALTZ7LsIOmwjriwoeY4vFgARr0,16046
 floe/config/project.py,sha256=zgTOhTDlbMr1chViMEZyv0drozZdumTaIoKDL0vv9sw,14141
 floe/config/storage.py,sha256=BK3P65ukFqNgmbjCcuYmXR04OScf1yD8f19vGJq-aDI,6234
 floe/config/types.py,sha256=4eveoOlyGmNxyZpI1PSjli9lnXLPc1-jYj4fL8qHN_8,5038
-floe/config/validate.py,sha256=4lU6yJ1qpI8AIr42SGy-xJR2brz_DBG6pNS_dkZw4zo,36065
+floe/config/validate.py,sha256=7tyofwoTBL-BBCNxnmkXeSOybwmMNngeEcj8zviRyLs,36176
 floe/config/vars.py,sha256=7Bq0IU0KrFeRlyA8W9-F0gDSDKfXCfFpBrX08H8Flvw,4577
 floe/config/yaml.py,sha256=HeFGhKFxYkiqQOwljYOgUD5naJyxL_ZIX6hUIS36tzU,10071
 floe/iceberg/catalog.py,sha256=n2NlgqRgnTe3Etdyra70Ib2aYtpR7zXWrajKl7spyYg,15340
 floe/iceberg/compact.py,sha256=5O372vAIsvgRc2PGihryAsrQ0otn0F30AQu-cetRCP0,7123
 floe/iceberg/merge.py,sha256=VE1Odf2B34Gxj7ZcuhbrfbgTGM9AdiDz6_2BbcQ-kSQ,12448
-floe/iceberg/sink.py,sha256=ySEAsKYpaKnqF9t7WjOW03KwpKynNIqLxOP1kQ-0Tbg,11723
+floe/iceberg/sink.py,sha256=WyT6yrp5JQyfaqWES96NWh3V-EzgTD4Y-IjB84mZ9Sg,11842
 floe/iceberg/unique.py,sha256=OmiPQthhB5jGTaXsCu9ZztNFEFUf6eT0MLU2Gnh-C2A,1095
 floe/iceberg/write.py,sha256=SRX5tb17zUeWwpg6uXKkVQ_muNJ2GqlQd0n-kmjWPVc,11141
 floe/lineage/events.py,sha256=4txd8bGUwKBZd-8Aw_ebTurhlxAYk8SVMgH8vSjC3bs,6682
@@ -45,8 +45,8 @@ floe/read/xlsx.py,sha256=dengSY8HuNh9ahyR_LR8beqXTZQubqQ1Jx8qEak5jrY,3009
 floe/read/xml.py,sha256=7m6DtoU8iExuLROavyojysqOUE_Bd_N0uwFaLAzjI_Y,8004
 floe/report/entity.py,sha256=FRwLtXrlJXDNXrOKscitm-NCNJWQXd0vr_JIf_cUp20,9867
 floe/report/summary.py,sha256=tb547VRkM5WGWHQ-4vZB2DUW4VTWgv9FUoym-NpuWE4,4452
-floe/run/__init__.py,sha256=6teCk4GZoX7grwtyhQUbqwbx_TSQebzViUJA19CmEl4,8107
-floe/run/entity.py,sha256=dML8VR-UyLUNVupkKaHeFIbR06TdPP2qcJ2rAMW4GeU,47568
+floe/run/__init__.py,sha256=Nm1OVIJF3p3DrkH3MCbBOzKcRFuFlMDSxhhdGyrM5N8,8189
+floe/run/entity.py,sha256=xw9RPspLM9ZrY5CmQfk5qBJXcxUvO5aaosLs5UdOiPs,48710
 floe/run/events.py,sha256=OE7zYkt0Dz2xrvgNavT1nHfZJ-97IIJ8LpHcameUJZU,2859
 floe/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/storage/archive.py,sha256=1gPImm9PDSJFCpWjg37ax4ubb0wwc6FZm38quQXhORg,1101
@@ -59,8 +59,8 @@ floe/storage/s3.py,sha256=XcG1qJTEY3KLSMLT950mUdrAxuCA8-Gx3fpNSzyV_Xo,7508
 floe/storage/types.py,sha256=FvAEXWMdVdopRK276JyETRYGOva2-uR8pZvsWj6Xvy0,993
 floe/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/write/outputs.py,sha256=cXxGUIJnZ38XPaqEI6a83GFUsapBRdY52XQm3GlUPcI,6937
-floe_python-0.8.4.dist-info/METADATA,sha256=kGo2pIkok-Gpky0VSm-2S5ttVTHx3tDcE21ynZE-HM4,442
-floe_python-0.8.4.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-floe_python-0.8.4.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
-floe_python-0.8.4.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
-floe_python-0.8.4.dist-info/RECORD,,
+floe_python-0.8.6.dist-info/METADATA,sha256=eUy15Jwu4CdMPA1EJDxZYVGwz3gRcJN02PyqJW-5Qjo,442
+floe_python-0.8.6.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+floe_python-0.8.6.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
+floe_python-0.8.6.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
+floe_python-0.8.6.dist-info/RECORD,,