floe-python 0.8.6__py3-none-any.whl → 0.8.7__py3-none-any.whl

floe/__init__.py

@@ -46,6 +46,7 @@ from floe.run.events import (
 )
 from floe.state import dump_state, read_state, reset
 from floe.storage.paths import normalize_local_path
+from floe.storage.resolve import clear_storage_clients
 from floe.storage.types import StorageClient
 
 _observer: Callable[[RunEvent], None] | None = None
@@ -281,6 +282,7 @@ def extract_config_env_vars(config_path: str) -> dict[str, str]:
 
 
 def _state(config_path: str, entity_name: str) -> tuple[str, str, str, StorageClient]:
+    clear_storage_clients()
     path, root = _load(config_path)
     [entity] = _run.select_entities(root, [entity_name])
     uri, client = state_uri(entity, normalize_local_path(path.parent), root.storages)

floe/checks/normalize.py

@@ -115,4 +115,6 @@ def normalize_columns(df: pl.DataFrame, strategy: str | None) -> pl.DataFrame:
                 f"normalized input column collision: {seen[new]} and {old} -> {new}"
             )
         seen[new] = old
-    return df.rename(names)
+    renamed = df.clone()
+    renamed.columns = list(names.values())  # a rename is a query plan, this is not
+    return renamed

floe/checks/rows.py

@@ -109,6 +109,10 @@ def split_rows(
     sources: Mapping[str, str] | None = None,
 ) -> tuple[pl.DataFrame, pl.DataFrame]:
     """`(accepted, rejected)`; rejected adds `__floe_row_index` (UInt64, row position in `df`) and `__floe_errors`."""
+    # a null mask passes the row, as in `error_column`
+    if not any(c.mask.any() for c in checks):
+        none = [pl.Series(ROW_INDEX, [], pl.UInt64), pl.Series(ERRORS, [], pl.String)]
+        return df, df.clear().with_columns(none)  # replaces a clashing column, as below
     errors = error_column(df.height, checks, formatter, sources)
     rejected = df.with_columns(
         pl.int_range(pl.len(), dtype=pl.UInt64).alias(ROW_INDEX), errors

floe/iceberg/catalog.py

@@ -4,6 +4,7 @@ import os
 import re
 import shutil
 import tempfile
+import threading
 import weakref
 from collections.abc import Generator, Sequence
 from contextlib import contextmanager
@@ -83,8 +84,34 @@ def _credential_props(catalog: str, credential: str) -> dict[str, str]:
     return {"credential": credential.removeprefix("client_credentials:")}
 
 
+_CATALOGS: dict[tuple[CatalogDefinition, StorageDefinition | None], Catalog] = {}
+_CATALOGS_LOCK = threading.Lock()
+
+
+def clear_catalogs() -> None:
+    """Called when a run starts: a catalog is reused within a run only, never across runs (#1101)."""
+    with _CATALOGS_LOCK:
+        _CATALOGS.clear()
+
+
 def open_catalog(
     definition: CatalogDefinition, storage: StorageDefinition | None = None
+) -> Catalog:
+    """One catalog per definition and storage until `clear_catalogs`, so the REST init (config, OAuth token) happens once.
+
+    Safe to call from several threads (one init per key). PyIceberg 0.12 refetches an expired OAuth token
+    itself: its REST calls retry once on a 401 after a new client-credentials request; a static `token:` has
+    nothing to refetch, as with a freshly opened catalog. A failed init is not kept.
+    """
+    key = (definition, storage)
+    with _CATALOGS_LOCK:
+        if key not in _CATALOGS:
+            _CATALOGS[key] = _open_catalog(definition, storage)
+        return _CATALOGS[key]
+
+
+def _open_catalog(
+    definition: CatalogDefinition, storage: StorageDefinition | None
 ) -> Catalog:
     """Credentials for S3 and Glue come from the environment; the catalog never vends them (as in Rust)."""
     props = {

floe/read/csv.py

@@ -79,15 +79,16 @@ def _scan_args(options: SourceOptions) -> _Args:
 
 def _columns(path: Path, args: _Args, declared: Sequence[str]) -> list[str]:
     try:
-        probe = args.scan(path, n_rows=0 if args.has_header else 1).collect()
+        probe = args.scan(path, n_rows=0 if args.has_header else 1)
+        if args.has_header:  # the schema is the header: no plan to run
+            return probe.collect_schema().names()
+        width = probe.collect().width
     except (pl.exceptions.PolarsError, OSError) as err:
         # Rust's polars prints the error kind: `no data: empty CSV`
         kind = "no data: " if isinstance(err, pl.exceptions.NoDataError) else ""
         raise FloeIoError(f"csv header read failed: {kind}{err}") from err
-    if args.has_header:
-        return probe.columns
-    names = list(declared[: probe.width])
-    return names + [f"extra_column_{i + 1}" for i in range(len(declared), probe.width)]
+    names = list(declared[:width])
+    return names + [f"extra_column_{i + 1}" for i in range(len(declared), width)]
 
 
 def _field_counts(path: Path, args: _Args, skip: int) -> Iterator[int]:

floe/run/__init__.py

@@ -11,6 +11,7 @@ from floe.checks.rows import Formatter
 from floe.config.project import RootConfig
 from floe.config.types import EntityConfig
 from floe.errors import FloeConfigError, FloeError
+from floe.iceberg.catalog import clear_catalogs
 from floe.report.summary import (
     build_summary,
     id_path,
@@ -24,7 +25,7 @@ from floe.run.entity import EntityRun, list_source, resolve_source, run_entity
 from floe.run.events import Emitter, Observer
 from floe.storage.inputs import ResolvedInputs
 from floe.storage.paths import normalize_local_path
-from floe.storage.resolve import resolve_path, storage_client
+from floe.storage.resolve import clear_storage_clients, resolve_path, storage_client
 
 
 @dataclass(frozen=True)
@@ -104,7 +105,9 @@ def run(
     config_uri: str | None = None,
 ) -> RunOutcome:
     """Run the selected entities in config order, writing each entity's report as it finishes. A failing entity raises: no summary, as Rust."""
+    clear_storage_clients()
     started, timer = _rfc3339(), time.monotonic()
+    clear_catalogs()
     selected = select_entities(root, entities)
     if full_refresh:
         selected = [_refresh(e) for e in selected]

floe/run/entity.py

@@ -5,7 +5,7 @@ import tempfile
 import threading
 import time
 from collections.abc import Callable, Generator, Iterable, Iterator
-from contextlib import suppress
+from contextlib import closing, suppress
 from dataclasses import dataclass, field, replace
 from datetime import datetime
 from functools import partial
@@ -85,7 +85,6 @@ from floe.storage.inputs import (
     InputFile,
     NoInputsError,
     ResolvedInputs,
-    localize_input,
     resolve_inputs,
     suffixes_for_format,
 )
@@ -99,6 +98,7 @@ from floe.storage.partitioning import (
     window,
 )
 from floe.storage.paths import normalize_local_path
+from floe.storage.prefetch import localized_inputs
 from floe.storage.resolve import resolve_path, storage_client
 from floe.storage.s3 import S3Client
 from floe.storage.types import StorageClient
@@ -335,27 +335,31 @@ def _typed(
 ) -> tuple[pl.DataFrame, list[RowCheck]]:
     entity, src = plan.entity, plan.entity.source
     # D28 (#913): the derived columns join after the file's own columns are normalised and matched
-    df = apply_mismatch(normalize_columns(raw, plan.strategy), mismatch).with_columns(
-        derived
-    )
-    df = df.select(
-        pl.col(n) if n in df.columns else pl.lit(None, pl.String).alias(n)
-        for n in plan.names
-    )
+    df = apply_mismatch(normalize_columns(raw, plan.strategy), mismatch)
+    if derived:
+        df = df.with_columns(derived)
+    if df.columns != plan.names:
+        df = df.select(
+            pl.col(n) if n in df.columns else pl.lit(None, pl.String).alias(n)
+            for n in plan.names
+        )
     columns = entity.schema_.columns
+    tag = "_e_cast_"  # the mask columns' prefix: no declared column may start with it
+    while any(n.startswith(tag) for n in plan.names):
+        tag = f"_{tag}"
     try:
         casts = [
             cast_column(n, c.column_type, src.cast_mode, df.schema[n])
             for n, c in zip(plan.names, columns, strict=True)
         ]
-        typed = df.with_columns(t for t, _ in casts)
-        masks = df.with_columns(
-            m for _, m in casts
-        )  # unlike select, broadcasts constant masks
+        # one plan for both; unlike select, with_columns broadcasts the constant masks
+        masks = [m.alias(f"{tag}{n}") for n, (_, m) in zip(plan.names, casts, strict=True)]  # fmt: skip
+        both = df.with_columns([t for t, _ in casts] + masks)
+        typed = both[plan.names]
     except pl.exceptions.InvalidOperationError as err:
         raise FloeRunError(f"entity.name={entity.name} cast failed: {err}") from err
     cast_checks = [
-        RowCheck("cast_error", n, CAST_ERROR_MESSAGE, masks[f"_e_cast_{n}"])
+        RowCheck("cast_error", n, CAST_ERROR_MESSAGE, both[f"{tag}{n}"])
         for n, c in zip(plan.names, columns, strict=True)
         if c.column_type != "string"
     ]  # fmt: skip
@@ -940,14 +944,18 @@ def _run_entity(
     def accepted_frames() -> Generator[pl.DataFrame]:
         """Files are read as the sink pulls (#1056); `files` and `complete` are final once it is exhausted."""
         try:
-            # Rust `localize_input`: a remote input is read from a local copy
-            with tempfile.TemporaryDirectory(prefix="floe-") as temp_dir:
+            queue = pending[::-1] if latest_first else pending
+            # Rust `localize_input`: a remote input is read from a local copy; #1102: downloaded ahead, and closed before the temp dir goes
+            with (
+                tempfile.TemporaryDirectory(prefix="floe-") as temp_dir,
+                closing(localized_inputs(queue, downloader, Path(temp_dir))) as inputs,
+            ):
                 if pending and sink.write_mode == "append" and plan.tracker.keys and (target is None or plan.table is not None):  # fmt: skip
                     plan.history = _key_cache(plan, accepted_client, base, Path(temp_dir))  # fmt: skip
-                for f in pending[::-1] if latest_first else pending:
+                for f in queue:
                     emit("file_started", entity=entity.name, input=f.source_uri)
                     timer = time.monotonic()
-                    local = localize_input(f, downloader, Path(temp_dir))
+                    local = next(inputs)
                     report, kept = _run_file(plan, f, local.local_path, Path(temp_dir))
                     if local.is_ephemeral:
                         local.local_path.unlink()

floe/storage/prefetch.py

@@ -0,0 +1,46 @@
+"""Download the inputs of one entity a few files ahead of the one being read (#1102)."""
+
+from collections import deque
+from collections.abc import Generator, Iterable
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+
+from floe.storage.inputs import InputFile, LocalInputFile, localize_input
+from floe.storage.types import Downloader
+
+# ponytail: a fixed window of 8 files and 512 MiB (unknown size counts as 0; the next file is always allowed) and one pool per entity; a config knob or a run-wide pool once a bucket needs more
+PREFETCH_WINDOW = 8
+PREFETCH_BYTES = 512 * 1024 * 1024
+
+
+def localized_inputs(
+    files: Iterable[InputFile], client: Downloader | None, temp_dir: Path
+) -> Generator[LocalInputFile]:
+    """`localize_input` for each file, in order. Close it to stop: pending downloads are cancelled and the running ones awaited."""
+    if client is None:  # local inputs are used in place; no pool
+        for f in files:
+            yield localize_input(f, None, temp_dir)
+        return
+    todo = deque(files)
+    ahead: deque[tuple[Future[LocalInputFile], int]] = deque()
+    pool = ThreadPoolExecutor(PREFETCH_WINDOW, thread_name_prefix="floe-prefetch")
+
+    def refill() -> None:
+        held = sum(size for _, size in ahead)
+        while todo and len(ahead) < PREFETCH_WINDOW:
+            size = todo[0].source_size or 0
+            if ahead and held + size > PREFETCH_BYTES:
+                break
+            held += size
+            ahead.append(
+                (pool.submit(localize_input, todo.popleft(), client, temp_dir), size)
+            )
+
+    try:
+        refill()
+        while ahead:
+            local = ahead.popleft()[0].result()
+            refill()
+            yield local
+    finally:
+        pool.shutdown(cancel_futures=True)

floe/storage/resolve.py

@@ -1,6 +1,7 @@
 """Storage reference resolution: name, else `storages.default`, else local (Rust `StorageResolver::resolve_path`)."""
 
 from pathlib import Path
+from threading import Lock
 
 from floe.config.storage import StorageDefinition, StoragesConfig
 from floe.errors import FloeConfigError, FloeRunError
@@ -48,8 +49,23 @@ def resolve_path(
     return f"s3://{definition.bucket}/{key}" if key else f"s3://{definition.bucket}", definition  # fmt: skip
 
 
+_clients: dict[StorageDefinition, S3Client] = {}
+_clients_lock = Lock()
+
+
+def clear_storage_clients() -> None:
+    """Start of a run: the clients below read credentials and `AWS_*` env when built."""
+    with _clients_lock:
+        _clients.clear()
+
+
 def storage_client(definition: StorageDefinition | None) -> LocalClient | S3Client:
-    """S3 credentials come from the environment, as in Rust."""
+    """S3 credentials come from the environment, as in Rust. One shared client per definition between `clear_storage_clients` calls."""
     if definition is None or definition.type == "local":
         return LocalClient()
-    return S3Client(definition.bucket or "", definition.region, definition.endpoint, definition.path_style_access)  # fmt: skip
+    with (
+        _clients_lock
+    ):  # boto3 clients are thread-safe; the lock only stops a duplicate build
+        if definition not in _clients:
+            _clients[definition] = S3Client(definition.bucket or "", definition.region, definition.endpoint, definition.path_style_access)  # fmt: skip
+        return _clients[definition]

{floe_python-0.8.6.dist-info → floe_python-0.8.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: floe-python
-Version: 0.8.6
+Version: 0.8.7
 Summary: Floe: a data-contract runtime for file ingestion
 License-File: LICENSE
 Requires-Python: >=3.11

{floe_python-0.8.6.dist-info → floe_python-0.8.7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-floe/__init__.py,sha256=0X9pm8uWiwoDslKZyyjsEAYd6g1mlvIWhkWgE5uhGng,13279
+floe/__init__.py,sha256=BMJil7rW2sI2OGGcEPUcz_mkZpyKe77sbdjGh7kjPAo,13362
 floe/__main__.py,sha256=Nzz6wrGXRpJiO59qlX7WEJRTlb-XcRFswmP0TA2KjyM,57
 floe/_cli.py,sha256=hwast3uMX0iUtkIpcChSIVHYkoYi4fR4jwCwQT53lXE,14101
 floe/_cli_output.py,sha256=VLxWQr-EPtIwklo9xr2CnjMD17U-7JrMUWdTtAP3jrY,8809
@@ -13,8 +13,8 @@ floe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/state.py,sha256=XFHF6zVxbdGg1bkONZDLryTD4noDF9IgaimaEI-3Nlg,11104
 floe/checks/cast.py,sha256=lM40xtKEggWI8FW3jijM3T2JFQtaXs8FhsFXo3vy5oM,6788
 floe/checks/mismatch.py,sha256=2Gz-tiWKxy-iehSjAlsGzPLtYeNIfBWvi4yQPlASVYQ,5309
-floe/checks/normalize.py,sha256=4g-Mw-UOtLnitavD3lPWUUYweGqGMUx_8dhq2AJU8oc,4331
-floe/checks/rows.py,sha256=E-YUZUvBUyOB7zat2dtbPWFM-0T1-01lYKuQU1A6wto,4156
+floe/checks/normalize.py,sha256=9fdRYFjWGXBAEw1Po71N5tsUWG8eekmSWIQktPduOig,4431
+floe/checks/rows.py,sha256=-yPieSus0-7ZewLv2vR8waHqQ9DGMbVvmJh1_hK4eDs,4433
 floe/checks/unique.py,sha256=qoZVqS_uG_mWZlV7-LdxT6_dMVQRlklEvMutO_V0qaY,12396
 floe/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/config/entity.py,sha256=CLXLXzkUbQhxenD_LwjBt0eoPreaYaxiBVEsdmnFIco,15883
@@ -27,7 +27,7 @@ floe/config/types.py,sha256=4eveoOlyGmNxyZpI1PSjli9lnXLPc1-jYj4fL8qHN_8,5038
 floe/config/validate.py,sha256=7tyofwoTBL-BBCNxnmkXeSOybwmMNngeEcj8zviRyLs,36176
 floe/config/vars.py,sha256=7Bq0IU0KrFeRlyA8W9-F0gDSDKfXCfFpBrX08H8Flvw,4577
 floe/config/yaml.py,sha256=HeFGhKFxYkiqQOwljYOgUD5naJyxL_ZIX6hUIS36tzU,10071
-floe/iceberg/catalog.py,sha256=n2NlgqRgnTe3Etdyra70Ib2aYtpR7zXWrajKl7spyYg,15340
+floe/iceberg/catalog.py,sha256=xjRCn6MFZCzleeM9DYkc8f6_In1qEQX_HQpDj1QSFWk,16380
 floe/iceberg/compact.py,sha256=5O372vAIsvgRc2PGihryAsrQ0otn0F30AQu-cetRCP0,7123
 floe/iceberg/merge.py,sha256=VE1Odf2B34Gxj7ZcuhbrfbgTGM9AdiDz6_2BbcQ-kSQ,12448
 floe/iceberg/sink.py,sha256=WyT6yrp5JQyfaqWES96NWh3V-EzgTD4Y-IjB84mZ9Sg,11842
@@ -37,7 +37,7 @@ floe/lineage/events.py,sha256=4txd8bGUwKBZd-8Aw_ebTurhlxAYk8SVMgH8vSjC3bs,6682
 floe/lineage/observer.py,sha256=dDYfWvPpFGjdqvFqQDZ2pTjj7uDfqji2-BTID9hzg90,10075
 floe/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/read/avro.py,sha256=rhGYlrWGFnqt_tyLIph57qV2TH91ULjoJ5qUlpW_6jI,1419
-floe/read/csv.py,sha256=5nXUGBa2s2Tt790UvOcrvfDym906JYjyhgunV6xPzr4,5153
+floe/read/csv.py,sha256=1lNLh6MGrInWi3TADERA--rOG1iOVcgGdo7JPtt5LdI,5237
 floe/read/json.py,sha256=dPUXNlQUmMeYId3lef8ssEQeHi0tIOoEblvMQDmNVt4,8284
 floe/read/orc.py,sha256=TWgdlLBR2NF_oVZ5zEmRKNXoTkq5QfF2BOFzKyzSjhY,1553
 floe/read/parquet.py,sha256=JwfuWaHj2TxBKJeaKignq8p0Op_pUfsOdVh7dDC3wcE,1491
@@ -45,8 +45,8 @@ floe/read/xlsx.py,sha256=dengSY8HuNh9ahyR_LR8beqXTZQubqQ1Jx8qEak5jrY,3009
 floe/read/xml.py,sha256=7m6DtoU8iExuLROavyojysqOUE_Bd_N0uwFaLAzjI_Y,8004
 floe/report/entity.py,sha256=FRwLtXrlJXDNXrOKscitm-NCNJWQXd0vr_JIf_cUp20,9867
 floe/report/summary.py,sha256=tb547VRkM5WGWHQ-4vZB2DUW4VTWgv9FUoym-NpuWE4,4452
-floe/run/__init__.py,sha256=Nm1OVIJF3p3DrkH3MCbBOzKcRFuFlMDSxhhdGyrM5N8,8189
-floe/run/entity.py,sha256=xw9RPspLM9ZrY5CmQfk5qBJXcxUvO5aaosLs5UdOiPs,48710
+floe/run/__init__.py,sha256=0QGSqgVTCcEH_Upp7A-Df376aN-pQc4G3F2N6uVebMw,8309
+floe/run/entity.py,sha256=lSQYCAP-d-1O9R44wJF8kgO3U2B60d_PM37NmxLJJfs,49269
 floe/run/events.py,sha256=OE7zYkt0Dz2xrvgNavT1nHfZJ-97IIJ8LpHcameUJZU,2859
 floe/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/storage/archive.py,sha256=1gPImm9PDSJFCpWjg37ax4ubb0wwc6FZm38quQXhORg,1101
@@ -54,13 +54,14 @@ floe/storage/inputs.py,sha256=nmgEo2ub3hPUghLUXHU1wnKzpL4aYZbHb6S0XPYvMew,10410
 floe/storage/local.py,sha256=UiGFYEY33j45CQspWzRyJBW0HC2Nk9_zG--wL-aUuJI,6388
 floe/storage/partitioning.py,sha256=MG8RJDrwgZhucTswlXC2RwCmzrYIoOE6SH_aBM11KW8,6178
 floe/storage/paths.py,sha256=Z0fry3Gp0Tu2DpUeKqNXdqYP6n_KcKzK2llGx0TGHB8,2343
-floe/storage/resolve.py,sha256=pFAuVytTfnSEcTAVlJjFTj7S_Vipjry1ormWZP3Egmc,2593
+floe/storage/prefetch.py,sha256=b-_be13xkJIw3LVndOb6SouTd1Pf8uZwG6Wy0Nk1pr8,1801
+floe/storage/resolve.py,sha256=0hGSdU_p10j9vzET25Kcnd-QF6H7uN-hzpo_SC-DV_c,3154
 floe/storage/s3.py,sha256=XcG1qJTEY3KLSMLT950mUdrAxuCA8-Gx3fpNSzyV_Xo,7508
 floe/storage/types.py,sha256=FvAEXWMdVdopRK276JyETRYGOva2-uR8pZvsWj6Xvy0,993
 floe/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/write/outputs.py,sha256=cXxGUIJnZ38XPaqEI6a83GFUsapBRdY52XQm3GlUPcI,6937
-floe_python-0.8.6.dist-info/METADATA,sha256=eUy15Jwu4CdMPA1EJDxZYVGwz3gRcJN02PyqJW-5Qjo,442
-floe_python-0.8.6.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-floe_python-0.8.6.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
-floe_python-0.8.6.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
-floe_python-0.8.6.dist-info/RECORD,,
+floe_python-0.8.7.dist-info/METADATA,sha256=xk8WIhLuQqvT4lPVlzSISa5FRfsyhIpL3TjcfVUGGoo,442
+floe_python-0.8.7.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+floe_python-0.8.7.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
+floe_python-0.8.7.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
+floe_python-0.8.7.dist-info/RECORD,,