floe-python 0.8.6__py3-none-any.whl → 0.8.8__py3-none-any.whl

floe/__init__.py

@@ -46,6 +46,7 @@ from floe.run.events import (
 )
 from floe.state import dump_state, read_state, reset
 from floe.storage.paths import normalize_local_path
+from floe.storage.resolve import clear_storage_clients
 from floe.storage.types import StorageClient
 
 _observer: Callable[[RunEvent], None] | None = None
@@ -135,8 +136,7 @@ def _profile(
         profile = parse_profile(path)
     except FloeError as e:
         raise FloeConfigError(f"failed to load profile {_debug(path)}: {e}") from e
-    return ProfileConfig(profile.name, profile.runner, profile.variables | (variables or {}),
-                         profile.catalogs, profile.storages, profile.lineage)  # fmt: skip
+    return replace(profile, variables=profile.variables | (variables or {}))
 
 
 def _load(config_path: str, profile: ProfileConfig | None = None) -> tuple[Path, _Root]:
@@ -281,6 +281,7 @@ def extract_config_env_vars(config_path: str) -> dict[str, str]:
 
 
 def _state(config_path: str, entity_name: str) -> tuple[str, str, str, StorageClient]:
+    clear_storage_clients()
     path, root = _load(config_path)
     [entity] = _run.select_entities(root, [entity_name])
     uri, client = state_uri(entity, normalize_local_path(path.parent), root.storages)

floe/_cli_output.py

@@ -1,6 +1,7 @@
 """`floe run` output: run events as log lines and the run summary (Rust: `crates/floe-cli/src/logging.rs`, `output.rs`)."""
 
 import json
+import threading
 import time
 from typing import Any
 
@@ -73,18 +74,20 @@ class Console:
     def __init__(self, log_format: str, run_id: str) -> None:
         self.log_format, self.run_id = log_format, run_id
         self.started = False
-        self.timings: list[dict[str, int]] = []  # per entity: input -> elapsed_ms
+        self.timings: dict[
+            str, dict[str, int]
+        ] = {}  # entity name -> input -> elapsed_ms
+        self._lock = threading.Lock()  # entities may run in parallel threads
 
     def __call__(self, event: dict[str, Any]) -> None:
         kind = event["event"]
-        self.started |= kind == "run_started"
-        if kind == "entity_started":
-            self.timings.append({})
-        elif kind == "file_finished" and self.timings:
-            self.timings[-1][event["input"]] = event["elapsed_ms"]
-        if self.log_format != "off":
-            fmt = event_json if self.log_format == "json" else event_text
-            print(fmt(event), flush=True)
+        with self._lock:
+            self.started |= kind == "run_started"
+            if kind == "file_finished":
+                self.timings.setdefault(event["entity"], {})[event["input"]] = event["elapsed_ms"]  # fmt: skip
+            if self.log_format != "off":
+                fmt = event_json if self.log_format == "json" else event_text
+                print(fmt(event), flush=True)
 
     def failed(self, err: FloeError, config: str | None) -> None:
         """Rust's `emit_failed_run_events`, after a `run_started` for `config` when the run never started.
@@ -154,7 +157,7 @@ def run_output(
     outcome: RunOutcome,
     entities: list[EntityConfig],
     mode: str,
-    timings: list[dict[str, int]],
+    timings: dict[str, dict[str, int]],
 ) -> str:
     """Rust's `format_run_output`; `mode` is `default`, `quiet` or `verbose`."""
     run_dir = outcome.run_dir  # a URI on remote storage: join with `/`, never `Path`
@@ -166,7 +169,17 @@ def run_output(
         results = dict.fromkeys(("files_total", "rows_total", "accepted_total", "rejected_total", "files_skipped"), 0)  # fmt: skip
         status, exit_code = "success", 0
     else:
-        blocks = [_entity(r, t, mode == "verbose") for r, t in zip(outcome.entity_reports, timings, strict=True)]  # fmt: skip
+        names = {
+            e.id: e.name for e in entities
+        }  # reports carry the id, `file_finished` events the name
+
+        def shown(report: dict[str, Any]) -> list[str]:
+            ident: str = report["entity"]["name"]
+            return _entity(
+                report, timings.get(names.get(ident, ident), {}), mode == "verbose"
+            )
+
+        blocks = [shown(r) for r in outcome.entity_reports]
         results, run = outcome.summary["results"], outcome.summary["run"]
         status, exit_code = run["status"], run["exit_code"]
     lines = ["DRY RUN MODE - No actual execution performed"] if outcome.dry_run is not None else []  # fmt: skip

floe/checks/mismatch.py

@@ -119,19 +119,34 @@ def apply_mismatch(df: pl.DataFrame, plan: MismatchOutcome) -> pl.DataFrame:
     return df
 
 
-def json_input_columns(path: Path, mode: str = "array") -> list[str] | None:
-    """Sorted top-level keys of every record, read exactly as `read_json` reads them.
+def json_probe(
+    path: Path, mode: str, limit: int
+) -> tuple[list[str] | None, list[tuple[str, Any]] | None]:
+    """`json_input_columns` plus the file's parsed records when it has at most `limit` (else `None`).
 
-    D40: the union over all records, so a key only a later record carries is still an input
-    column (Rust 0.7 probed the first object only). `None` for a file without records (D41), which
-    differs from records that carry no keys (`[]`): those are still checked for missing columns.
+    A file that fits one chunk is then parsed once: the caller reads the records it got here.
     """
     if mode not in ("array", "ndjson"):
         raise FloeIoError(
             f"unsupported source.options.json_mode={mode} (allowed: array, ndjson)"
         )
-    # ponytail: the file is parsed twice (probe, then read); fold the probe into the read if it costs
     keys: set[str] | None = None
-    for _, record in json_records(path, mode):
-        keys = (keys or set[str]()) | {str(k) for k in record}
-    return None if keys is None else sorted(keys)
+    head: list[tuple[str, Any]] | None = []
+    for located in json_records(path, mode):
+        keys = keys or set[str]()
+        keys.update(str(k) for k in located[1])
+        if head is not None and len(head) < limit:
+            head.append(located)
+        else:  # a longer file is read again, chunk by chunk, so it keeps nothing
+            head = None
+    return (None if keys is None else sorted(keys)), head
+
+
+def json_input_columns(path: Path, mode: str = "array") -> list[str] | None:
+    """Sorted top-level keys of every record, read exactly as `read_json` reads them.
+
+    D40: the union over all records, so a key only a later record carries is still an input
+    column (Rust 0.7 probed the first object only). `None` for a file without records (D41), which
+    differs from records that carry no keys (`[]`): those are still checked for missing columns.
+    """
+    return json_probe(path, mode, 0)[0]

floe/checks/normalize.py

@@ -115,4 +115,6 @@ def normalize_columns(df: pl.DataFrame, strategy: str | None) -> pl.DataFrame:
                 f"normalized input column collision: {seen[new]} and {old} -> {new}"
             )
         seen[new] = old
-    return df.rename(names)
+    renamed = df.clone()
+    renamed.columns = list(names.values())  # a rename is a query plan, this is not
+    return renamed

floe/checks/rows.py

@@ -109,6 +109,10 @@ def split_rows(
     sources: Mapping[str, str] | None = None,
 ) -> tuple[pl.DataFrame, pl.DataFrame]:
     """`(accepted, rejected)`; rejected adds `__floe_row_index` (UInt64, row position in `df`) and `__floe_errors`."""
+    # a null mask passes the row, as in `error_column`
+    if not any(c.mask.any() for c in checks):
+        none = [pl.Series(ROW_INDEX, [], pl.UInt64), pl.Series(ERRORS, [], pl.String)]
+        return df, df.clear().with_columns(none)  # replaces a clashing column, as below
     errors = error_column(df.height, checks, formatter, sources)
     rejected = df.with_columns(
         pl.int_range(pl.len(), dtype=pl.UInt64).alias(ROW_INDEX), errors

floe/config/profile.py

@@ -1,7 +1,7 @@
 """Environment profiles, ported from `profile/parse.rs` and `profile/validate.rs`."""
 
 import os
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from dataclasses import dataclass
 from pathlib import Path
 from typing import cast
@@ -395,12 +395,21 @@ def validate_profile(profile: ProfileConfig) -> None:
         check_lineage(profile.lineage, "profile.lineage")
 
 
+def entity_workers(orchestration: Mapping[str, object] | None) -> int:
+    """How many entities a run may run at once: 1 unless `strategy` is `parallel`, then `max_concurrent_entities` or `min(4, cpus)`."""
+    if not orchestration or orchestration.get("strategy") != "parallel":
+        return 1
+    limit = orchestration.get("max_concurrent_entities")
+    return limit if isinstance(limit, int) else min(4, os.cpu_count() or 1)
+
+
 def apply_profile(config: RootConfig, profile: ProfileConfig) -> RootConfig:
     """The profile's blocks replace the config's, as `apply_profile_catalogs`, `_storages` and `_lineage` do."""
     blocks = {
         "catalogs": profile.catalogs,
         "storages": profile.storages,
         "lineage": profile.lineage,
+        "entity_workers": entity_workers(profile.orchestration),
     }
     return config.model_copy(update={k: v for k, v in blocks.items() if v is not None})
 

floe/config/project.py

@@ -93,6 +93,7 @@ class RootConfig(BaseModel):
     lineage: LineageConfig | None
     entities: list[EntityConfig]
     config_files: list[Path]
+    entity_workers: int = 1  # profile/manifest orchestration; not a config key
 
 
 @dataclass

floe/iceberg/catalog.py

@@ -4,13 +4,15 @@ import os
 import re
 import shutil
 import tempfile
+import threading
 import weakref
 from collections.abc import Generator, Sequence
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from urllib.parse import urlsplit
 
 import duckdb
 from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import NamespaceAlreadyExistsError
 from pyiceberg.partitioning import PartitionField, PartitionSpec
 from pyiceberg.schema import Schema
 from pyiceberg.table import Table
@@ -83,8 +85,34 @@ def _credential_props(catalog: str, credential: str) -> dict[str, str]:
     return {"credential": credential.removeprefix("client_credentials:")}
 
 
+_CATALOGS: dict[tuple[CatalogDefinition, StorageDefinition | None], Catalog] = {}
+_CATALOGS_LOCK = threading.Lock()
+
+
+def clear_catalogs() -> None:
+    """Called when a run starts: a catalog is reused within a run only, never across runs (#1101)."""
+    with _CATALOGS_LOCK:
+        _CATALOGS.clear()
+
+
 def open_catalog(
     definition: CatalogDefinition, storage: StorageDefinition | None = None
+) -> Catalog:
+    """One catalog per definition and storage until `clear_catalogs`, so the REST init (config, OAuth token) happens once.
+
+    Safe to call from several threads (one init per key). PyIceberg 0.12 refetches an expired OAuth token
+    itself: its REST calls retry once on a 401 after a new client-credentials request; a static `token:` has
+    nothing to refetch, as with a freshly opened catalog. A failed init is not kept.
+    """
+    key = (definition, storage)
+    with _CATALOGS_LOCK:
+        if key not in _CATALOGS:
+            _CATALOGS[key] = _open_catalog(definition, storage)
+        return _CATALOGS[key]
+
+
+def _open_catalog(
+    definition: CatalogDefinition, storage: StorageDefinition | None
 ) -> Catalog:
     """Credentials for S3 and Glue come from the environment; the catalog never vends them (as in Rust)."""
     props = {
@@ -323,7 +351,10 @@ def ensure_table(
     if not namespace_exists:
         if not create_namespace:
             raise FloeRunError(f"entity.name={entity} iceberg namespace {namespace} does not exist and create_database_if_missing is false")  # fmt: skip
-        with run_error("rest catalog namespace create failed"):
+        with (
+            run_error("rest catalog namespace create failed"),
+            suppress(NamespaceAlreadyExistsError),
+        ):  # another entity or process created it first
             catalog.create_namespace(ident)
     identifier = (namespace, name)
     with run_error("rest catalog table_exists check failed"):

floe/lineage/observer.py

@@ -110,7 +110,6 @@ def _warn(message: str) -> None:
 class OpenLineageObserver:
     """A run observer (`floe.run.events.Observer`) posting OpenLineage `RunEvent`s."""
 
-    # ponytail: unsynchronised breaker counters; add a lock when entities run in parallel.
     def __init__(
         self, config: LineageConfig, entities: Sequence[EntityConfig], config_path: str
     ) -> None:
@@ -127,12 +126,14 @@ class OpenLineageObserver:
         self.run_uuid: str | None = None
         self.consecutive_failures = 0
         self.circuit_open = False
+        self._lock = threading.Lock()  # entities may run in parallel threads
 
     def __call__(self, event: dict[str, Any]) -> None:
         kind, ts = event["event"], event["ts_ms"]
         job = self.events.run_job_name
         if kind == "run_started":
-            self.consecutive_failures, self.circuit_open = 0, False
+            with self._lock:
+                self.consecutive_failures, self.circuit_open = 0, False
             self.run_uuid = str(uuid.uuid4())
             self._post("START", ts, self.run_uuid, job)
         elif kind == "entity_started":
@@ -171,19 +172,19 @@ class OpenLineageObserver:
             time.sleep(delay)
             retryable = self._attempt(data)
             if retryable is None:
-                self.consecutive_failures = 0
+                with self._lock:
+                    self.consecutive_failures = 0
                 return
             if not retryable:
                 break
         _warn(f"POST {self.url} failed — event dropped")
-        self.consecutive_failures += 1
-        failures = self.consecutive_failures
-        if (
-            failures
-            >= (3 if self.config.max_failures is None else self.config.max_failures)
-            and not self.circuit_open
-        ):
-            self.circuit_open = True
+        with self._lock:
+            self.consecutive_failures += 1
+            failures = self.consecutive_failures
+            limit = 3 if self.config.max_failures is None else self.config.max_failures
+            opening = failures >= limit and not self.circuit_open
+            self.circuit_open |= opening
+        if opening:
             _warn(f"disabled for this run after {failures} consecutive failures — check endpoint {self.url}")  # fmt: skip
 
     def _attempt(self, data: bytes) -> bool | None:
@@ -251,6 +252,7 @@ def run_with_lineage(
         return run(
             path, root, run_id=run_id, entities=entities, dry_run=dry_run,
             full_refresh=full_refresh, observer=observe, config_uri=config_uri,
+            workers=root.entity_workers,
         )  # fmt: skip
     except FloeError:
         if lineage:

floe/manifest_run.py

@@ -12,6 +12,7 @@ from typing import Any, TypeVar, cast, get_args
 
 from pydantic import BaseModel, ValidationError
 
+from floe.config.profile import entity_workers
 from floe.config.project import (
     LineageConfig,
     ProjectMetadata,
@@ -265,6 +266,9 @@ def load_manifest(location: str | Path) -> tuple[RootConfig, list[EntityConfig],
         spec = m["spec_version"]
         entities = [_entity(e, _named_local(m)) for e in m["entities"]]
         report_base = m["report_base_uri"]
+        workers = entity_workers(
+            cast(Json, m.get("execution") or {}).get("orchestration")
+        )
         meta = m.get("metadata")  # D20: dumped without its None fields
         metadata = None if meta is None else ProjectMetadata.model_validate(_omitted_as_none(ProjectMetadata, meta))  # fmt: skip
     except (ValueError, KeyError, TypeError, AttributeError) as e:
@@ -289,6 +293,7 @@ def load_manifest(location: str | Path) -> tuple[RootConfig, list[EntityConfig],
         lineage=_section(LineageConfig, m, "lineage"),
         entities=entities,
         config_files=[],
+        entity_workers=workers,
     )
     observed = [_lineage_entity(e, me) for e, me in zip(entities, m["entities"])]
     return root, observed, _replay_base(location, m)

floe/read/csv.py

@@ -79,15 +79,16 @@ def _scan_args(options: SourceOptions) -> _Args:
 
 def _columns(path: Path, args: _Args, declared: Sequence[str]) -> list[str]:
     try:
-        probe = args.scan(path, n_rows=0 if args.has_header else 1).collect()
+        probe = args.scan(path, n_rows=0 if args.has_header else 1)
+        if args.has_header:  # the schema is the header: no plan to run
+            return probe.collect_schema().names()
+        width = probe.collect().width
     except (pl.exceptions.PolarsError, OSError) as err:
         # Rust's polars prints the error kind: `no data: empty CSV`
         kind = "no data: " if isinstance(err, pl.exceptions.NoDataError) else ""
         raise FloeIoError(f"csv header read failed: {kind}{err}") from err
-    if args.has_header:
-        return probe.columns
-    names = list(declared[: probe.width])
-    return names + [f"extra_column_{i + 1}" for i in range(len(declared), probe.width)]
+    names = list(declared[:width])
+    return names + [f"extra_column_{i + 1}" for i in range(len(declared), width)]
 
 
 def _field_counts(path: Path, args: _Args, skip: int) -> Iterator[int]:

floe/read/json.py

@@ -15,6 +15,11 @@ from floe.config.types import ColumnConfig, SourceOptions
 from floe.errors import FloeIoError
 
 Selector = list[str | int]  # a field name or an array index
+JSON_CHUNK_ROWS = 1_000_000
+# ponytail: a file of at most this many bytes is parsed once, its dicts held until read (about 16x its size
+# in memory, so 8 MiB costs about 130 MiB); larger files parse twice (key pass, then chunks).
+# Upgrade path: build the first chunk's rows while probing.
+JSON_SINGLE_PASS_BYTES = 8 * 1024 * 1024
 
 
 @dataclass(frozen=True)
@@ -188,13 +193,15 @@ def read_json(
     options: SourceOptions,
     columns: Sequence[ColumnConfig],
     cast_mode: str = "strict",
-    chunk_rows: int = 1_000_000,
+    chunk_rows: int = JSON_CHUNK_ROWS,
+    records: Sequence[tuple[str, Any]] | None = None,
 ) -> Iterator[JsonChunk]:
     """Yield the rows in chunks of at most `chunk_rows`, columns named by selector; always at least one.
 
     `options.json_mode` is `array` (default) or `ndjson`. A selector that finds nothing is null.
     Objects and arrays are an error unless `cast_mode` is `coerce`, which keeps their compact JSON.
     Scalars are stringified with Python's `str`; Polars would render `1.0` as `1`, so it is not used.
+    `records` are the file's already parsed `json_records`, read instead of parsing the file again.
     """
     mode = options.json_mode or "array"
     if mode not in ("array", "ndjson"):
@@ -221,7 +228,7 @@ def read_json(
     exact = [decimal_type(c.column_type) is not None for c in columns]
     rows: list[list[str | None]] = []
     done = 0
-    for where, value in json_records(path, mode):
+    for where, value in json_records(path, mode) if records is None else records:
         rows.append(
             [
                 _cell(value, p, n, cast_mode, where, e)

floe/run/__init__.py

@@ -1,4 +1,4 @@
-"""A run: entity selection, run context, sequential or parallel entities, dry run, full refresh, events (Rust: `run/mod.rs`, `run/context.rs`)."""
+"""A run: entity selection, run context, entities in config order or in parallel threads (`workers` > 1), dry run, full refresh, events (Rust: `run/mod.rs`, `run/context.rs`)."""
 
 import time
 from collections.abc import Sequence
@@ -11,6 +11,7 @@ from floe.checks.rows import Formatter
 from floe.config.project import RootConfig
 from floe.config.types import EntityConfig
 from floe.errors import FloeConfigError, FloeError
+from floe.iceberg.catalog import clear_catalogs
 from floe.report.summary import (
     build_summary,
     id_path,
@@ -22,9 +23,10 @@ from floe.report.summary import (
 )
 from floe.run.entity import EntityRun, list_source, resolve_source, run_entity
 from floe.run.events import Emitter, Observer
+from floe.run.parallel import exclusive_keys, run_parallel
 from floe.storage.inputs import ResolvedInputs
 from floe.storage.paths import normalize_local_path
-from floe.storage.resolve import resolve_path, storage_client
+from floe.storage.resolve import clear_storage_clients, resolve_path, storage_client
 
 
 @dataclass(frozen=True)
@@ -102,9 +104,15 @@ def run(
     full_refresh: bool = False,
     observer: Observer | None = None,
     config_uri: str | None = None,
+    workers: int = 1,
 ) -> RunOutcome:
-    """Run the selected entities in config order, writing each entity's report as it finishes. A failing entity raises: no summary, as Rust."""
+    """Run the selected entities, writing each entity's report as it finishes. A failing entity raises: no summary, as Rust.
+
+    One worker runs them in config order. More run them in parallel threads (observers are called from those threads), never two on the same target; reports and summary stay in config order.
+    """
+    clear_storage_clients()
     started, timer = _rfc3339(), time.monotonic()
+    clear_catalogs()
     selected = select_entities(root, entities)
     if full_refresh:
         selected = [_refresh(e) for e in selected]
@@ -156,14 +164,21 @@ def run(
         archive = full_refresh and e.sink.archive and e.incremental_mode in ("none", "archive")  # fmt: skip
         if not archive:
             listed[e.id] = inputs(e)
-    reports: list[tuple[str, dict[str, Any]]] = []
     client = storage_client(report_storage)
-    # D36: a policy abort fails only its entity; a FloeError still stops the run
-    for e in selected:
+
+    def finish(e: EntityConfig) -> dict[str, Any]:
         report = one(e).report
-        reports.append((id_path(e), report))
         write_entity_report(run_path, id_path(e), report, client)
         _finished(emit, e.id, report)
+        return report
+
+    # D36: a policy abort fails only its entity; a FloeError still stops the run
+    if workers > 1 and len(selected) > 1:
+        keys = [exclusive_keys(e, config_dir, root.storages, root.catalogs) for e in selected]  # fmt: skip
+        done = run_parallel(selected, keys, workers, finish)
+    else:
+        done = [finish(e) for e in selected]
+    reports = [(id_path(e), r) for e, r in zip(selected, done, strict=True)]
     summary = build_summary(
         root.version, reports,
         run_id=run_id, started_at=started, finished_at=_rfc3339(),

floe/run/entity.py

@@ -5,7 +5,7 @@ import tempfile
 import threading
 import time
 from collections.abc import Callable, Generator, Iterable, Iterator
-from contextlib import suppress
+from contextlib import closing, suppress
 from dataclasses import dataclass, field, replace
 from datetime import datetime
 from functools import partial
@@ -21,7 +21,7 @@ from floe.checks.mismatch import (
     MismatchOutcome,
     apply_mismatch,
     declared_names,
-    json_input_columns,
+    json_probe,
     plan_mismatch,
 )
 from floe.checks.normalize import (
@@ -59,7 +59,7 @@ from floe.iceberg.write import find_committed
 from floe.pii import apply_pii_masking
 from floe.read.avro import read_avro
 from floe.read.csv import RAGGED_MESSAGE, read_csv
-from floe.read.json import read_json
+from floe.read.json import JSON_CHUNK_ROWS, JSON_SINGLE_PASS_BYTES, read_json
 from floe.read.orc import read_orc
 from floe.read.parquet import read_parquet
 from floe.read.xlsx import read_xlsx
@@ -85,7 +85,6 @@ from floe.storage.inputs import (
     InputFile,
     NoInputsError,
     ResolvedInputs,
-    localize_input,
     resolve_inputs,
     suffixes_for_format,
 )
@@ -99,6 +98,7 @@ from floe.storage.partitioning import (
     window,
 )
 from floe.storage.paths import normalize_local_path
+from floe.storage.prefetch import localized_inputs
 from floe.storage.resolve import resolve_path, storage_client
 from floe.storage.s3 import S3Client
 from floe.storage.types import StorageClient
@@ -118,7 +118,7 @@ class EntityRun:
     aborted: bool
 
 
-def _chunks(entity: EntityConfig, path: Path, names: list[str]) -> Chunks:
+def _chunks(entity: EntityConfig, path: Path, names: list[str], head: list[tuple[str, Any]] | None = None) -> Chunks:  # fmt: skip
     src = entity.source
     match src.format:
         case "csv" | "tsv":
@@ -139,7 +139,7 @@ def _chunks(entity: EntityConfig, path: Path, names: list[str]) -> Chunks:
             for x in read_xml(path, src.options, entity.schema_.columns[: len(names)]):
                 yield x.offset, x.df, None
         case "json":  # #998
-            for j in read_json(path, src.options, entity.schema_.columns[: len(names)], src.cast_mode):  # fmt: skip
+            for j in read_json(path, src.options, entity.schema_.columns[: len(names)], src.cast_mode, JSON_CHUNK_ROWS, head):  # fmt: skip
                 yield j.offset, j.df, None
         case other:
             raise FloeRunError(
@@ -335,27 +335,31 @@ def _typed(
 ) -> tuple[pl.DataFrame, list[RowCheck]]:
     entity, src = plan.entity, plan.entity.source
     # D28 (#913): the derived columns join after the file's own columns are normalised and matched
-    df = apply_mismatch(normalize_columns(raw, plan.strategy), mismatch).with_columns(
-        derived
-    )
-    df = df.select(
-        pl.col(n) if n in df.columns else pl.lit(None, pl.String).alias(n)
-        for n in plan.names
-    )
+    df = apply_mismatch(normalize_columns(raw, plan.strategy), mismatch)
+    if derived:
+        df = df.with_columns(derived)
+    if df.columns != plan.names:
+        df = df.select(
+            pl.col(n) if n in df.columns else pl.lit(None, pl.String).alias(n)
+            for n in plan.names
+        )
     columns = entity.schema_.columns
+    tag = "_e_cast_"  # the mask columns' prefix: no declared column may start with it
+    while any(n.startswith(tag) for n in plan.names):
+        tag = f"_{tag}"
     try:
         casts = [
             cast_column(n, c.column_type, src.cast_mode, df.schema[n])
             for n, c in zip(plan.names, columns, strict=True)
         ]
-        typed = df.with_columns(t for t, _ in casts)
-        masks = df.with_columns(
-            m for _, m in casts
-        )  # unlike select, broadcasts constant masks
+        # one plan for both; unlike select, with_columns broadcasts the constant masks
+        masks = [m.alias(f"{tag}{n}") for n, (_, m) in zip(plan.names, casts, strict=True)]  # fmt: skip
+        both = df.with_columns([t for t, _ in casts] + masks)
+        typed = both[plan.names]
     except pl.exceptions.InvalidOperationError as err:
         raise FloeRunError(f"entity.name={entity.name} cast failed: {err}") from err
     cast_checks = [
-        RowCheck("cast_error", n, CAST_ERROR_MESSAGE, masks[f"_e_cast_{n}"])
+        RowCheck("cast_error", n, CAST_ERROR_MESSAGE, both[f"{tag}{n}"])
         for n, c in zip(plan.names, columns, strict=True)
         if c.column_type != "string"
     ]  # fmt: skip
@@ -407,7 +411,9 @@ def _run_file(plan: _Plan, file: InputFile, path: Path, spill: Path) -> tuple[di
             first, probe = [], xml_columns(path, entity.source.options)
         elif entity.source.format == "json":
             # D40: every record's top-level keys; D41: only a file without records has nothing to compare
-            json_keys = json_input_columns(path, entity.source.options.json_mode or "array")  # fmt: skip
+            # a file listed as small is parsed once: the read reuses the probe's records (limit 0 keeps none)
+            json_keys, head = json_probe(path, entity.source.options.json_mode or "array", JSON_CHUNK_ROWS if file.source_size is not None and file.source_size <= JSON_SINGLE_PASS_BYTES else 0)  # fmt: skip
+            chunks = _chunks(entity, path, file_names, head)
             first, probe = [], declared if json_keys is None else json_keys
         else:  # the first chunk's columns are the input columns the plan compares
             first = [next(chunks)]
@@ -940,14 +946,18 @@ def _run_entity(
     def accepted_frames() -> Generator[pl.DataFrame]:
         """Files are read as the sink pulls (#1056); `files` and `complete` are final once it is exhausted."""
         try:
-            # Rust `localize_input`: a remote input is read from a local copy
-            with tempfile.TemporaryDirectory(prefix="floe-") as temp_dir:
+            queue = pending[::-1] if latest_first else pending
+            # Rust `localize_input`: a remote input is read from a local copy; #1102: downloaded ahead, and closed before the temp dir goes
+            with (
+                tempfile.TemporaryDirectory(prefix="floe-") as temp_dir,
+                closing(localized_inputs(queue, downloader, Path(temp_dir))) as inputs,
+            ):
                 if pending and sink.write_mode == "append" and plan.tracker.keys and (target is None or plan.table is not None):  # fmt: skip
                     plan.history = _key_cache(plan, accepted_client, base, Path(temp_dir))  # fmt: skip
-                for f in pending[::-1] if latest_first else pending:
+                for f in queue:
                     emit("file_started", entity=entity.name, input=f.source_uri)
                     timer = time.monotonic()
-                    local = localize_input(f, downloader, Path(temp_dir))
+                    local = next(inputs)
                     report, kept = _run_file(plan, f, local.local_path, Path(temp_dir))
                     if local.is_ephemeral:
                         local.local_path.unlink()

floe/run/parallel.py

@@ -0,0 +1,71 @@
+"""Entities in parallel threads (#1105): the schedule keeps entities that share a resource apart and the failure rule of D36."""
+
+from collections.abc import Callable, Sequence
+from concurrent.futures import Future, ThreadPoolExecutor, wait
+from pathlib import Path
+from typing import Any
+
+from floe.config.storage import CatalogsConfig, StoragesConfig
+from floe.config.types import EntityConfig
+from floe.errors import FloeError
+from floe.iceberg.sink import resolve_target
+from floe.run.entity import state_uri
+from floe.storage.resolve import resolve_path
+
+MERGE = "merge"  # every merge entity shares this key: DuckDB memory, and Polaris conflicts on one table
+
+
+def _canonical(uri: str) -> str:
+    """One spelling per destination: a local path resolved through symlinks, a remote URI without a trailing `/`."""
+    if uri.startswith("local://"):
+        return f"local://{Path(uri.removeprefix('local://')).resolve()}"
+    return uri.rstrip("/")
+
+
+def exclusive_keys(e: EntityConfig, config_dir: Path, storages: StoragesConfig | None, catalogs: CatalogsConfig | None) -> frozenset[str]:  # fmt: skip
+    """What `e` writes that no other running entity may touch: its accepted path, Iceberg table, state file, and the merge slot."""
+    sink = e.sink
+    try:
+        keys = {_canonical(resolve_path(storages, config_dir, sink.accepted.storage, sink.accepted.path, f"entity.name={e.name} sink.accepted.storage")[0])}  # fmt: skip
+        if sink.accepted.format == "iceberg":
+            t = resolve_target(storages, catalogs, config_dir, e)
+            keys.add(f"iceberg://{t.catalog.name}/{t.namespace}/{t.table}")
+        if e.incremental_mode == "file":
+            keys.add(_canonical(state_uri(e, config_dir, storages)[0]))
+    except FloeError:
+        return frozenset()  # `run_entity` raises the same error when it runs
+    if sink.write_mode.startswith("merge"):
+        keys.add(MERGE)
+    return frozenset(keys)
+
+
+def run_parallel(entities: Sequence[EntityConfig], keys: Sequence[frozenset[str]], workers: int, task: Callable[[EntityConfig], dict[str, Any]]) -> list[dict[str, Any]]:  # fmt: skip
+    """Run `task` over `entities` on `workers` threads, never two with a key in common; the reports in `entities` order.
+
+    After a failure no entity starts, the running ones finish (and release their claims), and the first error in `entities` order is raised.
+    """
+    reports: dict[int, dict[str, Any]] = {}
+    errors: dict[int, BaseException] = {}
+    pending = list(range(len(entities)))
+    running: dict[Future[dict[str, Any]], int] = {}
+    busy: set[str] = set()
+    with ThreadPoolExecutor(workers, thread_name_prefix="floe-entity") as pool:
+        while pending or running:
+            for i in list(pending):
+                if len(running) == workers:
+                    break
+                if not keys[i] & busy:
+                    pending.remove(i)
+                    busy |= keys[i]
+                    running[pool.submit(task, entities[i])] = i
+            for future in wait(running, return_when="FIRST_COMPLETED").done:
+                i = running.pop(future)
+                busy -= keys[i]
+                if error := future.exception():
+                    errors[i] = error
+                    pending.clear()
+                else:
+                    reports[i] = future.result()
+    if errors:
+        raise errors[min(errors)]
+    return [reports[i] for i in range(len(entities))]

floe/storage/prefetch.py

@@ -0,0 +1,46 @@
+"""Download the inputs of one entity a few files ahead of the one being read (#1102)."""
+
+from collections import deque
+from collections.abc import Generator, Iterable
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+
+from floe.storage.inputs import InputFile, LocalInputFile, localize_input
+from floe.storage.types import Downloader
+
+# ponytail: a fixed window of 8 files and 512 MiB (unknown size counts as 0; the next file is always allowed) and one pool per entity; a config knob or a run-wide pool once a bucket needs more
+PREFETCH_WINDOW = 8
+PREFETCH_BYTES = 512 * 1024 * 1024
+
+
+def localized_inputs(
+    files: Iterable[InputFile], client: Downloader | None, temp_dir: Path
+) -> Generator[LocalInputFile]:
+    """`localize_input` for each file, in order. Close it to stop: pending downloads are cancelled and the running ones awaited."""
+    if client is None:  # local inputs are used in place; no pool
+        for f in files:
+            yield localize_input(f, None, temp_dir)
+        return
+    todo = deque(files)
+    ahead: deque[tuple[Future[LocalInputFile], int]] = deque()
+    pool = ThreadPoolExecutor(PREFETCH_WINDOW, thread_name_prefix="floe-prefetch")
+
+    def refill() -> None:
+        held = sum(size for _, size in ahead)
+        while todo and len(ahead) < PREFETCH_WINDOW:
+            size = todo[0].source_size or 0
+            if ahead and held + size > PREFETCH_BYTES:
+                break
+            held += size
+            ahead.append(
+                (pool.submit(localize_input, todo.popleft(), client, temp_dir), size)
+            )
+
+    try:
+        refill()
+        while ahead:
+            local = ahead.popleft()[0].result()
+            refill()
+            yield local
+    finally:
+        pool.shutdown(cancel_futures=True)

floe/storage/resolve.py

@@ -1,6 +1,7 @@
 """Storage reference resolution: name, else `storages.default`, else local (Rust `StorageResolver::resolve_path`)."""
 
 from pathlib import Path
+from threading import Lock
 
 from floe.config.storage import StorageDefinition, StoragesConfig
 from floe.errors import FloeConfigError, FloeRunError
@@ -48,8 +49,23 @@ def resolve_path(
     return f"s3://{definition.bucket}/{key}" if key else f"s3://{definition.bucket}", definition  # fmt: skip
 
 
+_clients: dict[StorageDefinition, S3Client] = {}
+_clients_lock = Lock()
+
+
+def clear_storage_clients() -> None:
+    """Start of a run: the clients below read credentials and `AWS_*` env when built."""
+    with _clients_lock:
+        _clients.clear()
+
+
 def storage_client(definition: StorageDefinition | None) -> LocalClient | S3Client:
-    """S3 credentials come from the environment, as in Rust."""
+    """S3 credentials come from the environment, as in Rust. One shared client per definition between `clear_storage_clients` calls."""
     if definition is None or definition.type == "local":
         return LocalClient()
-    return S3Client(definition.bucket or "", definition.region, definition.endpoint, definition.path_style_access)  # fmt: skip
+    with (
+        _clients_lock
+    ):  # boto3 clients are thread-safe; the lock only stops a duplicate build
+        if definition not in _clients:
+            _clients[definition] = S3Client(definition.bucket or "", definition.region, definition.endpoint, definition.path_style_access)  # fmt: skip
+        return _clients[definition]

{floe_python-0.8.6.dist-info → floe_python-0.8.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: floe-python
-Version: 0.8.6
+Version: 0.8.8
 Summary: Floe: a data-contract runtime for file ingestion
 License-File: LICENSE
 Requires-Python: >=3.11

{floe_python-0.8.6.dist-info → floe_python-0.8.8.dist-info}/RECORD

@@ -1,66 +1,68 @@
-floe/__init__.py,sha256=0X9pm8uWiwoDslKZyyjsEAYd6g1mlvIWhkWgE5uhGng,13279
+floe/__init__.py,sha256=jEhTn9_w_xbOK0QPixbRMOBbZ2MiGAz2oW4I9n3z6_s,13254
 floe/__main__.py,sha256=Nzz6wrGXRpJiO59qlX7WEJRTlb-XcRFswmP0TA2KjyM,57
 floe/_cli.py,sha256=hwast3uMX0iUtkIpcChSIVHYkoYi4fR4jwCwQT53lXE,14101
-floe/_cli_output.py,sha256=VLxWQr-EPtIwklo9xr2CnjMD17U-7JrMUWdTtAP3jrY,8809
+floe/_cli_output.py,sha256=6DBAR3cNW2kFW_QGkwP1PpVC8AbvOjQW1LMaGnmC3qs,9232
 floe/add_entity.py,sha256=42b7l8o55dzgaR_DDgpW7YWdnU2weYMf4U_rf7qCY6M,21986
 floe/errors.py,sha256=hLSkgPQATBnjY1CAQqX4aB0oI0-WKoHW09bDXAgzC7k,186
 floe/glob.py,sha256=A0HdePLXLGl1T6heJbgdrNHr8pZfdidG7pQ1QsXWOl8,10579
 floe/maintain.py,sha256=CGSoZhO7p0u14o10FCKCO4FprzpVO548A71eq5FowF8,7343
 floe/manifest.py,sha256=sAWjv6yS4GokQDTnZwutyplLz1ICPBYl2qJmm9IORdM,20462
-floe/manifest_run.py,sha256=1XLInkpaCofMezMBs_5uIFoR5W7lbwCBFZeqRcnwUAw,12398
+floe/manifest_run.py,sha256=8rkHX2RuxHyp02NI2NLBsU6xq6m5yebR0eoYETV_7Tw,12591
 floe/pii.py,sha256=uJjU-C4c7h45E7rEL9K3Zp1LTTLh-FNKfUV5av2QMO4,4351
 floe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/state.py,sha256=XFHF6zVxbdGg1bkONZDLryTD4noDF9IgaimaEI-3Nlg,11104
 floe/checks/cast.py,sha256=lM40xtKEggWI8FW3jijM3T2JFQtaXs8FhsFXo3vy5oM,6788
-floe/checks/mismatch.py,sha256=2Gz-tiWKxy-iehSjAlsGzPLtYeNIfBWvi4yQPlASVYQ,5309
-floe/checks/normalize.py,sha256=4g-Mw-UOtLnitavD3lPWUUYweGqGMUx_8dhq2AJU8oc,4331
-floe/checks/rows.py,sha256=E-YUZUvBUyOB7zat2dtbPWFM-0T1-01lYKuQU1A6wto,4156
+floe/checks/mismatch.py,sha256=RNqRL1nihH6Y83RI-dxPXf0WTWZccXq6mnR4m6V0IIo,5827
+floe/checks/normalize.py,sha256=9fdRYFjWGXBAEw1Po71N5tsUWG8eekmSWIQktPduOig,4431
+floe/checks/rows.py,sha256=-yPieSus0-7ZewLv2vR8waHqQ9DGMbVvmJh1_hK4eDs,4433
 floe/checks/unique.py,sha256=qoZVqS_uG_mWZlV7-LdxT6_dMVQRlklEvMutO_V0qaY,12396
 floe/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/config/entity.py,sha256=CLXLXzkUbQhxenD_LwjBt0eoPreaYaxiBVEsdmnFIco,15883
 floe/config/include.py,sha256=XKYhp2QK_VUmAODAXJ35Wl4FCpQEMQm24m-WKlpI_og,5033
 floe/config/parse.py,sha256=B3gOi80-M0y9Y4OyNjM4MYmM2mFHKGVpqipX1JqaLjM,5290
-floe/config/profile.py,sha256=jWackztS-0H3dTMXHALTZ7LsIOmwjriwoeY4vFgARr0,16046
-floe/config/project.py,sha256=zgTOhTDlbMr1chViMEZyv0drozZdumTaIoKDL0vv9sw,14141
+floe/config/profile.py,sha256=4imKVdZ-Ehbj71OA3K4PrShE-gOYuicMDnTD-QHIkA0,16552
+floe/config/project.py,sha256=hUsnjbNAEiiy_iVJYA1IDquZ4OY4OF7JpNaEVJ_ocCg,14221
 floe/config/storage.py,sha256=BK3P65ukFqNgmbjCcuYmXR04OScf1yD8f19vGJq-aDI,6234
 floe/config/types.py,sha256=4eveoOlyGmNxyZpI1PSjli9lnXLPc1-jYj4fL8qHN_8,5038
 floe/config/validate.py,sha256=7tyofwoTBL-BBCNxnmkXeSOybwmMNngeEcj8zviRyLs,36176
 floe/config/vars.py,sha256=7Bq0IU0KrFeRlyA8W9-F0gDSDKfXCfFpBrX08H8Flvw,4577
 floe/config/yaml.py,sha256=HeFGhKFxYkiqQOwljYOgUD5naJyxL_ZIX6hUIS36tzU,10071
-floe/iceberg/catalog.py,sha256=n2NlgqRgnTe3Etdyra70Ib2aYtpR7zXWrajKl7spyYg,15340
+floe/iceberg/catalog.py,sha256=R0IBTaLe80k49SUuVMoAw0j7Ira2EDgjjk3XHSISIGg,16573
 floe/iceberg/compact.py,sha256=5O372vAIsvgRc2PGihryAsrQ0otn0F30AQu-cetRCP0,7123
 floe/iceberg/merge.py,sha256=VE1Odf2B34Gxj7ZcuhbrfbgTGM9AdiDz6_2BbcQ-kSQ,12448
 floe/iceberg/sink.py,sha256=WyT6yrp5JQyfaqWES96NWh3V-EzgTD4Y-IjB84mZ9Sg,11842
 floe/iceberg/unique.py,sha256=OmiPQthhB5jGTaXsCu9ZztNFEFUf6eT0MLU2Gnh-C2A,1095
 floe/iceberg/write.py,sha256=SRX5tb17zUeWwpg6uXKkVQ_muNJ2GqlQd0n-kmjWPVc,11141
 floe/lineage/events.py,sha256=4txd8bGUwKBZd-8Aw_ebTurhlxAYk8SVMgH8vSjC3bs,6682
-floe/lineage/observer.py,sha256=dDYfWvPpFGjdqvFqQDZ2pTjj7uDfqji2-BTID9hzg90,10075
+floe/lineage/observer.py,sha256=IFHjzsT65L-NkcQ8Noa_8CDVpLu4iGadRB2-K0Lj8t8,10216
 floe/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/read/avro.py,sha256=rhGYlrWGFnqt_tyLIph57qV2TH91ULjoJ5qUlpW_6jI,1419
-floe/read/csv.py,sha256=5nXUGBa2s2Tt790UvOcrvfDym906JYjyhgunV6xPzr4,5153
-floe/read/json.py,sha256=dPUXNlQUmMeYId3lef8ssEQeHi0tIOoEblvMQDmNVt4,8284
+floe/read/csv.py,sha256=1lNLh6MGrInWi3TADERA--rOG1iOVcgGdo7JPtt5LdI,5237
+floe/read/json.py,sha256=3wU24pKYIhheY7DRIjyb59NA4lQRPjRisXpsMH8cfdY,8807
 floe/read/orc.py,sha256=TWgdlLBR2NF_oVZ5zEmRKNXoTkq5QfF2BOFzKyzSjhY,1553
 floe/read/parquet.py,sha256=JwfuWaHj2TxBKJeaKignq8p0Op_pUfsOdVh7dDC3wcE,1491
 floe/read/xlsx.py,sha256=dengSY8HuNh9ahyR_LR8beqXTZQubqQ1Jx8qEak5jrY,3009
 floe/read/xml.py,sha256=7m6DtoU8iExuLROavyojysqOUE_Bd_N0uwFaLAzjI_Y,8004
 floe/report/entity.py,sha256=FRwLtXrlJXDNXrOKscitm-NCNJWQXd0vr_JIf_cUp20,9867
 floe/report/summary.py,sha256=tb547VRkM5WGWHQ-4vZB2DUW4VTWgv9FUoym-NpuWE4,4452
-floe/run/__init__.py,sha256=Nm1OVIJF3p3DrkH3MCbBOzKcRFuFlMDSxhhdGyrM5N8,8189
-floe/run/entity.py,sha256=xw9RPspLM9ZrY5CmQfk5qBJXcxUvO5aaosLs5UdOiPs,48710
+floe/run/__init__.py,sha256=leKCs7YOoPADLFzKzU8m-mdfmStbGz_VH9kAg12kzyo,8902
+floe/run/entity.py,sha256=e-IfES1ajZtIBrX9UnWDOpFHSoxjfYRtCBp63QavCFY,49653
 floe/run/events.py,sha256=OE7zYkt0Dz2xrvgNavT1nHfZJ-97IIJ8LpHcameUJZU,2859
+floe/run/parallel.py,sha256=lKBW9NOpOu6KFOcEo-RQ56ukEOKHXPPoLI3o6gmlY_g,3435
 floe/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/storage/archive.py,sha256=1gPImm9PDSJFCpWjg37ax4ubb0wwc6FZm38quQXhORg,1101
 floe/storage/inputs.py,sha256=nmgEo2ub3hPUghLUXHU1wnKzpL4aYZbHb6S0XPYvMew,10410
 floe/storage/local.py,sha256=UiGFYEY33j45CQspWzRyJBW0HC2Nk9_zG--wL-aUuJI,6388
 floe/storage/partitioning.py,sha256=MG8RJDrwgZhucTswlXC2RwCmzrYIoOE6SH_aBM11KW8,6178
 floe/storage/paths.py,sha256=Z0fry3Gp0Tu2DpUeKqNXdqYP6n_KcKzK2llGx0TGHB8,2343
-floe/storage/resolve.py,sha256=pFAuVytTfnSEcTAVlJjFTj7S_Vipjry1ormWZP3Egmc,2593
+floe/storage/prefetch.py,sha256=b-_be13xkJIw3LVndOb6SouTd1Pf8uZwG6Wy0Nk1pr8,1801
+floe/storage/resolve.py,sha256=0hGSdU_p10j9vzET25Kcnd-QF6H7uN-hzpo_SC-DV_c,3154
 floe/storage/s3.py,sha256=XcG1qJTEY3KLSMLT950mUdrAxuCA8-Gx3fpNSzyV_Xo,7508
 floe/storage/types.py,sha256=FvAEXWMdVdopRK276JyETRYGOva2-uR8pZvsWj6Xvy0,993
 floe/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/write/outputs.py,sha256=cXxGUIJnZ38XPaqEI6a83GFUsapBRdY52XQm3GlUPcI,6937
-floe_python-0.8.6.dist-info/METADATA,sha256=eUy15Jwu4CdMPA1EJDxZYVGwz3gRcJN02PyqJW-5Qjo,442
-floe_python-0.8.6.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-floe_python-0.8.6.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
-floe_python-0.8.6.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
-floe_python-0.8.6.dist-info/RECORD,,
+floe_python-0.8.8.dist-info/METADATA,sha256=nkx-wlv0W9muP3wviq1E6TBG-fS7wPa1Szy7qjGig_E,442
+floe_python-0.8.8.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+floe_python-0.8.8.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
+floe_python-0.8.8.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
+floe_python-0.8.8.dist-info/RECORD,,