floe-python 0.8.7__py3-none-any.whl → 0.8.9__py3-none-any.whl

floe/__init__.py

@@ -136,8 +136,7 @@ def _profile(
         profile = parse_profile(path)
     except FloeError as e:
         raise FloeConfigError(f"failed to load profile {_debug(path)}: {e}") from e
-    return ProfileConfig(profile.name, profile.runner, profile.variables | (variables or {}),
-                         profile.catalogs, profile.storages, profile.lineage)  # fmt: skip
+    return replace(profile, variables=profile.variables | (variables or {}))
 
 
 def _load(config_path: str, profile: ProfileConfig | None = None) -> tuple[Path, _Root]:

floe/_cli_output.py

@@ -1,6 +1,7 @@
 """`floe run` output: run events as log lines and the run summary (Rust: `crates/floe-cli/src/logging.rs`, `output.rs`)."""
 
 import json
+import threading
 import time
 from typing import Any
 
@@ -73,18 +74,20 @@ class Console:
     def __init__(self, log_format: str, run_id: str) -> None:
         self.log_format, self.run_id = log_format, run_id
         self.started = False
-        self.timings: list[dict[str, int]] = []  # per entity: input -> elapsed_ms
+        self.timings: dict[
+            str, dict[str, int]
+        ] = {}  # entity name -> input -> elapsed_ms
+        self._lock = threading.Lock()  # entities may run in parallel threads
 
     def __call__(self, event: dict[str, Any]) -> None:
         kind = event["event"]
-        self.started |= kind == "run_started"
-        if kind == "entity_started":
-            self.timings.append({})
-        elif kind == "file_finished" and self.timings:
-            self.timings[-1][event["input"]] = event["elapsed_ms"]
-        if self.log_format != "off":
-            fmt = event_json if self.log_format == "json" else event_text
-            print(fmt(event), flush=True)
+        with self._lock:
+            self.started |= kind == "run_started"
+            if kind == "file_finished":
+                self.timings.setdefault(event["entity"], {})[event["input"]] = event["elapsed_ms"]  # fmt: skip
+            if self.log_format != "off":
+                fmt = event_json if self.log_format == "json" else event_text
+                print(fmt(event), flush=True)
 
     def failed(self, err: FloeError, config: str | None) -> None:
         """Rust's `emit_failed_run_events`, after a `run_started` for `config` when the run never started.
@@ -154,7 +157,7 @@ def run_output(
     outcome: RunOutcome,
     entities: list[EntityConfig],
     mode: str,
-    timings: list[dict[str, int]],
+    timings: dict[str, dict[str, int]],
 ) -> str:
     """Rust's `format_run_output`; `mode` is `default`, `quiet` or `verbose`."""
     run_dir = outcome.run_dir  # a URI on remote storage: join with `/`, never `Path`
@@ -166,7 +169,17 @@ def run_output(
         results = dict.fromkeys(("files_total", "rows_total", "accepted_total", "rejected_total", "files_skipped"), 0)  # fmt: skip
         status, exit_code = "success", 0
     else:
-        blocks = [_entity(r, t, mode == "verbose") for r, t in zip(outcome.entity_reports, timings, strict=True)]  # fmt: skip
+        names = {
+            e.id: e.name for e in entities
+        }  # reports carry the id, `file_finished` events the name
+
+        def shown(report: dict[str, Any]) -> list[str]:
+            ident: str = report["entity"]["name"]
+            return _entity(
+                report, timings.get(names.get(ident, ident), {}), mode == "verbose"
+            )
+
+        blocks = [shown(r) for r in outcome.entity_reports]
         results, run = outcome.summary["results"], outcome.summary["run"]
         status, exit_code = run["status"], run["exit_code"]
     lines = ["DRY RUN MODE - No actual execution performed"] if outcome.dry_run is not None else []  # fmt: skip

floe/checks/mismatch.py

@@ -119,19 +119,34 @@ def apply_mismatch(df: pl.DataFrame, plan: MismatchOutcome) -> pl.DataFrame:
     return df
 
 
-def json_input_columns(path: Path, mode: str = "array") -> list[str] | None:
-    """Sorted top-level keys of every record, read exactly as `read_json` reads them.
+def json_probe(
+    path: Path, mode: str, limit: int
+) -> tuple[list[str] | None, list[tuple[str, Any]] | None]:
+    """`json_input_columns` plus the file's parsed records when it has at most `limit` (else `None`).
 
-    D40: the union over all records, so a key only a later record carries is still an input
-    column (Rust 0.7 probed the first object only). `None` for a file without records (D41), which
-    differs from records that carry no keys (`[]`): those are still checked for missing columns.
+    A file that fits one chunk is then parsed once: the caller reads the records it got here.
     """
     if mode not in ("array", "ndjson"):
         raise FloeIoError(
             f"unsupported source.options.json_mode={mode} (allowed: array, ndjson)"
         )
-    # ponytail: the file is parsed twice (probe, then read); fold the probe into the read if it costs
     keys: set[str] | None = None
-    for _, record in json_records(path, mode):
-        keys = (keys or set[str]()) | {str(k) for k in record}
-    return None if keys is None else sorted(keys)
+    head: list[tuple[str, Any]] | None = []
+    for located in json_records(path, mode):
+        keys = keys or set[str]()
+        keys.update(str(k) for k in located[1])
+        if head is not None and len(head) < limit:
+            head.append(located)
+        else:  # a longer file is read again, chunk by chunk, so it keeps nothing
+            head = None
+    return (None if keys is None else sorted(keys)), head
+
+
+def json_input_columns(path: Path, mode: str = "array") -> list[str] | None:
+    """Sorted top-level keys of every record, read exactly as `read_json` reads them.
+
+    D40: the union over all records, so a key only a later record carries is still an input
+    column (Rust 0.7 probed the first object only). `None` for a file without records (D41), which
+    differs from records that carry no keys (`[]`): those are still checked for missing columns.
+    """
+    return json_probe(path, mode, 0)[0]

floe/checks/unique.py

@@ -17,7 +17,6 @@ from floe.checks.rows import RowCheck
 
 SAMPLE_LIMIT = 5
 _SEEN = "__floe_seen"
-_DEBUG = "__floe_debug"
 _COUNT = "__floe_count"
 _BITS = "_bits"  # suffix of the bit-pattern column that keys a float column
 
@@ -56,45 +55,11 @@ def resolve_unique_keys(
     return resolved
 
 
-# `str` Debug; ponytail: non-ASCII non-printable characters (Rust `\\u{..}`) are left raw
-_STR_ESCAPES = {
-    "\\": "\\\\",
-    '"': '\\"',
-    "\n": "\\n",
-    "\r": "\\r",
-    "\t": "\\t",
-    "\0": "\\0",
-    **{chr(i): f"\\u{{{i:x}}}" for i in [*range(1, 32), 127] if chr(i) not in "\n\r\t"},
-}
-
-
 def _float_bits(s: pl.Series) -> pl.Series:
     """Rust's `to_bits()`: Polars cannot reinterpret a float, so view the Arrow buffer."""
     return pl.Series(s.cast(pl.Float64).to_arrow().view(pa.uint64()))
 
 
-def _debug(name: str, dtype: pl.DataType) -> pl.Expr:
-    """Rust's `{:?}` of the `UniqueKey` variant: the sample tie-break sorts on these strings."""
-    c = pl.col(name)
-    if dtype == pl.String:
-        escaped = c
-        for raw, esc in _STR_ESCAPES.items():
-            escaped = escaped.str.replace_all(raw, esc, literal=True)
-        return pl.format('String("{}")', escaped)
-    if dtype == pl.Boolean:
-        return pl.format("Bool({})", c.cast(pl.String))
-    if dtype.is_signed_integer():
-        return pl.format("I64({})", c.cast(pl.String))
-    if dtype.is_unsigned_integer():
-        return pl.format("U64({})", c.cast(pl.String))
-    if dtype.is_float():
-        return pl.format("F64({})", pl.col(name + _BITS).cast(pl.String))
-    if isinstance(dtype, pl.Datetime):  # chrono prints the fraction only when not zero
-        return pl.format('Other("{}")', c.dt.to_string("%Y-%m-%d %H:%M:%S%.f"))
-    # ponytail: date keys use `Other(display)`, unverified against Rust; ties may order differently
-    return pl.format('Other("{}")', c.cast(pl.String))
-
-
 def _text(value: object) -> str:
     """Rust's `to_string` of a key value."""
     if isinstance(value, bool):
@@ -221,7 +186,7 @@ class UniqueTracker:
         return RowCheck("unique", name, message, dup)
 
     def results(self) -> list[UniqueResult]:
-        """Per constraint, the top `SAMPLE_LIMIT` keys by count, ties by Rust's Debug string."""
+        """Per constraint, the top `SAMPLE_LIMIT` keys by count, ties by the key values (numbers numerically, strings by code point)."""
         return [self._result(c) for c in self._constraints]
 
     def _result(self, c: _Constraint) -> UniqueResult:
@@ -229,14 +194,9 @@ class UniqueTracker:
             return UniqueResult(c.columns, 0, [])
         parts = pl.concat(c.dup_counts)
         counts = parts.group_by(parts.columns[:-1]).agg(pl.col(_COUNT).sum())
-        debug = pl.format(
-            "CompositeKey([{}])",
-            pl.concat_str([_debug(k, counts.schema[k]) for k in c.ids], separator=", "),
-        )
-        top = (
-            counts.with_columns(debug.alias(_DEBUG))
-            .sort([_COUNT, _DEBUG], descending=[True, False])
-            .head(SAMPLE_LIMIT)
+        by = [_COUNT, *counts.columns[:-1]]
+        top = counts.sort(by, descending=[True] + [False] * (len(by) - 1)).head(
+            SAMPLE_LIMIT
         )
         samples = [  # at most SAMPLE_LIMIT rows
             UniqueSample(

floe/config/profile.py

@@ -1,7 +1,7 @@
 """Environment profiles, ported from `profile/parse.rs` and `profile/validate.rs`."""
 
 import os
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from dataclasses import dataclass
 from pathlib import Path
 from typing import cast
@@ -395,12 +395,21 @@ def validate_profile(profile: ProfileConfig) -> None:
         check_lineage(profile.lineage, "profile.lineage")
 
 
+def entity_workers(orchestration: Mapping[str, object] | None) -> int:
+    """How many entities a run may run at once: 1 unless `strategy` is `parallel`, then `max_concurrent_entities` or `min(4, cpus)`."""
+    if not orchestration or orchestration.get("strategy") != "parallel":
+        return 1
+    limit = orchestration.get("max_concurrent_entities")
+    return limit if isinstance(limit, int) else min(4, os.cpu_count() or 1)
+
+
 def apply_profile(config: RootConfig, profile: ProfileConfig) -> RootConfig:
     """The profile's blocks replace the config's, as `apply_profile_catalogs`, `_storages` and `_lineage` do."""
     blocks = {
         "catalogs": profile.catalogs,
         "storages": profile.storages,
         "lineage": profile.lineage,
+        "entity_workers": entity_workers(profile.orchestration),
     }
     return config.model_copy(update={k: v for k, v in blocks.items() if v is not None})
 

floe/config/project.py

@@ -93,6 +93,7 @@ class RootConfig(BaseModel):
     lineage: LineageConfig | None
     entities: list[EntityConfig]
     config_files: list[Path]
+    entity_workers: int = 1  # profile/manifest orchestration; not a config key
 
 
 @dataclass

floe/iceberg/catalog.py

@@ -7,11 +7,16 @@ import tempfile
 import threading
 import weakref
 from collections.abc import Generator, Sequence
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from urllib.parse import urlsplit
 
 import duckdb
 from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import (
+    NamespaceAlreadyExistsError,
+    NoSuchNamespaceError,
+    NoSuchTableError,
+)
 from pyiceberg.partitioning import PartitionField, PartitionSpec
 from pyiceberg.schema import Schema
 from pyiceberg.table import Table
@@ -344,52 +349,59 @@ def ensure_table(
 
     With `add_columns`, columns new in `schema` are added as optional (D13) and returned by name; partition changes stay rejected.
     """
-    ident = (namespace,)
-    with run_error("rest catalog namespace_exists check failed"):
-        namespace_exists = catalog.namespace_exists(ident)
-    if not namespace_exists:
-        if not create_namespace:
-            raise FloeRunError(f"entity.name={entity} iceberg namespace {namespace} does not exist and create_database_if_missing is false")  # fmt: skip
-        with run_error("rest catalog namespace create failed"):
-            catalog.create_namespace(ident)
     identifier = (namespace, name)
-    with run_error("rest catalog table_exists check failed"):
-        exists = catalog.table_exists(identifier)
-    if exists:
-        with run_error("rest catalog load_table failed"):
+    with run_error("rest catalog load_table failed"):
+        try:
             table = catalog.load_table(identifier)
-        added: list[NestedField] = []
-        if schema_evolution == "add_columns":
-            added = additive_columns(table.schema(), schema, entity)
-            # field ids of `schema` may differ from the table's once columns are added
-            check_schema = table.schema()
-        else:
-            check_schema_matches(table.schema(), schema, entity)
-            check_schema = schema
-        check_partition_spec_matches(
-            table.spec(),
+        except (
+            NoSuchTableError,
+            NoSuchNamespaceError,
+        ):  # fmt: skip -- a missing namespace raises either, per catalog
+            table = None
+    if table is None:
+        ident = (namespace,)
+        with run_error("rest catalog namespace_exists check failed"):
+            namespace_exists = catalog.namespace_exists(ident)
+        if not namespace_exists:
+            if not create_namespace:
+                raise FloeRunError(f"entity.name={entity} iceberg namespace {namespace} does not exist and create_database_if_missing is false")  # fmt: skip
+            with (
+                run_error("rest catalog namespace create failed"),
+                suppress(NamespaceAlreadyExistsError),
+            ):  # another entity or process created it first
+                catalog.create_namespace(ident)
+        spec = (
             None
             if partition_fields is None
-            else build_partition_spec(check_schema, partition_fields, entity),
-            entity,
+            else build_partition_spec(schema, partition_fields, entity)
         )
-        if added:
-            with run_error("rest catalog update_schema failed"):
-                with table.update_schema() as update:
-                    for field in added:
-                        update.add_column(field.name, field.field_type, required=False)
-                table.refresh()
-        return table, [f.name for f in added]
-    spec = (
+        with run_error("rest catalog create_table failed"):
+            created = catalog.create_table(  # pyright: ignore[reportUnknownMemberType] -- PyIceberg's `schema` is `Schema | Unknown`
+                identifier,
+                schema,
+                location=location,
+                partition_spec=spec if spec is not None else PartitionSpec(),
+            )
+        return created, []
+    added: list[NestedField] = []
+    if schema_evolution == "add_columns":
+        added = additive_columns(table.schema(), schema, entity)
+        # field ids of `schema` may differ from the table's once columns are added
+        check_schema = table.schema()
+    else:
+        check_schema_matches(table.schema(), schema, entity)
+        check_schema = schema
+    check_partition_spec_matches(
+        table.spec(),
         None
         if partition_fields is None
-        else build_partition_spec(schema, partition_fields, entity)
+        else build_partition_spec(check_schema, partition_fields, entity),
+        entity,
     )
-    with run_error("rest catalog create_table failed"):
-        created = catalog.create_table(  # pyright: ignore[reportUnknownMemberType] -- PyIceberg's `schema` is `Schema | Unknown`
-            identifier,
-            schema,
-            location=location,
-            partition_spec=spec if spec is not None else PartitionSpec(),
-        )
-    return created, []
+    if added:
+        with run_error("rest catalog update_schema failed"):
+            with table.update_schema() as update:
+                for field in added:
+                    update.add_column(field.name, field.field_type, required=False)
+            table.refresh()
+    return table, [f.name for f in added]

floe/iceberg/sink.py

@@ -15,6 +15,7 @@ from typing import Any
 import polars as pl
 import pyarrow as pa
 from pyiceberg.catalog import Catalog
+from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchTableError
 from pyiceberg.schema import Schema, assign_fresh_schema_ids
 from pyiceberg.table import Table
 
@@ -39,6 +40,7 @@ from floe.iceberg.merge import merge_scd1, merge_scd2
 from floe.iceberg.write import (
     SMALL_FILE_THRESHOLD_BYTES,
     IcebergWriteResult,
+    parquet_properties,
     write_table,
 )
 from floe.storage.inputs import InputFile
@@ -176,13 +178,15 @@ def write_iceberg(
     primary_key = entity.schema_.primary_key or []  # validated non-empty for merges
     key = [output.get(k.strip(), k.strip()) for k in primary_key]
     identifier = (target.namespace, target.table)
-    with run_error("rest catalog table_exists check failed"):
-        # the configured mode: a refreshed merge writes as overwrite and keeps the key guard (#1091)
-        exists = entity.sink.write_mode in ("merge_scd1", "merge_scd2") and catalog.table_exists(identifier)  # fmt: skip
-    if exists:
+    # the configured mode: a refreshed merge writes as overwrite and keeps the key guard (#1091)
+    if entity.sink.write_mode in ("merge_scd1", "merge_scd2"):
         with run_error("rest catalog load_table failed"):
-            existing = set(catalog.load_table(identifier).schema().column_names)
-        if added := [k for k in key if k not in existing]:  # Rust `merge/shared.rs`
+            try:
+                existing = set(catalog.load_table(identifier).schema().column_names)
+            except (NoSuchTableError, NoSuchNamespaceError):
+                existing = None
+        # Rust `merge/shared.rs`
+        if existing is not None and (added := [k for k in key if k not in existing]):
             raise FloeRunError(f"entity.name={entity.name} iceberg schema evolution failed: merge key columns cannot be added: {', '.join(added)}")  # fmt: skip
     table, added = ensure_table(
         catalog, target.namespace, target.table, schema, entity.name, target.location.removeprefix("local://"),
@@ -215,7 +219,7 @@ def write_iceberg(
         threshold = SMALL_FILE_THRESHOLD_BYTES if size is None else max(1, size // 2)
         append = "overwrite" if mode == "overwrite" else "append"
         unique = [[output.get(c, c) for c in k] for k in resolve_unique_keys(entity.schema_.unique_keys, [c.name for c in entity.schema_.columns if c.unique], entity.schema_.primary_key)]  # fmt: skip
-        result = write_table(table, reader, append, run_id, digest, threshold, unique_keys=unique)  # fmt: skip
+        result = write_table(table, reader, append, run_id, digest, threshold, unique_keys=unique, parquet=parquet_properties(options and options.compression, options and options.row_group_size))  # fmt: skip
     ids = {"table_version": result.metadata_version, "snapshot_id": result.snapshot_id}
     fields = {k: v for k, v in ids.items() if v is not None} | {
         "iceberg_catalog_name": target.catalog.name,

floe/iceberg/write.py

@@ -5,7 +5,7 @@
 
 import itertools
 import uuid
-from collections.abc import Callable, Iterator, Sequence
+from collections.abc import Callable, Iterator, Mapping, Sequence
 from dataclasses import dataclass, replace
 from datetime import datetime
 from pathlib import PurePosixPath
@@ -18,9 +18,17 @@ from pyiceberg.io.pyarrow import (
     _check_pyarrow_schema_compatible,  # pyright: ignore[reportPrivateUsage] -- same check `Transaction.append` makes
     _dataframe_to_data_files,  # pyright: ignore[reportPrivateUsage] -- feeds one snapshot producer from many chunks
 )
-from pyiceberg.manifest import DataFile, DataFileContent, ManifestEntryStatus
+from pyiceberg.manifest import (
+    DataFile,
+    DataFileContent,
+    ManifestEntryStatus,
+    ManifestFile,
+)
 from pyiceberg.table import Table, TableProperties
-from pyiceberg.table.snapshots import ancestors_of
+from pyiceberg.table.snapshots import Operation, ancestors_of
+from pyiceberg.table.update.snapshot import (
+    _OverwriteFiles,  # pyright: ignore[reportPrivateUsage] -- PyIceberg 0.12 has no public hook on the manifest list
+)
 
 from floe.errors import FloeRunError
 from floe.iceberg.catalog import run_error
@@ -55,6 +63,14 @@ class IcebergWriteResult:
     )
 
 
+class _LiveOverwrite(_OverwriteFiles):
+    """PyIceberg 0.12 keeps the all-DELETED manifest of every earlier overwrite in the manifest list, one more per
+    overwrite, so the scan and the commit read more manifests each time (#1132); Java drops them, so do we."""
+
+    def _existing_manifests(self) -> list[ManifestFile]:
+        return [m for m in super()._existing_manifests() if m.has_added_files() or m.has_existing_files()]  # fmt: skip
+
+
 def _chunks(reader: pa.RecordBatchReader, chunk_rows: int) -> Iterator[pa.Table]:
     pending: list[pa.RecordBatch] = []
     rows = 0
@@ -72,6 +88,16 @@ def _chunks(reader: pa.RecordBatchReader, chunk_rows: int) -> Iterator[pa.Table]
         yield pa.Table.from_batches(pending, reader.schema)
 
 
+def parquet_properties(
+    compression: str | None, row_group_size: int | None
+) -> dict[str, str]:
+    """The sink's `compression` and `row_group_size` as PyIceberg's write properties; unset keeps the table's."""
+    props = {TableProperties.PARQUET_COMPRESSION: compression} if compression else {}
+    if row_group_size:
+        props[TableProperties.PARQUET_ROW_GROUP_LIMIT] = str(row_group_size)
+    return props
+
+
 def _metadata_version(location: str | None) -> int | None:
     prefix = PurePosixPath(location or "").name.split("-", 1)[0]
     return int(prefix) if prefix.isdigit() else None
@@ -86,22 +112,25 @@ def write_table(
     small_file_threshold_bytes: int = SMALL_FILE_THRESHOLD_BYTES,
     chunk_rows: int = CHUNK_ROWS,
     unique_keys: Sequence[Sequence[str]] = (),
+    parquet: Mapping[str, str] | None = None,
 ) -> IcebergWriteResult:
     """Data files are written chunk by chunk (partitioned tables split each chunk), then committed once.
 
     Overwrite replaces every existing file in the same snapshot. An append of no rows commits nothing.
     On a conflict an append checks `unique_keys` (table column names) against the files added since the base
     snapshot and fails closed, committing nothing, when a rival already holds a key (#549).
+    `parquet` holds `write.parquet.*` properties for these data files only; the table's own properties are not changed.
     """
     with run_error("iceberg data write failed"):
         _check_pyarrow_schema_compatible(table.schema(), reader.schema)
+        metadata = table.metadata.model_copy(update={"properties": {**table.properties, **(parquet or {})}})  # fmt: skip
         counter = itertools.count()
         write_uuid = uuid.uuid4()
         data_files: list[DataFile] = [
             data_file
             for chunk in _chunks(reader, chunk_rows)
             for data_file in _dataframe_to_data_files(
-                table.metadata, chunk, table.io, write_uuid, counter
+                metadata, chunk, table.io, write_uuid, counter
             )
         ]
     digest = (
@@ -131,13 +160,21 @@ def write_table(
                 # PyIceberg retries a 409 itself (refresh, rebuild) and would skip the guard; in memory only, never sent
                 table.metadata = table.metadata.model_copy(update={"properties": {**table.properties, TableProperties.COMMIT_NUM_RETRIES: "0"}})  # fmt: skip
                 tx = table.transaction()
-                update = tx.update_snapshot(
-                    snapshot_properties={RUN_ID: run_id, INPUT_HASH: digest}
-                )
+                props = {RUN_ID: run_id, INPUT_HASH: digest}
                 with (
-                    update.overwrite()
+                    _LiveOverwrite(
+                        operation=Operation.OVERWRITE
+                        if table.current_snapshot()
+                        else Operation.APPEND,
+                        transaction=tx,
+                        io=table.io,
+                        branch="main",
+                        snapshot_properties=props,
+                    )
                     if mode == "overwrite"
-                    else update.fast_append() as snap
+                    else tx.update_snapshot(
+                        snapshot_properties=props
+                    ).fast_append() as snap
                 ):
                     for old in replaced:
                         snap.delete_data_file(old)

floe/lineage/observer.py

@@ -110,7 +110,6 @@ def _warn(message: str) -> None:
 class OpenLineageObserver:
     """A run observer (`floe.run.events.Observer`) posting OpenLineage `RunEvent`s."""
 
-    # ponytail: unsynchronised breaker counters; add a lock when entities run in parallel.
     def __init__(
         self, config: LineageConfig, entities: Sequence[EntityConfig], config_path: str
     ) -> None:
@@ -127,12 +126,14 @@ class OpenLineageObserver:
         self.run_uuid: str | None = None
         self.consecutive_failures = 0
         self.circuit_open = False
+        self._lock = threading.Lock()  # entities may run in parallel threads
 
     def __call__(self, event: dict[str, Any]) -> None:
         kind, ts = event["event"], event["ts_ms"]
         job = self.events.run_job_name
         if kind == "run_started":
-            self.consecutive_failures, self.circuit_open = 0, False
+            with self._lock:
+                self.consecutive_failures, self.circuit_open = 0, False
             self.run_uuid = str(uuid.uuid4())
             self._post("START", ts, self.run_uuid, job)
         elif kind == "entity_started":
@@ -171,19 +172,19 @@ class OpenLineageObserver:
             time.sleep(delay)
             retryable = self._attempt(data)
             if retryable is None:
-                self.consecutive_failures = 0
+                with self._lock:
+                    self.consecutive_failures = 0
                 return
             if not retryable:
                 break
         _warn(f"POST {self.url} failed — event dropped")
-        self.consecutive_failures += 1
-        failures = self.consecutive_failures
-        if (
-            failures
-            >= (3 if self.config.max_failures is None else self.config.max_failures)
-            and not self.circuit_open
-        ):
-            self.circuit_open = True
+        with self._lock:
+            self.consecutive_failures += 1
+            failures = self.consecutive_failures
+            limit = 3 if self.config.max_failures is None else self.config.max_failures
+            opening = failures >= limit and not self.circuit_open
+            self.circuit_open |= opening
+        if opening:
             _warn(f"disabled for this run after {failures} consecutive failures — check endpoint {self.url}")  # fmt: skip
 
     def _attempt(self, data: bytes) -> bool | None:
@@ -251,6 +252,7 @@ def run_with_lineage(
         return run(
             path, root, run_id=run_id, entities=entities, dry_run=dry_run,
             full_refresh=full_refresh, observer=observe, config_uri=config_uri,
+            workers=root.entity_workers,
         )  # fmt: skip
     except FloeError:
         if lineage:

floe/manifest_run.py

@@ -12,6 +12,7 @@ from typing import Any, TypeVar, cast, get_args
 
 from pydantic import BaseModel, ValidationError
 
+from floe.config.profile import entity_workers
 from floe.config.project import (
     LineageConfig,
     ProjectMetadata,
@@ -265,6 +266,9 @@ def load_manifest(location: str | Path) -> tuple[RootConfig, list[EntityConfig],
         spec = m["spec_version"]
         entities = [_entity(e, _named_local(m)) for e in m["entities"]]
         report_base = m["report_base_uri"]
+        workers = entity_workers(
+            cast(Json, m.get("execution") or {}).get("orchestration")
+        )
         meta = m.get("metadata")  # D20: dumped without its None fields
         metadata = None if meta is None else ProjectMetadata.model_validate(_omitted_as_none(ProjectMetadata, meta))  # fmt: skip
     except (ValueError, KeyError, TypeError, AttributeError) as e:
@@ -289,6 +293,7 @@ def load_manifest(location: str | Path) -> tuple[RootConfig, list[EntityConfig],
         lineage=_section(LineageConfig, m, "lineage"),
         entities=entities,
         config_files=[],
+        entity_workers=workers,
     )
     observed = [_lineage_entity(e, me) for e, me in zip(entities, m["entities"])]
     return root, observed, _replay_base(location, m)

floe/read/json.py

@@ -15,6 +15,11 @@ from floe.config.types import ColumnConfig, SourceOptions
 from floe.errors import FloeIoError
 
 Selector = list[str | int]  # a field name or an array index
+JSON_CHUNK_ROWS = 1_000_000
+# ponytail: a file of at most this many bytes is parsed once, its dicts held until read (about 16x its size
+# in memory, so 8 MiB costs about 130 MiB); larger files parse twice (key pass, then chunks).
+# Upgrade path: build the first chunk's rows while probing.
+JSON_SINGLE_PASS_BYTES = 8 * 1024 * 1024
 
 
 @dataclass(frozen=True)
@@ -188,13 +193,15 @@ def read_json(
     options: SourceOptions,
     columns: Sequence[ColumnConfig],
     cast_mode: str = "strict",
-    chunk_rows: int = 1_000_000,
+    chunk_rows: int = JSON_CHUNK_ROWS,
+    records: Sequence[tuple[str, Any]] | None = None,
 ) -> Iterator[JsonChunk]:
     """Yield the rows in chunks of at most `chunk_rows`, columns named by selector; always at least one.
 
     `options.json_mode` is `array` (default) or `ndjson`. A selector that finds nothing is null.
     Objects and arrays are an error unless `cast_mode` is `coerce`, which keeps their compact JSON.
     Scalars are stringified with Python's `str`; Polars would render `1.0` as `1`, so it is not used.
+    `records` are the file's already parsed `json_records`, read instead of parsing the file again.
     """
     mode = options.json_mode or "array"
     if mode not in ("array", "ndjson"):
@@ -221,7 +228,7 @@ def read_json(
     exact = [decimal_type(c.column_type) is not None for c in columns]
     rows: list[list[str | None]] = []
     done = 0
-    for where, value in json_records(path, mode):
+    for where, value in json_records(path, mode) if records is None else records:
         rows.append(
             [
                 _cell(value, p, n, cast_mode, where, e)

floe/run/__init__.py

@@ -1,7 +1,8 @@
-"""A run: entity selection, run context, sequential or parallel entities, dry run, full refresh, events (Rust: `run/mod.rs`, `run/context.rs`)."""
+"""A run: entity selection, run context, entities in config order or in parallel threads (`workers` > 1), dry run, full refresh, events (Rust: `run/mod.rs`, `run/context.rs`)."""
 
 import time
 from collections.abc import Sequence
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from pathlib import Path
@@ -23,6 +24,7 @@ from floe.report.summary import (
 )
 from floe.run.entity import EntityRun, list_source, resolve_source, run_entity
 from floe.run.events import Emitter, Observer
+from floe.run.parallel import exclusive_keys, run_parallel
 from floe.storage.inputs import ResolvedInputs
 from floe.storage.paths import normalize_local_path
 from floe.storage.resolve import clear_storage_clients, resolve_path, storage_client
@@ -103,8 +105,12 @@ def run(
     full_refresh: bool = False,
     observer: Observer | None = None,
     config_uri: str | None = None,
+    workers: int = 1,
 ) -> RunOutcome:
-    """Run the selected entities in config order, writing each entity's report as it finishes. A failing entity raises: no summary, as Rust."""
+    """Run the selected entities, writing each entity's report as it finishes. A failing entity raises: no summary, as Rust.
+
+    One worker runs them in config order. More run them in parallel threads (observers are called from those threads), never two on the same target; reports and summary stay in config order.
+    """
     clear_storage_clients()
     started, timer = _rfc3339(), time.monotonic()
     clear_catalogs()
@@ -153,20 +159,30 @@ def run(
 
     emit("run_started", config=config, report_base=report_path)
     # Rust resolves every entity's inputs before the first entity_started (#864); run_entity reuses them
-    listed: dict[str, ResolvedInputs] = {}
-    for e in selected:
-        # D24: a full refresh may rebuild from the archive alone, so run_entity decides
-        archive = full_refresh and e.sink.archive and e.incremental_mode in ("none", "archive")  # fmt: skip
-        if not archive:
-            listed[e.id] = inputs(e)
-    reports: list[tuple[str, dict[str, Any]]] = []
+    # D24: a full refresh may rebuild from the archive alone, so run_entity decides
+    to_list = [e for e in selected if not (full_refresh and e.sink.archive and e.incremental_mode in ("none", "archive"))]  # fmt: skip
+    # `map` yields in config order, so the first failure in config order is raised
+    if workers > 1 and len(to_list) > 1:
+        with ThreadPoolExecutor(workers, thread_name_prefix="floe-list") as pool:
+            found = list(pool.map(inputs, to_list))
+    else:
+        found = [inputs(e) for e in to_list]
+    listed = {e.id: f for e, f in zip(to_list, found, strict=True)}
     client = storage_client(report_storage)
-    # D36: a policy abort fails only its entity; a FloeError still stops the run
-    for e in selected:
+
+    def finish(e: EntityConfig) -> dict[str, Any]:
         report = one(e).report
-        reports.append((id_path(e), report))
         write_entity_report(run_path, id_path(e), report, client)
         _finished(emit, e.id, report)
+        return report
+
+    # D36: a policy abort fails only its entity; a FloeError still stops the run
+    if workers > 1 and len(selected) > 1:
+        keys = [exclusive_keys(e, config_dir, root.storages, root.catalogs) for e in selected]  # fmt: skip
+        done = run_parallel(selected, keys, workers, finish)
+    else:
+        done = [finish(e) for e in selected]
+    reports = [(id_path(e), r) for e, r in zip(selected, done, strict=True)]
     summary = build_summary(
         root.version, reports,
         run_id=run_id, started_at=started, finished_at=_rfc3339(),

floe/run/entity.py

@@ -21,7 +21,7 @@ from floe.checks.mismatch import (
     MismatchOutcome,
     apply_mismatch,
     declared_names,
-    json_input_columns,
+    json_probe,
     plan_mismatch,
 )
 from floe.checks.normalize import (
@@ -59,7 +59,7 @@ from floe.iceberg.write import find_committed
 from floe.pii import apply_pii_masking
 from floe.read.avro import read_avro
 from floe.read.csv import RAGGED_MESSAGE, read_csv
-from floe.read.json import read_json
+from floe.read.json import JSON_CHUNK_ROWS, JSON_SINGLE_PASS_BYTES, read_json
 from floe.read.orc import read_orc
 from floe.read.parquet import read_parquet
 from floe.read.xlsx import read_xlsx
@@ -118,7 +118,7 @@ class EntityRun:
     aborted: bool
 
 
-def _chunks(entity: EntityConfig, path: Path, names: list[str]) -> Chunks:
+def _chunks(entity: EntityConfig, path: Path, names: list[str], head: list[tuple[str, Any]] | None = None) -> Chunks:  # fmt: skip
     src = entity.source
     match src.format:
         case "csv" | "tsv":
@@ -139,7 +139,7 @@ def _chunks(entity: EntityConfig, path: Path, names: list[str]) -> Chunks:
             for x in read_xml(path, src.options, entity.schema_.columns[: len(names)]):
                 yield x.offset, x.df, None
         case "json":  # #998
-            for j in read_json(path, src.options, entity.schema_.columns[: len(names)], src.cast_mode):  # fmt: skip
+            for j in read_json(path, src.options, entity.schema_.columns[: len(names)], src.cast_mode, JSON_CHUNK_ROWS, head):  # fmt: skip
                 yield j.offset, j.df, None
         case other:
             raise FloeRunError(
@@ -402,7 +402,8 @@ def _run_file(plan: _Plan, file: InputFile, path: Path, spill: Path) -> tuple[di
     def failed(err: FloeIoError, mismatch: MismatchOutcome | None) -> tuple[dict[str, Any], Iterable[pl.DataFrame]]:  # fmt: skip
         # Rust skips the raw copy under PII: the file cannot be masked
         plan.tracker.restore(keys)  # the unreadable file leaves no keys behind
-        keep = entity.pii is None and entity.sink.rejected is not None
+        # a file removed after listing has nothing to copy: the read error alone is reported
+        keep = entity.pii is None and entity.sink.rejected is not None and path.exists()
         raw = _copy_raw(plan, path, severity, file.source_name) if keep else None
         return _failed_file(entity, uri, len(declared) + len(derived), mismatch, err, raw), []  # fmt: skip
 
@@ -411,7 +412,9 @@ def _run_file(plan: _Plan, file: InputFile, path: Path, spill: Path) -> tuple[di
             first, probe = [], xml_columns(path, entity.source.options)
         elif entity.source.format == "json":
             # D40: every record's top-level keys; D41: only a file without records has nothing to compare
-            json_keys = json_input_columns(path, entity.source.options.json_mode or "array")  # fmt: skip
+            # a file listed as small is parsed once: the read reuses the probe's records (limit 0 keeps none)
+            json_keys, head = json_probe(path, entity.source.options.json_mode or "array", JSON_CHUNK_ROWS if file.source_size is not None and file.source_size <= JSON_SINGLE_PASS_BYTES else 0)  # fmt: skip
+            chunks = _chunks(entity, path, file_names, head)
             first, probe = [], declared if json_keys is None else json_keys
         else:  # the first chunk's columns are the input columns the plan compares
             first = [next(chunks)]

floe/run/parallel.py

@@ -0,0 +1,71 @@
+"""Entities in parallel threads (#1105): the schedule keeps entities that share a resource apart and the failure rule of D36."""
+
+from collections.abc import Callable, Sequence
+from concurrent.futures import Future, ThreadPoolExecutor, wait
+from pathlib import Path
+from typing import Any
+
+from floe.config.storage import CatalogsConfig, StoragesConfig
+from floe.config.types import EntityConfig
+from floe.errors import FloeError
+from floe.iceberg.sink import resolve_target
+from floe.run.entity import state_uri
+from floe.storage.resolve import resolve_path
+
+MERGE = "merge"  # every merge entity shares this key: DuckDB memory, and Polaris conflicts on one table
+
+
+def _canonical(uri: str) -> str:
+    """One spelling per destination: a local path resolved through symlinks, a remote URI without a trailing `/`."""
+    if uri.startswith("local://"):
+        return f"local://{Path(uri.removeprefix('local://')).resolve()}"
+    return uri.rstrip("/")
+
+
+def exclusive_keys(e: EntityConfig, config_dir: Path, storages: StoragesConfig | None, catalogs: CatalogsConfig | None) -> frozenset[str]:  # fmt: skip
+    """What `e` writes that no other running entity may touch: its accepted path, Iceberg table, state file, and the merge slot."""
+    sink = e.sink
+    try:
+        keys = {_canonical(resolve_path(storages, config_dir, sink.accepted.storage, sink.accepted.path, f"entity.name={e.name} sink.accepted.storage")[0])}  # fmt: skip
+        if sink.accepted.format == "iceberg":
+            t = resolve_target(storages, catalogs, config_dir, e)
+            keys.add(f"iceberg://{t.catalog.name}/{t.namespace}/{t.table}")
+        if e.incremental_mode == "file":
+            keys.add(_canonical(state_uri(e, config_dir, storages)[0]))
+    except FloeError:
+        return frozenset()  # `run_entity` raises the same error when it runs
+    if sink.write_mode.startswith("merge"):
+        keys.add(MERGE)
+    return frozenset(keys)
+
+
+def run_parallel(entities: Sequence[EntityConfig], keys: Sequence[frozenset[str]], workers: int, task: Callable[[EntityConfig], dict[str, Any]]) -> list[dict[str, Any]]:  # fmt: skip
+    """Run `task` over `entities` on `workers` threads, never two with a key in common; the reports in `entities` order.
+
+    After a failure no entity starts, the running ones finish (and release their claims), and the first error in `entities` order is raised.
+    """
+    reports: dict[int, dict[str, Any]] = {}
+    errors: dict[int, BaseException] = {}
+    pending = list(range(len(entities)))
+    running: dict[Future[dict[str, Any]], int] = {}
+    busy: set[str] = set()
+    with ThreadPoolExecutor(workers, thread_name_prefix="floe-entity") as pool:
+        while pending or running:
+            for i in list(pending):
+                if len(running) == workers:
+                    break
+                if not keys[i] & busy:
+                    pending.remove(i)
+                    busy |= keys[i]
+                    running[pool.submit(task, entities[i])] = i
+            for future in wait(running, return_when="FIRST_COMPLETED").done:
+                i = running.pop(future)
+                busy -= keys[i]
+                if error := future.exception():
+                    errors[i] = error
+                    pending.clear()
+                else:
+                    reports[i] = future.result()
+    if errors:
+        raise errors[min(errors)]
+    return [reports[i] for i in range(len(entities))]

{floe_python-0.8.7.dist-info → floe_python-0.8.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: floe-python
-Version: 0.8.7
+Version: 0.8.9
 Summary: Floe: a data-contract runtime for file ingestion
 License-File: LICENSE
 Requires-Python: >=3.11

{floe_python-0.8.7.dist-info → floe_python-0.8.9.dist-info}/RECORD

@@ -1,53 +1,54 @@
-floe/__init__.py,sha256=BMJil7rW2sI2OGGcEPUcz_mkZpyKe77sbdjGh7kjPAo,13362
+floe/__init__.py,sha256=jEhTn9_w_xbOK0QPixbRMOBbZ2MiGAz2oW4I9n3z6_s,13254
 floe/__main__.py,sha256=Nzz6wrGXRpJiO59qlX7WEJRTlb-XcRFswmP0TA2KjyM,57
 floe/_cli.py,sha256=hwast3uMX0iUtkIpcChSIVHYkoYi4fR4jwCwQT53lXE,14101
-floe/_cli_output.py,sha256=VLxWQr-EPtIwklo9xr2CnjMD17U-7JrMUWdTtAP3jrY,8809
+floe/_cli_output.py,sha256=6DBAR3cNW2kFW_QGkwP1PpVC8AbvOjQW1LMaGnmC3qs,9232
 floe/add_entity.py,sha256=42b7l8o55dzgaR_DDgpW7YWdnU2weYMf4U_rf7qCY6M,21986
 floe/errors.py,sha256=hLSkgPQATBnjY1CAQqX4aB0oI0-WKoHW09bDXAgzC7k,186
 floe/glob.py,sha256=A0HdePLXLGl1T6heJbgdrNHr8pZfdidG7pQ1QsXWOl8,10579
 floe/maintain.py,sha256=CGSoZhO7p0u14o10FCKCO4FprzpVO548A71eq5FowF8,7343
 floe/manifest.py,sha256=sAWjv6yS4GokQDTnZwutyplLz1ICPBYl2qJmm9IORdM,20462
-floe/manifest_run.py,sha256=1XLInkpaCofMezMBs_5uIFoR5W7lbwCBFZeqRcnwUAw,12398
+floe/manifest_run.py,sha256=8rkHX2RuxHyp02NI2NLBsU6xq6m5yebR0eoYETV_7Tw,12591
 floe/pii.py,sha256=uJjU-C4c7h45E7rEL9K3Zp1LTTLh-FNKfUV5av2QMO4,4351
 floe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/state.py,sha256=XFHF6zVxbdGg1bkONZDLryTD4noDF9IgaimaEI-3Nlg,11104
 floe/checks/cast.py,sha256=lM40xtKEggWI8FW3jijM3T2JFQtaXs8FhsFXo3vy5oM,6788
-floe/checks/mismatch.py,sha256=2Gz-tiWKxy-iehSjAlsGzPLtYeNIfBWvi4yQPlASVYQ,5309
+floe/checks/mismatch.py,sha256=RNqRL1nihH6Y83RI-dxPXf0WTWZccXq6mnR4m6V0IIo,5827
 floe/checks/normalize.py,sha256=9fdRYFjWGXBAEw1Po71N5tsUWG8eekmSWIQktPduOig,4431
 floe/checks/rows.py,sha256=-yPieSus0-7ZewLv2vR8waHqQ9DGMbVvmJh1_hK4eDs,4433
-floe/checks/unique.py,sha256=qoZVqS_uG_mWZlV7-LdxT6_dMVQRlklEvMutO_V0qaY,12396
+floe/checks/unique.py,sha256=ZVckMOoGuKIzq3M84TGeZ1ks0nab9YTaI54w5Q_rjZc,10864
 floe/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/config/entity.py,sha256=CLXLXzkUbQhxenD_LwjBt0eoPreaYaxiBVEsdmnFIco,15883
 floe/config/include.py,sha256=XKYhp2QK_VUmAODAXJ35Wl4FCpQEMQm24m-WKlpI_og,5033
 floe/config/parse.py,sha256=B3gOi80-M0y9Y4OyNjM4MYmM2mFHKGVpqipX1JqaLjM,5290
-floe/config/profile.py,sha256=jWackztS-0H3dTMXHALTZ7LsIOmwjriwoeY4vFgARr0,16046
-floe/config/project.py,sha256=zgTOhTDlbMr1chViMEZyv0drozZdumTaIoKDL0vv9sw,14141
+floe/config/profile.py,sha256=4imKVdZ-Ehbj71OA3K4PrShE-gOYuicMDnTD-QHIkA0,16552
+floe/config/project.py,sha256=hUsnjbNAEiiy_iVJYA1IDquZ4OY4OF7JpNaEVJ_ocCg,14221
 floe/config/storage.py,sha256=BK3P65ukFqNgmbjCcuYmXR04OScf1yD8f19vGJq-aDI,6234
 floe/config/types.py,sha256=4eveoOlyGmNxyZpI1PSjli9lnXLPc1-jYj4fL8qHN_8,5038
 floe/config/validate.py,sha256=7tyofwoTBL-BBCNxnmkXeSOybwmMNngeEcj8zviRyLs,36176
 floe/config/vars.py,sha256=7Bq0IU0KrFeRlyA8W9-F0gDSDKfXCfFpBrX08H8Flvw,4577
 floe/config/yaml.py,sha256=HeFGhKFxYkiqQOwljYOgUD5naJyxL_ZIX6hUIS36tzU,10071
-floe/iceberg/catalog.py,sha256=xjRCn6MFZCzleeM9DYkc8f6_In1qEQX_HQpDj1QSFWk,16380
+floe/iceberg/catalog.py,sha256=qEhMvH4NFBd4GxqBEg4jqgTNxtqWkL_RRzIlgXr2GmA,16722
 floe/iceberg/compact.py,sha256=5O372vAIsvgRc2PGihryAsrQ0otn0F30AQu-cetRCP0,7123
 floe/iceberg/merge.py,sha256=VE1Odf2B34Gxj7ZcuhbrfbgTGM9AdiDz6_2BbcQ-kSQ,12448
-floe/iceberg/sink.py,sha256=WyT6yrp5JQyfaqWES96NWh3V-EzgTD4Y-IjB84mZ9Sg,11842
+floe/iceberg/sink.py,sha256=vdkgLjqvWH7opOrfc-147xCMQhLnxRjUzLjHLchHlAM,12043
 floe/iceberg/unique.py,sha256=OmiPQthhB5jGTaXsCu9ZztNFEFUf6eT0MLU2Gnh-C2A,1095
-floe/iceberg/write.py,sha256=SRX5tb17zUeWwpg6uXKkVQ_muNJ2GqlQd0n-kmjWPVc,11141
+floe/iceberg/write.py,sha256=YNE8IwLV9jK3vksO0XCj36HKHqW6ZEHCu4173Xpi6hg,12854
 floe/lineage/events.py,sha256=4txd8bGUwKBZd-8Aw_ebTurhlxAYk8SVMgH8vSjC3bs,6682
-floe/lineage/observer.py,sha256=dDYfWvPpFGjdqvFqQDZ2pTjj7uDfqji2-BTID9hzg90,10075
+floe/lineage/observer.py,sha256=IFHjzsT65L-NkcQ8Noa_8CDVpLu4iGadRB2-K0Lj8t8,10216
 floe/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/read/avro.py,sha256=rhGYlrWGFnqt_tyLIph57qV2TH91ULjoJ5qUlpW_6jI,1419
 floe/read/csv.py,sha256=1lNLh6MGrInWi3TADERA--rOG1iOVcgGdo7JPtt5LdI,5237
-floe/read/json.py,sha256=dPUXNlQUmMeYId3lef8ssEQeHi0tIOoEblvMQDmNVt4,8284
+floe/read/json.py,sha256=3wU24pKYIhheY7DRIjyb59NA4lQRPjRisXpsMH8cfdY,8807
 floe/read/orc.py,sha256=TWgdlLBR2NF_oVZ5zEmRKNXoTkq5QfF2BOFzKyzSjhY,1553
 floe/read/parquet.py,sha256=JwfuWaHj2TxBKJeaKignq8p0Op_pUfsOdVh7dDC3wcE,1491
 floe/read/xlsx.py,sha256=dengSY8HuNh9ahyR_LR8beqXTZQubqQ1Jx8qEak5jrY,3009
 floe/read/xml.py,sha256=7m6DtoU8iExuLROavyojysqOUE_Bd_N0uwFaLAzjI_Y,8004
 floe/report/entity.py,sha256=FRwLtXrlJXDNXrOKscitm-NCNJWQXd0vr_JIf_cUp20,9867
 floe/report/summary.py,sha256=tb547VRkM5WGWHQ-4vZB2DUW4VTWgv9FUoym-NpuWE4,4452
-floe/run/__init__.py,sha256=0QGSqgVTCcEH_Upp7A-Df376aN-pQc4G3F2N6uVebMw,8309
-floe/run/entity.py,sha256=lSQYCAP-d-1O9R44wJF8kgO3U2B60d_PM37NmxLJJfs,49269
+floe/run/__init__.py,sha256=yCLCoLbtA_Iuy7wabkFSImeuU8xrLp8Q0ZgYmw8WDRg,9229
+floe/run/entity.py,sha256=2NxCKNVUn0v2Oat4FslZfNjWAu3YhGKnSks3NRwkexU,49764
 floe/run/events.py,sha256=OE7zYkt0Dz2xrvgNavT1nHfZJ-97IIJ8LpHcameUJZU,2859
+floe/run/parallel.py,sha256=lKBW9NOpOu6KFOcEo-RQ56ukEOKHXPPoLI3o6gmlY_g,3435
 floe/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/storage/archive.py,sha256=1gPImm9PDSJFCpWjg37ax4ubb0wwc6FZm38quQXhORg,1101
 floe/storage/inputs.py,sha256=nmgEo2ub3hPUghLUXHU1wnKzpL4aYZbHb6S0XPYvMew,10410
@@ -60,8 +61,8 @@ floe/storage/s3.py,sha256=XcG1qJTEY3KLSMLT950mUdrAxuCA8-Gx3fpNSzyV_Xo,7508
 floe/storage/types.py,sha256=FvAEXWMdVdopRK276JyETRYGOva2-uR8pZvsWj6Xvy0,993
 floe/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/write/outputs.py,sha256=cXxGUIJnZ38XPaqEI6a83GFUsapBRdY52XQm3GlUPcI,6937
-floe_python-0.8.7.dist-info/METADATA,sha256=xk8WIhLuQqvT4lPVlzSISa5FRfsyhIpL3TjcfVUGGoo,442
-floe_python-0.8.7.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-floe_python-0.8.7.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
-floe_python-0.8.7.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
-floe_python-0.8.7.dist-info/RECORD,,
+floe_python-0.8.9.dist-info/METADATA,sha256=jwfiIw93AklPynvV_-ABBNgnNCZjBArFMK4APsKu9Bk,442
+floe_python-0.8.9.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+floe_python-0.8.9.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
+floe_python-0.8.9.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
+floe_python-0.8.9.dist-info/RECORD,,