floe-python 0.8.0__py3-none-any.whl → 0.8.1__py3-none-any.whl

floe/__init__.py

@@ -132,7 +132,7 @@ def _profile(
     if path is None:
         return None if variables is None else ProfileConfig("", {}, variables, None, None, None)  # fmt: skip
     try:
-        profile = parse_profile(Path(path))
+        profile = parse_profile(path)
     except FloeError as e:
         raise FloeConfigError(f"failed to load profile {_debug(path)}: {e}") from e
     return ProfileConfig(profile.name, profile.runner, profile.variables | (variables or {}),
@@ -225,10 +225,11 @@ def build_manifest(
 ) -> dict[str, Any]:
     """What `floe manifest generate` writes; `manifest_uri` is a remote `-o` (where it is deployed)."""
     from floe._cli import _load as _cli_load  # pyright: ignore[reportPrivateUsage] -- the CLI's config loading, so both build the same manifest
+    from floe._cli import _profile as _cli_profile  # pyright: ignore[reportPrivateUsage] -- as above
     from floe._cli import _realpath  # pyright: ignore[reportPrivateUsage] -- as above
 
     path = _realpath(config_path)
-    profile = parse_profile(_realpath(profile_path)) if profile_path else None
+    profile = _cli_profile(profile_path)
     root = _cli_load(path, profile)
     validate_config(root)
     options = _manifest.Options(

floe/_cli.py

@@ -23,7 +23,7 @@ from floe.errors import FloeConfigError, FloeError, FloeIoError
 from floe.lineage.observer import run_with_lineage
 from floe.maintain import FLOOR_DAYS, FLOOR_SNAPSHOTS, ORPHAN_DELAY_DAYS, maintain
 from floe.manifest_run import load_manifest, shown
-from floe.run import select_entities
+from floe.run import domain_entities, select_entities
 from floe.run.entity import state_uri
 from floe.state import dump_state, read_state, reset
 from floe.storage.paths import normalize_local_path, parse_s3_uri
@@ -37,6 +37,12 @@ def _realpath(path: str) -> Path:
         raise FloeIoError(f"{e.strerror} (os error {e.errno})") from e
 
 
+def _profile(location: str | None) -> ProfileConfig | None:
+    if not location:
+        return None
+    return parse_profile(location if location.startswith("s3://") else _realpath(location))  # fmt: skip
+
+
 def _load(path: Path, profile: ProfileConfig | None) -> RootConfig:
     variables = (
         resolve_vars(profile.variables, {}, extract_config_env_vars(path))
@@ -50,7 +56,7 @@ def _load(path: Path, profile: ProfileConfig | None) -> RootConfig:
 
 
 def _validate(args: argparse.Namespace) -> int:
-    profile = parse_profile(_realpath(args.profile)) if args.profile else None
+    profile = _profile(args.profile)
     if args.config is None:
         if profile is None:
             raise FloeConfigError(
@@ -75,11 +81,21 @@ def _validate(args: argparse.Namespace) -> int:
         if e.sink.rejected:
             print(f"  Sink rejected: {e.sink.rejected.format} ({e.sink.rejected.path})")  # fmt: skip
         print(f"  Severity: {e.policy.severity}")
-    profile_arg = f" -p {os.path.realpath(args.profile)}" if args.profile else ""
+    profile_arg = ""
+    if args.profile:
+        profile_arg = f" -p {args.profile if args.profile.startswith('s3://') else os.path.realpath(args.profile)}"  # fmt: skip
     print(f"Next: floe run -c {path}{profile_arg}")
     return 0
 
 
+def _selectors(root: RootConfig, args: argparse.Namespace) -> list[str]:
+    if not args.domain:
+        return args.entities
+    if args.entities:
+        raise FloeConfigError("--domain cannot be combined with --entities")
+    return domain_entities(root, args.domain)
+
+
 def _run(args: argparse.Namespace) -> int:
     run_id = args.run_id or datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S.%fZ")
     console = Console(args.log_format, run_id)
@@ -97,7 +113,7 @@ def _run(args: argparse.Namespace) -> int:
     elif args.config is None:
         raise FloeConfigError("one of --config or --manifest is required")
     else:
-        profile = parse_profile(_realpath(args.profile)) if args.profile else None
+        profile = _profile(args.profile)
         try:  # Rust logs a missing config without `run_started`
             path = _realpath(args.config)
         except FloeError as e:
@@ -112,7 +128,7 @@ def _run(args: argparse.Namespace) -> int:
     try:
         outcome = run_with_lineage(
             path, root, observed, job, console, validate=args.manifest is None,
-            run_id=run_id, entities=args.entities, dry_run=args.dry_run, full_refresh=args.full_refresh,
+            run_id=run_id, entities=_selectors(root, args), dry_run=args.dry_run, full_refresh=args.full_refresh,
             config_uri=config_uri,
         )  # fmt: skip
     except FloeError as e:
@@ -177,7 +193,7 @@ def _add_entity(args: argparse.Namespace) -> int:
 def _state(args: argparse.Namespace) -> int:
     if args.state_command == "reset" and not args.yes:
         raise FloeConfigError("state reset is destructive, rerun with --yes to confirm")
-    profile = parse_profile(_realpath(args.profile)) if args.profile else None
+    profile = _profile(args.profile)
     path = _realpath(args.config)
     root = _load(path, profile)
     [entity] = select_entities(root, [args.entity])
@@ -199,7 +215,7 @@ def _state(args: argparse.Namespace) -> int:
 
 
 def _maintain(args: argparse.Namespace) -> int:
-    profile = parse_profile(_realpath(args.profile)) if args.profile else None
+    profile = _profile(args.profile)
     path = _realpath(args.config)
     results = maintain(
         path, _load(path, profile), args.entities, args.older_than_days, args.retain_last,
@@ -226,6 +242,7 @@ def _parser() -> argparse.ArgumentParser:
     run_source = run.add_mutually_exclusive_group()
     run_source.add_argument("-c", "--config")
     run_source.add_argument("--manifest")
+    run.add_argument("--domain", action="extend", type=lambda s: s.split(","), default=[])  # fmt: skip
     run.add_argument("--run-id")
     run.add_argument("-F", "--full-refresh", action="store_true")
     run.add_argument("--dry-run", action="store_true")

floe/_cli_output.py

@@ -2,7 +2,6 @@
 
 import json
 import time
-from pathlib import Path
 from typing import Any
 
 from floe.config.types import EntityConfig
@@ -158,11 +157,11 @@ def run_output(
     timings: list[dict[str, int]],
 ) -> str:
     """Rust's `format_run_output`; `mode` is `default`, `quiet` or `verbose`."""
-    run_dir = Path(outcome.run_dir)
+    run_dir = outcome.run_dir  # a URI on remote storage: join with `/`, never `Path`
     head = [f"run id: {outcome.run_id}", f"report base: {outcome.report_base_path}"]
     if outcome.dry_run is not None:
         by_id = {e.id: e for e in entities}
-        blocks = [_preview(by_id[i], files, str(run_dir / id_path(by_id[i]) / "run.json"), mode == "verbose")
+        blocks = [_preview(by_id[i], files, f"{run_dir}/{id_path(by_id[i])}/run.json", mode == "verbose")
                   for i, files in outcome.dry_run.items()]  # fmt: skip
         results = dict.fromkeys(("files_total", "rows_total", "accepted_total", "rejected_total", "files_skipped"), 0)  # fmt: skip
         status, exit_code = "success", 0
@@ -182,6 +181,6 @@ def run_output(
     lines += [
         f"Totals: files={results['files_total']} rows={results['rows_total']} accepted={results['accepted_total']} rejected={results['rejected_total']}{skipped}",
         f"Overall: {status} (exit_code={exit_code})",
-        f"Run summary: {run_dir / 'run.summary.json'}",
+        f"Run summary: {run_dir}/run.summary.json",
     ]  # fmt: skip
     return "\n".join(lines)

floe/config/profile.py

@@ -16,13 +16,15 @@ from floe.config.validate import check_lineage
 from floe.config.vars import string_map
 from floe.config.yaml import (
     load_yaml,
+    parse_yaml,
     rust_debug,
     validate_known_keys,
     yaml_array,
     yaml_hash,
     yaml_string,
 )
-from floe.errors import FloeConfigError
+from floe.errors import FloeConfigError, FloeIoError
+from floe.storage.s3 import read_s3
 
 Hash = dict[object, object]
 Reader = Callable[[Hash, str, str], object]
@@ -65,6 +67,13 @@ def _strings(hash: Hash, key: str, ctx: str) -> list[str] | None:
     return [yaml_string(item, f"{ctx}.{key}[]") for item in items]
 
 
+def _command(hash: Hash, key: str, ctx: str) -> str | list[str] | None:
+    """A string, or a list as the manifest and the connectors take it."""
+    if isinstance(hash.get(key), list):
+        return _strings(hash, key, ctx)
+    return _string(hash, key, ctx)
+
+
 def _u64(hash: Hash, key: str, ctx: str) -> int | None:
     if key not in hash:
         return None
@@ -141,6 +150,23 @@ def _job_scope(hash: Hash, key: str, ctx: str) -> str | None:
     return scope
 
 
+def _str_keyed(value: object, where: str) -> dict[str, object]:
+    hash = yaml_hash(value, where)
+    validate_known_keys(hash, where, hash.keys())  # every key must be a string
+    return cast(dict[str, object], hash)
+
+
+def _object(hash: Hash, key: str, ctx: str) -> dict[str, object] | None:
+    return _str_keyed(hash[key], f"{ctx}.{key}") if key in hash else None
+
+
+def _objects(hash: Hash, key: str, ctx: str) -> list[dict[str, object]] | None:
+    if key not in hash:
+        return None
+    where = f"{ctx}.{key}"
+    return [_str_keyed(item, f"{where}[]") for item in yaml_array(hash[key], where)]
+
+
 def _auth(hash: Hash, key: str, ctx: str) -> dict[str, str | None] | None:
     if key not in hash:
         return None
@@ -153,7 +179,7 @@ def _auth(hash: Hash, key: str, ctx: str) -> dict[str, str | None] | None:
 
 # Rust reads the runner fields in this order, so the first malformed one is the one reported.
 _RUNNER_FIELDS: tuple[tuple[str, Reader], ...] = (
-    ("command", _string),
+    ("command", _command),
     ("args", _strings),
     ("timeout_seconds", _u64),
     ("ttl_seconds_after_finished", _u64),
@@ -173,6 +199,14 @@ _RUNNER_FIELDS: tuple[tuple[str, Reader], ...] = (
     ("env_parameters", _map),
     ("job_scope", _job_scope),
     ("domain_resources", _domain_resources),
+    ("labels", _map),
+    ("annotations", _map),
+    ("image_pull_policy", _string),
+    ("security_context", _object),
+    ("node_selector", _map),
+    ("tolerations", _objects),
+    ("backoff_limit", _u64),
+    ("env_from", _objects),
 )
 
 
@@ -370,8 +404,17 @@ def apply_profile(config: RootConfig, profile: ProfileConfig) -> RootConfig:
     return config.model_copy(update={k: v for k, v in blocks.items() if v is not None})
 
 
-def parse_profile(path: Path) -> ProfileConfig:
-    docs = load_yaml(path)
+def _docs(location: str | Path) -> list[object]:
+    if isinstance(location, Path) or not location.startswith("s3://"):
+        return load_yaml(Path(location))
+    try:
+        return parse_yaml(read_s3(location, "profile").decode())
+    except UnicodeDecodeError as e:
+        raise FloeIoError("stream did not contain valid UTF-8") from e
+
+
+def parse_profile(location: str | Path) -> ProfileConfig:
+    docs = _docs(location)
     if not docs:
         raise FloeConfigError("profile YAML is empty")
     if len(docs) > 1:

floe/config/storage.py

@@ -1,5 +1,7 @@
 """Storage and catalog definitions, ported from `config/parse.rs` (`parse_storages`, `parse_catalogs_with_context`)."""
 
+from typing import Literal
+
 from pydantic import BaseModel, ConfigDict, SecretStr
 
 from floe.config.yaml import (
@@ -37,16 +39,22 @@ class StoragesConfig(_Model):
 
 
 class CatalogDefinition(_Model):
-    """An Iceberg REST catalog; Glue and Unity are rejected (D1)."""
+    """An Iceberg REST or AWS Glue catalog; Unity is rejected (D1)."""
 
     name: str
-    uri: str
-    credential: SecretStr | None
-    warehouse: str | None
-    oauth2_server_uri: str | None
-    scope: str | None
+    type: Literal["rest", "glue"] = "rest"
+    # REST only
+    uri: str = ""
+    credential: SecretStr | None = None
+    warehouse: str | None = None
+    oauth2_server_uri: str | None = None
+    scope: str | None = None
     warehouse_storage: str | None
     warehouse_prefix: str | None
+    # Glue only; Rust 0.7's `allow_takeover` is accepted and ignored (D42)
+    region: str | None = None
+    database: str | None = None
+    create_database_if_missing: bool = True
 
 
 class CatalogsConfig(_Model):
@@ -88,11 +96,11 @@ _CATALOG_KEYS = (
     "scope",
     "warehouse_storage",
     "warehouse_prefix",
-    # Glue and Unity keys stay allowed, as in Rust, so a typo reports the key first
     "region",
     "database",
     "create_database_if_missing",
     "allow_takeover",
+    # Unity keys stay allowed, as in Rust, so a typo reports the key first
     "host",
     "catalog",
     "schema",
@@ -127,13 +135,24 @@ def _catalog(value: object, ctx: str) -> CatalogDefinition:
     validate_known_keys(hash, ctx, _CATALOG_KEYS)
     name = get_string(hash, "name", ctx)
     catalog_type = get_string(hash, "type", ctx)
-    if catalog_type in ("glue", "unity"):
+    if catalog_type == "unity":
         raise _Unsupported(
             f"{ctx} name={name} type={catalog_type} is not supported in 0.8"
         )
+    if catalog_type == "glue":
+        return CatalogDefinition(
+            name=name,
+            type="glue",
+            region=get_string(hash, "region", ctx),
+            database=get_string(hash, "database", ctx),
+            create_database_if_missing=opt_bool(hash, "create_database_if_missing", ctx)
+            is not False,
+            warehouse_storage=opt_string(hash, "warehouse_storage", ctx),
+            warehouse_prefix=opt_string(hash, "warehouse_prefix", ctx),
+        )
     if catalog_type != "rest":
         raise FloeConfigError(
-            f"{ctx} name={name} has unsupported type={catalog_type} (supported: rest)"
+            f"{ctx} name={name} has unsupported type={catalog_type} (supported: glue, rest)"
         )
     uri = get_string(hash, "uri", ctx)
     credential = opt_string(hash, "credential", ctx)

floe/config/validate.py

@@ -330,7 +330,10 @@ def _check_iceberg(
         raise _fail(
             entity, f"sink.accepted.iceberg.catalog references unknown catalog {name}"
         )
-    # Unity and Glue are rejected at parse (D1); the registry checked `warehouse_storage`.
+    # Unity is rejected at parse (D1); the registry checked `warehouse_storage`.
+    if definition.type == "glue" and entity.sink.write_mode.startswith("merge"):
+        # ponytail: merges run through DuckDB, attached to REST catalogs only; attach Glue when a merge needs it
+        raise _fail(entity, f"sink.write_mode={entity.sink.write_mode} requires a rest catalog (glue supports overwrite and append)")  # fmt: skip
     if definition.warehouse_storage is None:
         kind = _storage_type(config, accepted_storage)
         if kind not in ("s3", "gcs"):
@@ -708,7 +711,13 @@ def _check_storages(config: RootConfig) -> None:
                     f"catalogs.definitions name={c.name} warehouse_storage "
                     f"references unknown storage {wh}"
                 )
-            if (kind := _storage_type(config, wh)) not in ("s3", "gcs"):
+            kind = _storage_type(config, wh)
+            if c.type == "glue" and kind != "s3":
+                raise FloeConfigError(
+                    f"catalogs.definitions name={c.name} warehouse_storage must "
+                    f"reference s3 storage for glue catalog (got {kind})"
+                )
+            if kind not in ("s3", "gcs"):
                 raise FloeConfigError(
                     f"catalogs.definitions name={c.name} warehouse_storage must "
                     f"reference s3 or gcs storage for rest catalog (got {kind})"

floe/iceberg/catalog.py

@@ -74,12 +74,14 @@ def _credential_props(catalog: str, credential: str) -> dict[str, str]:
 def open_catalog(
     definition: CatalogDefinition, storage: StorageDefinition | None = None
 ) -> Catalog:
-    """Credentials for S3 come from the environment; the catalog never vends them (as in Rust)."""
+    """Credentials for S3 and Glue come from the environment; the catalog never vends them (as in Rust)."""
     props = {
         "type": "rest",
         "uri": definition.uri,
         "header.X-Iceberg-Access-Delegation": "",
     }
+    if definition.type == "glue":
+        props = {"type": "glue", "glue.region": definition.region or ""}
     if definition.warehouse is not None:
         props["warehouse"] = definition.warehouse
     if definition.credential is not None:
@@ -99,7 +101,7 @@ def open_catalog(
             props["s3.force-virtual-addressing"] = str(
                 not storage.path_style_access
             ).lower()
-    with run_error("rest catalog init failed"):
+    with run_error(f"{definition.type} catalog init failed"):
         return load_catalog(definition.name, **props)
 
 
@@ -167,6 +169,9 @@ def open_duckdb(
     definition: CatalogDefinition, storage: StorageDefinition | None = None
 ) -> duckdb.DuckDBPyConnection:
     """DuckDB with the catalog attached as `DUCKDB_CATALOG`."""
+    # validation rejects Glue merges; compaction lands here too
+    if definition.type != "rest":
+        raise FloeConfigError(f"iceberg catalog {definition.name}: merges and compaction require a rest catalog (got {definition.type})")  # fmt: skip
     statements = _duckdb_setup(definition, storage)
     with run_error("duckdb catalog attach failed"):
         # ponytail: DuckDB's default memory_limit and spill dir; set the ADR's limits when the run wires merges
@@ -292,8 +297,9 @@ def ensure_table(
     location: str | None = None,
     partition_fields: PartitionFields | None = None,
     schema_evolution: SchemaEvolutionMode = "strict",
+    create_namespace: bool = True,
 ) -> Table:
-    """Create the namespace and table if missing; otherwise check the table matches (Rust's append path).
+    """Create the namespace (unless `create_namespace` is off) and table if missing; otherwise check the table matches (Rust's append path).
 
     With `add_columns`, columns new in `schema` are added as optional (D13); partition changes stay rejected.
     """
@@ -301,6 +307,8 @@ def ensure_table(
     with run_error("rest catalog namespace_exists check failed"):
         namespace_exists = catalog.namespace_exists(ident)
     if not namespace_exists:
+        if not create_namespace:
+            raise FloeRunError(f"entity.name={entity} iceberg namespace {namespace} does not exist and create_database_if_missing is false")  # fmt: skip
         with run_error("rest catalog namespace create failed"):
             catalog.create_namespace(ident)
     identifier = (namespace, name)

floe/iceberg/sink.py

@@ -70,7 +70,9 @@ def resolve_target(storages: StoragesConfig | None, catalogs: CatalogsConfig | N
     catalog = next((c for c in (catalogs.definitions if catalogs else []) if c.name == name), None)  # fmt: skip
     if catalog is None:  # validated; kept for callers that skip validation
         raise FloeConfigError(f"entity.name={entity.name} sink.accepted.iceberg.catalog references unknown catalog {name}")  # fmt: skip
-    namespace = catalog_ident(sink.iceberg.namespace or entity.domain or "default")
+    namespace = catalog_ident(
+        sink.iceberg.namespace or entity.domain or catalog.database or "default"
+    )
     table = catalog_ident(sink.iceberg.table or entity.name)
     field = f"entity.name={entity.name} sink.accepted.storage"
     accepted_uri, storage = resolve_path(storages, config_dir, sink.storage, sink.path, field)  # fmt: skip
@@ -82,6 +84,9 @@ def resolve_target(storages: StoragesConfig | None, catalogs: CatalogsConfig | N
         location, storage = resolve_path(storages, config_dir, where, "/".join(p for p in parts if p), field)  # fmt: skip
     else:
         location = accepted_uri
+    # Rust: the Glue database holds the table; the namespace only names its location
+    if catalog.database is not None:
+        namespace = catalog.database
     return IcebergTarget(catalog, storage, namespace, table, location, accepted_uri)  # fmt: skip
 
 
@@ -169,7 +174,7 @@ def write_iceberg(
             raise FloeRunError(f"entity.name={entity.name} iceberg schema evolution failed: merge key columns cannot be added: {', '.join(added)}")  # fmt: skip
     table = ensure_table(
         catalog, target.namespace, target.table, schema, entity.name, target.location.removeprefix("local://"),
-        partition_fields(entity), evolution.mode if evolution else "strict",
+        partition_fields(entity), evolution.mode if evolution else "strict", target.catalog.create_database_if_missing,
     )  # fmt: skip
     merge: dict[str, Any] | None = None
     if mode in ("merge_scd1", "merge_scd2"):

floe/manifest.py

@@ -26,6 +26,7 @@ from floe.errors import FloeConfigError, FloeError
 from floe.pii import PiiColumn
 from floe.run import select_entities
 from floe.storage.paths import normalize_local_path
+from floe.storage.s3 import read_s3
 
 Json = dict[str, Any]
 _REMOTE = ("s3://", "gs://", "gcs://", "abfs://", "abfss://", "az://")
@@ -274,7 +275,10 @@ _RUNNER_FIELDS = (
     *("resources", "env", "workspace_url", "existing_cluster_id", "config_uri"),
     *("python_file_uri", "job_name", "auth", "env_parameters"),
 )
-_PYTHON_RUNNER_FIELDS = ("job_scope", "domain_resources")  # carried only when set
+_PYTHON_RUNNER_FIELDS = (  # carried only when set
+    *("job_scope", "domain_resources", "labels", "annotations", "image_pull_policy"),
+    *("security_context", "node_selector", "tolerations", "backoff_limit", "env_from"),
+)
 
 
 # Rust `runners_contract`: the fields each profile runner type carries into the manifest
@@ -351,6 +355,13 @@ def _blocks(root: RootConfig) -> Json:
                     {
                         "name": c.name,
                         "type_config": {
+                            "type": "glue",
+                            "region": c.region,
+                            "database": c.database,
+                            "create_database_if_missing": c.create_database_if_missing,
+                        }
+                        if c.type == "glue"
+                        else {
                             "type": "rest",
                             "uri": c.uri,
                             "credential": c.credential
@@ -452,9 +463,14 @@ def build_manifest(
     report = _Target(ctx, root.report)
     profile_uri = profile_checksum = None
     if options.profile_arg is not None:  # recorded like `config_uri` and its checksum
-        profile = Path(os.path.realpath(options.profile_arg))
-        profile_uri = _config_uri(options.profile_arg, profile, work_root)
-        profile_checksum = _sha256(profile.read_bytes())
+        # an s3 profile is read from there by the run, so it is recorded as typed
+        if options.profile_arg.startswith("s3://"):
+            profile_uri = options.profile_arg
+            profile_checksum = _sha256(read_s3(profile_uri, "profile"))
+        else:
+            profile = Path(os.path.realpath(options.profile_arg))
+            profile_uri = _config_uri(options.profile_arg, profile, work_root)
+            profile_checksum = _sha256(profile.read_bytes())
     manifest: Json = _put(
         {
             "generated_at_ts_ms": 0,

floe/manifest_run.py

@@ -38,7 +38,7 @@ from floe.config.types import (
 from floe.config.validate import FLOE_VERSION
 from floe.errors import FloeConfigError, FloeIoError
 from floe.storage.paths import normalize_local_path, parse_s3_uri
-from floe.storage.s3 import S3Client
+from floe.storage.s3 import read_s3
 
 Json = dict[str, Any]
 _M = TypeVar("_M", bound=BaseModel)
@@ -180,8 +180,7 @@ def _storage_def(d: Json) -> Json:
 
 
 def _catalog_def(d: Json) -> Json:
-    rest = {k: v for k, v in d["type_config"].items() if k != "type"}  # D1: REST only
-    return {k: v for k, v in d.items() if k != "type_config"} | rest
+    return {k: v for k, v in d.items() if k != "type_config"} | d["type_config"]
 
 
 def _report(base: str, storages: StoragesConfig | None) -> tuple[ReportConfig, StoragesConfig | None]:  # fmt: skip
@@ -226,12 +225,8 @@ def _named_local(m: Json) -> bool:
 
 
 def _read(location: str) -> bytes:
-    # S3 settings come from the environment (AWS_ENDPOINT_URL, ...), as for `-o s3://`
     if location.startswith("s3://"):
-        found = S3Client(parse_s3_uri(location)[0]).read(location)
-        if found is None:
-            raise FloeIoError(f"manifest not found: {location}")
-        return found.body
+        return read_s3(location, "manifest")
     if "://" in location:  # D1: gcs and adls are dropped
         raise FloeConfigError(f"remote manifest is not supported: {location}")
     try:

floe/run/__init__.py

@@ -59,6 +59,14 @@ def select_entities(root: RootConfig, selectors: Sequence[str]) -> list[EntityCo
     return [e for i, e in enumerate(root.entities) if i in picked]
 
 
+def domain_entities(root: RootConfig, domains: Sequence[str]) -> list[str]:
+    """Ids of the entities of these domains, as `select_entities` selectors."""
+    missing = [d for d in dict.fromkeys(domains) if all(e.domain != d for e in root.entities)]  # fmt: skip
+    if missing:
+        raise FloeConfigError(f"domains not found: {', '.join(missing)}")
+    return [e.id for e in root.entities if e.domain in domains]
+
+
 def _rfc3339() -> str:
     return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
 

floe/storage/s3.py

@@ -9,7 +9,7 @@ import boto3
 from botocore.config import Config
 from botocore.exceptions import BotoCoreError, ClientError
 
-from floe.errors import FloeStorageError
+from floe.errors import FloeIoError, FloeStorageError
 from floe.storage.paths import fnv1a_hex, parse_s3_uri
 from floe.storage.types import ObjectRef, StoredObject
 
@@ -180,3 +180,11 @@ class S3Client:
             raise FloeStorageError(f"s3 delete object failed: {e}") from e
         except BotoCoreError as e:
             raise FloeStorageError(f"s3 delete object failed: {e}") from e
+
+
+def read_s3(uri: str, what: str) -> bytes:
+    # S3 settings come from the environment (AWS_ENDPOINT_URL, ...), as for `-o s3://`
+    found = S3Client(parse_s3_uri(uri)[0]).read(uri)
+    if found is None:
+        raise FloeIoError(f"{what} not found: {uri}")
+    return found.body

{floe_python-0.8.0.dist-info → floe_python-0.8.1.dist-info}/METADATA

@@ -1,10 +1,10 @@
 Metadata-Version: 2.5
 Name: floe-python
-Version: 0.8.0
+Version: 0.8.1
 Summary: Floe: a data-contract runtime for file ingestion
 License-File: LICENSE
 Requires-Python: >=3.11
-Requires-Dist: boto3>=1.43.118
+Requires-Dist: boto3>=1.36.0
 Requires-Dist: duckdb==1.5.6
 Requires-Dist: fastexcel>=0.21.0
 Requires-Dist: polars>=2.0.0

{floe_python-0.8.0.dist-info → floe_python-0.8.1.dist-info}/RECORD

@@ -1,13 +1,13 @@
-floe/__init__.py,sha256=VkSVWF7vzIICTkxaNI2SVXfD2JQmrHtV_NiTBCUwzaM,13221
+floe/__init__.py,sha256=0X9pm8uWiwoDslKZyyjsEAYd6g1mlvIWhkWgE5uhGng,13279
 floe/__main__.py,sha256=Nzz6wrGXRpJiO59qlX7WEJRTlb-XcRFswmP0TA2KjyM,57
-floe/_cli.py,sha256=2ecJIMELxfXkm3ISzvQsS-CnnjUBj1Kqh9HANlUlMCM,13549
-floe/_cli_output.py,sha256=fwUqMkBvUEojri8WnXGzTIHMZLcoOHQfByeOz5L6ugA,8792
+floe/_cli.py,sha256=6zQRRczAgARgc7PcLrWS7lEzJa6GKsbpf8Zz8GsDjFo,14081
+floe/_cli_output.py,sha256=VLxWQr-EPtIwklo9xr2CnjMD17U-7JrMUWdTtAP3jrY,8809
 floe/add_entity.py,sha256=42b7l8o55dzgaR_DDgpW7YWdnU2weYMf4U_rf7qCY6M,21986
 floe/errors.py,sha256=hLSkgPQATBnjY1CAQqX4aB0oI0-WKoHW09bDXAgzC7k,186
 floe/glob.py,sha256=A0HdePLXLGl1T6heJbgdrNHr8pZfdidG7pQ1QsXWOl8,10579
 floe/maintain.py,sha256=CGSoZhO7p0u14o10FCKCO4FprzpVO548A71eq5FowF8,7343
-floe/manifest.py,sha256=HV3y-WzZeorVpTAGrNU9LuZCbvX8ALfGC9J3Q2jtnYY,19425
-floe/manifest_run.py,sha256=8m4LXaymkw3ZX706sTjZK5L1hOYDT231CWl9tjoBe1c,12700
+floe/manifest.py,sha256=GDaa01K9n5vxI0kRQSkM_xjjIwoeF6JN-N1Fktas8Xc,20216
+floe/manifest_run.py,sha256=1XLInkpaCofMezMBs_5uIFoR5W7lbwCBFZeqRcnwUAw,12398
 floe/pii.py,sha256=uJjU-C4c7h45E7rEL9K3Zp1LTTLh-FNKfUV5av2QMO4,4351
 floe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/state.py,sha256=XFHF6zVxbdGg1bkONZDLryTD4noDF9IgaimaEI-3Nlg,11104
@@ -20,17 +20,17 @@ floe/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/config/entity.py,sha256=CLXLXzkUbQhxenD_LwjBt0eoPreaYaxiBVEsdmnFIco,15883
 floe/config/include.py,sha256=XKYhp2QK_VUmAODAXJ35Wl4FCpQEMQm24m-WKlpI_og,5033
 floe/config/parse.py,sha256=B3gOi80-M0y9Y4OyNjM4MYmM2mFHKGVpqipX1JqaLjM,5290
-floe/config/profile.py,sha256=xlLbrMbD_prJ0UteoSUNwbmjOq5NMm6XmQvxQNg4WxI,14035
+floe/config/profile.py,sha256=mYbyFse2pBUFuV5TMtl0RFrfyPUnJhdklbAX5eSAM88,15564
 floe/config/project.py,sha256=zgTOhTDlbMr1chViMEZyv0drozZdumTaIoKDL0vv9sw,14141
-floe/config/storage.py,sha256=Yjia1mfxWvIq7DpS24AFSCQDeJ4aqxY9s76WDDrqMv4,5452
+floe/config/storage.py,sha256=BK3P65ukFqNgmbjCcuYmXR04OScf1yD8f19vGJq-aDI,6234
 floe/config/types.py,sha256=4eveoOlyGmNxyZpI1PSjli9lnXLPc1-jYj4fL8qHN_8,5038
-floe/config/validate.py,sha256=4m71oZgvKFH61Ts6TASlLZ_JDnPEjWo4cfGcoJOLGQY,34963
+floe/config/validate.py,sha256=MRwGztx0xbcqGjpIKjfCMDMeyTz-pd4Mr9XrfyiW5CQ,35571
 floe/config/vars.py,sha256=7Bq0IU0KrFeRlyA8W9-F0gDSDKfXCfFpBrX08H8Flvw,4577
 floe/config/yaml.py,sha256=HeFGhKFxYkiqQOwljYOgUD5naJyxL_ZIX6hUIS36tzU,10071
-floe/iceberg/catalog.py,sha256=nRRC4VyjKEXATlifmlE2394e8b5vAO_HiIp0LugS5tI,13819
+floe/iceberg/catalog.py,sha256=TeINK-jVZJAurROEeovVNK3s4PY-czeWih1y_KWohP8,14455
 floe/iceberg/compact.py,sha256=5O372vAIsvgRc2PGihryAsrQ0otn0F30AQu-cetRCP0,7123
 floe/iceberg/merge.py,sha256=VE1Odf2B34Gxj7ZcuhbrfbgTGM9AdiDz6_2BbcQ-kSQ,12448
-floe/iceberg/sink.py,sha256=2pWrixvqC9_ZNKvGAJju4-TfKOqFoYR28agllUt9AXA,10804
+floe/iceberg/sink.py,sha256=854HAAz4UmsgBGR6TrBy3tcqWk-ss1RcuLK-ynQe0gA,11040
 floe/iceberg/unique.py,sha256=PKUFEcCkU71k66__KfUE7NTos73ZYlZbb8GJPn0yFoo,2958
 floe/iceberg/write.py,sha256=zrFeeJu-YT4Pxo9u-fRujUtFb1Epx1KFoq2qFyP1iGs,11036
 floe/lineage/events.py,sha256=4txd8bGUwKBZd-8Aw_ebTurhlxAYk8SVMgH8vSjC3bs,6682
@@ -45,7 +45,7 @@ floe/read/xlsx.py,sha256=dengSY8HuNh9ahyR_LR8beqXTZQubqQ1Jx8qEak5jrY,3009
 floe/read/xml.py,sha256=7m6DtoU8iExuLROavyojysqOUE_Bd_N0uwFaLAzjI_Y,8004
 floe/report/entity.py,sha256=ekuB-1JDATvxIFnBTCQpku8uXtrSQNwvryjdTidXk6I,9707
 floe/report/summary.py,sha256=tb547VRkM5WGWHQ-4vZB2DUW4VTWgv9FUoym-NpuWE4,4452
-floe/run/__init__.py,sha256=PON2NYpvsRej6cXBPP8GDRZMcinVTtnKpwLT93Vzg0c,7687
+floe/run/__init__.py,sha256=6teCk4GZoX7grwtyhQUbqwbx_TSQebzViUJA19CmEl4,8107
 floe/run/entity.py,sha256=4UGhUtjYR84MOe3jjwFmLGWAaDln1z5KOHC0MOvDLpQ,45204
 floe/run/events.py,sha256=OE7zYkt0Dz2xrvgNavT1nHfZJ-97IIJ8LpHcameUJZU,2859
 floe/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -55,12 +55,12 @@ floe/storage/local.py,sha256=NKjBf_-PkDnbZ3FzXWE7KZ1xY0G_o3tRy0_7nxW7FLs,6307
 floe/storage/partitioning.py,sha256=MG8RJDrwgZhucTswlXC2RwCmzrYIoOE6SH_aBM11KW8,6178
 floe/storage/paths.py,sha256=Z0fry3Gp0Tu2DpUeKqNXdqYP6n_KcKzK2llGx0TGHB8,2343
 floe/storage/resolve.py,sha256=pFAuVytTfnSEcTAVlJjFTj7S_Vipjry1ormWZP3Egmc,2593
-floe/storage/s3.py,sha256=iWLDr3-qFH-Of3HKUiaKFD-1HQRip4FSQ-q9xxv7VNI,7212
+floe/storage/s3.py,sha256=XcG1qJTEY3KLSMLT950mUdrAxuCA8-Gx3fpNSzyV_Xo,7508
 floe/storage/types.py,sha256=FvAEXWMdVdopRK276JyETRYGOva2-uR8pZvsWj6Xvy0,993
 floe/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/write/outputs.py,sha256=5LgN_8PMPPOgkRVZw_KvxYXmc02-meL7iZLeMKVtufQ,4932
-floe_python-0.8.0.dist-info/METADATA,sha256=jQyy8j3vSQ6Qs047OQASPHOEnq5UV02wpXavZ3MHf7U,444
-floe_python-0.8.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-floe_python-0.8.0.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
-floe_python-0.8.0.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
-floe_python-0.8.0.dist-info/RECORD,,
+floe_python-0.8.1.dist-info/METADATA,sha256=t7zQL0mXQAhLksYIkXMkuuYolrQL3046WRDPjZurnQI,442
+floe_python-0.8.1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+floe_python-0.8.1.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
+floe_python-0.8.1.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
+floe_python-0.8.1.dist-info/RECORD,,