floe-python 0.8.9__py3-none-any.whl → 0.10.0__py3-none-any.whl

floe/__init__.py

@@ -15,7 +15,12 @@ __version__ = version(
 from floe import manifest as _manifest
 from floe import run as _run
 from floe.config.parse import parse_config
-from floe.config.profile import ProfileConfig, apply_profile, parse_profile
+from floe.config.profile import (
+    ProfileConfig,
+    apply_profile,
+    parse_profile,
+    validate_runners,
+)
 from floe.config.project import RootConfig as _Root
 from floe.config.validate import validate_config
 from floe.config.vars import extract_config_env_vars as _env_vars
@@ -32,7 +37,6 @@ from floe.maintain import FLOOR_DAYS, FLOOR_SNAPSHOTS, ORPHAN_DELAY_DAYS
 from floe.maintain import maintain as _maintain
 from floe.manifest_run import load_manifest, shown
 from floe.report.summary import id_path
-from floe.run.entity import state_uri
 from floe.run.events import (
     EntityFinishedEvent,
     EntityStartedEvent,
@@ -44,6 +48,7 @@ from floe.run.events import (
     RunStartedEvent,
     SchemaEvolutionAppliedEvent,
 )
+from floe.run.incremental import state_uri
 from floe.state import dump_state, read_state, reset
 from floe.storage.paths import normalize_local_path
 from floe.storage.resolve import clear_storage_clients
@@ -151,7 +156,9 @@ def validate(
     profile_vars: dict[str, str] | None = None,
     profile_path: str | None = None,
 ) -> None:
-    _, root = _load(config_path, _profile(profile_path, profile_vars))
+    profile = _profile(profile_path, profile_vars)
+    _, root = _load(config_path, profile)
+    validate_runners(root, profile)
     validate_config(root)
     _run.select_entities(root, entities or [])
 
@@ -169,6 +176,7 @@ def run(
     if profile:  # Rust `run_with_runtime`: profile vars resolved against the config's env vars first
         profile = replace(profile, variables=resolve_vars(profile.variables, {}, _env_vars(Path(config_path).absolute())))  # fmt: skip
     path, root = _load(config_path, profile)
+    validate_runners(root, profile)
     # as `floe run -c`: OpenLineage events, and validation after the observer so lineage sees a failure
     out = run_with_lineage(
         path, root, root.entities, str(path), _emit, validate=True, run_id=run_id,
@@ -190,7 +198,9 @@ def maintain(
     experimental: bool = False,
 ) -> list[dict[str, Any]]:
     """`floe maintain`: expire Iceberg snapshots behind the retention floor (7 days, 10 snapshots), then remove orphan files."""
-    path, root = _load(config_path, _profile(profile_path, profile_vars))
+    profile = _profile(profile_path, profile_vars)
+    path, root = _load(config_path, profile)
+    validate_runners(root, profile)
     return _maintain(path, root, entities or [], older_than_days, retain_last, orphan_delay_days, dry_run, compact, experimental)  # fmt: skip
 
 

floe/_cli.py

@@ -15,6 +15,7 @@ from floe.config.profile import (
     ProfileConfig,
     apply_profile,
     parse_profile,
+    validate_runners,
 )
 from floe.config.project import RootConfig
 from floe.config.validate import validate_config
@@ -24,13 +25,17 @@ from floe.lineage.observer import run_with_lineage
 from floe.maintain import FLOOR_DAYS, FLOOR_SNAPSHOTS, ORPHAN_DELAY_DAYS, maintain
 from floe.manifest_run import load_manifest, shown
 from floe.run import domain_entities, select_entities
-from floe.run.entity import state_uri
+from floe.run.incremental import state_uri
 from floe.state import dump_state, read_state, reset
 from floe.storage.paths import normalize_local_path, parse_s3_uri
 from floe.storage.s3 import S3Client
 
 
 def _realpath(path: str) -> Path:
+    if path.startswith("s3://"):
+        raise FloeConfigError(
+            f"{path}: a config must be a local path; download it first"
+        )
     try:
         return Path(os.path.realpath(path, strict=True))
     except OSError as e:
@@ -43,7 +48,9 @@ def _profile(location: str | None) -> ProfileConfig | None:
     return parse_profile(location if location.startswith(("s3://", "env:")) else _realpath(location))  # fmt: skip
 
 
-def _load(path: Path, profile: ProfileConfig | None) -> RootConfig:
+def _load(
+    path: Path, profile: ProfileConfig | None, *, check_runners: bool = True
+) -> RootConfig:
     variables = (
         resolve_vars(profile.variables, {}, extract_config_env_vars(path))
         if profile
@@ -52,6 +59,8 @@ def _load(path: Path, profile: ProfileConfig | None) -> RootConfig:
     root = parse_config(path, variables)
     if profile:
         root = apply_profile(root, profile)
+    if check_runners:
+        validate_runners(root, profile)
     return root
 
 
@@ -99,7 +108,7 @@ def _selectors(root: RootConfig, args: argparse.Namespace) -> list[str]:
 def _run(args: argparse.Namespace) -> int:
     run_id = args.run_id or datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S.%fZ")
     console = Console(args.log_format, run_id)
-    if args.manifest is not None:  # Rust ignores `-p` here: the manifest is compiled
+    if args.manifest is not None:
         try:  # Rust logs a manifest that fails to load after a `run_started`
             root, observed, path = load_manifest(args.manifest)
         except FloeIoError as e:  # Rust: a bare io error, logged with code `error`
@@ -195,7 +204,7 @@ def _state(args: argparse.Namespace) -> int:
         raise FloeConfigError("state reset is destructive, rerun with --yes to confirm")
     profile = _profile(args.profile)
     path = _realpath(args.config)
-    root = _load(path, profile)
+    root = _load(path, profile, check_runners=False)
     [entity] = select_entities(root, [args.entity])
     uri, client = state_uri(entity, normalize_local_path(path.parent), root.storages)
     if args.state_command == "reset":
@@ -300,7 +309,12 @@ def _parser() -> argparse.ArgumentParser:
 
 
 def main() -> int:
-    args, extra = _parser().parse_known_args()
+    parser = _parser()
+    args, extra = parser.parse_known_args()
+    if args.command == "run" and args.manifest is not None and args.profile is not None:
+        parser.error(
+            "argument -p/--profile: not allowed with argument --manifest (the manifest is already compiled)"
+        )
     if extra:  # clap's wording for an unknown flag
         print(f"error: unexpected argument '{extra[0]}' found\n\nFor more information, try '--help'.", file=sys.stderr)  # fmt: skip
         return 2

floe/add_entity.py

@@ -21,7 +21,7 @@ from floe.config.parse import parse_config
 from floe.config.project import V0_3, config_version, validate_domains
 from floe.config.types import SourceOptions
 from floe.config.validate import validate_entities
-from floe.config.yaml import _FLOAT, load_yaml, parse_yaml, rust_debug  # pyright: ignore[reportPrivateUsage] -- the YAML float grammar the emitter must quote
+from floe.config.yaml import _FLOAT, load_yaml, parse_yaml  # pyright: ignore[reportPrivateUsage] -- the YAML float grammar the emitter must quote
 from floe.errors import FloeConfigError, FloeError, FloeIoError
 from floe.glob import MatchOptions, Pattern, PatternError, glob
 
@@ -355,9 +355,9 @@ def _included_domain(
         try:
             matches = glob(os.path.join(base, pattern))
         except PatternError as e:
-            raise FloeConfigError(f"invalid include.domains pattern {rust_debug(pattern)} in {config}: {e}") from e  # fmt: skip
+            raise FloeConfigError(f"invalid include.domains pattern {pattern!r} in {config}: {e}") from e  # fmt: skip
         except FloeError as e:
-            raise FloeConfigError(f"failed to match include.domains pattern {rust_debug(pattern)} in {config}: {e}") from e  # fmt: skip
+            raise FloeConfigError(f"failed to match include.domains pattern {pattern!r} in {config}: {e}") from e  # fmt: skip
         for file in matches:
             try:
                 docs: list[object] = load_yaml(Path(file))

floe/checks/normalize.py

@@ -72,13 +72,15 @@ def source_names(columns: Sequence[ColumnConfig], strategy: str | None) -> list[
     """Normalised `source or name` of each column; two columns on one name are an error."""
     seen: dict[str, str] = {}
     for column in columns:
-        source = column.source or column.name
-        name = normalize_name(source, strategy)
+        name = normalize_name(column.source or column.name, strategy)
+        label = (
+            f"{column.name} (source {column.source})" if column.source else column.name
+        )
         if name in seen:
             raise FloeConfigError(
-                f"column source collision: {seen[name]} and {source} -> {name}"
+                f"column source collision: {seen[name]} and {label} -> {name}"
             )
-        seen[name] = source
+        seen[name] = label
     return list(seen)
 
 

floe/checks/rows.py

@@ -107,14 +107,20 @@ def split_rows(
     checks: Sequence[RowCheck],
     formatter: Formatter = "json",
     sources: Mapping[str, str] | None = None,
+    raw: pl.DataFrame | None = None,
 ) -> tuple[pl.DataFrame, pl.DataFrame]:
-    """`(accepted, rejected)`; rejected adds `__floe_row_index` (UInt64, row position in `df`) and `__floe_errors`."""
+    """`(accepted, rejected)`; rejected adds `__floe_row_index` (UInt64, row position in `df`) and `__floe_errors`.
+
+    `raw` is the row-aligned frame before the casts: the rejected rows are taken from it, so a failed cast keeps its source value.
+    """
+    src = df if raw is None else raw
     # a null mask passes the row, as in `error_column`
     if not any(c.mask.any() for c in checks):
         none = [pl.Series(ROW_INDEX, [], pl.UInt64), pl.Series(ERRORS, [], pl.String)]
-        return df, df.clear().with_columns(none)  # replaces a clashing column, as below
+        # with_columns replaces a clashing column, as below
+        return df, src.clear().with_columns(none)
     errors = error_column(df.height, checks, formatter, sources)
-    rejected = df.with_columns(
+    rejected = src.with_columns(
         pl.int_range(pl.len(), dtype=pl.UInt64).alias(ROW_INDEX), errors
     ).filter(errors.is_not_null())
     return df.filter(errors.is_null()), rejected

floe/config/entity.py

@@ -44,12 +44,11 @@ from floe.config.yaml import (
     opt_u64,
     opt_vec_string,
     opt_vec_vec_string,
-    rust_debug,
     validate_known_keys,
     yaml_array,
     yaml_hash,
 )
-from floe.errors import FloeConfigError, FloeError
+from floe.errors import NOT_SUPPORTED, FloeConfigError, FloeError
 from floe.pii import PiiColumn, Strategy
 
 _T = TypeVar("_T", bound=str)
@@ -164,17 +163,16 @@ def _schema_evolution(value: object) -> SchemaEvolutionConfig:
 def _column(value: object) -> ColumnConfig:
     ctx = "schema.columns"
     h = yaml_hash(value, ctx)
-    validate_known_keys(
-        h, ctx, ("name", "source", "type", "nullable", "unique", "width", "trim")
-    )
+    for key in ("width", "trim"):
+        if key in h:
+            raise FloeConfigError(f"{ctx}.{key} {NOT_SUPPORTED}")
+    validate_known_keys(h, ctx, ("name", "source", "type", "nullable", "unique"))
     return ColumnConfig(
         name=get_string(h, "name", ctx),
         source=opt_string(h, "source", ctx),
         column_type=get_string(h, "type", ctx),
         nullable=opt_bool(h, "nullable", ctx),
         unique=opt_bool(h, "unique", ctx),
-        width=opt_u64(h, "width", ctx),
-        trim=opt_bool(h, "trim", ctx),
     )
 
 
@@ -367,9 +365,9 @@ def _sink_target(
     value: object, ctx: str, allow_options: bool, write_mode: WriteMode
 ) -> SinkTarget:
     h = yaml_hash(value, ctx)
-    allowed = ("format", "path", "storage", "filesystem")
+    allowed = ("format", "path", "storage", "filesystem", "iceberg")
     if allow_options:
-        allowed += ("options", "merge", "iceberg", "delta", "duckdb")
+        allowed += ("options", "merge", "delta", "duckdb")
         allowed += ("partition_by", "partition_spec")
     validate_known_keys(h, ctx, allowed)
     storage = _storage(h, ctx)
@@ -380,7 +378,7 @@ def _sink_target(
     fmt = get_string(h, "format", ctx)
     if "duckdb" in h and fmt != "duckdb":
         raise FloeConfigError(
-            f'{ctx}.duckdb is only valid when format is "duckdb" (found format={rust_debug(fmt)})'
+            f'{ctx}.duckdb is only valid when format is "duckdb" (found format={fmt!r})'
         )
     spec = _partition_spec(h, ctx)
     return SinkTarget(
@@ -440,6 +438,7 @@ def parse_entity(
             "name",
             "metadata",
             "domain",
+            "runner",
             "incremental_mode",
             "state",
             "source",
@@ -453,6 +452,8 @@ def parse_entity(
     metadata = _metadata(h["metadata"]) if "metadata" in h else None
     domain = opt_string(h, "domain", "entity")
     mode = opt_string(h, "incremental_mode", "entity")
+    if mode is not None and _norm(mode) == "row":
+        raise FloeConfigError(f"incremental_mode=row {NOT_SUPPORTED}")
     incremental_mode: IncrementalMode = (
         "none"
         if mode is None
@@ -469,6 +470,7 @@ def parse_entity(
         id=name,
         metadata=metadata,
         domain=domain,
+        runner=opt_string(h, "runner", "entity"),
         incremental_mode=incremental_mode,
         state=state,
         source=source,

floe/config/include.py

@@ -11,7 +11,6 @@ from floe.config.yaml import (
     get_value,
     load_single_doc,
     opt_string,
-    rust_debug,
     validate_known_keys,
     yaml_array,
     yaml_hash,
@@ -75,7 +74,7 @@ def assemble_includes(
         matches = glob(os.path.join(glob_base(config_dir), pattern))
         if not matches:
             raise FloeConfigError(
-                f"include.domains pattern {rust_debug(pattern)} matched no file "
+                f"include.domains pattern {pattern!r} matched no file "
                 "(includes resolve against local files only; a remote project cannot include)"
             )
         for match in matches:
@@ -103,7 +102,7 @@ def assemble_includes(
                     entity["domain"] = name
                 elif declared != name:
                     raise FloeConfigError(
-                        f"{entity_file} has domain {rust_debug(declared)} but its _domain.yml names {rust_debug(name)}"
+                        f"{entity_file} has domain {declared!r} but its _domain.yml names {name!r}"
                     )
                 config_files.append(entity_file)
                 entities.append((entity, entity_file))

floe/config/profile.py

@@ -1,5 +1,6 @@
 """Environment profiles, ported from `profile/parse.rs` and `profile/validate.rs`."""
 
+import math
 import os
 from collections.abc import Callable, Mapping
 from dataclasses import dataclass
@@ -17,14 +18,14 @@ from floe.config.validate import check_lineage
 from floe.config.vars import string_map
 from floe.config.yaml import (
     load_yaml,
+    merge_yaml,
     parse_yaml,
-    rust_debug,
     validate_known_keys,
     yaml_array,
     yaml_hash,
     yaml_string,
 )
-from floe.errors import FloeConfigError, FloeIoError
+from floe.errors import NOT_SUPPORTED, FloeConfigError, FloeIoError
 from floe.storage.s3 import read_s3
 
 Hash = dict[object, object]
@@ -45,12 +46,14 @@ CONTRACT = {
 @dataclass(frozen=True)
 class ProfileConfig:
     name: str
-    runner: dict[str, object]  # `type` and every field, None when unset
+    # `type` and every field, None when unset; then a kubernetes_job's pass-through keys
+    runner: dict[str, object]
     variables: dict[str, str]
     catalogs: CatalogsConfig | None
     storages: StoragesConfig | None
     lineage: LineageConfig | None
     orchestration: dict[str, object] | None = None  # its set fields only, as Rust
+    runners: dict[str, dict[str, object]] | None = None
 
 
 def _string(hash: Hash, key: str, ctx: str, required: bool = False) -> str | None:
@@ -157,17 +160,6 @@ def _str_keyed(value: object, where: str) -> dict[str, object]:
     return cast(dict[str, object], hash)
 
 
-def _object(hash: Hash, key: str, ctx: str) -> dict[str, object] | None:
-    return _str_keyed(hash[key], f"{ctx}.{key}") if key in hash else None
-
-
-def _objects(hash: Hash, key: str, ctx: str) -> list[dict[str, object]] | None:
-    if key not in hash:
-        return None
-    where = f"{ctx}.{key}"
-    return [_str_keyed(item, f"{where}[]") for item in yaml_array(hash[key], where)]
-
-
 def _auth(hash: Hash, key: str, ctx: str) -> dict[str, str | None] | None:
     if key not in hash:
         return None
@@ -200,24 +192,41 @@ _RUNNER_FIELDS: tuple[tuple[str, Reader], ...] = (
     ("env_parameters", _map),
     ("job_scope", _job_scope),
     ("domain_resources", _domain_resources),
-    ("labels", _map),
-    ("annotations", _map),
-    ("image_pull_policy", _string),
-    ("security_context", _object),
-    ("node_selector", _map),
-    ("tolerations", _objects),
-    ("backoff_limit", _u64),
-    ("env_from", _objects),
+)
+# Legacy: the keys any runner type accepted before #1065. A non-k8s runner still accepts and ignores them,
+# so a profile copied from a kubernetes_job keeps loading; they are not validated here.
+_LEGACY_IGNORED = (
+    *("labels", "annotations", "image_pull_policy", "security_context"),
+    *("node_selector", "tolerations", "backoff_limit", "env_from"),
 )
 
 
+def _json(value: object, where: str) -> object:
+    """A pass-through value as written, once JSON can carry it: string mapping keys, finite numbers."""
+    if isinstance(value, dict):
+        hash = _str_keyed(cast(object, value), where)
+        return {k: _json(v, f"{where}.{k}") for k, v in hash.items()}
+    if isinstance(value, list):
+        return [_json(v, f"{where}[]") for v in cast(list[object], value)]
+    if isinstance(value, float) and not math.isfinite(value):
+        raise FloeConfigError(f"{where} must be a finite number")
+    return value
+
+
 def _runner(value: object) -> dict[str, object]:
     ctx = "profile.execution.runner"
     hash = yaml_hash(value, ctx)
-    validate_known_keys(hash, ctx, ("type", *(name for name, _ in _RUNNER_FIELDS)))
+    known = ("type", *(name for name, _ in _RUNNER_FIELDS))
+    # #1065: a kubernetes_job's other keys are pod options its orchestrator validates; they pass through
+    open_ = hash.get("type") == "kubernetes_job"
+    validate_known_keys(hash, ctx, hash.keys() if open_ else (*known, *_LEGACY_IGNORED))
     read: dict[str, object] = {"type": _string(hash, "type", ctx, True)}
     for name, reader in _RUNNER_FIELDS:
         read[name] = reader(hash, name, ctx)
+    if open_:
+        read |= {
+            str(k): _json(v, f"{ctx}.{k}") for k, v in hash.items() if k not in known
+        }
     return read
 
 
@@ -252,9 +261,9 @@ def _metadata(value: object) -> str:
 def _validation(value: object) -> None:
     ctx = "profile.validation"
     hash = yaml_hash(value, ctx)
-    validate_known_keys(hash, ctx, ("strict",))
-    if "strict" in hash and not isinstance(hash["strict"], bool):
-        raise FloeConfigError(f"{ctx}.strict must be a boolean")
+    if "strict" in hash:
+        raise FloeConfigError(f"{ctx}.strict {NOT_SUPPORTED}")
+    validate_known_keys(hash, ctx, ())
 
 
 def _parse_doc(doc: object) -> ProfileConfig:
@@ -276,13 +285,23 @@ def _parse_doc(doc: object) -> ProfileConfig:
     name = _metadata(root["metadata"])
     runner: dict[str, object] = {}
     orchestration = None
+    runners: dict[str, dict[str, object]] = {}
     if "execution" in root:
         ctx = "profile.execution"
         execution = yaml_hash(root["execution"], ctx)
-        validate_known_keys(execution, ctx, ("runner", "orchestration"))
+        validate_known_keys(execution, ctx, ("runner", "runners", "orchestration"))
         if "runner" not in execution:
             raise FloeConfigError(f"{ctx}.runner is required")
         runner = _runner(execution["runner"])
+        if "runners" in execution:
+            named = yaml_hash(execution["runners"], f"{ctx}.runners")
+            for key, value in named.items():
+                runner_name = yaml_string(key, f"{ctx}.runners name")
+                if not runner_name.strip() or runner_name in ("default", "local"):
+                    raise FloeConfigError(
+                        f"{ctx}.runners: invalid runner name {runner_name!r}"
+                    )
+                runners[runner_name] = _runner(merge_yaml(execution["runner"], value))
         if "orchestration" in execution:
             orchestration = _orchestration(execution["orchestration"])
     variables = (
@@ -300,7 +319,14 @@ def _parse_doc(doc: object) -> ProfileConfig:
     if "validation" in root:
         _validation(root["validation"])
     return ProfileConfig(
-        name, runner, variables, catalogs, storages, lineage, orchestration
+        name,
+        runner,
+        variables,
+        catalogs,
+        storages,
+        lineage,
+        orchestration=orchestration,
+        runners=runners,
     )
 
 
@@ -311,7 +337,7 @@ def _malformed(value: str) -> str | None:
         rest = rest[start + 2 :]
         end = rest.find("}")
         if end == -1:
-            return f"${{... (unclosed placeholder in: {rust_debug(value)})}}"[:120]
+            return f"${{... (unclosed placeholder in: {value!r})}}"[:120]
         if not rest[:end].strip():
             return "${} (empty placeholder)"
         rest = rest[end + 1 :]
@@ -382,6 +408,8 @@ def validate_profile(profile: ProfileConfig) -> None:
                 value = cast(dict[str, object], value).get(sub)
             if not isinstance(value, str) or not value.strip():
                 raise FloeConfigError(f"{ctx}.{field} is required for {kind}")
+    for name, runner in (profile.runners or {}).items():
+        validate_profile(ProfileConfig(name, runner, {}, None, None, None))
     for key, value in profile.variables.items():
         if (placeholder := _malformed(value)) is not None:
             raise FloeConfigError(
@@ -403,8 +431,22 @@ def entity_workers(orchestration: Mapping[str, object] | None) -> int:
     return limit if isinstance(limit, int) else min(4, os.cpu_count() or 1)
 
 
+def validate_runners(config: RootConfig, profile: ProfileConfig | None) -> None:
+    default = (
+        "default"
+        if profile and profile.runner.get("type") not in (None, "local")
+        else "local"
+    )
+    declared = {default, *((profile.runners or {}) if profile else {})}
+    for entity in config.entities:
+        if entity.runner is not None and entity.runner not in declared:
+            available = ", ".join(sorted(declared))
+            raise FloeConfigError(
+                f"entity {entity.id} runner {entity.runner!r} is not declared; available: {available}"
+            )
+
+
 def apply_profile(config: RootConfig, profile: ProfileConfig) -> RootConfig:
-    """The profile's blocks replace the config's, as `apply_profile_catalogs`, `_storages` and `_lineage` do."""
     blocks = {
         "catalogs": profile.catalogs,
         "storages": profile.storages,

floe/config/project.py

@@ -78,6 +78,7 @@ class LineageConfig(BaseModel):
 class DomainConfig(BaseModel):
     name: str
     incoming_dir: str | None
+    runner: str | None = None
     defaults: Hash | None  # raw; `validate_domains` rejects the keys D7 forbids
     resolved_incoming_dir: str | None = None  # set by the template pass
 
@@ -150,7 +151,7 @@ def _hint(entity: object, key: str) -> str | None:
 
 def parse_domain(item: object, version: str) -> DomainConfig:
     hash = yaml_hash(item, "domains")
-    validate_known_keys(hash, "domains", ("name", "incoming_dir", "defaults"))
+    validate_known_keys(hash, "domains", ("name", "incoming_dir", "defaults", "runner"))
     name = get_string(hash, "name", "domains")
     defaults = None
     if "defaults" in hash:
@@ -160,7 +161,12 @@ def parse_domain(item: object, version: str) -> DomainConfig:
     incoming_dir = opt_string(hash, "incoming_dir", "domains")
     if incoming_dir is None and config_version(version) < V0_3:
         raise FloeConfigError("missing required field domains.incoming_dir")
-    return DomainConfig(name=name, incoming_dir=incoming_dir, defaults=defaults)
+    return DomainConfig(
+        name=name,
+        incoming_dir=incoming_dir,
+        defaults=defaults,
+        runner=opt_string(hash, "runner", "domains"),
+    )
 
 
 def parse_sources(value: object, version: str, origin: Path | None) -> dict[str, Hash]:
@@ -241,6 +247,11 @@ def parse_entities(
     for index, entity in enumerate(entities):
         try:
             parsed = parse_entity(*_merge_entity(entity, sources, defaults))
+            if parsed.runner is None:
+                runner = next(
+                    (d.runner for d in domains if d.name == parsed.domain), None
+                )
+                parsed = parsed.model_copy(update={"runner": runner})
             if qualified and parsed.domain is not None:
                 parsed = parsed.model_copy(
                     update={"id": f"{parsed.domain}.{parsed.name}"}

floe/config/types.py

@@ -6,7 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field
 
 from floe.pii import PiiColumn
 
-IncrementalMode = Literal["none", "archive", "file", "row"]
+IncrementalMode = Literal["none", "archive", "file"]
 PolicySeverity = Literal["warn", "reject", "abort"]
 SchemaEvolutionMode = Literal["strict", "add_columns"]
 SchemaEvolutionIncompatibleAction = Literal["fail"]
@@ -54,8 +54,6 @@ class ColumnConfig(_Model):
     column_type: str
     nullable: bool | None
     unique: bool | None
-    width: int | None
-    trim: bool | None
 
 
 class SchemaConfig(_Model):
@@ -126,6 +124,15 @@ class IcebergSinkOptions(_Model):
     location: str | None
 
 
+def catalog_ident(value: str) -> str:
+    """Rust `normalize_catalog_ident`."""
+    out = "".join(c.lower() if c.isascii() and (c.isalnum() or c in "_-") else "_" for c in value)  # fmt: skip
+    return out.strip("_") or "default"
+
+
+NO_ICEBERG = IcebergSinkOptions(catalog=None, namespace=None, table=None, location=None)  # fmt: skip
+
+
 class IcebergPartitionField(_Model):
     column: str
     transform: str
@@ -167,6 +174,7 @@ class EntityConfig(_Model):
     id: str
     metadata: EntityMetadata | None
     domain: str | None
+    runner: str | None = None
     incremental_mode: IncrementalMode
     state: EntityStateConfig | None
     source: SourceConfig
@@ -185,7 +193,7 @@ def with_partition_columns(entity: EntityConfig) -> EntityConfig:
     date_type = "datetime" if "{HH}" in p.pattern else "date"
     derived = [(p.date_column, date_type), (p.batch_column, "string")]
     columns = [
-        ColumnConfig(name=n, source=n, column_type=t, nullable=False, unique=None, width=None, trim=None)
+        ColumnConfig(name=n, source=n, column_type=t, nullable=False, unique=None)
         for n, t in derived
         if n is not None
     ]  # fmt: skip  # `source` keeps the name out of normalize_columns