floe-python 0.9.0__py3-none-any.whl → 0.10.1__py3-none-any.whl

floe/__init__.py

@@ -15,7 +15,12 @@ __version__ = version(
 from floe import manifest as _manifest
 from floe import run as _run
 from floe.config.parse import parse_config
-from floe.config.profile import ProfileConfig, apply_profile, parse_profile
+from floe.config.profile import (
+    ProfileConfig,
+    apply_profile,
+    parse_profile,
+    validate_runners,
+)
 from floe.config.project import RootConfig as _Root
 from floe.config.validate import validate_config
 from floe.config.vars import extract_config_env_vars as _env_vars
@@ -32,7 +37,6 @@ from floe.maintain import FLOOR_DAYS, FLOOR_SNAPSHOTS, ORPHAN_DELAY_DAYS
 from floe.maintain import maintain as _maintain
 from floe.manifest_run import load_manifest, shown
 from floe.report.summary import id_path
-from floe.run.entity import state_uri
 from floe.run.events import (
     EntityFinishedEvent,
     EntityStartedEvent,
@@ -44,6 +48,7 @@ from floe.run.events import (
     RunStartedEvent,
     SchemaEvolutionAppliedEvent,
 )
+from floe.run.incremental import state_uri
 from floe.state import dump_state, read_state, reset
 from floe.storage.paths import normalize_local_path
 from floe.storage.resolve import clear_storage_clients
@@ -151,7 +156,9 @@ def validate(
     profile_vars: dict[str, str] | None = None,
     profile_path: str | None = None,
 ) -> None:
-    _, root = _load(config_path, _profile(profile_path, profile_vars))
+    profile = _profile(profile_path, profile_vars)
+    _, root = _load(config_path, profile)
+    validate_runners(root, profile)
     validate_config(root)
     _run.select_entities(root, entities or [])
 
@@ -169,6 +176,7 @@ def run(
     if profile:  # Rust `run_with_runtime`: profile vars resolved against the config's env vars first
         profile = replace(profile, variables=resolve_vars(profile.variables, {}, _env_vars(Path(config_path).absolute())))  # fmt: skip
     path, root = _load(config_path, profile)
+    validate_runners(root, profile)
     # as `floe run -c`: OpenLineage events, and validation after the observer so lineage sees a failure
     out = run_with_lineage(
         path, root, root.entities, str(path), _emit, validate=True, run_id=run_id,
@@ -190,7 +198,9 @@ def maintain(
     experimental: bool = False,
 ) -> list[dict[str, Any]]:
     """`floe maintain`: expire Iceberg snapshots behind the retention floor (7 days, 10 snapshots), then remove orphan files."""
-    path, root = _load(config_path, _profile(profile_path, profile_vars))
+    profile = _profile(profile_path, profile_vars)
+    path, root = _load(config_path, profile)
+    validate_runners(root, profile)
     return _maintain(path, root, entities or [], older_than_days, retain_last, orphan_delay_days, dry_run, compact, experimental)  # fmt: skip
 
 

floe/_cli.py

@@ -15,6 +15,7 @@ from floe.config.profile import (
     ProfileConfig,
     apply_profile,
     parse_profile,
+    validate_runners,
 )
 from floe.config.project import RootConfig
 from floe.config.validate import validate_config
@@ -24,13 +25,17 @@ from floe.lineage.observer import run_with_lineage
 from floe.maintain import FLOOR_DAYS, FLOOR_SNAPSHOTS, ORPHAN_DELAY_DAYS, maintain
 from floe.manifest_run import load_manifest, shown
 from floe.run import domain_entities, select_entities
-from floe.run.entity import state_uri
+from floe.run.incremental import state_uri
 from floe.state import dump_state, read_state, reset
 from floe.storage.paths import normalize_local_path, parse_s3_uri
 from floe.storage.s3 import S3Client
 
 
 def _realpath(path: str) -> Path:
+    if path.startswith("s3://"):
+        raise FloeConfigError(
+            f"{path}: a config must be a local path; download it first"
+        )
     try:
         return Path(os.path.realpath(path, strict=True))
     except OSError as e:
@@ -43,7 +48,9 @@ def _profile(location: str | None) -> ProfileConfig | None:
     return parse_profile(location if location.startswith(("s3://", "env:")) else _realpath(location))  # fmt: skip
 
 
-def _load(path: Path, profile: ProfileConfig | None) -> RootConfig:
+def _load(
+    path: Path, profile: ProfileConfig | None, *, check_runners: bool = True
+) -> RootConfig:
     variables = (
         resolve_vars(profile.variables, {}, extract_config_env_vars(path))
         if profile
@@ -52,6 +59,8 @@ def _load(path: Path, profile: ProfileConfig | None) -> RootConfig:
     root = parse_config(path, variables)
     if profile:
         root = apply_profile(root, profile)
+    if check_runners:
+        validate_runners(root, profile)
     return root
 
 
@@ -99,7 +108,7 @@ def _selectors(root: RootConfig, args: argparse.Namespace) -> list[str]:
 def _run(args: argparse.Namespace) -> int:
     run_id = args.run_id or datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S.%fZ")
     console = Console(args.log_format, run_id)
-    if args.manifest is not None:  # Rust ignores `-p` here: the manifest is compiled
+    if args.manifest is not None:
         try:  # Rust logs a manifest that fails to load after a `run_started`
             root, observed, path = load_manifest(args.manifest)
         except FloeIoError as e:  # Rust: a bare io error, logged with code `error`
@@ -195,7 +204,7 @@ def _state(args: argparse.Namespace) -> int:
         raise FloeConfigError("state reset is destructive, rerun with --yes to confirm")
     profile = _profile(args.profile)
     path = _realpath(args.config)
-    root = _load(path, profile)
+    root = _load(path, profile, check_runners=False)
     [entity] = select_entities(root, [args.entity])
     uri, client = state_uri(entity, normalize_local_path(path.parent), root.storages)
     if args.state_command == "reset":
@@ -300,7 +309,12 @@ def _parser() -> argparse.ArgumentParser:
 
 
 def main() -> int:
-    args, extra = _parser().parse_known_args()
+    parser = _parser()
+    args, extra = parser.parse_known_args()
+    if args.command == "run" and args.manifest is not None and args.profile is not None:
+        parser.error(
+            "argument -p/--profile: not allowed with argument --manifest (the manifest is already compiled)"
+        )
     if extra:  # clap's wording for an unknown flag
         print(f"error: unexpected argument '{extra[0]}' found\n\nFor more information, try '--help'.", file=sys.stderr)  # fmt: skip
         return 2

floe/add_entity.py

@@ -21,7 +21,7 @@ from floe.config.parse import parse_config
 from floe.config.project import V0_3, config_version, validate_domains
 from floe.config.types import SourceOptions
 from floe.config.validate import validate_entities
-from floe.config.yaml import _FLOAT, load_yaml, parse_yaml, rust_debug  # pyright: ignore[reportPrivateUsage] -- the YAML float grammar the emitter must quote
+from floe.config.yaml import _FLOAT, load_yaml, parse_yaml  # pyright: ignore[reportPrivateUsage] -- the YAML float grammar the emitter must quote
 from floe.errors import FloeConfigError, FloeError, FloeIoError
 from floe.glob import MatchOptions, Pattern, PatternError, glob
 
@@ -355,9 +355,9 @@ def _included_domain(
         try:
             matches = glob(os.path.join(base, pattern))
         except PatternError as e:
-            raise FloeConfigError(f"invalid include.domains pattern {rust_debug(pattern)} in {config}: {e}") from e  # fmt: skip
+            raise FloeConfigError(f"invalid include.domains pattern {pattern!r} in {config}: {e}") from e  # fmt: skip
         except FloeError as e:
-            raise FloeConfigError(f"failed to match include.domains pattern {rust_debug(pattern)} in {config}: {e}") from e  # fmt: skip
+            raise FloeConfigError(f"failed to match include.domains pattern {pattern!r} in {config}: {e}") from e  # fmt: skip
         for file in matches:
             try:
                 docs: list[object] = load_yaml(Path(file))
@@ -367,6 +367,9 @@ def _included_domain(
                 raise FloeConfigError(f"failed to parse domain file {file}: {e}") from e
             doc = docs[0] if docs else None
             if isinstance(doc, dict) and cast(Hash, doc).get("name") == domain:
+                # ponytail: a sibling would not load; writing into the glob's folder is the upgrade
+                if "entities" in doc:
+                    raise FloeConfigError(f"domain file {file} sets entities; add-entity writes only sibling entity files, so create this one by hand")  # fmt: skip
                 return os.path.dirname(file), cast(Hash, doc).get("defaults", {})
     return None
 

floe/checks/normalize.py

@@ -72,13 +72,15 @@ def source_names(columns: Sequence[ColumnConfig], strategy: str | None) -> list[
     """Normalised `source or name` of each column; two columns on one name are an error."""
     seen: dict[str, str] = {}
     for column in columns:
-        source = column.source or column.name
-        name = normalize_name(source, strategy)
+        name = normalize_name(column.source or column.name, strategy)
+        label = (
+            f"{column.name} (source {column.source})" if column.source else column.name
+        )
         if name in seen:
             raise FloeConfigError(
-                f"column source collision: {seen[name]} and {source} -> {name}"
+                f"column source collision: {seen[name]} and {label} -> {name}"
             )
-        seen[name] = source
+        seen[name] = label
     return list(seen)
 
 

floe/checks/rows.py

@@ -107,14 +107,20 @@ def split_rows(
     checks: Sequence[RowCheck],
     formatter: Formatter = "json",
     sources: Mapping[str, str] | None = None,
+    raw: pl.DataFrame | None = None,
 ) -> tuple[pl.DataFrame, pl.DataFrame]:
-    """`(accepted, rejected)`; rejected adds `__floe_row_index` (UInt64, row position in `df`) and `__floe_errors`."""
+    """`(accepted, rejected)`; rejected adds `__floe_row_index` (UInt64, row position in `df`) and `__floe_errors`.
+
+    `raw` is the row-aligned frame before the casts: the rejected rows are taken from it, so a failed cast keeps its source value.
+    """
+    src = df if raw is None else raw
     # a null mask passes the row, as in `error_column`
     if not any(c.mask.any() for c in checks):
         none = [pl.Series(ROW_INDEX, [], pl.UInt64), pl.Series(ERRORS, [], pl.String)]
-        return df, df.clear().with_columns(none)  # replaces a clashing column, as below
+        # with_columns replaces a clashing column, as below
+        return df, src.clear().with_columns(none)
     errors = error_column(df.height, checks, formatter, sources)
-    rejected = df.with_columns(
+    rejected = src.with_columns(
         pl.int_range(pl.len(), dtype=pl.UInt64).alias(ROW_INDEX), errors
     ).filter(errors.is_not_null())
     return df.filter(errors.is_null()), rejected

floe/config/entity.py

@@ -44,12 +44,11 @@ from floe.config.yaml import (
     opt_u64,
     opt_vec_string,
     opt_vec_vec_string,
-    rust_debug,
     validate_known_keys,
     yaml_array,
     yaml_hash,
 )
-from floe.errors import FloeConfigError, FloeError
+from floe.errors import NOT_SUPPORTED, FloeConfigError, FloeError
 from floe.pii import PiiColumn, Strategy
 
 _T = TypeVar("_T", bound=str)
@@ -164,17 +163,16 @@ def _schema_evolution(value: object) -> SchemaEvolutionConfig:
 def _column(value: object) -> ColumnConfig:
     ctx = "schema.columns"
     h = yaml_hash(value, ctx)
-    validate_known_keys(
-        h, ctx, ("name", "source", "type", "nullable", "unique", "width", "trim")
-    )
+    for key in ("width", "trim"):
+        if key in h:
+            raise FloeConfigError(f"{ctx}.{key} {NOT_SUPPORTED}")
+    validate_known_keys(h, ctx, ("name", "source", "type", "nullable", "unique"))
     return ColumnConfig(
         name=get_string(h, "name", ctx),
         source=opt_string(h, "source", ctx),
         column_type=get_string(h, "type", ctx),
         nullable=opt_bool(h, "nullable", ctx),
         unique=opt_bool(h, "unique", ctx),
-        width=opt_u64(h, "width", ctx),
-        trim=opt_bool(h, "trim", ctx),
     )
 
 
@@ -367,9 +365,9 @@ def _sink_target(
     value: object, ctx: str, allow_options: bool, write_mode: WriteMode
 ) -> SinkTarget:
     h = yaml_hash(value, ctx)
-    allowed = ("format", "path", "storage", "filesystem")
+    allowed = ("format", "path", "storage", "filesystem", "iceberg")
     if allow_options:
-        allowed += ("options", "merge", "iceberg", "delta", "duckdb")
+        allowed += ("options", "merge", "delta", "duckdb")
         allowed += ("partition_by", "partition_spec")
     validate_known_keys(h, ctx, allowed)
     storage = _storage(h, ctx)
@@ -380,7 +378,7 @@ def _sink_target(
     fmt = get_string(h, "format", ctx)
     if "duckdb" in h and fmt != "duckdb":
         raise FloeConfigError(
-            f'{ctx}.duckdb is only valid when format is "duckdb" (found format={rust_debug(fmt)})'
+            f'{ctx}.duckdb is only valid when format is "duckdb" (found format={fmt!r})'
         )
     spec = _partition_spec(h, ctx)
     return SinkTarget(
@@ -440,6 +438,7 @@ def parse_entity(
             "name",
             "metadata",
             "domain",
+            "runner",
             "incremental_mode",
             "state",
             "source",
@@ -453,6 +452,8 @@ def parse_entity(
     metadata = _metadata(h["metadata"]) if "metadata" in h else None
     domain = opt_string(h, "domain", "entity")
     mode = opt_string(h, "incremental_mode", "entity")
+    if mode is not None and _norm(mode) == "row":
+        raise FloeConfigError(f"incremental_mode=row {NOT_SUPPORTED}")
     incremental_mode: IncrementalMode = (
         "none"
         if mode is None
@@ -469,6 +470,7 @@ def parse_entity(
         id=name,
         metadata=metadata,
         domain=domain,
+        runner=opt_string(h, "runner", "entity"),
         incremental_mode=incremental_mode,
         state=state,
         source=source,

floe/config/include.py

@@ -11,7 +11,6 @@ from floe.config.yaml import (
     get_value,
     load_single_doc,
     opt_string,
-    rust_debug,
     validate_known_keys,
     yaml_array,
     yaml_hash,
@@ -31,6 +30,21 @@ def glob_base(directory: Path) -> str:
     return Pattern.escape(text)
 
 
+def _entity_files(domain: Hash, domain_file: Path) -> list[Path]:
+    """Its `entities` glob (#1213) if the domain file sets one, else its sibling `*.yml` files."""
+    directory = domain_file.parent
+    if "entities" not in domain:
+        files = [p for p in directory.iterdir() if p.suffix == ".yml"]
+    else:
+        pattern = yaml_string(domain.pop("entities"), "domains.entities")
+        files = [Path(m) for m in glob(os.path.join(glob_base(directory), pattern))]
+        if not files:
+            raise FloeConfigError(
+                f"domains.entities pattern {pattern!r} matched no file"
+            )
+    return sorted(p for p in files if p.resolve() != domain_file.resolve())
+
+
 def assemble_includes(
     doc: object, config_dir: Path, config_files: list[Path]
 ) -> IncludeOrigins:
@@ -75,23 +89,19 @@ def assemble_includes(
         matches = glob(os.path.join(glob_base(config_dir), pattern))
         if not matches:
             raise FloeConfigError(
-                f"include.domains pattern {rust_debug(pattern)} matched no file "
+                f"include.domains pattern {pattern!r} matched no file "
                 "(includes resolve against local files only; a remote project cannot include)"
             )
         for match in matches:
             domain_file = Path(match)
             with at_file(domain_file):
-                domain = load_single_doc(domain_file)
-                name = get_string(yaml_hash(domain, "domains"), "name", "domains")
+                domain = yaml_hash(load_single_doc(domain_file), "domains")
+                name = get_string(domain, "name", "domains")
+                entity_files = _entity_files(domain, domain_file)
             config_files.append(domain_file)
             domains.append((domain, domain_file))
             # D4: an entity's name comes from its YAML, never from its file name, so `con.yml` loads like any other
-            siblings = [
-                p
-                for p in domain_file.parent.iterdir()
-                if p != domain_file and p.suffix == ".yml"
-            ]
-            for entity_file in sorted(siblings):
+            for entity_file in entity_files:
                 with at_file(entity_file):
                     loaded = load_single_doc(entity_file)
                     if not isinstance(loaded, dict):
@@ -103,7 +113,7 @@ def assemble_includes(
                     entity["domain"] = name
                 elif declared != name:
                     raise FloeConfigError(
-                        f"{entity_file} has domain {rust_debug(declared)} but its _domain.yml names {rust_debug(name)}"
+                        f"{entity_file} has domain {declared!r} but its _domain.yml names {name!r}"
                     )
                 config_files.append(entity_file)
                 entities.append((entity, entity_file))

floe/config/profile.py

@@ -18,14 +18,14 @@ from floe.config.validate import check_lineage
 from floe.config.vars import string_map
 from floe.config.yaml import (
     load_yaml,
+    merge_yaml,
     parse_yaml,
-    rust_debug,
     validate_known_keys,
     yaml_array,
     yaml_hash,
     yaml_string,
 )
-from floe.errors import FloeConfigError, FloeIoError
+from floe.errors import NOT_SUPPORTED, FloeConfigError, FloeIoError
 from floe.storage.s3 import read_s3
 
 Hash = dict[object, object]
@@ -53,6 +53,7 @@ class ProfileConfig:
     storages: StoragesConfig | None
     lineage: LineageConfig | None
     orchestration: dict[str, object] | None = None  # its set fields only, as Rust
+    runners: dict[str, dict[str, object]] | None = None
 
 
 def _string(hash: Hash, key: str, ctx: str, required: bool = False) -> str | None:
@@ -260,9 +261,9 @@ def _metadata(value: object) -> str:
 def _validation(value: object) -> None:
     ctx = "profile.validation"
     hash = yaml_hash(value, ctx)
-    validate_known_keys(hash, ctx, ("strict",))
-    if "strict" in hash and not isinstance(hash["strict"], bool):
-        raise FloeConfigError(f"{ctx}.strict must be a boolean")
+    if "strict" in hash:
+        raise FloeConfigError(f"{ctx}.strict {NOT_SUPPORTED}")
+    validate_known_keys(hash, ctx, ())
 
 
 def _parse_doc(doc: object) -> ProfileConfig:
@@ -284,13 +285,23 @@ def _parse_doc(doc: object) -> ProfileConfig:
     name = _metadata(root["metadata"])
     runner: dict[str, object] = {}
     orchestration = None
+    runners: dict[str, dict[str, object]] = {}
     if "execution" in root:
         ctx = "profile.execution"
         execution = yaml_hash(root["execution"], ctx)
-        validate_known_keys(execution, ctx, ("runner", "orchestration"))
+        validate_known_keys(execution, ctx, ("runner", "runners", "orchestration"))
         if "runner" not in execution:
             raise FloeConfigError(f"{ctx}.runner is required")
         runner = _runner(execution["runner"])
+        if "runners" in execution:
+            named = yaml_hash(execution["runners"], f"{ctx}.runners")
+            for key, value in named.items():
+                runner_name = yaml_string(key, f"{ctx}.runners name")
+                if not runner_name.strip() or runner_name in ("default", "local"):
+                    raise FloeConfigError(
+                        f"{ctx}.runners: invalid runner name {runner_name!r}"
+                    )
+                runners[runner_name] = _runner(merge_yaml(execution["runner"], value))
         if "orchestration" in execution:
             orchestration = _orchestration(execution["orchestration"])
     variables = (
@@ -308,7 +319,14 @@ def _parse_doc(doc: object) -> ProfileConfig:
     if "validation" in root:
         _validation(root["validation"])
     return ProfileConfig(
-        name, runner, variables, catalogs, storages, lineage, orchestration
+        name,
+        runner,
+        variables,
+        catalogs,
+        storages,
+        lineage,
+        orchestration=orchestration,
+        runners=runners,
     )
 
 
@@ -319,7 +337,7 @@ def _malformed(value: str) -> str | None:
         rest = rest[start + 2 :]
         end = rest.find("}")
         if end == -1:
-            return f"${{... (unclosed placeholder in: {rust_debug(value)})}}"[:120]
+            return f"${{... (unclosed placeholder in: {value!r})}}"[:120]
         if not rest[:end].strip():
             return "${} (empty placeholder)"
         rest = rest[end + 1 :]
@@ -390,6 +408,8 @@ def validate_profile(profile: ProfileConfig) -> None:
                 value = cast(dict[str, object], value).get(sub)
             if not isinstance(value, str) or not value.strip():
                 raise FloeConfigError(f"{ctx}.{field} is required for {kind}")
+    for name, runner in (profile.runners or {}).items():
+        validate_profile(ProfileConfig(name, runner, {}, None, None, None))
     for key, value in profile.variables.items():
         if (placeholder := _malformed(value)) is not None:
             raise FloeConfigError(
@@ -411,8 +431,22 @@ def entity_workers(orchestration: Mapping[str, object] | None) -> int:
     return limit if isinstance(limit, int) else min(4, os.cpu_count() or 1)
 
 
+def validate_runners(config: RootConfig, profile: ProfileConfig | None) -> None:
+    default = (
+        "default"
+        if profile and profile.runner.get("type") not in (None, "local")
+        else "local"
+    )
+    declared = {default, *((profile.runners or {}) if profile else {})}
+    for entity in config.entities:
+        if entity.runner is not None and entity.runner not in declared:
+            available = ", ".join(sorted(declared))
+            raise FloeConfigError(
+                f"entity {entity.id} runner {entity.runner!r} is not declared; available: {available}"
+            )
+
+
 def apply_profile(config: RootConfig, profile: ProfileConfig) -> RootConfig:
-    """The profile's blocks replace the config's, as `apply_profile_catalogs`, `_storages` and `_lineage` do."""
     blocks = {
         "catalogs": profile.catalogs,
         "storages": profile.storages,

floe/config/project.py

@@ -78,6 +78,7 @@ class LineageConfig(BaseModel):
 class DomainConfig(BaseModel):
     name: str
     incoming_dir: str | None
+    runner: str | None = None
     defaults: Hash | None  # raw; `validate_domains` rejects the keys D7 forbids
     resolved_incoming_dir: str | None = None  # set by the template pass
 
@@ -150,7 +151,7 @@ def _hint(entity: object, key: str) -> str | None:
 
 def parse_domain(item: object, version: str) -> DomainConfig:
     hash = yaml_hash(item, "domains")
-    validate_known_keys(hash, "domains", ("name", "incoming_dir", "defaults"))
+    validate_known_keys(hash, "domains", ("name", "incoming_dir", "defaults", "runner"))
     name = get_string(hash, "name", "domains")
     defaults = None
     if "defaults" in hash:
@@ -160,7 +161,12 @@ def parse_domain(item: object, version: str) -> DomainConfig:
     incoming_dir = opt_string(hash, "incoming_dir", "domains")
     if incoming_dir is None and config_version(version) < V0_3:
         raise FloeConfigError("missing required field domains.incoming_dir")
-    return DomainConfig(name=name, incoming_dir=incoming_dir, defaults=defaults)
+    return DomainConfig(
+        name=name,
+        incoming_dir=incoming_dir,
+        defaults=defaults,
+        runner=opt_string(hash, "runner", "domains"),
+    )
 
 
 def parse_sources(value: object, version: str, origin: Path | None) -> dict[str, Hash]:
@@ -241,6 +247,11 @@ def parse_entities(
     for index, entity in enumerate(entities):
         try:
             parsed = parse_entity(*_merge_entity(entity, sources, defaults))
+            if parsed.runner is None:
+                runner = next(
+                    (d.runner for d in domains if d.name == parsed.domain), None
+                )
+                parsed = parsed.model_copy(update={"runner": runner})
             if qualified and parsed.domain is not None:
                 parsed = parsed.model_copy(
                     update={"id": f"{parsed.domain}.{parsed.name}"}

floe/config/types.py

@@ -6,7 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field
 
 from floe.pii import PiiColumn
 
-IncrementalMode = Literal["none", "archive", "file", "row"]
+IncrementalMode = Literal["none", "archive", "file"]
 PolicySeverity = Literal["warn", "reject", "abort"]
 SchemaEvolutionMode = Literal["strict", "add_columns"]
 SchemaEvolutionIncompatibleAction = Literal["fail"]
@@ -54,8 +54,6 @@ class ColumnConfig(_Model):
     column_type: str
     nullable: bool | None
     unique: bool | None
-    width: int | None
-    trim: bool | None
 
 
 class SchemaConfig(_Model):
@@ -126,6 +124,15 @@ class IcebergSinkOptions(_Model):
     location: str | None
 
 
+def catalog_ident(value: str) -> str:
+    """Rust `normalize_catalog_ident`."""
+    out = "".join(c.lower() if c.isascii() and (c.isalnum() or c in "_-") else "_" for c in value)  # fmt: skip
+    return out.strip("_") or "default"
+
+
+NO_ICEBERG = IcebergSinkOptions(catalog=None, namespace=None, table=None, location=None)  # fmt: skip
+
+
 class IcebergPartitionField(_Model):
     column: str
     transform: str
@@ -167,6 +174,7 @@ class EntityConfig(_Model):
     id: str
     metadata: EntityMetadata | None
     domain: str | None
+    runner: str | None = None
     incremental_mode: IncrementalMode
     state: EntityStateConfig | None
     source: SourceConfig
@@ -185,7 +193,7 @@ def with_partition_columns(entity: EntityConfig) -> EntityConfig:
     date_type = "datetime" if "{HH}" in p.pattern else "date"
     derived = [(p.date_column, date_type), (p.batch_column, "string")]
     columns = [
-        ColumnConfig(name=n, source=n, column_type=t, nullable=False, unique=None, width=None, trim=None)
+        ColumnConfig(name=n, source=n, column_type=t, nullable=False, unique=None)
         for n, t in derived
         if n is not None
     ]  # fmt: skip  # `source` keeps the name out of normalize_columns