floe-python 0.8.0__py3-none-any.whl

floe/config/validate.py

@@ -0,0 +1,803 @@
+"""Entity validation, ported from `config/validate.rs` (`validate_sink`, `validate_schema` and their helpers)."""
+
+import re
+
+from floe import __version__
+from floe.checks.cast import decimal_type
+from floe.checks.normalize import normalize_name, output_names
+from floe.config.project import (
+    LineageConfig,
+    RootConfig,
+    config_version,
+    validate_domains,
+)
+from floe.config.types import EntityConfig, WriteMode, with_partition_columns
+from floe.config.yaml import rust_debug
+from floe.errors import FloeConfigError
+from floe.pii import token_n
+from floe.read.json import parse_selector as parse_json_selector
+from floe.read.xml import parse_selector
+
+MIN_VERSION = (0, 1)
+QUALIFIED_VERSION = (0, 3)
+# D21: root.version is the minimum Floe release (major.minor) the config targets.
+FLOE_VERSION = config_version(".".join(__version__.split(".")[:2]))
+MAX_JSON_COLUMNS = 1024
+CAST_MODES = ("strict", "coerce")
+SOURCE_FORMATS = ("csv", "tsv", "parquet", "json", "orc", "avro", "xlsx", "xml")
+REMOTE_URIS = tuple(f"{s}://" for s in ("s3", "gs", "gcs", "abfs", "abfss", "az"))
+STORAGE_TYPES = ("local", "s3", "adls", "gcs")
+NORMALIZE_STRATEGIES = ("snake_case", "lower", "camel_case", "none")
+MISMATCH_MISSING = ("reject_file", "fill_nulls")
+MISMATCH_EXTRA = ("reject_file", "ignore")
+# D10 (#619): Rust 0.7 also lists `time`; it is not supported in 0.8.
+COLUMN_TYPES = ("string", "number", "boolean", "datetime", "date")
+SINK_FORMATS = ("parquet", "iceberg")
+PARQUET_COMPRESSIONS = ("snappy", "gzip", "zstd", "uncompressed")
+PARTITION_TRANSFORMS = ("identity", "year", "month", "day", "hour", "bucket")  # D12
+PARTITION_TOKENS = ("yyyy", "MM", "dd", "HH", "batch")  # D28
+PARTITION_TOKEN = re.compile(r"\{([^{}]*)\}")
+# D2 (#619): Iceberg accepts the merge modes; Rust 0.7 allows them for Delta (D1) and DuckDB (D1) only.
+SINK_WRITE_MODES: dict[str, tuple[WriteMode, ...]] = {
+    "parquet": ("overwrite", "append"),
+    "iceberg": ("overwrite", "append", "merge_scd1", "merge_scd2"),
+}
+SCD2_DEFAULTS = {
+    "current_flag_column": "__floe_is_current",
+    "valid_from_column": "__floe_valid_from",
+    "valid_to_column": "__floe_valid_to",
+}
+# Same normalisation as `checks/cast.py`, so every accepted type casts.
+_TYPE_NAMES = {
+    *("string", "str", "text", "boolean", "bool", "datetime", "timestamp", "date"),
+    *("number", "float", "float32", "float64", "double", "int"),
+    *("int8", "int16", "int32", "int64", "integer", "long"),
+    *("uint8", "uint16", "uint32", "uint64"),
+}  # fmt: skip
+
+
+def _canonical(value: str) -> str:
+    # D7 (#584 S1, S2): no trim, unlike Rust's `normalize_value`; the run-time paths never trimmed.
+    return value.lower().replace("-", "").replace("_", "")
+
+
+def _fail(entity: EntityConfig, message: str) -> FloeConfigError:
+    return FloeConfigError(f"entity.name={entity.name} {message}")
+
+
+def _check_primary_key(entity: EntityConfig) -> None:
+    primary_key = entity.schema_.primary_key
+    if primary_key is None:
+        return
+    if not primary_key:
+        raise _fail(entity, "schema.primary_key must not be empty")
+    columns = {c.name: c for c in reversed(entity.schema_.columns)}
+    seen: set[str] = set()
+    for index, name in enumerate(primary_key):
+        value = name.strip()
+        if not value:
+            raise _fail(entity, f"schema.primary_key[{index}] must not be empty")
+        if value in seen:
+            raise _fail(entity, f"schema.primary_key has duplicate column {value}")
+        seen.add(value)
+        if value not in columns:
+            raise _fail(
+                entity,
+                f"schema.primary_key[{index}]={value} references unknown schema column",
+            )
+        if columns[value].nullable is True:
+            raise _fail(
+                entity, f"schema.primary_key column {value} cannot set nullable=true"
+            )
+
+
+def _check_unique_keys(entity: EntityConfig) -> None:
+    # ponytail: Rust's override and dedup warnings are not ported; they are not errors.
+    unique_keys = entity.schema_.unique_keys
+    if unique_keys is None:
+        return
+    if not unique_keys:
+        raise _fail(entity, "schema.unique_keys must not be empty")
+    names = {c.name for c in entity.schema_.columns}
+    for index, key in enumerate(unique_keys):
+        where = f"schema.unique_keys[{index}]"
+        if not key:
+            raise _fail(entity, f"{where} must not be empty")
+        seen: set[str] = set()
+        for column_index, name in enumerate(key):
+            value = name.strip()
+            if not value:
+                raise _fail(entity, f"{where}[{column_index}] must not be empty")
+            if value not in names:
+                raise _fail(
+                    entity,
+                    f"{where}[{column_index}]={value} references unknown schema column",
+                )
+            if value in seen:
+                raise _fail(entity, f"{where} has duplicate column {value}")
+            seen.add(value)
+
+
+def _check_evolution(entity: EntityConfig, version: str) -> None:
+    evolution = entity.schema_.schema_evolution
+    if evolution is None:
+        return
+    if config_version(version) < (0, 2):
+        raise _fail(entity, 'schema.schema_evolution requires root.version >= "0.2"')
+    if evolution.mode == "add_columns" and entity.sink.accepted.format != "iceberg":
+        raise _fail(
+            entity,
+            "schema.schema_evolution.mode=add_columns requires sink.accepted.format=iceberg",
+        )
+
+
+def _merge_list(entity: EntityConfig, field: str, values: list[str]) -> None:
+    names = {c.name for c in entity.schema_.columns}
+    primary_key = {c.strip() for c in entity.schema_.primary_key or []}
+    where = f"sink.accepted.merge.{field}"
+    seen: set[str] = set()
+    for index, raw in enumerate(values):
+        value = raw.strip()
+        if not value:
+            raise _fail(entity, f"{where}[{index}] must not be empty")
+        if value not in names:
+            raise _fail(
+                entity, f"{where}[{index}]={value} references unknown schema column"
+            )
+        if value in seen:
+            raise _fail(entity, f"{where} has duplicate column {value}")
+        seen.add(value)
+    for index, raw in enumerate(values):
+        if raw.strip() in primary_key:
+            raise _fail(
+                entity,
+                f"{where}[{index}]={raw.strip()} cannot reference schema.primary_key column",
+            )
+
+
+def _check_merge(entity: EntityConfig) -> None:
+    merge = entity.sink.accepted.merge
+    if merge is None:
+        return
+    mode = entity.sink.write_mode
+    if not any(
+        m.startswith("merge") for m in SINK_WRITE_MODES[entity.sink.accepted.format]
+    ):
+        raise _fail(entity, "sink.accepted.merge requires sink.accepted.format=iceberg")
+    if mode not in ("merge_scd1", "merge_scd2"):
+        raise _fail(
+            entity,
+            "sink.accepted.merge is only supported with sink.write_mode=merge_scd1 or merge_scd2",
+        )
+    for field in ("ignore_columns", "compare_columns"):
+        values = getattr(merge, field)
+        if values is not None:
+            _merge_list(entity, field, values)
+    if merge.scd2 is None:
+        return
+    if mode != "merge_scd2":
+        raise _fail(
+            entity,
+            "sink.accepted.merge.scd2 is only supported with sink.write_mode=merge_scd2",
+        )
+    nc = (
+        entity.schema_.normalize_columns
+    )  # raw strategy: an unsupported one is validate_schema's error
+    strategy = (nc.strategy or "snake_case") if nc and nc.enabled else None
+    names = set(output_names(entity.schema_.columns, strategy))
+    resolved = {
+        field: (default if (v := getattr(merge.scd2, field)) is None else v).strip()
+        for field, default in SCD2_DEFAULTS.items()
+    }
+    for field, value in resolved.items():
+        if not value:
+            raise _fail(entity, f"sink.accepted.merge.scd2.{field} must not be empty")
+        if value in names:
+            raise _fail(
+                entity,
+                f"sink.accepted.merge.scd2.{field}={value} collides with schema column name",
+            )
+    if len(set(resolved.values())) != len(resolved):
+        raise _fail(entity, "sink.accepted.merge.scd2 column names must be unique")
+
+
+def _check_write_mode(entity: EntityConfig) -> None:
+    mode, fmt = entity.sink.write_mode, entity.sink.accepted.format
+    if mode not in SINK_WRITE_MODES[fmt]:
+        raise _fail(
+            entity,
+            f"sink.write_mode={mode} is not supported by sink.accepted.format={fmt}",
+        )
+    if mode in ("merge_scd1", "merge_scd2"):
+        primary_key = entity.schema_.primary_key
+        if primary_key is None:
+            raise _fail(entity, f"sink.write_mode={mode} requires schema.primary_key")
+        if not primary_key:
+            raise _fail(
+                entity, f"sink.write_mode={mode} requires non-empty schema.primary_key"
+            )
+    _check_merge(entity)
+
+
+def _check_partitioning(entity: EntityConfig) -> None:
+    accepted = entity.sink.accepted
+    if accepted.partition_by is not None and accepted.partition_spec is not None:
+        raise _fail(
+            entity,
+            "sink.accepted.partition_by and sink.accepted.partition_spec are mutually exclusive",
+        )
+    # ponytail: `partition_by` is Delta-only (D1), so its per-column checks are unreachable.
+    if accepted.partition_by is not None:
+        raise _fail(
+            entity,
+            "sink.accepted.partition_by is not supported in 0.8; "
+            "use sink.accepted.partition_spec with sink.accepted.format=iceberg",
+        )
+    spec = accepted.partition_spec
+    if spec is None:
+        return
+    where = "sink.accepted.partition_spec"
+    if accepted.format != "iceberg":
+        raise _fail(
+            entity, f"{where} is only supported for sink.accepted.format=iceberg"
+        )
+    if not spec:
+        raise _fail(entity, f"{where} must not be empty")
+    names = {c.name for c in entity.schema_.columns}
+    seen: set[tuple[str, str]] = set()
+    for index, field in enumerate(spec):
+        column = field.column.strip()
+        if not column:
+            raise _fail(entity, f"{where}[{index}].column must not be empty")
+        if column not in names:
+            raise _fail(
+                entity,
+                f"{where}[{index}].column={column} references unknown schema column",
+            )
+        transform = field.transform.strip().lower()
+        if transform not in PARTITION_TRANSFORMS:
+            raise _fail(
+                entity,
+                f"{where}[{index}].transform={field.transform} is unsupported "
+                f"(allowed: {', '.join(PARTITION_TRANSFORMS)})",
+            )
+        if transform == "bucket" and field.num_buckets is None:
+            raise _fail(
+                entity, f"{where}[{index}].num_buckets is required for transform=bucket"
+            )
+        if transform != "bucket" and field.num_buckets is not None:
+            raise _fail(
+                entity,
+                f"{where}[{index}].num_buckets is only supported with transform=bucket",
+            )
+        if field.num_buckets == 0:
+            raise _fail(entity, f"{where}[{index}].num_buckets must be greater than 0")
+        if (column, transform) in seen:
+            raise _fail(entity, f"{where} has duplicate entry ({column}, {transform})")
+        seen.add((column, transform))
+
+
+def _storage_ref(
+    config: RootConfig,
+    name: str | None,
+    field: str,
+    who: str,
+) -> str:
+    """`resolve_name` then `validate_reference`; `who` is the `entity.name=X ` prefix, empty for the report."""
+    storages = config.storages
+    if name is None:
+        name = (storages and storages.default) or "local"
+    if storages is None and name != "local":
+        raise FloeConfigError(
+            f"{who}{field} references unknown storage {name} (no storages block)"
+        )
+    if storages and name not in {d.name for d in storages.definitions}:
+        raise FloeConfigError(f"{who}{field} references unknown storage {name}")
+    return name
+
+
+def _storage_type(config: RootConfig, name: str) -> str:
+    definitions = config.storages.definitions if config.storages else []
+    return next((d.type for d in definitions if d.name == name), "local")
+
+
+def _check_iceberg(
+    entity: EntityConfig, config: RootConfig, accepted_storage: str
+) -> None:
+    accepted = entity.sink.accepted
+    iceberg = accepted.iceberg
+    if iceberg is None:
+        return
+    if accepted.format != "iceberg":
+        raise _fail(
+            entity,
+            "sink.accepted.iceberg is only supported for sink.accepted.format=iceberg",
+        )
+    catalogs = config.catalogs
+    name = (
+        iceberg.catalog
+        if iceberg.catalog is not None
+        else catalogs and catalogs.default
+    )
+    if name is None:
+        raise _fail(
+            entity,
+            "sink.accepted.iceberg.catalog is required (or set catalogs.default)",
+        )
+    definitions = catalogs.definitions if catalogs else []
+    definition = next((c for c in definitions if c.name == name), None)
+    if definition is None:
+        raise _fail(
+            entity, f"sink.accepted.iceberg.catalog references unknown catalog {name}"
+        )
+    # Unity and Glue are rejected at parse (D1); the registry checked `warehouse_storage`.
+    if definition.warehouse_storage is None:
+        kind = _storage_type(config, accepted_storage)
+        if kind not in ("s3", "gcs"):
+            raise _fail(
+                entity,
+                f"sink.accepted.iceberg.catalog requires sink.accepted storage type s3 or gcs (got {kind})",
+            )
+
+
+def _check_sink(entity: EntityConfig, config: RootConfig) -> None:
+    sink, accepted = entity.sink, entity.sink.accepted
+    if accepted.format in ("delta", "duckdb"):  # D1 (#619): reported first
+        raise _fail(
+            entity, f"sink.accepted.format={accepted.format} is not supported in 0.8"
+        )
+    if accepted.format not in SINK_FORMATS:
+        raise _fail(entity, f"sink.accepted.format={accepted.format} is unsupported")
+    options = accepted.options
+    if options is not None and accepted.format == "parquet":
+        if options.compression not in (None, *PARQUET_COMPRESSIONS):
+            raise _fail(
+                entity,
+                f"sink.accepted.options.compression={options.compression} is unsupported "
+                f"(allowed: {', '.join(PARQUET_COMPRESSIONS)})",
+            )
+        for field in ("row_group_size", "max_size_per_file"):
+            if getattr(options, field) == 0:
+                raise _fail(
+                    entity, f"sink.accepted.options.{field} must be greater than 0"
+                )
+    # D39 (#1007): abort writes the failing file to sink.rejected too; Rust 0.7 only failed at run time
+    if entity.policy.severity in ("reject", "abort") and sink.rejected is None:
+        raise _fail(entity, f"sink.rejected is required when policy.severity={entity.policy.severity}")  # fmt: skip
+    if sink.rejected is not None and sink.rejected.format != "csv":
+        raise _fail(
+            entity, f"sink.rejected.format={sink.rejected.format} is unsupported"
+        )
+    who = f"entity.name={entity.name} "
+    accepted_storage = _storage_ref(
+        config, accepted.storage, "sink.accepted.storage", who
+    )
+    _check_iceberg(entity, config, accepted_storage)
+    _check_partitioning(entity)
+    _check_write_mode(entity)
+    if sink.rejected is not None:
+        _storage_ref(config, sink.rejected.storage, "sink.rejected.storage", who)
+    archive = sink.archive
+    source = _storage_ref(config, entity.source.storage, "source.storage", who)
+    if archive is not None and archive.storage not in (None, source):
+        raise _fail(
+            entity, f"sink.archive.storage must match source.storage ({source})"
+        )
+
+
+def validate_schema(entity: EntityConfig, version: str) -> None:
+    schema = entity.schema_
+    part = entity.source.partitioning  # D28: derived columns are not read from the file
+    derived = {part.date_column, part.batch_column} if part else set[str | None]()
+    inputs = [c for c in schema.columns if c.name not in derived]
+    if entity.source.format == "json" and len(inputs) > MAX_JSON_COLUMNS:
+        raise _fail(
+            entity,
+            f"schema.columns has {len(inputs)} entries which exceeds "
+            f"the JSON selector limit of {MAX_JSON_COLUMNS}",
+        )
+    strategy = schema.normalize_columns and schema.normalize_columns.strategy
+    if strategy is not None and _canonical(strategy) not in {
+        _canonical(s) for s in NORMALIZE_STRATEGIES
+    }:
+        raise _fail(
+            entity,
+            f"schema.normalize_columns.strategy={strategy} is unsupported "
+            f"(allowed: {', '.join(NORMALIZE_STRATEGIES)})",
+        )
+    mismatch = schema.mismatch
+    if mismatch is not None:
+        for field, policy, allowed in (
+            ("missing_columns", mismatch.missing_columns, MISMATCH_MISSING),
+            ("extra_columns", mismatch.extra_columns, MISMATCH_EXTRA),
+        ):
+            if policy is not None and policy not in allowed:
+                raise _fail(
+                    entity,
+                    f"schema.mismatch.{field}={policy} is unsupported "
+                    f"(allowed: {', '.join(allowed)})",
+                )
+    for index, column in enumerate(inputs):
+        kind = _canonical(column.column_type)
+        try:
+            exact = decimal_type(column.column_type)
+        except ValueError as err:
+            raise _fail(entity, f"schema.columns[{index}].type: {err}") from err
+        if kind == "decimal":  # Float64 cannot hold exact decimals; scale is required
+            raise _fail(
+                entity,
+                f"schema.columns[{index}].type=decimal needs a precision and scale, "
+                "e.g. decimal(19,2); use float64 for approximate numbers",
+            )
+        if exact is None and kind not in _TYPE_NAMES:
+            raise _fail(
+                entity,
+                f"schema.columns[{index}].type={column.column_type} is unsupported "
+                f"for column {column.name} (allowed: {', '.join(COLUMN_TYPES)})",
+            )
+        parse = {"xml": parse_selector, "json": parse_json_selector}.get(
+            entity.source.format
+        )
+        if parse:
+            selector = column.source if column.source is not None else column.name
+            try:
+                parse(selector)
+            except ValueError as err:
+                raise _fail(
+                    entity,
+                    f"schema.columns[{index}].source={selector} is invalid: {err}",
+                ) from err
+    _check_primary_key(entity)
+    _check_unique_keys(entity)
+    _check_evolution(entity, version)
+
+
+def _check_source(entity: EntityConfig, config: RootConfig) -> None:
+    source = entity.source
+    if source.format == "fixed":  # D1 (#619): fixed-width is dropped
+        raise _fail(entity, f"source.format={source.format} is not supported in 0.8")
+    if source.format not in SOURCE_FORMATS:
+        raise _fail(entity, f"source.format={source.format} is unsupported")
+    if source.cast_mode not in CAST_MODES:
+        raise _fail(
+            entity,
+            f"source.cast_mode={source.cast_mode} is unsupported "
+            f"(allowed: {', '.join(CAST_MODES)})",
+        )
+    who = f"entity.name={entity.name} "
+    _storage_ref(config, source.storage, "source.storage", who)
+    options = source.options
+    mode = options.json_mode or "array"
+    if source.format == "json" and mode not in ("array", "ndjson"):
+        raise _fail(
+            entity,
+            f"source.options.json_mode={mode} is unsupported (allowed: array, ndjson)",
+        )
+    if source.format == "xlsx":
+        sheet, header_row = options.sheet, options.header_row
+        if sheet is not None and not sheet.strip():
+            raise _fail(entity, "source.options.sheet must not be empty")
+        header_row = 1 if header_row is None else header_row
+        if header_row == 0:
+            raise _fail(entity, "source.options.header_row must be greater than 0")
+        data_row = header_row + 1 if options.data_row is None else options.data_row
+        if data_row <= header_row:
+            raise _fail(
+                entity,
+                "source.options.data_row must be greater than source.options.header_row",
+            )
+    if source.format == "xml":
+        if not (options.row_tag or "").strip():
+            raise _fail(entity, "source.options.row_tag is required for xml input")
+        for field in ("namespace", "value_tag"):
+            value = getattr(options, field)
+            if value is not None and not value.strip():
+                raise _fail(entity, f"source.options.{field} must not be empty")
+    # D7 (#584 S3, S4): Rust 0.7 only rejects these when the reader opens the file.
+    if source.format in ("csv", "tsv"):
+        separator, encoding = options.separator or "", options.encoding or "utf8"
+        if len(separator.encode()) != 1:
+            raise _fail(
+                entity, f"separator must be a single byte, got {rust_debug(separator)}"
+            )
+        if _canonical(encoding) not in ("utf8", "lossyutf8"):
+            raise _fail(entity, f"unsupported encoding: {encoding}")
+
+
+def _check_source_partitioning(entity: EntityConfig) -> None:
+    p = entity.source.partitioning
+    if p is None:
+        return
+    where, pattern = "source.partitioning", f"source.partitioning.pattern={p.pattern}"
+    tokens = PARTITION_TOKEN.findall(p.pattern)
+    if unknown := [t for t in tokens if t not in PARTITION_TOKENS]:
+        allowed = ", ".join(f"{{{t}}}" for t in PARTITION_TOKENS)
+        raise _fail(entity, f"{pattern} has unknown token {{{unknown[0]}}} (allowed: {allowed})")  # fmt: skip
+    if set("{}") & set(PARTITION_TOKEN.sub("", p.pattern)):
+        raise _fail(entity, f"{pattern} has an unmatched brace")
+    if repeated := [t for t in PARTITION_TOKENS if tokens.count(t) > 1]:
+        raise _fail(entity, f"{pattern} repeats token {{{repeated[0]}}}")
+    if not {"yyyy", "MM", "dd"} <= set(tokens):
+        raise _fail(entity, f"{pattern} must contain {{yyyy}}, {{MM}} and {{dd}}")
+    # #913: also the names after normalize_columns (the input names and the output names)
+    nc, columns = entity.schema_.normalize_columns, entity.schema_.columns
+    strategy = (nc.strategy or "snake_case") if nc and nc.enabled else None
+    names = {c.name for c in columns} | set(output_names(columns, strategy))
+    inputs = {normalize_name(c.source or c.name, strategy) for c in columns}
+    for field, name in (("date_column", p.date_column), ("batch_column", p.batch_column)):  # fmt: skip
+        if name is None:
+            continue
+        if not name.strip():
+            raise _fail(entity, f"{where}.{field} must not be empty")
+        if name in names or normalize_name(name, strategy) in inputs:
+            raise _fail(entity, f"{where}.{field}={name} is already a column in schema.columns")  # fmt: skip
+    if p.batch_column is not None and "batch" not in tokens:
+        raise _fail(entity, f"{where}.batch_column requires {{batch}} in {where}.pattern")  # fmt: skip
+    if p.batch_column == p.date_column:
+        raise _fail(entity, f"{where}.batch_column={p.batch_column} must differ from {where}.date_column")  # fmt: skip
+
+
+def _check_state(entity: EntityConfig) -> None:
+    state, archive = entity.state, entity.sink.archive
+    if state and state.path is not None and not state.path.strip():
+        raise _fail(entity, "entity.state.path must not be empty")
+    mode = entity.incremental_mode
+    if mode == "archive" and archive is None:
+        raise _fail(entity, "incremental_mode=archive requires sink.archive")
+    # ponytail: Rust's deprecation warning for `sink.archive` without a mode is not ported.
+    if archive is not None and mode in ("file", "row"):
+        raise _fail(
+            entity,
+            f"sink.archive conflicts with incremental_mode={mode}; "
+            "use incremental_mode=archive or remove sink.archive",
+        )
+
+
+def _pii_error(entity: EntityConfig, name: str, message: str) -> FloeConfigError:
+    return _fail(entity, f"pii.columns[name={name}]{message}")
+
+
+def _check_pii(entity: EntityConfig) -> None:
+    pii = entity.pii
+    if pii is None:
+        return
+    if entity.policy.severity == "abort":
+        raise _fail(
+            entity,
+            "pii: masking is not applied when policy.severity=abort because the raw "
+            "file is written to sink.rejected without DataFrame processing",
+        )
+    if entity.sink.archive is not None:
+        raise _fail(
+            entity,
+            "pii: sink.archive copies the original unmasked source file to the archive "
+            "sink; remove sink.archive or disable pii masking",
+        )
+    schema = entity.schema_
+    mismatch = schema.mismatch
+    for field, policy, fix in (
+        ("missing_columns", mismatch and mismatch.missing_columns, "fill_nulls"),
+        ("extra_columns", mismatch and mismatch.extra_columns, "ignore"),
+    ):
+        if policy == "reject_file":
+            raise _fail(
+                entity,
+                f"pii: schema.mismatch.{field}=reject_file writes the raw unmasked file "
+                "to sink.rejected before PII masking can be applied; "
+                f"use {field}={fix} instead",
+            )
+    columns = {c.name: c for c in schema.columns}
+    primary_key = set(schema.primary_key or [])
+    mode = entity.sink.write_mode
+    unique = primary_key | {c for key in schema.unique_keys or [] for c in key}
+    if (
+        schema.unique_keys is None
+    ):  # legacy `unique: true` is ignored once `unique_keys` is set
+        unique |= {c.name for c in schema.columns if c.unique is True}
+    seen: set[str] = set()
+    for col in pii.columns:
+        name, strategy = col.name, col.strategy
+        if name in seen:
+            raise _fail(entity, f"pii.columns: column name {name} is duplicated")
+        seen.add(name)
+        if name not in columns:
+            raise _fail(entity, f"pii.columns: {name} is not a schema column")
+        if strategy == "tokenize":
+            raise _pii_error(entity, name, ".strategy=tokenize is not yet supported")
+        declared = columns[name]
+        if strategy in ("hash", "redact", "mask") and _canonical(
+            declared.column_type
+        ) not in ("string", "str", "text"):
+            raise _pii_error(
+                entity,
+                name,
+                f".strategy={strategy} can only be applied to string columns "
+                f"(declared type={declared.column_type}); "
+                "use strategy=nullify or strategy=drop instead",
+            )
+        if strategy == "nullify" and declared.nullable is False:
+            raise _pii_error(
+                entity,
+                name,
+                ".strategy=nullify cannot be applied to a nullable=false column",
+            )
+        if strategy == "drop" and name in primary_key:
+            raise _pii_error(
+                entity, name, ".strategy=drop cannot be applied to a primary_key column"
+            )
+        if name in primary_key and mode in ("merge_scd1", "merge_scd2"):
+            raise _pii_error(
+                entity,
+                name,
+                f": PII strategy cannot be applied to a primary_key column with write_mode={mode}: "
+                "masking the merge key corrupts the merge predicate",
+            )
+        if name in unique and strategy in ("drop", "nullify", "redact", "mask"):
+            raise _pii_error(
+                entity,
+                name,
+                f".strategy={strategy} cannot be applied to a unique-key column: the strategy "
+                "collapses or removes values, violating uniqueness constraints",
+            )
+        if name in unique and strategy == "hash":
+            raise _pii_error(
+                entity,
+                name,
+                ".strategy=hash cannot be applied to a unique-key column: uniqueness checks "
+                "record raw values in the run report before masking, leaking unmasked PII",
+            )
+        # ponytail: Rust's warning for an unkeyed `hash` is not ported.
+        fmt = entity.sink.accepted.format
+        if strategy == "drop" and fmt in ("iceberg", "delta"):
+            raise _pii_error(
+                entity,
+                name,
+                f".strategy=drop is not supported with format={fmt} "
+                "because the sink enforces the full declared schema",
+            )
+        if strategy == "mask":
+            pattern = col.mask_pattern
+            if pattern is None:
+                raise _pii_error(entity, name, ".strategy=mask requires mask_pattern")
+            if not (token_n(pattern, "{last") or token_n(pattern, "{first")):
+                raise _pii_error(
+                    entity,
+                    name,
+                    ".mask_pattern must contain at least one token ({firstN} or {lastN})",
+                )
+
+
+def _check_storages(config: RootConfig) -> None:
+    storages, catalogs = config.storages, config.catalogs
+    names: set[str] = set()
+    if storages is not None:
+        if not storages.definitions:
+            raise FloeConfigError("storages.definitions must not be empty")
+        for d in storages.definitions:
+            if d.type not in STORAGE_TYPES:
+                raise FloeConfigError(
+                    f"storages.definitions name={d.name} type={d.type} is unsupported "
+                    f"(allowed: {', '.join(STORAGE_TYPES)})"
+                )
+            if d.type == "s3" and (d.bucket is None or d.region is None):
+                missing = "bucket" if d.bucket is None else "region"
+                raise FloeConfigError(
+                    f"storages.definitions name={d.name} requires {missing} for type s3"
+                )
+            if d.name in names:
+                raise FloeConfigError(
+                    f"storages.definitions name={d.name} is duplicated"
+                )
+            names.add(d.name)
+        if storages.default is None:
+            raise FloeConfigError("storages.default is required when storages is set")
+        if storages.default not in names:
+            raise FloeConfigError(
+                f"storages.default={storages.default} does not match any definition"
+            )
+    if catalogs is None:
+        return
+    if not catalogs.definitions:
+        raise FloeConfigError("catalogs.definitions must not be empty")
+    seen: set[str] = set()
+    for c in catalogs.definitions:
+        wh = c.warehouse_storage
+        if wh is not None:
+            if wh not in names:
+                raise FloeConfigError(
+                    f"catalogs.definitions name={c.name} warehouse_storage "
+                    f"references unknown storage {wh}"
+                )
+            if (kind := _storage_type(config, wh)) not in ("s3", "gcs"):
+                raise FloeConfigError(
+                    f"catalogs.definitions name={c.name} warehouse_storage must "
+                    f"reference s3 or gcs storage for rest catalog (got {kind})"
+                )
+        if c.name in seen:
+            raise FloeConfigError(f"catalogs.definitions name={c.name} is duplicated")
+        seen.add(c.name)
+    if catalogs.default is not None and catalogs.default not in seen:
+        raise FloeConfigError(
+            f"catalogs.default={catalogs.default} does not match any definition"
+        )
+
+
+def check_lineage(lineage: LineageConfig, ctx: str) -> None:
+    if not lineage.url.strip():
+        raise FloeConfigError(f"{ctx}.url must not be empty")
+    if not lineage.namespace.strip():
+        raise FloeConfigError(f"{ctx}.namespace must not be empty")
+    if lineage.max_failures == 0:
+        raise FloeConfigError(f"{ctx}.max_failures must be at least 1")
+
+
+def _check_report_lineage(config: RootConfig) -> None:
+    report, lineage = config.report, config.lineage
+    name = _storage_ref(config, report.storage, "report.storage", "")
+    if _storage_type(config, name) == "local" and report.path.startswith(REMOTE_URIS):
+        raise FloeConfigError(f"report.path must be a local path (got {report.path})")
+    # D7 (#584 S5): Rust 0.7 only rejects an unknown formatter when a run starts.
+    if report.formatter not in (None, "json", "csv", "text"):
+        raise FloeConfigError(f"unsupported report.formatter: {report.formatter}")
+    if lineage is not None:
+        check_lineage(lineage, "lineage")
+
+
+def validate_entities(config: RootConfig) -> None:
+    version = config_version(config.version)
+    if version < MIN_VERSION:
+        raise FloeConfigError(
+            f"root.version={config.version} is unsupported "
+            f"(minimum supported version: {MIN_VERSION[0]}.{MIN_VERSION[1]})"
+        )
+    if version > FLOE_VERSION:
+        raise FloeConfigError(
+            f"root.version={config.version} is unsupported "
+            f"(newer than Floe {FLOE_VERSION[0]}.{FLOE_VERSION[1]})"
+        )
+    if not config.entities:
+        raise FloeConfigError(
+            "entities list is empty (at least one entity is required)"
+        )
+    _check_storages(config)
+    _check_report_lineage(config)
+    ids: dict[str, int] = {}
+    for index, entity in enumerate(config.entities):
+        _check_source(entity, config)
+        _check_source_partitioning(entity)
+        entity = with_partition_columns(entity)
+        _check_state(entity)
+        _check_sink(entity, config)
+        validate_schema(entity, config.version)
+        _check_pii(entity)
+        for field, value in (("name", entity.name), ("domain", entity.domain)):
+            if (
+                version >= QUALIFIED_VERSION
+                and value is not None
+                and (not value or any(c in value for c in "./\\"))
+            ):
+                raise _fail(
+                    entity,
+                    f"entity.{field} must be non-empty without '.', '/' or '\\' "
+                    'when root.version >= "0.3"',
+                )
+        if entity.id in ids:
+            raise FloeConfigError(
+                f"entity.name={entity.id} is duplicated in config "
+                f"(entities[{ids[entity.id]}] and entities[{index}])"
+            )
+        ids[entity.id] = index
+    # D26 (#844): Rust 0.7's catalog-less Iceberg tables are dropped; checked last so
+    # configs Rust rejects keep its message. `_check_iceberg` resolved the catalog.
+    for entity in config.entities:
+        if entity.sink.accepted.format == "iceberg" and entity.sink.accepted.iceberg is None:  # fmt: skip
+            raise _fail(
+                entity,
+                "sink.accepted.format=iceberg requires sink.accepted.iceberg with a catalog in 0.8",
+            )
+
+
+def validate_config(config: RootConfig) -> None:
+    """Rust `config::validate_config`."""
+    validate_domains(config)
+    validate_entities(config)