floe-python 0.8.0__py3-none-any.whl

floe/add_entity.py

@@ -0,0 +1,516 @@
+"""`floe add-entity`, ported from `crates/floe-core/src/add_entity.rs`: infer an entity from a sample file."""
+
+import io
+import json
+import math
+import os
+import re
+import secrets
+import stat
+import tempfile
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import NamedTuple, cast
+
+import polars as pl
+
+from floe import __version__
+from floe.config.include import glob_base
+from floe.config.parse import parse_config
+from floe.config.project import V0_3, config_version, validate_domains
+from floe.config.types import SourceOptions
+from floe.config.validate import validate_entities
+from floe.config.yaml import _FLOAT, load_yaml, parse_yaml, rust_debug  # pyright: ignore[reportPrivateUsage] -- the YAML float grammar the emitter must quote
+from floe.errors import FloeConfigError, FloeError, FloeIoError
+from floe.glob import MatchOptions, Pattern, PatternError, glob
+
+SAMPLE_ROWS = 200
+Hash = dict[object, object]
+_COMPONENT_WISE = MatchOptions(
+    require_literal_separator=True
+)  # as the glob walker: `*` stays in one component
+
+
+def _os(e: OSError) -> str:
+    return f"{e.strerror} (os error {e.errno})"
+
+
+def _file_uri_path(input: str) -> str:
+    url = urllib.parse.urlparse(input)
+    if url.netloc not in ("", "localhost"):
+        raise FloeConfigError(f"invalid file:// URI for add-entity input: {input}")
+    return urllib.request.url2pathname(url.path)
+
+
+def _entity_name(input: str) -> str:
+    scheme, sep, _ = input.partition("://")
+    if sep and scheme.lower() != "file":
+        raw = input.rstrip("/").rsplit("/", 1)[-1].split("?")[0]
+    else:
+        path = Path(_file_uri_path(input) if sep else input)
+        raw = path.stem if path.name not in ("", "..") else "entity"
+    name = re.sub("[^A-Za-z0-9]+", "_", raw).strip("_").lower()
+    if not name:
+        raise FloeConfigError(f"failed to derive entity name from input: {input}")
+    return name
+
+
+def _local_path(input: str) -> str:
+    scheme, sep, _ = input.partition("://")
+    if sep:
+        if scheme.lower() != "file":
+            raise FloeConfigError(
+                f"remote URI inference is not supported yet for add-entity input: {input}"
+            )
+        return _file_uri_path(input)
+    path = os.path.join(os.getcwd(), input)
+    try:
+        mode = os.stat(path).st_mode
+    except OSError as e:
+        raise FloeIoError(f"failed to access input at {path}: {_os(e)}") from e
+    if stat.S_ISDIR(mode):
+        raise FloeConfigError(
+            f"add-entity expects a file input for schema inference, got directory: {path}"
+        )
+    return path
+
+
+def _sniff_separator(path: str) -> str:
+    """Rust `sniff_delimiter`: the candidate most consistent over the first non-blank lines."""
+    lines: list[str] = []
+    try:
+        with open(path, "rb") as f:
+            for _, raw in zip(range(12), f, strict=False):
+                if line := raw.decode().strip():
+                    lines.append(line)
+                if len(lines) >= 6:
+                    break
+    except (OSError, UnicodeDecodeError):
+        return ","
+    best: tuple[int, str] | None = None
+    for candidate in ",;\t|":
+        counts = [line.count(candidate) for line in lines]
+        if not counts or counts[0] == 0:
+            continue
+        bonus = 10_000 if min(counts) > 0 and max(counts) == min(counts) else 0
+        score = bonus + min(counts) * 100 + sum(counts)
+        if best is None or score > best[0]:
+            best = (score, candidate)
+    return best[1] if best else ","
+
+
+def _floe_type(dtype: pl.DataType) -> str:
+    # D27b (#619): a `time` column becomes `string`; Rust writes `time`, which 0.8 rejects (D10)
+    for kind, name in ((pl.Boolean, "boolean"), (pl.Date, "date"), (pl.Datetime, "datetime")):  # fmt: skip
+        if dtype == kind:
+            return name
+    return "number" if dtype.is_numeric() else "string"
+
+
+def _json_sample(path: str) -> tuple[str, list[dict[str, object]]]:
+    """Rust's JSON sampling: the `json_mode` and the first 200 top-level objects."""
+    try:
+        text = Path(path).read_bytes().decode()
+    except OSError as e:
+        raise FloeIoError(f"failed to read json at {path}: {_os(e)}") from e
+    except UnicodeDecodeError as e:
+        raise FloeIoError(f"failed to read json at {path}: stream did not contain valid UTF-8") from e  # fmt: skip
+    if not (head := text.lstrip()[:1]):
+        raise FloeConfigError(f"json input {path} is empty")
+    if head == "[":
+        try:
+            rows = json.loads(text)
+        except json.JSONDecodeError as e:
+            raise FloeIoError(f"json parse error: {e}") from e
+        if not isinstance(rows, list):
+            raise FloeConfigError("expected json array at root")
+        rows = cast("list[object]", rows)[:SAMPLE_ROWS]
+        if not all(isinstance(row, dict) for row in rows):
+            raise FloeConfigError("expected top-level json objects inside array")
+        return "array", cast("list[dict[str, object]]", rows)
+    objects: list[dict[str, object]] = []
+    for n, line in enumerate(text.split("\n"), 1):
+        if len(objects) >= SAMPLE_ROWS:
+            break
+        if not (line := line.strip()):
+            continue
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError as e:
+            raise FloeIoError(f"json parse error at line {n}: {e}") from e
+        if not isinstance(row, dict):
+            raise FloeConfigError(f"expected top-level json object at line {n}")
+        objects.append(cast("dict[str, object]", row))
+    return "ndjson", objects
+
+
+def _read_json(rows: list[dict[str, object]]) -> pl.DataFrame:
+    """Polars' schema for the sampled objects; a key Polars can't type (mixed kinds) is read as text."""
+    text = "\n".join(map(json.dumps, rows))
+
+    def read(strings: list[str]) -> pl.DataFrame:
+        overrides = dict.fromkeys(strings, pl.String())
+        return pl.read_ndjson(io.StringIO(text), infer_schema_length=None, schema_overrides=overrides)  # fmt: skip
+
+    try:
+        return read([])
+    except pl.exceptions.PolarsError:
+        keys = list(dict.fromkeys(k for row in rows for k in row))
+        mixed: list[str] = []
+        for key in keys:
+            try:
+                read([k for k in keys if k != key])
+            except pl.exceptions.PolarsError:
+                mixed.append(key)
+        return read(mixed)
+
+
+def _temporal(column: pl.Series) -> pl.DataType:
+    """`Date` or `Datetime` when Polars parses every value of a text column, else `String`."""
+    values = column.drop_nulls()
+    for parse in (values.str.to_date, values.str.to_datetime):
+        try:
+            parsed = parse(strict=False)
+        except pl.exceptions.PolarsError:  # no format fits the first value
+            continue
+        if len(values) and parsed.null_count() == 0:
+            return parsed.dtype
+    return pl.String()
+
+
+def _columns(fmt: str, path: str) -> tuple[Hash | None, list[Hash], list[str]]:
+    """(source options, schema columns, notes), from Polars' own schema inference."""
+    notes: list[str] = []
+    if fmt == "csv":
+        separator = _sniff_separator(path)
+        try:
+            df = pl.read_csv(
+                path, separator=separator, n_rows=SAMPLE_ROWS, infer_schema_length=SAMPLE_ROWS,
+                null_values=SourceOptions.model_fields["null_values"].default, glob=False,
+            )  # fmt: skip
+        except (OSError, pl.exceptions.PolarsError) as e:
+            raise FloeIoError(f"csv schema inference failed: {e}") from e
+        nullable = {c.name: df.height == 0 or c.null_count() > 0 for c in df.iter_columns()}  # fmt: skip
+        schema, options = df.schema, cast(Hash, {"separator": separator})
+    elif fmt == "parquet":
+        try:  # a file object: Polars would expand glob characters in a path
+            with open(path, "rb") as f:
+                schema = pl.read_parquet_schema(f)
+        except OSError as e:
+            raise FloeIoError(f"failed to open parquet at {path}: {_os(e)}") from e
+        except pl.exceptions.PolarsError as e:
+            raise FloeIoError(f"failed to read parquet schema at {path}: {e}") from e
+        nullable, options = dict.fromkeys(schema, True), None
+    elif fmt == "json":  # D27a (#619): Polars types the values; Rust walks them by hand
+        mode, rows = _json_sample(path)
+        if not rows:
+            raise FloeConfigError(f"no json objects found in {path}")
+        df = _read_json(rows)
+        nullable = {c.name: c.null_count() > 0 for c in df.iter_columns()}
+        schema = {n: _temporal(df[n]) if t == pl.String else t for n, t in sorted(df.schema.items())}  # fmt: skip
+        options = cast(Hash, {"json_mode": mode})
+        notes.append(f"json inference sampled {len(rows)} row(s) in {mode} mode")
+        if nested := sorted({k for row in rows for k, v in row.items() if isinstance(v, (list, dict))}):  # fmt: skip
+            notes.append(f"nested JSON values inferred as string for top-level key(s): {', '.join(nested)}")  # fmt: skip
+    else:
+        raise FloeConfigError(f"unsupported add-entity format: {fmt} (allowed: csv, json, parquet)")  # fmt: skip
+    return (
+        options,
+        [
+            {
+                "name": n,
+                "source": n,
+                "type": _floe_type(t),
+                "nullable": nullable[n],
+                "unique": False,
+            }
+            for n, t in schema.items()
+        ],
+        notes,
+    )
+
+
+def infer_entity(
+    input: str, fmt: str | None, name: str | None, domain: str | None,
+    source_ref: str | None, resource: str | None,
+) -> tuple[Hash, str, list[str]]:  # fmt: skip
+    """The entity generated for a sample file, its format and notes (Rust `infer_entity_from_input`)."""
+    name = _entity_name(input) if name is None else name
+    if not name.strip():
+        raise FloeConfigError("entity name must not be empty")
+    path = _local_path(input)
+    if fmt is None:
+        # `.csv` alone has no extension, as in Rust
+        fmt = os.path.splitext(path)[1][1:].lower()
+        if fmt not in ("csv", "json", "parquet"):
+            raise FloeConfigError(
+                f"could not infer add-entity format from input {input} "
+                "(supported extensions: .csv, .json, .parquet); use --format"
+            )
+    options, columns, notes = _columns(fmt, path)
+    if not columns:
+        raise FloeConfigError(f"no columns inferred from {path}")
+    if source_ref is not None:
+        source: Hash = {"ref": source_ref, "resource": resource}
+    else:
+        persisted = path if input.partition("://")[1] else input
+        source = {"format": fmt, "path": persisted}
+        if options is not None:
+            source["options"] = options
+    entity: Hash = {
+        "name": name,
+        **({} if domain is None else {"domain": domain}),
+        "source": source,
+        "sink": {
+            "accepted": {"format": "parquet", "path": f"out/accepted/{name}/"},
+            "rejected": {"format": "csv", "path": f"out/rejected/{name}/"},
+        },
+        "policy": {"severity": "reject"},
+        "schema": {
+            "mismatch": {"missing_columns": "reject_file", "extra_columns": "ignore"},
+            "columns": columns,
+        },
+    }
+    return entity, fmt, notes
+
+
+class Added(NamedTuple):
+    name: str
+    format: str
+    columns: int
+    output: str
+    rendered: str | None  # the YAML of a dry run
+    notes: list[str]
+
+
+_WORDS = {"true", "false", "True", "False", "TRUE", "FALSE", "null", "Null", "NULL", "~", "y", "Y", "n", "N"} | {
+    "yes", "Yes", "YES", "no", "No", "NO", "on", "On", "ON", "off", "Off", "OFF"
+}  # fmt: skip
+_SPECIAL = re.compile("[:{}\\[\\],#`\"'\\\\\x00-\x06\t\n\r\x0e-\x1a\x1c-\x1f]")
+_ESCAPES = {i: f"\\u{i:04x}" for i in (*range(0x20), 0x7F)} | {
+    0x22: '\\"', 0x5C: "\\\\", 0x08: "\\b", 0x09: "\\t", 0x0A: "\\n", 0x0C: "\\f", 0x0D: "\\r"
+}  # fmt: skip
+
+
+def _scalar(value: object) -> str:
+    """yaml-rust2 `YamlEmitter::emit_node` for a scalar."""
+    if isinstance(value, str):
+        quote = (
+            not value or value[0] == " " or value[-1] == " " or value[0] in "&*?|-<>=!%@"
+            or bool(_SPECIAL.search(value)) or value in _WORDS or value.startswith((".", "0x"))
+            or bool(_FLOAT.fullmatch(value)) or value.lstrip("+-").lower() in ("inf", "infinity", "nan")
+        )  # fmt: skip
+        return f'"{value.translate(_ESCAPES)}"' if quote else value
+    if isinstance(value, bool):
+        return str(value).lower()
+    if isinstance(value, float):
+        # ponytail: yaml-rust2 re-emits a float's source text (`1.50`); this writes `1.5`
+        if math.isnan(value):
+            return ".nan"
+        return {math.inf: ".inf", -math.inf: "-.inf"}.get(value, repr(value))
+    return "~" if value is None else str(value)
+
+
+def _node(value: object, level: int) -> str:
+    if isinstance(value, list):
+        items = cast("list[object]", value)
+        if not items:
+            return "[]"
+        sep = "\n" + "  " * (level + 1)
+        return sep.join("-" + _val(True, x, level + 1) for x in items)
+    if isinstance(value, dict):
+        pairs = cast(Hash, value)
+        if not pairs:
+            return "{}"
+        sep = "\n" + "  " * (level + 1)
+        return sep.join(_scalar(k) + ":" + _val(False, v, level + 1) for k, v in pairs.items())  # fmt: skip
+    return _scalar(value)
+
+
+def _val(inline: bool, value: object, level: int) -> str:
+    text = _node(value, level)
+    block = not inline and text not in ("[]", "{}") and isinstance(value, (list, dict))
+    return ("\n" + "  " * (level + 1) if block else " ") + text
+
+
+def _render(doc: object) -> str:
+    """yaml-rust2 `YamlEmitter::dump` without the `---` line, as Rust's `render_yaml`."""
+    return _node(doc, -1) + "\n"
+
+
+def _included_domain(
+    config: str, root: object, domain: str
+) -> tuple[str, object] | None:
+    """The directory and `defaults` of the included domain file named `domain`."""
+    include = cast(Hash, root).get("include") if isinstance(root, dict) else None
+    patterns = cast(Hash, include).get("domains") if isinstance(include, dict) else None
+    if not isinstance(patterns, list):
+        return None
+    parent = os.path.dirname(config)
+    base = glob_base(Path(parent)) if parent else ""
+    for pattern in cast("list[object]", patterns):
+        if not isinstance(pattern, str):
+            continue
+        try:
+            matches = glob(os.path.join(base, pattern))
+        except PatternError as e:
+            raise FloeConfigError(f"invalid include.domains pattern {rust_debug(pattern)} in {config}: {e}") from e  # fmt: skip
+        except FloeError as e:
+            raise FloeConfigError(f"failed to match include.domains pattern {rust_debug(pattern)} in {config}: {e}") from e  # fmt: skip
+        for file in matches:
+            try:
+                docs: list[object] = load_yaml(Path(file))
+            except FloeIoError as e:
+                raise FloeIoError(f"failed to read domain file: {e}") from e
+            except FloeError as e:
+                raise FloeConfigError(f"failed to parse domain file {file}: {e}") from e
+            doc = docs[0] if docs else None
+            if isinstance(doc, dict) and cast(Hash, doc).get("name") == domain:
+                return os.path.dirname(file), cast(Hash, doc).get("defaults", {})
+    return None
+
+
+def _matches_domain_include(config: str, root: object, path: str) -> bool:
+    """Whether `assemble_includes` would load `path` as a domain file."""
+    include = cast(Hash, root).get("include") if isinstance(root, dict) else None
+    patterns = cast(Hash, include).get("domains") if isinstance(include, dict) else None
+    base = glob_base(Path(os.path.dirname(config)))
+    return any(
+        Pattern(os.path.join(base, p)).matches_with(path, _COMPONENT_WISE)
+        for p in cast("list[object]", patterns)
+        if isinstance(p, str)
+    )
+
+
+def _strip_defaults(entity: Hash, defaults: Hash) -> None:
+    for key, default in defaults.items():
+        if key not in entity:
+            continue
+        value = entity[key]
+        if isinstance(value, dict) and isinstance(default, dict):
+            _strip_defaults(cast(Hash, value), cast(Hash, default))
+            if value:
+                continue
+        del entity[key]
+
+
+def _validate(config: Path, extra_entity: Hash | None = None) -> None:
+    root = parse_config(config, extra_entity=extra_entity)
+    validate_domains(root)
+    validate_entities(root)
+
+
+def _validate_text(text: str, target: str) -> None:
+    """Validate `text` as a config placed next to `target` (Rust `validate_generated_config`)."""
+    try:
+        fd, tmp = tempfile.mkstemp(prefix=".tmp", dir=os.path.dirname(target) or ".")
+    except OSError:
+        fd, tmp = tempfile.mkstemp(prefix=".tmp")
+    try:
+        with os.fdopen(fd, "wb") as f:
+            f.write(text.encode())
+        _validate(Path(tmp))
+    finally:
+        os.unlink(tmp)
+
+
+def _write_new(path: str, text: str) -> None:
+    """Stage a temp file and hard-link it in, so the entity file appears whole and is never clobbered."""
+    tmp = os.path.join(
+        os.path.dirname(path), f".floe-add-entity-{secrets.token_hex(6)}.yml"
+    )
+    try:
+        fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666)
+    except OSError as e:
+        raise FloeIoError(f"failed to stage entity file: {_os(e)}") from e
+    try:  # a partial `.yml` left behind would be loaded as an entity
+        try:
+            with os.fdopen(fd, "wb") as f:
+                f.write(text.encode())
+        except OSError as e:
+            raise FloeIoError(f"failed to stage entity file: {_os(e)}") from e
+        try:
+            os.link(tmp, path)
+        except OSError as e:
+            raise FloeIoError(f"failed to write entity file: {_os(e)}") from e
+    finally:
+        os.unlink(tmp)
+
+
+def add_entity(
+    config: str, input: str, fmt: str | None, name: str | None, domain: str | None,
+    source_ref: str | None, resource: str | None, output: str | None, dry_run: bool,
+) -> Added:  # fmt: skip
+    if (source_ref is None) != (resource is None):
+        raise FloeConfigError("--source and --resource must be provided together")
+    entity, fmt, notes = infer_entity(input, fmt, name, domain, source_ref, resource)
+    name = cast(str, entity["name"])
+    columns = len(cast("list[object]", cast(Hash, entity["schema"])["columns"]))
+
+    exists = os.path.exists(config)
+    try:  # D21 (#619): a new config targets this Floe major.minor
+        docs: list[object] = load_yaml(Path(config)) if exists else [{"version": ".".join(__version__.split(".")[:2]), "entities": []}]  # fmt: skip
+    except FloeIoError as e:
+        raise FloeIoError(f"failed to read config at {config}: {e}") from e
+    included = None
+    if output is None and domain is not None and exists:
+        included = _included_domain(config, docs[0] if docs else None, domain)
+    if included is not None:
+        directory, defaults = included
+        if "/" in name or "\\" in name:
+            raise FloeConfigError(f"entity name cannot contain a path separator for an included domain at {directory}")  # fmt: skip
+        output = os.path.join(directory, f"{name}.yml")
+        if os.path.exists(output):
+            raise FloeIoError(f"refusing to overwrite existing entity file at {output}")
+        if _matches_domain_include(config, docs[0], output):
+            raise FloeConfigError(f"entity file {output} would also match an include.domains pattern and load as a domain; use --output or a different name")  # fmt: skip
+        if isinstance(defaults, dict):
+            # Rust first drops `name`, `schema.columns` and `primary_key` from the defaults; D7 rejects those
+            _strip_defaults(entity, cast(Hash, defaults))
+            entity.pop("domain", None)
+        rendered = _render(entity)
+        try:
+            # the reloaded bytes, as Rust: a name like `0o123` is written bare and reloads as an integer
+            [reloaded] = parse_yaml(rendered)
+            _validate(Path(config), {**cast(Hash, reloaded), "domain": domain})
+        except FloeError as e:
+            raise FloeConfigError(f"generated entity file {output} failed validation: {e}") from e  # fmt: skip
+        if not dry_run:
+            _write_new(output, rendered)
+        return Added(name, fmt, columns, output, rendered if dry_run else None, notes)
+
+    if len(docs) > 1:
+        raise FloeConfigError("YAML contains multiple documents; expected one")
+    root = docs[0] if docs else None
+    if not isinstance(root, dict):
+        raise FloeConfigError("config root must be a YAML mapping")
+    root = cast(Hash, root)
+    version = root.get("version")
+    try:  # 0.3+ entity identity is (domain, name) (#464); older configs compare names only
+        qualified = isinstance(version, str) and config_version(version) >= V0_3
+    except FloeConfigError:
+        qualified = False
+    if not isinstance(entities := root.setdefault("entities", []), list):
+        raise FloeConfigError("root.entities must be a YAML sequence")
+    entities = cast("list[object]", entities)
+
+    def field(e: object, key: str) -> object:
+        value = cast(Hash, e).get(key) if isinstance(e, dict) else None
+        return value if isinstance(value, str) else None
+
+    for e in entities:
+        if field(e, "name") == name and (not qualified or field(e, "domain") == domain):
+            raise FloeConfigError(f"entity already exists: {name}")
+    entities.append(entity)
+    rendered = _render(root)
+    output = config if output is None else output
+    if not dry_run and (parent := os.path.dirname(output)):
+        os.makedirs(parent, exist_ok=True)
+    _validate_text(rendered, config if dry_run else output)
+    if not dry_run:
+        try:
+            Path(output).write_bytes(rendered.encode())
+        except OSError as e:
+            raise FloeIoError(f"failed to write config at {output}: {_os(e)}") from e
+    return Added(name, fmt, columns, output, rendered if dry_run else None, notes)

floe/checks/cast.py

@@ -0,0 +1,160 @@
+"""String to typed casts and cast-mismatch errors (Rust: `checks/cast.rs`, `io/format.rs`)."""
+
+import json
+import re
+
+import polars as pl
+
+CAST_ERROR_MESSAGE = "invalid value for target type"
+
+_DTYPES: dict[str, pl.DataType] = {
+    "int8": pl.Int8(), "int16": pl.Int16(), "int32": pl.Int32(), "int64": pl.Int64(),
+    "int": pl.Int64(), "integer": pl.Int64(), "long": pl.Int64(),
+    "uint8": pl.UInt8(), "uint16": pl.UInt16(), "uint32": pl.UInt32(), "uint64": pl.UInt64(),
+    "float32": pl.Float32(),
+    "float64": pl.Float64(), "float": pl.Float64(), "double": pl.Float64(),
+    "number": pl.Float64(),
+}  # fmt: skip
+
+_DECIMAL = re.compile(r"decimal\(\s*(\d+)\s*,\s*(\d+)\s*\)")
+
+
+def decimal_type(column_type: str) -> pl.Decimal | None:
+    """`decimal(p,s)` as an exact `pl.Decimal`, or None when `kind` is not one.
+
+    Raises ValueError for a precision outside 1..38 or a scale outside 0..precision.
+    """
+    # matched on the declared type: `_canonical` would drop the sign of `decimal(4,-2)`
+    if not (m := _DECIMAL.fullmatch(column_type.lower())):
+        return None
+    precision, scale = int(m[1]), int(m[2])
+    if not 1 <= precision <= 38 or scale > precision:
+        raise ValueError(
+            f"decimal(p,s) needs 1 <= p <= 38 and 0 <= s <= p, got {column_type}"
+        )
+    return pl.Decimal(precision, scale)
+
+
+# Polars 2.0 has no string -> date/datetime cast, so both are parsed by regex (decision D9, #619):
+# groups 1-3 year/month/day, 4-7 hour/minute/second/fraction, 8-10 offset sign/hour/minute.
+_DATE = r"\+?([0-9]{1,4})-([0-9]{1,2})-([0-9]{1,2})"
+_TIME = r"T([0-9]{1,2}):([0-9]{1,2})(?::([0-9]{1,2})(?:\.([0-9]*))?)?"
+_OFFSET = r"(?:[Zz]|([+-])([0-9]{2})(?::?([0-9]{2}))?)?"
+_DATE_RE = f"^{_DATE}$"
+_DATETIME_RE = f"^{_DATE}{_TIME}{_OFFSET}$"
+
+
+def _groups(raw: pl.Expr, pattern: str, count: int) -> list[pl.Expr]:
+    groups = raw.str.strip_chars().str.extract_groups(pattern)
+    return [groups.struct.field(str(i)) for i in range(1, count + 1)]
+
+
+def _int(field: pl.Expr) -> pl.Expr:
+    return field.cast(pl.Int32)
+
+
+def _ymd(g: list[pl.Expr]) -> pl.Expr:
+    return pl.format(
+        "{}-{}-{}", g[0].str.zfill(4), g[1].str.zfill(2), g[2].str.zfill(2)
+    )
+
+
+def _date(raw: pl.Expr) -> pl.Expr:
+    g = _groups(raw, _DATE_RE, 3)
+    return pl.when(_int(g[0]) >= 1).then(_ymd(g).str.to_date("%Y-%m-%d", strict=False))
+
+
+def _datetime(raw: pl.Expr) -> pl.Expr:
+    g = _groups(raw, _DATETIME_RE, 10)
+    second = _int(g[5]).fill_null(0)
+    # fraction truncated (not rounded) to milliseconds
+    millis = _int(g[6].str.slice(0, 3).str.pad_end(3, "0")).fill_null(0)
+    off_hour, off_minute = _int(g[8]).fill_null(0), _int(g[9]).fill_null(0)
+    off_sign = pl.when(g[7] == "-").then(-1).otherwise(1)
+    valid = (_int(g[0]) >= 1) & (second <= 60) & (off_hour <= 23) & (off_minute <= 59)
+    local = pl.format(
+        "{} {}:{}", _ymd(g), g[3].str.zfill(2), g[4].str.zfill(2)
+    ).str.to_datetime("%Y-%m-%d %H:%M", time_unit="ms", strict=False)
+    # second 60 rolls over into the next minute; the offset is subtracted to reach UTC
+    shift = pl.duration(
+        seconds=second,
+        milliseconds=millis,
+        hours=-off_sign * off_hour,
+        minutes=-off_sign * off_minute,
+        time_unit="ms",
+    )
+    utc = local + shift
+    # the 1..9999 year range applies to the final value (rollover, offset), not only the input
+    return pl.when(valid & utc.dt.year().is_between(1, 9999)).then(utc)
+
+
+def _boolean(raw: pl.Expr) -> pl.Expr:
+    value = raw.str.strip_chars().str.to_lowercase()
+    return (
+        pl.when(value.is_in(["true", "1"]))
+        .then(True)
+        .when(value.is_in(["false", "0"]))
+        .then(False)
+    )
+
+
+def cast_column(
+    name: str,
+    column_type: str,
+    cast_mode: str = "strict",
+    dtype: pl.DataType | type[pl.DataType] = pl.String,
+) -> tuple[pl.Expr, pl.Expr]:
+    """Return `(typed, cast_error_mask)` expressions over the column `name`, read as `dtype`.
+
+    A value that is present but does not parse becomes null; the mask (named `_e_cast_<name>`)
+    flags it, except in `cast_mode: coerce` and for string columns.
+
+    `dtype` is the input column's dtype. Parquet, ORC and Avro keep native types (D15): anything
+    but `pl.String` is cast natively, as Rust's `cast_df_to_schema` does. That is Polars'
+    non-strict cast, so overflow, a float outside the integer range or NaN, and a date that does
+    not fit become null (a `cast_error`), while 2.5 into an int truncates to 2 and an int into a
+    bool is `!= 0`. A pair Polars cannot cast (date or datetime into bool, a list) raises, and
+    aborts the run as in Rust. D17: unlike Rust, a float with a fraction into an integer type is a
+    `cast_error` in strict mode (2.0 is accepted); in coerce mode it truncates as in Rust.
+    Timezone-aware datetimes lose the zone and keep their UTC value.
+    """
+    raw = pl.col(name)
+    native = dtype != pl.String
+    kind = column_type.lower().replace("-", "").replace("_", "")
+    if kind in ("string", "str", "text"):
+        text = raw.cast(pl.String) if native else raw
+        if native and dtype.is_float():  # Rust writes 1e20, Polars 1e+20
+            text = text.str.replace("e+", "e", literal=True)
+        return text, pl.lit(False).alias(f"_e_cast_{name}")
+    fractional = None
+    if (exact := decimal_type(column_type)) is not None:
+        typed = raw.cast(exact, strict=False)
+        if not native:  # strict: an exponent or digits beyond the scale are a cast_error, not a silent rounding
+            fractional = raw.str.contains(
+                rf"[eE]|\.[0-9]{{{exact.scale}}}[0-9]*[1-9]"
+            ).fill_null(False)
+    elif kind in _DTYPES:
+        typed = raw.cast(_DTYPES[kind], strict=False)
+        if native and dtype.is_float() and _DTYPES[kind].is_integer():
+            # D17; NaN and inf are null already
+            fractional = (raw != raw.floor()).fill_null(False)
+    elif kind in ("boolean", "bool"):
+        typed = raw.cast(pl.Boolean, strict=False) if native else _boolean(raw)
+    elif kind == "date":
+        typed = raw.cast(pl.Date, strict=False) if native else _date(raw)
+    elif kind in ("datetime", "timestamp"):
+        typed = raw.cast(pl.Datetime("ms"), strict=False) if native else _datetime(raw)
+    else:
+        raise ValueError(f"unsupported column type: {column_type}")
+    typed = typed.alias(name)
+    mask = pl.lit(False)
+    if cast_mode != "coerce":
+        mask = raw.is_not_null() & typed.is_null()
+        if fractional is not None:
+            mask = mask | fractional
+    return typed, mask.alias(f"_e_cast_{name}")
+
+
+def json_escape(value: str) -> str:
+    """The body of a JSON string: every control character escaped (#983), non-ASCII kept raw."""
+    return json.dumps(value, ensure_ascii=False)[1:-1]