floe-python 0.8.0__py3-none-any.whl

floe/glob.py

@@ -0,0 +1,300 @@
+"""Port of the Rust `glob` crate 0.3.4: `Pattern` matcher and the `glob` walker.
+
+Behaviour (error text, ordering, `**` collapsing, no silently skipped directories)
+follows the crate so a pattern selects the same files in Rust and Python.
+"""
+
+import os
+from dataclasses import dataclass
+from pathlib import PurePath
+
+from floe.errors import FloeError
+
+_SEPS = frozenset(s for s in (os.sep, os.altsep) if s)
+_MATCH, _SUB_NO, _ALL_NO = 0, 1, 2  # Match / SubPattern / EntirePattern DoesntMatch
+_DONE = -1  # the crate's usize::MAX index: already matched by fill_todo
+
+_E_WILD = "wildcards are either regular `*` or recursive `**`"
+_E_REC = "recursive wildcards must form a single path component"
+_E_RANGE = "invalid range pattern"
+
+_Spec = tuple[str, str | None]  # (char, None) or (start, end)
+_Token = tuple[str, str, tuple[_Spec, ...]]  # (kind, char, specs); kind: c ? * ** in ex
+
+
+class PatternError(FloeError):
+    def __init__(self, pos: int, msg: str) -> None:
+        super().__init__(f"Pattern syntax error near position {pos}: {msg}")
+        self.pos = pos
+        self.msg = msg
+
+
+@dataclass(frozen=True)
+class MatchOptions:
+    """Defaults are the crate's `MatchOptions::new()`."""
+
+    case_sensitive: bool = True
+    require_literal_separator: bool = False
+    require_literal_leading_dot: bool = False
+
+
+_DEFAULT = MatchOptions()
+
+
+def _specs(s: str) -> tuple[_Spec, ...]:
+    out: list[_Spec] = []
+    i = 0
+    while i < len(s):
+        if i + 3 <= len(s) and s[i + 1] == "-":
+            out.append((s[i], s[i + 2]))
+            i += 3
+        else:
+            out.append((s[i], None))
+            i += 1
+    return tuple(out)
+
+
+def _chars_eq(a: str, b: str, case_sensitive: bool) -> bool:
+    if os.name == "nt" and a in _SEPS and b in _SEPS:
+        return True
+    if not case_sensitive and a.isascii() and b.isascii():
+        return a.lower() == b.lower()
+    return a == b
+
+
+def _in_specs(specs: tuple[_Spec, ...], c: str, o: MatchOptions) -> bool:
+    for start, end in specs:
+        if end is None:
+            if _chars_eq(c, start, o.case_sensitive):
+                return True
+            continue
+        if not o.case_sensitive and c.isascii() and start.isascii() and end.isascii():
+            lo, hi = start.lower(), end.lower()
+            # only fold case when both ends are letters, as the crate does
+            if lo != lo.upper() and hi != hi.upper() and lo <= c.lower() <= hi:
+                return True
+        if start <= c <= end:
+            return True
+    return False
+
+
+class Pattern:
+    def __init__(self, pattern: str) -> None:
+        s, n, i = pattern, len(pattern), 0
+        tokens: list[_Token] = []
+        self.is_recursive = self.has_metachars = False
+        while i < n:
+            c = s[i]
+            if c == "?":
+                self.has_metachars = True
+                tokens.append(("?", "", ()))
+                i += 1
+            elif c == "*":
+                self.has_metachars = True
+                old = i
+                while i < n and s[i] == "*":
+                    i += 1
+                count = i - old
+                if count > 2:
+                    raise PatternError(old + 2, _E_WILD)
+                if count < 2:
+                    tokens.append(("*", "", ()))
+                    continue
+                if i == 2 or s[i - count - 1] in _SEPS:
+                    if i < n and s[i] in _SEPS:
+                        i += 1
+                    elif i != n:
+                        raise PatternError(i, _E_REC)
+                else:
+                    raise PatternError(old - 1, _E_REC)
+                # the crate's `tokens_len > 1` is kept: "**/**" in one string stays 2
+                if not (len(tokens) > 1 and tokens[-1][0] == "**"):
+                    self.is_recursive = True
+                    tokens.append(("**", "", ()))
+            elif c == "[":
+                self.has_metachars = True
+                if i + 4 <= n and s[i + 1] == "!":
+                    k = s.find("]", i + 3)
+                    if k >= 0:
+                        tokens.append(("ex", "", _specs(s[i + 2 : k])))
+                        i = k + 1
+                        continue
+                elif i + 3 <= n and s[i + 1] != "!":
+                    k = s.find("]", i + 2)
+                    if k >= 0:
+                        tokens.append(("in", "", _specs(s[i + 1 : k])))
+                        i = k + 1
+                        continue
+                raise PatternError(i, _E_RANGE)
+            else:
+                tokens.append(("c", c, ()))
+                i += 1
+        self.original = pattern
+        self.tokens = tokens
+
+    @staticmethod
+    def escape(s: str) -> str:
+        return "".join(f"[{c}]" if c in "?*[]" else c for c in s)
+
+    def matches_with(self, path: str, options: MatchOptions = _DEFAULT) -> bool:
+        return self._from(True, path, 0, 0, options) == _MATCH
+
+    def _from(self, follows: bool, s: str, p: int, ti: int, o: MatchOptions) -> int:
+        for t in range(ti, len(self.tokens)):
+            kind, ch, specs = self.tokens[t]
+            if kind in ("*", "**"):
+                m = self._from(follows, s, p, t + 1, o)
+                if m != _SUB_NO:
+                    return m
+                while p < len(s):
+                    c = s[p]
+                    p += 1
+                    if follows and o.require_literal_leading_dot and c == ".":
+                        return _SUB_NO
+                    follows = c in _SEPS
+                    if kind == "**" and not follows:
+                        continue
+                    if kind == "*" and o.require_literal_separator and follows:
+                        return _SUB_NO
+                    m = self._from(follows, s, p, t + 1, o)
+                    if m != _SUB_NO:
+                        return m
+                continue
+            if p >= len(s):
+                return _ALL_NO
+            c = s[p]
+            p += 1
+            is_sep = c in _SEPS
+            if kind != "c" and (
+                (o.require_literal_separator and is_sep)
+                or (follows and o.require_literal_leading_dot and c == ".")
+            ):
+                return _SUB_NO
+            if kind == "in":
+                ok = _in_specs(specs, c, o)
+            elif kind == "ex":
+                ok = not _in_specs(specs, c, o)
+            elif kind == "c":
+                ok = _chars_eq(c, ch, o.case_sensitive)
+            else:
+                ok = True
+            if not ok:
+                return _SUB_NO
+            follows = is_sep
+        return _MATCH if p >= len(s) else _SUB_NO
+
+
+_Entry = tuple[str, bool, int]  # (path, is_directory, pattern index)
+_Todo = list[_Entry | FloeError]
+
+
+def _glob_error(path: str, e: OSError) -> FloeError:
+    msg = f"{e.strerror} (os error {e.errno})" if e.errno else str(e)
+    return FloeError(f"attempting to read `{path}` resulted in an error: {msg}")
+
+
+def _is_dir(e: "os.DirEntry[str]") -> bool:
+    try:
+        return e.is_dir()
+    except OSError:
+        return False
+
+
+def _is_utf8(name: str) -> bool:
+    return not any("\udc80" <= c <= "\udcff" for c in name)  # surrogate-escaped bytes
+
+
+def _fill_todo(
+    todo: _Todo, pats: list[Pattern], idx: int, path: str, is_dir: bool, o: MatchOptions
+) -> None:
+    def add(p: str) -> None:
+        d = os.path.isdir(p)
+        if idx + 1 == len(pats):
+            todo.append((p, d, _DONE))
+        else:
+            _fill_todo(todo, pats, idx + 1, p, d, o)
+
+    pat = pats[idx]
+    if not pat.has_metachars:
+        s = pat.original
+        special = s in (".", "..")
+        nxt = s if path == "." else os.path.join(path, s)
+        if (special and is_dir) or (not special and os.path.lexists(nxt)):
+            add(nxt)
+    elif is_dir:
+        try:
+            with os.scandir(path) as it:
+                kids = [(e.name, _is_dir(e)) for e in it]
+        except OSError as e:
+            todo.append(_glob_error(path, e))
+            return
+        if o.require_literal_leading_dot:
+            kids = [k for k in kids if not k[0].startswith(".")]
+        kids.sort(key=lambda k: os.fsencode(k[0]), reverse=True)
+        todo.extend(
+            (k[0] if path == "." else os.path.join(path, k[0]), k[1], idx) for k in kids
+        )
+        # `.` and `..` only match a pattern with an explicit leading dot
+        if pat.tokens and pat.tokens[0] == ("c", ".", ()):
+            for special in (".", ".."):
+                if pat.matches_with(special, o):
+                    add(os.path.join(path, special))
+
+
+def glob(pattern: str, options: MatchOptions = _DEFAULT) -> list[str]:
+    """Paths matching `pattern` in the crate's order.
+
+    Raises `PatternError` on bad syntax and `FloeError` (the crate's `GlobError`
+    text) on the first directory that cannot be read.
+    """
+    Pattern(pattern)  # whole-pattern syntax check first, as the crate does
+    drive = os.path.splitdrive(pattern)[0]
+    root_len = len(drive) + (1 if pattern[len(drive) : len(drive) + 1] in _SEPS else 0)
+    body = pattern[root_len:]
+    if os.altsep:
+        body = body.replace(os.altsep, os.sep)
+    parts = body.split(os.sep)
+    if parts[-1] == "":
+        parts.pop()
+    pats = [Pattern(c) for c in parts]
+    if root_len == len(pattern):
+        pats.append(Pattern(""))
+    require_dir = pattern[-1:] in _SEPS
+    scope = pattern[:root_len] or "."
+    todo: _Todo = []
+    _fill_todo(todo, pats, 0, scope, os.path.isdir(scope), options)
+    out: list[str] = []
+    last = len(pats) - 1
+    while todo:
+        item = todo.pop()
+        if isinstance(item, FloeError):
+            raise item
+        path, is_dir, idx = item
+        if idx == _DONE:
+            if not require_dir or is_dir:
+                out.append(path)
+            continue
+        if pats[idx].is_recursive:
+            nxt = idx
+            while nxt + 1 < len(pats) and pats[nxt + 1].is_recursive:
+                nxt += 1
+            if is_dir:
+                _fill_todo(todo, pats, nxt, path, is_dir, options)
+                if nxt == last:
+                    out.append(path)
+                    continue
+            elif nxt == last:
+                continue
+            idx = nxt + 1
+        name = PurePath(path).name
+        # the crate skips non-UTF-8 names
+        if name in ("", "..") or not _is_utf8(name):
+            continue
+        if not pats[idx].matches_with(name, options):
+            continue
+        if idx == last:
+            if not require_dir or is_dir:
+                out.append(path)
+        else:
+            _fill_todo(todo, pats, idx + 1, path, is_dir, options)
+    return out

floe/iceberg/catalog.py

@@ -0,0 +1,345 @@
+"""Polaris REST catalog access, ported from `io/write/iceberg/rest.rs` and `iceberg/schema.rs`."""
+
+import os
+import re
+from collections.abc import Generator, Sequence
+from contextlib import contextmanager
+from urllib.parse import urlsplit
+
+import duckdb
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.partitioning import PartitionField, PartitionSpec
+from pyiceberg.schema import Schema
+from pyiceberg.table import Table
+from pyiceberg.transforms import parse_transform
+from pyiceberg.types import NestedField
+
+from floe.config.storage import CatalogDefinition, StorageDefinition
+from floe.config.types import SchemaEvolutionMode
+from floe.errors import FloeConfigError, FloeRunError
+
+# (column, transform) pairs, as in the entity's `sink.accepted.partition_spec`;
+# a bucket is `bucket[N]` (PyIceberg's notation) with N its `num_buckets` (D12)
+PartitionFields = Sequence[tuple[str, str]]
+
+_TRANSFORMS = frozenset({"identity", "year", "month", "day", "hour"})
+# first id of a partition field, as in the Iceberg spec
+_PARTITION_FIELD_ID_START = 1000
+DUCKDB_CATALOG = "floe_catalog"
+
+
+def transform_name(field: PartitionField) -> str:
+    return str(field.transform)  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] -- PyIceberg leaves Transform unparametrized
+
+
+@contextmanager
+def run_error(context: str) -> Generator[None]:
+    try:
+        yield
+    except Exception as e:  # PyIceberg has no common base exception
+        raise FloeRunError(f"{context}: {e}") from e
+
+
+def _expand_env_part(part: str, catalog: str) -> str:
+    if not part.startswith("${"):
+        return part
+    if not part.endswith("}"):
+        raise FloeRunError(
+            f"rest iceberg catalog {catalog} credential has unclosed env placeholder"
+        )
+    name = part[2:-1]
+    if not name or "{" in name or "}" in name:
+        raise FloeRunError(
+            f"rest iceberg catalog {catalog} credential has invalid env placeholder"
+        )
+    value = os.environ.get(name)
+    if value is None:
+        raise FloeRunError(
+            f"rest iceberg catalog {catalog} credential references env var {name} which is not set"
+        )
+    return value
+
+
+def _credential_props(catalog: str, credential: str) -> dict[str, str]:
+    """`token:<v>` is a bearer token; `client_credentials:<id>:<secret>` and a plain value are OAuth2."""
+    if "${" in credential:
+        credential = ":".join(
+            _expand_env_part(p, catalog) for p in credential.split(":")
+        )
+    if credential.startswith("token:"):
+        return {"token": credential.removeprefix("token:")}
+    return {"credential": credential.removeprefix("client_credentials:")}
+
+
+def open_catalog(
+    definition: CatalogDefinition, storage: StorageDefinition | None = None
+) -> Catalog:
+    """Credentials for S3 come from the environment; the catalog never vends them (as in Rust)."""
+    props = {
+        "type": "rest",
+        "uri": definition.uri,
+        "header.X-Iceberg-Access-Delegation": "",
+    }
+    if definition.warehouse is not None:
+        props["warehouse"] = definition.warehouse
+    if definition.credential is not None:
+        props.update(
+            _credential_props(definition.name, definition.credential.get_secret_value())
+        )
+    if definition.oauth2_server_uri is not None:
+        props["oauth2-server-uri"] = definition.oauth2_server_uri
+    if definition.scope is not None:
+        props["scope"] = definition.scope
+    if storage is not None:
+        if storage.endpoint is not None:
+            props["s3.endpoint"] = storage.endpoint
+        if storage.region is not None:
+            props["s3.region"] = storage.region
+        if storage.path_style_access is not None:
+            props["s3.force-virtual-addressing"] = str(
+                not storage.path_style_access
+            ).lower()
+    with run_error("rest catalog init failed"):
+        return load_catalog(definition.name, **props)
+
+
+def _sql_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _sql_options(options: dict[str, str]) -> str:
+    return ", ".join(f"{k} {_sql_literal(v)}" for k, v in options.items())
+
+
+def _duckdb_setup(
+    definition: CatalogDefinition, storage: StorageDefinition | None
+) -> list[str]:
+    """Secrets, then the ATTACH; S3 credentials come from the AWS chain, as for PyIceberg."""
+    statements: list[str] = []
+    auth: dict[str, str] = {}
+    if definition.credential is not None:
+        props = _credential_props(
+            definition.name, definition.credential.get_secret_value()
+        )
+        if "token" in props:
+            auth["TOKEN"] = props["token"]
+        else:
+            client_id, colon, secret = props["credential"].partition(":")
+            if (
+                not colon
+            ):  # PyIceberg sends it as a secret alone; DuckDB requires a client id
+                raise FloeConfigError(
+                    f"rest iceberg catalog {definition.name} credential needs client_id:client_secret "
+                    "for merges (DuckDB has no secret-only OAuth2)"
+                )
+            auth |= {
+                "CLIENT_ID": client_id,
+                "CLIENT_SECRET": secret,
+                "OAUTH2_SERVER_URI": definition.oauth2_server_uri
+                or f"{definition.uri.rstrip('/')}/v1/oauth/tokens",
+            }
+            if definition.scope is not None:
+                auth["OAUTH2_SCOPE"] = definition.scope
+    if storage is not None and storage.type == "s3":
+        s3 = {"PROVIDER": "credential_chain"}
+        if storage.region is not None:
+            s3["REGION"] = storage.region
+        if storage.endpoint is not None:
+            endpoint = urlsplit(storage.endpoint)
+            s3["ENDPOINT"] = endpoint.netloc or endpoint.path
+            s3["USE_SSL"] = str(endpoint.scheme != "http").lower()
+        if storage.path_style_access:
+            s3["URL_STYLE"] = "path"
+        statements.append(f"CREATE SECRET (TYPE s3, {_sql_options(s3)})")
+    # no access delegation, as in `open_catalog`
+    attach = {"TYPE": "iceberg", "ENDPOINT": definition.uri, "ACCESS_DELEGATION_MODE": "none"}  # fmt: skip
+    if auth:
+        statements.append(f"CREATE SECRET floe_auth (TYPE iceberg, {_sql_options(auth)})")  # fmt: skip
+        attach["SECRET"] = "floe_auth"
+    else:
+        attach["AUTHORIZATION_TYPE"] = "none"
+    warehouse = _sql_literal(definition.warehouse or "")
+    statements.append(f"ATTACH {warehouse} AS {DUCKDB_CATALOG} ({_sql_options(attach)})")  # fmt: skip
+    return statements
+
+
+def open_duckdb(
+    definition: CatalogDefinition, storage: StorageDefinition | None = None
+) -> duckdb.DuckDBPyConnection:
+    """DuckDB with the catalog attached as `DUCKDB_CATALOG`."""
+    statements = _duckdb_setup(definition, storage)
+    with run_error("duckdb catalog attach failed"):
+        # ponytail: DuckDB's default memory_limit and spill dir; set the ADR's limits when the run wires merges
+        con = duckdb.connect()
+        for statement in statements:
+            con.execute(statement)
+    return con
+
+
+def build_partition_spec(
+    schema: Schema, fields: PartitionFields, entity: str
+) -> PartitionSpec:
+    spec_fields: list[PartitionField] = []
+    for index, (column, transform) in enumerate(fields):
+        column = column.strip()
+        try:
+            source = schema.find_field(column)
+        except ValueError:
+            raise FloeRunError(
+                f"entity.name={entity} iceberg partition_spec column {column} was not found in runtime schema"
+            ) from None
+        transform = transform.strip().lower()
+        bucket = re.fullmatch(r"bucket\[[1-9][0-9]*\]", transform)
+        if bucket is None and transform not in _TRANSFORMS:
+            raise FloeRunError(
+                f"entity.name={entity} iceberg partition_spec column {column} has unsupported runtime transform {transform}"
+            )
+        spec_fields.append(
+            PartitionField(
+                source_id=source.field_id,
+                field_id=_PARTITION_FIELD_ID_START + index,
+                transform=parse_transform(transform),
+                name=column
+                if transform == "identity"
+                else f"{column}_{'bucket' if bucket else transform}",
+            )
+        )
+    return PartitionSpec(*spec_fields)
+
+
+def _describe_field(field: NestedField) -> str:
+    required = "required" if field.required else "optional"
+    return f"{field.name}:{field.field_type}:{required}"
+
+
+def check_schema_matches(existing: Schema, expected: Schema, entity: str) -> None:
+    """Rust has no Iceberg schema evolution: any difference fails (see the questions on #650)."""
+    if len(existing.fields) != len(expected.fields):
+        raise FloeRunError(
+            f"entity.name={entity} iceberg schema evolution is not supported "
+            f"(column count differs: existing={len(existing.fields)}, incoming={len(expected.fields)})"
+        )
+    for index, (old, new) in enumerate(zip(existing.fields, expected.fields)):
+        if (old.name, old.required, old.field_type) != (
+            new.name,
+            new.required,
+            new.field_type,
+        ):
+            raise FloeRunError(
+                f"entity.name={entity} iceberg schema evolution is not supported "
+                f"(column {index} differs: existing={_describe_field(old)} incoming={_describe_field(new)})"
+            )
+
+
+def additive_columns(
+    existing: Schema, expected: Schema, entity: str
+) -> list[NestedField]:
+    """Rust's Delta rules (`merge/shared.rs`) for `add_columns`: new columns only."""
+    incoming = {f.name: f for f in expected.fields}
+    incompatible: list[str] = []
+    for old in existing.fields:
+        new = incoming.get(old.name)
+        if new is None:
+            incompatible.append(f"missing existing column {old.name}")
+            continue
+        if new.field_type != old.field_type:
+            incompatible.append(
+                f"column {old.name} type changed from {old.field_type} to {new.field_type}"
+            )
+        if old.required and not new.required:
+            incompatible.append(
+                f"column {old.name} nullability changed from non-nullable to nullable"
+            )
+    if incompatible:
+        raise FloeRunError(
+            f"entity.name={entity} iceberg schema evolution failed: add_columns supports additive changes only; "
+            f"incompatible changes detected: {'; '.join(incompatible)}"
+        )
+    known = {f.name for f in existing.fields}
+    return [f for f in expected.fields if f.name not in known]
+
+
+def _describe_spec(spec: PartitionSpec) -> str:
+    if not spec.fields:
+        return "unpartitioned"
+    return ",".join(
+        f"{f.source_id}:{f.name}:{transform_name(f).capitalize()}" for f in spec.fields
+    )
+
+
+def _key(spec: PartitionSpec) -> list[tuple[int, str, str]]:
+    return [(f.source_id, f.name, transform_name(f)) for f in spec.fields]
+
+
+def check_partition_spec_matches(
+    existing: PartitionSpec, expected: PartitionSpec | None, entity: str
+) -> None:
+    if expected is None:
+        return
+    if _key(existing) != _key(expected):
+        raise FloeRunError(
+            f"entity.name={entity} iceberg partition spec evolution is not supported "
+            f"(existing={_describe_spec(existing)} incoming={_describe_spec(expected)})"
+        )
+
+
+def ensure_table(
+    catalog: Catalog,
+    namespace: str,
+    name: str,
+    schema: Schema,
+    entity: str,
+    location: str | None = None,
+    partition_fields: PartitionFields | None = None,
+    schema_evolution: SchemaEvolutionMode = "strict",
+) -> Table:
+    """Create the namespace and table if missing; otherwise check the table matches (Rust's append path).
+
+    With `add_columns`, columns new in `schema` are added as optional (D13); partition changes stay rejected.
+    """
+    ident = (namespace,)
+    with run_error("rest catalog namespace_exists check failed"):
+        namespace_exists = catalog.namespace_exists(ident)
+    if not namespace_exists:
+        with run_error("rest catalog namespace create failed"):
+            catalog.create_namespace(ident)
+    identifier = (namespace, name)
+    with run_error("rest catalog table_exists check failed"):
+        exists = catalog.table_exists(identifier)
+    if exists:
+        with run_error("rest catalog load_table failed"):
+            table = catalog.load_table(identifier)
+        added: list[NestedField] = []
+        if schema_evolution == "add_columns":
+            added = additive_columns(table.schema(), schema, entity)
+            # field ids of `schema` may differ from the table's once columns are added
+            check_schema = table.schema()
+        else:
+            check_schema_matches(table.schema(), schema, entity)
+            check_schema = schema
+        check_partition_spec_matches(
+            table.spec(),
+            None
+            if partition_fields is None
+            else build_partition_spec(check_schema, partition_fields, entity),
+            entity,
+        )
+        if added:
+            with run_error("rest catalog update_schema failed"):
+                with table.update_schema() as update:
+                    for field in added:
+                        update.add_column(field.name, field.field_type, required=False)
+                table.refresh()
+        return table
+    spec = (
+        None
+        if partition_fields is None
+        else build_partition_spec(schema, partition_fields, entity)
+    )
+    with run_error("rest catalog create_table failed"):
+        return catalog.create_table(  # pyright: ignore[reportUnknownMemberType] -- PyIceberg's `schema` is `Schema | Unknown`
+            identifier,
+            schema,
+            location=location,
+            partition_spec=spec if spec is not None else PartitionSpec(),
+        )