floe-python 0.8.0__py3-none-any.whl

floe/run/events.py

@@ -0,0 +1,42 @@
+"""Observer events: the shapes of `crates/floe-python/python/floe/_floe.pyi` (Rust: `run/events.rs`)."""
+
+import time
+from collections.abc import Callable
+from contextlib import suppress
+from dataclasses import dataclass
+from typing import Any, Literal, TypedDict
+
+Observer = Callable[[dict[str, Any]], None]
+
+
+@dataclass(frozen=True)
+class Emitter:
+    """Stamps `run_id` and `ts_ms`; with parallel entities the observer is called from worker threads."""
+
+    observer: Observer | None = None
+    run_id: str = ""
+
+    def __call__(self, event: str, **fields: Any) -> None:
+        if self.observer:
+            with suppress(
+                Exception
+            ):  # an observer must not abort ingestion (Rust ignores callback errors)
+                self.observer(
+                    {"event": event, "run_id": self.run_id, **fields}
+                    | {"ts_ms": time.time_ns() // 1_000_000}
+                )
+
+
+NO_EVENTS = Emitter()
+
+
+# The `RunEvent` dicts an observer receives, as typed in `_floe.pyi`.
+RunStartedEvent = TypedDict("RunStartedEvent", {"event": Literal["run_started"], "run_id": str, "config": str, "report_base": str | None, "ts_ms": int})  # fmt: skip
+EntityStartedEvent = TypedDict("EntityStartedEvent", {"event": Literal["entity_started"], "run_id": str, "name": str, "ts_ms": int})  # fmt: skip
+FileStartedEvent = TypedDict("FileStartedEvent", {"event": Literal["file_started"], "run_id": str, "entity": str, "input": str, "ts_ms": int})  # fmt: skip
+FileFinishedEvent = TypedDict("FileFinishedEvent", {"event": Literal["file_finished"], "run_id": str, "entity": str, "input": str, "status": str, "rows": int, "accepted": int, "rejected": int, "elapsed_ms": int, "ts_ms": int})  # fmt: skip
+SchemaEvolutionAppliedEvent = TypedDict("SchemaEvolutionAppliedEvent", {"event": Literal["schema_evolution_applied"], "run_id": str, "entity": str, "mode": str, "added_columns": list[str], "ts_ms": int})  # fmt: skip
+EntityFinishedEvent = TypedDict("EntityFinishedEvent", {"event": Literal["entity_finished"], "run_id": str, "name": str, "status": str, "files": int, "rows": int, "accepted": int, "rejected": int, "warnings": int, "errors": int, "ts_ms": int})  # fmt: skip
+RunFinishedEvent = TypedDict("RunFinishedEvent", {"event": Literal["run_finished"], "run_id": str, "status": str, "exit_code": int, "files": int, "rows": int, "accepted": int, "rejected": int, "warnings": int, "errors": int, "summary_uri": str | None, "ts_ms": int})  # fmt: skip
+LogEvent = TypedDict("LogEvent", {"event": Literal["log"], "run_id": str, "log_level": str, "code": str | None, "message": str, "entity": str | None, "input": str | None, "ts_ms": int})  # fmt: skip
+RunEvent = RunStartedEvent | EntityStartedEvent | FileStartedEvent | FileFinishedEvent | SchemaEvolutionAppliedEvent | EntityFinishedEvent | RunFinishedEvent | LogEvent  # fmt: skip

floe/state.py

@@ -0,0 +1,311 @@
+"""Incremental file state, ported from `state/mod.rs` (CAS claim, promote, release, renew, reset)."""
+
+import copy
+import json
+import random
+import time
+from collections.abc import Callable
+from dataclasses import asdict, dataclass
+from datetime import UTC, datetime, timedelta
+from typing import NamedTuple
+
+from floe.errors import FloeConfigError
+from floe.storage.inputs import InputFile
+from floe.storage.types import StorageClient
+
+SCHEMA_V1 = "floe.state.file-ingest.v1"
+SCHEMA_V2 = "floe.state.file-ingest.v2"
+CAS_RETRIES = 8
+_BASE_BACKOFF_MS = 50
+_MAX_BACKOFF_MS = 2_000
+CLAIM_TTL_SECONDS = 60 * 60
+
+
+@dataclass
+class FileState:
+    processed_at: str
+    size: int | None = None
+    mtime: str | None = None
+
+
+@dataclass
+class Claim:
+    run_id: str
+    acquired_at: str
+    expires_at: str
+    size: int | None = None
+    mtime: str | None = None
+
+
+@dataclass
+class EntityState:
+    entity: str
+    updated_at: str | None
+    files: dict[str, FileState]
+    claims: dict[str, Claim]
+    watermark: str | None = None  # D30: the latest partition date ingested (ISO)
+
+
+class Loaded(NamedTuple):
+    state: EntityState
+    version: str | None
+
+
+class ClaimOutcome(NamedTuple):
+    pending: list[InputFile]
+    claimed: Loaded | None
+    active_claims: list[str]
+    already_processed: list[tuple[InputFile, FileState]]
+    expired: dict[str, Claim]  # dropped claims of dead runs, by URI (#545 retry guard)
+
+
+def _now(offset_s: int = 0) -> str:
+    at = datetime.now(UTC) + timedelta(seconds=offset_s)
+    return at.isoformat().replace("+00:00", "Z")
+
+
+def dump_state(state: EntityState) -> bytes:
+    doc = {
+        "schema": SCHEMA_V2,
+        "entity": state.entity,
+        "updated_at": state.updated_at,
+        "files": {k: asdict(v) for k, v in sorted(state.files.items())},
+        "claims": {k: asdict(v) for k, v in sorted(state.claims.items())},
+    }
+    if state.watermark:  # D30: additive, absent without partitioning
+        doc["watermark"] = state.watermark
+    return json.dumps(doc, indent=2, ensure_ascii=False).encode()
+
+
+def parse_state(entity: str, body: bytes) -> EntityState:
+    try:
+        doc = json.loads(body)
+        schema: str = doc["schema"]
+        state = EntityState(
+            doc["entity"],
+            doc.get("updated_at"),
+            {k: FileState(**v) for k, v in doc.get("files", {}).items()},
+            {k: Claim(**v) for k, v in doc.get("claims", {}).items()}
+            if schema != SCHEMA_V1
+            else {},
+            doc.get("watermark"),
+        )
+    except (ValueError, KeyError, TypeError, AttributeError) as e:
+        raise FloeConfigError(f"entity.name={entity} invalid state file: {e}") from e
+    if schema not in (SCHEMA_V1, SCHEMA_V2):
+        raise FloeConfigError(
+            f"entity.name={entity} state schema mismatch: expected {SCHEMA_V1} or {SCHEMA_V2}, got {schema}"
+        )
+    if state.entity != entity:
+        raise FloeConfigError(
+            f"entity.name={entity} state entity mismatch: expected {entity}, got {state.entity}"
+        )
+    return state
+
+
+def read_state(client: StorageClient, uri: str, entity: str) -> EntityState | None:
+    stored = client.read(uri)
+    return None if stored is None else parse_state(entity, stored.body)
+
+
+def _load(client: StorageClient, uri: str, entity: str) -> Loaded:
+    stored = client.read(uri)
+    if stored is None:
+        return Loaded(EntityState(entity, None, {}, {}), None)
+    return Loaded(parse_state(entity, stored.body), stored.version)
+
+
+def _backoff(attempt: int) -> None:
+    window = min(_BASE_BACKOFF_MS * 2 ** min(attempt, 20), _MAX_BACKOFF_MS)
+    time.sleep(random.uniform(0, window) / 1000)
+
+
+def _is_live(claim: Claim, now: datetime) -> bool:
+    try:
+        return datetime.fromisoformat(claim.expires_at) > now
+    except (ValueError, TypeError):
+        return False
+
+
+def _persist(client: StorageClient, uri: str, loaded: Loaded) -> str | None:
+    return client.write_conditional(uri, loaded.version, dump_state(loaded.state))
+
+
+def claim_inputs(
+    client: StorageClient,
+    uri: str,
+    entity: str,
+    run_id: str,
+    inputs: list[InputFile],
+    barrier: bool = False,
+) -> ClaimOutcome:
+    """`barrier` (merge modes, #545 S7): while another run's claim is live, every input waits as `active_claim`."""
+    if not inputs:
+        return ClaimOutcome([], None, [], [], {})
+    for attempt in range(CAS_RETRIES):
+        if attempt:
+            _backoff(attempt - 1)
+        loaded = _load(client, uri, entity)
+        state = loaded.state
+        now = datetime.now(UTC)
+        expired = {u: c for u, c in state.claims.items() if not _is_live(c, now)}
+        state.claims = {u: c for u, c in state.claims.items() if u not in expired}
+        # only live claims are left, and none is ours
+        blocked = barrier and bool(state.claims)
+        pending: list[InputFile] = []
+        active: list[str] = []
+        processed: list[tuple[InputFile, FileState]] = []
+        acquired_at, expires_at = _now(), _now(CLAIM_TTL_SECONDS)
+        for f in inputs:
+            if f.source_uri in state.files:
+                processed.append((f, state.files[f.source_uri]))
+            elif blocked or f.source_uri in state.claims:
+                active.append(f.source_uri)
+            else:
+                # a dead run's batch can't have committed before its first claim (#545): keep that bound
+                was = expired.get(f.source_uri)
+                since = was.acquired_at if was else acquired_at
+                state.claims[f.source_uri] = Claim(
+                    run_id, since, expires_at, f.source_size, f.source_mtime
+                )
+                pending.append(f)
+        if not pending:
+            if not active:
+                _persist(client, uri, loaded)
+            return ClaimOutcome([], None, active, processed, expired)
+        state.updated_at = acquired_at
+        version = _persist(client, uri, loaded)
+        if version is not None:
+            return ClaimOutcome(
+                pending, Loaded(state, version), active, processed, expired
+            )
+    raise FloeConfigError(
+        f"entity.name={entity} incremental state update conflicted after {CAS_RETRIES} retries"
+    )
+
+
+def claim_all(
+    client: StorageClient, uri: str, entity: str, run_id: str, inputs: list[InputFile]
+) -> Loaded:
+    """Full refresh: claim every input, dropping other claims (Rust `claim_all_entity_inputs`)."""
+    for attempt in range(CAS_RETRIES):
+        if attempt:
+            _backoff(attempt - 1)
+        loaded = _load(client, uri, entity)
+        acquired_at, expires_at = _now(), _now(CLAIM_TTL_SECONDS)
+        claims = {
+            f.source_uri: Claim(run_id, acquired_at, expires_at, f.source_size, f.source_mtime)
+            for f in inputs
+        }  # fmt: skip
+        # D30: no watermark until the refresh promotes, so a failed one leaves the next run listing everything
+        fresh = Loaded(EntityState(entity, acquired_at, loaded.state.files, claims), loaded.version)  # fmt: skip
+        version = _persist(client, uri, fresh)
+        if version is not None:
+            return Loaded(fresh.state, version)
+    raise FloeConfigError(
+        f"entity.name={entity} full-refresh state write conflicted after {CAS_RETRIES} retries"
+    )
+
+
+def _mutate(
+    client: StorageClient,
+    uri: str,
+    claimed: Loaded,
+    mutate: Callable[[EntityState, set[str]], None],
+) -> None:
+    entity = claimed.state.entity
+    ours = set(claimed.state.claims)
+    for attempt in range(CAS_RETRIES):
+        if attempt:
+            _backoff(attempt - 1)
+        loaded = (
+            Loaded(copy.deepcopy(claimed.state), claimed.version)
+            if attempt == 0
+            else _load(client, uri, entity)
+        )
+        mutate(loaded.state, ours)
+        state = loaded.state
+        if state.files or state.claims or state.watermark:
+            done = _persist(client, uri, loaded) is not None
+        else:  # Rust deletes an emptied state
+            done = client.delete_conditional(uri, loaded.version)
+        if done:
+            return
+    raise FloeConfigError(
+        f"entity.name={entity} incremental state update conflicted after {CAS_RETRIES} retries"
+    )
+
+
+def promote(
+    client: StorageClient,
+    uri: str,
+    run_id: str,
+    claimed: Loaded,
+    full_refresh: bool = False,
+    only: set[str] | None = None,
+    advance: Callable[[EntityState, list[str]], None] | None = None,
+) -> None:
+    """A full refresh replaces `files` with this run's (Rust `promote_full_refresh_claimed_entity_state`); `only` limits the URIs.
+
+    `advance` (D30) gets the promoted URIs, in the same state write.
+    """
+
+    def apply(state: EntityState, ours: set[str]) -> None:
+        processed_at = _now()
+        if full_refresh:
+            state.files.clear()
+        mine = ours if only is None else ours & only
+        promoted = [
+            u for u, c in state.claims.items() if c.run_id == run_id and u in mine
+        ]
+        for u in promoted:
+            c = state.claims.pop(u)
+            state.files[u] = FileState(processed_at, c.size, c.mtime)
+        if advance:
+            advance(state, promoted)
+        state.updated_at = processed_at
+
+    _mutate(client, uri, claimed, apply)
+
+
+def release(client: StorageClient, uri: str, run_id: str, claimed: Loaded, restore: dict[str, Claim] | None = None, expire: bool = False) -> None:  # fmt: skip
+    """`restore`: the dead runs' expired claims this run took over, put back as they were (#545).
+
+    `expire` (D30): this run's other claims stay, expired, so they hold the listing window open until a run claims the files again.
+    """
+
+    def apply(state: EntityState, ours: set[str]) -> None:
+        mine = {u: c for u, c in state.claims.items() if c.run_id == run_id and u in ours}  # fmt: skip
+        state.claims = {u: c for u, c in state.claims.items() if u not in mine}
+        for u, c in (restore or {}).items():
+            if u in mine:  # not one it promoted: D30 may have pruned it
+                state.claims.setdefault(u, c)
+        for u, c in mine.items() if expire else ():
+            c.expires_at = c.acquired_at
+            state.claims.setdefault(u, c)
+        state.updated_at = _now()
+
+    _mutate(client, uri, claimed, apply)
+
+
+def renew(client: StorageClient, uri: str, run_id: str, claimed: Loaded) -> None:
+    def apply(state: EntityState, ours: set[str]) -> None:
+        expires_at = _now(CLAIM_TTL_SECONDS)
+        for u, c in state.claims.items():
+            if c.run_id == run_id and u in ours:
+                c.expires_at = expires_at
+        state.updated_at = _now()
+
+    _mutate(client, uri, claimed, apply)
+
+
+def reset(client: StorageClient, uri: str, entity: str) -> bool:
+    """Delete the state file, even a malformed one; False when there is none (Rust `reset_entity_state`)."""
+    stored = client.read(uri)
+    if stored is None:
+        return False
+    if not client.delete_conditional(uri, stored.version):
+        raise FloeConfigError(
+            f"entity.name={entity} remote state changed while resetting: {uri}"
+        )
+    return True

floe/storage/archive.py

@@ -0,0 +1,31 @@
+"""Input archiving, ported from `io/storage/ops/archive.rs`."""
+
+from collections.abc import Callable
+
+from floe.errors import FloeError, FloeRunError
+from floe.storage.inputs import InputFile
+from floe.storage.paths import archive_relative_path_for_run
+from floe.storage.types import StorageClient
+
+
+def archive_input_file(
+    client: StorageClient,
+    run_id: str,
+    entity_name: str,
+    join_relative: Callable[[str], str],
+    input_file: InputFile,
+    subdir: str = "",
+) -> str:
+    """`subdir` (D29): the partition sub-path kept between the entity folder and the file name."""
+    relative = archive_relative_path_for_run(
+        f"{entity_name}/{subdir}" if subdir else entity_name, input_file.source_name, run_id, input_file.source_uri
+    )  # fmt: skip
+    dest_uri = join_relative(relative)
+    client.copy(input_file.source_uri, dest_uri)
+    try:
+        client.delete(input_file.source_uri)
+    except FloeError as e:
+        raise FloeRunError(
+            f"entity.name={entity_name} archive delete failed for {input_file.source_uri}: {e}"
+        ) from e
+    return dest_uri

floe/storage/inputs.py

@@ -0,0 +1,302 @@
+"""Input resolution, ported from `io/storage/ops/inputs.rs` (`resolve_inputs`)."""
+
+import os
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path, PurePath
+from typing import Literal
+
+from floe.errors import FloeConfigError, FloeError, FloeRunError
+from floe.glob import MatchOptions, Pattern, PatternError, glob
+from floe.storage.paths import normalize_local_path, parse_s3_uri
+from floe.storage.types import Downloader, ObjectRef, StorageClient
+
+_SUFFIXES = {
+    "csv": [".csv"],
+    "tsv": [".tsv"],
+    "parquet": [".parquet"],
+    "json": [".json", ".jsonl", ".ndjson", ".djson"],
+    "avro": [".avro"],
+    "orc": [".orc"],
+    "xlsx": [".xlsx"],
+    "xml": [".xml"],
+}
+
+
+class NoInputsError(FloeRunError):
+    """No input file or object matched the source; any other listing error is not this."""
+
+
+@dataclass(frozen=True)
+class InputFile:
+    source_uri: str
+    source_name: str
+    source_stem: str
+    source_size: int | None
+    source_mtime: str | None
+
+
+@dataclass(frozen=True)
+class LocalInputFile:
+    file: InputFile
+    local_path: Path
+    is_ephemeral: bool
+
+
+@dataclass(frozen=True)
+class ResolvedInputs:
+    files: list[InputFile]
+    listed: list[str]
+    mode: Literal["file", "directory"]
+    warning: str | None = None  # D30: the listing window, logged once per run
+
+
+def suffixes_for_format(fmt: str) -> list[str]:
+    if fmt not in _SUFFIXES:
+        raise FloeConfigError(f"unsupported source format for input resolution: {fmt}")
+    return _SUFFIXES[fmt]
+
+
+def _rfc3339(ns: int) -> str:
+    secs, nanos = divmod(ns, 10**9)
+    text = datetime.fromtimestamp(secs, UTC).strftime("%Y-%m-%dT%H:%M:%S")
+    return f"{text}.{nanos:09d}".rstrip("0") + "Z" if nanos else f"{text}Z"
+
+
+def _glob_files(pattern: str) -> list[Path]:
+    try:
+        found = glob(pattern)
+    except PatternError as e:
+        raise FloeConfigError(f'invalid glob pattern "{pattern}": {e}') from e
+    except FloeError as e:
+        raise FloeConfigError(f'glob match failed for "{pattern}": {e}') from e
+    return [normalize_local_path(Path(p)) for p in found if Path(p).is_file()]
+
+
+def _visible(files: list[Path], root: Path, exclude: Path | None) -> list[Path]:
+    """D23: drop anything under a `.floe` directory (the default state location), as the cloud listing does; D31: and under `exclude`."""
+    root = normalize_local_path(
+        root
+    )  # the matches are normalised, so `..` in the root must be too
+    return [
+        f
+        for f in files
+        if ".floe" not in f.relative_to(root).parts[:-1]
+        and not (exclude and f.is_relative_to(exclude))
+    ]
+
+
+def _local_files(
+    config_dir: Path,
+    raw: str,
+    fmt: str,
+    entity: str,
+    storage: str,
+    glob_override: str | None,
+    recursive: bool,
+    exclude: Path | None,
+    after: str = "",
+) -> tuple[list[Path], Literal["file", "directory"]]:
+    def no_match(base: object, used: str) -> NoInputsError:
+        return NoInputsError(
+            f"entity.name={entity} source.storage={storage} no input files matched "
+            f"(base_path={base}, glob={used}, recursive={str(recursive).lower()})"
+        )
+
+    if any(c in raw for c in "*?["):
+        pattern = os.path.join(config_dir, raw)
+        prefix = pattern[: next(i for i, c in enumerate(pattern) if c in "*?[")]
+        root = Path(prefix) if prefix.endswith(os.sep) else Path(prefix).parent
+        files = _visible(sorted(set(_glob_files(pattern)), key=str), root, exclude)
+        if not files:
+            raise no_match(
+                os.path.dirname(pattern), os.path.basename(pattern)
+            )  # as written: Rust keeps a leading `./`
+        return files, "directory"
+    base = normalize_local_path(config_dir / raw)
+    if base.is_file():
+        return [base], "file"
+    globs = (
+        [glob_override]
+        if glob_override is not None
+        else [
+            "*" + "".join(f"[{c.lower()}{c.upper()}]" if c.isalpha() else c for c in s)
+            for s in suffixes_for_format(fmt)
+        ]
+    )
+    if not base.is_dir():
+        raise no_match(base, ",".join(globs))
+    patterns = [
+        os.path.join(d, "**", g) if recursive else os.path.join(d, g)
+        # D30: a glob override with a `/` keeps its meaning only from `base`, so it gets the after-listing filter alone
+        for d in (
+            _window_dirs(base, after)
+            if after and "/" not in (glob_override or "")
+            else [base]
+        )
+        for g in globs
+    ]
+    files = _visible(
+        sorted({f for p in patterns for f in _glob_files(p)}, key=str), base, exclude
+    )
+    if not files:
+        raise no_match(base, ",".join(globs))
+    return files, "directory"
+
+
+def _window_dirs(base: Path, after: str) -> list[Path]:
+    """D30: the directories at the depth of `after` (relative to `base`) that do not sort before it; older ones are never walked."""
+    dirs = [base]
+    for _ in after.split("/"):
+        dirs = sorted(d for p in dirs for d in p.iterdir() if d.is_dir() and (r := d.relative_to(base).as_posix()) >= after[: len(r)])  # fmt: skip
+    return dirs
+
+
+_REMOTE_SCHEMES = ("s3://", "gs://", "gcs://", "abfs://", "abfss://", "az://")
+_CLOUD = MatchOptions(case_sensitive=True, require_literal_separator=True)
+
+
+def _first_glob(value: str) -> int:
+    escaped = False
+    for i, ch in enumerate(value):
+        if escaped:
+            escaped = False
+        elif ch == "\\":
+            escaped = True
+        elif ch in "*?":
+            return i
+    return -1
+
+
+def _cloud_refs(
+    client: StorageClient,
+    key: str,
+    suffixes: list[str],
+    entity: str,
+    storage: str,
+    location: str,
+    exclude: str | None,
+    after: str = "",
+) -> list[ObjectRef]:
+    where = f"entity.name={entity} source.storage={storage}"
+    first = _first_glob(key)
+    prefix = key if first == -1 else key[:first]
+    matcher = None
+    if first != -1:
+        try:
+            if not prefix.strip("/"):
+                raise FloeRunError(
+                    "glob patterns for cloud sources must include a non-empty "
+                    "literal prefix before the first wildcard"
+                )
+            try:
+                matcher = Pattern(key)
+            except PatternError as e:
+                raise FloeRunError(f'invalid cloud glob pattern "{key}": {e}') from e
+        except FloeRunError as e:
+            raise FloeRunError(
+                f"{where} invalid cloud source path ({location}, path={key}): {e}"
+            ) from e
+    lowered = tuple(s.lower() for s in suffixes)
+    refs = [
+        r
+        for r in client.list(prefix, after)
+        if (matcher is None or matcher.matches_with(r.key, _CLOUD))
+        and not (r.key.startswith(".floe/") or "/.floe/" in r.key)
+        and not (exclude and r.uri.startswith(f"{exclude}/"))
+        and r.uri.lower().endswith(lowered)
+    ]
+    if not refs:
+        used = "glob=<none>" if matcher is None else f"glob={key}"
+        raise NoInputsError(
+            f"{where} no input objects matched ({location}, prefix={prefix}, "
+            f"{used}, suffixes={','.join(suffixes)})"
+        )
+    return sorted(refs, key=lambda r: r.uri)
+
+
+def resolve_inputs(
+    config_dir: Path,
+    path: str,
+    fmt: str,
+    entity: str,
+    storage: str,
+    client: StorageClient | None,
+    *,
+    list_only: bool = False,
+    recursive: bool = False,
+    glob_override: str | None = None,
+    exclude: str | None = None,
+    after: str = "",
+) -> ResolvedInputs:
+    """`path` is the source path, or a full `s3://` URI for a remote storage; nothing under the `exclude` URI is listed (D31).
+
+    D30: `after`, a URI, skips what sorts before it: S3 `StartAfter`, or local directories never walked.
+    """
+    files: list[InputFile] = []
+    if path.startswith("s3://"):
+        if client is None:
+            raise FloeRunError("s3 storage client missing")
+        bucket, key = parse_s3_uri(path)
+        refs = _cloud_refs(
+            client,
+            key,
+            suffixes_for_format(fmt),
+            entity,
+            storage,
+            f"bucket={bucket}",
+            exclude,
+            parse_s3_uri(after)[1] if after else "",
+        )
+        if not list_only:
+            for r in refs:
+                name = PurePath(r.key).name or entity
+                files.append(
+                    InputFile(
+                        r.uri,
+                        name,
+                        PurePath(name).stem or entity,
+                        r.size,
+                        r.last_modified,
+                    )
+                )
+        return ResolvedInputs(files, [r.uri for r in refs], "directory")
+    if "://" in path:
+        raise FloeConfigError(f"unsupported storage uri: {path}")
+    skip = Path(exclude.removeprefix("local://")) if exclude and exclude.startswith("local://") else None  # fmt: skip
+    start = Path(after.removeprefix("local://")) if after.startswith("local://") else None  # fmt: skip
+    base = normalize_local_path(config_dir / path)
+    paths, mode = _local_files(
+        config_dir, path, fmt, entity, storage, glob_override, recursive, skip,
+        start.relative_to(base).as_posix() if start and start.is_relative_to(base) else "",
+    )  # fmt: skip
+    uris = [client.resolve_uri(str(p)) if client else str(p) for p in paths]
+    if not list_only:
+        for p, uri in zip(paths, uris, strict=True):
+            st = p.stat()
+            name = p.name or entity
+            files.append(
+                InputFile(
+                    uri,
+                    name,
+                    PurePath(name).stem or entity,
+                    st.st_size,
+                    _rfc3339(st.st_mtime_ns),
+                )
+            )
+    return ResolvedInputs(files, uris, mode)
+
+
+def localize_input(
+    file: InputFile, client: Downloader | None, temp_dir: Path | None
+) -> LocalInputFile:
+    """Download a cloud input to `temp_dir`; a local input is used in place."""
+    if not file.source_uri.startswith(_REMOTE_SCHEMES):
+        return LocalInputFile(
+            file, Path(file.source_uri.removeprefix("local://")), False
+        )
+    if client is None:
+        raise FloeRunError(f"storage client required to download {file.source_uri}")
+    if temp_dir is None:
+        raise FloeRunError(f"temp_dir required to download {file.source_uri}")
+    return LocalInputFile(file, client.download(file.source_uri, temp_dir), True)