floe-python 0.4.4__tar.gz → 0.4.6__tar.gz

{floe_python-0.4.4 → floe_python-0.4.6}/Cargo.lock

@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.4.4"
+version = "0.4.6"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3412,7 +3412,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.4.4"
+version = "0.4.6"
 dependencies = [
  "apache-avro 0.16.0",
  "arrow",
@@ -3455,7 +3455,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.4.4"
+version = "0.4.6"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.4.4 → floe_python-0.4.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.4.4
+Version: 0.4.6
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.4.4"
+version = "0.4.6"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"
@@ -14,7 +14,7 @@ path = "src/lib.rs"
 
 [dependencies]
 yaml-rust2 = "0.11"
-polars = { version = "0.52.0", features = ["csv", "parquet", "lazy", "timezones", "dtype-date", "dtype-datetime", "dtype-time", "polars-ops", "is_unique", "is_first_distinct"] }
+polars = { version = "0.52.0", features = ["csv", "parquet", "lazy", "new_streaming", "timezones", "dtype-date", "dtype-datetime", "dtype-time", "polars-ops", "is_unique", "is_first_distinct"] }
 calamine = "0.24"
 rayon = "1"
 deltalake = { version = "0.30.1", features = ["datafusion", "s3", "azure", "gcs"] }

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/src/config/location.rs

@@ -45,7 +45,7 @@ pub fn resolve_config_location(input: &str) -> FloeResult<ConfigLocation> {
 fn download_remote_config(uri: &str, temp_dir: &Path) -> FloeResult<PathBuf> {
     if uri.starts_with("s3://") {
         let location = storage::s3::parse_s3_uri(uri)?;
-        let client = storage::s3::S3Client::new(location.bucket, None)?;
+        let client = storage::s3::S3Client::new(location.bucket, None, None, None)?;
         return client.download_to_temp(uri, temp_dir);
     }
     if uri.starts_with("gs://") {
@@ -63,6 +63,8 @@ fn download_remote_config(uri: &str, temp_dir: &Path) -> FloeResult<PathBuf> {
             account: Some(location.account),
             container: Some(location.container),
             prefix: None,
+            endpoint: None,
+            path_style_access: None,
         };
         let client = storage::adls::AdlsClient::new(&definition)?;
         return client.download_to_temp(uri, temp_dir);
@@ -70,6 +72,44 @@ fn download_remote_config(uri: &str, temp_dir: &Path) -> FloeResult<PathBuf> {
     Err(format!("unsupported config uri: {}", uri).into())
 }
 
+/// Write `bytes` to a remote URI by staging them in a temp file then uploading.
+pub fn write_bytes_to_remote_uri(bytes: &[u8], uri: &str) -> FloeResult<()> {
+    let temp_dir = TempDir::new()?;
+    let local_path = temp_dir.path().join("upload");
+    std::fs::write(&local_path, bytes)?;
+    upload_to_remote_uri(&local_path, uri)
+}
+
+pub fn upload_to_remote_uri(local_path: &Path, uri: &str) -> FloeResult<()> {
+    if uri.starts_with("s3://") {
+        let location = storage::s3::parse_s3_uri(uri)?;
+        let client = storage::s3::S3Client::new(location.bucket, None, None, None)?;
+        return client.upload_from_path(local_path, uri);
+    }
+    if uri.starts_with("gs://") {
+        let location = storage::gcs::parse_gcs_uri(uri)?;
+        let client = storage::gcs::GcsClient::new(location.bucket)?;
+        return client.upload_from_path(local_path, uri);
+    }
+    if uri.starts_with("abfs://") {
+        let location = storage::adls::parse_adls_uri(uri)?;
+        let definition = StorageDefinition {
+            name: "manifest".to_string(),
+            fs_type: "adls".to_string(),
+            bucket: None,
+            region: None,
+            account: Some(location.account),
+            container: Some(location.container),
+            prefix: None,
+            endpoint: None,
+            path_style_access: None,
+        };
+        let client = storage::adls::AdlsClient::new(&definition)?;
+        return client.upload_from_path(local_path, uri);
+    }
+    Err(format!("unsupported manifest output uri: {uri}").into())
+}
+
 pub(crate) fn is_remote_uri(value: &str) -> bool {
     value.starts_with("s3://") || value.starts_with("gs://") || value.starts_with("abfs://")
 }

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/src/config/mod.rs

@@ -9,7 +9,9 @@ pub(crate) mod yaml_decode;
 
 pub use catalog::{CatalogResolver, ResolvedDeltaCatalogTarget, ResolvedIcebergCatalogTarget};
 pub(crate) use location::is_remote_uri;
-pub use location::{resolve_config_location, ConfigLocation};
+pub use location::{
+    resolve_config_location, upload_to_remote_uri, write_bytes_to_remote_uri, ConfigLocation,
+};
 pub use storage::{resolve_local_path, ConfigBase, ResolvedPath, StorageResolver};
 pub use types::*;
 

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/src/config/parse.rs

@@ -695,6 +695,8 @@ fn parse_storage_definition(value: &Yaml) -> FloeResult<StorageDefinition> {
             "account",
             "container",
             "prefix",
+            "endpoint",
+            "path_style_access",
         ],
     )?;
     Ok(StorageDefinition {
@@ -705,6 +707,8 @@ fn parse_storage_definition(value: &Yaml) -> FloeResult<StorageDefinition> {
         account: opt_string(hash, "account", "storages.definitions")?,
         container: opt_string(hash, "container", "storages.definitions")?,
         prefix: opt_string(hash, "prefix", "storages.definitions")?,
+        endpoint: opt_string(hash, "endpoint", "storages.definitions")?,
+        path_style_access: opt_bool(hash, "path_style_access", "storages.definitions")?,
     })
 }
 

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/src/config/storage.rs

@@ -215,7 +215,7 @@ impl StorageResolver {
         raw_path: &str,
     ) -> FloeResult<ResolvedPath> {
         let name = storage_name.unwrap_or(self.default_name.as_str());
-        if !self.has_config && name != "local" {
+        if !self.has_config && name != "local" && !self.definitions.contains_key(name) {
             return Err(Box::new(ConfigError(format!(
                 "entity.name={} {field} references unknown storage {} (no storages block)",
                 entity_name, name
@@ -238,6 +238,8 @@ impl StorageResolver {
                 account: None,
                 container: None,
                 prefix: None,
+                endpoint: None,
+                path_style_access: None,
             }
         };
 
@@ -304,7 +306,7 @@ impl StorageResolver {
         raw_path: &str,
     ) -> FloeResult<ResolvedPath> {
         let name = storage_name.unwrap_or(self.default_name.as_str());
-        if !self.has_config && name != "local" {
+        if !self.has_config && name != "local" && !self.definitions.contains_key(name) {
             return Err(Box::new(ConfigError(format!(
                 "report.storage references unknown storage {} (no storages block)",
                 name
@@ -327,6 +329,8 @@ impl StorageResolver {
                 account: None,
                 container: None,
                 prefix: None,
+                endpoint: None,
+                path_style_access: None,
             }
         };
 
@@ -386,9 +390,48 @@ impl StorageResolver {
         }
     }
 
+    /// Scan definitions for the first one whose scheme and bucket/account match `uri`.
+    /// Used in manifest mode to resolve a bare report URI back to a named definition.
+    pub fn find_definition_name_for_uri(&self, uri: &str) -> Option<String> {
+        for (name, def) in &self.definitions {
+            if uri.starts_with("s3://") && def.fs_type == "s3" {
+                if let Some(b) = &def.bucket {
+                    if uri.starts_with(&format!("s3://{b}/")) || uri == format!("s3://{b}") {
+                        return Some(name.clone());
+                    }
+                }
+            }
+            if uri.starts_with("gs://") && def.fs_type == "gcs" {
+                if let Some(b) = &def.bucket {
+                    if uri.starts_with(&format!("gs://{b}/")) || uri == format!("gs://{b}") {
+                        return Some(name.clone());
+                    }
+                }
+            }
+            if uri.starts_with("abfs://") && def.fs_type == "adls" {
+                if let (Some(c), Some(a)) = (&def.container, &def.account) {
+                    if uri.starts_with(&format!("abfs://{c}@{a}.dfs.core.windows.net")) {
+                        return Some(name.clone());
+                    }
+                }
+            }
+        }
+        None
+    }
+
+    /// Register a synthetic `StorageDefinition` into this resolver.
+    /// Used in manifest mode when the report URI has no matching definition in the config.
+    /// Does NOT flip `has_config`; entity resolution keeps its implicit-local fallback.
+    pub fn register_definition(&mut self, definition: StorageDefinition) {
+        self.definitions.insert(definition.name.clone(), definition);
+    }
+
     pub fn definition(&self, name: &str) -> Option<StorageDefinition> {
         if self.has_config {
             self.definitions.get(name).cloned()
+        } else if let Some(def) = self.definitions.get(name) {
+            // Synthetic definition registered by register_definition (e.g. report target).
+            Some(def.clone())
         } else if name == "local" {
             Some(StorageDefinition {
                 name: "local".to_string(),
@@ -398,6 +441,8 @@ impl StorageResolver {
                 account: None,
                 container: None,
                 prefix: None,
+                endpoint: None,
+                path_style_access: None,
             })
         } else {
             None

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/src/config/types.rs

@@ -366,6 +366,8 @@ pub struct StorageDefinition {
     pub account: Option<String>,
     pub container: Option<String>,
     pub prefix: Option<String>,
+    pub endpoint: Option<String>,
+    pub path_style_access: Option<bool>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/src/io/format.rs

@@ -119,6 +119,182 @@ pub struct AcceptedWriteOutput {
     pub perf: Option<AcceptedWritePerfBreakdown>,
 }
 
+/// Per-write sinks cap their reported `part_files` list at this many entries
+/// (see `parquet.rs`). The reducer applies the same cap across flushes so
+/// the run report does not grow to N × 50 entries for high-fanout entities.
+pub const MAX_REPORTED_PART_FILES: usize = 50;
+
+impl AcceptedWriteOutput {
+    /// Fold a later flush's output into this one. The receiver represents the
+    /// running total across N completed flushes; `next` is the output of the
+    /// (N+1)th flush.
+    ///
+    /// Field semantics across flushes:
+    /// - `parts_written` (always known, the count of successful sink writes)
+    ///   sums.
+    /// - `files_written` and the `Option<u64>` metric fields
+    ///   (`total_bytes_written`, `small_files_count`, perf entries) sum
+    ///   when *both* sides are `Some`; if either side is `None` the merged
+    ///   result is `None`. "Unknown poisons" matches the per-flush
+    ///   semantics: when any single flush could not determine its file
+    ///   count (for example a remote Delta commit whose post-commit log
+    ///   could not be read), reporting a partial sum would silently
+    ///   under-count the total. The run report instead surfaces the value
+    ///   as unknown.
+    /// - `part_files` concatenates and is capped at `MAX_REPORTED_PART_FILES`
+    ///   so the reducer preserves the same cap the individual sink writers
+    ///   apply per-flush.
+    /// - `table_version` / `snapshot_id` take the latest (Delta commit /
+    ///   Iceberg snapshot move forward with every commit; the final state
+    ///   is what readers see).
+    /// - `table_root_uri`, `catalog`, `schema_evolution` take the first
+    ///   non-default value seen — table location and catalog registration
+    ///   are established by the first write; schema evolution only fires on
+    ///   the first (Overwrite) write because subsequent flushes are Append.
+    /// - `avg_file_size_mb` is recomputed from `total_bytes_written` divided
+    ///   by `files_written` when available (so it matches the per-flush
+    ///   semantics: for Parquet/Iceberg `files == parts`, but for Delta one
+    ///   commit can write multiple `add` files and `parts != files`).
+    ///   Falls back to `parts_written` when `files_written` is unknown.
+    /// - `perf` accumulates by summing each `Option<u64>` field.
+    /// - `merge` is unreachable in the buffered path (merge modes use the
+    ///   legacy accumulate-then-write code path); the running value is
+    ///   preserved if anything ever does pass one.
+    pub fn merge_in(&mut self, next: AcceptedWriteOutput) {
+        let AcceptedWriteOutput {
+            files_written,
+            parts_written,
+            part_files,
+            table_version,
+            snapshot_id,
+            table_root_uri,
+            catalog,
+            metrics,
+            merge,
+            schema_evolution,
+            perf,
+        } = next;
+
+        // `parts_written == 0` on the receiver means no prior flush has been
+        // merged. In that case `Option<u64>` fields on `self` start at `None`
+        // not because a flush returned unknown but because nothing has been
+        // recorded yet — distinguishing "vacuous" from "poisoned" matters
+        // because adopting the next flush's value verbatim on the first merge
+        // is correct, while applying poison-on-unknown semantics from `None`
+        // would always poison the very first merge.
+        let first_merge = self.parts_written == 0;
+
+        self.files_written = merge_option_u64(self.files_written, files_written, first_merge);
+        self.parts_written += parts_written;
+        let remaining = MAX_REPORTED_PART_FILES.saturating_sub(self.part_files.len());
+        if remaining > 0 {
+            self.part_files
+                .extend(part_files.into_iter().take(remaining));
+        }
+
+        if table_version.is_some() {
+            self.table_version = table_version;
+        }
+        if snapshot_id.is_some() {
+            self.snapshot_id = snapshot_id;
+        }
+
+        if self.table_root_uri.is_none() {
+            self.table_root_uri = table_root_uri;
+        }
+        if self.catalog.is_none() {
+            self.catalog = catalog;
+        }
+        if !self.schema_evolution.enabled
+            && !self.schema_evolution.applied
+            && self.schema_evolution.added_columns.is_empty()
+            && !self.schema_evolution.incompatible_changes_detected
+            && self.schema_evolution.mode.is_empty()
+        {
+            self.schema_evolution = schema_evolution;
+        }
+
+        self.metrics.total_bytes_written = merge_option_u64(
+            self.metrics.total_bytes_written,
+            metrics.total_bytes_written,
+            first_merge,
+        );
+        self.metrics.small_files_count = merge_option_u64(
+            self.metrics.small_files_count,
+            metrics.small_files_count,
+            first_merge,
+        );
+        self.metrics.avg_file_size_mb = recompute_avg_file_size_mb(
+            self.metrics.total_bytes_written,
+            self.files_written,
+            self.parts_written,
+        );
+
+        if self.merge.is_none() {
+            self.merge = merge;
+        }
+
+        match (self.perf.take(), perf) {
+            (Some(a), Some(b)) => self.perf = Some(sum_perf_breakdown(a, b)),
+            (Some(a), None) => self.perf = Some(a),
+            (None, Some(b)) => self.perf = Some(b),
+            (None, None) => self.perf = None,
+        }
+    }
+}
+
+/// Sum two `Option<u64>` values with poison-on-unknown semantics: if either
+/// side is `None`, the result is `None`. Reporting a partial sum as if it
+/// were the total would silently under-count for any aggregation across
+/// flushes where one flush could not determine the underlying count
+/// (e.g. remote Delta commit-log read failures).
+fn sum_option_u64(a: Option<u64>, b: Option<u64>) -> Option<u64> {
+    match (a, b) {
+        (Some(a), Some(b)) => Some(a + b),
+        _ => None,
+    }
+}
+
+/// Progressive `Option<u64>` merge used by `merge_in`. On the first merge
+/// (when the accumulator has no flush recorded yet) the next flush's value is
+/// taken verbatim; on subsequent merges `sum_option_u64`'s poison-on-unknown
+/// semantics apply.
+fn merge_option_u64(acc: Option<u64>, next: Option<u64>, first_merge: bool) -> Option<u64> {
+    if first_merge {
+        next
+    } else {
+        sum_option_u64(acc, next)
+    }
+}
+
+fn recompute_avg_file_size_mb(
+    total_bytes: Option<u64>,
+    files_written: Option<u64>,
+    parts_written: u64,
+) -> Option<f64> {
+    let bytes = total_bytes?;
+    let denominator = files_written.unwrap_or(parts_written);
+    if denominator == 0 {
+        return None;
+    }
+    let mb = (bytes as f64) / (denominator as f64) / (1024.0 * 1024.0);
+    Some(mb)
+}
+
+fn sum_perf_breakdown(
+    a: AcceptedWritePerfBreakdown,
+    b: AcceptedWritePerfBreakdown,
+) -> AcceptedWritePerfBreakdown {
+    AcceptedWritePerfBreakdown {
+        conversion_ms: sum_option_u64(a.conversion_ms, b.conversion_ms),
+        source_df_build_ms: sum_option_u64(a.source_df_build_ms, b.source_df_build_ms),
+        merge_exec_ms: sum_option_u64(a.merge_exec_ms, b.merge_exec_ms),
+        data_write_ms: sum_option_u64(a.data_write_ms, b.data_write_ms),
+        commit_ms: sum_option_u64(a.commit_ms, b.commit_ms),
+        metrics_read_ms: sum_option_u64(a.metrics_read_ms, b.metrics_read_ms),
+    }
+}
+
 pub trait InputAdapter: Send + Sync {
     fn format(&self) -> &'static str;
 

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/src/io/storage/mod.rs

@@ -129,7 +129,12 @@ fn build_client(definition: &config::StorageDefinition) -> FloeResult<Box<dyn St
         "s3" => {
             let bucket =
                 validation::require_field(definition, definition.bucket.as_ref(), "bucket", "s3")?;
-            Box::new(s3::S3Client::new(bucket, definition.region.as_deref())?)
+            Box::new(s3::S3Client::new(
+                bucket,
+                definition.region.as_deref(),
+                definition.endpoint.as_deref(),
+                definition.path_style_access,
+            )?)
         }
         "adls" => Box::new(adls::AdlsClient::new(definition)?),
         "gcs" => {

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/src/io/storage/object_store.rs

@@ -111,9 +111,16 @@ pub fn iceberg_store_config(
         Target::S3 { storage, uri, .. } => {
             let mut file_io_props = HashMap::new();
             if let Some(definition) = resolver.definition(storage) {
-                if let Some(region) = definition.region {
+                if let Some(region) = &definition.region {
                     file_io_props.insert(S3_REGION.to_string(), region.clone());
-                    file_io_props.insert(CLIENT_REGION.to_string(), region);
+                    file_io_props.insert(CLIENT_REGION.to_string(), region.clone());
+                }
+                if let Some(endpoint) = &definition.endpoint {
+                    file_io_props.insert("s3.endpoint".to_string(), endpoint.clone());
+                }
+                if let Some(path_style) = definition.path_style_access {
+                    file_io_props
+                        .insert("s3.path-style-access".to_string(), path_style.to_string());
                 }
             }
             Ok(IcebergStoreConfig {

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/src/io/storage/providers/s3.rs

@@ -19,23 +19,35 @@ pub struct S3Client {
 }
 
 impl S3Client {
-    pub fn new(bucket: String, region: Option<&str>) -> FloeResult<Self> {
+    pub fn new(
+        bucket: String,
+        region: Option<&str>,
+        endpoint: Option<&str>,
+        path_style_access: Option<bool>,
+    ) -> FloeResult<Self> {
         let runtime = tokio::runtime::Builder::new_current_thread()
             .enable_all()
             .build()
             .map_err(|err| Box::new(StorageError(format!("failed to build aws runtime: {err}"))))?;
+        let endpoint = endpoint.map(ToOwned::to_owned);
         let config = runtime.block_on(async {
             let region_provider = match region {
                 Some(region) => RegionProviderChain::first_try(Region::new(region.to_string()))
                     .or_default_provider(),
                 None => RegionProviderChain::default_provider(),
             };
-            aws_config::defaults(aws_config::BehaviorVersion::latest())
-                .region(region_provider)
-                .load()
-                .await
+            let mut builder =
+                aws_config::defaults(aws_config::BehaviorVersion::latest()).region(region_provider);
+            if let Some(ep) = endpoint {
+                builder = builder.endpoint_url(ep);
+            }
+            builder.load().await
         });
-        let client = Client::new(&config);
+        let mut s3_builder = aws_sdk_s3::config::Builder::from(&config);
+        if path_style_access.unwrap_or(false) {
+            s3_builder = s3_builder.force_path_style(true);
+        }
+        let client = Client::from_conf(s3_builder.build());
         Ok(Self {
             bucket,
             client,

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/src/io/write/iceberg/context.rs

@@ -76,7 +76,8 @@ pub(super) fn build_iceberg_write_context(
                         latest_s3_metadata_location(client, base_key)?
                     }
                     None => {
-                        let mut client = io::storage::s3::S3Client::new(bucket.clone(), None)?;
+                        let mut client =
+                            io::storage::s3::S3Client::new(bucket.clone(), None, None, None)?;
                         latest_s3_metadata_location(&mut client, base_key)?
                     }
                 }

{floe_python-0.4.4 → floe_python-0.4.6}/crates/floe-core/src/io/write/iceberg/rest.rs

@@ -45,6 +45,7 @@ pub(crate) async fn build_rest_catalog(
     }
 
     if let Some(credential) = rest_cfg.credential.as_deref() {
+        let credential = expand_env_refs(credential, &rest_cfg.catalog_name)?;
         if let Some(token_value) = credential.strip_prefix("token:") {
             // Bearer PAT (Unity Catalog / Nessie)
             props.insert("token".to_string(), token_value.to_string());
@@ -140,6 +141,39 @@ impl RestIcebergCatalogConfig {
     }
 }
 
+fn expand_env_refs(value: &str, catalog_name: &str) -> FloeResult<String> {
+    if !value.contains("${") {
+        return Ok(value.to_string());
+    }
+
+    let mut parts = Vec::new();
+    for part in value.split(':') {
+        parts.push(expand_env_ref_part(part, catalog_name)?);
+    }
+    Ok(parts.join(":"))
+}
+
+fn expand_env_ref_part(part: &str, catalog_name: &str) -> FloeResult<String> {
+    let Some(inner) = part.strip_prefix("${") else {
+        return Ok(part.to_string());
+    };
+    let Some(name) = inner.strip_suffix('}') else {
+        return Err(Box::new(RunError(format!(
+            "rest iceberg catalog {catalog_name} credential has unclosed env placeholder"
+        ))));
+    };
+    if name.is_empty() || name.contains('{') || name.contains('}') {
+        return Err(Box::new(RunError(format!(
+            "rest iceberg catalog {catalog_name} credential has invalid env placeholder"
+        ))));
+    }
+    std::env::var(name).map_err(|_| {
+        Box::new(RunError(format!(
+            "rest iceberg catalog {catalog_name} credential references env var {name} which is not set"
+        ))) as Box<dyn std::error::Error + Send + Sync>
+    })
+}
+
 pub(crate) async fn write_via_rest_catalog(
     rest_cfg: &RestIcebergCatalogConfig,
     table_root_uri: String,
@@ -342,3 +376,77 @@ async fn create_rest_table(
         .await
         .map_err(map_iceberg_err("rest catalog create_table failed"))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::expand_env_refs;
+
+    #[test]
+    fn expands_partial_env_refs_in_client_credentials() {
+        std::env::set_var("FLOE_TEST_REST_CLIENT_ID", "client-id");
+        std::env::set_var("FLOE_TEST_REST_CLIENT_SECRET", "client-secret");
+
+        let expanded = expand_env_refs(
+            "client_credentials:${FLOE_TEST_REST_CLIENT_ID}:${FLOE_TEST_REST_CLIENT_SECRET}",
+            "polaris",
+        )
+        .expect("expand credential");
+
+        assert_eq!(expanded, "client_credentials:client-id:client-secret");
+        std::env::remove_var("FLOE_TEST_REST_CLIENT_ID");
+        std::env::remove_var("FLOE_TEST_REST_CLIENT_SECRET");
+    }
+
+    #[test]
+    fn expands_exact_env_ref_in_token_credential() {
+        std::env::set_var("FLOE_TEST_REST_TOKEN", "pat-token");
+
+        let expanded =
+            expand_env_refs("token:${FLOE_TEST_REST_TOKEN}", "nessie").expect("expand token");
+
+        assert_eq!(expanded, "token:pat-token");
+        std::env::remove_var("FLOE_TEST_REST_TOKEN");
+    }
+
+    #[test]
+    fn preserves_literal_credential_text_that_contains_env_ref_syntax() {
+        let expanded =
+            expand_env_refs("token:abc${def}ghi", "nessie").expect("preserve literal credential");
+
+        assert_eq!(expanded, "token:abc${def}ghi");
+    }
+
+    #[test]
+    fn errors_when_env_ref_is_missing() {
+        std::env::remove_var("FLOE_TEST_REST_MISSING");
+
+        let err = expand_env_refs(
+            "client_credentials:${FLOE_TEST_REST_MISSING}:secret",
+            "polaris",
+        )
+        .unwrap_err();
+
+        assert_eq!(
+            err.to_string(),
+            "rest iceberg catalog polaris credential references env var FLOE_TEST_REST_MISSING which is not set"
+        );
+    }
+
+    #[test]
+    fn errors_on_malformed_env_ref() {
+        std::env::set_var("ID", "client-id");
+
+        let err = expand_env_refs(
+            "client_credentials:${ID}:literal-secret:${UNCLOSED",
+            "polaris",
+        )
+        .unwrap_err();
+
+        assert_eq!(
+            err.to_string(),
+            "rest iceberg catalog polaris credential has unclosed env placeholder"
+        );
+        assert!(!err.to_string().contains("literal-secret"));
+        std::env::remove_var("ID");
+    }
+}