floe-python 0.4.5__tar.gz → 0.4.6__tar.gz

{floe_python-0.4.5 → floe_python-0.4.6}/Cargo.lock

@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.4.5"
+version = "0.4.6"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3412,7 +3412,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.4.5"
+version = "0.4.6"
 dependencies = [
  "apache-avro 0.16.0",
  "arrow",
@@ -3455,7 +3455,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.4.5"
+version = "0.4.6"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.4.5 → floe_python-0.4.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.4.5
+Version: 0.4.6
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.4.5 → floe_python-0.4.6}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.4.5"
+version = "0.4.6"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"
@@ -14,7 +14,7 @@ path = "src/lib.rs"
 
 [dependencies]
 yaml-rust2 = "0.11"
-polars = { version = "0.52.0", features = ["csv", "parquet", "lazy", "timezones", "dtype-date", "dtype-datetime", "dtype-time", "polars-ops", "is_unique", "is_first_distinct"] }
+polars = { version = "0.52.0", features = ["csv", "parquet", "lazy", "new_streaming", "timezones", "dtype-date", "dtype-datetime", "dtype-time", "polars-ops", "is_unique", "is_first_distinct"] }
 calamine = "0.24"
 rayon = "1"
 deltalake = { version = "0.30.1", features = ["datafusion", "s3", "azure", "gcs"] }

{floe_python-0.4.5 → floe_python-0.4.6}/crates/floe-core/src/io/format.rs

@@ -119,6 +119,182 @@ pub struct AcceptedWriteOutput {
     pub perf: Option<AcceptedWritePerfBreakdown>,
 }
 
+/// Per-write sinks cap their reported `part_files` list at this many entries
+/// (see `parquet.rs`). The reducer applies the same cap across flushes so
+/// the run report does not grow to N × 50 entries for high-fanout entities.
+pub const MAX_REPORTED_PART_FILES: usize = 50;
+
+impl AcceptedWriteOutput {
+    /// Fold a later flush's output into this one. The receiver represents the
+    /// running total across N completed flushes; `next` is the output of the
+    /// (N+1)th flush.
+    ///
+    /// Field semantics across flushes:
+    /// - `parts_written` (always known, the count of successful sink writes)
+    ///   sums.
+    /// - `files_written` and the `Option<u64>` metric fields
+    ///   (`total_bytes_written`, `small_files_count`, perf entries) sum
+    ///   when *both* sides are `Some`; if either side is `None` the merged
+    ///   result is `None`. "Unknown poisons" matches the per-flush
+    ///   semantics: when any single flush could not determine its file
+    ///   count (for example a remote Delta commit whose post-commit log
+    ///   could not be read), reporting a partial sum would silently
+    ///   under-count the total. The run report instead surfaces the value
+    ///   as unknown.
+    /// - `part_files` concatenates and is capped at `MAX_REPORTED_PART_FILES`
+    ///   so the reducer preserves the same cap the individual sink writers
+    ///   apply per-flush.
+    /// - `table_version` / `snapshot_id` take the latest (Delta commit /
+    ///   Iceberg snapshot move forward with every commit; the final state
+    ///   is what readers see).
+    /// - `table_root_uri`, `catalog`, `schema_evolution` take the first
+    ///   non-default value seen — table location and catalog registration
+    ///   are established by the first write; schema evolution only fires on
+    ///   the first (Overwrite) write because subsequent flushes are Append.
+    /// - `avg_file_size_mb` is recomputed from `total_bytes_written` divided
+    ///   by `files_written` when available (so it matches the per-flush
+    ///   semantics: for Parquet/Iceberg `files == parts`, but for Delta one
+    ///   commit can write multiple `add` files and `parts != files`).
+    ///   Falls back to `parts_written` when `files_written` is unknown.
+    /// - `perf` accumulates by summing each `Option<u64>` field.
+    /// - `merge` is unreachable in the buffered path (merge modes use the
+    ///   legacy accumulate-then-write code path); the running value is
+    ///   preserved if anything ever does pass one.
+    pub fn merge_in(&mut self, next: AcceptedWriteOutput) {
+        let AcceptedWriteOutput {
+            files_written,
+            parts_written,
+            part_files,
+            table_version,
+            snapshot_id,
+            table_root_uri,
+            catalog,
+            metrics,
+            merge,
+            schema_evolution,
+            perf,
+        } = next;
+
+        // `parts_written == 0` on the receiver means no prior flush has been
+        // merged. In that case `Option<u64>` fields on `self` start at `None`
+        // not because a flush returned unknown but because nothing has been
+        // recorded yet — distinguishing "vacuous" from "poisoned" matters
+        // because adopting the next flush's value verbatim on the first merge
+        // is correct, while applying poison-on-unknown semantics from `None`
+        // would always poison the very first merge.
+        let first_merge = self.parts_written == 0;
+
+        self.files_written = merge_option_u64(self.files_written, files_written, first_merge);
+        self.parts_written += parts_written;
+        let remaining = MAX_REPORTED_PART_FILES.saturating_sub(self.part_files.len());
+        if remaining > 0 {
+            self.part_files
+                .extend(part_files.into_iter().take(remaining));
+        }
+
+        if table_version.is_some() {
+            self.table_version = table_version;
+        }
+        if snapshot_id.is_some() {
+            self.snapshot_id = snapshot_id;
+        }
+
+        if self.table_root_uri.is_none() {
+            self.table_root_uri = table_root_uri;
+        }
+        if self.catalog.is_none() {
+            self.catalog = catalog;
+        }
+        if !self.schema_evolution.enabled
+            && !self.schema_evolution.applied
+            && self.schema_evolution.added_columns.is_empty()
+            && !self.schema_evolution.incompatible_changes_detected
+            && self.schema_evolution.mode.is_empty()
+        {
+            self.schema_evolution = schema_evolution;
+        }
+
+        self.metrics.total_bytes_written = merge_option_u64(
+            self.metrics.total_bytes_written,
+            metrics.total_bytes_written,
+            first_merge,
+        );
+        self.metrics.small_files_count = merge_option_u64(
+            self.metrics.small_files_count,
+            metrics.small_files_count,
+            first_merge,
+        );
+        self.metrics.avg_file_size_mb = recompute_avg_file_size_mb(
+            self.metrics.total_bytes_written,
+            self.files_written,
+            self.parts_written,
+        );
+
+        if self.merge.is_none() {
+            self.merge = merge;
+        }
+
+        match (self.perf.take(), perf) {
+            (Some(a), Some(b)) => self.perf = Some(sum_perf_breakdown(a, b)),
+            (Some(a), None) => self.perf = Some(a),
+            (None, Some(b)) => self.perf = Some(b),
+            (None, None) => self.perf = None,
+        }
+    }
+}
+
+/// Sum two `Option<u64>` values with poison-on-unknown semantics: if either
+/// side is `None`, the result is `None`. Reporting a partial sum as if it
+/// were the total would silently under-count for any aggregation across
+/// flushes where one flush could not determine the underlying count
+/// (e.g. remote Delta commit-log read failures).
+fn sum_option_u64(a: Option<u64>, b: Option<u64>) -> Option<u64> {
+    match (a, b) {
+        (Some(a), Some(b)) => Some(a + b),
+        _ => None,
+    }
+}
+
+/// Progressive `Option<u64>` merge used by `merge_in`. On the first merge
+/// (when the accumulator has no flush recorded yet) the next flush's value is
+/// taken verbatim; on subsequent merges `sum_option_u64`'s poison-on-unknown
+/// semantics apply.
+fn merge_option_u64(acc: Option<u64>, next: Option<u64>, first_merge: bool) -> Option<u64> {
+    if first_merge {
+        next
+    } else {
+        sum_option_u64(acc, next)
+    }
+}
+
+fn recompute_avg_file_size_mb(
+    total_bytes: Option<u64>,
+    files_written: Option<u64>,
+    parts_written: u64,
+) -> Option<f64> {
+    let bytes = total_bytes?;
+    let denominator = files_written.unwrap_or(parts_written);
+    if denominator == 0 {
+        return None;
+    }
+    let mb = (bytes as f64) / (denominator as f64) / (1024.0 * 1024.0);
+    Some(mb)
+}
+
+fn sum_perf_breakdown(
+    a: AcceptedWritePerfBreakdown,
+    b: AcceptedWritePerfBreakdown,
+) -> AcceptedWritePerfBreakdown {
+    AcceptedWritePerfBreakdown {
+        conversion_ms: sum_option_u64(a.conversion_ms, b.conversion_ms),
+        source_df_build_ms: sum_option_u64(a.source_df_build_ms, b.source_df_build_ms),
+        merge_exec_ms: sum_option_u64(a.merge_exec_ms, b.merge_exec_ms),
+        data_write_ms: sum_option_u64(a.data_write_ms, b.data_write_ms),
+        commit_ms: sum_option_u64(a.commit_ms, b.commit_ms),
+        metrics_read_ms: sum_option_u64(a.metrics_read_ms, b.metrics_read_ms),
+    }
+}
+
 pub trait InputAdapter: Send + Sync {
     fn format(&self) -> &'static str;
 

{floe_python-0.4.5 → floe_python-0.4.6}/crates/floe-core/src/io/write/iceberg/rest.rs

@@ -45,6 +45,7 @@ pub(crate) async fn build_rest_catalog(
     }
 
     if let Some(credential) = rest_cfg.credential.as_deref() {
+        let credential = expand_env_refs(credential, &rest_cfg.catalog_name)?;
         if let Some(token_value) = credential.strip_prefix("token:") {
             // Bearer PAT (Unity Catalog / Nessie)
             props.insert("token".to_string(), token_value.to_string());
@@ -140,6 +141,39 @@ impl RestIcebergCatalogConfig {
     }
 }
 
+fn expand_env_refs(value: &str, catalog_name: &str) -> FloeResult<String> {
+    if !value.contains("${") {
+        return Ok(value.to_string());
+    }
+
+    let mut parts = Vec::new();
+    for part in value.split(':') {
+        parts.push(expand_env_ref_part(part, catalog_name)?);
+    }
+    Ok(parts.join(":"))
+}
+
+fn expand_env_ref_part(part: &str, catalog_name: &str) -> FloeResult<String> {
+    let Some(inner) = part.strip_prefix("${") else {
+        return Ok(part.to_string());
+    };
+    let Some(name) = inner.strip_suffix('}') else {
+        return Err(Box::new(RunError(format!(
+            "rest iceberg catalog {catalog_name} credential has unclosed env placeholder"
+        ))));
+    };
+    if name.is_empty() || name.contains('{') || name.contains('}') {
+        return Err(Box::new(RunError(format!(
+            "rest iceberg catalog {catalog_name} credential has invalid env placeholder"
+        ))));
+    }
+    std::env::var(name).map_err(|_| {
+        Box::new(RunError(format!(
+            "rest iceberg catalog {catalog_name} credential references env var {name} which is not set"
+        ))) as Box<dyn std::error::Error + Send + Sync>
+    })
+}
+
 pub(crate) async fn write_via_rest_catalog(
     rest_cfg: &RestIcebergCatalogConfig,
     table_root_uri: String,
@@ -342,3 +376,77 @@ async fn create_rest_table(
         .await
         .map_err(map_iceberg_err("rest catalog create_table failed"))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::expand_env_refs;
+
+    #[test]
+    fn expands_partial_env_refs_in_client_credentials() {
+        std::env::set_var("FLOE_TEST_REST_CLIENT_ID", "client-id");
+        std::env::set_var("FLOE_TEST_REST_CLIENT_SECRET", "client-secret");
+
+        let expanded = expand_env_refs(
+            "client_credentials:${FLOE_TEST_REST_CLIENT_ID}:${FLOE_TEST_REST_CLIENT_SECRET}",
+            "polaris",
+        )
+        .expect("expand credential");
+
+        assert_eq!(expanded, "client_credentials:client-id:client-secret");
+        std::env::remove_var("FLOE_TEST_REST_CLIENT_ID");
+        std::env::remove_var("FLOE_TEST_REST_CLIENT_SECRET");
+    }
+
+    #[test]
+    fn expands_exact_env_ref_in_token_credential() {
+        std::env::set_var("FLOE_TEST_REST_TOKEN", "pat-token");
+
+        let expanded =
+            expand_env_refs("token:${FLOE_TEST_REST_TOKEN}", "nessie").expect("expand token");
+
+        assert_eq!(expanded, "token:pat-token");
+        std::env::remove_var("FLOE_TEST_REST_TOKEN");
+    }
+
+    #[test]
+    fn preserves_literal_credential_text_that_contains_env_ref_syntax() {
+        let expanded =
+            expand_env_refs("token:abc${def}ghi", "nessie").expect("preserve literal credential");
+
+        assert_eq!(expanded, "token:abc${def}ghi");
+    }
+
+    #[test]
+    fn errors_when_env_ref_is_missing() {
+        std::env::remove_var("FLOE_TEST_REST_MISSING");
+
+        let err = expand_env_refs(
+            "client_credentials:${FLOE_TEST_REST_MISSING}:secret",
+            "polaris",
+        )
+        .unwrap_err();
+
+        assert_eq!(
+            err.to_string(),
+            "rest iceberg catalog polaris credential references env var FLOE_TEST_REST_MISSING which is not set"
+        );
+    }
+
+    #[test]
+    fn errors_on_malformed_env_ref() {
+        std::env::set_var("ID", "client-id");
+
+        let err = expand_env_refs(
+            "client_credentials:${ID}:literal-secret:${UNCLOSED",
+            "polaris",
+        )
+        .unwrap_err();
+
+        assert_eq!(
+            err.to_string(),
+            "rest iceberg catalog polaris credential has unclosed env placeholder"
+        );
+        assert!(!err.to_string().contains("literal-secret"));
+        std::env::remove_var("ID");
+    }
+}

{floe_python-0.4.5 → floe_python-0.4.6}/crates/floe-core/src/io/write/parquet.rs

@@ -1,6 +1,10 @@
 use std::path::Path;
 
-use polars::prelude::{DataFrame, ParquetCompression, ParquetWriter};
+use polars::polars_utils::plpath::PlPathRef;
+use polars::prelude::{
+    DataFrame, IntoLazy, ParquetCompression, ParquetWriteOptions, SinkOptions as PolarsSinkOptions,
+    SinkTarget,
+};
 
 use crate::checks::normalize::rename_output_columns;
 use crate::errors::{IoError, StorageError};
@@ -46,11 +50,32 @@ pub fn write_parquet_to_path(
     if let Some(parent) = output_path.parent() {
         std::fs::create_dir_all(parent)?;
     }
-    let file = std::fs::File::create(output_path)?;
-    let mut writer = ParquetWriter::new(file);
+    let write_options = build_parquet_write_options(options)?;
+    let sink_options = PolarsSinkOptions {
+        mkdir: false,
+        ..PolarsSinkOptions::default()
+    };
+    let target = SinkTarget::Path(PlPathRef::from_local_path(output_path).into_owned());
+    // The outer chunking loop in `ParquetSinkFormat::write` discards each
+    // chunk after the write, so taking the DataFrame here is safe. Using
+    // `std::mem::take` lets us hand ownership to `LazyFrame` without an
+    // extra clone of the Arrow buffers.
+    let frame = std::mem::take(df);
+    frame
+        .lazy()
+        .sink_parquet(target, write_options, None, sink_options)
+        .and_then(|lf| lf.with_new_streaming(true).collect())
+        .map_err(|err| Box::new(IoError(format!("parquet write failed: {err}"))))?;
+    Ok(())
+}
+
+fn build_parquet_write_options(
+    options: Option<&config::SinkOptions>,
+) -> FloeResult<ParquetWriteOptions> {
+    let mut write_options = ParquetWriteOptions::default();
     if let Some(options) = options {
         if let Some(compression) = &options.compression {
-            writer = writer.with_compression(parse_parquet_compression(compression)?);
+            write_options.compression = parse_parquet_compression(compression)?;
         }
         if let Some(row_group_size) = options.row_group_size {
             let row_group_size = usize::try_from(row_group_size).map_err(|_| {
@@ -58,13 +83,10 @@ pub fn write_parquet_to_path(
                     "parquet row_group_size is too large: {row_group_size}"
                 )))
             })?;
-            writer = writer.with_row_group_size(Some(row_group_size));
+            write_options.row_group_size = Some(row_group_size);
         }
     }
-    writer
-        .finish(df)
-        .map_err(|err| Box::new(IoError(format!("parquet write failed: {err}"))))?;
-    Ok(())
+    Ok(write_options)
 }
 
 impl SinkFormat for ParquetSinkFormat {

{floe_python-0.4.5 → floe_python-0.4.6}/crates/floe-core/src/manifest/builder.rs

@@ -6,9 +6,9 @@ use sha2::{Digest, Sha256};
 use crate::config::{ConfigLocation, RootConfig, SourceOptions, StorageResolver};
 use crate::manifest::model::{
     CommonManifest, ManifestArchiveTarget, ManifestColumnDef, ManifestDomain, ManifestEntity,
-    ManifestEntitySchema, ManifestExecution, ManifestExecutionDefaults, ManifestResultContract,
-    ManifestRunnerAuth, ManifestRunnerDefinition, ManifestRunnerResources, ManifestRunnerSecret,
-    ManifestRunners, ManifestSinkTarget, ManifestSinks, ManifestSource,
+    ManifestEntitySchema, ManifestExecution, ManifestExecutionDefaults, ManifestOrchestration,
+    ManifestResultContract, ManifestRunnerAuth, ManifestRunnerDefinition, ManifestRunnerResources,
+    ManifestRunnerSecret, ManifestRunners, ManifestSinkTarget, ManifestSinks, ManifestSource,
 };
 use crate::profile::ProfileConfig;
 use crate::FloeResult;
@@ -415,7 +415,7 @@ fn build_common_manifest(
                     .unwrap_or_else(|| domain.incoming_dir.clone()),
             })
             .collect(),
-        execution: default_execution_contract(options),
+        execution: default_execution_contract(options, profile),
         runners: runners_contract(profile),
         entities: manifest_entities,
         storages,
@@ -525,7 +525,10 @@ fn map_source_options(options: Option<&SourceOptions>) -> Option<serde_json::Val
     Some(serde_json::Value::Object(map))
 }
 
-fn default_execution_contract(options: &ManifestOptions) -> ManifestExecution {
+fn default_execution_contract(
+    options: &ManifestOptions,
+    profile: Option<&ProfileConfig>,
+) -> ManifestExecution {
     let mut exit_codes = BTreeMap::new();
     exit_codes.insert("0", "success_or_rejected");
     exit_codes.insert("1", "technical_failure");
@@ -554,6 +557,14 @@ fn default_execution_contract(options: &ManifestOptions) -> ManifestExecution {
     })
     .collect();
 
+    let orchestration = profile
+        .and_then(|p| p.execution.as_ref())
+        .and_then(|e| e.orchestration.as_ref())
+        .map(|o| ManifestOrchestration {
+            max_concurrent_entities: o.max_concurrent_entities,
+            strategy: o.strategy.clone(),
+        });
+
     ManifestExecution {
         entrypoint: "floe",
         base_args,
@@ -568,6 +579,7 @@ fn default_execution_contract(options: &ManifestOptions) -> ManifestExecution {
             env: BTreeMap::new(),
             workdir: None,
         },
+        orchestration,
     }
 }
 

{floe_python-0.4.5 → floe_python-0.4.6}/crates/floe-core/src/manifest/model.rs

@@ -49,6 +49,16 @@ pub struct ManifestExecution {
     pub log_format: &'static str,
     pub result_contract: ManifestResultContract,
     pub defaults: ManifestExecutionDefaults,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub orchestration: Option<ManifestOrchestration>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct ManifestOrchestration {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub max_concurrent_entities: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub strategy: Option<String>,
 }
 
 #[derive(Debug, Serialize)]

{floe_python-0.4.5 → floe_python-0.4.6}/crates/floe-core/src/profile/parse.rs

@@ -8,9 +8,9 @@ use crate::config::yaml_decode::{
     hash_get, load_yaml, validate_known_keys, yaml_array, yaml_hash, yaml_string,
 };
 use crate::profile::types::{
-    ProfileConfig, ProfileExecution, ProfileMetadata, ProfileRunner, ProfileRunnerAuth,
-    ProfileRunnerResources, ProfileRunnerSecret, ProfileValidation, PROFILE_API_VERSION,
-    PROFILE_KIND,
+    ProfileConfig, ProfileExecution, ProfileMetadata, ProfileOrchestration, ProfileRunner,
+    ProfileRunnerAuth, ProfileRunnerResources, ProfileRunnerSecret, ProfileValidation,
+    PROFILE_API_VERSION, PROFILE_KIND,
 };
 use crate::{ConfigError, FloeResult};
 
@@ -162,7 +162,7 @@ fn parse_metadata(value: &Yaml) -> FloeResult<ProfileMetadata> {
 
 fn parse_execution(value: &Yaml) -> FloeResult<ProfileExecution> {
     let hash = yaml_hash(value, "profile.execution")?;
-    validate_known_keys(hash, "profile.execution", &["runner"])?;
+    validate_known_keys(hash, "profile.execution", &["runner", "orchestration"])?;
 
     let runner_yaml = hash_get(hash, "runner").ok_or_else(|| {
         Box::new(ConfigError(
@@ -171,7 +171,51 @@ fn parse_execution(value: &Yaml) -> FloeResult<ProfileExecution> {
     })?;
     let runner = parse_runner(runner_yaml)?;
 
-    Ok(ProfileExecution { runner })
+    let orchestration = match hash_get(hash, "orchestration") {
+        Some(value) => Some(parse_orchestration(value)?),
+        None => None,
+    };
+
+    Ok(ProfileExecution {
+        runner,
+        orchestration,
+    })
+}
+
+fn parse_orchestration(value: &Yaml) -> FloeResult<ProfileOrchestration> {
+    let hash = yaml_hash(value, "profile.execution.orchestration")?;
+    validate_known_keys(
+        hash,
+        "profile.execution.orchestration",
+        &["max_concurrent_entities", "strategy"],
+    )?;
+
+    let max_concurrent_entities = get_optional_u64(
+        hash,
+        "max_concurrent_entities",
+        "profile.execution.orchestration",
+    )?;
+
+    if let Some(0) = max_concurrent_entities {
+        return Err(Box::new(ConfigError(
+            "profile.execution.orchestration.max_concurrent_entities: must be >= 1".to_string(),
+        )));
+    }
+
+    let strategy = get_optional_string(hash, "strategy", "profile.execution.orchestration")?;
+
+    if let Some(ref s) = strategy {
+        if s != "sequential" && s != "parallel" {
+            return Err(Box::new(ConfigError(format!(
+                "profile.execution.orchestration.strategy: expected \"sequential\" or \"parallel\", got \"{s}\""
+            ))));
+        }
+    }
+
+    Ok(ProfileOrchestration {
+        max_concurrent_entities,
+        strategy,
+    })
 }
 
 fn parse_runner(value: &Yaml) -> FloeResult<ProfileRunner> {

{floe_python-0.4.5 → floe_python-0.4.6}/crates/floe-core/src/profile/types.rs

@@ -27,6 +27,14 @@ pub struct ProfileMetadata {
 #[derive(Debug, Clone)]
 pub struct ProfileExecution {
     pub runner: ProfileRunner,
+    pub orchestration: Option<ProfileOrchestration>,
+}
+
+#[derive(Debug, Clone)]
+pub struct ProfileOrchestration {
+    pub max_concurrent_entities: Option<u64>,
+    /// "sequential" | "parallel"
+    pub strategy: Option<String>,
 }
 
 #[derive(Debug, Clone)]