floe-python 0.4.0__tar.gz → 0.4.1__tar.gz

{floe_python-0.4.0 → floe_python-0.4.1}/Cargo.lock

@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3412,7 +3412,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "apache-avro 0.16.0",
  "arrow",
@@ -3455,7 +3455,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.4.0 → floe_python-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.4.0
+Version: 0.4.1
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.4.0"
+version = "0.4.1"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"
@@ -41,7 +41,7 @@ uuid = "1"
 arrow = "57"
 iceberg = "0.9.0"
 iceberg-catalog-rest = "0.9.0"
-iceberg-storage-opendal = { version = "0.9.1", features = ["opendal-gcs"] }
+iceberg-storage-opendal = { version = "0.9.1", features = ["opendal-s3", "opendal-gcs"] }
 df-interchange = { version = "0.3.2", features = ["arrow_57", "polars_0_52"] }
 orc-rust = "0.7.1"
 

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/src/config/mod.rs

@@ -13,6 +13,9 @@ pub use storage::{resolve_local_path, ConfigBase, ResolvedPath, StorageResolver}
 pub use types::*;
 
 pub use parse::extract_raw_env_vars;
-pub(crate) use parse::{parse_catalogs_with_context, parse_config, parse_config_with_vars};
+pub(crate) use parse::{
+    parse_catalogs_with_context, parse_config, parse_config_with_vars, parse_lineage_config,
+    parse_storages,
+};
 pub(crate) use template::apply_templates_with_vars;
 pub(crate) use validate::{extract_first_n, extract_last_n, validate_config};

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/src/config/parse.rs

@@ -658,7 +658,7 @@ fn parse_sink_delta_options(value: &Yaml, ctx: &str) -> FloeResult<DeltaSinkTarg
     })
 }
 
-fn parse_storages(value: &Yaml) -> FloeResult<StoragesConfig> {
+pub(crate) fn parse_storages(value: &Yaml) -> FloeResult<StoragesConfig> {
     let hash = yaml_hash(value, "storages")?;
     validate_known_keys(hash, "storages", &["default", "definitions"])?;
     let definitions_yaml = match hash_get(hash, "definitions") {
@@ -1129,7 +1129,7 @@ fn parse_pii_column(value: &Yaml) -> FloeResult<PiiColumnConfig> {
     })
 }
 
-fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
+pub(crate) fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
     let hash = yaml_hash(value, "lineage")?;
     validate_known_keys(
         hash,

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/src/config/storage.rs

@@ -615,7 +615,7 @@ fn parent_prefix(key: &str) -> String {
     }
 }
 
-fn is_remote_uri(value: &str) -> bool {
+pub(crate) fn is_remote_uri(value: &str) -> bool {
     value.starts_with("s3://") || value.starts_with("gs://") || value.starts_with("abfs://")
 }
 

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/src/config/validate.rs

@@ -1,5 +1,6 @@
 use std::collections::HashSet;
 
+use crate::config::storage::is_remote_uri;
 use crate::config::{
     CatalogDefinition, CatalogTypeConfig, EntityConfig, IncrementalMode, PolicySeverity,
     RootConfig, SourceOptions, StorageDefinition,
@@ -128,6 +129,12 @@ fn validate_report(
 ) -> FloeResult<()> {
     let storage_name = storages.resolve_report_name(report.storage.as_deref())?;
     storages.validate_report_reference("report.storage", &storage_name)?;
+    if storages.definition_type(&storage_name) == Some("local") && is_remote_uri(&report.path) {
+        return Err(Box::new(ConfigError(format!(
+            "report.path must be a local path (got {})",
+            report.path
+        ))));
+    }
     Ok(())
 }
 

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/src/io/write/iceberg.rs

@@ -8,6 +8,7 @@ use iceberg::memory::{MemoryCatalogBuilder, MEMORY_CATALOG_WAREHOUSE};
 use iceberg::spec::{Schema, UnboundPartitionSpec};
 use iceberg::transaction::{ApplyTransactionAction, Transaction};
 use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableIdent};
+use iceberg_storage_opendal::OpenDalStorageFactory;
 use polars::prelude::DataFrame;
 
 use crate::errors::RunError;
@@ -375,6 +376,7 @@ async fn write_iceberg_table_async(
     catalog_props.insert(MEMORY_CATALOG_WAREHOUSE.to_string(), table_root_uri.clone());
 
     let is_local = !table_root_uri.starts_with("s3://")
+        && !table_root_uri.starts_with("s3a://")
         && !table_root_uri.starts_with("gs://")
         && !table_root_uri.starts_with("az://")
         && !table_root_uri.starts_with("abfss://");
@@ -382,6 +384,20 @@ async fn write_iceberg_table_async(
     if is_local {
         catalog_builder =
             catalog_builder.with_storage_factory(std::sync::Arc::new(LocalFsStorageFactory));
+    } else if table_root_uri.starts_with("s3://") || table_root_uri.starts_with("s3a://") {
+        let scheme = table_root_uri
+            .split("://")
+            .next()
+            .unwrap_or("s3")
+            .to_string();
+        catalog_builder =
+            catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::S3 {
+                configured_scheme: scheme,
+                customized_credential_load: None,
+            }));
+    } else if table_root_uri.starts_with("gs://") {
+        catalog_builder =
+            catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::Gcs));
     }
     let catalog = catalog_builder
         .load(catalog_name, catalog_props)
@@ -698,17 +714,35 @@ async fn collect_iceberg_batches(
     use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableIdent};
 
     let is_local = !warehouse_location.starts_with("s3://")
+        && !warehouse_location.starts_with("s3a://")
         && !warehouse_location.starts_with("gs://")
         && !warehouse_location.starts_with("az://")
         && !warehouse_location.starts_with("abfss://");
 
     let mut props = catalog_props;
-    props.insert(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse_location);
+    props.insert(
+        MEMORY_CATALOG_WAREHOUSE.to_string(),
+        warehouse_location.clone(),
+    );
 
     let mut catalog_builder = MemoryCatalogBuilder::default();
     if is_local {
         catalog_builder =
             catalog_builder.with_storage_factory(std::sync::Arc::new(LocalFsStorageFactory));
+    } else if warehouse_location.starts_with("s3://") || warehouse_location.starts_with("s3a://") {
+        let scheme = warehouse_location
+            .split("://")
+            .next()
+            .unwrap_or("s3")
+            .to_string();
+        catalog_builder =
+            catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::S3 {
+                configured_scheme: scheme,
+                customized_credential_load: None,
+            }));
+    } else if warehouse_location.starts_with("gs://") {
+        catalog_builder =
+            catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::Gcs));
     }
     let catalog = catalog_builder.load(ICEBERG_CATALOG_NAME, props).await?;
 

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/src/lib.rs

@@ -40,6 +40,8 @@ pub struct ValidateOptions {
     pub entities: Vec<String>,
     pub profile_vars: std::collections::HashMap<String, String>,
     pub profile_catalogs: Option<config::CatalogsConfig>,
+    pub profile_storages: Option<config::StoragesConfig>,
+    pub profile_lineage: Option<config::LineageConfig>,
 }
 
 #[derive(Debug, Default)]
@@ -62,6 +64,8 @@ pub fn validate_with_base(
 ) -> FloeResult<()> {
     let mut config = config::parse_config_with_vars(config_path, &options.profile_vars)?;
     apply_profile_catalogs(&mut config, options.profile_catalogs.as_ref());
+    apply_profile_storages(&mut config, options.profile_storages.as_ref());
+    apply_profile_lineage(&mut config, options.profile_lineage.as_ref());
     config::validate_config(&config)?;
 
     if !options.entities.is_empty() {
@@ -86,9 +90,13 @@ pub fn load_config_with_profile_overrides(
     config_path: &Path,
     profile_vars: &std::collections::HashMap<String, String>,
     profile_catalogs: Option<&config::CatalogsConfig>,
+    profile_storages: Option<&config::StoragesConfig>,
+    profile_lineage: Option<&config::LineageConfig>,
 ) -> FloeResult<config::RootConfig> {
     let mut config = config::parse_config_with_vars(config_path, profile_vars)?;
     apply_profile_catalogs(&mut config, profile_catalogs);
+    apply_profile_storages(&mut config, profile_storages);
+    apply_profile_lineage(&mut config, profile_lineage);
     Ok(config)
 }
 
@@ -107,6 +115,24 @@ pub(crate) fn apply_profile_catalogs(
     }
 }
 
+pub(crate) fn apply_profile_storages(
+    config: &mut config::RootConfig,
+    profile_storages: Option<&config::StoragesConfig>,
+) {
+    if let Some(storages) = profile_storages {
+        config.storages = Some(storages.clone());
+    }
+}
+
+pub(crate) fn apply_profile_lineage(
+    config: &mut config::RootConfig,
+    profile_lineage: Option<&config::LineageConfig>,
+) {
+    if let Some(lineage) = profile_lineage {
+        config.lineage = Some(lineage.clone());
+    }
+}
+
 pub fn extract_config_env_vars(
     config_path: &Path,
 ) -> FloeResult<std::collections::HashMap<String, String>> {

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/src/profile/parse.rs

@@ -56,6 +56,8 @@ fn parse_profile_doc(doc: &Yaml) -> FloeResult<ProfileConfig> {
             "execution",
             "variables",
             "catalogs",
+            "storages",
+            "lineage",
             "validation",
         ],
     )?;
@@ -98,6 +100,16 @@ fn parse_profile_doc(doc: &Yaml) -> FloeResult<ProfileConfig> {
         None => None,
     };
 
+    let storages = match hash_get(root, "storages") {
+        Some(value) => Some(crate::config::parse_storages(value)?),
+        None => None,
+    };
+
+    let lineage = match hash_get(root, "lineage") {
+        Some(value) => Some(crate::config::parse_lineage_config(value)?),
+        None => None,
+    };
+
     let validation = match hash_get(root, "validation") {
         Some(value) => Some(parse_validation(value)?),
         None => None,
@@ -110,6 +122,8 @@ fn parse_profile_doc(doc: &Yaml) -> FloeResult<ProfileConfig> {
         execution,
         variables,
         catalogs,
+        storages,
+        lineage,
         validation,
     })
 }

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/src/profile/types.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 
-use crate::config::CatalogsConfig;
+use crate::config::{CatalogsConfig, LineageConfig, StoragesConfig};
 
 /// Top-level profile document (apiVersion + kind + sections).
 #[derive(Debug, Clone)]
@@ -11,6 +11,8 @@ pub struct ProfileConfig {
     pub execution: Option<ProfileExecution>,
     pub variables: HashMap<String, String>,
     pub catalogs: Option<CatalogsConfig>,
+    pub storages: Option<StoragesConfig>,
+    pub lineage: Option<LineageConfig>,
     pub validation: Option<ProfileValidation>,
 }
 

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/src/profile/validate.rs

@@ -1,5 +1,6 @@
 use std::collections::{HashMap, HashSet};
 
+use crate::config::{LineageConfig, StoragesConfig};
 use crate::profile::types::{ProfileConfig, ProfileRunner};
 use crate::{ConfigError, FloeResult};
 
@@ -25,6 +26,12 @@ pub fn validate_profile(profile: &ProfileConfig) -> FloeResult<()> {
 
     validate_no_malformed_vars(&profile.variables)?;
     validate_profile_catalogs(profile)?;
+    if let Some(storages) = &profile.storages {
+        validate_profile_storages(storages)?;
+    }
+    if let Some(lineage) = &profile.lineage {
+        validate_profile_lineage(lineage)?;
+    }
 
     Ok(())
 }
@@ -66,6 +73,101 @@ fn validate_profile_catalogs(profile: &ProfileConfig) -> FloeResult<()> {
     Ok(())
 }
 
+fn validate_profile_storages(storages: &StoragesConfig) -> FloeResult<()> {
+    if storages.definitions.is_empty() {
+        return Err(Box::new(ConfigError(
+            "profile.storages.definitions must not be empty".to_string(),
+        )));
+    }
+    const ALLOWED_STORAGE_TYPES: &[&str] = &["local", "s3", "adls", "gcs"];
+    let mut names = HashSet::new();
+    for definition in &storages.definitions {
+        if definition.name.trim().is_empty() {
+            return Err(Box::new(ConfigError(
+                "profile.storages.definitions.name must not be empty".to_string(),
+            )));
+        }
+        if !names.insert(definition.name.as_str()) {
+            return Err(Box::new(ConfigError(format!(
+                "profile.storages.definitions name={} is duplicated",
+                definition.name
+            ))));
+        }
+        if !ALLOWED_STORAGE_TYPES.contains(&definition.fs_type.as_str()) {
+            return Err(Box::new(ConfigError(format!(
+                "profile.storages.definitions name={} type={} is unsupported (allowed: {})",
+                definition.name,
+                definition.fs_type,
+                ALLOWED_STORAGE_TYPES.join(", ")
+            ))));
+        }
+        if definition.fs_type == "s3" {
+            if definition.bucket.is_none() {
+                return Err(Box::new(ConfigError(format!(
+                    "profile.storages.definitions name={} requires bucket for type s3",
+                    definition.name
+                ))));
+            }
+            if definition.region.is_none() {
+                return Err(Box::new(ConfigError(format!(
+                    "profile.storages.definitions name={} requires region for type s3",
+                    definition.name
+                ))));
+            }
+        }
+        if definition.fs_type == "adls" {
+            if definition.account.is_none() {
+                return Err(Box::new(ConfigError(format!(
+                    "profile.storages.definitions name={} requires account for type adls",
+                    definition.name
+                ))));
+            }
+            if definition.container.is_none() {
+                return Err(Box::new(ConfigError(format!(
+                    "profile.storages.definitions name={} requires container for type adls",
+                    definition.name
+                ))));
+            }
+        }
+        if definition.fs_type == "gcs" && definition.bucket.is_none() {
+            return Err(Box::new(ConfigError(format!(
+                "profile.storages.definitions name={} requires bucket for type gcs",
+                definition.name
+            ))));
+        }
+    }
+    let Some(default_name) = &storages.default else {
+        return Err(Box::new(ConfigError(
+            "profile.storages.default is required when storages is set".to_string(),
+        )));
+    };
+    if !names.contains(default_name.as_str()) {
+        return Err(Box::new(ConfigError(format!(
+            "profile.storages.default={default_name} does not match any definition"
+        ))));
+    }
+    Ok(())
+}
+
+fn validate_profile_lineage(lineage: &LineageConfig) -> FloeResult<()> {
+    if lineage.url.trim().is_empty() {
+        return Err(Box::new(ConfigError(
+            "profile.lineage.url must not be empty".to_string(),
+        )));
+    }
+    if lineage.namespace.trim().is_empty() {
+        return Err(Box::new(ConfigError(
+            "profile.lineage.namespace must not be empty".to_string(),
+        )));
+    }
+    if lineage.max_failures == Some(0) {
+        return Err(Box::new(ConfigError(
+            "profile.lineage.max_failures must be at least 1".to_string(),
+        )));
+    }
+    Ok(())
+}
+
 /// Validate a variable map produced by merging sources in precedence order:
 ///
 ///   config variables  >  CLI overrides  >  profile variables

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/src/run/context.rs

@@ -33,6 +33,20 @@ impl RunContext {
                 .as_ref()
                 .and_then(|profile| profile.catalogs.as_ref()),
         );
+        crate::apply_profile_storages(
+            &mut config,
+            options
+                .profile
+                .as_ref()
+                .and_then(|profile| profile.storages.as_ref()),
+        );
+        crate::apply_profile_lineage(
+            &mut config,
+            options
+                .profile
+                .as_ref()
+                .and_then(|profile| profile.lineage.as_ref()),
+        );
         let storage_resolver = config::StorageResolver::new(&config, config_base)?;
         let catalog_resolver = config::CatalogResolver::new(&config)?;
         let config_dir =

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/src/run/mod.rs

@@ -120,6 +120,14 @@ pub fn run_with_runtime(
             .profile
             .as_ref()
             .and_then(|profile| profile.catalogs.clone()),
+        profile_storages: options
+            .profile
+            .as_ref()
+            .and_then(|profile| profile.storages.clone()),
+        profile_lineage: options
+            .profile
+            .as_ref()
+            .and_then(|profile| profile.lineage.clone()),
     };
     crate::validate_with_base(config_path, config_base.clone(), validate_options)?;
     let context = RunContext::new(config_path, config_base, &options, profile_vars)?;

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-core/tests/unit/config/config_validation.rs

@@ -109,6 +109,38 @@ entities:
     assert_validation_ok(&yaml);
 }
 
+#[test]
+fn remote_report_path_rejected_when_storage_is_local() {
+    let yaml = format!(
+        r#"version: "0.1"
+report:
+  path: "s3://floe-demo-bucket/reports"
+entities:
+{}"#,
+        base_entity("customer")
+    );
+    assert_validation_error(
+        &yaml,
+        &["report.path must be a local path (got s3://floe-demo-bucket/reports)"],
+    );
+}
+
+#[test]
+fn remote_report_path_rejected_for_gcs() {
+    let yaml = format!(
+        r#"version: "0.1"
+report:
+  path: "gs://floe-demo-bucket/reports"
+entities:
+{}"#,
+        base_entity("customer")
+    );
+    assert_validation_error(
+        &yaml,
+        &["report.path must be a local path (got gs://floe-demo-bucket/reports)"],
+    );
+}
+
 #[test]
 fn supported_config_versions_are_valid() {
     for version in &["0.2", "0.3"] {

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-python"
-version = "0.4.0"
+version = "0.4.1"
 edition = "2021"
 description = "Python bindings for floe-core (PyO3-based)"
 license = "Apache-2.0"
@@ -14,7 +14,7 @@ name = "_floe"
 crate-type = ["cdylib"]
 
 [dependencies]
-floe-core = { path = "../floe-core", version = "0.4.0" }
+floe-core = { path = "../floe-core", version = "0.4.1" }
 pyo3 = { version = "0.22", features = ["extension-module", "abi3-py310"] }
 serde_json = "1"
 

{floe_python-0.4.0 → floe_python-0.4.1}/crates/floe-python/src/functions.rs

@@ -38,6 +38,8 @@ fn load_optional_profile(
             variables: vars,
             validation: None,
             catalogs: None,
+            storages: None,
+            lineage: None,
         })),
     }
 }
@@ -59,7 +61,9 @@ pub fn validate(
             .as_ref()
             .map(|p| p.variables.clone())
             .unwrap_or_default(),
-        profile_catalogs: profile.and_then(|p| p.catalogs),
+        profile_catalogs: profile.as_ref().and_then(|p| p.catalogs.clone()),
+        profile_storages: profile.as_ref().and_then(|p| p.storages.clone()),
+        profile_lineage: profile.and_then(|p| p.lineage),
     };
     py.allow_threads(|| floe_core::validate(&path, options))
         .map_err(to_py_err)

{floe_python-0.4.0 → floe_python-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "floe-python"
-version = "0.4.0"
+version = "0.4.1"
 description = "Python bindings for floe-core — run floe pipelines at Rust speed from Python and notebooks"
 readme = "README.md"
 requires-python = ">=3.10"