floe-python 0.4.0__tar.gz → 0.4.2__tar.gz

{floe_python-0.4.0 → floe_python-0.4.2}/Cargo.lock

@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.4.0"
+version = "0.4.2"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3412,7 +3412,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.4.0"
+version = "0.4.2"
 dependencies = [
  "apache-avro 0.16.0",
  "arrow",
@@ -3455,7 +3455,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.4.0"
+version = "0.4.2"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.4.0 → floe_python-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.4.0
+Version: 0.4.2
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.4.0 → floe_python-0.4.2}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.4.0"
+version = "0.4.2"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"
@@ -41,7 +41,7 @@ uuid = "1"
 arrow = "57"
 iceberg = "0.9.0"
 iceberg-catalog-rest = "0.9.0"
-iceberg-storage-opendal = { version = "0.9.1", features = ["opendal-gcs"] }
+iceberg-storage-opendal = { version = "0.9.1", features = ["opendal-s3", "opendal-gcs"] }
 df-interchange = { version = "0.3.2", features = ["arrow_57", "polars_0_52"] }
 orc-rust = "0.7.1"
 

{floe_python-0.4.0 → floe_python-0.4.2}/crates/floe-core/README.md

@@ -73,7 +73,7 @@ floe run      -c config.yml   # run the pipeline
 | S3 storage | [docs/storages/s3.md](docs/storages/s3.md) |
 | ADLS storage | [docs/storages/adls.md](docs/storages/adls.md) |
 | GCS storage | [docs/storages/gcs.md](docs/storages/gcs.md) |
-| Incremental ingestion | [docs/profiles.md](docs/profiles.md) |
+| Environement specific profile config | [docs/profiles.md](docs/profiles.md) [docs/variables.md](docs/variables.md) |
 | Run reports | [docs/report.md](docs/report.md) |
 | CLI reference | [docs/cli.md](docs/cli.md) |
 | Orchestration (Dagster / Airflow) | [docs/summary.md](docs/summary.md) |

{floe_python-0.4.0 → floe_python-0.4.2}/crates/floe-core/src/config/mod.rs

@@ -13,6 +13,9 @@ pub use storage::{resolve_local_path, ConfigBase, ResolvedPath, StorageResolver}
 pub use types::*;
 
 pub use parse::extract_raw_env_vars;
-pub(crate) use parse::{parse_catalogs_with_context, parse_config, parse_config_with_vars};
+pub(crate) use parse::{
+    parse_catalogs_with_context, parse_config, parse_config_with_vars, parse_lineage_config,
+    parse_storages,
+};
 pub(crate) use template::apply_templates_with_vars;
 pub(crate) use validate::{extract_first_n, extract_last_n, validate_config};

{floe_python-0.4.0 → floe_python-0.4.2}/crates/floe-core/src/config/parse.rs

@@ -658,7 +658,7 @@ fn parse_sink_delta_options(value: &Yaml, ctx: &str) -> FloeResult<DeltaSinkTarg
     })
 }
 
-fn parse_storages(value: &Yaml) -> FloeResult<StoragesConfig> {
+pub(crate) fn parse_storages(value: &Yaml) -> FloeResult<StoragesConfig> {
     let hash = yaml_hash(value, "storages")?;
     validate_known_keys(hash, "storages", &["default", "definitions"])?;
     let definitions_yaml = match hash_get(hash, "definitions") {
@@ -1129,7 +1129,7 @@ fn parse_pii_column(value: &Yaml) -> FloeResult<PiiColumnConfig> {
     })
 }
 
-fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
+pub(crate) fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
     let hash = yaml_hash(value, "lineage")?;
     validate_known_keys(
         hash,

{floe_python-0.4.0 → floe_python-0.4.2}/crates/floe-core/src/config/storage.rs

@@ -615,7 +615,7 @@ fn parent_prefix(key: &str) -> String {
     }
 }
 
-fn is_remote_uri(value: &str) -> bool {
+pub(crate) fn is_remote_uri(value: &str) -> bool {
     value.starts_with("s3://") || value.starts_with("gs://") || value.starts_with("abfs://")
 }
 

{floe_python-0.4.0 → floe_python-0.4.2}/crates/floe-core/src/config/types.rs

@@ -1,6 +1,8 @@
 use std::collections::HashMap;
 use std::path::Path;
 
+use serde::{Deserialize, Serialize};
+
 use polars::polars_utils::pl_str::PlSmallStr;
 use polars::prelude::{
     CsvEncoding, CsvParseOptions, CsvReadOptions, DataType, NullValues, TimeUnit,
@@ -21,7 +23,7 @@ pub struct RootConfig {
     pub entities: Vec<EntityConfig>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct LineageConfig {
     pub url: String,
     pub api_key: Option<String>,
@@ -66,12 +68,12 @@ pub struct EntityConfig {
     pub pii: Option<PiiConfig>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct PiiConfig {
     pub columns: Vec<PiiColumnConfig>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct PiiColumnConfig {
     pub name: String,
     pub strategy: PiiStrategy,
@@ -79,7 +81,8 @@ pub struct PiiColumnConfig {
     pub redact_value: Option<String>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum PiiStrategy {
     Hash,
     Drop,
@@ -103,7 +106,8 @@ impl EntityConfig {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum IncrementalMode {
     #[default]
     None,
@@ -146,7 +150,7 @@ pub struct SourceConfig {
     pub cast_mode: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Deserialize)]
 pub struct SourceOptions {
     pub header: Option<bool>,
     pub separator: Option<String>,
@@ -262,7 +266,8 @@ pub struct SinkConfig {
     pub archive: Option<ArchiveTarget>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
 pub enum WriteMode {
     #[default]
     Overwrite,
@@ -300,34 +305,34 @@ pub const DEFAULT_SCD2_CURRENT_FLAG_COLUMN: &str = "__floe_is_current";
 pub const DEFAULT_SCD2_VALID_FROM_COLUMN: &str = "__floe_valid_from";
 pub const DEFAULT_SCD2_VALID_TO_COLUMN: &str = "__floe_valid_to";
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct MergeOptionsConfig {
     pub ignore_columns: Option<Vec<String>>,
     pub compare_columns: Option<Vec<String>>,
     pub scd2: Option<MergeScd2OptionsConfig>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct MergeScd2OptionsConfig {
     pub current_flag_column: Option<String>,
     pub valid_from_column: Option<String>,
     pub valid_to_column: Option<String>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct SinkOptions {
     pub compression: Option<String>,
     pub row_group_size: Option<u64>,
     pub max_size_per_file: Option<u64>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct IcebergPartitionFieldConfig {
     pub column: String,
     pub transform: String,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct IcebergSinkTargetConfig {
     pub catalog: Option<String>,
     pub namespace: Option<String>,
@@ -335,7 +340,7 @@ pub struct IcebergSinkTargetConfig {
     pub location: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DeltaSinkTargetConfig {
     /// Name of the catalog definition to use (falls back to `catalogs.default`).
     pub catalog: Option<String>,
@@ -345,13 +350,13 @@ pub struct DeltaSinkTargetConfig {
     pub table: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct StoragesConfig {
     pub default: Option<String>,
     pub definitions: Vec<StorageDefinition>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct StorageDefinition {
     pub name: String,
     pub fs_type: String,
@@ -362,7 +367,7 @@ pub struct StorageDefinition {
     pub prefix: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct CatalogsConfig {
     pub default: Option<String>,
     pub definitions: Vec<CatalogDefinition>,
@@ -371,7 +376,8 @@ pub struct CatalogsConfig {
 /// Type-specific configuration for a catalog definition.
 /// Each variant carries only the fields relevant to that catalog type.
 /// Add a new variant here when supporting a new catalog type.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "lowercase")]
 pub enum CatalogTypeConfig {
     Glue {
         region: String,
@@ -419,7 +425,7 @@ impl CatalogTypeConfig {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct CatalogDefinition {
     pub name: String,
     pub type_config: CatalogTypeConfig,
@@ -440,7 +446,8 @@ pub struct ArchiveTarget {
     pub storage: Option<String>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum PolicySeverity {
     #[default]
     Warn,
@@ -469,7 +476,7 @@ pub struct PolicyConfig {
     pub severity: PolicySeverity,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize)]
 pub struct SchemaConfig {
     pub normalize_columns: Option<NormalizeColumnsConfig>,
     pub mismatch: Option<SchemaMismatchConfig>,
@@ -485,19 +492,20 @@ impl SchemaConfig {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct NormalizeColumnsConfig {
     pub enabled: Option<bool>,
     pub strategy: Option<String>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
 pub struct SchemaEvolutionConfig {
     pub mode: SchemaEvolutionMode,
     pub on_incompatible: SchemaEvolutionIncompatibleAction,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
 pub enum SchemaEvolutionMode {
     #[default]
     Strict,
@@ -513,7 +521,8 @@ impl SchemaEvolutionMode {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum SchemaEvolutionIncompatibleAction {
     #[default]
     Fail,
@@ -527,13 +536,13 @@ impl SchemaEvolutionIncompatibleAction {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct SchemaMismatchConfig {
     pub missing_columns: Option<String>,
     pub extra_columns: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize)]
 pub struct ColumnConfig {
     pub name: String,
     pub source: Option<String>,

{floe_python-0.4.0 → floe_python-0.4.2}/crates/floe-core/src/config/validate.rs

@@ -1,5 +1,6 @@
 use std::collections::HashSet;
 
+use crate::config::storage::is_remote_uri;
 use crate::config::{
     CatalogDefinition, CatalogTypeConfig, EntityConfig, IncrementalMode, PolicySeverity,
     RootConfig, SourceOptions, StorageDefinition,
@@ -128,6 +129,12 @@ fn validate_report(
 ) -> FloeResult<()> {
     let storage_name = storages.resolve_report_name(report.storage.as_deref())?;
     storages.validate_report_reference("report.storage", &storage_name)?;
+    if storages.definition_type(&storage_name) == Some("local") && is_remote_uri(&report.path) {
+        return Err(Box::new(ConfigError(format!(
+            "report.path must be a local path (got {})",
+            report.path
+        ))));
+    }
     Ok(())
 }
 

{floe_python-0.4.0 → floe_python-0.4.2}/crates/floe-core/src/io/write/iceberg.rs

@@ -8,6 +8,7 @@ use iceberg::memory::{MemoryCatalogBuilder, MEMORY_CATALOG_WAREHOUSE};
 use iceberg::spec::{Schema, UnboundPartitionSpec};
 use iceberg::transaction::{ApplyTransactionAction, Transaction};
 use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableIdent};
+use iceberg_storage_opendal::OpenDalStorageFactory;
 use polars::prelude::DataFrame;
 
 use crate::errors::RunError;
@@ -375,6 +376,7 @@ async fn write_iceberg_table_async(
     catalog_props.insert(MEMORY_CATALOG_WAREHOUSE.to_string(), table_root_uri.clone());
 
     let is_local = !table_root_uri.starts_with("s3://")
+        && !table_root_uri.starts_with("s3a://")
         && !table_root_uri.starts_with("gs://")
         && !table_root_uri.starts_with("az://")
         && !table_root_uri.starts_with("abfss://");
@@ -382,6 +384,20 @@ async fn write_iceberg_table_async(
     if is_local {
         catalog_builder =
             catalog_builder.with_storage_factory(std::sync::Arc::new(LocalFsStorageFactory));
+    } else if table_root_uri.starts_with("s3://") || table_root_uri.starts_with("s3a://") {
+        let scheme = table_root_uri
+            .split("://")
+            .next()
+            .unwrap_or("s3")
+            .to_string();
+        catalog_builder =
+            catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::S3 {
+                configured_scheme: scheme,
+                customized_credential_load: None,
+            }));
+    } else if table_root_uri.starts_with("gs://") {
+        catalog_builder =
+            catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::Gcs));
     }
     let catalog = catalog_builder
         .load(catalog_name, catalog_props)
@@ -698,17 +714,35 @@ async fn collect_iceberg_batches(
     use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableIdent};
 
     let is_local = !warehouse_location.starts_with("s3://")
+        && !warehouse_location.starts_with("s3a://")
         && !warehouse_location.starts_with("gs://")
         && !warehouse_location.starts_with("az://")
         && !warehouse_location.starts_with("abfss://");
 
     let mut props = catalog_props;
-    props.insert(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse_location);
+    props.insert(
+        MEMORY_CATALOG_WAREHOUSE.to_string(),
+        warehouse_location.clone(),
+    );
 
     let mut catalog_builder = MemoryCatalogBuilder::default();
     if is_local {
         catalog_builder =
             catalog_builder.with_storage_factory(std::sync::Arc::new(LocalFsStorageFactory));
+    } else if warehouse_location.starts_with("s3://") || warehouse_location.starts_with("s3a://") {
+        let scheme = warehouse_location
+            .split("://")
+            .next()
+            .unwrap_or("s3")
+            .to_string();
+        catalog_builder =
+            catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::S3 {
+                configured_scheme: scheme,
+                customized_credential_load: None,
+            }));
+    } else if warehouse_location.starts_with("gs://") {
+        catalog_builder =
+            catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::Gcs));
     }
     let catalog = catalog_builder.load(ICEBERG_CATALOG_NAME, props).await?;
 

{floe_python-0.4.0 → floe_python-0.4.2}/crates/floe-core/src/lib.rs

@@ -17,18 +17,23 @@ pub mod state;
 pub mod vars;
 pub mod warnings;
 
-pub use crate::state::{inspect_entity_state_with_base, reset_entity_state_with_base};
+pub use crate::state::{
+    inspect_entity_state, inspect_entity_state_with_base, reset_entity_state,
+    reset_entity_state_with_base,
+};
 pub use add_entity::{add_entity_to_config, AddEntityOptions, AddEntityOutcome};
 pub use checks as check;
 pub use config::{resolve_config_location, ConfigLocation};
 pub use errors::ConfigError;
-pub use manifest::build_common_manifest_json;
+pub use manifest::{build_common_manifest_json, config_from_manifest_json};
 pub use profile::{
     detect_malformed_placeholder, detect_unresolved_placeholders, parse_profile,
     parse_profile_from_str, validate_merged_vars, validate_profile, ProfileConfig,
 };
 pub use run::events::{set_observer, MultiObserver, RunEvent, RunObserver};
-pub use run::{run, run_with_base, DryRunEntityPreview, EntityOutcome, RunOutcome};
+pub use run::{
+    run, run_with_base, run_with_manifest_path, DryRunEntityPreview, EntityOutcome, RunOutcome,
+};
 pub use runner::{parse_run_status_from_logs, ConnectorRunStatus};
 pub use runtime::{DefaultRuntime, Runtime};
 pub use vars::{resolve_vars, VarSources};
@@ -40,6 +45,8 @@ pub struct ValidateOptions {
     pub entities: Vec<String>,
     pub profile_vars: std::collections::HashMap<String, String>,
     pub profile_catalogs: Option<config::CatalogsConfig>,
+    pub profile_storages: Option<config::StoragesConfig>,
+    pub profile_lineage: Option<config::LineageConfig>,
 }
 
 #[derive(Debug, Default)]
@@ -62,6 +69,8 @@ pub fn validate_with_base(
 ) -> FloeResult<()> {
     let mut config = config::parse_config_with_vars(config_path, &options.profile_vars)?;
     apply_profile_catalogs(&mut config, options.profile_catalogs.as_ref());
+    apply_profile_storages(&mut config, options.profile_storages.as_ref());
+    apply_profile_lineage(&mut config, options.profile_lineage.as_ref());
     config::validate_config(&config)?;
 
     if !options.entities.is_empty() {
@@ -86,9 +95,13 @@ pub fn load_config_with_profile_overrides(
     config_path: &Path,
     profile_vars: &std::collections::HashMap<String, String>,
     profile_catalogs: Option<&config::CatalogsConfig>,
+    profile_storages: Option<&config::StoragesConfig>,
+    profile_lineage: Option<&config::LineageConfig>,
 ) -> FloeResult<config::RootConfig> {
     let mut config = config::parse_config_with_vars(config_path, profile_vars)?;
     apply_profile_catalogs(&mut config, profile_catalogs);
+    apply_profile_storages(&mut config, profile_storages);
+    apply_profile_lineage(&mut config, profile_lineage);
     Ok(config)
 }
 
@@ -107,6 +120,24 @@ pub(crate) fn apply_profile_catalogs(
     }
 }
 
+pub(crate) fn apply_profile_storages(
+    config: &mut config::RootConfig,
+    profile_storages: Option<&config::StoragesConfig>,
+) {
+    if let Some(storages) = profile_storages {
+        config.storages = Some(storages.clone());
+    }
+}
+
+pub(crate) fn apply_profile_lineage(
+    config: &mut config::RootConfig,
+    profile_lineage: Option<&config::LineageConfig>,
+) {
+    if let Some(lineage) = profile_lineage {
+        config.lineage = Some(lineage.clone());
+    }
+}
+
 pub fn extract_config_env_vars(
     config_path: &Path,
 ) -> FloeResult<std::collections::HashMap<String, String>> {