floe-python 0.4.4__tar.gz → 0.4.5__tar.gz

{floe_python-0.4.4 → floe_python-0.4.5}/Cargo.lock

@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.4.4"
+version = "0.4.5"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3412,7 +3412,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.4.4"
+version = "0.4.5"
 dependencies = [
  "apache-avro 0.16.0",
  "arrow",
@@ -3455,7 +3455,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.4.4"
+version = "0.4.5"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.4.4 → floe_python-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.4.4
+Version: 0.4.5
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.4.4 → floe_python-0.4.5}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.4.4"
+version = "0.4.5"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"

{floe_python-0.4.4 → floe_python-0.4.5}/crates/floe-core/src/config/location.rs

@@ -45,7 +45,7 @@ pub fn resolve_config_location(input: &str) -> FloeResult<ConfigLocation> {
 fn download_remote_config(uri: &str, temp_dir: &Path) -> FloeResult<PathBuf> {
     if uri.starts_with("s3://") {
         let location = storage::s3::parse_s3_uri(uri)?;
-        let client = storage::s3::S3Client::new(location.bucket, None)?;
+        let client = storage::s3::S3Client::new(location.bucket, None, None, None)?;
         return client.download_to_temp(uri, temp_dir);
     }
     if uri.starts_with("gs://") {
@@ -63,6 +63,8 @@ fn download_remote_config(uri: &str, temp_dir: &Path) -> FloeResult<PathBuf> {
             account: Some(location.account),
             container: Some(location.container),
             prefix: None,
+            endpoint: None,
+            path_style_access: None,
         };
         let client = storage::adls::AdlsClient::new(&definition)?;
         return client.download_to_temp(uri, temp_dir);
@@ -70,6 +72,44 @@ fn download_remote_config(uri: &str, temp_dir: &Path) -> FloeResult<PathBuf> {
     Err(format!("unsupported config uri: {}", uri).into())
 }
 
+/// Write `bytes` to a remote URI by staging them in a temp file then uploading.
+pub fn write_bytes_to_remote_uri(bytes: &[u8], uri: &str) -> FloeResult<()> {
+    let temp_dir = TempDir::new()?;
+    let local_path = temp_dir.path().join("upload");
+    std::fs::write(&local_path, bytes)?;
+    upload_to_remote_uri(&local_path, uri)
+}
+
+pub fn upload_to_remote_uri(local_path: &Path, uri: &str) -> FloeResult<()> {
+    if uri.starts_with("s3://") {
+        let location = storage::s3::parse_s3_uri(uri)?;
+        let client = storage::s3::S3Client::new(location.bucket, None, None, None)?;
+        return client.upload_from_path(local_path, uri);
+    }
+    if uri.starts_with("gs://") {
+        let location = storage::gcs::parse_gcs_uri(uri)?;
+        let client = storage::gcs::GcsClient::new(location.bucket)?;
+        return client.upload_from_path(local_path, uri);
+    }
+    if uri.starts_with("abfs://") {
+        let location = storage::adls::parse_adls_uri(uri)?;
+        let definition = StorageDefinition {
+            name: "manifest".to_string(),
+            fs_type: "adls".to_string(),
+            bucket: None,
+            region: None,
+            account: Some(location.account),
+            container: Some(location.container),
+            prefix: None,
+            endpoint: None,
+            path_style_access: None,
+        };
+        let client = storage::adls::AdlsClient::new(&definition)?;
+        return client.upload_from_path(local_path, uri);
+    }
+    Err(format!("unsupported manifest output uri: {uri}").into())
+}
+
 pub(crate) fn is_remote_uri(value: &str) -> bool {
     value.starts_with("s3://") || value.starts_with("gs://") || value.starts_with("abfs://")
 }

{floe_python-0.4.4 → floe_python-0.4.5}/crates/floe-core/src/config/mod.rs

@@ -9,7 +9,9 @@ pub(crate) mod yaml_decode;
 
 pub use catalog::{CatalogResolver, ResolvedDeltaCatalogTarget, ResolvedIcebergCatalogTarget};
 pub(crate) use location::is_remote_uri;
-pub use location::{resolve_config_location, ConfigLocation};
+pub use location::{
+    resolve_config_location, upload_to_remote_uri, write_bytes_to_remote_uri, ConfigLocation,
+};
 pub use storage::{resolve_local_path, ConfigBase, ResolvedPath, StorageResolver};
 pub use types::*;
 

{floe_python-0.4.4 → floe_python-0.4.5}/crates/floe-core/src/config/parse.rs

@@ -695,6 +695,8 @@ fn parse_storage_definition(value: &Yaml) -> FloeResult<StorageDefinition> {
             "account",
             "container",
             "prefix",
+            "endpoint",
+            "path_style_access",
         ],
     )?;
     Ok(StorageDefinition {
@@ -705,6 +707,8 @@ fn parse_storage_definition(value: &Yaml) -> FloeResult<StorageDefinition> {
         account: opt_string(hash, "account", "storages.definitions")?,
         container: opt_string(hash, "container", "storages.definitions")?,
         prefix: opt_string(hash, "prefix", "storages.definitions")?,
+        endpoint: opt_string(hash, "endpoint", "storages.definitions")?,
+        path_style_access: opt_bool(hash, "path_style_access", "storages.definitions")?,
     })
 }
 

{floe_python-0.4.4 → floe_python-0.4.5}/crates/floe-core/src/config/storage.rs

@@ -215,7 +215,7 @@ impl StorageResolver {
         raw_path: &str,
     ) -> FloeResult<ResolvedPath> {
         let name = storage_name.unwrap_or(self.default_name.as_str());
-        if !self.has_config && name != "local" {
+        if !self.has_config && name != "local" && !self.definitions.contains_key(name) {
             return Err(Box::new(ConfigError(format!(
                 "entity.name={} {field} references unknown storage {} (no storages block)",
                 entity_name, name
@@ -238,6 +238,8 @@ impl StorageResolver {
                 account: None,
                 container: None,
                 prefix: None,
+                endpoint: None,
+                path_style_access: None,
             }
         };
 
@@ -304,7 +306,7 @@ impl StorageResolver {
         raw_path: &str,
     ) -> FloeResult<ResolvedPath> {
         let name = storage_name.unwrap_or(self.default_name.as_str());
-        if !self.has_config && name != "local" {
+        if !self.has_config && name != "local" && !self.definitions.contains_key(name) {
             return Err(Box::new(ConfigError(format!(
                 "report.storage references unknown storage {} (no storages block)",
                 name
@@ -327,6 +329,8 @@ impl StorageResolver {
                 account: None,
                 container: None,
                 prefix: None,
+                endpoint: None,
+                path_style_access: None,
             }
         };
 
@@ -386,9 +390,48 @@ impl StorageResolver {
         }
     }
 
+    /// Scan definitions for the first one whose scheme and bucket/account match `uri`.
+    /// Used in manifest mode to resolve a bare report URI back to a named definition.
+    pub fn find_definition_name_for_uri(&self, uri: &str) -> Option<String> {
+        for (name, def) in &self.definitions {
+            if uri.starts_with("s3://") && def.fs_type == "s3" {
+                if let Some(b) = &def.bucket {
+                    if uri.starts_with(&format!("s3://{b}/")) || uri == format!("s3://{b}") {
+                        return Some(name.clone());
+                    }
+                }
+            }
+            if uri.starts_with("gs://") && def.fs_type == "gcs" {
+                if let Some(b) = &def.bucket {
+                    if uri.starts_with(&format!("gs://{b}/")) || uri == format!("gs://{b}") {
+                        return Some(name.clone());
+                    }
+                }
+            }
+            if uri.starts_with("abfs://") && def.fs_type == "adls" {
+                if let (Some(c), Some(a)) = (&def.container, &def.account) {
+                    if uri.starts_with(&format!("abfs://{c}@{a}.dfs.core.windows.net")) {
+                        return Some(name.clone());
+                    }
+                }
+            }
+        }
+        None
+    }
+
+    /// Register a synthetic `StorageDefinition` into this resolver.
+    /// Used in manifest mode when the report URI has no matching definition in the config.
+    /// Does NOT flip `has_config`; entity resolution keeps its implicit-local fallback.
+    pub fn register_definition(&mut self, definition: StorageDefinition) {
+        self.definitions.insert(definition.name.clone(), definition);
+    }
+
     pub fn definition(&self, name: &str) -> Option<StorageDefinition> {
         if self.has_config {
             self.definitions.get(name).cloned()
+        } else if let Some(def) = self.definitions.get(name) {
+            // Synthetic definition registered by register_definition (e.g. report target).
+            Some(def.clone())
         } else if name == "local" {
             Some(StorageDefinition {
                 name: "local".to_string(),
@@ -398,6 +441,8 @@ impl StorageResolver {
                 account: None,
                 container: None,
                 prefix: None,
+                endpoint: None,
+                path_style_access: None,
             })
         } else {
             None

{floe_python-0.4.4 → floe_python-0.4.5}/crates/floe-core/src/config/types.rs

@@ -366,6 +366,8 @@ pub struct StorageDefinition {
     pub account: Option<String>,
     pub container: Option<String>,
     pub prefix: Option<String>,
+    pub endpoint: Option<String>,
+    pub path_style_access: Option<bool>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

{floe_python-0.4.4 → floe_python-0.4.5}/crates/floe-core/src/io/storage/mod.rs

@@ -129,7 +129,12 @@ fn build_client(definition: &config::StorageDefinition) -> FloeResult<Box<dyn St
         "s3" => {
             let bucket =
                 validation::require_field(definition, definition.bucket.as_ref(), "bucket", "s3")?;
-            Box::new(s3::S3Client::new(bucket, definition.region.as_deref())?)
+            Box::new(s3::S3Client::new(
+                bucket,
+                definition.region.as_deref(),
+                definition.endpoint.as_deref(),
+                definition.path_style_access,
+            )?)
         }
         "adls" => Box::new(adls::AdlsClient::new(definition)?),
         "gcs" => {

{floe_python-0.4.4 → floe_python-0.4.5}/crates/floe-core/src/io/storage/object_store.rs

@@ -111,9 +111,16 @@ pub fn iceberg_store_config(
         Target::S3 { storage, uri, .. } => {
             let mut file_io_props = HashMap::new();
             if let Some(definition) = resolver.definition(storage) {
-                if let Some(region) = definition.region {
+                if let Some(region) = &definition.region {
                     file_io_props.insert(S3_REGION.to_string(), region.clone());
-                    file_io_props.insert(CLIENT_REGION.to_string(), region);
+                    file_io_props.insert(CLIENT_REGION.to_string(), region.clone());
+                }
+                if let Some(endpoint) = &definition.endpoint {
+                    file_io_props.insert("s3.endpoint".to_string(), endpoint.clone());
+                }
+                if let Some(path_style) = definition.path_style_access {
+                    file_io_props
+                        .insert("s3.path-style-access".to_string(), path_style.to_string());
                 }
             }
             Ok(IcebergStoreConfig {

{floe_python-0.4.4 → floe_python-0.4.5}/crates/floe-core/src/io/storage/providers/s3.rs

@@ -19,23 +19,35 @@ pub struct S3Client {
 }
 
 impl S3Client {
-    pub fn new(bucket: String, region: Option<&str>) -> FloeResult<Self> {
+    pub fn new(
+        bucket: String,
+        region: Option<&str>,
+        endpoint: Option<&str>,
+        path_style_access: Option<bool>,
+    ) -> FloeResult<Self> {
         let runtime = tokio::runtime::Builder::new_current_thread()
             .enable_all()
             .build()
             .map_err(|err| Box::new(StorageError(format!("failed to build aws runtime: {err}"))))?;
+        let endpoint = endpoint.map(ToOwned::to_owned);
         let config = runtime.block_on(async {
             let region_provider = match region {
                 Some(region) => RegionProviderChain::first_try(Region::new(region.to_string()))
                     .or_default_provider(),
                 None => RegionProviderChain::default_provider(),
             };
-            aws_config::defaults(aws_config::BehaviorVersion::latest())
-                .region(region_provider)
-                .load()
-                .await
+            let mut builder =
+                aws_config::defaults(aws_config::BehaviorVersion::latest()).region(region_provider);
+            if let Some(ep) = endpoint {
+                builder = builder.endpoint_url(ep);
+            }
+            builder.load().await
         });
-        let client = Client::new(&config);
+        let mut s3_builder = aws_sdk_s3::config::Builder::from(&config);
+        if path_style_access.unwrap_or(false) {
+            s3_builder = s3_builder.force_path_style(true);
+        }
+        let client = Client::from_conf(s3_builder.build());
         Ok(Self {
             bucket,
             client,

{floe_python-0.4.4 → floe_python-0.4.5}/crates/floe-core/src/io/write/iceberg/context.rs

@@ -76,7 +76,8 @@ pub(super) fn build_iceberg_write_context(
                         latest_s3_metadata_location(client, base_key)?
                     }
                     None => {
-                        let mut client = io::storage::s3::S3Client::new(bucket.clone(), None)?;
+                        let mut client =
+                            io::storage::s3::S3Client::new(bucket.clone(), None, None, None)?;
                         latest_s3_metadata_location(&mut client, base_key)?
                     }
                 }

{floe_python-0.4.4 → floe_python-0.4.5}/crates/floe-core/src/lib.rs

@@ -23,9 +23,13 @@ pub use crate::state::{
 };
 pub use add_entity::{add_entity_to_config, AddEntityOptions, AddEntityOutcome};
 pub use checks as check;
-pub use config::{resolve_config_location, ConfigLocation};
+pub use config::{
+    resolve_config_location, upload_to_remote_uri, write_bytes_to_remote_uri, ConfigLocation,
+};
 pub use errors::ConfigError;
-pub use manifest::{build_common_manifest_json, config_from_manifest_json};
+pub use manifest::{
+    build_common_manifest_json, config_from_manifest_json, ManifestOptions, PathMode,
+};
 pub use profile::{
     detect_malformed_placeholder, detect_unresolved_placeholders, parse_profile,
     parse_profile_from_str, validate_merged_vars, validate_profile, ProfileConfig,