floe-python 0.6.7__tar.gz → 0.6.9__tar.gz

{floe_python-0.6.7 → floe_python-0.6.9}/Cargo.lock

@@ -1805,7 +1805,7 @@ dependencies = [
  "crc",
  "digest",
  "rustversion",
- "spin 0.10.0",
+ "spin 0.10.1",
 ]
 
 [[package]]
@@ -1847,9 +1847,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -3205,9 +3205,9 @@ dependencies = [
 
 [[package]]
 name = "ethnum"
-version = "1.5.2"
+version = "1.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b"
+checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f"
 
 [[package]]
 name = "event-listener"
@@ -3340,7 +3340,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.6.7"
+version = "0.6.9"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3353,7 +3353,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.6.7"
+version = "0.6.9"
 dependencies = [
  "apache-avro",
  "arrow",
@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.6.7"
+version = "0.6.9"
 dependencies = [
  "floe-core",
  "pyo3",
@@ -4535,7 +4535,7 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 dependencies = [
- "spin 0.9.8",
+ "spin 0.9.9",
 ]
 
 [[package]]
@@ -7535,15 +7535,15 @@ dependencies = [
 
 [[package]]
 name = "spin"
-version = "0.9.8"
+version = "0.9.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"
 
 [[package]]
 name = "spin"
-version = "0.10.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591"
+checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
 
 [[package]]
 name = "spki"

{floe_python-0.6.7 → floe_python-0.6.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.6.7
+Version: 0.6.9
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.6.7"
+version = "0.6.9"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-core/src/config/parse.rs

@@ -1190,9 +1190,11 @@ pub(crate) fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
         "lineage",
         &[
             "url",
+            "endpoint",
             "api_key",
             "timeout_secs",
             "namespace",
+            "dataset_namespace",
             "producer",
             "max_failures",
             "job_name",
@@ -1200,9 +1202,11 @@ pub(crate) fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
     )?;
     Ok(LineageConfig {
         url: get_string(hash, "url", "lineage")?,
+        endpoint: opt_string(hash, "endpoint", "lineage")?,
         api_key: opt_string(hash, "api_key", "lineage")?,
         timeout_secs: opt_u64(hash, "timeout_secs", "lineage")?,
         namespace: get_string(hash, "namespace", "lineage")?,
+        dataset_namespace: opt_string(hash, "dataset_namespace", "lineage")?,
         producer: opt_string(hash, "producer", "lineage")?,
         max_failures: opt_u32(hash, "max_failures", "lineage")?,
         job_name: opt_string(hash, "job_name", "lineage")?,

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-core/src/config/template.rs

@@ -34,9 +34,11 @@ pub fn apply_templates_with_vars(
 
     if let Some(lineage) = config.lineage.as_mut() {
         lineage.url = replace_placeholders(&lineage.url, &vars, "lineage.url", None)?;
+        replace_lineage_field(&mut lineage.endpoint, &vars, "endpoint")?;
         if let Some(api_key) = lineage.api_key.as_mut() {
             *api_key = replace_placeholders(api_key, &vars, "lineage.api_key", None)?;
         }
+        replace_lineage_field(&mut lineage.dataset_namespace, &vars, "dataset_namespace")?;
     }
 
     if let Some(storages) = config.storages.as_mut() {
@@ -96,6 +98,18 @@ pub fn apply_templates_with_vars(
     Ok(())
 }
 
+fn replace_lineage_field(
+    value: &mut Option<String>,
+    vars: &HashMap<String, String>,
+    field_name: &str,
+) -> FloeResult<()> {
+    if let Some(current) = value.as_mut() {
+        let field = format!("lineage.{field_name}");
+        *current = replace_placeholders(current, vars, &field, None)?;
+    }
+    Ok(())
+}
+
 fn replace_storage_definition_field(
     value: &mut Option<String>,
     vars: &HashMap<String, String>,

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-core/src/config/types.rs

@@ -28,9 +28,11 @@ pub struct RootConfig {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct LineageConfig {
     pub url: String,
+    pub endpoint: Option<String>,
     pub api_key: Option<String>,
     pub timeout_secs: Option<u64>,
     pub namespace: String,
+    pub dataset_namespace: Option<String>,
     pub producer: Option<String>,
     pub max_failures: Option<u32>,
     pub job_name: Option<String>,

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-core/src/io/storage/providers/s3.rs

@@ -12,6 +12,8 @@ use crate::io::storage::uri::{format_bucket_uri, parse_bucket_uri, BucketLocatio
 use crate::io::storage::{planner, ConditionalWrite, ObjectRef, StorageClient, StoredObject};
 use crate::FloeResult;
 
+const AWS_S3_FORCE_PATH_STYLE: &str = "AWS_S3_FORCE_PATH_STYLE";
+
 pub struct S3Client {
     bucket: String,
     client: Client,
@@ -44,7 +46,7 @@ impl S3Client {
             builder.load().await
         });
         let mut s3_builder = aws_sdk_s3::config::Builder::from(&config);
-        if path_style_access.unwrap_or(false) {
+        if resolve_path_style_access(path_style_access) {
             s3_builder = s3_builder.force_path_style(true);
         }
         let client = Client::from_conf(s3_builder.build());
@@ -290,6 +292,25 @@ impl StorageClient for S3Client {
     }
 }
 
+#[doc(hidden)]
+pub fn resolve_path_style_access(configured: Option<bool>) -> bool {
+    let env_value = std::env::var(AWS_S3_FORCE_PATH_STYLE).ok();
+    resolve_path_style_access_from_env(configured, env_value.as_deref())
+}
+
+#[doc(hidden)]
+pub fn resolve_path_style_access_from_env(
+    configured: Option<bool>,
+    env_value: Option<&str>,
+) -> bool {
+    configured.unwrap_or_else(|| {
+        env_value.is_some_and(|value| {
+            let value = value.trim();
+            value == "1" || value.eq_ignore_ascii_case("true")
+        })
+    })
+}
+
 fn is_not_found<E: std::fmt::Display>(err: &E) -> bool {
     let text = err.to_string();
     text.contains("NoSuchKey") || text.contains("NotFound") || text.contains("404")

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-core/src/lineage/mod.rs

@@ -32,6 +32,18 @@ struct EntityUris {
     rejected: Option<OlDataset>,
 }
 
+fn resolve_lineage_url(base: &str, endpoint: Option<&str>) -> String {
+    let path = endpoint
+        .map(str::trim)
+        .filter(|s| !s.is_empty())
+        .unwrap_or("api/v1/lineage");
+    format!(
+        "{}/{}",
+        base.trim_end_matches('/'),
+        path.trim_start_matches('/')
+    )
+}
+
 pub struct OpenLineageObserver {
     client: reqwest::blocking::Client,
     config: LineageConfig,
@@ -91,6 +103,12 @@ impl OpenLineageObserver {
             })
             .collect();
 
+        let dataset_namespace = config
+            .dataset_namespace
+            .clone()
+            .filter(|s| !s.trim().is_empty())
+            .unwrap_or_else(|| config.namespace.clone());
+
         let entity_uris = entities
             .iter()
             .map(|e| {
@@ -105,7 +123,7 @@ impl OpenLineageObserver {
                         let ns = iceberg.namespace.as_deref().unwrap_or(e.name.as_str());
                         let tbl = iceberg.table.as_deref().unwrap_or(e.name.as_str());
                         OlDataset {
-                            namespace: config.namespace.clone(),
+                            namespace: dataset_namespace.clone(),
                             name: format!("{ns}.{tbl}"),
                         }
                     }
@@ -177,7 +195,7 @@ impl OpenLineageObserver {
             return;
         }
 
-        let url = format!("{}/api/v1/lineage", self.config.url.trim_end_matches('/'));
+        let url = resolve_lineage_url(&self.config.url, self.config.endpoint.as_deref());
         let max_failures = self.config.max_failures.unwrap_or(3) as usize;
         let retry_delays_ms: &[u64] = &[0, 100, 500];
 

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-core/tests/unit/config/lineage_validation.rs

@@ -74,7 +74,9 @@ report:
   path: "/tmp/reports"
 lineage:
   url: "http://marquez:5000"
+  endpoint: "/api/v1/openlineage/lineage"
   namespace: "my-namespace"
+  dataset_namespace: "iceberg.prod"
   timeout_secs: 10
   producer: "https://github.com/myorg/floe"
 entities:
@@ -95,6 +97,20 @@ entities:
 "#;
     let path = write_temp_config(config);
     validate(&path, ValidateOptions::default()).expect("valid lineage config");
+    let parsed = load_config_with_profile_overrides(
+        &path,
+        &std::collections::HashMap::new(),
+        None,
+        None,
+        None,
+    )
+    .expect("lineage config should parse");
+    let lineage = parsed.lineage.expect("lineage should be present");
+    assert_eq!(
+        lineage.endpoint.as_deref(),
+        Some("/api/v1/openlineage/lineage")
+    );
+    assert_eq!(lineage.dataset_namespace.as_deref(), Some("iceberg.prod"));
 }
 
 #[test]
@@ -180,7 +196,9 @@ entities:
     let path = write_temp_config(base);
     let profile_lineage = LineageConfig {
         url: "http://localhost:5000".to_string(),
+        endpoint: None,
         namespace: "floe-demo-local".to_string(),
+        dataset_namespace: None,
         api_key: None,
         timeout_secs: Some(2),
         producer: None,

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-core/tests/unit/config/templating.rs

@@ -88,6 +88,85 @@ entities:
     );
 }
 
+#[test]
+fn env_vars_apply_to_lineage_endpoint_and_dataset_namespace() {
+    let root = temp_dir("floe-lineage-vars");
+    let config_yaml = format!(
+        r#"version: "0.1"
+env:
+  vars:
+    LINEAGE_ENDPOINT: "api/v1/openlineage/lineage"
+    DATASET_NAMESPACE: "iceberg.prod"
+lineage:
+  url: "http://openmetadata:8585"
+  endpoint: "{{{{LINEAGE_ENDPOINT}}}}"
+  namespace: "dagster"
+  dataset_namespace: "{{{{DATASET_NAMESPACE}}}}"
+entities:
+  - name: "orders"
+    source:
+      format: "csv"
+      path: "{root}/in/orders.csv"
+    sink:
+      accepted:
+        format: "parquet"
+        path: "{root}/out/orders"
+    policy:
+      severity: "warn"
+    schema:
+      columns:
+        - name: "id"
+          type: "string"
+"#,
+        root = root.display(),
+    );
+    let config_path = write_config(&root, &config_yaml);
+    let parsed = load_config(&config_path).expect("parse config");
+    let lineage = parsed.lineage.expect("lineage");
+
+    assert_eq!(
+        lineage.endpoint.as_deref(),
+        Some("api/v1/openlineage/lineage")
+    );
+    assert_eq!(lineage.dataset_namespace.as_deref(), Some("iceberg.prod"));
+}
+
+#[test]
+fn unresolved_lineage_placeholder_errors() {
+    let root = temp_dir("floe-lineage-unresolved");
+    let config_yaml = format!(
+        r#"version: "0.1"
+lineage:
+  url: "http://openmetadata:8585"
+  endpoint: "{{{{OPENLINEAGE_ENDPOINT}}}}"
+  namespace: "dagster"
+entities:
+  - name: "orders"
+    source:
+      format: "csv"
+      path: "{root}/in/orders.csv"
+    sink:
+      accepted:
+        format: "parquet"
+        path: "{root}/out/orders"
+    policy:
+      severity: "warn"
+    schema:
+      columns:
+        - name: "id"
+          type: "string"
+"#,
+        root = root.display(),
+    );
+    let config_path = write_config(&root, &config_yaml);
+
+    let err = load_config(&config_path).expect_err("placeholder error");
+    assert_eq!(
+        err.to_string(),
+        "lineage.endpoint references unknown variable OPENLINEAGE_ENDPOINT"
+    );
+}
+
 #[test]
 fn env_file_vars_apply_to_source_path() {
     let root = temp_dir("floe-env-file");

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-core/tests/unit/io/storage/s3.rs

@@ -1,6 +1,6 @@
 use std::path::Path;
 
-use floe_core::io::storage::s3::parse_s3_uri;
+use floe_core::io::storage::s3::{parse_s3_uri, resolve_path_style_access_from_env};
 use floe_core::io::storage::{filter_by_suffixes, stable_sort_refs, temp_path_for_key, ObjectRef};
 
 #[test]
@@ -17,6 +17,27 @@ fn parse_s3_uri_allows_bucket_only() {
     assert_eq!(loc.key, "");
 }
 
+#[test]
+fn path_style_access_prefers_explicit_config() {
+    let resolve = resolve_path_style_access_from_env;
+
+    assert!(resolve(Some(true), None));
+    assert!(resolve(Some(true), Some("false")));
+    assert!(!resolve(Some(false), Some("true")));
+}
+
+#[test]
+fn path_style_access_falls_back_to_aws_env_convention() {
+    let resolve = resolve_path_style_access_from_env;
+
+    assert!(!resolve(None, None));
+    assert!(resolve(None, Some("true")));
+    assert!(resolve(None, Some("1")));
+    assert!(resolve(None, Some(" TRUE ")));
+    assert!(!resolve(None, Some("false")));
+    assert!(!resolve(None, Some("garbage")));
+}
+
 #[test]
 fn filter_keys_by_suffix_sorts_and_filters() {
     let refs = vec![

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-core/tests/unit/run/lineage.rs

@@ -9,15 +9,34 @@ use serde_json::json;
 fn make_config(server_url: &str, max_failures: Option<u32>) -> LineageConfig {
     LineageConfig {
         url: server_url.to_string(),
+        endpoint: None,
         api_key: None,
         timeout_secs: Some(2),
         namespace: "test-ns".to_string(),
+        dataset_namespace: None,
         producer: None,
         max_failures,
         job_name: None,
     }
 }
 
+#[test]
+fn custom_endpoint_is_used_for_posts() {
+    let mut server = mockito::Server::new();
+    let mock = server
+        .mock("POST", "/api/v1/openlineage/lineage")
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let mut config = make_config(&format!("{}/", server.url()), None);
+    config.endpoint = Some("/api/v1/openlineage/lineage".to_string());
+    let obs = OpenLineageObserver::new(&config, &[], "config.yml").unwrap();
+
+    obs.on_event(run_started_event());
+    mock.assert();
+}
+
 fn run_started_event() -> RunEvent {
     RunEvent::RunStarted {
         run_id: "test-run-1".to_string(),
@@ -987,7 +1006,8 @@ fn iceberg_accepted_uses_catalog_namespace_and_table_name() {
         .match_body(mockito::Matcher::PartialJson(json!({
             "eventType": "COMPLETE",
             "inputs": [{ "namespace": "s3://iceberg-data", "name": "bronze/sales/customers" }],
-            "outputs": [{ "namespace": "test-ns", "name": "sales_dev.customers" }]
+            "job": { "namespace": "test-ns" },
+            "outputs": [{ "namespace": "iceberg.prod", "name": "sales_dev.customers" }]
         })))
         .with_status(200)
         .expect(1)
@@ -1006,7 +1026,8 @@ fn iceberg_accepted_uses_catalog_namespace_and_table_name() {
         location: None,
     });
 
-    let config = make_config(&server.url(), None);
+    let mut config = make_config(&server.url(), None);
+    config.dataset_namespace = Some("iceberg.prod".to_string());
     let obs = OpenLineageObserver::new(&config, &[entity], "config.yml").unwrap();
 
     obs.on_event(RunEvent::EntityStarted {
@@ -1031,3 +1052,43 @@ fn iceberg_accepted_uses_catalog_namespace_and_table_name() {
     _start_mock.assert();
     _complete_mock.assert();
 }
+
+#[test]
+fn iceberg_accepted_dataset_namespace_defaults_to_job_namespace() {
+    let mut server = mockito::Server::new();
+    let _start_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .with_status(200)
+        .expect(1)
+        .create();
+    let _complete_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(json!({
+            "eventType": "COMPLETE",
+            "job": { "namespace": "test-ns" },
+            "outputs": [{ "namespace": "test-ns", "name": "sales_dev.customers" }]
+        })))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let mut entity = make_entity(
+        "customers",
+        "s3://iceberg-data/bronze/sales/customers",
+        "s3://warehouse/silver/customers",
+        None,
+    );
+    entity.sink.accepted.iceberg = Some(IcebergSinkTargetConfig {
+        catalog: None,
+        namespace: Some("sales_dev".to_string()),
+        table: Some("customers".to_string()),
+        location: None,
+    });
+
+    let config = make_config(&server.url(), None);
+    let obs = OpenLineageObserver::new(&config, &[entity], "config.yml").unwrap();
+    obs.on_event(entity_started_event());
+    obs.on_event(entity_finished_event("customers", "success"));
+    _start_mock.assert();
+    _complete_mock.assert();
+}

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-python"
-version = "0.6.7"
+version = "0.6.9"
 edition = "2021"
 description = "Python bindings for floe-core (PyO3-based)"
 license = "Apache-2.0"
@@ -14,7 +14,7 @@ name = "_floe"
 crate-type = ["cdylib"]
 
 [dependencies]
-floe-core = { path = "../floe-core", version = "0.6.7", default-features = false }
+floe-core = { path = "../floe-core", version = "0.6.9", default-features = false }
 pyo3 = { version = "0.29", features = ["extension-module", "abi3-py310"] }
 serde_json = "1"
 

{floe_python-0.6.7 → floe_python-0.6.9}/crates/floe-python/pyproject.duckdb.toml

@@ -22,7 +22,7 @@ build-backend = "maturin"
 
 [project]
 name = "floe-duckdb"
-version = "0.6.7"
+version = "0.6.9"
 description = "DuckDB companion extension for floe — adds the native DuckDB sink to the floe Python package"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -46,7 +46,7 @@ classifiers = [
 # only contributes the compiled DuckDB extension module into that package. Pin to
 # the exact lean version so the companion's `_floe_duckdb` ABI always matches the
 # `floe` package it plugs into.
-dependencies = ["floe-python==0.6.7"]
+dependencies = ["floe-python==0.6.9"]
 
 [project.urls]
 Homepage = "https://github.com/malon64/floe"

{floe_python-0.6.7 → floe_python-0.6.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "floe-python"
-version = "0.6.7"
+version = "0.6.9"
 description = "Python bindings for floe-core — run floe pipelines at Rust speed from Python and notebooks"
 readme = "README.md"
 requires-python = ">=3.10"