floe-python 0.6.9__tar.gz → 0.6.11__tar.gz

{floe_python-0.6.9 → floe_python-0.6.11}/Cargo.lock

@@ -3340,7 +3340,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.6.9"
+version = "0.6.11"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3353,7 +3353,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.6.9"
+version = "0.6.11"
 dependencies = [
  "apache-avro",
  "arrow",
@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.6.9"
+version = "0.6.11"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.6.9 → floe_python-0.6.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.6.9
+Version: 0.6.11
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.6.9 → floe_python-0.6.11}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.6.9"
+version = "0.6.11"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"

{floe_python-0.6.9 → floe_python-0.6.11}/crates/floe-core/src/lineage/mod.rs

@@ -8,12 +8,15 @@ use uuid::Uuid;
 
 use crate::config::{EntityConfig, LineageConfig};
 use crate::run::events::{RunEvent, RunObserver};
+use crate::secret::Secret;
 
 const DEFAULT_PRODUCER: &str = concat!(
     "https://github.com/malon64/floe/releases/tag/v",
     env!("CARGO_PKG_VERSION")
 );
 
+const OPENLINEAGE_API_KEY_ENV: &str = "OPENLINEAGE_API_KEY";
+
 #[derive(Clone)]
 struct ColumnMapping {
     output_name: String,
@@ -44,9 +47,36 @@ fn resolve_lineage_url(base: &str, endpoint: Option<&str>) -> String {
     )
 }
 
+fn is_unresolved_placeholder(value: &str) -> bool {
+    let value = value.trim();
+    value.starts_with("{{") && value.ends_with("}}")
+}
+
+/// Effective Bearer token for lineage POSTs.
+///
+/// An explicit `lineage.api_key` wins for backward compatibility. Otherwise Floe falls back
+/// to the ambient `OPENLINEAGE_API_KEY`, mirroring the OpenLineage Python client so a replayed
+/// manifest authenticates from a runner-mounted Secret without the credential ever being
+/// persisted in the manifest. An unexpanded `{{...}}` placeholder is treated as absent:
+/// profile-supplied lineage is merged after config templating (see `apply_profile_lineage`),
+/// so the raw placeholder would otherwise be sent verbatim as the token.
+fn resolve_api_key(configured: Option<&str>, ambient: Option<String>) -> Option<Secret> {
+    let explicit = configured
+        .map(str::trim)
+        .filter(|key| !key.is_empty() && !is_unresolved_placeholder(key));
+    match explicit {
+        Some(key) => Some(Secret::from(key)),
+        None => ambient
+            .map(|value| value.trim().to_string())
+            .filter(|value| !value.is_empty())
+            .map(Secret::from),
+    }
+}
+
 pub struct OpenLineageObserver {
     client: reqwest::blocking::Client,
     config: LineageConfig,
+    api_key: Option<Secret>,
     entity_start_ms: Mutex<HashMap<String, u128>>,
     entity_run_ids: Mutex<HashMap<String, String>>,
     run_start_ms: Mutex<Option<u128>>,
@@ -155,9 +185,15 @@ impl OpenLineageObserver {
             })
             .collect();
 
+        let api_key = resolve_api_key(
+            config.api_key.as_deref(),
+            std::env::var(OPENLINEAGE_API_KEY_ENV).ok(),
+        );
+
         Ok(Self {
             client,
             config: config.clone(),
+            api_key,
             entity_start_ms: Mutex::new(HashMap::new()),
             entity_run_ids: Mutex::new(HashMap::new()),
             run_start_ms: Mutex::new(None),
@@ -172,8 +208,8 @@ impl OpenLineageObserver {
 
     fn attempt_post(&self, url: &str, body: &Value) -> Result<(), bool> {
         let mut req = self.client.post(url).json(body);
-        if let Some(api_key) = self.config.api_key.as_deref() {
-            req = req.bearer_auth(api_key);
+        if let Some(api_key) = self.api_key.as_ref() {
+            req = req.bearer_auth(api_key.expose());
         }
         match req.send() {
             Err(_) => Err(true),
@@ -599,6 +635,23 @@ pub fn build_observer(
     Ok(Arc::new(obs))
 }
 
+/// Build a lineage observer for a `--manifest` replay from the manifest's
+/// embedded lineage block and its entities' resolved cloud datasets, so replay
+/// emits the same input/output lineage as a direct config run. `Ok(None)` when
+/// the manifest has no `lineage` block. See
+/// [`crate::manifest::reconstruct::lineage_inputs_from_manifest_json`].
+pub fn build_observer_from_manifest_json(
+    manifest_json: &str,
+    config_path: &str,
+) -> crate::FloeResult<Option<Arc<dyn RunObserver>>> {
+    match crate::manifest::reconstruct::lineage_inputs_from_manifest_json(manifest_json)? {
+        Some((lineage_cfg, entities)) => {
+            build_observer(&lineage_cfg, &entities, config_path).map(Some)
+        }
+        None => Ok(None),
+    }
+}
+
 impl OpenLineageObserver {
     pub fn is_circuit_open(&self) -> bool {
         self.circuit_open.load(Ordering::Relaxed)
@@ -608,3 +661,48 @@ impl OpenLineageObserver {
         self.consecutive_failures.load(Ordering::Relaxed)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn expose(key: Option<Secret>) -> Option<String> {
+        key.map(|s| s.expose().to_string())
+    }
+
+    #[test]
+    fn explicit_config_key_wins_over_env() {
+        let key = resolve_api_key(Some("cfg-token"), Some("env-token".to_string()));
+        assert_eq!(expose(key), Some("cfg-token".to_string()));
+    }
+
+    #[test]
+    fn falls_back_to_env_when_config_absent() {
+        let key = resolve_api_key(None, Some("env-token".to_string()));
+        assert_eq!(expose(key), Some("env-token".to_string()));
+    }
+
+    #[test]
+    fn unresolved_placeholder_falls_back_to_env() {
+        let key = resolve_api_key(
+            Some("{{OPENLINEAGE_API_KEY}}"),
+            Some("env-token".to_string()),
+        );
+        assert_eq!(expose(key), Some("env-token".to_string()));
+    }
+
+    #[test]
+    fn unresolved_placeholder_without_env_is_none() {
+        assert!(resolve_api_key(Some("{{OPENLINEAGE_API_KEY}}"), None).is_none());
+    }
+
+    #[test]
+    fn none_when_neither_source_present() {
+        assert!(resolve_api_key(None, None).is_none());
+    }
+
+    #[test]
+    fn blank_values_are_ignored() {
+        assert!(resolve_api_key(Some("   "), Some("   ".to_string())).is_none());
+    }
+}

{floe_python-0.6.9 → floe_python-0.6.11}/crates/floe-core/src/manifest/reconstruct.rs

@@ -56,6 +56,10 @@ pub struct ManifestSinksForRun {
 pub struct ManifestSinkTargetForRun {
     pub format: String,
     pub storage: String,
+    /// Resolved cloud URI (e.g. `s3://bucket/key`). Optional so hand-authored
+    /// manifests that omit it still parse; the generator always writes it.
+    #[serde(default)]
+    pub uri: Option<String>,
     pub path: String,
     pub options: Option<serde_json::Value>,
     pub partition_by: Option<Vec<String>>,
@@ -172,6 +176,71 @@ pub fn config_from_manifest_json(json: &str) -> FloeResult<(crate::config::RootC
     Ok((config, manifest.report_base_uri))
 }
 
+/// Reconstruct the lineage config and entities for a `--manifest` replay's
+/// OpenLineage observer, with each source/sink dataset path resolved to the
+/// manifest's cloud `uri`.
+///
+/// The run reconstruction (`config_from_manifest_json`) intentionally keeps the
+/// raw `path` fields — in the default manifest path mode those hold the original
+/// config paths (e.g. `sales/orders`) while the resolved cloud identity lives in
+/// `uri` (e.g. `s3://bucket/sales/orders`). The observer derives a dataset's
+/// namespace/name from the path, so for lineage it must see the resolved cloud
+/// URI; otherwise a named S3/GCS/ABFS source is reported under the `file`
+/// namespace. Local and relative paths are left as reconstructed.
+///
+/// Returns `Ok(None)` when the manifest has no `lineage` block.
+pub fn lineage_inputs_from_manifest_json(
+    json: &str,
+) -> FloeResult<Option<(LineageConfig, Vec<EntityConfig>)>> {
+    let manifest: ManifestForRun =
+        serde_json::from_str(json).map_err(|err| -> Box<dyn std::error::Error + Send + Sync> {
+            FloeError::config(format!("manifest parse error: {err}")).into()
+        })?;
+
+    let lineage = match deserialize_manifest_section::<LineageConfig>(
+        manifest.lineage.as_ref(),
+        "lineage",
+    )? {
+        Some(lineage) => lineage,
+        None => return Ok(None),
+    };
+
+    let mut entities = manifest
+        .entities
+        .iter()
+        .map(entity_from_manifest)
+        .collect::<FloeResult<Vec<_>>>()?;
+
+    // Reconstructed entities are in manifest order, so zip pairs each with its
+    // source manifest entry to overlay the resolved cloud URIs.
+    for (entity, m) in entities.iter_mut().zip(manifest.entities.iter()) {
+        overlay_cloud_uri(&mut entity.source.path, Some(m.source.uri.as_str()));
+        overlay_cloud_uri(
+            &mut entity.sink.accepted.path,
+            m.sinks.accepted.uri.as_deref(),
+        );
+        if let (Some(rejected), Some(m_rejected)) =
+            (entity.sink.rejected.as_mut(), m.sinks.rejected.as_ref())
+        {
+            overlay_cloud_uri(&mut rejected.path, m_rejected.uri.as_deref());
+        }
+    }
+
+    Ok(Some((lineage, entities)))
+}
+
+/// Replace `path` with `uri` when `uri` names a cloud object store; other
+/// schemes (local / relative) keep the reconstructed path. The prefix list
+/// mirrors `lineage::split_storage_uri` and must stay in sync with it.
+fn overlay_cloud_uri(path: &mut String, uri: Option<&str>) {
+    const CLOUD_SCHEMES: [&str; 6] = ["s3://", "gs://", "gcs://", "az://", "abfss://", "abfs://"];
+    if let Some(uri) = uri {
+        if CLOUD_SCHEMES.iter().any(|scheme| uri.starts_with(scheme)) {
+            *path = uri.to_string();
+        }
+    }
+}
+
 fn entity_from_manifest(m: &ManifestEntityForRun) -> FloeResult<EntityConfig> {
     let policy_severity = parse_policy_severity(m.policy_severity.as_deref().unwrap_or("warn"));
     let write_mode = parse_write_mode(m.write_mode.as_deref().unwrap_or("overwrite"));

{floe_python-0.6.9 → floe_python-0.6.11}/crates/floe-core/tests/unit/run/lineage.rs

@@ -1092,3 +1092,294 @@ fn iceberg_accepted_dataset_namespace_defaults_to_job_namespace() {
     _start_mock.assert();
     _complete_mock.assert();
 }
+
+// Sole owner of the process-wide OPENLINEAGE_API_KEY variable. The crate has no
+// serial-test harness, so all three auth paths are asserted inside one test that
+// controls the variable's lifecycle; no other test reads it, and every other lineage
+// test either sets an explicit api_key (which wins over the env regardless) or does not
+// match on request headers, so this cannot race them.
+#[test]
+fn api_key_resolution_over_the_wire() {
+    std::env::remove_var("OPENLINEAGE_API_KEY");
+
+    // No config api_key and no env var => request carries no Authorization header.
+    {
+        let mut server = mockito::Server::new();
+        let mock = server
+            .mock("POST", "/api/v1/lineage")
+            .match_header("authorization", mockito::Matcher::Missing)
+            .with_status(200)
+            .expect(1)
+            .create();
+        let config = make_config(&server.url(), None);
+        let obs = OpenLineageObserver::new(&config, &[], "config.yml").unwrap();
+        obs.on_event(run_started_event());
+        mock.assert();
+    }
+
+    // Manifest replay with api_key omitted + env var set => Bearer token from the env.
+    {
+        let mut server = mockito::Server::new();
+        let mock = server
+            .mock("POST", "/api/v1/lineage")
+            .match_header("authorization", "Bearer env-replay-token")
+            .with_status(200)
+            .expect(1)
+            .create();
+        let manifest = format!(
+            r#"{{"spec_version":"0.3","report_base_uri":"file:///tmp","entities":[],"lineage":{{"url":"{}","namespace":"test-ns"}}}}"#,
+            server.url()
+        );
+        std::env::set_var("OPENLINEAGE_API_KEY", "env-replay-token");
+        let (config, _) =
+            floe_core::config_from_manifest_json(&manifest).expect("reconstruct manifest config");
+        let lineage = config.lineage.expect("manifest lineage block");
+        let obs = OpenLineageObserver::new(&lineage, &[], "").unwrap();
+        obs.on_event(run_started_event());
+        mock.assert();
+    }
+
+    // Explicit config api_key wins even when the env var is present (backward compatible).
+    {
+        let mut server = mockito::Server::new();
+        let mock = server
+            .mock("POST", "/api/v1/lineage")
+            .match_header("authorization", "Bearer cfg-token")
+            .with_status(200)
+            .expect(1)
+            .create();
+        let mut config = make_config(&server.url(), None);
+        config.api_key = Some("cfg-token".to_string());
+        let obs = OpenLineageObserver::new(&config, &[], "config.yml").unwrap();
+        obs.on_event(run_started_event());
+        mock.assert();
+    }
+
+    std::env::remove_var("OPENLINEAGE_API_KEY");
+}
+
+// ---- Manifest-replay dataset lineage (issue #455) ----
+// build_observer_from_manifest_json must feed the manifest's entities and their
+// resolved cloud URIs to the observer so replay emits non-empty inputs/outputs.
+
+fn replay_entity(name: &str, source_uri: &str, accepted_uri: &str) -> serde_json::Value {
+    json!({
+        "name": name,
+        "source": { "format": "csv", "storage": "bronze", "uri": source_uri, "path": source_uri },
+        "sinks": {
+            "accepted": {
+                "format": "parquet",
+                "storage": "warehouse",
+                "uri": accepted_uri,
+                "path": accepted_uri
+            }
+        },
+        "schema": { "columns": [], "primary_key": [], "unique_keys": [] }
+    })
+}
+
+fn replay_manifest(server_url: &str, entities: serde_json::Value) -> String {
+    json!({
+        "spec_version": "0.3",
+        "report_base_uri": "file:///tmp",
+        "entities": entities,
+        "lineage": { "url": server_url, "namespace": "test-ns" }
+    })
+    .to_string()
+}
+
+fn entity_started(name: &str) -> RunEvent {
+    RunEvent::EntityStarted {
+        run_id: "test-run-1".to_string(),
+        name: name.to_string(),
+        ts_ms: 1_001_000,
+    }
+}
+
+// Replaying a manifest with an entity emits the entity's source input and
+// accepted output — the regression from issue #455. With entities dropped
+// (the previous `&[]`), inputs/outputs would be empty and the subset match on
+// inputs[0]/outputs[0] would fail.
+#[test]
+fn manifest_replay_passes_entities_to_observer() {
+    let mut server = mockito::Server::new();
+
+    let start_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(
+            json!({ "eventType": "START" }),
+        ))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let complete_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(json!({
+            "eventType": "COMPLETE",
+            "inputs": [{
+                "namespace": "s3://lakehouse-bronze",
+                "name": "sales/customer_health/support_tickets"
+            }],
+            "outputs": [{
+                "namespace": "s3://lakehouse-warehouse",
+                "name": "silver/support_tickets"
+            }]
+        })))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let manifest = replay_manifest(
+        &server.url(),
+        json!([replay_entity(
+            "support_tickets",
+            "s3://lakehouse-bronze/sales/customer_health/support_tickets",
+            "s3://lakehouse-warehouse/silver/support_tickets",
+        )]),
+    );
+
+    let obs = floe_core::lineage::build_observer_from_manifest_json(&manifest, "")
+        .expect("observer builds from manifest")
+        .expect("manifest has a lineage block => Some observer");
+
+    obs.on_event(entity_started("support_tickets"));
+    obs.on_event(entity_finished_event("support_tickets", "success"));
+
+    start_mock.assert();
+    complete_mock.assert();
+}
+
+// Default manifest path mode records raw config paths in `path` and the resolved
+// cloud identity in `uri`. Lineage must report the cloud dataset from `uri`, not
+// a `file` namespace derived from the raw path (PR #456 review).
+#[test]
+fn manifest_replay_default_path_mode_uses_resolved_cloud_uri() {
+    let mut server = mockito::Server::new();
+
+    let start_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(
+            json!({ "eventType": "START" }),
+        ))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let complete_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(json!({
+            "eventType": "COMPLETE",
+            "inputs": [{ "namespace": "s3://lakehouse-bronze", "name": "sales/orders" }],
+            "outputs": [{ "namespace": "s3://lakehouse-warehouse", "name": "silver/orders" }]
+        })))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    // path = raw config paths (default mode); uri = resolved cloud identities.
+    let entity = json!({
+        "name": "orders",
+        "source": {
+            "format": "csv",
+            "storage": "bronze",
+            "uri": "s3://lakehouse-bronze/sales/orders",
+            "path": "sales/orders"
+        },
+        "sinks": {
+            "accepted": {
+                "format": "parquet",
+                "storage": "warehouse",
+                "uri": "s3://lakehouse-warehouse/silver/orders",
+                "path": "silver/orders"
+            }
+        },
+        "schema": { "columns": [], "primary_key": [], "unique_keys": [] }
+    });
+    let manifest = replay_manifest(&server.url(), json!([entity]));
+
+    let obs = floe_core::lineage::build_observer_from_manifest_json(&manifest, "")
+        .expect("observer builds from manifest")
+        .expect("manifest has a lineage block => Some observer");
+
+    obs.on_event(entity_started("orders"));
+    obs.on_event(entity_finished_event("orders", "success"));
+
+    start_mock.assert();
+    complete_mock.assert();
+}
+
+// Firing events for only the selected entity (as `--entities orders` does)
+// still carries that entity's dataset context, because the observer holds all
+// manifest entities keyed by name.
+#[test]
+fn manifest_replay_entity_selection_keeps_dataset_context() {
+    let mut server = mockito::Server::new();
+
+    let start_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(
+            json!({ "eventType": "START" }),
+        ))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let complete_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(json!({
+            "eventType": "COMPLETE",
+            "inputs": [{ "namespace": "s3://lakehouse-bronze", "name": "sales/orders" }],
+            "outputs": [{ "namespace": "s3://lakehouse-warehouse", "name": "silver/orders" }]
+        })))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let manifest = replay_manifest(
+        &server.url(),
+        json!([
+            replay_entity(
+                "orders",
+                "s3://lakehouse-bronze/sales/orders",
+                "s3://lakehouse-warehouse/silver/orders",
+            ),
+            replay_entity(
+                "customers",
+                "s3://lakehouse-bronze/sales/customers",
+                "s3://lakehouse-warehouse/silver/customers",
+            )
+        ]),
+    );
+
+    let obs = floe_core::lineage::build_observer_from_manifest_json(&manifest, "")
+        .expect("observer builds from manifest")
+        .expect("manifest has a lineage block => Some observer");
+
+    // Only "orders" runs; "customers" is not selected.
+    obs.on_event(entity_started("orders"));
+    obs.on_event(entity_finished_event("orders", "success"));
+
+    start_mock.assert();
+    complete_mock.assert();
+}
+
+// A manifest without a lineage block yields no observer (lineage disabled),
+// rather than erroring.
+#[test]
+fn manifest_replay_without_lineage_block_yields_no_observer() {
+    let manifest = json!({
+        "spec_version": "0.3",
+        "report_base_uri": "file:///tmp",
+        "entities": []
+    })
+    .to_string();
+
+    let obs = floe_core::lineage::build_observer_from_manifest_json(&manifest, "")
+        .expect("no error when lineage block is absent");
+
+    assert!(
+        obs.is_none(),
+        "manifest without lineage block => no observer"
+    );
+}

{floe_python-0.6.9 → floe_python-0.6.11}/crates/floe-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-python"
-version = "0.6.9"
+version = "0.6.11"
 edition = "2021"
 description = "Python bindings for floe-core (PyO3-based)"
 license = "Apache-2.0"
@@ -14,7 +14,7 @@ name = "_floe"
 crate-type = ["cdylib"]
 
 [dependencies]
-floe-core = { path = "../floe-core", version = "0.6.9", default-features = false }
+floe-core = { path = "../floe-core", version = "0.6.11", default-features = false }
 pyo3 = { version = "0.29", features = ["extension-module", "abi3-py310"] }
 serde_json = "1"
 

{floe_python-0.6.9 → floe_python-0.6.11}/crates/floe-python/pyproject.duckdb.toml

@@ -22,7 +22,7 @@ build-backend = "maturin"
 
 [project]
 name = "floe-duckdb"
-version = "0.6.9"
+version = "0.6.11"
 description = "DuckDB companion extension for floe — adds the native DuckDB sink to the floe Python package"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -46,7 +46,7 @@ classifiers = [
 # only contributes the compiled DuckDB extension module into that package. Pin to
 # the exact lean version so the companion's `_floe_duckdb` ABI always matches the
 # `floe` package it plugs into.
-dependencies = ["floe-python==0.6.9"]
+dependencies = ["floe-python==0.6.11"]
 
 [project.urls]
 Homepage = "https://github.com/malon64/floe"

{floe_python-0.6.9 → floe_python-0.6.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "floe-python"
-version = "0.6.9"
+version = "0.6.11"
 description = "Python bindings for floe-core — run floe pipelines at Rust speed from Python and notebooks"
 readme = "README.md"
 requires-python = ">=3.10"