floe-python 0.6.10__tar.gz → 0.6.11__tar.gz

{floe_python-0.6.10 → floe_python-0.6.11}/Cargo.lock

@@ -3340,7 +3340,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.6.10"
+version = "0.6.11"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3353,7 +3353,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.6.10"
+version = "0.6.11"
 dependencies = [
  "apache-avro",
  "arrow",
@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.6.10"
+version = "0.6.11"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.6.10 → floe_python-0.6.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.6.10
+Version: 0.6.11
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.6.10 → floe_python-0.6.11}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.6.10"
+version = "0.6.11"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"

{floe_python-0.6.10 → floe_python-0.6.11}/crates/floe-core/src/lineage/mod.rs

@@ -635,6 +635,23 @@ pub fn build_observer(
     Ok(Arc::new(obs))
 }
 
+/// Build a lineage observer for a `--manifest` replay from the manifest's
+/// embedded lineage block and its entities' resolved cloud datasets, so replay
+/// emits the same input/output lineage as a direct config run. `Ok(None)` when
+/// the manifest has no `lineage` block. See
+/// [`crate::manifest::reconstruct::lineage_inputs_from_manifest_json`].
+pub fn build_observer_from_manifest_json(
+    manifest_json: &str,
+    config_path: &str,
+) -> crate::FloeResult<Option<Arc<dyn RunObserver>>> {
+    match crate::manifest::reconstruct::lineage_inputs_from_manifest_json(manifest_json)? {
+        Some((lineage_cfg, entities)) => {
+            build_observer(&lineage_cfg, &entities, config_path).map(Some)
+        }
+        None => Ok(None),
+    }
+}
+
 impl OpenLineageObserver {
     pub fn is_circuit_open(&self) -> bool {
         self.circuit_open.load(Ordering::Relaxed)

{floe_python-0.6.10 → floe_python-0.6.11}/crates/floe-core/src/manifest/reconstruct.rs

@@ -56,6 +56,10 @@ pub struct ManifestSinksForRun {
 pub struct ManifestSinkTargetForRun {
     pub format: String,
     pub storage: String,
+    /// Resolved cloud URI (e.g. `s3://bucket/key`). Optional so hand-authored
+    /// manifests that omit it still parse; the generator always writes it.
+    #[serde(default)]
+    pub uri: Option<String>,
     pub path: String,
     pub options: Option<serde_json::Value>,
     pub partition_by: Option<Vec<String>>,
@@ -172,6 +176,71 @@ pub fn config_from_manifest_json(json: &str) -> FloeResult<(crate::config::RootC
     Ok((config, manifest.report_base_uri))
 }
 
+/// Reconstruct the lineage config and entities for a `--manifest` replay's
+/// OpenLineage observer, with each source/sink dataset path resolved to the
+/// manifest's cloud `uri`.
+///
+/// The run reconstruction (`config_from_manifest_json`) intentionally keeps the
+/// raw `path` fields — in the default manifest path mode those hold the original
+/// config paths (e.g. `sales/orders`) while the resolved cloud identity lives in
+/// `uri` (e.g. `s3://bucket/sales/orders`). The observer derives a dataset's
+/// namespace/name from the path, so for lineage it must see the resolved cloud
+/// URI; otherwise a named S3/GCS/ABFS source is reported under the `file`
+/// namespace. Local and relative paths are left as reconstructed.
+///
+/// Returns `Ok(None)` when the manifest has no `lineage` block.
+pub fn lineage_inputs_from_manifest_json(
+    json: &str,
+) -> FloeResult<Option<(LineageConfig, Vec<EntityConfig>)>> {
+    let manifest: ManifestForRun =
+        serde_json::from_str(json).map_err(|err| -> Box<dyn std::error::Error + Send + Sync> {
+            FloeError::config(format!("manifest parse error: {err}")).into()
+        })?;
+
+    let lineage = match deserialize_manifest_section::<LineageConfig>(
+        manifest.lineage.as_ref(),
+        "lineage",
+    )? {
+        Some(lineage) => lineage,
+        None => return Ok(None),
+    };
+
+    let mut entities = manifest
+        .entities
+        .iter()
+        .map(entity_from_manifest)
+        .collect::<FloeResult<Vec<_>>>()?;
+
+    // Reconstructed entities are in manifest order, so zip pairs each with its
+    // source manifest entry to overlay the resolved cloud URIs.
+    for (entity, m) in entities.iter_mut().zip(manifest.entities.iter()) {
+        overlay_cloud_uri(&mut entity.source.path, Some(m.source.uri.as_str()));
+        overlay_cloud_uri(
+            &mut entity.sink.accepted.path,
+            m.sinks.accepted.uri.as_deref(),
+        );
+        if let (Some(rejected), Some(m_rejected)) =
+            (entity.sink.rejected.as_mut(), m.sinks.rejected.as_ref())
+        {
+            overlay_cloud_uri(&mut rejected.path, m_rejected.uri.as_deref());
+        }
+    }
+
+    Ok(Some((lineage, entities)))
+}
+
+/// Replace `path` with `uri` when `uri` names a cloud object store; other
+/// schemes (local / relative) keep the reconstructed path. The prefix list
+/// mirrors `lineage::split_storage_uri` and must stay in sync with it.
+fn overlay_cloud_uri(path: &mut String, uri: Option<&str>) {
+    const CLOUD_SCHEMES: [&str; 6] = ["s3://", "gs://", "gcs://", "az://", "abfss://", "abfs://"];
+    if let Some(uri) = uri {
+        if CLOUD_SCHEMES.iter().any(|scheme| uri.starts_with(scheme)) {
+            *path = uri.to_string();
+        }
+    }
+}
+
 fn entity_from_manifest(m: &ManifestEntityForRun) -> FloeResult<EntityConfig> {
     let policy_severity = parse_policy_severity(m.policy_severity.as_deref().unwrap_or("warn"));
     let write_mode = parse_write_mode(m.write_mode.as_deref().unwrap_or("overwrite"));

{floe_python-0.6.10 → floe_python-0.6.11}/crates/floe-core/tests/unit/run/lineage.rs

@@ -1157,3 +1157,229 @@ fn api_key_resolution_over_the_wire() {
 
     std::env::remove_var("OPENLINEAGE_API_KEY");
 }
+
+// ---- Manifest-replay dataset lineage (issue #455) ----
+// build_observer_from_manifest_json must feed the manifest's entities and their
+// resolved cloud URIs to the observer so replay emits non-empty inputs/outputs.
+
+fn replay_entity(name: &str, source_uri: &str, accepted_uri: &str) -> serde_json::Value {
+    json!({
+        "name": name,
+        "source": { "format": "csv", "storage": "bronze", "uri": source_uri, "path": source_uri },
+        "sinks": {
+            "accepted": {
+                "format": "parquet",
+                "storage": "warehouse",
+                "uri": accepted_uri,
+                "path": accepted_uri
+            }
+        },
+        "schema": { "columns": [], "primary_key": [], "unique_keys": [] }
+    })
+}
+
+fn replay_manifest(server_url: &str, entities: serde_json::Value) -> String {
+    json!({
+        "spec_version": "0.3",
+        "report_base_uri": "file:///tmp",
+        "entities": entities,
+        "lineage": { "url": server_url, "namespace": "test-ns" }
+    })
+    .to_string()
+}
+
+fn entity_started(name: &str) -> RunEvent {
+    RunEvent::EntityStarted {
+        run_id: "test-run-1".to_string(),
+        name: name.to_string(),
+        ts_ms: 1_001_000,
+    }
+}
+
+// Replaying a manifest with an entity emits the entity's source input and
+// accepted output — the regression from issue #455. With entities dropped
+// (the previous `&[]`), inputs/outputs would be empty and the subset match on
+// inputs[0]/outputs[0] would fail.
+#[test]
+fn manifest_replay_passes_entities_to_observer() {
+    let mut server = mockito::Server::new();
+
+    let start_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(
+            json!({ "eventType": "START" }),
+        ))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let complete_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(json!({
+            "eventType": "COMPLETE",
+            "inputs": [{
+                "namespace": "s3://lakehouse-bronze",
+                "name": "sales/customer_health/support_tickets"
+            }],
+            "outputs": [{
+                "namespace": "s3://lakehouse-warehouse",
+                "name": "silver/support_tickets"
+            }]
+        })))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let manifest = replay_manifest(
+        &server.url(),
+        json!([replay_entity(
+            "support_tickets",
+            "s3://lakehouse-bronze/sales/customer_health/support_tickets",
+            "s3://lakehouse-warehouse/silver/support_tickets",
+        )]),
+    );
+
+    let obs = floe_core::lineage::build_observer_from_manifest_json(&manifest, "")
+        .expect("observer builds from manifest")
+        .expect("manifest has a lineage block => Some observer");
+
+    obs.on_event(entity_started("support_tickets"));
+    obs.on_event(entity_finished_event("support_tickets", "success"));
+
+    start_mock.assert();
+    complete_mock.assert();
+}
+
+// Default manifest path mode records raw config paths in `path` and the resolved
+// cloud identity in `uri`. Lineage must report the cloud dataset from `uri`, not
+// a `file` namespace derived from the raw path (PR #456 review).
+#[test]
+fn manifest_replay_default_path_mode_uses_resolved_cloud_uri() {
+    let mut server = mockito::Server::new();
+
+    let start_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(
+            json!({ "eventType": "START" }),
+        ))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let complete_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(json!({
+            "eventType": "COMPLETE",
+            "inputs": [{ "namespace": "s3://lakehouse-bronze", "name": "sales/orders" }],
+            "outputs": [{ "namespace": "s3://lakehouse-warehouse", "name": "silver/orders" }]
+        })))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    // path = raw config paths (default mode); uri = resolved cloud identities.
+    let entity = json!({
+        "name": "orders",
+        "source": {
+            "format": "csv",
+            "storage": "bronze",
+            "uri": "s3://lakehouse-bronze/sales/orders",
+            "path": "sales/orders"
+        },
+        "sinks": {
+            "accepted": {
+                "format": "parquet",
+                "storage": "warehouse",
+                "uri": "s3://lakehouse-warehouse/silver/orders",
+                "path": "silver/orders"
+            }
+        },
+        "schema": { "columns": [], "primary_key": [], "unique_keys": [] }
+    });
+    let manifest = replay_manifest(&server.url(), json!([entity]));
+
+    let obs = floe_core::lineage::build_observer_from_manifest_json(&manifest, "")
+        .expect("observer builds from manifest")
+        .expect("manifest has a lineage block => Some observer");
+
+    obs.on_event(entity_started("orders"));
+    obs.on_event(entity_finished_event("orders", "success"));
+
+    start_mock.assert();
+    complete_mock.assert();
+}
+
+// Firing events for only the selected entity (as `--entities orders` does)
+// still carries that entity's dataset context, because the observer holds all
+// manifest entities keyed by name.
+#[test]
+fn manifest_replay_entity_selection_keeps_dataset_context() {
+    let mut server = mockito::Server::new();
+
+    let start_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(
+            json!({ "eventType": "START" }),
+        ))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let complete_mock = server
+        .mock("POST", "/api/v1/lineage")
+        .match_body(mockito::Matcher::PartialJson(json!({
+            "eventType": "COMPLETE",
+            "inputs": [{ "namespace": "s3://lakehouse-bronze", "name": "sales/orders" }],
+            "outputs": [{ "namespace": "s3://lakehouse-warehouse", "name": "silver/orders" }]
+        })))
+        .with_status(200)
+        .expect(1)
+        .create();
+
+    let manifest = replay_manifest(
+        &server.url(),
+        json!([
+            replay_entity(
+                "orders",
+                "s3://lakehouse-bronze/sales/orders",
+                "s3://lakehouse-warehouse/silver/orders",
+            ),
+            replay_entity(
+                "customers",
+                "s3://lakehouse-bronze/sales/customers",
+                "s3://lakehouse-warehouse/silver/customers",
+            )
+        ]),
+    );
+
+    let obs = floe_core::lineage::build_observer_from_manifest_json(&manifest, "")
+        .expect("observer builds from manifest")
+        .expect("manifest has a lineage block => Some observer");
+
+    // Only "orders" runs; "customers" is not selected.
+    obs.on_event(entity_started("orders"));
+    obs.on_event(entity_finished_event("orders", "success"));
+
+    start_mock.assert();
+    complete_mock.assert();
+}
+
+// A manifest without a lineage block yields no observer (lineage disabled),
+// rather than erroring.
+#[test]
+fn manifest_replay_without_lineage_block_yields_no_observer() {
+    let manifest = json!({
+        "spec_version": "0.3",
+        "report_base_uri": "file:///tmp",
+        "entities": []
+    })
+    .to_string();
+
+    let obs = floe_core::lineage::build_observer_from_manifest_json(&manifest, "")
+        .expect("no error when lineage block is absent");
+
+    assert!(
+        obs.is_none(),
+        "manifest without lineage block => no observer"
+    );
+}

{floe_python-0.6.10 → floe_python-0.6.11}/crates/floe-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-python"
-version = "0.6.10"
+version = "0.6.11"
 edition = "2021"
 description = "Python bindings for floe-core (PyO3-based)"
 license = "Apache-2.0"
@@ -14,7 +14,7 @@ name = "_floe"
 crate-type = ["cdylib"]
 
 [dependencies]
-floe-core = { path = "../floe-core", version = "0.6.10", default-features = false }
+floe-core = { path = "../floe-core", version = "0.6.11", default-features = false }
 pyo3 = { version = "0.29", features = ["extension-module", "abi3-py310"] }
 serde_json = "1"
 

{floe_python-0.6.10 → floe_python-0.6.11}/crates/floe-python/pyproject.duckdb.toml

@@ -22,7 +22,7 @@ build-backend = "maturin"
 
 [project]
 name = "floe-duckdb"
-version = "0.6.10"
+version = "0.6.11"
 description = "DuckDB companion extension for floe — adds the native DuckDB sink to the floe Python package"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -46,7 +46,7 @@ classifiers = [
 # only contributes the compiled DuckDB extension module into that package. Pin to
 # the exact lean version so the companion's `_floe_duckdb` ABI always matches the
 # `floe` package it plugs into.
-dependencies = ["floe-python==0.6.10"]
+dependencies = ["floe-python==0.6.11"]
 
 [project.urls]
 Homepage = "https://github.com/malon64/floe"

{floe_python-0.6.10 → floe_python-0.6.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "floe-python"
-version = "0.6.10"
+version = "0.6.11"
 description = "Python bindings for floe-core — run floe pipelines at Rust speed from Python and notebooks"
 readme = "README.md"
 requires-python = ">=3.10"