floe-python 0.4.2__tar.gz → 0.4.3__tar.gz

{floe_python-0.4.2 → floe_python-0.4.3}/Cargo.lock

@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.4.2"
+version = "0.4.3"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3412,7 +3412,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.4.2"
+version = "0.4.3"
 dependencies = [
  "apache-avro 0.16.0",
  "arrow",
@@ -3455,7 +3455,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.4.2"
+version = "0.4.3"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.4.2 → floe_python-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.4.2
+Version: 0.4.3
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -98,6 +98,43 @@ except floe.FloeError as e:
 | `set_observer(callback)` | Register a live-event callback |
 | `clear_observer()` | Remove the current callback |
 
+## Jupyter
+
+`RunOutcome` renders as a color-coded HTML table automatically in Jupyter — no extra code needed. Just end a cell with the variable:
+
+```python
+outcome = floe.run("orders.yml")
+outcome  # renders inline HTML table with per-entity status, accepted/rejected counts
+```
+
+Use `outcome.to_dict()` to turn results into a plain dict for pandas:
+
+```python
+import pandas as pd
+df = pd.DataFrame(outcome.entity_reports)
+```
+
+## Observing progress
+
+Register a callback to receive live events as the run proceeds:
+
+```python
+floe.set_observer(lambda e: print(f"[{e['event']}]", e.get("name", e.get("entity", ""))))
+outcome = floe.run("orders.yml")
+floe.clear_observer()
+```
+
+Event types: `run_started`, `entity_started`, `file_started`, `file_finished`, `schema_evolution_applied`, `entity_finished`, `run_finished`, `log`. See the [full guide](../../docs/python-bindings.md#observing-runs-in-real-time) for all event fields.
+
+## Profile overrides
+
+Override config variables or cloud credentials without editing the YAML:
+
+```python
+floe.run("orders.yml", profile_vars={"incoming_root": "s3://my-bucket/incoming"})
+floe.run("orders.yml", profile_path="prod.yml")
+```
+
 ## Building from source
 
 ```bash
@@ -111,3 +148,7 @@ maturin develop
 
 Apache 2.0
 
+---
+
+→ Full API reference and examples: [docs/python-bindings.md](../../docs/python-bindings.md)
+

{floe_python-0.4.2/crates/floe-python → floe_python-0.4.3}/README.md

@@ -70,6 +70,43 @@ except floe.FloeError as e:
 | `set_observer(callback)` | Register a live-event callback |
 | `clear_observer()` | Remove the current callback |
 
+## Jupyter
+
+`RunOutcome` renders as a color-coded HTML table automatically in Jupyter — no extra code needed. Just end a cell with the variable:
+
+```python
+outcome = floe.run("orders.yml")
+outcome  # renders inline HTML table with per-entity status, accepted/rejected counts
+```
+
+Use `outcome.to_dict()` to turn results into a plain dict for pandas:
+
+```python
+import pandas as pd
+df = pd.DataFrame(outcome.entity_reports)
+```
+
+## Observing progress
+
+Register a callback to receive live events as the run proceeds:
+
+```python
+floe.set_observer(lambda e: print(f"[{e['event']}]", e.get("name", e.get("entity", ""))))
+outcome = floe.run("orders.yml")
+floe.clear_observer()
+```
+
+Event types: `run_started`, `entity_started`, `file_started`, `file_finished`, `schema_evolution_applied`, `entity_finished`, `run_finished`, `log`. See the [full guide](../../docs/python-bindings.md#observing-runs-in-real-time) for all event fields.
+
+## Profile overrides
+
+Override config variables or cloud credentials without editing the YAML:
+
+```python
+floe.run("orders.yml", profile_vars={"incoming_root": "s3://my-bucket/incoming"})
+floe.run("orders.yml", profile_path="prod.yml")
+```
+
 ## Building from source
 
 ```bash
@@ -82,3 +119,7 @@ maturin develop
 ## License
 
 Apache 2.0
+
+---
+
+→ Full API reference and examples: [docs/python-bindings.md](../../docs/python-bindings.md)

{floe_python-0.4.2 → floe_python-0.4.3}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.4.2"
+version = "0.4.3"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"

{floe_python-0.4.2 → floe_python-0.4.3}/crates/floe-core/src/config/location.rs

@@ -70,6 +70,6 @@ fn download_remote_config(uri: &str, temp_dir: &Path) -> FloeResult<PathBuf> {
     Err(format!("unsupported config uri: {}", uri).into())
 }
 
-fn is_remote_uri(value: &str) -> bool {
+pub(crate) fn is_remote_uri(value: &str) -> bool {
     value.starts_with("s3://") || value.starts_with("gs://") || value.starts_with("abfs://")
 }

{floe_python-0.4.2 → floe_python-0.4.3}/crates/floe-core/src/config/mod.rs

@@ -8,6 +8,7 @@ mod validate;
 pub(crate) mod yaml_decode;
 
 pub use catalog::{CatalogResolver, ResolvedDeltaCatalogTarget, ResolvedIcebergCatalogTarget};
+pub(crate) use location::is_remote_uri;
 pub use location::{resolve_config_location, ConfigLocation};
 pub use storage::{resolve_local_path, ConfigBase, ResolvedPath, StorageResolver};
 pub use types::*;

{floe_python-0.4.2 → floe_python-0.4.3}/crates/floe-core/src/config/parse.rs

@@ -1141,6 +1141,7 @@ pub(crate) fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
             "namespace",
             "producer",
             "max_failures",
+            "job_name",
         ],
     )?;
     Ok(LineageConfig {
@@ -1150,5 +1151,6 @@ pub(crate) fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
         namespace: get_string(hash, "namespace", "lineage")?,
         producer: opt_string(hash, "producer", "lineage")?,
         max_failures: opt_u32(hash, "max_failures", "lineage")?,
+        job_name: opt_string(hash, "job_name", "lineage")?,
     })
 }

{floe_python-0.4.2 → floe_python-0.4.3}/crates/floe-core/src/config/types.rs

@@ -31,6 +31,7 @@ pub struct LineageConfig {
     pub namespace: String,
     pub producer: Option<String>,
     pub max_failures: Option<u32>,
+    pub job_name: Option<String>,
 }
 
 #[derive(Debug)]

{floe_python-0.4.2 → floe_python-0.4.3}/crates/floe-core/src/lib.rs

@@ -84,6 +84,15 @@ pub fn load_config(config_path: &Path) -> FloeResult<config::RootConfig> {
     config::parse_config(config_path)
 }
 
+/// Read manifest JSON from any supported URI (local path, `s3://`, `gs://`, `abfs://`).
+/// For remote URIs the file is downloaded to a temp directory that is cleaned up before
+/// this function returns; the caller receives the raw JSON text as a `String`.
+pub fn read_manifest_text(uri: &str) -> FloeResult<String> {
+    let location = config::resolve_config_location(uri)?;
+    let text = std::fs::read_to_string(&location.path)?;
+    Ok(text)
+}
+
 pub fn load_config_with_profile_vars(
     config_path: &Path,
     profile_vars: &std::collections::HashMap<String, String>,

{floe_python-0.4.2 → floe_python-0.4.3}/crates/floe-core/src/lineage/mod.rs

@@ -8,6 +8,18 @@ use serde_json::{json, Value};
 use crate::config::{EntityConfig, LineageConfig};
 use crate::run::events::{RunEvent, RunObserver};
 
+const DEFAULT_PRODUCER: &str = concat!(
+    "https://github.com/malon64/floe/releases/tag/v",
+    env!("CARGO_PKG_VERSION")
+);
+
+#[derive(Clone)]
+struct ColumnMapping {
+    output_name: String,
+    column_type: String,
+    source_field: Option<String>,
+}
+
 struct EntityUris {
     source: String,
     accepted: String,
@@ -20,14 +32,19 @@ pub struct OpenLineageObserver {
     entity_start_ms: Mutex<HashMap<String, u128>>,
     entity_run_ids: Mutex<HashMap<String, String>>,
     run_start_ms: Mutex<Option<u128>>,
-    entity_schemas: HashMap<String, Vec<(String, String)>>,
+    entity_schemas: HashMap<String, Vec<ColumnMapping>>,
     entity_uris: HashMap<String, EntityUris>,
+    run_job_name: String,
     consecutive_failures: AtomicUsize,
     circuit_open: AtomicBool,
 }
 
 impl OpenLineageObserver {
-    pub fn new(config: &LineageConfig, entities: &[EntityConfig]) -> crate::FloeResult<Self> {
+    pub fn new(
+        config: &LineageConfig,
+        entities: &[EntityConfig],
+        config_path: &str,
+    ) -> crate::FloeResult<Self> {
         let timeout = Duration::from_secs(config.timeout_secs.unwrap_or(5));
         let client = reqwest::blocking::Client::builder()
             .timeout(timeout)
@@ -38,14 +55,30 @@ impl OpenLineageObserver {
                 ))) as Box<dyn std::error::Error + Send + Sync>
             })?;
 
+        let run_job_name = config
+            .job_name
+            .clone()
+            .filter(|s| !s.is_empty())
+            .unwrap_or_else(|| {
+                std::path::Path::new(config_path)
+                    .file_stem()
+                    .and_then(|s| s.to_str())
+                    .unwrap_or("floe-run")
+                    .to_string()
+            });
+
         let entity_schemas = entities
             .iter()
             .map(|e| {
-                let fields: Vec<(String, String)> = e
+                let fields: Vec<ColumnMapping> = e
                     .schema
                     .columns
                     .iter()
-                    .map(|c| (c.name.clone(), c.column_type.clone()))
+                    .map(|c| ColumnMapping {
+                        output_name: c.name.clone(),
+                        column_type: c.column_type.clone(),
+                        source_field: c.source.clone(),
+                    })
                     .collect();
                 (e.name.clone(), fields)
             })
@@ -73,6 +106,7 @@ impl OpenLineageObserver {
             run_start_ms: Mutex::new(None),
             entity_schemas,
             entity_uris,
+            run_job_name,
             consecutive_failures: AtomicUsize::new(0),
             circuit_open: AtomicBool::new(false),
         })
@@ -155,10 +189,7 @@ impl OpenLineageObserver {
     }
 
     fn producer(&self) -> &str {
-        self.config
-            .producer
-            .as_deref()
-            .unwrap_or("https://github.com/malon64/floe")
+        self.config.producer.as_deref().unwrap_or(DEFAULT_PRODUCER)
     }
 
     fn parent_run_facet(&self) -> Option<Value> {
@@ -212,71 +243,112 @@ impl OpenLineageObserver {
             run_facets["parent"] = parent;
         }
 
-        // Build inputs: source dataset with schema and quality facets on COMPLETE/FAIL.
-        let inputs = match (stats.as_ref(), uris) {
+        // Build inputs/outputs based on whether stats and uris are present (COMPLETE/FAIL)
+        // or absent (START — keep both empty).
+        let (inputs, outputs) = match (stats.as_ref(), uris) {
             (Some(s), Some(u)) => {
                 let rejection_rate = if s.rows > 0 {
                     s.rejected as f64 / s.rows as f64
                 } else {
                     0.0
                 };
+
+                // Input: source dataset — sub-namespace avoids collision with real entity names.
+                let (src_ns, src_path) = split_storage_uri(&u.source);
+                let inputs = json!([{
+                    "namespace": format!("{}.source", self.config.namespace),
+                    "name": name,
+                    "facets": {
+                        "symlinks": symlinks_facet(self.producer(), &src_ns, &src_path, "DIRECTORY")
+                    }
+                }]);
+
+                // Accepted output: entity name as logical identifier, TABLE type.
+                let (acc_ns, acc_path) = split_storage_uri(&u.accepted);
                 let schema_facet = json!({
-                    "fields": s.schema_fields.iter().map(|(col_name, col_type)| {
-                        json!({ "name": col_name, "type": col_type })
+                    "fields": s.schema_fields.iter().map(|col| {
+                        json!({ "name": col.output_name, "type": col.column_type })
                     }).collect::<Vec<_>>(),
                     "_producer": self.producer(),
                     "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json"
                 });
-                let dq_facet = json!({
-                    "rowCount": s.rows,
+                let accepted_dq_facet = json!({
+                    "rowCount": s.accepted,
                     "validCount": s.accepted,
-                    "invalidCount": s.rejected,
+                    "invalidCount": 0u64,
                     "_producer": self.producer(),
-                    "_schemaURL": "https://openlineage.io/spec/facets/1-0-2/DataQualityMetricsInputDatasetFacet.json"
+                    "_schemaURL": "https://openlineage.io/spec/facets/1-0-2/DataQualityMetricsOutputDatasetFacet.json"
                 });
                 let floe_facet = json!({
                     "entity": name,
                     "rejectionRate": rejection_rate,
                     "files": s.files,
                     "rows": s.rows,
-                    "accepted": s.accepted,
-                    "rejected": s.rejected,
                     "warnings": s.warnings,
                     "errors": s.errors,
                     "_producer": self.producer(),
                     "_schemaURL": "https://github.com/malon64/floe/schemas/FloeQualityRunFacet.json"
                 });
-                json!([{
-                    "namespace": self.config.namespace,
-                    "name": u.source,
-                    "facets": {
-                        "schema": schema_facet,
-                        "dataQualityMetrics": dq_facet,
-                        "floeQualityRun": floe_facet
-                    }
-                }])
-            }
-            _ => json!([]),
-        };
 
-        // Build outputs: accepted sink always present; rejected sink when configured.
-        let outputs = match uris {
-            Some(u) => {
+                let mut accepted_facets = json!({
+                    "symlinks": symlinks_facet(self.producer(), &acc_ns, &acc_path, "TABLE"),
+                    "schema": schema_facet,
+                    "dataQualityMetrics": accepted_dq_facet,
+                    "floeQualityRun": floe_facet
+                });
+
+                if !s.schema_fields.is_empty() {
+                    let fields_map: serde_json::Map<String, Value> = s
+                        .schema_fields
+                        .iter()
+                        .map(|col| {
+                            let src = col.source_field.as_deref().unwrap_or(&col.output_name);
+                            let entry = json!({
+                                "inputFields": [{
+                                    "namespace": format!("{}.source", self.config.namespace),
+                                    "name": name,
+                                    "field": src
+                                }]
+                            });
+                            (col.output_name.clone(), entry)
+                        })
+                        .collect();
+                    accepted_facets["columnLineage"] = json!({
+                        "fields": fields_map,
+                        "_producer": self.producer(),
+                        "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/ColumnLineageDatasetFacet.json"
+                    });
+                }
+
                 let mut out = vec![json!({
                     "namespace": self.config.namespace,
-                    "name": u.accepted,
-                    "facets": {}
+                    "name": name,
+                    "facets": accepted_facets
                 })];
+
+                // Rejected output (when configured): DIRECTORY type, rejected-row quality metrics.
                 if let Some(ref rej) = u.rejected {
+                    let (rej_ns, rej_path) = split_storage_uri(rej);
+                    let rejected_dq_facet = json!({
+                        "rowCount": s.rejected,
+                        "validCount": 0u64,
+                        "invalidCount": s.rejected,
+                        "_producer": self.producer(),
+                        "_schemaURL": "https://openlineage.io/spec/facets/1-0-2/DataQualityMetricsOutputDatasetFacet.json"
+                    });
                     out.push(json!({
-                        "namespace": self.config.namespace,
-                        "name": rej,
-                        "facets": {}
+                        "namespace": format!("{}.rejected", self.config.namespace),
+                        "name": name,
+                        "facets": {
+                            "symlinks": symlinks_facet(self.producer(), &rej_ns, &rej_path, "DIRECTORY"),
+                            "dataQualityMetrics": rejected_dq_facet
+                        }
                     }));
                 }
-                json!(out)
+
+                (inputs, json!(out))
             }
-            None => json!([]),
+            _ => (json!([]), json!([])),
         };
 
         let body = json!({
@@ -307,7 +379,7 @@ struct EntityStats {
     rejected: u64,
     warnings: u64,
     errors: u64,
-    schema_fields: Vec<(String, String)>,
+    schema_fields: Vec<ColumnMapping>,
 }
 
 fn ms_to_iso8601(ms: u128) -> String {
@@ -324,6 +396,30 @@ fn ms_to_iso8601(ms: u128) -> String {
     }
 }
 
+fn split_storage_uri(uri: &str) -> (String, String) {
+    // abfss:// must precede abfs:// so the longer prefix matches first.
+    let cloud_prefixes = ["s3://", "gs://", "gcs://", "az://", "abfss://", "abfs://"];
+    for prefix in cloud_prefixes {
+        if let Some(after_scheme) = uri.strip_prefix(prefix) {
+            if let Some(slash) = after_scheme.find('/') {
+                let authority = uri[..prefix.len() + slash].to_string();
+                let path = after_scheme[slash..].to_string();
+                return (authority, path);
+            }
+            return (uri.to_string(), "/".to_string());
+        }
+    }
+    ("file".to_string(), uri.to_string())
+}
+
+fn symlinks_facet(producer: &str, namespace: &str, name: &str, ds_type: &str) -> Value {
+    json!({
+        "identifiers": [{ "namespace": namespace, "name": name, "type": ds_type }],
+        "_producer": producer,
+        "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json"
+    })
+}
+
 impl RunObserver for OpenLineageObserver {
     fn on_event(&self, event: RunEvent) {
         match event {
@@ -349,7 +445,7 @@ impl RunObserver for OpenLineageObserver {
                     },
                     "job": {
                         "namespace": self.config.namespace,
-                        "name": run_id,
+                        "name": self.run_job_name,
                         "facets": {}
                     },
                     "inputs": [],
@@ -444,7 +540,7 @@ impl RunObserver for OpenLineageObserver {
                     },
                     "job": {
                         "namespace": self.config.namespace,
-                        "name": run_id,
+                        "name": self.run_job_name,
                         "facets": {}
                     },
                     "inputs": [],
@@ -462,8 +558,9 @@ impl RunObserver for OpenLineageObserver {
 pub fn build_observer(
     config: &LineageConfig,
     entities: &[EntityConfig],
+    config_path: &str,
 ) -> crate::FloeResult<Arc<dyn RunObserver>> {
-    let obs = OpenLineageObserver::new(config, entities)?;
+    let obs = OpenLineageObserver::new(config, entities, config_path)?;
     Ok(Arc::new(obs))
 }
 

{floe_python-0.4.2 → floe_python-0.4.3}/crates/floe-core/src/run/context.rs

@@ -98,7 +98,13 @@ impl RunContext {
         let catalog_resolver = config::CatalogResolver::new(&config)?;
         let config_dir =
             crate::io::storage::paths::normalize_local_path(storage_resolver.config_dir());
-        let config_path = crate::io::storage::paths::normalize_local_path(manifest_path);
+        let manifest_str = manifest_path.to_string_lossy();
+        let config_path = if config::is_remote_uri(&manifest_str) {
+            // Preserve the URI string as-is; normalize_local_path would collapse s3:// → s3:/
+            std::path::PathBuf::from(manifest_str.as_ref())
+        } else {
+            crate::io::storage::paths::normalize_local_path(manifest_path)
+        };
 
         // The manifest embeds report_base_uri; resolve it to a target if it looks local.
         let (report_target, report_base_path) =
@@ -148,3 +154,24 @@ impl RunContext {
         })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::path::Path;
+
+    #[test]
+    fn remote_uri_preserved_via_pathbuf_from() {
+        // normalize_local_path iterates Path::components(), which collapses the double slash
+        // in "s3://..." producing "s3:/..." — this documents the bug that the fix avoids.
+        let uri = "s3://bucket/manifests/prod.json";
+        let normalized = crate::io::storage::paths::normalize_local_path(Path::new(uri));
+        assert_ne!(
+            normalized.display().to_string(),
+            uri,
+            "normalize_local_path should mangle s3:// (confirming the bug we guard against)"
+        );
+        // PathBuf::from preserves the raw bytes, so display() round-trips the URI correctly.
+        let preserved = std::path::PathBuf::from(uri);
+        assert_eq!(preserved.display().to_string(), uri);
+    }
+}

{floe_python-0.4.2 → floe_python-0.4.3}/crates/floe-core/src/run/mod.rs

@@ -104,9 +104,11 @@ pub(crate) fn run_with_manifest_runtime(
     runtime: &mut dyn Runtime,
 ) -> FloeResult<RunOutcome> {
     init_thread_pool();
-    let json = std::fs::read_to_string(manifest_path)?;
+    let manifest_str = manifest_path.to_string_lossy();
+    let location = config::resolve_config_location(&manifest_str)?;
+    let json = std::fs::read_to_string(&location.path)?;
     let (config, report_base_uri) = crate::manifest::config_from_manifest_json(&json)?;
-    let config_base = config::ConfigBase::local_from_path(manifest_path);
+    let config_base = location.base.clone();
     if !options.entities.is_empty() {
         validate_entities(&config, &options.entities)?;
     }

{floe_python-0.4.2 → floe_python-0.4.3}/crates/floe-core/tests/unit/config/lineage_validation.rs

@@ -185,6 +185,7 @@ entities:
         timeout_secs: Some(2),
         producer: None,
         max_failures: None,
+        job_name: None,
     };
     let config = load_config_with_profile_overrides(
         &path,