floe-python 0.4.1__tar.gz → 0.4.2__tar.gz

{floe_python-0.4.1 → floe_python-0.4.2}/Cargo.lock

@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3412,7 +3412,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "apache-avro 0.16.0",
  "arrow",
@@ -3455,7 +3455,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.4.1 → floe_python-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.4.1
+Version: 0.4.2
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.4.1 → floe_python-0.4.2}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.4.1"
+version = "0.4.2"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"

{floe_python-0.4.1 → floe_python-0.4.2}/crates/floe-core/README.md

@@ -73,7 +73,7 @@ floe run      -c config.yml   # run the pipeline
 | S3 storage | [docs/storages/s3.md](docs/storages/s3.md) |
 | ADLS storage | [docs/storages/adls.md](docs/storages/adls.md) |
 | GCS storage | [docs/storages/gcs.md](docs/storages/gcs.md) |
-| Incremental ingestion | [docs/profiles.md](docs/profiles.md) |
+| Environement specific profile config | [docs/profiles.md](docs/profiles.md) [docs/variables.md](docs/variables.md) |
 | Run reports | [docs/report.md](docs/report.md) |
 | CLI reference | [docs/cli.md](docs/cli.md) |
 | Orchestration (Dagster / Airflow) | [docs/summary.md](docs/summary.md) |

{floe_python-0.4.1 → floe_python-0.4.2}/crates/floe-core/src/config/types.rs

@@ -1,6 +1,8 @@
 use std::collections::HashMap;
 use std::path::Path;
 
+use serde::{Deserialize, Serialize};
+
 use polars::polars_utils::pl_str::PlSmallStr;
 use polars::prelude::{
     CsvEncoding, CsvParseOptions, CsvReadOptions, DataType, NullValues, TimeUnit,
@@ -21,7 +23,7 @@ pub struct RootConfig {
     pub entities: Vec<EntityConfig>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct LineageConfig {
     pub url: String,
     pub api_key: Option<String>,
@@ -66,12 +68,12 @@ pub struct EntityConfig {
     pub pii: Option<PiiConfig>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct PiiConfig {
     pub columns: Vec<PiiColumnConfig>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct PiiColumnConfig {
     pub name: String,
     pub strategy: PiiStrategy,
@@ -79,7 +81,8 @@ pub struct PiiColumnConfig {
     pub redact_value: Option<String>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum PiiStrategy {
     Hash,
     Drop,
@@ -103,7 +106,8 @@ impl EntityConfig {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum IncrementalMode {
     #[default]
     None,
@@ -146,7 +150,7 @@ pub struct SourceConfig {
     pub cast_mode: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Deserialize)]
 pub struct SourceOptions {
     pub header: Option<bool>,
     pub separator: Option<String>,
@@ -262,7 +266,8 @@ pub struct SinkConfig {
     pub archive: Option<ArchiveTarget>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
 pub enum WriteMode {
     #[default]
     Overwrite,
@@ -300,34 +305,34 @@ pub const DEFAULT_SCD2_CURRENT_FLAG_COLUMN: &str = "__floe_is_current";
 pub const DEFAULT_SCD2_VALID_FROM_COLUMN: &str = "__floe_valid_from";
 pub const DEFAULT_SCD2_VALID_TO_COLUMN: &str = "__floe_valid_to";
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct MergeOptionsConfig {
     pub ignore_columns: Option<Vec<String>>,
     pub compare_columns: Option<Vec<String>>,
     pub scd2: Option<MergeScd2OptionsConfig>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct MergeScd2OptionsConfig {
     pub current_flag_column: Option<String>,
     pub valid_from_column: Option<String>,
     pub valid_to_column: Option<String>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct SinkOptions {
     pub compression: Option<String>,
     pub row_group_size: Option<u64>,
     pub max_size_per_file: Option<u64>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct IcebergPartitionFieldConfig {
     pub column: String,
     pub transform: String,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct IcebergSinkTargetConfig {
     pub catalog: Option<String>,
     pub namespace: Option<String>,
@@ -335,7 +340,7 @@ pub struct IcebergSinkTargetConfig {
     pub location: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DeltaSinkTargetConfig {
     /// Name of the catalog definition to use (falls back to `catalogs.default`).
     pub catalog: Option<String>,
@@ -345,13 +350,13 @@ pub struct DeltaSinkTargetConfig {
     pub table: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct StoragesConfig {
     pub default: Option<String>,
     pub definitions: Vec<StorageDefinition>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct StorageDefinition {
     pub name: String,
     pub fs_type: String,
@@ -362,7 +367,7 @@ pub struct StorageDefinition {
     pub prefix: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct CatalogsConfig {
     pub default: Option<String>,
     pub definitions: Vec<CatalogDefinition>,
@@ -371,7 +376,8 @@ pub struct CatalogsConfig {
 /// Type-specific configuration for a catalog definition.
 /// Each variant carries only the fields relevant to that catalog type.
 /// Add a new variant here when supporting a new catalog type.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "lowercase")]
 pub enum CatalogTypeConfig {
     Glue {
         region: String,
@@ -419,7 +425,7 @@ impl CatalogTypeConfig {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct CatalogDefinition {
     pub name: String,
     pub type_config: CatalogTypeConfig,
@@ -440,7 +446,8 @@ pub struct ArchiveTarget {
     pub storage: Option<String>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum PolicySeverity {
     #[default]
     Warn,
@@ -469,7 +476,7 @@ pub struct PolicyConfig {
     pub severity: PolicySeverity,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize)]
 pub struct SchemaConfig {
     pub normalize_columns: Option<NormalizeColumnsConfig>,
     pub mismatch: Option<SchemaMismatchConfig>,
@@ -485,19 +492,20 @@ impl SchemaConfig {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct NormalizeColumnsConfig {
     pub enabled: Option<bool>,
     pub strategy: Option<String>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
 pub struct SchemaEvolutionConfig {
     pub mode: SchemaEvolutionMode,
     pub on_incompatible: SchemaEvolutionIncompatibleAction,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
 pub enum SchemaEvolutionMode {
     #[default]
     Strict,
@@ -513,7 +521,8 @@ impl SchemaEvolutionMode {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum SchemaEvolutionIncompatibleAction {
     #[default]
     Fail,
@@ -527,13 +536,13 @@ impl SchemaEvolutionIncompatibleAction {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct SchemaMismatchConfig {
     pub missing_columns: Option<String>,
     pub extra_columns: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize)]
 pub struct ColumnConfig {
     pub name: String,
     pub source: Option<String>,

{floe_python-0.4.1 → floe_python-0.4.2}/crates/floe-core/src/lib.rs

@@ -17,18 +17,23 @@ pub mod state;
 pub mod vars;
 pub mod warnings;
 
-pub use crate::state::{inspect_entity_state_with_base, reset_entity_state_with_base};
+pub use crate::state::{
+    inspect_entity_state, inspect_entity_state_with_base, reset_entity_state,
+    reset_entity_state_with_base,
+};
 pub use add_entity::{add_entity_to_config, AddEntityOptions, AddEntityOutcome};
 pub use checks as check;
 pub use config::{resolve_config_location, ConfigLocation};
 pub use errors::ConfigError;
-pub use manifest::build_common_manifest_json;
+pub use manifest::{build_common_manifest_json, config_from_manifest_json};
 pub use profile::{
     detect_malformed_placeholder, detect_unresolved_placeholders, parse_profile,
     parse_profile_from_str, validate_merged_vars, validate_profile, ProfileConfig,
 };
 pub use run::events::{set_observer, MultiObserver, RunEvent, RunObserver};
-pub use run::{run, run_with_base, DryRunEntityPreview, EntityOutcome, RunOutcome};
+pub use run::{
+    run, run_with_base, run_with_manifest_path, DryRunEntityPreview, EntityOutcome, RunOutcome,
+};
 pub use runner::{parse_run_status_from_logs, ConnectorRunStatus};
 pub use runtime::{DefaultRuntime, Runtime};
 pub use vars::{resolve_vars, VarSources};

{floe_python-0.4.1 → floe_python-0.4.2}/crates/floe-core/src/lineage/mod.rs

@@ -8,6 +8,12 @@ use serde_json::{json, Value};
 use crate::config::{EntityConfig, LineageConfig};
 use crate::run::events::{RunEvent, RunObserver};
 
+struct EntityUris {
+    source: String,
+    accepted: String,
+    rejected: Option<String>,
+}
+
 pub struct OpenLineageObserver {
     client: reqwest::blocking::Client,
     config: LineageConfig,
@@ -15,6 +21,7 @@ pub struct OpenLineageObserver {
     entity_run_ids: Mutex<HashMap<String, String>>,
     run_start_ms: Mutex<Option<u128>>,
     entity_schemas: HashMap<String, Vec<(String, String)>>,
+    entity_uris: HashMap<String, EntityUris>,
     consecutive_failures: AtomicUsize,
     circuit_open: AtomicBool,
 }
@@ -44,6 +51,20 @@ impl OpenLineageObserver {
             })
             .collect();
 
+        let entity_uris = entities
+            .iter()
+            .map(|e| {
+                (
+                    e.name.clone(),
+                    EntityUris {
+                        source: e.source.path.clone(),
+                        accepted: e.sink.accepted.path.clone(),
+                        rejected: e.sink.rejected.as_ref().map(|r| r.path.clone()),
+                    },
+                )
+            })
+            .collect();
+
         Ok(Self {
             client,
             config: config.clone(),
@@ -51,6 +72,7 @@ impl OpenLineageObserver {
             entity_run_ids: Mutex::new(HashMap::new()),
             run_start_ms: Mutex::new(None),
             entity_schemas,
+            entity_uris,
             consecutive_failures: AtomicUsize::new(0),
             circuit_open: AtomicBool::new(false),
         })
@@ -180,6 +202,7 @@ impl OpenLineageObserver {
         event_type: &str,
         ts_ms: u128,
         stats: Option<EntityStats>,
+        uris: Option<&EntityUris>,
     ) {
         let event_time = ms_to_iso8601(ts_ms);
         let job_name = format!("{}.{name}", self.config.namespace);
@@ -189,50 +212,71 @@ impl OpenLineageObserver {
             run_facets["parent"] = parent;
         }
 
-        // Build inputs with all facets for COMPLETE events; empty for START.
-        let inputs = if let Some(ref s) = stats {
-            let rejection_rate = if s.rows > 0 {
-                s.rejected as f64 / s.rows as f64
-            } else {
-                0.0
-            };
-            let schema_facet = json!({
-                "fields": s.schema_fields.iter().map(|(col_name, col_type)| {
-                    json!({ "name": col_name, "type": col_type })
-                }).collect::<Vec<_>>(),
-                "_producer": self.producer(),
-                "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json"
-            });
-            let dq_facet = json!({
-                "rowCount": s.rows,
-                "validCount": s.accepted,
-                "invalidCount": s.rejected,
-                "_producer": self.producer(),
-                "_schemaURL": "https://openlineage.io/spec/facets/1-0-2/DataQualityMetricsInputDatasetFacet.json"
-            });
-            let floe_facet = json!({
-                "entity": name,
-                "rejectionRate": rejection_rate,
-                "files": s.files,
-                "rows": s.rows,
-                "accepted": s.accepted,
-                "rejected": s.rejected,
-                "warnings": s.warnings,
-                "errors": s.errors,
-                "_producer": self.producer(),
-                "_schemaURL": "https://github.com/malon64/floe/schemas/FloeQualityRunFacet.json"
-            });
-            json!([{
-                "namespace": self.config.namespace,
-                "name": name,
-                "facets": {
-                    "schema": schema_facet,
-                    "dataQualityMetrics": dq_facet,
-                    "floeQualityRun": floe_facet
+        // Build inputs: source dataset with schema and quality facets on COMPLETE/FAIL.
+        let inputs = match (stats.as_ref(), uris) {
+            (Some(s), Some(u)) => {
+                let rejection_rate = if s.rows > 0 {
+                    s.rejected as f64 / s.rows as f64
+                } else {
+                    0.0
+                };
+                let schema_facet = json!({
+                    "fields": s.schema_fields.iter().map(|(col_name, col_type)| {
+                        json!({ "name": col_name, "type": col_type })
+                    }).collect::<Vec<_>>(),
+                    "_producer": self.producer(),
+                    "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json"
+                });
+                let dq_facet = json!({
+                    "rowCount": s.rows,
+                    "validCount": s.accepted,
+                    "invalidCount": s.rejected,
+                    "_producer": self.producer(),
+                    "_schemaURL": "https://openlineage.io/spec/facets/1-0-2/DataQualityMetricsInputDatasetFacet.json"
+                });
+                let floe_facet = json!({
+                    "entity": name,
+                    "rejectionRate": rejection_rate,
+                    "files": s.files,
+                    "rows": s.rows,
+                    "accepted": s.accepted,
+                    "rejected": s.rejected,
+                    "warnings": s.warnings,
+                    "errors": s.errors,
+                    "_producer": self.producer(),
+                    "_schemaURL": "https://github.com/malon64/floe/schemas/FloeQualityRunFacet.json"
+                });
+                json!([{
+                    "namespace": self.config.namespace,
+                    "name": u.source,
+                    "facets": {
+                        "schema": schema_facet,
+                        "dataQualityMetrics": dq_facet,
+                        "floeQualityRun": floe_facet
+                    }
+                }])
+            }
+            _ => json!([]),
+        };
+
+        // Build outputs: accepted sink always present; rejected sink when configured.
+        let outputs = match uris {
+            Some(u) => {
+                let mut out = vec![json!({
+                    "namespace": self.config.namespace,
+                    "name": u.accepted,
+                    "facets": {}
+                })];
+                if let Some(ref rej) = u.rejected {
+                    out.push(json!({
+                        "namespace": self.config.namespace,
+                        "name": rej,
+                        "facets": {}
+                    }));
                 }
-            }])
-        } else {
-            json!([])
+                json!(out)
+            }
+            None => json!([]),
         };
 
         let body = json!({
@@ -248,7 +292,7 @@ impl OpenLineageObserver {
                 "facets": {}
             },
             "inputs": inputs,
-            "outputs": [],
+            "outputs": outputs,
             "producer": self.producer(),
             "schemaURL": "https://openlineage.io/spec/1-0-5/OpenLineage.json#/$defs/RunEvent"
         });
@@ -329,13 +373,14 @@ impl RunObserver for OpenLineageObserver {
                 if let Ok(mut guard) = self.entity_run_ids.lock() {
                     guard.insert(name.clone(), entity_run_id.clone());
                 }
-                self.emit_entity_run_event(&entity_run_id, &name, "START", ts_ms, None);
+                self.emit_entity_run_event(&entity_run_id, &name, "START", ts_ms, None, None);
             }
             RunEvent::EntityFinished {
                 run_id,
                 name,
                 status,
                 files,
+                files_skipped: _,
                 rows,
                 accepted,
                 rejected,
@@ -364,7 +409,15 @@ impl RunObserver for OpenLineageObserver {
                     errors,
                     schema_fields,
                 };
-                self.emit_entity_run_event(&entity_run_id, &name, event_type, ts_ms, Some(stats));
+                let uris = self.entity_uris.get(&name);
+                self.emit_entity_run_event(
+                    &entity_run_id,
+                    &name,
+                    event_type,
+                    ts_ms,
+                    Some(stats),
+                    uris,
+                );
             }
             RunEvent::RunFinished {
                 run_id,