floe-python 0.3.10__tar.gz → 0.4.1__tar.gz

{floe_python-0.3.10 → floe_python-0.4.1}/Cargo.lock

@@ -476,6 +476,16 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063"
 
+[[package]]
+name = "assert-json-diff"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "assert_cmd"
 version = "2.1.2"
@@ -1708,6 +1718,15 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
 
+[[package]]
+name = "colored"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "comfy-table"
 version = "7.2.2"
@@ -3380,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.3.10"
+version = "0.4.1"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3393,7 +3412,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.3.10"
+version = "0.4.1"
 dependencies = [
  "apache-avro 0.16.0",
  "arrow",
@@ -3414,6 +3433,7 @@ dependencies = [
  "iceberg",
  "iceberg-catalog-rest",
  "iceberg-storage-opendal",
+ "mockito",
  "orc-rust",
  "polars",
  "rayon",
@@ -3425,6 +3445,7 @@ dependencies = [
  "serde_yaml",
  "sha2",
  "tempfile",
+ "thiserror 1.0.69",
  "time",
  "tokio",
  "url",
@@ -3434,7 +3455,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.3.10"
+version = "0.4.1"
 dependencies = [
  "floe-core",
  "pyo3",
@@ -4046,6 +4067,7 @@ dependencies = [
  "http 1.4.0",
  "http-body 1.0.1",
  "httparse",
+ "httpdate",
  "itoa",
  "pin-project-lite",
  "pin-utils",
@@ -4861,6 +4883,31 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "mockito"
+version = "1.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90820618712cab19cfc46b274c6c22546a82affcb3c3bdf0f29e3db8e1bb92c0"
+dependencies = [
+ "assert-json-diff",
+ "bytes",
+ "colored",
+ "futures-core",
+ "http 1.4.0",
+ "http-body 1.0.1",
+ "http-body-util",
+ "hyper 1.8.1",
+ "hyper-util",
+ "log",
+ "pin-project-lite",
+ "rand 0.9.4",
+ "regex",
+ "serde_json",
+ "serde_urlencoded",
+ "similar",
+ "tokio",
+]
+
 [[package]]
 name = "moka"
 version = "0.12.13"
@@ -7420,6 +7467,12 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
 
+[[package]]
+name = "similar"
+version = "2.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
+
 [[package]]
 name = "simple_asn1"
 version = "0.6.3"

{floe_python-0.3.10 → floe_python-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.3.10
+Version: 0.4.1
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.3.10 → floe_python-0.4.1}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.3.10"
+version = "0.4.1"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"
@@ -41,16 +41,18 @@ uuid = "1"
 arrow = "57"
 iceberg = "0.9.0"
 iceberg-catalog-rest = "0.9.0"
-iceberg-storage-opendal = { version = "0.9.1", features = ["opendal-gcs"] }
+iceberg-storage-opendal = { version = "0.9.1", features = ["opendal-s3", "opendal-gcs"] }
 df-interchange = { version = "0.3.2", features = ["arrow_57", "polars_0_52"] }
 orc-rust = "0.7.1"
 
 reqwest = { version = "0.12", default-features = true, features = ["native-tls-vendored", "json", "blocking"] }
 sha2 = "0.10"
 hex = "0.4"
+thiserror = "1"
 
 [dev-dependencies]
 rust_xlsxwriter = "0.67"
+mockito = "1"
 
 [features]
 vendored-openssl = []

{floe_python-0.3.10 → floe_python-0.4.1}/crates/floe-core/src/checks/mismatch.rs

@@ -2,6 +2,7 @@ use std::collections::HashMap;
 
 use polars::prelude::{DataFrame, DataType, Series};
 
+use crate::config::PolicySeverity;
 use crate::errors::RunError;
 use crate::{config, report, ConfigError, FloeResult};
 
@@ -124,7 +125,7 @@ pub fn plan_schema_mismatch(
     let mut warning = None;
     let rejection_requested = (effective_missing == "reject_file" && !missing.is_empty())
         || (effective_extra == "reject_file" && !extra.is_empty());
-    if rejection_requested && entity.policy.severity == "warn" {
+    if rejection_requested && entity.policy.severity == PolicySeverity::Warn {
         warning = Some(format!(
             "entity.name={} schema mismatch requested reject_file but policy.severity=warn; continuing",
             entity.name
@@ -139,10 +140,10 @@ pub fn plan_schema_mismatch(
     if (effective_missing == "reject_file" && !missing.is_empty())
         || (effective_extra == "reject_file" && !extra.is_empty())
     {
-        if entity.policy.severity == "abort" {
+        if entity.policy.severity == PolicySeverity::Abort {
             aborted = true;
             action = report::MismatchAction::Aborted;
-        } else if entity.policy.severity == "reject" {
+        } else if entity.policy.severity == PolicySeverity::Reject {
             rejected = true;
             action = report::MismatchAction::RejectedFile;
         }

{floe_python-0.3.10 → floe_python-0.4.1}/crates/floe-core/src/config/mod.rs

@@ -13,6 +13,9 @@ pub use storage::{resolve_local_path, ConfigBase, ResolvedPath, StorageResolver}
 pub use types::*;
 
 pub use parse::extract_raw_env_vars;
-pub(crate) use parse::{parse_catalogs_with_context, parse_config, parse_config_with_vars};
+pub(crate) use parse::{
+    parse_catalogs_with_context, parse_config, parse_config_with_vars, parse_lineage_config,
+    parse_storages,
+};
 pub(crate) use template::apply_templates_with_vars;
 pub(crate) use validate::{extract_first_n, extract_last_n, validate_config};

{floe_python-0.3.10 → floe_python-0.4.1}/crates/floe-core/src/config/parse.rs

@@ -15,8 +15,8 @@ use crate::config::{
     DomainConfig, EntityConfig, EntityMetadata, EntityStateConfig, EnvConfig,
     IcebergPartitionFieldConfig, IcebergSinkTargetConfig, IncrementalMode, LineageConfig,
     MergeOptionsConfig, MergeScd2OptionsConfig, NormalizeColumnsConfig, PiiColumnConfig, PiiConfig,
-    PiiStrategy, PolicyConfig, ProjectMetadata, ReportConfig, RootConfig, SchemaConfig,
-    SchemaEvolutionConfig, SchemaEvolutionIncompatibleAction, SchemaEvolutionMode,
+    PiiStrategy, PolicyConfig, PolicySeverity, ProjectMetadata, ReportConfig, RootConfig,
+    SchemaConfig, SchemaEvolutionConfig, SchemaEvolutionIncompatibleAction, SchemaEvolutionMode,
     SchemaMismatchConfig, SinkConfig, SinkOptions, SinkTarget, SourceConfig, SourceOptions,
     StorageDefinition, StoragesConfig, WriteMode,
 };
@@ -658,7 +658,7 @@ fn parse_sink_delta_options(value: &Yaml, ctx: &str) -> FloeResult<DeltaSinkTarg
     })
 }
 
-fn parse_storages(value: &Yaml) -> FloeResult<StoragesConfig> {
+pub(crate) fn parse_storages(value: &Yaml) -> FloeResult<StoragesConfig> {
     let hash = yaml_hash(value, "storages")?;
     validate_known_keys(hash, "storages", &["default", "definitions"])?;
     let definitions_yaml = match hash_get(hash, "definitions") {
@@ -830,9 +830,18 @@ fn parse_archive_target(value: &Yaml) -> FloeResult<ArchiveTarget> {
 fn parse_policy(value: &Yaml) -> FloeResult<PolicyConfig> {
     let hash = yaml_hash(value, "policy")?;
     validate_known_keys(hash, "policy", &["severity"])?;
-    Ok(PolicyConfig {
-        severity: get_string(hash, "severity", "policy")?,
-    })
+    let severity_str = get_string(hash, "severity", "policy")?;
+    let severity = match severity_str.as_str() {
+        "warn" => PolicySeverity::Warn,
+        "reject" => PolicySeverity::Reject,
+        "abort" => PolicySeverity::Abort,
+        other => {
+            return Err(Box::new(ConfigError(format!(
+                "policy.severity={other} is unsupported (allowed: warn, reject, abort)"
+            ))))
+        }
+    };
+    Ok(PolicyConfig { severity })
 }
 
 fn parse_schema(value: &Yaml) -> FloeResult<SchemaConfig> {
@@ -1061,6 +1070,20 @@ fn opt_u64(hash: &Hash, key: &str, ctx: &str) -> FloeResult<Option<u64>> {
     }
 }
 
+fn opt_u32(hash: &Hash, key: &str, ctx: &str) -> FloeResult<Option<u32>> {
+    match opt_u64(hash, key, ctx)? {
+        None => Ok(None),
+        Some(v) => {
+            if v > u32::MAX as u64 {
+                return Err(Box::new(ConfigError(format!(
+                    "value at {ctx}.{key} exceeds maximum allowed value"
+                ))));
+            }
+            Ok(Some(v as u32))
+        }
+    }
+}
+
 fn parse_pii_config(value: &Yaml) -> FloeResult<PiiConfig> {
     let hash = yaml_hash(value, "pii")?;
     validate_known_keys(hash, "pii", &["columns"])?;
@@ -1106,12 +1129,19 @@ fn parse_pii_column(value: &Yaml) -> FloeResult<PiiColumnConfig> {
     })
 }
 
-fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
+pub(crate) fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
     let hash = yaml_hash(value, "lineage")?;
     validate_known_keys(
         hash,
         "lineage",
-        &["url", "api_key", "timeout_secs", "namespace", "producer"],
+        &[
+            "url",
+            "api_key",
+            "timeout_secs",
+            "namespace",
+            "producer",
+            "max_failures",
+        ],
     )?;
     Ok(LineageConfig {
         url: get_string(hash, "url", "lineage")?,
@@ -1119,5 +1149,6 @@ fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
         timeout_secs: opt_u64(hash, "timeout_secs", "lineage")?,
         namespace: get_string(hash, "namespace", "lineage")?,
         producer: opt_string(hash, "producer", "lineage")?,
+        max_failures: opt_u32(hash, "max_failures", "lineage")?,
     })
 }

{floe_python-0.3.10 → floe_python-0.4.1}/crates/floe-core/src/config/storage.rs

@@ -126,6 +126,7 @@ pub struct ResolvedPath {
     pub local_path: Option<PathBuf>,
 }
 
+#[derive(Clone)]
 pub struct StorageResolver {
     config_base: ConfigBase,
     default_name: String,
@@ -614,7 +615,7 @@ fn parent_prefix(key: &str) -> String {
     }
 }
 
-fn is_remote_uri(value: &str) -> bool {
+pub(crate) fn is_remote_uri(value: &str) -> bool {
     value.starts_with("s3://") || value.starts_with("gs://") || value.starts_with("abfs://")
 }
 

{floe_python-0.3.10 → floe_python-0.4.1}/crates/floe-core/src/config/types.rs

@@ -3,7 +3,7 @@ use std::path::Path;
 
 use polars::polars_utils::pl_str::PlSmallStr;
 use polars::prelude::{
-    CsvEncoding, CsvParseOptions, CsvReadOptions, DataType, NullValues, Schema, TimeUnit,
+    CsvEncoding, CsvParseOptions, CsvReadOptions, DataType, NullValues, TimeUnit,
 };
 
 use crate::{ConfigError, FloeResult};
@@ -28,6 +28,7 @@ pub struct LineageConfig {
     pub timeout_secs: Option<u64>,
     pub namespace: String,
     pub producer: Option<String>,
+    pub max_failures: Option<u32>,
 }
 
 #[derive(Debug)]
@@ -261,12 +262,6 @@ pub struct SinkConfig {
     pub archive: Option<ArchiveTarget>,
 }
 
-impl SinkConfig {
-    pub fn resolved_write_mode(&self) -> WriteMode {
-        self.write_mode
-    }
-}
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub enum WriteMode {
     #[default]
@@ -445,9 +440,33 @@ pub struct ArchiveTarget {
     pub storage: Option<String>,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum PolicySeverity {
+    #[default]
+    Warn,
+    Reject,
+    Abort,
+}
+
+impl PolicySeverity {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Warn => "warn",
+            Self::Reject => "reject",
+            Self::Abort => "abort",
+        }
+    }
+}
+
+impl std::fmt::Display for PolicySeverity {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.as_str())
+    }
+}
+
 #[derive(Debug)]
 pub struct PolicyConfig {
-    pub severity: String,
+    pub severity: PolicySeverity,
 }
 
 #[derive(Debug)]
@@ -464,36 +483,6 @@ impl SchemaConfig {
     pub fn resolved_schema_evolution(&self) -> SchemaEvolutionConfig {
         self.schema_evolution.unwrap_or_default()
     }
-
-    pub fn to_polars_schema(&self) -> FloeResult<Schema> {
-        let mut schema = Schema::with_capacity(self.columns.len());
-        for column in &self.columns {
-            let dtype = parse_data_type(&column.column_type)?;
-            if schema.insert(column.name.as_str().into(), dtype).is_some() {
-                return Err(Box::new(ConfigError(format!(
-                    "duplicate column name in schema: {}",
-                    column.name
-                ))));
-            }
-        }
-        Ok(schema)
-    }
-
-    pub fn to_polars_string_schema(&self) -> FloeResult<Schema> {
-        let mut schema = Schema::with_capacity(self.columns.len());
-        for column in &self.columns {
-            if schema
-                .insert(column.name.as_str().into(), DataType::String)
-                .is_some()
-            {
-                return Err(Box::new(ConfigError(format!(
-                    "duplicate column name in schema: {}",
-                    column.name
-                ))));
-            }
-        }
-        Ok(schema)
-    }
 }
 
 #[derive(Debug)]

{floe_python-0.3.10 → floe_python-0.4.1}/crates/floe-core/src/config/validate.rs

@@ -1,18 +1,19 @@
 use std::collections::HashSet;
 
+use crate::config::storage::is_remote_uri;
 use crate::config::{
-    CatalogDefinition, CatalogTypeConfig, EntityConfig, IncrementalMode, RootConfig, SourceOptions,
-    StorageDefinition,
+    CatalogDefinition, CatalogTypeConfig, EntityConfig, IncrementalMode, PolicySeverity,
+    RootConfig, SourceOptions, StorageDefinition,
 };
 use crate::io::format;
 use crate::io::read::json_selector::parse_selector;
 use crate::io::read::xml_selector;
+use crate::io::write::sink_format::sink_format;
 use crate::{warnings, ConfigError, FloeResult};
 
 const ALLOWED_COLUMN_TYPES: &[&str] = &["string", "number", "boolean", "datetime", "date", "time"];
 const ALLOWED_CAST_MODES: &[&str] = &["strict", "coerce"];
 const ALLOWED_NORMALIZE_STRATEGIES: &[&str] = &["snake_case", "lower", "camel_case", "none"];
-const ALLOWED_POLICY_SEVERITIES: &[&str] = &["warn", "reject", "abort"];
 const ALLOWED_MISSING_POLICIES: &[&str] = &["reject_file", "fill_nulls"];
 const ALLOWED_EXTRA_POLICIES: &[&str] = &["reject_file", "ignore"];
 const ALLOWED_STORAGE_TYPES: &[&str] = &["local", "s3", "adls", "gcs"];
@@ -103,6 +104,11 @@ fn validate_lineage(lineage: &crate::config::LineageConfig) -> FloeResult<()> {
             "lineage.namespace must not be empty".to_string(),
         )));
     }
+    if lineage.max_failures == Some(0) {
+        return Err(Box::new(ConfigError(
+            "lineage.max_failures must be at least 1".to_string(),
+        )));
+    }
     Ok(())
 }
 
@@ -123,6 +129,12 @@ fn validate_report(
 ) -> FloeResult<()> {
     let storage_name = storages.resolve_report_name(report.storage.as_deref())?;
     storages.validate_report_reference("report.storage", &storage_name)?;
+    if storages.definition_type(&storage_name) == Some("local") && is_remote_uri(&report.path) {
+        return Err(Box::new(ConfigError(format!(
+            "report.path must be a local path (got {})",
+            report.path
+        ))));
+    }
     Ok(())
 }
 
@@ -134,7 +146,6 @@ fn validate_entity(
 ) -> FloeResult<()> {
     validate_source(entity, storages)?;
     validate_state(entity)?;
-    validate_policy(entity)?;
     validate_sink(entity, storages, catalogs)?;
     validate_schema(entity, config_version)?;
     if let Some(pii) = &entity.pii {
@@ -147,7 +158,7 @@ fn validate_pii(entity: &EntityConfig, pii: &crate::config::PiiConfig) -> FloeRe
     use crate::config::PiiStrategy;
     // Abort severity writes the raw input file to the rejected sink without
     // loading a DataFrame, bypassing masking entirely.
-    if entity.policy.severity == "abort" {
+    if entity.policy.severity == PolicySeverity::Abort {
         return Err(Box::new(ConfigError(format!(
             "entity.name={} pii: masking is not applied when policy.severity=abort \
              because the raw file is written to sink.rejected without DataFrame processing",
@@ -197,7 +208,7 @@ fn validate_pii(entity: &EntityConfig, pii: &crate::config::PiiConfig) -> FloeRe
         .flatten()
         .map(|s| s.as_str())
         .collect();
-    let write_mode = entity.sink.resolved_write_mode();
+    let write_mode = entity.sink.write_mode;
     let is_merge_mode = matches!(
         write_mode,
         crate::config::WriteMode::MergeScd1 | crate::config::WriteMode::MergeScd2
@@ -515,7 +526,7 @@ fn validate_sink(
         entity.sink.accepted.options.as_ref(),
     )?;
 
-    if entity.policy.severity == "reject" && entity.sink.rejected.is_none() {
+    if entity.policy.severity == PolicySeverity::Reject && entity.sink.rejected.is_none() {
         return Err(Box::new(ConfigError(format!(
             "entity.name={} sink.rejected is required when policy.severity=reject",
             entity.name
@@ -534,28 +545,14 @@ fn validate_sink(
         entity.sink.accepted.storage.as_deref(),
     )?;
     storages.validate_reference(entity, "sink.accepted.storage", &accepted_storage)?;
-    if entity.sink.accepted.format == "delta" {
-        if let Some(storage_type) = storages.definition_type(&accepted_storage) {
-            if storage_type != "local"
-                && storage_type != "s3"
-                && storage_type != "adls"
-                && storage_type != "gcs"
-            {
-                return Err(Box::new(ConfigError(format!(
-                    "entity.name={} sink.accepted.format=delta is only supported on local, s3, adls, or gcs storage (got {})",
-                    entity.name, storage_type
-                ))));
-            }
-        }
-    }
-    if entity.sink.accepted.format == "iceberg" {
-        if let Some(storage_type) = storages.definition_type(&accepted_storage) {
-            if storage_type != "local" && storage_type != "s3" && storage_type != "gcs" {
-                return Err(Box::new(ConfigError(format!(
-                    "entity.name={} sink.accepted.format=iceberg is only supported on local, s3, or gcs storage for now (got {})",
-                    entity.name, storage_type
-                ))));
-            }
+    if let Some(storage_type) = storages.definition_type(&accepted_storage) {
+        let fmt = sink_format(entity.sink.accepted.format.as_str())?;
+        if !fmt.supported_storages().contains(&storage_type) {
+            let supported = fmt.supported_storages().join(", ");
+            return Err(Box::new(ConfigError(format!(
+                "entity.name={} sink.accepted.format={} is not supported on {} storage (supported: {})",
+                entity.name, entity.sink.accepted.format, storage_type, supported
+            ))));
         }
     }
     validate_iceberg_catalog_binding(entity, storages, catalogs, &accepted_storage)?;
@@ -589,20 +586,23 @@ fn validate_sink(
 }
 
 fn validate_sink_write_mode(entity: &EntityConfig) -> FloeResult<()> {
-    let write_mode = entity.sink.resolved_write_mode();
+    let write_mode = entity.sink.write_mode;
+    let fmt = sink_format(entity.sink.accepted.format.as_str())?;
+    if !fmt.supported_modes().contains(&write_mode) {
+        return Err(Box::new(ConfigError(format!(
+            "entity.name={} sink.write_mode={} is not supported by sink.accepted.format={}",
+            entity.name,
+            write_mode.as_str(),
+            entity.sink.accepted.format
+        ))));
+    }
+
     let is_merge_mode = matches!(
         write_mode,
         crate::config::WriteMode::MergeScd1 | crate::config::WriteMode::MergeScd2
     );
     if is_merge_mode {
         let mode_name = write_mode.as_str();
-        if entity.sink.accepted.format != "delta" {
-            return Err(Box::new(ConfigError(format!(
-                "entity.name={} sink.write_mode={} requires sink.accepted.format=delta",
-                entity.name, mode_name
-            ))));
-        }
-
         let primary_key = entity.schema.primary_key.as_ref().ok_or_else(|| {
             Box::new(ConfigError(format!(
                 "entity.name={} sink.write_mode={} requires schema.primary_key",
@@ -628,9 +628,16 @@ fn validate_merge_options(
         return Ok(());
     };
 
-    if entity.sink.accepted.format != "delta" {
+    let fmt = sink_format(entity.sink.accepted.format.as_str())?;
+    let supports_merge = fmt.supported_modes().iter().any(|m| {
+        matches!(
+            m,
+            crate::config::WriteMode::MergeScd1 | crate::config::WriteMode::MergeScd2
+        )
+    });
+    if !supports_merge {
         return Err(Box::new(ConfigError(format!(
-            "entity.name={} sink.accepted.merge is only supported when sink.accepted.format=delta",
+            "entity.name={} sink.accepted.merge is only supported when sink.accepted.format supports merge (e.g. delta)",
             entity.name
         ))));
     }
@@ -1063,18 +1070,6 @@ fn validate_sink_partitioning(entity: &EntityConfig) -> FloeResult<()> {
     Ok(())
 }
 
-fn validate_policy(entity: &EntityConfig) -> FloeResult<()> {
-    if !ALLOWED_POLICY_SEVERITIES.contains(&entity.policy.severity.as_str()) {
-        return Err(Box::new(ConfigError(format!(
-            "entity.name={} policy.severity={} is unsupported (allowed: {})",
-            entity.name,
-            entity.policy.severity,
-            ALLOWED_POLICY_SEVERITIES.join(", ")
-        ))));
-    }
-    Ok(())
-}
-
 fn validate_schema(entity: &EntityConfig, config_version: ConfigVersion) -> FloeResult<()> {
     if entity.source.format == "json" && entity.schema.columns.len() > MAX_JSON_COLUMNS {
         return Err(Box::new(ConfigError(format!(

floe_python-0.4.1/crates/floe-core/src/errors.rs

@@ -0,0 +1,27 @@
+use crate::log::emit_log;
+
+#[derive(Debug, thiserror::Error)]
+#[error("{0}")]
+pub struct ConfigError(pub String);
+
+#[derive(Debug, thiserror::Error)]
+#[error("{0}")]
+pub struct RunError(pub String);
+
+#[derive(Debug, thiserror::Error)]
+#[error("{0}")]
+pub struct StorageError(pub String);
+
+#[derive(Debug, thiserror::Error)]
+#[error("{0}")]
+pub struct IoError(pub String);
+
+pub fn emit(
+    run_id: &str,
+    entity: Option<&str>,
+    input: Option<&str>,
+    code: Option<&str>,
+    message: &str,
+) {
+    emit_log("error", run_id, entity, input, code, message);
+}