floe-python 0.4.1__tar.gz → 0.4.3__tar.gz

{floe_python-0.4.1 → floe_python-0.4.3}/Cargo.lock

@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3412,7 +3412,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "apache-avro 0.16.0",
  "arrow",
@@ -3455,7 +3455,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.4.1 → floe_python-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.4.1
+Version: 0.4.3
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -98,6 +98,43 @@ except floe.FloeError as e:
 | `set_observer(callback)` | Register a live-event callback |
 | `clear_observer()` | Remove the current callback |
 
+## Jupyter
+
+`RunOutcome` renders as a color-coded HTML table automatically in Jupyter — no extra code needed. Just end a cell with the variable:
+
+```python
+outcome = floe.run("orders.yml")
+outcome  # renders inline HTML table with per-entity status, accepted/rejected counts
+```
+
+Use `outcome.to_dict()` to turn results into a plain dict for pandas:
+
+```python
+import pandas as pd
+df = pd.DataFrame(outcome.entity_reports)
+```
+
+## Observing progress
+
+Register a callback to receive live events as the run proceeds:
+
+```python
+floe.set_observer(lambda e: print(f"[{e['event']}]", e.get("name", e.get("entity", ""))))
+outcome = floe.run("orders.yml")
+floe.clear_observer()
+```
+
+Event types: `run_started`, `entity_started`, `file_started`, `file_finished`, `schema_evolution_applied`, `entity_finished`, `run_finished`, `log`. See the [full guide](../../docs/python-bindings.md#observing-runs-in-real-time) for all event fields.
+
+## Profile overrides
+
+Override config variables or cloud credentials without editing the YAML:
+
+```python
+floe.run("orders.yml", profile_vars={"incoming_root": "s3://my-bucket/incoming"})
+floe.run("orders.yml", profile_path="prod.yml")
+```
+
 ## Building from source
 
 ```bash
@@ -111,3 +148,7 @@ maturin develop
 
 Apache 2.0
 
+---
+
+→ Full API reference and examples: [docs/python-bindings.md](../../docs/python-bindings.md)
+

{floe_python-0.4.1/crates/floe-python → floe_python-0.4.3}/README.md

@@ -70,6 +70,43 @@ except floe.FloeError as e:
 | `set_observer(callback)` | Register a live-event callback |
 | `clear_observer()` | Remove the current callback |
 
+## Jupyter
+
+`RunOutcome` renders as a color-coded HTML table automatically in Jupyter — no extra code needed. Just end a cell with the variable:
+
+```python
+outcome = floe.run("orders.yml")
+outcome  # renders inline HTML table with per-entity status, accepted/rejected counts
+```
+
+Use `outcome.to_dict()` to turn results into a plain dict for pandas:
+
+```python
+import pandas as pd
+df = pd.DataFrame(outcome.entity_reports)
+```
+
+## Observing progress
+
+Register a callback to receive live events as the run proceeds:
+
+```python
+floe.set_observer(lambda e: print(f"[{e['event']}]", e.get("name", e.get("entity", ""))))
+outcome = floe.run("orders.yml")
+floe.clear_observer()
+```
+
+Event types: `run_started`, `entity_started`, `file_started`, `file_finished`, `schema_evolution_applied`, `entity_finished`, `run_finished`, `log`. See the [full guide](../../docs/python-bindings.md#observing-runs-in-real-time) for all event fields.
+
+## Profile overrides
+
+Override config variables or cloud credentials without editing the YAML:
+
+```python
+floe.run("orders.yml", profile_vars={"incoming_root": "s3://my-bucket/incoming"})
+floe.run("orders.yml", profile_path="prod.yml")
+```
+
 ## Building from source
 
 ```bash
@@ -82,3 +119,7 @@ maturin develop
 ## License
 
 Apache 2.0
+
+---
+
+→ Full API reference and examples: [docs/python-bindings.md](../../docs/python-bindings.md)

{floe_python-0.4.1 → floe_python-0.4.3}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.4.1"
+version = "0.4.3"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"

{floe_python-0.4.1 → floe_python-0.4.3}/crates/floe-core/README.md

@@ -73,7 +73,7 @@ floe run      -c config.yml   # run the pipeline
 | S3 storage | [docs/storages/s3.md](docs/storages/s3.md) |
 | ADLS storage | [docs/storages/adls.md](docs/storages/adls.md) |
 | GCS storage | [docs/storages/gcs.md](docs/storages/gcs.md) |
-| Incremental ingestion | [docs/profiles.md](docs/profiles.md) |
+| Environement specific profile config | [docs/profiles.md](docs/profiles.md) [docs/variables.md](docs/variables.md) |
 | Run reports | [docs/report.md](docs/report.md) |
 | CLI reference | [docs/cli.md](docs/cli.md) |
 | Orchestration (Dagster / Airflow) | [docs/summary.md](docs/summary.md) |

{floe_python-0.4.1 → floe_python-0.4.3}/crates/floe-core/src/config/location.rs

@@ -70,6 +70,6 @@ fn download_remote_config(uri: &str, temp_dir: &Path) -> FloeResult<PathBuf> {
     Err(format!("unsupported config uri: {}", uri).into())
 }
 
-fn is_remote_uri(value: &str) -> bool {
+pub(crate) fn is_remote_uri(value: &str) -> bool {
     value.starts_with("s3://") || value.starts_with("gs://") || value.starts_with("abfs://")
 }

{floe_python-0.4.1 → floe_python-0.4.3}/crates/floe-core/src/config/mod.rs

@@ -8,6 +8,7 @@ mod validate;
 pub(crate) mod yaml_decode;
 
 pub use catalog::{CatalogResolver, ResolvedDeltaCatalogTarget, ResolvedIcebergCatalogTarget};
+pub(crate) use location::is_remote_uri;
 pub use location::{resolve_config_location, ConfigLocation};
 pub use storage::{resolve_local_path, ConfigBase, ResolvedPath, StorageResolver};
 pub use types::*;

{floe_python-0.4.1 → floe_python-0.4.3}/crates/floe-core/src/config/parse.rs

@@ -1141,6 +1141,7 @@ pub(crate) fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
             "namespace",
             "producer",
             "max_failures",
+            "job_name",
         ],
     )?;
     Ok(LineageConfig {
@@ -1150,5 +1151,6 @@ pub(crate) fn parse_lineage_config(value: &Yaml) -> FloeResult<LineageConfig> {
         namespace: get_string(hash, "namespace", "lineage")?,
         producer: opt_string(hash, "producer", "lineage")?,
         max_failures: opt_u32(hash, "max_failures", "lineage")?,
+        job_name: opt_string(hash, "job_name", "lineage")?,
     })
 }

{floe_python-0.4.1 → floe_python-0.4.3}/crates/floe-core/src/config/types.rs

@@ -1,6 +1,8 @@
 use std::collections::HashMap;
 use std::path::Path;
 
+use serde::{Deserialize, Serialize};
+
 use polars::polars_utils::pl_str::PlSmallStr;
 use polars::prelude::{
     CsvEncoding, CsvParseOptions, CsvReadOptions, DataType, NullValues, TimeUnit,
@@ -21,7 +23,7 @@ pub struct RootConfig {
     pub entities: Vec<EntityConfig>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct LineageConfig {
     pub url: String,
     pub api_key: Option<String>,
@@ -29,6 +31,7 @@ pub struct LineageConfig {
     pub namespace: String,
     pub producer: Option<String>,
     pub max_failures: Option<u32>,
+    pub job_name: Option<String>,
 }
 
 #[derive(Debug)]
@@ -66,12 +69,12 @@ pub struct EntityConfig {
     pub pii: Option<PiiConfig>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct PiiConfig {
     pub columns: Vec<PiiColumnConfig>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct PiiColumnConfig {
     pub name: String,
     pub strategy: PiiStrategy,
@@ -79,7 +82,8 @@ pub struct PiiColumnConfig {
     pub redact_value: Option<String>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum PiiStrategy {
     Hash,
     Drop,
@@ -103,7 +107,8 @@ impl EntityConfig {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum IncrementalMode {
     #[default]
     None,
@@ -146,7 +151,7 @@ pub struct SourceConfig {
     pub cast_mode: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Deserialize)]
 pub struct SourceOptions {
     pub header: Option<bool>,
     pub separator: Option<String>,
@@ -262,7 +267,8 @@ pub struct SinkConfig {
     pub archive: Option<ArchiveTarget>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
 pub enum WriteMode {
     #[default]
     Overwrite,
@@ -300,34 +306,34 @@ pub const DEFAULT_SCD2_CURRENT_FLAG_COLUMN: &str = "__floe_is_current";
 pub const DEFAULT_SCD2_VALID_FROM_COLUMN: &str = "__floe_valid_from";
 pub const DEFAULT_SCD2_VALID_TO_COLUMN: &str = "__floe_valid_to";
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct MergeOptionsConfig {
     pub ignore_columns: Option<Vec<String>>,
     pub compare_columns: Option<Vec<String>>,
     pub scd2: Option<MergeScd2OptionsConfig>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct MergeScd2OptionsConfig {
     pub current_flag_column: Option<String>,
     pub valid_from_column: Option<String>,
     pub valid_to_column: Option<String>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct SinkOptions {
     pub compression: Option<String>,
     pub row_group_size: Option<u64>,
     pub max_size_per_file: Option<u64>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct IcebergPartitionFieldConfig {
     pub column: String,
     pub transform: String,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct IcebergSinkTargetConfig {
     pub catalog: Option<String>,
     pub namespace: Option<String>,
@@ -335,7 +341,7 @@ pub struct IcebergSinkTargetConfig {
     pub location: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DeltaSinkTargetConfig {
     /// Name of the catalog definition to use (falls back to `catalogs.default`).
     pub catalog: Option<String>,
@@ -345,13 +351,13 @@ pub struct DeltaSinkTargetConfig {
     pub table: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct StoragesConfig {
     pub default: Option<String>,
     pub definitions: Vec<StorageDefinition>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct StorageDefinition {
     pub name: String,
     pub fs_type: String,
@@ -362,7 +368,7 @@ pub struct StorageDefinition {
     pub prefix: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct CatalogsConfig {
     pub default: Option<String>,
     pub definitions: Vec<CatalogDefinition>,
@@ -371,7 +377,8 @@ pub struct CatalogsConfig {
 /// Type-specific configuration for a catalog definition.
 /// Each variant carries only the fields relevant to that catalog type.
 /// Add a new variant here when supporting a new catalog type.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "lowercase")]
 pub enum CatalogTypeConfig {
     Glue {
         region: String,
@@ -419,7 +426,7 @@ impl CatalogTypeConfig {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct CatalogDefinition {
     pub name: String,
     pub type_config: CatalogTypeConfig,
@@ -440,7 +447,8 @@ pub struct ArchiveTarget {
     pub storage: Option<String>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum PolicySeverity {
     #[default]
     Warn,
@@ -469,7 +477,7 @@ pub struct PolicyConfig {
     pub severity: PolicySeverity,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize)]
 pub struct SchemaConfig {
     pub normalize_columns: Option<NormalizeColumnsConfig>,
     pub mismatch: Option<SchemaMismatchConfig>,
@@ -485,19 +493,20 @@ impl SchemaConfig {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct NormalizeColumnsConfig {
     pub enabled: Option<bool>,
     pub strategy: Option<String>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
 pub struct SchemaEvolutionConfig {
     pub mode: SchemaEvolutionMode,
     pub on_incompatible: SchemaEvolutionIncompatibleAction,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
 pub enum SchemaEvolutionMode {
     #[default]
     Strict,
@@ -513,7 +522,8 @@ impl SchemaEvolutionMode {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum SchemaEvolutionIncompatibleAction {
     #[default]
     Fail,
@@ -527,13 +537,13 @@ impl SchemaEvolutionIncompatibleAction {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct SchemaMismatchConfig {
     pub missing_columns: Option<String>,
     pub extra_columns: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize)]
 pub struct ColumnConfig {
     pub name: String,
     pub source: Option<String>,

{floe_python-0.4.1 → floe_python-0.4.3}/crates/floe-core/src/lib.rs

@@ -17,18 +17,23 @@ pub mod state;
 pub mod vars;
 pub mod warnings;
 
-pub use crate::state::{inspect_entity_state_with_base, reset_entity_state_with_base};
+pub use crate::state::{
+    inspect_entity_state, inspect_entity_state_with_base, reset_entity_state,
+    reset_entity_state_with_base,
+};
 pub use add_entity::{add_entity_to_config, AddEntityOptions, AddEntityOutcome};
 pub use checks as check;
 pub use config::{resolve_config_location, ConfigLocation};
 pub use errors::ConfigError;
-pub use manifest::build_common_manifest_json;
+pub use manifest::{build_common_manifest_json, config_from_manifest_json};
 pub use profile::{
     detect_malformed_placeholder, detect_unresolved_placeholders, parse_profile,
     parse_profile_from_str, validate_merged_vars, validate_profile, ProfileConfig,
 };
 pub use run::events::{set_observer, MultiObserver, RunEvent, RunObserver};
-pub use run::{run, run_with_base, DryRunEntityPreview, EntityOutcome, RunOutcome};
+pub use run::{
+    run, run_with_base, run_with_manifest_path, DryRunEntityPreview, EntityOutcome, RunOutcome,
+};
 pub use runner::{parse_run_status_from_logs, ConnectorRunStatus};
 pub use runtime::{DefaultRuntime, Runtime};
 pub use vars::{resolve_vars, VarSources};
@@ -79,6 +84,15 @@ pub fn load_config(config_path: &Path) -> FloeResult<config::RootConfig> {
     config::parse_config(config_path)
 }
 
+/// Read manifest JSON from any supported URI (local path, `s3://`, `gs://`, `abfs://`).
+/// For remote URIs the file is downloaded to a temp directory that is cleaned up before
+/// this function returns; the caller receives the raw JSON text as a `String`.
+pub fn read_manifest_text(uri: &str) -> FloeResult<String> {
+    let location = config::resolve_config_location(uri)?;
+    let text = std::fs::read_to_string(&location.path)?;
+    Ok(text)
+}
+
 pub fn load_config_with_profile_vars(
     config_path: &Path,
     profile_vars: &std::collections::HashMap<String, String>,