floe-python 0.4.3__tar.gz → 0.4.4__tar.gz

{floe_python-0.4.3 → floe_python-0.4.4}/Cargo.lock

@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.4.3"
+version = "0.4.4"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3412,7 +3412,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.4.3"
+version = "0.4.4"
 dependencies = [
  "apache-avro 0.16.0",
  "arrow",
@@ -3455,7 +3455,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.4.3"
+version = "0.4.4"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.4.3 → floe_python-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.4.3
+Version: 0.4.4
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.4.3"
+version = "0.4.4"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/src/io/write/iceberg/rest.rs

@@ -9,6 +9,7 @@ use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableIdent};
 use iceberg_catalog_rest::{
     RestCatalogBuilder, REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE,
 };
+use iceberg_storage_opendal::OpenDalStorageFactory;
 
 use crate::config::CatalogTypeConfig;
 use crate::errors::RunError;
@@ -28,6 +29,7 @@ use super::{map_iceberg_err, IcebergWriteResult, PreparedIcebergWrite};
 pub(crate) async fn build_rest_catalog(
     rest_cfg: &RestIcebergCatalogConfig,
     file_io_props: HashMap<String, String>,
+    table_location: &str,
 ) -> FloeResult<iceberg_catalog_rest::RestCatalog> {
     let mut props: HashMap<String, String> = file_io_props;
 
@@ -63,7 +65,29 @@ pub(crate) async fn build_rest_catalog(
         props.insert("scope".to_string(), scope.to_string());
     }
 
-    let storage_factory: Arc<dyn StorageFactory> = Arc::new(LocalFsStorageFactory);
+    // Prefer the concrete table location for storage factory dispatch; fall back to the
+    // warehouse field for cases where the caller only has catalog-level config (e.g. seeding).
+    let effective_uri = if !table_location.is_empty() {
+        table_location
+    } else {
+        rest_cfg.warehouse.as_deref().unwrap_or("")
+    };
+    let storage_factory: Arc<dyn StorageFactory> =
+        if effective_uri.starts_with("s3://") || effective_uri.starts_with("s3a://") {
+            let scheme = effective_uri
+                .split("://")
+                .next()
+                .unwrap_or("s3")
+                .to_string();
+            Arc::new(OpenDalStorageFactory::S3 {
+                configured_scheme: scheme,
+                customized_credential_load: None,
+            })
+        } else if effective_uri.starts_with("gs://") {
+            Arc::new(OpenDalStorageFactory::Gcs)
+        } else {
+            Arc::new(LocalFsStorageFactory)
+        };
 
     RestCatalogBuilder::default()
         .with_storage_factory(storage_factory)
@@ -125,7 +149,7 @@ pub(crate) async fn write_via_rest_catalog(
     mode: config::WriteMode,
     small_file_threshold_bytes: u64,
 ) -> FloeResult<IcebergWriteResult> {
-    let catalog = build_rest_catalog(rest_cfg, file_io_props).await?;
+    let catalog = build_rest_catalog(rest_cfg, file_io_props, &table_root_uri).await?;
 
     let namespace_name = rest_cfg.namespace.clone();
     let namespace = NamespaceIdent::new(namespace_name);

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/src/io/write/iceberg.rs

@@ -627,9 +627,14 @@ fn seed_iceberg_from_catalog(
                 .map_err(|err| Box::new(RunError(format!("glue iceberg seed failed: {err}"))))?;
             seed_from_batches(tracker, batches, rename_back)
         }
-        IcebergCatalogConfig::Rest(rest_cfg) => {
-            seed_iceberg_from_rest(tracker, rest_cfg, file_io_props, scan_cols, rename_back)
-        }
+        IcebergCatalogConfig::Rest(rest_cfg) => seed_iceberg_from_rest(
+            tracker,
+            rest_cfg,
+            file_io_props,
+            &warehouse_location,
+            scan_cols,
+            rename_back,
+        ),
     }
 }
 
@@ -637,6 +642,7 @@ fn seed_iceberg_from_rest(
     tracker: &mut check::UniqueTracker,
     rest_cfg: &RestIcebergCatalogConfig,
     file_io_props: HashMap<String, String>,
+    warehouse_location: &str,
     scan_cols: &[String],
     rename_back: &HashMap<String, String>,
 ) -> FloeResult<()> {
@@ -653,7 +659,7 @@ fn seed_iceberg_from_rest(
 
     let batches = runtime
         .block_on(async {
-            let catalog = build_rest_catalog(rest_cfg, file_io_props).await?;
+            let catalog = build_rest_catalog(rest_cfg, file_io_props, warehouse_location).await?;
             let namespace = NamespaceIdent::new(rest_cfg.namespace.clone());
 
             if !catalog

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/src/lib.rs

@@ -54,6 +54,7 @@ pub struct RunOptions {
     pub run_id: Option<String>,
     pub entities: Vec<String>,
     pub dry_run: bool,
+    pub full_refresh: bool,
     pub profile: Option<ProfileConfig>,
 }
 

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/src/run/context.rs

@@ -16,6 +16,7 @@ pub struct RunContext {
     pub run_id: String,
     pub started_at: String,
     pub run_timer: Instant,
+    pub full_refresh: bool,
 }
 
 impl RunContext {
@@ -82,6 +83,7 @@ impl RunContext {
             run_id,
             started_at,
             run_timer: Instant::now(),
+            full_refresh: options.full_refresh,
         })
     }
 
@@ -151,6 +153,7 @@ impl RunContext {
             run_id,
             started_at,
             run_timer: Instant::now(),
+            full_refresh: options.full_refresh,
         })
     }
 }

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/src/run/entity/accepted_write.rs

@@ -20,6 +20,7 @@ pub(super) struct AcceptedWritePhaseContext<'a> {
     pub(super) accepted_target: &'a Target,
     pub(super) temp_dir: Option<&'a Path>,
     pub(super) write_mode: config::WriteMode,
+    pub(super) full_refresh: bool,
     pub(super) perf_enabled: bool,
     pub(super) phase_timings: &'a mut EntityPhaseTimings,
     pub(super) pending_input_count: usize,
@@ -37,6 +38,7 @@ pub(super) fn run_accepted_write_phase(
         accepted_target,
         temp_dir,
         write_mode,
+        full_refresh,
         perf_enabled,
         phase_timings,
         pending_input_count,
@@ -47,7 +49,7 @@ pub(super) fn run_accepted_write_phase(
         crate::io::write::strategy::merge::shared::default_schema_evolution_summary(
             entity, write_mode,
         );
-    if pending_input_count == 0 {
+    if pending_input_count == 0 && !(full_refresh && write_mode == config::WriteMode::Overwrite) {
         return Ok(io::format::AcceptedWriteOutput {
             files_written: Some(0),
             schema_evolution: default_schema_evolution,

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/src/run/entity/incremental.rs

@@ -11,7 +11,8 @@ use crate::report::{
 };
 use crate::run::RunContext;
 use crate::state::{
-    claim_entity_inputs, promote_claimed_entity_state, release_claimed_entity_state,
+    claim_all_entity_inputs, claim_entity_inputs, promote_claimed_entity_state,
+    promote_full_refresh_claimed_entity_state, release_claimed_entity_state,
     renew_claimed_entity_state, ClaimedEntityState, EntityFileState, CLAIM_TTL_SECONDS,
 };
 use crate::{config, warnings, FloeResult};
@@ -28,6 +29,34 @@ pub(super) fn prepare_incremental_context(
     entity: &config::EntityConfig,
     input_files: Vec<InputFile>,
 ) -> FloeResult<IncrementalContext> {
+    if context.full_refresh {
+        let pending_state = if entity.incremental_mode == IncrementalMode::File {
+            claim_all_entity_inputs(
+                &context.storage_resolver,
+                cloud,
+                entity,
+                &context.run_id,
+                input_files.clone(),
+            )?
+            .map(|claimed| {
+                PendingEntityState::new(
+                    claimed,
+                    context.storage_resolver.clone(),
+                    entity.name.clone(),
+                    context.run_id.clone(),
+                    true,
+                )
+            })
+        } else {
+            None
+        };
+        return Ok(IncrementalContext {
+            pending_inputs: input_files,
+            skipped_reports: Vec::new(),
+            pending_state,
+        });
+    }
+
     if entity.incremental_mode != IncrementalMode::File {
         return Ok(IncrementalContext {
             pending_inputs: input_files,
@@ -104,6 +133,7 @@ pub(super) fn prepare_incremental_context(
                 context.storage_resolver.clone(),
                 entity.name.clone(),
                 context.run_id.clone(),
+                false,
             )
         }),
     })
@@ -156,6 +186,7 @@ pub(super) struct PendingEntityState {
     run_id: String,
     heartbeat: Option<ClaimHeartbeat>,
     finalized: bool,
+    is_full_refresh: bool,
 }
 
 impl PendingEntityState {
@@ -164,6 +195,7 @@ impl PendingEntityState {
         resolver: StorageResolver,
         entity_name: String,
         run_id: String,
+        is_full_refresh: bool,
     ) -> Self {
         let heartbeat = ClaimHeartbeat::start(
             resolver.clone(),
@@ -178,6 +210,7 @@ impl PendingEntityState {
             run_id,
             heartbeat: Some(heartbeat),
             finalized: false,
+            is_full_refresh,
         }
     }
 
@@ -188,13 +221,23 @@ impl PendingEntityState {
         _entity: &config::EntityConfig,
     ) -> FloeResult<()> {
         self.stop_heartbeat();
-        promote_claimed_entity_state(
-            &self.resolver,
-            cloud,
-            &self.entity_name,
-            &self.run_id,
-            &self.claimed,
-        )?;
+        if self.is_full_refresh {
+            promote_full_refresh_claimed_entity_state(
+                &self.resolver,
+                cloud,
+                &self.entity_name,
+                &self.run_id,
+                &self.claimed,
+            )?;
+        } else {
+            promote_claimed_entity_state(
+                &self.resolver,
+                cloud,
+                &self.entity_name,
+                &self.run_id,
+                &self.claimed,
+            )?;
+        }
         self.finalized = true;
         Ok(())
     }

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/src/run/entity/mod.rs

@@ -87,7 +87,12 @@ pub(super) fn run_entity(
     let entity_start = perf_enabled.then(Instant::now);
     let mut phase_timings = EntityPhaseTimings::default();
     let input = &entity.source;
-    let write_mode = entity.sink.write_mode;
+    let write_mode =
+        if context.full_refresh && entity.sink.write_mode == crate::config::WriteMode::Append {
+            crate::config::WriteMode::Overwrite
+        } else {
+            entity.sink.write_mode
+        };
     let input_adapter = runtime.input_adapter(input.format.as_str())?;
     let resolved_targets = plan.resolved_targets;
     let formatter_name = context
@@ -284,6 +289,7 @@ pub(super) fn run_entity(
         accepted_target: &accepted_target,
         temp_dir: temp_dir_path,
         write_mode,
+        full_refresh: context.full_refresh,
         perf_enabled,
         phase_timings: &mut phase_timings,
         pending_input_count,

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/src/run/mod.rs

@@ -175,6 +175,20 @@ fn run_from_context(
     let observer = default_observer();
     let perf_enabled = perf::phase_timing_enabled();
     let selected_entities = select_entities(&context, &options);
+    if options.full_refresh {
+        for entity in &selected_entities {
+            if matches!(
+                entity.sink.write_mode,
+                config::WriteMode::MergeScd1 | config::WriteMode::MergeScd2
+            ) {
+                return Err(Box::new(ConfigError(format!(
+                    "entity '{}': --full-refresh is not supported with write_mode '{}'",
+                    entity.name,
+                    entity.sink.write_mode.as_str()
+                ))));
+            }
+        }
+    }
     let resolution_mode = if options.dry_run {
         io::storage::inputs::ResolveInputsMode::ListOnly
     } else {

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/src/state/mod.rs

@@ -241,6 +241,68 @@ pub fn claim_entity_inputs(
     ))))
 }
 
+/// Full-refresh variant of `claim_entity_inputs`.
+///
+/// Loads existing state to get the current CAS version, then writes a blank
+/// `EntityState` containing fresh claims for ALL `input_files` — discarding
+/// the historical `files` and `claims` maps. On successful commit the state
+/// file will contain only the files processed in this run.
+///
+/// Returns `None` when `input_files` is empty (nothing to claim).
+pub fn claim_all_entity_inputs(
+    resolver: &StorageResolver,
+    cloud: &mut CloudClient,
+    entity: &EntityConfig,
+    run_id: &str,
+    input_files: Vec<crate::io::format::InputFile>,
+) -> FloeResult<Option<ClaimedEntityState>> {
+    let acquired_at = now_rfc3339();
+    let expires_at = rfc3339_after_seconds(CLAIM_TTL_SECONDS);
+
+    for _ in 0..STATE_CAS_RETRIES {
+        // Read only to obtain the current CAS version; content is discarded.
+        let loaded = load_entity_state(resolver, cloud, entity)?;
+
+        let mut fresh_state = EntityState::new(&entity.name);
+        fresh_state.files = loaded.state.files.clone();
+        fresh_state.updated_at = Some(acquired_at.clone());
+        for input_file in &input_files {
+            fresh_state.claims.insert(
+                input_file.source_uri.clone(),
+                EntityFileClaim {
+                    run_id: run_id.to_string(),
+                    acquired_at: acquired_at.clone(),
+                    expires_at: expires_at.clone(),
+                    size: input_file.source_size,
+                    mtime: input_file.source_mtime.clone(),
+                },
+            );
+        }
+
+        let fresh_loaded = LoadedEntityState {
+            target: loaded.target,
+            state: fresh_state,
+            version: loaded.version,
+            existed: loaded.existed,
+        };
+        match persist_loaded_state(cloud, resolver, &fresh_loaded)? {
+            Some(version) => {
+                return Ok(Some(ClaimedEntityState {
+                    target: fresh_loaded.target,
+                    state: fresh_loaded.state,
+                    version,
+                }));
+            }
+            None => continue,
+        }
+    }
+
+    Err(Box::new(ConfigError(format!(
+        "entity.name={} full-refresh state write conflicted after {STATE_CAS_RETRIES} retries",
+        entity.name
+    ))))
+}
+
 pub fn promote_claimed_entity_state(
     resolver: &StorageResolver,
     cloud: &mut CloudClient,
@@ -272,6 +334,38 @@ pub fn promote_claimed_entity_state(
     })
 }
 
+pub fn promote_full_refresh_claimed_entity_state(
+    resolver: &StorageResolver,
+    cloud: &mut CloudClient,
+    entity_name: &str,
+    run_id: &str,
+    claimed: &ClaimedEntityState,
+) -> FloeResult<()> {
+    mutate_claimed_state(resolver, cloud, entity_name, claimed, |state, our_uris| {
+        let processed_at = now_rfc3339();
+        let claimed_files: Vec<String> = state
+            .claims
+            .iter()
+            .filter(|(uri, claim)| claim.run_id == run_id && our_uris.contains(*uri))
+            .map(|(uri, _)| uri.clone())
+            .collect();
+        state.files.clear();
+        for source_uri in claimed_files {
+            if let Some(claim) = state.claims.remove(&source_uri) {
+                state.files.insert(
+                    source_uri,
+                    EntityFileState {
+                        processed_at: processed_at.clone(),
+                        size: claim.size,
+                        mtime: claim.mtime,
+                    },
+                );
+            }
+        }
+        state.updated_at = Some(processed_at);
+    })
+}
+
 pub fn release_claimed_entity_state(
     resolver: &StorageResolver,
     cloud: &mut CloudClient,

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/tests/integration/archive_run.rs

@@ -105,6 +105,7 @@ fn archive_moves_only_processed_input_not_local_siblings() {
             run_id: Some("archive-sibling-it".to_string()),
             entities: Vec::new(),
             dry_run: false,
+            full_refresh: false,
         },
     )
     .expect("run config");
@@ -163,6 +164,7 @@ fn archive_repeated_runs_do_not_overwrite_same_source_filename() {
             run_id: Some("archive-run-1".to_string()),
             entities: Vec::new(),
             dry_run: false,
+            full_refresh: false,
         },
     )
     .expect("first run");
@@ -175,6 +177,7 @@ fn archive_repeated_runs_do_not_overwrite_same_source_filename() {
             run_id: Some("archive-run-2".to_string()),
             entities: Vec::new(),
             dry_run: false,
+            full_refresh: false,
         },
     )
     .expect("second run");
@@ -227,6 +230,7 @@ fn legacy_archive_config_still_archives_inputs() {
             run_id: Some("archive-legacy-compat".to_string()),
             entities: Vec::new(),
             dry_run: false,
+            full_refresh: false,
         },
     )
     .expect("run config");

{floe_python-0.4.3 → floe_python-0.4.4}/crates/floe-core/tests/integration/composite_unique.rs

@@ -81,6 +81,7 @@ entities:
             run_id: Some("it-composite-unique".to_string()),
             entities: Vec::new(),
             dry_run: false,
+            full_refresh: false,
         },
     )
     .expect("run config");