floe-python 0.6.4__tar.gz → 0.6.6__tar.gz

{floe_python-0.6.4 → floe_python-0.6.6}/Cargo.lock

@@ -3340,7 +3340,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.6.4"
+version = "0.6.6"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3353,7 +3353,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.6.4"
+version = "0.6.6"
 dependencies = [
  "apache-avro",
  "arrow",
@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.6.4"
+version = "0.6.6"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.6.4 → floe_python-0.6.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.6.4
+Version: 0.6.6
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.6.4 → floe_python-0.6.6}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.6.4"
+version = "0.6.6"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"

{floe_python-0.6.4 → floe_python-0.6.6}/crates/floe-core/src/config/location.rs

@@ -10,6 +10,11 @@ pub struct ConfigLocation {
     pub path: PathBuf,
     pub base: ConfigBase,
     pub display: String,
+    /// URI recorded as `config_uri` / `profile_uri` in the manifest:
+    /// `local://<path-as-typed>` for local configs (normalized, not canonicalized,
+    /// so a relative `-c` stays relative and the manifest does not depend on where
+    /// the file physically lives), or the scheme-normalized remote URI otherwise.
+    pub uri: String,
     _temp_dir: Option<TempDir>,
 }
 
@@ -18,10 +23,13 @@ pub fn resolve_config_location(input: &str) -> FloeResult<ConfigLocation> {
         let temp_dir = TempDir::new()?;
         let local_path = download_remote_config(input, temp_dir.path())?;
         let base = ConfigBase::remote_from_uri(temp_dir.path().to_path_buf(), input)?;
+        // Remote URIs are already environment-independent; keep them absolute.
+        let uri = storage::uri::normalize_remote_uri(input).into_owned();
         Ok(ConfigLocation {
             path: local_path,
             base,
             display: input.to_string(),
+            uri,
             _temp_dir: Some(temp_dir),
         })
     } else {
@@ -37,11 +45,46 @@ pub fn resolve_config_location(input: &str) -> FloeResult<ConfigLocation> {
             path: canonical.clone(),
             base,
             display: canonical.display().to_string(),
+            uri: format!("local://{}", normalize_input_path(input)),
             _temp_dir: None,
         })
     }
 }
 
+/// Lexically normalize a local path for `config_uri` / `profile_uri` without
+/// touching the filesystem, so a relative `-c` stays relative and the same path is
+/// produced on any machine. Uses `/` as the separator.
+fn normalize_input_path(input: &str) -> String {
+    use std::path::Component;
+
+    let mut parts: Vec<String> = Vec::new();
+    let mut has_prefix = false;
+    for comp in Path::new(input).components() {
+        match comp {
+            Component::CurDir => {}
+            Component::Prefix(prefix) => {
+                has_prefix = true;
+                parts.push(prefix.as_os_str().to_string_lossy().into_owned());
+            }
+            // A Windows drive/UNC prefix already carries the root; skip the trailing
+            // RootDir so we emit `C:/...`, not `C://...` (which looks like a scheme).
+            Component::RootDir => {
+                if !has_prefix {
+                    parts.push(String::new());
+                }
+            }
+            other => parts.push(other.as_os_str().to_string_lossy().into_owned()),
+        }
+    }
+
+    let joined = parts.join("/");
+    if joined.is_empty() {
+        ".".to_string()
+    } else {
+        joined
+    }
+}
+
 fn download_remote_config(uri: &str, temp_dir: &Path) -> FloeResult<PathBuf> {
     let normalized = storage::uri::normalize_remote_uri(uri);
     let uri = normalized.as_ref();

{floe_python-0.6.4 → floe_python-0.6.6}/crates/floe-core/src/io/write/iceberg/glue.rs

@@ -2,10 +2,11 @@ use crate::errors::FloeError;
 use aws_config::meta::region::RegionProviderChain;
 use aws_sdk_glue::config::Region as GlueRegion;
 use aws_sdk_glue::types::{
-    DatabaseInput as GlueDatabaseInput, StorageDescriptor as GlueStorageDescriptor,
-    TableInput as GlueTableInput,
+    Column as GlueColumn, DatabaseInput as GlueDatabaseInput,
+    StorageDescriptor as GlueStorageDescriptor, TableInput as GlueTableInput,
 };
 use aws_sdk_glue::Client as GlueClient;
+use iceberg::spec::{PrimitiveType, Schema, Type};
 
 use crate::{warnings, FloeResult};
 
@@ -127,6 +128,7 @@ pub(super) async fn upsert_glue_table(
     table_root_uri: &str,
     metadata_location: &str,
     version_id: Option<&str>,
+    schema: &Schema,
 ) -> FloeResult<()> {
     let client = build_glue_client(&glue_cfg.region).await?;
     ensure_glue_database(
@@ -136,7 +138,7 @@ pub(super) async fn upsert_glue_table(
     )
     .await?;
 
-    let table_input = build_glue_table_input(glue_cfg, table_root_uri, metadata_location);
+    let table_input = build_glue_table_input(glue_cfg, table_root_uri, metadata_location, schema);
 
     let create_result = client
         .create_table()
@@ -259,13 +261,54 @@ async fn ensure_glue_database(
     }
 }
 
+fn iceberg_type_to_glue_type(t: &Type) -> String {
+    match t {
+        Type::Primitive(p) => match p {
+            PrimitiveType::Boolean => "boolean".into(),
+            PrimitiveType::Int => "int".into(),
+            PrimitiveType::Long => "bigint".into(),
+            PrimitiveType::Float => "float".into(),
+            PrimitiveType::Double => "double".into(),
+            PrimitiveType::Decimal { precision, scale } => {
+                format!("decimal({precision},{scale})")
+            }
+            PrimitiveType::Date => "date".into(),
+            PrimitiveType::Time => "string".into(),
+            PrimitiveType::Timestamp
+            | PrimitiveType::Timestamptz
+            | PrimitiveType::TimestampNs
+            | PrimitiveType::TimestamptzNs => "timestamp".into(),
+            PrimitiveType::String | PrimitiveType::Uuid => "string".into(),
+            PrimitiveType::Fixed(_) | PrimitiveType::Binary => "binary".into(),
+        },
+        Type::Struct(_) => "struct<>".into(),
+        Type::List(_) => "array<string>".into(),
+        Type::Map(_) => "map<string,string>".into(),
+    }
+}
+
 fn build_glue_table_input(
     glue_cfg: &GlueIcebergCatalogConfig,
     table_root_uri: &str,
     metadata_location: &str,
+    schema: &Schema,
 ) -> GlueTableInput {
+    let columns: Vec<GlueColumn> = schema
+        .as_struct()
+        .fields()
+        .iter()
+        .map(|f| {
+            GlueColumn::builder()
+                .name(f.name.as_str())
+                .r#type(iceberg_type_to_glue_type(&f.field_type))
+                .build()
+                .expect("glue column builder validated")
+        })
+        .collect();
+
     let storage_descriptor = GlueStorageDescriptor::builder()
         .location(table_root_uri)
+        .set_columns(Some(columns))
         .build();
 
     GlueTableInput::builder()

{floe_python-0.6.4 → floe_python-0.6.6}/crates/floe-core/src/io/write/iceberg.rs

@@ -572,6 +572,7 @@ async fn write_iceberg_table_async(
             glue_state
                 .as_ref()
                 .and_then(|state| state.version_id.as_deref()),
+            &prepared.iceberg_schema,
         )
         .await?;
     }

{floe_python-0.6.4 → floe_python-0.6.6}/crates/floe-core/src/manifest/builder.rs

@@ -378,7 +378,10 @@ fn build_common_manifest(
         });
     }
 
-    let config_uri = canonical_config_uri(&config_location.display);
+    // Use `uri` (the path as typed / resolved remote URI), not the host-absolute
+    // `display`, so config_uri and the manifest_id derived from it do not depend on
+    // where the config file physically lives.
+    let config_uri = config_location.uri.clone();
     let config_checksum = std::fs::read(&config_location.path)
         .ok()
         .map(|b| sha256_hex(&b));
@@ -536,14 +539,6 @@ fn resolved_uri_to_path(uri: &str) -> String {
     }
 }
 
-fn canonical_config_uri(display: &str) -> String {
-    if display.contains("://") {
-        display.to_string()
-    } else {
-        format!("local://{}", display)
-    }
-}
-
 fn build_manifest_id(config_uri: &str, config_checksum: Option<&str>) -> String {
     const FNV_OFFSET_BASIS: u64 = 0xcbf29ce484222325;
     const FNV_PRIME: u64 = 0x100000001b3;

floe_python-0.6.6/crates/floe-core/tests/unit/config/location.rs

@@ -0,0 +1,44 @@
+use floe_core::resolve_config_location;
+
+fn write_config(dir: &std::path::Path) -> std::path::PathBuf {
+    let path = dir.join("config.yml");
+    std::fs::write(&path, "version: \"0.1\"\nentities: []\n").expect("write config");
+    path
+}
+
+#[test]
+fn local_config_uri_uses_local_scheme() {
+    let tmp = tempfile::TempDir::new().expect("temp dir");
+    let cfg = write_config(tmp.path());
+    let loc = resolve_config_location(cfg.to_str().expect("utf8")).expect("resolve");
+    assert!(loc.uri.starts_with("local://"), "uri: {}", loc.uri);
+    assert!(loc.uri.ends_with("/config.yml"), "uri: {}", loc.uri);
+}
+
+// The path is preserved lexically, not canonicalized: a `.` is dropped and a `..`
+// bounce survives (canonicalize would collapse it), and separators become `/`.
+#[test]
+fn local_config_uri_is_lexically_normalized() {
+    let tmp = tempfile::TempDir::new().expect("temp dir");
+    let cfg_dir = tmp.path().join("cfg");
+    std::fs::create_dir_all(&cfg_dir).expect("cfg dir");
+    write_config(&cfg_dir);
+
+    let bouncy = format!("{}/./../cfg/config.yml", cfg_dir.display());
+    let loc = resolve_config_location(&bouncy).expect("resolve");
+    assert!(loc.uri.contains("/../cfg/config.yml"), "uri: {}", loc.uri);
+    assert!(!loc.uri.contains("/./"), "uri: {}", loc.uri);
+}
+
+// A Windows drive path must yield `local://C:/...`, not `local://C://...` (which the
+// `local://` prefix would turn into a scheme-colliding, invalid URI). Windows-only
+// because drive prefixes are only parsed on Windows targets.
+#[cfg(windows)]
+#[test]
+fn windows_drive_config_uri_has_single_slash_after_drive() {
+    let tmp = tempfile::TempDir::new().expect("temp dir");
+    let cfg = write_config(tmp.path());
+    let loc = resolve_config_location(cfg.to_str().expect("utf8")).expect("resolve");
+    assert!(loc.uri.starts_with("local://"), "uri: {}", loc.uri);
+    assert!(!loc.uri.contains("C://"), "uri: {}", loc.uri);
+}

{floe_python-0.6.4 → floe_python-0.6.6}/crates/floe-core/tests/unit/config/mod.rs

@@ -12,6 +12,7 @@ pub mod gcs_storage;
 pub mod gcs_validation;
 pub mod lineage_validation;
 pub mod local_storage;
+pub mod location;
 pub mod parse;
 pub mod pii_validation;
 pub mod remote_base;

{floe_python-0.6.4 → floe_python-0.6.6}/crates/floe-core/tests/unit/manifest/mod.rs

@@ -16,13 +16,10 @@ fn repo_root() -> PathBuf {
 
 #[test]
 fn manifest_uses_local_uri_for_local_config() {
-    let config_path = repo_root().join("example/config.yml");
-    let expected_config_uri = format!(
-        "local://{}",
-        std::fs::canonicalize(&config_path)
-            .expect("canonicalize config path")
-            .display()
-    );
+    // Pass a canonical path so the expected value is independent of the test's cwd.
+    let config_path = std::fs::canonicalize(repo_root().join("example/config.yml"))
+        .expect("canonicalize config path");
+    let expected_config_uri = format!("local://{}", config_path.display());
 
     let config_location = resolve_config_location(
         config_path
@@ -46,6 +43,38 @@ fn manifest_uses_local_uri_for_local_config() {
     assert_eq!(value["config_uri"], expected_config_uri);
 }
 
+#[test]
+fn manifest_config_uri_is_not_canonicalized() {
+    // config_uri reflects the path as typed, not the canonical path: a lexical `..`
+    // bounce (which canonicalize() would collapse) surviving in the output proves
+    // the path was preserved as-typed.
+    let temp_dir = tempfile::TempDir::new().expect("temp dir");
+    let cfg_dir = temp_dir.path().join("cfg");
+    std::fs::create_dir_all(&cfg_dir).expect("cfg dir");
+    let config_path = cfg_dir.join("config.yml");
+    std::fs::write(
+        &config_path,
+        "version: \"0.1\"\nentities:\n  - name: orders\n    source:\n      format: csv\n      path: ./in/orders\n    sink:\n      write_mode: overwrite\n      accepted:\n        format: parquet\n        path: ./out/orders\n    policy:\n      severity: warn\n    schema:\n      columns:\n        - name: id\n          type: string\n",
+    )
+    .expect("write config");
+
+    // <cfg_dir>/../cfg/config.yml — canonicalize() would resolve this to
+    // <cfg_dir>/config.yml, dropping the "/../".
+    let bouncy = format!("{}/../cfg/config.yml", cfg_dir.display());
+    let loc = resolve_config_location(&bouncy).expect("resolve config location");
+    let config = load_config(&loc.path).expect("load config");
+    let payload = build_common_manifest_json(&loc, &config, &[], None, &ManifestOptions::default())
+        .expect("manifest");
+    let value: Value = serde_json::from_str(&payload).expect("valid json");
+
+    let uri = value["config_uri"].as_str().expect("config_uri string");
+    assert!(
+        uri.contains("/../cfg/config.yml"),
+        "config_uri was canonicalized (lost the as-typed form): {uri}"
+    );
+    assert!(value["manifest_id"].as_str().unwrap().starts_with("mfv1-"));
+}
+
 #[test]
 fn manifest_id_is_stable_for_same_config() {
     let config_path = repo_root().join("example/config.yml");

{floe_python-0.6.4 → floe_python-0.6.6}/crates/floe-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-python"
-version = "0.6.4"
+version = "0.6.6"
 edition = "2021"
 description = "Python bindings for floe-core (PyO3-based)"
 license = "Apache-2.0"
@@ -14,7 +14,7 @@ name = "_floe"
 crate-type = ["cdylib"]
 
 [dependencies]
-floe-core = { path = "../floe-core", version = "0.6.4", default-features = false }
+floe-core = { path = "../floe-core", version = "0.6.6", default-features = false }
 pyo3 = { version = "0.29", features = ["extension-module", "abi3-py310"] }
 serde_json = "1"
 

{floe_python-0.6.4 → floe_python-0.6.6}/crates/floe-python/pyproject.duckdb.toml

@@ -22,7 +22,7 @@ build-backend = "maturin"
 
 [project]
 name = "floe-duckdb"
-version = "0.6.4"
+version = "0.6.6"
 description = "DuckDB companion extension for floe — adds the native DuckDB sink to the floe Python package"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -46,7 +46,7 @@ classifiers = [
 # only contributes the compiled DuckDB extension module into that package. Pin to
 # the exact lean version so the companion's `_floe_duckdb` ABI always matches the
 # `floe` package it plugs into.
-dependencies = ["floe-python==0.6.4"]
+dependencies = ["floe-python==0.6.6"]
 
 [project.urls]
 Homepage = "https://github.com/malon64/floe"

{floe_python-0.6.4 → floe_python-0.6.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "floe-python"
-version = "0.6.4"
+version = "0.6.6"
 description = "Python bindings for floe-core — run floe pipelines at Rust speed from Python and notebooks"
 readme = "README.md"
 requires-python = ">=3.10"