floe-python 0.6.4__tar.gz → 0.6.5__tar.gz

{floe_python-0.6.4 → floe_python-0.6.5}/Cargo.lock

@@ -3340,7 +3340,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.6.4"
+version = "0.6.5"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3353,7 +3353,7 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.6.4"
+version = "0.6.5"
 dependencies = [
  "apache-avro",
  "arrow",
@@ -3399,7 +3399,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.6.4"
+version = "0.6.5"
 dependencies = [
  "floe-core",
  "pyo3",

{floe_python-0.6.4 → floe_python-0.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.6.4
+Version: 0.6.5
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.6.4 → floe_python-0.6.5}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.6.4"
+version = "0.6.5"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"

{floe_python-0.6.4 → floe_python-0.6.5}/crates/floe-core/src/io/write/iceberg/glue.rs

@@ -2,10 +2,11 @@ use crate::errors::FloeError;
 use aws_config::meta::region::RegionProviderChain;
 use aws_sdk_glue::config::Region as GlueRegion;
 use aws_sdk_glue::types::{
-    DatabaseInput as GlueDatabaseInput, StorageDescriptor as GlueStorageDescriptor,
-    TableInput as GlueTableInput,
+    Column as GlueColumn, DatabaseInput as GlueDatabaseInput,
+    StorageDescriptor as GlueStorageDescriptor, TableInput as GlueTableInput,
 };
 use aws_sdk_glue::Client as GlueClient;
+use iceberg::spec::{PrimitiveType, Schema, Type};
 
 use crate::{warnings, FloeResult};
 
@@ -127,6 +128,7 @@ pub(super) async fn upsert_glue_table(
     table_root_uri: &str,
     metadata_location: &str,
     version_id: Option<&str>,
+    schema: &Schema,
 ) -> FloeResult<()> {
     let client = build_glue_client(&glue_cfg.region).await?;
     ensure_glue_database(
@@ -136,7 +138,7 @@ pub(super) async fn upsert_glue_table(
     )
     .await?;
 
-    let table_input = build_glue_table_input(glue_cfg, table_root_uri, metadata_location);
+    let table_input = build_glue_table_input(glue_cfg, table_root_uri, metadata_location, schema);
 
     let create_result = client
         .create_table()
@@ -259,13 +261,54 @@ async fn ensure_glue_database(
     }
 }
 
+fn iceberg_type_to_glue_type(t: &Type) -> String {
+    match t {
+        Type::Primitive(p) => match p {
+            PrimitiveType::Boolean => "boolean".into(),
+            PrimitiveType::Int => "int".into(),
+            PrimitiveType::Long => "bigint".into(),
+            PrimitiveType::Float => "float".into(),
+            PrimitiveType::Double => "double".into(),
+            PrimitiveType::Decimal { precision, scale } => {
+                format!("decimal({precision},{scale})")
+            }
+            PrimitiveType::Date => "date".into(),
+            PrimitiveType::Time => "string".into(),
+            PrimitiveType::Timestamp
+            | PrimitiveType::Timestamptz
+            | PrimitiveType::TimestampNs
+            | PrimitiveType::TimestamptzNs => "timestamp".into(),
+            PrimitiveType::String | PrimitiveType::Uuid => "string".into(),
+            PrimitiveType::Fixed(_) | PrimitiveType::Binary => "binary".into(),
+        },
+        Type::Struct(_) => "struct<>".into(),
+        Type::List(_) => "array<string>".into(),
+        Type::Map(_) => "map<string,string>".into(),
+    }
+}
+
 fn build_glue_table_input(
     glue_cfg: &GlueIcebergCatalogConfig,
     table_root_uri: &str,
     metadata_location: &str,
+    schema: &Schema,
 ) -> GlueTableInput {
+    let columns: Vec<GlueColumn> = schema
+        .as_struct()
+        .fields()
+        .iter()
+        .map(|f| {
+            GlueColumn::builder()
+                .name(f.name.as_str())
+                .r#type(iceberg_type_to_glue_type(&f.field_type))
+                .build()
+                .expect("glue column builder validated")
+        })
+        .collect();
+
     let storage_descriptor = GlueStorageDescriptor::builder()
         .location(table_root_uri)
+        .set_columns(Some(columns))
         .build();
 
     GlueTableInput::builder()

{floe_python-0.6.4 → floe_python-0.6.5}/crates/floe-core/src/io/write/iceberg.rs

@@ -572,6 +572,7 @@ async fn write_iceberg_table_async(
             glue_state
                 .as_ref()
                 .and_then(|state| state.version_id.as_deref()),
+            &prepared.iceberg_schema,
         )
         .await?;
     }

{floe_python-0.6.4 → floe_python-0.6.5}/crates/floe-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-python"
-version = "0.6.4"
+version = "0.6.5"
 edition = "2021"
 description = "Python bindings for floe-core (PyO3-based)"
 license = "Apache-2.0"
@@ -14,7 +14,7 @@ name = "_floe"
 crate-type = ["cdylib"]
 
 [dependencies]
-floe-core = { path = "../floe-core", version = "0.6.4", default-features = false }
+floe-core = { path = "../floe-core", version = "0.6.5", default-features = false }
 pyo3 = { version = "0.29", features = ["extension-module", "abi3-py310"] }
 serde_json = "1"
 

{floe_python-0.6.4 → floe_python-0.6.5}/crates/floe-python/pyproject.duckdb.toml

@@ -22,7 +22,7 @@ build-backend = "maturin"
 
 [project]
 name = "floe-duckdb"
-version = "0.6.4"
+version = "0.6.5"
 description = "DuckDB companion extension for floe — adds the native DuckDB sink to the floe Python package"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -46,7 +46,7 @@ classifiers = [
 # only contributes the compiled DuckDB extension module into that package. Pin to
 # the exact lean version so the companion's `_floe_duckdb` ABI always matches the
 # `floe` package it plugs into.
-dependencies = ["floe-python==0.6.4"]
+dependencies = ["floe-python==0.6.5"]
 
 [project.urls]
 Homepage = "https://github.com/malon64/floe"

{floe_python-0.6.4 → floe_python-0.6.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "floe-python"
-version = "0.6.4"
+version = "0.6.5"
 description = "Python bindings for floe-core — run floe pipelines at Rust speed from Python and notebooks"
 readme = "README.md"
 requires-python = ">=3.10"