floe-python 0.5.6__tar.gz → 0.6.2__tar.gz

{floe_python-0.5.6 → floe_python-0.6.2}/Cargo.lock

@@ -3384,7 +3384,7 @@ dependencies = [
 
 [[package]]
 name = "floe-cli"
-version = "0.5.6"
+version = "0.6.2"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -3397,12 +3397,13 @@ dependencies = [
 
 [[package]]
 name = "floe-core"
-version = "0.5.6"
+version = "0.6.2"
 dependencies = [
  "apache-avro 0.16.0",
  "arrow",
  "async-trait",
  "aws-config",
+ "aws-credential-types",
  "aws-sdk-glue",
  "aws-sdk-s3",
  "azure_core",
@@ -3416,6 +3417,7 @@ dependencies = [
  "gcloud-storage",
  "glob",
  "hex",
+ "hmac",
  "iceberg",
  "iceberg-catalog-rest",
  "iceberg-storage-opendal",
@@ -3423,7 +3425,7 @@ dependencies = [
  "orc-rust",
  "polars",
  "rayon",
- "reqwest 0.12.28",
+ "reqwest 0.13.4",
  "roxmltree",
  "rust_xlsxwriter",
  "serde",
@@ -3440,7 +3442,7 @@ dependencies = [
 
 [[package]]
 name = "floe-python"
-version = "0.5.6"
+version = "0.6.2"
 dependencies = [
  "floe-core",
  "pyo3",
@@ -4363,15 +4365,6 @@ dependencies = [
  "serde_core",
 ]
 
-[[package]]
-name = "indoc"
-version = "2.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
-dependencies = [
- "rustversion",
-]
-
 [[package]]
 name = "inout"
 version = "0.1.4"
@@ -4857,15 +4850,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "memoffset"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -4956,17 +4940,17 @@ checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b"
 
 [[package]]
 name = "native-tls"
-version = "0.2.14"
+version = "0.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e"
+checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
 dependencies = [
  "libc",
  "log",
  "openssl",
- "openssl-probe 0.1.6",
+ "openssl-probe",
  "openssl-sys",
  "schannel",
- "security-framework 2.11.1",
+ "security-framework",
  "security-framework-sys",
  "tempfile",
 ]
@@ -5209,15 +5193,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-probe"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
-
-[[package]]
-name = "openssl-probe"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391"
+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
 
 [[package]]
 name = "openssl-src"
@@ -6296,37 +6274,32 @@ dependencies = [
 
 [[package]]
 name = "pyo3"
-version = "0.22.6"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884"
+checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c"
 dependencies = [
- "cfg-if",
- "indoc",
  "libc",
- "memoffset",
  "once_cell",
  "portable-atomic",
  "pyo3-build-config",
  "pyo3-ffi",
  "pyo3-macros",
- "unindent",
 ]
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.22.6"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38"
+checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078"
 dependencies = [
- "once_cell",
  "target-lexicon",
 ]
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.22.6"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636"
+checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -6334,9 +6307,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.22.6"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453"
+checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -6346,13 +6319,12 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.22.6"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe"
+checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
- "pyo3-build-config",
  "quote",
  "syn 2.0.117",
 ]
@@ -6800,24 +6772,33 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
 dependencies = [
  "base64",
  "bytes",
+ "encoding_rs",
+ "futures-channel",
  "futures-core",
  "futures-util",
+ "h2 0.4.13",
  "http 1.4.0",
  "http-body 1.0.1",
  "http-body-util",
  "hyper 1.8.1",
  "hyper-rustls 0.27.7",
+ "hyper-tls",
  "hyper-util",
  "js-sys",
  "log",
+ "mime",
+ "native-tls",
  "percent-encoding",
  "pin-project-lite",
  "quinn",
  "rustls 0.23.36",
  "rustls-pki-types",
  "rustls-platform-verifier",
+ "serde",
+ "serde_json",
  "sync_wrapper",
  "tokio",
+ "tokio-native-tls",
  "tokio-rustls 0.26.4",
  "tokio-util",
  "tower",
@@ -7019,12 +7000,11 @@ dependencies = [
 
 [[package]]
 name = "rust_xlsxwriter"
-version = "0.67.0"
+version = "0.95.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05208955a4ca51dfad36960bdae24d9d257d43225711c6f70ed15c5115478384"
+checksum = "f281b687352597d29efaad39701d1167d5c48aa76fb973e392bc13e9d44e7f36"
 dependencies = [
- "regex",
- "zip 2.4.2",
+ "zip 7.2.0",
 ]
 
 [[package]]
@@ -7088,10 +7068,10 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
 dependencies = [
- "openssl-probe 0.2.0",
+ "openssl-probe",
  "rustls-pki-types",
  "schannel",
- "security-framework 3.5.1",
+ "security-framework",
 ]
 
 [[package]]
@@ -7128,7 +7108,7 @@ dependencies = [
  "rustls-native-certs",
  "rustls-platform-verifier-android",
  "rustls-webpki 0.103.13",
- "security-framework 3.5.1",
+ "security-framework",
  "security-framework-sys",
  "webpki-root-certs",
  "windows-sys 0.61.2",
@@ -7272,19 +7252,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "security-framework"
-version = "2.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
-dependencies = [
- "bitflags",
- "core-foundation 0.9.4",
- "core-foundation-sys",
- "libc",
- "security-framework-sys",
-]
-
 [[package]]
 name = "security-framework"
 version = "3.5.1"
@@ -7613,7 +7580,7 @@ version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
  "proc-macro2",
  "quote",
  "syn 2.0.117",
@@ -7894,9 +7861,9 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.16"
+version = "0.13.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tempfile"
@@ -8297,6 +8264,12 @@ dependencies = [
  "syn 2.0.117",
 ]
 
+[[package]]
+name = "typed-path"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
+
 [[package]]
 name = "typeid"
 version = "1.0.3"
@@ -8433,12 +8406,6 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
-[[package]]
-name = "unindent"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
-
 [[package]]
 name = "untrusted"
 version = "0.7.1"
@@ -9320,32 +9287,29 @@ dependencies = [
 
 [[package]]
 name = "zip"
-version = "2.4.2"
+version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50"
+checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b"
 dependencies = [
  "arbitrary",
  "crc32fast",
- "crossbeam-utils",
- "displaydoc",
  "flate2",
  "indexmap 2.13.0",
  "memchr",
- "thiserror 2.0.18",
  "zopfli",
 ]
 
 [[package]]
 name = "zip"
-version = "6.0.0"
+version = "7.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b"
+checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0"
 dependencies = [
- "arbitrary",
  "crc32fast",
  "flate2",
  "indexmap 2.13.0",
  "memchr",
+ "typed-path",
  "zopfli",
 ]
 

{floe_python-0.5.6 → floe_python-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: floe-python
-Version: 0.5.6
+Version: 0.6.2
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{floe_python-0.5.6 → floe_python-0.6.2}/crates/floe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "floe-core"
-version = "0.5.6"
+version = "0.6.2"
 edition = "2021"
 description = "Core library for Floe, a YAML-driven technical ingestion tool."
 license = "MIT"
@@ -26,6 +26,9 @@ glob = "0.3"
 roxmltree = "0.20"
 aws-config = "1"
 aws-sdk-s3 = "1"
+# `ProvideCredentials` trait — used to resolve EKS Pod Identity / container credentials for
+# the Iceberg S3 write path (#426).
+aws-credential-types = "1"
 # Only used by the Iceberg Glue catalog path; gated under the `iceberg` feature.
 aws-sdk-glue = { version = "1", optional = true }
 apache-avro = "0.16"
@@ -51,13 +54,14 @@ orc-rust = "0.7.1"
 # split the RecordBatch type and break `dataframe_to_record_batch` ingestion.
 duckdb = { version = "=1.10500.0", features = ["bundled", "appender-arrow"], optional = true }
 
-reqwest = { version = "0.12", default-features = true, features = ["native-tls-vendored", "json", "blocking"] }
+reqwest = { version = "0.13", default-features = true, features = ["native-tls-vendored", "json", "blocking"] }
 sha2 = "0.10"
+hmac = "0.12"
 hex = "0.4"
 thiserror = "1"
 
 [dev-dependencies]
-rust_xlsxwriter = "0.67"
+rust_xlsxwriter = "0.95"
 mockito = "1"
 
 [features]

floe_python-0.6.2/crates/floe-core/README.md

@@ -0,0 +1,114 @@
+![Floe logo](docs/assets/floe-banner.png)
+
+# Floe
+
+**Floe** is a Polars-powered data contract runtime for reliable file ingestion.
+It validates raw files or extracted datasets before they enter your trusted
+layer, routing accepted rows to lakehouse sinks and rejected rows to quarantine
+with audit reports.
+
+Use Floe when you already have a platform such as Databricks, Fabric-style
+lakehouses, Snowflake/Open Catalog, MotherDuck, Airflow, or Dagster, but you
+need a lightweight entry gate for file contracts, quality checks, rejected rows,
+and run evidence.
+
+Floe complements extract/load tools such as dlt, ingestr, and Airbyte: they get
+data out of source systems; Floe decides what is allowed into trusted storage.
+
+## What Floe does
+
+- Defines source, schema, checks, accepted output, rejected output, write mode,
+  metadata, and reporting in one human-readable YAML contract.
+- Reads common file exports and extracted datasets from local or cloud storage.
+- Applies schema, casting, nullability, and uniqueness checks before write.
+- Writes accepted rows to Parquet, Delta Lake, Apache Iceberg, or DuckDB.
+- Writes invalid rows separately and emits deterministic JSON run reports.
+- Runs as a CLI binary, Docker image, Python library, or orchestrated job.
+
+## How it works
+
+![Floe architecture](docs/assets/architecture.png)
+
+Each `floe run` executes a deterministic gate per entity:
+
+| Stage | What happens |
+|---|---|
+| **1. Resolve inputs** | Discover and download source files from local or cloud storage |
+| **2. File-level checks** | Validate schema structure, file format, and headers |
+| **3. Row-level checks** | Apply type casting and `not_null` checks row by row |
+| **4. Entity-level checks** | Apply `unique` / primary-key checks across all input rows plus existing accepted data |
+| **5. Write outputs** | Route valid rows to accepted sinks, invalid rows to rejected sinks, and write reports |
+
+Floe uses Rust, Polars, and Arrow for single-node columnar execution. At the
+sink boundary, Arrow RecordBatches are handed to table-format writers without an
+extra serialization hop.
+
+- **Inputs:** CSV · TSV · JSON · Parquet · ORC · Avro · XLSX · XML · Fixed-width
+- **Accepted outputs:** Parquet · Delta Lake · Apache Iceberg · DuckDB / MotherDuck
+- **Storage:** local · S3 · ADLS · GCS
+- **Catalogs:** AWS Glue · Iceberg REST (Polaris, Nessie, Snowflake) · Databricks Unity Catalog
+
+## Feature index
+
+| Capability | Start here |
+|---|---|
+| Contracts and full YAML reference | [docs/config.md](docs/config.md) |
+| Pipeline phases and execution details | [docs/how-it-works.md](docs/how-it-works.md) |
+| Checks: schema mismatch, cast, `not_null`, `unique` | [docs/checks.md](docs/checks.md) |
+| Supported inputs, outputs, storage, and catalogs | [docs/support-matrix.md](docs/support-matrix.md) |
+| Write modes: `overwrite`, `append`, `merge_scd1`, `merge_scd2` | [docs/write_modes.md](docs/write_modes.md) |
+| Parquet, Delta, Iceberg, and DuckDB sinks | [docs/sinks/parquet.md](docs/sinks/parquet.md), [docs/sinks/delta.md](docs/sinks/delta.md), [docs/sinks/iceberg.md](docs/sinks/iceberg.md), [docs/sinks/duckdb.md](docs/sinks/duckdb.md) |
+| S3, ADLS, and GCS storage | [docs/storages/s3.md](docs/storages/s3.md), [docs/storages/adls.md](docs/storages/adls.md), [docs/storages/gcs.md](docs/storages/gcs.md) |
+| Incremental file state | [docs/incremental.md](docs/incremental.md) |
+| Profiles and variables | [docs/profiles.md](docs/profiles.md), [docs/variables.md](docs/variables.md) |
+| PII masking | [docs/pii.md](docs/pii.md) |
+| Reports, logs, and OpenLineage | [docs/report.md](docs/report.md), [docs/logging.md](docs/logging.md), [docs/lineage.md](docs/lineage.md) |
+| Airflow and Dagster manifests | [docs/manifest.md](docs/manifest.md), [orchestrators/airflow-floe/README.md](orchestrators/airflow-floe/README.md), [orchestrators/dagster-floe/README.md](orchestrators/dagster-floe/README.md) |
+| Python and notebooks | [docs/python-bindings.md](docs/python-bindings.md) |
+| Installation and CLI usage | [docs/installation.md](docs/installation.md), [docs/cli.md](docs/cli.md) |
+
+## Install
+
+**macOS / Linux — [Homebrew](https://brew.sh)**
+
+```bash
+brew tap malon64/floe
+brew install floe
+```
+
+**Windows — [Scoop](https://scoop.sh)**
+
+```bash
+scoop bucket add floe https://github.com/malon64/scoop-floe
+scoop install floe
+```
+
+**Docker**
+
+```bash
+docker pull ghcr.io/malon64/floe:latest
+docker run --rm -v "$PWD:/work" ghcr.io/malon64/floe:latest run -c /work/config.yml
+```
+
+Or download a prebuilt binary from [GitHub Releases](https://github.com/malon64/floe/releases), or `cargo install floe-cli`.  
+→ [Full installation guide](docs/installation.md)
+
+**DuckDB sink** is shipped as a companion (the default artifacts are lean): use the
+`ghcr.io/malon64/floe-duckdb` image, a `floe-duckdb` binary on your `PATH`, or the
+off-PyPI `floe-duckdb` wheel. The lean `floe` auto-delegates DuckDB-sink runs to it.
+→ [DuckDB support](docs/installation.md#duckdb-support-companion-distribution)
+
+## Quick start
+
+```bash
+floe validate -c config.yml   # validate config and schema
+floe run      -c config.yml   # run the pipeline
+```
+
+→ [Config reference](docs/config.md) · [Example config](example/config.yml)
+
+For the full documentation entry point, see [docs/summary.md](docs/summary.md).
+
+## License
+
+MIT