floe-python 0.6.1__tar.gz → 0.6.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {floe_python-0.6.1 → floe_python-0.6.2}/Cargo.lock +4 -3
- {floe_python-0.6.1 → floe_python-0.6.2}/PKG-INFO +1 -1
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/Cargo.toml +4 -1
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/template.rs +20 -4
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/object_store.rs +127 -1
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/rest.rs +11 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg.rs +14 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/merge/mod.rs +0 -1
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/manifest/builder.rs +13 -7
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/manifest/reconstruct.rs +27 -12
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/templating.rs +59 -1
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/manifest/mod.rs +110 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/Cargo.toml +2 -2
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/pyproject.duckdb.toml +2 -2
- {floe_python-0.6.1 → floe_python-0.6.2}/pyproject.toml +1 -1
- {floe_python-0.6.1 → floe_python-0.6.2}/Cargo.toml +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/README.md +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/README.md +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/add_entity.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/cast.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/mismatch.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/normalize.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/not_null.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/unique.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/catalog.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/location.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/parse.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/storage.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/types.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/validate.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/yaml_decode.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/errors.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/format.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/avro.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/csv.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/fixed_width.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/json.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/json_selector.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/orc.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/parquet.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/xlsx.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/xml.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/xml_selector.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/extensions.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/paths.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/placement.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/planner.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/uri.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/validation.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/ops/archive.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/ops/inputs.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/ops/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/ops/output.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/providers/adls.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/providers/gcs.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/providers/local.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/providers/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/providers/s3.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/target.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/unique_seed/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/accepted.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/arrow_convert.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/csv.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta/commit_metrics.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta/options.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta/record_batch.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta/unity.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta/unity_tests.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/duckdb/conn.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/duckdb/merge.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/duckdb.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/context.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/data_files.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/glue.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/metadata.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/schema.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/metrics.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/parquet.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/parts.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/sink_format.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/append.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/merge/keys.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/merge/scd1.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/merge/scd2.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/merge/shared.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/overwrite.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/lib.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/lineage/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/log.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/manifest/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/manifest/model.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/profile/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/profile/parse.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/profile/types.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/profile/validate.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/report/build.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/report/entity.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/report/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/report/output.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/context.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/accepted_buffer.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/accepted_write.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/incremental.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/pii.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/precheck.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/process.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/resolve.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/validate_split.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/events.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/file.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/output.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/perf.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/runner/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/runner/outcome.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/runtime.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/state/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/vars/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/vars/resolve.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/warnings.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/archive_run.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/composite_unique.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/delta_run.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/dry_run.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/duckdb_run.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/fixed_width.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/iceberg_gcs_run.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/iceberg_glue_run.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/iceberg_run.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/iceberg_s3_run.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/json_selectors.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/local_run.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/path_normalization.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/run_entities_filter.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/common.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/add_entity.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/adls_storage.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/adls_validation.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/catalogs.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/config_validation.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/duckdb_validation.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/gcs_storage.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/gcs_validation.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/lineage_validation.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/local_storage.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/parse.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/pii_validation.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/remote_base.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/storage_resolver_uri.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/format.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/avro_input.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/csv_nulls.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/json_array.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/json_ndjson.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/json_selector.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/orc_input.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/parquet_input.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/tsv.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/xlsx_input.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/xml.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/xml_selector.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/adls.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/adls_integration.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/gcs.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/inputs.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/local.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/paths.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/planner.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/s3.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/target.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/delta_merge.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/delta_write.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/iceberg_write.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/metrics.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/object_store.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/parquet_write.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/parts.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/rejected_csv.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/profile/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/profile/parse.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/profile/validate.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/report/accepted_output.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/report/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/report/storage.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/check_order.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/checks.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/entity/accepted_output.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/entity/incremental.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/entity/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/lineage.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/normalize.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/pii.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/report.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/schema_mismatch.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/runner/adapter.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/runner/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/state/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/vars/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/vars/resolve.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/.gitignore +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/README.md +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/functions.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/lib.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/observer.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/types/config.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/types/errors.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/types/mod.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/types/outcome.rs +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/tests/fixtures/config.yml +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/tests/fixtures/in/customer/customers_valid.csv +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/tests/fixtures/invalid_config.yml +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/tests/fixtures/profile.yml +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/tests/test_floe.py +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/python/floe/__init__.py +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/python/floe/_floe.pyi +0 -0
- {floe_python-0.6.1 → floe_python-0.6.2}/python/floe/py.typed +0 -0
|
@@ -3384,7 +3384,7 @@ dependencies = [
|
|
|
3384
3384
|
|
|
3385
3385
|
[[package]]
|
|
3386
3386
|
name = "floe-cli"
|
|
3387
|
-
version = "0.6.
|
|
3387
|
+
version = "0.6.2"
|
|
3388
3388
|
dependencies = [
|
|
3389
3389
|
"assert_cmd",
|
|
3390
3390
|
"clap",
|
|
@@ -3397,12 +3397,13 @@ dependencies = [
|
|
|
3397
3397
|
|
|
3398
3398
|
[[package]]
|
|
3399
3399
|
name = "floe-core"
|
|
3400
|
-
version = "0.6.
|
|
3400
|
+
version = "0.6.2"
|
|
3401
3401
|
dependencies = [
|
|
3402
3402
|
"apache-avro 0.16.0",
|
|
3403
3403
|
"arrow",
|
|
3404
3404
|
"async-trait",
|
|
3405
3405
|
"aws-config",
|
|
3406
|
+
"aws-credential-types",
|
|
3406
3407
|
"aws-sdk-glue",
|
|
3407
3408
|
"aws-sdk-s3",
|
|
3408
3409
|
"azure_core",
|
|
@@ -3441,7 +3442,7 @@ dependencies = [
|
|
|
3441
3442
|
|
|
3442
3443
|
[[package]]
|
|
3443
3444
|
name = "floe-python"
|
|
3444
|
-
version = "0.6.
|
|
3445
|
+
version = "0.6.2"
|
|
3445
3446
|
dependencies = [
|
|
3446
3447
|
"floe-core",
|
|
3447
3448
|
"pyo3",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "floe-core"
|
|
3
|
-
version = "0.6.
|
|
3
|
+
version = "0.6.2"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
description = "Core library for Floe, a YAML-driven technical ingestion tool."
|
|
6
6
|
license = "MIT"
|
|
@@ -26,6 +26,9 @@ glob = "0.3"
|
|
|
26
26
|
roxmltree = "0.20"
|
|
27
27
|
aws-config = "1"
|
|
28
28
|
aws-sdk-s3 = "1"
|
|
29
|
+
# `ProvideCredentials` trait — used to resolve EKS Pod Identity / container credentials for
|
|
30
|
+
# the Iceberg S3 write path (#426).
|
|
31
|
+
aws-credential-types = "1"
|
|
29
32
|
# Only used by the Iceberg Glue catalog path; gated under the `iceberg` feature.
|
|
30
33
|
aws-sdk-glue = { version = "1", optional = true }
|
|
31
34
|
apache-avro = "0.16"
|
|
@@ -41,10 +41,13 @@ pub fn apply_templates_with_vars(
|
|
|
41
41
|
|
|
42
42
|
if let Some(storages) = config.storages.as_mut() {
|
|
43
43
|
for definition in storages.definitions.iter_mut() {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
let name = definition.name.clone();
|
|
45
|
+
replace_storage_definition_field(&mut definition.bucket, &vars, &name, "bucket")?;
|
|
46
|
+
replace_storage_definition_field(&mut definition.region, &vars, &name, "region")?;
|
|
47
|
+
replace_storage_definition_field(&mut definition.account, &vars, &name, "account")?;
|
|
48
|
+
replace_storage_definition_field(&mut definition.container, &vars, &name, "container")?;
|
|
49
|
+
replace_storage_definition_field(&mut definition.prefix, &vars, &name, "prefix")?;
|
|
50
|
+
replace_storage_definition_field(&mut definition.endpoint, &vars, &name, "endpoint")?;
|
|
48
51
|
}
|
|
49
52
|
}
|
|
50
53
|
|
|
@@ -93,6 +96,19 @@ pub fn apply_templates_with_vars(
|
|
|
93
96
|
Ok(())
|
|
94
97
|
}
|
|
95
98
|
|
|
99
|
+
fn replace_storage_definition_field(
|
|
100
|
+
value: &mut Option<String>,
|
|
101
|
+
vars: &HashMap<String, String>,
|
|
102
|
+
definition_name: &str,
|
|
103
|
+
field_name: &str,
|
|
104
|
+
) -> FloeResult<()> {
|
|
105
|
+
if let Some(current) = value.as_mut() {
|
|
106
|
+
let field = format!("storages.definitions.{definition_name}.{field_name}");
|
|
107
|
+
*current = replace_placeholders(current, vars, &field, None)?;
|
|
108
|
+
}
|
|
109
|
+
Ok(())
|
|
110
|
+
}
|
|
111
|
+
|
|
96
112
|
fn build_env_vars(
|
|
97
113
|
config_dir: &Path,
|
|
98
114
|
env: Option<&EnvConfig>,
|
|
@@ -3,7 +3,9 @@ use crate::errors::FloeError;
|
|
|
3
3
|
use std::collections::HashMap;
|
|
4
4
|
|
|
5
5
|
#[cfg(feature = "iceberg")]
|
|
6
|
-
use iceberg::io::{
|
|
6
|
+
use iceberg::io::{
|
|
7
|
+
CLIENT_REGION, S3_ACCESS_KEY_ID, S3_REGION, S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN,
|
|
8
|
+
};
|
|
7
9
|
#[cfg(feature = "delta")]
|
|
8
10
|
use url::Url;
|
|
9
11
|
|
|
@@ -149,3 +151,127 @@ pub fn iceberg_store_config(
|
|
|
149
151
|
.into()),
|
|
150
152
|
}
|
|
151
153
|
}
|
|
154
|
+
|
|
155
|
+
/// Resolve AWS credentials through the AWS SDK default provider chain and inject them as
|
|
156
|
+
/// static `s3.*` properties for the opendal-backed Iceberg S3 writer.
|
|
157
|
+
///
|
|
158
|
+
/// The Iceberg write path builds its S3 operator through opendal + reqsign, whose default
|
|
159
|
+
/// credential chain does **not** understand EKS Pod Identity / ECS container credentials
|
|
160
|
+
/// (`AWS_CONTAINER_CREDENTIALS_FULL_URI` + token file) and falls back to EC2 IMDS — which
|
|
161
|
+
/// fails inside EKS Pod Identity pods (#426). The AWS SDK chain (`aws-config`, the same one
|
|
162
|
+
/// the Glue catalog and S3 reads already use successfully) does resolve those credentials, so
|
|
163
|
+
/// we resolve them here and hand opendal explicit static credentials.
|
|
164
|
+
///
|
|
165
|
+
/// Best-effort and non-regressive: if no provider is configured or resolution fails, the props
|
|
166
|
+
/// are left untouched so existing static-env / IMDS setups keep working. Caller-supplied
|
|
167
|
+
/// credential props are never overwritten.
|
|
168
|
+
#[cfg(feature = "iceberg")]
|
|
169
|
+
pub async fn inject_aws_static_credentials(
|
|
170
|
+
props: &mut HashMap<String, String>,
|
|
171
|
+
region: Option<&str>,
|
|
172
|
+
) {
|
|
173
|
+
use aws_credential_types::provider::ProvideCredentials;
|
|
174
|
+
use aws_sdk_s3::config::Region as S3Region;
|
|
175
|
+
|
|
176
|
+
// Respect explicitly-configured credentials; skip the resolver entirely.
|
|
177
|
+
if props.contains_key(S3_ACCESS_KEY_ID) {
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
|
|
182
|
+
if let Some(region) = region {
|
|
183
|
+
loader = loader.region(S3Region::new(region.to_string()));
|
|
184
|
+
}
|
|
185
|
+
let sdk_config = loader.load().await;
|
|
186
|
+
|
|
187
|
+
let Some(provider) = sdk_config.credentials_provider() else {
|
|
188
|
+
return;
|
|
189
|
+
};
|
|
190
|
+
let Ok(creds) = provider.provide_credentials().await else {
|
|
191
|
+
return;
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
apply_aws_credentials_to_props(props, &creds);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/// Map resolved AWS credentials onto the `s3.*` properties consumed by the opendal-backed
|
|
198
|
+
/// Iceberg S3 writer. Existing credential props are left untouched.
|
|
199
|
+
#[cfg(feature = "iceberg")]
|
|
200
|
+
fn apply_aws_credentials_to_props(
|
|
201
|
+
props: &mut HashMap<String, String>,
|
|
202
|
+
creds: &aws_credential_types::Credentials,
|
|
203
|
+
) {
|
|
204
|
+
if props.contains_key(S3_ACCESS_KEY_ID) {
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
props.insert(
|
|
208
|
+
S3_ACCESS_KEY_ID.to_string(),
|
|
209
|
+
creds.access_key_id().to_string(),
|
|
210
|
+
);
|
|
211
|
+
props.insert(
|
|
212
|
+
S3_SECRET_ACCESS_KEY.to_string(),
|
|
213
|
+
creds.secret_access_key().to_string(),
|
|
214
|
+
);
|
|
215
|
+
if let Some(token) = creds.session_token() {
|
|
216
|
+
props.insert(S3_SESSION_TOKEN.to_string(), token.to_string());
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
#[cfg(all(test, feature = "iceberg"))]
|
|
221
|
+
mod credential_tests {
|
|
222
|
+
use super::*;
|
|
223
|
+
use aws_credential_types::Credentials;
|
|
224
|
+
|
|
225
|
+
#[test]
|
|
226
|
+
fn maps_temporary_credentials_to_s3_props() {
|
|
227
|
+
let creds = Credentials::new(
|
|
228
|
+
"AKIDEXAMPLE",
|
|
229
|
+
"SECRET",
|
|
230
|
+
Some("SESSION".into()),
|
|
231
|
+
None,
|
|
232
|
+
"test",
|
|
233
|
+
);
|
|
234
|
+
let mut props = HashMap::new();
|
|
235
|
+
apply_aws_credentials_to_props(&mut props, &creds);
|
|
236
|
+
|
|
237
|
+
assert_eq!(
|
|
238
|
+
props.get(S3_ACCESS_KEY_ID).map(String::as_str),
|
|
239
|
+
Some("AKIDEXAMPLE")
|
|
240
|
+
);
|
|
241
|
+
assert_eq!(
|
|
242
|
+
props.get(S3_SECRET_ACCESS_KEY).map(String::as_str),
|
|
243
|
+
Some("SECRET")
|
|
244
|
+
);
|
|
245
|
+
assert_eq!(
|
|
246
|
+
props.get(S3_SESSION_TOKEN).map(String::as_str),
|
|
247
|
+
Some("SESSION")
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
#[test]
|
|
252
|
+
fn omits_session_token_for_long_lived_credentials() {
|
|
253
|
+
let creds = Credentials::new("AKIDEXAMPLE", "SECRET", None, None, "test");
|
|
254
|
+
let mut props = HashMap::new();
|
|
255
|
+
apply_aws_credentials_to_props(&mut props, &creds);
|
|
256
|
+
|
|
257
|
+
assert_eq!(
|
|
258
|
+
props.get(S3_ACCESS_KEY_ID).map(String::as_str),
|
|
259
|
+
Some("AKIDEXAMPLE")
|
|
260
|
+
);
|
|
261
|
+
assert!(!props.contains_key(S3_SESSION_TOKEN));
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
#[test]
|
|
265
|
+
fn does_not_overwrite_caller_supplied_credentials() {
|
|
266
|
+
let creds = Credentials::new("RESOLVED", "RESOLVED_SECRET", None, None, "test");
|
|
267
|
+
let mut props = HashMap::new();
|
|
268
|
+
props.insert(S3_ACCESS_KEY_ID.to_string(), "USER_KEY".to_string());
|
|
269
|
+
apply_aws_credentials_to_props(&mut props, &creds);
|
|
270
|
+
|
|
271
|
+
assert_eq!(
|
|
272
|
+
props.get(S3_ACCESS_KEY_ID).map(String::as_str),
|
|
273
|
+
Some("USER_KEY")
|
|
274
|
+
);
|
|
275
|
+
assert!(!props.contains_key(S3_SECRET_ACCESS_KEY));
|
|
276
|
+
}
|
|
277
|
+
}
|
|
@@ -80,6 +80,17 @@ pub(crate) async fn build_rest_catalog(
|
|
|
80
80
|
.next()
|
|
81
81
|
.unwrap_or("s3")
|
|
82
82
|
.to_string();
|
|
83
|
+
// Resolve EKS Pod Identity / container credentials so the opendal S3 writer does
|
|
84
|
+
// not fall back to EC2 IMDS (#426).
|
|
85
|
+
let region = props
|
|
86
|
+
.get(iceberg::io::S3_REGION)
|
|
87
|
+
.or_else(|| props.get(iceberg::io::CLIENT_REGION))
|
|
88
|
+
.cloned();
|
|
89
|
+
crate::io::storage::object_store::inject_aws_static_credentials(
|
|
90
|
+
&mut props,
|
|
91
|
+
region.as_deref(),
|
|
92
|
+
)
|
|
93
|
+
.await;
|
|
83
94
|
Arc::new(OpenDalStorageFactory::S3 {
|
|
84
95
|
configured_scheme: scheme,
|
|
85
96
|
customized_credential_load: None,
|
|
@@ -389,6 +389,13 @@ async fn write_iceberg_table_async(
|
|
|
389
389
|
.next()
|
|
390
390
|
.unwrap_or("s3")
|
|
391
391
|
.to_string();
|
|
392
|
+
// Resolve EKS Pod Identity / container credentials into static s3.* props so the
|
|
393
|
+
// opendal writer does not fall back to EC2 IMDS (#426).
|
|
394
|
+
let region = catalog_props
|
|
395
|
+
.get(iceberg::io::S3_REGION)
|
|
396
|
+
.or_else(|| catalog_props.get(iceberg::io::CLIENT_REGION))
|
|
397
|
+
.cloned();
|
|
398
|
+
object_store::inject_aws_static_credentials(&mut catalog_props, region.as_deref()).await;
|
|
392
399
|
catalog_builder =
|
|
393
400
|
catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::S3 {
|
|
394
401
|
configured_scheme: scheme,
|
|
@@ -731,6 +738,13 @@ async fn collect_iceberg_batches(
|
|
|
731
738
|
.next()
|
|
732
739
|
.unwrap_or("s3")
|
|
733
740
|
.to_string();
|
|
741
|
+
// Resolve EKS Pod Identity / container credentials so the opendal S3 reader does not
|
|
742
|
+
// fall back to EC2 IMDS when seeding from existing Iceberg metadata (#426).
|
|
743
|
+
let region = props
|
|
744
|
+
.get(iceberg::io::S3_REGION)
|
|
745
|
+
.or_else(|| props.get(iceberg::io::CLIENT_REGION))
|
|
746
|
+
.cloned();
|
|
747
|
+
object_store::inject_aws_static_credentials(&mut props, region.as_deref()).await;
|
|
734
748
|
catalog_builder =
|
|
735
749
|
catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::S3 {
|
|
736
750
|
configured_scheme: scheme,
|
|
@@ -395,15 +395,21 @@ fn build_common_manifest(
|
|
|
395
395
|
now_ts_ms()
|
|
396
396
|
};
|
|
397
397
|
|
|
398
|
-
// Serialize
|
|
399
|
-
|
|
400
|
-
|
|
398
|
+
// Serialize the effective config sections (config-level definitions already merged with
|
|
399
|
+
// the profile via `apply_profile_*`, and `{{VAR}}` placeholders resolved) as opaque JSON
|
|
400
|
+
// values so the manifest is self-contained and can be replayed by `floe run --manifest`
|
|
401
|
+
// without re-supplying the config or profile.
|
|
402
|
+
let storages = config
|
|
403
|
+
.storages
|
|
404
|
+
.as_ref()
|
|
401
405
|
.and_then(|v| serde_json::to_value(v).ok());
|
|
402
|
-
let catalogs =
|
|
403
|
-
.
|
|
406
|
+
let catalogs = config
|
|
407
|
+
.catalogs
|
|
408
|
+
.as_ref()
|
|
404
409
|
.and_then(|v| serde_json::to_value(v).ok());
|
|
405
|
-
let lineage =
|
|
406
|
-
.
|
|
410
|
+
let lineage = config
|
|
411
|
+
.lineage
|
|
412
|
+
.as_ref()
|
|
407
413
|
.and_then(|v| serde_json::to_value(v).ok());
|
|
408
414
|
|
|
409
415
|
CommonManifest {
|
|
@@ -93,6 +93,23 @@ pub struct ManifestColumnDefForRun {
|
|
|
93
93
|
pub trim: Option<bool>,
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
/// Deserialize an optional embedded manifest section (storages/catalogs/lineage),
|
|
97
|
+
/// returning a clear error if the block is present but malformed rather than silently
|
|
98
|
+
/// discarding it.
|
|
99
|
+
fn deserialize_manifest_section<T: serde::de::DeserializeOwned>(
|
|
100
|
+
value: Option<&serde_json::Value>,
|
|
101
|
+
section: &str,
|
|
102
|
+
) -> FloeResult<Option<T>> {
|
|
103
|
+
match value {
|
|
104
|
+
Some(v) => serde_json::from_value::<T>(v.clone()).map(Some).map_err(
|
|
105
|
+
|err| -> Box<dyn std::error::Error + Send + Sync> {
|
|
106
|
+
FloeError::config(format!("manifest {section} block is malformed: {err}")).into()
|
|
107
|
+
},
|
|
108
|
+
),
|
|
109
|
+
None => Ok(None),
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
96
113
|
/// Parse a manifest JSON string and reconstruct a minimal RootConfig.
|
|
97
114
|
/// Returns (config, report_base_uri).
|
|
98
115
|
pub fn config_from_manifest_json(json: &str) -> FloeResult<(crate::config::RootConfig, String)> {
|
|
@@ -101,18 +118,16 @@ pub fn config_from_manifest_json(json: &str) -> FloeResult<(crate::config::RootC
|
|
|
101
118
|
FloeError::config(format!("manifest parse error: {err}")).into()
|
|
102
119
|
})?;
|
|
103
120
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
let
|
|
109
|
-
.
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
let lineage =
|
|
113
|
-
.lineage
|
|
114
|
-
.as_ref()
|
|
115
|
-
.and_then(|v| serde_json::from_value::<LineageConfig>(v.clone()).ok());
|
|
121
|
+
// Deserialize the embedded sections, surfacing malformed blocks as a clear error instead
|
|
122
|
+
// of silently dropping them (a swallowed error here previously surfaced downstream as the
|
|
123
|
+
// confusing "no storages block" failure when a hand-edited manifest had a slightly-off
|
|
124
|
+
// storages shape).
|
|
125
|
+
let storages =
|
|
126
|
+
deserialize_manifest_section::<StoragesConfig>(manifest.storages.as_ref(), "storages")?;
|
|
127
|
+
let catalogs =
|
|
128
|
+
deserialize_manifest_section::<CatalogsConfig>(manifest.catalogs.as_ref(), "catalogs")?;
|
|
129
|
+
let lineage =
|
|
130
|
+
deserialize_manifest_section::<LineageConfig>(manifest.lineage.as_ref(), "lineage")?;
|
|
116
131
|
|
|
117
132
|
let entities = manifest
|
|
118
133
|
.entities
|
|
@@ -2,7 +2,7 @@ use std::fs;
|
|
|
2
2
|
use std::path::{Path, PathBuf};
|
|
3
3
|
use std::time::{SystemTime, UNIX_EPOCH};
|
|
4
4
|
|
|
5
|
-
use floe_core::load_config;
|
|
5
|
+
use floe_core::{load_config, load_config_with_profile_vars};
|
|
6
6
|
|
|
7
7
|
fn temp_dir(prefix: &str) -> PathBuf {
|
|
8
8
|
let mut path = std::env::temp_dir();
|
|
@@ -30,6 +30,64 @@ fn write_env(dir: &Path, contents: &str) -> PathBuf {
|
|
|
30
30
|
write_file(&path, contents)
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
#[test]
|
|
34
|
+
fn profile_vars_apply_to_storage_definition_fields() {
|
|
35
|
+
let root = temp_dir("floe-storage-vars");
|
|
36
|
+
let config_yaml = format!(
|
|
37
|
+
r#"version: "0.1"
|
|
38
|
+
storages:
|
|
39
|
+
default: "lakehouse_bronze"
|
|
40
|
+
definitions:
|
|
41
|
+
- name: "lakehouse_bronze"
|
|
42
|
+
type: "s3"
|
|
43
|
+
bucket: "{{{{BRONZE_BUCKET}}}}"
|
|
44
|
+
region: "{{{{STORAGE_REGION}}}}"
|
|
45
|
+
prefix: "{{{{STORAGE_PREFIX}}}}"
|
|
46
|
+
endpoint: "https://{{{{STORAGE_ENDPOINT}}}}"
|
|
47
|
+
entities:
|
|
48
|
+
- name: "orders"
|
|
49
|
+
source:
|
|
50
|
+
format: "csv"
|
|
51
|
+
path: "{root}/in/orders.csv"
|
|
52
|
+
sink:
|
|
53
|
+
accepted:
|
|
54
|
+
format: "parquet"
|
|
55
|
+
storage: "lakehouse_bronze"
|
|
56
|
+
path: "sales/orders"
|
|
57
|
+
policy:
|
|
58
|
+
severity: "warn"
|
|
59
|
+
schema:
|
|
60
|
+
columns:
|
|
61
|
+
- name: "id"
|
|
62
|
+
type: "string"
|
|
63
|
+
"#,
|
|
64
|
+
root = root.display(),
|
|
65
|
+
);
|
|
66
|
+
let config_path = write_config(&root, &config_yaml);
|
|
67
|
+
let profile_vars = [
|
|
68
|
+
("BRONZE_BUCKET".to_string(), "lakehouse-bronze".to_string()),
|
|
69
|
+
("STORAGE_REGION".to_string(), "us-east-1".to_string()),
|
|
70
|
+
("STORAGE_PREFIX".to_string(), "bronze/root".to_string()),
|
|
71
|
+
(
|
|
72
|
+
"STORAGE_ENDPOINT".to_string(),
|
|
73
|
+
"s3.local.example".to_string(),
|
|
74
|
+
),
|
|
75
|
+
]
|
|
76
|
+
.into_iter()
|
|
77
|
+
.collect();
|
|
78
|
+
|
|
79
|
+
let parsed = load_config_with_profile_vars(&config_path, &profile_vars).expect("parse config");
|
|
80
|
+
let definition = &parsed.storages.as_ref().expect("storages").definitions[0];
|
|
81
|
+
|
|
82
|
+
assert_eq!(definition.bucket.as_deref(), Some("lakehouse-bronze"));
|
|
83
|
+
assert_eq!(definition.region.as_deref(), Some("us-east-1"));
|
|
84
|
+
assert_eq!(definition.prefix.as_deref(), Some("bronze/root"));
|
|
85
|
+
assert_eq!(
|
|
86
|
+
definition.endpoint.as_deref(),
|
|
87
|
+
Some("https://s3.local.example")
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
|
|
33
91
|
#[test]
|
|
34
92
|
fn env_file_vars_apply_to_source_path() {
|
|
35
93
|
let root = temp_dir("floe-env-file");
|
|
@@ -1076,6 +1076,116 @@ entities:
|
|
|
1076
1076
|
assert_eq!(accepted.storage.as_deref(), Some("motherduck"));
|
|
1077
1077
|
}
|
|
1078
1078
|
|
|
1079
|
+
#[test]
|
|
1080
|
+
fn manifest_embeds_config_level_storages_for_replay() {
|
|
1081
|
+
// Storages defined at config level (not in a profile) must be embedded in the generated
|
|
1082
|
+
// manifest so `floe run --manifest` is self-contained and can resolve named storages.
|
|
1083
|
+
// Regression for #425 ("no storages block").
|
|
1084
|
+
let temp_dir = tempfile::TempDir::new().expect("temp dir");
|
|
1085
|
+
let cfg_dir = temp_dir.path().join("cfg");
|
|
1086
|
+
std::fs::create_dir_all(&cfg_dir).expect("cfg dir");
|
|
1087
|
+
let config_path = cfg_dir.join("config.yml");
|
|
1088
|
+
|
|
1089
|
+
let yaml = r#"version: "0.1"
|
|
1090
|
+
storages:
|
|
1091
|
+
default: "local"
|
|
1092
|
+
definitions:
|
|
1093
|
+
- name: "local"
|
|
1094
|
+
type: "local"
|
|
1095
|
+
- name: "lakehouse_bronze"
|
|
1096
|
+
type: "s3"
|
|
1097
|
+
bucket: "example-bronze"
|
|
1098
|
+
region: "eu-west-1"
|
|
1099
|
+
entities:
|
|
1100
|
+
- name: "accounts"
|
|
1101
|
+
source:
|
|
1102
|
+
format: "csv"
|
|
1103
|
+
storage: "local"
|
|
1104
|
+
path: "./in/accounts.csv"
|
|
1105
|
+
sink:
|
|
1106
|
+
accepted:
|
|
1107
|
+
format: "parquet"
|
|
1108
|
+
storage: "lakehouse_bronze"
|
|
1109
|
+
path: "sales/accounts"
|
|
1110
|
+
policy:
|
|
1111
|
+
severity: "warn"
|
|
1112
|
+
schema:
|
|
1113
|
+
columns:
|
|
1114
|
+
- name: "id"
|
|
1115
|
+
type: "string"
|
|
1116
|
+
"#;
|
|
1117
|
+
std::fs::write(&config_path, yaml).expect("write config");
|
|
1118
|
+
|
|
1119
|
+
let config_location = resolve_config_location(config_path.to_str().expect("utf8"))
|
|
1120
|
+
.expect("resolve config location");
|
|
1121
|
+
let config = load_config(&config_location.path).expect("load config");
|
|
1122
|
+
let opts = ManifestOptions {
|
|
1123
|
+
path_mode: PathMode::ResolvedUri,
|
|
1124
|
+
..ManifestOptions::default()
|
|
1125
|
+
};
|
|
1126
|
+
|
|
1127
|
+
let payload =
|
|
1128
|
+
build_common_manifest_json(&config_location, &config, &[], None, &opts).expect("manifest");
|
|
1129
|
+
let value: Value = serde_json::from_str(&payload).expect("valid json");
|
|
1130
|
+
|
|
1131
|
+
// The manifest must carry the config-level storages even without a profile.
|
|
1132
|
+
let definitions = value["storages"]["definitions"]
|
|
1133
|
+
.as_array()
|
|
1134
|
+
.expect("storages.definitions should be present in the manifest");
|
|
1135
|
+
assert!(
|
|
1136
|
+
definitions
|
|
1137
|
+
.iter()
|
|
1138
|
+
.any(|d| d["name"] == "lakehouse_bronze" && d["bucket"] == "example-bronze"),
|
|
1139
|
+
"config-level storage definition must be embedded in the manifest: {value:#}"
|
|
1140
|
+
);
|
|
1141
|
+
|
|
1142
|
+
// ...and survive reconstruction so manifest replay can resolve the named storage.
|
|
1143
|
+
let (reconstructed, _base) =
|
|
1144
|
+
config_from_manifest_json(&payload).expect("reconstruct config from manifest");
|
|
1145
|
+
let storages = reconstructed
|
|
1146
|
+
.storages
|
|
1147
|
+
.expect("storages should be populated after manifest round-trip");
|
|
1148
|
+
let bronze = storages
|
|
1149
|
+
.definitions
|
|
1150
|
+
.iter()
|
|
1151
|
+
.find(|d| d.name == "lakehouse_bronze")
|
|
1152
|
+
.expect("lakehouse_bronze definition should survive round-trip");
|
|
1153
|
+
assert_eq!(bronze.bucket.as_deref(), Some("example-bronze"));
|
|
1154
|
+
assert_eq!(bronze.region.as_deref(), Some("eu-west-1"));
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
#[test]
|
|
1158
|
+
fn manifest_reconstruct_rejects_malformed_storages_block() {
|
|
1159
|
+
// A present-but-malformed storages block must surface a clear error rather than being
|
|
1160
|
+
// silently dropped (which previously resurfaced downstream as the confusing
|
|
1161
|
+
// "no storages block" failure). Regression for #425.
|
|
1162
|
+
let config_path = repo_root().join("example/config.yml");
|
|
1163
|
+
let config_location = resolve_config_location(config_path.to_str().expect("utf8"))
|
|
1164
|
+
.expect("resolve config location");
|
|
1165
|
+
let config = load_config(&config_location.path).expect("load config");
|
|
1166
|
+
let payload = build_common_manifest_json(
|
|
1167
|
+
&config_location,
|
|
1168
|
+
&config,
|
|
1169
|
+
&[],
|
|
1170
|
+
None,
|
|
1171
|
+
&ManifestOptions::default(),
|
|
1172
|
+
)
|
|
1173
|
+
.expect("manifest");
|
|
1174
|
+
|
|
1175
|
+
let mut value: Value = serde_json::from_str(&payload).expect("valid json");
|
|
1176
|
+
// A bare string is not a valid storages object.
|
|
1177
|
+
value["storages"] = Value::String("totally-not-a-storages-block".into());
|
|
1178
|
+
let corrupted = serde_json::to_string(&value).expect("reserialize");
|
|
1179
|
+
|
|
1180
|
+
let err =
|
|
1181
|
+
config_from_manifest_json(&corrupted).expect_err("malformed storages block must error");
|
|
1182
|
+
let msg = err.to_string();
|
|
1183
|
+
assert!(
|
|
1184
|
+
msg.contains("storages") && msg.contains("malformed"),
|
|
1185
|
+
"expected a clear 'malformed storages' error, got: {msg}"
|
|
1186
|
+
);
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1079
1189
|
#[test]
|
|
1080
1190
|
fn manifest_path_mode_resolved_uri_sets_path_from_uri() {
|
|
1081
1191
|
let temp_dir = tempfile::TempDir::new().expect("temp dir");
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "floe-python"
|
|
3
|
-
version = "0.6.
|
|
3
|
+
version = "0.6.2"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
description = "Python bindings for floe-core (PyO3-based)"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -14,7 +14,7 @@ name = "_floe"
|
|
|
14
14
|
crate-type = ["cdylib"]
|
|
15
15
|
|
|
16
16
|
[dependencies]
|
|
17
|
-
floe-core = { path = "../floe-core", version = "0.6.
|
|
17
|
+
floe-core = { path = "../floe-core", version = "0.6.2", default-features = false }
|
|
18
18
|
pyo3 = { version = "0.29", features = ["extension-module", "abi3-py310"] }
|
|
19
19
|
serde_json = "1"
|
|
20
20
|
|
|
@@ -22,7 +22,7 @@ build-backend = "maturin"
|
|
|
22
22
|
|
|
23
23
|
[project]
|
|
24
24
|
name = "floe-duckdb"
|
|
25
|
-
version = "0.6.
|
|
25
|
+
version = "0.6.2"
|
|
26
26
|
description = "DuckDB companion extension for floe — adds the native DuckDB sink to the floe Python package"
|
|
27
27
|
readme = "README.md"
|
|
28
28
|
requires-python = ">=3.10"
|
|
@@ -46,7 +46,7 @@ classifiers = [
|
|
|
46
46
|
# only contributes the compiled DuckDB extension module into that package. Pin to
|
|
47
47
|
# the exact lean version so the companion's `_floe_duckdb` ABI always matches the
|
|
48
48
|
# `floe` package it plugs into.
|
|
49
|
-
dependencies = ["floe-python==0.6.
|
|
49
|
+
dependencies = ["floe-python==0.6.2"]
|
|
50
50
|
|
|
51
51
|
[project.urls]
|
|
52
52
|
Homepage = "https://github.com/malon64/floe"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|