floe-python 0.6.1__tar.gz → 0.6.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. {floe_python-0.6.1 → floe_python-0.6.2}/Cargo.lock +4 -3
  2. {floe_python-0.6.1 → floe_python-0.6.2}/PKG-INFO +1 -1
  3. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/Cargo.toml +4 -1
  4. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/template.rs +20 -4
  5. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/object_store.rs +127 -1
  6. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/rest.rs +11 -0
  7. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg.rs +14 -0
  8. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/merge/mod.rs +0 -1
  9. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/manifest/builder.rs +13 -7
  10. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/manifest/reconstruct.rs +27 -12
  11. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/templating.rs +59 -1
  12. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/manifest/mod.rs +110 -0
  13. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/Cargo.toml +2 -2
  14. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/pyproject.duckdb.toml +2 -2
  15. {floe_python-0.6.1 → floe_python-0.6.2}/pyproject.toml +1 -1
  16. {floe_python-0.6.1 → floe_python-0.6.2}/Cargo.toml +0 -0
  17. {floe_python-0.6.1 → floe_python-0.6.2}/README.md +0 -0
  18. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/README.md +0 -0
  19. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/add_entity.rs +0 -0
  20. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/cast.rs +0 -0
  21. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/mismatch.rs +0 -0
  22. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/mod.rs +0 -0
  23. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/normalize.rs +0 -0
  24. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/not_null.rs +0 -0
  25. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/checks/unique.rs +0 -0
  26. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/catalog.rs +0 -0
  27. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/location.rs +0 -0
  28. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/mod.rs +0 -0
  29. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/parse.rs +0 -0
  30. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/storage.rs +0 -0
  31. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/types.rs +0 -0
  32. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/validate.rs +0 -0
  33. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/config/yaml_decode.rs +0 -0
  34. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/errors.rs +0 -0
  35. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/format.rs +0 -0
  36. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/mod.rs +0 -0
  37. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/avro.rs +0 -0
  38. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/csv.rs +0 -0
  39. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/fixed_width.rs +0 -0
  40. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/json.rs +0 -0
  41. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/json_selector.rs +0 -0
  42. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/mod.rs +0 -0
  43. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/orc.rs +0 -0
  44. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/parquet.rs +0 -0
  45. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/xlsx.rs +0 -0
  46. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/xml.rs +0 -0
  47. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/read/xml_selector.rs +0 -0
  48. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/extensions.rs +0 -0
  49. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/mod.rs +0 -0
  50. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/paths.rs +0 -0
  51. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/placement.rs +0 -0
  52. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/planner.rs +0 -0
  53. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/uri.rs +0 -0
  54. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/core/validation.rs +0 -0
  55. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/mod.rs +0 -0
  56. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/ops/archive.rs +0 -0
  57. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/ops/inputs.rs +0 -0
  58. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/ops/mod.rs +0 -0
  59. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/ops/output.rs +0 -0
  60. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/providers/adls.rs +0 -0
  61. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/providers/gcs.rs +0 -0
  62. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/providers/local.rs +0 -0
  63. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/providers/mod.rs +0 -0
  64. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/providers/s3.rs +0 -0
  65. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/storage/target.rs +0 -0
  66. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/unique_seed/mod.rs +0 -0
  67. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/accepted.rs +0 -0
  68. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/arrow_convert.rs +0 -0
  69. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/csv.rs +0 -0
  70. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta/commit_metrics.rs +0 -0
  71. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta/options.rs +0 -0
  72. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta/record_batch.rs +0 -0
  73. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta/unity.rs +0 -0
  74. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta/unity_tests.rs +0 -0
  75. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/delta.rs +0 -0
  76. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/duckdb/conn.rs +0 -0
  77. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/duckdb/merge.rs +0 -0
  78. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/duckdb.rs +0 -0
  79. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/context.rs +0 -0
  80. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/data_files.rs +0 -0
  81. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/glue.rs +0 -0
  82. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/metadata.rs +0 -0
  83. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/iceberg/schema.rs +0 -0
  84. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/metrics.rs +0 -0
  85. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/mod.rs +0 -0
  86. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/parquet.rs +0 -0
  87. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/parts.rs +0 -0
  88. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/sink_format.rs +0 -0
  89. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/append.rs +0 -0
  90. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/merge/keys.rs +0 -0
  91. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/merge/scd1.rs +0 -0
  92. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/merge/scd2.rs +0 -0
  93. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/merge/shared.rs +0 -0
  94. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/mod.rs +0 -0
  95. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/io/write/strategy/overwrite.rs +0 -0
  96. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/lib.rs +0 -0
  97. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/lineage/mod.rs +0 -0
  98. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/log.rs +0 -0
  99. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/manifest/mod.rs +0 -0
  100. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/manifest/model.rs +0 -0
  101. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/profile/mod.rs +0 -0
  102. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/profile/parse.rs +0 -0
  103. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/profile/types.rs +0 -0
  104. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/profile/validate.rs +0 -0
  105. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/report/build.rs +0 -0
  106. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/report/entity.rs +0 -0
  107. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/report/mod.rs +0 -0
  108. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/report/output.rs +0 -0
  109. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/context.rs +0 -0
  110. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/accepted_buffer.rs +0 -0
  111. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/accepted_write.rs +0 -0
  112. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/incremental.rs +0 -0
  113. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/mod.rs +0 -0
  114. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/pii.rs +0 -0
  115. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/precheck.rs +0 -0
  116. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/process.rs +0 -0
  117. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/resolve.rs +0 -0
  118. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/entity/validate_split.rs +0 -0
  119. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/events.rs +0 -0
  120. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/file.rs +0 -0
  121. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/mod.rs +0 -0
  122. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/output.rs +0 -0
  123. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/run/perf.rs +0 -0
  124. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/runner/mod.rs +0 -0
  125. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/runner/outcome.rs +0 -0
  126. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/runtime.rs +0 -0
  127. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/state/mod.rs +0 -0
  128. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/vars/mod.rs +0 -0
  129. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/vars/resolve.rs +0 -0
  130. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/src/warnings.rs +0 -0
  131. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/archive_run.rs +0 -0
  132. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/composite_unique.rs +0 -0
  133. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/delta_run.rs +0 -0
  134. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/dry_run.rs +0 -0
  135. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/duckdb_run.rs +0 -0
  136. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/fixed_width.rs +0 -0
  137. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/iceberg_gcs_run.rs +0 -0
  138. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/iceberg_glue_run.rs +0 -0
  139. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/iceberg_run.rs +0 -0
  140. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/iceberg_s3_run.rs +0 -0
  141. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/json_selectors.rs +0 -0
  142. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/local_run.rs +0 -0
  143. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/mod.rs +0 -0
  144. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/path_normalization.rs +0 -0
  145. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration/run_entities_filter.rs +0 -0
  146. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/integration.rs +0 -0
  147. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/common.rs +0 -0
  148. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/add_entity.rs +0 -0
  149. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/adls_storage.rs +0 -0
  150. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/adls_validation.rs +0 -0
  151. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/catalogs.rs +0 -0
  152. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/config_validation.rs +0 -0
  153. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/duckdb_validation.rs +0 -0
  154. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/gcs_storage.rs +0 -0
  155. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/gcs_validation.rs +0 -0
  156. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/lineage_validation.rs +0 -0
  157. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/local_storage.rs +0 -0
  158. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/mod.rs +0 -0
  159. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/parse.rs +0 -0
  160. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/pii_validation.rs +0 -0
  161. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/remote_base.rs +0 -0
  162. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/config/storage_resolver_uri.rs +0 -0
  163. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/format.rs +0 -0
  164. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/mod.rs +0 -0
  165. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/avro_input.rs +0 -0
  166. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/csv_nulls.rs +0 -0
  167. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/json_array.rs +0 -0
  168. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/json_ndjson.rs +0 -0
  169. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/json_selector.rs +0 -0
  170. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/mod.rs +0 -0
  171. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/orc_input.rs +0 -0
  172. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/parquet_input.rs +0 -0
  173. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/tsv.rs +0 -0
  174. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/xlsx_input.rs +0 -0
  175. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/xml.rs +0 -0
  176. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/read/xml_selector.rs +0 -0
  177. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/adls.rs +0 -0
  178. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/adls_integration.rs +0 -0
  179. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/gcs.rs +0 -0
  180. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/inputs.rs +0 -0
  181. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/local.rs +0 -0
  182. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/mod.rs +0 -0
  183. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/paths.rs +0 -0
  184. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/planner.rs +0 -0
  185. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/s3.rs +0 -0
  186. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/storage/target.rs +0 -0
  187. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/delta_merge.rs +0 -0
  188. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/delta_write.rs +0 -0
  189. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/iceberg_write.rs +0 -0
  190. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/metrics.rs +0 -0
  191. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/mod.rs +0 -0
  192. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/object_store.rs +0 -0
  193. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/parquet_write.rs +0 -0
  194. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/parts.rs +0 -0
  195. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/io/write/rejected_csv.rs +0 -0
  196. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/mod.rs +0 -0
  197. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/profile/mod.rs +0 -0
  198. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/profile/parse.rs +0 -0
  199. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/profile/validate.rs +0 -0
  200. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/report/accepted_output.rs +0 -0
  201. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/report/mod.rs +0 -0
  202. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/report/storage.rs +0 -0
  203. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/check_order.rs +0 -0
  204. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/checks.rs +0 -0
  205. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/entity/accepted_output.rs +0 -0
  206. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/entity/incremental.rs +0 -0
  207. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/entity/mod.rs +0 -0
  208. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/lineage.rs +0 -0
  209. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/mod.rs +0 -0
  210. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/normalize.rs +0 -0
  211. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/pii.rs +0 -0
  212. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/report.rs +0 -0
  213. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/run/schema_mismatch.rs +0 -0
  214. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/runner/adapter.rs +0 -0
  215. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/runner/mod.rs +0 -0
  216. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/state/mod.rs +0 -0
  217. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/vars/mod.rs +0 -0
  218. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit/vars/resolve.rs +0 -0
  219. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-core/tests/unit.rs +0 -0
  220. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/.gitignore +0 -0
  221. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/README.md +0 -0
  222. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/functions.rs +0 -0
  223. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/lib.rs +0 -0
  224. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/observer.rs +0 -0
  225. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/types/config.rs +0 -0
  226. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/types/errors.rs +0 -0
  227. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/types/mod.rs +0 -0
  228. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/src/types/outcome.rs +0 -0
  229. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/tests/fixtures/config.yml +0 -0
  230. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/tests/fixtures/in/customer/customers_valid.csv +0 -0
  231. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/tests/fixtures/invalid_config.yml +0 -0
  232. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/tests/fixtures/profile.yml +0 -0
  233. {floe_python-0.6.1 → floe_python-0.6.2}/crates/floe-python/tests/test_floe.py +0 -0
  234. {floe_python-0.6.1 → floe_python-0.6.2}/python/floe/__init__.py +0 -0
  235. {floe_python-0.6.1 → floe_python-0.6.2}/python/floe/_floe.pyi +0 -0
  236. {floe_python-0.6.1 → floe_python-0.6.2}/python/floe/py.typed +0 -0
@@ -3384,7 +3384,7 @@ dependencies = [
3384
3384
 
3385
3385
  [[package]]
3386
3386
  name = "floe-cli"
3387
- version = "0.6.1"
3387
+ version = "0.6.2"
3388
3388
  dependencies = [
3389
3389
  "assert_cmd",
3390
3390
  "clap",
@@ -3397,12 +3397,13 @@ dependencies = [
3397
3397
 
3398
3398
  [[package]]
3399
3399
  name = "floe-core"
3400
- version = "0.6.1"
3400
+ version = "0.6.2"
3401
3401
  dependencies = [
3402
3402
  "apache-avro 0.16.0",
3403
3403
  "arrow",
3404
3404
  "async-trait",
3405
3405
  "aws-config",
3406
+ "aws-credential-types",
3406
3407
  "aws-sdk-glue",
3407
3408
  "aws-sdk-s3",
3408
3409
  "azure_core",
@@ -3441,7 +3442,7 @@ dependencies = [
3441
3442
 
3442
3443
  [[package]]
3443
3444
  name = "floe-python"
3444
- version = "0.6.1"
3445
+ version = "0.6.2"
3445
3446
  dependencies = [
3446
3447
  "floe-core",
3447
3448
  "pyo3",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: floe-python
3
- Version: 0.6.1
3
+ Version: 0.6.2
4
4
  Classifier: Development Status :: 4 - Beta
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: Intended Audience :: Science/Research
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "floe-core"
3
- version = "0.6.1"
3
+ version = "0.6.2"
4
4
  edition = "2021"
5
5
  description = "Core library for Floe, a YAML-driven technical ingestion tool."
6
6
  license = "MIT"
@@ -26,6 +26,9 @@ glob = "0.3"
26
26
  roxmltree = "0.20"
27
27
  aws-config = "1"
28
28
  aws-sdk-s3 = "1"
29
+ # `ProvideCredentials` trait — used to resolve EKS Pod Identity / container credentials for
30
+ # the Iceberg S3 write path (#426).
31
+ aws-credential-types = "1"
29
32
  # Only used by the Iceberg Glue catalog path; gated under the `iceberg` feature.
30
33
  aws-sdk-glue = { version = "1", optional = true }
31
34
  apache-avro = "0.16"
@@ -41,10 +41,13 @@ pub fn apply_templates_with_vars(
41
41
 
42
42
  if let Some(storages) = config.storages.as_mut() {
43
43
  for definition in storages.definitions.iter_mut() {
44
- if let Some(prefix) = definition.prefix.as_mut() {
45
- let field = format!("storages.definitions.{}.prefix", definition.name);
46
- *prefix = replace_placeholders(prefix, &vars, &field, None)?;
47
- }
44
+ let name = definition.name.clone();
45
+ replace_storage_definition_field(&mut definition.bucket, &vars, &name, "bucket")?;
46
+ replace_storage_definition_field(&mut definition.region, &vars, &name, "region")?;
47
+ replace_storage_definition_field(&mut definition.account, &vars, &name, "account")?;
48
+ replace_storage_definition_field(&mut definition.container, &vars, &name, "container")?;
49
+ replace_storage_definition_field(&mut definition.prefix, &vars, &name, "prefix")?;
50
+ replace_storage_definition_field(&mut definition.endpoint, &vars, &name, "endpoint")?;
48
51
  }
49
52
  }
50
53
 
@@ -93,6 +96,19 @@ pub fn apply_templates_with_vars(
93
96
  Ok(())
94
97
  }
95
98
 
99
+ fn replace_storage_definition_field(
100
+ value: &mut Option<String>,
101
+ vars: &HashMap<String, String>,
102
+ definition_name: &str,
103
+ field_name: &str,
104
+ ) -> FloeResult<()> {
105
+ if let Some(current) = value.as_mut() {
106
+ let field = format!("storages.definitions.{definition_name}.{field_name}");
107
+ *current = replace_placeholders(current, vars, &field, None)?;
108
+ }
109
+ Ok(())
110
+ }
111
+
96
112
  fn build_env_vars(
97
113
  config_dir: &Path,
98
114
  env: Option<&EnvConfig>,
@@ -3,7 +3,9 @@ use crate::errors::FloeError;
3
3
  use std::collections::HashMap;
4
4
 
5
5
  #[cfg(feature = "iceberg")]
6
- use iceberg::io::{CLIENT_REGION, S3_REGION};
6
+ use iceberg::io::{
7
+ CLIENT_REGION, S3_ACCESS_KEY_ID, S3_REGION, S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN,
8
+ };
7
9
  #[cfg(feature = "delta")]
8
10
  use url::Url;
9
11
 
@@ -149,3 +151,127 @@ pub fn iceberg_store_config(
149
151
  .into()),
150
152
  }
151
153
  }
154
+
155
+ /// Resolve AWS credentials through the AWS SDK default provider chain and inject them as
156
+ /// static `s3.*` properties for the opendal-backed Iceberg S3 writer.
157
+ ///
158
+ /// The Iceberg write path builds its S3 operator through opendal + reqsign, whose default
159
+ /// credential chain does **not** understand EKS Pod Identity / ECS container credentials
160
+ /// (`AWS_CONTAINER_CREDENTIALS_FULL_URI` + token file) and falls back to EC2 IMDS — which
161
+ /// fails inside EKS Pod Identity pods (#426). The AWS SDK chain (`aws-config`, the same one
162
+ /// the Glue catalog and S3 reads already use successfully) does resolve those credentials, so
163
+ /// we resolve them here and hand opendal explicit static credentials.
164
+ ///
165
+ /// Best-effort and non-regressive: if no provider is configured or resolution fails, the props
166
+ /// are left untouched so existing static-env / IMDS setups keep working. Caller-supplied
167
+ /// credential props are never overwritten.
168
+ #[cfg(feature = "iceberg")]
169
+ pub async fn inject_aws_static_credentials(
170
+ props: &mut HashMap<String, String>,
171
+ region: Option<&str>,
172
+ ) {
173
+ use aws_credential_types::provider::ProvideCredentials;
174
+ use aws_sdk_s3::config::Region as S3Region;
175
+
176
+ // Respect explicitly-configured credentials; skip the resolver entirely.
177
+ if props.contains_key(S3_ACCESS_KEY_ID) {
178
+ return;
179
+ }
180
+
181
+ let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
182
+ if let Some(region) = region {
183
+ loader = loader.region(S3Region::new(region.to_string()));
184
+ }
185
+ let sdk_config = loader.load().await;
186
+
187
+ let Some(provider) = sdk_config.credentials_provider() else {
188
+ return;
189
+ };
190
+ let Ok(creds) = provider.provide_credentials().await else {
191
+ return;
192
+ };
193
+
194
+ apply_aws_credentials_to_props(props, &creds);
195
+ }
196
+
197
+ /// Map resolved AWS credentials onto the `s3.*` properties consumed by the opendal-backed
198
+ /// Iceberg S3 writer. Existing credential props are left untouched.
199
+ #[cfg(feature = "iceberg")]
200
+ fn apply_aws_credentials_to_props(
201
+ props: &mut HashMap<String, String>,
202
+ creds: &aws_credential_types::Credentials,
203
+ ) {
204
+ if props.contains_key(S3_ACCESS_KEY_ID) {
205
+ return;
206
+ }
207
+ props.insert(
208
+ S3_ACCESS_KEY_ID.to_string(),
209
+ creds.access_key_id().to_string(),
210
+ );
211
+ props.insert(
212
+ S3_SECRET_ACCESS_KEY.to_string(),
213
+ creds.secret_access_key().to_string(),
214
+ );
215
+ if let Some(token) = creds.session_token() {
216
+ props.insert(S3_SESSION_TOKEN.to_string(), token.to_string());
217
+ }
218
+ }
219
+
220
+ #[cfg(all(test, feature = "iceberg"))]
221
+ mod credential_tests {
222
+ use super::*;
223
+ use aws_credential_types::Credentials;
224
+
225
+ #[test]
226
+ fn maps_temporary_credentials_to_s3_props() {
227
+ let creds = Credentials::new(
228
+ "AKIDEXAMPLE",
229
+ "SECRET",
230
+ Some("SESSION".into()),
231
+ None,
232
+ "test",
233
+ );
234
+ let mut props = HashMap::new();
235
+ apply_aws_credentials_to_props(&mut props, &creds);
236
+
237
+ assert_eq!(
238
+ props.get(S3_ACCESS_KEY_ID).map(String::as_str),
239
+ Some("AKIDEXAMPLE")
240
+ );
241
+ assert_eq!(
242
+ props.get(S3_SECRET_ACCESS_KEY).map(String::as_str),
243
+ Some("SECRET")
244
+ );
245
+ assert_eq!(
246
+ props.get(S3_SESSION_TOKEN).map(String::as_str),
247
+ Some("SESSION")
248
+ );
249
+ }
250
+
251
+ #[test]
252
+ fn omits_session_token_for_long_lived_credentials() {
253
+ let creds = Credentials::new("AKIDEXAMPLE", "SECRET", None, None, "test");
254
+ let mut props = HashMap::new();
255
+ apply_aws_credentials_to_props(&mut props, &creds);
256
+
257
+ assert_eq!(
258
+ props.get(S3_ACCESS_KEY_ID).map(String::as_str),
259
+ Some("AKIDEXAMPLE")
260
+ );
261
+ assert!(!props.contains_key(S3_SESSION_TOKEN));
262
+ }
263
+
264
+ #[test]
265
+ fn does_not_overwrite_caller_supplied_credentials() {
266
+ let creds = Credentials::new("RESOLVED", "RESOLVED_SECRET", None, None, "test");
267
+ let mut props = HashMap::new();
268
+ props.insert(S3_ACCESS_KEY_ID.to_string(), "USER_KEY".to_string());
269
+ apply_aws_credentials_to_props(&mut props, &creds);
270
+
271
+ assert_eq!(
272
+ props.get(S3_ACCESS_KEY_ID).map(String::as_str),
273
+ Some("USER_KEY")
274
+ );
275
+ assert!(!props.contains_key(S3_SECRET_ACCESS_KEY));
276
+ }
277
+ }
@@ -80,6 +80,17 @@ pub(crate) async fn build_rest_catalog(
80
80
  .next()
81
81
  .unwrap_or("s3")
82
82
  .to_string();
83
+ // Resolve EKS Pod Identity / container credentials so the opendal S3 writer does
84
+ // not fall back to EC2 IMDS (#426).
85
+ let region = props
86
+ .get(iceberg::io::S3_REGION)
87
+ .or_else(|| props.get(iceberg::io::CLIENT_REGION))
88
+ .cloned();
89
+ crate::io::storage::object_store::inject_aws_static_credentials(
90
+ &mut props,
91
+ region.as_deref(),
92
+ )
93
+ .await;
83
94
  Arc::new(OpenDalStorageFactory::S3 {
84
95
  configured_scheme: scheme,
85
96
  customized_credential_load: None,
@@ -389,6 +389,13 @@ async fn write_iceberg_table_async(
389
389
  .next()
390
390
  .unwrap_or("s3")
391
391
  .to_string();
392
+ // Resolve EKS Pod Identity / container credentials into static s3.* props so the
393
+ // opendal writer does not fall back to EC2 IMDS (#426).
394
+ let region = catalog_props
395
+ .get(iceberg::io::S3_REGION)
396
+ .or_else(|| catalog_props.get(iceberg::io::CLIENT_REGION))
397
+ .cloned();
398
+ object_store::inject_aws_static_credentials(&mut catalog_props, region.as_deref()).await;
392
399
  catalog_builder =
393
400
  catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::S3 {
394
401
  configured_scheme: scheme,
@@ -731,6 +738,13 @@ async fn collect_iceberg_batches(
731
738
  .next()
732
739
  .unwrap_or("s3")
733
740
  .to_string();
741
+ // Resolve EKS Pod Identity / container credentials so the opendal S3 reader does not
742
+ // fall back to EC2 IMDS when seeding from existing Iceberg metadata (#426).
743
+ let region = props
744
+ .get(iceberg::io::S3_REGION)
745
+ .or_else(|| props.get(iceberg::io::CLIENT_REGION))
746
+ .cloned();
747
+ object_store::inject_aws_static_credentials(&mut props, region.as_deref()).await;
734
748
  catalog_builder =
735
749
  catalog_builder.with_storage_factory(std::sync::Arc::new(OpenDalStorageFactory::S3 {
736
750
  configured_scheme: scheme,
@@ -2,7 +2,6 @@
2
2
  use crate::errors::FloeError;
3
3
  use polars::prelude::DataFrame;
4
4
 
5
- #[cfg(feature = "delta")]
6
5
  #[cfg(feature = "delta")]
7
6
  use crate::io::format::AcceptedMergeMetrics;
8
7
  #[cfg(feature = "delta")]
@@ -395,15 +395,21 @@ fn build_common_manifest(
395
395
  now_ts_ms()
396
396
  };
397
397
 
398
- // Serialize profile sections as opaque JSON values so they can be re-applied at run time.
399
- let storages = profile
400
- .and_then(|p| p.storages.as_ref())
398
+ // Serialize the effective config sections (config-level definitions already merged with
399
+ // the profile via `apply_profile_*`, and `{{VAR}}` placeholders resolved) as opaque JSON
400
+ // values so the manifest is self-contained and can be replayed by `floe run --manifest`
401
+ // without re-supplying the config or profile.
402
+ let storages = config
403
+ .storages
404
+ .as_ref()
401
405
  .and_then(|v| serde_json::to_value(v).ok());
402
- let catalogs = profile
403
- .and_then(|p| p.catalogs.as_ref())
406
+ let catalogs = config
407
+ .catalogs
408
+ .as_ref()
404
409
  .and_then(|v| serde_json::to_value(v).ok());
405
- let lineage = profile
406
- .and_then(|p| p.lineage.as_ref())
410
+ let lineage = config
411
+ .lineage
412
+ .as_ref()
407
413
  .and_then(|v| serde_json::to_value(v).ok());
408
414
 
409
415
  CommonManifest {
@@ -93,6 +93,23 @@ pub struct ManifestColumnDefForRun {
93
93
  pub trim: Option<bool>,
94
94
  }
95
95
 
96
+ /// Deserialize an optional embedded manifest section (storages/catalogs/lineage),
97
+ /// returning a clear error if the block is present but malformed rather than silently
98
+ /// discarding it.
99
+ fn deserialize_manifest_section<T: serde::de::DeserializeOwned>(
100
+ value: Option<&serde_json::Value>,
101
+ section: &str,
102
+ ) -> FloeResult<Option<T>> {
103
+ match value {
104
+ Some(v) => serde_json::from_value::<T>(v.clone()).map(Some).map_err(
105
+ |err| -> Box<dyn std::error::Error + Send + Sync> {
106
+ FloeError::config(format!("manifest {section} block is malformed: {err}")).into()
107
+ },
108
+ ),
109
+ None => Ok(None),
110
+ }
111
+ }
112
+
96
113
  /// Parse a manifest JSON string and reconstruct a minimal RootConfig.
97
114
  /// Returns (config, report_base_uri).
98
115
  pub fn config_from_manifest_json(json: &str) -> FloeResult<(crate::config::RootConfig, String)> {
@@ -101,18 +118,16 @@ pub fn config_from_manifest_json(json: &str) -> FloeResult<(crate::config::RootC
101
118
  FloeError::config(format!("manifest parse error: {err}")).into()
102
119
  })?;
103
120
 
104
- let storages = manifest
105
- .storages
106
- .as_ref()
107
- .and_then(|v| serde_json::from_value::<StoragesConfig>(v.clone()).ok());
108
- let catalogs = manifest
109
- .catalogs
110
- .as_ref()
111
- .and_then(|v| serde_json::from_value::<CatalogsConfig>(v.clone()).ok());
112
- let lineage = manifest
113
- .lineage
114
- .as_ref()
115
- .and_then(|v| serde_json::from_value::<LineageConfig>(v.clone()).ok());
121
+ // Deserialize the embedded sections, surfacing malformed blocks as a clear error instead
122
+ // of silently dropping them (a swallowed error here previously surfaced downstream as the
123
+ // confusing "no storages block" failure when a hand-edited manifest had a slightly-off
124
+ // storages shape).
125
+ let storages =
126
+ deserialize_manifest_section::<StoragesConfig>(manifest.storages.as_ref(), "storages")?;
127
+ let catalogs =
128
+ deserialize_manifest_section::<CatalogsConfig>(manifest.catalogs.as_ref(), "catalogs")?;
129
+ let lineage =
130
+ deserialize_manifest_section::<LineageConfig>(manifest.lineage.as_ref(), "lineage")?;
116
131
 
117
132
  let entities = manifest
118
133
  .entities
@@ -2,7 +2,7 @@ use std::fs;
2
2
  use std::path::{Path, PathBuf};
3
3
  use std::time::{SystemTime, UNIX_EPOCH};
4
4
 
5
- use floe_core::load_config;
5
+ use floe_core::{load_config, load_config_with_profile_vars};
6
6
 
7
7
  fn temp_dir(prefix: &str) -> PathBuf {
8
8
  let mut path = std::env::temp_dir();
@@ -30,6 +30,64 @@ fn write_env(dir: &Path, contents: &str) -> PathBuf {
30
30
  write_file(&path, contents)
31
31
  }
32
32
 
33
+ #[test]
34
+ fn profile_vars_apply_to_storage_definition_fields() {
35
+ let root = temp_dir("floe-storage-vars");
36
+ let config_yaml = format!(
37
+ r#"version: "0.1"
38
+ storages:
39
+ default: "lakehouse_bronze"
40
+ definitions:
41
+ - name: "lakehouse_bronze"
42
+ type: "s3"
43
+ bucket: "{{{{BRONZE_BUCKET}}}}"
44
+ region: "{{{{STORAGE_REGION}}}}"
45
+ prefix: "{{{{STORAGE_PREFIX}}}}"
46
+ endpoint: "https://{{{{STORAGE_ENDPOINT}}}}"
47
+ entities:
48
+ - name: "orders"
49
+ source:
50
+ format: "csv"
51
+ path: "{root}/in/orders.csv"
52
+ sink:
53
+ accepted:
54
+ format: "parquet"
55
+ storage: "lakehouse_bronze"
56
+ path: "sales/orders"
57
+ policy:
58
+ severity: "warn"
59
+ schema:
60
+ columns:
61
+ - name: "id"
62
+ type: "string"
63
+ "#,
64
+ root = root.display(),
65
+ );
66
+ let config_path = write_config(&root, &config_yaml);
67
+ let profile_vars = [
68
+ ("BRONZE_BUCKET".to_string(), "lakehouse-bronze".to_string()),
69
+ ("STORAGE_REGION".to_string(), "us-east-1".to_string()),
70
+ ("STORAGE_PREFIX".to_string(), "bronze/root".to_string()),
71
+ (
72
+ "STORAGE_ENDPOINT".to_string(),
73
+ "s3.local.example".to_string(),
74
+ ),
75
+ ]
76
+ .into_iter()
77
+ .collect();
78
+
79
+ let parsed = load_config_with_profile_vars(&config_path, &profile_vars).expect("parse config");
80
+ let definition = &parsed.storages.as_ref().expect("storages").definitions[0];
81
+
82
+ assert_eq!(definition.bucket.as_deref(), Some("lakehouse-bronze"));
83
+ assert_eq!(definition.region.as_deref(), Some("us-east-1"));
84
+ assert_eq!(definition.prefix.as_deref(), Some("bronze/root"));
85
+ assert_eq!(
86
+ definition.endpoint.as_deref(),
87
+ Some("https://s3.local.example")
88
+ );
89
+ }
90
+
33
91
  #[test]
34
92
  fn env_file_vars_apply_to_source_path() {
35
93
  let root = temp_dir("floe-env-file");
@@ -1076,6 +1076,116 @@ entities:
1076
1076
  assert_eq!(accepted.storage.as_deref(), Some("motherduck"));
1077
1077
  }
1078
1078
 
1079
+ #[test]
1080
+ fn manifest_embeds_config_level_storages_for_replay() {
1081
+ // Storages defined at config level (not in a profile) must be embedded in the generated
1082
+ // manifest so `floe run --manifest` is self-contained and can resolve named storages.
1083
+ // Regression for #425 ("no storages block").
1084
+ let temp_dir = tempfile::TempDir::new().expect("temp dir");
1085
+ let cfg_dir = temp_dir.path().join("cfg");
1086
+ std::fs::create_dir_all(&cfg_dir).expect("cfg dir");
1087
+ let config_path = cfg_dir.join("config.yml");
1088
+
1089
+ let yaml = r#"version: "0.1"
1090
+ storages:
1091
+ default: "local"
1092
+ definitions:
1093
+ - name: "local"
1094
+ type: "local"
1095
+ - name: "lakehouse_bronze"
1096
+ type: "s3"
1097
+ bucket: "example-bronze"
1098
+ region: "eu-west-1"
1099
+ entities:
1100
+ - name: "accounts"
1101
+ source:
1102
+ format: "csv"
1103
+ storage: "local"
1104
+ path: "./in/accounts.csv"
1105
+ sink:
1106
+ accepted:
1107
+ format: "parquet"
1108
+ storage: "lakehouse_bronze"
1109
+ path: "sales/accounts"
1110
+ policy:
1111
+ severity: "warn"
1112
+ schema:
1113
+ columns:
1114
+ - name: "id"
1115
+ type: "string"
1116
+ "#;
1117
+ std::fs::write(&config_path, yaml).expect("write config");
1118
+
1119
+ let config_location = resolve_config_location(config_path.to_str().expect("utf8"))
1120
+ .expect("resolve config location");
1121
+ let config = load_config(&config_location.path).expect("load config");
1122
+ let opts = ManifestOptions {
1123
+ path_mode: PathMode::ResolvedUri,
1124
+ ..ManifestOptions::default()
1125
+ };
1126
+
1127
+ let payload =
1128
+ build_common_manifest_json(&config_location, &config, &[], None, &opts).expect("manifest");
1129
+ let value: Value = serde_json::from_str(&payload).expect("valid json");
1130
+
1131
+ // The manifest must carry the config-level storages even without a profile.
1132
+ let definitions = value["storages"]["definitions"]
1133
+ .as_array()
1134
+ .expect("storages.definitions should be present in the manifest");
1135
+ assert!(
1136
+ definitions
1137
+ .iter()
1138
+ .any(|d| d["name"] == "lakehouse_bronze" && d["bucket"] == "example-bronze"),
1139
+ "config-level storage definition must be embedded in the manifest: {value:#}"
1140
+ );
1141
+
1142
+ // ...and survive reconstruction so manifest replay can resolve the named storage.
1143
+ let (reconstructed, _base) =
1144
+ config_from_manifest_json(&payload).expect("reconstruct config from manifest");
1145
+ let storages = reconstructed
1146
+ .storages
1147
+ .expect("storages should be populated after manifest round-trip");
1148
+ let bronze = storages
1149
+ .definitions
1150
+ .iter()
1151
+ .find(|d| d.name == "lakehouse_bronze")
1152
+ .expect("lakehouse_bronze definition should survive round-trip");
1153
+ assert_eq!(bronze.bucket.as_deref(), Some("example-bronze"));
1154
+ assert_eq!(bronze.region.as_deref(), Some("eu-west-1"));
1155
+ }
1156
+
1157
+ #[test]
1158
+ fn manifest_reconstruct_rejects_malformed_storages_block() {
1159
+ // A present-but-malformed storages block must surface a clear error rather than being
1160
+ // silently dropped (which previously resurfaced downstream as the confusing
1161
+ // "no storages block" failure). Regression for #425.
1162
+ let config_path = repo_root().join("example/config.yml");
1163
+ let config_location = resolve_config_location(config_path.to_str().expect("utf8"))
1164
+ .expect("resolve config location");
1165
+ let config = load_config(&config_location.path).expect("load config");
1166
+ let payload = build_common_manifest_json(
1167
+ &config_location,
1168
+ &config,
1169
+ &[],
1170
+ None,
1171
+ &ManifestOptions::default(),
1172
+ )
1173
+ .expect("manifest");
1174
+
1175
+ let mut value: Value = serde_json::from_str(&payload).expect("valid json");
1176
+ // A bare string is not a valid storages object.
1177
+ value["storages"] = Value::String("totally-not-a-storages-block".into());
1178
+ let corrupted = serde_json::to_string(&value).expect("reserialize");
1179
+
1180
+ let err =
1181
+ config_from_manifest_json(&corrupted).expect_err("malformed storages block must error");
1182
+ let msg = err.to_string();
1183
+ assert!(
1184
+ msg.contains("storages") && msg.contains("malformed"),
1185
+ "expected a clear 'malformed storages' error, got: {msg}"
1186
+ );
1187
+ }
1188
+
1079
1189
  #[test]
1080
1190
  fn manifest_path_mode_resolved_uri_sets_path_from_uri() {
1081
1191
  let temp_dir = tempfile::TempDir::new().expect("temp dir");
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "floe-python"
3
- version = "0.6.1"
3
+ version = "0.6.2"
4
4
  edition = "2021"
5
5
  description = "Python bindings for floe-core (PyO3-based)"
6
6
  license = "Apache-2.0"
@@ -14,7 +14,7 @@ name = "_floe"
14
14
  crate-type = ["cdylib"]
15
15
 
16
16
  [dependencies]
17
- floe-core = { path = "../floe-core", version = "0.6.1", default-features = false }
17
+ floe-core = { path = "../floe-core", version = "0.6.2", default-features = false }
18
18
  pyo3 = { version = "0.29", features = ["extension-module", "abi3-py310"] }
19
19
  serde_json = "1"
20
20
 
@@ -22,7 +22,7 @@ build-backend = "maturin"
22
22
 
23
23
  [project]
24
24
  name = "floe-duckdb"
25
- version = "0.6.1"
25
+ version = "0.6.2"
26
26
  description = "DuckDB companion extension for floe — adds the native DuckDB sink to the floe Python package"
27
27
  readme = "README.md"
28
28
  requires-python = ">=3.10"
@@ -46,7 +46,7 @@ classifiers = [
46
46
  # only contributes the compiled DuckDB extension module into that package. Pin to
47
47
  # the exact lean version so the companion's `_floe_duckdb` ABI always matches the
48
48
  # `floe` package it plugs into.
49
- dependencies = ["floe-python==0.6.1"]
49
+ dependencies = ["floe-python==0.6.2"]
50
50
 
51
51
  [project.urls]
52
52
  Homepage = "https://github.com/malon64/floe"
@@ -4,7 +4,7 @@ build-backend = "maturin"
4
4
 
5
5
  [project]
6
6
  name = "floe-python"
7
- version = "0.6.1"
7
+ version = "0.6.2"
8
8
  description = "Python bindings for floe-core — run floe pipelines at Rust speed from Python and notebooks"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
File without changes
File without changes