Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (346) hide show
  1. build_backends/main.py +25 -22
  2. build_backends/main_prd.py +10 -19
  3. flowfile/__init__.py +194 -74
  4. flowfile/__main__.py +10 -7
  5. flowfile/api.py +51 -57
  6. flowfile/web/__init__.py +14 -9
  7. flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
  8. flowfile/web/static/assets/AdminView-f9847d67.js +713 -0
  9. flowfile/web/static/assets/CloudConnectionView-cf85f943.css +72 -0
  10. flowfile/web/static/assets/{CloudConnectionManager-0dfba9f2.js → CloudConnectionView-faace55b.js} +11 -11
  11. flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
  12. flowfile/web/static/assets/{CloudStorageReader-d5b1b6c9.js → CloudStorageReader-d86ecaa7.js} +10 -8
  13. flowfile/web/static/assets/{CloudStorageWriter-00d87aad.js → CloudStorageWriter-0f4d9a44.js} +10 -8
  14. flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
  15. flowfile/web/static/assets/ColumnActionInput-c44b7aee.css +159 -0
  16. flowfile/web/static/assets/ColumnActionInput-f4189ae0.js +330 -0
  17. flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
  18. flowfile/web/static/assets/{ColumnSelector-4685e75d.js → ColumnSelector-e66b33da.js} +3 -5
  19. flowfile/web/static/assets/ContextMenu-49463352.js +9 -0
  20. flowfile/web/static/assets/ContextMenu-dd5f3f25.js +9 -0
  21. flowfile/web/static/assets/ContextMenu-f709b884.js +9 -0
  22. flowfile/web/static/assets/ContextMenu.vue_vue_type_script_setup_true_lang-a1bd6314.js +59 -0
  23. flowfile/web/static/assets/{CrossJoin-702a3edd.js → CrossJoin-24694b8f.js} +12 -10
  24. flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
  25. flowfile/web/static/assets/{CustomNode-b1519993.js → CustomNode-569d45ff.js} +43 -24
  26. flowfile/web/static/assets/CustomNode-edb9b939.css +42 -0
  27. flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-c20a1e16.css} +23 -21
  28. flowfile/web/static/assets/{DatabaseConnectionSettings-6f3e4ea5.js → DatabaseConnectionSettings-cfc08938.js} +5 -4
  29. flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-5bf8c75b.css} +41 -46
  30. flowfile/web/static/assets/{DatabaseReader-d38c7295.js → DatabaseReader-701feabb.js} +25 -15
  31. flowfile/web/static/assets/{DatabaseManager-cf5ef661.js → DatabaseView-0482e5b5.js} +11 -11
  32. flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
  33. flowfile/web/static/assets/{DatabaseWriter-b04ef46a.js → DatabaseWriter-16721989.js} +17 -10
  34. flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-bdcf2c8b.css} +29 -27
  35. flowfile/web/static/assets/{designer-8da3ba3a.css → DesignerView-49abb835.css} +783 -663
  36. flowfile/web/static/assets/{designer-9633482a.js → DesignerView-f64749fb.js} +1292 -3253
  37. flowfile/web/static/assets/{documentation-ca400224.js → DocumentationView-61bd2990.js} +5 -5
  38. flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-9ea6e871.css} +9 -9
  39. flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
  40. flowfile/web/static/assets/{ExploreData-5fa10ed8.js → ExploreData-e2735b13.js} +18 -9
  41. flowfile/web/static/assets/{ExternalSource-d39af878.js → ExternalSource-2535c3b2.js} +9 -7
  42. flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-7ac7373f.css} +20 -20
  43. flowfile/web/static/assets/Filter-2cdbc93c.js +287 -0
  44. flowfile/web/static/assets/Filter-7494ea97.css +48 -0
  45. flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
  46. flowfile/web/static/assets/{Formula-6b04fb1d.js → Formula-fcda3c2c.js} +13 -11
  47. flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
  48. flowfile/web/static/assets/{FuzzyMatch-999521f4.js → FuzzyMatch-f8d3b7d3.js} +12 -10
  49. flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-4b4d7db9.css} +5 -5
  50. flowfile/web/static/assets/{GraphSolver-17dd2198.js → GraphSolver-72eaa695.js} +14 -12
  51. flowfile/web/static/assets/GroupBy-5792782d.css +9 -0
  52. flowfile/web/static/assets/{GroupBy-6b039e18.js → GroupBy-8aa0598b.js} +9 -7
  53. flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
  54. flowfile/web/static/assets/{Join-24d0f113.js → Join-e40f0ffa.js} +13 -11
  55. flowfile/web/static/assets/LoginView-5111c9ae.js +134 -0
  56. flowfile/web/static/assets/LoginView-d325d632.css +172 -0
  57. flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
  58. flowfile/web/static/assets/{ManualInput-34639209.js → ManualInput-9b6f3224.js} +170 -116
  59. flowfile/web/static/assets/{MultiSelect-0e8724a3.js → MultiSelect-ef28e19e.js} +2 -2
  60. flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-b0e538c2.js → MultiSelect.vue_vue_type_script_setup_true_lang-83b3bbfd.js} +1 -1
  61. flowfile/web/static/assets/NodeDesigner-94cd4dd3.css +1429 -0
  62. flowfile/web/static/assets/NodeDesigner-d2b7ee2b.js +2712 -0
  63. flowfile/web/static/assets/{NumericInput-3d63a470.js → NumericInput-1d789794.js} +2 -2
  64. flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-e0edeccc.js → NumericInput.vue_vue_type_script_setup_true_lang-7775f83e.js} +5 -2
  65. flowfile/web/static/assets/Output-692dd25d.css +37 -0
  66. flowfile/web/static/assets/{Output-edea9802.js → Output-cefef801.js} +14 -10
  67. flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
  68. flowfile/web/static/assets/{Pivot-61d19301.js → Pivot-bab1b75b.js} +12 -10
  69. flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
  70. flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
  71. flowfile/web/static/assets/{PivotValidation-f97fec5b.js → PivotValidation-e7941f91.js} +3 -3
  72. flowfile/web/static/assets/{PivotValidation-de9f43fe.js → PivotValidation-fba09336.js} +3 -3
  73. flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
  74. flowfile/web/static/assets/{PolarsCode-bc3c9984.js → PolarsCode-740e40fa.js} +18 -9
  75. flowfile/web/static/assets/PopOver-862d7e28.js +939 -0
  76. flowfile/web/static/assets/PopOver-d96599db.css +33 -0
  77. flowfile/web/static/assets/{Read-64a3f259.js → Read-225cc63f.js} +16 -12
  78. flowfile/web/static/assets/{Read-e808b239.css → Read-90f366bc.css} +15 -15
  79. flowfile/web/static/assets/{RecordCount-3d5039be.js → RecordCount-ffc71eca.js} +6 -4
  80. flowfile/web/static/assets/{RecordId-597510e0.js → RecordId-a70bb8df.js} +9 -7
  81. flowfile/web/static/assets/{SQLQueryComponent-df51adbe.js → SQLQueryComponent-15a421f5.js} +3 -3
  82. flowfile/web/static/assets/SQLQueryComponent-edb90b98.css +29 -0
  83. flowfile/web/static/assets/{Sample-4be0a507.js → Sample-6c26afc7.js} +6 -4
  84. flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
  85. flowfile/web/static/assets/SecretSelector-ceed9496.js +113 -0
  86. flowfile/web/static/assets/{SecretManager-4839be57.js → SecretsView-214d255a.js} +35 -36
  87. flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
  88. flowfile/web/static/assets/{Select-9b72f201.js → Select-8fc29999.js} +9 -7
  89. flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
  90. flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
  91. flowfile/web/static/assets/{SettingsSection-7ded385d.js → SettingsSection-3f70e4c3.js} +3 -3
  92. flowfile/web/static/assets/{SettingsSection-f0f75a42.js → SettingsSection-83090218.js} +3 -3
  93. flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
  94. flowfile/web/static/assets/{SettingsSection-e1e9c953.js → SettingsSection-9f0d1725.js} +3 -3
  95. flowfile/web/static/assets/SetupView-3fa0aa03.js +160 -0
  96. flowfile/web/static/assets/SetupView-e2da3442.css +230 -0
  97. flowfile/web/static/assets/{SingleSelect-6c777aac.js → SingleSelect-a4a568cb.js} +2 -2
  98. flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-33e3ff9b.js → SingleSelect.vue_vue_type_script_setup_true_lang-c8ebdd33.js} +1 -1
  99. flowfile/web/static/assets/{SliderInput-7cb93e62.js → SliderInput-be533e71.js} +7 -4
  100. flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
  101. flowfile/web/static/assets/{Sort-6cbde21a.js → Sort-154dad81.js} +9 -7
  102. flowfile/web/static/assets/Sort-4abb7fae.css +9 -0
  103. flowfile/web/static/assets/{TextInput-d9a40c11.js → TextInput-454e2bda.js} +2 -2
  104. flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-5896c375.js → TextInput.vue_vue_type_script_setup_true_lang-e86510d0.js} +5 -2
  105. flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
  106. flowfile/web/static/assets/{TextToRows-c4fcbf4d.js → TextToRows-ea73433d.js} +11 -10
  107. flowfile/web/static/assets/{ToggleSwitch-4ef91d19.js → ToggleSwitch-9d7b30f1.js} +2 -2
  108. flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-38478c20.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-00f2580e.js} +1 -1
  109. flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-394a1f78.css} +14 -14
  110. flowfile/web/static/assets/{UnavailableFields-a03f512c.js → UnavailableFields-b72a2c72.js} +4 -4
  111. flowfile/web/static/assets/{Union-bfe9b996.js → Union-1e44f263.js} +8 -6
  112. flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
  113. flowfile/web/static/assets/Unique-2b705521.css +3 -0
  114. flowfile/web/static/assets/{Unique-5d023a27.js → Unique-a3bc6d0a.js} +13 -10
  115. flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-b6ad6427.css} +7 -7
  116. flowfile/web/static/assets/{Unpivot-91cc5354.js → Unpivot-e27935fc.js} +11 -9
  117. flowfile/web/static/assets/{UnpivotValidation-7ee2de44.js → UnpivotValidation-72497680.js} +3 -3
  118. flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
  119. flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
  120. flowfile/web/static/assets/{VueGraphicWalker-e51b9924.js → VueGraphicWalker-d9ab70a3.js} +4 -4
  121. flowfile/web/static/assets/{api-cf1221f0.js → api-a2102880.js} +1 -1
  122. flowfile/web/static/assets/{api-c1bad5ca.js → api-f75042b0.js} +1 -1
  123. flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-1d6acbd9.css} +41 -41
  124. flowfile/web/static/assets/{dropDown-614b998d.js → dropDown-2798a109.js} +3 -3
  125. flowfile/web/static/assets/{fullEditor-f7971590.js → fullEditor-cf7d7d93.js} +11 -10
  126. flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-fe9f7e18.css} +77 -65
  127. flowfile/web/static/assets/{genericNodeSettings-4fe5f36b.js → genericNodeSettings-14eac1c3.js} +5 -5
  128. flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
  129. flowfile/web/static/assets/{index-5429bbf8.js → index-387a6f18.js} +41806 -40958
  130. flowfile/web/static/assets/index-6b367bb5.js +38 -0
  131. flowfile/web/static/assets/{index-50508d4d.css → index-e96ab018.css} +2184 -569
  132. flowfile/web/static/assets/index-f0a6e5a5.js +2696 -0
  133. flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
  134. flowfile/web/static/assets/nodeInput-ed2ae8d7.js +2 -0
  135. flowfile/web/static/assets/{outputCsv-076b85ab.js → outputCsv-3c1757e8.js} +3 -3
  136. flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
  137. flowfile/web/static/assets/{outputExcel-0fd17dbe.js → outputExcel-686e1f48.js} +3 -3
  138. flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
  139. flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
  140. flowfile/web/static/assets/{outputParquet-b61e0847.js → outputParquet-df28faa7.js} +4 -4
  141. flowfile/web/static/assets/{readCsv-c767cb37.css → readCsv-3bfac4c3.css} +15 -15
  142. flowfile/web/static/assets/{readCsv-a8bb8b61.js → readCsv-e37eee21.js} +3 -3
  143. flowfile/web/static/assets/{readExcel-806d2826.css → readExcel-3db6b763.css} +13 -13
  144. flowfile/web/static/assets/{readExcel-67b4aee0.js → readExcel-a13f14bb.js} +5 -5
  145. flowfile/web/static/assets/{readParquet-92ce1dbc.js → readParquet-344cf746.js} +3 -3
  146. flowfile/web/static/assets/{readParquet-48c81530.css → readParquet-c5244ad5.css} +4 -4
  147. flowfile/web/static/assets/secrets.api-ae198c5c.js +65 -0
  148. flowfile/web/static/assets/{selectDynamic-92e25ee3.js → selectDynamic-6b4b0767.js} +5 -5
  149. flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
  150. flowfile/web/static/assets/{vue-codemirror.esm-41b0e0d7.js → vue-codemirror.esm-31ba0e0b.js} +31 -640
  151. flowfile/web/static/assets/{vue-content-loader.es-2c8e608f.js → vue-content-loader.es-4469c8ff.js} +1 -1
  152. flowfile/web/static/index.html +2 -2
  153. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/METADATA +3 -4
  154. flowfile-0.5.4.dist-info/RECORD +407 -0
  155. flowfile_core/__init__.py +13 -6
  156. flowfile_core/auth/jwt.py +51 -16
  157. flowfile_core/auth/models.py +32 -7
  158. flowfile_core/auth/password.py +89 -0
  159. flowfile_core/auth/secrets.py +64 -19
  160. flowfile_core/configs/__init__.py +9 -7
  161. flowfile_core/configs/flow_logger.py +15 -14
  162. flowfile_core/configs/node_store/__init__.py +72 -4
  163. flowfile_core/configs/node_store/nodes.py +155 -172
  164. flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
  165. flowfile_core/configs/settings.py +28 -15
  166. flowfile_core/database/connection.py +7 -6
  167. flowfile_core/database/init_db.py +96 -2
  168. flowfile_core/database/models.py +3 -1
  169. flowfile_core/fileExplorer/__init__.py +17 -0
  170. flowfile_core/fileExplorer/funcs.py +145 -57
  171. flowfile_core/fileExplorer/utils.py +10 -11
  172. flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
  173. flowfile_core/flowfile/analytics/analytics_processor.py +26 -24
  174. flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
  175. flowfile_core/flowfile/analytics/utils.py +1 -1
  176. flowfile_core/flowfile/code_generator/__init__.py +11 -0
  177. flowfile_core/flowfile/code_generator/code_generator.py +706 -247
  178. flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
  179. flowfile_core/flowfile/connection_manager/models.py +1 -1
  180. flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
  181. flowfile_core/flowfile/database_connection_manager/models.py +1 -1
  182. flowfile_core/flowfile/extensions.py +17 -12
  183. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
  184. flowfile_core/flowfile/flow_data_engine/create/funcs.py +115 -83
  185. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +493 -423
  186. flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
  187. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
  188. flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
  189. flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
  190. flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
  191. flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +31 -20
  192. flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
  193. flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
  194. flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +14 -15
  195. flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
  196. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
  197. flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
  198. flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
  199. flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
  200. flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
  201. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +190 -127
  202. flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
  203. flowfile_core/flowfile/flow_data_engine/utils.py +99 -67
  204. flowfile_core/flowfile/flow_graph.py +920 -571
  205. flowfile_core/flowfile/flow_graph_utils.py +31 -49
  206. flowfile_core/flowfile/flow_node/flow_node.py +379 -258
  207. flowfile_core/flowfile/flow_node/models.py +53 -41
  208. flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
  209. flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
  210. flowfile_core/flowfile/handler.py +80 -30
  211. flowfile_core/flowfile/manage/compatibility_enhancements.py +209 -126
  212. flowfile_core/flowfile/manage/io_flowfile.py +54 -57
  213. flowfile_core/flowfile/node_designer/__init__.py +19 -13
  214. flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
  215. flowfile_core/flowfile/node_designer/custom_node.py +162 -36
  216. flowfile_core/flowfile/node_designer/ui_components.py +278 -34
  217. flowfile_core/flowfile/schema_callbacks.py +71 -51
  218. flowfile_core/flowfile/setting_generator/__init__.py +0 -1
  219. flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
  220. flowfile_core/flowfile/setting_generator/settings.py +64 -53
  221. flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
  222. flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
  223. flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
  224. flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
  225. flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
  226. flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
  227. flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
  228. flowfile_core/flowfile/util/calculate_layout.py +9 -13
  229. flowfile_core/flowfile/util/execution_orderer.py +25 -17
  230. flowfile_core/flowfile/util/node_skipper.py +4 -4
  231. flowfile_core/flowfile/utils.py +19 -21
  232. flowfile_core/main.py +26 -19
  233. flowfile_core/routes/auth.py +284 -11
  234. flowfile_core/routes/cloud_connections.py +25 -25
  235. flowfile_core/routes/logs.py +21 -29
  236. flowfile_core/routes/public.py +46 -4
  237. flowfile_core/routes/routes.py +70 -34
  238. flowfile_core/routes/secrets.py +25 -27
  239. flowfile_core/routes/user_defined_components.py +483 -4
  240. flowfile_core/run_lock.py +0 -1
  241. flowfile_core/schemas/__init__.py +4 -6
  242. flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
  243. flowfile_core/schemas/cloud_storage_schemas.py +96 -66
  244. flowfile_core/schemas/input_schema.py +231 -144
  245. flowfile_core/schemas/output_model.py +49 -34
  246. flowfile_core/schemas/schemas.py +116 -89
  247. flowfile_core/schemas/transform_schema.py +518 -263
  248. flowfile_core/schemas/yaml_types.py +21 -7
  249. flowfile_core/secret_manager/secret_manager.py +123 -18
  250. flowfile_core/types.py +29 -9
  251. flowfile_core/utils/arrow_reader.py +7 -6
  252. flowfile_core/utils/excel_file_manager.py +3 -3
  253. flowfile_core/utils/fileManager.py +7 -7
  254. flowfile_core/utils/fl_executor.py +8 -10
  255. flowfile_core/utils/utils.py +4 -4
  256. flowfile_core/utils/validate_setup.py +5 -4
  257. flowfile_frame/__init__.py +117 -51
  258. flowfile_frame/adapters.py +2 -9
  259. flowfile_frame/adding_expr.py +73 -32
  260. flowfile_frame/cloud_storage/frame_helpers.py +27 -23
  261. flowfile_frame/cloud_storage/secret_manager.py +12 -26
  262. flowfile_frame/config.py +2 -5
  263. flowfile_frame/database/__init__.py +36 -0
  264. flowfile_frame/database/connection_manager.py +205 -0
  265. flowfile_frame/database/frame_helpers.py +249 -0
  266. flowfile_frame/expr.py +311 -218
  267. flowfile_frame/expr.pyi +160 -159
  268. flowfile_frame/expr_name.py +23 -23
  269. flowfile_frame/flow_frame.py +571 -476
  270. flowfile_frame/flow_frame.pyi +123 -104
  271. flowfile_frame/flow_frame_methods.py +227 -246
  272. flowfile_frame/group_frame.py +50 -20
  273. flowfile_frame/join.py +2 -2
  274. flowfile_frame/lazy.py +129 -87
  275. flowfile_frame/lazy_methods.py +83 -30
  276. flowfile_frame/list_name_space.py +55 -50
  277. flowfile_frame/selectors.py +148 -68
  278. flowfile_frame/series.py +9 -7
  279. flowfile_frame/utils.py +19 -21
  280. flowfile_worker/__init__.py +12 -7
  281. flowfile_worker/configs.py +41 -33
  282. flowfile_worker/create/__init__.py +14 -9
  283. flowfile_worker/create/funcs.py +114 -77
  284. flowfile_worker/create/models.py +46 -43
  285. flowfile_worker/create/pl_types.py +14 -15
  286. flowfile_worker/create/read_excel_tables.py +34 -41
  287. flowfile_worker/create/utils.py +22 -19
  288. flowfile_worker/external_sources/s3_source/main.py +18 -51
  289. flowfile_worker/external_sources/s3_source/models.py +34 -27
  290. flowfile_worker/external_sources/sql_source/main.py +8 -5
  291. flowfile_worker/external_sources/sql_source/models.py +13 -9
  292. flowfile_worker/flow_logger.py +10 -8
  293. flowfile_worker/funcs.py +214 -155
  294. flowfile_worker/main.py +11 -17
  295. flowfile_worker/models.py +35 -28
  296. flowfile_worker/process_manager.py +2 -3
  297. flowfile_worker/routes.py +121 -90
  298. flowfile_worker/secrets.py +114 -21
  299. flowfile_worker/spawner.py +89 -54
  300. flowfile_worker/utils.py +3 -2
  301. shared/__init__.py +2 -7
  302. shared/storage_config.py +25 -13
  303. test_utils/postgres/commands.py +3 -2
  304. test_utils/postgres/fixtures.py +9 -9
  305. test_utils/s3/commands.py +1 -1
  306. test_utils/s3/data_generator.py +3 -4
  307. test_utils/s3/demo_data_generator.py +4 -7
  308. test_utils/s3/fixtures.py +7 -5
  309. tools/migrate/__init__.py +1 -1
  310. tools/migrate/__main__.py +16 -29
  311. tools/migrate/legacy_schemas.py +251 -190
  312. tools/migrate/migrate.py +193 -181
  313. tools/migrate/tests/conftest.py +1 -3
  314. tools/migrate/tests/test_migrate.py +36 -41
  315. tools/migrate/tests/test_migration_e2e.py +28 -29
  316. tools/migrate/tests/test_node_migrations.py +50 -20
  317. flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
  318. flowfile/web/static/assets/ContextMenu-23e909da.js +0 -41
  319. flowfile/web/static/assets/ContextMenu-4c74eef1.css +0 -26
  320. flowfile/web/static/assets/ContextMenu-63cfa99b.css +0 -26
  321. flowfile/web/static/assets/ContextMenu-70ae0c79.js +0 -41
  322. flowfile/web/static/assets/ContextMenu-c13f91d0.css +0 -26
  323. flowfile/web/static/assets/ContextMenu-f149cf7c.js +0 -41
  324. flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
  325. flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
  326. flowfile/web/static/assets/Filter-9b6d08db.js +0 -164
  327. flowfile/web/static/assets/Filter-f62091b3.css +0 -20
  328. flowfile/web/static/assets/GroupBy-b9505323.css +0 -51
  329. flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
  330. flowfile/web/static/assets/Output-283fe388.css +0 -37
  331. flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
  332. flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
  333. flowfile/web/static/assets/SQLQueryComponent-36cef432.css +0 -27
  334. flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
  335. flowfile/web/static/assets/Sort-3643d625.css +0 -51
  336. flowfile/web/static/assets/Unique-f9fb0809.css +0 -51
  337. flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
  338. flowfile/web/static/assets/nodeInput-5d0d6b79.js +0 -41
  339. flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
  340. flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
  341. flowfile/web/static/assets/secretApi-68435402.js +0 -46
  342. flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
  343. flowfile-0.5.1.dist-info/RECORD +0 -388
  344. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/WHEEL +0 -0
  345. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/entry_points.txt +0 -0
  346. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/licenses/LICENSE +0 -0
@@ -1,19 +1,40 @@
1
- # ui_components.py - Updated ColumnSelector
1
+ from typing import Any, Literal, NamedTuple
2
2
 
3
- from typing import List, Optional, Any, Literal, Union, Type, Tuple, Dict
4
-
5
- from pydantic import Field, BaseModel, computed_field
3
+ from pydantic import BaseModel, Field, SecretStr, computed_field
6
4
 
7
5
  from flowfile_core.flowfile.node_designer._type_registry import normalize_type_spec
6
+ from flowfile_core.secret_manager.secret_manager import decrypt_secret, get_encrypted_secret
7
+
8
8
  # Public API import
9
9
  from flowfile_core.types import DataType, TypeSpec
10
10
 
11
11
  InputType = Literal["text", "number", "secret", "array", "date", "boolean"]
12
12
 
13
13
 
14
- def normalize_input_to_data_types(
15
- v: Any
16
- ) -> Union[Literal["ALL"], List[DataType]]:
14
+ class ActionOption(NamedTuple):
15
+ """
16
+ A named tuple representing an action option with a value and display label.
17
+
18
+ Use this to define actions with custom labels in ColumnActionInput:
19
+ actions=[
20
+ ActionOption("sum", "Sum"),
21
+ ActionOption("avg", "Average"),
22
+ "count" # plain strings also work
23
+ ]
24
+ """
25
+
26
+ value: str
27
+ """The internal value used in the data."""
28
+
29
+ label: str
30
+ """The display label shown in the UI."""
31
+
32
+
33
+ # Type alias for action specifications - accepts strings or ActionOption tuples
34
+ ActionSpec = str | ActionOption
35
+
36
+
37
+ def normalize_input_to_data_types(v: Any) -> Literal["ALL"] | list[DataType]:
17
38
  """
18
39
  Normalizes a wide variety of inputs to either 'ALL' or a sorted list of DataType enums.
19
40
  This function is used as a Pydantic BeforeValidator.
@@ -46,9 +67,10 @@ class FlowfileInComponent(BaseModel):
46
67
  It's not meant to be used directly, but rather to be inherited by specific
47
68
  component classes.
48
69
  """
70
+
49
71
  component_type: str = Field(..., description="Type of the UI component")
50
72
  value: Any = None
51
- label: Optional[str] = None
73
+ label: str | None = None
52
74
  input_type: InputType
53
75
 
54
76
  def set_value(self, value: Any):
@@ -83,6 +105,7 @@ class IncomingColumns:
83
105
  options=IncomingColumns
84
106
  )
85
107
  """
108
+
86
109
  pass
87
110
 
88
111
 
@@ -94,25 +117,21 @@ class ColumnSelector(FlowfileInComponent):
94
117
  This is particularly useful when a node operation should only be applied
95
118
  to columns of a specific type (e.g., numeric, string, date).
96
119
  """
120
+
97
121
  component_type: Literal["ColumnSelector"] = "ColumnSelector"
98
122
  required: bool = False
99
123
  multiple: bool = False
100
124
  input_type: InputType = "text"
101
125
 
102
126
  # Normalized output: either "ALL" or list of DataType enums
103
- data_type_filter_input: TypeSpec = Field(
104
- default="ALL",
105
- alias="data_types",
106
- repr=False,
107
- exclude=True
108
- )
127
+ data_type_filter_input: TypeSpec = Field(default="ALL", alias="data_types", repr=False, exclude=True)
109
128
 
110
129
  class Config:
111
130
  arbitrary_types_allowed = True
112
131
 
113
132
  @computed_field
114
133
  @property
115
- def data_types_filter(self) -> Union[Literal["ALL"], List[DataType]]:
134
+ def data_types_filter(self) -> Literal["ALL"] | list[DataType]:
116
135
  """
117
136
  A computed field that normalizes the `data_type_filter_input` into a
118
137
  standardized format for the frontend.
@@ -125,16 +144,17 @@ class ColumnSelector(FlowfileInComponent):
125
144
  correct format for the frontend.
126
145
  """
127
146
  data = super().model_dump(**kwargs)
128
- if 'data_types_filter' in data and data['data_types_filter'] != "ALL":
129
- data['data_types'] = sorted([dt.value for dt in data['data_types_filter']])
147
+ if "data_types_filter" in data and data["data_types_filter"] != "ALL":
148
+ data["data_types"] = sorted([dt.value for dt in data["data_types_filter"]])
130
149
  return data
131
150
 
132
151
 
133
152
  class TextInput(FlowfileInComponent):
134
153
  """A standard text input field for capturing string values."""
154
+
135
155
  component_type: Literal["TextInput"] = "TextInput"
136
- default: Optional[str] = ""
137
- placeholder: Optional[str] = ""
156
+ default: str | None = ""
157
+ placeholder: str | None = ""
138
158
  input_type: InputType = "text"
139
159
 
140
160
  def __init__(self, **data):
@@ -145,10 +165,11 @@ class TextInput(FlowfileInComponent):
145
165
 
146
166
  class NumericInput(FlowfileInComponent):
147
167
  """A numeric input field with optional minimum and maximum value validation."""
168
+
148
169
  component_type: Literal["NumericInput"] = "NumericInput"
149
- default: Optional[float] = None
150
- min_value: Optional[float] = None
151
- max_value: Optional[float] = None
170
+ default: float | None = None
171
+ min_value: float | None = None
172
+ max_value: float | None = None
152
173
  input_type: InputType = "number"
153
174
 
154
175
  def __init__(self, **data):
@@ -157,11 +178,30 @@ class NumericInput(FlowfileInComponent):
157
178
  self.value = self.default
158
179
 
159
180
 
181
+ class SliderInput(FlowfileInComponent):
182
+ """A slider input for selecting a numeric value within a range."""
183
+
184
+ component_type: Literal["SliderInput"] = "SliderInput"
185
+ default: float | None = None
186
+ min_value: float = 0
187
+ max_value: float = 100
188
+ step: float = 1
189
+ input_type: InputType = "number"
190
+
191
+ def __init__(self, **data):
192
+ super().__init__(**data)
193
+ if self.value is None and self.default is not None:
194
+ self.value = self.default
195
+ elif self.value is None:
196
+ self.value = self.min_value
197
+
198
+
160
199
  class ToggleSwitch(FlowfileInComponent):
161
200
  """A boolean toggle switch, typically used for enabling or disabling a feature."""
201
+
162
202
  component_type: Literal["ToggleSwitch"] = "ToggleSwitch"
163
203
  default: bool = False
164
- description: Optional[str] = None
204
+ description: str | None = None
165
205
  input_type: InputType = "boolean"
166
206
 
167
207
  def __init__(self, **data):
@@ -182,9 +222,10 @@ class SingleSelect(FlowfileInComponent):
182
222
  dynamically populated from the input dataframe's columns by using the
183
223
  `IncomingColumns` marker.
184
224
  """
225
+
185
226
  component_type: Literal["SingleSelect"] = "SingleSelect"
186
- options: Union[List[Union[str, Tuple[str, Any]]], Type[IncomingColumns]]
187
- default: Optional[Any] = None
227
+ options: list[str | tuple[str, Any]] | type[IncomingColumns]
228
+ default: Any | None = None
188
229
  input_type: InputType = "text"
189
230
 
190
231
  def __init__(self, **data):
@@ -200,9 +241,10 @@ class MultiSelect(FlowfileInComponent):
200
241
  Like `SingleSelect`, the options can be static or dynamically populated
201
242
  from the input columns using the `IncomingColumns` marker.
202
243
  """
244
+
203
245
  component_type: Literal["MultiSelect"] = "MultiSelect"
204
- options: Union[List[Union[str, Tuple[str, Any]]], Type[IncomingColumns]]
205
- default: List[Any] = Field(default_factory=list)
246
+ options: list[str | tuple[str, Any]] | type[IncomingColumns]
247
+ default: list[Any] = Field(default_factory=list)
206
248
  input_type: InputType = "array"
207
249
 
208
250
  def __init__(self, **data):
@@ -226,12 +268,13 @@ class Section(BaseModel):
226
268
  my_text_input=TextInput(label="Enter a value")
227
269
  )
228
270
  """
229
- title: Optional[str] = None
230
- description: Optional[str] = None
271
+
272
+ title: str | None = None
273
+ description: str | None = None
231
274
  hidden: bool = False
232
275
 
233
276
  class Config:
234
- extra = 'allow'
277
+ extra = "allow"
235
278
  arbitrary_types_allowed = True
236
279
 
237
280
  def __init__(self, **data):
@@ -240,7 +283,7 @@ class Section(BaseModel):
240
283
  """
241
284
  super().__init__(**data)
242
285
 
243
- def __call__(self, **kwargs) -> 'Section':
286
+ def __call__(self, **kwargs) -> "Section":
244
287
  """
245
288
  Allows adding components to the section after initialization.
246
289
 
@@ -250,7 +293,7 @@ class Section(BaseModel):
250
293
  setattr(self, key, value)
251
294
  return self
252
295
 
253
- def get_components(self) -> Dict[str, FlowfileInComponent]:
296
+ def get_components(self) -> dict[str, FlowfileInComponent]:
254
297
  """
255
298
  Get all FlowfileInComponent instances from the section.
256
299
 
@@ -263,15 +306,216 @@ class Section(BaseModel):
263
306
  components = {}
264
307
 
265
308
  # Get from extra fields
266
- for key, value in getattr(self, '__pydantic_extra__', {}).items():
309
+ for key, value in getattr(self, "__pydantic_extra__", {}).items():
267
310
  if isinstance(value, FlowfileInComponent):
268
311
  components[key] = value
269
312
 
270
313
  # Get from defined fields (excluding metadata)
271
314
  for field_name in self.model_fields:
272
- if field_name not in {'title', 'description', 'hidden'}:
315
+ if field_name not in {"title", "description", "hidden"}:
273
316
  value = getattr(self, field_name, None)
274
317
  if isinstance(value, FlowfileInComponent):
275
318
  components[field_name] = value
276
319
 
277
320
  return components
321
+
322
+
323
+ class AvailableSecrets:
324
+ """
325
+ A marker class used in `SecretSelector` components.
326
+
327
+ When `options` is set to this class, the component will be dynamically
328
+ populated with the secret names available to the current user.
329
+ This allows users to select from available secrets at runtime.
330
+
331
+ Example:
332
+ class MyNodeSettings(NodeSettings):
333
+ api_key = SecretSelector(
334
+ label="Select an API Key",
335
+ options=AvailableSecrets
336
+ )
337
+ """
338
+
339
+ pass
340
+
341
+
342
+ class SecretSelector(FlowfileInComponent):
343
+ component_type: Literal["SecretSelector"] = "SecretSelector"
344
+ options: type[AvailableSecrets] = AvailableSecrets
345
+ required: bool = False
346
+ description: str | None = None
347
+ input_type: InputType = "secret"
348
+ name_prefix: str | None = None
349
+
350
+ # Private fields for runtime context
351
+ _user_id: int | None = None
352
+ _accessed_secrets: set | None = None # Reference to node's tracking set
353
+
354
+ def set_execution_context(self, user_id: int, accessed_secrets: set):
355
+ """Called by framework before process() runs."""
356
+ self._user_id = user_id
357
+ self._accessed_secrets = accessed_secrets
358
+
359
+ @property
360
+ def secret_value(self) -> SecretStr | None:
361
+ """
362
+ Get the decrypted secret value.
363
+
364
+ Can only be called during node execution (after context is set).
365
+ Returns None if no secret is selected.
366
+ """
367
+ if self.value is None:
368
+ return None
369
+
370
+ if self._user_id is None:
371
+ raise ValueError(
372
+ "Secret can only be accessed during node execution. "
373
+ "Ensure you're calling this from within the process() method."
374
+ )
375
+
376
+ encrypted = get_encrypted_secret(current_user_id=self._user_id, secret_name=self.value)
377
+
378
+ if encrypted is None:
379
+ raise ValueError(
380
+ f"Secret '{self.value}' not found for user. " f"Please ensure the secret exists in your secrets store."
381
+ )
382
+
383
+ decrypted = decrypt_secret(encrypted)
384
+
385
+ if self._accessed_secrets is not None:
386
+ self._accessed_secrets.add(decrypted.get_secret_value())
387
+ else:
388
+ self._accessed_secrets = {decrypted.get_secret_value()}
389
+ return decrypted
390
+
391
+ def model_dump(self, **kwargs) -> dict:
392
+ """
393
+ Overrides the default `model_dump` to signal to the frontend
394
+ that this needs dynamic population from available secrets.
395
+ """
396
+ data = super().model_dump(**kwargs)
397
+ # Signal to frontend that options should be fetched from /secrets endpoint
398
+ data["options"] = {"__type__": "AvailableSecrets"}
399
+ if self.name_prefix:
400
+ data["name_prefix"] = self.name_prefix
401
+ return data
402
+
403
+
404
+ class ColumnActionInput(FlowfileInComponent):
405
+ """
406
+ A generic UI component for configuring column-based transformations.
407
+
408
+ This component allows users to select columns, choose an action/transformation,
409
+ and optionally rename the output. It can be configured for many use cases:
410
+ rolling windows, aggregations, string transformations, type conversions, etc.
411
+
412
+ The component displays:
413
+ - A list of available columns (filterable by data type)
414
+ - A table of configured operations with: Column, Action, Output Name
415
+ - Optional group by and order by selectors
416
+
417
+ Example - Rolling Window:
418
+ ColumnActionInput(
419
+ label="Rolling Calculations",
420
+ actions=["sum", "mean", "min", "max"],
421
+ output_name_template="{column}_rolling_{action}",
422
+ show_group_by=True,
423
+ show_order_by=True,
424
+ data_types="Numeric"
425
+ )
426
+
427
+ Example - String Transformations:
428
+ ColumnActionInput(
429
+ label="String Operations",
430
+ actions=["upper", "lower", "trim", "reverse"],
431
+ output_name_template="{column}_{action}",
432
+ data_types="String"
433
+ )
434
+
435
+ Example - Aggregations with ActionOption:
436
+ ColumnActionInput(
437
+ label="Aggregations",
438
+ actions=[
439
+ ActionOption("sum", "Sum"),
440
+ ActionOption("count", "Count"),
441
+ ActionOption("mean", "Average"),
442
+ "min", # plain strings also work
443
+ ],
444
+ output_name_template="{column}_{action}",
445
+ show_group_by=True
446
+ )
447
+ """
448
+
449
+ component_type: Literal["ColumnActionInput"] = "ColumnActionInput"
450
+ input_type: InputType = "array"
451
+
452
+ # Configurable actions - list of action names or ActionOption tuples
453
+ actions: list[ActionSpec] = Field(default_factory=list)
454
+ """Actions available for selection. Can be strings or ActionOption(value, label) tuples."""
455
+
456
+ # Template for auto-generating output names
457
+ # Supports placeholders: {column}, {action}
458
+ output_name_template: str = "{column}_{action}"
459
+ """Template for generating default output names. Use {column} and {action} placeholders."""
460
+
461
+ # Optional grouping/ordering support
462
+ show_group_by: bool = False
463
+ """Whether to show the group by column selector."""
464
+
465
+ show_order_by: bool = False
466
+ """Whether to show the order by column selector."""
467
+
468
+ # Type filtering for column selection
469
+ data_type_filter_input: TypeSpec = Field(default="ALL", alias="data_types", repr=False, exclude=True)
470
+ """Filter columns by data type. Defaults to ALL."""
471
+
472
+ class Config:
473
+ arbitrary_types_allowed = True
474
+
475
+ def __init__(self, **data):
476
+ super().__init__(**data)
477
+ # Initialize value if not set
478
+ if self.value is None:
479
+ self.value = {
480
+ "rows": [],
481
+ "group_by_columns": [],
482
+ "order_by_column": None,
483
+ }
484
+
485
+ def set_value(self, value: Any):
486
+ """
487
+ Sets the value from frontend.
488
+ """
489
+ self.value = value
490
+ return self
491
+
492
+ @computed_field
493
+ @property
494
+ def data_types_filter(self) -> Literal["ALL"] | list[DataType]:
495
+ """
496
+ A computed field that normalizes the `data_type_filter_input` into a
497
+ standardized format for the frontend.
498
+ """
499
+ return normalize_input_to_data_types(self.data_type_filter_input)
500
+
501
+ def model_dump(self, **kwargs) -> dict:
502
+ """
503
+ Serializes the component for the frontend.
504
+ """
505
+ data = super().model_dump(**kwargs)
506
+ # Normalize actions to list of {value, label} objects for frontend
507
+ normalized_actions = []
508
+ for action in self.actions:
509
+ if isinstance(action, tuple):
510
+ normalized_actions.append({"value": action[0], "label": action[1]})
511
+ else:
512
+ normalized_actions.append({"value": action, "label": action})
513
+ data["actions"] = normalized_actions
514
+ data["output_name_template"] = self.output_name_template
515
+ data["show_group_by"] = self.show_group_by
516
+ data["show_order_by"] = self.show_order_by
517
+ if "data_types_filter" in data and data["data_types_filter"] != "ALL":
518
+ data["data_types"] = sorted([dt.value for dt in data["data_types_filter"]])
519
+ else:
520
+ data["data_types"] = "ALL"
521
+ return data
@@ -1,23 +1,17 @@
1
-
2
- from typing import List
3
-
4
- from polars import datatypes
5
1
  import polars as pl
6
-
7
2
  from pl_fuzzy_frame_match.output_column_name_utils import set_name_in_fuzzy_mappings
8
3
  from pl_fuzzy_frame_match.pre_process import rename_fuzzy_right_mapping
4
+ from polars import datatypes
9
5
 
10
- from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import fetch_unique_values
11
6
  from flowfile_core.configs.flow_logger import main_logger
12
7
  from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn, PlType
13
- from flowfile_core.schemas import transform_schema
14
- from flowfile_core.schemas import input_schema
15
- from flowfile_core.schemas.transform_schema import FuzzyMatchInputManager
8
+ from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import fetch_unique_values
9
+ from flowfile_core.schemas import input_schema, transform_schema
16
10
 
17
11
 
18
- def _ensure_all_columns_have_select(left_cols: List[str],
19
- right_cols: List[str],
20
- fuzzy_match_input: transform_schema.FuzzyMatchInputManager):
12
+ def _ensure_all_columns_have_select(
13
+ left_cols: list[str], right_cols: list[str], fuzzy_match_input: transform_schema.FuzzyMatchInputManager
14
+ ):
21
15
  """
22
16
  Ensure that all columns in the left and right FlowDataEngines are included in the fuzzy match input's select
23
17
  statements.
@@ -33,13 +27,16 @@ def _ensure_all_columns_have_select(left_cols: List[str],
33
27
  left_cols_in_select = {c.old_name for c in fuzzy_match_input.left_select.renames}
34
28
 
35
29
  fuzzy_match_input.left_select.renames.extend(
36
- [transform_schema.SelectInput(col) for col in left_cols if col not in left_cols_in_select])
30
+ [transform_schema.SelectInput(col) for col in left_cols if col not in left_cols_in_select]
31
+ )
37
32
  fuzzy_match_input.right_select.renames.extend(
38
33
  [transform_schema.SelectInput(col) for col in right_cols if col not in right_cols_in_select]
39
34
  )
40
35
 
41
36
 
42
- def _order_join_inputs_based_on_col_order(col_order: List[str], join_inputs: transform_schema.JoinInputsManager) -> None:
37
+ def _order_join_inputs_based_on_col_order(
38
+ col_order: list[str], join_inputs: transform_schema.JoinInputsManager
39
+ ) -> None:
43
40
  """
44
41
  Ensure that the select columns in the fuzzy match input match the order of the incoming columns.
45
42
  This function modifies the join_inputs object in-place.
@@ -52,17 +49,23 @@ def _order_join_inputs_based_on_col_order(col_order: List[str], join_inputs: tra
52
49
  join_inputs.select_inputs.renames = ordered_renames
53
50
 
54
51
 
55
- def calculate_fuzzy_match_schema(fm_input: transform_schema.FuzzyMatchInputManager,
56
- left_schema: List[FlowfileColumn],
57
- right_schema: List[FlowfileColumn]):
58
- _ensure_all_columns_have_select(left_cols=[col.column_name for col in left_schema],
59
- right_cols=[col.column_name for col in right_schema],
60
- fuzzy_match_input=fm_input)
52
+ def calculate_fuzzy_match_schema(
53
+ fm_input: transform_schema.FuzzyMatchInputManager,
54
+ left_schema: list[FlowfileColumn],
55
+ right_schema: list[FlowfileColumn],
56
+ ):
57
+ _ensure_all_columns_have_select(
58
+ left_cols=[col.column_name for col in left_schema],
59
+ right_cols=[col.column_name for col in right_schema],
60
+ fuzzy_match_input=fm_input,
61
+ )
61
62
 
62
- _order_join_inputs_based_on_col_order(col_order=[col.column_name for col in left_schema],
63
- join_inputs=fm_input.left_select)
64
- _order_join_inputs_based_on_col_order(col_order=[col.column_name for col in right_schema],
65
- join_inputs=fm_input.right_select)
63
+ _order_join_inputs_based_on_col_order(
64
+ col_order=[col.column_name for col in left_schema], join_inputs=fm_input.left_select
65
+ )
66
+ _order_join_inputs_based_on_col_order(
67
+ col_order=[col.column_name for col in right_schema], join_inputs=fm_input.right_select
68
+ )
66
69
  for column in fm_input.left_select.renames:
67
70
  if column.join_key:
68
71
  column.keep = True
@@ -78,20 +81,27 @@ def calculate_fuzzy_match_schema(fm_input: transform_schema.FuzzyMatchInputManag
78
81
  for column in fm_input.left_select.renames:
79
82
  column_schema = left_schema_dict.get(column.old_name)
80
83
  if column_schema and (column.keep or column.join_key):
81
- output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
82
- example_values=column_schema.example_values))
84
+ output_schema.append(
85
+ FlowfileColumn.from_input(
86
+ column.new_name, column_schema.data_type, example_values=column_schema.example_values
87
+ )
88
+ )
83
89
  for column in fm_input.right_select.renames:
84
90
  column_schema = right_schema_dict.get(column.old_name)
85
91
  if column_schema and (column.keep or column.join_key):
86
- output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
87
- example_values=column_schema.example_values))
92
+ output_schema.append(
93
+ FlowfileColumn.from_input(
94
+ column.new_name, column_schema.data_type, example_values=column_schema.example_values
95
+ )
96
+ )
88
97
  set_name_in_fuzzy_mappings(new_join_mapping)
89
- output_schema.extend([FlowfileColumn.from_input(fuzzy_mapping.output_column_name, 'Float64')
90
- for fuzzy_mapping in new_join_mapping])
98
+ output_schema.extend(
99
+ [FlowfileColumn.from_input(fuzzy_mapping.output_column_name, "Float64") for fuzzy_mapping in new_join_mapping]
100
+ )
91
101
  return output_schema
92
102
 
93
103
 
94
- def get_schema_of_column(node_input_schema: List[FlowfileColumn], col_name: str) -> FlowfileColumn|None:
104
+ def get_schema_of_column(node_input_schema: list[FlowfileColumn], col_name: str) -> FlowfileColumn | None:
95
105
  for s in node_input_schema:
96
106
  if s.name == col_name:
97
107
  return s
@@ -99,44 +109,54 @@ def get_schema_of_column(node_input_schema: List[FlowfileColumn], col_name: str)
99
109
 
100
110
  class InvalidSetup(ValueError):
101
111
  """Error raised when pivot column has too many unique values."""
112
+
102
113
  pass
103
114
 
104
115
 
105
116
  def get_output_data_type_pivot(schema: FlowfileColumn, agg_type: str) -> datatypes:
106
- if agg_type in ('count', 'n_unique'):
117
+ if agg_type in ("count", "n_unique"):
107
118
  output_type = datatypes.Float64 # count is always float
108
- elif schema.generic_datatype() == 'numeric':
119
+ elif schema.generic_datatype() == "numeric":
109
120
  output_type = datatypes.Float64
110
- elif schema.generic_datatype() == 'string':
121
+ elif schema.generic_datatype() == "string":
111
122
  output_type = datatypes.Utf8
112
- elif schema.generic_datatype() == 'date':
123
+ elif schema.generic_datatype() == "date":
113
124
  output_type = datatypes.Datetime
114
125
  else:
115
126
  output_type = datatypes.Utf8
116
127
  return output_type
117
128
 
118
129
 
119
- def pre_calculate_pivot_schema(node_input_schema: List[FlowfileColumn],
120
- pivot_input: transform_schema.PivotInput,
121
- output_fields: List[input_schema.MinimalFieldInfo] = None,
122
- input_lf: pl.LazyFrame = None) -> List[FlowfileColumn]:
123
- index_columns_schema = [get_schema_of_column(node_input_schema, index_col) for index_col in
124
- pivot_input.index_columns]
130
+ def pre_calculate_pivot_schema(
131
+ node_input_schema: list[FlowfileColumn],
132
+ pivot_input: transform_schema.PivotInput,
133
+ output_fields: list[input_schema.MinimalFieldInfo] = None,
134
+ input_lf: pl.LazyFrame = None,
135
+ ) -> list[FlowfileColumn]:
136
+ index_columns_schema = [
137
+ get_schema_of_column(node_input_schema, index_col) for index_col in pivot_input.index_columns
138
+ ]
125
139
  val_column_schema = get_schema_of_column(node_input_schema, pivot_input.value_col)
126
140
  if output_fields is not None and len(output_fields) > 0:
127
- return index_columns_schema+[FlowfileColumn(PlType(column_name=output_field.name,
128
- pl_datatype=output_field.data_type)) for output_field in
129
- output_fields]
141
+ return index_columns_schema + [
142
+ FlowfileColumn(PlType(column_name=output_field.name, pl_datatype=output_field.data_type))
143
+ for output_field in output_fields
144
+ ]
130
145
 
131
146
  else:
132
147
  max_unique_vals = 200
133
- unique_vals = fetch_unique_values(input_lf.select(pivot_input.pivot_column)
134
- .unique()
135
- .sort(pivot_input.pivot_column)
136
- .limit(max_unique_vals).cast(pl.String))
148
+ unique_vals = fetch_unique_values(
149
+ input_lf.select(pivot_input.pivot_column)
150
+ .unique()
151
+ .sort(pivot_input.pivot_column)
152
+ .limit(max_unique_vals)
153
+ .cast(pl.String)
154
+ )
137
155
  if len(unique_vals) >= max_unique_vals:
138
- main_logger.warning('Pivot column has too many unique values. Please consider using a different column.'
139
- f' Max unique values: {max_unique_vals}')
156
+ main_logger.warning(
157
+ "Pivot column has too many unique values. Please consider using a different column."
158
+ f" Max unique values: {max_unique_vals}"
159
+ )
140
160
  pl_output_fields = []
141
161
  for val in unique_vals:
142
162
  if len(pivot_input.aggregations) == 1:
@@ -145,5 +165,5 @@ def pre_calculate_pivot_schema(node_input_schema: List[FlowfileColumn],
145
165
  else:
146
166
  for agg in pivot_input.aggregations:
147
167
  output_type = get_output_data_type_pivot(val_column_schema, agg)
148
- pl_output_fields.append(PlType(column_name=f'{val}_{agg}', pl_datatype=output_type))
168
+ pl_output_fields.append(PlType(column_name=f"{val}_{agg}", pl_datatype=output_type))
149
169
  return index_columns_schema + [FlowfileColumn(pl_output_field) for pl_output_field in pl_output_fields]
@@ -1,2 +1 @@
1
-
2
1
  from flowfile_core.flowfile.setting_generator.settings import setting_generator, setting_updator
@@ -1,5 +1,6 @@
1
+ from collections.abc import Callable
2
+
1
3
  from flowfile_core.configs import logger
2
- from typing import Callable
3
4
 
4
5
 
5
6
  class SettingGenerator:
@@ -13,10 +14,10 @@ class SettingGenerator:
13
14
  setattr(self, f.__name__, f)
14
15
 
15
16
  def get_setting_generator(self, node_type: str) -> Callable:
16
- logger.info('getting setting generator for ' + node_type)
17
+ logger.info("getting setting generator for " + node_type)
17
18
 
18
19
  if node_type in self.setting_generator_set:
19
- logger.info('setting generator found')
20
+ logger.info("setting generator found")
20
21
  return getattr(self, node_type)
21
22
  else:
22
23
  return lambda x: x
@@ -33,9 +34,9 @@ class SettingUpdator:
33
34
  setattr(self, f.__name__, f)
34
35
 
35
36
  def get_setting_updator(self, node_type: str) -> Callable:
36
- logger.info('getting setting updator for ' + node_type)
37
+ logger.info("getting setting updator for " + node_type)
37
38
  if node_type in self.setting_updator_set:
38
- logger.info('setting updator found')
39
+ logger.info("setting updator found")
39
40
  return getattr(self, node_type)
40
41
  else:
41
42
  return lambda x: x