Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (346) hide show
  1. build_backends/main.py +25 -22
  2. build_backends/main_prd.py +10 -19
  3. flowfile/__init__.py +194 -74
  4. flowfile/__main__.py +10 -7
  5. flowfile/api.py +51 -57
  6. flowfile/web/__init__.py +14 -9
  7. flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
  8. flowfile/web/static/assets/AdminView-f9847d67.js +713 -0
  9. flowfile/web/static/assets/CloudConnectionView-cf85f943.css +72 -0
  10. flowfile/web/static/assets/{CloudConnectionManager-0dfba9f2.js → CloudConnectionView-faace55b.js} +11 -11
  11. flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
  12. flowfile/web/static/assets/{CloudStorageReader-d5b1b6c9.js → CloudStorageReader-d86ecaa7.js} +10 -8
  13. flowfile/web/static/assets/{CloudStorageWriter-00d87aad.js → CloudStorageWriter-0f4d9a44.js} +10 -8
  14. flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
  15. flowfile/web/static/assets/ColumnActionInput-c44b7aee.css +159 -0
  16. flowfile/web/static/assets/ColumnActionInput-f4189ae0.js +330 -0
  17. flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
  18. flowfile/web/static/assets/{ColumnSelector-4685e75d.js → ColumnSelector-e66b33da.js} +3 -5
  19. flowfile/web/static/assets/ContextMenu-49463352.js +9 -0
  20. flowfile/web/static/assets/ContextMenu-dd5f3f25.js +9 -0
  21. flowfile/web/static/assets/ContextMenu-f709b884.js +9 -0
  22. flowfile/web/static/assets/ContextMenu.vue_vue_type_script_setup_true_lang-a1bd6314.js +59 -0
  23. flowfile/web/static/assets/{CrossJoin-702a3edd.js → CrossJoin-24694b8f.js} +12 -10
  24. flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
  25. flowfile/web/static/assets/{CustomNode-b1519993.js → CustomNode-569d45ff.js} +43 -24
  26. flowfile/web/static/assets/CustomNode-edb9b939.css +42 -0
  27. flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-c20a1e16.css} +23 -21
  28. flowfile/web/static/assets/{DatabaseConnectionSettings-6f3e4ea5.js → DatabaseConnectionSettings-cfc08938.js} +5 -4
  29. flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-5bf8c75b.css} +41 -46
  30. flowfile/web/static/assets/{DatabaseReader-d38c7295.js → DatabaseReader-701feabb.js} +25 -15
  31. flowfile/web/static/assets/{DatabaseManager-cf5ef661.js → DatabaseView-0482e5b5.js} +11 -11
  32. flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
  33. flowfile/web/static/assets/{DatabaseWriter-b04ef46a.js → DatabaseWriter-16721989.js} +17 -10
  34. flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-bdcf2c8b.css} +29 -27
  35. flowfile/web/static/assets/{designer-8da3ba3a.css → DesignerView-49abb835.css} +783 -663
  36. flowfile/web/static/assets/{designer-9633482a.js → DesignerView-f64749fb.js} +1292 -3253
  37. flowfile/web/static/assets/{documentation-ca400224.js → DocumentationView-61bd2990.js} +5 -5
  38. flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-9ea6e871.css} +9 -9
  39. flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
  40. flowfile/web/static/assets/{ExploreData-5fa10ed8.js → ExploreData-e2735b13.js} +18 -9
  41. flowfile/web/static/assets/{ExternalSource-d39af878.js → ExternalSource-2535c3b2.js} +9 -7
  42. flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-7ac7373f.css} +20 -20
  43. flowfile/web/static/assets/Filter-2cdbc93c.js +287 -0
  44. flowfile/web/static/assets/Filter-7494ea97.css +48 -0
  45. flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
  46. flowfile/web/static/assets/{Formula-6b04fb1d.js → Formula-fcda3c2c.js} +13 -11
  47. flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
  48. flowfile/web/static/assets/{FuzzyMatch-999521f4.js → FuzzyMatch-f8d3b7d3.js} +12 -10
  49. flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-4b4d7db9.css} +5 -5
  50. flowfile/web/static/assets/{GraphSolver-17dd2198.js → GraphSolver-72eaa695.js} +14 -12
  51. flowfile/web/static/assets/GroupBy-5792782d.css +9 -0
  52. flowfile/web/static/assets/{GroupBy-6b039e18.js → GroupBy-8aa0598b.js} +9 -7
  53. flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
  54. flowfile/web/static/assets/{Join-24d0f113.js → Join-e40f0ffa.js} +13 -11
  55. flowfile/web/static/assets/LoginView-5111c9ae.js +134 -0
  56. flowfile/web/static/assets/LoginView-d325d632.css +172 -0
  57. flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
  58. flowfile/web/static/assets/{ManualInput-34639209.js → ManualInput-9b6f3224.js} +170 -116
  59. flowfile/web/static/assets/{MultiSelect-0e8724a3.js → MultiSelect-ef28e19e.js} +2 -2
  60. flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-b0e538c2.js → MultiSelect.vue_vue_type_script_setup_true_lang-83b3bbfd.js} +1 -1
  61. flowfile/web/static/assets/NodeDesigner-94cd4dd3.css +1429 -0
  62. flowfile/web/static/assets/NodeDesigner-d2b7ee2b.js +2712 -0
  63. flowfile/web/static/assets/{NumericInput-3d63a470.js → NumericInput-1d789794.js} +2 -2
  64. flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-e0edeccc.js → NumericInput.vue_vue_type_script_setup_true_lang-7775f83e.js} +5 -2
  65. flowfile/web/static/assets/Output-692dd25d.css +37 -0
  66. flowfile/web/static/assets/{Output-edea9802.js → Output-cefef801.js} +14 -10
  67. flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
  68. flowfile/web/static/assets/{Pivot-61d19301.js → Pivot-bab1b75b.js} +12 -10
  69. flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
  70. flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
  71. flowfile/web/static/assets/{PivotValidation-f97fec5b.js → PivotValidation-e7941f91.js} +3 -3
  72. flowfile/web/static/assets/{PivotValidation-de9f43fe.js → PivotValidation-fba09336.js} +3 -3
  73. flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
  74. flowfile/web/static/assets/{PolarsCode-bc3c9984.js → PolarsCode-740e40fa.js} +18 -9
  75. flowfile/web/static/assets/PopOver-862d7e28.js +939 -0
  76. flowfile/web/static/assets/PopOver-d96599db.css +33 -0
  77. flowfile/web/static/assets/{Read-64a3f259.js → Read-225cc63f.js} +16 -12
  78. flowfile/web/static/assets/{Read-e808b239.css → Read-90f366bc.css} +15 -15
  79. flowfile/web/static/assets/{RecordCount-3d5039be.js → RecordCount-ffc71eca.js} +6 -4
  80. flowfile/web/static/assets/{RecordId-597510e0.js → RecordId-a70bb8df.js} +9 -7
  81. flowfile/web/static/assets/{SQLQueryComponent-df51adbe.js → SQLQueryComponent-15a421f5.js} +3 -3
  82. flowfile/web/static/assets/SQLQueryComponent-edb90b98.css +29 -0
  83. flowfile/web/static/assets/{Sample-4be0a507.js → Sample-6c26afc7.js} +6 -4
  84. flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
  85. flowfile/web/static/assets/SecretSelector-ceed9496.js +113 -0
  86. flowfile/web/static/assets/{SecretManager-4839be57.js → SecretsView-214d255a.js} +35 -36
  87. flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
  88. flowfile/web/static/assets/{Select-9b72f201.js → Select-8fc29999.js} +9 -7
  89. flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
  90. flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
  91. flowfile/web/static/assets/{SettingsSection-7ded385d.js → SettingsSection-3f70e4c3.js} +3 -3
  92. flowfile/web/static/assets/{SettingsSection-f0f75a42.js → SettingsSection-83090218.js} +3 -3
  93. flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
  94. flowfile/web/static/assets/{SettingsSection-e1e9c953.js → SettingsSection-9f0d1725.js} +3 -3
  95. flowfile/web/static/assets/SetupView-3fa0aa03.js +160 -0
  96. flowfile/web/static/assets/SetupView-e2da3442.css +230 -0
  97. flowfile/web/static/assets/{SingleSelect-6c777aac.js → SingleSelect-a4a568cb.js} +2 -2
  98. flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-33e3ff9b.js → SingleSelect.vue_vue_type_script_setup_true_lang-c8ebdd33.js} +1 -1
  99. flowfile/web/static/assets/{SliderInput-7cb93e62.js → SliderInput-be533e71.js} +7 -4
  100. flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
  101. flowfile/web/static/assets/{Sort-6cbde21a.js → Sort-154dad81.js} +9 -7
  102. flowfile/web/static/assets/Sort-4abb7fae.css +9 -0
  103. flowfile/web/static/assets/{TextInput-d9a40c11.js → TextInput-454e2bda.js} +2 -2
  104. flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-5896c375.js → TextInput.vue_vue_type_script_setup_true_lang-e86510d0.js} +5 -2
  105. flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
  106. flowfile/web/static/assets/{TextToRows-c4fcbf4d.js → TextToRows-ea73433d.js} +11 -10
  107. flowfile/web/static/assets/{ToggleSwitch-4ef91d19.js → ToggleSwitch-9d7b30f1.js} +2 -2
  108. flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-38478c20.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-00f2580e.js} +1 -1
  109. flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-394a1f78.css} +14 -14
  110. flowfile/web/static/assets/{UnavailableFields-a03f512c.js → UnavailableFields-b72a2c72.js} +4 -4
  111. flowfile/web/static/assets/{Union-bfe9b996.js → Union-1e44f263.js} +8 -6
  112. flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
  113. flowfile/web/static/assets/Unique-2b705521.css +3 -0
  114. flowfile/web/static/assets/{Unique-5d023a27.js → Unique-a3bc6d0a.js} +13 -10
  115. flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-b6ad6427.css} +7 -7
  116. flowfile/web/static/assets/{Unpivot-91cc5354.js → Unpivot-e27935fc.js} +11 -9
  117. flowfile/web/static/assets/{UnpivotValidation-7ee2de44.js → UnpivotValidation-72497680.js} +3 -3
  118. flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
  119. flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
  120. flowfile/web/static/assets/{VueGraphicWalker-e51b9924.js → VueGraphicWalker-d9ab70a3.js} +4 -4
  121. flowfile/web/static/assets/{api-cf1221f0.js → api-a2102880.js} +1 -1
  122. flowfile/web/static/assets/{api-c1bad5ca.js → api-f75042b0.js} +1 -1
  123. flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-1d6acbd9.css} +41 -41
  124. flowfile/web/static/assets/{dropDown-614b998d.js → dropDown-2798a109.js} +3 -3
  125. flowfile/web/static/assets/{fullEditor-f7971590.js → fullEditor-cf7d7d93.js} +11 -10
  126. flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-fe9f7e18.css} +77 -65
  127. flowfile/web/static/assets/{genericNodeSettings-4fe5f36b.js → genericNodeSettings-14eac1c3.js} +5 -5
  128. flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
  129. flowfile/web/static/assets/{index-5429bbf8.js → index-387a6f18.js} +41806 -40958
  130. flowfile/web/static/assets/index-6b367bb5.js +38 -0
  131. flowfile/web/static/assets/{index-50508d4d.css → index-e96ab018.css} +2184 -569
  132. flowfile/web/static/assets/index-f0a6e5a5.js +2696 -0
  133. flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
  134. flowfile/web/static/assets/nodeInput-ed2ae8d7.js +2 -0
  135. flowfile/web/static/assets/{outputCsv-076b85ab.js → outputCsv-3c1757e8.js} +3 -3
  136. flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
  137. flowfile/web/static/assets/{outputExcel-0fd17dbe.js → outputExcel-686e1f48.js} +3 -3
  138. flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
  139. flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
  140. flowfile/web/static/assets/{outputParquet-b61e0847.js → outputParquet-df28faa7.js} +4 -4
  141. flowfile/web/static/assets/{readCsv-c767cb37.css → readCsv-3bfac4c3.css} +15 -15
  142. flowfile/web/static/assets/{readCsv-a8bb8b61.js → readCsv-e37eee21.js} +3 -3
  143. flowfile/web/static/assets/{readExcel-806d2826.css → readExcel-3db6b763.css} +13 -13
  144. flowfile/web/static/assets/{readExcel-67b4aee0.js → readExcel-a13f14bb.js} +5 -5
  145. flowfile/web/static/assets/{readParquet-92ce1dbc.js → readParquet-344cf746.js} +3 -3
  146. flowfile/web/static/assets/{readParquet-48c81530.css → readParquet-c5244ad5.css} +4 -4
  147. flowfile/web/static/assets/secrets.api-ae198c5c.js +65 -0
  148. flowfile/web/static/assets/{selectDynamic-92e25ee3.js → selectDynamic-6b4b0767.js} +5 -5
  149. flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
  150. flowfile/web/static/assets/{vue-codemirror.esm-41b0e0d7.js → vue-codemirror.esm-31ba0e0b.js} +31 -640
  151. flowfile/web/static/assets/{vue-content-loader.es-2c8e608f.js → vue-content-loader.es-4469c8ff.js} +1 -1
  152. flowfile/web/static/index.html +2 -2
  153. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/METADATA +3 -4
  154. flowfile-0.5.4.dist-info/RECORD +407 -0
  155. flowfile_core/__init__.py +13 -6
  156. flowfile_core/auth/jwt.py +51 -16
  157. flowfile_core/auth/models.py +32 -7
  158. flowfile_core/auth/password.py +89 -0
  159. flowfile_core/auth/secrets.py +64 -19
  160. flowfile_core/configs/__init__.py +9 -7
  161. flowfile_core/configs/flow_logger.py +15 -14
  162. flowfile_core/configs/node_store/__init__.py +72 -4
  163. flowfile_core/configs/node_store/nodes.py +155 -172
  164. flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
  165. flowfile_core/configs/settings.py +28 -15
  166. flowfile_core/database/connection.py +7 -6
  167. flowfile_core/database/init_db.py +96 -2
  168. flowfile_core/database/models.py +3 -1
  169. flowfile_core/fileExplorer/__init__.py +17 -0
  170. flowfile_core/fileExplorer/funcs.py +145 -57
  171. flowfile_core/fileExplorer/utils.py +10 -11
  172. flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
  173. flowfile_core/flowfile/analytics/analytics_processor.py +26 -24
  174. flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
  175. flowfile_core/flowfile/analytics/utils.py +1 -1
  176. flowfile_core/flowfile/code_generator/__init__.py +11 -0
  177. flowfile_core/flowfile/code_generator/code_generator.py +706 -247
  178. flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
  179. flowfile_core/flowfile/connection_manager/models.py +1 -1
  180. flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
  181. flowfile_core/flowfile/database_connection_manager/models.py +1 -1
  182. flowfile_core/flowfile/extensions.py +17 -12
  183. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
  184. flowfile_core/flowfile/flow_data_engine/create/funcs.py +115 -83
  185. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +493 -423
  186. flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
  187. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
  188. flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
  189. flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
  190. flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
  191. flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +31 -20
  192. flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
  193. flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
  194. flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +14 -15
  195. flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
  196. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
  197. flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
  198. flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
  199. flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
  200. flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
  201. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +190 -127
  202. flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
  203. flowfile_core/flowfile/flow_data_engine/utils.py +99 -67
  204. flowfile_core/flowfile/flow_graph.py +920 -571
  205. flowfile_core/flowfile/flow_graph_utils.py +31 -49
  206. flowfile_core/flowfile/flow_node/flow_node.py +379 -258
  207. flowfile_core/flowfile/flow_node/models.py +53 -41
  208. flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
  209. flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
  210. flowfile_core/flowfile/handler.py +80 -30
  211. flowfile_core/flowfile/manage/compatibility_enhancements.py +209 -126
  212. flowfile_core/flowfile/manage/io_flowfile.py +54 -57
  213. flowfile_core/flowfile/node_designer/__init__.py +19 -13
  214. flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
  215. flowfile_core/flowfile/node_designer/custom_node.py +162 -36
  216. flowfile_core/flowfile/node_designer/ui_components.py +278 -34
  217. flowfile_core/flowfile/schema_callbacks.py +71 -51
  218. flowfile_core/flowfile/setting_generator/__init__.py +0 -1
  219. flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
  220. flowfile_core/flowfile/setting_generator/settings.py +64 -53
  221. flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
  222. flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
  223. flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
  224. flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
  225. flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
  226. flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
  227. flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
  228. flowfile_core/flowfile/util/calculate_layout.py +9 -13
  229. flowfile_core/flowfile/util/execution_orderer.py +25 -17
  230. flowfile_core/flowfile/util/node_skipper.py +4 -4
  231. flowfile_core/flowfile/utils.py +19 -21
  232. flowfile_core/main.py +26 -19
  233. flowfile_core/routes/auth.py +284 -11
  234. flowfile_core/routes/cloud_connections.py +25 -25
  235. flowfile_core/routes/logs.py +21 -29
  236. flowfile_core/routes/public.py +46 -4
  237. flowfile_core/routes/routes.py +70 -34
  238. flowfile_core/routes/secrets.py +25 -27
  239. flowfile_core/routes/user_defined_components.py +483 -4
  240. flowfile_core/run_lock.py +0 -1
  241. flowfile_core/schemas/__init__.py +4 -6
  242. flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
  243. flowfile_core/schemas/cloud_storage_schemas.py +96 -66
  244. flowfile_core/schemas/input_schema.py +231 -144
  245. flowfile_core/schemas/output_model.py +49 -34
  246. flowfile_core/schemas/schemas.py +116 -89
  247. flowfile_core/schemas/transform_schema.py +518 -263
  248. flowfile_core/schemas/yaml_types.py +21 -7
  249. flowfile_core/secret_manager/secret_manager.py +123 -18
  250. flowfile_core/types.py +29 -9
  251. flowfile_core/utils/arrow_reader.py +7 -6
  252. flowfile_core/utils/excel_file_manager.py +3 -3
  253. flowfile_core/utils/fileManager.py +7 -7
  254. flowfile_core/utils/fl_executor.py +8 -10
  255. flowfile_core/utils/utils.py +4 -4
  256. flowfile_core/utils/validate_setup.py +5 -4
  257. flowfile_frame/__init__.py +117 -51
  258. flowfile_frame/adapters.py +2 -9
  259. flowfile_frame/adding_expr.py +73 -32
  260. flowfile_frame/cloud_storage/frame_helpers.py +27 -23
  261. flowfile_frame/cloud_storage/secret_manager.py +12 -26
  262. flowfile_frame/config.py +2 -5
  263. flowfile_frame/database/__init__.py +36 -0
  264. flowfile_frame/database/connection_manager.py +205 -0
  265. flowfile_frame/database/frame_helpers.py +249 -0
  266. flowfile_frame/expr.py +311 -218
  267. flowfile_frame/expr.pyi +160 -159
  268. flowfile_frame/expr_name.py +23 -23
  269. flowfile_frame/flow_frame.py +571 -476
  270. flowfile_frame/flow_frame.pyi +123 -104
  271. flowfile_frame/flow_frame_methods.py +227 -246
  272. flowfile_frame/group_frame.py +50 -20
  273. flowfile_frame/join.py +2 -2
  274. flowfile_frame/lazy.py +129 -87
  275. flowfile_frame/lazy_methods.py +83 -30
  276. flowfile_frame/list_name_space.py +55 -50
  277. flowfile_frame/selectors.py +148 -68
  278. flowfile_frame/series.py +9 -7
  279. flowfile_frame/utils.py +19 -21
  280. flowfile_worker/__init__.py +12 -7
  281. flowfile_worker/configs.py +41 -33
  282. flowfile_worker/create/__init__.py +14 -9
  283. flowfile_worker/create/funcs.py +114 -77
  284. flowfile_worker/create/models.py +46 -43
  285. flowfile_worker/create/pl_types.py +14 -15
  286. flowfile_worker/create/read_excel_tables.py +34 -41
  287. flowfile_worker/create/utils.py +22 -19
  288. flowfile_worker/external_sources/s3_source/main.py +18 -51
  289. flowfile_worker/external_sources/s3_source/models.py +34 -27
  290. flowfile_worker/external_sources/sql_source/main.py +8 -5
  291. flowfile_worker/external_sources/sql_source/models.py +13 -9
  292. flowfile_worker/flow_logger.py +10 -8
  293. flowfile_worker/funcs.py +214 -155
  294. flowfile_worker/main.py +11 -17
  295. flowfile_worker/models.py +35 -28
  296. flowfile_worker/process_manager.py +2 -3
  297. flowfile_worker/routes.py +121 -90
  298. flowfile_worker/secrets.py +114 -21
  299. flowfile_worker/spawner.py +89 -54
  300. flowfile_worker/utils.py +3 -2
  301. shared/__init__.py +2 -7
  302. shared/storage_config.py +25 -13
  303. test_utils/postgres/commands.py +3 -2
  304. test_utils/postgres/fixtures.py +9 -9
  305. test_utils/s3/commands.py +1 -1
  306. test_utils/s3/data_generator.py +3 -4
  307. test_utils/s3/demo_data_generator.py +4 -7
  308. test_utils/s3/fixtures.py +7 -5
  309. tools/migrate/__init__.py +1 -1
  310. tools/migrate/__main__.py +16 -29
  311. tools/migrate/legacy_schemas.py +251 -190
  312. tools/migrate/migrate.py +193 -181
  313. tools/migrate/tests/conftest.py +1 -3
  314. tools/migrate/tests/test_migrate.py +36 -41
  315. tools/migrate/tests/test_migration_e2e.py +28 -29
  316. tools/migrate/tests/test_node_migrations.py +50 -20
  317. flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
  318. flowfile/web/static/assets/ContextMenu-23e909da.js +0 -41
  319. flowfile/web/static/assets/ContextMenu-4c74eef1.css +0 -26
  320. flowfile/web/static/assets/ContextMenu-63cfa99b.css +0 -26
  321. flowfile/web/static/assets/ContextMenu-70ae0c79.js +0 -41
  322. flowfile/web/static/assets/ContextMenu-c13f91d0.css +0 -26
  323. flowfile/web/static/assets/ContextMenu-f149cf7c.js +0 -41
  324. flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
  325. flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
  326. flowfile/web/static/assets/Filter-9b6d08db.js +0 -164
  327. flowfile/web/static/assets/Filter-f62091b3.css +0 -20
  328. flowfile/web/static/assets/GroupBy-b9505323.css +0 -51
  329. flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
  330. flowfile/web/static/assets/Output-283fe388.css +0 -37
  331. flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
  332. flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
  333. flowfile/web/static/assets/SQLQueryComponent-36cef432.css +0 -27
  334. flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
  335. flowfile/web/static/assets/Sort-3643d625.css +0 -51
  336. flowfile/web/static/assets/Unique-f9fb0809.css +0 -51
  337. flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
  338. flowfile/web/static/assets/nodeInput-5d0d6b79.js +0 -41
  339. flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
  340. flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
  341. flowfile/web/static/assets/secretApi-68435402.js +0 -46
  342. flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
  343. flowfile-0.5.1.dist-info/RECORD +0 -388
  344. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/WHEEL +0 -0
  345. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/entry_points.txt +0 -0
  346. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/licenses/LICENSE +0 -0
@@ -1,7 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
- import inspect
4
- from typing import TYPE_CHECKING, Callable, Optional
3
+ from collections.abc import Callable
4
+ from typing import TYPE_CHECKING
5
5
 
6
6
  if TYPE_CHECKING:
7
7
  from flowfile_frame.expr import Expr
@@ -10,7 +10,7 @@ if TYPE_CHECKING:
10
10
  class ExprNameNameSpace:
11
11
  """Namespace for expression name operations in FlowFrame."""
12
12
 
13
- def __init__(self, parent_expr: 'Expr', parent_repr_str: str):
13
+ def __init__(self, parent_expr: Expr, parent_repr_str: str):
14
14
  """
15
15
  Initialize the namespace with parent expression reference.
16
16
 
@@ -25,11 +25,11 @@ class ExprNameNameSpace:
25
25
  self.expr = parent_expr.expr.name if parent_expr.expr is not None else None
26
26
  self.parent_repr_str = parent_repr_str
27
27
 
28
- def _create_next_expr(self, method_name: str, *args, result_expr: Optional[any] = None, **kwargs) -> 'Expr':
28
+ def _create_next_expr(self, method_name: str, *args, result_expr: any | None = None, **kwargs) -> Expr:
29
29
  """Create a new expression with name operation applied."""
30
30
  from flowfile_frame.expr import Expr
31
31
 
32
- args_repr = ''
32
+ args_repr = ""
33
33
  # Format positional args for repr
34
34
  if args:
35
35
  args_strs = []
@@ -44,20 +44,21 @@ class ExprNameNameSpace:
44
44
  if method_name == "map":
45
45
  # For map function specifically, try to extract the function body
46
46
  import inspect
47
+
47
48
  try:
48
49
  source = inspect.getsource(arg).strip()
49
50
  # Handle multiline lambdas and functions
50
- if '\n' in source:
51
+ if "\n" in source:
51
52
  # Try to extract just the lambda expression if possible
52
- if 'lambda' in source:
53
- lambda_parts = source.split('lambda')
53
+ if "lambda" in source:
54
+ lambda_parts = source.split("lambda")
54
55
  if len(lambda_parts) > 1:
55
56
  lambda_expr = f"lambda{lambda_parts[1].split(':')[0]}:"
56
- body_parts = source.split(':')
57
+ body_parts = source.split(":")
57
58
  if len(body_parts) > 1:
58
59
  body = body_parts[1].strip()
59
60
  # Remove trailing characters like parentheses, commas
60
- body = body.rstrip(')\n, ')
61
+ body = body.rstrip(")\n, ")
61
62
  lambda_expr += f" {body}"
62
63
  args_strs.append(lambda_expr)
63
64
  else:
@@ -69,8 +70,8 @@ class ExprNameNameSpace:
69
70
  args_strs.append("lambda x: x.upper()") # Simplified fallback
70
71
  else:
71
72
  # Single line function, extract it directly
72
- if 'lambda' in source:
73
- lambda_expr = source.split('=')[-1].strip()
73
+ if "lambda" in source:
74
+ lambda_expr = source.split("=")[-1].strip()
74
75
  args_strs.append(lambda_expr)
75
76
  else:
76
77
  args_strs.append("lambda x: x.upper()") # Fallback
@@ -102,11 +103,11 @@ class ExprNameNameSpace:
102
103
  initial_column_name=self.parent._initial_column_name,
103
104
  selector=None,
104
105
  agg_func=self.parent.agg_func,
105
- is_complex=True
106
+ is_complex=True,
106
107
  )
107
108
  return new_expr
108
109
 
109
- def keep(self) -> 'Expr':
110
+ def keep(self) -> Expr:
110
111
  """
111
112
  Keep the original root name of the expression.
112
113
 
@@ -120,7 +121,7 @@ class ExprNameNameSpace:
120
121
  result_expr = self.expr.keep() if self.expr is not None else None
121
122
  return self._create_next_expr("keep", result_expr=result_expr)
122
123
 
123
- def map(self, function: Callable[[str], str]) -> 'Expr':
124
+ def map(self, function: Callable[[str], str]) -> Expr:
124
125
  """
125
126
  Rename the output of an expression by mapping a function over the root name.
126
127
 
@@ -147,7 +148,7 @@ class ExprNameNameSpace:
147
148
  # The representation is handled by _create_next_expr with special lambda handling
148
149
  return self._create_next_expr("map", function, result_expr=result_expr)
149
150
 
150
- def prefix(self, prefix: str) -> 'Expr':
151
+ def prefix(self, prefix: str) -> Expr:
151
152
  """
152
153
  Add a prefix to the root column name of the expression.
153
154
 
@@ -164,7 +165,7 @@ class ExprNameNameSpace:
164
165
  result_expr = self.expr.prefix(prefix) if self.expr is not None else None
165
166
  return self._create_next_expr("prefix", prefix, result_expr=result_expr)
166
167
 
167
- def suffix(self, suffix: str) -> 'Expr':
168
+ def suffix(self, suffix: str) -> Expr:
168
169
  """
169
170
  Add a suffix to the root column name of the expression.
170
171
 
@@ -181,7 +182,7 @@ class ExprNameNameSpace:
181
182
  result_expr = self.expr.suffix(suffix) if self.expr is not None else None
182
183
  return self._create_next_expr("suffix", suffix, result_expr=result_expr)
183
184
 
184
- def to_lowercase(self) -> 'Expr':
185
+ def to_lowercase(self) -> Expr:
185
186
  """
186
187
  Make the root column name lowercase.
187
188
 
@@ -193,7 +194,7 @@ class ExprNameNameSpace:
193
194
  result_expr = self.expr.to_lowercase() if self.expr is not None else None
194
195
  return self._create_next_expr("to_lowercase", result_expr=result_expr)
195
196
 
196
- def to_uppercase(self) -> 'Expr':
197
+ def to_uppercase(self) -> Expr:
197
198
  """
198
199
  Make the root column name uppercase.
199
200
 
@@ -205,7 +206,7 @@ class ExprNameNameSpace:
205
206
  result_expr = self.expr.to_uppercase() if self.expr is not None else None
206
207
  return self._create_next_expr("to_uppercase", result_expr=result_expr)
207
208
 
208
- def map_fields(self, function: Callable[[str], str]) -> 'Expr':
209
+ def map_fields(self, function: Callable[[str], str]) -> Expr:
209
210
  """
210
211
  Rename fields of a struct by mapping a function over the field name(s).
211
212
 
@@ -222,7 +223,7 @@ class ExprNameNameSpace:
222
223
  result_expr = self.expr.map_fields(function) if self.expr is not None else None
223
224
  return self._create_next_expr("map_fields", function, result_expr=result_expr)
224
225
 
225
- def prefix_fields(self, prefix: str) -> 'Expr':
226
+ def prefix_fields(self, prefix: str) -> Expr:
226
227
  """
227
228
  Add a prefix to all field names of a struct.
228
229
 
@@ -239,7 +240,7 @@ class ExprNameNameSpace:
239
240
  result_expr = self.expr.prefix_fields(prefix) if self.expr is not None else None
240
241
  return self._create_next_expr("prefix_fields", prefix, result_expr=result_expr)
241
242
 
242
- def suffix_fields(self, suffix: str) -> 'Expr':
243
+ def suffix_fields(self, suffix: str) -> Expr:
243
244
  """
244
245
  Add a suffix to all field names of a struct.
245
246
 
@@ -255,4 +256,3 @@ class ExprNameNameSpace:
255
256
  """
256
257
  result_expr = self.expr.suffix_fields(suffix) if self.expr is not None else None
257
258
  return self._create_next_expr("suffix_fields", suffix, result_expr=result_expr)
258
-