Flowfile 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (329) hide show
  1. build_backends/main.py +25 -22
  2. build_backends/main_prd.py +10 -19
  3. flowfile/__init__.py +178 -74
  4. flowfile/__main__.py +10 -7
  5. flowfile/api.py +51 -57
  6. flowfile/web/__init__.py +14 -9
  7. flowfile/web/static/assets/AdminView-49392a9a.js +713 -0
  8. flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
  9. flowfile/web/static/assets/CloudConnectionView-36bcd6df.css +72 -0
  10. flowfile/web/static/assets/{CloudConnectionManager-0dfba9f2.js → CloudConnectionView-f13f202b.js} +11 -11
  11. flowfile/web/static/assets/{CloudStorageReader-d5b1b6c9.js → CloudStorageReader-0023d4a5.js} +10 -8
  12. flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
  13. flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
  14. flowfile/web/static/assets/{CloudStorageWriter-00d87aad.js → CloudStorageWriter-8e781e11.js} +10 -8
  15. flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
  16. flowfile/web/static/assets/{ColumnSelector-4685e75d.js → ColumnSelector-8ad68ea9.js} +3 -5
  17. flowfile/web/static/assets/{ContextMenu-c13f91d0.css → ContextMenu-26d4dd27.css} +6 -6
  18. flowfile/web/static/assets/{ContextMenu-23e909da.js → ContextMenu-31ee57f0.js} +3 -3
  19. flowfile/web/static/assets/{ContextMenu-70ae0c79.js → ContextMenu-69a74055.js} +3 -3
  20. flowfile/web/static/assets/{ContextMenu-f149cf7c.js → ContextMenu-8e2051c6.js} +3 -3
  21. flowfile/web/static/assets/{ContextMenu-4c74eef1.css → ContextMenu-8ec1729e.css} +6 -6
  22. flowfile/web/static/assets/{ContextMenu-63cfa99b.css → ContextMenu-9b310c60.css} +6 -6
  23. flowfile/web/static/assets/{CrossJoin-702a3edd.js → CrossJoin-03df6938.js} +12 -10
  24. flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
  25. flowfile/web/static/assets/CustomNode-59e99a86.css +32 -0
  26. flowfile/web/static/assets/{CustomNode-b1519993.js → CustomNode-8479239b.js} +36 -24
  27. flowfile/web/static/assets/{DatabaseConnectionSettings-6f3e4ea5.js → DatabaseConnectionSettings-869e3efd.js} +5 -4
  28. flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-e91df89a.css} +13 -13
  29. flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-36898a00.css} +24 -24
  30. flowfile/web/static/assets/{DatabaseReader-d38c7295.js → DatabaseReader-c58b9552.js} +25 -15
  31. flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
  32. flowfile/web/static/assets/{DatabaseManager-cf5ef661.js → DatabaseView-d26a9140.js} +11 -11
  33. flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-217a99f1.css} +19 -19
  34. flowfile/web/static/assets/{DatabaseWriter-b04ef46a.js → DatabaseWriter-4d05ddc7.js} +17 -10
  35. flowfile/web/static/assets/{designer-8da3ba3a.css → DesignerView-a6d0ee84.css} +614 -546
  36. flowfile/web/static/assets/{designer-9633482a.js → DesignerView-e6f5c0e8.js} +1107 -3170
  37. flowfile/web/static/assets/{documentation-ca400224.js → DocumentationView-2e78ef1b.js} +5 -5
  38. flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-fd46c656.css} +7 -7
  39. flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
  40. flowfile/web/static/assets/{ExploreData-5fa10ed8.js → ExploreData-7b54caca.js} +18 -9
  41. flowfile/web/static/assets/{ExternalSource-d39af878.js → ExternalSource-3fa399b2.js} +9 -7
  42. flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-47ab05a3.css} +17 -17
  43. flowfile/web/static/assets/Filter-7494ea97.css +48 -0
  44. flowfile/web/static/assets/Filter-8cbbdbf3.js +287 -0
  45. flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
  46. flowfile/web/static/assets/{Formula-6b04fb1d.js → Formula-aac42b1e.js} +13 -11
  47. flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
  48. flowfile/web/static/assets/{FuzzyMatch-999521f4.js → FuzzyMatch-cd9bbfca.js} +12 -10
  49. flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-c24dec17.css} +5 -5
  50. flowfile/web/static/assets/{GraphSolver-17dd2198.js → GraphSolver-c7e6780e.js} +13 -11
  51. flowfile/web/static/assets/{GroupBy-6b039e18.js → GroupBy-93c5d22b.js} +9 -7
  52. flowfile/web/static/assets/{GroupBy-b9505323.css → GroupBy-be7ac0bf.css} +10 -10
  53. flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
  54. flowfile/web/static/assets/{Join-24d0f113.js → Join-a19b2de2.js} +13 -11
  55. flowfile/web/static/assets/LoginView-0df4ed0a.js +134 -0
  56. flowfile/web/static/assets/LoginView-d325d632.css +172 -0
  57. flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
  58. flowfile/web/static/assets/{ManualInput-34639209.js → ManualInput-8d3374b2.js} +170 -116
  59. flowfile/web/static/assets/{MultiSelect-0e8724a3.js → MultiSelect-ad1b6243.js} +2 -2
  60. flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-b0e538c2.js → MultiSelect.vue_vue_type_script_setup_true_lang-e278950d.js} +1 -1
  61. flowfile/web/static/assets/NodeDesigner-40b647c9.js +2610 -0
  62. flowfile/web/static/assets/NodeDesigner-5f53be3f.css +1429 -0
  63. flowfile/web/static/assets/{NumericInput-3d63a470.js → NumericInput-7100234c.js} +2 -2
  64. flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-e0edeccc.js → NumericInput.vue_vue_type_script_setup_true_lang-5130219f.js} +5 -2
  65. flowfile/web/static/assets/{Output-283fe388.css → Output-35e97000.css} +6 -6
  66. flowfile/web/static/assets/{Output-edea9802.js → Output-f5efd2aa.js} +12 -9
  67. flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
  68. flowfile/web/static/assets/{Pivot-61d19301.js → Pivot-d981d23c.js} +11 -9
  69. flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
  70. flowfile/web/static/assets/{PivotValidation-f97fec5b.js → PivotValidation-39386e95.js} +3 -3
  71. flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
  72. flowfile/web/static/assets/{PivotValidation-de9f43fe.js → PivotValidation-63de1f73.js} +3 -3
  73. flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
  74. flowfile/web/static/assets/{PolarsCode-bc3c9984.js → PolarsCode-f9d69217.js} +18 -9
  75. flowfile/web/static/assets/PopOver-b22f049e.js +939 -0
  76. flowfile/web/static/assets/PopOver-d96599db.css +33 -0
  77. flowfile/web/static/assets/{Read-e808b239.css → Read-36e7bd51.css} +12 -12
  78. flowfile/web/static/assets/{Read-64a3f259.js → Read-aec2e377.js} +14 -11
  79. flowfile/web/static/assets/{RecordCount-3d5039be.js → RecordCount-78ed6845.js} +6 -4
  80. flowfile/web/static/assets/{RecordId-597510e0.js → RecordId-2156e890.js} +8 -6
  81. flowfile/web/static/assets/{SQLQueryComponent-36cef432.css → SQLQueryComponent-1c2f26b4.css} +5 -5
  82. flowfile/web/static/assets/{SQLQueryComponent-df51adbe.js → SQLQueryComponent-48c72f5b.js} +3 -3
  83. flowfile/web/static/assets/{Sample-4be0a507.js → Sample-1352ca74.js} +6 -4
  84. flowfile/web/static/assets/SecretSelector-22b5ff89.js +113 -0
  85. flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
  86. flowfile/web/static/assets/{SecretManager-4839be57.js → SecretsView-17df66ee.js} +35 -36
  87. flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
  88. flowfile/web/static/assets/{Select-9b72f201.js → Select-0aee4c54.js} +9 -7
  89. flowfile/web/static/assets/{SettingsSection-f0f75a42.js → SettingsSection-0784e157.js} +3 -3
  90. flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
  91. flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
  92. flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
  93. flowfile/web/static/assets/{SettingsSection-e1e9c953.js → SettingsSection-cd341bb6.js} +3 -3
  94. flowfile/web/static/assets/{SettingsSection-7ded385d.js → SettingsSection-f2002a6d.js} +3 -3
  95. flowfile/web/static/assets/{SingleSelect-6c777aac.js → SingleSelect-460cc0ea.js} +2 -2
  96. flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-33e3ff9b.js → SingleSelect.vue_vue_type_script_setup_true_lang-30741bb2.js} +1 -1
  97. flowfile/web/static/assets/{SliderInput-7cb93e62.js → SliderInput-5d926864.js} +7 -4
  98. flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
  99. flowfile/web/static/assets/{Sort-6cbde21a.js → Sort-3cdc971b.js} +9 -7
  100. flowfile/web/static/assets/{Unique-f9fb0809.css → Sort-8a871341.css} +10 -10
  101. flowfile/web/static/assets/{TextInput-d9a40c11.js → TextInput-a2d0bfbd.js} +2 -2
  102. flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-5896c375.js → TextInput.vue_vue_type_script_setup_true_lang-abad1ca2.js} +5 -2
  103. flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
  104. flowfile/web/static/assets/{TextToRows-c4fcbf4d.js → TextToRows-918945f7.js} +11 -10
  105. flowfile/web/static/assets/{ToggleSwitch-4ef91d19.js → ToggleSwitch-f0ef5196.js} +2 -2
  106. flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-38478c20.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-5605c793.js} +1 -1
  107. flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-54d2f518.css} +6 -6
  108. flowfile/web/static/assets/{UnavailableFields-a03f512c.js → UnavailableFields-bdad6144.js} +4 -4
  109. flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
  110. flowfile/web/static/assets/{Union-bfe9b996.js → Union-e8ab8c86.js} +8 -6
  111. flowfile/web/static/assets/{Unique-5d023a27.js → Unique-8cd4f976.js} +13 -10
  112. flowfile/web/static/assets/{Sort-3643d625.css → Unique-9fb2f567.css} +10 -10
  113. flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-710a2948.css} +7 -7
  114. flowfile/web/static/assets/{Unpivot-91cc5354.js → Unpivot-8da14095.js} +10 -8
  115. flowfile/web/static/assets/{UnpivotValidation-7ee2de44.js → UnpivotValidation-6f7d89ff.js} +3 -3
  116. flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
  117. flowfile/web/static/assets/{VueGraphicWalker-e51b9924.js → VueGraphicWalker-3fb312e1.js} +4 -4
  118. flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
  119. flowfile/web/static/assets/{api-cf1221f0.js → api-24483f0d.js} +1 -1
  120. flowfile/web/static/assets/{api-c1bad5ca.js → api-8b81fa73.js} +1 -1
  121. flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-3d8dc5fa.css} +40 -40
  122. flowfile/web/static/assets/{dropDown-614b998d.js → dropDown-ac0fda9d.js} +3 -3
  123. flowfile/web/static/assets/{fullEditor-f7971590.js → fullEditor-5497a84a.js} +11 -10
  124. flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-a0be62b3.css} +74 -62
  125. flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
  126. flowfile/web/static/assets/{genericNodeSettings-4fe5f36b.js → genericNodeSettings-99014e1d.js} +5 -5
  127. flowfile/web/static/assets/index-07dda503.js +38 -0
  128. flowfile/web/static/assets/index-3ba44389.js +2696 -0
  129. flowfile/web/static/assets/{index-50508d4d.css → index-e6289dd0.css} +1945 -569
  130. flowfile/web/static/assets/{index-5429bbf8.js → index-fb6493ae.js} +41626 -40867
  131. flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
  132. flowfile/web/static/assets/nodeInput-0eb13f1a.js +2 -0
  133. flowfile/web/static/assets/{outputCsv-076b85ab.js → outputCsv-8f8ba42d.js} +3 -3
  134. flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
  135. flowfile/web/static/assets/{outputExcel-0fd17dbe.js → outputExcel-393f4fef.js} +3 -3
  136. flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
  137. flowfile/web/static/assets/{outputParquet-b61e0847.js → outputParquet-07c81f65.js} +4 -4
  138. flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
  139. flowfile/web/static/assets/{readCsv-a8bb8b61.js → readCsv-07f6d9ad.js} +3 -3
  140. flowfile/web/static/assets/{readCsv-c767cb37.css → readCsv-3bfac4c3.css} +15 -15
  141. flowfile/web/static/assets/{readExcel-806d2826.css → readExcel-3db6b763.css} +13 -13
  142. flowfile/web/static/assets/{readExcel-67b4aee0.js → readExcel-ed69bc8f.js} +5 -5
  143. flowfile/web/static/assets/{readParquet-48c81530.css → readParquet-c5244ad5.css} +4 -4
  144. flowfile/web/static/assets/{readParquet-92ce1dbc.js → readParquet-e3ed4528.js} +3 -3
  145. flowfile/web/static/assets/secrets.api-002e7d7e.js +65 -0
  146. flowfile/web/static/assets/{selectDynamic-92e25ee3.js → selectDynamic-80b92899.js} +5 -5
  147. flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
  148. flowfile/web/static/assets/{vue-codemirror.esm-41b0e0d7.js → vue-codemirror.esm-0965f39f.js} +31 -640
  149. flowfile/web/static/assets/{vue-content-loader.es-2c8e608f.js → vue-content-loader.es-c506ad97.js} +1 -1
  150. flowfile/web/static/index.html +2 -2
  151. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/METADATA +2 -3
  152. flowfile-0.5.3.dist-info/RECORD +402 -0
  153. flowfile_core/__init__.py +13 -6
  154. flowfile_core/auth/jwt.py +51 -16
  155. flowfile_core/auth/models.py +32 -7
  156. flowfile_core/auth/password.py +89 -0
  157. flowfile_core/auth/secrets.py +8 -6
  158. flowfile_core/configs/__init__.py +9 -7
  159. flowfile_core/configs/flow_logger.py +15 -14
  160. flowfile_core/configs/node_store/__init__.py +72 -4
  161. flowfile_core/configs/node_store/nodes.py +155 -172
  162. flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
  163. flowfile_core/configs/settings.py +28 -15
  164. flowfile_core/database/connection.py +7 -6
  165. flowfile_core/database/init_db.py +96 -2
  166. flowfile_core/database/models.py +3 -1
  167. flowfile_core/fileExplorer/__init__.py +17 -0
  168. flowfile_core/fileExplorer/funcs.py +123 -57
  169. flowfile_core/fileExplorer/utils.py +10 -11
  170. flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
  171. flowfile_core/flowfile/analytics/analytics_processor.py +26 -24
  172. flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
  173. flowfile_core/flowfile/analytics/utils.py +1 -1
  174. flowfile_core/flowfile/code_generator/code_generator.py +358 -244
  175. flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
  176. flowfile_core/flowfile/connection_manager/models.py +1 -1
  177. flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
  178. flowfile_core/flowfile/database_connection_manager/models.py +1 -1
  179. flowfile_core/flowfile/extensions.py +17 -12
  180. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
  181. flowfile_core/flowfile/flow_data_engine/create/funcs.py +115 -83
  182. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +481 -423
  183. flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
  184. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
  185. flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
  186. flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
  187. flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
  188. flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +31 -20
  189. flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
  190. flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
  191. flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +14 -15
  192. flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
  193. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
  194. flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
  195. flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
  196. flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
  197. flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
  198. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +190 -127
  199. flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
  200. flowfile_core/flowfile/flow_data_engine/utils.py +99 -67
  201. flowfile_core/flowfile/flow_graph.py +918 -571
  202. flowfile_core/flowfile/flow_graph_utils.py +31 -49
  203. flowfile_core/flowfile/flow_node/flow_node.py +330 -233
  204. flowfile_core/flowfile/flow_node/models.py +53 -41
  205. flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
  206. flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
  207. flowfile_core/flowfile/handler.py +80 -30
  208. flowfile_core/flowfile/manage/compatibility_enhancements.py +209 -126
  209. flowfile_core/flowfile/manage/io_flowfile.py +54 -57
  210. flowfile_core/flowfile/node_designer/__init__.py +15 -13
  211. flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
  212. flowfile_core/flowfile/node_designer/custom_node.py +162 -36
  213. flowfile_core/flowfile/node_designer/ui_components.py +135 -34
  214. flowfile_core/flowfile/schema_callbacks.py +71 -51
  215. flowfile_core/flowfile/setting_generator/__init__.py +0 -1
  216. flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
  217. flowfile_core/flowfile/setting_generator/settings.py +64 -53
  218. flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
  219. flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
  220. flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
  221. flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
  222. flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
  223. flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
  224. flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
  225. flowfile_core/flowfile/util/calculate_layout.py +9 -13
  226. flowfile_core/flowfile/util/execution_orderer.py +25 -17
  227. flowfile_core/flowfile/util/node_skipper.py +4 -4
  228. flowfile_core/flowfile/utils.py +19 -21
  229. flowfile_core/main.py +26 -19
  230. flowfile_core/routes/auth.py +284 -11
  231. flowfile_core/routes/cloud_connections.py +25 -25
  232. flowfile_core/routes/logs.py +21 -29
  233. flowfile_core/routes/public.py +3 -3
  234. flowfile_core/routes/routes.py +70 -34
  235. flowfile_core/routes/secrets.py +25 -27
  236. flowfile_core/routes/user_defined_components.py +483 -4
  237. flowfile_core/run_lock.py +0 -1
  238. flowfile_core/schemas/__init__.py +4 -6
  239. flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
  240. flowfile_core/schemas/cloud_storage_schemas.py +59 -53
  241. flowfile_core/schemas/input_schema.py +231 -144
  242. flowfile_core/schemas/output_model.py +49 -34
  243. flowfile_core/schemas/schemas.py +116 -89
  244. flowfile_core/schemas/transform_schema.py +518 -263
  245. flowfile_core/schemas/yaml_types.py +21 -7
  246. flowfile_core/secret_manager/secret_manager.py +17 -13
  247. flowfile_core/types.py +29 -9
  248. flowfile_core/utils/arrow_reader.py +7 -6
  249. flowfile_core/utils/excel_file_manager.py +3 -3
  250. flowfile_core/utils/fileManager.py +7 -7
  251. flowfile_core/utils/fl_executor.py +8 -10
  252. flowfile_core/utils/utils.py +4 -4
  253. flowfile_core/utils/validate_setup.py +5 -4
  254. flowfile_frame/__init__.py +106 -51
  255. flowfile_frame/adapters.py +2 -9
  256. flowfile_frame/adding_expr.py +73 -32
  257. flowfile_frame/cloud_storage/frame_helpers.py +27 -23
  258. flowfile_frame/cloud_storage/secret_manager.py +12 -26
  259. flowfile_frame/config.py +2 -5
  260. flowfile_frame/expr.py +311 -218
  261. flowfile_frame/expr.pyi +160 -159
  262. flowfile_frame/expr_name.py +23 -23
  263. flowfile_frame/flow_frame.py +571 -476
  264. flowfile_frame/flow_frame.pyi +123 -104
  265. flowfile_frame/flow_frame_methods.py +227 -246
  266. flowfile_frame/group_frame.py +50 -20
  267. flowfile_frame/join.py +2 -2
  268. flowfile_frame/lazy.py +129 -87
  269. flowfile_frame/lazy_methods.py +83 -30
  270. flowfile_frame/list_name_space.py +55 -50
  271. flowfile_frame/selectors.py +148 -68
  272. flowfile_frame/series.py +9 -7
  273. flowfile_frame/utils.py +19 -21
  274. flowfile_worker/__init__.py +12 -7
  275. flowfile_worker/configs.py +11 -19
  276. flowfile_worker/create/__init__.py +14 -9
  277. flowfile_worker/create/funcs.py +114 -77
  278. flowfile_worker/create/models.py +46 -43
  279. flowfile_worker/create/pl_types.py +14 -15
  280. flowfile_worker/create/read_excel_tables.py +34 -41
  281. flowfile_worker/create/utils.py +22 -19
  282. flowfile_worker/external_sources/s3_source/main.py +18 -51
  283. flowfile_worker/external_sources/s3_source/models.py +34 -27
  284. flowfile_worker/external_sources/sql_source/main.py +8 -5
  285. flowfile_worker/external_sources/sql_source/models.py +13 -9
  286. flowfile_worker/flow_logger.py +10 -8
  287. flowfile_worker/funcs.py +214 -155
  288. flowfile_worker/main.py +11 -17
  289. flowfile_worker/models.py +35 -28
  290. flowfile_worker/process_manager.py +2 -3
  291. flowfile_worker/routes.py +121 -90
  292. flowfile_worker/secrets.py +9 -6
  293. flowfile_worker/spawner.py +80 -49
  294. flowfile_worker/utils.py +3 -2
  295. shared/__init__.py +2 -7
  296. shared/storage_config.py +25 -13
  297. test_utils/postgres/commands.py +3 -2
  298. test_utils/postgres/fixtures.py +9 -9
  299. test_utils/s3/commands.py +1 -1
  300. test_utils/s3/data_generator.py +3 -4
  301. test_utils/s3/demo_data_generator.py +4 -7
  302. test_utils/s3/fixtures.py +7 -5
  303. tools/migrate/__init__.py +1 -1
  304. tools/migrate/__main__.py +16 -29
  305. tools/migrate/legacy_schemas.py +251 -190
  306. tools/migrate/migrate.py +193 -181
  307. tools/migrate/tests/conftest.py +1 -3
  308. tools/migrate/tests/test_migrate.py +36 -41
  309. tools/migrate/tests/test_migration_e2e.py +28 -29
  310. tools/migrate/tests/test_node_migrations.py +50 -20
  311. flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
  312. flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
  313. flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
  314. flowfile/web/static/assets/Filter-9b6d08db.js +0 -164
  315. flowfile/web/static/assets/Filter-f62091b3.css +0 -20
  316. flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
  317. flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
  318. flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
  319. flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
  320. flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
  321. flowfile/web/static/assets/nodeInput-5d0d6b79.js +0 -41
  322. flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
  323. flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
  324. flowfile/web/static/assets/secretApi-68435402.js +0 -46
  325. flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
  326. flowfile-0.5.1.dist-info/RECORD +0 -388
  327. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/WHEEL +0 -0
  328. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/entry_points.txt +0 -0
  329. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/licenses/LICENSE +0 -0
@@ -1,15 +1,15 @@
1
+ from collections.abc import Callable
2
+ from dataclasses import dataclass
3
+ from typing import Literal
1
4
 
2
5
  import pyarrow as pa
3
- from typing import List, Union, Callable, Optional, Literal
4
- from dataclasses import dataclass
5
6
 
6
7
  # Forward declaration for type hints to avoid circular imports
7
8
  if False:
8
9
  from flowfile_core.flowfile.flow_node.flow_node import FlowNode
9
10
 
10
- from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
11
11
  from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
12
- from flowfile_core.schemas import schemas
12
+ from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
13
13
 
14
14
 
15
15
  @dataclass
@@ -27,30 +27,35 @@ class NodeStepPromise:
27
27
  right_input: The ID of the node connected to the right input port.
28
28
  depends_on: A list of node IDs that this node depends on for main inputs.
29
29
  """
30
- node_id: Union[str, int]
30
+
31
+ node_id: str | int
31
32
  name: str
32
33
  is_start: bool
33
- leads_to_id: Optional[List[Union[str, int]]] = None
34
- left_input: Optional[Union[str, int]] = None
35
- right_input: Optional[Union[str, int]] = None
36
- depends_on: Optional[List[Union[str, int]]] = None
34
+ leads_to_id: list[str | int] | None = None
35
+ left_input: str | int | None = None
36
+ right_input: str | int | None = None
37
+ depends_on: list[str | int] | None = None
37
38
 
38
39
 
39
40
  class NodeStepStats:
40
41
  """
41
42
  Tracks the execution status and statistics of a `FlowNode`.
42
43
  """
44
+
43
45
  error: str = None
44
46
  _has_run_with_current_setup: bool = False
45
47
  has_completed_last_run: bool = False
46
48
  active: bool = True
47
49
  is_canceled: bool = False
48
50
 
49
- def __init__(self, error: str = None,
50
- has_run_with_current_setup: bool = False,
51
- has_completed_last_run: bool = False,
52
- active: bool = True,
53
- is_canceled: bool = False):
51
+ def __init__(
52
+ self,
53
+ error: str = None,
54
+ has_run_with_current_setup: bool = False,
55
+ has_completed_last_run: bool = False,
56
+ active: bool = True,
57
+ is_canceled: bool = False,
58
+ ):
54
59
  """
55
60
  Initializes the node's statistics.
56
61
 
@@ -71,9 +76,11 @@ class NodeStepStats:
71
76
  Provides a string representation of the node's stats.
72
77
  :return: A string detailing the current stats.
73
78
  """
74
- return (f"NodeStepStats(error={self.error}, has_run_with_current_setup={self.has_run_with_current_setup}, "
75
- f"has_completed_last_run={self.has_completed_last_run}, "
76
- f"active={self.active}, is_canceled={self.is_canceled})")
79
+ return (
80
+ f"NodeStepStats(error={self.error}, has_run_with_current_setup={self.has_run_with_current_setup}, "
81
+ f"has_completed_last_run={self.has_completed_last_run}, "
82
+ f"active={self.active}, is_canceled={self.is_canceled})"
83
+ )
77
84
 
78
85
  @property
79
86
  def has_run_with_current_setup(self) -> bool:
@@ -109,6 +116,7 @@ class NodeStepSettings:
109
116
  setup_errors: If True, indicates a non-blocking error occurred during setup.
110
117
  breaking_setup_errors: If True, indicates an error occurred that prevents execution.
111
118
  """
119
+
112
120
  cache_results: bool = False
113
121
  renew_schema: bool = True
114
122
  streamable: bool = True
@@ -125,12 +133,13 @@ class NodeStepInputs:
125
133
  right_input: The `FlowNode` connected to the right input port.
126
134
  main_inputs: A list of `FlowNode` objects connected to the main input port(s).
127
135
  """
136
+
128
137
  left_input: "FlowNode" = None
129
138
  right_input: "FlowNode" = None
130
- main_inputs: List["FlowNode"] = None
139
+ main_inputs: list["FlowNode"] = None
131
140
 
132
141
  @property
133
- def input_ids(self) -> List[int]:
142
+ def input_ids(self) -> list[int]:
134
143
  """
135
144
  Gets the IDs of all connected input nodes.
136
145
  :return: A list of integer node IDs.
@@ -139,7 +148,7 @@ class NodeStepInputs:
139
148
  return [node_input.node_information.id for node_input in self.get_all_inputs()]
140
149
  return []
141
150
 
142
- def get_all_inputs(self) -> List["FlowNode"]:
151
+ def get_all_inputs(self) -> list["FlowNode"]:
143
152
  """
144
153
  Retrieves a single list containing all input nodes (main, left, and right).
145
154
  :return: A list of all connected `FlowNode` objects.
@@ -157,8 +166,9 @@ class NodeStepInputs:
157
166
  main_inputs_repr = f"Main Inputs: {self.main_inputs}" if self.main_inputs else "Main Inputs: None"
158
167
  return f"{self.__class__.__name__}({left_repr}, {right_repr}, {main_inputs_repr})"
159
168
 
160
- def validate_if_input_connection_exists(self, node_input_id: int,
161
- connection_name: Literal['main', 'left', 'right']) -> bool:
169
+ def validate_if_input_connection_exists(
170
+ self, node_input_id: int, connection_name: Literal["main", "left", "right"]
171
+ ) -> bool:
162
172
  """
163
173
  Checks if a connection from a specific node ID exists on a given port.
164
174
 
@@ -166,11 +176,11 @@ class NodeStepInputs:
166
176
  :param connection_name: The name of the input port ('main', 'left', 'right').
167
177
  :return: True if the connection exists, False otherwise.
168
178
  """
169
- if connection_name == 'main' and self.main_inputs:
179
+ if connection_name == "main" and self.main_inputs:
170
180
  return any(node_input.node_information.id == node_input_id for node_input in self.main_inputs)
171
- if connection_name == 'left' and self.left_input:
181
+ if connection_name == "left" and self.left_input:
172
182
  return self.left_input.node_information.id == node_input_id
173
- if connection_name == 'right':
183
+ if connection_name == "right":
174
184
  return self.right_input.node_information.id == node_input_id
175
185
 
176
186
 
@@ -185,25 +195,27 @@ class NodeSchemaInformation:
185
195
  drop_columns: A list of column names that will be dropped by the node.
186
196
  output_columns: A list of `FlowfileColumn` objects that will be added by the node.
187
197
  """
188
- result_schema: Optional[List[FlowfileColumn]] = None
189
- predicted_schema: Optional[List[FlowfileColumn]] = None
190
- input_columns: List[str] = []
191
- drop_columns: List[str] = []
192
- output_columns: List[FlowfileColumn] = []
198
+
199
+ result_schema: list[FlowfileColumn] | None = None
200
+ predicted_schema: list[FlowfileColumn] | None = None
201
+ input_columns: list[str] = []
202
+ drop_columns: list[str] = []
203
+ output_columns: list[FlowfileColumn] = []
193
204
 
194
205
 
195
206
  class NodeResults:
196
207
  """
197
208
  Stores the outputs of a `FlowNode`'s execution, including data, errors, and metadata.
198
209
  """
199
- _resulting_data: Optional[FlowDataEngine] = None
200
- example_data: Optional[FlowDataEngine] = None
201
- example_data_path: Optional[str] = None
202
- example_data_generator: Optional[Callable[[], pa.Table]] = None
210
+
211
+ _resulting_data: FlowDataEngine | None = None
212
+ example_data: FlowDataEngine | None = None
213
+ example_data_path: str | None = None
214
+ example_data_generator: Callable[[], pa.Table] | None = None
203
215
  run_time: int = -1
204
- errors: Optional[str] = None
205
- warnings: Optional[str] = None
206
- analysis_data_generator: Optional[Callable[[], pa.Table]] = None
216
+ errors: str | None = None
217
+ warnings: str | None = None
218
+ analysis_data_generator: Callable[[], pa.Table] | None = None
207
219
 
208
220
  def __init__(self):
209
221
  self._resulting_data = None
@@ -214,7 +226,7 @@ class NodeResults:
214
226
  self.example_data_generator = None
215
227
  self.analysis_data_generator = None
216
228
 
217
- def get_example_data(self) -> Optional[pa.Table]:
229
+ def get_example_data(self) -> pa.Table | None:
218
230
  """
219
231
  Executes the generator to fetch a sample of the resulting data.
220
232
  :return: A PyArrow Table containing a sample of the data, or None.
@@ -223,7 +235,7 @@ class NodeResults:
223
235
  return self.example_data_generator()
224
236
 
225
237
  @property
226
- def resulting_data(self) -> Optional[FlowDataEngine]:
238
+ def resulting_data(self) -> FlowDataEngine | None:
227
239
  """
228
240
  Gets the full resulting data from the node's execution.
229
241
  :return: A `FlowDataEngine` instance containing the result, or None.
@@ -231,7 +243,7 @@ class NodeResults:
231
243
  return self._resulting_data
232
244
 
233
245
  @resulting_data.setter
234
- def resulting_data(self, d: Optional[FlowDataEngine]):
246
+ def resulting_data(self, d: FlowDataEngine | None):
235
247
  """
236
248
  Sets the resulting data.
237
249
  :param d: The `FlowDataEngine` instance to store.
@@ -241,4 +253,4 @@ class NodeResults:
241
253
  def reset(self):
242
254
  """Resets all result attributes to their default, empty state."""
243
255
  self._resulting_data = None
244
- self.run_time = -1
256
+ self.run_time = -1
@@ -1,9 +1,11 @@
1
- from typing import Callable, Any, Optional, Generic, TypeVar
2
- from concurrent.futures import ThreadPoolExecutor, Future
3
1
  import threading
2
+ from collections.abc import Callable
3
+ from concurrent.futures import Future, ThreadPoolExecutor
4
+ from typing import Any, Generic, TypeVar
5
+
4
6
  from flowfile_core.configs import logger
5
7
 
6
- T = TypeVar('T')
8
+ T = TypeVar("T")
7
9
 
8
10
 
9
11
  class SingleExecutionFuture(Generic[T]):
@@ -14,20 +16,16 @@ class SingleExecutionFuture(Generic[T]):
14
16
  """
15
17
 
16
18
  func: Callable[[], T]
17
- on_error: Optional[Callable[[Exception], Any]]
19
+ on_error: Callable[[Exception], Any] | None
18
20
  _lock: threading.RLock
19
- _executor: Optional[ThreadPoolExecutor]
20
- _future: Optional[Future[T]]
21
- _result_value: Optional[T]
22
- _exception: Optional[Exception]
21
+ _executor: ThreadPoolExecutor | None
22
+ _future: Future[T] | None
23
+ _result_value: T | None
24
+ _exception: Exception | None
23
25
  _has_completed: bool
24
26
  _has_started: bool
25
27
 
26
- def __init__(
27
- self,
28
- func: Callable[[], T],
29
- on_error: Optional[Callable[[Exception], Any]] = None
30
- ) -> None:
28
+ def __init__(self, func: Callable[[], T], on_error: Callable[[Exception], Any] | None = None) -> None:
31
29
  """Initialize with function and optional error handler."""
32
30
  self.func = func
33
31
  self.on_error = on_error
@@ -81,7 +79,7 @@ class SingleExecutionFuture(Generic[T]):
81
79
  if self._executor and not self._executor._shutdown:
82
80
  self._executor.shutdown(wait=False)
83
81
 
84
- def __call__(self) -> Optional[T]:
82
+ def __call__(self) -> T | None:
85
83
  """Execute function if not running and return its result."""
86
84
  with self._lock:
87
85
  # If already completed, return cached result or raise cached exception
@@ -137,10 +135,7 @@ class SingleExecutionFuture(Generic[T]):
137
135
  """Check if the function is currently executing."""
138
136
  with self._lock:
139
137
  return bool(
140
- self._has_started and
141
- not self._has_completed and
142
- self._future is not None and
143
- not self._future.done()
138
+ self._has_started and not self._has_completed and self._future is not None and not self._future.done()
144
139
  )
145
140
 
146
141
  def is_completed(self) -> bool:
@@ -148,7 +143,7 @@ class SingleExecutionFuture(Generic[T]):
148
143
  with self._lock:
149
144
  return self._has_completed
150
145
 
151
- def get_result(self) -> Optional[T]:
146
+ def get_result(self) -> T | None:
152
147
  """Get the cached result without triggering execution."""
153
148
  with self._lock:
154
149
  if self._exception:
@@ -1,7 +1,4 @@
1
- from pydantic import BaseModel
2
-
3
1
  from flowfile_core.flowfile.flow_node.flow_node import FlowNode
4
-
5
2
  from flowfile_core.flowfile.graph_tree.models import BranchInfo, InputInfo
6
3
 
7
4
 
@@ -13,35 +10,39 @@ def calculate_depth(node_id: int, node_info: dict[int, BranchInfo], visited: set
13
10
  if node_id in visited:
14
11
  return node_info[node_id].depth
15
12
  visited.add(node_id)
16
-
13
+
17
14
  max_input_depth = -1
18
15
  inputs = node_info[node_id].inputs
19
-
16
+
20
17
  for main_id in inputs.main:
21
18
  max_input_depth = max(max_input_depth, calculate_depth(main_id, node_info, visited))
22
19
  if inputs.left:
23
20
  max_input_depth = max(max_input_depth, calculate_depth(inputs.left, node_info, visited))
24
21
  if inputs.right:
25
22
  max_input_depth = max(max_input_depth, calculate_depth(inputs.right, node_info, visited))
26
-
23
+
27
24
  node_info[node_id].depth = max_input_depth + 1
28
25
  return node_info[node_id].depth
29
26
 
30
27
 
31
28
  # Trace paths from each root
32
- def trace_path(node_id: int, node_info: dict[int, BranchInfo], merge_points: dict[int, list[int]],
33
- current_path: list[int] | None = None):
29
+ def trace_path(
30
+ node_id: int,
31
+ node_info: dict[int, BranchInfo],
32
+ merge_points: dict[int, list[int]],
33
+ current_path: list[int] | None = None,
34
+ ):
34
35
  """Define the trace of each node path"""
35
36
  if current_path is None:
36
37
  current_path = []
37
-
38
+
38
39
  current_path = current_path + [node_id]
39
40
  outputs = node_info[node_id].outputs
40
-
41
+
41
42
  if not outputs:
42
43
  # End of path
43
44
  return [current_path]
44
-
45
+
45
46
  # If this node has multiple outputs or connects to a merge point, branch
46
47
  all_paths = []
47
48
  for output_id in outputs:
@@ -64,7 +65,7 @@ def build_node_info(nodes: list[FlowNode]) -> dict[int, BranchInfo]:
64
65
  # Get node label
65
66
  operation = node.node_type.replace("_", " ").title() if node.node_type else "Unknown"
66
67
  label = f"{operation} (id={node_id})"
67
- if hasattr(node, 'setting_input') and hasattr(node.setting_input, 'description'):
68
+ if hasattr(node, "setting_input") and hasattr(node.setting_input, "description"):
68
69
  if node.setting_input.description:
69
70
  desc = node.setting_input.description
70
71
  if len(desc) > 20: # Truncate long descriptions
@@ -75,18 +76,14 @@ def build_node_info(nodes: list[FlowNode]) -> dict[int, BranchInfo]:
75
76
  inputs = InputInfo(
76
77
  main=[n.node_id for n in (node.node_inputs.main_inputs or [])],
77
78
  left=node.node_inputs.left_input.node_id if node.node_inputs.left_input else None,
78
- right=node.node_inputs.right_input.node_id if node.node_inputs.right_input else None
79
+ right=node.node_inputs.right_input.node_id if node.node_inputs.right_input else None,
79
80
  )
80
81
  outputs = [n.node_id for n in node.leads_to_nodes]
81
82
 
82
83
  node_info[node_id] = BranchInfo(
83
- label=label,
84
- short_label=f"{operation} ({node_id})",
85
- inputs=inputs,
86
- outputs=outputs,
87
- depth=0
84
+ label=label, short_label=f"{operation} ({node_id})", inputs=inputs, outputs=outputs, depth=0
88
85
  )
89
-
86
+
90
87
  return node_info
91
88
 
92
89
 
@@ -112,19 +109,20 @@ def define_node_connections(node_info: dict[int, BranchInfo]) -> dict[int, list[
112
109
  if output_id not in merge_points:
113
110
  merge_points[output_id] = []
114
111
  merge_points[output_id].append(node_id)
115
-
112
+
116
113
  return merge_points
117
114
 
118
115
 
119
- def build_flow_paths(node_info: dict[int, BranchInfo], flow_starts: list[FlowNode],
120
- merge_points: dict[int, list[int]]):
116
+ def build_flow_paths(node_info: dict[int, BranchInfo], flow_starts: list[FlowNode], merge_points: dict[int, list[int]]):
121
117
  """Build the flow paths to be drawn"""
122
118
 
123
-
124
119
  # Find all root nodes (no inputs)
125
- root_nodes = [nid for nid, info in node_info.items()
126
- if not info.inputs.main and not info.inputs.left and not info.inputs.right]
127
-
120
+ root_nodes = [
121
+ nid
122
+ for nid, info in node_info.items()
123
+ if not info.inputs.main and not info.inputs.left and not info.inputs.right
124
+ ]
125
+
128
126
  if not root_nodes and flow_starts:
129
127
  root_nodes = [n.node_id for n in flow_starts]
130
128
  paths = [] # List of paths through the graph
@@ -136,7 +134,7 @@ def build_flow_paths(node_info: dict[int, BranchInfo], flow_starts: list[FlowNod
136
134
  return paths
137
135
 
138
136
 
139
- def group_paths(paths:list, merge_points:dict):
137
+ def group_paths(paths: list, merge_points: dict):
140
138
  """Groups each node path."""
141
139
  paths_by_merge = {}
142
140
  standalone_paths = []
@@ -152,12 +150,14 @@ def group_paths(paths:list, merge_points:dict):
152
150
  return paths_by_merge, standalone_paths
153
151
 
154
152
 
155
- def draw_merged_paths(node_info: dict[int, BranchInfo],
156
- merge_points: dict[int, list[int]],
157
- paths_by_merge: dict[int, list[list[int]]],
158
- merge_drawn: set,
159
- drawn_nodes: set,
160
- lines: list[str]):
153
+ def draw_merged_paths(
154
+ node_info: dict[int, BranchInfo],
155
+ merge_points: dict[int, list[int]],
156
+ paths_by_merge: dict[int, list[list[int]]],
157
+ merge_drawn: set,
158
+ drawn_nodes: set,
159
+ lines: list[str],
160
+ ):
161
161
  """Draws paths for each node that merges."""
162
162
  for merge_id, merge_paths in paths_by_merge.items():
163
163
  if merge_id in merge_drawn:
@@ -171,8 +171,7 @@ def draw_merged_paths(node_info: dict[int, BranchInfo],
171
171
  source_path = None
172
172
  for path in merge_paths:
173
173
  if source_id in path:
174
-
175
- source_path = path[:path.index(source_id) + 1]
174
+ source_path = path[: path.index(source_id) + 1]
176
175
  break
177
176
 
178
177
  if source_path:
@@ -215,14 +214,15 @@ def draw_merged_paths(node_info: dict[int, BranchInfo],
215
214
  return paths_by_merge
216
215
 
217
216
 
218
- def draw_standalone_paths(drawn_nodes: set[int], standalone_paths: list[list[int]], lines: list[str],
219
- node_info: dict[int, BranchInfo]):
220
- """ Draws paths that do not merge."""
217
+ def draw_standalone_paths(
218
+ drawn_nodes: set[int], standalone_paths: list[list[int]], lines: list[str], node_info: dict[int, BranchInfo]
219
+ ):
220
+ """Draws paths that do not merge."""
221
221
  # Draw standalone paths
222
222
  for path in standalone_paths:
223
223
  if all(nid in drawn_nodes for nid in path):
224
224
  continue
225
-
225
+
226
226
  line_parts = []
227
227
  for i, node_id in enumerate(path):
228
228
  if node_id not in drawn_nodes:
@@ -231,12 +231,12 @@ def draw_standalone_paths(drawn_nodes: set[int], standalone_paths: list[list[int
231
231
  else:
232
232
  line_parts.append(f" ──> {node_info[node_id].short_label}")
233
233
  drawn_nodes.add(node_id)
234
-
234
+
235
235
  if line_parts:
236
236
  lines.append("".join(line_parts))
237
237
 
238
238
 
239
- def add_un_drawn_nodes(drawn_nodes: set[int], node_info: dict[int, BranchInfo], lines: list[str]):
239
+ def add_un_drawn_nodes(drawn_nodes: set[int], node_info: dict[int, BranchInfo], lines: list[str]):
240
240
  """Adds isolated nodes if exists."""
241
241
  # Add any remaining undrawn nodes
242
242
 
@@ -1,14 +1,12 @@
1
-
2
- from dataclasses import dataclass
3
- from typing import Dict, List
4
1
  import os
5
- from pathlib import Path
2
+ from dataclasses import dataclass
6
3
  from datetime import datetime
4
+ from pathlib import Path
7
5
 
8
- from flowfile_core.flowfile.manage.io_flowfile import open_flow
9
6
  from flowfile_core.flowfile.flow_graph import FlowGraph
10
- from flowfile_core.schemas.schemas import FlowSettings
7
+ from flowfile_core.flowfile.manage.io_flowfile import open_flow
11
8
  from flowfile_core.flowfile.utils import create_unique_id
9
+ from flowfile_core.schemas.schemas import FlowSettings
12
10
  from shared.storage_config import storage
13
11
 
14
12
 
@@ -21,62 +19,111 @@ def get_flow_save_location(flow_name: str) -> Path:
21
19
 
22
20
  def create_flow_name() -> str:
23
21
  """Creates a unique flow name"""
24
- return datetime.now().strftime("%Y%m%d_%H_%M_%S")+"_flow.yaml"
22
+ return datetime.now().strftime("%Y%m%d_%H_%M_%S") + "_flow.yaml"
25
23
 
26
24
 
27
25
  @dataclass
28
26
  class FlowfileHandler:
29
- _flows: Dict[int, FlowGraph]
27
+ _flows: dict[int, FlowGraph]
28
+ _user_sessions: dict[int, set[int]] # Maps user_id -> set of flow_ids
30
29
 
31
30
  def __init__(self):
32
31
  self._flows = {}
32
+ self._user_sessions = {}
33
33
 
34
34
  @property
35
- def flowfile_flows(self) -> List[FlowGraph]:
35
+ def flowfile_flows(self) -> list[FlowGraph]:
36
36
  return list(self._flows.values())
37
37
 
38
+ def _register_user_session(self, user_id: int | None, flow_id: int):
39
+ """Register a flow as belonging to a user's session."""
40
+ if user_id is not None:
41
+ if user_id not in self._user_sessions:
42
+ self._user_sessions[user_id] = set()
43
+ self._user_sessions[user_id].add(flow_id)
44
+
45
+ def _unregister_user_session(self, user_id: int | None, flow_id: int):
46
+ """Remove a flow from a user's session."""
47
+ if user_id is not None and user_id in self._user_sessions:
48
+ self._user_sessions[user_id].discard(flow_id)
49
+
50
+ def get_user_flows(self, user_id: int | None) -> list[FlowGraph]:
51
+ """Get all flows belonging to a specific user's session."""
52
+ if user_id is None:
53
+ return self.flowfile_flows
54
+ user_flow_ids = self._user_sessions.get(user_id, set())
55
+ return [f for f in self._flows.values() if f.flow_id in user_flow_ids]
56
+
57
+ def user_has_flow(self, user_id: int | None, flow_id: int) -> bool:
58
+ """Check if a user has access to a specific flow."""
59
+ if user_id is None:
60
+ return flow_id in self._flows
61
+ return flow_id in self._user_sessions.get(user_id, set())
62
+
38
63
  def __add__(self, other: FlowGraph) -> int:
39
64
  self._flows[other.flow_id] = other
40
65
  return other.flow_id
41
66
 
42
- def import_flow(self, flow_path: Path | str) -> int:
67
+ def import_flow(self, flow_path: Path | str, user_id: int | None = None) -> int:
43
68
  if isinstance(flow_path, str):
44
69
  flow_path = Path(flow_path)
45
70
  imported_flow = open_flow(flow_path)
46
71
  self._flows[imported_flow.flow_id] = imported_flow
47
72
  imported_flow.flow_settings = self.get_flow_info(imported_flow.flow_id)
48
73
  imported_flow.flow_settings.is_running = False
74
+ self._register_user_session(user_id, imported_flow.flow_id)
49
75
  return imported_flow.flow_id
50
76
 
51
- def register_flow(self, flow_settings: FlowSettings):
77
+ def register_flow(self, flow_settings: FlowSettings, user_id: int | None = None) -> FlowGraph:
78
+ """Register a flow with the handler and associate it with a user session."""
52
79
  if flow_settings.flow_id in self._flows:
53
80
  self.delete_flow(flow_settings.flow_id)
54
- raise 'flow already registered'
55
- else:
56
- name = flow_settings.name if flow_settings.name else flow_settings.flow_id
57
- self._flows[flow_settings.flow_id] = FlowGraph(name=name, flow_settings=flow_settings)
81
+ raise ValueError("Flow already registered")
82
+ name = flow_settings.name if flow_settings.name else str(flow_settings.flow_id)
83
+ self._flows[flow_settings.flow_id] = FlowGraph(name=name, flow_settings=flow_settings)
84
+ self._register_user_session(user_id, flow_settings.flow_id)
58
85
  return self.get_flow(flow_settings.flow_id)
59
86
 
60
- def get_flow(self, flow_id: int) -> FlowGraph | None:
61
- return self._flows.get(flow_id, None)
62
-
63
- def delete_flow(self, flow_id: int):
64
- flow = self._flows.pop(flow_id)
65
- del flow
87
+ def get_flow(self, flow_id: int, user_id: int | None = None) -> FlowGraph | None:
88
+ """Get a flow by ID, optionally checking user access."""
89
+ flow = self._flows.get(flow_id, None)
90
+ if flow and user_id is not None:
91
+ # Only return the flow if user has access
92
+ if not self.user_has_flow(user_id, flow_id):
93
+ return None
94
+ return flow
95
+
96
+ def delete_flow(self, flow_id: int, user_id: int | None = None):
97
+ """Remove flow from user's session. Flow data remains until all users close it."""
98
+ if user_id is not None:
99
+ if not self.user_has_flow(user_id, flow_id):
100
+ raise Exception(f"Flow {flow_id} not found in user's session")
101
+ self._unregister_user_session(user_id, flow_id)
102
+ # Check if any user still has this flow open
103
+ flow_still_open = any(flow_id in flows for flows in self._user_sessions.values())
104
+ if not flow_still_open and flow_id in self._flows:
105
+ flow = self._flows.pop(flow_id)
106
+ del flow
107
+ else:
108
+ # No user context - delete directly
109
+ if flow_id in self._flows:
110
+ flow = self._flows.pop(flow_id)
111
+ del flow
66
112
 
67
- def save_flow(self, flow_id: int, flow_path: str):
68
- flow = self.get_flow(flow_id)
113
+ def save_flow(self, flow_id: int, flow_path: str, user_id: int | None = None):
114
+ flow = self.get_flow(flow_id, user_id)
69
115
  if flow:
70
116
  flow.save_flow(flow_path)
71
117
  else:
72
- raise Exception('Flow not found')
118
+ raise Exception("Flow not found or not accessible by user")
73
119
 
74
- def add_flow(self, name: str = None, flow_path: str = None) -> int:
120
+ def add_flow(self, name: str = None, flow_path: str = None, user_id: int | None = None) -> int:
75
121
  """
76
122
  Creates a new flow with a reference to the flow path
77
123
  Args:
78
124
  name (str): The name of the flow
79
125
  flow_path (str): The path to the flow file
126
+ user_id (int): The ID of the user creating the flow
80
127
 
81
128
  Returns:
82
129
  int: The flow id
@@ -87,15 +134,18 @@ class FlowfileHandler:
87
134
  name = create_flow_name()
88
135
  if not flow_path:
89
136
  flow_path = get_flow_save_location(name)
90
- flow_info = FlowSettings(name=name, flow_id=next_id, save_location=str(flow_path), path=str(flow_path))
91
- flow = self.register_flow(flow_info)
137
+ flow_info = FlowSettings(
138
+ name=name, flow_id=next_id, save_location=str(flow_path),
139
+ path=str(flow_path)
140
+ )
141
+ flow = self.register_flow(flow_info, user_id=user_id)
92
142
  flow.save_flow(flow.flow_settings.path)
93
143
  return next_id
94
144
 
95
145
  def get_flow_info(self, flow_id: int) -> FlowSettings:
96
146
  flow = self.get_flow(flow_id)
97
147
  if not flow:
98
- raise Exception(f'Flow {flow_id} not found')
148
+ raise Exception(f"Flow {flow_id} not found")
99
149
  flow_exists = os.path.exists(flow.flow_settings.path)
100
150
  last_modified_ts = os.path.getmtime(flow.flow_settings.path) if flow_exists else -1
101
151
  flow.flow_settings.modified_on = last_modified_ts
@@ -104,8 +154,8 @@ class FlowfileHandler:
104
154
  def get_node(self, flow_id: int, node_id: int):
105
155
  flow = self.get_flow(flow_id)
106
156
  if not flow:
107
- raise Exception(f'Flow {flow_id} not found')
157
+ raise Exception(f"Flow {flow_id} not found")
108
158
  node = flow.get_node(node_id)
109
159
  if not node:
110
- raise Exception(f'Node {node_id} not found in flow {flow_id}')
160
+ raise Exception(f"Node {node_id} not found in flow {flow_id}")
111
161
  return node