Flowfile 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (329) hide show
  1. build_backends/main.py +25 -22
  2. build_backends/main_prd.py +10 -19
  3. flowfile/__init__.py +178 -74
  4. flowfile/__main__.py +10 -7
  5. flowfile/api.py +51 -57
  6. flowfile/web/__init__.py +14 -9
  7. flowfile/web/static/assets/AdminView-49392a9a.js +713 -0
  8. flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
  9. flowfile/web/static/assets/CloudConnectionView-36bcd6df.css +72 -0
  10. flowfile/web/static/assets/{CloudConnectionManager-0dfba9f2.js → CloudConnectionView-f13f202b.js} +11 -11
  11. flowfile/web/static/assets/{CloudStorageReader-d5b1b6c9.js → CloudStorageReader-0023d4a5.js} +10 -8
  12. flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
  13. flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
  14. flowfile/web/static/assets/{CloudStorageWriter-00d87aad.js → CloudStorageWriter-8e781e11.js} +10 -8
  15. flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
  16. flowfile/web/static/assets/{ColumnSelector-4685e75d.js → ColumnSelector-8ad68ea9.js} +3 -5
  17. flowfile/web/static/assets/{ContextMenu-c13f91d0.css → ContextMenu-26d4dd27.css} +6 -6
  18. flowfile/web/static/assets/{ContextMenu-23e909da.js → ContextMenu-31ee57f0.js} +3 -3
  19. flowfile/web/static/assets/{ContextMenu-70ae0c79.js → ContextMenu-69a74055.js} +3 -3
  20. flowfile/web/static/assets/{ContextMenu-f149cf7c.js → ContextMenu-8e2051c6.js} +3 -3
  21. flowfile/web/static/assets/{ContextMenu-4c74eef1.css → ContextMenu-8ec1729e.css} +6 -6
  22. flowfile/web/static/assets/{ContextMenu-63cfa99b.css → ContextMenu-9b310c60.css} +6 -6
  23. flowfile/web/static/assets/{CrossJoin-702a3edd.js → CrossJoin-03df6938.js} +12 -10
  24. flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
  25. flowfile/web/static/assets/CustomNode-59e99a86.css +32 -0
  26. flowfile/web/static/assets/{CustomNode-b1519993.js → CustomNode-8479239b.js} +36 -24
  27. flowfile/web/static/assets/{DatabaseConnectionSettings-6f3e4ea5.js → DatabaseConnectionSettings-869e3efd.js} +5 -4
  28. flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-e91df89a.css} +13 -13
  29. flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-36898a00.css} +24 -24
  30. flowfile/web/static/assets/{DatabaseReader-d38c7295.js → DatabaseReader-c58b9552.js} +25 -15
  31. flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
  32. flowfile/web/static/assets/{DatabaseManager-cf5ef661.js → DatabaseView-d26a9140.js} +11 -11
  33. flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-217a99f1.css} +19 -19
  34. flowfile/web/static/assets/{DatabaseWriter-b04ef46a.js → DatabaseWriter-4d05ddc7.js} +17 -10
  35. flowfile/web/static/assets/{designer-8da3ba3a.css → DesignerView-a6d0ee84.css} +614 -546
  36. flowfile/web/static/assets/{designer-9633482a.js → DesignerView-e6f5c0e8.js} +1107 -3170
  37. flowfile/web/static/assets/{documentation-ca400224.js → DocumentationView-2e78ef1b.js} +5 -5
  38. flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-fd46c656.css} +7 -7
  39. flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
  40. flowfile/web/static/assets/{ExploreData-5fa10ed8.js → ExploreData-7b54caca.js} +18 -9
  41. flowfile/web/static/assets/{ExternalSource-d39af878.js → ExternalSource-3fa399b2.js} +9 -7
  42. flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-47ab05a3.css} +17 -17
  43. flowfile/web/static/assets/Filter-7494ea97.css +48 -0
  44. flowfile/web/static/assets/Filter-8cbbdbf3.js +287 -0
  45. flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
  46. flowfile/web/static/assets/{Formula-6b04fb1d.js → Formula-aac42b1e.js} +13 -11
  47. flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
  48. flowfile/web/static/assets/{FuzzyMatch-999521f4.js → FuzzyMatch-cd9bbfca.js} +12 -10
  49. flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-c24dec17.css} +5 -5
  50. flowfile/web/static/assets/{GraphSolver-17dd2198.js → GraphSolver-c7e6780e.js} +13 -11
  51. flowfile/web/static/assets/{GroupBy-6b039e18.js → GroupBy-93c5d22b.js} +9 -7
  52. flowfile/web/static/assets/{GroupBy-b9505323.css → GroupBy-be7ac0bf.css} +10 -10
  53. flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
  54. flowfile/web/static/assets/{Join-24d0f113.js → Join-a19b2de2.js} +13 -11
  55. flowfile/web/static/assets/LoginView-0df4ed0a.js +134 -0
  56. flowfile/web/static/assets/LoginView-d325d632.css +172 -0
  57. flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
  58. flowfile/web/static/assets/{ManualInput-34639209.js → ManualInput-8d3374b2.js} +170 -116
  59. flowfile/web/static/assets/{MultiSelect-0e8724a3.js → MultiSelect-ad1b6243.js} +2 -2
  60. flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-b0e538c2.js → MultiSelect.vue_vue_type_script_setup_true_lang-e278950d.js} +1 -1
  61. flowfile/web/static/assets/NodeDesigner-40b647c9.js +2610 -0
  62. flowfile/web/static/assets/NodeDesigner-5f53be3f.css +1429 -0
  63. flowfile/web/static/assets/{NumericInput-3d63a470.js → NumericInput-7100234c.js} +2 -2
  64. flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-e0edeccc.js → NumericInput.vue_vue_type_script_setup_true_lang-5130219f.js} +5 -2
  65. flowfile/web/static/assets/{Output-283fe388.css → Output-35e97000.css} +6 -6
  66. flowfile/web/static/assets/{Output-edea9802.js → Output-f5efd2aa.js} +12 -9
  67. flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
  68. flowfile/web/static/assets/{Pivot-61d19301.js → Pivot-d981d23c.js} +11 -9
  69. flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
  70. flowfile/web/static/assets/{PivotValidation-f97fec5b.js → PivotValidation-39386e95.js} +3 -3
  71. flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
  72. flowfile/web/static/assets/{PivotValidation-de9f43fe.js → PivotValidation-63de1f73.js} +3 -3
  73. flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
  74. flowfile/web/static/assets/{PolarsCode-bc3c9984.js → PolarsCode-f9d69217.js} +18 -9
  75. flowfile/web/static/assets/PopOver-b22f049e.js +939 -0
  76. flowfile/web/static/assets/PopOver-d96599db.css +33 -0
  77. flowfile/web/static/assets/{Read-e808b239.css → Read-36e7bd51.css} +12 -12
  78. flowfile/web/static/assets/{Read-64a3f259.js → Read-aec2e377.js} +14 -11
  79. flowfile/web/static/assets/{RecordCount-3d5039be.js → RecordCount-78ed6845.js} +6 -4
  80. flowfile/web/static/assets/{RecordId-597510e0.js → RecordId-2156e890.js} +8 -6
  81. flowfile/web/static/assets/{SQLQueryComponent-36cef432.css → SQLQueryComponent-1c2f26b4.css} +5 -5
  82. flowfile/web/static/assets/{SQLQueryComponent-df51adbe.js → SQLQueryComponent-48c72f5b.js} +3 -3
  83. flowfile/web/static/assets/{Sample-4be0a507.js → Sample-1352ca74.js} +6 -4
  84. flowfile/web/static/assets/SecretSelector-22b5ff89.js +113 -0
  85. flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
  86. flowfile/web/static/assets/{SecretManager-4839be57.js → SecretsView-17df66ee.js} +35 -36
  87. flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
  88. flowfile/web/static/assets/{Select-9b72f201.js → Select-0aee4c54.js} +9 -7
  89. flowfile/web/static/assets/{SettingsSection-f0f75a42.js → SettingsSection-0784e157.js} +3 -3
  90. flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
  91. flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
  92. flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
  93. flowfile/web/static/assets/{SettingsSection-e1e9c953.js → SettingsSection-cd341bb6.js} +3 -3
  94. flowfile/web/static/assets/{SettingsSection-7ded385d.js → SettingsSection-f2002a6d.js} +3 -3
  95. flowfile/web/static/assets/{SingleSelect-6c777aac.js → SingleSelect-460cc0ea.js} +2 -2
  96. flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-33e3ff9b.js → SingleSelect.vue_vue_type_script_setup_true_lang-30741bb2.js} +1 -1
  97. flowfile/web/static/assets/{SliderInput-7cb93e62.js → SliderInput-5d926864.js} +7 -4
  98. flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
  99. flowfile/web/static/assets/{Sort-6cbde21a.js → Sort-3cdc971b.js} +9 -7
  100. flowfile/web/static/assets/{Unique-f9fb0809.css → Sort-8a871341.css} +10 -10
  101. flowfile/web/static/assets/{TextInput-d9a40c11.js → TextInput-a2d0bfbd.js} +2 -2
  102. flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-5896c375.js → TextInput.vue_vue_type_script_setup_true_lang-abad1ca2.js} +5 -2
  103. flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
  104. flowfile/web/static/assets/{TextToRows-c4fcbf4d.js → TextToRows-918945f7.js} +11 -10
  105. flowfile/web/static/assets/{ToggleSwitch-4ef91d19.js → ToggleSwitch-f0ef5196.js} +2 -2
  106. flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-38478c20.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-5605c793.js} +1 -1
  107. flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-54d2f518.css} +6 -6
  108. flowfile/web/static/assets/{UnavailableFields-a03f512c.js → UnavailableFields-bdad6144.js} +4 -4
  109. flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
  110. flowfile/web/static/assets/{Union-bfe9b996.js → Union-e8ab8c86.js} +8 -6
  111. flowfile/web/static/assets/{Unique-5d023a27.js → Unique-8cd4f976.js} +13 -10
  112. flowfile/web/static/assets/{Sort-3643d625.css → Unique-9fb2f567.css} +10 -10
  113. flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-710a2948.css} +7 -7
  114. flowfile/web/static/assets/{Unpivot-91cc5354.js → Unpivot-8da14095.js} +10 -8
  115. flowfile/web/static/assets/{UnpivotValidation-7ee2de44.js → UnpivotValidation-6f7d89ff.js} +3 -3
  116. flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
  117. flowfile/web/static/assets/{VueGraphicWalker-e51b9924.js → VueGraphicWalker-3fb312e1.js} +4 -4
  118. flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
  119. flowfile/web/static/assets/{api-cf1221f0.js → api-24483f0d.js} +1 -1
  120. flowfile/web/static/assets/{api-c1bad5ca.js → api-8b81fa73.js} +1 -1
  121. flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-3d8dc5fa.css} +40 -40
  122. flowfile/web/static/assets/{dropDown-614b998d.js → dropDown-ac0fda9d.js} +3 -3
  123. flowfile/web/static/assets/{fullEditor-f7971590.js → fullEditor-5497a84a.js} +11 -10
  124. flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-a0be62b3.css} +74 -62
  125. flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
  126. flowfile/web/static/assets/{genericNodeSettings-4fe5f36b.js → genericNodeSettings-99014e1d.js} +5 -5
  127. flowfile/web/static/assets/index-07dda503.js +38 -0
  128. flowfile/web/static/assets/index-3ba44389.js +2696 -0
  129. flowfile/web/static/assets/{index-50508d4d.css → index-e6289dd0.css} +1945 -569
  130. flowfile/web/static/assets/{index-5429bbf8.js → index-fb6493ae.js} +41626 -40867
  131. flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
  132. flowfile/web/static/assets/nodeInput-0eb13f1a.js +2 -0
  133. flowfile/web/static/assets/{outputCsv-076b85ab.js → outputCsv-8f8ba42d.js} +3 -3
  134. flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
  135. flowfile/web/static/assets/{outputExcel-0fd17dbe.js → outputExcel-393f4fef.js} +3 -3
  136. flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
  137. flowfile/web/static/assets/{outputParquet-b61e0847.js → outputParquet-07c81f65.js} +4 -4
  138. flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
  139. flowfile/web/static/assets/{readCsv-a8bb8b61.js → readCsv-07f6d9ad.js} +3 -3
  140. flowfile/web/static/assets/{readCsv-c767cb37.css → readCsv-3bfac4c3.css} +15 -15
  141. flowfile/web/static/assets/{readExcel-806d2826.css → readExcel-3db6b763.css} +13 -13
  142. flowfile/web/static/assets/{readExcel-67b4aee0.js → readExcel-ed69bc8f.js} +5 -5
  143. flowfile/web/static/assets/{readParquet-48c81530.css → readParquet-c5244ad5.css} +4 -4
  144. flowfile/web/static/assets/{readParquet-92ce1dbc.js → readParquet-e3ed4528.js} +3 -3
  145. flowfile/web/static/assets/secrets.api-002e7d7e.js +65 -0
  146. flowfile/web/static/assets/{selectDynamic-92e25ee3.js → selectDynamic-80b92899.js} +5 -5
  147. flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
  148. flowfile/web/static/assets/{vue-codemirror.esm-41b0e0d7.js → vue-codemirror.esm-0965f39f.js} +31 -640
  149. flowfile/web/static/assets/{vue-content-loader.es-2c8e608f.js → vue-content-loader.es-c506ad97.js} +1 -1
  150. flowfile/web/static/index.html +2 -2
  151. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/METADATA +2 -3
  152. flowfile-0.5.3.dist-info/RECORD +402 -0
  153. flowfile_core/__init__.py +13 -6
  154. flowfile_core/auth/jwt.py +51 -16
  155. flowfile_core/auth/models.py +32 -7
  156. flowfile_core/auth/password.py +89 -0
  157. flowfile_core/auth/secrets.py +8 -6
  158. flowfile_core/configs/__init__.py +9 -7
  159. flowfile_core/configs/flow_logger.py +15 -14
  160. flowfile_core/configs/node_store/__init__.py +72 -4
  161. flowfile_core/configs/node_store/nodes.py +155 -172
  162. flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
  163. flowfile_core/configs/settings.py +28 -15
  164. flowfile_core/database/connection.py +7 -6
  165. flowfile_core/database/init_db.py +96 -2
  166. flowfile_core/database/models.py +3 -1
  167. flowfile_core/fileExplorer/__init__.py +17 -0
  168. flowfile_core/fileExplorer/funcs.py +123 -57
  169. flowfile_core/fileExplorer/utils.py +10 -11
  170. flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
  171. flowfile_core/flowfile/analytics/analytics_processor.py +26 -24
  172. flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
  173. flowfile_core/flowfile/analytics/utils.py +1 -1
  174. flowfile_core/flowfile/code_generator/code_generator.py +358 -244
  175. flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
  176. flowfile_core/flowfile/connection_manager/models.py +1 -1
  177. flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
  178. flowfile_core/flowfile/database_connection_manager/models.py +1 -1
  179. flowfile_core/flowfile/extensions.py +17 -12
  180. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
  181. flowfile_core/flowfile/flow_data_engine/create/funcs.py +115 -83
  182. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +481 -423
  183. flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
  184. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
  185. flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
  186. flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
  187. flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
  188. flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +31 -20
  189. flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
  190. flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
  191. flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +14 -15
  192. flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
  193. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
  194. flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
  195. flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
  196. flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
  197. flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
  198. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +190 -127
  199. flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
  200. flowfile_core/flowfile/flow_data_engine/utils.py +99 -67
  201. flowfile_core/flowfile/flow_graph.py +918 -571
  202. flowfile_core/flowfile/flow_graph_utils.py +31 -49
  203. flowfile_core/flowfile/flow_node/flow_node.py +330 -233
  204. flowfile_core/flowfile/flow_node/models.py +53 -41
  205. flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
  206. flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
  207. flowfile_core/flowfile/handler.py +80 -30
  208. flowfile_core/flowfile/manage/compatibility_enhancements.py +209 -126
  209. flowfile_core/flowfile/manage/io_flowfile.py +54 -57
  210. flowfile_core/flowfile/node_designer/__init__.py +15 -13
  211. flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
  212. flowfile_core/flowfile/node_designer/custom_node.py +162 -36
  213. flowfile_core/flowfile/node_designer/ui_components.py +135 -34
  214. flowfile_core/flowfile/schema_callbacks.py +71 -51
  215. flowfile_core/flowfile/setting_generator/__init__.py +0 -1
  216. flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
  217. flowfile_core/flowfile/setting_generator/settings.py +64 -53
  218. flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
  219. flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
  220. flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
  221. flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
  222. flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
  223. flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
  224. flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
  225. flowfile_core/flowfile/util/calculate_layout.py +9 -13
  226. flowfile_core/flowfile/util/execution_orderer.py +25 -17
  227. flowfile_core/flowfile/util/node_skipper.py +4 -4
  228. flowfile_core/flowfile/utils.py +19 -21
  229. flowfile_core/main.py +26 -19
  230. flowfile_core/routes/auth.py +284 -11
  231. flowfile_core/routes/cloud_connections.py +25 -25
  232. flowfile_core/routes/logs.py +21 -29
  233. flowfile_core/routes/public.py +3 -3
  234. flowfile_core/routes/routes.py +70 -34
  235. flowfile_core/routes/secrets.py +25 -27
  236. flowfile_core/routes/user_defined_components.py +483 -4
  237. flowfile_core/run_lock.py +0 -1
  238. flowfile_core/schemas/__init__.py +4 -6
  239. flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
  240. flowfile_core/schemas/cloud_storage_schemas.py +59 -53
  241. flowfile_core/schemas/input_schema.py +231 -144
  242. flowfile_core/schemas/output_model.py +49 -34
  243. flowfile_core/schemas/schemas.py +116 -89
  244. flowfile_core/schemas/transform_schema.py +518 -263
  245. flowfile_core/schemas/yaml_types.py +21 -7
  246. flowfile_core/secret_manager/secret_manager.py +17 -13
  247. flowfile_core/types.py +29 -9
  248. flowfile_core/utils/arrow_reader.py +7 -6
  249. flowfile_core/utils/excel_file_manager.py +3 -3
  250. flowfile_core/utils/fileManager.py +7 -7
  251. flowfile_core/utils/fl_executor.py +8 -10
  252. flowfile_core/utils/utils.py +4 -4
  253. flowfile_core/utils/validate_setup.py +5 -4
  254. flowfile_frame/__init__.py +106 -51
  255. flowfile_frame/adapters.py +2 -9
  256. flowfile_frame/adding_expr.py +73 -32
  257. flowfile_frame/cloud_storage/frame_helpers.py +27 -23
  258. flowfile_frame/cloud_storage/secret_manager.py +12 -26
  259. flowfile_frame/config.py +2 -5
  260. flowfile_frame/expr.py +311 -218
  261. flowfile_frame/expr.pyi +160 -159
  262. flowfile_frame/expr_name.py +23 -23
  263. flowfile_frame/flow_frame.py +571 -476
  264. flowfile_frame/flow_frame.pyi +123 -104
  265. flowfile_frame/flow_frame_methods.py +227 -246
  266. flowfile_frame/group_frame.py +50 -20
  267. flowfile_frame/join.py +2 -2
  268. flowfile_frame/lazy.py +129 -87
  269. flowfile_frame/lazy_methods.py +83 -30
  270. flowfile_frame/list_name_space.py +55 -50
  271. flowfile_frame/selectors.py +148 -68
  272. flowfile_frame/series.py +9 -7
  273. flowfile_frame/utils.py +19 -21
  274. flowfile_worker/__init__.py +12 -7
  275. flowfile_worker/configs.py +11 -19
  276. flowfile_worker/create/__init__.py +14 -9
  277. flowfile_worker/create/funcs.py +114 -77
  278. flowfile_worker/create/models.py +46 -43
  279. flowfile_worker/create/pl_types.py +14 -15
  280. flowfile_worker/create/read_excel_tables.py +34 -41
  281. flowfile_worker/create/utils.py +22 -19
  282. flowfile_worker/external_sources/s3_source/main.py +18 -51
  283. flowfile_worker/external_sources/s3_source/models.py +34 -27
  284. flowfile_worker/external_sources/sql_source/main.py +8 -5
  285. flowfile_worker/external_sources/sql_source/models.py +13 -9
  286. flowfile_worker/flow_logger.py +10 -8
  287. flowfile_worker/funcs.py +214 -155
  288. flowfile_worker/main.py +11 -17
  289. flowfile_worker/models.py +35 -28
  290. flowfile_worker/process_manager.py +2 -3
  291. flowfile_worker/routes.py +121 -90
  292. flowfile_worker/secrets.py +9 -6
  293. flowfile_worker/spawner.py +80 -49
  294. flowfile_worker/utils.py +3 -2
  295. shared/__init__.py +2 -7
  296. shared/storage_config.py +25 -13
  297. test_utils/postgres/commands.py +3 -2
  298. test_utils/postgres/fixtures.py +9 -9
  299. test_utils/s3/commands.py +1 -1
  300. test_utils/s3/data_generator.py +3 -4
  301. test_utils/s3/demo_data_generator.py +4 -7
  302. test_utils/s3/fixtures.py +7 -5
  303. tools/migrate/__init__.py +1 -1
  304. tools/migrate/__main__.py +16 -29
  305. tools/migrate/legacy_schemas.py +251 -190
  306. tools/migrate/migrate.py +193 -181
  307. tools/migrate/tests/conftest.py +1 -3
  308. tools/migrate/tests/test_migrate.py +36 -41
  309. tools/migrate/tests/test_migration_e2e.py +28 -29
  310. tools/migrate/tests/test_node_migrations.py +50 -20
  311. flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
  312. flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
  313. flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
  314. flowfile/web/static/assets/Filter-9b6d08db.js +0 -164
  315. flowfile/web/static/assets/Filter-f62091b3.css +0 -20
  316. flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
  317. flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
  318. flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
  319. flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
  320. flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
  321. flowfile/web/static/assets/nodeInput-5d0d6b79.js +0 -41
  322. flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
  323. flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
  324. flowfile/web/static/assets/secretApi-68435402.js +0 -46
  325. flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
  326. flowfile-0.5.1.dist-info/RECORD +0 -388
  327. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/WHEEL +0 -0
  328. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/entry_points.txt +0 -0
  329. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/licenses/LICENSE +0 -0
@@ -1,51 +1,138 @@
1
1
  # Comprehensive mapping from SQLAlchemy types to Polars types
2
- from typing import Dict, Type, Union, cast, TYPE_CHECKING, Any
3
- from pydantic import SecretStr
2
+ from typing import TYPE_CHECKING, Any, Union, cast
3
+ from urllib.parse import quote_plus
4
4
 
5
5
  import polars as pl
6
6
  from polars import DataType as PolarsType
7
+ from pydantic import SecretStr
7
8
  from sqlalchemy.sql.sqltypes import (
8
- _Binary, ARRAY, BIGINT, BigInteger, BINARY, BLOB, BOOLEAN, Boolean,
9
- CHAR, CLOB, Concatenable, DATE, Date, DATETIME, DateTime,
10
- DECIMAL, DOUBLE, Double, DOUBLE_PRECISION, Enum, FLOAT, Float,
11
- Indexable, INT, INTEGER, Integer, Interval, JSON, LargeBinary,
12
- MatchType, NCHAR, NULLTYPE, NullType, NUMERIC, Numeric, NVARCHAR,
13
- PickleType, REAL, SchemaType, SMALLINT, SmallInteger, String,
14
- STRINGTYPE, TEXT, Text, TIME, Time, TIMESTAMP, TupleType,
15
- Unicode, UnicodeText, UUID, Uuid, VARBINARY, VARCHAR
9
+ ARRAY,
10
+ BIGINT,
11
+ BINARY,
12
+ BLOB,
13
+ BOOLEAN,
14
+ CHAR,
15
+ CLOB,
16
+ DATE,
17
+ DATETIME,
18
+ DECIMAL,
19
+ DOUBLE,
20
+ DOUBLE_PRECISION,
21
+ FLOAT,
22
+ INT,
23
+ INTEGER,
24
+ JSON,
25
+ NCHAR,
26
+ NULLTYPE,
27
+ NUMERIC,
28
+ NVARCHAR,
29
+ REAL,
30
+ SMALLINT,
31
+ STRINGTYPE,
32
+ TEXT,
33
+ TIME,
34
+ TIMESTAMP,
35
+ UUID,
36
+ VARBINARY,
37
+ VARCHAR,
38
+ BigInteger,
39
+ Boolean,
40
+ Concatenable,
41
+ Date,
42
+ DateTime,
43
+ Double,
44
+ Enum,
45
+ Float,
46
+ Indexable,
47
+ Integer,
48
+ Interval,
49
+ LargeBinary,
50
+ MatchType,
51
+ NullType,
52
+ Numeric,
53
+ PickleType,
54
+ SchemaType,
55
+ SmallInteger,
56
+ String,
57
+ Text,
58
+ Time,
59
+ TupleType,
60
+ Unicode,
61
+ UnicodeText,
62
+ Uuid,
63
+ _Binary,
16
64
  )
17
- from sqlalchemy.sql.type_api import (
18
- ExternalType, TypeDecorator,
19
- TypeEngine, UserDefinedType, Variant
20
- )
21
-
22
-
23
- from typing import Optional
24
- from urllib.parse import quote_plus
25
-
65
+ from sqlalchemy.sql.type_api import ExternalType, TypeDecorator, TypeEngine, UserDefinedType, Variant
26
66
 
27
67
  if TYPE_CHECKING:
28
68
  SqlType = Union[
29
- Type[_Binary], Type[ARRAY], Type[BIGINT], Type[BigInteger], Type[BINARY],
30
- Type[BLOB], Type[BOOLEAN], Type[Boolean], Type[CHAR], Type[CLOB],
31
- Type[Concatenable], Type[DATE], Type[Date], Type[DATETIME], Type[DateTime],
32
- Type[DECIMAL], Type[DOUBLE], Type[Double], Type[DOUBLE_PRECISION], Type[Enum],
33
- Type[FLOAT], Type[Float], Type[Indexable], Type[INT], Type[INTEGER],
34
- Type[Integer], Type[Interval], Type[JSON], Type[LargeBinary], Type[MatchType],
35
- Type[NCHAR], Type[NULLTYPE], Type[NullType], Type[NUMERIC], Type[Numeric],
36
- Type[NVARCHAR], Type[PickleType], Type[REAL], Type[SchemaType], Type[SMALLINT],
37
- Type[SmallInteger], Type[String], Type[STRINGTYPE], Type[TEXT], Type[Text],
38
- Type[TIME], Type[Time], Type[TIMESTAMP], Type[TupleType], Type[Unicode],
39
- Type[UnicodeText], Type[UUID], Type[Uuid], Type[VARBINARY], Type[VARCHAR],
40
- Type[TypeDecorator], Type[TypeEngine], Type[UserDefinedType], Type[Variant],
41
- Type[ExternalType]
69
+ type[_Binary],
70
+ type[ARRAY],
71
+ type[BIGINT],
72
+ type[BigInteger],
73
+ type[BINARY],
74
+ type[BLOB],
75
+ type[BOOLEAN],
76
+ type[Boolean],
77
+ type[CHAR],
78
+ type[CLOB],
79
+ type[Concatenable],
80
+ type[DATE],
81
+ type[Date],
82
+ type[DATETIME],
83
+ type[DateTime],
84
+ type[DECIMAL],
85
+ type[DOUBLE],
86
+ type[Double],
87
+ type[DOUBLE_PRECISION],
88
+ type[Enum],
89
+ type[FLOAT],
90
+ type[Float],
91
+ type[Indexable],
92
+ type[INT],
93
+ type[INTEGER],
94
+ type[Integer],
95
+ type[Interval],
96
+ type[JSON],
97
+ type[LargeBinary],
98
+ type[MatchType],
99
+ type[NCHAR],
100
+ type[NULLTYPE],
101
+ type[NullType],
102
+ type[NUMERIC],
103
+ type[Numeric],
104
+ type[NVARCHAR],
105
+ type[PickleType],
106
+ type[REAL],
107
+ type[SchemaType],
108
+ type[SMALLINT],
109
+ type[SmallInteger],
110
+ type[String],
111
+ type[STRINGTYPE],
112
+ type[TEXT],
113
+ type[Text],
114
+ type[TIME],
115
+ type[Time],
116
+ type[TIMESTAMP],
117
+ type[TupleType],
118
+ type[Unicode],
119
+ type[UnicodeText],
120
+ type[UUID],
121
+ type[Uuid],
122
+ type[VARBINARY],
123
+ type[VARCHAR],
124
+ type[TypeDecorator],
125
+ type[TypeEngine],
126
+ type[UserDefinedType],
127
+ type[Variant],
128
+ type[ExternalType],
42
129
  ]
43
130
  else:
44
131
  SqlType = Any
45
132
 
46
133
 
47
134
  # Comprehensive mapping from SQLAlchemy types to Polars types
48
- sqlalchemy_to_polars: Dict[SqlType, PolarsType] = {
135
+ sqlalchemy_to_polars: dict[SqlType, PolarsType] = {
49
136
  # Numeric types
50
137
  Integer: pl.Int64,
51
138
  INTEGER: pl.Int64,
@@ -65,7 +152,6 @@ sqlalchemy_to_polars: Dict[SqlType, PolarsType] = {
65
152
  DECIMAL: pl.Decimal,
66
153
  Boolean: pl.Boolean,
67
154
  BOOLEAN: pl.Boolean,
68
-
69
155
  # String types
70
156
  String: pl.Utf8,
71
157
  VARCHAR: pl.Utf8,
@@ -78,7 +164,6 @@ sqlalchemy_to_polars: Dict[SqlType, PolarsType] = {
78
164
  STRINGTYPE: pl.Utf8,
79
165
  Unicode: pl.Utf8,
80
166
  UnicodeText: pl.Utf8,
81
-
82
167
  # Date/Time types
83
168
  Date: pl.Date,
84
169
  DATE: pl.Date,
@@ -88,27 +173,22 @@ sqlalchemy_to_polars: Dict[SqlType, PolarsType] = {
88
173
  Time: pl.Time,
89
174
  TIME: pl.Time,
90
175
  Interval: pl.Duration,
91
-
92
176
  # Binary types
93
177
  _Binary: pl.Binary,
94
178
  LargeBinary: pl.Binary,
95
179
  BINARY: pl.Binary,
96
180
  VARBINARY: pl.Binary,
97
181
  BLOB: pl.Binary,
98
-
99
182
  # JSON types
100
183
  JSON: pl.Utf8, # Polars doesn't have a dedicated JSON type, using Utf8
101
-
102
184
  # UUID types
103
185
  UUID: pl.Utf8, # Mapped to string
104
186
  Uuid: pl.Utf8, # Mapped to string
105
-
106
187
  # Other types
107
188
  ARRAY: pl.List, # Approx mapping
108
189
  Enum: pl.String, # Approx mapping
109
190
  PickleType: pl.Object, # For storing Python objects
110
191
  TupleType: pl.Struct, # Mapped to struct
111
-
112
192
  # Special/Abstract types
113
193
  NULLTYPE: None,
114
194
  NullType: None,
@@ -124,165 +204,139 @@ sqlalchemy_to_polars: Dict[SqlType, PolarsType] = {
124
204
  }
125
205
 
126
206
  # Create string mappings, filtering out None values
127
- sqlalchemy_to_polars_str: Dict[str, str] = {
207
+ sqlalchemy_to_polars_str: dict[str, str] = {
128
208
  k.__name__: v.__name__
129
209
  for k, v in sqlalchemy_to_polars.items()
130
- if v is not None and hasattr(k, '__name__') and hasattr(v, '__name__')
210
+ if v is not None and hasattr(k, "__name__") and hasattr(v, "__name__")
131
211
  }
132
212
 
133
213
  # Additional string mappings for common SQL type names
134
- sql_type_name_to_polars: Dict[str, PolarsType] = {
135
- # PostgreSQL types
136
- 'integer': pl.Int64,
137
- 'bigint': pl.Int64,
138
- 'smallint': pl.Int64,
139
- 'numeric': pl.Decimal,
140
- 'real': pl.Float32,
141
- 'double precision': pl.Float64,
142
- 'boolean': pl.Boolean,
143
- 'varchar': pl.Utf8,
144
- 'character varying': pl.Utf8,
145
- 'character': pl.Utf8,
146
- 'text': pl.Utf8,
147
- 'date': pl.Date,
148
- 'timestamp': pl.Datetime,
149
- 'timestamp without time zone': pl.Datetime,
150
- 'timestamp with time zone': pl.Datetime,
151
- 'time': pl.Time,
152
- 'time without time zone': pl.Time,
153
- 'time with time zone': pl.Time,
154
- 'interval': pl.Duration,
155
- 'bytea': pl.Binary,
156
- 'jsonb': pl.Utf8,
157
- 'json': pl.Utf8,
158
- 'uuid': pl.Utf8,
159
- 'cidr': pl.Utf8,
160
- 'inet': pl.Utf8,
161
- 'macaddr': pl.Utf8,
162
- 'bit': pl.Utf8,
163
- 'bit varying': pl.Utf8,
164
- 'money': pl.Decimal,
165
- 'xml': pl.Utf8,
166
- 'tsquery': pl.Utf8,
167
- 'tsvector': pl.Utf8,
168
- 'hstore': pl.Utf8,
169
-
170
- # MySQL types
171
- 'int': pl.Int32,
172
- 'int unsigned': pl.UInt64,
173
- 'bigint unsigned': pl.UInt64,
174
- 'smallint unsigned': pl.UInt16,
175
- 'tinyint': pl.Int8,
176
- 'tinyint unsigned': pl.UInt8,
177
- 'mediumint': pl.Int32,
178
- 'mediumint unsigned': pl.UInt32,
179
- 'decimal': pl.Decimal,
180
- 'float': pl.Float32,
181
- 'double': pl.Float64,
182
- 'bit': pl.Boolean,
183
- 'char': pl.Utf8,
184
- 'varchar': pl.Utf8,
185
- 'binary': pl.Binary,
186
- 'varbinary': pl.Binary,
187
- 'tinyblob': pl.Binary,
188
- 'blob': pl.Binary,
189
- 'mediumblob': pl.Binary,
190
- 'longblob': pl.Binary,
191
- 'tinytext': pl.Utf8,
192
- 'text': pl.Utf8,
193
- 'mediumtext': pl.Utf8,
194
- 'longtext': pl.Utf8,
195
- 'datetime': pl.Datetime,
196
- 'timestamp': pl.Datetime,
197
- 'year': pl.Int16,
198
- 'enum': pl.String,
199
- 'set': pl.List,
200
- 'json': pl.Utf8,
201
-
202
- # SQLite types
203
- 'integer': pl.Int64, # SQLite's INTEGER is 64-bit
204
- 'real': pl.Float64,
205
- 'text': pl.Utf8,
206
- 'blob': pl.Binary,
207
- 'null': None,
208
-
209
- # Oracle types
210
- 'number': pl.Decimal,
211
- 'float': pl.Float64,
212
- 'binary_float': pl.Float32,
213
- 'binary_double': pl.Float64,
214
- 'varchar2': pl.Utf8,
215
- 'nvarchar2': pl.Utf8,
216
- 'char': pl.Utf8,
217
- 'nchar': pl.Utf8,
218
- 'clob': pl.Utf8,
219
- 'nclob': pl.Utf8,
220
- 'long': pl.Utf8,
221
- 'raw': pl.Binary,
222
- 'long raw': pl.Binary,
223
- 'rowid': pl.Utf8,
224
- 'urowid': pl.Utf8,
225
- 'date': pl.Datetime, # Oracle DATE includes time
226
- 'timestamp': pl.Datetime,
227
- 'timestamp with time zone': pl.Datetime,
228
- 'timestamp with local time zone': pl.Datetime,
229
- 'interval year to month': pl.Duration,
230
- 'interval day to second': pl.Duration,
231
- 'bfile': pl.Binary,
232
- 'xmltype': pl.Utf8,
233
-
234
- # SQL Server types
235
- 'bit': pl.Boolean,
236
- 'tinyint': pl.Int8,
237
- 'smallint': pl.Int16,
238
- 'int': pl.Int32,
239
- 'bigint': pl.Int64,
240
- 'numeric': pl.Decimal,
241
- 'decimal': pl.Decimal,
242
- 'smallmoney': pl.Decimal,
243
- 'money': pl.Decimal,
244
- 'float': pl.Float64,
245
- 'real': pl.Float32,
246
- 'datetime': pl.Datetime,
247
- 'datetime2': pl.Datetime,
248
- 'smalldatetime': pl.Datetime,
249
- 'date': pl.Date,
250
- 'time': pl.Time,
251
- 'datetimeoffset': pl.Datetime,
252
- 'char': pl.Utf8,
253
- 'varchar': pl.Utf8,
254
- 'text': pl.Utf8,
255
- 'nchar': pl.Utf8,
256
- 'nvarchar': pl.Utf8,
257
- 'ntext': pl.Utf8,
258
- 'binary': pl.Binary,
259
- 'varbinary': pl.Binary,
260
- 'image': pl.Binary,
261
- 'uniqueidentifier': pl.Utf8,
262
- 'xml': pl.Utf8,
263
- 'sql_variant': pl.Object,
264
- 'hierarchyid': pl.Utf8,
265
- 'geometry': pl.Utf8,
266
- 'geography': pl.Utf8,
267
-
268
- # Common abbreviations and aliases
269
- 'int4': pl.Int32,
270
- 'int8': pl.Int64,
271
- 'float4': pl.Float32,
272
- 'float8': pl.Float64,
273
- 'bool': pl.Boolean,
274
- 'serial': pl.Int32, # PostgreSQL auto-incrementing integer
275
- 'bigserial': pl.Int64, # PostgreSQL auto-incrementing bigint
276
- 'smallserial': pl.Int16, # PostgreSQL auto-incrementing smallint
214
+ sql_type_name_to_polars: dict[str, PolarsType] = {
215
+ # --- Integers ---
216
+ "int": pl.Int32,
217
+ "integer": pl.Int64,
218
+ "int4": pl.Int32,
219
+ "int8": pl.Int64,
220
+ "bigint": pl.Int64,
221
+ "short": pl.Int16,
222
+ "smallint": pl.Int16,
223
+ "tinyint": pl.Int8,
224
+ "mediumint": pl.Int32,
225
+ "serial": pl.Int32,
226
+ "bigserial": pl.Int64,
227
+ "smallserial": pl.Int16,
228
+ # Unsigned (MySQL specific)
229
+ "int unsigned": pl.UInt64,
230
+ "bigint unsigned": pl.UInt64,
231
+ "smallint unsigned": pl.UInt16,
232
+ "tinyint unsigned": pl.UInt8,
233
+ "mediumint unsigned": pl.UInt32,
234
+ "year": pl.Int16,
235
+
236
+ # --- Floats & Decimals ---
237
+ "numeric": pl.Decimal,
238
+ "decimal": pl.Decimal,
239
+ "number": pl.Decimal, # Oracle
240
+ "money": pl.Decimal,
241
+ "smallmoney": pl.Decimal,
242
+ "real": pl.Float32,
243
+ "float": pl.Float64,
244
+ "float4": pl.Float32,
245
+ "float8": pl.Float64,
246
+ "double": pl.Float64,
247
+ "double precision": pl.Float64,
248
+ "binary_float": pl.Float32, # Oracle
249
+ "binary_double": pl.Float64, # Oracle
250
+
251
+ # --- Booleans ---
252
+ "boolean": pl.Boolean,
253
+ "bool": pl.Boolean,
254
+ "bit": pl.Boolean, # Note: PostgreSQL 'bit' is varying, but MSSQL/MySQL 'bit' is boolean. Defaulting to Bool.
255
+
256
+ # --- Strings / Text ---
257
+ "varchar": pl.Utf8,
258
+ "varchar2": pl.Utf8, # Oracle
259
+ "nvarchar": pl.Utf8,
260
+ "nvarchar2": pl.Utf8, # Oracle
261
+ "char": pl.Utf8,
262
+ "nchar": pl.Utf8,
263
+ "character": pl.Utf8,
264
+ "character varying": pl.Utf8,
265
+ "text": pl.Utf8,
266
+ "tinytext": pl.Utf8,
267
+ "mediumtext": pl.Utf8,
268
+ "longtext": pl.Utf8,
269
+ "ntext": pl.Utf8,
270
+ "clob": pl.Utf8,
271
+ "nclob": pl.Utf8,
272
+ "long": pl.Utf8, # Oracle
273
+ "enum": pl.String,
274
+ "set": pl.List,
275
+ "rowid": pl.Utf8, # Oracle
276
+ "urowid": pl.Utf8, # Oracle
277
+ "uniqueidentifier": pl.Utf8, # MSSQL
278
+ "xml": pl.Utf8,
279
+ "xmltype": pl.Utf8,
280
+ "json": pl.Utf8,
281
+ "jsonb": pl.Utf8,
282
+
283
+ # --- Network / Specialized Strings (Postgres) ---
284
+ "uuid": pl.Utf8,
285
+ "cidr": pl.Utf8,
286
+ "inet": pl.Utf8,
287
+ "macaddr": pl.Utf8,
288
+ "tsquery": pl.Utf8,
289
+ "tsvector": pl.Utf8,
290
+ "hstore": pl.Utf8,
291
+ "geometry": pl.Utf8,
292
+ "geography": pl.Utf8,
293
+ "hierarchyid": pl.Utf8,
294
+ "bit varying": pl.Utf8,
295
+
296
+ # --- Dates & Times ---
297
+ "date": pl.Date,
298
+ "datetime": pl.Datetime,
299
+ "datetime2": pl.Datetime, # MSSQL
300
+ "smalldatetime": pl.Datetime, # MSSQL
301
+ "timestamp": pl.Datetime,
302
+ "timestamp without time zone": pl.Datetime,
303
+ "timestamp with time zone": pl.Datetime,
304
+ "timestamp with local time zone": pl.Datetime,
305
+ "datetimeoffset": pl.Datetime, # MSSQL
306
+ "time": pl.Time,
307
+ "time without time zone": pl.Time,
308
+ "time with time zone": pl.Time,
309
+
310
+ # --- Durations / Intervals ---
311
+ "interval": pl.Duration,
312
+ "interval year to month": pl.Duration, # Oracle
313
+ "interval day to second": pl.Duration, # Oracle
314
+
315
+ # --- Binary ---
316
+ "bytea": pl.Binary, # Postgres
317
+ "binary": pl.Binary,
318
+ "varbinary": pl.Binary,
319
+ "blob": pl.Binary,
320
+ "tinyblob": pl.Binary,
321
+ "mediumblob": pl.Binary,
322
+ "longblob": pl.Binary,
323
+ "raw": pl.Binary, # Oracle
324
+ "long raw": pl.Binary, # Oracle
325
+ "bfile": pl.Binary, # Oracle
326
+ "image": pl.Binary, # MSSQL
327
+
328
+ # --- Other ---
329
+ "null": None,
330
+ "sql_variant": pl.Object,
277
331
  }
278
332
 
279
333
  # String to string mapping
280
- sql_type_name_to_polars_str: Dict[str, str] = {
334
+ sql_type_name_to_polars_str: dict[str, str] = {
281
335
  k: v.__name__ for k, v in sql_type_name_to_polars.items() if v is not None
282
336
  }
283
337
 
284
338
 
285
- def get_polars_type(sqlalchemy_type: Union[SqlType, str]):
339
+ def get_polars_type(sqlalchemy_type: SqlType | str):
286
340
  """
287
341
  Get the corresponding Polars type from a SQLAlchemy type or string type name.
288
342
 
@@ -309,14 +363,14 @@ def get_polars_type(sqlalchemy_type: Union[SqlType, str]):
309
363
 
310
364
 
311
365
  def construct_sql_uri(
312
- database_type: str = "postgresql",
313
- host: Optional[str] = None,
314
- port: Optional[int] = None,
315
- username: Optional[str] = None,
316
- password: Optional[SecretStr] = None,
317
- database: Optional[str] = None,
318
- url: Optional[str] = None,
319
- **kwargs
366
+ database_type: str = "postgresql",
367
+ host: str | None = None,
368
+ port: int | None = None,
369
+ username: str | None = None,
370
+ password: SecretStr | None = None,
371
+ database: str | None = None,
372
+ url: str | None = None,
373
+ **kwargs,
320
374
  ) -> str:
321
375
  """
322
376
  Constructs a SQL URI string from the provided parameters.
@@ -376,4 +430,4 @@ def construct_sql_uri(
376
430
  params = "&".join(f"{key}={quote_plus(str(value))}" for key, value in kwargs.items())
377
431
  base_uri += f"?{params}"
378
432
 
379
- return base_uri
433
+ return base_uri
@@ -1,18 +1,14 @@
1
1
  import logging
2
- import math
3
2
  from collections import defaultdict, deque
4
- from typing import List, Dict, Set, Tuple, TYPE_CHECKING
3
+ from typing import TYPE_CHECKING
5
4
 
6
5
  if TYPE_CHECKING:
7
6
  from flowfile_core.flowfile.flow_graph import FlowGraph
8
7
 
9
8
 
10
9
  def calculate_layered_layout(
11
- graph: 'FlowGraph',
12
- x_spacing: int = 250,
13
- y_spacing: int = 100,
14
- initial_y: int = 50
15
- ) -> Dict[int, Tuple[int, int]]:
10
+ graph: "FlowGraph", x_spacing: int = 250, y_spacing: int = 100, initial_y: int = 50
11
+ ) -> dict[int, tuple[int, int]]:
16
12
  """
17
13
  Calculates node positions using a simplified layered approach for a
18
14
  LEFT-TO-RIGHT flow. Stages determine horizontal position (X), and
@@ -32,9 +28,9 @@ def calculate_layered_layout(
32
28
 
33
29
  nodes = graph.nodes
34
30
  node_ids = {node.node_id for node in nodes}
35
- adj: Dict[int, List[int]] = defaultdict(list)
36
- rev_adj: Dict[int, List[int]] = defaultdict(list)
37
- in_degree: Dict[int, int] = defaultdict(int)
31
+ adj: dict[int, list[int]] = defaultdict(list)
32
+ rev_adj: dict[int, list[int]] = defaultdict(list)
33
+ in_degree: dict[int, int] = defaultdict(int)
38
34
 
39
35
  # --- Graph Building Stage ---
40
36
  try:
@@ -65,8 +61,8 @@ def calculate_layered_layout(
65
61
  adj[node.node_id].append(child_node.node_id)
66
62
  in_degree[child_node.node_id] += 1
67
63
 
68
- stages: Dict[int, List[int]] = defaultdict(list)
69
- node_stage: Dict[int, int] = {}
64
+ stages: dict[int, list[int]] = defaultdict(list)
65
+ node_stage: dict[int, int] = {}
70
66
  initial_sources = sorted([node_id for node_id in node_ids if in_degree.get(node_id, 0) == 0])
71
67
  queue = deque(initial_sources)
72
68
  current_stage = 0
@@ -105,7 +101,7 @@ def calculate_layered_layout(
105
101
  current_stage += 1
106
102
 
107
103
  # --- Coordinate Assignment Stage ---
108
- positions: Dict[int, Tuple[int, int]] = {}
104
+ positions: dict[int, tuple[int, int]] = {}
109
105
  max_stage_height = 0
110
106
 
111
107
  for stage_index, node_ids_in_stage in stages.items():
@@ -1,19 +1,20 @@
1
- from typing import List, Dict, Set
2
- from flowfile_core.flowfile.flow_node.flow_node import FlowNode
1
+ from collections import defaultdict, deque
2
+
3
3
  from flowfile_core.configs import logger
4
- from collections import deque, defaultdict
4
+ from flowfile_core.flowfile.flow_node.flow_node import FlowNode
5
5
  from flowfile_core.flowfile.util.node_skipper import determine_nodes_to_skip
6
6
 
7
- def compute_execution_plan(nodes: List[FlowNode], flow_starts: List[FlowNode] = None):
8
- """ Computes the execution order after finding the nodes to skip on the execution step."""
7
+
8
+ def compute_execution_plan(nodes: list[FlowNode], flow_starts: list[FlowNode] = None):
9
+ """Computes the execution order after finding the nodes to skip on the execution step."""
9
10
  skip_nodes = determine_nodes_to_skip(nodes=nodes)
10
- computed_execution_order = determine_execution_order(all_nodes=[node for node in nodes if node not in skip_nodes],
11
- flow_starts=flow_starts)
11
+ computed_execution_order = determine_execution_order(
12
+ all_nodes=[node for node in nodes if node not in skip_nodes], flow_starts=flow_starts
13
+ )
12
14
  return skip_nodes, computed_execution_order
13
15
 
14
16
 
15
-
16
- def determine_execution_order(all_nodes: List[FlowNode], flow_starts: List[FlowNode] = None) -> List[FlowNode]:
17
+ def determine_execution_order(all_nodes: list[FlowNode], flow_starts: list[FlowNode] = None) -> list[FlowNode]:
17
18
  """
18
19
  Determines the execution order of nodes using topological sorting based on node dependencies.
19
20
 
@@ -41,7 +42,7 @@ def determine_execution_order(all_nodes: List[FlowNode], flow_starts: List[FlowN
41
42
  return execution_order
42
43
 
43
44
 
44
- def build_node_map(all_nodes: List[FlowNode]) -> Dict[str, FlowNode]:
45
+ def build_node_map(all_nodes: list[FlowNode]) -> dict[str, FlowNode]:
45
46
  """
46
47
  Creates a mapping from node ID to node object.
47
48
 
@@ -54,8 +55,9 @@ def build_node_map(all_nodes: List[FlowNode]) -> Dict[str, FlowNode]:
54
55
  return {node.node_id: node for node in all_nodes}
55
56
 
56
57
 
57
- def compute_in_degrees_and_adjacency_list(all_nodes: List[FlowNode],
58
- node_map: Dict[str, FlowNode]) -> (Dict[str, int], Dict[str, List[str]]):
58
+ def compute_in_degrees_and_adjacency_list(
59
+ all_nodes: list[FlowNode], node_map: dict[str, FlowNode]
60
+ ) -> (dict[str, int], dict[str, list[str]]):
59
61
  """
60
62
  Computes the in-degree and adjacency list for all nodes.
61
63
 
@@ -81,8 +83,9 @@ def compute_in_degrees_and_adjacency_list(all_nodes: List[FlowNode],
81
83
  return in_degree, adjacency_list
82
84
 
83
85
 
84
- def initialize_queue(flow_starts: List[FlowNode], all_nodes: List[FlowNode], in_degree: Dict[str, int]) -> (
85
- deque, Set[str]):
86
+ def initialize_queue(
87
+ flow_starts: list[FlowNode], all_nodes: list[FlowNode], in_degree: dict[str, int]
88
+ ) -> (deque, set[str]):
86
89
  """
87
90
  Initializes the queue with nodes that have zero in-degree or based on specified flow start nodes.
88
91
 
@@ -115,8 +118,13 @@ deque, Set[str]):
115
118
  return queue, visited_nodes
116
119
 
117
120
 
118
- def perform_topological_sort(queue: deque, node_map: Dict[str, FlowNode], in_degree: Dict[str, int],
119
- adjacency_list: Dict[str, List[str]], visited_nodes: Set[str]) -> List[FlowNode]:
121
+ def perform_topological_sort(
122
+ queue: deque,
123
+ node_map: dict[str, FlowNode],
124
+ in_degree: dict[str, int],
125
+ adjacency_list: dict[str, list[str]],
126
+ visited_nodes: set[str],
127
+ ) -> list[FlowNode]:
120
128
  """
121
129
  Performs topological sorting to determine the execution order of nodes.
122
130
 
@@ -131,7 +139,7 @@ def perform_topological_sort(queue: deque, node_map: Dict[str, FlowNode], in_deg
131
139
  List[FlowNode]: A list of nodes in the order they should be executed.
132
140
  """
133
141
  execution_order = []
134
- logger.info('Starting topological sort to determine execution order')
142
+ logger.info("Starting topological sort to determine execution order")
135
143
 
136
144
  while queue:
137
145
  current_node_id = queue.popleft()