Flowfile 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (329) hide show
  1. build_backends/main.py +25 -22
  2. build_backends/main_prd.py +10 -19
  3. flowfile/__init__.py +178 -74
  4. flowfile/__main__.py +10 -7
  5. flowfile/api.py +51 -57
  6. flowfile/web/__init__.py +14 -9
  7. flowfile/web/static/assets/AdminView-49392a9a.js +713 -0
  8. flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
  9. flowfile/web/static/assets/CloudConnectionView-36bcd6df.css +72 -0
  10. flowfile/web/static/assets/{CloudConnectionManager-0dfba9f2.js → CloudConnectionView-f13f202b.js} +11 -11
  11. flowfile/web/static/assets/{CloudStorageReader-d5b1b6c9.js → CloudStorageReader-0023d4a5.js} +10 -8
  12. flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
  13. flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
  14. flowfile/web/static/assets/{CloudStorageWriter-00d87aad.js → CloudStorageWriter-8e781e11.js} +10 -8
  15. flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
  16. flowfile/web/static/assets/{ColumnSelector-4685e75d.js → ColumnSelector-8ad68ea9.js} +3 -5
  17. flowfile/web/static/assets/{ContextMenu-c13f91d0.css → ContextMenu-26d4dd27.css} +6 -6
  18. flowfile/web/static/assets/{ContextMenu-23e909da.js → ContextMenu-31ee57f0.js} +3 -3
  19. flowfile/web/static/assets/{ContextMenu-70ae0c79.js → ContextMenu-69a74055.js} +3 -3
  20. flowfile/web/static/assets/{ContextMenu-f149cf7c.js → ContextMenu-8e2051c6.js} +3 -3
  21. flowfile/web/static/assets/{ContextMenu-4c74eef1.css → ContextMenu-8ec1729e.css} +6 -6
  22. flowfile/web/static/assets/{ContextMenu-63cfa99b.css → ContextMenu-9b310c60.css} +6 -6
  23. flowfile/web/static/assets/{CrossJoin-702a3edd.js → CrossJoin-03df6938.js} +12 -10
  24. flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
  25. flowfile/web/static/assets/CustomNode-59e99a86.css +32 -0
  26. flowfile/web/static/assets/{CustomNode-b1519993.js → CustomNode-8479239b.js} +36 -24
  27. flowfile/web/static/assets/{DatabaseConnectionSettings-6f3e4ea5.js → DatabaseConnectionSettings-869e3efd.js} +5 -4
  28. flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-e91df89a.css} +13 -13
  29. flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-36898a00.css} +24 -24
  30. flowfile/web/static/assets/{DatabaseReader-d38c7295.js → DatabaseReader-c58b9552.js} +25 -15
  31. flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
  32. flowfile/web/static/assets/{DatabaseManager-cf5ef661.js → DatabaseView-d26a9140.js} +11 -11
  33. flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-217a99f1.css} +19 -19
  34. flowfile/web/static/assets/{DatabaseWriter-b04ef46a.js → DatabaseWriter-4d05ddc7.js} +17 -10
  35. flowfile/web/static/assets/{designer-8da3ba3a.css → DesignerView-a6d0ee84.css} +614 -546
  36. flowfile/web/static/assets/{designer-9633482a.js → DesignerView-e6f5c0e8.js} +1107 -3170
  37. flowfile/web/static/assets/{documentation-ca400224.js → DocumentationView-2e78ef1b.js} +5 -5
  38. flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-fd46c656.css} +7 -7
  39. flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
  40. flowfile/web/static/assets/{ExploreData-5fa10ed8.js → ExploreData-7b54caca.js} +18 -9
  41. flowfile/web/static/assets/{ExternalSource-d39af878.js → ExternalSource-3fa399b2.js} +9 -7
  42. flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-47ab05a3.css} +17 -17
  43. flowfile/web/static/assets/Filter-7494ea97.css +48 -0
  44. flowfile/web/static/assets/Filter-8cbbdbf3.js +287 -0
  45. flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
  46. flowfile/web/static/assets/{Formula-6b04fb1d.js → Formula-aac42b1e.js} +13 -11
  47. flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
  48. flowfile/web/static/assets/{FuzzyMatch-999521f4.js → FuzzyMatch-cd9bbfca.js} +12 -10
  49. flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-c24dec17.css} +5 -5
  50. flowfile/web/static/assets/{GraphSolver-17dd2198.js → GraphSolver-c7e6780e.js} +13 -11
  51. flowfile/web/static/assets/{GroupBy-6b039e18.js → GroupBy-93c5d22b.js} +9 -7
  52. flowfile/web/static/assets/{GroupBy-b9505323.css → GroupBy-be7ac0bf.css} +10 -10
  53. flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
  54. flowfile/web/static/assets/{Join-24d0f113.js → Join-a19b2de2.js} +13 -11
  55. flowfile/web/static/assets/LoginView-0df4ed0a.js +134 -0
  56. flowfile/web/static/assets/LoginView-d325d632.css +172 -0
  57. flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
  58. flowfile/web/static/assets/{ManualInput-34639209.js → ManualInput-8d3374b2.js} +170 -116
  59. flowfile/web/static/assets/{MultiSelect-0e8724a3.js → MultiSelect-ad1b6243.js} +2 -2
  60. flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-b0e538c2.js → MultiSelect.vue_vue_type_script_setup_true_lang-e278950d.js} +1 -1
  61. flowfile/web/static/assets/NodeDesigner-40b647c9.js +2610 -0
  62. flowfile/web/static/assets/NodeDesigner-5f53be3f.css +1429 -0
  63. flowfile/web/static/assets/{NumericInput-3d63a470.js → NumericInput-7100234c.js} +2 -2
  64. flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-e0edeccc.js → NumericInput.vue_vue_type_script_setup_true_lang-5130219f.js} +5 -2
  65. flowfile/web/static/assets/{Output-283fe388.css → Output-35e97000.css} +6 -6
  66. flowfile/web/static/assets/{Output-edea9802.js → Output-f5efd2aa.js} +12 -9
  67. flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
  68. flowfile/web/static/assets/{Pivot-61d19301.js → Pivot-d981d23c.js} +11 -9
  69. flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
  70. flowfile/web/static/assets/{PivotValidation-f97fec5b.js → PivotValidation-39386e95.js} +3 -3
  71. flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
  72. flowfile/web/static/assets/{PivotValidation-de9f43fe.js → PivotValidation-63de1f73.js} +3 -3
  73. flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
  74. flowfile/web/static/assets/{PolarsCode-bc3c9984.js → PolarsCode-f9d69217.js} +18 -9
  75. flowfile/web/static/assets/PopOver-b22f049e.js +939 -0
  76. flowfile/web/static/assets/PopOver-d96599db.css +33 -0
  77. flowfile/web/static/assets/{Read-e808b239.css → Read-36e7bd51.css} +12 -12
  78. flowfile/web/static/assets/{Read-64a3f259.js → Read-aec2e377.js} +14 -11
  79. flowfile/web/static/assets/{RecordCount-3d5039be.js → RecordCount-78ed6845.js} +6 -4
  80. flowfile/web/static/assets/{RecordId-597510e0.js → RecordId-2156e890.js} +8 -6
  81. flowfile/web/static/assets/{SQLQueryComponent-36cef432.css → SQLQueryComponent-1c2f26b4.css} +5 -5
  82. flowfile/web/static/assets/{SQLQueryComponent-df51adbe.js → SQLQueryComponent-48c72f5b.js} +3 -3
  83. flowfile/web/static/assets/{Sample-4be0a507.js → Sample-1352ca74.js} +6 -4
  84. flowfile/web/static/assets/SecretSelector-22b5ff89.js +113 -0
  85. flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
  86. flowfile/web/static/assets/{SecretManager-4839be57.js → SecretsView-17df66ee.js} +35 -36
  87. flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
  88. flowfile/web/static/assets/{Select-9b72f201.js → Select-0aee4c54.js} +9 -7
  89. flowfile/web/static/assets/{SettingsSection-f0f75a42.js → SettingsSection-0784e157.js} +3 -3
  90. flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
  91. flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
  92. flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
  93. flowfile/web/static/assets/{SettingsSection-e1e9c953.js → SettingsSection-cd341bb6.js} +3 -3
  94. flowfile/web/static/assets/{SettingsSection-7ded385d.js → SettingsSection-f2002a6d.js} +3 -3
  95. flowfile/web/static/assets/{SingleSelect-6c777aac.js → SingleSelect-460cc0ea.js} +2 -2
  96. flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-33e3ff9b.js → SingleSelect.vue_vue_type_script_setup_true_lang-30741bb2.js} +1 -1
  97. flowfile/web/static/assets/{SliderInput-7cb93e62.js → SliderInput-5d926864.js} +7 -4
  98. flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
  99. flowfile/web/static/assets/{Sort-6cbde21a.js → Sort-3cdc971b.js} +9 -7
  100. flowfile/web/static/assets/{Unique-f9fb0809.css → Sort-8a871341.css} +10 -10
  101. flowfile/web/static/assets/{TextInput-d9a40c11.js → TextInput-a2d0bfbd.js} +2 -2
  102. flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-5896c375.js → TextInput.vue_vue_type_script_setup_true_lang-abad1ca2.js} +5 -2
  103. flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
  104. flowfile/web/static/assets/{TextToRows-c4fcbf4d.js → TextToRows-918945f7.js} +11 -10
  105. flowfile/web/static/assets/{ToggleSwitch-4ef91d19.js → ToggleSwitch-f0ef5196.js} +2 -2
  106. flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-38478c20.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-5605c793.js} +1 -1
  107. flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-54d2f518.css} +6 -6
  108. flowfile/web/static/assets/{UnavailableFields-a03f512c.js → UnavailableFields-bdad6144.js} +4 -4
  109. flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
  110. flowfile/web/static/assets/{Union-bfe9b996.js → Union-e8ab8c86.js} +8 -6
  111. flowfile/web/static/assets/{Unique-5d023a27.js → Unique-8cd4f976.js} +13 -10
  112. flowfile/web/static/assets/{Sort-3643d625.css → Unique-9fb2f567.css} +10 -10
  113. flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-710a2948.css} +7 -7
  114. flowfile/web/static/assets/{Unpivot-91cc5354.js → Unpivot-8da14095.js} +10 -8
  115. flowfile/web/static/assets/{UnpivotValidation-7ee2de44.js → UnpivotValidation-6f7d89ff.js} +3 -3
  116. flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
  117. flowfile/web/static/assets/{VueGraphicWalker-e51b9924.js → VueGraphicWalker-3fb312e1.js} +4 -4
  118. flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
  119. flowfile/web/static/assets/{api-cf1221f0.js → api-24483f0d.js} +1 -1
  120. flowfile/web/static/assets/{api-c1bad5ca.js → api-8b81fa73.js} +1 -1
  121. flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-3d8dc5fa.css} +40 -40
  122. flowfile/web/static/assets/{dropDown-614b998d.js → dropDown-ac0fda9d.js} +3 -3
  123. flowfile/web/static/assets/{fullEditor-f7971590.js → fullEditor-5497a84a.js} +11 -10
  124. flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-a0be62b3.css} +74 -62
  125. flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
  126. flowfile/web/static/assets/{genericNodeSettings-4fe5f36b.js → genericNodeSettings-99014e1d.js} +5 -5
  127. flowfile/web/static/assets/index-07dda503.js +38 -0
  128. flowfile/web/static/assets/index-3ba44389.js +2696 -0
  129. flowfile/web/static/assets/{index-50508d4d.css → index-e6289dd0.css} +1945 -569
  130. flowfile/web/static/assets/{index-5429bbf8.js → index-fb6493ae.js} +41626 -40867
  131. flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
  132. flowfile/web/static/assets/nodeInput-0eb13f1a.js +2 -0
  133. flowfile/web/static/assets/{outputCsv-076b85ab.js → outputCsv-8f8ba42d.js} +3 -3
  134. flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
  135. flowfile/web/static/assets/{outputExcel-0fd17dbe.js → outputExcel-393f4fef.js} +3 -3
  136. flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
  137. flowfile/web/static/assets/{outputParquet-b61e0847.js → outputParquet-07c81f65.js} +4 -4
  138. flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
  139. flowfile/web/static/assets/{readCsv-a8bb8b61.js → readCsv-07f6d9ad.js} +3 -3
  140. flowfile/web/static/assets/{readCsv-c767cb37.css → readCsv-3bfac4c3.css} +15 -15
  141. flowfile/web/static/assets/{readExcel-806d2826.css → readExcel-3db6b763.css} +13 -13
  142. flowfile/web/static/assets/{readExcel-67b4aee0.js → readExcel-ed69bc8f.js} +5 -5
  143. flowfile/web/static/assets/{readParquet-48c81530.css → readParquet-c5244ad5.css} +4 -4
  144. flowfile/web/static/assets/{readParquet-92ce1dbc.js → readParquet-e3ed4528.js} +3 -3
  145. flowfile/web/static/assets/secrets.api-002e7d7e.js +65 -0
  146. flowfile/web/static/assets/{selectDynamic-92e25ee3.js → selectDynamic-80b92899.js} +5 -5
  147. flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
  148. flowfile/web/static/assets/{vue-codemirror.esm-41b0e0d7.js → vue-codemirror.esm-0965f39f.js} +31 -640
  149. flowfile/web/static/assets/{vue-content-loader.es-2c8e608f.js → vue-content-loader.es-c506ad97.js} +1 -1
  150. flowfile/web/static/index.html +2 -2
  151. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/METADATA +2 -3
  152. flowfile-0.5.3.dist-info/RECORD +402 -0
  153. flowfile_core/__init__.py +13 -6
  154. flowfile_core/auth/jwt.py +51 -16
  155. flowfile_core/auth/models.py +32 -7
  156. flowfile_core/auth/password.py +89 -0
  157. flowfile_core/auth/secrets.py +8 -6
  158. flowfile_core/configs/__init__.py +9 -7
  159. flowfile_core/configs/flow_logger.py +15 -14
  160. flowfile_core/configs/node_store/__init__.py +72 -4
  161. flowfile_core/configs/node_store/nodes.py +155 -172
  162. flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
  163. flowfile_core/configs/settings.py +28 -15
  164. flowfile_core/database/connection.py +7 -6
  165. flowfile_core/database/init_db.py +96 -2
  166. flowfile_core/database/models.py +3 -1
  167. flowfile_core/fileExplorer/__init__.py +17 -0
  168. flowfile_core/fileExplorer/funcs.py +123 -57
  169. flowfile_core/fileExplorer/utils.py +10 -11
  170. flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
  171. flowfile_core/flowfile/analytics/analytics_processor.py +26 -24
  172. flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
  173. flowfile_core/flowfile/analytics/utils.py +1 -1
  174. flowfile_core/flowfile/code_generator/code_generator.py +358 -244
  175. flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
  176. flowfile_core/flowfile/connection_manager/models.py +1 -1
  177. flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
  178. flowfile_core/flowfile/database_connection_manager/models.py +1 -1
  179. flowfile_core/flowfile/extensions.py +17 -12
  180. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
  181. flowfile_core/flowfile/flow_data_engine/create/funcs.py +115 -83
  182. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +481 -423
  183. flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
  184. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
  185. flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
  186. flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
  187. flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
  188. flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +31 -20
  189. flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
  190. flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
  191. flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +14 -15
  192. flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
  193. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
  194. flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
  195. flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
  196. flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
  197. flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
  198. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +190 -127
  199. flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
  200. flowfile_core/flowfile/flow_data_engine/utils.py +99 -67
  201. flowfile_core/flowfile/flow_graph.py +918 -571
  202. flowfile_core/flowfile/flow_graph_utils.py +31 -49
  203. flowfile_core/flowfile/flow_node/flow_node.py +330 -233
  204. flowfile_core/flowfile/flow_node/models.py +53 -41
  205. flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
  206. flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
  207. flowfile_core/flowfile/handler.py +80 -30
  208. flowfile_core/flowfile/manage/compatibility_enhancements.py +209 -126
  209. flowfile_core/flowfile/manage/io_flowfile.py +54 -57
  210. flowfile_core/flowfile/node_designer/__init__.py +15 -13
  211. flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
  212. flowfile_core/flowfile/node_designer/custom_node.py +162 -36
  213. flowfile_core/flowfile/node_designer/ui_components.py +135 -34
  214. flowfile_core/flowfile/schema_callbacks.py +71 -51
  215. flowfile_core/flowfile/setting_generator/__init__.py +0 -1
  216. flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
  217. flowfile_core/flowfile/setting_generator/settings.py +64 -53
  218. flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
  219. flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
  220. flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
  221. flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
  222. flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
  223. flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
  224. flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
  225. flowfile_core/flowfile/util/calculate_layout.py +9 -13
  226. flowfile_core/flowfile/util/execution_orderer.py +25 -17
  227. flowfile_core/flowfile/util/node_skipper.py +4 -4
  228. flowfile_core/flowfile/utils.py +19 -21
  229. flowfile_core/main.py +26 -19
  230. flowfile_core/routes/auth.py +284 -11
  231. flowfile_core/routes/cloud_connections.py +25 -25
  232. flowfile_core/routes/logs.py +21 -29
  233. flowfile_core/routes/public.py +3 -3
  234. flowfile_core/routes/routes.py +70 -34
  235. flowfile_core/routes/secrets.py +25 -27
  236. flowfile_core/routes/user_defined_components.py +483 -4
  237. flowfile_core/run_lock.py +0 -1
  238. flowfile_core/schemas/__init__.py +4 -6
  239. flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
  240. flowfile_core/schemas/cloud_storage_schemas.py +59 -53
  241. flowfile_core/schemas/input_schema.py +231 -144
  242. flowfile_core/schemas/output_model.py +49 -34
  243. flowfile_core/schemas/schemas.py +116 -89
  244. flowfile_core/schemas/transform_schema.py +518 -263
  245. flowfile_core/schemas/yaml_types.py +21 -7
  246. flowfile_core/secret_manager/secret_manager.py +17 -13
  247. flowfile_core/types.py +29 -9
  248. flowfile_core/utils/arrow_reader.py +7 -6
  249. flowfile_core/utils/excel_file_manager.py +3 -3
  250. flowfile_core/utils/fileManager.py +7 -7
  251. flowfile_core/utils/fl_executor.py +8 -10
  252. flowfile_core/utils/utils.py +4 -4
  253. flowfile_core/utils/validate_setup.py +5 -4
  254. flowfile_frame/__init__.py +106 -51
  255. flowfile_frame/adapters.py +2 -9
  256. flowfile_frame/adding_expr.py +73 -32
  257. flowfile_frame/cloud_storage/frame_helpers.py +27 -23
  258. flowfile_frame/cloud_storage/secret_manager.py +12 -26
  259. flowfile_frame/config.py +2 -5
  260. flowfile_frame/expr.py +311 -218
  261. flowfile_frame/expr.pyi +160 -159
  262. flowfile_frame/expr_name.py +23 -23
  263. flowfile_frame/flow_frame.py +571 -476
  264. flowfile_frame/flow_frame.pyi +123 -104
  265. flowfile_frame/flow_frame_methods.py +227 -246
  266. flowfile_frame/group_frame.py +50 -20
  267. flowfile_frame/join.py +2 -2
  268. flowfile_frame/lazy.py +129 -87
  269. flowfile_frame/lazy_methods.py +83 -30
  270. flowfile_frame/list_name_space.py +55 -50
  271. flowfile_frame/selectors.py +148 -68
  272. flowfile_frame/series.py +9 -7
  273. flowfile_frame/utils.py +19 -21
  274. flowfile_worker/__init__.py +12 -7
  275. flowfile_worker/configs.py +11 -19
  276. flowfile_worker/create/__init__.py +14 -9
  277. flowfile_worker/create/funcs.py +114 -77
  278. flowfile_worker/create/models.py +46 -43
  279. flowfile_worker/create/pl_types.py +14 -15
  280. flowfile_worker/create/read_excel_tables.py +34 -41
  281. flowfile_worker/create/utils.py +22 -19
  282. flowfile_worker/external_sources/s3_source/main.py +18 -51
  283. flowfile_worker/external_sources/s3_source/models.py +34 -27
  284. flowfile_worker/external_sources/sql_source/main.py +8 -5
  285. flowfile_worker/external_sources/sql_source/models.py +13 -9
  286. flowfile_worker/flow_logger.py +10 -8
  287. flowfile_worker/funcs.py +214 -155
  288. flowfile_worker/main.py +11 -17
  289. flowfile_worker/models.py +35 -28
  290. flowfile_worker/process_manager.py +2 -3
  291. flowfile_worker/routes.py +121 -90
  292. flowfile_worker/secrets.py +9 -6
  293. flowfile_worker/spawner.py +80 -49
  294. flowfile_worker/utils.py +3 -2
  295. shared/__init__.py +2 -7
  296. shared/storage_config.py +25 -13
  297. test_utils/postgres/commands.py +3 -2
  298. test_utils/postgres/fixtures.py +9 -9
  299. test_utils/s3/commands.py +1 -1
  300. test_utils/s3/data_generator.py +3 -4
  301. test_utils/s3/demo_data_generator.py +4 -7
  302. test_utils/s3/fixtures.py +7 -5
  303. tools/migrate/__init__.py +1 -1
  304. tools/migrate/__main__.py +16 -29
  305. tools/migrate/legacy_schemas.py +251 -190
  306. tools/migrate/migrate.py +193 -181
  307. tools/migrate/tests/conftest.py +1 -3
  308. tools/migrate/tests/test_migrate.py +36 -41
  309. tools/migrate/tests/test_migration_e2e.py +28 -29
  310. tools/migrate/tests/test_node_migrations.py +50 -20
  311. flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
  312. flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
  313. flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
  314. flowfile/web/static/assets/Filter-9b6d08db.js +0 -164
  315. flowfile/web/static/assets/Filter-f62091b3.css +0 -20
  316. flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
  317. flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
  318. flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
  319. flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
  320. flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
  321. flowfile/web/static/assets/nodeInput-5d0d6b79.js +0 -41
  322. flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
  323. flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
  324. flowfile/web/static/assets/secretApi-68435402.js +0 -46
  325. flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
  326. flowfile-0.5.1.dist-info/RECORD +0 -388
  327. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/WHEEL +0 -0
  328. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/entry_points.txt +0 -0
  329. {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/licenses/LICENSE +0 -0
@@ -16,37 +16,42 @@ DO NOT USE THESE IN PRODUCTION CODE - use the actual schemas from flowfile_core.
16
16
  """
17
17
 
18
18
  from dataclasses import dataclass, field
19
- from typing import List, Optional, Set, Any, Literal, Dict, Tuple
20
- from pydantic import BaseModel, Field
19
+ from typing import Any, Literal
21
20
 
21
+ from pydantic import BaseModel, Field
22
22
 
23
23
  # =============================================================================
24
24
  # OLD INPUT/OUTPUT SCHEMAS (before table_settings migration)
25
25
  # These mirror the structure that exists in old pickle files
26
26
  # =============================================================================
27
27
 
28
+
28
29
  class MinimalFieldInfo(BaseModel):
29
30
  """Represents the most basic information about a data field (column)."""
31
+
30
32
  name: str
31
33
  data_type: str = "String"
32
34
 
33
35
 
34
36
  class OutputCsvTable(BaseModel):
35
37
  """OLD: Settings for writing a CSV file."""
36
- file_type: str = 'csv'
37
- delimiter: str = ','
38
- encoding: str = 'utf-8'
38
+
39
+ file_type: str = "csv"
40
+ delimiter: str = ","
41
+ encoding: str = "utf-8"
39
42
 
40
43
 
41
44
  class OutputParquetTable(BaseModel):
42
45
  """OLD: Settings for writing a Parquet file."""
43
- file_type: str = 'parquet'
46
+
47
+ file_type: str = "parquet"
44
48
 
45
49
 
46
50
  class OutputExcelTable(BaseModel):
47
51
  """OLD: Settings for writing an Excel file."""
48
- file_type: str = 'excel'
49
- sheet_name: str = 'Sheet1'
52
+
53
+ file_type: str = "excel"
54
+ sheet_name: str = "Sheet1"
50
55
 
51
56
 
52
57
  class OutputSettings(BaseModel):
@@ -55,16 +60,17 @@ class OutputSettings(BaseModel):
55
60
  This is the OLD format where CSV, Parquet, and Excel settings
56
61
  were stored in separate fields rather than a unified table_settings.
57
62
  """
63
+
58
64
  name: str
59
65
  directory: str
60
66
  file_type: str
61
- fields: Optional[List[str]] = Field(default_factory=list)
62
- write_mode: str = 'overwrite'
67
+ fields: list[str] | None = Field(default_factory=list)
68
+ write_mode: str = "overwrite"
63
69
  # OLD: Separate fields for each output type
64
- output_csv_table: Optional[OutputCsvTable] = Field(default_factory=OutputCsvTable)
70
+ output_csv_table: OutputCsvTable | None = Field(default_factory=OutputCsvTable)
65
71
  output_parquet_table: OutputParquetTable = Field(default_factory=OutputParquetTable)
66
72
  output_excel_table: OutputExcelTable = Field(default_factory=OutputExcelTable)
67
- abs_file_path: Optional[str] = None
73
+ abs_file_path: str | None = None
68
74
 
69
75
 
70
76
  class ReceivedTable(BaseModel):
@@ -73,32 +79,33 @@ class ReceivedTable(BaseModel):
73
79
  This is the OLD format where all settings (CSV, Excel, Parquet)
74
80
  were stored as flat fields on the model rather than nested in table_settings.
75
81
  """
82
+
76
83
  # Metadata fields
77
- id: Optional[int] = None
78
- name: Optional[str] = None
79
- path: str = ''
80
- directory: Optional[str] = None
84
+ id: int | None = None
85
+ name: str | None = None
86
+ path: str = ""
87
+ directory: str | None = None
81
88
  analysis_file_available: bool = False
82
- status: Optional[str] = None
83
- file_type: Optional[str] = None
84
- fields: List[MinimalFieldInfo] = Field(default_factory=list)
85
- abs_file_path: Optional[str] = None
89
+ status: str | None = None
90
+ file_type: str | None = None
91
+ fields: list[MinimalFieldInfo] = Field(default_factory=list)
92
+ abs_file_path: str | None = None
86
93
 
87
94
  # OLD: CSV/JSON fields at top level (not nested)
88
- reference: str = ''
95
+ reference: str = ""
89
96
  starting_from_line: int = 0
90
- delimiter: str = ','
97
+ delimiter: str = ","
91
98
  has_headers: bool = True
92
- encoding: Optional[str] = 'utf-8'
93
- parquet_ref: Optional[str] = None
94
- row_delimiter: str = '\n'
99
+ encoding: str | None = "utf-8"
100
+ parquet_ref: str | None = None
101
+ row_delimiter: str = "\n"
95
102
  quote_char: str = '"'
96
103
  infer_schema_length: int = 10_000
97
104
  truncate_ragged_lines: bool = False
98
105
  ignore_errors: bool = False
99
106
 
100
107
  # OLD: Excel fields at top level (not nested)
101
- sheet_name: Optional[str] = None
108
+ sheet_name: str | None = None
102
109
  start_row: int = 0
103
110
  start_column: int = 0
104
111
  end_row: int = 0
@@ -110,21 +117,24 @@ class ReceivedTable(BaseModel):
110
117
  # FLOW AND NODE SCHEMAS (Pydantic - structure unchanged, just re-exported)
111
118
  # =============================================================================
112
119
 
120
+
113
121
  class FlowGraphConfig(BaseModel):
114
122
  """Configuration model for a flow graph's basic properties."""
123
+
115
124
  flow_id: int = 1
116
- description: Optional[str] = None
117
- save_location: Optional[str] = None
118
- name: str = ''
119
- path: str = ''
120
- execution_mode: str = 'Performance'
121
- execution_location: str = 'local'
125
+ description: str | None = None
126
+ save_location: str | None = None
127
+ name: str = ""
128
+ path: str = ""
129
+ execution_mode: str = "Performance"
130
+ execution_location: str = "local"
122
131
 
123
132
 
124
133
  class FlowSettings(FlowGraphConfig):
125
134
  """Extends FlowGraphConfig with additional operational settings."""
135
+
126
136
  auto_save: bool = False
127
- modified_on: Optional[float] = None
137
+ modified_on: float | None = None
128
138
  show_detailed_progress: bool = True
129
139
  is_running: bool = False
130
140
  is_canceled: bool = False
@@ -132,52 +142,60 @@ class FlowSettings(FlowGraphConfig):
132
142
 
133
143
  class NodeBase(BaseModel):
134
144
  """Base model for all nodes in a FlowGraph."""
145
+
135
146
  flow_id: int
136
147
  node_id: int
137
- cache_results: Optional[bool] = False
138
- pos_x: Optional[float] = 0
139
- pos_y: Optional[float] = 0
140
- is_setup: Optional[bool] = True
141
- description: Optional[str] = ''
142
- user_id: Optional[int] = None
143
- is_flow_output: Optional[bool] = False
144
- is_user_defined: Optional[bool] = False
148
+ cache_results: bool | None = False
149
+ pos_x: float | None = 0
150
+ pos_y: float | None = 0
151
+ is_setup: bool | None = True
152
+ description: str | None = ""
153
+ user_id: int | None = None
154
+ is_flow_output: bool | None = False
155
+ is_user_defined: bool | None = False
145
156
 
146
157
 
147
158
  class NodeSingleInput(NodeBase):
148
159
  """A base model for any node that takes a single data input."""
149
- depending_on_id: Optional[int] = -1
160
+
161
+ depending_on_id: int | None = -1
150
162
 
151
163
 
152
164
  class NodeMultiInput(NodeBase):
153
165
  """A base model for any node that takes multiple data inputs."""
154
- depending_on_ids: Optional[List[int]] = Field(default_factory=lambda: [-1])
166
+
167
+ depending_on_ids: list[int] | None = Field(default_factory=lambda: [-1])
155
168
 
156
169
 
157
170
  class NodeRead(NodeBase):
158
171
  """Settings for a node that reads data from a file."""
172
+
159
173
  received_file: ReceivedTable
160
174
 
161
175
 
162
176
  class NodeSelect(NodeSingleInput):
163
177
  """Settings for a node that selects, renames, and reorders columns."""
178
+
164
179
  keep_missing: bool = True
165
- select_input: List[Any] = Field(default_factory=list)
166
- sorted_by: Optional[str] = 'none'
180
+ select_input: list[Any] = Field(default_factory=list)
181
+ sorted_by: str | None = "none"
167
182
 
168
183
 
169
184
  class NodeFilter(NodeSingleInput):
170
185
  """Settings for a node that filters rows based on a condition."""
186
+
171
187
  filter_input: Any = None
172
188
 
173
189
 
174
190
  class NodeFormula(NodeSingleInput):
175
191
  """Settings for a node that applies a formula to create/modify a column."""
192
+
176
193
  function: Any = None
177
194
 
178
195
 
179
196
  class NodeJoin(NodeMultiInput):
180
197
  """Settings for a node that performs a standard SQL-style join."""
198
+
181
199
  auto_generate_selection: bool = True
182
200
  verify_integrity: bool = True
183
201
  join_input: Any = None
@@ -188,6 +206,7 @@ class NodeJoin(NodeMultiInput):
188
206
 
189
207
  class NodeCrossJoin(NodeMultiInput):
190
208
  """Settings for a node that performs a cross join."""
209
+
191
210
  auto_generate_selection: bool = True
192
211
  verify_integrity: bool = True
193
212
  cross_join_input: Any = None
@@ -198,6 +217,7 @@ class NodeCrossJoin(NodeMultiInput):
198
217
 
199
218
  class NodeFuzzyMatch(NodeMultiInput):
200
219
  """Settings for a node that performs a fuzzy join."""
220
+
201
221
  auto_generate_selection: bool = True
202
222
  verify_integrity: bool = True
203
223
  join_input: Any = None # FuzzyMatchInput
@@ -208,141 +228,161 @@ class NodeFuzzyMatch(NodeMultiInput):
208
228
 
209
229
  class NodePolarsCode(NodeMultiInput):
210
230
  """Settings for a node that executes arbitrary Polars code."""
231
+
211
232
  polars_code_input: Any = None
212
233
 
213
234
 
214
235
  class NodeOutput(NodeSingleInput):
215
236
  """Settings for a node that writes its input to a file."""
237
+
216
238
  output_settings: OutputSettings
217
239
 
218
240
 
219
241
  class NodeGroupBy(NodeSingleInput):
220
242
  """Settings for a node that performs a group-by and aggregation."""
243
+
221
244
  groupby_input: Any = None
222
245
 
223
246
 
224
247
  class NodeSort(NodeSingleInput):
225
248
  """Settings for a node that sorts the data."""
226
- sort_input: List[Any] = Field(default_factory=list)
249
+
250
+ sort_input: list[Any] = Field(default_factory=list)
227
251
 
228
252
 
229
253
  class NodeUnion(NodeMultiInput):
230
254
  """Settings for a node that concatenates multiple inputs."""
255
+
231
256
  union_input: Any = None
232
257
 
233
258
 
234
259
  class NodeUnique(NodeSingleInput):
235
260
  """Settings for a node that returns unique rows."""
261
+
236
262
  unique_input: Any = None
237
263
 
238
264
 
239
265
  class NodePivot(NodeSingleInput):
240
266
  """Settings for a node that pivots data."""
267
+
241
268
  pivot_input: Any = None
242
- output_fields: Optional[List[MinimalFieldInfo]] = None
269
+ output_fields: list[MinimalFieldInfo] | None = None
243
270
 
244
271
 
245
272
  class NodeUnpivot(NodeSingleInput):
246
273
  """Settings for a node that unpivots data."""
274
+
247
275
  unpivot_input: Any = None
248
276
 
249
277
 
250
278
  class NodeRecordId(NodeSingleInput):
251
279
  """Settings for adding a record ID column."""
280
+
252
281
  record_id_input: Any = None
253
282
 
254
283
 
255
284
  class NodeTextToRows(NodeSingleInput):
256
285
  """Settings for splitting text into rows."""
286
+
257
287
  text_to_rows_input: Any = None
258
288
 
259
289
 
260
290
  class NodeGraphSolver(NodeSingleInput):
261
291
  """Settings for graph-solving operations."""
292
+
262
293
  graph_solver_input: Any = None
263
294
 
264
295
 
265
296
  class NodeSample(NodeSingleInput):
266
297
  """Settings for sampling data."""
298
+
267
299
  sample_size: int = 1000
268
300
 
269
301
 
270
302
  class NodePromise(NodeBase):
271
303
  """A placeholder node not yet configured."""
304
+
272
305
  is_setup: bool = False
273
- node_type: str = ''
306
+ node_type: str = ""
274
307
 
275
308
 
276
309
  class DatabaseConnection(BaseModel):
277
310
  """Defines database connection parameters."""
311
+
278
312
  database_type: str = "postgresql"
279
- username: Optional[str] = None
280
- password_ref: Optional[str] = None
281
- host: Optional[str] = None
282
- port: Optional[int] = None
283
- database: Optional[str] = None
284
- url: Optional[str] = None
313
+ username: str | None = None
314
+ password_ref: str | None = None
315
+ host: str | None = None
316
+ port: int | None = None
317
+ database: str | None = None
318
+ url: str | None = None
285
319
 
286
320
 
287
321
  class DatabaseSettings(BaseModel):
288
322
  """Defines settings for reading from a database."""
289
- connection_mode: Optional[str] = 'inline'
290
- database_connection: Optional[DatabaseConnection] = None
291
- database_connection_name: Optional[str] = None
292
- schema_name: Optional[str] = None
293
- table_name: Optional[str] = None
294
- query: Optional[str] = None
295
- query_mode: str = 'table'
323
+
324
+ connection_mode: str | None = "inline"
325
+ database_connection: DatabaseConnection | None = None
326
+ database_connection_name: str | None = None
327
+ schema_name: str | None = None
328
+ table_name: str | None = None
329
+ query: str | None = None
330
+ query_mode: str = "table"
296
331
 
297
332
 
298
333
  class NodeDatabaseReader(NodeBase):
299
334
  """Settings for reading from a database."""
335
+
300
336
  database_settings: DatabaseSettings
301
- fields: Optional[List[MinimalFieldInfo]] = None
337
+ fields: list[MinimalFieldInfo] | None = None
302
338
 
303
339
 
304
340
  class NodeInformation(BaseModel):
305
341
  """Stores the state and configuration of a node instance."""
306
- id: Optional[int] = None
307
- type: Optional[str] = None
308
- is_setup: Optional[bool] = None
309
- description: Optional[str] = ''
310
- x_position: Optional[int] = 0
311
- y_position: Optional[int] = 0
312
- left_input_id: Optional[int] = None
313
- right_input_id: Optional[int] = None
314
- input_ids: Optional[List[int]] = Field(default_factory=lambda: [-1])
315
- outputs: Optional[List[int]] = Field(default_factory=lambda: [-1])
316
- setting_input: Optional[Any] = None
342
+
343
+ id: int | None = None
344
+ type: str | None = None
345
+ is_setup: bool | None = None
346
+ description: str | None = ""
347
+ x_position: int | None = 0
348
+ y_position: int | None = 0
349
+ left_input_id: int | None = None
350
+ right_input_id: int | None = None
351
+ input_ids: list[int] | None = Field(default_factory=lambda: [-1])
352
+ outputs: list[int] | None = Field(default_factory=lambda: [-1])
353
+ setting_input: Any | None = None
317
354
 
318
355
 
319
356
  class FlowInformation(BaseModel):
320
357
  """Represents the complete state of a flow."""
358
+
321
359
  flow_id: int
322
- flow_name: Optional[str] = ''
323
- flow_settings: Optional[FlowSettings] = None
324
- data: Dict[int, NodeInformation] = Field(default_factory=dict)
325
- node_starts: List[int] = Field(default_factory=list)
326
- node_connections: List[Tuple[int, int]] = Field(default_factory=list)
360
+ flow_name: str | None = ""
361
+ flow_settings: FlowSettings | None = None
362
+ data: dict[int, NodeInformation] = Field(default_factory=dict)
363
+ node_starts: list[int] = Field(default_factory=list)
364
+ node_connections: list[tuple[int, int]] = Field(default_factory=list)
327
365
 
328
366
 
329
367
  # =============================================================================
330
368
  # TRANSFORM SCHEMAS (dataclasses - these changed from @dataclass to BaseModel)
331
369
  # =============================================================================
332
370
 
371
+
333
372
  @dataclass
334
373
  class SelectInput:
335
374
  """Defines how a single column should be selected, renamed, or type-cast."""
375
+
336
376
  old_name: str
337
- original_position: Optional[int] = None
338
- new_name: Optional[str] = None
339
- data_type: Optional[str] = None
340
- data_type_change: Optional[bool] = False
341
- join_key: Optional[bool] = False
342
- is_altered: Optional[bool] = False
343
- position: Optional[int] = None
344
- is_available: Optional[bool] = True
345
- keep: Optional[bool] = True
377
+ original_position: int | None = None
378
+ new_name: str | None = None
379
+ data_type: str | None = None
380
+ data_type_change: bool | None = False
381
+ join_key: bool | None = False
382
+ is_altered: bool | None = False
383
+ position: int | None = None
384
+ is_available: bool | None = True
385
+ keep: bool | None = True
346
386
 
347
387
  def __post_init__(self):
348
388
  if self.new_name is None:
@@ -352,56 +392,63 @@ class SelectInput:
352
392
  @dataclass
353
393
  class FieldInput:
354
394
  """Represents a single field with its name and data type."""
395
+
355
396
  name: str
356
- data_type: Optional[str] = None
397
+ data_type: str | None = None
357
398
 
358
399
 
359
400
  @dataclass
360
401
  class FunctionInput:
361
402
  """Defines a formula to be applied."""
403
+
362
404
  field: FieldInput = None
363
- function: str = ''
405
+ function: str = ""
364
406
 
365
407
 
366
408
  @dataclass
367
409
  class BasicFilter:
368
410
  """Defines a simple, single-condition filter."""
369
- field: str = ''
370
- filter_type: str = ''
371
- filter_value: str = ''
411
+
412
+ field: str = ""
413
+ filter_type: str = ""
414
+ filter_value: str = ""
372
415
 
373
416
 
374
417
  @dataclass
375
418
  class FilterInput:
376
419
  """Defines the settings for a filter operation."""
377
- advanced_filter: str = ''
420
+
421
+ advanced_filter: str = ""
378
422
  basic_filter: BasicFilter = None
379
- filter_type: str = 'basic'
423
+ filter_type: str = "basic"
380
424
 
381
425
 
382
426
  @dataclass
383
427
  class SelectInputs:
384
428
  """A container for a list of SelectInput objects."""
385
- renames: List[SelectInput] = field(default_factory=list)
429
+
430
+ renames: list[SelectInput] = field(default_factory=list)
386
431
 
387
432
  @property
388
- def old_cols(self) -> Set:
433
+ def old_cols(self) -> set:
389
434
  return set(v.old_name for v in self.renames if v.keep)
390
435
 
391
436
  @property
392
- def new_cols(self) -> Set:
437
+ def new_cols(self) -> set:
393
438
  return set(v.new_name for v in self.renames if v.keep)
394
439
 
395
440
 
396
441
  @dataclass
397
442
  class JoinInputs:
398
443
  """Extends SelectInputs with functionality specific to join operations."""
399
- renames: List[SelectInput] = field(default_factory=list)
444
+
445
+ renames: list[SelectInput] = field(default_factory=list)
400
446
 
401
447
 
402
448
  @dataclass
403
449
  class JoinMap:
404
450
  """Defines a single mapping between a left and right column for a join key."""
451
+
405
452
  left_col: str = None
406
453
  right_col: str = None
407
454
 
@@ -409,6 +456,7 @@ class JoinMap:
409
456
  @dataclass
410
457
  class CrossJoinInput:
411
458
  """Defines the settings for a cross join operation."""
459
+
412
460
  left_select: Any = None
413
461
  right_select: Any = None
414
462
 
@@ -416,88 +464,98 @@ class CrossJoinInput:
416
464
  @dataclass
417
465
  class JoinInput:
418
466
  """Defines the settings for a standard SQL-style join."""
419
- join_mapping: List[JoinMap] = field(default_factory=list)
467
+
468
+ join_mapping: list[JoinMap] = field(default_factory=list)
420
469
  left_select: Any = None
421
470
  right_select: Any = None
422
- how: str = 'inner'
471
+ how: str = "inner"
423
472
 
424
473
 
425
474
  @dataclass
426
475
  class FuzzyMapping:
427
476
  """Defines a fuzzy match column mapping with threshold."""
477
+
428
478
  left_col: str = None
429
479
  right_col: str = None
430
480
  threshold_score: int = 80
431
- fuzzy_type: str = 'levenshtein'
481
+ fuzzy_type: str = "levenshtein"
432
482
 
433
483
 
434
484
  @dataclass
435
485
  class FuzzyMatchInput:
436
486
  """Extends JoinInput with settings specific to fuzzy matching."""
437
- join_mapping: List[FuzzyMapping] = field(default_factory=list)
487
+
488
+ join_mapping: list[FuzzyMapping] = field(default_factory=list)
438
489
  left_select: Any = None
439
490
  right_select: Any = None
440
- how: str = 'inner'
491
+ how: str = "inner"
441
492
  aggregate_output: bool = False
442
493
 
443
494
 
444
495
  @dataclass
445
496
  class AggColl:
446
497
  """Represents a single aggregation operation."""
498
+
447
499
  old_name: str = None
448
500
  agg: str = None
449
- new_name: Optional[str] = None
450
- output_type: Optional[str] = None
501
+ new_name: str | None = None
502
+ output_type: str | None = None
451
503
 
452
504
 
453
505
  @dataclass
454
506
  class GroupByInput:
455
507
  """Represents the input for a group by operation."""
456
- agg_cols: List[AggColl] = field(default_factory=list)
508
+
509
+ agg_cols: list[AggColl] = field(default_factory=list)
457
510
 
458
511
 
459
512
  @dataclass
460
513
  class PivotInput:
461
514
  """Defines the settings for a pivot operation."""
462
- index_columns: List[str] = field(default_factory=list)
515
+
516
+ index_columns: list[str] = field(default_factory=list)
463
517
  pivot_column: str = None
464
518
  value_col: str = None
465
- aggregations: List[str] = field(default_factory=list)
519
+ aggregations: list[str] = field(default_factory=list)
466
520
 
467
521
 
468
522
  @dataclass
469
523
  class SortByInput:
470
524
  """Defines a single sort condition on a column."""
525
+
471
526
  column: str = None
472
- how: str = 'asc'
527
+ how: str = "asc"
473
528
 
474
529
 
475
530
  @dataclass
476
531
  class RecordIdInput:
477
532
  """Defines settings for adding a record ID column."""
478
- output_column_name: str = 'record_id'
533
+
534
+ output_column_name: str = "record_id"
479
535
  offset: int = 1
480
- group_by: Optional[bool] = False
481
- group_by_columns: Optional[List[str]] = field(default_factory=list)
536
+ group_by: bool | None = False
537
+ group_by_columns: list[str] | None = field(default_factory=list)
482
538
 
483
539
 
484
540
  @dataclass
485
541
  class TextToRowsInput:
486
542
  """Defines settings for splitting a text column into multiple rows."""
543
+
487
544
  column_to_split: str = None
488
- output_column_name: Optional[str] = None
489
- split_by_fixed_value: Optional[bool] = True
490
- split_fixed_value: Optional[str] = ','
491
- split_by_column: Optional[str] = None
545
+ output_column_name: str | None = None
546
+ split_by_fixed_value: bool | None = True
547
+ split_fixed_value: str | None = ","
548
+ split_by_column: str | None = None
492
549
 
493
550
 
494
551
  @dataclass
495
552
  class UnpivotInput:
496
553
  """Defines settings for an unpivot operation."""
497
- index_columns: Optional[List[str]] = field(default_factory=list)
498
- value_columns: Optional[List[str]] = field(default_factory=list)
499
- data_type_selector: Optional[Literal['float', 'all', 'date', 'numeric', 'string']] = None
500
- data_type_selector_mode: Optional[Literal['data_type', 'column']] = 'column'
554
+
555
+ index_columns: list[str] | None = field(default_factory=list)
556
+ value_columns: list[str] | None = field(default_factory=list)
557
+ data_type_selector: Literal["float", "all", "date", "numeric", "string"] | None = None
558
+ data_type_selector_mode: Literal["data_type", "column"] | None = "column"
501
559
 
502
560
  def __post_init__(self):
503
561
  if self.index_columns is None:
@@ -509,38 +567,43 @@ class UnpivotInput:
509
567
  @dataclass
510
568
  class UnionInput:
511
569
  """Defines settings for a union operation."""
512
- mode: Literal['selective', 'relaxed'] = 'relaxed'
570
+
571
+ mode: Literal["selective", "relaxed"] = "relaxed"
513
572
 
514
573
 
515
574
  @dataclass
516
575
  class UniqueInput:
517
576
  """Defines settings for a uniqueness operation."""
518
- columns: Optional[List[str]] = None
577
+
578
+ columns: list[str] | None = None
519
579
  strategy: str = "any"
520
580
 
521
581
 
522
582
  @dataclass
523
583
  class GraphSolverInput:
524
584
  """Defines settings for a graph-solving operation."""
585
+
525
586
  col_from: str = None
526
587
  col_to: str = None
527
- output_column_name: Optional[str] = 'graph_group'
588
+ output_column_name: str | None = "graph_group"
528
589
 
529
590
 
530
591
  @dataclass
531
592
  class PolarsCodeInput:
532
593
  """A simple container for user-provided Polars code."""
533
- polars_code: str = ''
594
+
595
+ polars_code: str = ""
534
596
 
535
597
 
536
598
  @dataclass
537
599
  class SampleInput:
538
600
  """Defines settings for sampling rows."""
539
- n: Optional[int] = None
540
- fraction: Optional[float] = None
601
+
602
+ n: int | None = None
603
+ fraction: float | None = None
541
604
  with_replacement: bool = False
542
605
  shuffle: bool = False
543
- seed: Optional[int] = None
606
+ seed: int | None = None
544
607
 
545
608
 
546
609
  # =============================================================================
@@ -550,72 +613,70 @@ class SampleInput:
550
613
 
551
614
  LEGACY_CLASS_MAP = {
552
615
  # Transform schema dataclasses
553
- 'SelectInput': SelectInput,
554
- 'FieldInput': FieldInput,
555
- 'FunctionInput': FunctionInput,
556
- 'BasicFilter': BasicFilter,
557
- 'FilterInput': FilterInput,
558
- 'SelectInputs': SelectInputs,
559
- 'JoinInputs': JoinInputs,
560
- 'JoinMap': JoinMap,
561
- 'CrossJoinInput': CrossJoinInput,
562
- 'JoinInput': JoinInput,
563
- 'FuzzyMapping': FuzzyMapping,
564
- 'FuzzyMatchInput': FuzzyMatchInput,
565
- 'AggColl': AggColl,
566
- 'GroupByInput': GroupByInput,
567
- 'PivotInput': PivotInput,
568
- 'SortByInput': SortByInput,
569
- 'RecordIdInput': RecordIdInput,
570
- 'TextToRowsInput': TextToRowsInput,
571
- 'UnpivotInput': UnpivotInput,
572
- 'UnionInput': UnionInput,
573
- 'UniqueInput': UniqueInput,
574
- 'GraphSolverInput': GraphSolverInput,
575
- 'PolarsCodeInput': PolarsCodeInput,
576
- 'SampleInput': SampleInput,
577
-
616
+ "SelectInput": SelectInput,
617
+ "FieldInput": FieldInput,
618
+ "FunctionInput": FunctionInput,
619
+ "BasicFilter": BasicFilter,
620
+ "FilterInput": FilterInput,
621
+ "SelectInputs": SelectInputs,
622
+ "JoinInputs": JoinInputs,
623
+ "JoinMap": JoinMap,
624
+ "CrossJoinInput": CrossJoinInput,
625
+ "JoinInput": JoinInput,
626
+ "FuzzyMapping": FuzzyMapping,
627
+ "FuzzyMatchInput": FuzzyMatchInput,
628
+ "AggColl": AggColl,
629
+ "GroupByInput": GroupByInput,
630
+ "PivotInput": PivotInput,
631
+ "SortByInput": SortByInput,
632
+ "RecordIdInput": RecordIdInput,
633
+ "TextToRowsInput": TextToRowsInput,
634
+ "UnpivotInput": UnpivotInput,
635
+ "UnionInput": UnionInput,
636
+ "UniqueInput": UniqueInput,
637
+ "GraphSolverInput": GraphSolverInput,
638
+ "PolarsCodeInput": PolarsCodeInput,
639
+ "SampleInput": SampleInput,
578
640
  # OLD Input/Output schemas (before table_settings)
579
- 'ReceivedTable': ReceivedTable,
580
- 'OutputSettings': OutputSettings,
581
- 'OutputCsvTable': OutputCsvTable,
582
- 'OutputParquetTable': OutputParquetTable,
583
- 'OutputExcelTable': OutputExcelTable,
584
- 'MinimalFieldInfo': MinimalFieldInfo,
585
-
641
+ "ReceivedTable": ReceivedTable,
642
+ "OutputSettings": OutputSettings,
643
+ "OutputCsvTable": OutputCsvTable,
644
+ "OutputParquetTable": OutputParquetTable,
645
+ "OutputExcelTable": OutputExcelTable,
646
+ "MinimalFieldInfo": MinimalFieldInfo,
586
647
  # Flow and Node schemas
587
- 'FlowSettings': FlowSettings,
588
- 'FlowGraphConfig': FlowGraphConfig,
589
- 'FlowInformation': FlowInformation,
590
- 'NodeInformation': NodeInformation,
591
- 'NodeBase': NodeBase,
592
- 'NodeSingleInput': NodeSingleInput,
593
- 'NodeMultiInput': NodeMultiInput,
594
- 'NodeRead': NodeRead,
595
- 'NodeSelect': NodeSelect,
596
- 'NodeFilter': NodeFilter,
597
- 'NodeFormula': NodeFormula,
598
- 'NodeJoin': NodeJoin,
599
- 'NodeCrossJoin': NodeCrossJoin,
600
- 'NodeFuzzyMatch': NodeFuzzyMatch,
601
- 'NodePolarsCode': NodePolarsCode,
602
- 'NodeOutput': NodeOutput,
603
- 'NodeGroupBy': NodeGroupBy,
604
- 'NodeSort': NodeSort,
605
- 'NodeUnion': NodeUnion,
606
- 'NodeUnique': NodeUnique,
607
- 'NodePivot': NodePivot,
608
- 'NodeUnpivot': NodeUnpivot,
609
- 'NodeRecordId': NodeRecordId,
610
- 'NodeTextToRows': NodeTextToRows,
611
- 'NodeGraphSolver': NodeGraphSolver,
612
- 'NodeSample': NodeSample,
613
- 'NodePromise': NodePromise,
614
- 'DatabaseConnection': DatabaseConnection,
615
- 'DatabaseSettings': DatabaseSettings,
616
- 'NodeDatabaseReader': NodeDatabaseReader,
648
+ "FlowSettings": FlowSettings,
649
+ "FlowGraphConfig": FlowGraphConfig,
650
+ "FlowInformation": FlowInformation,
651
+ "NodeInformation": NodeInformation,
652
+ "NodeBase": NodeBase,
653
+ "NodeSingleInput": NodeSingleInput,
654
+ "NodeMultiInput": NodeMultiInput,
655
+ "NodeRead": NodeRead,
656
+ "NodeSelect": NodeSelect,
657
+ "NodeFilter": NodeFilter,
658
+ "NodeFormula": NodeFormula,
659
+ "NodeJoin": NodeJoin,
660
+ "NodeCrossJoin": NodeCrossJoin,
661
+ "NodeFuzzyMatch": NodeFuzzyMatch,
662
+ "NodePolarsCode": NodePolarsCode,
663
+ "NodeOutput": NodeOutput,
664
+ "NodeGroupBy": NodeGroupBy,
665
+ "NodeSort": NodeSort,
666
+ "NodeUnion": NodeUnion,
667
+ "NodeUnique": NodeUnique,
668
+ "NodePivot": NodePivot,
669
+ "NodeUnpivot": NodeUnpivot,
670
+ "NodeRecordId": NodeRecordId,
671
+ "NodeTextToRows": NodeTextToRows,
672
+ "NodeGraphSolver": NodeGraphSolver,
673
+ "NodeSample": NodeSample,
674
+ "NodePromise": NodePromise,
675
+ "DatabaseConnection": DatabaseConnection,
676
+ "DatabaseSettings": DatabaseSettings,
677
+ "NodeDatabaseReader": NodeDatabaseReader,
617
678
  }
618
679
 
619
680
 
620
681
  # Export all classes
621
- __all__ = list(LEGACY_CLASS_MAP.keys()) + ['LEGACY_CLASS_MAP']
682
+ __all__ = list(LEGACY_CLASS_MAP.keys()) + ["LEGACY_CLASS_MAP"]