Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (346) hide show
  1. build_backends/main.py +25 -22
  2. build_backends/main_prd.py +10 -19
  3. flowfile/__init__.py +194 -74
  4. flowfile/__main__.py +10 -7
  5. flowfile/api.py +51 -57
  6. flowfile/web/__init__.py +14 -9
  7. flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
  8. flowfile/web/static/assets/AdminView-f9847d67.js +713 -0
  9. flowfile/web/static/assets/CloudConnectionView-cf85f943.css +72 -0
  10. flowfile/web/static/assets/{CloudConnectionManager-0dfba9f2.js → CloudConnectionView-faace55b.js} +11 -11
  11. flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
  12. flowfile/web/static/assets/{CloudStorageReader-d5b1b6c9.js → CloudStorageReader-d86ecaa7.js} +10 -8
  13. flowfile/web/static/assets/{CloudStorageWriter-00d87aad.js → CloudStorageWriter-0f4d9a44.js} +10 -8
  14. flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
  15. flowfile/web/static/assets/ColumnActionInput-c44b7aee.css +159 -0
  16. flowfile/web/static/assets/ColumnActionInput-f4189ae0.js +330 -0
  17. flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
  18. flowfile/web/static/assets/{ColumnSelector-4685e75d.js → ColumnSelector-e66b33da.js} +3 -5
  19. flowfile/web/static/assets/ContextMenu-49463352.js +9 -0
  20. flowfile/web/static/assets/ContextMenu-dd5f3f25.js +9 -0
  21. flowfile/web/static/assets/ContextMenu-f709b884.js +9 -0
  22. flowfile/web/static/assets/ContextMenu.vue_vue_type_script_setup_true_lang-a1bd6314.js +59 -0
  23. flowfile/web/static/assets/{CrossJoin-702a3edd.js → CrossJoin-24694b8f.js} +12 -10
  24. flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
  25. flowfile/web/static/assets/{CustomNode-b1519993.js → CustomNode-569d45ff.js} +43 -24
  26. flowfile/web/static/assets/CustomNode-edb9b939.css +42 -0
  27. flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-c20a1e16.css} +23 -21
  28. flowfile/web/static/assets/{DatabaseConnectionSettings-6f3e4ea5.js → DatabaseConnectionSettings-cfc08938.js} +5 -4
  29. flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-5bf8c75b.css} +41 -46
  30. flowfile/web/static/assets/{DatabaseReader-d38c7295.js → DatabaseReader-701feabb.js} +25 -15
  31. flowfile/web/static/assets/{DatabaseManager-cf5ef661.js → DatabaseView-0482e5b5.js} +11 -11
  32. flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
  33. flowfile/web/static/assets/{DatabaseWriter-b04ef46a.js → DatabaseWriter-16721989.js} +17 -10
  34. flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-bdcf2c8b.css} +29 -27
  35. flowfile/web/static/assets/{designer-8da3ba3a.css → DesignerView-49abb835.css} +783 -663
  36. flowfile/web/static/assets/{designer-9633482a.js → DesignerView-f64749fb.js} +1292 -3253
  37. flowfile/web/static/assets/{documentation-ca400224.js → DocumentationView-61bd2990.js} +5 -5
  38. flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-9ea6e871.css} +9 -9
  39. flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
  40. flowfile/web/static/assets/{ExploreData-5fa10ed8.js → ExploreData-e2735b13.js} +18 -9
  41. flowfile/web/static/assets/{ExternalSource-d39af878.js → ExternalSource-2535c3b2.js} +9 -7
  42. flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-7ac7373f.css} +20 -20
  43. flowfile/web/static/assets/Filter-2cdbc93c.js +287 -0
  44. flowfile/web/static/assets/Filter-7494ea97.css +48 -0
  45. flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
  46. flowfile/web/static/assets/{Formula-6b04fb1d.js → Formula-fcda3c2c.js} +13 -11
  47. flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
  48. flowfile/web/static/assets/{FuzzyMatch-999521f4.js → FuzzyMatch-f8d3b7d3.js} +12 -10
  49. flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-4b4d7db9.css} +5 -5
  50. flowfile/web/static/assets/{GraphSolver-17dd2198.js → GraphSolver-72eaa695.js} +14 -12
  51. flowfile/web/static/assets/GroupBy-5792782d.css +9 -0
  52. flowfile/web/static/assets/{GroupBy-6b039e18.js → GroupBy-8aa0598b.js} +9 -7
  53. flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
  54. flowfile/web/static/assets/{Join-24d0f113.js → Join-e40f0ffa.js} +13 -11
  55. flowfile/web/static/assets/LoginView-5111c9ae.js +134 -0
  56. flowfile/web/static/assets/LoginView-d325d632.css +172 -0
  57. flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
  58. flowfile/web/static/assets/{ManualInput-34639209.js → ManualInput-9b6f3224.js} +170 -116
  59. flowfile/web/static/assets/{MultiSelect-0e8724a3.js → MultiSelect-ef28e19e.js} +2 -2
  60. flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-b0e538c2.js → MultiSelect.vue_vue_type_script_setup_true_lang-83b3bbfd.js} +1 -1
  61. flowfile/web/static/assets/NodeDesigner-94cd4dd3.css +1429 -0
  62. flowfile/web/static/assets/NodeDesigner-d2b7ee2b.js +2712 -0
  63. flowfile/web/static/assets/{NumericInput-3d63a470.js → NumericInput-1d789794.js} +2 -2
  64. flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-e0edeccc.js → NumericInput.vue_vue_type_script_setup_true_lang-7775f83e.js} +5 -2
  65. flowfile/web/static/assets/Output-692dd25d.css +37 -0
  66. flowfile/web/static/assets/{Output-edea9802.js → Output-cefef801.js} +14 -10
  67. flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
  68. flowfile/web/static/assets/{Pivot-61d19301.js → Pivot-bab1b75b.js} +12 -10
  69. flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
  70. flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
  71. flowfile/web/static/assets/{PivotValidation-f97fec5b.js → PivotValidation-e7941f91.js} +3 -3
  72. flowfile/web/static/assets/{PivotValidation-de9f43fe.js → PivotValidation-fba09336.js} +3 -3
  73. flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
  74. flowfile/web/static/assets/{PolarsCode-bc3c9984.js → PolarsCode-740e40fa.js} +18 -9
  75. flowfile/web/static/assets/PopOver-862d7e28.js +939 -0
  76. flowfile/web/static/assets/PopOver-d96599db.css +33 -0
  77. flowfile/web/static/assets/{Read-64a3f259.js → Read-225cc63f.js} +16 -12
  78. flowfile/web/static/assets/{Read-e808b239.css → Read-90f366bc.css} +15 -15
  79. flowfile/web/static/assets/{RecordCount-3d5039be.js → RecordCount-ffc71eca.js} +6 -4
  80. flowfile/web/static/assets/{RecordId-597510e0.js → RecordId-a70bb8df.js} +9 -7
  81. flowfile/web/static/assets/{SQLQueryComponent-df51adbe.js → SQLQueryComponent-15a421f5.js} +3 -3
  82. flowfile/web/static/assets/SQLQueryComponent-edb90b98.css +29 -0
  83. flowfile/web/static/assets/{Sample-4be0a507.js → Sample-6c26afc7.js} +6 -4
  84. flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
  85. flowfile/web/static/assets/SecretSelector-ceed9496.js +113 -0
  86. flowfile/web/static/assets/{SecretManager-4839be57.js → SecretsView-214d255a.js} +35 -36
  87. flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
  88. flowfile/web/static/assets/{Select-9b72f201.js → Select-8fc29999.js} +9 -7
  89. flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
  90. flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
  91. flowfile/web/static/assets/{SettingsSection-7ded385d.js → SettingsSection-3f70e4c3.js} +3 -3
  92. flowfile/web/static/assets/{SettingsSection-f0f75a42.js → SettingsSection-83090218.js} +3 -3
  93. flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
  94. flowfile/web/static/assets/{SettingsSection-e1e9c953.js → SettingsSection-9f0d1725.js} +3 -3
  95. flowfile/web/static/assets/SetupView-3fa0aa03.js +160 -0
  96. flowfile/web/static/assets/SetupView-e2da3442.css +230 -0
  97. flowfile/web/static/assets/{SingleSelect-6c777aac.js → SingleSelect-a4a568cb.js} +2 -2
  98. flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-33e3ff9b.js → SingleSelect.vue_vue_type_script_setup_true_lang-c8ebdd33.js} +1 -1
  99. flowfile/web/static/assets/{SliderInput-7cb93e62.js → SliderInput-be533e71.js} +7 -4
  100. flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
  101. flowfile/web/static/assets/{Sort-6cbde21a.js → Sort-154dad81.js} +9 -7
  102. flowfile/web/static/assets/Sort-4abb7fae.css +9 -0
  103. flowfile/web/static/assets/{TextInput-d9a40c11.js → TextInput-454e2bda.js} +2 -2
  104. flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-5896c375.js → TextInput.vue_vue_type_script_setup_true_lang-e86510d0.js} +5 -2
  105. flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
  106. flowfile/web/static/assets/{TextToRows-c4fcbf4d.js → TextToRows-ea73433d.js} +11 -10
  107. flowfile/web/static/assets/{ToggleSwitch-4ef91d19.js → ToggleSwitch-9d7b30f1.js} +2 -2
  108. flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-38478c20.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-00f2580e.js} +1 -1
  109. flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-394a1f78.css} +14 -14
  110. flowfile/web/static/assets/{UnavailableFields-a03f512c.js → UnavailableFields-b72a2c72.js} +4 -4
  111. flowfile/web/static/assets/{Union-bfe9b996.js → Union-1e44f263.js} +8 -6
  112. flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
  113. flowfile/web/static/assets/Unique-2b705521.css +3 -0
  114. flowfile/web/static/assets/{Unique-5d023a27.js → Unique-a3bc6d0a.js} +13 -10
  115. flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-b6ad6427.css} +7 -7
  116. flowfile/web/static/assets/{Unpivot-91cc5354.js → Unpivot-e27935fc.js} +11 -9
  117. flowfile/web/static/assets/{UnpivotValidation-7ee2de44.js → UnpivotValidation-72497680.js} +3 -3
  118. flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
  119. flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
  120. flowfile/web/static/assets/{VueGraphicWalker-e51b9924.js → VueGraphicWalker-d9ab70a3.js} +4 -4
  121. flowfile/web/static/assets/{api-cf1221f0.js → api-a2102880.js} +1 -1
  122. flowfile/web/static/assets/{api-c1bad5ca.js → api-f75042b0.js} +1 -1
  123. flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-1d6acbd9.css} +41 -41
  124. flowfile/web/static/assets/{dropDown-614b998d.js → dropDown-2798a109.js} +3 -3
  125. flowfile/web/static/assets/{fullEditor-f7971590.js → fullEditor-cf7d7d93.js} +11 -10
  126. flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-fe9f7e18.css} +77 -65
  127. flowfile/web/static/assets/{genericNodeSettings-4fe5f36b.js → genericNodeSettings-14eac1c3.js} +5 -5
  128. flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
  129. flowfile/web/static/assets/{index-5429bbf8.js → index-387a6f18.js} +41806 -40958
  130. flowfile/web/static/assets/index-6b367bb5.js +38 -0
  131. flowfile/web/static/assets/{index-50508d4d.css → index-e96ab018.css} +2184 -569
  132. flowfile/web/static/assets/index-f0a6e5a5.js +2696 -0
  133. flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
  134. flowfile/web/static/assets/nodeInput-ed2ae8d7.js +2 -0
  135. flowfile/web/static/assets/{outputCsv-076b85ab.js → outputCsv-3c1757e8.js} +3 -3
  136. flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
  137. flowfile/web/static/assets/{outputExcel-0fd17dbe.js → outputExcel-686e1f48.js} +3 -3
  138. flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
  139. flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
  140. flowfile/web/static/assets/{outputParquet-b61e0847.js → outputParquet-df28faa7.js} +4 -4
  141. flowfile/web/static/assets/{readCsv-c767cb37.css → readCsv-3bfac4c3.css} +15 -15
  142. flowfile/web/static/assets/{readCsv-a8bb8b61.js → readCsv-e37eee21.js} +3 -3
  143. flowfile/web/static/assets/{readExcel-806d2826.css → readExcel-3db6b763.css} +13 -13
  144. flowfile/web/static/assets/{readExcel-67b4aee0.js → readExcel-a13f14bb.js} +5 -5
  145. flowfile/web/static/assets/{readParquet-92ce1dbc.js → readParquet-344cf746.js} +3 -3
  146. flowfile/web/static/assets/{readParquet-48c81530.css → readParquet-c5244ad5.css} +4 -4
  147. flowfile/web/static/assets/secrets.api-ae198c5c.js +65 -0
  148. flowfile/web/static/assets/{selectDynamic-92e25ee3.js → selectDynamic-6b4b0767.js} +5 -5
  149. flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
  150. flowfile/web/static/assets/{vue-codemirror.esm-41b0e0d7.js → vue-codemirror.esm-31ba0e0b.js} +31 -640
  151. flowfile/web/static/assets/{vue-content-loader.es-2c8e608f.js → vue-content-loader.es-4469c8ff.js} +1 -1
  152. flowfile/web/static/index.html +2 -2
  153. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/METADATA +3 -4
  154. flowfile-0.5.4.dist-info/RECORD +407 -0
  155. flowfile_core/__init__.py +13 -6
  156. flowfile_core/auth/jwt.py +51 -16
  157. flowfile_core/auth/models.py +32 -7
  158. flowfile_core/auth/password.py +89 -0
  159. flowfile_core/auth/secrets.py +64 -19
  160. flowfile_core/configs/__init__.py +9 -7
  161. flowfile_core/configs/flow_logger.py +15 -14
  162. flowfile_core/configs/node_store/__init__.py +72 -4
  163. flowfile_core/configs/node_store/nodes.py +155 -172
  164. flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
  165. flowfile_core/configs/settings.py +28 -15
  166. flowfile_core/database/connection.py +7 -6
  167. flowfile_core/database/init_db.py +96 -2
  168. flowfile_core/database/models.py +3 -1
  169. flowfile_core/fileExplorer/__init__.py +17 -0
  170. flowfile_core/fileExplorer/funcs.py +145 -57
  171. flowfile_core/fileExplorer/utils.py +10 -11
  172. flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
  173. flowfile_core/flowfile/analytics/analytics_processor.py +26 -24
  174. flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
  175. flowfile_core/flowfile/analytics/utils.py +1 -1
  176. flowfile_core/flowfile/code_generator/__init__.py +11 -0
  177. flowfile_core/flowfile/code_generator/code_generator.py +706 -247
  178. flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
  179. flowfile_core/flowfile/connection_manager/models.py +1 -1
  180. flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
  181. flowfile_core/flowfile/database_connection_manager/models.py +1 -1
  182. flowfile_core/flowfile/extensions.py +17 -12
  183. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
  184. flowfile_core/flowfile/flow_data_engine/create/funcs.py +115 -83
  185. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +493 -423
  186. flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
  187. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
  188. flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
  189. flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
  190. flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
  191. flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +31 -20
  192. flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
  193. flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
  194. flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +14 -15
  195. flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
  196. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
  197. flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
  198. flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
  199. flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
  200. flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
  201. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +190 -127
  202. flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
  203. flowfile_core/flowfile/flow_data_engine/utils.py +99 -67
  204. flowfile_core/flowfile/flow_graph.py +920 -571
  205. flowfile_core/flowfile/flow_graph_utils.py +31 -49
  206. flowfile_core/flowfile/flow_node/flow_node.py +379 -258
  207. flowfile_core/flowfile/flow_node/models.py +53 -41
  208. flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
  209. flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
  210. flowfile_core/flowfile/handler.py +80 -30
  211. flowfile_core/flowfile/manage/compatibility_enhancements.py +209 -126
  212. flowfile_core/flowfile/manage/io_flowfile.py +54 -57
  213. flowfile_core/flowfile/node_designer/__init__.py +19 -13
  214. flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
  215. flowfile_core/flowfile/node_designer/custom_node.py +162 -36
  216. flowfile_core/flowfile/node_designer/ui_components.py +278 -34
  217. flowfile_core/flowfile/schema_callbacks.py +71 -51
  218. flowfile_core/flowfile/setting_generator/__init__.py +0 -1
  219. flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
  220. flowfile_core/flowfile/setting_generator/settings.py +64 -53
  221. flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
  222. flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
  223. flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
  224. flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
  225. flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
  226. flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
  227. flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
  228. flowfile_core/flowfile/util/calculate_layout.py +9 -13
  229. flowfile_core/flowfile/util/execution_orderer.py +25 -17
  230. flowfile_core/flowfile/util/node_skipper.py +4 -4
  231. flowfile_core/flowfile/utils.py +19 -21
  232. flowfile_core/main.py +26 -19
  233. flowfile_core/routes/auth.py +284 -11
  234. flowfile_core/routes/cloud_connections.py +25 -25
  235. flowfile_core/routes/logs.py +21 -29
  236. flowfile_core/routes/public.py +46 -4
  237. flowfile_core/routes/routes.py +70 -34
  238. flowfile_core/routes/secrets.py +25 -27
  239. flowfile_core/routes/user_defined_components.py +483 -4
  240. flowfile_core/run_lock.py +0 -1
  241. flowfile_core/schemas/__init__.py +4 -6
  242. flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
  243. flowfile_core/schemas/cloud_storage_schemas.py +96 -66
  244. flowfile_core/schemas/input_schema.py +231 -144
  245. flowfile_core/schemas/output_model.py +49 -34
  246. flowfile_core/schemas/schemas.py +116 -89
  247. flowfile_core/schemas/transform_schema.py +518 -263
  248. flowfile_core/schemas/yaml_types.py +21 -7
  249. flowfile_core/secret_manager/secret_manager.py +123 -18
  250. flowfile_core/types.py +29 -9
  251. flowfile_core/utils/arrow_reader.py +7 -6
  252. flowfile_core/utils/excel_file_manager.py +3 -3
  253. flowfile_core/utils/fileManager.py +7 -7
  254. flowfile_core/utils/fl_executor.py +8 -10
  255. flowfile_core/utils/utils.py +4 -4
  256. flowfile_core/utils/validate_setup.py +5 -4
  257. flowfile_frame/__init__.py +117 -51
  258. flowfile_frame/adapters.py +2 -9
  259. flowfile_frame/adding_expr.py +73 -32
  260. flowfile_frame/cloud_storage/frame_helpers.py +27 -23
  261. flowfile_frame/cloud_storage/secret_manager.py +12 -26
  262. flowfile_frame/config.py +2 -5
  263. flowfile_frame/database/__init__.py +36 -0
  264. flowfile_frame/database/connection_manager.py +205 -0
  265. flowfile_frame/database/frame_helpers.py +249 -0
  266. flowfile_frame/expr.py +311 -218
  267. flowfile_frame/expr.pyi +160 -159
  268. flowfile_frame/expr_name.py +23 -23
  269. flowfile_frame/flow_frame.py +571 -476
  270. flowfile_frame/flow_frame.pyi +123 -104
  271. flowfile_frame/flow_frame_methods.py +227 -246
  272. flowfile_frame/group_frame.py +50 -20
  273. flowfile_frame/join.py +2 -2
  274. flowfile_frame/lazy.py +129 -87
  275. flowfile_frame/lazy_methods.py +83 -30
  276. flowfile_frame/list_name_space.py +55 -50
  277. flowfile_frame/selectors.py +148 -68
  278. flowfile_frame/series.py +9 -7
  279. flowfile_frame/utils.py +19 -21
  280. flowfile_worker/__init__.py +12 -7
  281. flowfile_worker/configs.py +41 -33
  282. flowfile_worker/create/__init__.py +14 -9
  283. flowfile_worker/create/funcs.py +114 -77
  284. flowfile_worker/create/models.py +46 -43
  285. flowfile_worker/create/pl_types.py +14 -15
  286. flowfile_worker/create/read_excel_tables.py +34 -41
  287. flowfile_worker/create/utils.py +22 -19
  288. flowfile_worker/external_sources/s3_source/main.py +18 -51
  289. flowfile_worker/external_sources/s3_source/models.py +34 -27
  290. flowfile_worker/external_sources/sql_source/main.py +8 -5
  291. flowfile_worker/external_sources/sql_source/models.py +13 -9
  292. flowfile_worker/flow_logger.py +10 -8
  293. flowfile_worker/funcs.py +214 -155
  294. flowfile_worker/main.py +11 -17
  295. flowfile_worker/models.py +35 -28
  296. flowfile_worker/process_manager.py +2 -3
  297. flowfile_worker/routes.py +121 -90
  298. flowfile_worker/secrets.py +114 -21
  299. flowfile_worker/spawner.py +89 -54
  300. flowfile_worker/utils.py +3 -2
  301. shared/__init__.py +2 -7
  302. shared/storage_config.py +25 -13
  303. test_utils/postgres/commands.py +3 -2
  304. test_utils/postgres/fixtures.py +9 -9
  305. test_utils/s3/commands.py +1 -1
  306. test_utils/s3/data_generator.py +3 -4
  307. test_utils/s3/demo_data_generator.py +4 -7
  308. test_utils/s3/fixtures.py +7 -5
  309. tools/migrate/__init__.py +1 -1
  310. tools/migrate/__main__.py +16 -29
  311. tools/migrate/legacy_schemas.py +251 -190
  312. tools/migrate/migrate.py +193 -181
  313. tools/migrate/tests/conftest.py +1 -3
  314. tools/migrate/tests/test_migrate.py +36 -41
  315. tools/migrate/tests/test_migration_e2e.py +28 -29
  316. tools/migrate/tests/test_node_migrations.py +50 -20
  317. flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
  318. flowfile/web/static/assets/ContextMenu-23e909da.js +0 -41
  319. flowfile/web/static/assets/ContextMenu-4c74eef1.css +0 -26
  320. flowfile/web/static/assets/ContextMenu-63cfa99b.css +0 -26
  321. flowfile/web/static/assets/ContextMenu-70ae0c79.js +0 -41
  322. flowfile/web/static/assets/ContextMenu-c13f91d0.css +0 -26
  323. flowfile/web/static/assets/ContextMenu-f149cf7c.js +0 -41
  324. flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
  325. flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
  326. flowfile/web/static/assets/Filter-9b6d08db.js +0 -164
  327. flowfile/web/static/assets/Filter-f62091b3.css +0 -20
  328. flowfile/web/static/assets/GroupBy-b9505323.css +0 -51
  329. flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
  330. flowfile/web/static/assets/Output-283fe388.css +0 -37
  331. flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
  332. flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
  333. flowfile/web/static/assets/SQLQueryComponent-36cef432.css +0 -27
  334. flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
  335. flowfile/web/static/assets/Sort-3643d625.css +0 -51
  336. flowfile/web/static/assets/Unique-f9fb0809.css +0 -51
  337. flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
  338. flowfile/web/static/assets/nodeInput-5d0d6b79.js +0 -41
  339. flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
  340. flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
  341. flowfile/web/static/assets/secretApi-68435402.js +0 -46
  342. flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
  343. flowfile-0.5.1.dist-info/RECORD +0 -388
  344. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/WHEEL +0 -0
  345. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/entry_points.txt +0 -0
  346. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/licenses/LICENSE +0 -0
@@ -1,71 +1,102 @@
1
- from typing import List, Optional, Literal, Iterator, Any, Annotated
2
- from flowfile_core.schemas import transform_schema
3
- from pathlib import Path
4
1
  import os
2
+ from pathlib import Path
3
+ from typing import Annotated, Any, Literal
4
+
5
+ import polars as pl
6
+ from pydantic import (
7
+ BaseModel,
8
+ ConfigDict,
9
+ Field,
10
+ SecretStr,
11
+ StringConstraints,
12
+ ValidationInfo,
13
+ field_validator,
14
+ model_validator,
15
+ )
16
+
17
+ from flowfile_core.schemas import transform_schema
5
18
  from flowfile_core.schemas.analysis_schemas import graphic_walker_schemas as gs_schemas
6
19
  from flowfile_core.schemas.cloud_storage_schemas import CloudStorageReadSettings, CloudStorageWriteSettings
7
20
  from flowfile_core.schemas.yaml_types import (
8
- OutputSettingsYaml, NodeSelectYaml, NodeJoinYaml,
9
- NodeCrossJoinYaml, NodeFuzzyMatchYaml, NodeOutputYaml
21
+ NodeCrossJoinYaml,
22
+ NodeFuzzyMatchYaml,
23
+ NodeJoinYaml,
24
+ NodeOutputYaml,
25
+ NodeSelectYaml,
26
+ OutputSettingsYaml,
10
27
  )
11
28
  from flowfile_core.utils.utils import ensure_similarity_dicts, standardize_col_dtype
12
- from pydantic import (BaseModel, Field, model_validator, field_validator,
13
- SecretStr, ConfigDict, StringConstraints, ValidationInfo)
14
- import polars as pl
15
-
16
29
 
17
- SecretRef = Annotated[str, StringConstraints(min_length=1, max_length=100),
18
- Field(description="An ID referencing an encrypted secret.")]
30
+ SecretRef = Annotated[
31
+ str, StringConstraints(min_length=1, max_length=100), Field(description="An ID referencing an encrypted secret.")
32
+ ]
19
33
 
20
34
 
21
- OutputConnectionClass = Literal['output-0', 'output-1', 'output-2', 'output-3', 'output-4',
22
- 'output-5', 'output-6', 'output-7', 'output-8', 'output-9']
35
+ OutputConnectionClass = Literal[
36
+ "output-0",
37
+ "output-1",
38
+ "output-2",
39
+ "output-3",
40
+ "output-4",
41
+ "output-5",
42
+ "output-6",
43
+ "output-7",
44
+ "output-8",
45
+ "output-9",
46
+ ]
23
47
 
24
- InputConnectionClass = Literal['input-0', 'input-1', 'input-2', 'input-3', 'input-4',
25
- 'input-5', 'input-6', 'input-7', 'input-8', 'input-9']
48
+ InputConnectionClass = Literal[
49
+ "input-0", "input-1", "input-2", "input-3", "input-4", "input-5", "input-6", "input-7", "input-8", "input-9"
50
+ ]
26
51
 
27
52
  InputType = Literal["main", "left", "right"]
28
53
 
29
54
 
30
55
  class NewDirectory(BaseModel):
31
56
  """Defines the information required to create a new directory."""
57
+
32
58
  source_path: str
33
59
  dir_name: str
34
60
 
35
61
 
36
62
  class RemoveItem(BaseModel):
37
63
  """Represents a single item to be removed from a directory or list."""
64
+
38
65
  path: str
39
66
  id: int = -1
40
67
 
41
68
 
42
69
  class RemoveItemsInput(BaseModel):
43
70
  """Defines a list of items to be removed."""
44
- paths: List[RemoveItem]
71
+
72
+ paths: list[RemoveItem]
45
73
  source_path: str
46
74
 
47
75
 
48
76
  class MinimalFieldInfo(BaseModel):
49
77
  """Represents the most basic information about a data field (column)."""
78
+
50
79
  name: str
51
80
  data_type: str = "String"
52
81
 
53
82
 
54
83
  class InputTableBase(BaseModel):
55
84
  """Base settings for input file operations."""
85
+
56
86
  file_type: str # Will be overridden with Literal in subclasses
57
87
 
58
88
 
59
89
  class InputCsvTable(InputTableBase):
60
90
  """Defines settings for reading a CSV file."""
61
- file_type: Literal['csv'] = 'csv'
62
- reference: str = ''
91
+
92
+ file_type: Literal["csv"] = "csv"
93
+ reference: str = ""
63
94
  starting_from_line: int = 0
64
- delimiter: str = ','
95
+ delimiter: str = ","
65
96
  has_headers: bool = True
66
- encoding: str = 'utf-8'
67
- parquet_ref: Optional[str] = None
68
- row_delimiter: str = '\n'
97
+ encoding: str = "utf-8"
98
+ parquet_ref: str | None = None
99
+ row_delimiter: str = "\n"
69
100
  quote_char: str = '"'
70
101
  infer_schema_length: int = 10_000
71
102
  truncate_ragged_lines: bool = False
@@ -74,18 +105,21 @@ class InputCsvTable(InputTableBase):
74
105
 
75
106
  class InputJsonTable(InputCsvTable):
76
107
  """Defines settings for reading a JSON file."""
77
- file_type: Literal['json'] = 'json'
108
+
109
+ file_type: Literal["json"] = "json"
78
110
 
79
111
 
80
112
  class InputParquetTable(InputTableBase):
81
113
  """Defines settings for reading a Parquet file."""
82
- file_type: Literal['parquet'] = 'parquet'
114
+
115
+ file_type: Literal["parquet"] = "parquet"
83
116
 
84
117
 
85
118
  class InputExcelTable(InputTableBase):
86
119
  """Defines settings for reading an Excel file."""
87
- file_type: Literal['excel'] = 'excel'
88
- sheet_name: Optional[str] = None
120
+
121
+ file_type: Literal["excel"] = "excel"
122
+ sheet_name: str | None = None
89
123
  start_row: int = 0
90
124
  start_column: int = 0
91
125
  end_row: int = 0
@@ -93,60 +127,58 @@ class InputExcelTable(InputTableBase):
93
127
  has_headers: bool = True
94
128
  type_inference: bool = False
95
129
 
96
- @model_validator(mode='after')
130
+ @model_validator(mode="after")
97
131
  def validate_range_values(self):
98
132
  """Validates that the Excel cell range is logical."""
99
133
  for attribute in [self.start_row, self.start_column, self.end_row, self.end_column]:
100
134
  if not isinstance(attribute, int) or attribute < 0:
101
135
  raise ValueError("Row and column indices must be non-negative integers")
102
- if (self.end_row > 0 and self.start_row > self.end_row) or \
103
- (self.end_column > 0 and self.start_column > self.end_column):
136
+ if (self.end_row > 0 and self.start_row > self.end_row) or (
137
+ self.end_column > 0 and self.start_column > self.end_column
138
+ ):
104
139
  raise ValueError("Start row/column must not be greater than end row/column")
105
140
  return self
106
141
 
107
142
 
108
143
  # Create the discriminated union (similar to OutputTableSettings)
109
144
  InputTableSettings = Annotated[
110
- InputCsvTable | InputJsonTable | InputParquetTable | InputExcelTable,
111
- Field(discriminator='file_type')
145
+ InputCsvTable | InputJsonTable | InputParquetTable | InputExcelTable, Field(discriminator="file_type")
112
146
  ]
113
147
 
114
148
 
115
149
  # Now create the main ReceivedTable model
116
150
  class ReceivedTable(BaseModel):
117
151
  """Model for defining a table received from an external source."""
152
+
118
153
  # Metadata fields
119
- id: Optional[int] = None
120
- name: Optional[str] = None
154
+ id: int | None = None
155
+ name: str | None = None
121
156
  path: str # This can be an absolute or relative path
122
- directory: Optional[str] = None
157
+ directory: str | None = None
123
158
  analysis_file_available: bool = False
124
- status: Optional[str] = None
125
- fields: List[MinimalFieldInfo] = Field(default_factory=list)
126
- abs_file_path: Optional[str] = None
159
+ status: str | None = None
160
+ fields: list[MinimalFieldInfo] = Field(default_factory=list)
161
+ abs_file_path: str | None = None
127
162
 
128
- file_type: Literal['csv', 'json', 'parquet', 'excel']
163
+ file_type: Literal["csv", "json", "parquet", "excel"]
129
164
 
130
165
  table_settings: InputTableSettings
131
166
 
132
167
  @classmethod
133
- def create_from_path(cls, path: str, file_type: Literal['csv', 'json', 'parquet', 'excel'] = 'csv'):
168
+ def create_from_path(cls, path: str, file_type: Literal["csv", "json", "parquet", "excel"] = "csv"):
134
169
  """Creates an instance from a file path string."""
135
170
  filename = Path(path).name
136
171
 
137
172
  # Create appropriate table_settings based on file_type
138
173
  settings_map = {
139
- 'csv': InputCsvTable(),
140
- 'json': InputJsonTable(),
141
- 'parquet': InputParquetTable(),
142
- 'excel': InputExcelTable(),
174
+ "csv": InputCsvTable(),
175
+ "json": InputJsonTable(),
176
+ "parquet": InputParquetTable(),
177
+ "excel": InputExcelTable(),
143
178
  }
144
179
 
145
180
  return cls(
146
- name=filename,
147
- path=path,
148
- file_type=file_type,
149
- table_settings=settings_map.get(file_type, InputCsvTable())
181
+ name=filename, path=path, file_type=file_type, table_settings=settings_map.get(file_type, InputCsvTable())
150
182
  )
151
183
 
152
184
  @property
@@ -166,19 +198,19 @@ class ReceivedTable(BaseModel):
166
198
  base_path = base_path / self.name
167
199
  self.abs_file_path = str(base_path.resolve())
168
200
 
169
- @model_validator(mode='before')
201
+ @model_validator(mode="before")
170
202
  @classmethod
171
203
  def set_default_table_settings(cls, data):
172
204
  """Create default table_settings based on file_type if not provided."""
173
205
  if isinstance(data, dict):
174
- if 'table_settings' not in data or data['table_settings'] is None:
175
- data['table_settings'] = {}
206
+ if "table_settings" not in data or data["table_settings"] is None:
207
+ data["table_settings"] = {}
176
208
 
177
- if isinstance(data['table_settings'], dict) and 'file_type' not in data['table_settings']:
178
- data['table_settings']['file_type'] = data.get('file_type', 'csv')
209
+ if isinstance(data["table_settings"], dict) and "file_type" not in data["table_settings"]:
210
+ data["table_settings"]["file_type"] = data.get("file_type", "csv")
179
211
  return data
180
212
 
181
- @model_validator(mode='after')
213
+ @model_validator(mode="after")
182
214
  def populate_abs_file_path(self):
183
215
  """Ensures the absolute file path is populated after validation."""
184
216
  if not self.abs_file_path:
@@ -188,38 +220,41 @@ class ReceivedTable(BaseModel):
188
220
 
189
221
  class OutputCsvTable(BaseModel):
190
222
  """Defines settings for writing a CSV file."""
191
- file_type: Literal['csv'] = 'csv'
192
- delimiter: str = ','
193
- encoding: str = 'utf-8'
223
+
224
+ file_type: Literal["csv"] = "csv"
225
+ delimiter: str = ","
226
+ encoding: str = "utf-8"
194
227
 
195
228
 
196
229
  class OutputParquetTable(BaseModel):
197
230
  """Defines settings for writing a Parquet file."""
198
- file_type: Literal['parquet'] = 'parquet'
231
+
232
+ file_type: Literal["parquet"] = "parquet"
199
233
 
200
234
 
201
235
  class OutputExcelTable(BaseModel):
202
236
  """Defines settings for writing an Excel file."""
203
- file_type: Literal['excel'] = 'excel'
204
- sheet_name: str = 'Sheet1'
237
+
238
+ file_type: Literal["excel"] = "excel"
239
+ sheet_name: str = "Sheet1"
205
240
 
206
241
 
207
242
  # Create a discriminated union
208
243
  OutputTableSettings = Annotated[
209
- OutputCsvTable | OutputParquetTable | OutputExcelTable,
210
- Field(discriminator='file_type')
244
+ OutputCsvTable | OutputParquetTable | OutputExcelTable, Field(discriminator="file_type")
211
245
  ]
212
246
 
213
247
 
214
248
  class OutputSettings(BaseModel):
215
249
  """Defines the complete settings for an output node."""
250
+
216
251
  name: str
217
252
  directory: str
218
253
  file_type: str # This drives which table_settings to use
219
- fields: Optional[List[str]] = Field(default_factory=list)
220
- write_mode: str = 'overwrite'
254
+ fields: list[str] | None = Field(default_factory=list)
255
+ write_mode: str = "overwrite"
221
256
  table_settings: OutputTableSettings
222
- abs_file_path: Optional[str] = None
257
+ abs_file_path: str | None = None
223
258
 
224
259
  def to_yaml_dict(self) -> OutputSettingsYaml:
225
260
  """Converts the output settings to a dictionary suitable for YAML serialization."""
@@ -241,34 +276,34 @@ class OutputSettings(BaseModel):
241
276
 
242
277
  @property
243
278
  def sheet_name(self) -> str | None:
244
- if self.file_type == 'excel':
279
+ if self.file_type == "excel":
245
280
  return self.table_settings.sheet_name
246
281
 
247
282
  @property
248
283
  def delimiter(self) -> str | None:
249
- if self.file_type == 'csv':
284
+ if self.file_type == "csv":
250
285
  return self.table_settings.delimiter
251
286
 
252
- @field_validator('table_settings', mode='before')
287
+ @field_validator("table_settings", mode="before")
253
288
  @classmethod
254
289
  def validate_table_settings(cls, v, info: ValidationInfo):
255
290
  """Ensures table_settings matches the file_type."""
256
291
  if v is None:
257
- file_type = info.data.get('file_type', 'csv')
292
+ file_type = info.data.get("file_type", "csv")
258
293
  # Create default based on file_type
259
294
  match file_type:
260
- case 'csv':
295
+ case "csv":
261
296
  return OutputCsvTable()
262
- case 'parquet':
297
+ case "parquet":
263
298
  return OutputParquetTable()
264
- case 'excel':
299
+ case "excel":
265
300
  return OutputExcelTable()
266
301
  case _:
267
302
  return OutputCsvTable()
268
303
 
269
304
  # If it's a dict, add file_type if missing
270
- if isinstance(v, dict) and 'file_type' not in v:
271
- v['file_type'] = info.data.get('file_type', 'csv')
305
+ if isinstance(v, dict) and "file_type" not in v:
306
+ v["file_type"] = info.data.get("file_type", "csv")
272
307
 
273
308
  return v
274
309
 
@@ -281,7 +316,7 @@ class OutputSettings(BaseModel):
281
316
  base_path = base_path / self.name
282
317
  self.abs_file_path = str(base_path.resolve())
283
318
 
284
- @model_validator(mode='after')
319
+ @model_validator(mode="after")
285
320
  def populate_abs_file_path(self):
286
321
  """Ensures the absolute file path is populated after validation."""
287
322
  self.set_absolute_filepath()
@@ -290,34 +325,38 @@ class OutputSettings(BaseModel):
290
325
 
291
326
  class NodeBase(BaseModel):
292
327
  """Base model for all nodes in a FlowGraph. Contains common metadata."""
328
+
293
329
  model_config = ConfigDict(arbitrary_types_allowed=True)
294
330
  flow_id: int
295
331
  node_id: int
296
- cache_results: Optional[bool] = False
297
- pos_x: Optional[float] = 0
298
- pos_y: Optional[float] = 0
299
- is_setup: Optional[bool] = True
300
- description: Optional[str] = ''
301
- user_id: Optional[int] = None
302
- is_flow_output: Optional[bool] = False
303
- is_user_defined: Optional[bool] = False # Indicator if the node is a user defined node
332
+ cache_results: bool | None = False
333
+ pos_x: float | None = 0
334
+ pos_y: float | None = 0
335
+ is_setup: bool | None = True
336
+ description: str | None = ""
337
+ user_id: int | None = None
338
+ is_flow_output: bool | None = False
339
+ is_user_defined: bool | None = False # Indicator if the node is a user defined node
304
340
 
305
341
 
306
342
  class NodeSingleInput(NodeBase):
307
343
  """A base model for any node that takes a single data input."""
308
- depending_on_id: Optional[int] = -1
344
+
345
+ depending_on_id: int | None = -1
309
346
 
310
347
 
311
348
  class NodeMultiInput(NodeBase):
312
349
  """A base model for any node that takes multiple data inputs."""
313
- depending_on_ids: Optional[List[int]] = Field(default_factory=list)
350
+
351
+ depending_on_ids: list[int] | None = Field(default_factory=list)
314
352
 
315
353
 
316
354
  class NodeSelect(NodeSingleInput):
317
355
  """Settings for a node that selects, renames, and reorders columns."""
356
+
318
357
  keep_missing: bool = True
319
- select_input: List[transform_schema.SelectInput] = Field(default_factory=list)
320
- sorted_by: Optional[Literal['none', 'asc', 'desc']] = 'none'
358
+ select_input: list[transform_schema.SelectInput] = Field(default_factory=list)
359
+ sorted_by: Literal["none", "asc", "desc"] | None = "none"
321
360
 
322
361
  def to_yaml_dict(self) -> NodeSelectYaml:
323
362
  """Converts the select node settings to a dictionary for YAML serialization."""
@@ -331,31 +370,37 @@ class NodeSelect(NodeSingleInput):
331
370
 
332
371
  class NodeFilter(NodeSingleInput):
333
372
  """Settings for a node that filters rows based on a condition."""
373
+
334
374
  filter_input: transform_schema.FilterInput
335
375
 
336
376
 
337
377
  class NodeSort(NodeSingleInput):
338
378
  """Settings for a node that sorts the data by one or more columns."""
339
- sort_input: List[transform_schema.SortByInput] = Field(default_factory=list)
379
+
380
+ sort_input: list[transform_schema.SortByInput] = Field(default_factory=list)
340
381
 
341
382
 
342
383
  class NodeTextToRows(NodeSingleInput):
343
384
  """Settings for a node that splits a text column into multiple rows."""
385
+
344
386
  text_to_rows_input: transform_schema.TextToRowsInput
345
387
 
346
388
 
347
389
  class NodeSample(NodeSingleInput):
348
390
  """Settings for a node that samples a subset of the data."""
391
+
349
392
  sample_size: int = 1000
350
393
 
351
394
 
352
395
  class NodeRecordId(NodeSingleInput):
353
396
  """Settings for a node that adds a unique record ID column."""
397
+
354
398
  record_id_input: transform_schema.RecordIdInput
355
399
 
356
400
 
357
401
  class NodeJoin(NodeMultiInput):
358
402
  """Settings for a node that performs a standard SQL-style join."""
403
+
359
404
  auto_generate_selection: bool = True
360
405
  verify_integrity: bool = True
361
406
  join_input: transform_schema.JoinInput
@@ -378,6 +423,7 @@ class NodeJoin(NodeMultiInput):
378
423
 
379
424
  class NodeCrossJoin(NodeMultiInput):
380
425
  """Settings for a node that performs a cross join."""
426
+
381
427
  auto_generate_selection: bool = True
382
428
  verify_integrity: bool = True
383
429
  cross_join_input: transform_schema.CrossJoinInput
@@ -400,6 +446,7 @@ class NodeCrossJoin(NodeMultiInput):
400
446
 
401
447
  class NodeFuzzyMatch(NodeJoin):
402
448
  """Settings for a node that performs a fuzzy join based on string similarity."""
449
+
403
450
  join_input: transform_schema.FuzzyMatchInput
404
451
 
405
452
  def to_yaml_dict(self) -> NodeFuzzyMatchYaml:
@@ -417,98 +464,105 @@ class NodeFuzzyMatch(NodeJoin):
417
464
 
418
465
  class NodeDatasource(NodeBase):
419
466
  """Base settings for a node that acts as a data source."""
467
+
420
468
  file_ref: str = None
421
469
 
422
470
 
423
471
  class RawData(BaseModel):
424
472
  """Represents data in a raw, columnar format for manual input."""
425
- columns: List[MinimalFieldInfo] = None
426
- data: List[List]
473
+
474
+ columns: list[MinimalFieldInfo] = None
475
+ data: list[list]
427
476
 
428
477
  @classmethod
429
- def from_pylist(cls, pylist: List[dict]):
478
+ def from_pylist(cls, pylist: list[dict]):
430
479
  """Creates a RawData object from a list of Python dictionaries."""
431
480
  if len(pylist) == 0:
432
481
  return cls(columns=[], data=[])
433
482
  pylist = ensure_similarity_dicts(pylist)
434
- values = [standardize_col_dtype([vv for vv in c]) for c in
435
- zip(*(r.values() for r in pylist))]
483
+ values = [standardize_col_dtype([vv for vv in c]) for c in zip(*(r.values() for r in pylist), strict=False)]
436
484
  data_types = (pl.DataType.from_python(type(next((v for v in column_values), None))) for column_values in values)
437
485
  columns = [MinimalFieldInfo(name=c, data_type=str(next(data_types))) for c in pylist[0].keys()]
438
486
  return cls(columns=columns, data=values)
439
487
 
440
- def to_pylist(self) -> List[dict]:
488
+ def to_pylist(self) -> list[dict]:
441
489
  """Converts the RawData object back into a list of Python dictionaries."""
442
490
  return [{c.name: self.data[ci][ri] for ci, c in enumerate(self.columns)} for ri in range(len(self.data[0]))]
443
491
 
444
492
 
445
493
  class NodeManualInput(NodeBase):
446
494
  """Settings for a node that allows direct data entry in the UI."""
447
- raw_data_format: Optional[RawData] = None
495
+
496
+ raw_data_format: RawData | None = None
448
497
 
449
498
 
450
499
  class NodeRead(NodeBase):
451
500
  """Settings for a node that reads data from a file."""
501
+
452
502
  received_file: ReceivedTable
453
503
 
454
504
 
455
505
  class DatabaseConnection(BaseModel):
456
506
  """Defines the connection parameters for a database."""
507
+
457
508
  database_type: str = "postgresql"
458
- username: Optional[str] = None
459
- password_ref: Optional[SecretRef] = None
460
- host: Optional[str] = None
461
- port: Optional[int] = None
462
- database: Optional[str] = None
463
- url: Optional[str] = None
509
+ username: str | None = None
510
+ password_ref: SecretRef | None = None
511
+ host: str | None = None
512
+ port: int | None = None
513
+ database: str | None = None
514
+ url: str | None = None
464
515
 
465
516
 
466
517
  class FullDatabaseConnection(BaseModel):
467
518
  """A complete database connection model including the secret password."""
519
+
468
520
  connection_name: str
469
521
  database_type: str = "postgresql"
470
522
  username: str
471
523
  password: SecretStr
472
- host: Optional[str] = None
473
- port: Optional[int] = None
474
- database: Optional[str] = None
475
- ssl_enabled: Optional[bool] = False
476
- url: Optional[str] = None
524
+ host: str | None = None
525
+ port: int | None = None
526
+ database: str | None = None
527
+ ssl_enabled: bool | None = False
528
+ url: str | None = None
477
529
 
478
530
 
479
531
  class FullDatabaseConnectionInterface(BaseModel):
480
532
  """A database connection model intended for UI display, omitting the password."""
533
+
481
534
  connection_name: str
482
535
  database_type: str = "postgresql"
483
536
  username: str
484
- host: Optional[str] = None
485
- port: Optional[int] = None
486
- database: Optional[str] = None
487
- ssl_enabled: Optional[bool] = False
488
- url: Optional[str] = None
537
+ host: str | None = None
538
+ port: int | None = None
539
+ database: str | None = None
540
+ ssl_enabled: bool | None = False
541
+ url: str | None = None
489
542
 
490
543
 
491
544
  class DatabaseSettings(BaseModel):
492
545
  """Defines settings for reading from a database, either via table or query."""
493
- connection_mode: Optional[Literal['inline', 'reference']] = 'inline'
494
- database_connection: Optional[DatabaseConnection] = None
495
- database_connection_name: Optional[str] = None
496
- schema_name: Optional[str] = None
497
- table_name: Optional[str] = None
498
- query: Optional[str] = None
499
- query_mode: Literal['query', 'table', 'reference'] = 'table'
500
-
501
- @model_validator(mode='after')
546
+
547
+ connection_mode: Literal["inline", "reference"] | None = "inline"
548
+ database_connection: DatabaseConnection | None = None
549
+ database_connection_name: str | None = None
550
+ schema_name: str | None = None
551
+ table_name: str | None = None
552
+ query: str | None = None
553
+ query_mode: Literal["query", "table", "reference"] = "table"
554
+
555
+ @model_validator(mode="after")
502
556
  def validate_table_or_query(self):
503
557
  # Validate that either table_name or query is provided
504
- if (not self.table_name and not self.query) and self.query_mode == 'inline':
558
+ if (not self.table_name and not self.query) and self.query_mode == "inline":
505
559
  raise ValueError("Either 'table_name' or 'query' must be provided")
506
560
 
507
561
  # Validate correct connection information based on connection_mode
508
- if self.connection_mode == 'inline' and self.database_connection is None:
562
+ if self.connection_mode == "inline" and self.database_connection is None:
509
563
  raise ValueError("When 'connection_mode' is 'inline', 'database_connection' must be provided")
510
564
 
511
- if self.connection_mode == 'reference' and not self.database_connection_name:
565
+ if self.connection_mode == "reference" and not self.database_connection_name:
512
566
  raise ValueError("When 'connection_mode' is 'reference', 'database_connection_name' must be provided")
513
567
 
514
568
  return self
@@ -516,44 +570,51 @@ class DatabaseSettings(BaseModel):
516
570
 
517
571
  class DatabaseWriteSettings(BaseModel):
518
572
  """Defines settings for writing data to a database table."""
519
- connection_mode: Optional[Literal['inline', 'reference']] = 'inline'
520
- database_connection: Optional[DatabaseConnection] = None
521
- database_connection_name: Optional[str] = None
573
+
574
+ connection_mode: Literal["inline", "reference"] | None = "inline"
575
+ database_connection: DatabaseConnection | None = None
576
+ database_connection_name: str | None = None
522
577
  table_name: str
523
- schema_name: Optional[str] = None
524
- if_exists: Optional[Literal['append', 'replace', 'fail']] = 'append'
578
+ schema_name: str | None = None
579
+ if_exists: Literal["append", "replace", "fail"] | None = "append"
525
580
 
526
581
 
527
582
  class NodeDatabaseReader(NodeBase):
528
583
  """Settings for a node that reads from a database."""
584
+
529
585
  database_settings: DatabaseSettings
530
- fields: Optional[List[MinimalFieldInfo]] = None
586
+ fields: list[MinimalFieldInfo] | None = None
531
587
 
532
588
 
533
589
  class NodeDatabaseWriter(NodeSingleInput):
534
590
  """Settings for a node that writes data to a database."""
591
+
535
592
  database_write_settings: DatabaseWriteSettings
536
593
 
537
594
 
538
595
  class NodeCloudStorageReader(NodeBase):
539
596
  """Settings for a node that reads from a cloud storage service (S3, GCS, etc.)."""
597
+
540
598
  cloud_storage_settings: CloudStorageReadSettings
541
- fields: Optional[List[MinimalFieldInfo]] = None
599
+ fields: list[MinimalFieldInfo] | None = None
542
600
 
543
601
 
544
602
  class NodeCloudStorageWriter(NodeSingleInput):
545
603
  """Settings for a node that writes to a cloud storage service."""
604
+
546
605
  cloud_storage_settings: CloudStorageWriteSettings
547
606
 
548
607
 
549
608
  class ExternalSource(BaseModel):
550
609
  """Base model for data coming from a predefined external source."""
551
- orientation: str = 'row'
552
- fields: Optional[List[MinimalFieldInfo]] = None
610
+
611
+ orientation: str = "row"
612
+ fields: list[MinimalFieldInfo] | None = None
553
613
 
554
614
 
555
615
  class SampleUsers(ExternalSource):
556
616
  """Settings for generating a sample dataset of users."""
617
+
557
618
  SAMPLE_USERS: bool
558
619
  class_name: str = "sample_users"
559
620
  size: int = 100
@@ -561,58 +622,71 @@ class SampleUsers(ExternalSource):
561
622
 
562
623
  class NodeExternalSource(NodeBase):
563
624
  """Settings for a node that connects to a registered external data source."""
625
+
564
626
  identifier: str
565
627
  source_settings: SampleUsers
566
628
 
567
629
 
568
630
  class NodeFormula(NodeSingleInput):
569
631
  """Settings for a node that applies a formula to create/modify a column."""
632
+
570
633
  function: transform_schema.FunctionInput = None
571
634
 
572
635
 
573
636
  class NodeGroupBy(NodeSingleInput):
574
637
  """Settings for a node that performs a group-by and aggregation operation."""
638
+
575
639
  groupby_input: transform_schema.GroupByInput = None
576
640
 
577
641
 
578
642
  class NodePromise(NodeBase):
579
643
  """A placeholder node for an operation that has not yet been configured."""
644
+
580
645
  is_setup: bool = False
581
646
  node_type: str
582
647
 
583
648
 
584
649
  class NodeInputConnection(BaseModel):
585
650
  """Represents the input side of a connection between two nodes."""
651
+
586
652
  node_id: int
587
653
  connection_class: InputConnectionClass
588
654
 
589
- def get_node_input_connection_type(self) -> Literal['main', 'right', 'left']:
655
+ def get_node_input_connection_type(self) -> Literal["main", "right", "left"]:
590
656
  """Determines the semantic type of the input (e.g., for a join)."""
591
657
  match self.connection_class:
592
- case 'input-0': return 'main'
593
- case 'input-1': return 'right'
594
- case 'input-2': return 'left'
595
- case _: raise ValueError(f"Unexpected connection_class: {self.connection_class}")
658
+ case "input-0":
659
+ return "main"
660
+ case "input-1":
661
+ return "right"
662
+ case "input-2":
663
+ return "left"
664
+ case _:
665
+ raise ValueError(f"Unexpected connection_class: {self.connection_class}")
596
666
 
597
667
 
598
668
  class NodePivot(NodeSingleInput):
599
669
  """Settings for a node that pivots data from a long to a wide format."""
670
+
600
671
  pivot_input: transform_schema.PivotInput = None
601
- output_fields: Optional[List[MinimalFieldInfo]] = None
672
+ output_fields: list[MinimalFieldInfo] | None = None
602
673
 
603
674
 
604
675
  class NodeUnpivot(NodeSingleInput):
605
676
  """Settings for a node that unpivots data from a wide to a long format."""
677
+
606
678
  unpivot_input: transform_schema.UnpivotInput = None
607
679
 
608
680
 
609
681
  class NodeUnion(NodeMultiInput):
610
682
  """Settings for a node that concatenates multiple data inputs."""
683
+
611
684
  union_input: transform_schema.UnionInput = Field(default_factory=transform_schema.UnionInput)
612
685
 
613
686
 
614
687
  class NodeOutput(NodeSingleInput):
615
688
  """Settings for a node that writes its input to a file."""
689
+
616
690
  output_settings: OutputSettings
617
691
 
618
692
  def to_yaml_dict(self) -> NodeOutputYaml:
@@ -625,12 +699,14 @@ class NodeOutput(NodeSingleInput):
625
699
 
626
700
  class NodeOutputConnection(BaseModel):
627
701
  """Represents the output side of a connection between two nodes."""
702
+
628
703
  node_id: int
629
704
  connection_class: OutputConnectionClass
630
705
 
631
706
 
632
707
  class NodeConnection(BaseModel):
633
708
  """Represents a connection (edge) between two nodes in the graph."""
709
+
634
710
  input_connection: NodeInputConnection
635
711
  output_connection: NodeOutputConnection
636
712
 
@@ -638,45 +714,56 @@ class NodeConnection(BaseModel):
638
714
  def create_from_simple_input(cls, from_id: int, to_id: int, input_type: InputType = "input-0"):
639
715
  """Creates a standard connection between two nodes."""
640
716
  match input_type:
641
- case "main": connection_class: InputConnectionClass = "input-0"
642
- case "right": connection_class: InputConnectionClass = "input-1"
643
- case "left": connection_class: InputConnectionClass = "input-2"
644
- case _: connection_class: InputConnectionClass = "input-0"
717
+ case "main":
718
+ connection_class: InputConnectionClass = "input-0"
719
+ case "right":
720
+ connection_class: InputConnectionClass = "input-1"
721
+ case "left":
722
+ connection_class: InputConnectionClass = "input-2"
723
+ case _:
724
+ connection_class: InputConnectionClass = "input-0"
645
725
  node_input = NodeInputConnection(node_id=to_id, connection_class=connection_class)
646
- node_output = NodeOutputConnection(node_id=from_id, connection_class='output-0')
726
+ node_output = NodeOutputConnection(node_id=from_id, connection_class="output-0")
647
727
  return cls(input_connection=node_input, output_connection=node_output)
648
728
 
649
729
 
650
730
  class NodeDescription(BaseModel):
651
731
  """A simple model for updating a node's description text."""
652
- description: str = ''
732
+
733
+ description: str = ""
653
734
 
654
735
 
655
736
  class NodeExploreData(NodeBase):
656
737
  """Settings for a node that provides an interactive data exploration interface."""
657
- graphic_walker_input: Optional[gs_schemas.GraphicWalkerInput] = None
738
+
739
+ graphic_walker_input: gs_schemas.GraphicWalkerInput | None = None
658
740
 
659
741
 
660
742
  class NodeGraphSolver(NodeSingleInput):
661
743
  """Settings for a node that solves graph-based problems (e.g., connected components)."""
744
+
662
745
  graph_solver_input: transform_schema.GraphSolverInput
663
746
 
664
747
 
665
748
  class NodeUnique(NodeSingleInput):
666
749
  """Settings for a node that returns the unique rows from the data."""
750
+
667
751
  unique_input: transform_schema.UniqueInput
668
752
 
669
753
 
670
754
  class NodeRecordCount(NodeSingleInput):
671
755
  """Settings for a node that counts the number of records."""
756
+
672
757
  pass
673
758
 
674
759
 
675
760
  class NodePolarsCode(NodeMultiInput):
676
761
  """Settings for a node that executes arbitrary user-provided Polars code."""
762
+
677
763
  polars_code_input: transform_schema.PolarsCodeInput
678
764
 
679
765
 
680
766
  class UserDefinedNode(NodeMultiInput):
681
767
  """Settings for a node that contains the user defined node information"""
768
+
682
769
  settings: Any