Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (346) hide show
  1. build_backends/main.py +25 -22
  2. build_backends/main_prd.py +10 -19
  3. flowfile/__init__.py +194 -74
  4. flowfile/__main__.py +10 -7
  5. flowfile/api.py +51 -57
  6. flowfile/web/__init__.py +14 -9
  7. flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
  8. flowfile/web/static/assets/AdminView-f9847d67.js +713 -0
  9. flowfile/web/static/assets/CloudConnectionView-cf85f943.css +72 -0
  10. flowfile/web/static/assets/{CloudConnectionManager-0dfba9f2.js → CloudConnectionView-faace55b.js} +11 -11
  11. flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
  12. flowfile/web/static/assets/{CloudStorageReader-d5b1b6c9.js → CloudStorageReader-d86ecaa7.js} +10 -8
  13. flowfile/web/static/assets/{CloudStorageWriter-00d87aad.js → CloudStorageWriter-0f4d9a44.js} +10 -8
  14. flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
  15. flowfile/web/static/assets/ColumnActionInput-c44b7aee.css +159 -0
  16. flowfile/web/static/assets/ColumnActionInput-f4189ae0.js +330 -0
  17. flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
  18. flowfile/web/static/assets/{ColumnSelector-4685e75d.js → ColumnSelector-e66b33da.js} +3 -5
  19. flowfile/web/static/assets/ContextMenu-49463352.js +9 -0
  20. flowfile/web/static/assets/ContextMenu-dd5f3f25.js +9 -0
  21. flowfile/web/static/assets/ContextMenu-f709b884.js +9 -0
  22. flowfile/web/static/assets/ContextMenu.vue_vue_type_script_setup_true_lang-a1bd6314.js +59 -0
  23. flowfile/web/static/assets/{CrossJoin-702a3edd.js → CrossJoin-24694b8f.js} +12 -10
  24. flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
  25. flowfile/web/static/assets/{CustomNode-b1519993.js → CustomNode-569d45ff.js} +43 -24
  26. flowfile/web/static/assets/CustomNode-edb9b939.css +42 -0
  27. flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-c20a1e16.css} +23 -21
  28. flowfile/web/static/assets/{DatabaseConnectionSettings-6f3e4ea5.js → DatabaseConnectionSettings-cfc08938.js} +5 -4
  29. flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-5bf8c75b.css} +41 -46
  30. flowfile/web/static/assets/{DatabaseReader-d38c7295.js → DatabaseReader-701feabb.js} +25 -15
  31. flowfile/web/static/assets/{DatabaseManager-cf5ef661.js → DatabaseView-0482e5b5.js} +11 -11
  32. flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
  33. flowfile/web/static/assets/{DatabaseWriter-b04ef46a.js → DatabaseWriter-16721989.js} +17 -10
  34. flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-bdcf2c8b.css} +29 -27
  35. flowfile/web/static/assets/{designer-8da3ba3a.css → DesignerView-49abb835.css} +783 -663
  36. flowfile/web/static/assets/{designer-9633482a.js → DesignerView-f64749fb.js} +1292 -3253
  37. flowfile/web/static/assets/{documentation-ca400224.js → DocumentationView-61bd2990.js} +5 -5
  38. flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-9ea6e871.css} +9 -9
  39. flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
  40. flowfile/web/static/assets/{ExploreData-5fa10ed8.js → ExploreData-e2735b13.js} +18 -9
  41. flowfile/web/static/assets/{ExternalSource-d39af878.js → ExternalSource-2535c3b2.js} +9 -7
  42. flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-7ac7373f.css} +20 -20
  43. flowfile/web/static/assets/Filter-2cdbc93c.js +287 -0
  44. flowfile/web/static/assets/Filter-7494ea97.css +48 -0
  45. flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
  46. flowfile/web/static/assets/{Formula-6b04fb1d.js → Formula-fcda3c2c.js} +13 -11
  47. flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
  48. flowfile/web/static/assets/{FuzzyMatch-999521f4.js → FuzzyMatch-f8d3b7d3.js} +12 -10
  49. flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-4b4d7db9.css} +5 -5
  50. flowfile/web/static/assets/{GraphSolver-17dd2198.js → GraphSolver-72eaa695.js} +14 -12
  51. flowfile/web/static/assets/GroupBy-5792782d.css +9 -0
  52. flowfile/web/static/assets/{GroupBy-6b039e18.js → GroupBy-8aa0598b.js} +9 -7
  53. flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
  54. flowfile/web/static/assets/{Join-24d0f113.js → Join-e40f0ffa.js} +13 -11
  55. flowfile/web/static/assets/LoginView-5111c9ae.js +134 -0
  56. flowfile/web/static/assets/LoginView-d325d632.css +172 -0
  57. flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
  58. flowfile/web/static/assets/{ManualInput-34639209.js → ManualInput-9b6f3224.js} +170 -116
  59. flowfile/web/static/assets/{MultiSelect-0e8724a3.js → MultiSelect-ef28e19e.js} +2 -2
  60. flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-b0e538c2.js → MultiSelect.vue_vue_type_script_setup_true_lang-83b3bbfd.js} +1 -1
  61. flowfile/web/static/assets/NodeDesigner-94cd4dd3.css +1429 -0
  62. flowfile/web/static/assets/NodeDesigner-d2b7ee2b.js +2712 -0
  63. flowfile/web/static/assets/{NumericInput-3d63a470.js → NumericInput-1d789794.js} +2 -2
  64. flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-e0edeccc.js → NumericInput.vue_vue_type_script_setup_true_lang-7775f83e.js} +5 -2
  65. flowfile/web/static/assets/Output-692dd25d.css +37 -0
  66. flowfile/web/static/assets/{Output-edea9802.js → Output-cefef801.js} +14 -10
  67. flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
  68. flowfile/web/static/assets/{Pivot-61d19301.js → Pivot-bab1b75b.js} +12 -10
  69. flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
  70. flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
  71. flowfile/web/static/assets/{PivotValidation-f97fec5b.js → PivotValidation-e7941f91.js} +3 -3
  72. flowfile/web/static/assets/{PivotValidation-de9f43fe.js → PivotValidation-fba09336.js} +3 -3
  73. flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
  74. flowfile/web/static/assets/{PolarsCode-bc3c9984.js → PolarsCode-740e40fa.js} +18 -9
  75. flowfile/web/static/assets/PopOver-862d7e28.js +939 -0
  76. flowfile/web/static/assets/PopOver-d96599db.css +33 -0
  77. flowfile/web/static/assets/{Read-64a3f259.js → Read-225cc63f.js} +16 -12
  78. flowfile/web/static/assets/{Read-e808b239.css → Read-90f366bc.css} +15 -15
  79. flowfile/web/static/assets/{RecordCount-3d5039be.js → RecordCount-ffc71eca.js} +6 -4
  80. flowfile/web/static/assets/{RecordId-597510e0.js → RecordId-a70bb8df.js} +9 -7
  81. flowfile/web/static/assets/{SQLQueryComponent-df51adbe.js → SQLQueryComponent-15a421f5.js} +3 -3
  82. flowfile/web/static/assets/SQLQueryComponent-edb90b98.css +29 -0
  83. flowfile/web/static/assets/{Sample-4be0a507.js → Sample-6c26afc7.js} +6 -4
  84. flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
  85. flowfile/web/static/assets/SecretSelector-ceed9496.js +113 -0
  86. flowfile/web/static/assets/{SecretManager-4839be57.js → SecretsView-214d255a.js} +35 -36
  87. flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
  88. flowfile/web/static/assets/{Select-9b72f201.js → Select-8fc29999.js} +9 -7
  89. flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
  90. flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
  91. flowfile/web/static/assets/{SettingsSection-7ded385d.js → SettingsSection-3f70e4c3.js} +3 -3
  92. flowfile/web/static/assets/{SettingsSection-f0f75a42.js → SettingsSection-83090218.js} +3 -3
  93. flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
  94. flowfile/web/static/assets/{SettingsSection-e1e9c953.js → SettingsSection-9f0d1725.js} +3 -3
  95. flowfile/web/static/assets/SetupView-3fa0aa03.js +160 -0
  96. flowfile/web/static/assets/SetupView-e2da3442.css +230 -0
  97. flowfile/web/static/assets/{SingleSelect-6c777aac.js → SingleSelect-a4a568cb.js} +2 -2
  98. flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-33e3ff9b.js → SingleSelect.vue_vue_type_script_setup_true_lang-c8ebdd33.js} +1 -1
  99. flowfile/web/static/assets/{SliderInput-7cb93e62.js → SliderInput-be533e71.js} +7 -4
  100. flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
  101. flowfile/web/static/assets/{Sort-6cbde21a.js → Sort-154dad81.js} +9 -7
  102. flowfile/web/static/assets/Sort-4abb7fae.css +9 -0
  103. flowfile/web/static/assets/{TextInput-d9a40c11.js → TextInput-454e2bda.js} +2 -2
  104. flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-5896c375.js → TextInput.vue_vue_type_script_setup_true_lang-e86510d0.js} +5 -2
  105. flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
  106. flowfile/web/static/assets/{TextToRows-c4fcbf4d.js → TextToRows-ea73433d.js} +11 -10
  107. flowfile/web/static/assets/{ToggleSwitch-4ef91d19.js → ToggleSwitch-9d7b30f1.js} +2 -2
  108. flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-38478c20.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-00f2580e.js} +1 -1
  109. flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-394a1f78.css} +14 -14
  110. flowfile/web/static/assets/{UnavailableFields-a03f512c.js → UnavailableFields-b72a2c72.js} +4 -4
  111. flowfile/web/static/assets/{Union-bfe9b996.js → Union-1e44f263.js} +8 -6
  112. flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
  113. flowfile/web/static/assets/Unique-2b705521.css +3 -0
  114. flowfile/web/static/assets/{Unique-5d023a27.js → Unique-a3bc6d0a.js} +13 -10
  115. flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-b6ad6427.css} +7 -7
  116. flowfile/web/static/assets/{Unpivot-91cc5354.js → Unpivot-e27935fc.js} +11 -9
  117. flowfile/web/static/assets/{UnpivotValidation-7ee2de44.js → UnpivotValidation-72497680.js} +3 -3
  118. flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
  119. flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
  120. flowfile/web/static/assets/{VueGraphicWalker-e51b9924.js → VueGraphicWalker-d9ab70a3.js} +4 -4
  121. flowfile/web/static/assets/{api-cf1221f0.js → api-a2102880.js} +1 -1
  122. flowfile/web/static/assets/{api-c1bad5ca.js → api-f75042b0.js} +1 -1
  123. flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-1d6acbd9.css} +41 -41
  124. flowfile/web/static/assets/{dropDown-614b998d.js → dropDown-2798a109.js} +3 -3
  125. flowfile/web/static/assets/{fullEditor-f7971590.js → fullEditor-cf7d7d93.js} +11 -10
  126. flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-fe9f7e18.css} +77 -65
  127. flowfile/web/static/assets/{genericNodeSettings-4fe5f36b.js → genericNodeSettings-14eac1c3.js} +5 -5
  128. flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
  129. flowfile/web/static/assets/{index-5429bbf8.js → index-387a6f18.js} +41806 -40958
  130. flowfile/web/static/assets/index-6b367bb5.js +38 -0
  131. flowfile/web/static/assets/{index-50508d4d.css → index-e96ab018.css} +2184 -569
  132. flowfile/web/static/assets/index-f0a6e5a5.js +2696 -0
  133. flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
  134. flowfile/web/static/assets/nodeInput-ed2ae8d7.js +2 -0
  135. flowfile/web/static/assets/{outputCsv-076b85ab.js → outputCsv-3c1757e8.js} +3 -3
  136. flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
  137. flowfile/web/static/assets/{outputExcel-0fd17dbe.js → outputExcel-686e1f48.js} +3 -3
  138. flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
  139. flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
  140. flowfile/web/static/assets/{outputParquet-b61e0847.js → outputParquet-df28faa7.js} +4 -4
  141. flowfile/web/static/assets/{readCsv-c767cb37.css → readCsv-3bfac4c3.css} +15 -15
  142. flowfile/web/static/assets/{readCsv-a8bb8b61.js → readCsv-e37eee21.js} +3 -3
  143. flowfile/web/static/assets/{readExcel-806d2826.css → readExcel-3db6b763.css} +13 -13
  144. flowfile/web/static/assets/{readExcel-67b4aee0.js → readExcel-a13f14bb.js} +5 -5
  145. flowfile/web/static/assets/{readParquet-92ce1dbc.js → readParquet-344cf746.js} +3 -3
  146. flowfile/web/static/assets/{readParquet-48c81530.css → readParquet-c5244ad5.css} +4 -4
  147. flowfile/web/static/assets/secrets.api-ae198c5c.js +65 -0
  148. flowfile/web/static/assets/{selectDynamic-92e25ee3.js → selectDynamic-6b4b0767.js} +5 -5
  149. flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
  150. flowfile/web/static/assets/{vue-codemirror.esm-41b0e0d7.js → vue-codemirror.esm-31ba0e0b.js} +31 -640
  151. flowfile/web/static/assets/{vue-content-loader.es-2c8e608f.js → vue-content-loader.es-4469c8ff.js} +1 -1
  152. flowfile/web/static/index.html +2 -2
  153. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/METADATA +3 -4
  154. flowfile-0.5.4.dist-info/RECORD +407 -0
  155. flowfile_core/__init__.py +13 -6
  156. flowfile_core/auth/jwt.py +51 -16
  157. flowfile_core/auth/models.py +32 -7
  158. flowfile_core/auth/password.py +89 -0
  159. flowfile_core/auth/secrets.py +64 -19
  160. flowfile_core/configs/__init__.py +9 -7
  161. flowfile_core/configs/flow_logger.py +15 -14
  162. flowfile_core/configs/node_store/__init__.py +72 -4
  163. flowfile_core/configs/node_store/nodes.py +155 -172
  164. flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
  165. flowfile_core/configs/settings.py +28 -15
  166. flowfile_core/database/connection.py +7 -6
  167. flowfile_core/database/init_db.py +96 -2
  168. flowfile_core/database/models.py +3 -1
  169. flowfile_core/fileExplorer/__init__.py +17 -0
  170. flowfile_core/fileExplorer/funcs.py +145 -57
  171. flowfile_core/fileExplorer/utils.py +10 -11
  172. flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
  173. flowfile_core/flowfile/analytics/analytics_processor.py +26 -24
  174. flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
  175. flowfile_core/flowfile/analytics/utils.py +1 -1
  176. flowfile_core/flowfile/code_generator/__init__.py +11 -0
  177. flowfile_core/flowfile/code_generator/code_generator.py +706 -247
  178. flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
  179. flowfile_core/flowfile/connection_manager/models.py +1 -1
  180. flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
  181. flowfile_core/flowfile/database_connection_manager/models.py +1 -1
  182. flowfile_core/flowfile/extensions.py +17 -12
  183. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
  184. flowfile_core/flowfile/flow_data_engine/create/funcs.py +115 -83
  185. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +493 -423
  186. flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
  187. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
  188. flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
  189. flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
  190. flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
  191. flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +31 -20
  192. flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
  193. flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
  194. flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +14 -15
  195. flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
  196. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
  197. flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
  198. flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
  199. flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
  200. flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
  201. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +190 -127
  202. flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
  203. flowfile_core/flowfile/flow_data_engine/utils.py +99 -67
  204. flowfile_core/flowfile/flow_graph.py +920 -571
  205. flowfile_core/flowfile/flow_graph_utils.py +31 -49
  206. flowfile_core/flowfile/flow_node/flow_node.py +379 -258
  207. flowfile_core/flowfile/flow_node/models.py +53 -41
  208. flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
  209. flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
  210. flowfile_core/flowfile/handler.py +80 -30
  211. flowfile_core/flowfile/manage/compatibility_enhancements.py +209 -126
  212. flowfile_core/flowfile/manage/io_flowfile.py +54 -57
  213. flowfile_core/flowfile/node_designer/__init__.py +19 -13
  214. flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
  215. flowfile_core/flowfile/node_designer/custom_node.py +162 -36
  216. flowfile_core/flowfile/node_designer/ui_components.py +278 -34
  217. flowfile_core/flowfile/schema_callbacks.py +71 -51
  218. flowfile_core/flowfile/setting_generator/__init__.py +0 -1
  219. flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
  220. flowfile_core/flowfile/setting_generator/settings.py +64 -53
  221. flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
  222. flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
  223. flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
  224. flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
  225. flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
  226. flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
  227. flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
  228. flowfile_core/flowfile/util/calculate_layout.py +9 -13
  229. flowfile_core/flowfile/util/execution_orderer.py +25 -17
  230. flowfile_core/flowfile/util/node_skipper.py +4 -4
  231. flowfile_core/flowfile/utils.py +19 -21
  232. flowfile_core/main.py +26 -19
  233. flowfile_core/routes/auth.py +284 -11
  234. flowfile_core/routes/cloud_connections.py +25 -25
  235. flowfile_core/routes/logs.py +21 -29
  236. flowfile_core/routes/public.py +46 -4
  237. flowfile_core/routes/routes.py +70 -34
  238. flowfile_core/routes/secrets.py +25 -27
  239. flowfile_core/routes/user_defined_components.py +483 -4
  240. flowfile_core/run_lock.py +0 -1
  241. flowfile_core/schemas/__init__.py +4 -6
  242. flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
  243. flowfile_core/schemas/cloud_storage_schemas.py +96 -66
  244. flowfile_core/schemas/input_schema.py +231 -144
  245. flowfile_core/schemas/output_model.py +49 -34
  246. flowfile_core/schemas/schemas.py +116 -89
  247. flowfile_core/schemas/transform_schema.py +518 -263
  248. flowfile_core/schemas/yaml_types.py +21 -7
  249. flowfile_core/secret_manager/secret_manager.py +123 -18
  250. flowfile_core/types.py +29 -9
  251. flowfile_core/utils/arrow_reader.py +7 -6
  252. flowfile_core/utils/excel_file_manager.py +3 -3
  253. flowfile_core/utils/fileManager.py +7 -7
  254. flowfile_core/utils/fl_executor.py +8 -10
  255. flowfile_core/utils/utils.py +4 -4
  256. flowfile_core/utils/validate_setup.py +5 -4
  257. flowfile_frame/__init__.py +117 -51
  258. flowfile_frame/adapters.py +2 -9
  259. flowfile_frame/adding_expr.py +73 -32
  260. flowfile_frame/cloud_storage/frame_helpers.py +27 -23
  261. flowfile_frame/cloud_storage/secret_manager.py +12 -26
  262. flowfile_frame/config.py +2 -5
  263. flowfile_frame/database/__init__.py +36 -0
  264. flowfile_frame/database/connection_manager.py +205 -0
  265. flowfile_frame/database/frame_helpers.py +249 -0
  266. flowfile_frame/expr.py +311 -218
  267. flowfile_frame/expr.pyi +160 -159
  268. flowfile_frame/expr_name.py +23 -23
  269. flowfile_frame/flow_frame.py +571 -476
  270. flowfile_frame/flow_frame.pyi +123 -104
  271. flowfile_frame/flow_frame_methods.py +227 -246
  272. flowfile_frame/group_frame.py +50 -20
  273. flowfile_frame/join.py +2 -2
  274. flowfile_frame/lazy.py +129 -87
  275. flowfile_frame/lazy_methods.py +83 -30
  276. flowfile_frame/list_name_space.py +55 -50
  277. flowfile_frame/selectors.py +148 -68
  278. flowfile_frame/series.py +9 -7
  279. flowfile_frame/utils.py +19 -21
  280. flowfile_worker/__init__.py +12 -7
  281. flowfile_worker/configs.py +41 -33
  282. flowfile_worker/create/__init__.py +14 -9
  283. flowfile_worker/create/funcs.py +114 -77
  284. flowfile_worker/create/models.py +46 -43
  285. flowfile_worker/create/pl_types.py +14 -15
  286. flowfile_worker/create/read_excel_tables.py +34 -41
  287. flowfile_worker/create/utils.py +22 -19
  288. flowfile_worker/external_sources/s3_source/main.py +18 -51
  289. flowfile_worker/external_sources/s3_source/models.py +34 -27
  290. flowfile_worker/external_sources/sql_source/main.py +8 -5
  291. flowfile_worker/external_sources/sql_source/models.py +13 -9
  292. flowfile_worker/flow_logger.py +10 -8
  293. flowfile_worker/funcs.py +214 -155
  294. flowfile_worker/main.py +11 -17
  295. flowfile_worker/models.py +35 -28
  296. flowfile_worker/process_manager.py +2 -3
  297. flowfile_worker/routes.py +121 -90
  298. flowfile_worker/secrets.py +114 -21
  299. flowfile_worker/spawner.py +89 -54
  300. flowfile_worker/utils.py +3 -2
  301. shared/__init__.py +2 -7
  302. shared/storage_config.py +25 -13
  303. test_utils/postgres/commands.py +3 -2
  304. test_utils/postgres/fixtures.py +9 -9
  305. test_utils/s3/commands.py +1 -1
  306. test_utils/s3/data_generator.py +3 -4
  307. test_utils/s3/demo_data_generator.py +4 -7
  308. test_utils/s3/fixtures.py +7 -5
  309. tools/migrate/__init__.py +1 -1
  310. tools/migrate/__main__.py +16 -29
  311. tools/migrate/legacy_schemas.py +251 -190
  312. tools/migrate/migrate.py +193 -181
  313. tools/migrate/tests/conftest.py +1 -3
  314. tools/migrate/tests/test_migrate.py +36 -41
  315. tools/migrate/tests/test_migration_e2e.py +28 -29
  316. tools/migrate/tests/test_node_migrations.py +50 -20
  317. flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
  318. flowfile/web/static/assets/ContextMenu-23e909da.js +0 -41
  319. flowfile/web/static/assets/ContextMenu-4c74eef1.css +0 -26
  320. flowfile/web/static/assets/ContextMenu-63cfa99b.css +0 -26
  321. flowfile/web/static/assets/ContextMenu-70ae0c79.js +0 -41
  322. flowfile/web/static/assets/ContextMenu-c13f91d0.css +0 -26
  323. flowfile/web/static/assets/ContextMenu-f149cf7c.js +0 -41
  324. flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
  325. flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
  326. flowfile/web/static/assets/Filter-9b6d08db.js +0 -164
  327. flowfile/web/static/assets/Filter-f62091b3.css +0 -20
  328. flowfile/web/static/assets/GroupBy-b9505323.css +0 -51
  329. flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
  330. flowfile/web/static/assets/Output-283fe388.css +0 -37
  331. flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
  332. flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
  333. flowfile/web/static/assets/SQLQueryComponent-36cef432.css +0 -27
  334. flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
  335. flowfile/web/static/assets/Sort-3643d625.css +0 -51
  336. flowfile/web/static/assets/Unique-f9fb0809.css +0 -51
  337. flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
  338. flowfile/web/static/assets/nodeInput-5d0d6b79.js +0 -41
  339. flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
  340. flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
  341. flowfile/web/static/assets/secretApi-68435402.js +0 -46
  342. flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
  343. flowfile-0.5.1.dist-info/RECORD +0 -388
  344. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/WHEEL +0 -0
  345. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/entry_points.txt +0 -0
  346. {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/licenses/LICENSE +0 -0
@@ -1,5 +1,3 @@
1
- from typing import List, Union, TYPE_CHECKING # Added TYPE_CHECKING
2
-
3
1
  import polars as pl
4
2
 
5
3
  # --- TYPE CHECKING IMPORTS ---
@@ -7,11 +5,12 @@ import polars as pl
7
5
  # Import Expr only for type hints
8
6
  from flowfile_frame.expr import Expr
9
7
 
10
-
11
8
  # --- Selector Base Classes (Compound, Complement) ---
12
9
 
10
+
13
11
  class Selector:
14
12
  """Base class for column selectors, inspired by polars.selectors"""
13
+
15
14
  def __init__(self):
16
15
  self._repr_str = self._get_repr_str() # Use base repr calculation method
17
16
  # No agg_func state stored here anymore
@@ -28,25 +27,25 @@ class Selector:
28
27
  def __repr__(self) -> str:
29
28
  return self._repr_str
30
29
 
31
- def __or__(self, other: 'Selector') -> 'CompoundSelector':
32
- return CompoundSelector(self, other, 'union')
30
+ def __or__(self, other: "Selector") -> "CompoundSelector":
31
+ return CompoundSelector(self, other, "union")
33
32
 
34
- def __and__(self, other: 'Selector') -> 'CompoundSelector':
35
- return CompoundSelector(self, other, 'intersection')
33
+ def __and__(self, other: "Selector") -> "CompoundSelector":
34
+ return CompoundSelector(self, other, "intersection")
36
35
 
37
- def __sub__(self, other: 'Selector') -> 'CompoundSelector':
38
- return CompoundSelector(self, other, 'difference')
36
+ def __sub__(self, other: "Selector") -> "CompoundSelector":
37
+ return CompoundSelector(self, other, "difference")
39
38
 
40
- def __xor__(self, other: 'Selector') -> 'CompoundSelector':
41
- return CompoundSelector(self, other, 'symmetric_difference')
39
+ def __xor__(self, other: "Selector") -> "CompoundSelector":
40
+ return CompoundSelector(self, other, "symmetric_difference")
42
41
 
43
- def __invert__(self) -> 'ComplementSelector':
42
+ def __invert__(self) -> "ComplementSelector":
44
43
  return ComplementSelector(self)
45
44
 
46
45
  # --- Aggregation Methods ---
47
46
  # These methods now return Expr objects, importing Expr locally
48
47
 
49
- def sum(self) -> 'Expr':
48
+ def sum(self) -> "Expr":
50
49
  """Create an expression to sum columns selected by this selector."""
51
50
  # Expr init will handle creating the 'pl.sum(selector)' repr
52
51
  return Expr(expr=None, selector=self, agg_func="sum")
@@ -54,43 +53,43 @@ class Selector:
54
53
  def expr(self):
55
54
  return eval(self.repr_str)
56
55
 
57
- def mean(self) -> 'Expr':
56
+ def mean(self) -> "Expr":
58
57
  """Create an expression to average columns selected by this selector."""
59
58
  return Expr(expr=None, selector=self, agg_func="mean")
60
59
 
61
- def median(self) -> 'Expr':
60
+ def median(self) -> "Expr":
62
61
  """Create an expression to find the median of columns selected by this selector."""
63
62
  return Expr(expr=None, selector=self, agg_func="median")
64
63
 
65
- def min(self) -> 'Expr':
64
+ def min(self) -> "Expr":
66
65
  """Create an expression to find the minimum of columns selected by this selector."""
67
66
  return Expr(expr=None, selector=self, agg_func="min")
68
67
 
69
- def max(self) -> 'Expr':
68
+ def max(self) -> "Expr":
70
69
  """Create an expression to find the maximum of columns selected by this selector."""
71
70
  return Expr(expr=None, selector=self, agg_func="max")
72
71
 
73
- def std(self, ddof: int = 1) -> 'Expr':
72
+ def std(self, ddof: int = 1) -> "Expr":
74
73
  """Create an expression to find the standard deviation of columns selected by this selector."""
75
74
  return Expr(expr=None, selector=self, agg_func="std", ddof=ddof)
76
75
 
77
- def var(self, ddof: int = 1) -> 'Expr':
76
+ def var(self, ddof: int = 1) -> "Expr":
78
77
  """Create an expression to find the variance of columns selected by this selector."""
79
78
  return Expr(expr=None, selector=self, agg_func="var", ddof=ddof)
80
79
 
81
- def first(self) -> 'Expr':
80
+ def first(self) -> "Expr":
82
81
  """Create an expression to get the first element of columns selected by this selector."""
83
82
  return Expr(expr=None, selector=self, agg_func="first")
84
83
 
85
- def last(self) -> 'Expr':
84
+ def last(self) -> "Expr":
86
85
  """Create an expression to get the last element of columns selected by this selector."""
87
86
  return Expr(expr=None, selector=self, agg_func="last")
88
87
 
89
- def count(self) -> 'Expr':
88
+ def count(self) -> "Expr":
90
89
  """Create an expression to count elements in columns selected by this selector."""
91
90
  return Expr(expr=None, selector=self, agg_func="count")
92
91
 
93
- def n_unique(self) -> 'Expr':
92
+ def n_unique(self) -> "Expr":
94
93
  """Create an expression to count unique elements in columns selected by this selector."""
95
94
  return Expr(expr=None, selector=self, agg_func="n_unique")
96
95
 
@@ -99,6 +98,7 @@ class Selector:
99
98
 
100
99
  class CompoundSelector(Selector):
101
100
  """Selector representing a compound operation between two selectors"""
101
+
102
102
  def __init__(self, left: Selector, right: Selector, operation: str):
103
103
  self.left = left
104
104
  self.right = right
@@ -106,8 +106,8 @@ class CompoundSelector(Selector):
106
106
  super().__init__()
107
107
 
108
108
  def _get_repr_str(self) -> str:
109
- op_map = {'union': '|', 'intersection': '&', 'difference': '-', 'symmetric_difference': '^'}
110
- op_symbol = op_map.get(self.operation, '|')
109
+ op_map = {"union": "|", "intersection": "&", "difference": "-", "symmetric_difference": "^"}
110
+ op_symbol = op_map.get(self.operation, "|")
111
111
  # Use base repr (_repr_str) of operands
112
112
  left_repr = f"({self.left._repr_str})" if isinstance(self.left, CompoundSelector) else self.left._repr_str
113
113
  right_repr = f"({self.right._repr_str})" if isinstance(self.right, CompoundSelector) else self.right._repr_str
@@ -116,77 +116,95 @@ class CompoundSelector(Selector):
116
116
 
117
117
  class ComplementSelector(Selector):
118
118
  """Selector representing the complement (NOT) of another selector"""
119
+
119
120
  def __init__(self, selector: Selector):
120
121
  self.selector = selector
121
122
  super().__init__()
122
123
 
123
124
  def _get_repr_str(self) -> str:
124
- selector_repr = f"({self.selector._repr_str})" if isinstance(self.selector, CompoundSelector) else self.selector._repr_str
125
+ selector_repr = (
126
+ f"({self.selector._repr_str})" if isinstance(self.selector, CompoundSelector) else self.selector._repr_str
127
+ )
125
128
  return f"~{selector_repr}"
126
129
 
127
130
 
128
131
  class NumericSelector(Selector):
129
- def _get_repr_str(self) -> str: return "pl.selectors.numeric()"
132
+ def _get_repr_str(self) -> str:
133
+ return "pl.selectors.numeric()"
130
134
 
131
135
 
132
136
  class FloatSelector(Selector):
133
- def _get_repr_str(self) -> str: return "pl.selectors.float()"
137
+ def _get_repr_str(self) -> str:
138
+ return "pl.selectors.float()"
134
139
 
135
140
 
136
141
  class IntegerSelector(Selector):
137
- def _get_repr_str(self) -> str: return "pl.selectors.integer()"
142
+ def _get_repr_str(self) -> str:
143
+ return "pl.selectors.integer()"
138
144
 
139
145
 
140
146
  class StringSelector(Selector):
141
- def _get_repr_str(self) -> str: return "pl.selectors.string()"
147
+ def _get_repr_str(self) -> str:
148
+ return "pl.selectors.string()"
142
149
 
143
150
 
144
151
  class TemporalSelector(Selector):
145
- def _get_repr_str(self) -> str: return "pl.selectors.temporal()"
152
+ def _get_repr_str(self) -> str:
153
+ return "pl.selectors.temporal()"
146
154
 
147
155
 
148
156
  class DatetimeSelector(Selector):
149
- def _get_repr_str(self) -> str: return "pl.selectors.datetime()"
157
+ def _get_repr_str(self) -> str:
158
+ return "pl.selectors.datetime()"
150
159
 
151
160
 
152
161
  class DateSelector(Selector):
153
- def _get_repr_str(self) -> str: return "pl.selectors.date()"
162
+ def _get_repr_str(self) -> str:
163
+ return "pl.selectors.date()"
154
164
 
155
165
 
156
166
  class TimeSelector(Selector):
157
- def _get_repr_str(self) -> str: return "pl.selectors.time()"
167
+ def _get_repr_str(self) -> str:
168
+ return "pl.selectors.time()"
158
169
 
159
170
 
160
171
  class DurationSelector(Selector):
161
- def _get_repr_str(self) -> str: return "pl.selectors.duration()"
172
+ def _get_repr_str(self) -> str:
173
+ return "pl.selectors.duration()"
162
174
 
163
175
 
164
176
  class BooleanSelector(Selector):
165
- def _get_repr_str(self) -> str: return "pl.selectors.boolean()"
177
+ def _get_repr_str(self) -> str:
178
+ return "pl.selectors.boolean()"
166
179
 
167
180
 
168
181
  class CategoricalSelector(Selector):
169
- def _get_repr_str(self) -> str: return "pl.selectors.categorical()"
182
+ def _get_repr_str(self) -> str:
183
+ return "pl.selectors.categorical()"
170
184
 
171
185
 
172
186
  class ObjectSelector(Selector):
173
- def _get_repr_str(self) -> str: return "pl.selectors.object()"
187
+ def _get_repr_str(self) -> str:
188
+ return "pl.selectors.object()"
174
189
 
175
190
 
176
191
  class ListSelector(Selector):
177
- def _get_repr_str(self) -> str: return "pl.selectors.list()"
192
+ def _get_repr_str(self) -> str:
193
+ return "pl.selectors.list()"
178
194
 
179
195
 
180
196
  class StructSelector(Selector):
181
- def _get_repr_str(self) -> str: return "pl.selectors.struct()"
197
+ def _get_repr_str(self) -> str:
198
+ return "pl.selectors.struct()"
182
199
 
183
200
 
184
201
  class AllSelector(Selector):
185
- def _get_repr_str(self) -> str: return "pl.selectors.all()"
202
+ def _get_repr_str(self) -> str:
203
+ return "pl.selectors.all()"
186
204
 
187
205
 
188
206
  class DtypeSelector(Selector):
189
- def __init__(self, dtypes: Union[pl.DataType, List[pl.DataType]]):
207
+ def __init__(self, dtypes: pl.DataType | list[pl.DataType]):
190
208
  self.dtypes = dtypes if isinstance(dtypes, list) else [dtypes]
191
209
  super().__init__()
192
210
 
@@ -208,38 +226,100 @@ class PatternSelector(Selector):
208
226
 
209
227
 
210
228
  class ContainsSelector(PatternSelector):
211
- def _get_repr_str(self) -> str: return f"pl.selectors.contains({self.pattern!r})"
229
+ def _get_repr_str(self) -> str:
230
+ return f"pl.selectors.contains({self.pattern!r})"
212
231
 
213
232
 
214
233
  class StartsWithSelector(PatternSelector):
215
- def _get_repr_str(self) -> str: return f"pl.selectors.starts_with({self.pattern!r})"
234
+ def _get_repr_str(self) -> str:
235
+ return f"pl.selectors.starts_with({self.pattern!r})"
216
236
 
217
237
 
218
238
  class EndsWithSelector(PatternSelector):
219
- def _get_repr_str(self) -> str: return f"pl.selectors.ends_with({self.pattern!r})"
239
+ def _get_repr_str(self) -> str:
240
+ return f"pl.selectors.ends_with({self.pattern!r})"
220
241
 
221
242
 
222
243
  class MatchesSelector(PatternSelector):
223
- def _get_repr_str(self) -> str: return f"pl.selectors.matches({self.pattern!r})"
224
-
225
-
226
- def numeric() -> NumericSelector: return NumericSelector()
227
- def float_() -> FloatSelector: return FloatSelector()
228
- def integer() -> IntegerSelector: return IntegerSelector()
229
- def string() -> StringSelector: return StringSelector()
230
- def temporal() -> TemporalSelector: return TemporalSelector()
231
- def datetime() -> DatetimeSelector: return DatetimeSelector()
232
- def date() -> DateSelector: return DateSelector()
233
- def time() -> TimeSelector: return TimeSelector()
234
- def duration() -> DurationSelector: return DurationSelector()
235
- def boolean() -> BooleanSelector: return BooleanSelector()
236
- def categorical() -> CategoricalSelector: return CategoricalSelector()
237
- def object_() -> ObjectSelector: return ObjectSelector()
238
- def list_() -> ListSelector: return ListSelector()
239
- def struct() -> StructSelector: return StructSelector()
240
- def all_() -> AllSelector: return AllSelector()
241
- def by_dtype(dtypes: Union[pl.DataType, List[pl.DataType]]) -> DtypeSelector: return DtypeSelector(dtypes)
242
- def contains(pattern: str) -> ContainsSelector: return ContainsSelector(pattern)
243
- def starts_with(pattern: str) -> StartsWithSelector: return StartsWithSelector(pattern)
244
- def ends_with(pattern: str) -> EndsWithSelector: return EndsWithSelector(pattern)
245
- def matches(pattern: str) -> MatchesSelector: return MatchesSelector(pattern)
244
+ def _get_repr_str(self) -> str:
245
+ return f"pl.selectors.matches({self.pattern!r})"
246
+
247
+
248
+ def numeric() -> NumericSelector:
249
+ return NumericSelector()
250
+
251
+
252
+ def float_() -> FloatSelector:
253
+ return FloatSelector()
254
+
255
+
256
+ def integer() -> IntegerSelector:
257
+ return IntegerSelector()
258
+
259
+
260
+ def string() -> StringSelector:
261
+ return StringSelector()
262
+
263
+
264
+ def temporal() -> TemporalSelector:
265
+ return TemporalSelector()
266
+
267
+
268
+ def datetime() -> DatetimeSelector:
269
+ return DatetimeSelector()
270
+
271
+
272
+ def date() -> DateSelector:
273
+ return DateSelector()
274
+
275
+
276
+ def time() -> TimeSelector:
277
+ return TimeSelector()
278
+
279
+
280
+ def duration() -> DurationSelector:
281
+ return DurationSelector()
282
+
283
+
284
+ def boolean() -> BooleanSelector:
285
+ return BooleanSelector()
286
+
287
+
288
+ def categorical() -> CategoricalSelector:
289
+ return CategoricalSelector()
290
+
291
+
292
+ def object_() -> ObjectSelector:
293
+ return ObjectSelector()
294
+
295
+
296
+ def list_() -> ListSelector:
297
+ return ListSelector()
298
+
299
+
300
+ def struct() -> StructSelector:
301
+ return StructSelector()
302
+
303
+
304
+ def all_() -> AllSelector:
305
+ return AllSelector()
306
+
307
+
308
+ def by_dtype(dtypes: pl.DataType | list[pl.DataType]) -> DtypeSelector:
309
+ return DtypeSelector(dtypes)
310
+
311
+
312
+ def contains(pattern: str) -> ContainsSelector:
313
+ return ContainsSelector(pattern)
314
+
315
+
316
+ def starts_with(pattern: str) -> StartsWithSelector:
317
+ return StartsWithSelector(pattern)
318
+
319
+
320
+ def ends_with(pattern: str) -> EndsWithSelector:
321
+ return EndsWithSelector(pattern)
322
+
323
+
324
+ def matches(pattern: str) -> MatchesSelector:
325
+ return MatchesSelector(pattern)
flowfile_frame/series.py CHANGED
@@ -1,6 +1,8 @@
1
1
  from __future__ import annotations
2
+
3
+ from typing import Any
4
+
2
5
  import polars as pl
3
- from typing import Any, Optional, Union, List
4
6
 
5
7
 
6
8
  class Series:
@@ -9,11 +11,11 @@ class Series:
9
11
  """
10
12
 
11
13
  def __init__(
12
- self,
13
- name: str | list | pl.Series | None = None,
14
- values: list | None = None,
15
- dtype: Any = None,
16
- **kwargs # Ignored parameters
14
+ self,
15
+ name: str | list | pl.Series | None = None,
16
+ values: list | None = None,
17
+ dtype: Any = None,
18
+ **kwargs, # Ignored parameters
17
19
  ):
18
20
  """
19
21
  Initialize a FlowSeries with the same API as pl.Series.
@@ -67,4 +69,4 @@ class Series:
67
69
 
68
70
  def __str__(self) -> str:
69
71
  """Same as __repr__."""
70
- return self.__repr__()
72
+ return self.__repr__()
flowfile_frame/utils.py CHANGED
@@ -1,25 +1,23 @@
1
+ import inspect
2
+ import textwrap
1
3
  import uuid
2
- import time
3
- import os
4
- import requests
5
- import subprocess
6
- from pathlib import Path
4
+ from collections.abc import Iterable
5
+ from typing import Any
6
+
7
7
  import polars as pl
8
- from typing import Iterable, Any, List, Optional
9
8
 
10
9
  from flowfile_core.flowfile.flow_graph import FlowGraph
11
10
  from flowfile_core.schemas import schemas
12
- from tempfile import TemporaryDirectory
13
- import inspect
14
- import textwrap
11
+
15
12
 
16
13
  def _is_iterable(obj: Any) -> bool:
17
14
  # Avoid treating strings as iterables in this context
18
15
  return isinstance(obj, Iterable) and not isinstance(obj, (str, bytes))
19
16
 
20
17
 
21
- def _check_if_convertible_to_code(expressions: List[Any]) -> bool:
18
+ def _check_if_convertible_to_code(expressions: list[Any]) -> bool:
22
19
  from flowfile_frame.expr import Expr
20
+
23
21
  for expr in expressions:
24
22
  if isinstance(expr, Expr):
25
23
  if not expr.convertable_to_code:
@@ -27,7 +25,9 @@ def _check_if_convertible_to_code(expressions: List[Any]) -> bool:
27
25
  return True
28
26
 
29
27
 
30
- def _parse_inputs_as_iterable(inputs: tuple[Any, ...] | tuple[Iterable[Any]],) -> List[Any]:
28
+ def _parse_inputs_as_iterable(
29
+ inputs: tuple[Any, ...] | tuple[Iterable[Any]],
30
+ ) -> list[Any]:
31
31
  if not inputs:
32
32
  return []
33
33
 
@@ -55,7 +55,7 @@ def _get_function_source(func):
55
55
  source = inspect.getsource(func)
56
56
 
57
57
  # Check if it's a lambda
58
- if func.__name__ == '<lambda>':
58
+ if func.__name__ == "<lambda>":
59
59
  # Extract just the lambda expression
60
60
  # This is tricky as getsource returns the entire line
61
61
  return None, False
@@ -72,9 +72,9 @@ def _get_function_source(func):
72
72
  return None, False
73
73
 
74
74
 
75
- def ensure_inputs_as_iterable(inputs: Any | Iterable[Any]) -> List[Any]:
75
+ def ensure_inputs_as_iterable(inputs: Any | Iterable[Any]) -> list[Any]:
76
76
  """Convert inputs to list, treating strings as single items."""
77
- if inputs is None or (hasattr(inputs, '__len__') and len(inputs) == 0):
77
+ if inputs is None or (hasattr(inputs, "__len__") and len(inputs) == 0):
78
78
  return []
79
79
  # Treat strings/bytes as atomic items, everything else check if iterable
80
80
  if isinstance(inputs, (str, bytes)) or not _is_iterable(inputs):
@@ -99,13 +99,11 @@ def create_flow_graph(flow_id: int = None) -> FlowGraph:
99
99
  """
100
100
  if flow_id is None:
101
101
  flow_id = _generate_id()
102
- flow_settings = schemas.FlowSettings(
103
- flow_id=flow_id,
104
- name=f"Flow_{flow_id}",
105
- path=f"flow_{flow_id}"
106
- )
102
+ flow_settings = schemas.FlowSettings(flow_id=flow_id, name=f"Flow_{flow_id}", path=f"flow_{flow_id}")
107
103
  flow_graph = FlowGraph(flow_settings=flow_settings)
108
- flow_graph.flow_settings.execution_location = 'local' # always create a local frame so that the run time does not attempt to use the flowfile_worker process
104
+ flow_graph.flow_settings.execution_location = (
105
+ "local" # always create a local frame so that the run time does not attempt to use the flowfile_worker process
106
+ )
109
107
  return flow_graph
110
108
 
111
109
 
@@ -140,4 +138,4 @@ def generate_node_id() -> int:
140
138
 
141
139
  def set_node_id(node_id):
142
140
  """Set the node ID to a specific value."""
143
- data["c"] = node_id
141
+ data["c"] = node_id
@@ -1,19 +1,24 @@
1
- from typing import Dict
2
- import threading
1
+ # ruff: noqa: E402
2
+
3
3
  import multiprocessing
4
- from shared.storage_config import storage
5
- from importlib.metadata import version
4
+ import threading
5
+ from importlib.metadata import PackageNotFoundError, version
6
6
 
7
- __version__ = version("Flowfile")
7
+ from shared.storage_config import storage
8
8
 
9
+ try:
10
+ __version__ = version("Flowfile")
11
+ except PackageNotFoundError:
12
+ __version__ = "0.5.0"
9
13
  multiprocessing.set_start_method('spawn', force=True)
10
14
 
11
15
  from multiprocessing import get_context
16
+
12
17
  from flowfile_worker.models import Status
13
18
 
14
19
  mp_context = get_context("spawn")
15
20
 
16
- status_dict: Dict[str, Status] = dict()
21
+ status_dict: dict[str, Status] = dict()
17
22
  process_dict = dict()
18
23
 
19
24
  status_dict_lock = threading.Lock()
@@ -26,4 +31,4 @@ CACHE_EXPIRATION_TIME = 24 * 60 * 60
26
31
  CACHE_DIR = storage.cache_directory
27
32
 
28
33
 
29
- PROCESS_MEMORY_USAGE: Dict[str, float] = dict()
34
+ PROCESS_MEMORY_USAGE: dict[str, float] = dict()
@@ -1,34 +1,32 @@
1
1
  # flowfile_worker.configs
2
2
 
3
- import logging
4
- import platform
5
3
  import argparse
4
+ import logging
5
+ import multiprocessing
6
6
  import os
7
- from connectorx import __version__
7
+ import platform
8
8
 
9
+ from connectorx import __version__
9
10
 
10
11
  # Configure logging
11
- logging.basicConfig(format='%(asctime)s: %(message)s')
12
- logger = logging.getLogger('FlowfileWorker')
12
+ logging.basicConfig(format="%(asctime)s: %(message)s")
13
+ logger = logging.getLogger("FlowfileWorker")
13
14
  logger.setLevel(logging.INFO)
14
15
 
15
16
  # Constants for worker and core configuration
16
17
  DEFAULT_SERVICE_HOST = "0.0.0.0" if platform.system() != "Windows" else "127.0.0.1"
17
18
  DEFAULT_SERVICE_PORT = 63579
18
- DEFAULT_CORE_HOST = "0.0.0.0" if platform.system() != "Windows" else "127.0.0.1"
19
- DEFAULT_CORE_PORT = 63578
20
- TEST_MODE = True if 'TEST_MODE' in os.environ else False
19
+ # Check environment variable for core host (used in Docker mode)
20
+ DEFAULT_CORE_HOST = os.environ.get("CORE_HOST", "0.0.0.0" if platform.system() != "Windows" else "127.0.0.1")
21
+ DEFAULT_CORE_PORT = int(os.environ.get("CORE_PORT", 63578))
22
+ TEST_MODE = True if "TEST_MODE" in os.environ else False
21
23
 
22
24
 
23
25
  def parse_args():
24
26
  """Parse command line arguments"""
25
27
  parser = argparse.ArgumentParser(description="Flowfile Worker Server")
26
- parser.add_argument(
27
- "--host", type=str, default=DEFAULT_SERVICE_HOST, help="Host to bind worker to"
28
- )
29
- parser.add_argument(
30
- "--port", type=int, default=DEFAULT_SERVICE_PORT, help="Port to bind worker to"
31
- )
28
+ parser.add_argument("--host", type=str, default=DEFAULT_SERVICE_HOST, help="Host to bind worker to")
29
+ parser.add_argument("--port", type=int, default=DEFAULT_SERVICE_PORT, help="Port to bind worker to")
32
30
  parser.add_argument(
33
31
  "--core-host",
34
32
  type=str,
@@ -47,14 +45,10 @@ def parse_args():
47
45
 
48
46
  # Validate arguments
49
47
  if args.port < 1 or args.port > 65535:
50
- raise ValueError(
51
- f"Invalid port number: {args.port}. Port must be between 1 and 65535."
52
- )
48
+ raise ValueError(f"Invalid port number: {args.port}. Port must be between 1 and 65535.")
53
49
 
54
50
  if args.core_port < 1 or args.core_port > 65535:
55
- raise ValueError(
56
- f"Invalid core port number: {args.core_port}. Port must be between 1 and 65535."
57
- )
51
+ raise ValueError(f"Invalid core port number: {args.core_port}. Port must be between 1 and 65535.")
58
52
 
59
53
  # Check if hosts are valid (basic check)
60
54
  if not args.host:
@@ -77,19 +71,33 @@ def get_core_url(host, port):
77
71
  return f"http://{host}:{port}"
78
72
 
79
73
 
80
- # Parse arguments - defaults are already set in the argument parser
81
- args = parse_args()
74
+ def _is_main_process():
75
+ """Check if we're running in the main process (not a spawned child)"""
76
+ return multiprocessing.current_process().name == "MainProcess"
82
77
 
83
- # These variables will already use defaults from argparse if not provided
84
- SERVICE_HOST = args.host
85
- SERVICE_PORT = args.port
86
- CORE_HOST = args.core_host
87
- CORE_PORT = args.core_port
88
78
 
89
- # Generate the core URI
90
- FLOWFILE_CORE_URI = get_core_url(CORE_HOST, CORE_PORT)
79
+ # Only parse args and log in the main process
80
+ # Spawned child processes will use environment variables or defaults
81
+ if _is_main_process():
82
+ # Parse arguments - defaults are already set in the argument parser
83
+ args = parse_args()
91
84
 
92
- logger.info(f"ConnectorX version: {__version__}")
93
- # Log configuration
94
- logger.info(f"Worker configured at {SERVICE_HOST}:{SERVICE_PORT}")
95
- logger.info(f"Core service configured at {FLOWFILE_CORE_URI}")
85
+ # These variables will already use defaults from argparse if not provided
86
+ SERVICE_HOST = args.host
87
+ SERVICE_PORT = args.port
88
+ CORE_HOST = args.core_host
89
+ CORE_PORT = args.core_port
90
+
91
+ logger.info(f"ConnectorX version: {__version__}")
92
+ # Log configuration
93
+ logger.info(f"Worker configured at {SERVICE_HOST}:{SERVICE_PORT}")
94
+ logger.info(f"Core service configured at {get_core_url(CORE_HOST, CORE_PORT)}")
95
+ else:
96
+ # In spawned processes, use defaults from environment variables
97
+ SERVICE_HOST = DEFAULT_SERVICE_HOST
98
+ SERVICE_PORT = DEFAULT_SERVICE_PORT
99
+ CORE_HOST = DEFAULT_CORE_HOST
100
+ CORE_PORT = DEFAULT_CORE_PORT
101
+
102
+ # Generate the core URI (used by both main and spawned processes)
103
+ FLOWFILE_CORE_URI = get_core_url(CORE_HOST, CORE_PORT)
@@ -1,19 +1,24 @@
1
-
2
- from flowfile_worker.create.funcs import (create_from_path_csv, create_from_path_parquet, create_from_path_excel,
3
- create_from_path_json)
4
1
  from typing import Literal
5
2
 
6
- FileType = Literal['csv', 'parquet', 'json', 'excel']
3
+ from flowfile_worker.create.funcs import (
4
+ create_from_path_csv,
5
+ create_from_path_excel,
6
+ create_from_path_json,
7
+ create_from_path_parquet,
8
+ )
9
+
10
+ FileType = Literal["csv", "parquet", "json", "excel"]
11
+
7
12
 
8
13
  def table_creator_factory_method(file_type: FileType) -> callable:
9
14
  match file_type:
10
- case 'csv':
15
+ case "csv":
11
16
  return create_from_path_csv
12
- case 'parquet':
17
+ case "parquet":
13
18
  return create_from_path_parquet
14
- case 'excel':
19
+ case "excel":
15
20
  return create_from_path_excel
16
- case 'json':
21
+ case "json":
17
22
  return create_from_path_json
18
23
  case _:
19
- raise ValueError(f'Unsupported file type: {file_type}')
24
+ raise ValueError(f"Unsupported file type: {file_type}")