Flowfile 0.4.1__py3-none-any.whl → 0.5.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- build_backends/main.py +25 -22
- build_backends/main_prd.py +10 -19
- flowfile/__init__.py +179 -73
- flowfile/__main__.py +10 -7
- flowfile/api.py +52 -59
- flowfile/web/__init__.py +14 -9
- flowfile/web/static/assets/AdminView-49392a9a.js +713 -0
- flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
- flowfile/web/static/assets/CloudConnectionView-36bcd6df.css +72 -0
- flowfile/web/static/assets/{CloudConnectionManager-d3248f8d.js → CloudConnectionView-f13f202b.js} +11 -11
- flowfile/web/static/assets/{CloudStorageReader-d65bf041.js → CloudStorageReader-0023d4a5.js} +10 -8
- flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
- flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
- flowfile/web/static/assets/{CloudStorageWriter-e83be3ed.js → CloudStorageWriter-8e781e11.js} +10 -8
- flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
- flowfile/web/static/assets/{ColumnSelector-cce661cf.js → ColumnSelector-8ad68ea9.js} +3 -5
- flowfile/web/static/assets/{ContextMenu-c13f91d0.css → ContextMenu-26d4dd27.css} +6 -6
- flowfile/web/static/assets/{ContextMenu-11a4652a.js → ContextMenu-31ee57f0.js} +3 -3
- flowfile/web/static/assets/{ContextMenu-160afb08.js → ContextMenu-69a74055.js} +3 -3
- flowfile/web/static/assets/{ContextMenu-cf18d2cc.js → ContextMenu-8e2051c6.js} +3 -3
- flowfile/web/static/assets/{ContextMenu-4c74eef1.css → ContextMenu-8ec1729e.css} +6 -6
- flowfile/web/static/assets/{ContextMenu-63cfa99b.css → ContextMenu-9b310c60.css} +6 -6
- flowfile/web/static/assets/{CrossJoin-d395d38c.js → CrossJoin-03df6938.js} +12 -10
- flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
- flowfile/web/static/assets/CustomNode-59e99a86.css +32 -0
- flowfile/web/static/assets/{CustomNode-b812dc0b.js → CustomNode-8479239b.js} +36 -24
- flowfile/web/static/assets/{DatabaseConnectionSettings-7000bf2c.js → DatabaseConnectionSettings-869e3efd.js} +5 -4
- flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-e91df89a.css} +13 -13
- flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-36898a00.css} +24 -24
- flowfile/web/static/assets/{DatabaseReader-4f035d0c.js → DatabaseReader-c58b9552.js} +25 -15
- flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
- flowfile/web/static/assets/{DatabaseManager-9662ec5b.js → DatabaseView-d26a9140.js} +11 -11
- flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-217a99f1.css} +19 -19
- flowfile/web/static/assets/{DatabaseWriter-f65dcd54.js → DatabaseWriter-4d05ddc7.js} +17 -10
- flowfile/web/static/assets/{designer-e3c150ec.css → DesignerView-a6d0ee84.css} +629 -538
- flowfile/web/static/assets/{designer-f3656d8c.js → DesignerView-e6f5c0e8.js} +1214 -3209
- flowfile/web/static/assets/{documentation-52b241e7.js → DocumentationView-2e78ef1b.js} +5 -5
- flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-fd46c656.css} +7 -7
- flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
- flowfile/web/static/assets/{ExploreData-94c43dfc.js → ExploreData-7b54caca.js} +18 -9
- flowfile/web/static/assets/{ExternalSource-ac04b3cc.js → ExternalSource-3fa399b2.js} +9 -7
- flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-47ab05a3.css} +17 -17
- flowfile/web/static/assets/Filter-7494ea97.css +48 -0
- flowfile/web/static/assets/Filter-8cbbdbf3.js +287 -0
- flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
- flowfile/web/static/assets/{Formula-71472193.js → Formula-aac42b1e.js} +13 -11
- flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
- flowfile/web/static/assets/{FuzzyMatch-b317f631.js → FuzzyMatch-cd9bbfca.js} +12 -10
- flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-c24dec17.css} +5 -5
- flowfile/web/static/assets/{GraphSolver-754a234f.js → GraphSolver-c7e6780e.js} +13 -11
- flowfile/web/static/assets/{GroupBy-6c6f9802.js → GroupBy-93c5d22b.js} +9 -7
- flowfile/web/static/assets/{GroupBy-b9505323.css → GroupBy-be7ac0bf.css} +10 -10
- flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
- flowfile/web/static/assets/{Join-a1b800be.js → Join-a19b2de2.js} +13 -11
- flowfile/web/static/assets/LoginView-0df4ed0a.js +134 -0
- flowfile/web/static/assets/LoginView-d325d632.css +172 -0
- flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
- flowfile/web/static/assets/{ManualInput-a9640276.js → ManualInput-8d3374b2.js} +170 -116
- flowfile/web/static/assets/{MultiSelect-97213888.js → MultiSelect-ad1b6243.js} +2 -2
- flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-6ffe088a.js → MultiSelect.vue_vue_type_script_setup_true_lang-e278950d.js} +1 -1
- flowfile/web/static/assets/NodeDesigner-40b647c9.js +2610 -0
- flowfile/web/static/assets/NodeDesigner-5f53be3f.css +1429 -0
- flowfile/web/static/assets/{NumericInput-e638088a.js → NumericInput-7100234c.js} +2 -2
- flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-90eb2cba.js → NumericInput.vue_vue_type_script_setup_true_lang-5130219f.js} +5 -2
- flowfile/web/static/assets/{Output-ddc9079f.css → Output-35e97000.css} +6 -6
- flowfile/web/static/assets/{Output-76750610.js → Output-f5efd2aa.js} +60 -38
- flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
- flowfile/web/static/assets/{Pivot-7814803f.js → Pivot-d981d23c.js} +11 -9
- flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
- flowfile/web/static/assets/{PivotValidation-f92137d2.js → PivotValidation-39386e95.js} +3 -3
- flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
- flowfile/web/static/assets/{PivotValidation-76dd431a.js → PivotValidation-63de1f73.js} +3 -3
- flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
- flowfile/web/static/assets/{PolarsCode-889c3008.js → PolarsCode-f9d69217.js} +18 -9
- flowfile/web/static/assets/PopOver-b22f049e.js +939 -0
- flowfile/web/static/assets/PopOver-d96599db.css +33 -0
- flowfile/web/static/assets/{Read-6b17491f.css → Read-36e7bd51.css} +12 -12
- flowfile/web/static/assets/{Read-637b72a7.js → Read-aec2e377.js} +83 -105
- flowfile/web/static/assets/{RecordCount-2b050c41.js → RecordCount-78ed6845.js} +6 -4
- flowfile/web/static/assets/{RecordId-81df7784.js → RecordId-2156e890.js} +8 -6
- flowfile/web/static/assets/{SQLQueryComponent-36cef432.css → SQLQueryComponent-1c2f26b4.css} +5 -5
- flowfile/web/static/assets/{SQLQueryComponent-88dcfe53.js → SQLQueryComponent-48c72f5b.js} +3 -3
- flowfile/web/static/assets/{Sample-258ad2a9.js → Sample-1352ca74.js} +6 -4
- flowfile/web/static/assets/SecretSelector-22b5ff89.js +113 -0
- flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
- flowfile/web/static/assets/{SecretManager-2a2cb7e2.js → SecretsView-17df66ee.js} +35 -36
- flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
- flowfile/web/static/assets/{Select-850215fd.js → Select-0aee4c54.js} +9 -7
- flowfile/web/static/assets/{SettingsSection-55bae608.js → SettingsSection-0784e157.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-0e8d9123.js → SettingsSection-cd341bb6.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-29b4fa6b.js → SettingsSection-f2002a6d.js} +3 -3
- flowfile/web/static/assets/{SingleSelect-bebd408b.js → SingleSelect-460cc0ea.js} +2 -2
- flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-6093741c.js → SingleSelect.vue_vue_type_script_setup_true_lang-30741bb2.js} +1 -1
- flowfile/web/static/assets/{SliderInput-6a05ab61.js → SliderInput-5d926864.js} +7 -4
- flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
- flowfile/web/static/assets/{Sort-10ab48ed.js → Sort-3cdc971b.js} +9 -7
- flowfile/web/static/assets/{Unique-f9fb0809.css → Sort-8a871341.css} +10 -10
- flowfile/web/static/assets/{TextInput-df9d6259.js → TextInput-a2d0bfbd.js} +2 -2
- flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-000e1178.js → TextInput.vue_vue_type_script_setup_true_lang-abad1ca2.js} +5 -2
- flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
- flowfile/web/static/assets/{TextToRows-6c2d93d8.js → TextToRows-918945f7.js} +11 -10
- flowfile/web/static/assets/{ToggleSwitch-0ff7ac52.js → ToggleSwitch-f0ef5196.js} +2 -2
- flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-c6dc3029.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-5605c793.js} +1 -1
- flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-54d2f518.css} +6 -6
- flowfile/web/static/assets/{UnavailableFields-1bab97cb.js → UnavailableFields-bdad6144.js} +4 -4
- flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
- flowfile/web/static/assets/{Union-b563478a.js → Union-e8ab8c86.js} +8 -6
- flowfile/web/static/assets/{Unique-f90db5db.js → Unique-8cd4f976.js} +13 -22
- flowfile/web/static/assets/{Sort-3643d625.css → Unique-9fb2f567.css} +10 -10
- flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-710a2948.css} +7 -7
- flowfile/web/static/assets/{Unpivot-bcb0025f.js → Unpivot-8da14095.js} +10 -8
- flowfile/web/static/assets/{UnpivotValidation-c4e73b04.js → UnpivotValidation-6f7d89ff.js} +3 -3
- flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
- flowfile/web/static/assets/{VueGraphicWalker-bb8535e2.js → VueGraphicWalker-3fb312e1.js} +4 -4
- flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
- flowfile/web/static/assets/{api-4c8e3822.js → api-24483f0d.js} +1 -1
- flowfile/web/static/assets/{api-2d6adc4f.js → api-8b81fa73.js} +1 -1
- flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-3d8dc5fa.css} +40 -40
- flowfile/web/static/assets/{dropDown-1bca8a74.js → dropDown-ac0fda9d.js} +3 -3
- flowfile/web/static/assets/{fullEditor-2985687e.js → fullEditor-5497a84a.js} +11 -10
- flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-a0be62b3.css} +74 -62
- flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
- flowfile/web/static/assets/{genericNodeSettings-0476ba4e.js → genericNodeSettings-99014e1d.js} +5 -5
- flowfile/web/static/assets/index-07dda503.js +38 -0
- flowfile/web/static/assets/index-3ba44389.js +2696 -0
- flowfile/web/static/assets/{index-50508d4d.css → index-e6289dd0.css} +1945 -569
- flowfile/web/static/assets/{index-246f201c.js → index-fb6493ae.js} +41626 -40869
- flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
- flowfile/web/static/assets/nodeInput-0eb13f1a.js +2 -0
- flowfile/web/static/assets/{outputCsv-d686eeaf.js → outputCsv-8f8ba42d.js} +3 -3
- flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
- flowfile/web/static/assets/{outputExcel-8809ea2f.js → outputExcel-393f4fef.js} +3 -3
- flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
- flowfile/web/static/assets/{outputParquet-53ba645a.js → outputParquet-07c81f65.js} +4 -4
- flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
- flowfile/web/static/assets/{readCsv-053bf97b.js → readCsv-07f6d9ad.js} +21 -20
- flowfile/web/static/assets/{readCsv-bca3ed53.css → readCsv-3bfac4c3.css} +15 -15
- flowfile/web/static/assets/{readExcel-e1b381ea.css → readExcel-3db6b763.css} +13 -13
- flowfile/web/static/assets/{readExcel-ad531eab.js → readExcel-ed69bc8f.js} +10 -12
- flowfile/web/static/assets/{readParquet-cee068e2.css → readParquet-c5244ad5.css} +4 -4
- flowfile/web/static/assets/{readParquet-58e899a1.js → readParquet-e3ed4528.js} +4 -7
- flowfile/web/static/assets/secrets.api-002e7d7e.js +65 -0
- flowfile/web/static/assets/{selectDynamic-b38de2ba.js → selectDynamic-80b92899.js} +5 -5
- flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
- flowfile/web/static/assets/{vue-codemirror.esm-db9b8936.js → vue-codemirror.esm-0965f39f.js} +31 -637
- flowfile/web/static/assets/{vue-content-loader.es-b5f3ac30.js → vue-content-loader.es-c506ad97.js} +1 -1
- flowfile/web/static/index.html +2 -2
- {flowfile-0.4.1.dist-info → flowfile-0.5.3.dist-info}/METADATA +4 -4
- flowfile-0.5.3.dist-info/RECORD +402 -0
- {flowfile-0.4.1.dist-info → flowfile-0.5.3.dist-info}/WHEEL +1 -1
- {flowfile-0.4.1.dist-info → flowfile-0.5.3.dist-info}/entry_points.txt +1 -0
- flowfile_core/__init__.py +13 -3
- flowfile_core/auth/jwt.py +51 -16
- flowfile_core/auth/models.py +32 -7
- flowfile_core/auth/password.py +89 -0
- flowfile_core/auth/secrets.py +8 -6
- flowfile_core/configs/__init__.py +9 -7
- flowfile_core/configs/flow_logger.py +15 -14
- flowfile_core/configs/node_store/__init__.py +72 -4
- flowfile_core/configs/node_store/nodes.py +155 -172
- flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
- flowfile_core/configs/settings.py +28 -15
- flowfile_core/database/connection.py +7 -6
- flowfile_core/database/init_db.py +96 -2
- flowfile_core/database/models.py +3 -1
- flowfile_core/fileExplorer/__init__.py +17 -0
- flowfile_core/fileExplorer/funcs.py +123 -57
- flowfile_core/fileExplorer/utils.py +10 -11
- flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
- flowfile_core/flowfile/analytics/analytics_processor.py +27 -24
- flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
- flowfile_core/flowfile/analytics/utils.py +1 -1
- flowfile_core/flowfile/code_generator/code_generator.py +391 -279
- flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
- flowfile_core/flowfile/connection_manager/models.py +1 -1
- flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
- flowfile_core/flowfile/database_connection_manager/models.py +1 -1
- flowfile_core/flowfile/extensions.py +17 -12
- flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
- flowfile_core/flowfile/flow_data_engine/create/funcs.py +152 -103
- flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +526 -477
- flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
- flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
- flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
- flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
- flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
- flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +43 -32
- flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
- flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +15 -11
- flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
- flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
- flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
- flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +360 -191
- flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
- flowfile_core/flowfile/flow_data_engine/utils.py +101 -67
- flowfile_core/flowfile/flow_graph.py +1011 -561
- flowfile_core/flowfile/flow_graph_utils.py +31 -49
- flowfile_core/flowfile/flow_node/flow_node.py +332 -232
- flowfile_core/flowfile/flow_node/models.py +54 -41
- flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
- flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
- flowfile_core/flowfile/handler.py +82 -32
- flowfile_core/flowfile/manage/compatibility_enhancements.py +493 -47
- flowfile_core/flowfile/manage/io_flowfile.py +391 -0
- flowfile_core/flowfile/node_designer/__init__.py +15 -13
- flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
- flowfile_core/flowfile/node_designer/custom_node.py +162 -36
- flowfile_core/flowfile/node_designer/ui_components.py +136 -35
- flowfile_core/flowfile/schema_callbacks.py +77 -54
- flowfile_core/flowfile/setting_generator/__init__.py +0 -1
- flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
- flowfile_core/flowfile/setting_generator/settings.py +72 -55
- flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
- flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
- flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
- flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
- flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
- flowfile_core/flowfile/util/calculate_layout.py +9 -13
- flowfile_core/flowfile/util/execution_orderer.py +25 -17
- flowfile_core/flowfile/util/node_skipper.py +4 -4
- flowfile_core/flowfile/utils.py +19 -21
- flowfile_core/main.py +26 -19
- flowfile_core/routes/auth.py +284 -11
- flowfile_core/routes/cloud_connections.py +25 -25
- flowfile_core/routes/logs.py +21 -29
- flowfile_core/routes/public.py +3 -3
- flowfile_core/routes/routes.py +77 -43
- flowfile_core/routes/secrets.py +25 -27
- flowfile_core/routes/user_defined_components.py +483 -4
- flowfile_core/run_lock.py +0 -1
- flowfile_core/schemas/__init__.py +4 -6
- flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
- flowfile_core/schemas/cloud_storage_schemas.py +59 -55
- flowfile_core/schemas/input_schema.py +398 -154
- flowfile_core/schemas/output_model.py +50 -35
- flowfile_core/schemas/schemas.py +207 -67
- flowfile_core/schemas/transform_schema.py +1360 -435
- flowfile_core/schemas/yaml_types.py +117 -0
- flowfile_core/secret_manager/secret_manager.py +17 -13
- flowfile_core/{flowfile/node_designer/data_types.py → types.py} +33 -3
- flowfile_core/utils/arrow_reader.py +7 -6
- flowfile_core/utils/excel_file_manager.py +3 -3
- flowfile_core/utils/fileManager.py +7 -7
- flowfile_core/utils/fl_executor.py +8 -10
- flowfile_core/utils/utils.py +4 -4
- flowfile_core/utils/validate_setup.py +5 -4
- flowfile_frame/__init__.py +107 -50
- flowfile_frame/adapters.py +2 -9
- flowfile_frame/adding_expr.py +73 -32
- flowfile_frame/cloud_storage/frame_helpers.py +27 -23
- flowfile_frame/cloud_storage/secret_manager.py +12 -26
- flowfile_frame/config.py +2 -5
- flowfile_frame/expr.py +311 -218
- flowfile_frame/expr.pyi +160 -159
- flowfile_frame/expr_name.py +23 -23
- flowfile_frame/flow_frame.py +581 -489
- flowfile_frame/flow_frame.pyi +123 -104
- flowfile_frame/flow_frame_methods.py +236 -252
- flowfile_frame/group_frame.py +50 -20
- flowfile_frame/join.py +2 -2
- flowfile_frame/lazy.py +129 -87
- flowfile_frame/lazy_methods.py +83 -30
- flowfile_frame/list_name_space.py +55 -50
- flowfile_frame/selectors.py +148 -68
- flowfile_frame/series.py +9 -7
- flowfile_frame/utils.py +19 -21
- flowfile_worker/__init__.py +12 -4
- flowfile_worker/configs.py +11 -19
- flowfile_worker/create/__init__.py +14 -27
- flowfile_worker/create/funcs.py +143 -94
- flowfile_worker/create/models.py +139 -68
- flowfile_worker/create/pl_types.py +14 -15
- flowfile_worker/create/read_excel_tables.py +34 -41
- flowfile_worker/create/utils.py +22 -19
- flowfile_worker/external_sources/s3_source/main.py +18 -51
- flowfile_worker/external_sources/s3_source/models.py +34 -27
- flowfile_worker/external_sources/sql_source/main.py +8 -5
- flowfile_worker/external_sources/sql_source/models.py +13 -9
- flowfile_worker/flow_logger.py +10 -8
- flowfile_worker/funcs.py +214 -155
- flowfile_worker/main.py +11 -17
- flowfile_worker/models.py +35 -28
- flowfile_worker/process_manager.py +2 -3
- flowfile_worker/routes.py +121 -93
- flowfile_worker/secrets.py +9 -6
- flowfile_worker/spawner.py +80 -49
- flowfile_worker/utils.py +3 -2
- shared/__init__.py +2 -7
- shared/storage_config.py +25 -13
- test_utils/postgres/commands.py +3 -2
- test_utils/postgres/fixtures.py +9 -9
- test_utils/s3/commands.py +1 -1
- test_utils/s3/data_generator.py +3 -4
- test_utils/s3/demo_data_generator.py +4 -7
- test_utils/s3/fixtures.py +7 -5
- tools/migrate/README.md +56 -0
- tools/migrate/__init__.py +12 -0
- tools/migrate/__main__.py +118 -0
- tools/migrate/legacy_schemas.py +682 -0
- tools/migrate/migrate.py +610 -0
- tools/migrate/tests/__init__.py +0 -0
- tools/migrate/tests/conftest.py +21 -0
- tools/migrate/tests/test_migrate.py +622 -0
- tools/migrate/tests/test_migration_e2e.py +1009 -0
- tools/migrate/tests/test_node_migrations.py +843 -0
- flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
- flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
- flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
- flowfile/web/static/assets/Filter-812dcbca.js +0 -164
- flowfile/web/static/assets/Filter-f62091b3.css +0 -20
- flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
- flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
- flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
- flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
- flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
- flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
- flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
- flowfile/web/static/assets/secretApi-538058f3.js +0 -46
- flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
- flowfile-0.4.1.dist-info/RECORD +0 -376
- flowfile_core/flowfile/manage/open_flowfile.py +0 -143
- {flowfile-0.4.1.dist-info → flowfile-0.5.3.dist-info}/licenses/LICENSE +0 -0
- /flowfile_core/flowfile/manage/manage_flowfile.py → /tools/__init__.py +0 -0
|
@@ -1,15 +1,15 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Literal
|
|
1
4
|
|
|
2
5
|
import pyarrow as pa
|
|
3
|
-
from typing import List, Union, Callable, Optional, Literal
|
|
4
|
-
from dataclasses import dataclass
|
|
5
6
|
|
|
6
7
|
# Forward declaration for type hints to avoid circular imports
|
|
7
8
|
if False:
|
|
8
9
|
from flowfile_core.flowfile.flow_node.flow_node import FlowNode
|
|
9
10
|
|
|
10
|
-
from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
|
|
11
11
|
from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
|
|
12
|
-
from flowfile_core.
|
|
12
|
+
from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
@dataclass
|
|
@@ -27,30 +27,35 @@ class NodeStepPromise:
|
|
|
27
27
|
right_input: The ID of the node connected to the right input port.
|
|
28
28
|
depends_on: A list of node IDs that this node depends on for main inputs.
|
|
29
29
|
"""
|
|
30
|
-
|
|
30
|
+
|
|
31
|
+
node_id: str | int
|
|
31
32
|
name: str
|
|
32
33
|
is_start: bool
|
|
33
|
-
leads_to_id:
|
|
34
|
-
left_input:
|
|
35
|
-
right_input:
|
|
36
|
-
depends_on:
|
|
34
|
+
leads_to_id: list[str | int] | None = None
|
|
35
|
+
left_input: str | int | None = None
|
|
36
|
+
right_input: str | int | None = None
|
|
37
|
+
depends_on: list[str | int] | None = None
|
|
37
38
|
|
|
38
39
|
|
|
39
40
|
class NodeStepStats:
|
|
40
41
|
"""
|
|
41
42
|
Tracks the execution status and statistics of a `FlowNode`.
|
|
42
43
|
"""
|
|
44
|
+
|
|
43
45
|
error: str = None
|
|
44
46
|
_has_run_with_current_setup: bool = False
|
|
45
47
|
has_completed_last_run: bool = False
|
|
46
48
|
active: bool = True
|
|
47
49
|
is_canceled: bool = False
|
|
48
50
|
|
|
49
|
-
def __init__(
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
error: str = None,
|
|
54
|
+
has_run_with_current_setup: bool = False,
|
|
55
|
+
has_completed_last_run: bool = False,
|
|
56
|
+
active: bool = True,
|
|
57
|
+
is_canceled: bool = False,
|
|
58
|
+
):
|
|
54
59
|
"""
|
|
55
60
|
Initializes the node's statistics.
|
|
56
61
|
|
|
@@ -71,9 +76,11 @@ class NodeStepStats:
|
|
|
71
76
|
Provides a string representation of the node's stats.
|
|
72
77
|
:return: A string detailing the current stats.
|
|
73
78
|
"""
|
|
74
|
-
return (
|
|
75
|
-
|
|
76
|
-
|
|
79
|
+
return (
|
|
80
|
+
f"NodeStepStats(error={self.error}, has_run_with_current_setup={self.has_run_with_current_setup}, "
|
|
81
|
+
f"has_completed_last_run={self.has_completed_last_run}, "
|
|
82
|
+
f"active={self.active}, is_canceled={self.is_canceled})"
|
|
83
|
+
)
|
|
77
84
|
|
|
78
85
|
@property
|
|
79
86
|
def has_run_with_current_setup(self) -> bool:
|
|
@@ -109,6 +116,7 @@ class NodeStepSettings:
|
|
|
109
116
|
setup_errors: If True, indicates a non-blocking error occurred during setup.
|
|
110
117
|
breaking_setup_errors: If True, indicates an error occurred that prevents execution.
|
|
111
118
|
"""
|
|
119
|
+
|
|
112
120
|
cache_results: bool = False
|
|
113
121
|
renew_schema: bool = True
|
|
114
122
|
streamable: bool = True
|
|
@@ -125,20 +133,22 @@ class NodeStepInputs:
|
|
|
125
133
|
right_input: The `FlowNode` connected to the right input port.
|
|
126
134
|
main_inputs: A list of `FlowNode` objects connected to the main input port(s).
|
|
127
135
|
"""
|
|
136
|
+
|
|
128
137
|
left_input: "FlowNode" = None
|
|
129
138
|
right_input: "FlowNode" = None
|
|
130
|
-
main_inputs:
|
|
139
|
+
main_inputs: list["FlowNode"] = None
|
|
131
140
|
|
|
132
141
|
@property
|
|
133
|
-
def input_ids(self) ->
|
|
142
|
+
def input_ids(self) -> list[int]:
|
|
134
143
|
"""
|
|
135
144
|
Gets the IDs of all connected input nodes.
|
|
136
145
|
:return: A list of integer node IDs.
|
|
137
146
|
"""
|
|
138
147
|
if self.main_inputs is not None:
|
|
139
148
|
return [node_input.node_information.id for node_input in self.get_all_inputs()]
|
|
149
|
+
return []
|
|
140
150
|
|
|
141
|
-
def get_all_inputs(self) ->
|
|
151
|
+
def get_all_inputs(self) -> list["FlowNode"]:
|
|
142
152
|
"""
|
|
143
153
|
Retrieves a single list containing all input nodes (main, left, and right).
|
|
144
154
|
:return: A list of all connected `FlowNode` objects.
|
|
@@ -156,8 +166,9 @@ class NodeStepInputs:
|
|
|
156
166
|
main_inputs_repr = f"Main Inputs: {self.main_inputs}" if self.main_inputs else "Main Inputs: None"
|
|
157
167
|
return f"{self.__class__.__name__}({left_repr}, {right_repr}, {main_inputs_repr})"
|
|
158
168
|
|
|
159
|
-
def validate_if_input_connection_exists(
|
|
160
|
-
|
|
169
|
+
def validate_if_input_connection_exists(
|
|
170
|
+
self, node_input_id: int, connection_name: Literal["main", "left", "right"]
|
|
171
|
+
) -> bool:
|
|
161
172
|
"""
|
|
162
173
|
Checks if a connection from a specific node ID exists on a given port.
|
|
163
174
|
|
|
@@ -165,11 +176,11 @@ class NodeStepInputs:
|
|
|
165
176
|
:param connection_name: The name of the input port ('main', 'left', 'right').
|
|
166
177
|
:return: True if the connection exists, False otherwise.
|
|
167
178
|
"""
|
|
168
|
-
if connection_name ==
|
|
179
|
+
if connection_name == "main" and self.main_inputs:
|
|
169
180
|
return any(node_input.node_information.id == node_input_id for node_input in self.main_inputs)
|
|
170
|
-
if connection_name ==
|
|
181
|
+
if connection_name == "left" and self.left_input:
|
|
171
182
|
return self.left_input.node_information.id == node_input_id
|
|
172
|
-
if connection_name ==
|
|
183
|
+
if connection_name == "right":
|
|
173
184
|
return self.right_input.node_information.id == node_input_id
|
|
174
185
|
|
|
175
186
|
|
|
@@ -184,25 +195,27 @@ class NodeSchemaInformation:
|
|
|
184
195
|
drop_columns: A list of column names that will be dropped by the node.
|
|
185
196
|
output_columns: A list of `FlowfileColumn` objects that will be added by the node.
|
|
186
197
|
"""
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
198
|
+
|
|
199
|
+
result_schema: list[FlowfileColumn] | None = None
|
|
200
|
+
predicted_schema: list[FlowfileColumn] | None = None
|
|
201
|
+
input_columns: list[str] = []
|
|
202
|
+
drop_columns: list[str] = []
|
|
203
|
+
output_columns: list[FlowfileColumn] = []
|
|
192
204
|
|
|
193
205
|
|
|
194
206
|
class NodeResults:
|
|
195
207
|
"""
|
|
196
208
|
Stores the outputs of a `FlowNode`'s execution, including data, errors, and metadata.
|
|
197
209
|
"""
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
210
|
+
|
|
211
|
+
_resulting_data: FlowDataEngine | None = None
|
|
212
|
+
example_data: FlowDataEngine | None = None
|
|
213
|
+
example_data_path: str | None = None
|
|
214
|
+
example_data_generator: Callable[[], pa.Table] | None = None
|
|
202
215
|
run_time: int = -1
|
|
203
|
-
errors:
|
|
204
|
-
warnings:
|
|
205
|
-
analysis_data_generator:
|
|
216
|
+
errors: str | None = None
|
|
217
|
+
warnings: str | None = None
|
|
218
|
+
analysis_data_generator: Callable[[], pa.Table] | None = None
|
|
206
219
|
|
|
207
220
|
def __init__(self):
|
|
208
221
|
self._resulting_data = None
|
|
@@ -213,7 +226,7 @@ class NodeResults:
|
|
|
213
226
|
self.example_data_generator = None
|
|
214
227
|
self.analysis_data_generator = None
|
|
215
228
|
|
|
216
|
-
def get_example_data(self) ->
|
|
229
|
+
def get_example_data(self) -> pa.Table | None:
|
|
217
230
|
"""
|
|
218
231
|
Executes the generator to fetch a sample of the resulting data.
|
|
219
232
|
:return: A PyArrow Table containing a sample of the data, or None.
|
|
@@ -222,7 +235,7 @@ class NodeResults:
|
|
|
222
235
|
return self.example_data_generator()
|
|
223
236
|
|
|
224
237
|
@property
|
|
225
|
-
def resulting_data(self) ->
|
|
238
|
+
def resulting_data(self) -> FlowDataEngine | None:
|
|
226
239
|
"""
|
|
227
240
|
Gets the full resulting data from the node's execution.
|
|
228
241
|
:return: A `FlowDataEngine` instance containing the result, or None.
|
|
@@ -230,7 +243,7 @@ class NodeResults:
|
|
|
230
243
|
return self._resulting_data
|
|
231
244
|
|
|
232
245
|
@resulting_data.setter
|
|
233
|
-
def resulting_data(self, d:
|
|
246
|
+
def resulting_data(self, d: FlowDataEngine | None):
|
|
234
247
|
"""
|
|
235
248
|
Sets the resulting data.
|
|
236
249
|
:param d: The `FlowDataEngine` instance to store.
|
|
@@ -240,4 +253,4 @@ class NodeResults:
|
|
|
240
253
|
def reset(self):
|
|
241
254
|
"""Resets all result attributes to their default, empty state."""
|
|
242
255
|
self._resulting_data = None
|
|
243
|
-
self.run_time = -1
|
|
256
|
+
self.run_time = -1
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
from typing import Callable, Any, Optional, Generic, TypeVar
|
|
2
|
-
from concurrent.futures import ThreadPoolExecutor, Future
|
|
3
1
|
import threading
|
|
2
|
+
from collections.abc import Callable
|
|
3
|
+
from concurrent.futures import Future, ThreadPoolExecutor
|
|
4
|
+
from typing import Any, Generic, TypeVar
|
|
5
|
+
|
|
4
6
|
from flowfile_core.configs import logger
|
|
5
7
|
|
|
6
|
-
T = TypeVar(
|
|
8
|
+
T = TypeVar("T")
|
|
7
9
|
|
|
8
10
|
|
|
9
11
|
class SingleExecutionFuture(Generic[T]):
|
|
@@ -14,20 +16,16 @@ class SingleExecutionFuture(Generic[T]):
|
|
|
14
16
|
"""
|
|
15
17
|
|
|
16
18
|
func: Callable[[], T]
|
|
17
|
-
on_error:
|
|
19
|
+
on_error: Callable[[Exception], Any] | None
|
|
18
20
|
_lock: threading.RLock
|
|
19
|
-
_executor:
|
|
20
|
-
_future:
|
|
21
|
-
_result_value:
|
|
22
|
-
_exception:
|
|
21
|
+
_executor: ThreadPoolExecutor | None
|
|
22
|
+
_future: Future[T] | None
|
|
23
|
+
_result_value: T | None
|
|
24
|
+
_exception: Exception | None
|
|
23
25
|
_has_completed: bool
|
|
24
26
|
_has_started: bool
|
|
25
27
|
|
|
26
|
-
def __init__(
|
|
27
|
-
self,
|
|
28
|
-
func: Callable[[], T],
|
|
29
|
-
on_error: Optional[Callable[[Exception], Any]] = None
|
|
30
|
-
) -> None:
|
|
28
|
+
def __init__(self, func: Callable[[], T], on_error: Callable[[Exception], Any] | None = None) -> None:
|
|
31
29
|
"""Initialize with function and optional error handler."""
|
|
32
30
|
self.func = func
|
|
33
31
|
self.on_error = on_error
|
|
@@ -81,7 +79,7 @@ class SingleExecutionFuture(Generic[T]):
|
|
|
81
79
|
if self._executor and not self._executor._shutdown:
|
|
82
80
|
self._executor.shutdown(wait=False)
|
|
83
81
|
|
|
84
|
-
def __call__(self) ->
|
|
82
|
+
def __call__(self) -> T | None:
|
|
85
83
|
"""Execute function if not running and return its result."""
|
|
86
84
|
with self._lock:
|
|
87
85
|
# If already completed, return cached result or raise cached exception
|
|
@@ -137,10 +135,7 @@ class SingleExecutionFuture(Generic[T]):
|
|
|
137
135
|
"""Check if the function is currently executing."""
|
|
138
136
|
with self._lock:
|
|
139
137
|
return bool(
|
|
140
|
-
self._has_started and
|
|
141
|
-
not self._has_completed and
|
|
142
|
-
self._future is not None and
|
|
143
|
-
not self._future.done()
|
|
138
|
+
self._has_started and not self._has_completed and self._future is not None and not self._future.done()
|
|
144
139
|
)
|
|
145
140
|
|
|
146
141
|
def is_completed(self) -> bool:
|
|
@@ -148,7 +143,7 @@ class SingleExecutionFuture(Generic[T]):
|
|
|
148
143
|
with self._lock:
|
|
149
144
|
return self._has_completed
|
|
150
145
|
|
|
151
|
-
def get_result(self) ->
|
|
146
|
+
def get_result(self) -> T | None:
|
|
152
147
|
"""Get the cached result without triggering execution."""
|
|
153
148
|
with self._lock:
|
|
154
149
|
if self._exception:
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
from pydantic import BaseModel
|
|
2
|
-
|
|
3
1
|
from flowfile_core.flowfile.flow_node.flow_node import FlowNode
|
|
4
|
-
|
|
5
2
|
from flowfile_core.flowfile.graph_tree.models import BranchInfo, InputInfo
|
|
6
3
|
|
|
7
4
|
|
|
@@ -13,35 +10,39 @@ def calculate_depth(node_id: int, node_info: dict[int, BranchInfo], visited: set
|
|
|
13
10
|
if node_id in visited:
|
|
14
11
|
return node_info[node_id].depth
|
|
15
12
|
visited.add(node_id)
|
|
16
|
-
|
|
13
|
+
|
|
17
14
|
max_input_depth = -1
|
|
18
15
|
inputs = node_info[node_id].inputs
|
|
19
|
-
|
|
16
|
+
|
|
20
17
|
for main_id in inputs.main:
|
|
21
18
|
max_input_depth = max(max_input_depth, calculate_depth(main_id, node_info, visited))
|
|
22
19
|
if inputs.left:
|
|
23
20
|
max_input_depth = max(max_input_depth, calculate_depth(inputs.left, node_info, visited))
|
|
24
21
|
if inputs.right:
|
|
25
22
|
max_input_depth = max(max_input_depth, calculate_depth(inputs.right, node_info, visited))
|
|
26
|
-
|
|
23
|
+
|
|
27
24
|
node_info[node_id].depth = max_input_depth + 1
|
|
28
25
|
return node_info[node_id].depth
|
|
29
26
|
|
|
30
27
|
|
|
31
28
|
# Trace paths from each root
|
|
32
|
-
def trace_path(
|
|
33
|
-
|
|
29
|
+
def trace_path(
|
|
30
|
+
node_id: int,
|
|
31
|
+
node_info: dict[int, BranchInfo],
|
|
32
|
+
merge_points: dict[int, list[int]],
|
|
33
|
+
current_path: list[int] | None = None,
|
|
34
|
+
):
|
|
34
35
|
"""Define the trace of each node path"""
|
|
35
36
|
if current_path is None:
|
|
36
37
|
current_path = []
|
|
37
|
-
|
|
38
|
+
|
|
38
39
|
current_path = current_path + [node_id]
|
|
39
40
|
outputs = node_info[node_id].outputs
|
|
40
|
-
|
|
41
|
+
|
|
41
42
|
if not outputs:
|
|
42
43
|
# End of path
|
|
43
44
|
return [current_path]
|
|
44
|
-
|
|
45
|
+
|
|
45
46
|
# If this node has multiple outputs or connects to a merge point, branch
|
|
46
47
|
all_paths = []
|
|
47
48
|
for output_id in outputs:
|
|
@@ -64,7 +65,7 @@ def build_node_info(nodes: list[FlowNode]) -> dict[int, BranchInfo]:
|
|
|
64
65
|
# Get node label
|
|
65
66
|
operation = node.node_type.replace("_", " ").title() if node.node_type else "Unknown"
|
|
66
67
|
label = f"{operation} (id={node_id})"
|
|
67
|
-
if hasattr(node,
|
|
68
|
+
if hasattr(node, "setting_input") and hasattr(node.setting_input, "description"):
|
|
68
69
|
if node.setting_input.description:
|
|
69
70
|
desc = node.setting_input.description
|
|
70
71
|
if len(desc) > 20: # Truncate long descriptions
|
|
@@ -75,18 +76,14 @@ def build_node_info(nodes: list[FlowNode]) -> dict[int, BranchInfo]:
|
|
|
75
76
|
inputs = InputInfo(
|
|
76
77
|
main=[n.node_id for n in (node.node_inputs.main_inputs or [])],
|
|
77
78
|
left=node.node_inputs.left_input.node_id if node.node_inputs.left_input else None,
|
|
78
|
-
right=node.node_inputs.right_input.node_id if node.node_inputs.right_input else None
|
|
79
|
+
right=node.node_inputs.right_input.node_id if node.node_inputs.right_input else None,
|
|
79
80
|
)
|
|
80
81
|
outputs = [n.node_id for n in node.leads_to_nodes]
|
|
81
82
|
|
|
82
83
|
node_info[node_id] = BranchInfo(
|
|
83
|
-
label=label,
|
|
84
|
-
short_label=f"{operation} ({node_id})",
|
|
85
|
-
inputs=inputs,
|
|
86
|
-
outputs=outputs,
|
|
87
|
-
depth=0
|
|
84
|
+
label=label, short_label=f"{operation} ({node_id})", inputs=inputs, outputs=outputs, depth=0
|
|
88
85
|
)
|
|
89
|
-
|
|
86
|
+
|
|
90
87
|
return node_info
|
|
91
88
|
|
|
92
89
|
|
|
@@ -112,19 +109,20 @@ def define_node_connections(node_info: dict[int, BranchInfo]) -> dict[int, list[
|
|
|
112
109
|
if output_id not in merge_points:
|
|
113
110
|
merge_points[output_id] = []
|
|
114
111
|
merge_points[output_id].append(node_id)
|
|
115
|
-
|
|
112
|
+
|
|
116
113
|
return merge_points
|
|
117
114
|
|
|
118
115
|
|
|
119
|
-
def build_flow_paths(node_info: dict[int, BranchInfo], flow_starts: list[FlowNode],
|
|
120
|
-
merge_points: dict[int, list[int]]):
|
|
116
|
+
def build_flow_paths(node_info: dict[int, BranchInfo], flow_starts: list[FlowNode], merge_points: dict[int, list[int]]):
|
|
121
117
|
"""Build the flow paths to be drawn"""
|
|
122
118
|
|
|
123
|
-
|
|
124
119
|
# Find all root nodes (no inputs)
|
|
125
|
-
root_nodes = [
|
|
126
|
-
|
|
127
|
-
|
|
120
|
+
root_nodes = [
|
|
121
|
+
nid
|
|
122
|
+
for nid, info in node_info.items()
|
|
123
|
+
if not info.inputs.main and not info.inputs.left and not info.inputs.right
|
|
124
|
+
]
|
|
125
|
+
|
|
128
126
|
if not root_nodes and flow_starts:
|
|
129
127
|
root_nodes = [n.node_id for n in flow_starts]
|
|
130
128
|
paths = [] # List of paths through the graph
|
|
@@ -136,7 +134,7 @@ def build_flow_paths(node_info: dict[int, BranchInfo], flow_starts: list[FlowNod
|
|
|
136
134
|
return paths
|
|
137
135
|
|
|
138
136
|
|
|
139
|
-
def group_paths(paths:list, merge_points:dict):
|
|
137
|
+
def group_paths(paths: list, merge_points: dict):
|
|
140
138
|
"""Groups each node path."""
|
|
141
139
|
paths_by_merge = {}
|
|
142
140
|
standalone_paths = []
|
|
@@ -152,12 +150,14 @@ def group_paths(paths:list, merge_points:dict):
|
|
|
152
150
|
return paths_by_merge, standalone_paths
|
|
153
151
|
|
|
154
152
|
|
|
155
|
-
def draw_merged_paths(
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
153
|
+
def draw_merged_paths(
|
|
154
|
+
node_info: dict[int, BranchInfo],
|
|
155
|
+
merge_points: dict[int, list[int]],
|
|
156
|
+
paths_by_merge: dict[int, list[list[int]]],
|
|
157
|
+
merge_drawn: set,
|
|
158
|
+
drawn_nodes: set,
|
|
159
|
+
lines: list[str],
|
|
160
|
+
):
|
|
161
161
|
"""Draws paths for each node that merges."""
|
|
162
162
|
for merge_id, merge_paths in paths_by_merge.items():
|
|
163
163
|
if merge_id in merge_drawn:
|
|
@@ -171,8 +171,7 @@ def draw_merged_paths(node_info: dict[int, BranchInfo],
|
|
|
171
171
|
source_path = None
|
|
172
172
|
for path in merge_paths:
|
|
173
173
|
if source_id in path:
|
|
174
|
-
|
|
175
|
-
source_path = path[:path.index(source_id) + 1]
|
|
174
|
+
source_path = path[: path.index(source_id) + 1]
|
|
176
175
|
break
|
|
177
176
|
|
|
178
177
|
if source_path:
|
|
@@ -215,14 +214,15 @@ def draw_merged_paths(node_info: dict[int, BranchInfo],
|
|
|
215
214
|
return paths_by_merge
|
|
216
215
|
|
|
217
216
|
|
|
218
|
-
def draw_standalone_paths(
|
|
219
|
-
|
|
220
|
-
|
|
217
|
+
def draw_standalone_paths(
|
|
218
|
+
drawn_nodes: set[int], standalone_paths: list[list[int]], lines: list[str], node_info: dict[int, BranchInfo]
|
|
219
|
+
):
|
|
220
|
+
"""Draws paths that do not merge."""
|
|
221
221
|
# Draw standalone paths
|
|
222
222
|
for path in standalone_paths:
|
|
223
223
|
if all(nid in drawn_nodes for nid in path):
|
|
224
224
|
continue
|
|
225
|
-
|
|
225
|
+
|
|
226
226
|
line_parts = []
|
|
227
227
|
for i, node_id in enumerate(path):
|
|
228
228
|
if node_id not in drawn_nodes:
|
|
@@ -231,12 +231,12 @@ def draw_standalone_paths(drawn_nodes: set[int], standalone_paths: list[list[int
|
|
|
231
231
|
else:
|
|
232
232
|
line_parts.append(f" ──> {node_info[node_id].short_label}")
|
|
233
233
|
drawn_nodes.add(node_id)
|
|
234
|
-
|
|
234
|
+
|
|
235
235
|
if line_parts:
|
|
236
236
|
lines.append("".join(line_parts))
|
|
237
237
|
|
|
238
238
|
|
|
239
|
-
def add_un_drawn_nodes(drawn_nodes: set[int], node_info: dict[int, BranchInfo],
|
|
239
|
+
def add_un_drawn_nodes(drawn_nodes: set[int], node_info: dict[int, BranchInfo], lines: list[str]):
|
|
240
240
|
"""Adds isolated nodes if exists."""
|
|
241
241
|
# Add any remaining undrawn nodes
|
|
242
242
|
|
|
@@ -1,82 +1,129 @@
|
|
|
1
|
-
|
|
2
|
-
from dataclasses import dataclass
|
|
3
|
-
from typing import Dict, List
|
|
4
1
|
import os
|
|
5
|
-
from
|
|
2
|
+
from dataclasses import dataclass
|
|
6
3
|
from datetime import datetime
|
|
4
|
+
from pathlib import Path
|
|
7
5
|
|
|
8
|
-
from flowfile_core.flowfile.manage.open_flowfile import open_flow
|
|
9
6
|
from flowfile_core.flowfile.flow_graph import FlowGraph
|
|
10
|
-
from flowfile_core.
|
|
7
|
+
from flowfile_core.flowfile.manage.io_flowfile import open_flow
|
|
11
8
|
from flowfile_core.flowfile.utils import create_unique_id
|
|
9
|
+
from flowfile_core.schemas.schemas import FlowSettings
|
|
12
10
|
from shared.storage_config import storage
|
|
13
11
|
|
|
14
12
|
|
|
15
13
|
def get_flow_save_location(flow_name: str) -> Path:
|
|
16
14
|
"""Gets the initial save location for flow files"""
|
|
17
|
-
if ".
|
|
18
|
-
flow_name += ".
|
|
15
|
+
if ".yaml" not in flow_name and ".yml" not in flow_name:
|
|
16
|
+
flow_name += ".yaml"
|
|
19
17
|
return storage.temp_directory_for_flows / flow_name
|
|
20
18
|
|
|
21
19
|
|
|
22
20
|
def create_flow_name() -> str:
|
|
23
21
|
"""Creates a unique flow name"""
|
|
24
|
-
return datetime.now().strftime("%Y%m%d_%H_%M_%S")+"_flow.
|
|
22
|
+
return datetime.now().strftime("%Y%m%d_%H_%M_%S") + "_flow.yaml"
|
|
25
23
|
|
|
26
24
|
|
|
27
25
|
@dataclass
|
|
28
26
|
class FlowfileHandler:
|
|
29
|
-
_flows:
|
|
27
|
+
_flows: dict[int, FlowGraph]
|
|
28
|
+
_user_sessions: dict[int, set[int]] # Maps user_id -> set of flow_ids
|
|
30
29
|
|
|
31
30
|
def __init__(self):
|
|
32
31
|
self._flows = {}
|
|
32
|
+
self._user_sessions = {}
|
|
33
33
|
|
|
34
34
|
@property
|
|
35
|
-
def flowfile_flows(self) ->
|
|
35
|
+
def flowfile_flows(self) -> list[FlowGraph]:
|
|
36
36
|
return list(self._flows.values())
|
|
37
37
|
|
|
38
|
+
def _register_user_session(self, user_id: int | None, flow_id: int):
|
|
39
|
+
"""Register a flow as belonging to a user's session."""
|
|
40
|
+
if user_id is not None:
|
|
41
|
+
if user_id not in self._user_sessions:
|
|
42
|
+
self._user_sessions[user_id] = set()
|
|
43
|
+
self._user_sessions[user_id].add(flow_id)
|
|
44
|
+
|
|
45
|
+
def _unregister_user_session(self, user_id: int | None, flow_id: int):
|
|
46
|
+
"""Remove a flow from a user's session."""
|
|
47
|
+
if user_id is not None and user_id in self._user_sessions:
|
|
48
|
+
self._user_sessions[user_id].discard(flow_id)
|
|
49
|
+
|
|
50
|
+
def get_user_flows(self, user_id: int | None) -> list[FlowGraph]:
|
|
51
|
+
"""Get all flows belonging to a specific user's session."""
|
|
52
|
+
if user_id is None:
|
|
53
|
+
return self.flowfile_flows
|
|
54
|
+
user_flow_ids = self._user_sessions.get(user_id, set())
|
|
55
|
+
return [f for f in self._flows.values() if f.flow_id in user_flow_ids]
|
|
56
|
+
|
|
57
|
+
def user_has_flow(self, user_id: int | None, flow_id: int) -> bool:
|
|
58
|
+
"""Check if a user has access to a specific flow."""
|
|
59
|
+
if user_id is None:
|
|
60
|
+
return flow_id in self._flows
|
|
61
|
+
return flow_id in self._user_sessions.get(user_id, set())
|
|
62
|
+
|
|
38
63
|
def __add__(self, other: FlowGraph) -> int:
|
|
39
64
|
self._flows[other.flow_id] = other
|
|
40
65
|
return other.flow_id
|
|
41
66
|
|
|
42
|
-
def import_flow(self, flow_path: Path|str) -> int:
|
|
67
|
+
def import_flow(self, flow_path: Path | str, user_id: int | None = None) -> int:
|
|
43
68
|
if isinstance(flow_path, str):
|
|
44
69
|
flow_path = Path(flow_path)
|
|
45
70
|
imported_flow = open_flow(flow_path)
|
|
46
71
|
self._flows[imported_flow.flow_id] = imported_flow
|
|
47
72
|
imported_flow.flow_settings = self.get_flow_info(imported_flow.flow_id)
|
|
48
73
|
imported_flow.flow_settings.is_running = False
|
|
74
|
+
self._register_user_session(user_id, imported_flow.flow_id)
|
|
49
75
|
return imported_flow.flow_id
|
|
50
76
|
|
|
51
|
-
def register_flow(self, flow_settings: FlowSettings):
|
|
77
|
+
def register_flow(self, flow_settings: FlowSettings, user_id: int | None = None) -> FlowGraph:
|
|
78
|
+
"""Register a flow with the handler and associate it with a user session."""
|
|
52
79
|
if flow_settings.flow_id in self._flows:
|
|
53
80
|
self.delete_flow(flow_settings.flow_id)
|
|
54
|
-
raise
|
|
55
|
-
else
|
|
56
|
-
|
|
57
|
-
|
|
81
|
+
raise ValueError("Flow already registered")
|
|
82
|
+
name = flow_settings.name if flow_settings.name else str(flow_settings.flow_id)
|
|
83
|
+
self._flows[flow_settings.flow_id] = FlowGraph(name=name, flow_settings=flow_settings)
|
|
84
|
+
self._register_user_session(user_id, flow_settings.flow_id)
|
|
58
85
|
return self.get_flow(flow_settings.flow_id)
|
|
59
86
|
|
|
60
|
-
def get_flow(self, flow_id: int) -> FlowGraph | None:
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
87
|
+
def get_flow(self, flow_id: int, user_id: int | None = None) -> FlowGraph | None:
|
|
88
|
+
"""Get a flow by ID, optionally checking user access."""
|
|
89
|
+
flow = self._flows.get(flow_id, None)
|
|
90
|
+
if flow and user_id is not None:
|
|
91
|
+
# Only return the flow if user has access
|
|
92
|
+
if not self.user_has_flow(user_id, flow_id):
|
|
93
|
+
return None
|
|
94
|
+
return flow
|
|
95
|
+
|
|
96
|
+
def delete_flow(self, flow_id: int, user_id: int | None = None):
|
|
97
|
+
"""Remove flow from user's session. Flow data remains until all users close it."""
|
|
98
|
+
if user_id is not None:
|
|
99
|
+
if not self.user_has_flow(user_id, flow_id):
|
|
100
|
+
raise Exception(f"Flow {flow_id} not found in user's session")
|
|
101
|
+
self._unregister_user_session(user_id, flow_id)
|
|
102
|
+
# Check if any user still has this flow open
|
|
103
|
+
flow_still_open = any(flow_id in flows for flows in self._user_sessions.values())
|
|
104
|
+
if not flow_still_open and flow_id in self._flows:
|
|
105
|
+
flow = self._flows.pop(flow_id)
|
|
106
|
+
del flow
|
|
107
|
+
else:
|
|
108
|
+
# No user context - delete directly
|
|
109
|
+
if flow_id in self._flows:
|
|
110
|
+
flow = self._flows.pop(flow_id)
|
|
111
|
+
del flow
|
|
66
112
|
|
|
67
|
-
def save_flow(self, flow_id: int, flow_path: str):
|
|
68
|
-
flow = self.get_flow(flow_id)
|
|
113
|
+
def save_flow(self, flow_id: int, flow_path: str, user_id: int | None = None):
|
|
114
|
+
flow = self.get_flow(flow_id, user_id)
|
|
69
115
|
if flow:
|
|
70
116
|
flow.save_flow(flow_path)
|
|
71
117
|
else:
|
|
72
|
-
raise Exception(
|
|
118
|
+
raise Exception("Flow not found or not accessible by user")
|
|
73
119
|
|
|
74
|
-
def add_flow(self, name: str = None, flow_path: str = None) -> int:
|
|
120
|
+
def add_flow(self, name: str = None, flow_path: str = None, user_id: int | None = None) -> int:
|
|
75
121
|
"""
|
|
76
122
|
Creates a new flow with a reference to the flow path
|
|
77
123
|
Args:
|
|
78
124
|
name (str): The name of the flow
|
|
79
125
|
flow_path (str): The path to the flow file
|
|
126
|
+
user_id (int): The ID of the user creating the flow
|
|
80
127
|
|
|
81
128
|
Returns:
|
|
82
129
|
int: The flow id
|
|
@@ -87,15 +134,18 @@ class FlowfileHandler:
|
|
|
87
134
|
name = create_flow_name()
|
|
88
135
|
if not flow_path:
|
|
89
136
|
flow_path = get_flow_save_location(name)
|
|
90
|
-
flow_info = FlowSettings(
|
|
91
|
-
|
|
137
|
+
flow_info = FlowSettings(
|
|
138
|
+
name=name, flow_id=next_id, save_location=str(flow_path),
|
|
139
|
+
path=str(flow_path)
|
|
140
|
+
)
|
|
141
|
+
flow = self.register_flow(flow_info, user_id=user_id)
|
|
92
142
|
flow.save_flow(flow.flow_settings.path)
|
|
93
143
|
return next_id
|
|
94
144
|
|
|
95
145
|
def get_flow_info(self, flow_id: int) -> FlowSettings:
|
|
96
146
|
flow = self.get_flow(flow_id)
|
|
97
147
|
if not flow:
|
|
98
|
-
raise Exception(f
|
|
148
|
+
raise Exception(f"Flow {flow_id} not found")
|
|
99
149
|
flow_exists = os.path.exists(flow.flow_settings.path)
|
|
100
150
|
last_modified_ts = os.path.getmtime(flow.flow_settings.path) if flow_exists else -1
|
|
101
151
|
flow.flow_settings.modified_on = last_modified_ts
|
|
@@ -104,8 +154,8 @@ class FlowfileHandler:
|
|
|
104
154
|
def get_node(self, flow_id: int, node_id: int):
|
|
105
155
|
flow = self.get_flow(flow_id)
|
|
106
156
|
if not flow:
|
|
107
|
-
raise Exception(f
|
|
157
|
+
raise Exception(f"Flow {flow_id} not found")
|
|
108
158
|
node = flow.get_node(node_id)
|
|
109
159
|
if not node:
|
|
110
|
-
raise Exception(f
|
|
160
|
+
raise Exception(f"Node {node_id} not found in flow {flow_id}")
|
|
111
161
|
return node
|