Flowfile 0.4.1__py3-none-any.whl → 0.5.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- build_backends/main.py +25 -22
- build_backends/main_prd.py +10 -19
- flowfile/__init__.py +179 -73
- flowfile/__main__.py +10 -7
- flowfile/api.py +52 -59
- flowfile/web/__init__.py +14 -9
- flowfile/web/static/assets/AdminView-49392a9a.js +713 -0
- flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
- flowfile/web/static/assets/CloudConnectionView-36bcd6df.css +72 -0
- flowfile/web/static/assets/{CloudConnectionManager-d3248f8d.js → CloudConnectionView-f13f202b.js} +11 -11
- flowfile/web/static/assets/{CloudStorageReader-d65bf041.js → CloudStorageReader-0023d4a5.js} +10 -8
- flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
- flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
- flowfile/web/static/assets/{CloudStorageWriter-e83be3ed.js → CloudStorageWriter-8e781e11.js} +10 -8
- flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
- flowfile/web/static/assets/{ColumnSelector-cce661cf.js → ColumnSelector-8ad68ea9.js} +3 -5
- flowfile/web/static/assets/{ContextMenu-c13f91d0.css → ContextMenu-26d4dd27.css} +6 -6
- flowfile/web/static/assets/{ContextMenu-11a4652a.js → ContextMenu-31ee57f0.js} +3 -3
- flowfile/web/static/assets/{ContextMenu-160afb08.js → ContextMenu-69a74055.js} +3 -3
- flowfile/web/static/assets/{ContextMenu-cf18d2cc.js → ContextMenu-8e2051c6.js} +3 -3
- flowfile/web/static/assets/{ContextMenu-4c74eef1.css → ContextMenu-8ec1729e.css} +6 -6
- flowfile/web/static/assets/{ContextMenu-63cfa99b.css → ContextMenu-9b310c60.css} +6 -6
- flowfile/web/static/assets/{CrossJoin-d395d38c.js → CrossJoin-03df6938.js} +12 -10
- flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
- flowfile/web/static/assets/CustomNode-59e99a86.css +32 -0
- flowfile/web/static/assets/{CustomNode-b812dc0b.js → CustomNode-8479239b.js} +36 -24
- flowfile/web/static/assets/{DatabaseConnectionSettings-7000bf2c.js → DatabaseConnectionSettings-869e3efd.js} +5 -4
- flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-e91df89a.css} +13 -13
- flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-36898a00.css} +24 -24
- flowfile/web/static/assets/{DatabaseReader-4f035d0c.js → DatabaseReader-c58b9552.js} +25 -15
- flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
- flowfile/web/static/assets/{DatabaseManager-9662ec5b.js → DatabaseView-d26a9140.js} +11 -11
- flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-217a99f1.css} +19 -19
- flowfile/web/static/assets/{DatabaseWriter-f65dcd54.js → DatabaseWriter-4d05ddc7.js} +17 -10
- flowfile/web/static/assets/{designer-e3c150ec.css → DesignerView-a6d0ee84.css} +629 -538
- flowfile/web/static/assets/{designer-f3656d8c.js → DesignerView-e6f5c0e8.js} +1214 -3209
- flowfile/web/static/assets/{documentation-52b241e7.js → DocumentationView-2e78ef1b.js} +5 -5
- flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-fd46c656.css} +7 -7
- flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
- flowfile/web/static/assets/{ExploreData-94c43dfc.js → ExploreData-7b54caca.js} +18 -9
- flowfile/web/static/assets/{ExternalSource-ac04b3cc.js → ExternalSource-3fa399b2.js} +9 -7
- flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-47ab05a3.css} +17 -17
- flowfile/web/static/assets/Filter-7494ea97.css +48 -0
- flowfile/web/static/assets/Filter-8cbbdbf3.js +287 -0
- flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
- flowfile/web/static/assets/{Formula-71472193.js → Formula-aac42b1e.js} +13 -11
- flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
- flowfile/web/static/assets/{FuzzyMatch-b317f631.js → FuzzyMatch-cd9bbfca.js} +12 -10
- flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-c24dec17.css} +5 -5
- flowfile/web/static/assets/{GraphSolver-754a234f.js → GraphSolver-c7e6780e.js} +13 -11
- flowfile/web/static/assets/{GroupBy-6c6f9802.js → GroupBy-93c5d22b.js} +9 -7
- flowfile/web/static/assets/{GroupBy-b9505323.css → GroupBy-be7ac0bf.css} +10 -10
- flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
- flowfile/web/static/assets/{Join-a1b800be.js → Join-a19b2de2.js} +13 -11
- flowfile/web/static/assets/LoginView-0df4ed0a.js +134 -0
- flowfile/web/static/assets/LoginView-d325d632.css +172 -0
- flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
- flowfile/web/static/assets/{ManualInput-a9640276.js → ManualInput-8d3374b2.js} +170 -116
- flowfile/web/static/assets/{MultiSelect-97213888.js → MultiSelect-ad1b6243.js} +2 -2
- flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-6ffe088a.js → MultiSelect.vue_vue_type_script_setup_true_lang-e278950d.js} +1 -1
- flowfile/web/static/assets/NodeDesigner-40b647c9.js +2610 -0
- flowfile/web/static/assets/NodeDesigner-5f53be3f.css +1429 -0
- flowfile/web/static/assets/{NumericInput-e638088a.js → NumericInput-7100234c.js} +2 -2
- flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-90eb2cba.js → NumericInput.vue_vue_type_script_setup_true_lang-5130219f.js} +5 -2
- flowfile/web/static/assets/{Output-ddc9079f.css → Output-35e97000.css} +6 -6
- flowfile/web/static/assets/{Output-76750610.js → Output-f5efd2aa.js} +60 -38
- flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
- flowfile/web/static/assets/{Pivot-7814803f.js → Pivot-d981d23c.js} +11 -9
- flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
- flowfile/web/static/assets/{PivotValidation-f92137d2.js → PivotValidation-39386e95.js} +3 -3
- flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
- flowfile/web/static/assets/{PivotValidation-76dd431a.js → PivotValidation-63de1f73.js} +3 -3
- flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
- flowfile/web/static/assets/{PolarsCode-889c3008.js → PolarsCode-f9d69217.js} +18 -9
- flowfile/web/static/assets/PopOver-b22f049e.js +939 -0
- flowfile/web/static/assets/PopOver-d96599db.css +33 -0
- flowfile/web/static/assets/{Read-6b17491f.css → Read-36e7bd51.css} +12 -12
- flowfile/web/static/assets/{Read-637b72a7.js → Read-aec2e377.js} +83 -105
- flowfile/web/static/assets/{RecordCount-2b050c41.js → RecordCount-78ed6845.js} +6 -4
- flowfile/web/static/assets/{RecordId-81df7784.js → RecordId-2156e890.js} +8 -6
- flowfile/web/static/assets/{SQLQueryComponent-36cef432.css → SQLQueryComponent-1c2f26b4.css} +5 -5
- flowfile/web/static/assets/{SQLQueryComponent-88dcfe53.js → SQLQueryComponent-48c72f5b.js} +3 -3
- flowfile/web/static/assets/{Sample-258ad2a9.js → Sample-1352ca74.js} +6 -4
- flowfile/web/static/assets/SecretSelector-22b5ff89.js +113 -0
- flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
- flowfile/web/static/assets/{SecretManager-2a2cb7e2.js → SecretsView-17df66ee.js} +35 -36
- flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
- flowfile/web/static/assets/{Select-850215fd.js → Select-0aee4c54.js} +9 -7
- flowfile/web/static/assets/{SettingsSection-55bae608.js → SettingsSection-0784e157.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-0e8d9123.js → SettingsSection-cd341bb6.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-29b4fa6b.js → SettingsSection-f2002a6d.js} +3 -3
- flowfile/web/static/assets/{SingleSelect-bebd408b.js → SingleSelect-460cc0ea.js} +2 -2
- flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-6093741c.js → SingleSelect.vue_vue_type_script_setup_true_lang-30741bb2.js} +1 -1
- flowfile/web/static/assets/{SliderInput-6a05ab61.js → SliderInput-5d926864.js} +7 -4
- flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
- flowfile/web/static/assets/{Sort-10ab48ed.js → Sort-3cdc971b.js} +9 -7
- flowfile/web/static/assets/{Unique-f9fb0809.css → Sort-8a871341.css} +10 -10
- flowfile/web/static/assets/{TextInput-df9d6259.js → TextInput-a2d0bfbd.js} +2 -2
- flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-000e1178.js → TextInput.vue_vue_type_script_setup_true_lang-abad1ca2.js} +5 -2
- flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
- flowfile/web/static/assets/{TextToRows-6c2d93d8.js → TextToRows-918945f7.js} +11 -10
- flowfile/web/static/assets/{ToggleSwitch-0ff7ac52.js → ToggleSwitch-f0ef5196.js} +2 -2
- flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-c6dc3029.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-5605c793.js} +1 -1
- flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-54d2f518.css} +6 -6
- flowfile/web/static/assets/{UnavailableFields-1bab97cb.js → UnavailableFields-bdad6144.js} +4 -4
- flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
- flowfile/web/static/assets/{Union-b563478a.js → Union-e8ab8c86.js} +8 -6
- flowfile/web/static/assets/{Unique-f90db5db.js → Unique-8cd4f976.js} +13 -22
- flowfile/web/static/assets/{Sort-3643d625.css → Unique-9fb2f567.css} +10 -10
- flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-710a2948.css} +7 -7
- flowfile/web/static/assets/{Unpivot-bcb0025f.js → Unpivot-8da14095.js} +10 -8
- flowfile/web/static/assets/{UnpivotValidation-c4e73b04.js → UnpivotValidation-6f7d89ff.js} +3 -3
- flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
- flowfile/web/static/assets/{VueGraphicWalker-bb8535e2.js → VueGraphicWalker-3fb312e1.js} +4 -4
- flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
- flowfile/web/static/assets/{api-4c8e3822.js → api-24483f0d.js} +1 -1
- flowfile/web/static/assets/{api-2d6adc4f.js → api-8b81fa73.js} +1 -1
- flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-3d8dc5fa.css} +40 -40
- flowfile/web/static/assets/{dropDown-1bca8a74.js → dropDown-ac0fda9d.js} +3 -3
- flowfile/web/static/assets/{fullEditor-2985687e.js → fullEditor-5497a84a.js} +11 -10
- flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-a0be62b3.css} +74 -62
- flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
- flowfile/web/static/assets/{genericNodeSettings-0476ba4e.js → genericNodeSettings-99014e1d.js} +5 -5
- flowfile/web/static/assets/index-07dda503.js +38 -0
- flowfile/web/static/assets/index-3ba44389.js +2696 -0
- flowfile/web/static/assets/{index-50508d4d.css → index-e6289dd0.css} +1945 -569
- flowfile/web/static/assets/{index-246f201c.js → index-fb6493ae.js} +41626 -40869
- flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
- flowfile/web/static/assets/nodeInput-0eb13f1a.js +2 -0
- flowfile/web/static/assets/{outputCsv-d686eeaf.js → outputCsv-8f8ba42d.js} +3 -3
- flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
- flowfile/web/static/assets/{outputExcel-8809ea2f.js → outputExcel-393f4fef.js} +3 -3
- flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
- flowfile/web/static/assets/{outputParquet-53ba645a.js → outputParquet-07c81f65.js} +4 -4
- flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
- flowfile/web/static/assets/{readCsv-053bf97b.js → readCsv-07f6d9ad.js} +21 -20
- flowfile/web/static/assets/{readCsv-bca3ed53.css → readCsv-3bfac4c3.css} +15 -15
- flowfile/web/static/assets/{readExcel-e1b381ea.css → readExcel-3db6b763.css} +13 -13
- flowfile/web/static/assets/{readExcel-ad531eab.js → readExcel-ed69bc8f.js} +10 -12
- flowfile/web/static/assets/{readParquet-cee068e2.css → readParquet-c5244ad5.css} +4 -4
- flowfile/web/static/assets/{readParquet-58e899a1.js → readParquet-e3ed4528.js} +4 -7
- flowfile/web/static/assets/secrets.api-002e7d7e.js +65 -0
- flowfile/web/static/assets/{selectDynamic-b38de2ba.js → selectDynamic-80b92899.js} +5 -5
- flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
- flowfile/web/static/assets/{vue-codemirror.esm-db9b8936.js → vue-codemirror.esm-0965f39f.js} +31 -637
- flowfile/web/static/assets/{vue-content-loader.es-b5f3ac30.js → vue-content-loader.es-c506ad97.js} +1 -1
- flowfile/web/static/index.html +2 -2
- {flowfile-0.4.1.dist-info → flowfile-0.5.3.dist-info}/METADATA +4 -4
- flowfile-0.5.3.dist-info/RECORD +402 -0
- {flowfile-0.4.1.dist-info → flowfile-0.5.3.dist-info}/WHEEL +1 -1
- {flowfile-0.4.1.dist-info → flowfile-0.5.3.dist-info}/entry_points.txt +1 -0
- flowfile_core/__init__.py +13 -3
- flowfile_core/auth/jwt.py +51 -16
- flowfile_core/auth/models.py +32 -7
- flowfile_core/auth/password.py +89 -0
- flowfile_core/auth/secrets.py +8 -6
- flowfile_core/configs/__init__.py +9 -7
- flowfile_core/configs/flow_logger.py +15 -14
- flowfile_core/configs/node_store/__init__.py +72 -4
- flowfile_core/configs/node_store/nodes.py +155 -172
- flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
- flowfile_core/configs/settings.py +28 -15
- flowfile_core/database/connection.py +7 -6
- flowfile_core/database/init_db.py +96 -2
- flowfile_core/database/models.py +3 -1
- flowfile_core/fileExplorer/__init__.py +17 -0
- flowfile_core/fileExplorer/funcs.py +123 -57
- flowfile_core/fileExplorer/utils.py +10 -11
- flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
- flowfile_core/flowfile/analytics/analytics_processor.py +27 -24
- flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
- flowfile_core/flowfile/analytics/utils.py +1 -1
- flowfile_core/flowfile/code_generator/code_generator.py +391 -279
- flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
- flowfile_core/flowfile/connection_manager/models.py +1 -1
- flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
- flowfile_core/flowfile/database_connection_manager/models.py +1 -1
- flowfile_core/flowfile/extensions.py +17 -12
- flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
- flowfile_core/flowfile/flow_data_engine/create/funcs.py +152 -103
- flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +526 -477
- flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
- flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
- flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
- flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
- flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
- flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +43 -32
- flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
- flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +15 -11
- flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
- flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
- flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
- flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +360 -191
- flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
- flowfile_core/flowfile/flow_data_engine/utils.py +101 -67
- flowfile_core/flowfile/flow_graph.py +1011 -561
- flowfile_core/flowfile/flow_graph_utils.py +31 -49
- flowfile_core/flowfile/flow_node/flow_node.py +332 -232
- flowfile_core/flowfile/flow_node/models.py +54 -41
- flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
- flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
- flowfile_core/flowfile/handler.py +82 -32
- flowfile_core/flowfile/manage/compatibility_enhancements.py +493 -47
- flowfile_core/flowfile/manage/io_flowfile.py +391 -0
- flowfile_core/flowfile/node_designer/__init__.py +15 -13
- flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
- flowfile_core/flowfile/node_designer/custom_node.py +162 -36
- flowfile_core/flowfile/node_designer/ui_components.py +136 -35
- flowfile_core/flowfile/schema_callbacks.py +77 -54
- flowfile_core/flowfile/setting_generator/__init__.py +0 -1
- flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
- flowfile_core/flowfile/setting_generator/settings.py +72 -55
- flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
- flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
- flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
- flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
- flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
- flowfile_core/flowfile/util/calculate_layout.py +9 -13
- flowfile_core/flowfile/util/execution_orderer.py +25 -17
- flowfile_core/flowfile/util/node_skipper.py +4 -4
- flowfile_core/flowfile/utils.py +19 -21
- flowfile_core/main.py +26 -19
- flowfile_core/routes/auth.py +284 -11
- flowfile_core/routes/cloud_connections.py +25 -25
- flowfile_core/routes/logs.py +21 -29
- flowfile_core/routes/public.py +3 -3
- flowfile_core/routes/routes.py +77 -43
- flowfile_core/routes/secrets.py +25 -27
- flowfile_core/routes/user_defined_components.py +483 -4
- flowfile_core/run_lock.py +0 -1
- flowfile_core/schemas/__init__.py +4 -6
- flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
- flowfile_core/schemas/cloud_storage_schemas.py +59 -55
- flowfile_core/schemas/input_schema.py +398 -154
- flowfile_core/schemas/output_model.py +50 -35
- flowfile_core/schemas/schemas.py +207 -67
- flowfile_core/schemas/transform_schema.py +1360 -435
- flowfile_core/schemas/yaml_types.py +117 -0
- flowfile_core/secret_manager/secret_manager.py +17 -13
- flowfile_core/{flowfile/node_designer/data_types.py → types.py} +33 -3
- flowfile_core/utils/arrow_reader.py +7 -6
- flowfile_core/utils/excel_file_manager.py +3 -3
- flowfile_core/utils/fileManager.py +7 -7
- flowfile_core/utils/fl_executor.py +8 -10
- flowfile_core/utils/utils.py +4 -4
- flowfile_core/utils/validate_setup.py +5 -4
- flowfile_frame/__init__.py +107 -50
- flowfile_frame/adapters.py +2 -9
- flowfile_frame/adding_expr.py +73 -32
- flowfile_frame/cloud_storage/frame_helpers.py +27 -23
- flowfile_frame/cloud_storage/secret_manager.py +12 -26
- flowfile_frame/config.py +2 -5
- flowfile_frame/expr.py +311 -218
- flowfile_frame/expr.pyi +160 -159
- flowfile_frame/expr_name.py +23 -23
- flowfile_frame/flow_frame.py +581 -489
- flowfile_frame/flow_frame.pyi +123 -104
- flowfile_frame/flow_frame_methods.py +236 -252
- flowfile_frame/group_frame.py +50 -20
- flowfile_frame/join.py +2 -2
- flowfile_frame/lazy.py +129 -87
- flowfile_frame/lazy_methods.py +83 -30
- flowfile_frame/list_name_space.py +55 -50
- flowfile_frame/selectors.py +148 -68
- flowfile_frame/series.py +9 -7
- flowfile_frame/utils.py +19 -21
- flowfile_worker/__init__.py +12 -4
- flowfile_worker/configs.py +11 -19
- flowfile_worker/create/__init__.py +14 -27
- flowfile_worker/create/funcs.py +143 -94
- flowfile_worker/create/models.py +139 -68
- flowfile_worker/create/pl_types.py +14 -15
- flowfile_worker/create/read_excel_tables.py +34 -41
- flowfile_worker/create/utils.py +22 -19
- flowfile_worker/external_sources/s3_source/main.py +18 -51
- flowfile_worker/external_sources/s3_source/models.py +34 -27
- flowfile_worker/external_sources/sql_source/main.py +8 -5
- flowfile_worker/external_sources/sql_source/models.py +13 -9
- flowfile_worker/flow_logger.py +10 -8
- flowfile_worker/funcs.py +214 -155
- flowfile_worker/main.py +11 -17
- flowfile_worker/models.py +35 -28
- flowfile_worker/process_manager.py +2 -3
- flowfile_worker/routes.py +121 -93
- flowfile_worker/secrets.py +9 -6
- flowfile_worker/spawner.py +80 -49
- flowfile_worker/utils.py +3 -2
- shared/__init__.py +2 -7
- shared/storage_config.py +25 -13
- test_utils/postgres/commands.py +3 -2
- test_utils/postgres/fixtures.py +9 -9
- test_utils/s3/commands.py +1 -1
- test_utils/s3/data_generator.py +3 -4
- test_utils/s3/demo_data_generator.py +4 -7
- test_utils/s3/fixtures.py +7 -5
- tools/migrate/README.md +56 -0
- tools/migrate/__init__.py +12 -0
- tools/migrate/__main__.py +118 -0
- tools/migrate/legacy_schemas.py +682 -0
- tools/migrate/migrate.py +610 -0
- tools/migrate/tests/__init__.py +0 -0
- tools/migrate/tests/conftest.py +21 -0
- tools/migrate/tests/test_migrate.py +622 -0
- tools/migrate/tests/test_migration_e2e.py +1009 -0
- tools/migrate/tests/test_node_migrations.py +843 -0
- flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
- flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
- flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
- flowfile/web/static/assets/Filter-812dcbca.js +0 -164
- flowfile/web/static/assets/Filter-f62091b3.css +0 -20
- flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
- flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
- flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
- flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
- flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
- flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
- flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
- flowfile/web/static/assets/secretApi-538058f3.js +0 -46
- flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
- flowfile-0.4.1.dist-info/RECORD +0 -376
- flowfile_core/flowfile/manage/open_flowfile.py +0 -143
- {flowfile-0.4.1.dist-info → flowfile-0.5.3.dist-info}/licenses/LICENSE +0 -0
- /flowfile_core/flowfile/manage/manage_flowfile.py → /tools/__init__.py +0 -0
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
|
|
2
1
|
import polars as pl
|
|
3
|
-
|
|
2
|
+
|
|
4
3
|
from flowfile_core.flowfile.flow_data_engine import utils
|
|
4
|
+
from flowfile_core.utils.fl_executor import process_executor
|
|
5
5
|
|
|
6
6
|
# calculate_schema_threaded = process_executor(wait_on_completion=True, max_workers=1)(utils.calculate_schema)
|
|
7
7
|
write_threaded = process_executor(False, max_workers=1)(utils.write_polars_frame)
|
|
8
8
|
collect_threaded = process_executor(wait_on_completion=False, max_workers=1)(utils.collect)
|
|
9
9
|
cache_polars_frame_to_temp_thread = process_executor(wait_on_completion=True, max_workers=1)(
|
|
10
|
-
utils.cache_polars_frame_to_temp
|
|
10
|
+
utils.cache_polars_frame_to_temp
|
|
11
|
+
)
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
@process_executor(False, max_workers=1)
|
|
14
15
|
def do_something_random():
|
|
15
|
-
print(
|
|
16
|
+
print("10 seconds")
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
# @process_executor(False, max_workers=1)
|
|
19
20
|
def get_join_count(left: pl.LazyFrame, right: pl.LazyFrame, left_on_keys, right_on_keys, how):
|
|
20
21
|
left_joined_df = left.group_by(left_on_keys).count()
|
|
21
22
|
right_joined_df = right.group_by(right_on_keys).count()
|
|
22
|
-
data: pl.LazyFrame = left_joined_df.join(right_joined_df, left_on=left_on_keys,
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
result = data.select((pl.col('total') * pl.col('count') * pl.col('count_right'))).sum()
|
|
23
|
+
data: pl.LazyFrame = left_joined_df.join(right_joined_df, left_on=left_on_keys, right_on=right_on_keys, how=how)
|
|
24
|
+
data = data.with_columns(pl.lit(1).alias("total").cast(pl.UInt64))
|
|
25
|
+
result = data.select(pl.col("total") * pl.col("count") * pl.col("count_right")).sum()
|
|
26
26
|
n_records = result.collect().to_series().to_list()[0]
|
|
27
27
|
return n_records
|
|
@@ -1,13 +1,15 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from base64 import encodebytes
|
|
3
|
+
from collections.abc import Callable, Iterable
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
1
6
|
import polars as pl
|
|
2
|
-
|
|
7
|
+
import requests
|
|
8
|
+
|
|
3
9
|
from flowfile_core.configs import logger
|
|
4
|
-
from flowfile_core.
|
|
5
|
-
from flowfile_core.flowfile.flow_data_engine.subprocess_operations import Status
|
|
10
|
+
from flowfile_core.configs.settings import AVAILABLE_RAM, WORKER_URL
|
|
11
|
+
from flowfile_core.flowfile.flow_data_engine.subprocess_operations import ExternalDfFetcher, Status
|
|
6
12
|
from flowfile_core.utils.utils import standardize_col_dtype
|
|
7
|
-
import os
|
|
8
|
-
from typing import List, Dict, Iterable, Callable, Any
|
|
9
|
-
import requests
|
|
10
|
-
from base64 import encodebytes
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
def get_data_type(vals: Iterable[Any]):
|
|
@@ -15,23 +17,26 @@ def get_data_type(vals: Iterable[Any]):
|
|
|
15
17
|
if len(types) == 1:
|
|
16
18
|
return types.pop().__name__
|
|
17
19
|
elif types == {float, int}:
|
|
18
|
-
return
|
|
20
|
+
return "float"
|
|
19
21
|
else:
|
|
20
|
-
return
|
|
22
|
+
return "str"
|
|
21
23
|
|
|
22
24
|
|
|
23
|
-
def calculate_schema(lf: pl.LazyFrame) ->
|
|
24
|
-
r = ExternalDfFetcher(lf=lf, operation_type=
|
|
25
|
-
schema_stats:
|
|
25
|
+
def calculate_schema(lf: pl.LazyFrame) -> list[dict]:
|
|
26
|
+
r = ExternalDfFetcher(lf=lf, operation_type="calculate_schema", wait_on_completion=False, flow_id=-1, node_id=-1)
|
|
27
|
+
schema_stats: list[dict] = r.get_result()
|
|
28
|
+
|
|
26
29
|
for schema_stat in schema_stats:
|
|
27
|
-
schema_stat[
|
|
30
|
+
schema_stat["pl_datatype"] = getattr(pl.datatypes, schema_stat["pl_datatype"])
|
|
31
|
+
|
|
28
32
|
return schema_stats
|
|
29
33
|
|
|
30
34
|
|
|
31
|
-
def write_polars_frame(
|
|
32
|
-
|
|
35
|
+
def write_polars_frame(
|
|
36
|
+
_df: pl.LazyFrame | pl.DataFrame, path: str, data_type: str = "parquet", estimated_size: int = 0
|
|
37
|
+
):
|
|
33
38
|
is_lazy = isinstance(_df, pl.LazyFrame)
|
|
34
|
-
logger.info(
|
|
39
|
+
logger.info("Caching data frame")
|
|
35
40
|
if is_lazy:
|
|
36
41
|
if estimated_size > 0:
|
|
37
42
|
fit_memory = estimated_size / 1024 / 1000 / 1000 < AVAILABLE_RAM
|
|
@@ -41,16 +46,16 @@ def write_polars_frame(_df: pl.LazyFrame | pl.DataFrame, path: str, data_type: s
|
|
|
41
46
|
|
|
42
47
|
if is_lazy:
|
|
43
48
|
logger.info("Writing in memory efficient mode")
|
|
44
|
-
write_method = getattr(_df,
|
|
49
|
+
write_method = getattr(_df, "sink_" + data_type)
|
|
45
50
|
try:
|
|
46
51
|
write_method(path)
|
|
47
52
|
return True
|
|
48
|
-
except Exception
|
|
53
|
+
except Exception:
|
|
49
54
|
pass
|
|
50
55
|
if is_lazy:
|
|
51
56
|
_df = _df.collect()
|
|
52
57
|
try:
|
|
53
|
-
write_method = getattr(_df,
|
|
58
|
+
write_method = getattr(_df, "write_" + data_type)
|
|
54
59
|
write_method(path)
|
|
55
60
|
return True
|
|
56
61
|
except:
|
|
@@ -65,13 +70,13 @@ def collect(df: pl.LazyFrame, streamable: bool = True):
|
|
|
65
70
|
|
|
66
71
|
|
|
67
72
|
def cache_polars_frame_to_temp(_df: pl.LazyFrame | pl.DataFrame, tempdir: str = None) -> pl.LazyFrame:
|
|
68
|
-
path = f
|
|
73
|
+
path = f"{tempdir}\\fl_file_{id(_df)}"
|
|
69
74
|
result = write_polars_frame(_df, path)
|
|
70
75
|
if result:
|
|
71
76
|
df = pl.read_parquet(path)
|
|
72
77
|
return df.lazy()
|
|
73
78
|
else:
|
|
74
|
-
raise Exception(
|
|
79
|
+
raise Exception("Could not cache the data")
|
|
75
80
|
|
|
76
81
|
|
|
77
82
|
def define_pl_col_transformation(col_name: str, col_type: pl.DataType) -> pl.Expr:
|
|
@@ -83,64 +88,94 @@ def define_pl_col_transformation(col_name: str, col_type: pl.DataType) -> pl.Exp
|
|
|
83
88
|
return pl.col(col_name).cast(col_type, strict=False)
|
|
84
89
|
|
|
85
90
|
|
|
86
|
-
def execute_write_method(
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
+
def execute_write_method(
|
|
92
|
+
write_method: Callable,
|
|
93
|
+
path: str,
|
|
94
|
+
data_type: str = None,
|
|
95
|
+
sheet_name: str = None,
|
|
96
|
+
delimiter: str = None,
|
|
97
|
+
write_mode: str = "create",
|
|
98
|
+
):
|
|
99
|
+
if data_type == "excel":
|
|
100
|
+
logger.info("Writing as excel file")
|
|
91
101
|
write_method(path, worksheet=sheet_name)
|
|
92
|
-
elif data_type ==
|
|
93
|
-
logger.info(
|
|
94
|
-
if write_mode ==
|
|
95
|
-
with open(path,
|
|
96
|
-
write_method(path=f, separator=delimiter, quote_style=
|
|
102
|
+
elif data_type == "csv":
|
|
103
|
+
logger.info("Writing as csv file")
|
|
104
|
+
if write_mode == "append":
|
|
105
|
+
with open(path, "ab") as f:
|
|
106
|
+
write_method(path=f, separator=delimiter, quote_style="always")
|
|
97
107
|
else:
|
|
98
|
-
write_method(path=path, separator=delimiter, quote_style=
|
|
99
|
-
elif data_type ==
|
|
100
|
-
logger.info(
|
|
108
|
+
write_method(path=path, separator=delimiter, quote_style="always")
|
|
109
|
+
elif data_type == "parquet":
|
|
110
|
+
logger.info("Writing as parquet file")
|
|
101
111
|
write_method(path)
|
|
102
112
|
|
|
103
113
|
|
|
104
|
-
def write_output(
|
|
105
|
-
|
|
106
|
-
|
|
114
|
+
def write_output(
|
|
115
|
+
_df: pl.LazyFrame,
|
|
116
|
+
data_type: str,
|
|
117
|
+
path: str,
|
|
118
|
+
write_mode: str,
|
|
119
|
+
sheet_name: str = None,
|
|
120
|
+
delimiter: str = None,
|
|
121
|
+
flow_id: int = -1,
|
|
122
|
+
node_id: int | str = -1,
|
|
123
|
+
) -> Status:
|
|
107
124
|
serializable_df = _df.serialize()
|
|
108
|
-
r = requests.post(
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
125
|
+
r = requests.post(
|
|
126
|
+
f"{WORKER_URL}/write_results/",
|
|
127
|
+
json={
|
|
128
|
+
"operation": encodebytes(serializable_df).decode(),
|
|
129
|
+
"data_type": data_type,
|
|
130
|
+
"path": path,
|
|
131
|
+
"write_mode": write_mode,
|
|
132
|
+
"sheet_name": sheet_name,
|
|
133
|
+
"delimiter": delimiter,
|
|
134
|
+
"flowfile_node_id": node_id,
|
|
135
|
+
"flowfile_flow_id": flow_id,
|
|
136
|
+
},
|
|
137
|
+
)
|
|
117
138
|
if r.ok:
|
|
118
139
|
return Status(**r.json())
|
|
119
140
|
else:
|
|
120
|
-
raise Exception(f
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
def local_write_output(
|
|
124
|
-
|
|
141
|
+
raise Exception(f"Could not cache the data, {r.text}")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def local_write_output(
|
|
145
|
+
_df: pl.LazyFrame | pl.DataFrame,
|
|
146
|
+
data_type: str,
|
|
147
|
+
path: str,
|
|
148
|
+
write_mode: str,
|
|
149
|
+
sheet_name: str = None,
|
|
150
|
+
delimiter: str = None,
|
|
151
|
+
flow_id: int = -1,
|
|
152
|
+
node_id: int | str = -1,
|
|
153
|
+
):
|
|
125
154
|
is_lazy = isinstance(_df, pl.LazyFrame)
|
|
126
|
-
sink_method_str =
|
|
127
|
-
write_method_str =
|
|
155
|
+
sink_method_str = "sink_" + data_type
|
|
156
|
+
write_method_str = "write_" + data_type
|
|
128
157
|
has_sink_method = hasattr(_df, sink_method_str)
|
|
129
158
|
write_method = None
|
|
130
|
-
if os.path.exists(path) and write_mode ==
|
|
159
|
+
if os.path.exists(path) and write_mode == "create":
|
|
131
160
|
return None
|
|
132
161
|
if has_sink_method and is_lazy:
|
|
133
|
-
write_method = getattr(_df,
|
|
162
|
+
write_method = getattr(_df, "sink_" + data_type)
|
|
134
163
|
elif not is_lazy or not has_sink_method:
|
|
135
164
|
if is_lazy:
|
|
136
165
|
_df = _df.collect()
|
|
137
166
|
write_method = getattr(_df, write_method_str)
|
|
138
167
|
if write_method is not None:
|
|
139
|
-
execute_write_method(
|
|
140
|
-
|
|
168
|
+
execute_write_method(
|
|
169
|
+
write_method,
|
|
170
|
+
path=path,
|
|
171
|
+
data_type=data_type,
|
|
172
|
+
sheet_name=sheet_name,
|
|
173
|
+
delimiter=delimiter,
|
|
174
|
+
write_mode=write_mode,
|
|
175
|
+
)
|
|
141
176
|
|
|
142
177
|
|
|
143
|
-
def create_pl_df_type_save(raw_data: Iterable[Iterable], orient: str =
|
|
178
|
+
def create_pl_df_type_save(raw_data: Iterable[Iterable], orient: str = "row") -> pl.DataFrame:
|
|
144
179
|
"""
|
|
145
180
|
orient : {'col', 'row'}, default None
|
|
146
181
|
Whether to interpret two-dimensional data as columns or as rows. If None,
|
|
@@ -150,25 +185,24 @@ def create_pl_df_type_save(raw_data: Iterable[Iterable], orient: str = 'row') ->
|
|
|
150
185
|
:param orient:
|
|
151
186
|
:return: polars dataframe
|
|
152
187
|
"""
|
|
153
|
-
if orient ==
|
|
154
|
-
raw_data = zip(*raw_data)
|
|
188
|
+
if orient == "row":
|
|
189
|
+
raw_data = zip(*raw_data, strict=False)
|
|
155
190
|
raw_data = [standardize_col_dtype(values) for values in raw_data]
|
|
156
|
-
return pl.DataFrame(raw_data, orient=
|
|
191
|
+
return pl.DataFrame(raw_data, orient="col")
|
|
157
192
|
|
|
158
193
|
|
|
159
|
-
def find_first_positions(lst:
|
|
160
|
-
first_positions:
|
|
194
|
+
def find_first_positions(lst: list[str]) -> dict[str, int]:
|
|
195
|
+
first_positions: dict[str, int] = {}
|
|
161
196
|
for i, value in enumerate(lst):
|
|
162
197
|
if value not in first_positions:
|
|
163
198
|
first_positions[value] = i
|
|
164
199
|
return first_positions
|
|
165
200
|
|
|
166
201
|
|
|
167
|
-
def match_order(l:
|
|
202
|
+
def match_order(l: list[str], ref: list[str]) -> list[str]:
|
|
168
203
|
ref_order = find_first_positions(ref)
|
|
169
204
|
order = []
|
|
170
205
|
for v in l:
|
|
171
|
-
org_order = ref_order.get(v, float(
|
|
206
|
+
org_order = ref_order.get(v, float("inf"))
|
|
172
207
|
order.append(org_order)
|
|
173
|
-
return [v for _, v in sorted(zip(order, l))]
|
|
174
|
-
|
|
208
|
+
return [v for _, v in sorted(zip(order, l, strict=False))]
|