Flowfile 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- build_backends/main.py +25 -22
- build_backends/main_prd.py +10 -19
- flowfile/__init__.py +178 -74
- flowfile/__main__.py +10 -7
- flowfile/api.py +51 -57
- flowfile/web/__init__.py +14 -9
- flowfile/web/static/assets/AdminView-49392a9a.js +713 -0
- flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
- flowfile/web/static/assets/CloudConnectionView-36bcd6df.css +72 -0
- flowfile/web/static/assets/{CloudConnectionManager-0dfba9f2.js → CloudConnectionView-f13f202b.js} +11 -11
- flowfile/web/static/assets/{CloudStorageReader-d5b1b6c9.js → CloudStorageReader-0023d4a5.js} +10 -8
- flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
- flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
- flowfile/web/static/assets/{CloudStorageWriter-00d87aad.js → CloudStorageWriter-8e781e11.js} +10 -8
- flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
- flowfile/web/static/assets/{ColumnSelector-4685e75d.js → ColumnSelector-8ad68ea9.js} +3 -5
- flowfile/web/static/assets/{ContextMenu-c13f91d0.css → ContextMenu-26d4dd27.css} +6 -6
- flowfile/web/static/assets/{ContextMenu-23e909da.js → ContextMenu-31ee57f0.js} +3 -3
- flowfile/web/static/assets/{ContextMenu-70ae0c79.js → ContextMenu-69a74055.js} +3 -3
- flowfile/web/static/assets/{ContextMenu-f149cf7c.js → ContextMenu-8e2051c6.js} +3 -3
- flowfile/web/static/assets/{ContextMenu-4c74eef1.css → ContextMenu-8ec1729e.css} +6 -6
- flowfile/web/static/assets/{ContextMenu-63cfa99b.css → ContextMenu-9b310c60.css} +6 -6
- flowfile/web/static/assets/{CrossJoin-702a3edd.js → CrossJoin-03df6938.js} +12 -10
- flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
- flowfile/web/static/assets/CustomNode-59e99a86.css +32 -0
- flowfile/web/static/assets/{CustomNode-b1519993.js → CustomNode-8479239b.js} +36 -24
- flowfile/web/static/assets/{DatabaseConnectionSettings-6f3e4ea5.js → DatabaseConnectionSettings-869e3efd.js} +5 -4
- flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-e91df89a.css} +13 -13
- flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-36898a00.css} +24 -24
- flowfile/web/static/assets/{DatabaseReader-d38c7295.js → DatabaseReader-c58b9552.js} +25 -15
- flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
- flowfile/web/static/assets/{DatabaseManager-cf5ef661.js → DatabaseView-d26a9140.js} +11 -11
- flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-217a99f1.css} +19 -19
- flowfile/web/static/assets/{DatabaseWriter-b04ef46a.js → DatabaseWriter-4d05ddc7.js} +17 -10
- flowfile/web/static/assets/{designer-8da3ba3a.css → DesignerView-a6d0ee84.css} +614 -546
- flowfile/web/static/assets/{designer-9633482a.js → DesignerView-e6f5c0e8.js} +1107 -3170
- flowfile/web/static/assets/{documentation-ca400224.js → DocumentationView-2e78ef1b.js} +5 -5
- flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-fd46c656.css} +7 -7
- flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
- flowfile/web/static/assets/{ExploreData-5fa10ed8.js → ExploreData-7b54caca.js} +18 -9
- flowfile/web/static/assets/{ExternalSource-d39af878.js → ExternalSource-3fa399b2.js} +9 -7
- flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-47ab05a3.css} +17 -17
- flowfile/web/static/assets/Filter-7494ea97.css +48 -0
- flowfile/web/static/assets/Filter-8cbbdbf3.js +287 -0
- flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
- flowfile/web/static/assets/{Formula-6b04fb1d.js → Formula-aac42b1e.js} +13 -11
- flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
- flowfile/web/static/assets/{FuzzyMatch-999521f4.js → FuzzyMatch-cd9bbfca.js} +12 -10
- flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-c24dec17.css} +5 -5
- flowfile/web/static/assets/{GraphSolver-17dd2198.js → GraphSolver-c7e6780e.js} +13 -11
- flowfile/web/static/assets/{GroupBy-6b039e18.js → GroupBy-93c5d22b.js} +9 -7
- flowfile/web/static/assets/{GroupBy-b9505323.css → GroupBy-be7ac0bf.css} +10 -10
- flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
- flowfile/web/static/assets/{Join-24d0f113.js → Join-a19b2de2.js} +13 -11
- flowfile/web/static/assets/LoginView-0df4ed0a.js +134 -0
- flowfile/web/static/assets/LoginView-d325d632.css +172 -0
- flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
- flowfile/web/static/assets/{ManualInput-34639209.js → ManualInput-8d3374b2.js} +170 -116
- flowfile/web/static/assets/{MultiSelect-0e8724a3.js → MultiSelect-ad1b6243.js} +2 -2
- flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-b0e538c2.js → MultiSelect.vue_vue_type_script_setup_true_lang-e278950d.js} +1 -1
- flowfile/web/static/assets/NodeDesigner-40b647c9.js +2610 -0
- flowfile/web/static/assets/NodeDesigner-5f53be3f.css +1429 -0
- flowfile/web/static/assets/{NumericInput-3d63a470.js → NumericInput-7100234c.js} +2 -2
- flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-e0edeccc.js → NumericInput.vue_vue_type_script_setup_true_lang-5130219f.js} +5 -2
- flowfile/web/static/assets/{Output-283fe388.css → Output-35e97000.css} +6 -6
- flowfile/web/static/assets/{Output-edea9802.js → Output-f5efd2aa.js} +12 -9
- flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
- flowfile/web/static/assets/{Pivot-61d19301.js → Pivot-d981d23c.js} +11 -9
- flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
- flowfile/web/static/assets/{PivotValidation-f97fec5b.js → PivotValidation-39386e95.js} +3 -3
- flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
- flowfile/web/static/assets/{PivotValidation-de9f43fe.js → PivotValidation-63de1f73.js} +3 -3
- flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
- flowfile/web/static/assets/{PolarsCode-bc3c9984.js → PolarsCode-f9d69217.js} +18 -9
- flowfile/web/static/assets/PopOver-b22f049e.js +939 -0
- flowfile/web/static/assets/PopOver-d96599db.css +33 -0
- flowfile/web/static/assets/{Read-e808b239.css → Read-36e7bd51.css} +12 -12
- flowfile/web/static/assets/{Read-64a3f259.js → Read-aec2e377.js} +14 -11
- flowfile/web/static/assets/{RecordCount-3d5039be.js → RecordCount-78ed6845.js} +6 -4
- flowfile/web/static/assets/{RecordId-597510e0.js → RecordId-2156e890.js} +8 -6
- flowfile/web/static/assets/{SQLQueryComponent-36cef432.css → SQLQueryComponent-1c2f26b4.css} +5 -5
- flowfile/web/static/assets/{SQLQueryComponent-df51adbe.js → SQLQueryComponent-48c72f5b.js} +3 -3
- flowfile/web/static/assets/{Sample-4be0a507.js → Sample-1352ca74.js} +6 -4
- flowfile/web/static/assets/SecretSelector-22b5ff89.js +113 -0
- flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
- flowfile/web/static/assets/{SecretManager-4839be57.js → SecretsView-17df66ee.js} +35 -36
- flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
- flowfile/web/static/assets/{Select-9b72f201.js → Select-0aee4c54.js} +9 -7
- flowfile/web/static/assets/{SettingsSection-f0f75a42.js → SettingsSection-0784e157.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-e1e9c953.js → SettingsSection-cd341bb6.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-7ded385d.js → SettingsSection-f2002a6d.js} +3 -3
- flowfile/web/static/assets/{SingleSelect-6c777aac.js → SingleSelect-460cc0ea.js} +2 -2
- flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-33e3ff9b.js → SingleSelect.vue_vue_type_script_setup_true_lang-30741bb2.js} +1 -1
- flowfile/web/static/assets/{SliderInput-7cb93e62.js → SliderInput-5d926864.js} +7 -4
- flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
- flowfile/web/static/assets/{Sort-6cbde21a.js → Sort-3cdc971b.js} +9 -7
- flowfile/web/static/assets/{Unique-f9fb0809.css → Sort-8a871341.css} +10 -10
- flowfile/web/static/assets/{TextInput-d9a40c11.js → TextInput-a2d0bfbd.js} +2 -2
- flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-5896c375.js → TextInput.vue_vue_type_script_setup_true_lang-abad1ca2.js} +5 -2
- flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
- flowfile/web/static/assets/{TextToRows-c4fcbf4d.js → TextToRows-918945f7.js} +11 -10
- flowfile/web/static/assets/{ToggleSwitch-4ef91d19.js → ToggleSwitch-f0ef5196.js} +2 -2
- flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-38478c20.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-5605c793.js} +1 -1
- flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-54d2f518.css} +6 -6
- flowfile/web/static/assets/{UnavailableFields-a03f512c.js → UnavailableFields-bdad6144.js} +4 -4
- flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
- flowfile/web/static/assets/{Union-bfe9b996.js → Union-e8ab8c86.js} +8 -6
- flowfile/web/static/assets/{Unique-5d023a27.js → Unique-8cd4f976.js} +13 -10
- flowfile/web/static/assets/{Sort-3643d625.css → Unique-9fb2f567.css} +10 -10
- flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-710a2948.css} +7 -7
- flowfile/web/static/assets/{Unpivot-91cc5354.js → Unpivot-8da14095.js} +10 -8
- flowfile/web/static/assets/{UnpivotValidation-7ee2de44.js → UnpivotValidation-6f7d89ff.js} +3 -3
- flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
- flowfile/web/static/assets/{VueGraphicWalker-e51b9924.js → VueGraphicWalker-3fb312e1.js} +4 -4
- flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
- flowfile/web/static/assets/{api-cf1221f0.js → api-24483f0d.js} +1 -1
- flowfile/web/static/assets/{api-c1bad5ca.js → api-8b81fa73.js} +1 -1
- flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-3d8dc5fa.css} +40 -40
- flowfile/web/static/assets/{dropDown-614b998d.js → dropDown-ac0fda9d.js} +3 -3
- flowfile/web/static/assets/{fullEditor-f7971590.js → fullEditor-5497a84a.js} +11 -10
- flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-a0be62b3.css} +74 -62
- flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
- flowfile/web/static/assets/{genericNodeSettings-4fe5f36b.js → genericNodeSettings-99014e1d.js} +5 -5
- flowfile/web/static/assets/index-07dda503.js +38 -0
- flowfile/web/static/assets/index-3ba44389.js +2696 -0
- flowfile/web/static/assets/{index-50508d4d.css → index-e6289dd0.css} +1945 -569
- flowfile/web/static/assets/{index-5429bbf8.js → index-fb6493ae.js} +41626 -40867
- flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
- flowfile/web/static/assets/nodeInput-0eb13f1a.js +2 -0
- flowfile/web/static/assets/{outputCsv-076b85ab.js → outputCsv-8f8ba42d.js} +3 -3
- flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
- flowfile/web/static/assets/{outputExcel-0fd17dbe.js → outputExcel-393f4fef.js} +3 -3
- flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
- flowfile/web/static/assets/{outputParquet-b61e0847.js → outputParquet-07c81f65.js} +4 -4
- flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
- flowfile/web/static/assets/{readCsv-a8bb8b61.js → readCsv-07f6d9ad.js} +3 -3
- flowfile/web/static/assets/{readCsv-c767cb37.css → readCsv-3bfac4c3.css} +15 -15
- flowfile/web/static/assets/{readExcel-806d2826.css → readExcel-3db6b763.css} +13 -13
- flowfile/web/static/assets/{readExcel-67b4aee0.js → readExcel-ed69bc8f.js} +5 -5
- flowfile/web/static/assets/{readParquet-48c81530.css → readParquet-c5244ad5.css} +4 -4
- flowfile/web/static/assets/{readParquet-92ce1dbc.js → readParquet-e3ed4528.js} +3 -3
- flowfile/web/static/assets/secrets.api-002e7d7e.js +65 -0
- flowfile/web/static/assets/{selectDynamic-92e25ee3.js → selectDynamic-80b92899.js} +5 -5
- flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
- flowfile/web/static/assets/{vue-codemirror.esm-41b0e0d7.js → vue-codemirror.esm-0965f39f.js} +31 -640
- flowfile/web/static/assets/{vue-content-loader.es-2c8e608f.js → vue-content-loader.es-c506ad97.js} +1 -1
- flowfile/web/static/index.html +2 -2
- {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/METADATA +2 -3
- flowfile-0.5.3.dist-info/RECORD +402 -0
- flowfile_core/__init__.py +13 -6
- flowfile_core/auth/jwt.py +51 -16
- flowfile_core/auth/models.py +32 -7
- flowfile_core/auth/password.py +89 -0
- flowfile_core/auth/secrets.py +8 -6
- flowfile_core/configs/__init__.py +9 -7
- flowfile_core/configs/flow_logger.py +15 -14
- flowfile_core/configs/node_store/__init__.py +72 -4
- flowfile_core/configs/node_store/nodes.py +155 -172
- flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
- flowfile_core/configs/settings.py +28 -15
- flowfile_core/database/connection.py +7 -6
- flowfile_core/database/init_db.py +96 -2
- flowfile_core/database/models.py +3 -1
- flowfile_core/fileExplorer/__init__.py +17 -0
- flowfile_core/fileExplorer/funcs.py +123 -57
- flowfile_core/fileExplorer/utils.py +10 -11
- flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
- flowfile_core/flowfile/analytics/analytics_processor.py +26 -24
- flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
- flowfile_core/flowfile/analytics/utils.py +1 -1
- flowfile_core/flowfile/code_generator/code_generator.py +358 -244
- flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
- flowfile_core/flowfile/connection_manager/models.py +1 -1
- flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
- flowfile_core/flowfile/database_connection_manager/models.py +1 -1
- flowfile_core/flowfile/extensions.py +17 -12
- flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
- flowfile_core/flowfile/flow_data_engine/create/funcs.py +115 -83
- flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +481 -423
- flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
- flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
- flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
- flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
- flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
- flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +31 -20
- flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
- flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +14 -15
- flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
- flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
- flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
- flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +190 -127
- flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
- flowfile_core/flowfile/flow_data_engine/utils.py +99 -67
- flowfile_core/flowfile/flow_graph.py +918 -571
- flowfile_core/flowfile/flow_graph_utils.py +31 -49
- flowfile_core/flowfile/flow_node/flow_node.py +330 -233
- flowfile_core/flowfile/flow_node/models.py +53 -41
- flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
- flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
- flowfile_core/flowfile/handler.py +80 -30
- flowfile_core/flowfile/manage/compatibility_enhancements.py +209 -126
- flowfile_core/flowfile/manage/io_flowfile.py +54 -57
- flowfile_core/flowfile/node_designer/__init__.py +15 -13
- flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
- flowfile_core/flowfile/node_designer/custom_node.py +162 -36
- flowfile_core/flowfile/node_designer/ui_components.py +135 -34
- flowfile_core/flowfile/schema_callbacks.py +71 -51
- flowfile_core/flowfile/setting_generator/__init__.py +0 -1
- flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
- flowfile_core/flowfile/setting_generator/settings.py +64 -53
- flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
- flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
- flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
- flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
- flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
- flowfile_core/flowfile/util/calculate_layout.py +9 -13
- flowfile_core/flowfile/util/execution_orderer.py +25 -17
- flowfile_core/flowfile/util/node_skipper.py +4 -4
- flowfile_core/flowfile/utils.py +19 -21
- flowfile_core/main.py +26 -19
- flowfile_core/routes/auth.py +284 -11
- flowfile_core/routes/cloud_connections.py +25 -25
- flowfile_core/routes/logs.py +21 -29
- flowfile_core/routes/public.py +3 -3
- flowfile_core/routes/routes.py +70 -34
- flowfile_core/routes/secrets.py +25 -27
- flowfile_core/routes/user_defined_components.py +483 -4
- flowfile_core/run_lock.py +0 -1
- flowfile_core/schemas/__init__.py +4 -6
- flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
- flowfile_core/schemas/cloud_storage_schemas.py +59 -53
- flowfile_core/schemas/input_schema.py +231 -144
- flowfile_core/schemas/output_model.py +49 -34
- flowfile_core/schemas/schemas.py +116 -89
- flowfile_core/schemas/transform_schema.py +518 -263
- flowfile_core/schemas/yaml_types.py +21 -7
- flowfile_core/secret_manager/secret_manager.py +17 -13
- flowfile_core/types.py +29 -9
- flowfile_core/utils/arrow_reader.py +7 -6
- flowfile_core/utils/excel_file_manager.py +3 -3
- flowfile_core/utils/fileManager.py +7 -7
- flowfile_core/utils/fl_executor.py +8 -10
- flowfile_core/utils/utils.py +4 -4
- flowfile_core/utils/validate_setup.py +5 -4
- flowfile_frame/__init__.py +106 -51
- flowfile_frame/adapters.py +2 -9
- flowfile_frame/adding_expr.py +73 -32
- flowfile_frame/cloud_storage/frame_helpers.py +27 -23
- flowfile_frame/cloud_storage/secret_manager.py +12 -26
- flowfile_frame/config.py +2 -5
- flowfile_frame/expr.py +311 -218
- flowfile_frame/expr.pyi +160 -159
- flowfile_frame/expr_name.py +23 -23
- flowfile_frame/flow_frame.py +571 -476
- flowfile_frame/flow_frame.pyi +123 -104
- flowfile_frame/flow_frame_methods.py +227 -246
- flowfile_frame/group_frame.py +50 -20
- flowfile_frame/join.py +2 -2
- flowfile_frame/lazy.py +129 -87
- flowfile_frame/lazy_methods.py +83 -30
- flowfile_frame/list_name_space.py +55 -50
- flowfile_frame/selectors.py +148 -68
- flowfile_frame/series.py +9 -7
- flowfile_frame/utils.py +19 -21
- flowfile_worker/__init__.py +12 -7
- flowfile_worker/configs.py +11 -19
- flowfile_worker/create/__init__.py +14 -9
- flowfile_worker/create/funcs.py +114 -77
- flowfile_worker/create/models.py +46 -43
- flowfile_worker/create/pl_types.py +14 -15
- flowfile_worker/create/read_excel_tables.py +34 -41
- flowfile_worker/create/utils.py +22 -19
- flowfile_worker/external_sources/s3_source/main.py +18 -51
- flowfile_worker/external_sources/s3_source/models.py +34 -27
- flowfile_worker/external_sources/sql_source/main.py +8 -5
- flowfile_worker/external_sources/sql_source/models.py +13 -9
- flowfile_worker/flow_logger.py +10 -8
- flowfile_worker/funcs.py +214 -155
- flowfile_worker/main.py +11 -17
- flowfile_worker/models.py +35 -28
- flowfile_worker/process_manager.py +2 -3
- flowfile_worker/routes.py +121 -90
- flowfile_worker/secrets.py +9 -6
- flowfile_worker/spawner.py +80 -49
- flowfile_worker/utils.py +3 -2
- shared/__init__.py +2 -7
- shared/storage_config.py +25 -13
- test_utils/postgres/commands.py +3 -2
- test_utils/postgres/fixtures.py +9 -9
- test_utils/s3/commands.py +1 -1
- test_utils/s3/data_generator.py +3 -4
- test_utils/s3/demo_data_generator.py +4 -7
- test_utils/s3/fixtures.py +7 -5
- tools/migrate/__init__.py +1 -1
- tools/migrate/__main__.py +16 -29
- tools/migrate/legacy_schemas.py +251 -190
- tools/migrate/migrate.py +193 -181
- tools/migrate/tests/conftest.py +1 -3
- tools/migrate/tests/test_migrate.py +36 -41
- tools/migrate/tests/test_migration_e2e.py +28 -29
- tools/migrate/tests/test_node_migrations.py +50 -20
- flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
- flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
- flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
- flowfile/web/static/assets/Filter-9b6d08db.js +0 -164
- flowfile/web/static/assets/Filter-f62091b3.css +0 -20
- flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
- flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
- flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
- flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
- flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
- flowfile/web/static/assets/nodeInput-5d0d6b79.js +0 -41
- flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
- flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
- flowfile/web/static/assets/secretApi-68435402.js +0 -46
- flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
- flowfile-0.5.1.dist-info/RECORD +0 -388
- {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/WHEEL +0 -0
- {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/entry_points.txt +0 -0
- {flowfile-0.5.1.dist-info → flowfile-0.5.3.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,51 +1,138 @@
|
|
|
1
1
|
# Comprehensive mapping from SQLAlchemy types to Polars types
|
|
2
|
-
from typing import
|
|
3
|
-
from
|
|
2
|
+
from typing import TYPE_CHECKING, Any, Union, cast
|
|
3
|
+
from urllib.parse import quote_plus
|
|
4
4
|
|
|
5
5
|
import polars as pl
|
|
6
6
|
from polars import DataType as PolarsType
|
|
7
|
+
from pydantic import SecretStr
|
|
7
8
|
from sqlalchemy.sql.sqltypes import (
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
9
|
+
ARRAY,
|
|
10
|
+
BIGINT,
|
|
11
|
+
BINARY,
|
|
12
|
+
BLOB,
|
|
13
|
+
BOOLEAN,
|
|
14
|
+
CHAR,
|
|
15
|
+
CLOB,
|
|
16
|
+
DATE,
|
|
17
|
+
DATETIME,
|
|
18
|
+
DECIMAL,
|
|
19
|
+
DOUBLE,
|
|
20
|
+
DOUBLE_PRECISION,
|
|
21
|
+
FLOAT,
|
|
22
|
+
INT,
|
|
23
|
+
INTEGER,
|
|
24
|
+
JSON,
|
|
25
|
+
NCHAR,
|
|
26
|
+
NULLTYPE,
|
|
27
|
+
NUMERIC,
|
|
28
|
+
NVARCHAR,
|
|
29
|
+
REAL,
|
|
30
|
+
SMALLINT,
|
|
31
|
+
STRINGTYPE,
|
|
32
|
+
TEXT,
|
|
33
|
+
TIME,
|
|
34
|
+
TIMESTAMP,
|
|
35
|
+
UUID,
|
|
36
|
+
VARBINARY,
|
|
37
|
+
VARCHAR,
|
|
38
|
+
BigInteger,
|
|
39
|
+
Boolean,
|
|
40
|
+
Concatenable,
|
|
41
|
+
Date,
|
|
42
|
+
DateTime,
|
|
43
|
+
Double,
|
|
44
|
+
Enum,
|
|
45
|
+
Float,
|
|
46
|
+
Indexable,
|
|
47
|
+
Integer,
|
|
48
|
+
Interval,
|
|
49
|
+
LargeBinary,
|
|
50
|
+
MatchType,
|
|
51
|
+
NullType,
|
|
52
|
+
Numeric,
|
|
53
|
+
PickleType,
|
|
54
|
+
SchemaType,
|
|
55
|
+
SmallInteger,
|
|
56
|
+
String,
|
|
57
|
+
Text,
|
|
58
|
+
Time,
|
|
59
|
+
TupleType,
|
|
60
|
+
Unicode,
|
|
61
|
+
UnicodeText,
|
|
62
|
+
Uuid,
|
|
63
|
+
_Binary,
|
|
16
64
|
)
|
|
17
|
-
from sqlalchemy.sql.type_api import
|
|
18
|
-
ExternalType, TypeDecorator,
|
|
19
|
-
TypeEngine, UserDefinedType, Variant
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
from typing import Optional
|
|
24
|
-
from urllib.parse import quote_plus
|
|
25
|
-
|
|
65
|
+
from sqlalchemy.sql.type_api import ExternalType, TypeDecorator, TypeEngine, UserDefinedType, Variant
|
|
26
66
|
|
|
27
67
|
if TYPE_CHECKING:
|
|
28
68
|
SqlType = Union[
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
69
|
+
type[_Binary],
|
|
70
|
+
type[ARRAY],
|
|
71
|
+
type[BIGINT],
|
|
72
|
+
type[BigInteger],
|
|
73
|
+
type[BINARY],
|
|
74
|
+
type[BLOB],
|
|
75
|
+
type[BOOLEAN],
|
|
76
|
+
type[Boolean],
|
|
77
|
+
type[CHAR],
|
|
78
|
+
type[CLOB],
|
|
79
|
+
type[Concatenable],
|
|
80
|
+
type[DATE],
|
|
81
|
+
type[Date],
|
|
82
|
+
type[DATETIME],
|
|
83
|
+
type[DateTime],
|
|
84
|
+
type[DECIMAL],
|
|
85
|
+
type[DOUBLE],
|
|
86
|
+
type[Double],
|
|
87
|
+
type[DOUBLE_PRECISION],
|
|
88
|
+
type[Enum],
|
|
89
|
+
type[FLOAT],
|
|
90
|
+
type[Float],
|
|
91
|
+
type[Indexable],
|
|
92
|
+
type[INT],
|
|
93
|
+
type[INTEGER],
|
|
94
|
+
type[Integer],
|
|
95
|
+
type[Interval],
|
|
96
|
+
type[JSON],
|
|
97
|
+
type[LargeBinary],
|
|
98
|
+
type[MatchType],
|
|
99
|
+
type[NCHAR],
|
|
100
|
+
type[NULLTYPE],
|
|
101
|
+
type[NullType],
|
|
102
|
+
type[NUMERIC],
|
|
103
|
+
type[Numeric],
|
|
104
|
+
type[NVARCHAR],
|
|
105
|
+
type[PickleType],
|
|
106
|
+
type[REAL],
|
|
107
|
+
type[SchemaType],
|
|
108
|
+
type[SMALLINT],
|
|
109
|
+
type[SmallInteger],
|
|
110
|
+
type[String],
|
|
111
|
+
type[STRINGTYPE],
|
|
112
|
+
type[TEXT],
|
|
113
|
+
type[Text],
|
|
114
|
+
type[TIME],
|
|
115
|
+
type[Time],
|
|
116
|
+
type[TIMESTAMP],
|
|
117
|
+
type[TupleType],
|
|
118
|
+
type[Unicode],
|
|
119
|
+
type[UnicodeText],
|
|
120
|
+
type[UUID],
|
|
121
|
+
type[Uuid],
|
|
122
|
+
type[VARBINARY],
|
|
123
|
+
type[VARCHAR],
|
|
124
|
+
type[TypeDecorator],
|
|
125
|
+
type[TypeEngine],
|
|
126
|
+
type[UserDefinedType],
|
|
127
|
+
type[Variant],
|
|
128
|
+
type[ExternalType],
|
|
42
129
|
]
|
|
43
130
|
else:
|
|
44
131
|
SqlType = Any
|
|
45
132
|
|
|
46
133
|
|
|
47
134
|
# Comprehensive mapping from SQLAlchemy types to Polars types
|
|
48
|
-
sqlalchemy_to_polars:
|
|
135
|
+
sqlalchemy_to_polars: dict[SqlType, PolarsType] = {
|
|
49
136
|
# Numeric types
|
|
50
137
|
Integer: pl.Int64,
|
|
51
138
|
INTEGER: pl.Int64,
|
|
@@ -65,7 +152,6 @@ sqlalchemy_to_polars: Dict[SqlType, PolarsType] = {
|
|
|
65
152
|
DECIMAL: pl.Decimal,
|
|
66
153
|
Boolean: pl.Boolean,
|
|
67
154
|
BOOLEAN: pl.Boolean,
|
|
68
|
-
|
|
69
155
|
# String types
|
|
70
156
|
String: pl.Utf8,
|
|
71
157
|
VARCHAR: pl.Utf8,
|
|
@@ -78,7 +164,6 @@ sqlalchemy_to_polars: Dict[SqlType, PolarsType] = {
|
|
|
78
164
|
STRINGTYPE: pl.Utf8,
|
|
79
165
|
Unicode: pl.Utf8,
|
|
80
166
|
UnicodeText: pl.Utf8,
|
|
81
|
-
|
|
82
167
|
# Date/Time types
|
|
83
168
|
Date: pl.Date,
|
|
84
169
|
DATE: pl.Date,
|
|
@@ -88,27 +173,22 @@ sqlalchemy_to_polars: Dict[SqlType, PolarsType] = {
|
|
|
88
173
|
Time: pl.Time,
|
|
89
174
|
TIME: pl.Time,
|
|
90
175
|
Interval: pl.Duration,
|
|
91
|
-
|
|
92
176
|
# Binary types
|
|
93
177
|
_Binary: pl.Binary,
|
|
94
178
|
LargeBinary: pl.Binary,
|
|
95
179
|
BINARY: pl.Binary,
|
|
96
180
|
VARBINARY: pl.Binary,
|
|
97
181
|
BLOB: pl.Binary,
|
|
98
|
-
|
|
99
182
|
# JSON types
|
|
100
183
|
JSON: pl.Utf8, # Polars doesn't have a dedicated JSON type, using Utf8
|
|
101
|
-
|
|
102
184
|
# UUID types
|
|
103
185
|
UUID: pl.Utf8, # Mapped to string
|
|
104
186
|
Uuid: pl.Utf8, # Mapped to string
|
|
105
|
-
|
|
106
187
|
# Other types
|
|
107
188
|
ARRAY: pl.List, # Approx mapping
|
|
108
189
|
Enum: pl.String, # Approx mapping
|
|
109
190
|
PickleType: pl.Object, # For storing Python objects
|
|
110
191
|
TupleType: pl.Struct, # Mapped to struct
|
|
111
|
-
|
|
112
192
|
# Special/Abstract types
|
|
113
193
|
NULLTYPE: None,
|
|
114
194
|
NullType: None,
|
|
@@ -124,165 +204,139 @@ sqlalchemy_to_polars: Dict[SqlType, PolarsType] = {
|
|
|
124
204
|
}
|
|
125
205
|
|
|
126
206
|
# Create string mappings, filtering out None values
|
|
127
|
-
sqlalchemy_to_polars_str:
|
|
207
|
+
sqlalchemy_to_polars_str: dict[str, str] = {
|
|
128
208
|
k.__name__: v.__name__
|
|
129
209
|
for k, v in sqlalchemy_to_polars.items()
|
|
130
|
-
if v is not None and hasattr(k,
|
|
210
|
+
if v is not None and hasattr(k, "__name__") and hasattr(v, "__name__")
|
|
131
211
|
}
|
|
132
212
|
|
|
133
213
|
# Additional string mappings for common SQL type names
|
|
134
|
-
sql_type_name_to_polars:
|
|
135
|
-
#
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
'
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
'datetimeoffset': pl.Datetime,
|
|
252
|
-
'char': pl.Utf8,
|
|
253
|
-
'varchar': pl.Utf8,
|
|
254
|
-
'text': pl.Utf8,
|
|
255
|
-
'nchar': pl.Utf8,
|
|
256
|
-
'nvarchar': pl.Utf8,
|
|
257
|
-
'ntext': pl.Utf8,
|
|
258
|
-
'binary': pl.Binary,
|
|
259
|
-
'varbinary': pl.Binary,
|
|
260
|
-
'image': pl.Binary,
|
|
261
|
-
'uniqueidentifier': pl.Utf8,
|
|
262
|
-
'xml': pl.Utf8,
|
|
263
|
-
'sql_variant': pl.Object,
|
|
264
|
-
'hierarchyid': pl.Utf8,
|
|
265
|
-
'geometry': pl.Utf8,
|
|
266
|
-
'geography': pl.Utf8,
|
|
267
|
-
|
|
268
|
-
# Common abbreviations and aliases
|
|
269
|
-
'int4': pl.Int32,
|
|
270
|
-
'int8': pl.Int64,
|
|
271
|
-
'float4': pl.Float32,
|
|
272
|
-
'float8': pl.Float64,
|
|
273
|
-
'bool': pl.Boolean,
|
|
274
|
-
'serial': pl.Int32, # PostgreSQL auto-incrementing integer
|
|
275
|
-
'bigserial': pl.Int64, # PostgreSQL auto-incrementing bigint
|
|
276
|
-
'smallserial': pl.Int16, # PostgreSQL auto-incrementing smallint
|
|
214
|
+
sql_type_name_to_polars: dict[str, PolarsType] = {
|
|
215
|
+
# --- Integers ---
|
|
216
|
+
"int": pl.Int32,
|
|
217
|
+
"integer": pl.Int64,
|
|
218
|
+
"int4": pl.Int32,
|
|
219
|
+
"int8": pl.Int64,
|
|
220
|
+
"bigint": pl.Int64,
|
|
221
|
+
"short": pl.Int16,
|
|
222
|
+
"smallint": pl.Int16,
|
|
223
|
+
"tinyint": pl.Int8,
|
|
224
|
+
"mediumint": pl.Int32,
|
|
225
|
+
"serial": pl.Int32,
|
|
226
|
+
"bigserial": pl.Int64,
|
|
227
|
+
"smallserial": pl.Int16,
|
|
228
|
+
# Unsigned (MySQL specific)
|
|
229
|
+
"int unsigned": pl.UInt64,
|
|
230
|
+
"bigint unsigned": pl.UInt64,
|
|
231
|
+
"smallint unsigned": pl.UInt16,
|
|
232
|
+
"tinyint unsigned": pl.UInt8,
|
|
233
|
+
"mediumint unsigned": pl.UInt32,
|
|
234
|
+
"year": pl.Int16,
|
|
235
|
+
|
|
236
|
+
# --- Floats & Decimals ---
|
|
237
|
+
"numeric": pl.Decimal,
|
|
238
|
+
"decimal": pl.Decimal,
|
|
239
|
+
"number": pl.Decimal, # Oracle
|
|
240
|
+
"money": pl.Decimal,
|
|
241
|
+
"smallmoney": pl.Decimal,
|
|
242
|
+
"real": pl.Float32,
|
|
243
|
+
"float": pl.Float64,
|
|
244
|
+
"float4": pl.Float32,
|
|
245
|
+
"float8": pl.Float64,
|
|
246
|
+
"double": pl.Float64,
|
|
247
|
+
"double precision": pl.Float64,
|
|
248
|
+
"binary_float": pl.Float32, # Oracle
|
|
249
|
+
"binary_double": pl.Float64, # Oracle
|
|
250
|
+
|
|
251
|
+
# --- Booleans ---
|
|
252
|
+
"boolean": pl.Boolean,
|
|
253
|
+
"bool": pl.Boolean,
|
|
254
|
+
"bit": pl.Boolean, # Note: PostgreSQL 'bit' is varying, but MSSQL/MySQL 'bit' is boolean. Defaulting to Bool.
|
|
255
|
+
|
|
256
|
+
# --- Strings / Text ---
|
|
257
|
+
"varchar": pl.Utf8,
|
|
258
|
+
"varchar2": pl.Utf8, # Oracle
|
|
259
|
+
"nvarchar": pl.Utf8,
|
|
260
|
+
"nvarchar2": pl.Utf8, # Oracle
|
|
261
|
+
"char": pl.Utf8,
|
|
262
|
+
"nchar": pl.Utf8,
|
|
263
|
+
"character": pl.Utf8,
|
|
264
|
+
"character varying": pl.Utf8,
|
|
265
|
+
"text": pl.Utf8,
|
|
266
|
+
"tinytext": pl.Utf8,
|
|
267
|
+
"mediumtext": pl.Utf8,
|
|
268
|
+
"longtext": pl.Utf8,
|
|
269
|
+
"ntext": pl.Utf8,
|
|
270
|
+
"clob": pl.Utf8,
|
|
271
|
+
"nclob": pl.Utf8,
|
|
272
|
+
"long": pl.Utf8, # Oracle
|
|
273
|
+
"enum": pl.String,
|
|
274
|
+
"set": pl.List,
|
|
275
|
+
"rowid": pl.Utf8, # Oracle
|
|
276
|
+
"urowid": pl.Utf8, # Oracle
|
|
277
|
+
"uniqueidentifier": pl.Utf8, # MSSQL
|
|
278
|
+
"xml": pl.Utf8,
|
|
279
|
+
"xmltype": pl.Utf8,
|
|
280
|
+
"json": pl.Utf8,
|
|
281
|
+
"jsonb": pl.Utf8,
|
|
282
|
+
|
|
283
|
+
# --- Network / Specialized Strings (Postgres) ---
|
|
284
|
+
"uuid": pl.Utf8,
|
|
285
|
+
"cidr": pl.Utf8,
|
|
286
|
+
"inet": pl.Utf8,
|
|
287
|
+
"macaddr": pl.Utf8,
|
|
288
|
+
"tsquery": pl.Utf8,
|
|
289
|
+
"tsvector": pl.Utf8,
|
|
290
|
+
"hstore": pl.Utf8,
|
|
291
|
+
"geometry": pl.Utf8,
|
|
292
|
+
"geography": pl.Utf8,
|
|
293
|
+
"hierarchyid": pl.Utf8,
|
|
294
|
+
"bit varying": pl.Utf8,
|
|
295
|
+
|
|
296
|
+
# --- Dates & Times ---
|
|
297
|
+
"date": pl.Date,
|
|
298
|
+
"datetime": pl.Datetime,
|
|
299
|
+
"datetime2": pl.Datetime, # MSSQL
|
|
300
|
+
"smalldatetime": pl.Datetime, # MSSQL
|
|
301
|
+
"timestamp": pl.Datetime,
|
|
302
|
+
"timestamp without time zone": pl.Datetime,
|
|
303
|
+
"timestamp with time zone": pl.Datetime,
|
|
304
|
+
"timestamp with local time zone": pl.Datetime,
|
|
305
|
+
"datetimeoffset": pl.Datetime, # MSSQL
|
|
306
|
+
"time": pl.Time,
|
|
307
|
+
"time without time zone": pl.Time,
|
|
308
|
+
"time with time zone": pl.Time,
|
|
309
|
+
|
|
310
|
+
# --- Durations / Intervals ---
|
|
311
|
+
"interval": pl.Duration,
|
|
312
|
+
"interval year to month": pl.Duration, # Oracle
|
|
313
|
+
"interval day to second": pl.Duration, # Oracle
|
|
314
|
+
|
|
315
|
+
# --- Binary ---
|
|
316
|
+
"bytea": pl.Binary, # Postgres
|
|
317
|
+
"binary": pl.Binary,
|
|
318
|
+
"varbinary": pl.Binary,
|
|
319
|
+
"blob": pl.Binary,
|
|
320
|
+
"tinyblob": pl.Binary,
|
|
321
|
+
"mediumblob": pl.Binary,
|
|
322
|
+
"longblob": pl.Binary,
|
|
323
|
+
"raw": pl.Binary, # Oracle
|
|
324
|
+
"long raw": pl.Binary, # Oracle
|
|
325
|
+
"bfile": pl.Binary, # Oracle
|
|
326
|
+
"image": pl.Binary, # MSSQL
|
|
327
|
+
|
|
328
|
+
# --- Other ---
|
|
329
|
+
"null": None,
|
|
330
|
+
"sql_variant": pl.Object,
|
|
277
331
|
}
|
|
278
332
|
|
|
279
333
|
# String to string mapping
|
|
280
|
-
sql_type_name_to_polars_str:
|
|
334
|
+
sql_type_name_to_polars_str: dict[str, str] = {
|
|
281
335
|
k: v.__name__ for k, v in sql_type_name_to_polars.items() if v is not None
|
|
282
336
|
}
|
|
283
337
|
|
|
284
338
|
|
|
285
|
-
def get_polars_type(sqlalchemy_type:
|
|
339
|
+
def get_polars_type(sqlalchemy_type: SqlType | str):
|
|
286
340
|
"""
|
|
287
341
|
Get the corresponding Polars type from a SQLAlchemy type or string type name.
|
|
288
342
|
|
|
@@ -309,14 +363,14 @@ def get_polars_type(sqlalchemy_type: Union[SqlType, str]):
|
|
|
309
363
|
|
|
310
364
|
|
|
311
365
|
def construct_sql_uri(
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
366
|
+
database_type: str = "postgresql",
|
|
367
|
+
host: str | None = None,
|
|
368
|
+
port: int | None = None,
|
|
369
|
+
username: str | None = None,
|
|
370
|
+
password: SecretStr | None = None,
|
|
371
|
+
database: str | None = None,
|
|
372
|
+
url: str | None = None,
|
|
373
|
+
**kwargs,
|
|
320
374
|
) -> str:
|
|
321
375
|
"""
|
|
322
376
|
Constructs a SQL URI string from the provided parameters.
|
|
@@ -376,4 +430,4 @@ def construct_sql_uri(
|
|
|
376
430
|
params = "&".join(f"{key}={quote_plus(str(value))}" for key, value in kwargs.items())
|
|
377
431
|
base_uri += f"?{params}"
|
|
378
432
|
|
|
379
|
-
return base_uri
|
|
433
|
+
return base_uri
|
|
@@ -1,18 +1,14 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
import math
|
|
3
2
|
from collections import defaultdict, deque
|
|
4
|
-
from typing import
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
5
4
|
|
|
6
5
|
if TYPE_CHECKING:
|
|
7
6
|
from flowfile_core.flowfile.flow_graph import FlowGraph
|
|
8
7
|
|
|
9
8
|
|
|
10
9
|
def calculate_layered_layout(
|
|
11
|
-
graph:
|
|
12
|
-
|
|
13
|
-
y_spacing: int = 100,
|
|
14
|
-
initial_y: int = 50
|
|
15
|
-
) -> Dict[int, Tuple[int, int]]:
|
|
10
|
+
graph: "FlowGraph", x_spacing: int = 250, y_spacing: int = 100, initial_y: int = 50
|
|
11
|
+
) -> dict[int, tuple[int, int]]:
|
|
16
12
|
"""
|
|
17
13
|
Calculates node positions using a simplified layered approach for a
|
|
18
14
|
LEFT-TO-RIGHT flow. Stages determine horizontal position (X), and
|
|
@@ -32,9 +28,9 @@ def calculate_layered_layout(
|
|
|
32
28
|
|
|
33
29
|
nodes = graph.nodes
|
|
34
30
|
node_ids = {node.node_id for node in nodes}
|
|
35
|
-
adj:
|
|
36
|
-
rev_adj:
|
|
37
|
-
in_degree:
|
|
31
|
+
adj: dict[int, list[int]] = defaultdict(list)
|
|
32
|
+
rev_adj: dict[int, list[int]] = defaultdict(list)
|
|
33
|
+
in_degree: dict[int, int] = defaultdict(int)
|
|
38
34
|
|
|
39
35
|
# --- Graph Building Stage ---
|
|
40
36
|
try:
|
|
@@ -65,8 +61,8 @@ def calculate_layered_layout(
|
|
|
65
61
|
adj[node.node_id].append(child_node.node_id)
|
|
66
62
|
in_degree[child_node.node_id] += 1
|
|
67
63
|
|
|
68
|
-
stages:
|
|
69
|
-
node_stage:
|
|
64
|
+
stages: dict[int, list[int]] = defaultdict(list)
|
|
65
|
+
node_stage: dict[int, int] = {}
|
|
70
66
|
initial_sources = sorted([node_id for node_id in node_ids if in_degree.get(node_id, 0) == 0])
|
|
71
67
|
queue = deque(initial_sources)
|
|
72
68
|
current_stage = 0
|
|
@@ -105,7 +101,7 @@ def calculate_layered_layout(
|
|
|
105
101
|
current_stage += 1
|
|
106
102
|
|
|
107
103
|
# --- Coordinate Assignment Stage ---
|
|
108
|
-
positions:
|
|
104
|
+
positions: dict[int, tuple[int, int]] = {}
|
|
109
105
|
max_stage_height = 0
|
|
110
106
|
|
|
111
107
|
for stage_index, node_ids_in_stage in stages.items():
|
|
@@ -1,19 +1,20 @@
|
|
|
1
|
-
from
|
|
2
|
-
|
|
1
|
+
from collections import defaultdict, deque
|
|
2
|
+
|
|
3
3
|
from flowfile_core.configs import logger
|
|
4
|
-
from
|
|
4
|
+
from flowfile_core.flowfile.flow_node.flow_node import FlowNode
|
|
5
5
|
from flowfile_core.flowfile.util.node_skipper import determine_nodes_to_skip
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
|
|
8
|
+
def compute_execution_plan(nodes: list[FlowNode], flow_starts: list[FlowNode] = None):
|
|
9
|
+
"""Computes the execution order after finding the nodes to skip on the execution step."""
|
|
9
10
|
skip_nodes = determine_nodes_to_skip(nodes=nodes)
|
|
10
|
-
computed_execution_order = determine_execution_order(
|
|
11
|
-
|
|
11
|
+
computed_execution_order = determine_execution_order(
|
|
12
|
+
all_nodes=[node for node in nodes if node not in skip_nodes], flow_starts=flow_starts
|
|
13
|
+
)
|
|
12
14
|
return skip_nodes, computed_execution_order
|
|
13
15
|
|
|
14
16
|
|
|
15
|
-
|
|
16
|
-
def determine_execution_order(all_nodes: List[FlowNode], flow_starts: List[FlowNode] = None) -> List[FlowNode]:
|
|
17
|
+
def determine_execution_order(all_nodes: list[FlowNode], flow_starts: list[FlowNode] = None) -> list[FlowNode]:
|
|
17
18
|
"""
|
|
18
19
|
Determines the execution order of nodes using topological sorting based on node dependencies.
|
|
19
20
|
|
|
@@ -41,7 +42,7 @@ def determine_execution_order(all_nodes: List[FlowNode], flow_starts: List[FlowN
|
|
|
41
42
|
return execution_order
|
|
42
43
|
|
|
43
44
|
|
|
44
|
-
def build_node_map(all_nodes:
|
|
45
|
+
def build_node_map(all_nodes: list[FlowNode]) -> dict[str, FlowNode]:
|
|
45
46
|
"""
|
|
46
47
|
Creates a mapping from node ID to node object.
|
|
47
48
|
|
|
@@ -54,8 +55,9 @@ def build_node_map(all_nodes: List[FlowNode]) -> Dict[str, FlowNode]:
|
|
|
54
55
|
return {node.node_id: node for node in all_nodes}
|
|
55
56
|
|
|
56
57
|
|
|
57
|
-
def compute_in_degrees_and_adjacency_list(
|
|
58
|
-
|
|
58
|
+
def compute_in_degrees_and_adjacency_list(
|
|
59
|
+
all_nodes: list[FlowNode], node_map: dict[str, FlowNode]
|
|
60
|
+
) -> (dict[str, int], dict[str, list[str]]):
|
|
59
61
|
"""
|
|
60
62
|
Computes the in-degree and adjacency list for all nodes.
|
|
61
63
|
|
|
@@ -81,8 +83,9 @@ def compute_in_degrees_and_adjacency_list(all_nodes: List[FlowNode],
|
|
|
81
83
|
return in_degree, adjacency_list
|
|
82
84
|
|
|
83
85
|
|
|
84
|
-
def initialize_queue(
|
|
85
|
-
|
|
86
|
+
def initialize_queue(
|
|
87
|
+
flow_starts: list[FlowNode], all_nodes: list[FlowNode], in_degree: dict[str, int]
|
|
88
|
+
) -> (deque, set[str]):
|
|
86
89
|
"""
|
|
87
90
|
Initializes the queue with nodes that have zero in-degree or based on specified flow start nodes.
|
|
88
91
|
|
|
@@ -115,8 +118,13 @@ deque, Set[str]):
|
|
|
115
118
|
return queue, visited_nodes
|
|
116
119
|
|
|
117
120
|
|
|
118
|
-
def perform_topological_sort(
|
|
119
|
-
|
|
121
|
+
def perform_topological_sort(
|
|
122
|
+
queue: deque,
|
|
123
|
+
node_map: dict[str, FlowNode],
|
|
124
|
+
in_degree: dict[str, int],
|
|
125
|
+
adjacency_list: dict[str, list[str]],
|
|
126
|
+
visited_nodes: set[str],
|
|
127
|
+
) -> list[FlowNode]:
|
|
120
128
|
"""
|
|
121
129
|
Performs topological sorting to determine the execution order of nodes.
|
|
122
130
|
|
|
@@ -131,7 +139,7 @@ def perform_topological_sort(queue: deque, node_map: Dict[str, FlowNode], in_deg
|
|
|
131
139
|
List[FlowNode]: A list of nodes in the order they should be executed.
|
|
132
140
|
"""
|
|
133
141
|
execution_order = []
|
|
134
|
-
logger.info(
|
|
142
|
+
logger.info("Starting topological sort to determine execution order")
|
|
135
143
|
|
|
136
144
|
while queue:
|
|
137
145
|
current_node_id = queue.popleft()
|