Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- build_backends/main.py +25 -22
- build_backends/main_prd.py +10 -19
- flowfile/__init__.py +194 -74
- flowfile/__main__.py +10 -7
- flowfile/api.py +51 -57
- flowfile/web/__init__.py +14 -9
- flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
- flowfile/web/static/assets/AdminView-f9847d67.js +713 -0
- flowfile/web/static/assets/CloudConnectionView-cf85f943.css +72 -0
- flowfile/web/static/assets/{CloudConnectionManager-0dfba9f2.js → CloudConnectionView-faace55b.js} +11 -11
- flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
- flowfile/web/static/assets/{CloudStorageReader-d5b1b6c9.js → CloudStorageReader-d86ecaa7.js} +10 -8
- flowfile/web/static/assets/{CloudStorageWriter-00d87aad.js → CloudStorageWriter-0f4d9a44.js} +10 -8
- flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
- flowfile/web/static/assets/ColumnActionInput-c44b7aee.css +159 -0
- flowfile/web/static/assets/ColumnActionInput-f4189ae0.js +330 -0
- flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
- flowfile/web/static/assets/{ColumnSelector-4685e75d.js → ColumnSelector-e66b33da.js} +3 -5
- flowfile/web/static/assets/ContextMenu-49463352.js +9 -0
- flowfile/web/static/assets/ContextMenu-dd5f3f25.js +9 -0
- flowfile/web/static/assets/ContextMenu-f709b884.js +9 -0
- flowfile/web/static/assets/ContextMenu.vue_vue_type_script_setup_true_lang-a1bd6314.js +59 -0
- flowfile/web/static/assets/{CrossJoin-702a3edd.js → CrossJoin-24694b8f.js} +12 -10
- flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
- flowfile/web/static/assets/{CustomNode-b1519993.js → CustomNode-569d45ff.js} +43 -24
- flowfile/web/static/assets/CustomNode-edb9b939.css +42 -0
- flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-c20a1e16.css} +23 -21
- flowfile/web/static/assets/{DatabaseConnectionSettings-6f3e4ea5.js → DatabaseConnectionSettings-cfc08938.js} +5 -4
- flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-5bf8c75b.css} +41 -46
- flowfile/web/static/assets/{DatabaseReader-d38c7295.js → DatabaseReader-701feabb.js} +25 -15
- flowfile/web/static/assets/{DatabaseManager-cf5ef661.js → DatabaseView-0482e5b5.js} +11 -11
- flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
- flowfile/web/static/assets/{DatabaseWriter-b04ef46a.js → DatabaseWriter-16721989.js} +17 -10
- flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-bdcf2c8b.css} +29 -27
- flowfile/web/static/assets/{designer-8da3ba3a.css → DesignerView-49abb835.css} +783 -663
- flowfile/web/static/assets/{designer-9633482a.js → DesignerView-f64749fb.js} +1292 -3253
- flowfile/web/static/assets/{documentation-ca400224.js → DocumentationView-61bd2990.js} +5 -5
- flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-9ea6e871.css} +9 -9
- flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
- flowfile/web/static/assets/{ExploreData-5fa10ed8.js → ExploreData-e2735b13.js} +18 -9
- flowfile/web/static/assets/{ExternalSource-d39af878.js → ExternalSource-2535c3b2.js} +9 -7
- flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-7ac7373f.css} +20 -20
- flowfile/web/static/assets/Filter-2cdbc93c.js +287 -0
- flowfile/web/static/assets/Filter-7494ea97.css +48 -0
- flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
- flowfile/web/static/assets/{Formula-6b04fb1d.js → Formula-fcda3c2c.js} +13 -11
- flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
- flowfile/web/static/assets/{FuzzyMatch-999521f4.js → FuzzyMatch-f8d3b7d3.js} +12 -10
- flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-4b4d7db9.css} +5 -5
- flowfile/web/static/assets/{GraphSolver-17dd2198.js → GraphSolver-72eaa695.js} +14 -12
- flowfile/web/static/assets/GroupBy-5792782d.css +9 -0
- flowfile/web/static/assets/{GroupBy-6b039e18.js → GroupBy-8aa0598b.js} +9 -7
- flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
- flowfile/web/static/assets/{Join-24d0f113.js → Join-e40f0ffa.js} +13 -11
- flowfile/web/static/assets/LoginView-5111c9ae.js +134 -0
- flowfile/web/static/assets/LoginView-d325d632.css +172 -0
- flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
- flowfile/web/static/assets/{ManualInput-34639209.js → ManualInput-9b6f3224.js} +170 -116
- flowfile/web/static/assets/{MultiSelect-0e8724a3.js → MultiSelect-ef28e19e.js} +2 -2
- flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-b0e538c2.js → MultiSelect.vue_vue_type_script_setup_true_lang-83b3bbfd.js} +1 -1
- flowfile/web/static/assets/NodeDesigner-94cd4dd3.css +1429 -0
- flowfile/web/static/assets/NodeDesigner-d2b7ee2b.js +2712 -0
- flowfile/web/static/assets/{NumericInput-3d63a470.js → NumericInput-1d789794.js} +2 -2
- flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-e0edeccc.js → NumericInput.vue_vue_type_script_setup_true_lang-7775f83e.js} +5 -2
- flowfile/web/static/assets/Output-692dd25d.css +37 -0
- flowfile/web/static/assets/{Output-edea9802.js → Output-cefef801.js} +14 -10
- flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
- flowfile/web/static/assets/{Pivot-61d19301.js → Pivot-bab1b75b.js} +12 -10
- flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
- flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
- flowfile/web/static/assets/{PivotValidation-f97fec5b.js → PivotValidation-e7941f91.js} +3 -3
- flowfile/web/static/assets/{PivotValidation-de9f43fe.js → PivotValidation-fba09336.js} +3 -3
- flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
- flowfile/web/static/assets/{PolarsCode-bc3c9984.js → PolarsCode-740e40fa.js} +18 -9
- flowfile/web/static/assets/PopOver-862d7e28.js +939 -0
- flowfile/web/static/assets/PopOver-d96599db.css +33 -0
- flowfile/web/static/assets/{Read-64a3f259.js → Read-225cc63f.js} +16 -12
- flowfile/web/static/assets/{Read-e808b239.css → Read-90f366bc.css} +15 -15
- flowfile/web/static/assets/{RecordCount-3d5039be.js → RecordCount-ffc71eca.js} +6 -4
- flowfile/web/static/assets/{RecordId-597510e0.js → RecordId-a70bb8df.js} +9 -7
- flowfile/web/static/assets/{SQLQueryComponent-df51adbe.js → SQLQueryComponent-15a421f5.js} +3 -3
- flowfile/web/static/assets/SQLQueryComponent-edb90b98.css +29 -0
- flowfile/web/static/assets/{Sample-4be0a507.js → Sample-6c26afc7.js} +6 -4
- flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
- flowfile/web/static/assets/SecretSelector-ceed9496.js +113 -0
- flowfile/web/static/assets/{SecretManager-4839be57.js → SecretsView-214d255a.js} +35 -36
- flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
- flowfile/web/static/assets/{Select-9b72f201.js → Select-8fc29999.js} +9 -7
- flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-7ded385d.js → SettingsSection-3f70e4c3.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-f0f75a42.js → SettingsSection-83090218.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-e1e9c953.js → SettingsSection-9f0d1725.js} +3 -3
- flowfile/web/static/assets/SetupView-3fa0aa03.js +160 -0
- flowfile/web/static/assets/SetupView-e2da3442.css +230 -0
- flowfile/web/static/assets/{SingleSelect-6c777aac.js → SingleSelect-a4a568cb.js} +2 -2
- flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-33e3ff9b.js → SingleSelect.vue_vue_type_script_setup_true_lang-c8ebdd33.js} +1 -1
- flowfile/web/static/assets/{SliderInput-7cb93e62.js → SliderInput-be533e71.js} +7 -4
- flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
- flowfile/web/static/assets/{Sort-6cbde21a.js → Sort-154dad81.js} +9 -7
- flowfile/web/static/assets/Sort-4abb7fae.css +9 -0
- flowfile/web/static/assets/{TextInput-d9a40c11.js → TextInput-454e2bda.js} +2 -2
- flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-5896c375.js → TextInput.vue_vue_type_script_setup_true_lang-e86510d0.js} +5 -2
- flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
- flowfile/web/static/assets/{TextToRows-c4fcbf4d.js → TextToRows-ea73433d.js} +11 -10
- flowfile/web/static/assets/{ToggleSwitch-4ef91d19.js → ToggleSwitch-9d7b30f1.js} +2 -2
- flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-38478c20.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-00f2580e.js} +1 -1
- flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-394a1f78.css} +14 -14
- flowfile/web/static/assets/{UnavailableFields-a03f512c.js → UnavailableFields-b72a2c72.js} +4 -4
- flowfile/web/static/assets/{Union-bfe9b996.js → Union-1e44f263.js} +8 -6
- flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
- flowfile/web/static/assets/Unique-2b705521.css +3 -0
- flowfile/web/static/assets/{Unique-5d023a27.js → Unique-a3bc6d0a.js} +13 -10
- flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-b6ad6427.css} +7 -7
- flowfile/web/static/assets/{Unpivot-91cc5354.js → Unpivot-e27935fc.js} +11 -9
- flowfile/web/static/assets/{UnpivotValidation-7ee2de44.js → UnpivotValidation-72497680.js} +3 -3
- flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
- flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
- flowfile/web/static/assets/{VueGraphicWalker-e51b9924.js → VueGraphicWalker-d9ab70a3.js} +4 -4
- flowfile/web/static/assets/{api-cf1221f0.js → api-a2102880.js} +1 -1
- flowfile/web/static/assets/{api-c1bad5ca.js → api-f75042b0.js} +1 -1
- flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-1d6acbd9.css} +41 -41
- flowfile/web/static/assets/{dropDown-614b998d.js → dropDown-2798a109.js} +3 -3
- flowfile/web/static/assets/{fullEditor-f7971590.js → fullEditor-cf7d7d93.js} +11 -10
- flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-fe9f7e18.css} +77 -65
- flowfile/web/static/assets/{genericNodeSettings-4fe5f36b.js → genericNodeSettings-14eac1c3.js} +5 -5
- flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
- flowfile/web/static/assets/{index-5429bbf8.js → index-387a6f18.js} +41806 -40958
- flowfile/web/static/assets/index-6b367bb5.js +38 -0
- flowfile/web/static/assets/{index-50508d4d.css → index-e96ab018.css} +2184 -569
- flowfile/web/static/assets/index-f0a6e5a5.js +2696 -0
- flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
- flowfile/web/static/assets/nodeInput-ed2ae8d7.js +2 -0
- flowfile/web/static/assets/{outputCsv-076b85ab.js → outputCsv-3c1757e8.js} +3 -3
- flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
- flowfile/web/static/assets/{outputExcel-0fd17dbe.js → outputExcel-686e1f48.js} +3 -3
- flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
- flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
- flowfile/web/static/assets/{outputParquet-b61e0847.js → outputParquet-df28faa7.js} +4 -4
- flowfile/web/static/assets/{readCsv-c767cb37.css → readCsv-3bfac4c3.css} +15 -15
- flowfile/web/static/assets/{readCsv-a8bb8b61.js → readCsv-e37eee21.js} +3 -3
- flowfile/web/static/assets/{readExcel-806d2826.css → readExcel-3db6b763.css} +13 -13
- flowfile/web/static/assets/{readExcel-67b4aee0.js → readExcel-a13f14bb.js} +5 -5
- flowfile/web/static/assets/{readParquet-92ce1dbc.js → readParquet-344cf746.js} +3 -3
- flowfile/web/static/assets/{readParquet-48c81530.css → readParquet-c5244ad5.css} +4 -4
- flowfile/web/static/assets/secrets.api-ae198c5c.js +65 -0
- flowfile/web/static/assets/{selectDynamic-92e25ee3.js → selectDynamic-6b4b0767.js} +5 -5
- flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
- flowfile/web/static/assets/{vue-codemirror.esm-41b0e0d7.js → vue-codemirror.esm-31ba0e0b.js} +31 -640
- flowfile/web/static/assets/{vue-content-loader.es-2c8e608f.js → vue-content-loader.es-4469c8ff.js} +1 -1
- flowfile/web/static/index.html +2 -2
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/METADATA +3 -4
- flowfile-0.5.4.dist-info/RECORD +407 -0
- flowfile_core/__init__.py +13 -6
- flowfile_core/auth/jwt.py +51 -16
- flowfile_core/auth/models.py +32 -7
- flowfile_core/auth/password.py +89 -0
- flowfile_core/auth/secrets.py +64 -19
- flowfile_core/configs/__init__.py +9 -7
- flowfile_core/configs/flow_logger.py +15 -14
- flowfile_core/configs/node_store/__init__.py +72 -4
- flowfile_core/configs/node_store/nodes.py +155 -172
- flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
- flowfile_core/configs/settings.py +28 -15
- flowfile_core/database/connection.py +7 -6
- flowfile_core/database/init_db.py +96 -2
- flowfile_core/database/models.py +3 -1
- flowfile_core/fileExplorer/__init__.py +17 -0
- flowfile_core/fileExplorer/funcs.py +145 -57
- flowfile_core/fileExplorer/utils.py +10 -11
- flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
- flowfile_core/flowfile/analytics/analytics_processor.py +26 -24
- flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
- flowfile_core/flowfile/analytics/utils.py +1 -1
- flowfile_core/flowfile/code_generator/__init__.py +11 -0
- flowfile_core/flowfile/code_generator/code_generator.py +706 -247
- flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
- flowfile_core/flowfile/connection_manager/models.py +1 -1
- flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
- flowfile_core/flowfile/database_connection_manager/models.py +1 -1
- flowfile_core/flowfile/extensions.py +17 -12
- flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
- flowfile_core/flowfile/flow_data_engine/create/funcs.py +115 -83
- flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +493 -423
- flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
- flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
- flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
- flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
- flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
- flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +31 -20
- flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
- flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +14 -15
- flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
- flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
- flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
- flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +190 -127
- flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
- flowfile_core/flowfile/flow_data_engine/utils.py +99 -67
- flowfile_core/flowfile/flow_graph.py +920 -571
- flowfile_core/flowfile/flow_graph_utils.py +31 -49
- flowfile_core/flowfile/flow_node/flow_node.py +379 -258
- flowfile_core/flowfile/flow_node/models.py +53 -41
- flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
- flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
- flowfile_core/flowfile/handler.py +80 -30
- flowfile_core/flowfile/manage/compatibility_enhancements.py +209 -126
- flowfile_core/flowfile/manage/io_flowfile.py +54 -57
- flowfile_core/flowfile/node_designer/__init__.py +19 -13
- flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
- flowfile_core/flowfile/node_designer/custom_node.py +162 -36
- flowfile_core/flowfile/node_designer/ui_components.py +278 -34
- flowfile_core/flowfile/schema_callbacks.py +71 -51
- flowfile_core/flowfile/setting_generator/__init__.py +0 -1
- flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
- flowfile_core/flowfile/setting_generator/settings.py +64 -53
- flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
- flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
- flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
- flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
- flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
- flowfile_core/flowfile/util/calculate_layout.py +9 -13
- flowfile_core/flowfile/util/execution_orderer.py +25 -17
- flowfile_core/flowfile/util/node_skipper.py +4 -4
- flowfile_core/flowfile/utils.py +19 -21
- flowfile_core/main.py +26 -19
- flowfile_core/routes/auth.py +284 -11
- flowfile_core/routes/cloud_connections.py +25 -25
- flowfile_core/routes/logs.py +21 -29
- flowfile_core/routes/public.py +46 -4
- flowfile_core/routes/routes.py +70 -34
- flowfile_core/routes/secrets.py +25 -27
- flowfile_core/routes/user_defined_components.py +483 -4
- flowfile_core/run_lock.py +0 -1
- flowfile_core/schemas/__init__.py +4 -6
- flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
- flowfile_core/schemas/cloud_storage_schemas.py +96 -66
- flowfile_core/schemas/input_schema.py +231 -144
- flowfile_core/schemas/output_model.py +49 -34
- flowfile_core/schemas/schemas.py +116 -89
- flowfile_core/schemas/transform_schema.py +518 -263
- flowfile_core/schemas/yaml_types.py +21 -7
- flowfile_core/secret_manager/secret_manager.py +123 -18
- flowfile_core/types.py +29 -9
- flowfile_core/utils/arrow_reader.py +7 -6
- flowfile_core/utils/excel_file_manager.py +3 -3
- flowfile_core/utils/fileManager.py +7 -7
- flowfile_core/utils/fl_executor.py +8 -10
- flowfile_core/utils/utils.py +4 -4
- flowfile_core/utils/validate_setup.py +5 -4
- flowfile_frame/__init__.py +117 -51
- flowfile_frame/adapters.py +2 -9
- flowfile_frame/adding_expr.py +73 -32
- flowfile_frame/cloud_storage/frame_helpers.py +27 -23
- flowfile_frame/cloud_storage/secret_manager.py +12 -26
- flowfile_frame/config.py +2 -5
- flowfile_frame/database/__init__.py +36 -0
- flowfile_frame/database/connection_manager.py +205 -0
- flowfile_frame/database/frame_helpers.py +249 -0
- flowfile_frame/expr.py +311 -218
- flowfile_frame/expr.pyi +160 -159
- flowfile_frame/expr_name.py +23 -23
- flowfile_frame/flow_frame.py +571 -476
- flowfile_frame/flow_frame.pyi +123 -104
- flowfile_frame/flow_frame_methods.py +227 -246
- flowfile_frame/group_frame.py +50 -20
- flowfile_frame/join.py +2 -2
- flowfile_frame/lazy.py +129 -87
- flowfile_frame/lazy_methods.py +83 -30
- flowfile_frame/list_name_space.py +55 -50
- flowfile_frame/selectors.py +148 -68
- flowfile_frame/series.py +9 -7
- flowfile_frame/utils.py +19 -21
- flowfile_worker/__init__.py +12 -7
- flowfile_worker/configs.py +41 -33
- flowfile_worker/create/__init__.py +14 -9
- flowfile_worker/create/funcs.py +114 -77
- flowfile_worker/create/models.py +46 -43
- flowfile_worker/create/pl_types.py +14 -15
- flowfile_worker/create/read_excel_tables.py +34 -41
- flowfile_worker/create/utils.py +22 -19
- flowfile_worker/external_sources/s3_source/main.py +18 -51
- flowfile_worker/external_sources/s3_source/models.py +34 -27
- flowfile_worker/external_sources/sql_source/main.py +8 -5
- flowfile_worker/external_sources/sql_source/models.py +13 -9
- flowfile_worker/flow_logger.py +10 -8
- flowfile_worker/funcs.py +214 -155
- flowfile_worker/main.py +11 -17
- flowfile_worker/models.py +35 -28
- flowfile_worker/process_manager.py +2 -3
- flowfile_worker/routes.py +121 -90
- flowfile_worker/secrets.py +114 -21
- flowfile_worker/spawner.py +89 -54
- flowfile_worker/utils.py +3 -2
- shared/__init__.py +2 -7
- shared/storage_config.py +25 -13
- test_utils/postgres/commands.py +3 -2
- test_utils/postgres/fixtures.py +9 -9
- test_utils/s3/commands.py +1 -1
- test_utils/s3/data_generator.py +3 -4
- test_utils/s3/demo_data_generator.py +4 -7
- test_utils/s3/fixtures.py +7 -5
- tools/migrate/__init__.py +1 -1
- tools/migrate/__main__.py +16 -29
- tools/migrate/legacy_schemas.py +251 -190
- tools/migrate/migrate.py +193 -181
- tools/migrate/tests/conftest.py +1 -3
- tools/migrate/tests/test_migrate.py +36 -41
- tools/migrate/tests/test_migration_e2e.py +28 -29
- tools/migrate/tests/test_node_migrations.py +50 -20
- flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
- flowfile/web/static/assets/ContextMenu-23e909da.js +0 -41
- flowfile/web/static/assets/ContextMenu-4c74eef1.css +0 -26
- flowfile/web/static/assets/ContextMenu-63cfa99b.css +0 -26
- flowfile/web/static/assets/ContextMenu-70ae0c79.js +0 -41
- flowfile/web/static/assets/ContextMenu-c13f91d0.css +0 -26
- flowfile/web/static/assets/ContextMenu-f149cf7c.js +0 -41
- flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
- flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
- flowfile/web/static/assets/Filter-9b6d08db.js +0 -164
- flowfile/web/static/assets/Filter-f62091b3.css +0 -20
- flowfile/web/static/assets/GroupBy-b9505323.css +0 -51
- flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
- flowfile/web/static/assets/Output-283fe388.css +0 -37
- flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
- flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
- flowfile/web/static/assets/SQLQueryComponent-36cef432.css +0 -27
- flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
- flowfile/web/static/assets/Sort-3643d625.css +0 -51
- flowfile/web/static/assets/Unique-f9fb0809.css +0 -51
- flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
- flowfile/web/static/assets/nodeInput-5d0d6b79.js +0 -41
- flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
- flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
- flowfile/web/static/assets/secretApi-68435402.js +0 -46
- flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
- flowfile-0.5.1.dist-info/RECORD +0 -388
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/WHEEL +0 -0
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/entry_points.txt +0 -0
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,19 +1,40 @@
|
|
|
1
|
-
|
|
1
|
+
from typing import Any, Literal, NamedTuple
|
|
2
2
|
|
|
3
|
-
from
|
|
4
|
-
|
|
5
|
-
from pydantic import Field, BaseModel, computed_field
|
|
3
|
+
from pydantic import BaseModel, Field, SecretStr, computed_field
|
|
6
4
|
|
|
7
5
|
from flowfile_core.flowfile.node_designer._type_registry import normalize_type_spec
|
|
6
|
+
from flowfile_core.secret_manager.secret_manager import decrypt_secret, get_encrypted_secret
|
|
7
|
+
|
|
8
8
|
# Public API import
|
|
9
9
|
from flowfile_core.types import DataType, TypeSpec
|
|
10
10
|
|
|
11
11
|
InputType = Literal["text", "number", "secret", "array", "date", "boolean"]
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
class ActionOption(NamedTuple):
|
|
15
|
+
"""
|
|
16
|
+
A named tuple representing an action option with a value and display label.
|
|
17
|
+
|
|
18
|
+
Use this to define actions with custom labels in ColumnActionInput:
|
|
19
|
+
actions=[
|
|
20
|
+
ActionOption("sum", "Sum"),
|
|
21
|
+
ActionOption("avg", "Average"),
|
|
22
|
+
"count" # plain strings also work
|
|
23
|
+
]
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
value: str
|
|
27
|
+
"""The internal value used in the data."""
|
|
28
|
+
|
|
29
|
+
label: str
|
|
30
|
+
"""The display label shown in the UI."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# Type alias for action specifications - accepts strings or ActionOption tuples
|
|
34
|
+
ActionSpec = str | ActionOption
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def normalize_input_to_data_types(v: Any) -> Literal["ALL"] | list[DataType]:
|
|
17
38
|
"""
|
|
18
39
|
Normalizes a wide variety of inputs to either 'ALL' or a sorted list of DataType enums.
|
|
19
40
|
This function is used as a Pydantic BeforeValidator.
|
|
@@ -46,9 +67,10 @@ class FlowfileInComponent(BaseModel):
|
|
|
46
67
|
It's not meant to be used directly, but rather to be inherited by specific
|
|
47
68
|
component classes.
|
|
48
69
|
"""
|
|
70
|
+
|
|
49
71
|
component_type: str = Field(..., description="Type of the UI component")
|
|
50
72
|
value: Any = None
|
|
51
|
-
label:
|
|
73
|
+
label: str | None = None
|
|
52
74
|
input_type: InputType
|
|
53
75
|
|
|
54
76
|
def set_value(self, value: Any):
|
|
@@ -83,6 +105,7 @@ class IncomingColumns:
|
|
|
83
105
|
options=IncomingColumns
|
|
84
106
|
)
|
|
85
107
|
"""
|
|
108
|
+
|
|
86
109
|
pass
|
|
87
110
|
|
|
88
111
|
|
|
@@ -94,25 +117,21 @@ class ColumnSelector(FlowfileInComponent):
|
|
|
94
117
|
This is particularly useful when a node operation should only be applied
|
|
95
118
|
to columns of a specific type (e.g., numeric, string, date).
|
|
96
119
|
"""
|
|
120
|
+
|
|
97
121
|
component_type: Literal["ColumnSelector"] = "ColumnSelector"
|
|
98
122
|
required: bool = False
|
|
99
123
|
multiple: bool = False
|
|
100
124
|
input_type: InputType = "text"
|
|
101
125
|
|
|
102
126
|
# Normalized output: either "ALL" or list of DataType enums
|
|
103
|
-
data_type_filter_input: TypeSpec = Field(
|
|
104
|
-
default="ALL",
|
|
105
|
-
alias="data_types",
|
|
106
|
-
repr=False,
|
|
107
|
-
exclude=True
|
|
108
|
-
)
|
|
127
|
+
data_type_filter_input: TypeSpec = Field(default="ALL", alias="data_types", repr=False, exclude=True)
|
|
109
128
|
|
|
110
129
|
class Config:
|
|
111
130
|
arbitrary_types_allowed = True
|
|
112
131
|
|
|
113
132
|
@computed_field
|
|
114
133
|
@property
|
|
115
|
-
def data_types_filter(self) ->
|
|
134
|
+
def data_types_filter(self) -> Literal["ALL"] | list[DataType]:
|
|
116
135
|
"""
|
|
117
136
|
A computed field that normalizes the `data_type_filter_input` into a
|
|
118
137
|
standardized format for the frontend.
|
|
@@ -125,16 +144,17 @@ class ColumnSelector(FlowfileInComponent):
|
|
|
125
144
|
correct format for the frontend.
|
|
126
145
|
"""
|
|
127
146
|
data = super().model_dump(**kwargs)
|
|
128
|
-
if
|
|
129
|
-
data[
|
|
147
|
+
if "data_types_filter" in data and data["data_types_filter"] != "ALL":
|
|
148
|
+
data["data_types"] = sorted([dt.value for dt in data["data_types_filter"]])
|
|
130
149
|
return data
|
|
131
150
|
|
|
132
151
|
|
|
133
152
|
class TextInput(FlowfileInComponent):
|
|
134
153
|
"""A standard text input field for capturing string values."""
|
|
154
|
+
|
|
135
155
|
component_type: Literal["TextInput"] = "TextInput"
|
|
136
|
-
default:
|
|
137
|
-
placeholder:
|
|
156
|
+
default: str | None = ""
|
|
157
|
+
placeholder: str | None = ""
|
|
138
158
|
input_type: InputType = "text"
|
|
139
159
|
|
|
140
160
|
def __init__(self, **data):
|
|
@@ -145,10 +165,11 @@ class TextInput(FlowfileInComponent):
|
|
|
145
165
|
|
|
146
166
|
class NumericInput(FlowfileInComponent):
|
|
147
167
|
"""A numeric input field with optional minimum and maximum value validation."""
|
|
168
|
+
|
|
148
169
|
component_type: Literal["NumericInput"] = "NumericInput"
|
|
149
|
-
default:
|
|
150
|
-
min_value:
|
|
151
|
-
max_value:
|
|
170
|
+
default: float | None = None
|
|
171
|
+
min_value: float | None = None
|
|
172
|
+
max_value: float | None = None
|
|
152
173
|
input_type: InputType = "number"
|
|
153
174
|
|
|
154
175
|
def __init__(self, **data):
|
|
@@ -157,11 +178,30 @@ class NumericInput(FlowfileInComponent):
|
|
|
157
178
|
self.value = self.default
|
|
158
179
|
|
|
159
180
|
|
|
181
|
+
class SliderInput(FlowfileInComponent):
|
|
182
|
+
"""A slider input for selecting a numeric value within a range."""
|
|
183
|
+
|
|
184
|
+
component_type: Literal["SliderInput"] = "SliderInput"
|
|
185
|
+
default: float | None = None
|
|
186
|
+
min_value: float = 0
|
|
187
|
+
max_value: float = 100
|
|
188
|
+
step: float = 1
|
|
189
|
+
input_type: InputType = "number"
|
|
190
|
+
|
|
191
|
+
def __init__(self, **data):
|
|
192
|
+
super().__init__(**data)
|
|
193
|
+
if self.value is None and self.default is not None:
|
|
194
|
+
self.value = self.default
|
|
195
|
+
elif self.value is None:
|
|
196
|
+
self.value = self.min_value
|
|
197
|
+
|
|
198
|
+
|
|
160
199
|
class ToggleSwitch(FlowfileInComponent):
|
|
161
200
|
"""A boolean toggle switch, typically used for enabling or disabling a feature."""
|
|
201
|
+
|
|
162
202
|
component_type: Literal["ToggleSwitch"] = "ToggleSwitch"
|
|
163
203
|
default: bool = False
|
|
164
|
-
description:
|
|
204
|
+
description: str | None = None
|
|
165
205
|
input_type: InputType = "boolean"
|
|
166
206
|
|
|
167
207
|
def __init__(self, **data):
|
|
@@ -182,9 +222,10 @@ class SingleSelect(FlowfileInComponent):
|
|
|
182
222
|
dynamically populated from the input dataframe's columns by using the
|
|
183
223
|
`IncomingColumns` marker.
|
|
184
224
|
"""
|
|
225
|
+
|
|
185
226
|
component_type: Literal["SingleSelect"] = "SingleSelect"
|
|
186
|
-
options:
|
|
187
|
-
default:
|
|
227
|
+
options: list[str | tuple[str, Any]] | type[IncomingColumns]
|
|
228
|
+
default: Any | None = None
|
|
188
229
|
input_type: InputType = "text"
|
|
189
230
|
|
|
190
231
|
def __init__(self, **data):
|
|
@@ -200,9 +241,10 @@ class MultiSelect(FlowfileInComponent):
|
|
|
200
241
|
Like `SingleSelect`, the options can be static or dynamically populated
|
|
201
242
|
from the input columns using the `IncomingColumns` marker.
|
|
202
243
|
"""
|
|
244
|
+
|
|
203
245
|
component_type: Literal["MultiSelect"] = "MultiSelect"
|
|
204
|
-
options:
|
|
205
|
-
default:
|
|
246
|
+
options: list[str | tuple[str, Any]] | type[IncomingColumns]
|
|
247
|
+
default: list[Any] = Field(default_factory=list)
|
|
206
248
|
input_type: InputType = "array"
|
|
207
249
|
|
|
208
250
|
def __init__(self, **data):
|
|
@@ -226,12 +268,13 @@ class Section(BaseModel):
|
|
|
226
268
|
my_text_input=TextInput(label="Enter a value")
|
|
227
269
|
)
|
|
228
270
|
"""
|
|
229
|
-
|
|
230
|
-
|
|
271
|
+
|
|
272
|
+
title: str | None = None
|
|
273
|
+
description: str | None = None
|
|
231
274
|
hidden: bool = False
|
|
232
275
|
|
|
233
276
|
class Config:
|
|
234
|
-
extra =
|
|
277
|
+
extra = "allow"
|
|
235
278
|
arbitrary_types_allowed = True
|
|
236
279
|
|
|
237
280
|
def __init__(self, **data):
|
|
@@ -240,7 +283,7 @@ class Section(BaseModel):
|
|
|
240
283
|
"""
|
|
241
284
|
super().__init__(**data)
|
|
242
285
|
|
|
243
|
-
def __call__(self, **kwargs) ->
|
|
286
|
+
def __call__(self, **kwargs) -> "Section":
|
|
244
287
|
"""
|
|
245
288
|
Allows adding components to the section after initialization.
|
|
246
289
|
|
|
@@ -250,7 +293,7 @@ class Section(BaseModel):
|
|
|
250
293
|
setattr(self, key, value)
|
|
251
294
|
return self
|
|
252
295
|
|
|
253
|
-
def get_components(self) ->
|
|
296
|
+
def get_components(self) -> dict[str, FlowfileInComponent]:
|
|
254
297
|
"""
|
|
255
298
|
Get all FlowfileInComponent instances from the section.
|
|
256
299
|
|
|
@@ -263,15 +306,216 @@ class Section(BaseModel):
|
|
|
263
306
|
components = {}
|
|
264
307
|
|
|
265
308
|
# Get from extra fields
|
|
266
|
-
for key, value in getattr(self,
|
|
309
|
+
for key, value in getattr(self, "__pydantic_extra__", {}).items():
|
|
267
310
|
if isinstance(value, FlowfileInComponent):
|
|
268
311
|
components[key] = value
|
|
269
312
|
|
|
270
313
|
# Get from defined fields (excluding metadata)
|
|
271
314
|
for field_name in self.model_fields:
|
|
272
|
-
if field_name not in {
|
|
315
|
+
if field_name not in {"title", "description", "hidden"}:
|
|
273
316
|
value = getattr(self, field_name, None)
|
|
274
317
|
if isinstance(value, FlowfileInComponent):
|
|
275
318
|
components[field_name] = value
|
|
276
319
|
|
|
277
320
|
return components
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
class AvailableSecrets:
|
|
324
|
+
"""
|
|
325
|
+
A marker class used in `SecretSelector` components.
|
|
326
|
+
|
|
327
|
+
When `options` is set to this class, the component will be dynamically
|
|
328
|
+
populated with the secret names available to the current user.
|
|
329
|
+
This allows users to select from available secrets at runtime.
|
|
330
|
+
|
|
331
|
+
Example:
|
|
332
|
+
class MyNodeSettings(NodeSettings):
|
|
333
|
+
api_key = SecretSelector(
|
|
334
|
+
label="Select an API Key",
|
|
335
|
+
options=AvailableSecrets
|
|
336
|
+
)
|
|
337
|
+
"""
|
|
338
|
+
|
|
339
|
+
pass
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
class SecretSelector(FlowfileInComponent):
|
|
343
|
+
component_type: Literal["SecretSelector"] = "SecretSelector"
|
|
344
|
+
options: type[AvailableSecrets] = AvailableSecrets
|
|
345
|
+
required: bool = False
|
|
346
|
+
description: str | None = None
|
|
347
|
+
input_type: InputType = "secret"
|
|
348
|
+
name_prefix: str | None = None
|
|
349
|
+
|
|
350
|
+
# Private fields for runtime context
|
|
351
|
+
_user_id: int | None = None
|
|
352
|
+
_accessed_secrets: set | None = None # Reference to node's tracking set
|
|
353
|
+
|
|
354
|
+
def set_execution_context(self, user_id: int, accessed_secrets: set):
|
|
355
|
+
"""Called by framework before process() runs."""
|
|
356
|
+
self._user_id = user_id
|
|
357
|
+
self._accessed_secrets = accessed_secrets
|
|
358
|
+
|
|
359
|
+
@property
|
|
360
|
+
def secret_value(self) -> SecretStr | None:
|
|
361
|
+
"""
|
|
362
|
+
Get the decrypted secret value.
|
|
363
|
+
|
|
364
|
+
Can only be called during node execution (after context is set).
|
|
365
|
+
Returns None if no secret is selected.
|
|
366
|
+
"""
|
|
367
|
+
if self.value is None:
|
|
368
|
+
return None
|
|
369
|
+
|
|
370
|
+
if self._user_id is None:
|
|
371
|
+
raise ValueError(
|
|
372
|
+
"Secret can only be accessed during node execution. "
|
|
373
|
+
"Ensure you're calling this from within the process() method."
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
encrypted = get_encrypted_secret(current_user_id=self._user_id, secret_name=self.value)
|
|
377
|
+
|
|
378
|
+
if encrypted is None:
|
|
379
|
+
raise ValueError(
|
|
380
|
+
f"Secret '{self.value}' not found for user. " f"Please ensure the secret exists in your secrets store."
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
decrypted = decrypt_secret(encrypted)
|
|
384
|
+
|
|
385
|
+
if self._accessed_secrets is not None:
|
|
386
|
+
self._accessed_secrets.add(decrypted.get_secret_value())
|
|
387
|
+
else:
|
|
388
|
+
self._accessed_secrets = {decrypted.get_secret_value()}
|
|
389
|
+
return decrypted
|
|
390
|
+
|
|
391
|
+
def model_dump(self, **kwargs) -> dict:
|
|
392
|
+
"""
|
|
393
|
+
Overrides the default `model_dump` to signal to the frontend
|
|
394
|
+
that this needs dynamic population from available secrets.
|
|
395
|
+
"""
|
|
396
|
+
data = super().model_dump(**kwargs)
|
|
397
|
+
# Signal to frontend that options should be fetched from /secrets endpoint
|
|
398
|
+
data["options"] = {"__type__": "AvailableSecrets"}
|
|
399
|
+
if self.name_prefix:
|
|
400
|
+
data["name_prefix"] = self.name_prefix
|
|
401
|
+
return data
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
class ColumnActionInput(FlowfileInComponent):
|
|
405
|
+
"""
|
|
406
|
+
A generic UI component for configuring column-based transformations.
|
|
407
|
+
|
|
408
|
+
This component allows users to select columns, choose an action/transformation,
|
|
409
|
+
and optionally rename the output. It can be configured for many use cases:
|
|
410
|
+
rolling windows, aggregations, string transformations, type conversions, etc.
|
|
411
|
+
|
|
412
|
+
The component displays:
|
|
413
|
+
- A list of available columns (filterable by data type)
|
|
414
|
+
- A table of configured operations with: Column, Action, Output Name
|
|
415
|
+
- Optional group by and order by selectors
|
|
416
|
+
|
|
417
|
+
Example - Rolling Window:
|
|
418
|
+
ColumnActionInput(
|
|
419
|
+
label="Rolling Calculations",
|
|
420
|
+
actions=["sum", "mean", "min", "max"],
|
|
421
|
+
output_name_template="{column}_rolling_{action}",
|
|
422
|
+
show_group_by=True,
|
|
423
|
+
show_order_by=True,
|
|
424
|
+
data_types="Numeric"
|
|
425
|
+
)
|
|
426
|
+
|
|
427
|
+
Example - String Transformations:
|
|
428
|
+
ColumnActionInput(
|
|
429
|
+
label="String Operations",
|
|
430
|
+
actions=["upper", "lower", "trim", "reverse"],
|
|
431
|
+
output_name_template="{column}_{action}",
|
|
432
|
+
data_types="String"
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
Example - Aggregations with ActionOption:
|
|
436
|
+
ColumnActionInput(
|
|
437
|
+
label="Aggregations",
|
|
438
|
+
actions=[
|
|
439
|
+
ActionOption("sum", "Sum"),
|
|
440
|
+
ActionOption("count", "Count"),
|
|
441
|
+
ActionOption("mean", "Average"),
|
|
442
|
+
"min", # plain strings also work
|
|
443
|
+
],
|
|
444
|
+
output_name_template="{column}_{action}",
|
|
445
|
+
show_group_by=True
|
|
446
|
+
)
|
|
447
|
+
"""
|
|
448
|
+
|
|
449
|
+
component_type: Literal["ColumnActionInput"] = "ColumnActionInput"
|
|
450
|
+
input_type: InputType = "array"
|
|
451
|
+
|
|
452
|
+
# Configurable actions - list of action names or ActionOption tuples
|
|
453
|
+
actions: list[ActionSpec] = Field(default_factory=list)
|
|
454
|
+
"""Actions available for selection. Can be strings or ActionOption(value, label) tuples."""
|
|
455
|
+
|
|
456
|
+
# Template for auto-generating output names
|
|
457
|
+
# Supports placeholders: {column}, {action}
|
|
458
|
+
output_name_template: str = "{column}_{action}"
|
|
459
|
+
"""Template for generating default output names. Use {column} and {action} placeholders."""
|
|
460
|
+
|
|
461
|
+
# Optional grouping/ordering support
|
|
462
|
+
show_group_by: bool = False
|
|
463
|
+
"""Whether to show the group by column selector."""
|
|
464
|
+
|
|
465
|
+
show_order_by: bool = False
|
|
466
|
+
"""Whether to show the order by column selector."""
|
|
467
|
+
|
|
468
|
+
# Type filtering for column selection
|
|
469
|
+
data_type_filter_input: TypeSpec = Field(default="ALL", alias="data_types", repr=False, exclude=True)
|
|
470
|
+
"""Filter columns by data type. Defaults to ALL."""
|
|
471
|
+
|
|
472
|
+
class Config:
|
|
473
|
+
arbitrary_types_allowed = True
|
|
474
|
+
|
|
475
|
+
def __init__(self, **data):
|
|
476
|
+
super().__init__(**data)
|
|
477
|
+
# Initialize value if not set
|
|
478
|
+
if self.value is None:
|
|
479
|
+
self.value = {
|
|
480
|
+
"rows": [],
|
|
481
|
+
"group_by_columns": [],
|
|
482
|
+
"order_by_column": None,
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
def set_value(self, value: Any):
|
|
486
|
+
"""
|
|
487
|
+
Sets the value from frontend.
|
|
488
|
+
"""
|
|
489
|
+
self.value = value
|
|
490
|
+
return self
|
|
491
|
+
|
|
492
|
+
@computed_field
|
|
493
|
+
@property
|
|
494
|
+
def data_types_filter(self) -> Literal["ALL"] | list[DataType]:
|
|
495
|
+
"""
|
|
496
|
+
A computed field that normalizes the `data_type_filter_input` into a
|
|
497
|
+
standardized format for the frontend.
|
|
498
|
+
"""
|
|
499
|
+
return normalize_input_to_data_types(self.data_type_filter_input)
|
|
500
|
+
|
|
501
|
+
def model_dump(self, **kwargs) -> dict:
|
|
502
|
+
"""
|
|
503
|
+
Serializes the component for the frontend.
|
|
504
|
+
"""
|
|
505
|
+
data = super().model_dump(**kwargs)
|
|
506
|
+
# Normalize actions to list of {value, label} objects for frontend
|
|
507
|
+
normalized_actions = []
|
|
508
|
+
for action in self.actions:
|
|
509
|
+
if isinstance(action, tuple):
|
|
510
|
+
normalized_actions.append({"value": action[0], "label": action[1]})
|
|
511
|
+
else:
|
|
512
|
+
normalized_actions.append({"value": action, "label": action})
|
|
513
|
+
data["actions"] = normalized_actions
|
|
514
|
+
data["output_name_template"] = self.output_name_template
|
|
515
|
+
data["show_group_by"] = self.show_group_by
|
|
516
|
+
data["show_order_by"] = self.show_order_by
|
|
517
|
+
if "data_types_filter" in data and data["data_types_filter"] != "ALL":
|
|
518
|
+
data["data_types"] = sorted([dt.value for dt in data["data_types_filter"]])
|
|
519
|
+
else:
|
|
520
|
+
data["data_types"] = "ALL"
|
|
521
|
+
return data
|
|
@@ -1,23 +1,17 @@
|
|
|
1
|
-
|
|
2
|
-
from typing import List
|
|
3
|
-
|
|
4
|
-
from polars import datatypes
|
|
5
1
|
import polars as pl
|
|
6
|
-
|
|
7
2
|
from pl_fuzzy_frame_match.output_column_name_utils import set_name_in_fuzzy_mappings
|
|
8
3
|
from pl_fuzzy_frame_match.pre_process import rename_fuzzy_right_mapping
|
|
4
|
+
from polars import datatypes
|
|
9
5
|
|
|
10
|
-
from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import fetch_unique_values
|
|
11
6
|
from flowfile_core.configs.flow_logger import main_logger
|
|
12
7
|
from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn, PlType
|
|
13
|
-
from flowfile_core.
|
|
14
|
-
from flowfile_core.schemas import input_schema
|
|
15
|
-
from flowfile_core.schemas.transform_schema import FuzzyMatchInputManager
|
|
8
|
+
from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import fetch_unique_values
|
|
9
|
+
from flowfile_core.schemas import input_schema, transform_schema
|
|
16
10
|
|
|
17
11
|
|
|
18
|
-
def _ensure_all_columns_have_select(
|
|
19
|
-
|
|
20
|
-
|
|
12
|
+
def _ensure_all_columns_have_select(
|
|
13
|
+
left_cols: list[str], right_cols: list[str], fuzzy_match_input: transform_schema.FuzzyMatchInputManager
|
|
14
|
+
):
|
|
21
15
|
"""
|
|
22
16
|
Ensure that all columns in the left and right FlowDataEngines are included in the fuzzy match input's select
|
|
23
17
|
statements.
|
|
@@ -33,13 +27,16 @@ def _ensure_all_columns_have_select(left_cols: List[str],
|
|
|
33
27
|
left_cols_in_select = {c.old_name for c in fuzzy_match_input.left_select.renames}
|
|
34
28
|
|
|
35
29
|
fuzzy_match_input.left_select.renames.extend(
|
|
36
|
-
[transform_schema.SelectInput(col) for col in left_cols if col not in left_cols_in_select]
|
|
30
|
+
[transform_schema.SelectInput(col) for col in left_cols if col not in left_cols_in_select]
|
|
31
|
+
)
|
|
37
32
|
fuzzy_match_input.right_select.renames.extend(
|
|
38
33
|
[transform_schema.SelectInput(col) for col in right_cols if col not in right_cols_in_select]
|
|
39
34
|
)
|
|
40
35
|
|
|
41
36
|
|
|
42
|
-
def _order_join_inputs_based_on_col_order(
|
|
37
|
+
def _order_join_inputs_based_on_col_order(
|
|
38
|
+
col_order: list[str], join_inputs: transform_schema.JoinInputsManager
|
|
39
|
+
) -> None:
|
|
43
40
|
"""
|
|
44
41
|
Ensure that the select columns in the fuzzy match input match the order of the incoming columns.
|
|
45
42
|
This function modifies the join_inputs object in-place.
|
|
@@ -52,17 +49,23 @@ def _order_join_inputs_based_on_col_order(col_order: List[str], join_inputs: tra
|
|
|
52
49
|
join_inputs.select_inputs.renames = ordered_renames
|
|
53
50
|
|
|
54
51
|
|
|
55
|
-
def calculate_fuzzy_match_schema(
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
52
|
+
def calculate_fuzzy_match_schema(
|
|
53
|
+
fm_input: transform_schema.FuzzyMatchInputManager,
|
|
54
|
+
left_schema: list[FlowfileColumn],
|
|
55
|
+
right_schema: list[FlowfileColumn],
|
|
56
|
+
):
|
|
57
|
+
_ensure_all_columns_have_select(
|
|
58
|
+
left_cols=[col.column_name for col in left_schema],
|
|
59
|
+
right_cols=[col.column_name for col in right_schema],
|
|
60
|
+
fuzzy_match_input=fm_input,
|
|
61
|
+
)
|
|
61
62
|
|
|
62
|
-
_order_join_inputs_based_on_col_order(
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
_order_join_inputs_based_on_col_order(
|
|
64
|
+
col_order=[col.column_name for col in left_schema], join_inputs=fm_input.left_select
|
|
65
|
+
)
|
|
66
|
+
_order_join_inputs_based_on_col_order(
|
|
67
|
+
col_order=[col.column_name for col in right_schema], join_inputs=fm_input.right_select
|
|
68
|
+
)
|
|
66
69
|
for column in fm_input.left_select.renames:
|
|
67
70
|
if column.join_key:
|
|
68
71
|
column.keep = True
|
|
@@ -78,20 +81,27 @@ def calculate_fuzzy_match_schema(fm_input: transform_schema.FuzzyMatchInputManag
|
|
|
78
81
|
for column in fm_input.left_select.renames:
|
|
79
82
|
column_schema = left_schema_dict.get(column.old_name)
|
|
80
83
|
if column_schema and (column.keep or column.join_key):
|
|
81
|
-
output_schema.append(
|
|
82
|
-
|
|
84
|
+
output_schema.append(
|
|
85
|
+
FlowfileColumn.from_input(
|
|
86
|
+
column.new_name, column_schema.data_type, example_values=column_schema.example_values
|
|
87
|
+
)
|
|
88
|
+
)
|
|
83
89
|
for column in fm_input.right_select.renames:
|
|
84
90
|
column_schema = right_schema_dict.get(column.old_name)
|
|
85
91
|
if column_schema and (column.keep or column.join_key):
|
|
86
|
-
output_schema.append(
|
|
87
|
-
|
|
92
|
+
output_schema.append(
|
|
93
|
+
FlowfileColumn.from_input(
|
|
94
|
+
column.new_name, column_schema.data_type, example_values=column_schema.example_values
|
|
95
|
+
)
|
|
96
|
+
)
|
|
88
97
|
set_name_in_fuzzy_mappings(new_join_mapping)
|
|
89
|
-
output_schema.extend(
|
|
90
|
-
|
|
98
|
+
output_schema.extend(
|
|
99
|
+
[FlowfileColumn.from_input(fuzzy_mapping.output_column_name, "Float64") for fuzzy_mapping in new_join_mapping]
|
|
100
|
+
)
|
|
91
101
|
return output_schema
|
|
92
102
|
|
|
93
103
|
|
|
94
|
-
def get_schema_of_column(node_input_schema:
|
|
104
|
+
def get_schema_of_column(node_input_schema: list[FlowfileColumn], col_name: str) -> FlowfileColumn | None:
|
|
95
105
|
for s in node_input_schema:
|
|
96
106
|
if s.name == col_name:
|
|
97
107
|
return s
|
|
@@ -99,44 +109,54 @@ def get_schema_of_column(node_input_schema: List[FlowfileColumn], col_name: str)
|
|
|
99
109
|
|
|
100
110
|
class InvalidSetup(ValueError):
|
|
101
111
|
"""Error raised when pivot column has too many unique values."""
|
|
112
|
+
|
|
102
113
|
pass
|
|
103
114
|
|
|
104
115
|
|
|
105
116
|
def get_output_data_type_pivot(schema: FlowfileColumn, agg_type: str) -> datatypes:
|
|
106
|
-
if agg_type in (
|
|
117
|
+
if agg_type in ("count", "n_unique"):
|
|
107
118
|
output_type = datatypes.Float64 # count is always float
|
|
108
|
-
elif schema.generic_datatype() ==
|
|
119
|
+
elif schema.generic_datatype() == "numeric":
|
|
109
120
|
output_type = datatypes.Float64
|
|
110
|
-
elif schema.generic_datatype() ==
|
|
121
|
+
elif schema.generic_datatype() == "string":
|
|
111
122
|
output_type = datatypes.Utf8
|
|
112
|
-
elif schema.generic_datatype() ==
|
|
123
|
+
elif schema.generic_datatype() == "date":
|
|
113
124
|
output_type = datatypes.Datetime
|
|
114
125
|
else:
|
|
115
126
|
output_type = datatypes.Utf8
|
|
116
127
|
return output_type
|
|
117
128
|
|
|
118
129
|
|
|
119
|
-
def pre_calculate_pivot_schema(
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
130
|
+
def pre_calculate_pivot_schema(
|
|
131
|
+
node_input_schema: list[FlowfileColumn],
|
|
132
|
+
pivot_input: transform_schema.PivotInput,
|
|
133
|
+
output_fields: list[input_schema.MinimalFieldInfo] = None,
|
|
134
|
+
input_lf: pl.LazyFrame = None,
|
|
135
|
+
) -> list[FlowfileColumn]:
|
|
136
|
+
index_columns_schema = [
|
|
137
|
+
get_schema_of_column(node_input_schema, index_col) for index_col in pivot_input.index_columns
|
|
138
|
+
]
|
|
125
139
|
val_column_schema = get_schema_of_column(node_input_schema, pivot_input.value_col)
|
|
126
140
|
if output_fields is not None and len(output_fields) > 0:
|
|
127
|
-
return index_columns_schema+[
|
|
128
|
-
|
|
129
|
-
|
|
141
|
+
return index_columns_schema + [
|
|
142
|
+
FlowfileColumn(PlType(column_name=output_field.name, pl_datatype=output_field.data_type))
|
|
143
|
+
for output_field in output_fields
|
|
144
|
+
]
|
|
130
145
|
|
|
131
146
|
else:
|
|
132
147
|
max_unique_vals = 200
|
|
133
|
-
unique_vals = fetch_unique_values(
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
148
|
+
unique_vals = fetch_unique_values(
|
|
149
|
+
input_lf.select(pivot_input.pivot_column)
|
|
150
|
+
.unique()
|
|
151
|
+
.sort(pivot_input.pivot_column)
|
|
152
|
+
.limit(max_unique_vals)
|
|
153
|
+
.cast(pl.String)
|
|
154
|
+
)
|
|
137
155
|
if len(unique_vals) >= max_unique_vals:
|
|
138
|
-
main_logger.warning(
|
|
139
|
-
|
|
156
|
+
main_logger.warning(
|
|
157
|
+
"Pivot column has too many unique values. Please consider using a different column."
|
|
158
|
+
f" Max unique values: {max_unique_vals}"
|
|
159
|
+
)
|
|
140
160
|
pl_output_fields = []
|
|
141
161
|
for val in unique_vals:
|
|
142
162
|
if len(pivot_input.aggregations) == 1:
|
|
@@ -145,5 +165,5 @@ def pre_calculate_pivot_schema(node_input_schema: List[FlowfileColumn],
|
|
|
145
165
|
else:
|
|
146
166
|
for agg in pivot_input.aggregations:
|
|
147
167
|
output_type = get_output_data_type_pivot(val_column_schema, agg)
|
|
148
|
-
pl_output_fields.append(PlType(column_name=f
|
|
168
|
+
pl_output_fields.append(PlType(column_name=f"{val}_{agg}", pl_datatype=output_type))
|
|
149
169
|
return index_columns_schema + [FlowfileColumn(pl_output_field) for pl_output_field in pl_output_fields]
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
|
|
1
3
|
from flowfile_core.configs import logger
|
|
2
|
-
from typing import Callable
|
|
3
4
|
|
|
4
5
|
|
|
5
6
|
class SettingGenerator:
|
|
@@ -13,10 +14,10 @@ class SettingGenerator:
|
|
|
13
14
|
setattr(self, f.__name__, f)
|
|
14
15
|
|
|
15
16
|
def get_setting_generator(self, node_type: str) -> Callable:
|
|
16
|
-
logger.info(
|
|
17
|
+
logger.info("getting setting generator for " + node_type)
|
|
17
18
|
|
|
18
19
|
if node_type in self.setting_generator_set:
|
|
19
|
-
logger.info(
|
|
20
|
+
logger.info("setting generator found")
|
|
20
21
|
return getattr(self, node_type)
|
|
21
22
|
else:
|
|
22
23
|
return lambda x: x
|
|
@@ -33,9 +34,9 @@ class SettingUpdator:
|
|
|
33
34
|
setattr(self, f.__name__, f)
|
|
34
35
|
|
|
35
36
|
def get_setting_updator(self, node_type: str) -> Callable:
|
|
36
|
-
logger.info(
|
|
37
|
+
logger.info("getting setting updator for " + node_type)
|
|
37
38
|
if node_type in self.setting_updator_set:
|
|
38
|
-
logger.info(
|
|
39
|
+
logger.info("setting updator found")
|
|
39
40
|
return getattr(self, node_type)
|
|
40
41
|
else:
|
|
41
42
|
return lambda x: x
|