Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- build_backends/main.py +25 -22
- build_backends/main_prd.py +10 -19
- flowfile/__init__.py +194 -74
- flowfile/__main__.py +10 -7
- flowfile/api.py +51 -57
- flowfile/web/__init__.py +14 -9
- flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
- flowfile/web/static/assets/AdminView-f9847d67.js +713 -0
- flowfile/web/static/assets/CloudConnectionView-cf85f943.css +72 -0
- flowfile/web/static/assets/{CloudConnectionManager-0dfba9f2.js → CloudConnectionView-faace55b.js} +11 -11
- flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
- flowfile/web/static/assets/{CloudStorageReader-d5b1b6c9.js → CloudStorageReader-d86ecaa7.js} +10 -8
- flowfile/web/static/assets/{CloudStorageWriter-00d87aad.js → CloudStorageWriter-0f4d9a44.js} +10 -8
- flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
- flowfile/web/static/assets/ColumnActionInput-c44b7aee.css +159 -0
- flowfile/web/static/assets/ColumnActionInput-f4189ae0.js +330 -0
- flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
- flowfile/web/static/assets/{ColumnSelector-4685e75d.js → ColumnSelector-e66b33da.js} +3 -5
- flowfile/web/static/assets/ContextMenu-49463352.js +9 -0
- flowfile/web/static/assets/ContextMenu-dd5f3f25.js +9 -0
- flowfile/web/static/assets/ContextMenu-f709b884.js +9 -0
- flowfile/web/static/assets/ContextMenu.vue_vue_type_script_setup_true_lang-a1bd6314.js +59 -0
- flowfile/web/static/assets/{CrossJoin-702a3edd.js → CrossJoin-24694b8f.js} +12 -10
- flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
- flowfile/web/static/assets/{CustomNode-b1519993.js → CustomNode-569d45ff.js} +43 -24
- flowfile/web/static/assets/CustomNode-edb9b939.css +42 -0
- flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-c20a1e16.css} +23 -21
- flowfile/web/static/assets/{DatabaseConnectionSettings-6f3e4ea5.js → DatabaseConnectionSettings-cfc08938.js} +5 -4
- flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-5bf8c75b.css} +41 -46
- flowfile/web/static/assets/{DatabaseReader-d38c7295.js → DatabaseReader-701feabb.js} +25 -15
- flowfile/web/static/assets/{DatabaseManager-cf5ef661.js → DatabaseView-0482e5b5.js} +11 -11
- flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
- flowfile/web/static/assets/{DatabaseWriter-b04ef46a.js → DatabaseWriter-16721989.js} +17 -10
- flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-bdcf2c8b.css} +29 -27
- flowfile/web/static/assets/{designer-8da3ba3a.css → DesignerView-49abb835.css} +783 -663
- flowfile/web/static/assets/{designer-9633482a.js → DesignerView-f64749fb.js} +1292 -3253
- flowfile/web/static/assets/{documentation-ca400224.js → DocumentationView-61bd2990.js} +5 -5
- flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-9ea6e871.css} +9 -9
- flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
- flowfile/web/static/assets/{ExploreData-5fa10ed8.js → ExploreData-e2735b13.js} +18 -9
- flowfile/web/static/assets/{ExternalSource-d39af878.js → ExternalSource-2535c3b2.js} +9 -7
- flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-7ac7373f.css} +20 -20
- flowfile/web/static/assets/Filter-2cdbc93c.js +287 -0
- flowfile/web/static/assets/Filter-7494ea97.css +48 -0
- flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
- flowfile/web/static/assets/{Formula-6b04fb1d.js → Formula-fcda3c2c.js} +13 -11
- flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
- flowfile/web/static/assets/{FuzzyMatch-999521f4.js → FuzzyMatch-f8d3b7d3.js} +12 -10
- flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-4b4d7db9.css} +5 -5
- flowfile/web/static/assets/{GraphSolver-17dd2198.js → GraphSolver-72eaa695.js} +14 -12
- flowfile/web/static/assets/GroupBy-5792782d.css +9 -0
- flowfile/web/static/assets/{GroupBy-6b039e18.js → GroupBy-8aa0598b.js} +9 -7
- flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
- flowfile/web/static/assets/{Join-24d0f113.js → Join-e40f0ffa.js} +13 -11
- flowfile/web/static/assets/LoginView-5111c9ae.js +134 -0
- flowfile/web/static/assets/LoginView-d325d632.css +172 -0
- flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
- flowfile/web/static/assets/{ManualInput-34639209.js → ManualInput-9b6f3224.js} +170 -116
- flowfile/web/static/assets/{MultiSelect-0e8724a3.js → MultiSelect-ef28e19e.js} +2 -2
- flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-b0e538c2.js → MultiSelect.vue_vue_type_script_setup_true_lang-83b3bbfd.js} +1 -1
- flowfile/web/static/assets/NodeDesigner-94cd4dd3.css +1429 -0
- flowfile/web/static/assets/NodeDesigner-d2b7ee2b.js +2712 -0
- flowfile/web/static/assets/{NumericInput-3d63a470.js → NumericInput-1d789794.js} +2 -2
- flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-e0edeccc.js → NumericInput.vue_vue_type_script_setup_true_lang-7775f83e.js} +5 -2
- flowfile/web/static/assets/Output-692dd25d.css +37 -0
- flowfile/web/static/assets/{Output-edea9802.js → Output-cefef801.js} +14 -10
- flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
- flowfile/web/static/assets/{Pivot-61d19301.js → Pivot-bab1b75b.js} +12 -10
- flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
- flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
- flowfile/web/static/assets/{PivotValidation-f97fec5b.js → PivotValidation-e7941f91.js} +3 -3
- flowfile/web/static/assets/{PivotValidation-de9f43fe.js → PivotValidation-fba09336.js} +3 -3
- flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
- flowfile/web/static/assets/{PolarsCode-bc3c9984.js → PolarsCode-740e40fa.js} +18 -9
- flowfile/web/static/assets/PopOver-862d7e28.js +939 -0
- flowfile/web/static/assets/PopOver-d96599db.css +33 -0
- flowfile/web/static/assets/{Read-64a3f259.js → Read-225cc63f.js} +16 -12
- flowfile/web/static/assets/{Read-e808b239.css → Read-90f366bc.css} +15 -15
- flowfile/web/static/assets/{RecordCount-3d5039be.js → RecordCount-ffc71eca.js} +6 -4
- flowfile/web/static/assets/{RecordId-597510e0.js → RecordId-a70bb8df.js} +9 -7
- flowfile/web/static/assets/{SQLQueryComponent-df51adbe.js → SQLQueryComponent-15a421f5.js} +3 -3
- flowfile/web/static/assets/SQLQueryComponent-edb90b98.css +29 -0
- flowfile/web/static/assets/{Sample-4be0a507.js → Sample-6c26afc7.js} +6 -4
- flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
- flowfile/web/static/assets/SecretSelector-ceed9496.js +113 -0
- flowfile/web/static/assets/{SecretManager-4839be57.js → SecretsView-214d255a.js} +35 -36
- flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
- flowfile/web/static/assets/{Select-9b72f201.js → Select-8fc29999.js} +9 -7
- flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-7ded385d.js → SettingsSection-3f70e4c3.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-f0f75a42.js → SettingsSection-83090218.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-e1e9c953.js → SettingsSection-9f0d1725.js} +3 -3
- flowfile/web/static/assets/SetupView-3fa0aa03.js +160 -0
- flowfile/web/static/assets/SetupView-e2da3442.css +230 -0
- flowfile/web/static/assets/{SingleSelect-6c777aac.js → SingleSelect-a4a568cb.js} +2 -2
- flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-33e3ff9b.js → SingleSelect.vue_vue_type_script_setup_true_lang-c8ebdd33.js} +1 -1
- flowfile/web/static/assets/{SliderInput-7cb93e62.js → SliderInput-be533e71.js} +7 -4
- flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
- flowfile/web/static/assets/{Sort-6cbde21a.js → Sort-154dad81.js} +9 -7
- flowfile/web/static/assets/Sort-4abb7fae.css +9 -0
- flowfile/web/static/assets/{TextInput-d9a40c11.js → TextInput-454e2bda.js} +2 -2
- flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-5896c375.js → TextInput.vue_vue_type_script_setup_true_lang-e86510d0.js} +5 -2
- flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
- flowfile/web/static/assets/{TextToRows-c4fcbf4d.js → TextToRows-ea73433d.js} +11 -10
- flowfile/web/static/assets/{ToggleSwitch-4ef91d19.js → ToggleSwitch-9d7b30f1.js} +2 -2
- flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-38478c20.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-00f2580e.js} +1 -1
- flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-394a1f78.css} +14 -14
- flowfile/web/static/assets/{UnavailableFields-a03f512c.js → UnavailableFields-b72a2c72.js} +4 -4
- flowfile/web/static/assets/{Union-bfe9b996.js → Union-1e44f263.js} +8 -6
- flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
- flowfile/web/static/assets/Unique-2b705521.css +3 -0
- flowfile/web/static/assets/{Unique-5d023a27.js → Unique-a3bc6d0a.js} +13 -10
- flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-b6ad6427.css} +7 -7
- flowfile/web/static/assets/{Unpivot-91cc5354.js → Unpivot-e27935fc.js} +11 -9
- flowfile/web/static/assets/{UnpivotValidation-7ee2de44.js → UnpivotValidation-72497680.js} +3 -3
- flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
- flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
- flowfile/web/static/assets/{VueGraphicWalker-e51b9924.js → VueGraphicWalker-d9ab70a3.js} +4 -4
- flowfile/web/static/assets/{api-cf1221f0.js → api-a2102880.js} +1 -1
- flowfile/web/static/assets/{api-c1bad5ca.js → api-f75042b0.js} +1 -1
- flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-1d6acbd9.css} +41 -41
- flowfile/web/static/assets/{dropDown-614b998d.js → dropDown-2798a109.js} +3 -3
- flowfile/web/static/assets/{fullEditor-f7971590.js → fullEditor-cf7d7d93.js} +11 -10
- flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-fe9f7e18.css} +77 -65
- flowfile/web/static/assets/{genericNodeSettings-4fe5f36b.js → genericNodeSettings-14eac1c3.js} +5 -5
- flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
- flowfile/web/static/assets/{index-5429bbf8.js → index-387a6f18.js} +41806 -40958
- flowfile/web/static/assets/index-6b367bb5.js +38 -0
- flowfile/web/static/assets/{index-50508d4d.css → index-e96ab018.css} +2184 -569
- flowfile/web/static/assets/index-f0a6e5a5.js +2696 -0
- flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
- flowfile/web/static/assets/nodeInput-ed2ae8d7.js +2 -0
- flowfile/web/static/assets/{outputCsv-076b85ab.js → outputCsv-3c1757e8.js} +3 -3
- flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
- flowfile/web/static/assets/{outputExcel-0fd17dbe.js → outputExcel-686e1f48.js} +3 -3
- flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
- flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
- flowfile/web/static/assets/{outputParquet-b61e0847.js → outputParquet-df28faa7.js} +4 -4
- flowfile/web/static/assets/{readCsv-c767cb37.css → readCsv-3bfac4c3.css} +15 -15
- flowfile/web/static/assets/{readCsv-a8bb8b61.js → readCsv-e37eee21.js} +3 -3
- flowfile/web/static/assets/{readExcel-806d2826.css → readExcel-3db6b763.css} +13 -13
- flowfile/web/static/assets/{readExcel-67b4aee0.js → readExcel-a13f14bb.js} +5 -5
- flowfile/web/static/assets/{readParquet-92ce1dbc.js → readParquet-344cf746.js} +3 -3
- flowfile/web/static/assets/{readParquet-48c81530.css → readParquet-c5244ad5.css} +4 -4
- flowfile/web/static/assets/secrets.api-ae198c5c.js +65 -0
- flowfile/web/static/assets/{selectDynamic-92e25ee3.js → selectDynamic-6b4b0767.js} +5 -5
- flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
- flowfile/web/static/assets/{vue-codemirror.esm-41b0e0d7.js → vue-codemirror.esm-31ba0e0b.js} +31 -640
- flowfile/web/static/assets/{vue-content-loader.es-2c8e608f.js → vue-content-loader.es-4469c8ff.js} +1 -1
- flowfile/web/static/index.html +2 -2
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/METADATA +3 -4
- flowfile-0.5.4.dist-info/RECORD +407 -0
- flowfile_core/__init__.py +13 -6
- flowfile_core/auth/jwt.py +51 -16
- flowfile_core/auth/models.py +32 -7
- flowfile_core/auth/password.py +89 -0
- flowfile_core/auth/secrets.py +64 -19
- flowfile_core/configs/__init__.py +9 -7
- flowfile_core/configs/flow_logger.py +15 -14
- flowfile_core/configs/node_store/__init__.py +72 -4
- flowfile_core/configs/node_store/nodes.py +155 -172
- flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
- flowfile_core/configs/settings.py +28 -15
- flowfile_core/database/connection.py +7 -6
- flowfile_core/database/init_db.py +96 -2
- flowfile_core/database/models.py +3 -1
- flowfile_core/fileExplorer/__init__.py +17 -0
- flowfile_core/fileExplorer/funcs.py +145 -57
- flowfile_core/fileExplorer/utils.py +10 -11
- flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
- flowfile_core/flowfile/analytics/analytics_processor.py +26 -24
- flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
- flowfile_core/flowfile/analytics/utils.py +1 -1
- flowfile_core/flowfile/code_generator/__init__.py +11 -0
- flowfile_core/flowfile/code_generator/code_generator.py +706 -247
- flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
- flowfile_core/flowfile/connection_manager/models.py +1 -1
- flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
- flowfile_core/flowfile/database_connection_manager/models.py +1 -1
- flowfile_core/flowfile/extensions.py +17 -12
- flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
- flowfile_core/flowfile/flow_data_engine/create/funcs.py +115 -83
- flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +493 -423
- flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
- flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
- flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
- flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
- flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
- flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +31 -20
- flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
- flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +14 -15
- flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
- flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
- flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
- flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +190 -127
- flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
- flowfile_core/flowfile/flow_data_engine/utils.py +99 -67
- flowfile_core/flowfile/flow_graph.py +920 -571
- flowfile_core/flowfile/flow_graph_utils.py +31 -49
- flowfile_core/flowfile/flow_node/flow_node.py +379 -258
- flowfile_core/flowfile/flow_node/models.py +53 -41
- flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
- flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
- flowfile_core/flowfile/handler.py +80 -30
- flowfile_core/flowfile/manage/compatibility_enhancements.py +209 -126
- flowfile_core/flowfile/manage/io_flowfile.py +54 -57
- flowfile_core/flowfile/node_designer/__init__.py +19 -13
- flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
- flowfile_core/flowfile/node_designer/custom_node.py +162 -36
- flowfile_core/flowfile/node_designer/ui_components.py +278 -34
- flowfile_core/flowfile/schema_callbacks.py +71 -51
- flowfile_core/flowfile/setting_generator/__init__.py +0 -1
- flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
- flowfile_core/flowfile/setting_generator/settings.py +64 -53
- flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
- flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
- flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
- flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
- flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
- flowfile_core/flowfile/util/calculate_layout.py +9 -13
- flowfile_core/flowfile/util/execution_orderer.py +25 -17
- flowfile_core/flowfile/util/node_skipper.py +4 -4
- flowfile_core/flowfile/utils.py +19 -21
- flowfile_core/main.py +26 -19
- flowfile_core/routes/auth.py +284 -11
- flowfile_core/routes/cloud_connections.py +25 -25
- flowfile_core/routes/logs.py +21 -29
- flowfile_core/routes/public.py +46 -4
- flowfile_core/routes/routes.py +70 -34
- flowfile_core/routes/secrets.py +25 -27
- flowfile_core/routes/user_defined_components.py +483 -4
- flowfile_core/run_lock.py +0 -1
- flowfile_core/schemas/__init__.py +4 -6
- flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
- flowfile_core/schemas/cloud_storage_schemas.py +96 -66
- flowfile_core/schemas/input_schema.py +231 -144
- flowfile_core/schemas/output_model.py +49 -34
- flowfile_core/schemas/schemas.py +116 -89
- flowfile_core/schemas/transform_schema.py +518 -263
- flowfile_core/schemas/yaml_types.py +21 -7
- flowfile_core/secret_manager/secret_manager.py +123 -18
- flowfile_core/types.py +29 -9
- flowfile_core/utils/arrow_reader.py +7 -6
- flowfile_core/utils/excel_file_manager.py +3 -3
- flowfile_core/utils/fileManager.py +7 -7
- flowfile_core/utils/fl_executor.py +8 -10
- flowfile_core/utils/utils.py +4 -4
- flowfile_core/utils/validate_setup.py +5 -4
- flowfile_frame/__init__.py +117 -51
- flowfile_frame/adapters.py +2 -9
- flowfile_frame/adding_expr.py +73 -32
- flowfile_frame/cloud_storage/frame_helpers.py +27 -23
- flowfile_frame/cloud_storage/secret_manager.py +12 -26
- flowfile_frame/config.py +2 -5
- flowfile_frame/database/__init__.py +36 -0
- flowfile_frame/database/connection_manager.py +205 -0
- flowfile_frame/database/frame_helpers.py +249 -0
- flowfile_frame/expr.py +311 -218
- flowfile_frame/expr.pyi +160 -159
- flowfile_frame/expr_name.py +23 -23
- flowfile_frame/flow_frame.py +571 -476
- flowfile_frame/flow_frame.pyi +123 -104
- flowfile_frame/flow_frame_methods.py +227 -246
- flowfile_frame/group_frame.py +50 -20
- flowfile_frame/join.py +2 -2
- flowfile_frame/lazy.py +129 -87
- flowfile_frame/lazy_methods.py +83 -30
- flowfile_frame/list_name_space.py +55 -50
- flowfile_frame/selectors.py +148 -68
- flowfile_frame/series.py +9 -7
- flowfile_frame/utils.py +19 -21
- flowfile_worker/__init__.py +12 -7
- flowfile_worker/configs.py +41 -33
- flowfile_worker/create/__init__.py +14 -9
- flowfile_worker/create/funcs.py +114 -77
- flowfile_worker/create/models.py +46 -43
- flowfile_worker/create/pl_types.py +14 -15
- flowfile_worker/create/read_excel_tables.py +34 -41
- flowfile_worker/create/utils.py +22 -19
- flowfile_worker/external_sources/s3_source/main.py +18 -51
- flowfile_worker/external_sources/s3_source/models.py +34 -27
- flowfile_worker/external_sources/sql_source/main.py +8 -5
- flowfile_worker/external_sources/sql_source/models.py +13 -9
- flowfile_worker/flow_logger.py +10 -8
- flowfile_worker/funcs.py +214 -155
- flowfile_worker/main.py +11 -17
- flowfile_worker/models.py +35 -28
- flowfile_worker/process_manager.py +2 -3
- flowfile_worker/routes.py +121 -90
- flowfile_worker/secrets.py +114 -21
- flowfile_worker/spawner.py +89 -54
- flowfile_worker/utils.py +3 -2
- shared/__init__.py +2 -7
- shared/storage_config.py +25 -13
- test_utils/postgres/commands.py +3 -2
- test_utils/postgres/fixtures.py +9 -9
- test_utils/s3/commands.py +1 -1
- test_utils/s3/data_generator.py +3 -4
- test_utils/s3/demo_data_generator.py +4 -7
- test_utils/s3/fixtures.py +7 -5
- tools/migrate/__init__.py +1 -1
- tools/migrate/__main__.py +16 -29
- tools/migrate/legacy_schemas.py +251 -190
- tools/migrate/migrate.py +193 -181
- tools/migrate/tests/conftest.py +1 -3
- tools/migrate/tests/test_migrate.py +36 -41
- tools/migrate/tests/test_migration_e2e.py +28 -29
- tools/migrate/tests/test_node_migrations.py +50 -20
- flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
- flowfile/web/static/assets/ContextMenu-23e909da.js +0 -41
- flowfile/web/static/assets/ContextMenu-4c74eef1.css +0 -26
- flowfile/web/static/assets/ContextMenu-63cfa99b.css +0 -26
- flowfile/web/static/assets/ContextMenu-70ae0c79.js +0 -41
- flowfile/web/static/assets/ContextMenu-c13f91d0.css +0 -26
- flowfile/web/static/assets/ContextMenu-f149cf7c.js +0 -41
- flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
- flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
- flowfile/web/static/assets/Filter-9b6d08db.js +0 -164
- flowfile/web/static/assets/Filter-f62091b3.css +0 -20
- flowfile/web/static/assets/GroupBy-b9505323.css +0 -51
- flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
- flowfile/web/static/assets/Output-283fe388.css +0 -37
- flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
- flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
- flowfile/web/static/assets/SQLQueryComponent-36cef432.css +0 -27
- flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
- flowfile/web/static/assets/Sort-3643d625.css +0 -51
- flowfile/web/static/assets/Unique-f9fb0809.css +0 -51
- flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
- flowfile/web/static/assets/nodeInput-5d0d6b79.js +0 -41
- flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
- flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
- flowfile/web/static/assets/secretApi-68435402.js +0 -46
- flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
- flowfile-0.5.1.dist-info/RECORD +0 -388
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/WHEEL +0 -0
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/entry_points.txt +0 -0
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,71 +1,102 @@
|
|
|
1
|
-
from typing import List, Optional, Literal, Iterator, Any, Annotated
|
|
2
|
-
from flowfile_core.schemas import transform_schema
|
|
3
|
-
from pathlib import Path
|
|
4
1
|
import os
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Annotated, Any, Literal
|
|
4
|
+
|
|
5
|
+
import polars as pl
|
|
6
|
+
from pydantic import (
|
|
7
|
+
BaseModel,
|
|
8
|
+
ConfigDict,
|
|
9
|
+
Field,
|
|
10
|
+
SecretStr,
|
|
11
|
+
StringConstraints,
|
|
12
|
+
ValidationInfo,
|
|
13
|
+
field_validator,
|
|
14
|
+
model_validator,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
from flowfile_core.schemas import transform_schema
|
|
5
18
|
from flowfile_core.schemas.analysis_schemas import graphic_walker_schemas as gs_schemas
|
|
6
19
|
from flowfile_core.schemas.cloud_storage_schemas import CloudStorageReadSettings, CloudStorageWriteSettings
|
|
7
20
|
from flowfile_core.schemas.yaml_types import (
|
|
8
|
-
|
|
9
|
-
|
|
21
|
+
NodeCrossJoinYaml,
|
|
22
|
+
NodeFuzzyMatchYaml,
|
|
23
|
+
NodeJoinYaml,
|
|
24
|
+
NodeOutputYaml,
|
|
25
|
+
NodeSelectYaml,
|
|
26
|
+
OutputSettingsYaml,
|
|
10
27
|
)
|
|
11
28
|
from flowfile_core.utils.utils import ensure_similarity_dicts, standardize_col_dtype
|
|
12
|
-
from pydantic import (BaseModel, Field, model_validator, field_validator,
|
|
13
|
-
SecretStr, ConfigDict, StringConstraints, ValidationInfo)
|
|
14
|
-
import polars as pl
|
|
15
|
-
|
|
16
29
|
|
|
17
|
-
SecretRef = Annotated[
|
|
18
|
-
|
|
30
|
+
SecretRef = Annotated[
|
|
31
|
+
str, StringConstraints(min_length=1, max_length=100), Field(description="An ID referencing an encrypted secret.")
|
|
32
|
+
]
|
|
19
33
|
|
|
20
34
|
|
|
21
|
-
OutputConnectionClass = Literal[
|
|
22
|
-
|
|
35
|
+
OutputConnectionClass = Literal[
|
|
36
|
+
"output-0",
|
|
37
|
+
"output-1",
|
|
38
|
+
"output-2",
|
|
39
|
+
"output-3",
|
|
40
|
+
"output-4",
|
|
41
|
+
"output-5",
|
|
42
|
+
"output-6",
|
|
43
|
+
"output-7",
|
|
44
|
+
"output-8",
|
|
45
|
+
"output-9",
|
|
46
|
+
]
|
|
23
47
|
|
|
24
|
-
InputConnectionClass = Literal[
|
|
25
|
-
|
|
48
|
+
InputConnectionClass = Literal[
|
|
49
|
+
"input-0", "input-1", "input-2", "input-3", "input-4", "input-5", "input-6", "input-7", "input-8", "input-9"
|
|
50
|
+
]
|
|
26
51
|
|
|
27
52
|
InputType = Literal["main", "left", "right"]
|
|
28
53
|
|
|
29
54
|
|
|
30
55
|
class NewDirectory(BaseModel):
|
|
31
56
|
"""Defines the information required to create a new directory."""
|
|
57
|
+
|
|
32
58
|
source_path: str
|
|
33
59
|
dir_name: str
|
|
34
60
|
|
|
35
61
|
|
|
36
62
|
class RemoveItem(BaseModel):
|
|
37
63
|
"""Represents a single item to be removed from a directory or list."""
|
|
64
|
+
|
|
38
65
|
path: str
|
|
39
66
|
id: int = -1
|
|
40
67
|
|
|
41
68
|
|
|
42
69
|
class RemoveItemsInput(BaseModel):
|
|
43
70
|
"""Defines a list of items to be removed."""
|
|
44
|
-
|
|
71
|
+
|
|
72
|
+
paths: list[RemoveItem]
|
|
45
73
|
source_path: str
|
|
46
74
|
|
|
47
75
|
|
|
48
76
|
class MinimalFieldInfo(BaseModel):
|
|
49
77
|
"""Represents the most basic information about a data field (column)."""
|
|
78
|
+
|
|
50
79
|
name: str
|
|
51
80
|
data_type: str = "String"
|
|
52
81
|
|
|
53
82
|
|
|
54
83
|
class InputTableBase(BaseModel):
|
|
55
84
|
"""Base settings for input file operations."""
|
|
85
|
+
|
|
56
86
|
file_type: str # Will be overridden with Literal in subclasses
|
|
57
87
|
|
|
58
88
|
|
|
59
89
|
class InputCsvTable(InputTableBase):
|
|
60
90
|
"""Defines settings for reading a CSV file."""
|
|
61
|
-
|
|
62
|
-
|
|
91
|
+
|
|
92
|
+
file_type: Literal["csv"] = "csv"
|
|
93
|
+
reference: str = ""
|
|
63
94
|
starting_from_line: int = 0
|
|
64
|
-
delimiter: str =
|
|
95
|
+
delimiter: str = ","
|
|
65
96
|
has_headers: bool = True
|
|
66
|
-
encoding: str =
|
|
67
|
-
parquet_ref:
|
|
68
|
-
row_delimiter: str =
|
|
97
|
+
encoding: str = "utf-8"
|
|
98
|
+
parquet_ref: str | None = None
|
|
99
|
+
row_delimiter: str = "\n"
|
|
69
100
|
quote_char: str = '"'
|
|
70
101
|
infer_schema_length: int = 10_000
|
|
71
102
|
truncate_ragged_lines: bool = False
|
|
@@ -74,18 +105,21 @@ class InputCsvTable(InputTableBase):
|
|
|
74
105
|
|
|
75
106
|
class InputJsonTable(InputCsvTable):
|
|
76
107
|
"""Defines settings for reading a JSON file."""
|
|
77
|
-
|
|
108
|
+
|
|
109
|
+
file_type: Literal["json"] = "json"
|
|
78
110
|
|
|
79
111
|
|
|
80
112
|
class InputParquetTable(InputTableBase):
|
|
81
113
|
"""Defines settings for reading a Parquet file."""
|
|
82
|
-
|
|
114
|
+
|
|
115
|
+
file_type: Literal["parquet"] = "parquet"
|
|
83
116
|
|
|
84
117
|
|
|
85
118
|
class InputExcelTable(InputTableBase):
|
|
86
119
|
"""Defines settings for reading an Excel file."""
|
|
87
|
-
|
|
88
|
-
|
|
120
|
+
|
|
121
|
+
file_type: Literal["excel"] = "excel"
|
|
122
|
+
sheet_name: str | None = None
|
|
89
123
|
start_row: int = 0
|
|
90
124
|
start_column: int = 0
|
|
91
125
|
end_row: int = 0
|
|
@@ -93,60 +127,58 @@ class InputExcelTable(InputTableBase):
|
|
|
93
127
|
has_headers: bool = True
|
|
94
128
|
type_inference: bool = False
|
|
95
129
|
|
|
96
|
-
@model_validator(mode=
|
|
130
|
+
@model_validator(mode="after")
|
|
97
131
|
def validate_range_values(self):
|
|
98
132
|
"""Validates that the Excel cell range is logical."""
|
|
99
133
|
for attribute in [self.start_row, self.start_column, self.end_row, self.end_column]:
|
|
100
134
|
if not isinstance(attribute, int) or attribute < 0:
|
|
101
135
|
raise ValueError("Row and column indices must be non-negative integers")
|
|
102
|
-
if (self.end_row > 0 and self.start_row > self.end_row) or
|
|
103
|
-
|
|
136
|
+
if (self.end_row > 0 and self.start_row > self.end_row) or (
|
|
137
|
+
self.end_column > 0 and self.start_column > self.end_column
|
|
138
|
+
):
|
|
104
139
|
raise ValueError("Start row/column must not be greater than end row/column")
|
|
105
140
|
return self
|
|
106
141
|
|
|
107
142
|
|
|
108
143
|
# Create the discriminated union (similar to OutputTableSettings)
|
|
109
144
|
InputTableSettings = Annotated[
|
|
110
|
-
InputCsvTable | InputJsonTable | InputParquetTable | InputExcelTable,
|
|
111
|
-
Field(discriminator='file_type')
|
|
145
|
+
InputCsvTable | InputJsonTable | InputParquetTable | InputExcelTable, Field(discriminator="file_type")
|
|
112
146
|
]
|
|
113
147
|
|
|
114
148
|
|
|
115
149
|
# Now create the main ReceivedTable model
|
|
116
150
|
class ReceivedTable(BaseModel):
|
|
117
151
|
"""Model for defining a table received from an external source."""
|
|
152
|
+
|
|
118
153
|
# Metadata fields
|
|
119
|
-
id:
|
|
120
|
-
name:
|
|
154
|
+
id: int | None = None
|
|
155
|
+
name: str | None = None
|
|
121
156
|
path: str # This can be an absolute or relative path
|
|
122
|
-
directory:
|
|
157
|
+
directory: str | None = None
|
|
123
158
|
analysis_file_available: bool = False
|
|
124
|
-
status:
|
|
125
|
-
fields:
|
|
126
|
-
abs_file_path:
|
|
159
|
+
status: str | None = None
|
|
160
|
+
fields: list[MinimalFieldInfo] = Field(default_factory=list)
|
|
161
|
+
abs_file_path: str | None = None
|
|
127
162
|
|
|
128
|
-
file_type: Literal[
|
|
163
|
+
file_type: Literal["csv", "json", "parquet", "excel"]
|
|
129
164
|
|
|
130
165
|
table_settings: InputTableSettings
|
|
131
166
|
|
|
132
167
|
@classmethod
|
|
133
|
-
def create_from_path(cls, path: str, file_type: Literal[
|
|
168
|
+
def create_from_path(cls, path: str, file_type: Literal["csv", "json", "parquet", "excel"] = "csv"):
|
|
134
169
|
"""Creates an instance from a file path string."""
|
|
135
170
|
filename = Path(path).name
|
|
136
171
|
|
|
137
172
|
# Create appropriate table_settings based on file_type
|
|
138
173
|
settings_map = {
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
174
|
+
"csv": InputCsvTable(),
|
|
175
|
+
"json": InputJsonTable(),
|
|
176
|
+
"parquet": InputParquetTable(),
|
|
177
|
+
"excel": InputExcelTable(),
|
|
143
178
|
}
|
|
144
179
|
|
|
145
180
|
return cls(
|
|
146
|
-
name=filename,
|
|
147
|
-
path=path,
|
|
148
|
-
file_type=file_type,
|
|
149
|
-
table_settings=settings_map.get(file_type, InputCsvTable())
|
|
181
|
+
name=filename, path=path, file_type=file_type, table_settings=settings_map.get(file_type, InputCsvTable())
|
|
150
182
|
)
|
|
151
183
|
|
|
152
184
|
@property
|
|
@@ -166,19 +198,19 @@ class ReceivedTable(BaseModel):
|
|
|
166
198
|
base_path = base_path / self.name
|
|
167
199
|
self.abs_file_path = str(base_path.resolve())
|
|
168
200
|
|
|
169
|
-
@model_validator(mode=
|
|
201
|
+
@model_validator(mode="before")
|
|
170
202
|
@classmethod
|
|
171
203
|
def set_default_table_settings(cls, data):
|
|
172
204
|
"""Create default table_settings based on file_type if not provided."""
|
|
173
205
|
if isinstance(data, dict):
|
|
174
|
-
if
|
|
175
|
-
data[
|
|
206
|
+
if "table_settings" not in data or data["table_settings"] is None:
|
|
207
|
+
data["table_settings"] = {}
|
|
176
208
|
|
|
177
|
-
if isinstance(data[
|
|
178
|
-
data[
|
|
209
|
+
if isinstance(data["table_settings"], dict) and "file_type" not in data["table_settings"]:
|
|
210
|
+
data["table_settings"]["file_type"] = data.get("file_type", "csv")
|
|
179
211
|
return data
|
|
180
212
|
|
|
181
|
-
@model_validator(mode=
|
|
213
|
+
@model_validator(mode="after")
|
|
182
214
|
def populate_abs_file_path(self):
|
|
183
215
|
"""Ensures the absolute file path is populated after validation."""
|
|
184
216
|
if not self.abs_file_path:
|
|
@@ -188,38 +220,41 @@ class ReceivedTable(BaseModel):
|
|
|
188
220
|
|
|
189
221
|
class OutputCsvTable(BaseModel):
|
|
190
222
|
"""Defines settings for writing a CSV file."""
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
223
|
+
|
|
224
|
+
file_type: Literal["csv"] = "csv"
|
|
225
|
+
delimiter: str = ","
|
|
226
|
+
encoding: str = "utf-8"
|
|
194
227
|
|
|
195
228
|
|
|
196
229
|
class OutputParquetTable(BaseModel):
|
|
197
230
|
"""Defines settings for writing a Parquet file."""
|
|
198
|
-
|
|
231
|
+
|
|
232
|
+
file_type: Literal["parquet"] = "parquet"
|
|
199
233
|
|
|
200
234
|
|
|
201
235
|
class OutputExcelTable(BaseModel):
|
|
202
236
|
"""Defines settings for writing an Excel file."""
|
|
203
|
-
|
|
204
|
-
|
|
237
|
+
|
|
238
|
+
file_type: Literal["excel"] = "excel"
|
|
239
|
+
sheet_name: str = "Sheet1"
|
|
205
240
|
|
|
206
241
|
|
|
207
242
|
# Create a discriminated union
|
|
208
243
|
OutputTableSettings = Annotated[
|
|
209
|
-
OutputCsvTable | OutputParquetTable | OutputExcelTable,
|
|
210
|
-
Field(discriminator='file_type')
|
|
244
|
+
OutputCsvTable | OutputParquetTable | OutputExcelTable, Field(discriminator="file_type")
|
|
211
245
|
]
|
|
212
246
|
|
|
213
247
|
|
|
214
248
|
class OutputSettings(BaseModel):
|
|
215
249
|
"""Defines the complete settings for an output node."""
|
|
250
|
+
|
|
216
251
|
name: str
|
|
217
252
|
directory: str
|
|
218
253
|
file_type: str # This drives which table_settings to use
|
|
219
|
-
fields:
|
|
220
|
-
write_mode: str =
|
|
254
|
+
fields: list[str] | None = Field(default_factory=list)
|
|
255
|
+
write_mode: str = "overwrite"
|
|
221
256
|
table_settings: OutputTableSettings
|
|
222
|
-
abs_file_path:
|
|
257
|
+
abs_file_path: str | None = None
|
|
223
258
|
|
|
224
259
|
def to_yaml_dict(self) -> OutputSettingsYaml:
|
|
225
260
|
"""Converts the output settings to a dictionary suitable for YAML serialization."""
|
|
@@ -241,34 +276,34 @@ class OutputSettings(BaseModel):
|
|
|
241
276
|
|
|
242
277
|
@property
|
|
243
278
|
def sheet_name(self) -> str | None:
|
|
244
|
-
if self.file_type ==
|
|
279
|
+
if self.file_type == "excel":
|
|
245
280
|
return self.table_settings.sheet_name
|
|
246
281
|
|
|
247
282
|
@property
|
|
248
283
|
def delimiter(self) -> str | None:
|
|
249
|
-
if self.file_type ==
|
|
284
|
+
if self.file_type == "csv":
|
|
250
285
|
return self.table_settings.delimiter
|
|
251
286
|
|
|
252
|
-
@field_validator(
|
|
287
|
+
@field_validator("table_settings", mode="before")
|
|
253
288
|
@classmethod
|
|
254
289
|
def validate_table_settings(cls, v, info: ValidationInfo):
|
|
255
290
|
"""Ensures table_settings matches the file_type."""
|
|
256
291
|
if v is None:
|
|
257
|
-
file_type = info.data.get(
|
|
292
|
+
file_type = info.data.get("file_type", "csv")
|
|
258
293
|
# Create default based on file_type
|
|
259
294
|
match file_type:
|
|
260
|
-
case
|
|
295
|
+
case "csv":
|
|
261
296
|
return OutputCsvTable()
|
|
262
|
-
case
|
|
297
|
+
case "parquet":
|
|
263
298
|
return OutputParquetTable()
|
|
264
|
-
case
|
|
299
|
+
case "excel":
|
|
265
300
|
return OutputExcelTable()
|
|
266
301
|
case _:
|
|
267
302
|
return OutputCsvTable()
|
|
268
303
|
|
|
269
304
|
# If it's a dict, add file_type if missing
|
|
270
|
-
if isinstance(v, dict) and
|
|
271
|
-
v[
|
|
305
|
+
if isinstance(v, dict) and "file_type" not in v:
|
|
306
|
+
v["file_type"] = info.data.get("file_type", "csv")
|
|
272
307
|
|
|
273
308
|
return v
|
|
274
309
|
|
|
@@ -281,7 +316,7 @@ class OutputSettings(BaseModel):
|
|
|
281
316
|
base_path = base_path / self.name
|
|
282
317
|
self.abs_file_path = str(base_path.resolve())
|
|
283
318
|
|
|
284
|
-
@model_validator(mode=
|
|
319
|
+
@model_validator(mode="after")
|
|
285
320
|
def populate_abs_file_path(self):
|
|
286
321
|
"""Ensures the absolute file path is populated after validation."""
|
|
287
322
|
self.set_absolute_filepath()
|
|
@@ -290,34 +325,38 @@ class OutputSettings(BaseModel):
|
|
|
290
325
|
|
|
291
326
|
class NodeBase(BaseModel):
|
|
292
327
|
"""Base model for all nodes in a FlowGraph. Contains common metadata."""
|
|
328
|
+
|
|
293
329
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
294
330
|
flow_id: int
|
|
295
331
|
node_id: int
|
|
296
|
-
cache_results:
|
|
297
|
-
pos_x:
|
|
298
|
-
pos_y:
|
|
299
|
-
is_setup:
|
|
300
|
-
description:
|
|
301
|
-
user_id:
|
|
302
|
-
is_flow_output:
|
|
303
|
-
is_user_defined:
|
|
332
|
+
cache_results: bool | None = False
|
|
333
|
+
pos_x: float | None = 0
|
|
334
|
+
pos_y: float | None = 0
|
|
335
|
+
is_setup: bool | None = True
|
|
336
|
+
description: str | None = ""
|
|
337
|
+
user_id: int | None = None
|
|
338
|
+
is_flow_output: bool | None = False
|
|
339
|
+
is_user_defined: bool | None = False # Indicator if the node is a user defined node
|
|
304
340
|
|
|
305
341
|
|
|
306
342
|
class NodeSingleInput(NodeBase):
|
|
307
343
|
"""A base model for any node that takes a single data input."""
|
|
308
|
-
|
|
344
|
+
|
|
345
|
+
depending_on_id: int | None = -1
|
|
309
346
|
|
|
310
347
|
|
|
311
348
|
class NodeMultiInput(NodeBase):
|
|
312
349
|
"""A base model for any node that takes multiple data inputs."""
|
|
313
|
-
|
|
350
|
+
|
|
351
|
+
depending_on_ids: list[int] | None = Field(default_factory=list)
|
|
314
352
|
|
|
315
353
|
|
|
316
354
|
class NodeSelect(NodeSingleInput):
|
|
317
355
|
"""Settings for a node that selects, renames, and reorders columns."""
|
|
356
|
+
|
|
318
357
|
keep_missing: bool = True
|
|
319
|
-
select_input:
|
|
320
|
-
sorted_by:
|
|
358
|
+
select_input: list[transform_schema.SelectInput] = Field(default_factory=list)
|
|
359
|
+
sorted_by: Literal["none", "asc", "desc"] | None = "none"
|
|
321
360
|
|
|
322
361
|
def to_yaml_dict(self) -> NodeSelectYaml:
|
|
323
362
|
"""Converts the select node settings to a dictionary for YAML serialization."""
|
|
@@ -331,31 +370,37 @@ class NodeSelect(NodeSingleInput):
|
|
|
331
370
|
|
|
332
371
|
class NodeFilter(NodeSingleInput):
|
|
333
372
|
"""Settings for a node that filters rows based on a condition."""
|
|
373
|
+
|
|
334
374
|
filter_input: transform_schema.FilterInput
|
|
335
375
|
|
|
336
376
|
|
|
337
377
|
class NodeSort(NodeSingleInput):
|
|
338
378
|
"""Settings for a node that sorts the data by one or more columns."""
|
|
339
|
-
|
|
379
|
+
|
|
380
|
+
sort_input: list[transform_schema.SortByInput] = Field(default_factory=list)
|
|
340
381
|
|
|
341
382
|
|
|
342
383
|
class NodeTextToRows(NodeSingleInput):
|
|
343
384
|
"""Settings for a node that splits a text column into multiple rows."""
|
|
385
|
+
|
|
344
386
|
text_to_rows_input: transform_schema.TextToRowsInput
|
|
345
387
|
|
|
346
388
|
|
|
347
389
|
class NodeSample(NodeSingleInput):
|
|
348
390
|
"""Settings for a node that samples a subset of the data."""
|
|
391
|
+
|
|
349
392
|
sample_size: int = 1000
|
|
350
393
|
|
|
351
394
|
|
|
352
395
|
class NodeRecordId(NodeSingleInput):
|
|
353
396
|
"""Settings for a node that adds a unique record ID column."""
|
|
397
|
+
|
|
354
398
|
record_id_input: transform_schema.RecordIdInput
|
|
355
399
|
|
|
356
400
|
|
|
357
401
|
class NodeJoin(NodeMultiInput):
|
|
358
402
|
"""Settings for a node that performs a standard SQL-style join."""
|
|
403
|
+
|
|
359
404
|
auto_generate_selection: bool = True
|
|
360
405
|
verify_integrity: bool = True
|
|
361
406
|
join_input: transform_schema.JoinInput
|
|
@@ -378,6 +423,7 @@ class NodeJoin(NodeMultiInput):
|
|
|
378
423
|
|
|
379
424
|
class NodeCrossJoin(NodeMultiInput):
|
|
380
425
|
"""Settings for a node that performs a cross join."""
|
|
426
|
+
|
|
381
427
|
auto_generate_selection: bool = True
|
|
382
428
|
verify_integrity: bool = True
|
|
383
429
|
cross_join_input: transform_schema.CrossJoinInput
|
|
@@ -400,6 +446,7 @@ class NodeCrossJoin(NodeMultiInput):
|
|
|
400
446
|
|
|
401
447
|
class NodeFuzzyMatch(NodeJoin):
|
|
402
448
|
"""Settings for a node that performs a fuzzy join based on string similarity."""
|
|
449
|
+
|
|
403
450
|
join_input: transform_schema.FuzzyMatchInput
|
|
404
451
|
|
|
405
452
|
def to_yaml_dict(self) -> NodeFuzzyMatchYaml:
|
|
@@ -417,98 +464,105 @@ class NodeFuzzyMatch(NodeJoin):
|
|
|
417
464
|
|
|
418
465
|
class NodeDatasource(NodeBase):
|
|
419
466
|
"""Base settings for a node that acts as a data source."""
|
|
467
|
+
|
|
420
468
|
file_ref: str = None
|
|
421
469
|
|
|
422
470
|
|
|
423
471
|
class RawData(BaseModel):
|
|
424
472
|
"""Represents data in a raw, columnar format for manual input."""
|
|
425
|
-
|
|
426
|
-
|
|
473
|
+
|
|
474
|
+
columns: list[MinimalFieldInfo] = None
|
|
475
|
+
data: list[list]
|
|
427
476
|
|
|
428
477
|
@classmethod
|
|
429
|
-
def from_pylist(cls, pylist:
|
|
478
|
+
def from_pylist(cls, pylist: list[dict]):
|
|
430
479
|
"""Creates a RawData object from a list of Python dictionaries."""
|
|
431
480
|
if len(pylist) == 0:
|
|
432
481
|
return cls(columns=[], data=[])
|
|
433
482
|
pylist = ensure_similarity_dicts(pylist)
|
|
434
|
-
values = [standardize_col_dtype([vv for vv in c]) for c in
|
|
435
|
-
zip(*(r.values() for r in pylist))]
|
|
483
|
+
values = [standardize_col_dtype([vv for vv in c]) for c in zip(*(r.values() for r in pylist), strict=False)]
|
|
436
484
|
data_types = (pl.DataType.from_python(type(next((v for v in column_values), None))) for column_values in values)
|
|
437
485
|
columns = [MinimalFieldInfo(name=c, data_type=str(next(data_types))) for c in pylist[0].keys()]
|
|
438
486
|
return cls(columns=columns, data=values)
|
|
439
487
|
|
|
440
|
-
def to_pylist(self) ->
|
|
488
|
+
def to_pylist(self) -> list[dict]:
|
|
441
489
|
"""Converts the RawData object back into a list of Python dictionaries."""
|
|
442
490
|
return [{c.name: self.data[ci][ri] for ci, c in enumerate(self.columns)} for ri in range(len(self.data[0]))]
|
|
443
491
|
|
|
444
492
|
|
|
445
493
|
class NodeManualInput(NodeBase):
|
|
446
494
|
"""Settings for a node that allows direct data entry in the UI."""
|
|
447
|
-
|
|
495
|
+
|
|
496
|
+
raw_data_format: RawData | None = None
|
|
448
497
|
|
|
449
498
|
|
|
450
499
|
class NodeRead(NodeBase):
|
|
451
500
|
"""Settings for a node that reads data from a file."""
|
|
501
|
+
|
|
452
502
|
received_file: ReceivedTable
|
|
453
503
|
|
|
454
504
|
|
|
455
505
|
class DatabaseConnection(BaseModel):
|
|
456
506
|
"""Defines the connection parameters for a database."""
|
|
507
|
+
|
|
457
508
|
database_type: str = "postgresql"
|
|
458
|
-
username:
|
|
459
|
-
password_ref:
|
|
460
|
-
host:
|
|
461
|
-
port:
|
|
462
|
-
database:
|
|
463
|
-
url:
|
|
509
|
+
username: str | None = None
|
|
510
|
+
password_ref: SecretRef | None = None
|
|
511
|
+
host: str | None = None
|
|
512
|
+
port: int | None = None
|
|
513
|
+
database: str | None = None
|
|
514
|
+
url: str | None = None
|
|
464
515
|
|
|
465
516
|
|
|
466
517
|
class FullDatabaseConnection(BaseModel):
|
|
467
518
|
"""A complete database connection model including the secret password."""
|
|
519
|
+
|
|
468
520
|
connection_name: str
|
|
469
521
|
database_type: str = "postgresql"
|
|
470
522
|
username: str
|
|
471
523
|
password: SecretStr
|
|
472
|
-
host:
|
|
473
|
-
port:
|
|
474
|
-
database:
|
|
475
|
-
ssl_enabled:
|
|
476
|
-
url:
|
|
524
|
+
host: str | None = None
|
|
525
|
+
port: int | None = None
|
|
526
|
+
database: str | None = None
|
|
527
|
+
ssl_enabled: bool | None = False
|
|
528
|
+
url: str | None = None
|
|
477
529
|
|
|
478
530
|
|
|
479
531
|
class FullDatabaseConnectionInterface(BaseModel):
|
|
480
532
|
"""A database connection model intended for UI display, omitting the password."""
|
|
533
|
+
|
|
481
534
|
connection_name: str
|
|
482
535
|
database_type: str = "postgresql"
|
|
483
536
|
username: str
|
|
484
|
-
host:
|
|
485
|
-
port:
|
|
486
|
-
database:
|
|
487
|
-
ssl_enabled:
|
|
488
|
-
url:
|
|
537
|
+
host: str | None = None
|
|
538
|
+
port: int | None = None
|
|
539
|
+
database: str | None = None
|
|
540
|
+
ssl_enabled: bool | None = False
|
|
541
|
+
url: str | None = None
|
|
489
542
|
|
|
490
543
|
|
|
491
544
|
class DatabaseSettings(BaseModel):
|
|
492
545
|
"""Defines settings for reading from a database, either via table or query."""
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
546
|
+
|
|
547
|
+
connection_mode: Literal["inline", "reference"] | None = "inline"
|
|
548
|
+
database_connection: DatabaseConnection | None = None
|
|
549
|
+
database_connection_name: str | None = None
|
|
550
|
+
schema_name: str | None = None
|
|
551
|
+
table_name: str | None = None
|
|
552
|
+
query: str | None = None
|
|
553
|
+
query_mode: Literal["query", "table", "reference"] = "table"
|
|
554
|
+
|
|
555
|
+
@model_validator(mode="after")
|
|
502
556
|
def validate_table_or_query(self):
|
|
503
557
|
# Validate that either table_name or query is provided
|
|
504
|
-
if (not self.table_name and not self.query) and self.query_mode ==
|
|
558
|
+
if (not self.table_name and not self.query) and self.query_mode == "inline":
|
|
505
559
|
raise ValueError("Either 'table_name' or 'query' must be provided")
|
|
506
560
|
|
|
507
561
|
# Validate correct connection information based on connection_mode
|
|
508
|
-
if self.connection_mode ==
|
|
562
|
+
if self.connection_mode == "inline" and self.database_connection is None:
|
|
509
563
|
raise ValueError("When 'connection_mode' is 'inline', 'database_connection' must be provided")
|
|
510
564
|
|
|
511
|
-
if self.connection_mode ==
|
|
565
|
+
if self.connection_mode == "reference" and not self.database_connection_name:
|
|
512
566
|
raise ValueError("When 'connection_mode' is 'reference', 'database_connection_name' must be provided")
|
|
513
567
|
|
|
514
568
|
return self
|
|
@@ -516,44 +570,51 @@ class DatabaseSettings(BaseModel):
|
|
|
516
570
|
|
|
517
571
|
class DatabaseWriteSettings(BaseModel):
|
|
518
572
|
"""Defines settings for writing data to a database table."""
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
573
|
+
|
|
574
|
+
connection_mode: Literal["inline", "reference"] | None = "inline"
|
|
575
|
+
database_connection: DatabaseConnection | None = None
|
|
576
|
+
database_connection_name: str | None = None
|
|
522
577
|
table_name: str
|
|
523
|
-
schema_name:
|
|
524
|
-
if_exists:
|
|
578
|
+
schema_name: str | None = None
|
|
579
|
+
if_exists: Literal["append", "replace", "fail"] | None = "append"
|
|
525
580
|
|
|
526
581
|
|
|
527
582
|
class NodeDatabaseReader(NodeBase):
|
|
528
583
|
"""Settings for a node that reads from a database."""
|
|
584
|
+
|
|
529
585
|
database_settings: DatabaseSettings
|
|
530
|
-
fields:
|
|
586
|
+
fields: list[MinimalFieldInfo] | None = None
|
|
531
587
|
|
|
532
588
|
|
|
533
589
|
class NodeDatabaseWriter(NodeSingleInput):
|
|
534
590
|
"""Settings for a node that writes data to a database."""
|
|
591
|
+
|
|
535
592
|
database_write_settings: DatabaseWriteSettings
|
|
536
593
|
|
|
537
594
|
|
|
538
595
|
class NodeCloudStorageReader(NodeBase):
|
|
539
596
|
"""Settings for a node that reads from a cloud storage service (S3, GCS, etc.)."""
|
|
597
|
+
|
|
540
598
|
cloud_storage_settings: CloudStorageReadSettings
|
|
541
|
-
fields:
|
|
599
|
+
fields: list[MinimalFieldInfo] | None = None
|
|
542
600
|
|
|
543
601
|
|
|
544
602
|
class NodeCloudStorageWriter(NodeSingleInput):
|
|
545
603
|
"""Settings for a node that writes to a cloud storage service."""
|
|
604
|
+
|
|
546
605
|
cloud_storage_settings: CloudStorageWriteSettings
|
|
547
606
|
|
|
548
607
|
|
|
549
608
|
class ExternalSource(BaseModel):
|
|
550
609
|
"""Base model for data coming from a predefined external source."""
|
|
551
|
-
|
|
552
|
-
|
|
610
|
+
|
|
611
|
+
orientation: str = "row"
|
|
612
|
+
fields: list[MinimalFieldInfo] | None = None
|
|
553
613
|
|
|
554
614
|
|
|
555
615
|
class SampleUsers(ExternalSource):
|
|
556
616
|
"""Settings for generating a sample dataset of users."""
|
|
617
|
+
|
|
557
618
|
SAMPLE_USERS: bool
|
|
558
619
|
class_name: str = "sample_users"
|
|
559
620
|
size: int = 100
|
|
@@ -561,58 +622,71 @@ class SampleUsers(ExternalSource):
|
|
|
561
622
|
|
|
562
623
|
class NodeExternalSource(NodeBase):
|
|
563
624
|
"""Settings for a node that connects to a registered external data source."""
|
|
625
|
+
|
|
564
626
|
identifier: str
|
|
565
627
|
source_settings: SampleUsers
|
|
566
628
|
|
|
567
629
|
|
|
568
630
|
class NodeFormula(NodeSingleInput):
|
|
569
631
|
"""Settings for a node that applies a formula to create/modify a column."""
|
|
632
|
+
|
|
570
633
|
function: transform_schema.FunctionInput = None
|
|
571
634
|
|
|
572
635
|
|
|
573
636
|
class NodeGroupBy(NodeSingleInput):
|
|
574
637
|
"""Settings for a node that performs a group-by and aggregation operation."""
|
|
638
|
+
|
|
575
639
|
groupby_input: transform_schema.GroupByInput = None
|
|
576
640
|
|
|
577
641
|
|
|
578
642
|
class NodePromise(NodeBase):
|
|
579
643
|
"""A placeholder node for an operation that has not yet been configured."""
|
|
644
|
+
|
|
580
645
|
is_setup: bool = False
|
|
581
646
|
node_type: str
|
|
582
647
|
|
|
583
648
|
|
|
584
649
|
class NodeInputConnection(BaseModel):
|
|
585
650
|
"""Represents the input side of a connection between two nodes."""
|
|
651
|
+
|
|
586
652
|
node_id: int
|
|
587
653
|
connection_class: InputConnectionClass
|
|
588
654
|
|
|
589
|
-
def get_node_input_connection_type(self) -> Literal[
|
|
655
|
+
def get_node_input_connection_type(self) -> Literal["main", "right", "left"]:
|
|
590
656
|
"""Determines the semantic type of the input (e.g., for a join)."""
|
|
591
657
|
match self.connection_class:
|
|
592
|
-
case
|
|
593
|
-
|
|
594
|
-
case
|
|
595
|
-
|
|
658
|
+
case "input-0":
|
|
659
|
+
return "main"
|
|
660
|
+
case "input-1":
|
|
661
|
+
return "right"
|
|
662
|
+
case "input-2":
|
|
663
|
+
return "left"
|
|
664
|
+
case _:
|
|
665
|
+
raise ValueError(f"Unexpected connection_class: {self.connection_class}")
|
|
596
666
|
|
|
597
667
|
|
|
598
668
|
class NodePivot(NodeSingleInput):
|
|
599
669
|
"""Settings for a node that pivots data from a long to a wide format."""
|
|
670
|
+
|
|
600
671
|
pivot_input: transform_schema.PivotInput = None
|
|
601
|
-
output_fields:
|
|
672
|
+
output_fields: list[MinimalFieldInfo] | None = None
|
|
602
673
|
|
|
603
674
|
|
|
604
675
|
class NodeUnpivot(NodeSingleInput):
|
|
605
676
|
"""Settings for a node that unpivots data from a wide to a long format."""
|
|
677
|
+
|
|
606
678
|
unpivot_input: transform_schema.UnpivotInput = None
|
|
607
679
|
|
|
608
680
|
|
|
609
681
|
class NodeUnion(NodeMultiInput):
|
|
610
682
|
"""Settings for a node that concatenates multiple data inputs."""
|
|
683
|
+
|
|
611
684
|
union_input: transform_schema.UnionInput = Field(default_factory=transform_schema.UnionInput)
|
|
612
685
|
|
|
613
686
|
|
|
614
687
|
class NodeOutput(NodeSingleInput):
|
|
615
688
|
"""Settings for a node that writes its input to a file."""
|
|
689
|
+
|
|
616
690
|
output_settings: OutputSettings
|
|
617
691
|
|
|
618
692
|
def to_yaml_dict(self) -> NodeOutputYaml:
|
|
@@ -625,12 +699,14 @@ class NodeOutput(NodeSingleInput):
|
|
|
625
699
|
|
|
626
700
|
class NodeOutputConnection(BaseModel):
|
|
627
701
|
"""Represents the output side of a connection between two nodes."""
|
|
702
|
+
|
|
628
703
|
node_id: int
|
|
629
704
|
connection_class: OutputConnectionClass
|
|
630
705
|
|
|
631
706
|
|
|
632
707
|
class NodeConnection(BaseModel):
|
|
633
708
|
"""Represents a connection (edge) between two nodes in the graph."""
|
|
709
|
+
|
|
634
710
|
input_connection: NodeInputConnection
|
|
635
711
|
output_connection: NodeOutputConnection
|
|
636
712
|
|
|
@@ -638,45 +714,56 @@ class NodeConnection(BaseModel):
|
|
|
638
714
|
def create_from_simple_input(cls, from_id: int, to_id: int, input_type: InputType = "input-0"):
|
|
639
715
|
"""Creates a standard connection between two nodes."""
|
|
640
716
|
match input_type:
|
|
641
|
-
case "main":
|
|
642
|
-
|
|
643
|
-
case "
|
|
644
|
-
|
|
717
|
+
case "main":
|
|
718
|
+
connection_class: InputConnectionClass = "input-0"
|
|
719
|
+
case "right":
|
|
720
|
+
connection_class: InputConnectionClass = "input-1"
|
|
721
|
+
case "left":
|
|
722
|
+
connection_class: InputConnectionClass = "input-2"
|
|
723
|
+
case _:
|
|
724
|
+
connection_class: InputConnectionClass = "input-0"
|
|
645
725
|
node_input = NodeInputConnection(node_id=to_id, connection_class=connection_class)
|
|
646
|
-
node_output = NodeOutputConnection(node_id=from_id, connection_class=
|
|
726
|
+
node_output = NodeOutputConnection(node_id=from_id, connection_class="output-0")
|
|
647
727
|
return cls(input_connection=node_input, output_connection=node_output)
|
|
648
728
|
|
|
649
729
|
|
|
650
730
|
class NodeDescription(BaseModel):
|
|
651
731
|
"""A simple model for updating a node's description text."""
|
|
652
|
-
|
|
732
|
+
|
|
733
|
+
description: str = ""
|
|
653
734
|
|
|
654
735
|
|
|
655
736
|
class NodeExploreData(NodeBase):
|
|
656
737
|
"""Settings for a node that provides an interactive data exploration interface."""
|
|
657
|
-
|
|
738
|
+
|
|
739
|
+
graphic_walker_input: gs_schemas.GraphicWalkerInput | None = None
|
|
658
740
|
|
|
659
741
|
|
|
660
742
|
class NodeGraphSolver(NodeSingleInput):
|
|
661
743
|
"""Settings for a node that solves graph-based problems (e.g., connected components)."""
|
|
744
|
+
|
|
662
745
|
graph_solver_input: transform_schema.GraphSolverInput
|
|
663
746
|
|
|
664
747
|
|
|
665
748
|
class NodeUnique(NodeSingleInput):
|
|
666
749
|
"""Settings for a node that returns the unique rows from the data."""
|
|
750
|
+
|
|
667
751
|
unique_input: transform_schema.UniqueInput
|
|
668
752
|
|
|
669
753
|
|
|
670
754
|
class NodeRecordCount(NodeSingleInput):
|
|
671
755
|
"""Settings for a node that counts the number of records."""
|
|
756
|
+
|
|
672
757
|
pass
|
|
673
758
|
|
|
674
759
|
|
|
675
760
|
class NodePolarsCode(NodeMultiInput):
|
|
676
761
|
"""Settings for a node that executes arbitrary user-provided Polars code."""
|
|
762
|
+
|
|
677
763
|
polars_code_input: transform_schema.PolarsCodeInput
|
|
678
764
|
|
|
679
765
|
|
|
680
766
|
class UserDefinedNode(NodeMultiInput):
|
|
681
767
|
"""Settings for a node that contains the user defined node information"""
|
|
768
|
+
|
|
682
769
|
settings: Any
|