Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- build_backends/main.py +25 -22
- build_backends/main_prd.py +10 -19
- flowfile/__init__.py +194 -74
- flowfile/__main__.py +10 -7
- flowfile/api.py +51 -57
- flowfile/web/__init__.py +14 -9
- flowfile/web/static/assets/AdminView-f53bad23.css +129 -0
- flowfile/web/static/assets/AdminView-f9847d67.js +713 -0
- flowfile/web/static/assets/CloudConnectionView-cf85f943.css +72 -0
- flowfile/web/static/assets/{CloudConnectionManager-0dfba9f2.js → CloudConnectionView-faace55b.js} +11 -11
- flowfile/web/static/assets/{CloudStorageReader-29d14fcc.css → CloudStorageReader-24c54524.css} +27 -27
- flowfile/web/static/assets/{CloudStorageReader-d5b1b6c9.js → CloudStorageReader-d86ecaa7.js} +10 -8
- flowfile/web/static/assets/{CloudStorageWriter-00d87aad.js → CloudStorageWriter-0f4d9a44.js} +10 -8
- flowfile/web/static/assets/{CloudStorageWriter-b0ee067f.css → CloudStorageWriter-60547855.css} +26 -26
- flowfile/web/static/assets/ColumnActionInput-c44b7aee.css +159 -0
- flowfile/web/static/assets/ColumnActionInput-f4189ae0.js +330 -0
- flowfile/web/static/assets/{ColumnSelector-47996a16.css → ColumnSelector-371637fb.css} +2 -2
- flowfile/web/static/assets/{ColumnSelector-4685e75d.js → ColumnSelector-e66b33da.js} +3 -5
- flowfile/web/static/assets/ContextMenu-49463352.js +9 -0
- flowfile/web/static/assets/ContextMenu-dd5f3f25.js +9 -0
- flowfile/web/static/assets/ContextMenu-f709b884.js +9 -0
- flowfile/web/static/assets/ContextMenu.vue_vue_type_script_setup_true_lang-a1bd6314.js +59 -0
- flowfile/web/static/assets/{CrossJoin-702a3edd.js → CrossJoin-24694b8f.js} +12 -10
- flowfile/web/static/assets/{CrossJoin-1119d18e.css → CrossJoin-71b4cc10.css} +20 -20
- flowfile/web/static/assets/{CustomNode-b1519993.js → CustomNode-569d45ff.js} +43 -24
- flowfile/web/static/assets/CustomNode-edb9b939.css +42 -0
- flowfile/web/static/assets/{DatabaseConnectionSettings-0c04b2e5.css → DatabaseConnectionSettings-c20a1e16.css} +23 -21
- flowfile/web/static/assets/{DatabaseConnectionSettings-6f3e4ea5.js → DatabaseConnectionSettings-cfc08938.js} +5 -4
- flowfile/web/static/assets/{DatabaseReader-ae61773c.css → DatabaseReader-5bf8c75b.css} +41 -46
- flowfile/web/static/assets/{DatabaseReader-d38c7295.js → DatabaseReader-701feabb.js} +25 -15
- flowfile/web/static/assets/{DatabaseManager-cf5ef661.js → DatabaseView-0482e5b5.js} +11 -11
- flowfile/web/static/assets/DatabaseView-6655afd6.css +57 -0
- flowfile/web/static/assets/{DatabaseWriter-b04ef46a.js → DatabaseWriter-16721989.js} +17 -10
- flowfile/web/static/assets/{DatabaseWriter-2f570e53.css → DatabaseWriter-bdcf2c8b.css} +29 -27
- flowfile/web/static/assets/{designer-8da3ba3a.css → DesignerView-49abb835.css} +783 -663
- flowfile/web/static/assets/{designer-9633482a.js → DesignerView-f64749fb.js} +1292 -3253
- flowfile/web/static/assets/{documentation-ca400224.js → DocumentationView-61bd2990.js} +5 -5
- flowfile/web/static/assets/{documentation-12216a74.css → DocumentationView-9ea6e871.css} +9 -9
- flowfile/web/static/assets/{ExploreData-2d0cf4db.css → ExploreData-10c5acc8.css} +13 -12
- flowfile/web/static/assets/{ExploreData-5fa10ed8.js → ExploreData-e2735b13.js} +18 -9
- flowfile/web/static/assets/{ExternalSource-d39af878.js → ExternalSource-2535c3b2.js} +9 -7
- flowfile/web/static/assets/{ExternalSource-e37b6275.css → ExternalSource-7ac7373f.css} +20 -20
- flowfile/web/static/assets/Filter-2cdbc93c.js +287 -0
- flowfile/web/static/assets/Filter-7494ea97.css +48 -0
- flowfile/web/static/assets/{Formula-bb96803d.css → Formula-53d58c43.css} +7 -7
- flowfile/web/static/assets/{Formula-6b04fb1d.js → Formula-fcda3c2c.js} +13 -11
- flowfile/web/static/assets/{FuzzyMatch-1010f966.css → FuzzyMatch-ad6361d6.css} +68 -69
- flowfile/web/static/assets/{FuzzyMatch-999521f4.js → FuzzyMatch-f8d3b7d3.js} +12 -10
- flowfile/web/static/assets/{Pivot-cf333e3d.css → GraphSolver-4b4d7db9.css} +5 -5
- flowfile/web/static/assets/{GraphSolver-17dd2198.js → GraphSolver-72eaa695.js} +14 -12
- flowfile/web/static/assets/GroupBy-5792782d.css +9 -0
- flowfile/web/static/assets/{GroupBy-6b039e18.js → GroupBy-8aa0598b.js} +9 -7
- flowfile/web/static/assets/{Join-fd79b451.css → Join-28b5e18f.css} +22 -22
- flowfile/web/static/assets/{Join-24d0f113.js → Join-e40f0ffa.js} +13 -11
- flowfile/web/static/assets/LoginView-5111c9ae.js +134 -0
- flowfile/web/static/assets/LoginView-d325d632.css +172 -0
- flowfile/web/static/assets/ManualInput-3702e677.css +293 -0
- flowfile/web/static/assets/{ManualInput-34639209.js → ManualInput-9b6f3224.js} +170 -116
- flowfile/web/static/assets/{MultiSelect-0e8724a3.js → MultiSelect-ef28e19e.js} +2 -2
- flowfile/web/static/assets/{MultiSelect.vue_vue_type_script_setup_true_lang-b0e538c2.js → MultiSelect.vue_vue_type_script_setup_true_lang-83b3bbfd.js} +1 -1
- flowfile/web/static/assets/NodeDesigner-94cd4dd3.css +1429 -0
- flowfile/web/static/assets/NodeDesigner-d2b7ee2b.js +2712 -0
- flowfile/web/static/assets/{NumericInput-3d63a470.js → NumericInput-1d789794.js} +2 -2
- flowfile/web/static/assets/{NumericInput.vue_vue_type_script_setup_true_lang-e0edeccc.js → NumericInput.vue_vue_type_script_setup_true_lang-7775f83e.js} +5 -2
- flowfile/web/static/assets/Output-692dd25d.css +37 -0
- flowfile/web/static/assets/{Output-edea9802.js → Output-cefef801.js} +14 -10
- flowfile/web/static/assets/{GraphSolver-f0cb7bfb.css → Pivot-0eda81b4.css} +5 -5
- flowfile/web/static/assets/{Pivot-61d19301.js → Pivot-bab1b75b.js} +12 -10
- flowfile/web/static/assets/PivotValidation-0e905b1a.css +13 -0
- flowfile/web/static/assets/PivotValidation-41b57ad6.css +13 -0
- flowfile/web/static/assets/{PivotValidation-f97fec5b.js → PivotValidation-e7941f91.js} +3 -3
- flowfile/web/static/assets/{PivotValidation-de9f43fe.js → PivotValidation-fba09336.js} +3 -3
- flowfile/web/static/assets/{PolarsCode-650322d1.css → PolarsCode-2b1f1f23.css} +4 -4
- flowfile/web/static/assets/{PolarsCode-bc3c9984.js → PolarsCode-740e40fa.js} +18 -9
- flowfile/web/static/assets/PopOver-862d7e28.js +939 -0
- flowfile/web/static/assets/PopOver-d96599db.css +33 -0
- flowfile/web/static/assets/{Read-64a3f259.js → Read-225cc63f.js} +16 -12
- flowfile/web/static/assets/{Read-e808b239.css → Read-90f366bc.css} +15 -15
- flowfile/web/static/assets/{RecordCount-3d5039be.js → RecordCount-ffc71eca.js} +6 -4
- flowfile/web/static/assets/{RecordId-597510e0.js → RecordId-a70bb8df.js} +9 -7
- flowfile/web/static/assets/{SQLQueryComponent-df51adbe.js → SQLQueryComponent-15a421f5.js} +3 -3
- flowfile/web/static/assets/SQLQueryComponent-edb90b98.css +29 -0
- flowfile/web/static/assets/{Sample-4be0a507.js → Sample-6c26afc7.js} +6 -4
- flowfile/web/static/assets/SecretSelector-6329f743.css +43 -0
- flowfile/web/static/assets/SecretSelector-ceed9496.js +113 -0
- flowfile/web/static/assets/{SecretManager-4839be57.js → SecretsView-214d255a.js} +35 -36
- flowfile/web/static/assets/SecretsView-aa291340.css +38 -0
- flowfile/web/static/assets/{Select-9b72f201.js → Select-8fc29999.js} +9 -7
- flowfile/web/static/assets/{SettingsSection-71e6b7e3.css → SettingsSection-07fbbc39.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-5c696bee.css → SettingsSection-26fe48d4.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-7ded385d.js → SettingsSection-3f70e4c3.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-f0f75a42.js → SettingsSection-83090218.js} +3 -3
- flowfile/web/static/assets/{SettingsSection-2e4d03c4.css → SettingsSection-8f980839.css} +4 -4
- flowfile/web/static/assets/{SettingsSection-e1e9c953.js → SettingsSection-9f0d1725.js} +3 -3
- flowfile/web/static/assets/SetupView-3fa0aa03.js +160 -0
- flowfile/web/static/assets/SetupView-e2da3442.css +230 -0
- flowfile/web/static/assets/{SingleSelect-6c777aac.js → SingleSelect-a4a568cb.js} +2 -2
- flowfile/web/static/assets/{SingleSelect.vue_vue_type_script_setup_true_lang-33e3ff9b.js → SingleSelect.vue_vue_type_script_setup_true_lang-c8ebdd33.js} +1 -1
- flowfile/web/static/assets/{SliderInput-7cb93e62.js → SliderInput-be533e71.js} +7 -4
- flowfile/web/static/assets/SliderInput-f2e4f23c.css +4 -0
- flowfile/web/static/assets/{Sort-6cbde21a.js → Sort-154dad81.js} +9 -7
- flowfile/web/static/assets/Sort-4abb7fae.css +9 -0
- flowfile/web/static/assets/{TextInput-d9a40c11.js → TextInput-454e2bda.js} +2 -2
- flowfile/web/static/assets/{TextInput.vue_vue_type_script_setup_true_lang-5896c375.js → TextInput.vue_vue_type_script_setup_true_lang-e86510d0.js} +5 -2
- flowfile/web/static/assets/{TextToRows-5d2c1190.css → TextToRows-12afb4f4.css} +10 -10
- flowfile/web/static/assets/{TextToRows-c4fcbf4d.js → TextToRows-ea73433d.js} +11 -10
- flowfile/web/static/assets/{ToggleSwitch-4ef91d19.js → ToggleSwitch-9d7b30f1.js} +2 -2
- flowfile/web/static/assets/{ToggleSwitch.vue_vue_type_script_setup_true_lang-38478c20.js → ToggleSwitch.vue_vue_type_script_setup_true_lang-00f2580e.js} +1 -1
- flowfile/web/static/assets/{UnavailableFields-5edd5322.css → UnavailableFields-394a1f78.css} +14 -14
- flowfile/web/static/assets/{UnavailableFields-a03f512c.js → UnavailableFields-b72a2c72.js} +4 -4
- flowfile/web/static/assets/{Union-bfe9b996.js → Union-1e44f263.js} +8 -6
- flowfile/web/static/assets/{Union-af6c3d9b.css → Union-d6a8d7d5.css} +7 -7
- flowfile/web/static/assets/Unique-2b705521.css +3 -0
- flowfile/web/static/assets/{Unique-5d023a27.js → Unique-a3bc6d0a.js} +13 -10
- flowfile/web/static/assets/{Unpivot-1e422df3.css → Unpivot-b6ad6427.css} +7 -7
- flowfile/web/static/assets/{Unpivot-91cc5354.js → Unpivot-e27935fc.js} +11 -9
- flowfile/web/static/assets/{UnpivotValidation-7ee2de44.js → UnpivotValidation-72497680.js} +3 -3
- flowfile/web/static/assets/UnpivotValidation-d5ca3b7b.css +13 -0
- flowfile/web/static/assets/{VueGraphicWalker-ed5ab88b.css → VueGraphicWalker-430f0b86.css} +1 -1
- flowfile/web/static/assets/{VueGraphicWalker-e51b9924.js → VueGraphicWalker-d9ab70a3.js} +4 -4
- flowfile/web/static/assets/{api-cf1221f0.js → api-a2102880.js} +1 -1
- flowfile/web/static/assets/{api-c1bad5ca.js → api-f75042b0.js} +1 -1
- flowfile/web/static/assets/{dropDown-35135ba8.css → dropDown-1d6acbd9.css} +41 -41
- flowfile/web/static/assets/{dropDown-614b998d.js → dropDown-2798a109.js} +3 -3
- flowfile/web/static/assets/{fullEditor-f7971590.js → fullEditor-cf7d7d93.js} +11 -10
- flowfile/web/static/assets/{fullEditor-178376bb.css → fullEditor-fe9f7e18.css} +77 -65
- flowfile/web/static/assets/{genericNodeSettings-4fe5f36b.js → genericNodeSettings-14eac1c3.js} +5 -5
- flowfile/web/static/assets/{genericNodeSettings-924759c7.css → genericNodeSettings-3b2507ea.css} +10 -10
- flowfile/web/static/assets/{index-5429bbf8.js → index-387a6f18.js} +41806 -40958
- flowfile/web/static/assets/index-6b367bb5.js +38 -0
- flowfile/web/static/assets/{index-50508d4d.css → index-e96ab018.css} +2184 -569
- flowfile/web/static/assets/index-f0a6e5a5.js +2696 -0
- flowfile/web/static/assets/node.types-2c15bb7e.js +82 -0
- flowfile/web/static/assets/nodeInput-ed2ae8d7.js +2 -0
- flowfile/web/static/assets/{outputCsv-076b85ab.js → outputCsv-3c1757e8.js} +3 -3
- flowfile/web/static/assets/outputCsv-b9a072af.css +2499 -0
- flowfile/web/static/assets/{outputExcel-0fd17dbe.js → outputExcel-686e1f48.js} +3 -3
- flowfile/web/static/assets/{outputExcel-b41305c0.css → outputExcel-f5d272b2.css} +26 -26
- flowfile/web/static/assets/outputParquet-54597c3c.css +4 -0
- flowfile/web/static/assets/{outputParquet-b61e0847.js → outputParquet-df28faa7.js} +4 -4
- flowfile/web/static/assets/{readCsv-c767cb37.css → readCsv-3bfac4c3.css} +15 -15
- flowfile/web/static/assets/{readCsv-a8bb8b61.js → readCsv-e37eee21.js} +3 -3
- flowfile/web/static/assets/{readExcel-806d2826.css → readExcel-3db6b763.css} +13 -13
- flowfile/web/static/assets/{readExcel-67b4aee0.js → readExcel-a13f14bb.js} +5 -5
- flowfile/web/static/assets/{readParquet-92ce1dbc.js → readParquet-344cf746.js} +3 -3
- flowfile/web/static/assets/{readParquet-48c81530.css → readParquet-c5244ad5.css} +4 -4
- flowfile/web/static/assets/secrets.api-ae198c5c.js +65 -0
- flowfile/web/static/assets/{selectDynamic-92e25ee3.js → selectDynamic-6b4b0767.js} +5 -5
- flowfile/web/static/assets/{selectDynamic-aa913ff4.css → selectDynamic-f2fb394f.css} +21 -20
- flowfile/web/static/assets/{vue-codemirror.esm-41b0e0d7.js → vue-codemirror.esm-31ba0e0b.js} +31 -640
- flowfile/web/static/assets/{vue-content-loader.es-2c8e608f.js → vue-content-loader.es-4469c8ff.js} +1 -1
- flowfile/web/static/index.html +2 -2
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/METADATA +3 -4
- flowfile-0.5.4.dist-info/RECORD +407 -0
- flowfile_core/__init__.py +13 -6
- flowfile_core/auth/jwt.py +51 -16
- flowfile_core/auth/models.py +32 -7
- flowfile_core/auth/password.py +89 -0
- flowfile_core/auth/secrets.py +64 -19
- flowfile_core/configs/__init__.py +9 -7
- flowfile_core/configs/flow_logger.py +15 -14
- flowfile_core/configs/node_store/__init__.py +72 -4
- flowfile_core/configs/node_store/nodes.py +155 -172
- flowfile_core/configs/node_store/user_defined_node_registry.py +108 -27
- flowfile_core/configs/settings.py +28 -15
- flowfile_core/database/connection.py +7 -6
- flowfile_core/database/init_db.py +96 -2
- flowfile_core/database/models.py +3 -1
- flowfile_core/fileExplorer/__init__.py +17 -0
- flowfile_core/fileExplorer/funcs.py +145 -57
- flowfile_core/fileExplorer/utils.py +10 -11
- flowfile_core/flowfile/_extensions/real_time_interface.py +10 -8
- flowfile_core/flowfile/analytics/analytics_processor.py +26 -24
- flowfile_core/flowfile/analytics/graphic_walker.py +11 -12
- flowfile_core/flowfile/analytics/utils.py +1 -1
- flowfile_core/flowfile/code_generator/__init__.py +11 -0
- flowfile_core/flowfile/code_generator/code_generator.py +706 -247
- flowfile_core/flowfile/connection_manager/_connection_manager.py +6 -5
- flowfile_core/flowfile/connection_manager/models.py +1 -1
- flowfile_core/flowfile/database_connection_manager/db_connections.py +60 -44
- flowfile_core/flowfile/database_connection_manager/models.py +1 -1
- flowfile_core/flowfile/extensions.py +17 -12
- flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +34 -32
- flowfile_core/flowfile/flow_data_engine/create/funcs.py +115 -83
- flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +493 -423
- flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +2 -2
- flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +92 -52
- flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py +12 -11
- flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +6 -6
- flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +26 -30
- flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py +31 -20
- flowfile_core/flowfile/flow_data_engine/join/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/join/utils.py +11 -9
- flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py +14 -15
- flowfile_core/flowfile/flow_data_engine/pivot_table.py +5 -7
- flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +95 -82
- flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +66 -65
- flowfile_core/flowfile/flow_data_engine/sample_data.py +27 -21
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py +1 -1
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py +13 -11
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +190 -127
- flowfile_core/flowfile/flow_data_engine/threaded_processes.py +8 -8
- flowfile_core/flowfile/flow_data_engine/utils.py +99 -67
- flowfile_core/flowfile/flow_graph.py +920 -571
- flowfile_core/flowfile/flow_graph_utils.py +31 -49
- flowfile_core/flowfile/flow_node/flow_node.py +379 -258
- flowfile_core/flowfile/flow_node/models.py +53 -41
- flowfile_core/flowfile/flow_node/schema_callback.py +14 -19
- flowfile_core/flowfile/graph_tree/graph_tree.py +41 -41
- flowfile_core/flowfile/handler.py +80 -30
- flowfile_core/flowfile/manage/compatibility_enhancements.py +209 -126
- flowfile_core/flowfile/manage/io_flowfile.py +54 -57
- flowfile_core/flowfile/node_designer/__init__.py +19 -13
- flowfile_core/flowfile/node_designer/_type_registry.py +34 -37
- flowfile_core/flowfile/node_designer/custom_node.py +162 -36
- flowfile_core/flowfile/node_designer/ui_components.py +278 -34
- flowfile_core/flowfile/schema_callbacks.py +71 -51
- flowfile_core/flowfile/setting_generator/__init__.py +0 -1
- flowfile_core/flowfile/setting_generator/setting_generator.py +6 -5
- flowfile_core/flowfile/setting_generator/settings.py +64 -53
- flowfile_core/flowfile/sources/external_sources/base_class.py +12 -10
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +27 -17
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +9 -9
- flowfile_core/flowfile/sources/external_sources/factory.py +0 -1
- flowfile_core/flowfile/sources/external_sources/sql_source/models.py +45 -31
- flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +198 -73
- flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +250 -196
- flowfile_core/flowfile/util/calculate_layout.py +9 -13
- flowfile_core/flowfile/util/execution_orderer.py +25 -17
- flowfile_core/flowfile/util/node_skipper.py +4 -4
- flowfile_core/flowfile/utils.py +19 -21
- flowfile_core/main.py +26 -19
- flowfile_core/routes/auth.py +284 -11
- flowfile_core/routes/cloud_connections.py +25 -25
- flowfile_core/routes/logs.py +21 -29
- flowfile_core/routes/public.py +46 -4
- flowfile_core/routes/routes.py +70 -34
- flowfile_core/routes/secrets.py +25 -27
- flowfile_core/routes/user_defined_components.py +483 -4
- flowfile_core/run_lock.py +0 -1
- flowfile_core/schemas/__init__.py +4 -6
- flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py +55 -55
- flowfile_core/schemas/cloud_storage_schemas.py +96 -66
- flowfile_core/schemas/input_schema.py +231 -144
- flowfile_core/schemas/output_model.py +49 -34
- flowfile_core/schemas/schemas.py +116 -89
- flowfile_core/schemas/transform_schema.py +518 -263
- flowfile_core/schemas/yaml_types.py +21 -7
- flowfile_core/secret_manager/secret_manager.py +123 -18
- flowfile_core/types.py +29 -9
- flowfile_core/utils/arrow_reader.py +7 -6
- flowfile_core/utils/excel_file_manager.py +3 -3
- flowfile_core/utils/fileManager.py +7 -7
- flowfile_core/utils/fl_executor.py +8 -10
- flowfile_core/utils/utils.py +4 -4
- flowfile_core/utils/validate_setup.py +5 -4
- flowfile_frame/__init__.py +117 -51
- flowfile_frame/adapters.py +2 -9
- flowfile_frame/adding_expr.py +73 -32
- flowfile_frame/cloud_storage/frame_helpers.py +27 -23
- flowfile_frame/cloud_storage/secret_manager.py +12 -26
- flowfile_frame/config.py +2 -5
- flowfile_frame/database/__init__.py +36 -0
- flowfile_frame/database/connection_manager.py +205 -0
- flowfile_frame/database/frame_helpers.py +249 -0
- flowfile_frame/expr.py +311 -218
- flowfile_frame/expr.pyi +160 -159
- flowfile_frame/expr_name.py +23 -23
- flowfile_frame/flow_frame.py +571 -476
- flowfile_frame/flow_frame.pyi +123 -104
- flowfile_frame/flow_frame_methods.py +227 -246
- flowfile_frame/group_frame.py +50 -20
- flowfile_frame/join.py +2 -2
- flowfile_frame/lazy.py +129 -87
- flowfile_frame/lazy_methods.py +83 -30
- flowfile_frame/list_name_space.py +55 -50
- flowfile_frame/selectors.py +148 -68
- flowfile_frame/series.py +9 -7
- flowfile_frame/utils.py +19 -21
- flowfile_worker/__init__.py +12 -7
- flowfile_worker/configs.py +41 -33
- flowfile_worker/create/__init__.py +14 -9
- flowfile_worker/create/funcs.py +114 -77
- flowfile_worker/create/models.py +46 -43
- flowfile_worker/create/pl_types.py +14 -15
- flowfile_worker/create/read_excel_tables.py +34 -41
- flowfile_worker/create/utils.py +22 -19
- flowfile_worker/external_sources/s3_source/main.py +18 -51
- flowfile_worker/external_sources/s3_source/models.py +34 -27
- flowfile_worker/external_sources/sql_source/main.py +8 -5
- flowfile_worker/external_sources/sql_source/models.py +13 -9
- flowfile_worker/flow_logger.py +10 -8
- flowfile_worker/funcs.py +214 -155
- flowfile_worker/main.py +11 -17
- flowfile_worker/models.py +35 -28
- flowfile_worker/process_manager.py +2 -3
- flowfile_worker/routes.py +121 -90
- flowfile_worker/secrets.py +114 -21
- flowfile_worker/spawner.py +89 -54
- flowfile_worker/utils.py +3 -2
- shared/__init__.py +2 -7
- shared/storage_config.py +25 -13
- test_utils/postgres/commands.py +3 -2
- test_utils/postgres/fixtures.py +9 -9
- test_utils/s3/commands.py +1 -1
- test_utils/s3/data_generator.py +3 -4
- test_utils/s3/demo_data_generator.py +4 -7
- test_utils/s3/fixtures.py +7 -5
- tools/migrate/__init__.py +1 -1
- tools/migrate/__main__.py +16 -29
- tools/migrate/legacy_schemas.py +251 -190
- tools/migrate/migrate.py +193 -181
- tools/migrate/tests/conftest.py +1 -3
- tools/migrate/tests/test_migrate.py +36 -41
- tools/migrate/tests/test_migration_e2e.py +28 -29
- tools/migrate/tests/test_node_migrations.py +50 -20
- flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +0 -86
- flowfile/web/static/assets/ContextMenu-23e909da.js +0 -41
- flowfile/web/static/assets/ContextMenu-4c74eef1.css +0 -26
- flowfile/web/static/assets/ContextMenu-63cfa99b.css +0 -26
- flowfile/web/static/assets/ContextMenu-70ae0c79.js +0 -41
- flowfile/web/static/assets/ContextMenu-c13f91d0.css +0 -26
- flowfile/web/static/assets/ContextMenu-f149cf7c.js +0 -41
- flowfile/web/static/assets/CustomNode-74a37f74.css +0 -32
- flowfile/web/static/assets/DatabaseManager-30fa27e5.css +0 -64
- flowfile/web/static/assets/Filter-9b6d08db.js +0 -164
- flowfile/web/static/assets/Filter-f62091b3.css +0 -20
- flowfile/web/static/assets/GroupBy-b9505323.css +0 -51
- flowfile/web/static/assets/ManualInput-3246a08d.css +0 -96
- flowfile/web/static/assets/Output-283fe388.css +0 -37
- flowfile/web/static/assets/PivotValidation-891ddfb0.css +0 -13
- flowfile/web/static/assets/PivotValidation-c46cd420.css +0 -13
- flowfile/web/static/assets/SQLQueryComponent-36cef432.css +0 -27
- flowfile/web/static/assets/SliderInput-b8fb6a8c.css +0 -4
- flowfile/web/static/assets/Sort-3643d625.css +0 -51
- flowfile/web/static/assets/Unique-f9fb0809.css +0 -51
- flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +0 -13
- flowfile/web/static/assets/nodeInput-5d0d6b79.js +0 -41
- flowfile/web/static/assets/outputCsv-9cc59e0b.css +0 -2499
- flowfile/web/static/assets/outputParquet-cf8cf3f2.css +0 -4
- flowfile/web/static/assets/secretApi-68435402.js +0 -46
- flowfile/web/static/assets/vue-codemirror-bccfde04.css +0 -32
- flowfile-0.5.1.dist-info/RECORD +0 -388
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/WHEEL +0 -0
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/entry_points.txt +0 -0
- {flowfile-0.5.1.dist-info → flowfile-0.5.4.dist-info}/licenses/LICENSE +0 -0
tools/migrate/legacy_schemas.py
CHANGED
|
@@ -16,37 +16,42 @@ DO NOT USE THESE IN PRODUCTION CODE - use the actual schemas from flowfile_core.
|
|
|
16
16
|
"""
|
|
17
17
|
|
|
18
18
|
from dataclasses import dataclass, field
|
|
19
|
-
from typing import
|
|
20
|
-
from pydantic import BaseModel, Field
|
|
19
|
+
from typing import Any, Literal
|
|
21
20
|
|
|
21
|
+
from pydantic import BaseModel, Field
|
|
22
22
|
|
|
23
23
|
# =============================================================================
|
|
24
24
|
# OLD INPUT/OUTPUT SCHEMAS (before table_settings migration)
|
|
25
25
|
# These mirror the structure that exists in old pickle files
|
|
26
26
|
# =============================================================================
|
|
27
27
|
|
|
28
|
+
|
|
28
29
|
class MinimalFieldInfo(BaseModel):
|
|
29
30
|
"""Represents the most basic information about a data field (column)."""
|
|
31
|
+
|
|
30
32
|
name: str
|
|
31
33
|
data_type: str = "String"
|
|
32
34
|
|
|
33
35
|
|
|
34
36
|
class OutputCsvTable(BaseModel):
|
|
35
37
|
"""OLD: Settings for writing a CSV file."""
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
38
|
+
|
|
39
|
+
file_type: str = "csv"
|
|
40
|
+
delimiter: str = ","
|
|
41
|
+
encoding: str = "utf-8"
|
|
39
42
|
|
|
40
43
|
|
|
41
44
|
class OutputParquetTable(BaseModel):
|
|
42
45
|
"""OLD: Settings for writing a Parquet file."""
|
|
43
|
-
|
|
46
|
+
|
|
47
|
+
file_type: str = "parquet"
|
|
44
48
|
|
|
45
49
|
|
|
46
50
|
class OutputExcelTable(BaseModel):
|
|
47
51
|
"""OLD: Settings for writing an Excel file."""
|
|
48
|
-
|
|
49
|
-
|
|
52
|
+
|
|
53
|
+
file_type: str = "excel"
|
|
54
|
+
sheet_name: str = "Sheet1"
|
|
50
55
|
|
|
51
56
|
|
|
52
57
|
class OutputSettings(BaseModel):
|
|
@@ -55,16 +60,17 @@ class OutputSettings(BaseModel):
|
|
|
55
60
|
This is the OLD format where CSV, Parquet, and Excel settings
|
|
56
61
|
were stored in separate fields rather than a unified table_settings.
|
|
57
62
|
"""
|
|
63
|
+
|
|
58
64
|
name: str
|
|
59
65
|
directory: str
|
|
60
66
|
file_type: str
|
|
61
|
-
fields:
|
|
62
|
-
write_mode: str =
|
|
67
|
+
fields: list[str] | None = Field(default_factory=list)
|
|
68
|
+
write_mode: str = "overwrite"
|
|
63
69
|
# OLD: Separate fields for each output type
|
|
64
|
-
output_csv_table:
|
|
70
|
+
output_csv_table: OutputCsvTable | None = Field(default_factory=OutputCsvTable)
|
|
65
71
|
output_parquet_table: OutputParquetTable = Field(default_factory=OutputParquetTable)
|
|
66
72
|
output_excel_table: OutputExcelTable = Field(default_factory=OutputExcelTable)
|
|
67
|
-
abs_file_path:
|
|
73
|
+
abs_file_path: str | None = None
|
|
68
74
|
|
|
69
75
|
|
|
70
76
|
class ReceivedTable(BaseModel):
|
|
@@ -73,32 +79,33 @@ class ReceivedTable(BaseModel):
|
|
|
73
79
|
This is the OLD format where all settings (CSV, Excel, Parquet)
|
|
74
80
|
were stored as flat fields on the model rather than nested in table_settings.
|
|
75
81
|
"""
|
|
82
|
+
|
|
76
83
|
# Metadata fields
|
|
77
|
-
id:
|
|
78
|
-
name:
|
|
79
|
-
path: str =
|
|
80
|
-
directory:
|
|
84
|
+
id: int | None = None
|
|
85
|
+
name: str | None = None
|
|
86
|
+
path: str = ""
|
|
87
|
+
directory: str | None = None
|
|
81
88
|
analysis_file_available: bool = False
|
|
82
|
-
status:
|
|
83
|
-
file_type:
|
|
84
|
-
fields:
|
|
85
|
-
abs_file_path:
|
|
89
|
+
status: str | None = None
|
|
90
|
+
file_type: str | None = None
|
|
91
|
+
fields: list[MinimalFieldInfo] = Field(default_factory=list)
|
|
92
|
+
abs_file_path: str | None = None
|
|
86
93
|
|
|
87
94
|
# OLD: CSV/JSON fields at top level (not nested)
|
|
88
|
-
reference: str =
|
|
95
|
+
reference: str = ""
|
|
89
96
|
starting_from_line: int = 0
|
|
90
|
-
delimiter: str =
|
|
97
|
+
delimiter: str = ","
|
|
91
98
|
has_headers: bool = True
|
|
92
|
-
encoding:
|
|
93
|
-
parquet_ref:
|
|
94
|
-
row_delimiter: str =
|
|
99
|
+
encoding: str | None = "utf-8"
|
|
100
|
+
parquet_ref: str | None = None
|
|
101
|
+
row_delimiter: str = "\n"
|
|
95
102
|
quote_char: str = '"'
|
|
96
103
|
infer_schema_length: int = 10_000
|
|
97
104
|
truncate_ragged_lines: bool = False
|
|
98
105
|
ignore_errors: bool = False
|
|
99
106
|
|
|
100
107
|
# OLD: Excel fields at top level (not nested)
|
|
101
|
-
sheet_name:
|
|
108
|
+
sheet_name: str | None = None
|
|
102
109
|
start_row: int = 0
|
|
103
110
|
start_column: int = 0
|
|
104
111
|
end_row: int = 0
|
|
@@ -110,21 +117,24 @@ class ReceivedTable(BaseModel):
|
|
|
110
117
|
# FLOW AND NODE SCHEMAS (Pydantic - structure unchanged, just re-exported)
|
|
111
118
|
# =============================================================================
|
|
112
119
|
|
|
120
|
+
|
|
113
121
|
class FlowGraphConfig(BaseModel):
|
|
114
122
|
"""Configuration model for a flow graph's basic properties."""
|
|
123
|
+
|
|
115
124
|
flow_id: int = 1
|
|
116
|
-
description:
|
|
117
|
-
save_location:
|
|
118
|
-
name: str =
|
|
119
|
-
path: str =
|
|
120
|
-
execution_mode: str =
|
|
121
|
-
execution_location: str =
|
|
125
|
+
description: str | None = None
|
|
126
|
+
save_location: str | None = None
|
|
127
|
+
name: str = ""
|
|
128
|
+
path: str = ""
|
|
129
|
+
execution_mode: str = "Performance"
|
|
130
|
+
execution_location: str = "local"
|
|
122
131
|
|
|
123
132
|
|
|
124
133
|
class FlowSettings(FlowGraphConfig):
|
|
125
134
|
"""Extends FlowGraphConfig with additional operational settings."""
|
|
135
|
+
|
|
126
136
|
auto_save: bool = False
|
|
127
|
-
modified_on:
|
|
137
|
+
modified_on: float | None = None
|
|
128
138
|
show_detailed_progress: bool = True
|
|
129
139
|
is_running: bool = False
|
|
130
140
|
is_canceled: bool = False
|
|
@@ -132,52 +142,60 @@ class FlowSettings(FlowGraphConfig):
|
|
|
132
142
|
|
|
133
143
|
class NodeBase(BaseModel):
|
|
134
144
|
"""Base model for all nodes in a FlowGraph."""
|
|
145
|
+
|
|
135
146
|
flow_id: int
|
|
136
147
|
node_id: int
|
|
137
|
-
cache_results:
|
|
138
|
-
pos_x:
|
|
139
|
-
pos_y:
|
|
140
|
-
is_setup:
|
|
141
|
-
description:
|
|
142
|
-
user_id:
|
|
143
|
-
is_flow_output:
|
|
144
|
-
is_user_defined:
|
|
148
|
+
cache_results: bool | None = False
|
|
149
|
+
pos_x: float | None = 0
|
|
150
|
+
pos_y: float | None = 0
|
|
151
|
+
is_setup: bool | None = True
|
|
152
|
+
description: str | None = ""
|
|
153
|
+
user_id: int | None = None
|
|
154
|
+
is_flow_output: bool | None = False
|
|
155
|
+
is_user_defined: bool | None = False
|
|
145
156
|
|
|
146
157
|
|
|
147
158
|
class NodeSingleInput(NodeBase):
|
|
148
159
|
"""A base model for any node that takes a single data input."""
|
|
149
|
-
|
|
160
|
+
|
|
161
|
+
depending_on_id: int | None = -1
|
|
150
162
|
|
|
151
163
|
|
|
152
164
|
class NodeMultiInput(NodeBase):
|
|
153
165
|
"""A base model for any node that takes multiple data inputs."""
|
|
154
|
-
|
|
166
|
+
|
|
167
|
+
depending_on_ids: list[int] | None = Field(default_factory=lambda: [-1])
|
|
155
168
|
|
|
156
169
|
|
|
157
170
|
class NodeRead(NodeBase):
|
|
158
171
|
"""Settings for a node that reads data from a file."""
|
|
172
|
+
|
|
159
173
|
received_file: ReceivedTable
|
|
160
174
|
|
|
161
175
|
|
|
162
176
|
class NodeSelect(NodeSingleInput):
|
|
163
177
|
"""Settings for a node that selects, renames, and reorders columns."""
|
|
178
|
+
|
|
164
179
|
keep_missing: bool = True
|
|
165
|
-
select_input:
|
|
166
|
-
sorted_by:
|
|
180
|
+
select_input: list[Any] = Field(default_factory=list)
|
|
181
|
+
sorted_by: str | None = "none"
|
|
167
182
|
|
|
168
183
|
|
|
169
184
|
class NodeFilter(NodeSingleInput):
|
|
170
185
|
"""Settings for a node that filters rows based on a condition."""
|
|
186
|
+
|
|
171
187
|
filter_input: Any = None
|
|
172
188
|
|
|
173
189
|
|
|
174
190
|
class NodeFormula(NodeSingleInput):
|
|
175
191
|
"""Settings for a node that applies a formula to create/modify a column."""
|
|
192
|
+
|
|
176
193
|
function: Any = None
|
|
177
194
|
|
|
178
195
|
|
|
179
196
|
class NodeJoin(NodeMultiInput):
|
|
180
197
|
"""Settings for a node that performs a standard SQL-style join."""
|
|
198
|
+
|
|
181
199
|
auto_generate_selection: bool = True
|
|
182
200
|
verify_integrity: bool = True
|
|
183
201
|
join_input: Any = None
|
|
@@ -188,6 +206,7 @@ class NodeJoin(NodeMultiInput):
|
|
|
188
206
|
|
|
189
207
|
class NodeCrossJoin(NodeMultiInput):
|
|
190
208
|
"""Settings for a node that performs a cross join."""
|
|
209
|
+
|
|
191
210
|
auto_generate_selection: bool = True
|
|
192
211
|
verify_integrity: bool = True
|
|
193
212
|
cross_join_input: Any = None
|
|
@@ -198,6 +217,7 @@ class NodeCrossJoin(NodeMultiInput):
|
|
|
198
217
|
|
|
199
218
|
class NodeFuzzyMatch(NodeMultiInput):
|
|
200
219
|
"""Settings for a node that performs a fuzzy join."""
|
|
220
|
+
|
|
201
221
|
auto_generate_selection: bool = True
|
|
202
222
|
verify_integrity: bool = True
|
|
203
223
|
join_input: Any = None # FuzzyMatchInput
|
|
@@ -208,141 +228,161 @@ class NodeFuzzyMatch(NodeMultiInput):
|
|
|
208
228
|
|
|
209
229
|
class NodePolarsCode(NodeMultiInput):
|
|
210
230
|
"""Settings for a node that executes arbitrary Polars code."""
|
|
231
|
+
|
|
211
232
|
polars_code_input: Any = None
|
|
212
233
|
|
|
213
234
|
|
|
214
235
|
class NodeOutput(NodeSingleInput):
|
|
215
236
|
"""Settings for a node that writes its input to a file."""
|
|
237
|
+
|
|
216
238
|
output_settings: OutputSettings
|
|
217
239
|
|
|
218
240
|
|
|
219
241
|
class NodeGroupBy(NodeSingleInput):
|
|
220
242
|
"""Settings for a node that performs a group-by and aggregation."""
|
|
243
|
+
|
|
221
244
|
groupby_input: Any = None
|
|
222
245
|
|
|
223
246
|
|
|
224
247
|
class NodeSort(NodeSingleInput):
|
|
225
248
|
"""Settings for a node that sorts the data."""
|
|
226
|
-
|
|
249
|
+
|
|
250
|
+
sort_input: list[Any] = Field(default_factory=list)
|
|
227
251
|
|
|
228
252
|
|
|
229
253
|
class NodeUnion(NodeMultiInput):
|
|
230
254
|
"""Settings for a node that concatenates multiple inputs."""
|
|
255
|
+
|
|
231
256
|
union_input: Any = None
|
|
232
257
|
|
|
233
258
|
|
|
234
259
|
class NodeUnique(NodeSingleInput):
|
|
235
260
|
"""Settings for a node that returns unique rows."""
|
|
261
|
+
|
|
236
262
|
unique_input: Any = None
|
|
237
263
|
|
|
238
264
|
|
|
239
265
|
class NodePivot(NodeSingleInput):
|
|
240
266
|
"""Settings for a node that pivots data."""
|
|
267
|
+
|
|
241
268
|
pivot_input: Any = None
|
|
242
|
-
output_fields:
|
|
269
|
+
output_fields: list[MinimalFieldInfo] | None = None
|
|
243
270
|
|
|
244
271
|
|
|
245
272
|
class NodeUnpivot(NodeSingleInput):
|
|
246
273
|
"""Settings for a node that unpivots data."""
|
|
274
|
+
|
|
247
275
|
unpivot_input: Any = None
|
|
248
276
|
|
|
249
277
|
|
|
250
278
|
class NodeRecordId(NodeSingleInput):
|
|
251
279
|
"""Settings for adding a record ID column."""
|
|
280
|
+
|
|
252
281
|
record_id_input: Any = None
|
|
253
282
|
|
|
254
283
|
|
|
255
284
|
class NodeTextToRows(NodeSingleInput):
|
|
256
285
|
"""Settings for splitting text into rows."""
|
|
286
|
+
|
|
257
287
|
text_to_rows_input: Any = None
|
|
258
288
|
|
|
259
289
|
|
|
260
290
|
class NodeGraphSolver(NodeSingleInput):
|
|
261
291
|
"""Settings for graph-solving operations."""
|
|
292
|
+
|
|
262
293
|
graph_solver_input: Any = None
|
|
263
294
|
|
|
264
295
|
|
|
265
296
|
class NodeSample(NodeSingleInput):
|
|
266
297
|
"""Settings for sampling data."""
|
|
298
|
+
|
|
267
299
|
sample_size: int = 1000
|
|
268
300
|
|
|
269
301
|
|
|
270
302
|
class NodePromise(NodeBase):
|
|
271
303
|
"""A placeholder node not yet configured."""
|
|
304
|
+
|
|
272
305
|
is_setup: bool = False
|
|
273
|
-
node_type: str =
|
|
306
|
+
node_type: str = ""
|
|
274
307
|
|
|
275
308
|
|
|
276
309
|
class DatabaseConnection(BaseModel):
|
|
277
310
|
"""Defines database connection parameters."""
|
|
311
|
+
|
|
278
312
|
database_type: str = "postgresql"
|
|
279
|
-
username:
|
|
280
|
-
password_ref:
|
|
281
|
-
host:
|
|
282
|
-
port:
|
|
283
|
-
database:
|
|
284
|
-
url:
|
|
313
|
+
username: str | None = None
|
|
314
|
+
password_ref: str | None = None
|
|
315
|
+
host: str | None = None
|
|
316
|
+
port: int | None = None
|
|
317
|
+
database: str | None = None
|
|
318
|
+
url: str | None = None
|
|
285
319
|
|
|
286
320
|
|
|
287
321
|
class DatabaseSettings(BaseModel):
|
|
288
322
|
"""Defines settings for reading from a database."""
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
323
|
+
|
|
324
|
+
connection_mode: str | None = "inline"
|
|
325
|
+
database_connection: DatabaseConnection | None = None
|
|
326
|
+
database_connection_name: str | None = None
|
|
327
|
+
schema_name: str | None = None
|
|
328
|
+
table_name: str | None = None
|
|
329
|
+
query: str | None = None
|
|
330
|
+
query_mode: str = "table"
|
|
296
331
|
|
|
297
332
|
|
|
298
333
|
class NodeDatabaseReader(NodeBase):
|
|
299
334
|
"""Settings for reading from a database."""
|
|
335
|
+
|
|
300
336
|
database_settings: DatabaseSettings
|
|
301
|
-
fields:
|
|
337
|
+
fields: list[MinimalFieldInfo] | None = None
|
|
302
338
|
|
|
303
339
|
|
|
304
340
|
class NodeInformation(BaseModel):
|
|
305
341
|
"""Stores the state and configuration of a node instance."""
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
342
|
+
|
|
343
|
+
id: int | None = None
|
|
344
|
+
type: str | None = None
|
|
345
|
+
is_setup: bool | None = None
|
|
346
|
+
description: str | None = ""
|
|
347
|
+
x_position: int | None = 0
|
|
348
|
+
y_position: int | None = 0
|
|
349
|
+
left_input_id: int | None = None
|
|
350
|
+
right_input_id: int | None = None
|
|
351
|
+
input_ids: list[int] | None = Field(default_factory=lambda: [-1])
|
|
352
|
+
outputs: list[int] | None = Field(default_factory=lambda: [-1])
|
|
353
|
+
setting_input: Any | None = None
|
|
317
354
|
|
|
318
355
|
|
|
319
356
|
class FlowInformation(BaseModel):
|
|
320
357
|
"""Represents the complete state of a flow."""
|
|
358
|
+
|
|
321
359
|
flow_id: int
|
|
322
|
-
flow_name:
|
|
323
|
-
flow_settings:
|
|
324
|
-
data:
|
|
325
|
-
node_starts:
|
|
326
|
-
node_connections:
|
|
360
|
+
flow_name: str | None = ""
|
|
361
|
+
flow_settings: FlowSettings | None = None
|
|
362
|
+
data: dict[int, NodeInformation] = Field(default_factory=dict)
|
|
363
|
+
node_starts: list[int] = Field(default_factory=list)
|
|
364
|
+
node_connections: list[tuple[int, int]] = Field(default_factory=list)
|
|
327
365
|
|
|
328
366
|
|
|
329
367
|
# =============================================================================
|
|
330
368
|
# TRANSFORM SCHEMAS (dataclasses - these changed from @dataclass to BaseModel)
|
|
331
369
|
# =============================================================================
|
|
332
370
|
|
|
371
|
+
|
|
333
372
|
@dataclass
|
|
334
373
|
class SelectInput:
|
|
335
374
|
"""Defines how a single column should be selected, renamed, or type-cast."""
|
|
375
|
+
|
|
336
376
|
old_name: str
|
|
337
|
-
original_position:
|
|
338
|
-
new_name:
|
|
339
|
-
data_type:
|
|
340
|
-
data_type_change:
|
|
341
|
-
join_key:
|
|
342
|
-
is_altered:
|
|
343
|
-
position:
|
|
344
|
-
is_available:
|
|
345
|
-
keep:
|
|
377
|
+
original_position: int | None = None
|
|
378
|
+
new_name: str | None = None
|
|
379
|
+
data_type: str | None = None
|
|
380
|
+
data_type_change: bool | None = False
|
|
381
|
+
join_key: bool | None = False
|
|
382
|
+
is_altered: bool | None = False
|
|
383
|
+
position: int | None = None
|
|
384
|
+
is_available: bool | None = True
|
|
385
|
+
keep: bool | None = True
|
|
346
386
|
|
|
347
387
|
def __post_init__(self):
|
|
348
388
|
if self.new_name is None:
|
|
@@ -352,56 +392,63 @@ class SelectInput:
|
|
|
352
392
|
@dataclass
|
|
353
393
|
class FieldInput:
|
|
354
394
|
"""Represents a single field with its name and data type."""
|
|
395
|
+
|
|
355
396
|
name: str
|
|
356
|
-
data_type:
|
|
397
|
+
data_type: str | None = None
|
|
357
398
|
|
|
358
399
|
|
|
359
400
|
@dataclass
|
|
360
401
|
class FunctionInput:
|
|
361
402
|
"""Defines a formula to be applied."""
|
|
403
|
+
|
|
362
404
|
field: FieldInput = None
|
|
363
|
-
function: str =
|
|
405
|
+
function: str = ""
|
|
364
406
|
|
|
365
407
|
|
|
366
408
|
@dataclass
|
|
367
409
|
class BasicFilter:
|
|
368
410
|
"""Defines a simple, single-condition filter."""
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
411
|
+
|
|
412
|
+
field: str = ""
|
|
413
|
+
filter_type: str = ""
|
|
414
|
+
filter_value: str = ""
|
|
372
415
|
|
|
373
416
|
|
|
374
417
|
@dataclass
|
|
375
418
|
class FilterInput:
|
|
376
419
|
"""Defines the settings for a filter operation."""
|
|
377
|
-
|
|
420
|
+
|
|
421
|
+
advanced_filter: str = ""
|
|
378
422
|
basic_filter: BasicFilter = None
|
|
379
|
-
filter_type: str =
|
|
423
|
+
filter_type: str = "basic"
|
|
380
424
|
|
|
381
425
|
|
|
382
426
|
@dataclass
|
|
383
427
|
class SelectInputs:
|
|
384
428
|
"""A container for a list of SelectInput objects."""
|
|
385
|
-
|
|
429
|
+
|
|
430
|
+
renames: list[SelectInput] = field(default_factory=list)
|
|
386
431
|
|
|
387
432
|
@property
|
|
388
|
-
def old_cols(self) ->
|
|
433
|
+
def old_cols(self) -> set:
|
|
389
434
|
return set(v.old_name for v in self.renames if v.keep)
|
|
390
435
|
|
|
391
436
|
@property
|
|
392
|
-
def new_cols(self) ->
|
|
437
|
+
def new_cols(self) -> set:
|
|
393
438
|
return set(v.new_name for v in self.renames if v.keep)
|
|
394
439
|
|
|
395
440
|
|
|
396
441
|
@dataclass
|
|
397
442
|
class JoinInputs:
|
|
398
443
|
"""Extends SelectInputs with functionality specific to join operations."""
|
|
399
|
-
|
|
444
|
+
|
|
445
|
+
renames: list[SelectInput] = field(default_factory=list)
|
|
400
446
|
|
|
401
447
|
|
|
402
448
|
@dataclass
|
|
403
449
|
class JoinMap:
|
|
404
450
|
"""Defines a single mapping between a left and right column for a join key."""
|
|
451
|
+
|
|
405
452
|
left_col: str = None
|
|
406
453
|
right_col: str = None
|
|
407
454
|
|
|
@@ -409,6 +456,7 @@ class JoinMap:
|
|
|
409
456
|
@dataclass
|
|
410
457
|
class CrossJoinInput:
|
|
411
458
|
"""Defines the settings for a cross join operation."""
|
|
459
|
+
|
|
412
460
|
left_select: Any = None
|
|
413
461
|
right_select: Any = None
|
|
414
462
|
|
|
@@ -416,88 +464,98 @@ class CrossJoinInput:
|
|
|
416
464
|
@dataclass
|
|
417
465
|
class JoinInput:
|
|
418
466
|
"""Defines the settings for a standard SQL-style join."""
|
|
419
|
-
|
|
467
|
+
|
|
468
|
+
join_mapping: list[JoinMap] = field(default_factory=list)
|
|
420
469
|
left_select: Any = None
|
|
421
470
|
right_select: Any = None
|
|
422
|
-
how: str =
|
|
471
|
+
how: str = "inner"
|
|
423
472
|
|
|
424
473
|
|
|
425
474
|
@dataclass
|
|
426
475
|
class FuzzyMapping:
|
|
427
476
|
"""Defines a fuzzy match column mapping with threshold."""
|
|
477
|
+
|
|
428
478
|
left_col: str = None
|
|
429
479
|
right_col: str = None
|
|
430
480
|
threshold_score: int = 80
|
|
431
|
-
fuzzy_type: str =
|
|
481
|
+
fuzzy_type: str = "levenshtein"
|
|
432
482
|
|
|
433
483
|
|
|
434
484
|
@dataclass
|
|
435
485
|
class FuzzyMatchInput:
|
|
436
486
|
"""Extends JoinInput with settings specific to fuzzy matching."""
|
|
437
|
-
|
|
487
|
+
|
|
488
|
+
join_mapping: list[FuzzyMapping] = field(default_factory=list)
|
|
438
489
|
left_select: Any = None
|
|
439
490
|
right_select: Any = None
|
|
440
|
-
how: str =
|
|
491
|
+
how: str = "inner"
|
|
441
492
|
aggregate_output: bool = False
|
|
442
493
|
|
|
443
494
|
|
|
444
495
|
@dataclass
|
|
445
496
|
class AggColl:
|
|
446
497
|
"""Represents a single aggregation operation."""
|
|
498
|
+
|
|
447
499
|
old_name: str = None
|
|
448
500
|
agg: str = None
|
|
449
|
-
new_name:
|
|
450
|
-
output_type:
|
|
501
|
+
new_name: str | None = None
|
|
502
|
+
output_type: str | None = None
|
|
451
503
|
|
|
452
504
|
|
|
453
505
|
@dataclass
|
|
454
506
|
class GroupByInput:
|
|
455
507
|
"""Represents the input for a group by operation."""
|
|
456
|
-
|
|
508
|
+
|
|
509
|
+
agg_cols: list[AggColl] = field(default_factory=list)
|
|
457
510
|
|
|
458
511
|
|
|
459
512
|
@dataclass
|
|
460
513
|
class PivotInput:
|
|
461
514
|
"""Defines the settings for a pivot operation."""
|
|
462
|
-
|
|
515
|
+
|
|
516
|
+
index_columns: list[str] = field(default_factory=list)
|
|
463
517
|
pivot_column: str = None
|
|
464
518
|
value_col: str = None
|
|
465
|
-
aggregations:
|
|
519
|
+
aggregations: list[str] = field(default_factory=list)
|
|
466
520
|
|
|
467
521
|
|
|
468
522
|
@dataclass
|
|
469
523
|
class SortByInput:
|
|
470
524
|
"""Defines a single sort condition on a column."""
|
|
525
|
+
|
|
471
526
|
column: str = None
|
|
472
|
-
how: str =
|
|
527
|
+
how: str = "asc"
|
|
473
528
|
|
|
474
529
|
|
|
475
530
|
@dataclass
|
|
476
531
|
class RecordIdInput:
|
|
477
532
|
"""Defines settings for adding a record ID column."""
|
|
478
|
-
|
|
533
|
+
|
|
534
|
+
output_column_name: str = "record_id"
|
|
479
535
|
offset: int = 1
|
|
480
|
-
group_by:
|
|
481
|
-
group_by_columns:
|
|
536
|
+
group_by: bool | None = False
|
|
537
|
+
group_by_columns: list[str] | None = field(default_factory=list)
|
|
482
538
|
|
|
483
539
|
|
|
484
540
|
@dataclass
|
|
485
541
|
class TextToRowsInput:
|
|
486
542
|
"""Defines settings for splitting a text column into multiple rows."""
|
|
543
|
+
|
|
487
544
|
column_to_split: str = None
|
|
488
|
-
output_column_name:
|
|
489
|
-
split_by_fixed_value:
|
|
490
|
-
split_fixed_value:
|
|
491
|
-
split_by_column:
|
|
545
|
+
output_column_name: str | None = None
|
|
546
|
+
split_by_fixed_value: bool | None = True
|
|
547
|
+
split_fixed_value: str | None = ","
|
|
548
|
+
split_by_column: str | None = None
|
|
492
549
|
|
|
493
550
|
|
|
494
551
|
@dataclass
|
|
495
552
|
class UnpivotInput:
|
|
496
553
|
"""Defines settings for an unpivot operation."""
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
554
|
+
|
|
555
|
+
index_columns: list[str] | None = field(default_factory=list)
|
|
556
|
+
value_columns: list[str] | None = field(default_factory=list)
|
|
557
|
+
data_type_selector: Literal["float", "all", "date", "numeric", "string"] | None = None
|
|
558
|
+
data_type_selector_mode: Literal["data_type", "column"] | None = "column"
|
|
501
559
|
|
|
502
560
|
def __post_init__(self):
|
|
503
561
|
if self.index_columns is None:
|
|
@@ -509,38 +567,43 @@ class UnpivotInput:
|
|
|
509
567
|
@dataclass
|
|
510
568
|
class UnionInput:
|
|
511
569
|
"""Defines settings for a union operation."""
|
|
512
|
-
|
|
570
|
+
|
|
571
|
+
mode: Literal["selective", "relaxed"] = "relaxed"
|
|
513
572
|
|
|
514
573
|
|
|
515
574
|
@dataclass
|
|
516
575
|
class UniqueInput:
|
|
517
576
|
"""Defines settings for a uniqueness operation."""
|
|
518
|
-
|
|
577
|
+
|
|
578
|
+
columns: list[str] | None = None
|
|
519
579
|
strategy: str = "any"
|
|
520
580
|
|
|
521
581
|
|
|
522
582
|
@dataclass
|
|
523
583
|
class GraphSolverInput:
|
|
524
584
|
"""Defines settings for a graph-solving operation."""
|
|
585
|
+
|
|
525
586
|
col_from: str = None
|
|
526
587
|
col_to: str = None
|
|
527
|
-
output_column_name:
|
|
588
|
+
output_column_name: str | None = "graph_group"
|
|
528
589
|
|
|
529
590
|
|
|
530
591
|
@dataclass
|
|
531
592
|
class PolarsCodeInput:
|
|
532
593
|
"""A simple container for user-provided Polars code."""
|
|
533
|
-
|
|
594
|
+
|
|
595
|
+
polars_code: str = ""
|
|
534
596
|
|
|
535
597
|
|
|
536
598
|
@dataclass
|
|
537
599
|
class SampleInput:
|
|
538
600
|
"""Defines settings for sampling rows."""
|
|
539
|
-
|
|
540
|
-
|
|
601
|
+
|
|
602
|
+
n: int | None = None
|
|
603
|
+
fraction: float | None = None
|
|
541
604
|
with_replacement: bool = False
|
|
542
605
|
shuffle: bool = False
|
|
543
|
-
seed:
|
|
606
|
+
seed: int | None = None
|
|
544
607
|
|
|
545
608
|
|
|
546
609
|
# =============================================================================
|
|
@@ -550,72 +613,70 @@ class SampleInput:
|
|
|
550
613
|
|
|
551
614
|
LEGACY_CLASS_MAP = {
|
|
552
615
|
# Transform schema dataclasses
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
616
|
+
"SelectInput": SelectInput,
|
|
617
|
+
"FieldInput": FieldInput,
|
|
618
|
+
"FunctionInput": FunctionInput,
|
|
619
|
+
"BasicFilter": BasicFilter,
|
|
620
|
+
"FilterInput": FilterInput,
|
|
621
|
+
"SelectInputs": SelectInputs,
|
|
622
|
+
"JoinInputs": JoinInputs,
|
|
623
|
+
"JoinMap": JoinMap,
|
|
624
|
+
"CrossJoinInput": CrossJoinInput,
|
|
625
|
+
"JoinInput": JoinInput,
|
|
626
|
+
"FuzzyMapping": FuzzyMapping,
|
|
627
|
+
"FuzzyMatchInput": FuzzyMatchInput,
|
|
628
|
+
"AggColl": AggColl,
|
|
629
|
+
"GroupByInput": GroupByInput,
|
|
630
|
+
"PivotInput": PivotInput,
|
|
631
|
+
"SortByInput": SortByInput,
|
|
632
|
+
"RecordIdInput": RecordIdInput,
|
|
633
|
+
"TextToRowsInput": TextToRowsInput,
|
|
634
|
+
"UnpivotInput": UnpivotInput,
|
|
635
|
+
"UnionInput": UnionInput,
|
|
636
|
+
"UniqueInput": UniqueInput,
|
|
637
|
+
"GraphSolverInput": GraphSolverInput,
|
|
638
|
+
"PolarsCodeInput": PolarsCodeInput,
|
|
639
|
+
"SampleInput": SampleInput,
|
|
578
640
|
# OLD Input/Output schemas (before table_settings)
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
641
|
+
"ReceivedTable": ReceivedTable,
|
|
642
|
+
"OutputSettings": OutputSettings,
|
|
643
|
+
"OutputCsvTable": OutputCsvTable,
|
|
644
|
+
"OutputParquetTable": OutputParquetTable,
|
|
645
|
+
"OutputExcelTable": OutputExcelTable,
|
|
646
|
+
"MinimalFieldInfo": MinimalFieldInfo,
|
|
586
647
|
# Flow and Node schemas
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
648
|
+
"FlowSettings": FlowSettings,
|
|
649
|
+
"FlowGraphConfig": FlowGraphConfig,
|
|
650
|
+
"FlowInformation": FlowInformation,
|
|
651
|
+
"NodeInformation": NodeInformation,
|
|
652
|
+
"NodeBase": NodeBase,
|
|
653
|
+
"NodeSingleInput": NodeSingleInput,
|
|
654
|
+
"NodeMultiInput": NodeMultiInput,
|
|
655
|
+
"NodeRead": NodeRead,
|
|
656
|
+
"NodeSelect": NodeSelect,
|
|
657
|
+
"NodeFilter": NodeFilter,
|
|
658
|
+
"NodeFormula": NodeFormula,
|
|
659
|
+
"NodeJoin": NodeJoin,
|
|
660
|
+
"NodeCrossJoin": NodeCrossJoin,
|
|
661
|
+
"NodeFuzzyMatch": NodeFuzzyMatch,
|
|
662
|
+
"NodePolarsCode": NodePolarsCode,
|
|
663
|
+
"NodeOutput": NodeOutput,
|
|
664
|
+
"NodeGroupBy": NodeGroupBy,
|
|
665
|
+
"NodeSort": NodeSort,
|
|
666
|
+
"NodeUnion": NodeUnion,
|
|
667
|
+
"NodeUnique": NodeUnique,
|
|
668
|
+
"NodePivot": NodePivot,
|
|
669
|
+
"NodeUnpivot": NodeUnpivot,
|
|
670
|
+
"NodeRecordId": NodeRecordId,
|
|
671
|
+
"NodeTextToRows": NodeTextToRows,
|
|
672
|
+
"NodeGraphSolver": NodeGraphSolver,
|
|
673
|
+
"NodeSample": NodeSample,
|
|
674
|
+
"NodePromise": NodePromise,
|
|
675
|
+
"DatabaseConnection": DatabaseConnection,
|
|
676
|
+
"DatabaseSettings": DatabaseSettings,
|
|
677
|
+
"NodeDatabaseReader": NodeDatabaseReader,
|
|
617
678
|
}
|
|
618
679
|
|
|
619
680
|
|
|
620
681
|
# Export all classes
|
|
621
|
-
__all__ = list(LEGACY_CLASS_MAP.keys()) + [
|
|
682
|
+
__all__ = list(LEGACY_CLASS_MAP.keys()) + ["LEGACY_CLASS_MAP"]
|