Flowfile 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of Flowfile might be problematic. Click here for more details.
- flowfile/__init__.py +27 -6
- flowfile/api.py +1 -0
- flowfile/web/__init__.py +2 -2
- flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +86 -0
- flowfile/web/static/assets/CloudConnectionManager-c20a740f.js +783 -0
- flowfile/web/static/assets/CloudStorageReader-29d14fcc.css +143 -0
- flowfile/web/static/assets/CloudStorageReader-960b400a.js +437 -0
- flowfile/web/static/assets/CloudStorageWriter-49c9a4b2.css +138 -0
- flowfile/web/static/assets/CloudStorageWriter-e3decbdd.js +430 -0
- flowfile/web/static/assets/{CrossJoin-dfcf7351.js → CrossJoin-d67e2405.js} +8 -8
- flowfile/web/static/assets/{DatabaseConnectionSettings-b2afb1d7.js → DatabaseConnectionSettings-a81e0f7e.js} +2 -2
- flowfile/web/static/assets/{DatabaseManager-824a49b2.js → DatabaseManager-9ea35e84.js} +2 -2
- flowfile/web/static/assets/{DatabaseReader-a48124d8.js → DatabaseReader-9578bfa5.js} +9 -9
- flowfile/web/static/assets/{DatabaseWriter-b47cbae2.js → DatabaseWriter-19531098.js} +9 -9
- flowfile/web/static/assets/{ExploreData-fdfc45a4.js → ExploreData-40476474.js} +47141 -43697
- flowfile/web/static/assets/{ExternalSource-861b0e71.js → ExternalSource-2297ef96.js} +6 -6
- flowfile/web/static/assets/{Filter-f87bb897.js → Filter-f211c03a.js} +8 -8
- flowfile/web/static/assets/{Formula-b8cefc31.css → Formula-29f19d21.css} +10 -0
- flowfile/web/static/assets/{Formula-1e2ed720.js → Formula-4207ea31.js} +75 -9
- flowfile/web/static/assets/{FuzzyMatch-b6cc4fdd.js → FuzzyMatch-bf120df0.js} +9 -9
- flowfile/web/static/assets/{GraphSolver-6a371f4c.js → GraphSolver-5bb7497a.js} +5 -5
- flowfile/web/static/assets/{GroupBy-f7b7f472.js → GroupBy-92c81b65.js} +6 -6
- flowfile/web/static/assets/{Join-eec38203.js → Join-4e49a274.js} +23 -15
- flowfile/web/static/assets/{Join-41c0f331.css → Join-f45eff22.css} +20 -20
- flowfile/web/static/assets/{ManualInput-9aaa46fb.js → ManualInput-90998ae8.js} +106 -34
- flowfile/web/static/assets/{ManualInput-ac7b9972.css → ManualInput-a71b52c6.css} +29 -17
- flowfile/web/static/assets/{Output-3b2ca045.js → Output-81e3e917.js} +4 -4
- flowfile/web/static/assets/{Pivot-a4f5d88f.js → Pivot-a3419842.js} +6 -6
- flowfile/web/static/assets/{PolarsCode-49ce444f.js → PolarsCode-72710deb.js} +6 -6
- flowfile/web/static/assets/{Read-07acdc9a.js → Read-c4059daf.js} +6 -6
- flowfile/web/static/assets/{RecordCount-6a21da56.js → RecordCount-c2b5e095.js} +5 -5
- flowfile/web/static/assets/{RecordId-949bdc17.js → RecordId-10baf191.js} +6 -6
- flowfile/web/static/assets/{Sample-7afca6e1.js → Sample-3ed9a0ae.js} +5 -5
- flowfile/web/static/assets/{SecretManager-b41c029d.js → SecretManager-0d49c0e8.js} +2 -2
- flowfile/web/static/assets/{Select-32b28406.js → Select-8a02a0b3.js} +8 -8
- flowfile/web/static/assets/{SettingsSection-a0f15a05.js → SettingsSection-4c0f45f5.js} +1 -1
- flowfile/web/static/assets/{Sort-fc6ba0e2.js → Sort-f55c9f9d.js} +6 -6
- flowfile/web/static/assets/{TextToRows-23127596.js → TextToRows-5dbc2145.js} +8 -8
- flowfile/web/static/assets/{UnavailableFields-c42880a3.js → UnavailableFields-a1768e52.js} +2 -2
- flowfile/web/static/assets/{Union-39eecc6c.js → Union-f2aefdc9.js} +5 -5
- flowfile/web/static/assets/{Unique-a0e8fe61.js → Unique-46b250da.js} +8 -8
- flowfile/web/static/assets/{Unpivot-1e2d43f0.js → Unpivot-25ac84cc.js} +5 -5
- flowfile/web/static/assets/api-6ef0dcef.js +80 -0
- flowfile/web/static/assets/{api-44ca9e9c.js → api-a0abbdc7.js} +1 -1
- flowfile/web/static/assets/cloud_storage_reader-aa1415d6.png +0 -0
- flowfile/web/static/assets/{designer-267d44f1.js → designer-13eabd83.js} +36 -34
- flowfile/web/static/assets/{documentation-6c0810a2.js → documentation-b87e7f6f.js} +1 -1
- flowfile/web/static/assets/{dropDown-52790b15.js → dropDown-13564764.js} +1 -1
- flowfile/web/static/assets/{fullEditor-e272b506.js → fullEditor-fd2cd6f9.js} +2 -2
- flowfile/web/static/assets/{genericNodeSettings-4bdcf98e.js → genericNodeSettings-71e11604.js} +3 -3
- flowfile/web/static/assets/{index-e235a8bc.js → index-f6c15e76.js} +59 -22
- flowfile/web/static/assets/{nodeTitle-fc3fc4b7.js → nodeTitle-988d9efe.js} +3 -3
- flowfile/web/static/assets/{secretApi-cdc2a3fd.js → secretApi-dd636aa2.js} +1 -1
- flowfile/web/static/assets/{selectDynamic-96aa82cd.js → selectDynamic-af36165e.js} +3 -3
- flowfile/web/static/assets/{vue-codemirror.esm-25e75a08.js → vue-codemirror.esm-2847001e.js} +2 -1
- flowfile/web/static/assets/{vue-content-loader.es-6c4b1c24.js → vue-content-loader.es-0371da73.js} +1 -1
- flowfile/web/static/index.html +1 -1
- {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/METADATA +9 -4
- {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/RECORD +131 -124
- {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/entry_points.txt +2 -0
- flowfile_core/__init__.py +3 -0
- flowfile_core/auth/jwt.py +39 -0
- flowfile_core/configs/node_store/nodes.py +9 -6
- flowfile_core/configs/settings.py +6 -5
- flowfile_core/database/connection.py +63 -15
- flowfile_core/database/init_db.py +0 -1
- flowfile_core/database/models.py +49 -2
- flowfile_core/flowfile/code_generator/code_generator.py +472 -17
- flowfile_core/flowfile/connection_manager/models.py +1 -1
- flowfile_core/flowfile/database_connection_manager/db_connections.py +216 -2
- flowfile_core/flowfile/extensions.py +1 -1
- flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +259 -0
- flowfile_core/flowfile/flow_data_engine/create/funcs.py +19 -8
- flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +1062 -311
- flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +12 -2
- flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py +1 -1
- flowfile_core/flowfile/flow_data_engine/join/__init__.py +2 -1
- flowfile_core/flowfile/flow_data_engine/join/utils.py +25 -0
- flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +3 -1
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +29 -22
- flowfile_core/flowfile/flow_data_engine/utils.py +1 -40
- flowfile_core/flowfile/flow_graph.py +718 -253
- flowfile_core/flowfile/flow_graph_utils.py +2 -2
- flowfile_core/flowfile/flow_node/flow_node.py +563 -117
- flowfile_core/flowfile/flow_node/models.py +154 -20
- flowfile_core/flowfile/flow_node/schema_callback.py +3 -2
- flowfile_core/flowfile/handler.py +2 -33
- flowfile_core/flowfile/manage/open_flowfile.py +1 -2
- flowfile_core/flowfile/sources/external_sources/__init__.py +0 -2
- flowfile_core/flowfile/sources/external_sources/factory.py +4 -7
- flowfile_core/flowfile/util/calculate_layout.py +0 -2
- flowfile_core/flowfile/utils.py +35 -26
- flowfile_core/main.py +35 -15
- flowfile_core/routes/cloud_connections.py +77 -0
- flowfile_core/routes/logs.py +2 -7
- flowfile_core/routes/public.py +1 -0
- flowfile_core/routes/routes.py +130 -90
- flowfile_core/routes/secrets.py +72 -14
- flowfile_core/schemas/__init__.py +8 -0
- flowfile_core/schemas/cloud_storage_schemas.py +215 -0
- flowfile_core/schemas/input_schema.py +121 -71
- flowfile_core/schemas/output_model.py +19 -3
- flowfile_core/schemas/schemas.py +150 -12
- flowfile_core/schemas/transform_schema.py +175 -35
- flowfile_core/utils/utils.py +40 -1
- flowfile_core/utils/validate_setup.py +41 -0
- flowfile_frame/__init__.py +9 -1
- flowfile_frame/cloud_storage/frame_helpers.py +39 -0
- flowfile_frame/cloud_storage/secret_manager.py +73 -0
- flowfile_frame/expr.py +28 -1
- flowfile_frame/expr.pyi +76 -61
- flowfile_frame/flow_frame.py +481 -208
- flowfile_frame/flow_frame.pyi +140 -91
- flowfile_frame/flow_frame_methods.py +160 -22
- flowfile_frame/group_frame.py +3 -0
- flowfile_frame/utils.py +25 -3
- flowfile_worker/external_sources/s3_source/main.py +216 -0
- flowfile_worker/external_sources/s3_source/models.py +142 -0
- flowfile_worker/funcs.py +51 -6
- flowfile_worker/models.py +22 -2
- flowfile_worker/routes.py +40 -38
- flowfile_worker/utils.py +1 -1
- test_utils/s3/commands.py +46 -0
- test_utils/s3/data_generator.py +292 -0
- test_utils/s3/demo_data_generator.py +186 -0
- test_utils/s3/fixtures.py +214 -0
- flowfile/web/static/assets/AirbyteReader-1ac35765.css +0 -314
- flowfile/web/static/assets/AirbyteReader-e08044e5.js +0 -922
- flowfile/web/static/assets/dropDownGeneric-60f56a8a.js +0 -72
- flowfile/web/static/assets/dropDownGeneric-895680d6.css +0 -10
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/airbyte.py +0 -159
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py +0 -172
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/settings.py +0 -173
- flowfile_core/schemas/defaults.py +0 -9
- flowfile_core/schemas/external_sources/airbyte_schemas.py +0 -20
- flowfile_core/schemas/models.py +0 -193
- flowfile_worker/external_sources/airbyte_sources/cache_manager.py +0 -161
- flowfile_worker/external_sources/airbyte_sources/main.py +0 -89
- flowfile_worker/external_sources/airbyte_sources/models.py +0 -133
- flowfile_worker/external_sources/airbyte_sources/settings.py +0 -0
- {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/LICENSE +0 -0
- {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/WHEEL +0 -0
- {flowfile_core/flowfile/sources/external_sources/airbyte_sources → flowfile_frame/cloud_storage}/__init__.py +0 -0
- {flowfile_core/schemas/external_sources → flowfile_worker/external_sources/s3_source}/__init__.py +0 -0
- {flowfile_worker/external_sources/airbyte_sources → test_utils/s3}/__init__.py +0 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
|
|
2
|
+
.cloud-storage-container[data-v-8f1c7f30] {
|
|
3
|
+
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
|
4
|
+
max-width: 100%;
|
|
5
|
+
color: #333;
|
|
6
|
+
}
|
|
7
|
+
.section-subtitle[data-v-8f1c7f30] {
|
|
8
|
+
margin: 0 0 0.75rem 0;
|
|
9
|
+
font-size: 0.95rem;
|
|
10
|
+
font-weight: 600;
|
|
11
|
+
color: #4a5568;
|
|
12
|
+
}
|
|
13
|
+
.subsection-title[data-v-8f1c7f30] {
|
|
14
|
+
margin: 0.5rem 0 0.5rem 0;
|
|
15
|
+
font-size: 0.875rem;
|
|
16
|
+
font-weight: 600;
|
|
17
|
+
color: #718096;
|
|
18
|
+
}
|
|
19
|
+
.format-options[data-v-8f1c7f30] {
|
|
20
|
+
margin-top: 1rem;
|
|
21
|
+
padding: 1rem;
|
|
22
|
+
background-color: #f7fafc;
|
|
23
|
+
border-radius: 4px;
|
|
24
|
+
border: 1px solid #e2e8f0;
|
|
25
|
+
}
|
|
26
|
+
.form-row[data-v-8f1c7f30] {
|
|
27
|
+
display: flex;
|
|
28
|
+
gap: 0.75rem;
|
|
29
|
+
margin-bottom: 0.75rem;
|
|
30
|
+
width: 100%;
|
|
31
|
+
box-sizing: border-box;
|
|
32
|
+
}
|
|
33
|
+
.half[data-v-8f1c7f30] {
|
|
34
|
+
flex: 1;
|
|
35
|
+
min-width: 0;
|
|
36
|
+
max-width: calc(50% - 0.375rem);
|
|
37
|
+
}
|
|
38
|
+
.form-control[data-v-8f1c7f30] {
|
|
39
|
+
width: 100%;
|
|
40
|
+
padding: 0.5rem;
|
|
41
|
+
border: 1px solid #e2e8f0;
|
|
42
|
+
border-radius: 4px;
|
|
43
|
+
font-size: 0.875rem;
|
|
44
|
+
box-sizing: border-box;
|
|
45
|
+
}
|
|
46
|
+
.form-group[data-v-8f1c7f30] {
|
|
47
|
+
margin-bottom: 0.75rem;
|
|
48
|
+
width: 100%;
|
|
49
|
+
}
|
|
50
|
+
label[data-v-8f1c7f30] {
|
|
51
|
+
display: block;
|
|
52
|
+
margin-bottom: 0.25rem;
|
|
53
|
+
font-size: 0.875rem;
|
|
54
|
+
font-weight: 500;
|
|
55
|
+
color: #4a5568;
|
|
56
|
+
}
|
|
57
|
+
select.form-control[data-v-8f1c7f30] {
|
|
58
|
+
appearance: none;
|
|
59
|
+
background-image: url("data:image/svg+xml;charset=utf-8,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='none' stroke='%234a5568' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpolyline points='6 9 12 15 18 9'/%3E%3C/svg%3E");
|
|
60
|
+
background-repeat: no-repeat;
|
|
61
|
+
background-position: right 0.5rem center;
|
|
62
|
+
background-size: 1em;
|
|
63
|
+
padding-right: 2rem;
|
|
64
|
+
}
|
|
65
|
+
.checkbox-container[data-v-8f1c7f30] {
|
|
66
|
+
display: flex;
|
|
67
|
+
align-items: center;
|
|
68
|
+
gap: 0.5rem;
|
|
69
|
+
}
|
|
70
|
+
.checkbox-input[data-v-8f1c7f30] {
|
|
71
|
+
width: 1rem;
|
|
72
|
+
height: 1rem;
|
|
73
|
+
cursor: pointer;
|
|
74
|
+
}
|
|
75
|
+
.checkbox-label[data-v-8f1c7f30] {
|
|
76
|
+
margin: 0;
|
|
77
|
+
cursor: pointer;
|
|
78
|
+
}
|
|
79
|
+
.info-box[data-v-8f1c7f30] {
|
|
80
|
+
display: flex;
|
|
81
|
+
gap: 0.75rem;
|
|
82
|
+
padding: 0.75rem;
|
|
83
|
+
background-color: #e6f7ff;
|
|
84
|
+
border-left: 4px solid #1890ff;
|
|
85
|
+
border-radius: 4px;
|
|
86
|
+
margin-top: 1rem;
|
|
87
|
+
font-size: 0.875rem;
|
|
88
|
+
}
|
|
89
|
+
.info-box i[data-v-8f1c7f30] {
|
|
90
|
+
color: #1890ff;
|
|
91
|
+
font-size: 1.25rem;
|
|
92
|
+
flex-shrink: 0;
|
|
93
|
+
}
|
|
94
|
+
.info-box p[data-v-8f1c7f30] {
|
|
95
|
+
margin: 0;
|
|
96
|
+
color: #4a5568;
|
|
97
|
+
}
|
|
98
|
+
.helper-text[data-v-8f1c7f30] {
|
|
99
|
+
display: flex;
|
|
100
|
+
align-items: center;
|
|
101
|
+
gap: 0.5rem;
|
|
102
|
+
margin-top: 0.5rem;
|
|
103
|
+
font-size: 0.8125rem;
|
|
104
|
+
color: #718096;
|
|
105
|
+
}
|
|
106
|
+
.helper-text i[data-v-8f1c7f30] {
|
|
107
|
+
color: #4299e1;
|
|
108
|
+
font-size: 0.875rem;
|
|
109
|
+
}
|
|
110
|
+
.loading-state[data-v-8f1c7f30] {
|
|
111
|
+
display: flex;
|
|
112
|
+
flex-direction: column;
|
|
113
|
+
align-items: center;
|
|
114
|
+
gap: 0.5rem;
|
|
115
|
+
padding: 1rem;
|
|
116
|
+
}
|
|
117
|
+
.loading-state p[data-v-8f1c7f30] {
|
|
118
|
+
margin: 0;
|
|
119
|
+
color: #718096;
|
|
120
|
+
font-size: 0.875rem;
|
|
121
|
+
}
|
|
122
|
+
.loading-spinner[data-v-8f1c7f30] {
|
|
123
|
+
width: 2rem;
|
|
124
|
+
height: 2rem;
|
|
125
|
+
border: 2px solid #e2e8f0;
|
|
126
|
+
border-top-color: #4299e1;
|
|
127
|
+
border-radius: 50%;
|
|
128
|
+
animation: spin-8f1c7f30 0.8s linear infinite;
|
|
129
|
+
}
|
|
130
|
+
@keyframes spin-8f1c7f30 {
|
|
131
|
+
to {
|
|
132
|
+
transform: rotate(360deg);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
@media (max-width: 640px) {
|
|
136
|
+
.form-row[data-v-8f1c7f30] {
|
|
137
|
+
flex-direction: column;
|
|
138
|
+
gap: 0.5rem;
|
|
139
|
+
}
|
|
140
|
+
.half[data-v-8f1c7f30] {
|
|
141
|
+
max-width: 100%;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
import { d as defineComponent, r as ref, c as openBlock, e as createElementBlock, f as createVNode, w as withCtx, p as createBaseVNode, a5 as withDirectives, F as Fragment, q as renderList, t as toDisplayString, ag as vModelSelect, g as createTextVNode, i as createCommentVNode, a6 as vModelText, ai as vModelCheckbox, h as createBlock, u as unref, z as ElMessage, _ as _export_sfc, n as onMounted, R as nextTick, a7 as Teleport } from "./index-f6c15e76.js";
|
|
2
|
+
import { C as CodeLoader } from "./vue-content-loader.es-0371da73.js";
|
|
3
|
+
import { u as useNodeStore } from "./vue-codemirror.esm-2847001e.js";
|
|
4
|
+
import { f as fetchCloudStorageConnectionsInterfaces } from "./api-6ef0dcef.js";
|
|
5
|
+
import { G as GenericNodeSettings } from "./genericNodeSettings-71e11604.js";
|
|
6
|
+
import { N as NodeButton, a as NodeTitle } from "./nodeTitle-988d9efe.js";
|
|
7
|
+
import "./designer-13eabd83.js";
|
|
8
|
+
const createNodeCloudStorageReader = (flowId, nodeId) => {
|
|
9
|
+
const cloudStorageReadSettings = {
|
|
10
|
+
auth_mode: "aws-cli",
|
|
11
|
+
scan_mode: "directory",
|
|
12
|
+
resource_path: "",
|
|
13
|
+
file_format: void 0,
|
|
14
|
+
csv_has_header: false,
|
|
15
|
+
csv_encoding: "utf8",
|
|
16
|
+
delta_version: void 0
|
|
17
|
+
};
|
|
18
|
+
const nodePolarsCode = {
|
|
19
|
+
flow_id: flowId,
|
|
20
|
+
node_id: nodeId,
|
|
21
|
+
pos_x: 0,
|
|
22
|
+
pos_y: 0,
|
|
23
|
+
cloud_storage_settings: cloudStorageReadSettings,
|
|
24
|
+
cache_results: false,
|
|
25
|
+
fields: []
|
|
26
|
+
};
|
|
27
|
+
return nodePolarsCode;
|
|
28
|
+
};
|
|
29
|
+
const _hoisted_1$1 = {
|
|
30
|
+
key: 0,
|
|
31
|
+
class: "cloud-storage-container"
|
|
32
|
+
};
|
|
33
|
+
const _hoisted_2 = { class: "listbox-wrapper" };
|
|
34
|
+
const _hoisted_3 = { class: "form-group" };
|
|
35
|
+
const _hoisted_4 = {
|
|
36
|
+
key: 0,
|
|
37
|
+
class: "loading-state"
|
|
38
|
+
};
|
|
39
|
+
const _hoisted_5 = { key: 1 };
|
|
40
|
+
const _hoisted_6 = ["value"];
|
|
41
|
+
const _hoisted_7 = {
|
|
42
|
+
key: 0,
|
|
43
|
+
class: "helper-text"
|
|
44
|
+
};
|
|
45
|
+
const _hoisted_8 = { class: "listbox-wrapper" };
|
|
46
|
+
const _hoisted_9 = { class: "form-group" };
|
|
47
|
+
const _hoisted_10 = { class: "form-group" };
|
|
48
|
+
const _hoisted_11 = {
|
|
49
|
+
key: 0,
|
|
50
|
+
class: "form-group"
|
|
51
|
+
};
|
|
52
|
+
const _hoisted_12 = {
|
|
53
|
+
key: 1,
|
|
54
|
+
class: "format-options"
|
|
55
|
+
};
|
|
56
|
+
const _hoisted_13 = { class: "form-group" };
|
|
57
|
+
const _hoisted_14 = { class: "checkbox-container" };
|
|
58
|
+
const _hoisted_15 = { class: "form-row" };
|
|
59
|
+
const _hoisted_16 = { class: "form-group half" };
|
|
60
|
+
const _hoisted_17 = { class: "form-group half" };
|
|
61
|
+
const _hoisted_18 = {
|
|
62
|
+
key: 2,
|
|
63
|
+
class: "format-options"
|
|
64
|
+
};
|
|
65
|
+
const _hoisted_19 = { class: "form-group" };
|
|
66
|
+
const _hoisted_20 = {
|
|
67
|
+
key: 3,
|
|
68
|
+
class: "info-box"
|
|
69
|
+
};
|
|
70
|
+
const _sfc_main$1 = /* @__PURE__ */ defineComponent({
|
|
71
|
+
__name: "CloudStorageReader",
|
|
72
|
+
props: {
|
|
73
|
+
nodeId: {}
|
|
74
|
+
},
|
|
75
|
+
setup(__props, { expose: __expose }) {
|
|
76
|
+
const nodeStore = useNodeStore();
|
|
77
|
+
const dataLoaded = ref(false);
|
|
78
|
+
const nodeCloudStorageReader = ref(null);
|
|
79
|
+
const connectionInterfaces = ref([]);
|
|
80
|
+
const connectionsAreLoading = ref(false);
|
|
81
|
+
const selectedConnection = ref(null);
|
|
82
|
+
const getStorageTypeLabel = (storageType) => {
|
|
83
|
+
switch (storageType) {
|
|
84
|
+
case "s3":
|
|
85
|
+
return "AWS S3";
|
|
86
|
+
case "adls":
|
|
87
|
+
return "Azure ADLS";
|
|
88
|
+
case "gcs":
|
|
89
|
+
return "Google Cloud Storage";
|
|
90
|
+
default:
|
|
91
|
+
return storageType.toUpperCase();
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
const getAuthMethodLabel = (authMethod) => {
|
|
95
|
+
switch (authMethod) {
|
|
96
|
+
case "access_key":
|
|
97
|
+
return "Access Key";
|
|
98
|
+
case "iam_role":
|
|
99
|
+
return "IAM Role";
|
|
100
|
+
case "service_principal":
|
|
101
|
+
return "Service Principal";
|
|
102
|
+
case "managed_identity":
|
|
103
|
+
return "Managed Identity";
|
|
104
|
+
case "sas_token":
|
|
105
|
+
return "SAS Token";
|
|
106
|
+
case "aws-cli":
|
|
107
|
+
return "AWS CLI";
|
|
108
|
+
case "auto":
|
|
109
|
+
return "Auto";
|
|
110
|
+
default:
|
|
111
|
+
return authMethod;
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
const handleFileFormatChange = () => {
|
|
115
|
+
resetFields();
|
|
116
|
+
if (nodeCloudStorageReader.value) {
|
|
117
|
+
const format = nodeCloudStorageReader.value.cloud_storage_settings.file_format;
|
|
118
|
+
if (format === "csv") {
|
|
119
|
+
if (nodeCloudStorageReader.value.cloud_storage_settings.csv_has_header === void 0) {
|
|
120
|
+
nodeCloudStorageReader.value.cloud_storage_settings.csv_has_header = true;
|
|
121
|
+
}
|
|
122
|
+
if (!nodeCloudStorageReader.value.cloud_storage_settings.csv_delimiter) {
|
|
123
|
+
nodeCloudStorageReader.value.cloud_storage_settings.csv_delimiter = ",";
|
|
124
|
+
}
|
|
125
|
+
if (!nodeCloudStorageReader.value.cloud_storage_settings.csv_encoding) {
|
|
126
|
+
nodeCloudStorageReader.value.cloud_storage_settings.csv_encoding = "utf8";
|
|
127
|
+
}
|
|
128
|
+
} else {
|
|
129
|
+
nodeCloudStorageReader.value.cloud_storage_settings.csv_has_header = void 0;
|
|
130
|
+
nodeCloudStorageReader.value.cloud_storage_settings.csv_delimiter = void 0;
|
|
131
|
+
nodeCloudStorageReader.value.cloud_storage_settings.csv_encoding = void 0;
|
|
132
|
+
}
|
|
133
|
+
if (format !== "delta") {
|
|
134
|
+
nodeCloudStorageReader.value.cloud_storage_settings.delta_version = void 0;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
const resetFields = () => {
|
|
139
|
+
if (nodeCloudStorageReader.value) {
|
|
140
|
+
nodeCloudStorageReader.value.fields = [];
|
|
141
|
+
if (!selectedConnection.value) {
|
|
142
|
+
nodeCloudStorageReader.value.cloud_storage_settings.auth_mode = "aws-cli";
|
|
143
|
+
nodeCloudStorageReader.value.cloud_storage_settings.connection_name = void 0;
|
|
144
|
+
} else {
|
|
145
|
+
nodeCloudStorageReader.value.cloud_storage_settings.auth_mode = selectedConnection.value.authMethod;
|
|
146
|
+
nodeCloudStorageReader.value.cloud_storage_settings.connection_name = selectedConnection.value.connectionName;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
const setConnectionOnConnectionName = async (connectionName) => {
|
|
151
|
+
selectedConnection.value = connectionInterfaces.value.find(
|
|
152
|
+
(connectionInterface) => connectionInterface.connectionName === connectionName
|
|
153
|
+
// Use '===' for strict equality
|
|
154
|
+
) || null;
|
|
155
|
+
};
|
|
156
|
+
const loadNodeData = async (nodeId) => {
|
|
157
|
+
var _a, _b;
|
|
158
|
+
try {
|
|
159
|
+
const [nodeData] = await Promise.all([
|
|
160
|
+
nodeStore.getNodeData(nodeId, false),
|
|
161
|
+
fetchConnections()
|
|
162
|
+
]);
|
|
163
|
+
if (nodeData) {
|
|
164
|
+
const hasValidSetup = Boolean((_a = nodeData.setting_input) == null ? void 0 : _a.is_setup);
|
|
165
|
+
nodeCloudStorageReader.value = hasValidSetup ? nodeData.setting_input : createNodeCloudStorageReader(nodeStore.flow_id, nodeId);
|
|
166
|
+
if ((_b = nodeCloudStorageReader.value) == null ? void 0 : _b.cloud_storage_settings.connection_name) {
|
|
167
|
+
await setConnectionOnConnectionName(
|
|
168
|
+
nodeCloudStorageReader.value.cloud_storage_settings.connection_name
|
|
169
|
+
);
|
|
170
|
+
} else {
|
|
171
|
+
selectedConnection.value = null;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
dataLoaded.value = true;
|
|
175
|
+
} catch (error) {
|
|
176
|
+
console.error("Error loading node data:", error);
|
|
177
|
+
dataLoaded.value = false;
|
|
178
|
+
}
|
|
179
|
+
};
|
|
180
|
+
const pushNodeData = async () => {
|
|
181
|
+
if (!nodeCloudStorageReader.value || !nodeCloudStorageReader.value.cloud_storage_settings) {
|
|
182
|
+
console.log("Returning without pushing");
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
nodeCloudStorageReader.value.is_setup = true;
|
|
186
|
+
nodeStore.updateSettings(nodeCloudStorageReader);
|
|
187
|
+
dataLoaded.value = false;
|
|
188
|
+
};
|
|
189
|
+
const fetchConnections = async () => {
|
|
190
|
+
connectionsAreLoading.value = true;
|
|
191
|
+
try {
|
|
192
|
+
connectionInterfaces.value = await fetchCloudStorageConnectionsInterfaces();
|
|
193
|
+
} catch (error) {
|
|
194
|
+
console.error("Error fetching connections:", error);
|
|
195
|
+
ElMessage.error("Failed to load cloud storage connections");
|
|
196
|
+
} finally {
|
|
197
|
+
connectionsAreLoading.value = false;
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
__expose({
|
|
201
|
+
loadNodeData,
|
|
202
|
+
pushNodeData
|
|
203
|
+
});
|
|
204
|
+
return (_ctx, _cache) => {
|
|
205
|
+
return dataLoaded.value && nodeCloudStorageReader.value ? (openBlock(), createElementBlock("div", _hoisted_1$1, [
|
|
206
|
+
createVNode(GenericNodeSettings, {
|
|
207
|
+
modelValue: nodeCloudStorageReader.value,
|
|
208
|
+
"onUpdate:modelValue": _cache[8] || (_cache[8] = ($event) => nodeCloudStorageReader.value = $event)
|
|
209
|
+
}, {
|
|
210
|
+
default: withCtx(() => [
|
|
211
|
+
createBaseVNode("div", _hoisted_2, [
|
|
212
|
+
createBaseVNode("div", _hoisted_3, [
|
|
213
|
+
_cache[12] || (_cache[12] = createBaseVNode("label", { for: "connection-select" }, "Cloud Storage Connection", -1)),
|
|
214
|
+
connectionsAreLoading.value ? (openBlock(), createElementBlock("div", _hoisted_4, _cache[9] || (_cache[9] = [
|
|
215
|
+
createBaseVNode("div", { class: "loading-spinner" }, null, -1),
|
|
216
|
+
createBaseVNode("p", null, "Loading connections...", -1)
|
|
217
|
+
]))) : (openBlock(), createElementBlock("div", _hoisted_5, [
|
|
218
|
+
withDirectives(createBaseVNode("select", {
|
|
219
|
+
id: "connection-select",
|
|
220
|
+
"onUpdate:modelValue": _cache[0] || (_cache[0] = ($event) => selectedConnection.value = $event),
|
|
221
|
+
class: "form-control minimal-select",
|
|
222
|
+
onChange: resetFields
|
|
223
|
+
}, [
|
|
224
|
+
_cache[10] || (_cache[10] = createBaseVNode("option", { value: null }, "No connection (use local credentials)", -1)),
|
|
225
|
+
(openBlock(true), createElementBlock(Fragment, null, renderList(connectionInterfaces.value, (conn) => {
|
|
226
|
+
return openBlock(), createElementBlock("option", {
|
|
227
|
+
key: conn.connectionName,
|
|
228
|
+
value: conn
|
|
229
|
+
}, toDisplayString(conn.connectionName) + " (" + toDisplayString(getStorageTypeLabel(conn.storageType)) + " - " + toDisplayString(getAuthMethodLabel(conn.authMethod)) + ") ", 9, _hoisted_6);
|
|
230
|
+
}), 128))
|
|
231
|
+
], 544), [
|
|
232
|
+
[vModelSelect, selectedConnection.value]
|
|
233
|
+
]),
|
|
234
|
+
!nodeCloudStorageReader.value.cloud_storage_settings.connection_name ? (openBlock(), createElementBlock("div", _hoisted_7, _cache[11] || (_cache[11] = [
|
|
235
|
+
createBaseVNode("i", { class: "fa-solid fa-info-circle" }, null, -1),
|
|
236
|
+
createTextVNode(" Will use local AWS CLI credentials or environment variables ")
|
|
237
|
+
]))) : createCommentVNode("", true)
|
|
238
|
+
]))
|
|
239
|
+
])
|
|
240
|
+
]),
|
|
241
|
+
createBaseVNode("div", _hoisted_8, [
|
|
242
|
+
_cache[26] || (_cache[26] = createBaseVNode("h4", { class: "section-subtitle" }, "File Settings", -1)),
|
|
243
|
+
createBaseVNode("div", _hoisted_9, [
|
|
244
|
+
_cache[13] || (_cache[13] = createBaseVNode("label", { for: "file-path" }, "File Path", -1)),
|
|
245
|
+
withDirectives(createBaseVNode("input", {
|
|
246
|
+
id: "file-path",
|
|
247
|
+
"onUpdate:modelValue": _cache[1] || (_cache[1] = ($event) => nodeCloudStorageReader.value.cloud_storage_settings.resource_path = $event),
|
|
248
|
+
type: "text",
|
|
249
|
+
class: "form-control",
|
|
250
|
+
placeholder: "e.g., bucket-name/folder/file.csv or bucket-name/folder/",
|
|
251
|
+
onInput: resetFields
|
|
252
|
+
}, null, 544), [
|
|
253
|
+
[vModelText, nodeCloudStorageReader.value.cloud_storage_settings.resource_path]
|
|
254
|
+
])
|
|
255
|
+
]),
|
|
256
|
+
createBaseVNode("div", _hoisted_10, [
|
|
257
|
+
_cache[15] || (_cache[15] = createBaseVNode("label", { for: "file-format" }, "File Format", -1)),
|
|
258
|
+
withDirectives(createBaseVNode("select", {
|
|
259
|
+
id: "file-format",
|
|
260
|
+
"onUpdate:modelValue": _cache[2] || (_cache[2] = ($event) => nodeCloudStorageReader.value.cloud_storage_settings.file_format = $event),
|
|
261
|
+
class: "form-control",
|
|
262
|
+
onChange: handleFileFormatChange
|
|
263
|
+
}, _cache[14] || (_cache[14] = [
|
|
264
|
+
createBaseVNode("option", { value: "csv" }, "CSV", -1),
|
|
265
|
+
createBaseVNode("option", { value: "parquet" }, "Parquet", -1),
|
|
266
|
+
createBaseVNode("option", { value: "json" }, "JSON", -1),
|
|
267
|
+
createBaseVNode("option", { value: "delta" }, "Delta Lake", -1)
|
|
268
|
+
]), 544), [
|
|
269
|
+
[vModelSelect, nodeCloudStorageReader.value.cloud_storage_settings.file_format]
|
|
270
|
+
])
|
|
271
|
+
]),
|
|
272
|
+
nodeCloudStorageReader.value.cloud_storage_settings.file_format !== "delta" ? (openBlock(), createElementBlock("div", _hoisted_11, [
|
|
273
|
+
_cache[17] || (_cache[17] = createBaseVNode("label", { for: "scan-mode" }, "Scan Mode", -1)),
|
|
274
|
+
withDirectives(createBaseVNode("select", {
|
|
275
|
+
id: "scan-mode",
|
|
276
|
+
"onUpdate:modelValue": _cache[3] || (_cache[3] = ($event) => nodeCloudStorageReader.value.cloud_storage_settings.scan_mode = $event),
|
|
277
|
+
class: "form-control"
|
|
278
|
+
}, _cache[16] || (_cache[16] = [
|
|
279
|
+
createBaseVNode("option", { value: "single_file" }, "Single File", -1),
|
|
280
|
+
createBaseVNode("option", { value: "directory" }, "Directory", -1)
|
|
281
|
+
]), 512), [
|
|
282
|
+
[vModelSelect, nodeCloudStorageReader.value.cloud_storage_settings.scan_mode]
|
|
283
|
+
])
|
|
284
|
+
])) : createCommentVNode("", true),
|
|
285
|
+
nodeCloudStorageReader.value.cloud_storage_settings.file_format === "csv" ? (openBlock(), createElementBlock("div", _hoisted_12, [
|
|
286
|
+
_cache[22] || (_cache[22] = createBaseVNode("h5", { class: "subsection-title" }, "CSV Options", -1)),
|
|
287
|
+
createBaseVNode("div", _hoisted_13, [
|
|
288
|
+
createBaseVNode("div", _hoisted_14, [
|
|
289
|
+
withDirectives(createBaseVNode("input", {
|
|
290
|
+
id: "csv-has-header",
|
|
291
|
+
"onUpdate:modelValue": _cache[4] || (_cache[4] = ($event) => nodeCloudStorageReader.value.cloud_storage_settings.csv_has_header = $event),
|
|
292
|
+
type: "checkbox",
|
|
293
|
+
class: "checkbox-input"
|
|
294
|
+
}, null, 512), [
|
|
295
|
+
[vModelCheckbox, nodeCloudStorageReader.value.cloud_storage_settings.csv_has_header]
|
|
296
|
+
]),
|
|
297
|
+
_cache[18] || (_cache[18] = createBaseVNode("label", {
|
|
298
|
+
for: "csv-has-header",
|
|
299
|
+
class: "checkbox-label"
|
|
300
|
+
}, "First row contains headers", -1))
|
|
301
|
+
])
|
|
302
|
+
]),
|
|
303
|
+
createBaseVNode("div", _hoisted_15, [
|
|
304
|
+
createBaseVNode("div", _hoisted_16, [
|
|
305
|
+
_cache[19] || (_cache[19] = createBaseVNode("label", { for: "csv-delimiter" }, "Delimiter", -1)),
|
|
306
|
+
withDirectives(createBaseVNode("input", {
|
|
307
|
+
id: "csv-delimiter",
|
|
308
|
+
"onUpdate:modelValue": _cache[5] || (_cache[5] = ($event) => nodeCloudStorageReader.value.cloud_storage_settings.csv_delimiter = $event),
|
|
309
|
+
type: "text",
|
|
310
|
+
class: "form-control",
|
|
311
|
+
placeholder: ",",
|
|
312
|
+
maxlength: "1"
|
|
313
|
+
}, null, 512), [
|
|
314
|
+
[vModelText, nodeCloudStorageReader.value.cloud_storage_settings.csv_delimiter]
|
|
315
|
+
])
|
|
316
|
+
]),
|
|
317
|
+
createBaseVNode("div", _hoisted_17, [
|
|
318
|
+
_cache[21] || (_cache[21] = createBaseVNode("label", { for: "csv-encoding" }, "Encoding", -1)),
|
|
319
|
+
withDirectives(createBaseVNode("select", {
|
|
320
|
+
id: "csv-encoding",
|
|
321
|
+
"onUpdate:modelValue": _cache[6] || (_cache[6] = ($event) => nodeCloudStorageReader.value.cloud_storage_settings.csv_encoding = $event),
|
|
322
|
+
class: "form-control"
|
|
323
|
+
}, _cache[20] || (_cache[20] = [
|
|
324
|
+
createBaseVNode("option", { value: "utf8" }, "UTF-8", -1),
|
|
325
|
+
createBaseVNode("option", { value: "utf8-lossy" }, "UTF-8 Lossy", -1)
|
|
326
|
+
]), 512), [
|
|
327
|
+
[vModelSelect, nodeCloudStorageReader.value.cloud_storage_settings.csv_encoding]
|
|
328
|
+
])
|
|
329
|
+
])
|
|
330
|
+
])
|
|
331
|
+
])) : createCommentVNode("", true),
|
|
332
|
+
nodeCloudStorageReader.value.cloud_storage_settings.file_format === "delta" ? (openBlock(), createElementBlock("div", _hoisted_18, [
|
|
333
|
+
_cache[24] || (_cache[24] = createBaseVNode("h5", { class: "subsection-title" }, "Delta Lake Options", -1)),
|
|
334
|
+
createBaseVNode("div", _hoisted_19, [
|
|
335
|
+
_cache[23] || (_cache[23] = createBaseVNode("label", { for: "delta-version" }, "Version (optional)", -1)),
|
|
336
|
+
withDirectives(createBaseVNode("input", {
|
|
337
|
+
id: "delta-version",
|
|
338
|
+
"onUpdate:modelValue": _cache[7] || (_cache[7] = ($event) => nodeCloudStorageReader.value.cloud_storage_settings.delta_version = $event),
|
|
339
|
+
type: "number",
|
|
340
|
+
class: "form-control",
|
|
341
|
+
placeholder: "Latest version",
|
|
342
|
+
min: "0"
|
|
343
|
+
}, null, 512), [
|
|
344
|
+
[
|
|
345
|
+
vModelText,
|
|
346
|
+
nodeCloudStorageReader.value.cloud_storage_settings.delta_version,
|
|
347
|
+
void 0,
|
|
348
|
+
{ number: true }
|
|
349
|
+
]
|
|
350
|
+
])
|
|
351
|
+
])
|
|
352
|
+
])) : createCommentVNode("", true),
|
|
353
|
+
nodeCloudStorageReader.value.cloud_storage_settings.scan_mode === "directory" ? (openBlock(), createElementBlock("div", _hoisted_20, _cache[25] || (_cache[25] = [
|
|
354
|
+
createBaseVNode("i", { class: "fa-solid fa-info-circle" }, null, -1),
|
|
355
|
+
createBaseVNode("div", null, [
|
|
356
|
+
createBaseVNode("p", null, " Directory scan will read all files matching the selected format in the specified path. ")
|
|
357
|
+
], -1)
|
|
358
|
+
]))) : createCommentVNode("", true)
|
|
359
|
+
])
|
|
360
|
+
]),
|
|
361
|
+
_: 1
|
|
362
|
+
}, 8, ["modelValue"])
|
|
363
|
+
])) : (openBlock(), createBlock(unref(CodeLoader), { key: 1 }));
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
});
|
|
367
|
+
const CloudStorageReader_vue_vue_type_style_index_0_scoped_8f1c7f30_lang = "";
|
|
368
|
+
const CloudStorageReader = /* @__PURE__ */ _export_sfc(_sfc_main$1, [["__scopeId", "data-v-8f1c7f30"]]);
|
|
369
|
+
const _hoisted_1 = { ref: "el" };
|
|
370
|
+
const _sfc_main = /* @__PURE__ */ defineComponent({
|
|
371
|
+
__name: "CloudStorageReader",
|
|
372
|
+
props: {
|
|
373
|
+
nodeId: {
|
|
374
|
+
type: Number,
|
|
375
|
+
required: true
|
|
376
|
+
}
|
|
377
|
+
},
|
|
378
|
+
setup(__props) {
|
|
379
|
+
const nodeStore = useNodeStore();
|
|
380
|
+
const childComp = ref(null);
|
|
381
|
+
const props = __props;
|
|
382
|
+
const drawer = ref(false);
|
|
383
|
+
const closeOnDrawer = () => {
|
|
384
|
+
var _a;
|
|
385
|
+
drawer.value = false;
|
|
386
|
+
(_a = childComp.value) == null ? void 0 : _a.pushNodeData();
|
|
387
|
+
};
|
|
388
|
+
const openDrawer = async () => {
|
|
389
|
+
if (nodeStore.node_id === props.nodeId) {
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
nodeStore.closeDrawer();
|
|
393
|
+
drawer.value = true;
|
|
394
|
+
const drawerOpen = nodeStore.isDrawerOpen;
|
|
395
|
+
nodeStore.isDrawerOpen = true;
|
|
396
|
+
await nextTick();
|
|
397
|
+
if (nodeStore.node_id === props.nodeId && drawerOpen) {
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
if (childComp.value) {
|
|
401
|
+
childComp.value.loadNodeData(props.nodeId);
|
|
402
|
+
nodeStore.openDrawer(closeOnDrawer);
|
|
403
|
+
}
|
|
404
|
+
};
|
|
405
|
+
onMounted(async () => {
|
|
406
|
+
await nextTick();
|
|
407
|
+
});
|
|
408
|
+
return (_ctx, _cache) => {
|
|
409
|
+
return openBlock(), createElementBlock("div", _hoisted_1, [
|
|
410
|
+
createVNode(NodeButton, {
|
|
411
|
+
ref: "nodeButton",
|
|
412
|
+
"node-id": __props.nodeId,
|
|
413
|
+
"image-src": "cloud_storage_reader.png",
|
|
414
|
+
title: `${__props.nodeId}: Cloud storage reader`,
|
|
415
|
+
onClick: openDrawer
|
|
416
|
+
}, null, 8, ["node-id", "title"]),
|
|
417
|
+
drawer.value ? (openBlock(), createBlock(Teleport, {
|
|
418
|
+
key: 0,
|
|
419
|
+
to: "#nodesettings"
|
|
420
|
+
}, [
|
|
421
|
+
createVNode(NodeTitle, {
|
|
422
|
+
title: "Read data from cloud provider",
|
|
423
|
+
intro: "Read data from cloud provider"
|
|
424
|
+
}),
|
|
425
|
+
createVNode(CloudStorageReader, {
|
|
426
|
+
ref_key: "childComp",
|
|
427
|
+
ref: childComp,
|
|
428
|
+
"node-id": __props.nodeId
|
|
429
|
+
}, null, 8, ["node-id"])
|
|
430
|
+
])) : createCommentVNode("", true)
|
|
431
|
+
], 512);
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
});
|
|
435
|
+
export {
|
|
436
|
+
_sfc_main as default
|
|
437
|
+
};
|