Flowfile 0.3.4.1__py3-none-any.whl → 0.3.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of Flowfile might be problematic. Click here for more details.
- flowfile/__init__.py +3 -3
- flowfile/api.py +36 -15
- flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +86 -0
- flowfile/web/static/assets/CloudConnectionManager-d004942f.js +784 -0
- flowfile/web/static/assets/CloudStorageReader-29d14fcc.css +143 -0
- flowfile/web/static/assets/CloudStorageReader-eccf9fc2.js +437 -0
- flowfile/web/static/assets/CloudStorageWriter-49c9a4b2.css +138 -0
- flowfile/web/static/assets/CloudStorageWriter-b1ba6bba.js +430 -0
- flowfile/web/static/assets/{CrossJoin-dfcf7351.js → CrossJoin-68981877.js} +8 -8
- flowfile/web/static/assets/{DatabaseConnectionSettings-b2afb1d7.js → DatabaseConnectionSettings-0b06649c.js} +2 -2
- flowfile/web/static/assets/{DatabaseManager-824a49b2.js → DatabaseManager-8349a426.js} +2 -2
- flowfile/web/static/assets/{DatabaseReader-a48124d8.js → DatabaseReader-905344f8.js} +9 -9
- flowfile/web/static/assets/{DatabaseWriter-b47cbae2.js → DatabaseWriter-9f5b8638.js} +9 -9
- flowfile/web/static/assets/{ExploreData-fdfc45a4.js → ExploreData-131a6d53.js} +5 -5
- flowfile/web/static/assets/{ExternalSource-861b0e71.js → ExternalSource-e3549dcc.js} +6 -6
- flowfile/web/static/assets/{Filter-f87bb897.js → Filter-6e0730ae.js} +8 -8
- flowfile/web/static/assets/{Formula-1e2ed720.js → Formula-02f033e6.js} +75 -9
- flowfile/web/static/assets/{Formula-b8cefc31.css → Formula-29f19d21.css} +10 -0
- flowfile/web/static/assets/{FuzzyMatch-b6cc4fdd.js → FuzzyMatch-54c14036.js} +9 -9
- flowfile/web/static/assets/{GraphSolver-6a371f4c.js → GraphSolver-08a3f499.js} +5 -5
- flowfile/web/static/assets/{GroupBy-f7b7f472.js → GroupBy-2ae38139.js} +6 -6
- flowfile/web/static/assets/{Join-eec38203.js → Join-493b9772.js} +23 -15
- flowfile/web/static/assets/{Join-41c0f331.css → Join-f45eff22.css} +20 -20
- flowfile/web/static/assets/{ManualInput-9aaa46fb.js → ManualInput-4373d163.js} +106 -34
- flowfile/web/static/assets/{ManualInput-ac7b9972.css → ManualInput-a71b52c6.css} +29 -17
- flowfile/web/static/assets/{Output-3b2ca045.js → Output-b534f3c7.js} +4 -4
- flowfile/web/static/assets/{Pivot-a4f5d88f.js → Pivot-2968ff65.js} +6 -6
- flowfile/web/static/assets/{PolarsCode-49ce444f.js → PolarsCode-65136536.js} +6 -6
- flowfile/web/static/assets/{Read-07acdc9a.js → Read-c56339ed.js} +6 -6
- flowfile/web/static/assets/{RecordCount-6a21da56.js → RecordCount-1c641a5e.js} +5 -5
- flowfile/web/static/assets/{RecordId-949bdc17.js → RecordId-df308b8f.js} +6 -6
- flowfile/web/static/assets/{Sample-7afca6e1.js → Sample-293e8a64.js} +5 -5
- flowfile/web/static/assets/{SecretManager-b41c029d.js → SecretManager-03911655.js} +2 -2
- flowfile/web/static/assets/{Select-32b28406.js → Select-3058a13d.js} +8 -8
- flowfile/web/static/assets/{SettingsSection-a0f15a05.js → SettingsSection-fbf4fb39.js} +1 -1
- flowfile/web/static/assets/{Sort-fc6ba0e2.js → Sort-a29bbaf7.js} +6 -6
- flowfile/web/static/assets/{TextToRows-23127596.js → TextToRows-c7d7760e.js} +8 -8
- flowfile/web/static/assets/{UnavailableFields-c42880a3.js → UnavailableFields-118f1d20.js} +2 -2
- flowfile/web/static/assets/{Union-39eecc6c.js → Union-f0589571.js} +5 -5
- flowfile/web/static/assets/{Unique-a0e8fe61.js → Unique-7329a207.js} +8 -8
- flowfile/web/static/assets/{Unpivot-1e2d43f0.js → Unpivot-30b0be15.js} +5 -5
- flowfile/web/static/assets/{api-44ca9e9c.js → api-602fb95c.js} +1 -1
- flowfile/web/static/assets/api-fb67319c.js +80 -0
- flowfile/web/static/assets/cloud_storage_reader-aa1415d6.png +0 -0
- flowfile/web/static/assets/{designer-267d44f1.js → designer-94a6bf4d.js} +36 -34
- flowfile/web/static/assets/{documentation-6c0810a2.js → documentation-a224831e.js} +1 -1
- flowfile/web/static/assets/{dropDown-52790b15.js → dropDown-c2d2aa97.js} +1 -1
- flowfile/web/static/assets/{fullEditor-e272b506.js → fullEditor-921ac5fd.js} +2 -2
- flowfile/web/static/assets/{genericNodeSettings-4bdcf98e.js → genericNodeSettings-7013cc94.js} +3 -3
- flowfile/web/static/assets/{index-e235a8bc.js → index-3a75211d.js} +19 -6
- flowfile/web/static/assets/{nodeTitle-fc3fc4b7.js → nodeTitle-a63d4680.js} +3 -3
- flowfile/web/static/assets/{secretApi-cdc2a3fd.js → secretApi-763aec6e.js} +1 -1
- flowfile/web/static/assets/{selectDynamic-96aa82cd.js → selectDynamic-08464729.js} +3 -3
- flowfile/web/static/assets/{vue-codemirror.esm-25e75a08.js → vue-codemirror.esm-f15a5f87.js} +2 -1
- flowfile/web/static/assets/{vue-content-loader.es-6c4b1c24.js → vue-content-loader.es-93bd09d7.js} +1 -1
- flowfile/web/static/index.html +1 -1
- {flowfile-0.3.4.1.dist-info → flowfile-0.3.6.dist-info}/METADATA +8 -3
- {flowfile-0.3.4.1.dist-info → flowfile-0.3.6.dist-info}/RECORD +109 -104
- {flowfile-0.3.4.1.dist-info → flowfile-0.3.6.dist-info}/entry_points.txt +2 -0
- flowfile_core/__init__.py +2 -0
- flowfile_core/configs/node_store/nodes.py +8 -6
- flowfile_core/database/connection.py +63 -15
- flowfile_core/database/init_db.py +0 -1
- flowfile_core/database/models.py +49 -2
- flowfile_core/flowfile/code_generator/code_generator.py +402 -18
- flowfile_core/flowfile/connection_manager/models.py +1 -1
- flowfile_core/flowfile/database_connection_manager/db_connections.py +216 -2
- flowfile_core/flowfile/extensions.py +1 -1
- flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +259 -0
- flowfile_core/flowfile/flow_data_engine/create/funcs.py +19 -8
- flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +522 -59
- flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +12 -2
- flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py +1 -1
- flowfile_core/flowfile/flow_data_engine/join/__init__.py +2 -1
- flowfile_core/flowfile/flow_data_engine/join/utils.py +25 -0
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +29 -22
- flowfile_core/flowfile/flow_data_engine/utils.py +1 -40
- flowfile_core/flowfile/flow_graph.py +119 -82
- flowfile_core/flowfile/flow_node/flow_node.py +68 -33
- flowfile_core/flowfile/flow_node/models.py +32 -3
- flowfile_core/flowfile/flow_node/schema_callback.py +3 -2
- flowfile_core/flowfile/sources/external_sources/__init__.py +0 -2
- flowfile_core/flowfile/sources/external_sources/factory.py +4 -7
- flowfile_core/flowfile/utils.py +1 -23
- flowfile_core/main.py +3 -2
- flowfile_core/routes/cloud_connections.py +81 -0
- flowfile_core/routes/logs.py +0 -1
- flowfile_core/routes/routes.py +3 -39
- flowfile_core/schemas/cloud_storage_schemas.py +215 -0
- flowfile_core/schemas/input_schema.py +37 -15
- flowfile_core/schemas/schemas.py +7 -2
- flowfile_core/schemas/transform_schema.py +97 -22
- flowfile_core/utils/utils.py +40 -1
- flowfile_core/utils/validate_setup.py +41 -0
- flowfile_frame/flow_frame.py +253 -102
- flowfile_frame/flow_frame_methods.py +13 -13
- flowfile_worker/external_sources/s3_source/main.py +216 -0
- flowfile_worker/external_sources/s3_source/models.py +142 -0
- flowfile_worker/funcs.py +51 -6
- flowfile_worker/models.py +22 -2
- flowfile_worker/routes.py +40 -38
- flowfile_worker/utils.py +1 -1
- test_utils/s3/commands.py +46 -0
- test_utils/s3/data_generator.py +291 -0
- test_utils/s3/fixtures.py +209 -0
- flowfile/web/static/assets/AirbyteReader-1ac35765.css +0 -314
- flowfile/web/static/assets/AirbyteReader-e08044e5.js +0 -922
- flowfile/web/static/assets/dropDownGeneric-60f56a8a.js +0 -72
- flowfile/web/static/assets/dropDownGeneric-895680d6.css +0 -10
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/airbyte.py +0 -159
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py +0 -172
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/settings.py +0 -173
- flowfile_core/schemas/external_sources/airbyte_schemas.py +0 -20
- flowfile_worker/external_sources/airbyte_sources/__init__.py +0 -0
- flowfile_worker/external_sources/airbyte_sources/cache_manager.py +0 -161
- flowfile_worker/external_sources/airbyte_sources/main.py +0 -89
- flowfile_worker/external_sources/airbyte_sources/models.py +0 -133
- flowfile_worker/external_sources/airbyte_sources/settings.py +0 -0
- {flowfile-0.3.4.1.dist-info → flowfile-0.3.6.dist-info}/LICENSE +0 -0
- {flowfile-0.3.4.1.dist-info → flowfile-0.3.6.dist-info}/WHEEL +0 -0
- {flowfile_core/flowfile/sources/external_sources/airbyte_sources → flowfile_worker/external_sources/s3_source}/__init__.py +0 -0
- {flowfile_core/schemas/external_sources → test_utils/s3}/__init__.py +0 -0
flowfile/__init__.py
CHANGED
|
@@ -7,7 +7,7 @@ This package ties together the FlowFile ecosystem components:
|
|
|
7
7
|
- flowfile_worker: Computation engine
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
-
__version__ = "0.3.
|
|
10
|
+
__version__ = "0.3.5"
|
|
11
11
|
|
|
12
12
|
import os
|
|
13
13
|
import logging
|
|
@@ -20,7 +20,7 @@ from flowfile.api import (open_graph_in_editor)
|
|
|
20
20
|
from flowfile_frame.flow_frame import (
|
|
21
21
|
FlowFrame
|
|
22
22
|
)
|
|
23
|
-
from flowfile_frame import read_csv, read_parquet, from_dict, concat
|
|
23
|
+
from flowfile_frame import read_csv, read_parquet, from_dict, concat, scan_csv, scan_parquet
|
|
24
24
|
from flowfile_frame.expr import (
|
|
25
25
|
col, lit, column, cum_count, len,
|
|
26
26
|
sum, min, max, mean, count, when
|
|
@@ -50,7 +50,7 @@ __all__ = [
|
|
|
50
50
|
'FlowFrame', 'GroupByFrame',
|
|
51
51
|
|
|
52
52
|
# Main creation functions
|
|
53
|
-
'read_csv', 'read_parquet', 'from_dict', 'concat',
|
|
53
|
+
'read_csv', 'read_parquet', 'from_dict', 'concat', 'scan_csv', 'scan_parquet',
|
|
54
54
|
|
|
55
55
|
# Expression API
|
|
56
56
|
'col', 'lit', 'column', 'cum_count', 'len',
|
flowfile/api.py
CHANGED
|
@@ -147,9 +147,6 @@ def build_server_command(module_name: str) -> List[str]:
|
|
|
147
147
|
Build the appropriate command to start the server based on environment detection.
|
|
148
148
|
Tries Poetry first if in a Poetry environment, falls back to direct module execution.
|
|
149
149
|
"""
|
|
150
|
-
command: List[str] = []
|
|
151
|
-
|
|
152
|
-
# Case 1: Check if we're in a Poetry environment
|
|
153
150
|
if is_poetry_environment():
|
|
154
151
|
logger.info("Poetry environment detected.")
|
|
155
152
|
if is_command_available(POETRY_PATH):
|
|
@@ -166,17 +163,42 @@ def build_server_command(module_name: str) -> List[str]:
|
|
|
166
163
|
else:
|
|
167
164
|
logger.warning(f"Poetry command not found at '{POETRY_PATH}'. Falling back to Python module.")
|
|
168
165
|
|
|
169
|
-
# Case 2:
|
|
170
|
-
logger.info(
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
"
|
|
177
|
-
"
|
|
178
|
-
|
|
166
|
+
# Case 2: Fallback to direct script execution
|
|
167
|
+
logger.info("Falling back to direct script execution.")
|
|
168
|
+
python_parent_dir = Path(sys.executable).parent
|
|
169
|
+
command: List[str]
|
|
170
|
+
scripts_dir = Path(sys.executable).parent
|
|
171
|
+
|
|
172
|
+
if platform.system() == "Windows":
|
|
173
|
+
exe_path = scripts_dir / f"{module_name}.exe"
|
|
174
|
+
script_py_path = scripts_dir / f"{module_name}-script.py"
|
|
175
|
+
plain_script_path = scripts_dir / module_name
|
|
176
|
+
|
|
177
|
+
if exe_path.exists():
|
|
178
|
+
logger.info(f"Using .exe wrapper: {exe_path}")
|
|
179
|
+
command = [str(exe_path), "run", "ui", "--no-browser"]
|
|
180
|
+
elif script_py_path.exists():
|
|
181
|
+
logger.info(f"Using '-script.py' with interpreter: {script_py_path}")
|
|
182
|
+
command = [sys.executable, str(script_py_path), "run", "ui", "--no-browser"]
|
|
183
|
+
elif plain_script_path.exists():
|
|
184
|
+
logger.info(f"Using plain script with interpreter: {plain_script_path}")
|
|
185
|
+
command = [sys.executable, str(plain_script_path), "run", "ui", "--no-browser"]
|
|
186
|
+
else:
|
|
187
|
+
raise FileNotFoundError(
|
|
188
|
+
f"Could not find an executable script for '{module_name}' in '{scripts_dir}'. "
|
|
189
|
+
f"Checked for '{exe_path.name}', '{script_py_path.name}', and '{plain_script_path.name}'. "
|
|
190
|
+
"Ensure the package is installed correctly."
|
|
191
|
+
)
|
|
192
|
+
else:
|
|
193
|
+
# On Unix-like systems, the script in 'bin' is directly executable
|
|
194
|
+
script_path = python_parent_dir / "bin" / module_name
|
|
195
|
+
if not script_path.exists():
|
|
196
|
+
script_path = python_parent_dir / module_name # Fallback for different venv structures
|
|
197
|
+
|
|
198
|
+
logger.info(f"Using direct script execution path: {script_path}")
|
|
199
|
+
command = [str(script_path), "run", "ui", "--no-browser"]
|
|
179
200
|
|
|
201
|
+
logger.info(f"Built server command: {command}")
|
|
180
202
|
return command
|
|
181
203
|
|
|
182
204
|
|
|
@@ -210,7 +232,6 @@ def start_flowfile_server_process(module_name: str = DEFAULT_MODULE_NAME) -> Tup
|
|
|
210
232
|
# Build command automatically based on environment detection
|
|
211
233
|
command = build_server_command(module_name)
|
|
212
234
|
logger.info(f"Starting server with command: {' '.join(command)}")
|
|
213
|
-
|
|
214
235
|
try:
|
|
215
236
|
# Windows-specific subprocess creation
|
|
216
237
|
if platform.system() == "Windows":
|
|
@@ -244,7 +265,7 @@ def start_flowfile_server_process(module_name: str = DEFAULT_MODULE_NAME) -> Tup
|
|
|
244
265
|
time.sleep(1)
|
|
245
266
|
else:
|
|
246
267
|
logger.error("Failed to start server: API did not become responsive within 60 seconds. "
|
|
247
|
-
"Try again or
|
|
268
|
+
"Try again or start service by running\n"
|
|
248
269
|
"flowfile run ui")
|
|
249
270
|
if _server_process and _server_process.stderr:
|
|
250
271
|
try:
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
|
|
2
|
+
.description-text[data-v-23359c71] {
|
|
3
|
+
color: #6c757d;
|
|
4
|
+
margin-top: 0.5rem;
|
|
5
|
+
font-size: 0.95rem;
|
|
6
|
+
}
|
|
7
|
+
.info-box[data-v-23359c71] {
|
|
8
|
+
display: flex;
|
|
9
|
+
gap: 1rem;
|
|
10
|
+
padding: 1rem;
|
|
11
|
+
background-color: #f8f9fa;
|
|
12
|
+
border-left: 4px solid #17a2b8;
|
|
13
|
+
border-radius: 4px;
|
|
14
|
+
}
|
|
15
|
+
.info-box i[data-v-23359c71] {
|
|
16
|
+
color: #17a2b8;
|
|
17
|
+
font-size: 1.5rem;
|
|
18
|
+
margin-top: 0.5rem;
|
|
19
|
+
}
|
|
20
|
+
.info-box p[data-v-23359c71] {
|
|
21
|
+
margin: 0;
|
|
22
|
+
margin-bottom: 0.5rem;
|
|
23
|
+
}
|
|
24
|
+
.info-box p[data-v-23359c71]:last-child {
|
|
25
|
+
margin-bottom: 0;
|
|
26
|
+
}
|
|
27
|
+
.modal-description[data-v-23359c71] {
|
|
28
|
+
color: #6c757d;
|
|
29
|
+
font-size: 0.9rem;
|
|
30
|
+
}
|
|
31
|
+
.badge[data-v-23359c71] {
|
|
32
|
+
background-color: #e9ecef;
|
|
33
|
+
border-radius: 1rem;
|
|
34
|
+
padding: 0.25rem 0.75rem;
|
|
35
|
+
font-size: 0.75rem;
|
|
36
|
+
margin-left: 0.5rem;
|
|
37
|
+
}
|
|
38
|
+
.auth-badge[data-v-23359c71] {
|
|
39
|
+
background-color: #d1ecf1;
|
|
40
|
+
color: #0c5460;
|
|
41
|
+
}
|
|
42
|
+
.warning-text[data-v-23359c71] {
|
|
43
|
+
color: #dc3545;
|
|
44
|
+
font-size: 0.875rem;
|
|
45
|
+
margin-top: 0.5rem;
|
|
46
|
+
}
|
|
47
|
+
.dialog-footer[data-v-23359c71] {
|
|
48
|
+
display: flex;
|
|
49
|
+
justify-content: flex-end;
|
|
50
|
+
gap: 0.5rem;
|
|
51
|
+
margin-top: 1rem;
|
|
52
|
+
}
|
|
53
|
+
.connection-details[data-v-23359c71] {
|
|
54
|
+
font-size: 0.85rem;
|
|
55
|
+
color: #6c757d;
|
|
56
|
+
margin-top: 0.25rem;
|
|
57
|
+
}
|
|
58
|
+
.separator[data-v-23359c71] {
|
|
59
|
+
margin: 0 0.5rem;
|
|
60
|
+
}
|
|
61
|
+
.hint-text[data-v-23359c71] {
|
|
62
|
+
color: #6c757d;
|
|
63
|
+
font-size: 0.875rem;
|
|
64
|
+
margin-top: 0.5rem;
|
|
65
|
+
}
|
|
66
|
+
.mb-3[data-v-23359c71] {
|
|
67
|
+
margin-bottom: 1rem;
|
|
68
|
+
}
|
|
69
|
+
.flex-col[data-v-23359c71] {
|
|
70
|
+
display: flex;
|
|
71
|
+
flex-direction: column;
|
|
72
|
+
}
|
|
73
|
+
.gap-2[data-v-23359c71] {
|
|
74
|
+
gap: 0.5rem;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/* Cloud-specific styles */
|
|
78
|
+
.fa-brands[data-v-23359c71] {
|
|
79
|
+
font-size: 1.2rem;
|
|
80
|
+
}
|
|
81
|
+
.fa-aws[data-v-23359c71] {
|
|
82
|
+
color: #ff9900;
|
|
83
|
+
}
|
|
84
|
+
.fa-microsoft[data-v-23359c71] {
|
|
85
|
+
color: #0078d4;
|
|
86
|
+
}
|