Flowfile 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of Flowfile might be problematic. Click here for more details.
- flowfile/__init__.py +27 -6
- flowfile/api.py +1 -0
- flowfile/web/__init__.py +2 -2
- flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +86 -0
- flowfile/web/static/assets/CloudConnectionManager-c20a740f.js +783 -0
- flowfile/web/static/assets/CloudStorageReader-29d14fcc.css +143 -0
- flowfile/web/static/assets/CloudStorageReader-960b400a.js +437 -0
- flowfile/web/static/assets/CloudStorageWriter-49c9a4b2.css +138 -0
- flowfile/web/static/assets/CloudStorageWriter-e3decbdd.js +430 -0
- flowfile/web/static/assets/{CrossJoin-dfcf7351.js → CrossJoin-d67e2405.js} +8 -8
- flowfile/web/static/assets/{DatabaseConnectionSettings-b2afb1d7.js → DatabaseConnectionSettings-a81e0f7e.js} +2 -2
- flowfile/web/static/assets/{DatabaseManager-824a49b2.js → DatabaseManager-9ea35e84.js} +2 -2
- flowfile/web/static/assets/{DatabaseReader-a48124d8.js → DatabaseReader-9578bfa5.js} +9 -9
- flowfile/web/static/assets/{DatabaseWriter-b47cbae2.js → DatabaseWriter-19531098.js} +9 -9
- flowfile/web/static/assets/{ExploreData-fdfc45a4.js → ExploreData-40476474.js} +47141 -43697
- flowfile/web/static/assets/{ExternalSource-861b0e71.js → ExternalSource-2297ef96.js} +6 -6
- flowfile/web/static/assets/{Filter-f87bb897.js → Filter-f211c03a.js} +8 -8
- flowfile/web/static/assets/{Formula-b8cefc31.css → Formula-29f19d21.css} +10 -0
- flowfile/web/static/assets/{Formula-1e2ed720.js → Formula-4207ea31.js} +75 -9
- flowfile/web/static/assets/{FuzzyMatch-b6cc4fdd.js → FuzzyMatch-bf120df0.js} +9 -9
- flowfile/web/static/assets/{GraphSolver-6a371f4c.js → GraphSolver-5bb7497a.js} +5 -5
- flowfile/web/static/assets/{GroupBy-f7b7f472.js → GroupBy-92c81b65.js} +6 -6
- flowfile/web/static/assets/{Join-eec38203.js → Join-4e49a274.js} +23 -15
- flowfile/web/static/assets/{Join-41c0f331.css → Join-f45eff22.css} +20 -20
- flowfile/web/static/assets/{ManualInput-9aaa46fb.js → ManualInput-90998ae8.js} +106 -34
- flowfile/web/static/assets/{ManualInput-ac7b9972.css → ManualInput-a71b52c6.css} +29 -17
- flowfile/web/static/assets/{Output-3b2ca045.js → Output-81e3e917.js} +4 -4
- flowfile/web/static/assets/{Pivot-a4f5d88f.js → Pivot-a3419842.js} +6 -6
- flowfile/web/static/assets/{PolarsCode-49ce444f.js → PolarsCode-72710deb.js} +6 -6
- flowfile/web/static/assets/{Read-07acdc9a.js → Read-c4059daf.js} +6 -6
- flowfile/web/static/assets/{RecordCount-6a21da56.js → RecordCount-c2b5e095.js} +5 -5
- flowfile/web/static/assets/{RecordId-949bdc17.js → RecordId-10baf191.js} +6 -6
- flowfile/web/static/assets/{Sample-7afca6e1.js → Sample-3ed9a0ae.js} +5 -5
- flowfile/web/static/assets/{SecretManager-b41c029d.js → SecretManager-0d49c0e8.js} +2 -2
- flowfile/web/static/assets/{Select-32b28406.js → Select-8a02a0b3.js} +8 -8
- flowfile/web/static/assets/{SettingsSection-a0f15a05.js → SettingsSection-4c0f45f5.js} +1 -1
- flowfile/web/static/assets/{Sort-fc6ba0e2.js → Sort-f55c9f9d.js} +6 -6
- flowfile/web/static/assets/{TextToRows-23127596.js → TextToRows-5dbc2145.js} +8 -8
- flowfile/web/static/assets/{UnavailableFields-c42880a3.js → UnavailableFields-a1768e52.js} +2 -2
- flowfile/web/static/assets/{Union-39eecc6c.js → Union-f2aefdc9.js} +5 -5
- flowfile/web/static/assets/{Unique-a0e8fe61.js → Unique-46b250da.js} +8 -8
- flowfile/web/static/assets/{Unpivot-1e2d43f0.js → Unpivot-25ac84cc.js} +5 -5
- flowfile/web/static/assets/api-6ef0dcef.js +80 -0
- flowfile/web/static/assets/{api-44ca9e9c.js → api-a0abbdc7.js} +1 -1
- flowfile/web/static/assets/cloud_storage_reader-aa1415d6.png +0 -0
- flowfile/web/static/assets/{designer-267d44f1.js → designer-13eabd83.js} +36 -34
- flowfile/web/static/assets/{documentation-6c0810a2.js → documentation-b87e7f6f.js} +1 -1
- flowfile/web/static/assets/{dropDown-52790b15.js → dropDown-13564764.js} +1 -1
- flowfile/web/static/assets/{fullEditor-e272b506.js → fullEditor-fd2cd6f9.js} +2 -2
- flowfile/web/static/assets/{genericNodeSettings-4bdcf98e.js → genericNodeSettings-71e11604.js} +3 -3
- flowfile/web/static/assets/{index-e235a8bc.js → index-f6c15e76.js} +59 -22
- flowfile/web/static/assets/{nodeTitle-fc3fc4b7.js → nodeTitle-988d9efe.js} +3 -3
- flowfile/web/static/assets/{secretApi-cdc2a3fd.js → secretApi-dd636aa2.js} +1 -1
- flowfile/web/static/assets/{selectDynamic-96aa82cd.js → selectDynamic-af36165e.js} +3 -3
- flowfile/web/static/assets/{vue-codemirror.esm-25e75a08.js → vue-codemirror.esm-2847001e.js} +2 -1
- flowfile/web/static/assets/{vue-content-loader.es-6c4b1c24.js → vue-content-loader.es-0371da73.js} +1 -1
- flowfile/web/static/index.html +1 -1
- {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/METADATA +9 -4
- {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/RECORD +131 -124
- {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/entry_points.txt +2 -0
- flowfile_core/__init__.py +3 -0
- flowfile_core/auth/jwt.py +39 -0
- flowfile_core/configs/node_store/nodes.py +9 -6
- flowfile_core/configs/settings.py +6 -5
- flowfile_core/database/connection.py +63 -15
- flowfile_core/database/init_db.py +0 -1
- flowfile_core/database/models.py +49 -2
- flowfile_core/flowfile/code_generator/code_generator.py +472 -17
- flowfile_core/flowfile/connection_manager/models.py +1 -1
- flowfile_core/flowfile/database_connection_manager/db_connections.py +216 -2
- flowfile_core/flowfile/extensions.py +1 -1
- flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +259 -0
- flowfile_core/flowfile/flow_data_engine/create/funcs.py +19 -8
- flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +1062 -311
- flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +12 -2
- flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py +1 -1
- flowfile_core/flowfile/flow_data_engine/join/__init__.py +2 -1
- flowfile_core/flowfile/flow_data_engine/join/utils.py +25 -0
- flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +3 -1
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +29 -22
- flowfile_core/flowfile/flow_data_engine/utils.py +1 -40
- flowfile_core/flowfile/flow_graph.py +718 -253
- flowfile_core/flowfile/flow_graph_utils.py +2 -2
- flowfile_core/flowfile/flow_node/flow_node.py +563 -117
- flowfile_core/flowfile/flow_node/models.py +154 -20
- flowfile_core/flowfile/flow_node/schema_callback.py +3 -2
- flowfile_core/flowfile/handler.py +2 -33
- flowfile_core/flowfile/manage/open_flowfile.py +1 -2
- flowfile_core/flowfile/sources/external_sources/__init__.py +0 -2
- flowfile_core/flowfile/sources/external_sources/factory.py +4 -7
- flowfile_core/flowfile/util/calculate_layout.py +0 -2
- flowfile_core/flowfile/utils.py +35 -26
- flowfile_core/main.py +35 -15
- flowfile_core/routes/cloud_connections.py +77 -0
- flowfile_core/routes/logs.py +2 -7
- flowfile_core/routes/public.py +1 -0
- flowfile_core/routes/routes.py +130 -90
- flowfile_core/routes/secrets.py +72 -14
- flowfile_core/schemas/__init__.py +8 -0
- flowfile_core/schemas/cloud_storage_schemas.py +215 -0
- flowfile_core/schemas/input_schema.py +121 -71
- flowfile_core/schemas/output_model.py +19 -3
- flowfile_core/schemas/schemas.py +150 -12
- flowfile_core/schemas/transform_schema.py +175 -35
- flowfile_core/utils/utils.py +40 -1
- flowfile_core/utils/validate_setup.py +41 -0
- flowfile_frame/__init__.py +9 -1
- flowfile_frame/cloud_storage/frame_helpers.py +39 -0
- flowfile_frame/cloud_storage/secret_manager.py +73 -0
- flowfile_frame/expr.py +28 -1
- flowfile_frame/expr.pyi +76 -61
- flowfile_frame/flow_frame.py +481 -208
- flowfile_frame/flow_frame.pyi +140 -91
- flowfile_frame/flow_frame_methods.py +160 -22
- flowfile_frame/group_frame.py +3 -0
- flowfile_frame/utils.py +25 -3
- flowfile_worker/external_sources/s3_source/main.py +216 -0
- flowfile_worker/external_sources/s3_source/models.py +142 -0
- flowfile_worker/funcs.py +51 -6
- flowfile_worker/models.py +22 -2
- flowfile_worker/routes.py +40 -38
- flowfile_worker/utils.py +1 -1
- test_utils/s3/commands.py +46 -0
- test_utils/s3/data_generator.py +292 -0
- test_utils/s3/demo_data_generator.py +186 -0
- test_utils/s3/fixtures.py +214 -0
- flowfile/web/static/assets/AirbyteReader-1ac35765.css +0 -314
- flowfile/web/static/assets/AirbyteReader-e08044e5.js +0 -922
- flowfile/web/static/assets/dropDownGeneric-60f56a8a.js +0 -72
- flowfile/web/static/assets/dropDownGeneric-895680d6.css +0 -10
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/airbyte.py +0 -159
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py +0 -172
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/settings.py +0 -173
- flowfile_core/schemas/defaults.py +0 -9
- flowfile_core/schemas/external_sources/airbyte_schemas.py +0 -20
- flowfile_core/schemas/models.py +0 -193
- flowfile_worker/external_sources/airbyte_sources/cache_manager.py +0 -161
- flowfile_worker/external_sources/airbyte_sources/main.py +0 -89
- flowfile_worker/external_sources/airbyte_sources/models.py +0 -133
- flowfile_worker/external_sources/airbyte_sources/settings.py +0 -0
- {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/LICENSE +0 -0
- {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/WHEEL +0 -0
- {flowfile_core/flowfile/sources/external_sources/airbyte_sources → flowfile_frame/cloud_storage}/__init__.py +0 -0
- {flowfile_core/schemas/external_sources → flowfile_worker/external_sources/s3_source}/__init__.py +0 -0
- {flowfile_worker/external_sources/airbyte_sources → test_utils/s3}/__init__.py +0 -0
flowfile/__init__.py
CHANGED
|
@@ -7,20 +7,36 @@ This package ties together the FlowFile ecosystem components:
|
|
|
7
7
|
- flowfile_worker: Computation engine
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
-
__version__ = "0.3.
|
|
10
|
+
__version__ = "0.3.7"
|
|
11
11
|
|
|
12
12
|
import os
|
|
13
13
|
import logging
|
|
14
14
|
|
|
15
|
-
os.environ['
|
|
16
|
-
os.environ['
|
|
15
|
+
os.environ['FLOWFILE_WORKER_PORT'] = "63578"
|
|
16
|
+
os.environ['FLOWFILE_SINGLE_FILE_MODE'] = "1"
|
|
17
17
|
|
|
18
18
|
from flowfile.web import start_server as start_web_ui
|
|
19
19
|
from flowfile.api import (open_graph_in_editor)
|
|
20
20
|
from flowfile_frame.flow_frame import (
|
|
21
21
|
FlowFrame
|
|
22
22
|
)
|
|
23
|
-
from
|
|
23
|
+
from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection
|
|
24
|
+
from flowfile_core.flowfile.flow_graph import FlowGraph
|
|
25
|
+
from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
|
|
26
|
+
from flowfile_core.flowfile.flow_node.flow_node import FlowNode
|
|
27
|
+
from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
|
|
28
|
+
from flowfile_core.schemas import input_schema as node_interface
|
|
29
|
+
from flowfile_core.schemas.schemas import FlowSettings, FlowInformation
|
|
30
|
+
from flowfile_core.schemas import transform_schema
|
|
31
|
+
from flowfile_frame import (read_csv, read_parquet, from_dict, concat, scan_csv, scan_parquet,
|
|
32
|
+
scan_delta,
|
|
33
|
+
scan_parquet_from_cloud_storage,
|
|
34
|
+
scan_json_from_cloud_storage,
|
|
35
|
+
scan_csv_from_cloud_storage,
|
|
36
|
+
get_all_available_cloud_storage_connections,
|
|
37
|
+
create_cloud_storage_connection,
|
|
38
|
+
del_cloud_storage_connection,
|
|
39
|
+
create_cloud_storage_connection_if_not_exists)
|
|
24
40
|
from flowfile_frame.expr import (
|
|
25
41
|
col, lit, column, cum_count, len,
|
|
26
42
|
sum, min, max, mean, count, when
|
|
@@ -47,10 +63,15 @@ from polars.datatypes import (
|
|
|
47
63
|
|
|
48
64
|
__all__ = [
|
|
49
65
|
# Core FlowFrame classes
|
|
50
|
-
'FlowFrame', 'GroupByFrame',
|
|
66
|
+
'FlowFrame', 'GroupByFrame', 'FullCloudStorageConnection',
|
|
51
67
|
|
|
52
68
|
# Main creation functions
|
|
53
|
-
'read_csv', 'read_parquet', 'from_dict', 'concat',
|
|
69
|
+
'read_csv', 'read_parquet', 'from_dict', 'concat', 'scan_csv', 'scan_parquet', 'scan_delta',
|
|
70
|
+
'scan_parquet_from_cloud_storage', 'scan_json_from_cloud_storage',
|
|
71
|
+
'scan_csv_from_cloud_storage', 'get_all_available_cloud_storage_connections', 'create_cloud_storage_connection',
|
|
72
|
+
'del_cloud_storage_connection', 'create_cloud_storage_connection_if_not_exists',
|
|
73
|
+
'FlowGraph', 'FlowDataEngine', 'node_interface', 'FlowSettings', 'transform_schema',
|
|
74
|
+
'FlowNode', 'FlowfileColumn', 'FlowInformation',
|
|
54
75
|
|
|
55
76
|
# Expression API
|
|
56
77
|
'col', 'lit', 'column', 'cum_count', 'len',
|
flowfile/api.py
CHANGED
|
@@ -456,6 +456,7 @@ def open_graph_in_editor(flow_graph: FlowGraph, storage_location: Optional[str]
|
|
|
456
456
|
if flow_id is not None:
|
|
457
457
|
if flow_in_single_mode and automatically_open_browser:
|
|
458
458
|
_open_flow_in_browser(flow_id)
|
|
459
|
+
logger.info(f"Ui available at: http://127.0.0.1:63578/ui#/main/designer")
|
|
459
460
|
return True
|
|
460
461
|
else:
|
|
461
462
|
return False
|
flowfile/web/__init__.py
CHANGED
|
@@ -50,8 +50,8 @@ def extend_app(app: FastAPI):
|
|
|
50
50
|
return Response(status_code=404)
|
|
51
51
|
|
|
52
52
|
@app.get("/single_mode")
|
|
53
|
-
async def in_single_mode():
|
|
54
|
-
return os.environ.get('
|
|
53
|
+
async def in_single_mode() -> bool:
|
|
54
|
+
return os.environ.get('FLOWFILE_SINGLE_FILE_MODE', "0") == "1"
|
|
55
55
|
|
|
56
56
|
@app.get("/ui", include_in_schema=False)
|
|
57
57
|
async def web_ui_root():
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
|
|
2
|
+
.description-text[data-v-23359c71] {
|
|
3
|
+
color: #6c757d;
|
|
4
|
+
margin-top: 0.5rem;
|
|
5
|
+
font-size: 0.95rem;
|
|
6
|
+
}
|
|
7
|
+
.info-box[data-v-23359c71] {
|
|
8
|
+
display: flex;
|
|
9
|
+
gap: 1rem;
|
|
10
|
+
padding: 1rem;
|
|
11
|
+
background-color: #f8f9fa;
|
|
12
|
+
border-left: 4px solid #17a2b8;
|
|
13
|
+
border-radius: 4px;
|
|
14
|
+
}
|
|
15
|
+
.info-box i[data-v-23359c71] {
|
|
16
|
+
color: #17a2b8;
|
|
17
|
+
font-size: 1.5rem;
|
|
18
|
+
margin-top: 0.5rem;
|
|
19
|
+
}
|
|
20
|
+
.info-box p[data-v-23359c71] {
|
|
21
|
+
margin: 0;
|
|
22
|
+
margin-bottom: 0.5rem;
|
|
23
|
+
}
|
|
24
|
+
.info-box p[data-v-23359c71]:last-child {
|
|
25
|
+
margin-bottom: 0;
|
|
26
|
+
}
|
|
27
|
+
.modal-description[data-v-23359c71] {
|
|
28
|
+
color: #6c757d;
|
|
29
|
+
font-size: 0.9rem;
|
|
30
|
+
}
|
|
31
|
+
.badge[data-v-23359c71] {
|
|
32
|
+
background-color: #e9ecef;
|
|
33
|
+
border-radius: 1rem;
|
|
34
|
+
padding: 0.25rem 0.75rem;
|
|
35
|
+
font-size: 0.75rem;
|
|
36
|
+
margin-left: 0.5rem;
|
|
37
|
+
}
|
|
38
|
+
.auth-badge[data-v-23359c71] {
|
|
39
|
+
background-color: #d1ecf1;
|
|
40
|
+
color: #0c5460;
|
|
41
|
+
}
|
|
42
|
+
.warning-text[data-v-23359c71] {
|
|
43
|
+
color: #dc3545;
|
|
44
|
+
font-size: 0.875rem;
|
|
45
|
+
margin-top: 0.5rem;
|
|
46
|
+
}
|
|
47
|
+
.dialog-footer[data-v-23359c71] {
|
|
48
|
+
display: flex;
|
|
49
|
+
justify-content: flex-end;
|
|
50
|
+
gap: 0.5rem;
|
|
51
|
+
margin-top: 1rem;
|
|
52
|
+
}
|
|
53
|
+
.connection-details[data-v-23359c71] {
|
|
54
|
+
font-size: 0.85rem;
|
|
55
|
+
color: #6c757d;
|
|
56
|
+
margin-top: 0.25rem;
|
|
57
|
+
}
|
|
58
|
+
.separator[data-v-23359c71] {
|
|
59
|
+
margin: 0 0.5rem;
|
|
60
|
+
}
|
|
61
|
+
.hint-text[data-v-23359c71] {
|
|
62
|
+
color: #6c757d;
|
|
63
|
+
font-size: 0.875rem;
|
|
64
|
+
margin-top: 0.5rem;
|
|
65
|
+
}
|
|
66
|
+
.mb-3[data-v-23359c71] {
|
|
67
|
+
margin-bottom: 1rem;
|
|
68
|
+
}
|
|
69
|
+
.flex-col[data-v-23359c71] {
|
|
70
|
+
display: flex;
|
|
71
|
+
flex-direction: column;
|
|
72
|
+
}
|
|
73
|
+
.gap-2[data-v-23359c71] {
|
|
74
|
+
gap: 0.5rem;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/* Cloud-specific styles */
|
|
78
|
+
.fa-brands[data-v-23359c71] {
|
|
79
|
+
font-size: 1.2rem;
|
|
80
|
+
}
|
|
81
|
+
.fa-aws[data-v-23359c71] {
|
|
82
|
+
color: #ff9900;
|
|
83
|
+
}
|
|
84
|
+
.fa-microsoft[data-v-23359c71] {
|
|
85
|
+
color: #0078d4;
|
|
86
|
+
}
|