Flowfile 0.3.5__py3-none-any.whl → 0.3.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of Flowfile might be problematic. Click here for more details.
- flowfile/__init__.py +3 -3
- flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +86 -0
- flowfile/web/static/assets/CloudConnectionManager-d004942f.js +784 -0
- flowfile/web/static/assets/CloudStorageReader-29d14fcc.css +143 -0
- flowfile/web/static/assets/CloudStorageReader-eccf9fc2.js +437 -0
- flowfile/web/static/assets/CloudStorageWriter-49c9a4b2.css +138 -0
- flowfile/web/static/assets/CloudStorageWriter-b1ba6bba.js +430 -0
- flowfile/web/static/assets/{CrossJoin-dfcf7351.js → CrossJoin-68981877.js} +8 -8
- flowfile/web/static/assets/{DatabaseConnectionSettings-b2afb1d7.js → DatabaseConnectionSettings-0b06649c.js} +2 -2
- flowfile/web/static/assets/{DatabaseManager-824a49b2.js → DatabaseManager-8349a426.js} +2 -2
- flowfile/web/static/assets/{DatabaseReader-a48124d8.js → DatabaseReader-905344f8.js} +9 -9
- flowfile/web/static/assets/{DatabaseWriter-b47cbae2.js → DatabaseWriter-9f5b8638.js} +9 -9
- flowfile/web/static/assets/{ExploreData-fdfc45a4.js → ExploreData-131a6d53.js} +5 -5
- flowfile/web/static/assets/{ExternalSource-861b0e71.js → ExternalSource-e3549dcc.js} +6 -6
- flowfile/web/static/assets/{Filter-f87bb897.js → Filter-6e0730ae.js} +8 -8
- flowfile/web/static/assets/{Formula-1e2ed720.js → Formula-02f033e6.js} +75 -9
- flowfile/web/static/assets/{Formula-b8cefc31.css → Formula-29f19d21.css} +10 -0
- flowfile/web/static/assets/{FuzzyMatch-b6cc4fdd.js → FuzzyMatch-54c14036.js} +9 -9
- flowfile/web/static/assets/{GraphSolver-6a371f4c.js → GraphSolver-08a3f499.js} +5 -5
- flowfile/web/static/assets/{GroupBy-f7b7f472.js → GroupBy-2ae38139.js} +6 -6
- flowfile/web/static/assets/{Join-eec38203.js → Join-493b9772.js} +23 -15
- flowfile/web/static/assets/{Join-41c0f331.css → Join-f45eff22.css} +20 -20
- flowfile/web/static/assets/{ManualInput-9aaa46fb.js → ManualInput-4373d163.js} +106 -34
- flowfile/web/static/assets/{ManualInput-ac7b9972.css → ManualInput-a71b52c6.css} +29 -17
- flowfile/web/static/assets/{Output-3b2ca045.js → Output-b534f3c7.js} +4 -4
- flowfile/web/static/assets/{Pivot-a4f5d88f.js → Pivot-2968ff65.js} +6 -6
- flowfile/web/static/assets/{PolarsCode-49ce444f.js → PolarsCode-65136536.js} +6 -6
- flowfile/web/static/assets/{Read-07acdc9a.js → Read-c56339ed.js} +6 -6
- flowfile/web/static/assets/{RecordCount-6a21da56.js → RecordCount-1c641a5e.js} +5 -5
- flowfile/web/static/assets/{RecordId-949bdc17.js → RecordId-df308b8f.js} +6 -6
- flowfile/web/static/assets/{Sample-7afca6e1.js → Sample-293e8a64.js} +5 -5
- flowfile/web/static/assets/{SecretManager-b41c029d.js → SecretManager-03911655.js} +2 -2
- flowfile/web/static/assets/{Select-32b28406.js → Select-3058a13d.js} +8 -8
- flowfile/web/static/assets/{SettingsSection-a0f15a05.js → SettingsSection-fbf4fb39.js} +1 -1
- flowfile/web/static/assets/{Sort-fc6ba0e2.js → Sort-a29bbaf7.js} +6 -6
- flowfile/web/static/assets/{TextToRows-23127596.js → TextToRows-c7d7760e.js} +8 -8
- flowfile/web/static/assets/{UnavailableFields-c42880a3.js → UnavailableFields-118f1d20.js} +2 -2
- flowfile/web/static/assets/{Union-39eecc6c.js → Union-f0589571.js} +5 -5
- flowfile/web/static/assets/{Unique-a0e8fe61.js → Unique-7329a207.js} +8 -8
- flowfile/web/static/assets/{Unpivot-1e2d43f0.js → Unpivot-30b0be15.js} +5 -5
- flowfile/web/static/assets/{api-44ca9e9c.js → api-602fb95c.js} +1 -1
- flowfile/web/static/assets/api-fb67319c.js +80 -0
- flowfile/web/static/assets/cloud_storage_reader-aa1415d6.png +0 -0
- flowfile/web/static/assets/{designer-267d44f1.js → designer-94a6bf4d.js} +36 -34
- flowfile/web/static/assets/{documentation-6c0810a2.js → documentation-a224831e.js} +1 -1
- flowfile/web/static/assets/{dropDown-52790b15.js → dropDown-c2d2aa97.js} +1 -1
- flowfile/web/static/assets/{fullEditor-e272b506.js → fullEditor-921ac5fd.js} +2 -2
- flowfile/web/static/assets/{genericNodeSettings-4bdcf98e.js → genericNodeSettings-7013cc94.js} +3 -3
- flowfile/web/static/assets/{index-e235a8bc.js → index-3a75211d.js} +19 -6
- flowfile/web/static/assets/{nodeTitle-fc3fc4b7.js → nodeTitle-a63d4680.js} +3 -3
- flowfile/web/static/assets/{secretApi-cdc2a3fd.js → secretApi-763aec6e.js} +1 -1
- flowfile/web/static/assets/{selectDynamic-96aa82cd.js → selectDynamic-08464729.js} +3 -3
- flowfile/web/static/assets/{vue-codemirror.esm-25e75a08.js → vue-codemirror.esm-f15a5f87.js} +2 -1
- flowfile/web/static/assets/{vue-content-loader.es-6c4b1c24.js → vue-content-loader.es-93bd09d7.js} +1 -1
- flowfile/web/static/index.html +1 -1
- {flowfile-0.3.5.dist-info → flowfile-0.3.6.dist-info}/METADATA +8 -3
- {flowfile-0.3.5.dist-info → flowfile-0.3.6.dist-info}/RECORD +108 -103
- {flowfile-0.3.5.dist-info → flowfile-0.3.6.dist-info}/entry_points.txt +2 -0
- flowfile_core/__init__.py +2 -0
- flowfile_core/configs/node_store/nodes.py +8 -6
- flowfile_core/database/connection.py +63 -15
- flowfile_core/database/init_db.py +0 -1
- flowfile_core/database/models.py +49 -2
- flowfile_core/flowfile/code_generator/code_generator.py +401 -17
- flowfile_core/flowfile/connection_manager/models.py +1 -1
- flowfile_core/flowfile/database_connection_manager/db_connections.py +216 -2
- flowfile_core/flowfile/extensions.py +1 -1
- flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +259 -0
- flowfile_core/flowfile/flow_data_engine/create/funcs.py +19 -8
- flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +522 -59
- flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +12 -2
- flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py +1 -1
- flowfile_core/flowfile/flow_data_engine/join/__init__.py +2 -1
- flowfile_core/flowfile/flow_data_engine/join/utils.py +25 -0
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +29 -22
- flowfile_core/flowfile/flow_data_engine/utils.py +1 -40
- flowfile_core/flowfile/flow_graph.py +119 -82
- flowfile_core/flowfile/flow_node/flow_node.py +68 -33
- flowfile_core/flowfile/flow_node/models.py +32 -3
- flowfile_core/flowfile/flow_node/schema_callback.py +3 -2
- flowfile_core/flowfile/sources/external_sources/__init__.py +0 -2
- flowfile_core/flowfile/sources/external_sources/factory.py +4 -7
- flowfile_core/flowfile/utils.py +1 -23
- flowfile_core/main.py +3 -2
- flowfile_core/routes/cloud_connections.py +81 -0
- flowfile_core/routes/logs.py +0 -1
- flowfile_core/routes/routes.py +3 -39
- flowfile_core/schemas/cloud_storage_schemas.py +215 -0
- flowfile_core/schemas/input_schema.py +37 -15
- flowfile_core/schemas/schemas.py +7 -2
- flowfile_core/schemas/transform_schema.py +97 -22
- flowfile_core/utils/utils.py +40 -1
- flowfile_core/utils/validate_setup.py +41 -0
- flowfile_frame/flow_frame.py +253 -102
- flowfile_frame/flow_frame_methods.py +13 -13
- flowfile_worker/external_sources/s3_source/main.py +216 -0
- flowfile_worker/external_sources/s3_source/models.py +142 -0
- flowfile_worker/funcs.py +51 -6
- flowfile_worker/models.py +22 -2
- flowfile_worker/routes.py +40 -38
- flowfile_worker/utils.py +1 -1
- test_utils/s3/commands.py +46 -0
- test_utils/s3/data_generator.py +291 -0
- test_utils/s3/fixtures.py +209 -0
- flowfile/web/static/assets/AirbyteReader-1ac35765.css +0 -314
- flowfile/web/static/assets/AirbyteReader-e08044e5.js +0 -922
- flowfile/web/static/assets/dropDownGeneric-60f56a8a.js +0 -72
- flowfile/web/static/assets/dropDownGeneric-895680d6.css +0 -10
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/airbyte.py +0 -159
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py +0 -172
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/settings.py +0 -173
- flowfile_core/schemas/external_sources/airbyte_schemas.py +0 -20
- flowfile_worker/external_sources/airbyte_sources/__init__.py +0 -0
- flowfile_worker/external_sources/airbyte_sources/cache_manager.py +0 -161
- flowfile_worker/external_sources/airbyte_sources/main.py +0 -89
- flowfile_worker/external_sources/airbyte_sources/models.py +0 -133
- flowfile_worker/external_sources/airbyte_sources/settings.py +0 -0
- {flowfile-0.3.5.dist-info → flowfile-0.3.6.dist-info}/LICENSE +0 -0
- {flowfile-0.3.5.dist-info → flowfile-0.3.6.dist-info}/WHEEL +0 -0
- {flowfile_core/flowfile/sources/external_sources/airbyte_sources → flowfile_worker/external_sources/s3_source}/__init__.py +0 -0
- {flowfile_core/schemas/external_sources → test_utils/s3}/__init__.py +0 -0
flowfile/__init__.py
CHANGED
|
@@ -7,7 +7,7 @@ This package ties together the FlowFile ecosystem components:
|
|
|
7
7
|
- flowfile_worker: Computation engine
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
-
__version__ = "0.3.
|
|
10
|
+
__version__ = "0.3.5"
|
|
11
11
|
|
|
12
12
|
import os
|
|
13
13
|
import logging
|
|
@@ -20,7 +20,7 @@ from flowfile.api import (open_graph_in_editor)
|
|
|
20
20
|
from flowfile_frame.flow_frame import (
|
|
21
21
|
FlowFrame
|
|
22
22
|
)
|
|
23
|
-
from flowfile_frame import read_csv, read_parquet, from_dict, concat
|
|
23
|
+
from flowfile_frame import read_csv, read_parquet, from_dict, concat, scan_csv, scan_parquet
|
|
24
24
|
from flowfile_frame.expr import (
|
|
25
25
|
col, lit, column, cum_count, len,
|
|
26
26
|
sum, min, max, mean, count, when
|
|
@@ -50,7 +50,7 @@ __all__ = [
|
|
|
50
50
|
'FlowFrame', 'GroupByFrame',
|
|
51
51
|
|
|
52
52
|
# Main creation functions
|
|
53
|
-
'read_csv', 'read_parquet', 'from_dict', 'concat',
|
|
53
|
+
'read_csv', 'read_parquet', 'from_dict', 'concat', 'scan_csv', 'scan_parquet',
|
|
54
54
|
|
|
55
55
|
# Expression API
|
|
56
56
|
'col', 'lit', 'column', 'cum_count', 'len',
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
|
|
2
|
+
.description-text[data-v-23359c71] {
|
|
3
|
+
color: #6c757d;
|
|
4
|
+
margin-top: 0.5rem;
|
|
5
|
+
font-size: 0.95rem;
|
|
6
|
+
}
|
|
7
|
+
.info-box[data-v-23359c71] {
|
|
8
|
+
display: flex;
|
|
9
|
+
gap: 1rem;
|
|
10
|
+
padding: 1rem;
|
|
11
|
+
background-color: #f8f9fa;
|
|
12
|
+
border-left: 4px solid #17a2b8;
|
|
13
|
+
border-radius: 4px;
|
|
14
|
+
}
|
|
15
|
+
.info-box i[data-v-23359c71] {
|
|
16
|
+
color: #17a2b8;
|
|
17
|
+
font-size: 1.5rem;
|
|
18
|
+
margin-top: 0.5rem;
|
|
19
|
+
}
|
|
20
|
+
.info-box p[data-v-23359c71] {
|
|
21
|
+
margin: 0;
|
|
22
|
+
margin-bottom: 0.5rem;
|
|
23
|
+
}
|
|
24
|
+
.info-box p[data-v-23359c71]:last-child {
|
|
25
|
+
margin-bottom: 0;
|
|
26
|
+
}
|
|
27
|
+
.modal-description[data-v-23359c71] {
|
|
28
|
+
color: #6c757d;
|
|
29
|
+
font-size: 0.9rem;
|
|
30
|
+
}
|
|
31
|
+
.badge[data-v-23359c71] {
|
|
32
|
+
background-color: #e9ecef;
|
|
33
|
+
border-radius: 1rem;
|
|
34
|
+
padding: 0.25rem 0.75rem;
|
|
35
|
+
font-size: 0.75rem;
|
|
36
|
+
margin-left: 0.5rem;
|
|
37
|
+
}
|
|
38
|
+
.auth-badge[data-v-23359c71] {
|
|
39
|
+
background-color: #d1ecf1;
|
|
40
|
+
color: #0c5460;
|
|
41
|
+
}
|
|
42
|
+
.warning-text[data-v-23359c71] {
|
|
43
|
+
color: #dc3545;
|
|
44
|
+
font-size: 0.875rem;
|
|
45
|
+
margin-top: 0.5rem;
|
|
46
|
+
}
|
|
47
|
+
.dialog-footer[data-v-23359c71] {
|
|
48
|
+
display: flex;
|
|
49
|
+
justify-content: flex-end;
|
|
50
|
+
gap: 0.5rem;
|
|
51
|
+
margin-top: 1rem;
|
|
52
|
+
}
|
|
53
|
+
.connection-details[data-v-23359c71] {
|
|
54
|
+
font-size: 0.85rem;
|
|
55
|
+
color: #6c757d;
|
|
56
|
+
margin-top: 0.25rem;
|
|
57
|
+
}
|
|
58
|
+
.separator[data-v-23359c71] {
|
|
59
|
+
margin: 0 0.5rem;
|
|
60
|
+
}
|
|
61
|
+
.hint-text[data-v-23359c71] {
|
|
62
|
+
color: #6c757d;
|
|
63
|
+
font-size: 0.875rem;
|
|
64
|
+
margin-top: 0.5rem;
|
|
65
|
+
}
|
|
66
|
+
.mb-3[data-v-23359c71] {
|
|
67
|
+
margin-bottom: 1rem;
|
|
68
|
+
}
|
|
69
|
+
.flex-col[data-v-23359c71] {
|
|
70
|
+
display: flex;
|
|
71
|
+
flex-direction: column;
|
|
72
|
+
}
|
|
73
|
+
.gap-2[data-v-23359c71] {
|
|
74
|
+
gap: 0.5rem;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/* Cloud-specific styles */
|
|
78
|
+
.fa-brands[data-v-23359c71] {
|
|
79
|
+
font-size: 1.2rem;
|
|
80
|
+
}
|
|
81
|
+
.fa-aws[data-v-23359c71] {
|
|
82
|
+
color: #ff9900;
|
|
83
|
+
}
|
|
84
|
+
.fa-microsoft[data-v-23359c71] {
|
|
85
|
+
color: #0078d4;
|
|
86
|
+
}
|