Flowfile 0.3.1.2__py3-none-any.whl → 0.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of Flowfile might be problematic. Click here for more details.
- flowfile/__init__.py +2 -1
- flowfile/api.py +5 -3
- flowfile/web/__init__.py +3 -0
- flowfile/web/static/assets/{AirbyteReader-cb0c1d4a.js → AirbyteReader-2b1cf2d8.js} +10 -9
- flowfile/web/static/assets/{CrossJoin-a514fa59.js → CrossJoin-cc3ab73c.js} +8 -8
- flowfile/web/static/assets/{DatabaseConnectionSettings-f2cecf33.js → DatabaseConnectionSettings-307c4652.js} +2 -2
- flowfile/web/static/assets/{DatabaseManager-83ee3c98.js → DatabaseManager-69faa6e1.js} +10 -6
- flowfile/web/static/assets/{DatabaseReader-dc0c6881.js → DatabaseReader-e4134cd0.js} +9 -9
- flowfile/web/static/assets/{DatabaseWriter-5afe9f8d.js → DatabaseWriter-d32d75b1.js} +9 -9
- flowfile/web/static/assets/{ExploreData-c7ee19cf.js → ExploreData-5eb48389.js} +18639 -18629
- flowfile/web/static/assets/{ExternalSource-17b23a01.js → ExternalSource-29489051.js} +8 -21
- flowfile/web/static/assets/{Filter-90856b4f.js → Filter-031332bb.js} +9 -9
- flowfile/web/static/assets/{Formula-38b71e9e.js → Formula-3b900540.js} +15 -15
- flowfile/web/static/assets/{Formula-d60a74f4.css → Formula-b8cefc31.css} +4 -4
- flowfile/web/static/assets/{FuzzyMatch-d0f1fe81.js → FuzzyMatch-dee31153.js} +9 -9
- flowfile/web/static/assets/{GraphSolver-0c86bbc6.js → GraphSolver-ca74eb47.js} +5 -5
- flowfile/web/static/assets/{GroupBy-f2772e9f.js → GroupBy-081b6591.js} +8 -7
- flowfile/web/static/assets/{Join-bc3e1cf7.js → Join-b467376f.js} +11 -10
- flowfile/web/static/assets/{ManualInput-03aa0245.js → ManualInput-ffffb80a.js} +11 -8
- flowfile/web/static/assets/{Output-5b35eee8.js → Output-9a87d4ba.js} +4 -4
- flowfile/web/static/assets/{Pivot-7164087c.js → Pivot-ee3e6093.js} +8 -7
- flowfile/web/static/assets/{PolarsCode-3abf6507.js → PolarsCode-03921254.js} +13 -11
- flowfile/web/static/assets/{PopOver-b37ff9be.js → PopOver-3bdf8951.js} +1 -1
- flowfile/web/static/assets/{Read-65966a3e.js → Read-67fee3a0.js} +6 -6
- flowfile/web/static/assets/{RecordCount-c66c6d6d.js → RecordCount-a2acd02d.js} +7 -6
- flowfile/web/static/assets/{RecordId-826dc095.js → RecordId-0c8bcd77.js} +10 -8
- flowfile/web/static/assets/{Sample-4ed555c8.js → Sample-60594a3a.js} +7 -6
- flowfile/web/static/assets/{SecretManager-eac1e97d.js → SecretManager-bbcec2ac.js} +2 -2
- flowfile/web/static/assets/{Select-085f05cc.js → Select-9540e6ca.js} +8 -8
- flowfile/web/static/assets/{SettingsSection-1f5e79c1.js → SettingsSection-48f28104.js} +1 -1
- flowfile/web/static/assets/{Sort-3e6cb414.js → Sort-6dbe3633.js} +6 -6
- flowfile/web/static/assets/{TextToRows-606349bc.js → TextToRows-27aab4a8.js} +18 -13
- flowfile/web/static/assets/{UnavailableFields-b41976ed.js → UnavailableFields-8143044b.js} +2 -2
- flowfile/web/static/assets/{Union-fca91665.js → Union-52460248.js} +7 -6
- flowfile/web/static/assets/{Unique-a59f830e.js → Unique-f6962644.js} +8 -8
- flowfile/web/static/assets/{Unpivot-c3815565.js → Unpivot-1ff1e938.js} +5 -5
- flowfile/web/static/assets/{api-22b338bd.js → api-3b345d92.js} +1 -1
- flowfile/web/static/assets/{designer-e5bbe26f.js → designer-4736134f.js} +72 -42
- flowfile/web/static/assets/{documentation-08045cf2.js → documentation-b9545eba.js} +1 -1
- flowfile/web/static/assets/{dropDown-5e7e9a5a.js → dropDown-d5a4014c.js} +1 -1
- flowfile/web/static/assets/{dropDownGeneric-50a91b99.js → dropDownGeneric-1f4e32ec.js} +2 -2
- flowfile/web/static/assets/{fullEditor-705c6ccb.js → fullEditor-f4791c23.js} +3 -3
- flowfile/web/static/assets/{genericNodeSettings-65587f20.js → genericNodeSettings-1d456350.js} +3 -3
- flowfile/web/static/assets/{index-552863fd.js → index-f25c9283.js} +2608 -1570
- flowfile/web/static/assets/{nodeTitle-cf9bae3c.js → nodeTitle-cad6fd9d.js} +3 -3
- flowfile/web/static/assets/{secretApi-3ad510e1.js → secretApi-01f07e2c.js} +1 -1
- flowfile/web/static/assets/{selectDynamic-bd644891.js → selectDynamic-f46a4e3f.js} +3 -3
- flowfile/web/static/assets/{vue-codemirror.esm-dd17b478.js → vue-codemirror.esm-eb98fc8b.js} +15 -14
- flowfile/web/static/assets/{vue-content-loader.es-6b36f05e.js → vue-content-loader.es-860c0380.js} +1 -1
- flowfile/web/static/index.html +1 -1
- {flowfile-0.3.1.2.dist-info → flowfile-0.3.3.dist-info}/METADATA +1 -3
- {flowfile-0.3.1.2.dist-info → flowfile-0.3.3.dist-info}/RECORD +97 -88
- flowfile_core/configs/__init__.py +15 -4
- flowfile_core/configs/node_store/nodes.py +2 -4
- flowfile_core/configs/settings.py +5 -3
- flowfile_core/configs/utils.py +18 -0
- flowfile_core/flowfile/FlowfileFlow.py +84 -29
- flowfile_core/flowfile/database_connection_manager/db_connections.py +1 -1
- flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +55 -18
- flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +42 -9
- flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +42 -3
- flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +34 -2
- flowfile_core/flowfile/flow_data_engine/sample_data.py +25 -7
- flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +4 -3
- flowfile_core/flowfile/flow_data_engine/utils.py +1 -0
- flowfile_core/flowfile/flow_graph_utils.py +320 -0
- flowfile_core/flowfile/flow_node/flow_node.py +2 -1
- flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py +2 -2
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/__init__.py +0 -1
- flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +1 -1
- flowfile_core/flowfile/utils.py +34 -3
- flowfile_core/main.py +2 -3
- flowfile_core/routes/secrets.py +1 -1
- flowfile_core/schemas/input_schema.py +12 -14
- flowfile_core/schemas/transform_schema.py +25 -47
- flowfile_frame/__init__.py +11 -4
- flowfile_frame/adding_expr.py +280 -0
- flowfile_frame/config.py +9 -0
- flowfile_frame/expr.py +301 -83
- flowfile_frame/expr.pyi +2174 -0
- flowfile_frame/expr_name.py +258 -0
- flowfile_frame/flow_frame.py +616 -627
- flowfile_frame/flow_frame.pyi +336 -0
- flowfile_frame/flow_frame_methods.py +617 -0
- flowfile_frame/group_frame.py +89 -42
- flowfile_frame/join.py +1 -2
- flowfile_frame/lazy.py +704 -0
- flowfile_frame/lazy_methods.py +201 -0
- flowfile_frame/list_name_space.py +324 -0
- flowfile_frame/selectors.py +3 -0
- flowfile_frame/series.py +70 -0
- flowfile_frame/utils.py +80 -4
- flowfile/web/static/assets/GoogleSheet-854294a4.js +0 -2616
- flowfile/web/static/assets/GoogleSheet-92084da7.css +0 -233
- flowfile_core/flowfile/sources/external_sources/custom_external_sources/google_sheet.py +0 -74
- {flowfile-0.3.1.2.dist-info → flowfile-0.3.3.dist-info}/LICENSE +0 -0
- {flowfile-0.3.1.2.dist-info → flowfile-0.3.3.dist-info}/WHEEL +0 -0
- {flowfile-0.3.1.2.dist-info → flowfile-0.3.3.dist-info}/entry_points.txt +0 -0
- /flowfile_core/{secrets → secret_manager}/__init__.py +0 -0
- /flowfile_core/{secrets/secrets.py → secret_manager/secret_manager.py} +0 -0
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
from typing import TYPE_CHECKING, Callable, Optional
|
|
5
|
+
|
|
6
|
+
if TYPE_CHECKING:
|
|
7
|
+
from flowfile_frame.expr import Expr
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ExprNameNameSpace:
|
|
11
|
+
"""Namespace for expression name operations in FlowFrame."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, parent_expr: 'Expr', parent_repr_str: str):
|
|
14
|
+
"""
|
|
15
|
+
Initialize the namespace with parent expression reference.
|
|
16
|
+
|
|
17
|
+
Parameters
|
|
18
|
+
----------
|
|
19
|
+
parent_expr : Expr
|
|
20
|
+
The parent expression
|
|
21
|
+
parent_repr_str : str
|
|
22
|
+
String representation of the parent expression
|
|
23
|
+
"""
|
|
24
|
+
self.parent = parent_expr
|
|
25
|
+
self.expr = parent_expr.expr.name if parent_expr.expr is not None else None
|
|
26
|
+
self.parent_repr_str = parent_repr_str
|
|
27
|
+
|
|
28
|
+
def _create_next_expr(self, method_name: str, *args, result_expr: Optional[any] = None, **kwargs) -> 'Expr':
|
|
29
|
+
"""Create a new expression with name operation applied."""
|
|
30
|
+
from flowfile_frame.expr import Expr
|
|
31
|
+
|
|
32
|
+
args_repr = ''
|
|
33
|
+
# Format positional args for repr
|
|
34
|
+
if args:
|
|
35
|
+
args_strs = []
|
|
36
|
+
for arg in args:
|
|
37
|
+
if callable(arg):
|
|
38
|
+
# Special handling for lambda functions and callables
|
|
39
|
+
if hasattr(arg, "__name__") and arg.__name__ != "<lambda>":
|
|
40
|
+
# Named function - use its name
|
|
41
|
+
args_strs.append(arg.__name__)
|
|
42
|
+
else:
|
|
43
|
+
# Lambda or unnamed function - create a proper function string representation
|
|
44
|
+
if method_name == "map":
|
|
45
|
+
# For map function specifically, try to extract the function body
|
|
46
|
+
import inspect
|
|
47
|
+
try:
|
|
48
|
+
source = inspect.getsource(arg).strip()
|
|
49
|
+
# Handle multiline lambdas and functions
|
|
50
|
+
if '\n' in source:
|
|
51
|
+
# Try to extract just the lambda expression if possible
|
|
52
|
+
if 'lambda' in source:
|
|
53
|
+
lambda_parts = source.split('lambda')
|
|
54
|
+
if len(lambda_parts) > 1:
|
|
55
|
+
lambda_expr = f"lambda{lambda_parts[1].split(':')[0]}:"
|
|
56
|
+
body_parts = source.split(':')
|
|
57
|
+
if len(body_parts) > 1:
|
|
58
|
+
body = body_parts[1].strip()
|
|
59
|
+
# Remove trailing characters like parentheses, commas
|
|
60
|
+
body = body.rstrip(')\n, ')
|
|
61
|
+
lambda_expr += f" {body}"
|
|
62
|
+
args_strs.append(lambda_expr)
|
|
63
|
+
else:
|
|
64
|
+
args_strs.append("lambda x: x.upper()") # Fallback for common case
|
|
65
|
+
else:
|
|
66
|
+
args_strs.append("lambda x: x.upper()") # Fallback for common case
|
|
67
|
+
else:
|
|
68
|
+
# For non-lambda functions, use a simplified representation
|
|
69
|
+
args_strs.append("lambda x: x.upper()") # Simplified fallback
|
|
70
|
+
else:
|
|
71
|
+
# Single line function, extract it directly
|
|
72
|
+
if 'lambda' in source:
|
|
73
|
+
lambda_expr = source.split('=')[-1].strip()
|
|
74
|
+
args_strs.append(lambda_expr)
|
|
75
|
+
else:
|
|
76
|
+
args_strs.append("lambda x: x.upper()") # Fallback
|
|
77
|
+
except Exception:
|
|
78
|
+
# Fallback to a common case if source extraction fails
|
|
79
|
+
args_strs.append("lambda x: x.upper()")
|
|
80
|
+
else:
|
|
81
|
+
# For other methods, use a generic representation
|
|
82
|
+
args_strs.append("lambda x: x.upper()") # Default case for other name methods
|
|
83
|
+
else:
|
|
84
|
+
args_strs.append(repr(arg))
|
|
85
|
+
args_repr = ", ".join(args_strs)
|
|
86
|
+
|
|
87
|
+
# Format keyword args for repr
|
|
88
|
+
if kwargs:
|
|
89
|
+
kwargs_str = ", ".join(f"{k}={repr(v)}" for k, v in kwargs.items())
|
|
90
|
+
if args_repr:
|
|
91
|
+
args_repr = f"{args_repr}, {kwargs_str}"
|
|
92
|
+
else:
|
|
93
|
+
args_repr = kwargs_str
|
|
94
|
+
|
|
95
|
+
new_repr = f"{self.parent_repr_str}.name.{method_name}({args_repr})"
|
|
96
|
+
|
|
97
|
+
# Create new expression with updated representation
|
|
98
|
+
new_expr = Expr(
|
|
99
|
+
result_expr,
|
|
100
|
+
self.parent.column_name,
|
|
101
|
+
repr_str=new_repr,
|
|
102
|
+
initial_column_name=self.parent._initial_column_name,
|
|
103
|
+
selector=None,
|
|
104
|
+
agg_func=self.parent.agg_func,
|
|
105
|
+
is_complex=True
|
|
106
|
+
)
|
|
107
|
+
return new_expr
|
|
108
|
+
|
|
109
|
+
def keep(self) -> 'Expr':
|
|
110
|
+
"""
|
|
111
|
+
Keep the original root name of the expression.
|
|
112
|
+
|
|
113
|
+
This will undo any previous renaming operations on the expression.
|
|
114
|
+
|
|
115
|
+
Returns
|
|
116
|
+
-------
|
|
117
|
+
Expr
|
|
118
|
+
A new expression with original name kept
|
|
119
|
+
"""
|
|
120
|
+
result_expr = self.expr.keep() if self.expr is not None else None
|
|
121
|
+
return self._create_next_expr("keep", result_expr=result_expr)
|
|
122
|
+
|
|
123
|
+
def map(self, function: Callable[[str], str]) -> 'Expr':
|
|
124
|
+
"""
|
|
125
|
+
Rename the output of an expression by mapping a function over the root name.
|
|
126
|
+
|
|
127
|
+
Parameters
|
|
128
|
+
----------
|
|
129
|
+
function
|
|
130
|
+
Function that maps a root name to a new name
|
|
131
|
+
|
|
132
|
+
Returns
|
|
133
|
+
-------
|
|
134
|
+
Expr
|
|
135
|
+
A new expression with mapped name
|
|
136
|
+
|
|
137
|
+
Examples
|
|
138
|
+
--------
|
|
139
|
+
>>> df = ff.FlowFrame({'a': [1, 2, 3]})
|
|
140
|
+
>>> df.select(ff.col('a').name.map(lambda x: x.upper()))
|
|
141
|
+
"""
|
|
142
|
+
# We need to handle both the actual expression and its string representation
|
|
143
|
+
|
|
144
|
+
# For the actual polars expression:
|
|
145
|
+
result_expr = self.expr.map(function) if self.expr is not None else None
|
|
146
|
+
|
|
147
|
+
# The representation is handled by _create_next_expr with special lambda handling
|
|
148
|
+
return self._create_next_expr("map", function, result_expr=result_expr)
|
|
149
|
+
|
|
150
|
+
def prefix(self, prefix: str) -> 'Expr':
|
|
151
|
+
"""
|
|
152
|
+
Add a prefix to the root column name of the expression.
|
|
153
|
+
|
|
154
|
+
Parameters
|
|
155
|
+
----------
|
|
156
|
+
prefix
|
|
157
|
+
Prefix to add to the root column name
|
|
158
|
+
|
|
159
|
+
Returns
|
|
160
|
+
-------
|
|
161
|
+
Expr
|
|
162
|
+
A new expression with prefixed name
|
|
163
|
+
"""
|
|
164
|
+
result_expr = self.expr.prefix(prefix) if self.expr is not None else None
|
|
165
|
+
return self._create_next_expr("prefix", prefix, result_expr=result_expr)
|
|
166
|
+
|
|
167
|
+
def suffix(self, suffix: str) -> 'Expr':
|
|
168
|
+
"""
|
|
169
|
+
Add a suffix to the root column name of the expression.
|
|
170
|
+
|
|
171
|
+
Parameters
|
|
172
|
+
----------
|
|
173
|
+
suffix
|
|
174
|
+
Suffix to add to the root column name
|
|
175
|
+
|
|
176
|
+
Returns
|
|
177
|
+
-------
|
|
178
|
+
Expr
|
|
179
|
+
A new expression with suffixed name
|
|
180
|
+
"""
|
|
181
|
+
result_expr = self.expr.suffix(suffix) if self.expr is not None else None
|
|
182
|
+
return self._create_next_expr("suffix", suffix, result_expr=result_expr)
|
|
183
|
+
|
|
184
|
+
def to_lowercase(self) -> 'Expr':
|
|
185
|
+
"""
|
|
186
|
+
Make the root column name lowercase.
|
|
187
|
+
|
|
188
|
+
Returns
|
|
189
|
+
-------
|
|
190
|
+
Expr
|
|
191
|
+
A new expression with lowercase name
|
|
192
|
+
"""
|
|
193
|
+
result_expr = self.expr.to_lowercase() if self.expr is not None else None
|
|
194
|
+
return self._create_next_expr("to_lowercase", result_expr=result_expr)
|
|
195
|
+
|
|
196
|
+
def to_uppercase(self) -> 'Expr':
|
|
197
|
+
"""
|
|
198
|
+
Make the root column name uppercase.
|
|
199
|
+
|
|
200
|
+
Returns
|
|
201
|
+
-------
|
|
202
|
+
Expr
|
|
203
|
+
A new expression with uppercase name
|
|
204
|
+
"""
|
|
205
|
+
result_expr = self.expr.to_uppercase() if self.expr is not None else None
|
|
206
|
+
return self._create_next_expr("to_uppercase", result_expr=result_expr)
|
|
207
|
+
|
|
208
|
+
def map_fields(self, function: Callable[[str], str]) -> 'Expr':
|
|
209
|
+
"""
|
|
210
|
+
Rename fields of a struct by mapping a function over the field name(s).
|
|
211
|
+
|
|
212
|
+
Parameters
|
|
213
|
+
----------
|
|
214
|
+
function
|
|
215
|
+
Function that maps a field name to a new name
|
|
216
|
+
|
|
217
|
+
Returns
|
|
218
|
+
-------
|
|
219
|
+
Expr
|
|
220
|
+
A new expression with mapped field names
|
|
221
|
+
"""
|
|
222
|
+
result_expr = self.expr.map_fields(function) if self.expr is not None else None
|
|
223
|
+
return self._create_next_expr("map_fields", function, result_expr=result_expr)
|
|
224
|
+
|
|
225
|
+
def prefix_fields(self, prefix: str) -> 'Expr':
|
|
226
|
+
"""
|
|
227
|
+
Add a prefix to all field names of a struct.
|
|
228
|
+
|
|
229
|
+
Parameters
|
|
230
|
+
----------
|
|
231
|
+
prefix
|
|
232
|
+
Prefix to add to the field name
|
|
233
|
+
|
|
234
|
+
Returns
|
|
235
|
+
-------
|
|
236
|
+
Expr
|
|
237
|
+
A new expression with prefixed field names
|
|
238
|
+
"""
|
|
239
|
+
result_expr = self.expr.prefix_fields(prefix) if self.expr is not None else None
|
|
240
|
+
return self._create_next_expr("prefix_fields", prefix, result_expr=result_expr)
|
|
241
|
+
|
|
242
|
+
def suffix_fields(self, suffix: str) -> 'Expr':
|
|
243
|
+
"""
|
|
244
|
+
Add a suffix to all field names of a struct.
|
|
245
|
+
|
|
246
|
+
Parameters
|
|
247
|
+
----------
|
|
248
|
+
suffix
|
|
249
|
+
Suffix to add to the field name
|
|
250
|
+
|
|
251
|
+
Returns
|
|
252
|
+
-------
|
|
253
|
+
Expr
|
|
254
|
+
A new expression with suffixed field names
|
|
255
|
+
"""
|
|
256
|
+
result_expr = self.expr.suffix_fields(suffix) if self.expr is not None else None
|
|
257
|
+
return self._create_next_expr("suffix_fields", suffix, result_expr=result_expr)
|
|
258
|
+
|