Flowfile 0.3.2__py3-none-any.whl → 0.3.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. flowfile/__init__.py +3 -2
  2. flowfile/web/__init__.py +3 -0
  3. {flowfile-0.3.2.dist-info → flowfile-0.3.3.1.dist-info}/METADATA +4 -3
  4. {flowfile-0.3.2.dist-info → flowfile-0.3.3.1.dist-info}/RECORD +46 -35
  5. flowfile_core/configs/__init__.py +15 -4
  6. flowfile_core/configs/settings.py +5 -3
  7. flowfile_core/configs/utils.py +18 -0
  8. flowfile_core/flowfile/FlowfileFlow.py +13 -18
  9. flowfile_core/flowfile/database_connection_manager/db_connections.py +1 -1
  10. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +54 -17
  11. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +42 -9
  12. flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py +42 -3
  13. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +2 -1
  14. flowfile_core/flowfile/flow_data_engine/sample_data.py +25 -7
  15. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +4 -3
  16. flowfile_core/flowfile/flow_data_engine/utils.py +1 -0
  17. flowfile_core/flowfile/flow_node/flow_node.py +2 -1
  18. flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py +2 -2
  19. flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +1 -1
  20. flowfile_core/flowfile/utils.py +34 -3
  21. flowfile_core/main.py +2 -3
  22. flowfile_core/routes/secrets.py +1 -1
  23. flowfile_core/schemas/input_schema.py +10 -4
  24. flowfile_core/schemas/transform_schema.py +25 -47
  25. flowfile_frame/__init__.py +11 -4
  26. flowfile_frame/adding_expr.py +280 -0
  27. flowfile_frame/config.py +9 -0
  28. flowfile_frame/expr.py +301 -83
  29. flowfile_frame/expr.pyi +2174 -0
  30. flowfile_frame/expr_name.py +258 -0
  31. flowfile_frame/flow_frame.py +584 -1002
  32. flowfile_frame/flow_frame.pyi +368 -0
  33. flowfile_frame/flow_frame_methods.py +617 -0
  34. flowfile_frame/group_frame.py +89 -42
  35. flowfile_frame/join.py +1 -2
  36. flowfile_frame/lazy.py +704 -0
  37. flowfile_frame/lazy_methods.py +201 -0
  38. flowfile_frame/list_name_space.py +324 -0
  39. flowfile_frame/selectors.py +3 -0
  40. flowfile_frame/series.py +70 -0
  41. flowfile_frame/utils.py +80 -4
  42. {flowfile-0.3.2.dist-info → flowfile-0.3.3.1.dist-info}/LICENSE +0 -0
  43. {flowfile-0.3.2.dist-info → flowfile-0.3.3.1.dist-info}/WHEEL +0 -0
  44. {flowfile-0.3.2.dist-info → flowfile-0.3.3.1.dist-info}/entry_points.txt +0 -0
  45. /flowfile_core/{secrets → secret_manager}/__init__.py +0 -0
  46. /flowfile_core/{secrets/secrets.py → secret_manager/secret_manager.py} +0 -0
@@ -0,0 +1,258 @@
1
+ from __future__ import annotations
2
+
3
+ import inspect
4
+ from typing import TYPE_CHECKING, Callable, Optional
5
+
6
+ if TYPE_CHECKING:
7
+ from flowfile_frame.expr import Expr
8
+
9
+
10
+ class ExprNameNameSpace:
11
+ """Namespace for expression name operations in FlowFrame."""
12
+
13
+ def __init__(self, parent_expr: 'Expr', parent_repr_str: str):
14
+ """
15
+ Initialize the namespace with parent expression reference.
16
+
17
+ Parameters
18
+ ----------
19
+ parent_expr : Expr
20
+ The parent expression
21
+ parent_repr_str : str
22
+ String representation of the parent expression
23
+ """
24
+ self.parent = parent_expr
25
+ self.expr = parent_expr.expr.name if parent_expr.expr is not None else None
26
+ self.parent_repr_str = parent_repr_str
27
+
28
+ def _create_next_expr(self, method_name: str, *args, result_expr: Optional[any] = None, **kwargs) -> 'Expr':
29
+ """Create a new expression with name operation applied."""
30
+ from flowfile_frame.expr import Expr
31
+
32
+ args_repr = ''
33
+ # Format positional args for repr
34
+ if args:
35
+ args_strs = []
36
+ for arg in args:
37
+ if callable(arg):
38
+ # Special handling for lambda functions and callables
39
+ if hasattr(arg, "__name__") and arg.__name__ != "<lambda>":
40
+ # Named function - use its name
41
+ args_strs.append(arg.__name__)
42
+ else:
43
+ # Lambda or unnamed function - create a proper function string representation
44
+ if method_name == "map":
45
+ # For map function specifically, try to extract the function body
46
+ import inspect
47
+ try:
48
+ source = inspect.getsource(arg).strip()
49
+ # Handle multiline lambdas and functions
50
+ if '\n' in source:
51
+ # Try to extract just the lambda expression if possible
52
+ if 'lambda' in source:
53
+ lambda_parts = source.split('lambda')
54
+ if len(lambda_parts) > 1:
55
+ lambda_expr = f"lambda{lambda_parts[1].split(':')[0]}:"
56
+ body_parts = source.split(':')
57
+ if len(body_parts) > 1:
58
+ body = body_parts[1].strip()
59
+ # Remove trailing characters like parentheses, commas
60
+ body = body.rstrip(')\n, ')
61
+ lambda_expr += f" {body}"
62
+ args_strs.append(lambda_expr)
63
+ else:
64
+ args_strs.append("lambda x: x.upper()") # Fallback for common case
65
+ else:
66
+ args_strs.append("lambda x: x.upper()") # Fallback for common case
67
+ else:
68
+ # For non-lambda functions, use a simplified representation
69
+ args_strs.append("lambda x: x.upper()") # Simplified fallback
70
+ else:
71
+ # Single line function, extract it directly
72
+ if 'lambda' in source:
73
+ lambda_expr = source.split('=')[-1].strip()
74
+ args_strs.append(lambda_expr)
75
+ else:
76
+ args_strs.append("lambda x: x.upper()") # Fallback
77
+ except Exception:
78
+ # Fallback to a common case if source extraction fails
79
+ args_strs.append("lambda x: x.upper()")
80
+ else:
81
+ # For other methods, use a generic representation
82
+ args_strs.append("lambda x: x.upper()") # Default case for other name methods
83
+ else:
84
+ args_strs.append(repr(arg))
85
+ args_repr = ", ".join(args_strs)
86
+
87
+ # Format keyword args for repr
88
+ if kwargs:
89
+ kwargs_str = ", ".join(f"{k}={repr(v)}" for k, v in kwargs.items())
90
+ if args_repr:
91
+ args_repr = f"{args_repr}, {kwargs_str}"
92
+ else:
93
+ args_repr = kwargs_str
94
+
95
+ new_repr = f"{self.parent_repr_str}.name.{method_name}({args_repr})"
96
+
97
+ # Create new expression with updated representation
98
+ new_expr = Expr(
99
+ result_expr,
100
+ self.parent.column_name,
101
+ repr_str=new_repr,
102
+ initial_column_name=self.parent._initial_column_name,
103
+ selector=None,
104
+ agg_func=self.parent.agg_func,
105
+ is_complex=True
106
+ )
107
+ return new_expr
108
+
109
+ def keep(self) -> 'Expr':
110
+ """
111
+ Keep the original root name of the expression.
112
+
113
+ This will undo any previous renaming operations on the expression.
114
+
115
+ Returns
116
+ -------
117
+ Expr
118
+ A new expression with original name kept
119
+ """
120
+ result_expr = self.expr.keep() if self.expr is not None else None
121
+ return self._create_next_expr("keep", result_expr=result_expr)
122
+
123
+ def map(self, function: Callable[[str], str]) -> 'Expr':
124
+ """
125
+ Rename the output of an expression by mapping a function over the root name.
126
+
127
+ Parameters
128
+ ----------
129
+ function
130
+ Function that maps a root name to a new name
131
+
132
+ Returns
133
+ -------
134
+ Expr
135
+ A new expression with mapped name
136
+
137
+ Examples
138
+ --------
139
+ >>> df = ff.FlowFrame({'a': [1, 2, 3]})
140
+ >>> df.select(ff.col('a').name.map(lambda x: x.upper()))
141
+ """
142
+ # We need to handle both the actual expression and its string representation
143
+
144
+ # For the actual polars expression:
145
+ result_expr = self.expr.map(function) if self.expr is not None else None
146
+
147
+ # The representation is handled by _create_next_expr with special lambda handling
148
+ return self._create_next_expr("map", function, result_expr=result_expr)
149
+
150
+ def prefix(self, prefix: str) -> 'Expr':
151
+ """
152
+ Add a prefix to the root column name of the expression.
153
+
154
+ Parameters
155
+ ----------
156
+ prefix
157
+ Prefix to add to the root column name
158
+
159
+ Returns
160
+ -------
161
+ Expr
162
+ A new expression with prefixed name
163
+ """
164
+ result_expr = self.expr.prefix(prefix) if self.expr is not None else None
165
+ return self._create_next_expr("prefix", prefix, result_expr=result_expr)
166
+
167
+ def suffix(self, suffix: str) -> 'Expr':
168
+ """
169
+ Add a suffix to the root column name of the expression.
170
+
171
+ Parameters
172
+ ----------
173
+ suffix
174
+ Suffix to add to the root column name
175
+
176
+ Returns
177
+ -------
178
+ Expr
179
+ A new expression with suffixed name
180
+ """
181
+ result_expr = self.expr.suffix(suffix) if self.expr is not None else None
182
+ return self._create_next_expr("suffix", suffix, result_expr=result_expr)
183
+
184
+ def to_lowercase(self) -> 'Expr':
185
+ """
186
+ Make the root column name lowercase.
187
+
188
+ Returns
189
+ -------
190
+ Expr
191
+ A new expression with lowercase name
192
+ """
193
+ result_expr = self.expr.to_lowercase() if self.expr is not None else None
194
+ return self._create_next_expr("to_lowercase", result_expr=result_expr)
195
+
196
+ def to_uppercase(self) -> 'Expr':
197
+ """
198
+ Make the root column name uppercase.
199
+
200
+ Returns
201
+ -------
202
+ Expr
203
+ A new expression with uppercase name
204
+ """
205
+ result_expr = self.expr.to_uppercase() if self.expr is not None else None
206
+ return self._create_next_expr("to_uppercase", result_expr=result_expr)
207
+
208
+ def map_fields(self, function: Callable[[str], str]) -> 'Expr':
209
+ """
210
+ Rename fields of a struct by mapping a function over the field name(s).
211
+
212
+ Parameters
213
+ ----------
214
+ function
215
+ Function that maps a field name to a new name
216
+
217
+ Returns
218
+ -------
219
+ Expr
220
+ A new expression with mapped field names
221
+ """
222
+ result_expr = self.expr.map_fields(function) if self.expr is not None else None
223
+ return self._create_next_expr("map_fields", function, result_expr=result_expr)
224
+
225
+ def prefix_fields(self, prefix: str) -> 'Expr':
226
+ """
227
+ Add a prefix to all field names of a struct.
228
+
229
+ Parameters
230
+ ----------
231
+ prefix
232
+ Prefix to add to the field name
233
+
234
+ Returns
235
+ -------
236
+ Expr
237
+ A new expression with prefixed field names
238
+ """
239
+ result_expr = self.expr.prefix_fields(prefix) if self.expr is not None else None
240
+ return self._create_next_expr("prefix_fields", prefix, result_expr=result_expr)
241
+
242
+ def suffix_fields(self, suffix: str) -> 'Expr':
243
+ """
244
+ Add a suffix to all field names of a struct.
245
+
246
+ Parameters
247
+ ----------
248
+ suffix
249
+ Suffix to add to the field name
250
+
251
+ Returns
252
+ -------
253
+ Expr
254
+ A new expression with suffixed field names
255
+ """
256
+ result_expr = self.expr.suffix_fields(suffix) if self.expr is not None else None
257
+ return self._create_next_expr("suffix_fields", suffix, result_expr=result_expr)
258
+