Flowfile 0.3.9__py3-none-any.whl → 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of Flowfile might be problematic. Click here for more details.

Files changed (169) hide show
  1. flowfile/__init__.py +6 -1
  2. flowfile/api.py +0 -1
  3. flowfile/web/static/assets/{CloudConnectionManager-c97c25f8.js → CloudConnectionManager-109ecc3c.js} +2 -2
  4. flowfile/web/static/assets/{CloudStorageReader-f1ff509e.js → CloudStorageReader-19cdd67a.js} +11 -78
  5. flowfile/web/static/assets/{CloudStorageWriter-034f8b78.js → CloudStorageWriter-48e0ae20.js} +12 -79
  6. flowfile/web/static/assets/{CloudStorageWriter-49c9a4b2.css → CloudStorageWriter-b0ee067f.css} +24 -24
  7. flowfile/web/static/assets/ColumnSelector-47996a16.css +10 -0
  8. flowfile/web/static/assets/ColumnSelector-ecaf7c44.js +83 -0
  9. flowfile/web/static/assets/ContextMenu-2b348c4c.js +41 -0
  10. flowfile/web/static/assets/{SettingsSection-9c836ecc.css → ContextMenu-4c74eef1.css} +0 -21
  11. flowfile/web/static/assets/ContextMenu-63cfa99b.css +26 -0
  12. flowfile/web/static/assets/ContextMenu-a779eed7.js +41 -0
  13. flowfile/web/static/assets/ContextMenu-c13f91d0.css +26 -0
  14. flowfile/web/static/assets/ContextMenu-eca26a03.js +41 -0
  15. flowfile/web/static/assets/{CrossJoin-41efa4cb.css → CrossJoin-1119d18e.css} +18 -18
  16. flowfile/web/static/assets/{CrossJoin-9e156ebe.js → CrossJoin-a88f8142.js} +14 -84
  17. flowfile/web/static/assets/CustomNode-74a37f74.css +32 -0
  18. flowfile/web/static/assets/CustomNode-cb863dff.js +211 -0
  19. flowfile/web/static/assets/{DatabaseConnectionSettings-d5c625b3.js → DatabaseConnectionSettings-819d3267.js} +3 -3
  20. flowfile/web/static/assets/{DatabaseManager-265adc5e.js → DatabaseManager-84ee2834.js} +2 -2
  21. flowfile/web/static/assets/{DatabaseReader-0b10551e.js → DatabaseReader-060dd412.js} +14 -114
  22. flowfile/web/static/assets/{DatabaseReader-f50c6558.css → DatabaseReader-ae61773c.css} +0 -27
  23. flowfile/web/static/assets/{DatabaseWriter-c17c6916.js → DatabaseWriter-7fc7750f.js} +13 -74
  24. flowfile/web/static/assets/{ExploreData-5bdae813.css → ExploreData-2d0cf4db.css} +8 -14
  25. flowfile/web/static/assets/ExploreData-82c95991.js +192 -0
  26. flowfile/web/static/assets/{ExternalSource-3a66556c.js → ExternalSource-e1a6ddc7.js} +8 -79
  27. flowfile/web/static/assets/{Filter-91ad87e7.js → Filter-8aca894a.js} +12 -85
  28. flowfile/web/static/assets/{Filter-a9d08ba1.css → Filter-f62091b3.css} +3 -3
  29. flowfile/web/static/assets/{Formula-29f19d21.css → Formula-bb96803d.css} +4 -4
  30. flowfile/web/static/assets/{Formula-3c395ab1.js → Formula-e33686d9.js} +18 -85
  31. flowfile/web/static/assets/{FuzzyMatch-6857de82.css → FuzzyMatch-1010f966.css} +42 -42
  32. flowfile/web/static/assets/{FuzzyMatch-2df0d230.js → FuzzyMatch-abda150d.js} +16 -87
  33. flowfile/web/static/assets/{GraphSolver-d285877f.js → GraphSolver-4ecad1d7.js} +13 -159
  34. flowfile/web/static/assets/GraphSolver-f0cb7bfb.css +22 -0
  35. flowfile/web/static/assets/{GroupBy-0bd1cc6b.js → GroupBy-656d07f3.js} +12 -75
  36. flowfile/web/static/assets/{Unique-b5615727.css → GroupBy-b9505323.css} +8 -8
  37. flowfile/web/static/assets/{Join-5a78a203.js → Join-b84ec849.js} +15 -85
  38. flowfile/web/static/assets/{Join-f45eff22.css → Join-fd79b451.css} +20 -20
  39. flowfile/web/static/assets/{ManualInput-a71b52c6.css → ManualInput-3246a08d.css} +20 -20
  40. flowfile/web/static/assets/{ManualInput-93aef9d6.js → ManualInput-346f4135.js} +11 -82
  41. flowfile/web/static/assets/MultiSelect-61b98268.js +5 -0
  42. flowfile/web/static/assets/MultiSelect.vue_vue_type_script_setup_true_lang-2a7c8312.js +63 -0
  43. flowfile/web/static/assets/NumericInput-e36602c2.js +5 -0
  44. flowfile/web/static/assets/NumericInput.vue_vue_type_script_setup_true_lang-211a1990.js +35 -0
  45. flowfile/web/static/assets/Output-ddc9079f.css +37 -0
  46. flowfile/web/static/assets/{Output-411ecaee.js → Output-eb041599.js} +13 -243
  47. flowfile/web/static/assets/Pivot-cf333e3d.css +22 -0
  48. flowfile/web/static/assets/{Pivot-89db4b04.js → Pivot-f5c774f4.js} +14 -138
  49. flowfile/web/static/assets/PivotValidation-26546cbc.js +61 -0
  50. flowfile/web/static/assets/PivotValidation-891ddfb0.css +13 -0
  51. flowfile/web/static/assets/PivotValidation-c46cd420.css +13 -0
  52. flowfile/web/static/assets/PivotValidation-e150a24b.js +61 -0
  53. flowfile/web/static/assets/{PolarsCode-a9f974f8.js → PolarsCode-da3a7abf.js} +13 -80
  54. flowfile/web/static/assets/Read-0c768769.js +243 -0
  55. flowfile/web/static/assets/Read-6b17491f.css +62 -0
  56. flowfile/web/static/assets/RecordCount-84736276.js +53 -0
  57. flowfile/web/static/assets/{RecordId-55ae7d36.js → RecordId-60055e6d.js} +8 -80
  58. flowfile/web/static/assets/SQLQueryComponent-36cef432.css +27 -0
  59. flowfile/web/static/assets/SQLQueryComponent-8a486004.js +38 -0
  60. flowfile/web/static/assets/{Sample-b4a18476.js → Sample-2d662611.js} +8 -77
  61. flowfile/web/static/assets/{SecretManager-b066d13a.js → SecretManager-ef586cab.js} +2 -2
  62. flowfile/web/static/assets/{Select-727688dc.js → Select-2e4a6965.js} +11 -85
  63. flowfile/web/static/assets/SettingsSection-2e4d03c4.css +21 -0
  64. flowfile/web/static/assets/{SettingsSection-695ac487.js → SettingsSection-310b61c0.js} +2 -40
  65. flowfile/web/static/assets/SettingsSection-5634f439.js +45 -0
  66. flowfile/web/static/assets/SettingsSection-5c696bee.css +20 -0
  67. flowfile/web/static/assets/SettingsSection-71e6b7e3.css +21 -0
  68. flowfile/web/static/assets/SettingsSection-7c68b19f.js +53 -0
  69. flowfile/web/static/assets/SingleSelect-7298811a.js +5 -0
  70. flowfile/web/static/assets/SingleSelect.vue_vue_type_script_setup_true_lang-43807bad.js +62 -0
  71. flowfile/web/static/assets/SliderInput-53105476.js +40 -0
  72. flowfile/web/static/assets/SliderInput-b8fb6a8c.css +4 -0
  73. flowfile/web/static/assets/{GroupBy-ab1ea74b.css → Sort-3643d625.css} +8 -8
  74. flowfile/web/static/assets/{Sort-be3339a8.js → Sort-4fdebe74.js} +12 -97
  75. flowfile/web/static/assets/TextInput-28366b7e.js +5 -0
  76. flowfile/web/static/assets/TextInput.vue_vue_type_script_setup_true_lang-9cad14ba.js +32 -0
  77. flowfile/web/static/assets/{TextToRows-c92d1ec2.css → TextToRows-5d2c1190.css} +9 -9
  78. flowfile/web/static/assets/{TextToRows-7b8998da.js → TextToRows-73ffa692.js} +14 -83
  79. flowfile/web/static/assets/ToggleSwitch-598add30.js +5 -0
  80. flowfile/web/static/assets/ToggleSwitch.vue_vue_type_script_setup_true_lang-f620cd32.js +31 -0
  81. flowfile/web/static/assets/{UnavailableFields-8b0cb48e.js → UnavailableFields-66239e83.js} +2 -2
  82. flowfile/web/static/assets/Union-26b10614.js +77 -0
  83. flowfile/web/static/assets/{Union-8d9ac7f9.css → Union-af6c3d9b.css} +6 -6
  84. flowfile/web/static/assets/{Unique-af5a80b4.js → Unique-33b9edbb.js} +22 -91
  85. flowfile/web/static/assets/{Sort-7ccfa0fe.css → Unique-f9fb0809.css} +8 -8
  86. flowfile/web/static/assets/Unpivot-1e422df3.css +30 -0
  87. flowfile/web/static/assets/{Unpivot-5195d411.js → Unpivot-ef69d0e2.js} +12 -166
  88. flowfile/web/static/assets/UnpivotValidation-0d240eeb.css +13 -0
  89. flowfile/web/static/assets/UnpivotValidation-8658388e.js +51 -0
  90. flowfile/web/static/assets/{ExploreData-18a4fe52.js → VueGraphicWalker-4d7861f4.js} +4 -264
  91. flowfile/web/static/assets/VueGraphicWalker-ed5ab88b.css +6 -0
  92. flowfile/web/static/assets/{api-023d1733.js → api-2d1394bd.js} +1 -1
  93. flowfile/web/static/assets/{api-cb00cce6.js → api-c908fffe.js} +1 -1
  94. flowfile/web/static/assets/{designer-6c322d8e.js → designer-1667687d.js} +2201 -705
  95. flowfile/web/static/assets/{designer-2197d782.css → designer-665e9408.css} +836 -201
  96. flowfile/web/static/assets/{documentation-4d1fafe1.js → documentation-5eed779e.js} +1 -1
  97. flowfile/web/static/assets/{dropDown-0b46dd77.js → dropDown-41ebe3c2.js} +1 -1
  98. flowfile/web/static/assets/{fullEditor-ec4e4f95.js → fullEditor-0670d32d.js} +2 -2
  99. flowfile/web/static/assets/{genericNodeSettings-def5879b.js → genericNodeSettings-38410ebf.js} +3 -3
  100. flowfile/web/static/assets/{index-681a3ed0.css → index-50508d4d.css} +8 -0
  101. flowfile/web/static/assets/{index-683fc198.js → index-5ec791df.js} +210 -31
  102. flowfile/web/static/assets/outputCsv-059583b6.js +86 -0
  103. flowfile/web/static/assets/{Output-48f81019.css → outputCsv-9cc59e0b.css} +0 -143
  104. flowfile/web/static/assets/outputExcel-76b1e02c.js +56 -0
  105. flowfile/web/static/assets/outputExcel-b41305c0.css +102 -0
  106. flowfile/web/static/assets/outputParquet-440fd4c7.js +31 -0
  107. flowfile/web/static/assets/outputParquet-cf8cf3f2.css +4 -0
  108. flowfile/web/static/assets/readCsv-9813903a.js +178 -0
  109. flowfile/web/static/assets/readCsv-bca3ed53.css +52 -0
  110. flowfile/web/static/assets/readExcel-7f40d237.js +203 -0
  111. flowfile/web/static/assets/readExcel-e1b381ea.css +64 -0
  112. flowfile/web/static/assets/readParquet-22d56002.js +26 -0
  113. flowfile/web/static/assets/readParquet-cee068e2.css +19 -0
  114. flowfile/web/static/assets/{secretApi-baceb6f9.js → secretApi-b3cb072e.js} +1 -1
  115. flowfile/web/static/assets/{selectDynamic-de91449a.js → selectDynamic-7ad95bca.js} +7 -7
  116. flowfile/web/static/assets/{selectDynamic-b062bc9b.css → selectDynamic-aa913ff4.css} +16 -16
  117. flowfile/web/static/assets/user-defined-icon-0ae16c90.png +0 -0
  118. flowfile/web/static/assets/{vue-codemirror.esm-dc5e3348.js → vue-codemirror.esm-b1dfaa46.js} +59 -33
  119. flowfile/web/static/assets/{vue-content-loader.es-ba94b82f.js → vue-content-loader.es-22bac17c.js} +1 -1
  120. flowfile/web/static/index.html +2 -2
  121. {flowfile-0.3.9.dist-info → flowfile-0.4.0.dist-info}/METADATA +1 -1
  122. {flowfile-0.3.9.dist-info → flowfile-0.4.0.dist-info}/RECORD +160 -102
  123. flowfile_core/configs/flow_logger.py +5 -13
  124. flowfile_core/configs/node_store/__init__.py +30 -0
  125. flowfile_core/configs/node_store/nodes.py +383 -99
  126. flowfile_core/configs/node_store/user_defined_node_registry.py +193 -0
  127. flowfile_core/configs/settings.py +2 -1
  128. flowfile_core/database/connection.py +5 -21
  129. flowfile_core/fileExplorer/funcs.py +239 -121
  130. flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py +4 -0
  131. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +19 -34
  132. flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py +36 -0
  133. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +28 -8
  134. flowfile_core/flowfile/flow_graph.py +117 -34
  135. flowfile_core/flowfile/flow_node/flow_node.py +45 -13
  136. flowfile_core/flowfile/handler.py +22 -3
  137. flowfile_core/flowfile/manage/open_flowfile.py +9 -1
  138. flowfile_core/flowfile/node_designer/__init__.py +47 -0
  139. flowfile_core/flowfile/node_designer/_type_registry.py +197 -0
  140. flowfile_core/flowfile/node_designer/custom_node.py +371 -0
  141. flowfile_core/flowfile/node_designer/data_types.py +146 -0
  142. flowfile_core/flowfile/node_designer/ui_components.py +277 -0
  143. flowfile_core/flowfile/schema_callbacks.py +8 -4
  144. flowfile_core/flowfile/setting_generator/settings.py +0 -1
  145. flowfile_core/main.py +5 -1
  146. flowfile_core/routes/routes.py +73 -28
  147. flowfile_core/routes/user_defined_components.py +55 -0
  148. flowfile_core/schemas/input_schema.py +7 -1
  149. flowfile_core/schemas/output_model.py +5 -2
  150. flowfile_core/schemas/schemas.py +8 -3
  151. flowfile_core/schemas/transform_schema.py +1 -0
  152. flowfile_core/utils/validate_setup.py +3 -1
  153. flowfile_worker/__init__.py +6 -35
  154. flowfile_worker/main.py +5 -2
  155. flowfile_worker/routes.py +47 -5
  156. shared/__init__.py +15 -0
  157. shared/storage_config.py +258 -0
  158. flowfile/web/static/assets/GraphSolver-17fd26db.css +0 -68
  159. flowfile/web/static/assets/Pivot-f415e85f.css +0 -35
  160. flowfile/web/static/assets/Read-80dc1675.css +0 -197
  161. flowfile/web/static/assets/Read-c3b1929c.js +0 -701
  162. flowfile/web/static/assets/RecordCount-4e95f98e.js +0 -122
  163. flowfile/web/static/assets/Union-89fd73dc.js +0 -146
  164. flowfile/web/static/assets/Unpivot-246e9bbd.css +0 -77
  165. flowfile/web/static/assets/nodeTitle-a16db7c3.js +0 -227
  166. flowfile/web/static/assets/nodeTitle-f4b12bcb.css +0 -134
  167. {flowfile-0.3.9.dist-info → flowfile-0.4.0.dist-info}/LICENSE +0 -0
  168. {flowfile-0.3.9.dist-info → flowfile-0.4.0.dist-info}/WHEEL +0 -0
  169. {flowfile-0.3.9.dist-info → flowfile-0.4.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,146 @@
1
+ # types.py - Public API for type specifications
2
+ """
3
+ Public type system for column selection and data type specification.
4
+
5
+ Usage:
6
+ from flowfile_core.types import Types
7
+
8
+ # Use type groups
9
+ ColumnSelector(data_types=Types.Numeric)
10
+ ColumnSelector(data_types=Types.String)
11
+
12
+ # Use specific types
13
+ ColumnSelector(data_types=Types.Int64)
14
+ ColumnSelector(data_types=Types.Float)
15
+
16
+ # Mix and match
17
+ ColumnSelector(data_types=[Types.Numeric, Types.String])
18
+ """
19
+
20
+ from enum import Enum
21
+ from typing import List, Union
22
+ import polars as pl
23
+
24
+
25
+ class TypeGroup(str, Enum):
26
+ """High-level type groups for column selection."""
27
+ Numeric = "Numeric"
28
+ String = "String"
29
+ Date = "Date"
30
+ Boolean = "Boolean"
31
+ Binary = "Binary"
32
+ Complex = "Complex"
33
+ All = "ALL"
34
+
35
+ def __str__(self) -> str:
36
+ return self.value
37
+
38
+ def __repr__(self) -> str:
39
+ return f"Types.{self.name}"
40
+
41
+
42
+ class DataType(str, Enum):
43
+ """Specific data types for fine-grained control."""
44
+ # Numeric types
45
+ Int8 = "Int8"
46
+ Int16 = "Int16"
47
+ Int32 = "Int32"
48
+ Int64 = "Int64"
49
+ UInt8 = "UInt8"
50
+ UInt16 = "UInt16"
51
+ UInt32 = "UInt32"
52
+ UInt64 = "UInt64"
53
+ Float32 = "Float32"
54
+ Float64 = "Float64"
55
+ Decimal = "Decimal"
56
+
57
+ # String types
58
+ String = "String"
59
+ Categorical = "Categorical"
60
+
61
+ # Date types
62
+ Date = "Date"
63
+ Datetime = "Datetime"
64
+ Time = "Time"
65
+ Duration = "Duration"
66
+
67
+ # Other types
68
+ Boolean = "Boolean"
69
+ Binary = "Binary"
70
+ List = "List"
71
+ Struct = "Struct"
72
+ Array = "Array"
73
+
74
+ def __str__(self) -> str:
75
+ return self.value
76
+
77
+ def __repr__(self) -> str:
78
+ return f"Types.{self.name}"
79
+
80
+
81
+ class Types:
82
+ """
83
+ Main entry point for type specifications.
84
+
85
+ Examples:
86
+ Types.Numeric # All numeric columns
87
+ Types.String # All string columns
88
+ Types.Int64 # 64-bit integers only
89
+ Types.Float # Alias for Float64
90
+ Types.All # All column types
91
+ """
92
+
93
+ # Type groups (most common use case)
94
+ Numeric = TypeGroup.Numeric
95
+ String = TypeGroup.String
96
+ Date = TypeGroup.Date
97
+ Boolean = TypeGroup.Boolean
98
+ Binary = TypeGroup.Binary
99
+ Complex = TypeGroup.Complex
100
+ All = TypeGroup.All
101
+
102
+ # Specific numeric types
103
+ Int = DataType.Int64 # Default integer
104
+ Int8 = DataType.Int8
105
+ Int16 = DataType.Int16
106
+ Int32 = DataType.Int32
107
+ Int64 = DataType.Int64
108
+ UInt8 = DataType.UInt8
109
+ UInt16 = DataType.UInt16
110
+ UInt32 = DataType.UInt32
111
+ UInt64 = DataType.UInt64
112
+
113
+ Float = DataType.Float64 # Default float
114
+ Float32 = DataType.Float32
115
+ Float64 = DataType.Float64
116
+ Decimal = DataType.Decimal
117
+
118
+ # String types
119
+ Str = DataType.String
120
+ Text = DataType.String # Alias
121
+ Categorical = DataType.Categorical
122
+ Cat = DataType.Categorical # Short alias
123
+
124
+ # Date/time types
125
+ Date = DataType.Date
126
+ Datetime = DataType.Datetime
127
+ Time = DataType.Time
128
+ Duration = DataType.Duration
129
+
130
+ # Other types
131
+ Bool = DataType.Boolean
132
+ Bytes = DataType.Binary
133
+ List = DataType.List
134
+ Struct = DataType.Struct
135
+ Array = DataType.Array
136
+
137
+
138
+ # Type alias for better type hints
139
+ TypeSpec = Union[
140
+ TypeGroup,
141
+ DataType,
142
+ str,
143
+ List[Union[TypeGroup, DataType, str, type[pl.DataType], pl.DataType]],
144
+ type[pl.DataType],
145
+ pl.DataType
146
+ ]
@@ -0,0 +1,277 @@
1
+ # ui_components.py - Updated ColumnSelector
2
+
3
+ from typing import List, Optional, Any, Literal, Union, Type, Tuple, Dict
4
+
5
+ from pydantic import Field, BaseModel, computed_field
6
+
7
+ from flowfile_core.flowfile.node_designer._type_registry import normalize_type_spec
8
+ # Public API import
9
+ from flowfile_core.flowfile.node_designer.data_types import DataType, TypeSpec
10
+
11
+ InputType = Literal["text", "number", "secret", "array", "date", "boolean"]
12
+
13
+
14
+ def normalize_input_to_data_types(
15
+ v: Any
16
+ ) -> Union[Literal["ALL"], List[DataType]]:
17
+ """
18
+ Normalizes a wide variety of inputs to either 'ALL' or a sorted list of DataType enums.
19
+ This function is used as a Pydantic BeforeValidator.
20
+
21
+ Args:
22
+ v: The input value to normalize. Can be a string, a list of strings,
23
+ a DataType, a TypeGroup, or a list of those.
24
+
25
+ Returns:
26
+ Either the string "ALL" or a sorted list of unique DataType enums.
27
+ """
28
+ if v == "ALL":
29
+ return "ALL"
30
+ if isinstance(v, list) and all(isinstance(item, DataType) for item in v):
31
+ return v
32
+
33
+ normalized_set = normalize_type_spec(v)
34
+
35
+ if normalized_set == set(DataType):
36
+ return "ALL"
37
+
38
+ return sorted(list(normalized_set), key=lambda x: x.value)
39
+
40
+
41
+ class FlowfileInComponent(BaseModel):
42
+ """
43
+ Base class for all UI components in the node settings panel.
44
+
45
+ This class provides the common attributes and methods that all UI components share.
46
+ It's not meant to be used directly, but rather to be inherited by specific
47
+ component classes.
48
+ """
49
+ component_type: str = Field(..., description="Type of the UI component")
50
+ value: Any = None
51
+ label: Optional[str] = None
52
+ input_type: InputType
53
+
54
+ def set_value(self, value: Any):
55
+ """
56
+ Sets the value of the component, received from the frontend.
57
+
58
+ This method is used internally by the framework to populate the component's
59
+ value when a user interacts with the UI.
60
+
61
+ Args:
62
+ value: The new value for the component.
63
+
64
+ Returns:
65
+ The component instance with the updated value.
66
+ """
67
+ self.value = value
68
+ return self
69
+
70
+
71
+ class IncomingColumns:
72
+ """
73
+ A marker class used in `SingleSelect` and `MultiSelect` components.
74
+
75
+ When `options` is set to this class, the component will be dynamically
76
+ populated with the column names from the node's input dataframe.
77
+ This allows users to select from the available columns at runtime.
78
+
79
+ Example:
80
+ class MyNodeSettings(NodeSettings):
81
+ column_to_process = SingleSelect(
82
+ label="Select a column",
83
+ options=IncomingColumns
84
+ )
85
+ """
86
+ pass
87
+
88
+
89
+ class ColumnSelector(FlowfileInComponent):
90
+ """
91
+ A UI component that allows users to select one or more columns from the
92
+ input dataframe, with an optional filter based on column data types.
93
+
94
+ This is particularly useful when a node operation should only be applied
95
+ to columns of a specific type (e.g., numeric, string, date).
96
+ """
97
+ component_type: Literal["ColumnSelector"] = "ColumnSelector"
98
+ required: bool = False
99
+ multiple: bool = False
100
+ input_type: InputType = "text"
101
+
102
+ # Normalized output: either "ALL" or list of DataType enums
103
+ data_type_filter_input: TypeSpec = Field(
104
+ default="ALL",
105
+ alias="data_types",
106
+ repr=False,
107
+ exclude=True
108
+ )
109
+
110
+ class Config:
111
+ arbitrary_types_allowed = True
112
+
113
+ @computed_field
114
+ @property
115
+ def data_types_filter(self) -> Union[Literal["ALL"], List[DataType]]:
116
+ """
117
+ A computed field that normalizes the `data_type_filter_input` into a
118
+ standardized format for the frontend.
119
+ """
120
+ return normalize_input_to_data_types(self.data_type_filter_input)
121
+
122
+ def model_dump(self, **kwargs) -> dict:
123
+ """
124
+ Overrides the default `model_dump` to ensure `data_types` is in the
125
+ correct format for the frontend.
126
+ """
127
+ data = super().model_dump(**kwargs)
128
+ if 'data_types_filter' in data and data['data_types_filter'] != "ALL":
129
+ data['data_types'] = sorted([dt.value for dt in data['data_types_filter']])
130
+ return data
131
+
132
+
133
+ class TextInput(FlowfileInComponent):
134
+ """A standard text input field for capturing string values."""
135
+ component_type: Literal["TextInput"] = "TextInput"
136
+ default: Optional[str] = ""
137
+ placeholder: Optional[str] = ""
138
+ input_type: InputType = "text"
139
+
140
+ def __init__(self, **data):
141
+ super().__init__(**data)
142
+ if self.value is None and self.default is not None:
143
+ self.value = self.default
144
+
145
+
146
+ class NumericInput(FlowfileInComponent):
147
+ """A numeric input field with optional minimum and maximum value validation."""
148
+ component_type: Literal["NumericInput"] = "NumericInput"
149
+ default: Optional[float] = None
150
+ min_value: Optional[float] = None
151
+ max_value: Optional[float] = None
152
+ input_type: InputType = "number"
153
+
154
+ def __init__(self, **data):
155
+ super().__init__(**data)
156
+ if self.value is None and self.default is not None:
157
+ self.value = self.default
158
+
159
+
160
+ class ToggleSwitch(FlowfileInComponent):
161
+ """A boolean toggle switch, typically used for enabling or disabling a feature."""
162
+ component_type: Literal["ToggleSwitch"] = "ToggleSwitch"
163
+ default: bool = False
164
+ description: Optional[str] = None
165
+ input_type: InputType = "boolean"
166
+
167
+ def __init__(self, **data):
168
+ super().__init__(**data)
169
+ if self.value is None:
170
+ self.value = self.default
171
+
172
+ def __bool__(self):
173
+ """Allows the component instance to be evaluated as a boolean."""
174
+ return bool(self.value)
175
+
176
+
177
+ class SingleSelect(FlowfileInComponent):
178
+ """
179
+ A dropdown menu for selecting a single option from a list.
180
+
181
+ The options can be a static list of strings or tuples, or they can be
182
+ dynamically populated from the input dataframe's columns by using the
183
+ `IncomingColumns` marker.
184
+ """
185
+ component_type: Literal["SingleSelect"] = "SingleSelect"
186
+ options: Union[List[Union[str, Tuple[str, Any]]], Type[IncomingColumns]]
187
+ default: Optional[Any] = None
188
+ input_type: InputType = "text"
189
+
190
+ def __init__(self, **data):
191
+ super().__init__(**data)
192
+ if self.value is None and self.default is not None:
193
+ self.value = self.default
194
+
195
+
196
+ class MultiSelect(FlowfileInComponent):
197
+ """
198
+ A multi-select dropdown for choosing multiple options from a list.
199
+
200
+ Like `SingleSelect`, the options can be static or dynamically populated
201
+ from the input columns using the `IncomingColumns` marker.
202
+ """
203
+ component_type: Literal["MultiSelect"] = "MultiSelect"
204
+ options: Union[List[Union[str, Tuple[str, Any]]], Type[IncomingColumns]]
205
+ default: List[Any] = Field(default_factory=list)
206
+ input_type: InputType = "array"
207
+
208
+ def __init__(self, **data):
209
+ super().__init__(**data)
210
+ if self.value is None:
211
+ self.value = self.default if self.default else []
212
+
213
+
214
+ class Section(BaseModel):
215
+ """
216
+ A container for grouping related UI components in the node settings panel.
217
+
218
+ Sections help organize the UI by grouping components under a common title
219
+ and description. Components can be added as keyword arguments during
220
+ initialization or afterward.
221
+
222
+ Example:
223
+ main_section = Section(
224
+ title="Main Settings",
225
+ description="Configure the primary behavior of the node.",
226
+ my_text_input=TextInput(label="Enter a value")
227
+ )
228
+ """
229
+ title: Optional[str] = None
230
+ description: Optional[str] = None
231
+ hidden: bool = False
232
+
233
+ class Config:
234
+ extra = 'allow'
235
+ arbitrary_types_allowed = True
236
+
237
+ def __init__(self, **data):
238
+ """
239
+ Initialize a Section with components as keyword arguments.
240
+ """
241
+ super().__init__(**data)
242
+
243
+ def __call__(self, **kwargs) -> 'Section':
244
+ """
245
+ Allows adding components to the section after initialization.
246
+
247
+ This makes it possible to build up a section dynamically.
248
+ """
249
+ for key, value in kwargs.items():
250
+ setattr(self, key, value)
251
+ return self
252
+
253
+ def get_components(self) -> Dict[str, FlowfileInComponent]:
254
+ """
255
+ Get all FlowfileInComponent instances from the section.
256
+
257
+ This method collects all the UI components that have been added to the
258
+ section, whether as defined fields or as extra fields.
259
+
260
+ Returns:
261
+ A dictionary mapping component names to their instances.
262
+ """
263
+ components = {}
264
+
265
+ # Get from extra fields
266
+ for key, value in getattr(self, '__pydantic_extra__', {}).items():
267
+ if isinstance(value, FlowfileInComponent):
268
+ components[key] = value
269
+
270
+ # Get from defined fields (excluding metadata)
271
+ for field_name in self.model_fields:
272
+ if field_name not in {'title', 'description', 'hidden'}:
273
+ value = getattr(self, field_name, None)
274
+ if isinstance(value, FlowfileInComponent):
275
+ components[field_name] = value
276
+
277
+ return components
@@ -61,21 +61,25 @@ def calculate_fuzzy_match_schema(fm_input: transform_schema.FuzzyMatchInput,
61
61
  join_inputs=fm_input.left_select)
62
62
  _order_join_inputs_based_on_col_order(col_order=[col.column_name for col in right_schema],
63
63
  join_inputs=fm_input.right_select)
64
+ for column in fm_input.left_select.renames:
65
+ if column.join_key:
66
+ column.keep = True
67
+ for column in fm_input.right_select.renames:
68
+ if column.join_key:
69
+ column.keep = True
64
70
  left_schema_dict, right_schema_dict = ({ls.name: ls for ls in left_schema}, {rs.name: rs for rs in right_schema})
65
71
  fm_input.auto_rename()
66
-
67
72
  right_renames = {column.old_name: column.new_name for column in fm_input.right_select.renames}
68
73
  new_join_mapping = rename_fuzzy_right_mapping(fm_input.join_mapping, right_renames)
69
-
70
74
  output_schema = []
71
75
  for column in fm_input.left_select.renames:
72
76
  column_schema = left_schema_dict.get(column.old_name)
73
- if column_schema and column.keep:
77
+ if column_schema and (column.keep or column.join_key):
74
78
  output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
75
79
  example_values=column_schema.example_values))
76
80
  for column in fm_input.right_select.renames:
77
81
  column_schema = right_schema_dict.get(column.old_name)
78
- if column_schema and column.keep:
82
+ if column_schema and (column.keep or column.join_key):
79
83
  output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
80
84
  example_values=column_schema.example_values))
81
85
  set_name_in_fuzzy_mappings(new_join_mapping)
@@ -56,7 +56,6 @@ def cross_join(node_data: "NodeData") -> NodeData:
56
56
  ji = transform_schema.CrossJoinInput(left_select=node_data.main_input.columns,
57
57
  right_select=node_data.right_input.columns)
58
58
  ji.auto_rename()
59
- print(ji)
60
59
  node_data.setting_input = input_schema.NodeCrossJoin(flow_id=node_data.flow_id,
61
60
  node_id=node_data.node_id,
62
61
  cross_join_input=ji)
flowfile_core/main.py CHANGED
@@ -7,6 +7,8 @@ import uvicorn
7
7
  from fastapi import FastAPI
8
8
  from fastapi.middleware.cors import CORSMiddleware
9
9
 
10
+ from shared.storage_config import storage
11
+
10
12
  from flowfile_core import ServerRun
11
13
  from flowfile_core.configs.settings import (SERVER_HOST, SERVER_PORT, WORKER_HOST, WORKER_PORT, WORKER_URL,)
12
14
 
@@ -16,15 +18,16 @@ from flowfile_core.routes.routes import router
16
18
  from flowfile_core.routes.public import router as public_router
17
19
  from flowfile_core.routes.logs import router as logs_router
18
20
  from flowfile_core.routes.cloud_connections import router as cloud_connections_router
21
+ from flowfile_core.routes.user_defined_components import router as user_defined_components_router
19
22
 
20
23
  from flowfile_core.configs.flow_logger import clear_all_flow_logs
24
+ storage.cleanup_directories()
21
25
 
22
26
  os.environ["FLOWFILE_MODE"] = "electron"
23
27
 
24
28
  should_exit = False
25
29
  server_instance = None
26
30
 
27
-
28
31
  @asynccontextmanager
29
32
  async def shutdown_handler(app: FastAPI):
30
33
  """Handles the graceful startup and shutdown of the FastAPI application.
@@ -77,6 +80,7 @@ app.include_router(logs_router, tags=["logs"])
77
80
  app.include_router(auth_router, prefix="/auth", tags=["auth"])
78
81
  app.include_router(secrets_router, prefix="/secrets", tags=["secrets"])
79
82
  app.include_router(cloud_connections_router, prefix="/cloud_connections", tags=["cloud_connections"])
83
+ app.include_router(user_defined_components_router, prefix="/user_defined_components", tags=["user_defined_components"])
80
84
 
81
85
 
82
86
  @app.post("/shutdown")
@@ -12,49 +12,49 @@ import logging
12
12
  import os
13
13
  from pathlib import Path
14
14
  from typing import List, Dict, Any, Optional
15
- from sqlalchemy.orm import Session
16
15
 
17
16
  from fastapi import APIRouter, File, UploadFile, BackgroundTasks, HTTPException, status, Body, Depends
18
17
  from fastapi.responses import JSONResponse, Response
19
18
  # External dependencies
20
19
  from polars_expr_transformer.function_overview import get_all_expressions, get_expression_overview
20
+ from sqlalchemy.orm import Session
21
21
 
22
+ from flowfile_core import flow_file_handler
22
23
  # Core modules
23
24
  from flowfile_core.auth.jwt import get_current_active_user
24
25
  from flowfile_core.configs import logger
25
- from flowfile_core.configs.node_store import nodes
26
- from flowfile_core.configs.settings import IS_RUNNING_IN_DOCKER
26
+ from flowfile_core.configs.node_store import nodes_list, check_if_has_default_setting
27
+ from flowfile_core.database.connection import get_db
27
28
  # File handling
28
29
  from flowfile_core.fileExplorer.funcs import (
29
- FileExplorer,
30
+ SecureFileExplorer,
30
31
  FileInfo,
31
32
  get_files_from_directory
32
33
  )
33
- from flowfile_core.flowfile.flow_graph import add_connection, delete_connection
34
- from flowfile_core.flowfile.code_generator.code_generator import export_flow_to_polars
35
34
  from flowfile_core.flowfile.analytics.analytics_processor import AnalyticsProcessor
35
+ from flowfile_core.flowfile.code_generator.code_generator import export_flow_to_polars
36
+ from flowfile_core.flowfile.database_connection_manager.db_connections import (store_database_connection,
37
+ get_database_connection,
38
+ delete_database_connection,
39
+ get_all_database_connections_interface)
36
40
  from flowfile_core.flowfile.extensions import get_instant_func_results
37
- # Flow handling
38
-
41
+ from flowfile_core.flowfile.flow_graph import add_connection, delete_connection
39
42
  from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import create_sql_source_from_db_settings
40
43
  from flowfile_core.run_lock import get_flow_run_lock
41
- # Schema and models
42
44
  from flowfile_core.schemas import input_schema, schemas, output_model
43
45
  from flowfile_core.utils import excel_file_manager
44
- from flowfile_core.utils.fileManager import create_dir, remove_paths
46
+ from flowfile_core.utils.fileManager import create_dir
45
47
  from flowfile_core.utils.utils import camel_case_to_snake_case
46
- from flowfile_core import flow_file_handler
47
- from flowfile_core.flowfile.database_connection_manager.db_connections import (store_database_connection,
48
- get_database_connection,
49
- delete_database_connection,
50
- get_all_database_connections_interface)
51
- from flowfile_core.database.connection import get_db
48
+ from shared.storage_config import storage
52
49
 
53
50
 
54
51
  router = APIRouter(dependencies=[Depends(get_current_active_user)])
55
52
 
56
53
  # Initialize services
57
- file_explorer = FileExplorer('/app/shared' if IS_RUNNING_IN_DOCKER else None)
54
+ file_explorer = SecureFileExplorer(
55
+ start_path=storage.user_data_directory,
56
+ sandbox_root=storage.user_data_directory
57
+ )
58
58
 
59
59
 
60
60
  def get_node_model(setting_name_ref: str):
@@ -148,7 +148,7 @@ async def get_directory_contents(directory: str, file_types: List[str] = None,
148
148
  Returns:
149
149
  A list of `FileInfo` objects representing the directory's contents.
150
150
  """
151
- directory_explorer = FileExplorer(directory)
151
+ directory_explorer = SecureFileExplorer(directory, storage.user_data_directory)
152
152
  try:
153
153
  return directory_explorer.list_contents(show_hidden=include_hidden, file_types=file_types)
154
154
  except Exception as e:
@@ -198,6 +198,24 @@ async def get_active_flow_file_sessions() -> List[schemas.FlowSettings]:
198
198
  return [flf.flow_settings for flf in flow_file_handler.flowfile_flows]
199
199
 
200
200
 
201
+ @router.post("/node/trigger_fetch_data", tags=['editor'])
202
+ async def trigger_fetch_node_data(flow_id: int, node_id: int, background_tasks: BackgroundTasks):
203
+ """Fetches and refreshes the data for a specific node."""
204
+ flow = flow_file_handler.get_flow(flow_id)
205
+ lock = get_flow_run_lock(flow_id)
206
+ async with lock:
207
+ if flow.flow_settings.is_running:
208
+ raise HTTPException(422, 'Flow is already running')
209
+ try:
210
+ flow.validate_if_node_can_be_fetched(node_id)
211
+ except Exception as e:
212
+ raise HTTPException(422, str(e))
213
+ background_tasks.add_task(flow.trigger_fetch_node, node_id)
214
+ return JSONResponse(content={"message": "Data started",
215
+ "flow_id": flow_id,
216
+ "node_id": node_id}, status_code=status.HTTP_200_OK)
217
+
218
+
201
219
  @router.post('/flow/run/', tags=['editor'])
202
220
  async def run_flow(flow_id: int, background_tasks: BackgroundTasks) -> JSONResponse:
203
221
  """Executes a flow in a background task.
@@ -228,6 +246,16 @@ def cancel_flow(flow_id: int):
228
246
  flow.cancel()
229
247
 
230
248
 
249
+ @router.post("/flow/apply_standard_layout/", tags=["editor"])
250
+ def apply_standard_layout(flow_id: int):
251
+ flow = flow_file_handler.get_flow(flow_id)
252
+ if not flow:
253
+ raise HTTPException(status_code=404, detail="Flow not found")
254
+ if flow.flow_settings.is_running:
255
+ raise HTTPException(422, "Flow is running")
256
+ flow.apply_layout()
257
+
258
+
231
259
  @router.get('/flow/run_status/', tags=['editor'],
232
260
  response_model=output_model.RunInformation)
233
261
  def get_run_status(flow_id: int, response: Response):
@@ -238,10 +266,12 @@ def get_run_status(flow_id: int, response: Response):
238
266
  flow = flow_file_handler.get_flow(flow_id)
239
267
  if not flow:
240
268
  raise HTTPException(status_code=404, detail="Flow not found")
269
+ if flow.latest_run_info is None:
270
+ raise HTTPException(status_code=404, detail="No run information available")
241
271
  if flow.flow_settings.is_running:
242
272
  response.status_code = status.HTTP_202_ACCEPTED
243
- return flow.get_run_info()
244
- response.status_code = status.HTTP_200_OK
273
+ else:
274
+ response.status_code = status.HTTP_200_OK
245
275
  return flow.get_run_info()
246
276
 
247
277
 
@@ -324,7 +354,7 @@ def add_node(flow_id: int, node_id: int, node_type: str, pos_x: int = 0, pos_y:
324
354
  logger.info("Adding node")
325
355
  flow.add_node_promise(node_promise)
326
356
 
327
- if nodes.check_if_has_default_setting(node_type):
357
+ if check_if_has_default_setting(node_type):
328
358
  logger.info(f'Found standard settings for {node_type}, trying to upload them')
329
359
  setting_name_ref = 'node' + node_type.replace('_', '')
330
360
  node_model = get_node_model(setting_name_ref)
@@ -439,11 +469,24 @@ def get_generated_code(flow_id: int) -> str:
439
469
 
440
470
 
441
471
  @router.post('/editor/create_flow/', tags=['editor'])
442
- def create_flow(flow_path: str):
472
+ def create_flow(flow_path: str = None, name: str = None):
443
473
  """Creates a new, empty flow file at the specified path and registers a session for it."""
444
- flow_path = Path(flow_path)
445
- logger.info('Creating flow')
446
- return flow_file_handler.add_flow(name=flow_path.stem, flow_path=str(flow_path))
474
+ if flow_path is not None and name is None:
475
+ name = Path(flow_path).stem
476
+ elif flow_path is not None and name is not None:
477
+ if name not in flow_path and flow_path.endswith(".flowfile"):
478
+ raise HTTPException(422, 'The name must be part of the flow path when a full path is provided')
479
+ elif name in flow_path and not flow_path.endswith(".flowfile"):
480
+ flow_path = str(Path(flow_path) / (name + ".flowfile"))
481
+ elif name not in flow_path and name.endswith(".flowfile"):
482
+ flow_path = str(Path(flow_path) / name)
483
+ elif name not in flow_path and not name.endswith(".flowfile"):
484
+ flow_path = str(Path(flow_path) / (name + ".flowfile"))
485
+ if flow_path is not None:
486
+ flow_path_ref = Path(flow_path)
487
+ if not flow_path_ref.parent.exists():
488
+ raise HTTPException(422, 'The directory does not exist')
489
+ return flow_file_handler.add_flow(name=name, flow_path=flow_path)
447
490
 
448
491
 
449
492
  @router.post('/editor/close_flow/', tags=['editor'])
@@ -471,6 +514,7 @@ def add_generic_settings(input_data: Dict[str, Any], node_type: str, current_use
471
514
  add_func = getattr(flow, 'add_' + node_type)
472
515
  parsed_input = None
473
516
  setting_name_ref = 'node' + node_type.replace('_', '')
517
+
474
518
  if add_func is None:
475
519
  raise HTTPException(404, 'could not find the function')
476
520
  try:
@@ -496,10 +540,11 @@ def get_list_of_saved_flows(path: str):
496
540
  except:
497
541
  return []
498
542
 
499
- @router.get('/node_list', response_model=List[nodes.NodeTemplate])
500
- def get_node_list() -> List[nodes.NodeTemplate]:
543
+
544
+ @router.get('/node_list', response_model=List[schemas.NodeTemplate])
545
+ def get_node_list() -> List[schemas.NodeTemplate]:
501
546
  """Retrieves the list of all available node types and their templates."""
502
- return nodes.nodes_list
547
+ return nodes_list
503
548
 
504
549
 
505
550
  @router.get('/node', response_model=output_model.NodeData, tags=['editor'])