Flowfile 0.3.6__py3-none-any.whl → 0.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of Flowfile might be problematic. Click here for more details.

Files changed (98) hide show
  1. flowfile/__init__.py +27 -6
  2. flowfile/api.py +1 -0
  3. flowfile/web/__init__.py +2 -2
  4. flowfile/web/static/assets/{CloudConnectionManager-d004942f.js → CloudConnectionManager-c20a740f.js} +3 -4
  5. flowfile/web/static/assets/{CloudStorageReader-eccf9fc2.js → CloudStorageReader-960b400a.js} +7 -7
  6. flowfile/web/static/assets/{CloudStorageWriter-b1ba6bba.js → CloudStorageWriter-e3decbdd.js} +7 -7
  7. flowfile/web/static/assets/{CrossJoin-68981877.js → CrossJoin-d67e2405.js} +8 -8
  8. flowfile/web/static/assets/{DatabaseConnectionSettings-0b06649c.js → DatabaseConnectionSettings-a81e0f7e.js} +2 -2
  9. flowfile/web/static/assets/{DatabaseManager-8349a426.js → DatabaseManager-9ea35e84.js} +2 -2
  10. flowfile/web/static/assets/{DatabaseReader-905344f8.js → DatabaseReader-9578bfa5.js} +9 -9
  11. flowfile/web/static/assets/{DatabaseWriter-9f5b8638.js → DatabaseWriter-19531098.js} +9 -9
  12. flowfile/web/static/assets/{ExploreData-131a6d53.js → ExploreData-40476474.js} +47141 -43697
  13. flowfile/web/static/assets/{ExternalSource-e3549dcc.js → ExternalSource-2297ef96.js} +6 -6
  14. flowfile/web/static/assets/{Filter-6e0730ae.js → Filter-f211c03a.js} +8 -8
  15. flowfile/web/static/assets/{Formula-02f033e6.js → Formula-4207ea31.js} +8 -8
  16. flowfile/web/static/assets/{FuzzyMatch-54c14036.js → FuzzyMatch-bf120df0.js} +9 -9
  17. flowfile/web/static/assets/{GraphSolver-08a3f499.js → GraphSolver-5bb7497a.js} +5 -5
  18. flowfile/web/static/assets/{GroupBy-2ae38139.js → GroupBy-92c81b65.js} +6 -6
  19. flowfile/web/static/assets/{Join-493b9772.js → Join-4e49a274.js} +9 -9
  20. flowfile/web/static/assets/{ManualInput-4373d163.js → ManualInput-90998ae8.js} +5 -5
  21. flowfile/web/static/assets/{Output-b534f3c7.js → Output-81e3e917.js} +4 -4
  22. flowfile/web/static/assets/{Pivot-2968ff65.js → Pivot-a3419842.js} +6 -6
  23. flowfile/web/static/assets/{PolarsCode-65136536.js → PolarsCode-72710deb.js} +6 -6
  24. flowfile/web/static/assets/{Read-c56339ed.js → Read-c4059daf.js} +6 -6
  25. flowfile/web/static/assets/{RecordCount-1c641a5e.js → RecordCount-c2b5e095.js} +5 -5
  26. flowfile/web/static/assets/{RecordId-df308b8f.js → RecordId-10baf191.js} +6 -6
  27. flowfile/web/static/assets/{Sample-293e8a64.js → Sample-3ed9a0ae.js} +5 -5
  28. flowfile/web/static/assets/{SecretManager-03911655.js → SecretManager-0d49c0e8.js} +2 -2
  29. flowfile/web/static/assets/{Select-3058a13d.js → Select-8a02a0b3.js} +8 -8
  30. flowfile/web/static/assets/{SettingsSection-fbf4fb39.js → SettingsSection-4c0f45f5.js} +1 -1
  31. flowfile/web/static/assets/{Sort-a29bbaf7.js → Sort-f55c9f9d.js} +6 -6
  32. flowfile/web/static/assets/{TextToRows-c7d7760e.js → TextToRows-5dbc2145.js} +8 -8
  33. flowfile/web/static/assets/{UnavailableFields-118f1d20.js → UnavailableFields-a1768e52.js} +2 -2
  34. flowfile/web/static/assets/{Union-f0589571.js → Union-f2aefdc9.js} +5 -5
  35. flowfile/web/static/assets/{Unique-7329a207.js → Unique-46b250da.js} +8 -8
  36. flowfile/web/static/assets/{Unpivot-30b0be15.js → Unpivot-25ac84cc.js} +5 -5
  37. flowfile/web/static/assets/{api-fb67319c.js → api-6ef0dcef.js} +1 -1
  38. flowfile/web/static/assets/{api-602fb95c.js → api-a0abbdc7.js} +1 -1
  39. flowfile/web/static/assets/{designer-94a6bf4d.js → designer-13eabd83.js} +4 -4
  40. flowfile/web/static/assets/{documentation-a224831e.js → documentation-b87e7f6f.js} +1 -1
  41. flowfile/web/static/assets/{dropDown-c2d2aa97.js → dropDown-13564764.js} +1 -1
  42. flowfile/web/static/assets/{fullEditor-921ac5fd.js → fullEditor-fd2cd6f9.js} +2 -2
  43. flowfile/web/static/assets/{genericNodeSettings-7013cc94.js → genericNodeSettings-71e11604.js} +3 -3
  44. flowfile/web/static/assets/{index-3a75211d.js → index-f6c15e76.js} +46 -22
  45. flowfile/web/static/assets/{nodeTitle-a63d4680.js → nodeTitle-988d9efe.js} +3 -3
  46. flowfile/web/static/assets/{secretApi-763aec6e.js → secretApi-dd636aa2.js} +1 -1
  47. flowfile/web/static/assets/{selectDynamic-08464729.js → selectDynamic-af36165e.js} +3 -3
  48. flowfile/web/static/assets/{vue-codemirror.esm-f15a5f87.js → vue-codemirror.esm-2847001e.js} +1 -1
  49. flowfile/web/static/assets/{vue-content-loader.es-93bd09d7.js → vue-content-loader.es-0371da73.js} +1 -1
  50. flowfile/web/static/index.html +1 -1
  51. {flowfile-0.3.6.dist-info → flowfile-0.3.7.dist-info}/METADATA +2 -2
  52. {flowfile-0.3.6.dist-info → flowfile-0.3.7.dist-info}/RECORD +96 -94
  53. flowfile_core/__init__.py +1 -0
  54. flowfile_core/auth/jwt.py +39 -0
  55. flowfile_core/configs/node_store/nodes.py +1 -0
  56. flowfile_core/configs/settings.py +6 -5
  57. flowfile_core/flowfile/code_generator/code_generator.py +71 -0
  58. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +1 -1
  59. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +597 -309
  60. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +3 -1
  61. flowfile_core/flowfile/flow_graph.py +619 -191
  62. flowfile_core/flowfile/flow_graph_utils.py +2 -2
  63. flowfile_core/flowfile/flow_node/flow_node.py +500 -89
  64. flowfile_core/flowfile/flow_node/models.py +125 -20
  65. flowfile_core/flowfile/handler.py +2 -33
  66. flowfile_core/flowfile/manage/open_flowfile.py +1 -2
  67. flowfile_core/flowfile/util/calculate_layout.py +0 -2
  68. flowfile_core/flowfile/utils.py +36 -5
  69. flowfile_core/main.py +32 -13
  70. flowfile_core/routes/cloud_connections.py +7 -11
  71. flowfile_core/routes/logs.py +2 -6
  72. flowfile_core/routes/public.py +1 -0
  73. flowfile_core/routes/routes.py +127 -51
  74. flowfile_core/routes/secrets.py +72 -14
  75. flowfile_core/schemas/__init__.py +8 -0
  76. flowfile_core/schemas/input_schema.py +92 -64
  77. flowfile_core/schemas/output_model.py +19 -3
  78. flowfile_core/schemas/schemas.py +144 -11
  79. flowfile_core/schemas/transform_schema.py +82 -17
  80. flowfile_frame/__init__.py +9 -1
  81. flowfile_frame/cloud_storage/__init__.py +0 -0
  82. flowfile_frame/cloud_storage/frame_helpers.py +39 -0
  83. flowfile_frame/cloud_storage/secret_manager.py +73 -0
  84. flowfile_frame/expr.py +28 -1
  85. flowfile_frame/expr.pyi +76 -61
  86. flowfile_frame/flow_frame.py +232 -110
  87. flowfile_frame/flow_frame.pyi +140 -91
  88. flowfile_frame/flow_frame_methods.py +150 -12
  89. flowfile_frame/group_frame.py +3 -0
  90. flowfile_frame/utils.py +25 -3
  91. test_utils/s3/data_generator.py +1 -0
  92. test_utils/s3/demo_data_generator.py +186 -0
  93. test_utils/s3/fixtures.py +6 -1
  94. flowfile_core/schemas/defaults.py +0 -9
  95. flowfile_core/schemas/models.py +0 -193
  96. {flowfile-0.3.6.dist-info → flowfile-0.3.7.dist-info}/LICENSE +0 -0
  97. {flowfile-0.3.6.dist-info → flowfile-0.3.7.dist-info}/WHEEL +0 -0
  98. {flowfile-0.3.6.dist-info → flowfile-0.3.7.dist-info}/entry_points.txt +0 -0
@@ -1,3 +1,11 @@
1
+ """
2
+ Main API router and endpoint definitions for the Flowfile application.
3
+
4
+ This module sets up the FastAPI router, defines all the API endpoints for interacting
5
+ with flows, nodes, files, and other core components of the application. It handles
6
+ the logic for creating, reading, updating, and deleting these resources.
7
+ """
8
+
1
9
  import asyncio
2
10
  import inspect
3
11
  import logging
@@ -43,7 +51,6 @@ from flowfile_core.flowfile.database_connection_manager.db_connections import (s
43
51
  from flowfile_core.database.connection import get_db
44
52
 
45
53
 
46
-
47
54
  router = APIRouter(dependencies=[Depends(get_current_active_user)])
48
55
 
49
56
  # Initialize services
@@ -51,6 +58,7 @@ file_explorer = FileExplorer('/app/shared' if IS_RUNNING_IN_DOCKER else None)
51
58
 
52
59
 
53
60
  def get_node_model(setting_name_ref: str):
61
+ """(Internal) Retrieves a node's Pydantic model from the input_schema module by its name."""
54
62
  logger.info("Getting node model for: " + setting_name_ref)
55
63
  for ref_name, ref in inspect.getmodule(input_schema).__dict__.items():
56
64
  if ref_name.lower() == setting_name_ref:
@@ -59,7 +67,15 @@ def get_node_model(setting_name_ref: str):
59
67
 
60
68
 
61
69
  @router.post("/upload/")
62
- async def upload_file(file: UploadFile = File(...)):
70
+ async def upload_file(file: UploadFile = File(...)) -> JSONResponse:
71
+ """Uploads a file to the server's 'uploads' directory.
72
+
73
+ Args:
74
+ file: The file to be uploaded.
75
+
76
+ Returns:
77
+ A JSON response containing the filename and the path where it was saved.
78
+ """
63
79
  file_location = f"uploads/{file.filename}"
64
80
  with open(file_location, "wb+") as file_object:
65
81
  file_object.write(file.file.read())
@@ -68,6 +84,17 @@ async def upload_file(file: UploadFile = File(...)):
68
84
 
69
85
  @router.get('/files/files_in_local_directory/', response_model=List[FileInfo], tags=['file manager'])
70
86
  async def get_local_files(directory: str) -> List[FileInfo]:
87
+ """Retrieves a list of files from a specified local directory.
88
+
89
+ Args:
90
+ directory: The absolute path of the directory to scan.
91
+
92
+ Returns:
93
+ A list of `FileInfo` objects for each item in the directory.
94
+
95
+ Raises:
96
+ HTTPException: 404 if the directory does not exist.
97
+ """
71
98
  files = get_files_from_directory(directory)
72
99
  if files is None:
73
100
  raise HTTPException(404, 'Directory does not exist')
@@ -76,36 +103,51 @@ async def get_local_files(directory: str) -> List[FileInfo]:
76
103
 
77
104
  @router.get('/files/tree/', response_model=List[FileInfo], tags=['file manager'])
78
105
  async def get_current_files() -> List[FileInfo]:
106
+ """Gets the contents of the file explorer's current directory."""
79
107
  f = file_explorer.list_contents()
80
108
  return f
81
109
 
82
110
 
83
111
  @router.post('/files/navigate_up/', response_model=str, tags=['file manager'])
84
112
  async def navigate_up() -> str:
113
+ """Navigates the file explorer one directory level up."""
85
114
  file_explorer.navigate_up()
86
115
  return str(file_explorer.current_path)
87
116
 
88
117
 
89
118
  @router.post('/files/navigate_into/', response_model=str, tags=['file manager'])
90
119
  async def navigate_into_directory(directory_name: str) -> str:
120
+ """Navigates the file explorer into a specified subdirectory."""
91
121
  file_explorer.navigate_into(directory_name)
92
122
  return str(file_explorer.current_path)
93
123
 
94
124
 
95
125
  @router.post('/files/navigate_to/', tags=['file manager'])
96
126
  async def navigate_to_directory(directory_name: str) -> str:
127
+ """Navigates the file explorer to an absolute directory path."""
97
128
  file_explorer.navigate_to(directory_name)
98
129
  return str(file_explorer.current_path)
99
130
 
100
131
 
101
132
  @router.get('/files/current_path/', response_model=str, tags=['file manager'])
102
133
  async def get_current_path() -> str:
134
+ """Returns the current absolute path of the file explorer."""
103
135
  return str(file_explorer.current_path)
104
136
 
105
137
 
106
138
  @router.get('/files/directory_contents/', response_model=List[FileInfo], tags=['file manager'])
107
139
  async def get_directory_contents(directory: str, file_types: List[str] = None,
108
140
  include_hidden: bool = False) -> List[FileInfo]:
141
+ """Gets the contents of an arbitrary directory path.
142
+
143
+ Args:
144
+ directory: The absolute path to the directory.
145
+ file_types: An optional list of file extensions to filter by.
146
+ include_hidden: If True, includes hidden files and directories.
147
+
148
+ Returns:
149
+ A list of `FileInfo` objects representing the directory's contents.
150
+ """
109
151
  directory_explorer = FileExplorer(directory)
110
152
  try:
111
153
  return directory_explorer.list_contents(show_hidden=include_hidden, file_types=file_types)
@@ -116,11 +158,20 @@ async def get_directory_contents(directory: str, file_types: List[str] = None,
116
158
 
117
159
  @router.get('/files/current_directory_contents/', response_model=List[FileInfo], tags=['file manager'])
118
160
  async def get_current_directory_contents(file_types: List[str] = None, include_hidden: bool = False) -> List[FileInfo]:
161
+ """Gets the contents of the file explorer's current directory."""
119
162
  return file_explorer.list_contents(file_types=file_types, show_hidden=include_hidden)
120
163
 
121
164
 
122
165
  @router.post('/files/create_directory', response_model=output_model.OutputDir, tags=['file manager'])
123
166
  def create_directory(new_directory: input_schema.NewDirectory) -> bool:
167
+ """Creates a new directory at the specified path.
168
+
169
+ Args:
170
+ new_directory: An `input_schema.NewDirectory` object with the path and name.
171
+
172
+ Returns:
173
+ `True` if the directory was created successfully.
174
+ """
124
175
  result, error = create_dir(new_directory)
125
176
  if result:
126
177
  return True
@@ -129,17 +180,35 @@ def create_directory(new_directory: input_schema.NewDirectory) -> bool:
129
180
 
130
181
 
131
182
  @router.post('/flow/register/', tags=['editor'])
132
- def register_flow(flow_data: schemas.FlowSettings):
183
+ def register_flow(flow_data: schemas.FlowSettings) -> int:
184
+ """Registers a new flow session with the application.
185
+
186
+ Args:
187
+ flow_data: The `FlowSettings` for the new flow.
188
+
189
+ Returns:
190
+ The ID of the newly registered flow.
191
+ """
133
192
  return flow_file_handler.register_flow(flow_data)
134
193
 
135
194
 
136
195
  @router.get('/active_flowfile_sessions/', response_model=List[schemas.FlowSettings])
137
196
  async def get_active_flow_file_sessions() -> List[schemas.FlowSettings]:
197
+ """Retrieves a list of all currently active flow sessions."""
138
198
  return [flf.flow_settings for flf in flow_file_handler.flowfile_flows]
139
199
 
140
200
 
141
201
  @router.post('/flow/run/', tags=['editor'])
142
- async def run_flow(flow_id: int, background_tasks: BackgroundTasks):
202
+ async def run_flow(flow_id: int, background_tasks: BackgroundTasks) -> JSONResponse:
203
+ """Executes a flow in a background task.
204
+
205
+ Args:
206
+ flow_id: The ID of the flow to execute.
207
+ background_tasks: FastAPI's background task runner.
208
+
209
+ Returns:
210
+ A JSON response indicating that the flow has started.
211
+ """
143
212
  logger.info('starting to run...')
144
213
  flow = flow_file_handler.get_flow(flow_id)
145
214
  lock = get_flow_run_lock(flow_id)
@@ -147,11 +216,12 @@ async def run_flow(flow_id: int, background_tasks: BackgroundTasks):
147
216
  if flow.flow_settings.is_running:
148
217
  raise HTTPException(422, 'Flow is already running')
149
218
  background_tasks.add_task(flow.run_graph)
150
- JSONResponse(content={"message": "Data started", "flow_id": flow_id}, status_code=status.HTTP_202_ACCEPTED)
219
+ return JSONResponse(content={"message": "Data started", "flow_id": flow_id}, status_code=status.HTTP_200_OK)
151
220
 
152
221
 
153
222
  @router.post('/flow/cancel/', tags=['editor'])
154
223
  def cancel_flow(flow_id: int):
224
+ """Cancels a currently running flow execution."""
155
225
  flow = flow_file_handler.get_flow(flow_id)
156
226
  if not flow.flow_settings.is_running:
157
227
  raise HTTPException(422, 'Flow is not running')
@@ -161,6 +231,10 @@ def cancel_flow(flow_id: int):
161
231
  @router.get('/flow/run_status/', tags=['editor'],
162
232
  response_model=output_model.RunInformation)
163
233
  def get_run_status(flow_id: int, response: Response):
234
+ """Retrieves the run status information for a specific flow.
235
+
236
+ Returns a 202 Accepted status while the flow is running, and 200 OK when finished.
237
+ """
164
238
  flow = flow_file_handler.get_flow(flow_id)
165
239
  if not flow:
166
240
  raise HTTPException(status_code=404, detail="Flow not found")
@@ -189,15 +263,12 @@ def add_flow_input(input_data: input_schema.NodeDatasource):
189
263
 
190
264
  @router.post('/editor/copy_node', tags=['editor'])
191
265
  def copy_node(node_id_to_copy_from: int, flow_id_to_copy_from: int, node_promise: input_schema.NodePromise):
192
- """
193
- Add a node to the flow.
194
- Parameters
195
- ----------
196
- node_id_to_copy_from: int, the id of the node to copy
197
- flow_id_to_copy_from: int, the id of the flow to copy from
198
- node_promise: NodePromise, the node promise that contains all the data
199
- Returns
200
- -------
266
+ """Copies an existing node's settings to a new node promise.
267
+
268
+ Args:
269
+ node_id_to_copy_from: The ID of the node to copy the settings from.
270
+ flow_id_to_copy_from: The ID of the flow containing the source node.
271
+ node_promise: A `NodePromise` representing the new node to be created.
201
272
  """
202
273
  try:
203
274
  flow_to_copy_from = flow_file_handler.get_flow(flow_id_to_copy_from)
@@ -227,19 +298,14 @@ def copy_node(node_id_to_copy_from: int, flow_id_to_copy_from: int, node_promise
227
298
 
228
299
  @router.post('/editor/add_node/', tags=['editor'])
229
300
  def add_node(flow_id: int, node_id: int, node_type: str, pos_x: int = 0, pos_y: int = 0):
230
- """
231
- Add a node to the flow.
232
- Parameters
233
- ----------
234
- flow_id: int, the flow id
235
- node_id: int, the node id
236
- node_type: str, the node type
237
- pos_x: int, the x position of the node
238
- pos_y: int, the y position of the node
239
-
240
- Returns
241
- -------
242
-
301
+ """Adds a new, unconfigured node (a "promise") to the flow graph.
302
+
303
+ Args:
304
+ flow_id: The ID of the flow to add the node to.
305
+ node_id: The client-generated ID for the new node.
306
+ node_type: The type of the node to add (e.g., 'filter', 'join').
307
+ pos_x: The X coordinate for the node's position in the UI.
308
+ pos_y: The Y coordinate for the node's position in the UI.
243
309
  """
244
310
  flow = flow_file_handler.get_flow(flow_id)
245
311
  logger.info(f'Adding a promise for {node_type}')
@@ -270,6 +336,7 @@ def add_node(flow_id: int, node_id: int, node_type: str, pos_x: int = 0, pos_y:
270
336
 
271
337
  @router.post('/editor/delete_node/', tags=['editor'])
272
338
  def delete_node(flow_id: Optional[int], node_id: int):
339
+ """Deletes a node from the flow graph."""
273
340
  logger.info('Deleting node')
274
341
  flow = flow_file_handler.get_flow(flow_id)
275
342
  if flow.flow_settings.is_running:
@@ -279,6 +346,7 @@ def delete_node(flow_id: Optional[int], node_id: int):
279
346
 
280
347
  @router.post('/editor/delete_connection/', tags=['editor'])
281
348
  def delete_node_connection(flow_id: int, node_connection: input_schema.NodeConnection = None):
349
+ """Deletes a connection (edge) between two nodes."""
282
350
  flow_id = int(flow_id)
283
351
  logger.info(
284
352
  f'Deleting connection node {node_connection.output_connection.node_id} to node {node_connection.input_connection.node_id}')
@@ -293,9 +361,7 @@ def create_db_connection(input_connection: input_schema.FullDatabaseConnection,
293
361
  current_user=Depends(get_current_active_user),
294
362
  db: Session = Depends(get_db)
295
363
  ):
296
- """
297
- Create a database connection.
298
- """
364
+ """Creates and securely stores a new database connection."""
299
365
  logger.info(f'Creating database connection {input_connection.connection_name}')
300
366
  try:
301
367
  store_database_connection(db, input_connection, current_user.id)
@@ -312,9 +378,7 @@ def delete_db_connection(connection_name: str,
312
378
  current_user=Depends(get_current_active_user),
313
379
  db: Session = Depends(get_db)
314
380
  ):
315
- """
316
- Delete a database connection.
317
- """
381
+ """Deletes a stored database connection."""
318
382
  logger.info(f'Deleting database connection {connection_name}')
319
383
  db_connection = get_database_connection(db, connection_name, current_user.id)
320
384
  if db_connection is None:
@@ -328,11 +392,13 @@ def delete_db_connection(connection_name: str,
328
392
  def get_db_connections(
329
393
  db: Session = Depends(get_db),
330
394
  current_user=Depends(get_current_active_user)) -> List[input_schema.FullDatabaseConnectionInterface]:
395
+ """Retrieves all stored database connections for the current user (without passwords)."""
331
396
  return get_all_database_connections_interface(db, current_user.id)
332
397
 
333
398
 
334
399
  @router.post('/editor/connect_node/', tags=['editor'])
335
400
  def connect_node(flow_id: int, node_connection: input_schema.NodeConnection):
401
+ """Creates a connection (edge) between two nodes in the flow graph."""
336
402
  flow = flow_file_handler.get_flow(flow_id)
337
403
  if flow is None:
338
404
  logger.info('could not find the flow')
@@ -344,16 +410,19 @@ def connect_node(flow_id: int, node_connection: input_schema.NodeConnection):
344
410
 
345
411
  @router.get('/editor/expression_doc', tags=['editor'], response_model=List[output_model.ExpressionsOverview])
346
412
  def get_expression_doc() -> List[output_model.ExpressionsOverview]:
413
+ """Retrieves documentation for available Polars expressions."""
347
414
  return get_expression_overview()
348
415
 
349
416
 
350
417
  @router.get('/editor/expressions', tags=['editor'], response_model=List[str])
351
418
  def get_expressions() -> List[str]:
419
+ """Retrieves a list of all available Flowfile expression names."""
352
420
  return get_all_expressions()
353
421
 
354
422
 
355
423
  @router.get('/editor/flow', tags=['editor'], response_model=schemas.FlowSettings)
356
424
  def get_flow(flow_id: int):
425
+ """Retrieves the settings for a specific flow."""
357
426
  flow_id = int(flow_id)
358
427
  result = get_flow_settings(flow_id)
359
428
  return result
@@ -361,6 +430,7 @@ def get_flow(flow_id: int):
361
430
 
362
431
  @router.get("/editor/code_to_polars", tags=[], response_model=str)
363
432
  def get_generated_code(flow_id: int) -> str:
433
+ """Generates and returns a Python script with Polars code representing the flow."""
364
434
  flow_id = int(flow_id)
365
435
  flow = flow_file_handler.get_flow(flow_id)
366
436
  if flow is None:
@@ -370,6 +440,7 @@ def get_generated_code(flow_id: int) -> str:
370
440
 
371
441
  @router.post('/editor/create_flow/', tags=['editor'])
372
442
  def create_flow(flow_path: str):
443
+ """Creates a new, empty flow file at the specified path and registers a session for it."""
373
444
  flow_path = Path(flow_path)
374
445
  logger.info('Creating flow')
375
446
  return flow_file_handler.add_flow(name=flow_path.stem, flow_path=str(flow_path))
@@ -377,11 +448,17 @@ def create_flow(flow_path: str):
377
448
 
378
449
  @router.post('/editor/close_flow/', tags=['editor'])
379
450
  def close_flow(flow_id: int) -> None:
451
+ """Closes an active flow session."""
380
452
  flow_file_handler.delete_flow(flow_id)
381
453
 
382
454
 
383
455
  @router.post('/update_settings/', tags=['transform'])
384
456
  def add_generic_settings(input_data: Dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)):
457
+ """A generic endpoint to update the settings of any node.
458
+
459
+ This endpoint dynamically determines the correct Pydantic model and update
460
+ function based on the `node_type` parameter.
461
+ """
385
462
  input_data['user_id'] = current_user.id
386
463
  node_type = camel_case_to_snake_case(node_type)
387
464
  flow_id = int(input_data.get('flow_id'))
@@ -413,6 +490,7 @@ def add_generic_settings(input_data: Dict[str, Any], node_type: str, current_use
413
490
 
414
491
  @router.get('/files/available_flow_files', tags=['editor'], response_model=List[FileInfo])
415
492
  def get_list_of_saved_flows(path: str):
493
+ """Scans a directory for saved flow files (`.flowfile`)."""
416
494
  try:
417
495
  return get_files_from_directory(path, types=['flowfile'])
418
496
  except:
@@ -420,26 +498,13 @@ def get_list_of_saved_flows(path: str):
420
498
 
421
499
  @router.get('/node_list', response_model=List[nodes.NodeTemplate])
422
500
  def get_node_list() -> List[nodes.NodeTemplate]:
501
+ """Retrieves the list of all available node types and their templates."""
423
502
  return nodes.nodes_list
424
503
 
425
504
 
426
- # @router.post('/reset')
427
- # def reset():
428
- # flow_file_handler.delete_flow(1)
429
- # register_flow(schemas.FlowSettings(flow_id=1))
430
-
431
-
432
- @router.post('/files/remove_items', tags=['file manager'])
433
- def remove_items(remove_items_input: input_schema.RemoveItemsInput):
434
- result, error = remove_paths(remove_items_input)
435
- if result:
436
- return result
437
- else:
438
- raise error
439
-
440
-
441
505
  @router.get('/node', response_model=output_model.NodeData, tags=['editor'])
442
506
  def get_node(flow_id: int, node_id: int, get_data: bool = False):
507
+ """Retrieves the complete state and data preview for a single node."""
443
508
  logging.info(f'Getting node {node_id} from flow {flow_id}')
444
509
  flow = flow_file_handler.get_flow(flow_id)
445
510
  node = flow.get_node(node_id)
@@ -451,6 +516,7 @@ def get_node(flow_id: int, node_id: int, get_data: bool = False):
451
516
 
452
517
  @router.post('/node/description/', tags=['editor'])
453
518
  def update_description_node(flow_id: int, node_id: int, description: str = Body(...)):
519
+ """Updates the description text for a specific node."""
454
520
  try:
455
521
  node = flow_file_handler.get_flow(flow_id).get_node(node_id)
456
522
  except:
@@ -461,6 +527,7 @@ def update_description_node(flow_id: int, node_id: int, description: str = Body(
461
527
 
462
528
  @router.get('/node/description', tags=['editor'])
463
529
  def get_description_node(flow_id: int, node_id: int):
530
+ """Retrieves the description text for a specific node."""
464
531
  try:
465
532
  node = flow_file_handler.get_flow(flow_id).get_node(node_id)
466
533
  except:
@@ -472,6 +539,7 @@ def get_description_node(flow_id: int, node_id: int):
472
539
 
473
540
  @router.get('/node/data', response_model=output_model.TableExample, tags=['editor'])
474
541
  def get_table_example(flow_id: int, node_id: int):
542
+ """Retrieves a data preview (schema and sample rows) for a node's output."""
475
543
  flow = flow_file_handler.get_flow(flow_id)
476
544
  node = flow.get_node(node_id)
477
545
  return node.get_table_example(True)
@@ -479,6 +547,7 @@ def get_table_example(flow_id: int, node_id: int):
479
547
 
480
548
  @router.get('/node/downstream_node_ids', response_model=List[int], tags=['editor'])
481
549
  async def get_downstream_node_ids(flow_id: int, node_id: int) -> List[int]:
550
+ """Gets a list of all node IDs that are downstream dependencies of a given node."""
482
551
  flow = flow_file_handler.get_flow(flow_id)
483
552
  node = flow.get_node(node_id)
484
553
  return list(node.get_all_dependent_node_ids())
@@ -486,6 +555,7 @@ async def get_downstream_node_ids(flow_id: int, node_id: int) -> List[int]:
486
555
 
487
556
  @router.get('/import_flow/', tags=['editor'], response_model=int)
488
557
  def import_saved_flow(flow_path: str) -> int:
558
+ """Imports a flow from a saved `.flowfile` and registers it as a new session."""
489
559
  flow_path = Path(flow_path)
490
560
  if not flow_path.exists():
491
561
  raise HTTPException(404, 'File not found')
@@ -494,12 +564,14 @@ def import_saved_flow(flow_path: str) -> int:
494
564
 
495
565
  @router.get('/save_flow', tags=['editor'])
496
566
  def save_flow(flow_id: int, flow_path: str = None):
567
+ """Saves the current state of a flow to a `.flowfile`."""
497
568
  flow = flow_file_handler.get_flow(flow_id)
498
569
  flow.save_flow(flow_path=flow_path)
499
570
 
500
571
 
501
572
  @router.get('/flow_data', tags=['manager'])
502
573
  def get_flow_frontend_data(flow_id: Optional[int] = 1):
574
+ """Retrieves the data needed to render the flow graph in the frontend."""
503
575
  flow = flow_file_handler.get_flow(flow_id)
504
576
  if flow is None:
505
577
  raise HTTPException(404, 'could not find the flow')
@@ -508,6 +580,7 @@ def get_flow_frontend_data(flow_id: Optional[int] = 1):
508
580
 
509
581
  @router.get('/flow_settings', tags=['manager'], response_model=schemas.FlowSettings)
510
582
  def get_flow_settings(flow_id: Optional[int] = 1) -> schemas.FlowSettings:
583
+ """Retrieves the main settings for a flow."""
511
584
  flow = flow_file_handler.get_flow(flow_id)
512
585
  if flow is None:
513
586
  raise HTTPException(404, 'could not find the flow')
@@ -516,6 +589,7 @@ def get_flow_settings(flow_id: Optional[int] = 1) -> schemas.FlowSettings:
516
589
 
517
590
  @router.post('/flow_settings', tags=['manager'])
518
591
  def update_flow_settings(flow_settings: schemas.FlowSettings):
592
+ """Updates the main settings for a flow."""
519
593
  flow = flow_file_handler.get_flow(flow_settings.flow_id)
520
594
  if flow is None:
521
595
  raise HTTPException(404, 'could not find the flow')
@@ -524,6 +598,7 @@ def update_flow_settings(flow_settings: schemas.FlowSettings):
524
598
 
525
599
  @router.get('/flow_data/v2', tags=['manager'])
526
600
  def get_vue_flow_data(flow_id: int) -> schemas.VueFlowInput:
601
+ """Retrieves the flow data formatted for the Vue-based frontend."""
527
602
  flow = flow_file_handler.get_flow(flow_id)
528
603
  if flow is None:
529
604
  raise HTTPException(404, 'could not find the flow')
@@ -533,6 +608,7 @@ def get_vue_flow_data(flow_id: int) -> schemas.VueFlowInput:
533
608
 
534
609
  @router.get('/analysis_data/graphic_walker_input', tags=['analysis'], response_model=input_schema.NodeExploreData)
535
610
  def get_graphic_walker_input(flow_id: int, node_id: int):
611
+ """Gets the data and configuration for the Graphic Walker data exploration tool."""
536
612
  flow = flow_file_handler.get_flow(flow_id)
537
613
  node = flow.get_node(node_id)
538
614
  if node.results.analysis_data_generator is None:
@@ -543,6 +619,7 @@ def get_graphic_walker_input(flow_id: int, node_id: int):
543
619
 
544
620
  @router.get('/custom_functions/instant_result', tags=[])
545
621
  async def get_instant_function_result(flow_id: int, node_id: int, func_string: str):
622
+ """Executes a simple, instant function on a node's data and returns the result."""
546
623
  try:
547
624
  node = flow_file_handler.get_node(flow_id, node_id)
548
625
  result = await asyncio.to_thread(get_instant_func_results, node, func_string)
@@ -553,6 +630,7 @@ async def get_instant_function_result(flow_id: int, node_id: int, func_string: s
553
630
 
554
631
  @router.get('/api/get_xlsx_sheet_names', tags=['excel_reader'], response_model=List[str])
555
632
  async def get_excel_sheet_names(path: str) -> List[str] | None:
633
+ """Retrieves the sheet names from an Excel file."""
556
634
  sheet_names = excel_file_manager.get_sheet_names(path)
557
635
  if sheet_names:
558
636
  return sheet_names
@@ -565,9 +643,7 @@ async def validate_db_settings(
565
643
  database_settings: input_schema.DatabaseSettings,
566
644
  current_user=Depends(get_current_active_user)
567
645
  ):
568
- """
569
- Validate the query settings for a database connection.
570
- """
646
+ """Validates that a connection can be made to a database with the given settings."""
571
647
  # Validate the query settings
572
648
  try:
573
649
  sql_source = create_sql_source_from_db_settings(database_settings, user_id=current_user.id)
@@ -1,5 +1,10 @@
1
- # app_routes/secrets.py
1
+ """
2
+ Manages CRUD (Create, Read, Update, Delete) operations for secrets.
2
3
 
4
+ This router provides secure endpoints for creating, retrieving, and deleting
5
+ sensitive credentials for the authenticated user. Secrets are encrypted before
6
+ being stored and are associated with the user's ID.
7
+ """
3
8
  import os
4
9
  from typing import List
5
10
 
@@ -10,20 +15,31 @@ from flowfile_core.auth.jwt import get_current_active_user
10
15
  from flowfile_core.auth.models import Secret, SecretInput
11
16
  from flowfile_core.database import models as db_models
12
17
  from flowfile_core.database.connection import get_db
13
- from flowfile_core.secret_manager.secret_manager import encrypt_secret, store_secret, delete_secret as delete_secret_action
18
+ from flowfile_core.secret_manager.secret_manager import store_secret, delete_secret as delete_secret_action
14
19
 
15
20
  router = APIRouter(dependencies=[Depends(get_current_active_user)])
16
21
 
17
22
 
18
- # Get all secrets for current user
19
23
  @router.get("/secrets", response_model=List[Secret])
20
24
  async def get_secrets(current_user=Depends(get_current_active_user), db: Session = Depends(get_db)):
25
+ """Retrieves all secret names for the currently authenticated user.
26
+
27
+ Note: This endpoint returns the secret names and metadata but does not
28
+ expose the decrypted secret values.
29
+
30
+ Args:
31
+ current_user: The authenticated user object, injected by FastAPI.
32
+ db: The database session, injected by FastAPI.
33
+
34
+ Returns:
35
+ A list of `Secret` objects, each containing the name and encrypted value.
36
+ """
21
37
  user_id = current_user.id
22
38
 
23
39
  # Get secrets from database
24
40
  db_secrets = db.query(db_models.Secret).filter(db_models.Secret.user_id == user_id).all()
25
41
 
26
- # Decrypt secrets
42
+ # Prepare response model (without decrypting)
27
43
  secrets = []
28
44
  for db_secret in db_secrets:
29
45
  secrets.append(Secret(
@@ -35,13 +51,27 @@ async def get_secrets(current_user=Depends(get_current_active_user), db: Session
35
51
  return secrets
36
52
 
37
53
 
38
- # Create a new secret
39
54
  @router.post("/secrets", response_model=Secret)
40
55
  async def create_secret(secret: SecretInput, current_user=Depends(get_current_active_user),
41
- db: Session = Depends(get_db)):
42
- print('current_user', current_user)
56
+ db: Session = Depends(get_db)) -> Secret:
57
+ """Creates a new secret for the authenticated user.
58
+
59
+ The secret value is encrypted before being stored in the database. A secret
60
+ name must be unique for a given user.
61
+
62
+ Args:
63
+ secret: A `SecretInput` object containing the name and plaintext value of the secret.
64
+ current_user: The authenticated user object, injected by FastAPI.
65
+ db: The database session, injected by FastAPI.
66
+
67
+ Raises:
68
+ HTTPException: 400 if a secret with the same name already exists for the user.
69
+
70
+ Returns:
71
+ A `Secret` object containing the name and the *encrypted* value.
72
+ """
43
73
  # Get user ID
44
- user_id = 1 if os.environ.get("FLOWFILE_MODE") == "electron" or 1 == 1 else current_user.id
74
+ user_id = 1 if os.environ.get("FLOWFILE_MODE") == "electron" else current_user.id
45
75
 
46
76
  existing_secret = db.query(db_models.Secret).filter(
47
77
  db_models.Secret.user_id == user_id,
@@ -51,13 +81,30 @@ async def create_secret(secret: SecretInput, current_user=Depends(get_current_ac
51
81
  if existing_secret:
52
82
  raise HTTPException(status_code=400, detail="Secret with this name already exists")
53
83
 
54
- encrypted_value = store_secret(db, secret, user_id).encrypted_value
55
- return Secret(name=secret.name, value=encrypted_value, user_id=str(user_id))
84
+ # The store_secret function handles encryption and DB storage
85
+ stored_secret = store_secret(db, secret, user_id)
86
+ return Secret(name=stored_secret.name, value=stored_secret.encrypted_value, user_id=str(user_id))
56
87
 
57
88
 
58
- # Get a specific secret by name
59
89
  @router.get("/secrets/{secret_name}", response_model=Secret)
60
- async def get_secret(secret_name: str, current_user=Depends(get_current_active_user), db: Session = Depends(get_db)):
90
+ async def get_secret(secret_name: str,
91
+ current_user=Depends(get_current_active_user), db: Session = Depends(get_db)) -> Secret:
92
+ """Retrieves a specific secret by name for the authenticated user.
93
+
94
+ Note: This endpoint returns the secret name and metadata but does not
95
+ expose the decrypted secret value.
96
+
97
+ Args:
98
+ secret_name: The name of the secret to retrieve.
99
+ current_user: The authenticated user object, injected by FastAPI.
100
+ db: The database session, injected by FastAPI.
101
+
102
+ Raises:
103
+ HTTPException: 404 if the secret is not found.
104
+
105
+ Returns:
106
+ A `Secret` object containing the name and encrypted value.
107
+ """
61
108
  # Get user ID
62
109
  user_id = 1 if os.environ.get("FLOWFILE_MODE") == "electron" else current_user.id
63
110
 
@@ -78,8 +125,19 @@ async def get_secret(secret_name: str, current_user=Depends(get_current_active_u
78
125
 
79
126
 
80
127
  @router.delete("/secrets/{secret_name}", status_code=204)
81
- async def delete_secret(secret_name: str, current_user=Depends(get_current_active_user), db: Session = Depends(get_db)):
128
+ async def delete_secret(secret_name: str, current_user=Depends(get_current_active_user),
129
+ db: Session = Depends(get_db)) -> None:
130
+ """Deletes a secret by name for the authenticated user.
131
+
132
+ Args:
133
+ secret_name: The name of the secret to delete.
134
+ current_user: The authenticated user object, injected by FastAPI.
135
+ db: The database session, injected by FastAPI.
136
+
137
+ Returns:
138
+ An empty response with a 204 No Content status code upon success.
139
+ """
82
140
  # Get user ID
83
- user_id = 1 if os.environ.get("FLOWFILE_MODE") == "electron" or 1 == 1 else current_user.id
141
+ user_id = 1 if os.environ.get("FLOWFILE_MODE") == "electron" else current_user.id
84
142
  delete_secret_action(db, secret_name, user_id)
85
143
  return None
@@ -0,0 +1,8 @@
1
+ from flowfile_core.schemas import input_schema as node_interface, transform_schema as transformation_settings
2
+ from flowfile_core.schemas.schemas import FlowSettings, FlowInformation
3
+ from flowfile_core.schemas.input_schema import RawData
4
+
5
+
6
+ __all__ = [
7
+ "transformation_settings", "node_interface", "FlowSettings", "FlowInformation", "RawData"
8
+ ]