Flowfile 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of Flowfile might be problematic. Click here for more details.

Files changed (145) hide show
  1. flowfile/__init__.py +27 -6
  2. flowfile/api.py +1 -0
  3. flowfile/web/__init__.py +2 -2
  4. flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +86 -0
  5. flowfile/web/static/assets/CloudConnectionManager-c20a740f.js +783 -0
  6. flowfile/web/static/assets/CloudStorageReader-29d14fcc.css +143 -0
  7. flowfile/web/static/assets/CloudStorageReader-960b400a.js +437 -0
  8. flowfile/web/static/assets/CloudStorageWriter-49c9a4b2.css +138 -0
  9. flowfile/web/static/assets/CloudStorageWriter-e3decbdd.js +430 -0
  10. flowfile/web/static/assets/{CrossJoin-dfcf7351.js → CrossJoin-d67e2405.js} +8 -8
  11. flowfile/web/static/assets/{DatabaseConnectionSettings-b2afb1d7.js → DatabaseConnectionSettings-a81e0f7e.js} +2 -2
  12. flowfile/web/static/assets/{DatabaseManager-824a49b2.js → DatabaseManager-9ea35e84.js} +2 -2
  13. flowfile/web/static/assets/{DatabaseReader-a48124d8.js → DatabaseReader-9578bfa5.js} +9 -9
  14. flowfile/web/static/assets/{DatabaseWriter-b47cbae2.js → DatabaseWriter-19531098.js} +9 -9
  15. flowfile/web/static/assets/{ExploreData-fdfc45a4.js → ExploreData-40476474.js} +47141 -43697
  16. flowfile/web/static/assets/{ExternalSource-861b0e71.js → ExternalSource-2297ef96.js} +6 -6
  17. flowfile/web/static/assets/{Filter-f87bb897.js → Filter-f211c03a.js} +8 -8
  18. flowfile/web/static/assets/{Formula-b8cefc31.css → Formula-29f19d21.css} +10 -0
  19. flowfile/web/static/assets/{Formula-1e2ed720.js → Formula-4207ea31.js} +75 -9
  20. flowfile/web/static/assets/{FuzzyMatch-b6cc4fdd.js → FuzzyMatch-bf120df0.js} +9 -9
  21. flowfile/web/static/assets/{GraphSolver-6a371f4c.js → GraphSolver-5bb7497a.js} +5 -5
  22. flowfile/web/static/assets/{GroupBy-f7b7f472.js → GroupBy-92c81b65.js} +6 -6
  23. flowfile/web/static/assets/{Join-eec38203.js → Join-4e49a274.js} +23 -15
  24. flowfile/web/static/assets/{Join-41c0f331.css → Join-f45eff22.css} +20 -20
  25. flowfile/web/static/assets/{ManualInput-9aaa46fb.js → ManualInput-90998ae8.js} +106 -34
  26. flowfile/web/static/assets/{ManualInput-ac7b9972.css → ManualInput-a71b52c6.css} +29 -17
  27. flowfile/web/static/assets/{Output-3b2ca045.js → Output-81e3e917.js} +4 -4
  28. flowfile/web/static/assets/{Pivot-a4f5d88f.js → Pivot-a3419842.js} +6 -6
  29. flowfile/web/static/assets/{PolarsCode-49ce444f.js → PolarsCode-72710deb.js} +6 -6
  30. flowfile/web/static/assets/{Read-07acdc9a.js → Read-c4059daf.js} +6 -6
  31. flowfile/web/static/assets/{RecordCount-6a21da56.js → RecordCount-c2b5e095.js} +5 -5
  32. flowfile/web/static/assets/{RecordId-949bdc17.js → RecordId-10baf191.js} +6 -6
  33. flowfile/web/static/assets/{Sample-7afca6e1.js → Sample-3ed9a0ae.js} +5 -5
  34. flowfile/web/static/assets/{SecretManager-b41c029d.js → SecretManager-0d49c0e8.js} +2 -2
  35. flowfile/web/static/assets/{Select-32b28406.js → Select-8a02a0b3.js} +8 -8
  36. flowfile/web/static/assets/{SettingsSection-a0f15a05.js → SettingsSection-4c0f45f5.js} +1 -1
  37. flowfile/web/static/assets/{Sort-fc6ba0e2.js → Sort-f55c9f9d.js} +6 -6
  38. flowfile/web/static/assets/{TextToRows-23127596.js → TextToRows-5dbc2145.js} +8 -8
  39. flowfile/web/static/assets/{UnavailableFields-c42880a3.js → UnavailableFields-a1768e52.js} +2 -2
  40. flowfile/web/static/assets/{Union-39eecc6c.js → Union-f2aefdc9.js} +5 -5
  41. flowfile/web/static/assets/{Unique-a0e8fe61.js → Unique-46b250da.js} +8 -8
  42. flowfile/web/static/assets/{Unpivot-1e2d43f0.js → Unpivot-25ac84cc.js} +5 -5
  43. flowfile/web/static/assets/api-6ef0dcef.js +80 -0
  44. flowfile/web/static/assets/{api-44ca9e9c.js → api-a0abbdc7.js} +1 -1
  45. flowfile/web/static/assets/cloud_storage_reader-aa1415d6.png +0 -0
  46. flowfile/web/static/assets/{designer-267d44f1.js → designer-13eabd83.js} +36 -34
  47. flowfile/web/static/assets/{documentation-6c0810a2.js → documentation-b87e7f6f.js} +1 -1
  48. flowfile/web/static/assets/{dropDown-52790b15.js → dropDown-13564764.js} +1 -1
  49. flowfile/web/static/assets/{fullEditor-e272b506.js → fullEditor-fd2cd6f9.js} +2 -2
  50. flowfile/web/static/assets/{genericNodeSettings-4bdcf98e.js → genericNodeSettings-71e11604.js} +3 -3
  51. flowfile/web/static/assets/{index-e235a8bc.js → index-f6c15e76.js} +59 -22
  52. flowfile/web/static/assets/{nodeTitle-fc3fc4b7.js → nodeTitle-988d9efe.js} +3 -3
  53. flowfile/web/static/assets/{secretApi-cdc2a3fd.js → secretApi-dd636aa2.js} +1 -1
  54. flowfile/web/static/assets/{selectDynamic-96aa82cd.js → selectDynamic-af36165e.js} +3 -3
  55. flowfile/web/static/assets/{vue-codemirror.esm-25e75a08.js → vue-codemirror.esm-2847001e.js} +2 -1
  56. flowfile/web/static/assets/{vue-content-loader.es-6c4b1c24.js → vue-content-loader.es-0371da73.js} +1 -1
  57. flowfile/web/static/index.html +1 -1
  58. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/METADATA +9 -4
  59. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/RECORD +131 -124
  60. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/entry_points.txt +2 -0
  61. flowfile_core/__init__.py +3 -0
  62. flowfile_core/auth/jwt.py +39 -0
  63. flowfile_core/configs/node_store/nodes.py +9 -6
  64. flowfile_core/configs/settings.py +6 -5
  65. flowfile_core/database/connection.py +63 -15
  66. flowfile_core/database/init_db.py +0 -1
  67. flowfile_core/database/models.py +49 -2
  68. flowfile_core/flowfile/code_generator/code_generator.py +472 -17
  69. flowfile_core/flowfile/connection_manager/models.py +1 -1
  70. flowfile_core/flowfile/database_connection_manager/db_connections.py +216 -2
  71. flowfile_core/flowfile/extensions.py +1 -1
  72. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +259 -0
  73. flowfile_core/flowfile/flow_data_engine/create/funcs.py +19 -8
  74. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +1062 -311
  75. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +12 -2
  76. flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py +1 -1
  77. flowfile_core/flowfile/flow_data_engine/join/__init__.py +2 -1
  78. flowfile_core/flowfile/flow_data_engine/join/utils.py +25 -0
  79. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +3 -1
  80. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +29 -22
  81. flowfile_core/flowfile/flow_data_engine/utils.py +1 -40
  82. flowfile_core/flowfile/flow_graph.py +718 -253
  83. flowfile_core/flowfile/flow_graph_utils.py +2 -2
  84. flowfile_core/flowfile/flow_node/flow_node.py +563 -117
  85. flowfile_core/flowfile/flow_node/models.py +154 -20
  86. flowfile_core/flowfile/flow_node/schema_callback.py +3 -2
  87. flowfile_core/flowfile/handler.py +2 -33
  88. flowfile_core/flowfile/manage/open_flowfile.py +1 -2
  89. flowfile_core/flowfile/sources/external_sources/__init__.py +0 -2
  90. flowfile_core/flowfile/sources/external_sources/factory.py +4 -7
  91. flowfile_core/flowfile/util/calculate_layout.py +0 -2
  92. flowfile_core/flowfile/utils.py +35 -26
  93. flowfile_core/main.py +35 -15
  94. flowfile_core/routes/cloud_connections.py +77 -0
  95. flowfile_core/routes/logs.py +2 -7
  96. flowfile_core/routes/public.py +1 -0
  97. flowfile_core/routes/routes.py +130 -90
  98. flowfile_core/routes/secrets.py +72 -14
  99. flowfile_core/schemas/__init__.py +8 -0
  100. flowfile_core/schemas/cloud_storage_schemas.py +215 -0
  101. flowfile_core/schemas/input_schema.py +121 -71
  102. flowfile_core/schemas/output_model.py +19 -3
  103. flowfile_core/schemas/schemas.py +150 -12
  104. flowfile_core/schemas/transform_schema.py +175 -35
  105. flowfile_core/utils/utils.py +40 -1
  106. flowfile_core/utils/validate_setup.py +41 -0
  107. flowfile_frame/__init__.py +9 -1
  108. flowfile_frame/cloud_storage/frame_helpers.py +39 -0
  109. flowfile_frame/cloud_storage/secret_manager.py +73 -0
  110. flowfile_frame/expr.py +28 -1
  111. flowfile_frame/expr.pyi +76 -61
  112. flowfile_frame/flow_frame.py +481 -208
  113. flowfile_frame/flow_frame.pyi +140 -91
  114. flowfile_frame/flow_frame_methods.py +160 -22
  115. flowfile_frame/group_frame.py +3 -0
  116. flowfile_frame/utils.py +25 -3
  117. flowfile_worker/external_sources/s3_source/main.py +216 -0
  118. flowfile_worker/external_sources/s3_source/models.py +142 -0
  119. flowfile_worker/funcs.py +51 -6
  120. flowfile_worker/models.py +22 -2
  121. flowfile_worker/routes.py +40 -38
  122. flowfile_worker/utils.py +1 -1
  123. test_utils/s3/commands.py +46 -0
  124. test_utils/s3/data_generator.py +292 -0
  125. test_utils/s3/demo_data_generator.py +186 -0
  126. test_utils/s3/fixtures.py +214 -0
  127. flowfile/web/static/assets/AirbyteReader-1ac35765.css +0 -314
  128. flowfile/web/static/assets/AirbyteReader-e08044e5.js +0 -922
  129. flowfile/web/static/assets/dropDownGeneric-60f56a8a.js +0 -72
  130. flowfile/web/static/assets/dropDownGeneric-895680d6.css +0 -10
  131. flowfile_core/flowfile/sources/external_sources/airbyte_sources/airbyte.py +0 -159
  132. flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py +0 -172
  133. flowfile_core/flowfile/sources/external_sources/airbyte_sources/settings.py +0 -173
  134. flowfile_core/schemas/defaults.py +0 -9
  135. flowfile_core/schemas/external_sources/airbyte_schemas.py +0 -20
  136. flowfile_core/schemas/models.py +0 -193
  137. flowfile_worker/external_sources/airbyte_sources/cache_manager.py +0 -161
  138. flowfile_worker/external_sources/airbyte_sources/main.py +0 -89
  139. flowfile_worker/external_sources/airbyte_sources/models.py +0 -133
  140. flowfile_worker/external_sources/airbyte_sources/settings.py +0 -0
  141. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/LICENSE +0 -0
  142. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/WHEEL +0 -0
  143. {flowfile_core/flowfile/sources/external_sources/airbyte_sources → flowfile_frame/cloud_storage}/__init__.py +0 -0
  144. {flowfile_core/schemas/external_sources → flowfile_worker/external_sources/s3_source}/__init__.py +0 -0
  145. {flowfile_worker/external_sources/airbyte_sources → test_utils/s3}/__init__.py +0 -0
@@ -33,9 +33,7 @@ async def format_sse_message(data: str) -> str:
33
33
 
34
34
  @router.post("/logs/{flow_id}", tags=['flow_logging'])
35
35
  async def add_log(flow_id: int, log_message: str):
36
- """
37
- Adds a log message to the log file for a given flow_id.
38
- """
36
+ """Adds a log message to the log file for a given flow_id."""
39
37
  flow = flow_file_handler.get_flow(flow_id)
40
38
  if not flow:
41
39
  raise HTTPException(status_code=404, detail="Flow not found")
@@ -45,9 +43,7 @@ async def add_log(flow_id: int, log_message: str):
45
43
 
46
44
  @router.post("/raw_logs", tags=['flow_logging'])
47
45
  async def add_raw_log(raw_log_input: schemas.RawLogInput):
48
- """
49
- Adds a log message to the log file for a given flow_id.
50
- """
46
+ """Adds a log message to the log file for a given flow_id."""
51
47
  logger.info('Adding raw logs')
52
48
  flow = flow_file_handler.get_flow(raw_log_input.flowfile_flow_id)
53
49
  if not flow:
@@ -86,7 +82,6 @@ async def stream_log_file(
86
82
  line = await file.readline()
87
83
  if line:
88
84
  formatted_message = await format_sse_message(line.strip())
89
- logger.info(f'Yielding line: {line.strip()}')
90
85
  yield formatted_message
91
86
  last_active = time.monotonic() # Reset idle timer on activity
92
87
  else:
@@ -7,4 +7,5 @@ router = APIRouter()
7
7
 
8
8
  @router.get("/", tags=['admin'])
9
9
  async def docs_redirect():
10
+ """ Redirects to the documentation page."""
10
11
  return RedirectResponse(url='/docs')
@@ -1,3 +1,11 @@
1
+ """
2
+ Main API router and endpoint definitions for the Flowfile application.
3
+
4
+ This module sets up the FastAPI router, defines all the API endpoints for interacting
5
+ with flows, nodes, files, and other core components of the application. It handles
6
+ the logic for creating, reading, updating, and deleting these resources.
7
+ """
8
+
1
9
  import asyncio
2
10
  import inspect
3
11
  import logging
@@ -27,12 +35,7 @@ from flowfile_core.flowfile.code_generator.code_generator import export_flow_to_
27
35
  from flowfile_core.flowfile.analytics.analytics_processor import AnalyticsProcessor
28
36
  from flowfile_core.flowfile.extensions import get_instant_func_results
29
37
  # Flow handling
30
- from flowfile_core.flowfile.sources.external_sources.airbyte_sources.models import AirbyteConfigTemplate
31
- # Airbyte
32
- from flowfile_core.flowfile.sources.external_sources.airbyte_sources.settings import (
33
- airbyte_config_handler,
34
- AirbyteHandler
35
- )
38
+
36
39
  from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import create_sql_source_from_db_settings
37
40
  from flowfile_core.run_lock import get_flow_run_lock
38
41
  # Schema and models
@@ -48,8 +51,6 @@ from flowfile_core.flowfile.database_connection_manager.db_connections import (s
48
51
  from flowfile_core.database.connection import get_db
49
52
 
50
53
 
51
-
52
- # Router setup
53
54
  router = APIRouter(dependencies=[Depends(get_current_active_user)])
54
55
 
55
56
  # Initialize services
@@ -57,13 +58,24 @@ file_explorer = FileExplorer('/app/shared' if IS_RUNNING_IN_DOCKER else None)
57
58
 
58
59
 
59
60
  def get_node_model(setting_name_ref: str):
61
+ """(Internal) Retrieves a node's Pydantic model from the input_schema module by its name."""
62
+ logger.info("Getting node model for: " + setting_name_ref)
60
63
  for ref_name, ref in inspect.getmodule(input_schema).__dict__.items():
61
64
  if ref_name.lower() == setting_name_ref:
62
65
  return ref
66
+ logger.error(f"Could not find node model for: {setting_name_ref}")
63
67
 
64
68
 
65
69
  @router.post("/upload/")
66
- async def upload_file(file: UploadFile = File(...)):
70
+ async def upload_file(file: UploadFile = File(...)) -> JSONResponse:
71
+ """Uploads a file to the server's 'uploads' directory.
72
+
73
+ Args:
74
+ file: The file to be uploaded.
75
+
76
+ Returns:
77
+ A JSON response containing the filename and the path where it was saved.
78
+ """
67
79
  file_location = f"uploads/{file.filename}"
68
80
  with open(file_location, "wb+") as file_object:
69
81
  file_object.write(file.file.read())
@@ -72,6 +84,17 @@ async def upload_file(file: UploadFile = File(...)):
72
84
 
73
85
  @router.get('/files/files_in_local_directory/', response_model=List[FileInfo], tags=['file manager'])
74
86
  async def get_local_files(directory: str) -> List[FileInfo]:
87
+ """Retrieves a list of files from a specified local directory.
88
+
89
+ Args:
90
+ directory: The absolute path of the directory to scan.
91
+
92
+ Returns:
93
+ A list of `FileInfo` objects for each item in the directory.
94
+
95
+ Raises:
96
+ HTTPException: 404 if the directory does not exist.
97
+ """
75
98
  files = get_files_from_directory(directory)
76
99
  if files is None:
77
100
  raise HTTPException(404, 'Directory does not exist')
@@ -80,36 +103,51 @@ async def get_local_files(directory: str) -> List[FileInfo]:
80
103
 
81
104
  @router.get('/files/tree/', response_model=List[FileInfo], tags=['file manager'])
82
105
  async def get_current_files() -> List[FileInfo]:
106
+ """Gets the contents of the file explorer's current directory."""
83
107
  f = file_explorer.list_contents()
84
108
  return f
85
109
 
86
110
 
87
111
  @router.post('/files/navigate_up/', response_model=str, tags=['file manager'])
88
112
  async def navigate_up() -> str:
113
+ """Navigates the file explorer one directory level up."""
89
114
  file_explorer.navigate_up()
90
115
  return str(file_explorer.current_path)
91
116
 
92
117
 
93
118
  @router.post('/files/navigate_into/', response_model=str, tags=['file manager'])
94
119
  async def navigate_into_directory(directory_name: str) -> str:
120
+ """Navigates the file explorer into a specified subdirectory."""
95
121
  file_explorer.navigate_into(directory_name)
96
122
  return str(file_explorer.current_path)
97
123
 
98
124
 
99
125
  @router.post('/files/navigate_to/', tags=['file manager'])
100
126
  async def navigate_to_directory(directory_name: str) -> str:
127
+ """Navigates the file explorer to an absolute directory path."""
101
128
  file_explorer.navigate_to(directory_name)
102
129
  return str(file_explorer.current_path)
103
130
 
104
131
 
105
132
  @router.get('/files/current_path/', response_model=str, tags=['file manager'])
106
133
  async def get_current_path() -> str:
134
+ """Returns the current absolute path of the file explorer."""
107
135
  return str(file_explorer.current_path)
108
136
 
109
137
 
110
138
  @router.get('/files/directory_contents/', response_model=List[FileInfo], tags=['file manager'])
111
139
  async def get_directory_contents(directory: str, file_types: List[str] = None,
112
140
  include_hidden: bool = False) -> List[FileInfo]:
141
+ """Gets the contents of an arbitrary directory path.
142
+
143
+ Args:
144
+ directory: The absolute path to the directory.
145
+ file_types: An optional list of file extensions to filter by.
146
+ include_hidden: If True, includes hidden files and directories.
147
+
148
+ Returns:
149
+ A list of `FileInfo` objects representing the directory's contents.
150
+ """
113
151
  directory_explorer = FileExplorer(directory)
114
152
  try:
115
153
  return directory_explorer.list_contents(show_hidden=include_hidden, file_types=file_types)
@@ -120,11 +158,20 @@ async def get_directory_contents(directory: str, file_types: List[str] = None,
120
158
 
121
159
  @router.get('/files/current_directory_contents/', response_model=List[FileInfo], tags=['file manager'])
122
160
  async def get_current_directory_contents(file_types: List[str] = None, include_hidden: bool = False) -> List[FileInfo]:
161
+ """Gets the contents of the file explorer's current directory."""
123
162
  return file_explorer.list_contents(file_types=file_types, show_hidden=include_hidden)
124
163
 
125
164
 
126
165
  @router.post('/files/create_directory', response_model=output_model.OutputDir, tags=['file manager'])
127
166
  def create_directory(new_directory: input_schema.NewDirectory) -> bool:
167
+ """Creates a new directory at the specified path.
168
+
169
+ Args:
170
+ new_directory: An `input_schema.NewDirectory` object with the path and name.
171
+
172
+ Returns:
173
+ `True` if the directory was created successfully.
174
+ """
128
175
  result, error = create_dir(new_directory)
129
176
  if result:
130
177
  return True
@@ -133,17 +180,35 @@ def create_directory(new_directory: input_schema.NewDirectory) -> bool:
133
180
 
134
181
 
135
182
  @router.post('/flow/register/', tags=['editor'])
136
- def register_flow(flow_data: schemas.FlowSettings):
183
+ def register_flow(flow_data: schemas.FlowSettings) -> int:
184
+ """Registers a new flow session with the application.
185
+
186
+ Args:
187
+ flow_data: The `FlowSettings` for the new flow.
188
+
189
+ Returns:
190
+ The ID of the newly registered flow.
191
+ """
137
192
  return flow_file_handler.register_flow(flow_data)
138
193
 
139
194
 
140
195
  @router.get('/active_flowfile_sessions/', response_model=List[schemas.FlowSettings])
141
196
  async def get_active_flow_file_sessions() -> List[schemas.FlowSettings]:
197
+ """Retrieves a list of all currently active flow sessions."""
142
198
  return [flf.flow_settings for flf in flow_file_handler.flowfile_flows]
143
199
 
144
200
 
145
201
  @router.post('/flow/run/', tags=['editor'])
146
- async def run_flow(flow_id: int, background_tasks: BackgroundTasks):
202
+ async def run_flow(flow_id: int, background_tasks: BackgroundTasks) -> JSONResponse:
203
+ """Executes a flow in a background task.
204
+
205
+ Args:
206
+ flow_id: The ID of the flow to execute.
207
+ background_tasks: FastAPI's background task runner.
208
+
209
+ Returns:
210
+ A JSON response indicating that the flow has started.
211
+ """
147
212
  logger.info('starting to run...')
148
213
  flow = flow_file_handler.get_flow(flow_id)
149
214
  lock = get_flow_run_lock(flow_id)
@@ -151,11 +216,12 @@ async def run_flow(flow_id: int, background_tasks: BackgroundTasks):
151
216
  if flow.flow_settings.is_running:
152
217
  raise HTTPException(422, 'Flow is already running')
153
218
  background_tasks.add_task(flow.run_graph)
154
- JSONResponse(content={"message": "Data started", "flow_id": flow_id}, status_code=status.HTTP_202_ACCEPTED)
219
+ return JSONResponse(content={"message": "Data started", "flow_id": flow_id}, status_code=status.HTTP_200_OK)
155
220
 
156
221
 
157
222
  @router.post('/flow/cancel/', tags=['editor'])
158
223
  def cancel_flow(flow_id: int):
224
+ """Cancels a currently running flow execution."""
159
225
  flow = flow_file_handler.get_flow(flow_id)
160
226
  if not flow.flow_settings.is_running:
161
227
  raise HTTPException(422, 'Flow is not running')
@@ -165,6 +231,10 @@ def cancel_flow(flow_id: int):
165
231
  @router.get('/flow/run_status/', tags=['editor'],
166
232
  response_model=output_model.RunInformation)
167
233
  def get_run_status(flow_id: int, response: Response):
234
+ """Retrieves the run status information for a specific flow.
235
+
236
+ Returns a 202 Accepted status while the flow is running, and 200 OK when finished.
237
+ """
168
238
  flow = flow_file_handler.get_flow(flow_id)
169
239
  if not flow:
170
240
  raise HTTPException(status_code=404, detail="Flow not found")
@@ -193,15 +263,12 @@ def add_flow_input(input_data: input_schema.NodeDatasource):
193
263
 
194
264
  @router.post('/editor/copy_node', tags=['editor'])
195
265
  def copy_node(node_id_to_copy_from: int, flow_id_to_copy_from: int, node_promise: input_schema.NodePromise):
196
- """
197
- Add a node to the flow.
198
- Parameters
199
- ----------
200
- node_id_to_copy_from: int, the id of the node to copy
201
- flow_id_to_copy_from: int, the id of the flow to copy from
202
- node_promise: NodePromise, the node promise that contains all the data
203
- Returns
204
- -------
266
+ """Copies an existing node's settings to a new node promise.
267
+
268
+ Args:
269
+ node_id_to_copy_from: The ID of the node to copy the settings from.
270
+ flow_id_to_copy_from: The ID of the flow containing the source node.
271
+ node_promise: A `NodePromise` representing the new node to be created.
205
272
  """
206
273
  try:
207
274
  flow_to_copy_from = flow_file_handler.get_flow(flow_id_to_copy_from)
@@ -231,19 +298,14 @@ def copy_node(node_id_to_copy_from: int, flow_id_to_copy_from: int, node_promise
231
298
 
232
299
  @router.post('/editor/add_node/', tags=['editor'])
233
300
  def add_node(flow_id: int, node_id: int, node_type: str, pos_x: int = 0, pos_y: int = 0):
234
- """
235
- Add a node to the flow.
236
- Parameters
237
- ----------
238
- flow_id: int, the flow id
239
- node_id: int, the node id
240
- node_type: str, the node type
241
- pos_x: int, the x position of the node
242
- pos_y: int, the y position of the node
243
-
244
- Returns
245
- -------
246
-
301
+ """Adds a new, unconfigured node (a "promise") to the flow graph.
302
+
303
+ Args:
304
+ flow_id: The ID of the flow to add the node to.
305
+ node_id: The client-generated ID for the new node.
306
+ node_type: The type of the node to add (e.g., 'filter', 'join').
307
+ pos_x: The X coordinate for the node's position in the UI.
308
+ pos_y: The Y coordinate for the node's position in the UI.
247
309
  """
248
310
  flow = flow_file_handler.get_flow(flow_id)
249
311
  logger.info(f'Adding a promise for {node_type}')
@@ -274,6 +336,7 @@ def add_node(flow_id: int, node_id: int, node_type: str, pos_x: int = 0, pos_y:
274
336
 
275
337
  @router.post('/editor/delete_node/', tags=['editor'])
276
338
  def delete_node(flow_id: Optional[int], node_id: int):
339
+ """Deletes a node from the flow graph."""
277
340
  logger.info('Deleting node')
278
341
  flow = flow_file_handler.get_flow(flow_id)
279
342
  if flow.flow_settings.is_running:
@@ -283,6 +346,7 @@ def delete_node(flow_id: Optional[int], node_id: int):
283
346
 
284
347
  @router.post('/editor/delete_connection/', tags=['editor'])
285
348
  def delete_node_connection(flow_id: int, node_connection: input_schema.NodeConnection = None):
349
+ """Deletes a connection (edge) between two nodes."""
286
350
  flow_id = int(flow_id)
287
351
  logger.info(
288
352
  f'Deleting connection node {node_connection.output_connection.node_id} to node {node_connection.input_connection.node_id}')
@@ -297,9 +361,7 @@ def create_db_connection(input_connection: input_schema.FullDatabaseConnection,
297
361
  current_user=Depends(get_current_active_user),
298
362
  db: Session = Depends(get_db)
299
363
  ):
300
- """
301
- Create a database connection.
302
- """
364
+ """Creates and securely stores a new database connection."""
303
365
  logger.info(f'Creating database connection {input_connection.connection_name}')
304
366
  try:
305
367
  store_database_connection(db, input_connection, current_user.id)
@@ -316,9 +378,7 @@ def delete_db_connection(connection_name: str,
316
378
  current_user=Depends(get_current_active_user),
317
379
  db: Session = Depends(get_db)
318
380
  ):
319
- """
320
- Delete a database connection.
321
- """
381
+ """Deletes a stored database connection."""
322
382
  logger.info(f'Deleting database connection {connection_name}')
323
383
  db_connection = get_database_connection(db, connection_name, current_user.id)
324
384
  if db_connection is None:
@@ -332,11 +392,13 @@ def delete_db_connection(connection_name: str,
332
392
  def get_db_connections(
333
393
  db: Session = Depends(get_db),
334
394
  current_user=Depends(get_current_active_user)) -> List[input_schema.FullDatabaseConnectionInterface]:
395
+ """Retrieves all stored database connections for the current user (without passwords)."""
335
396
  return get_all_database_connections_interface(db, current_user.id)
336
397
 
337
398
 
338
399
  @router.post('/editor/connect_node/', tags=['editor'])
339
400
  def connect_node(flow_id: int, node_connection: input_schema.NodeConnection):
401
+ """Creates a connection (edge) between two nodes in the flow graph."""
340
402
  flow = flow_file_handler.get_flow(flow_id)
341
403
  if flow is None:
342
404
  logger.info('could not find the flow')
@@ -348,16 +410,19 @@ def connect_node(flow_id: int, node_connection: input_schema.NodeConnection):
348
410
 
349
411
  @router.get('/editor/expression_doc', tags=['editor'], response_model=List[output_model.ExpressionsOverview])
350
412
  def get_expression_doc() -> List[output_model.ExpressionsOverview]:
413
+ """Retrieves documentation for available Polars expressions."""
351
414
  return get_expression_overview()
352
415
 
353
416
 
354
417
  @router.get('/editor/expressions', tags=['editor'], response_model=List[str])
355
418
  def get_expressions() -> List[str]:
419
+ """Retrieves a list of all available Flowfile expression names."""
356
420
  return get_all_expressions()
357
421
 
358
422
 
359
423
  @router.get('/editor/flow', tags=['editor'], response_model=schemas.FlowSettings)
360
424
  def get_flow(flow_id: int):
425
+ """Retrieves the settings for a specific flow."""
361
426
  flow_id = int(flow_id)
362
427
  result = get_flow_settings(flow_id)
363
428
  return result
@@ -365,6 +430,7 @@ def get_flow(flow_id: int):
365
430
 
366
431
  @router.get("/editor/code_to_polars", tags=[], response_model=str)
367
432
  def get_generated_code(flow_id: int) -> str:
433
+ """Generates and returns a Python script with Polars code representing the flow."""
368
434
  flow_id = int(flow_id)
369
435
  flow = flow_file_handler.get_flow(flow_id)
370
436
  if flow is None:
@@ -374,6 +440,7 @@ def get_generated_code(flow_id: int) -> str:
374
440
 
375
441
  @router.post('/editor/create_flow/', tags=['editor'])
376
442
  def create_flow(flow_path: str):
443
+ """Creates a new, empty flow file at the specified path and registers a session for it."""
377
444
  flow_path = Path(flow_path)
378
445
  logger.info('Creating flow')
379
446
  return flow_file_handler.add_flow(name=flow_path.stem, flow_path=str(flow_path))
@@ -381,43 +448,17 @@ def create_flow(flow_path: str):
381
448
 
382
449
  @router.post('/editor/close_flow/', tags=['editor'])
383
450
  def close_flow(flow_id: int) -> None:
451
+ """Closes an active flow session."""
384
452
  flow_file_handler.delete_flow(flow_id)
385
453
 
386
454
 
387
- @router.get('/airbyte/available_connectors', tags=['airbyte'])
388
- def get_available_connectors():
389
- return airbyte_config_handler.available_connectors
390
-
391
-
392
- @router.get('/airbyte/available_configs', tags=['airbyte'])
393
- def get_available_configs() -> List[str]:
394
- """
395
- Get the available configurations for the airbyte connectors
396
- Returns: List of available configurations
397
- """
398
- return airbyte_config_handler.available_configs
399
-
400
-
401
- @router.get('/airbyte/config_template', tags=['airbyte'], response_model=AirbyteConfigTemplate)
402
- def get_config_spec(connector_name: str):
403
- a = airbyte_config_handler.get_config('source-' + connector_name)
404
- return a
405
-
406
-
407
- @router.post('/airbyte/set_airbyte_configs_for_streams', tags=['airbyte'])
408
- def set_airbyte_configs_for_streams(airbyte_config: input_schema.AirbyteConfig):
409
- logger.info('Setting airbyte config, update_style = ')
410
- logger.info(f'Setting config for {airbyte_config.source_name}')
411
- logger.debug(f'Config: {airbyte_config.mapped_config_spec}')
412
- airbyte_handler = AirbyteHandler(airbyte_config=airbyte_config)
413
- try:
414
- _ = airbyte_handler.get_available_streams()
415
- except Exception as e:
416
- raise HTTPException(404, str(e))
417
-
418
-
419
455
  @router.post('/update_settings/', tags=['transform'])
420
456
  def add_generic_settings(input_data: Dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)):
457
+ """A generic endpoint to update the settings of any node.
458
+
459
+ This endpoint dynamically determines the correct Pydantic model and update
460
+ function based on the `node_type` parameter.
461
+ """
421
462
  input_data['user_id'] = current_user.id
422
463
  node_type = camel_case_to_snake_case(node_type)
423
464
  flow_id = int(input_data.get('flow_id'))
@@ -449,6 +490,7 @@ def add_generic_settings(input_data: Dict[str, Any], node_type: str, current_use
449
490
 
450
491
  @router.get('/files/available_flow_files', tags=['editor'], response_model=List[FileInfo])
451
492
  def get_list_of_saved_flows(path: str):
493
+ """Scans a directory for saved flow files (`.flowfile`)."""
452
494
  try:
453
495
  return get_files_from_directory(path, types=['flowfile'])
454
496
  except:
@@ -456,26 +498,13 @@ def get_list_of_saved_flows(path: str):
456
498
 
457
499
  @router.get('/node_list', response_model=List[nodes.NodeTemplate])
458
500
  def get_node_list() -> List[nodes.NodeTemplate]:
501
+ """Retrieves the list of all available node types and their templates."""
459
502
  return nodes.nodes_list
460
503
 
461
504
 
462
- # @router.post('/reset')
463
- # def reset():
464
- # flow_file_handler.delete_flow(1)
465
- # register_flow(schemas.FlowSettings(flow_id=1))
466
-
467
-
468
- @router.post('/files/remove_items', tags=['file manager'])
469
- def remove_items(remove_items_input: input_schema.RemoveItemsInput):
470
- result, error = remove_paths(remove_items_input)
471
- if result:
472
- return result
473
- else:
474
- raise error
475
-
476
-
477
505
  @router.get('/node', response_model=output_model.NodeData, tags=['editor'])
478
506
  def get_node(flow_id: int, node_id: int, get_data: bool = False):
507
+ """Retrieves the complete state and data preview for a single node."""
479
508
  logging.info(f'Getting node {node_id} from flow {flow_id}')
480
509
  flow = flow_file_handler.get_flow(flow_id)
481
510
  node = flow.get_node(node_id)
@@ -487,6 +516,7 @@ def get_node(flow_id: int, node_id: int, get_data: bool = False):
487
516
 
488
517
  @router.post('/node/description/', tags=['editor'])
489
518
  def update_description_node(flow_id: int, node_id: int, description: str = Body(...)):
519
+ """Updates the description text for a specific node."""
490
520
  try:
491
521
  node = flow_file_handler.get_flow(flow_id).get_node(node_id)
492
522
  except:
@@ -497,6 +527,7 @@ def update_description_node(flow_id: int, node_id: int, description: str = Body(
497
527
 
498
528
  @router.get('/node/description', tags=['editor'])
499
529
  def get_description_node(flow_id: int, node_id: int):
530
+ """Retrieves the description text for a specific node."""
500
531
  try:
501
532
  node = flow_file_handler.get_flow(flow_id).get_node(node_id)
502
533
  except:
@@ -508,6 +539,7 @@ def get_description_node(flow_id: int, node_id: int):
508
539
 
509
540
  @router.get('/node/data', response_model=output_model.TableExample, tags=['editor'])
510
541
  def get_table_example(flow_id: int, node_id: int):
542
+ """Retrieves a data preview (schema and sample rows) for a node's output."""
511
543
  flow = flow_file_handler.get_flow(flow_id)
512
544
  node = flow.get_node(node_id)
513
545
  return node.get_table_example(True)
@@ -515,6 +547,7 @@ def get_table_example(flow_id: int, node_id: int):
515
547
 
516
548
  @router.get('/node/downstream_node_ids', response_model=List[int], tags=['editor'])
517
549
  async def get_downstream_node_ids(flow_id: int, node_id: int) -> List[int]:
550
+ """Gets a list of all node IDs that are downstream dependencies of a given node."""
518
551
  flow = flow_file_handler.get_flow(flow_id)
519
552
  node = flow.get_node(node_id)
520
553
  return list(node.get_all_dependent_node_ids())
@@ -522,6 +555,7 @@ async def get_downstream_node_ids(flow_id: int, node_id: int) -> List[int]:
522
555
 
523
556
  @router.get('/import_flow/', tags=['editor'], response_model=int)
524
557
  def import_saved_flow(flow_path: str) -> int:
558
+ """Imports a flow from a saved `.flowfile` and registers it as a new session."""
525
559
  flow_path = Path(flow_path)
526
560
  if not flow_path.exists():
527
561
  raise HTTPException(404, 'File not found')
@@ -530,12 +564,14 @@ def import_saved_flow(flow_path: str) -> int:
530
564
 
531
565
  @router.get('/save_flow', tags=['editor'])
532
566
  def save_flow(flow_id: int, flow_path: str = None):
567
+ """Saves the current state of a flow to a `.flowfile`."""
533
568
  flow = flow_file_handler.get_flow(flow_id)
534
569
  flow.save_flow(flow_path=flow_path)
535
570
 
536
571
 
537
572
  @router.get('/flow_data', tags=['manager'])
538
573
  def get_flow_frontend_data(flow_id: Optional[int] = 1):
574
+ """Retrieves the data needed to render the flow graph in the frontend."""
539
575
  flow = flow_file_handler.get_flow(flow_id)
540
576
  if flow is None:
541
577
  raise HTTPException(404, 'could not find the flow')
@@ -544,6 +580,7 @@ def get_flow_frontend_data(flow_id: Optional[int] = 1):
544
580
 
545
581
  @router.get('/flow_settings', tags=['manager'], response_model=schemas.FlowSettings)
546
582
  def get_flow_settings(flow_id: Optional[int] = 1) -> schemas.FlowSettings:
583
+ """Retrieves the main settings for a flow."""
547
584
  flow = flow_file_handler.get_flow(flow_id)
548
585
  if flow is None:
549
586
  raise HTTPException(404, 'could not find the flow')
@@ -552,6 +589,7 @@ def get_flow_settings(flow_id: Optional[int] = 1) -> schemas.FlowSettings:
552
589
 
553
590
  @router.post('/flow_settings', tags=['manager'])
554
591
  def update_flow_settings(flow_settings: schemas.FlowSettings):
592
+ """Updates the main settings for a flow."""
555
593
  flow = flow_file_handler.get_flow(flow_settings.flow_id)
556
594
  if flow is None:
557
595
  raise HTTPException(404, 'could not find the flow')
@@ -560,6 +598,7 @@ def update_flow_settings(flow_settings: schemas.FlowSettings):
560
598
 
561
599
  @router.get('/flow_data/v2', tags=['manager'])
562
600
  def get_vue_flow_data(flow_id: int) -> schemas.VueFlowInput:
601
+ """Retrieves the flow data formatted for the Vue-based frontend."""
563
602
  flow = flow_file_handler.get_flow(flow_id)
564
603
  if flow is None:
565
604
  raise HTTPException(404, 'could not find the flow')
@@ -569,6 +608,7 @@ def get_vue_flow_data(flow_id: int) -> schemas.VueFlowInput:
569
608
 
570
609
  @router.get('/analysis_data/graphic_walker_input', tags=['analysis'], response_model=input_schema.NodeExploreData)
571
610
  def get_graphic_walker_input(flow_id: int, node_id: int):
611
+ """Gets the data and configuration for the Graphic Walker data exploration tool."""
572
612
  flow = flow_file_handler.get_flow(flow_id)
573
613
  node = flow.get_node(node_id)
574
614
  if node.results.analysis_data_generator is None:
@@ -579,6 +619,7 @@ def get_graphic_walker_input(flow_id: int, node_id: int):
579
619
 
580
620
  @router.get('/custom_functions/instant_result', tags=[])
581
621
  async def get_instant_function_result(flow_id: int, node_id: int, func_string: str):
622
+ """Executes a simple, instant function on a node's data and returns the result."""
582
623
  try:
583
624
  node = flow_file_handler.get_node(flow_id, node_id)
584
625
  result = await asyncio.to_thread(get_instant_func_results, node, func_string)
@@ -589,6 +630,7 @@ async def get_instant_function_result(flow_id: int, node_id: int, func_string: s
589
630
 
590
631
  @router.get('/api/get_xlsx_sheet_names', tags=['excel_reader'], response_model=List[str])
591
632
  async def get_excel_sheet_names(path: str) -> List[str] | None:
633
+ """Retrieves the sheet names from an Excel file."""
592
634
  sheet_names = excel_file_manager.get_sheet_names(path)
593
635
  if sheet_names:
594
636
  return sheet_names
@@ -601,9 +643,7 @@ async def validate_db_settings(
601
643
  database_settings: input_schema.DatabaseSettings,
602
644
  current_user=Depends(get_current_active_user)
603
645
  ):
604
- """
605
- Validate the query settings for a database connection.
606
- """
646
+ """Validates that a connection can be made to a database with the given settings."""
607
647
  # Validate the query settings
608
648
  try:
609
649
  sql_source = create_sql_source_from_db_settings(database_settings, user_id=current_user.id)