griptape-nodes 0.65.6__py3-none-any.whl → 0.66.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- griptape_nodes/common/node_executor.py +352 -27
- griptape_nodes/drivers/storage/base_storage_driver.py +12 -3
- griptape_nodes/drivers/storage/griptape_cloud_storage_driver.py +18 -2
- griptape_nodes/drivers/storage/local_storage_driver.py +42 -5
- griptape_nodes/exe_types/base_iterative_nodes.py +0 -1
- griptape_nodes/exe_types/connections.py +42 -0
- griptape_nodes/exe_types/core_types.py +2 -2
- griptape_nodes/exe_types/node_groups/__init__.py +2 -1
- griptape_nodes/exe_types/node_groups/base_iterative_node_group.py +177 -0
- griptape_nodes/exe_types/node_groups/base_node_group.py +1 -0
- griptape_nodes/exe_types/node_groups/subflow_node_group.py +35 -2
- griptape_nodes/exe_types/param_types/parameter_audio.py +1 -1
- griptape_nodes/exe_types/param_types/parameter_bool.py +1 -1
- griptape_nodes/exe_types/param_types/parameter_button.py +1 -1
- griptape_nodes/exe_types/param_types/parameter_float.py +1 -1
- griptape_nodes/exe_types/param_types/parameter_image.py +1 -1
- griptape_nodes/exe_types/param_types/parameter_int.py +1 -1
- griptape_nodes/exe_types/param_types/parameter_number.py +1 -1
- griptape_nodes/exe_types/param_types/parameter_string.py +1 -1
- griptape_nodes/exe_types/param_types/parameter_three_d.py +1 -1
- griptape_nodes/exe_types/param_types/parameter_video.py +1 -1
- griptape_nodes/machines/control_flow.py +5 -4
- griptape_nodes/machines/dag_builder.py +121 -55
- griptape_nodes/machines/fsm.py +10 -0
- griptape_nodes/machines/parallel_resolution.py +39 -38
- griptape_nodes/machines/sequential_resolution.py +29 -3
- griptape_nodes/node_library/library_registry.py +41 -2
- griptape_nodes/retained_mode/events/library_events.py +147 -8
- griptape_nodes/retained_mode/events/os_events.py +12 -4
- griptape_nodes/retained_mode/managers/fitness_problems/libraries/__init__.py +2 -0
- griptape_nodes/retained_mode/managers/fitness_problems/libraries/incompatible_requirements_problem.py +34 -0
- griptape_nodes/retained_mode/managers/flow_manager.py +133 -20
- griptape_nodes/retained_mode/managers/library_manager.py +1324 -564
- griptape_nodes/retained_mode/managers/node_manager.py +9 -3
- griptape_nodes/retained_mode/managers/os_manager.py +429 -65
- griptape_nodes/retained_mode/managers/resource_types/compute_resource.py +82 -0
- griptape_nodes/retained_mode/managers/resource_types/os_resource.py +17 -0
- griptape_nodes/retained_mode/managers/static_files_manager.py +21 -8
- griptape_nodes/retained_mode/managers/version_compatibility_manager.py +3 -3
- griptape_nodes/version_compatibility/versions/v0_39_0/modified_parameters_set_removal.py +5 -5
- griptape_nodes/version_compatibility/versions/v0_65_4/__init__.py +5 -0
- griptape_nodes/version_compatibility/versions/v0_65_4/run_in_parallel_to_run_in_order.py +79 -0
- griptape_nodes/version_compatibility/versions/v0_65_5/__init__.py +5 -0
- griptape_nodes/version_compatibility/versions/v0_65_5/flux_2_removed_parameters.py +85 -0
- {griptape_nodes-0.65.6.dist-info → griptape_nodes-0.66.1.dist-info}/METADATA +1 -1
- {griptape_nodes-0.65.6.dist-info → griptape_nodes-0.66.1.dist-info}/RECORD +48 -53
- griptape_nodes/retained_mode/managers/library_lifecycle/__init__.py +0 -45
- griptape_nodes/retained_mode/managers/library_lifecycle/data_models.py +0 -191
- griptape_nodes/retained_mode/managers/library_lifecycle/library_directory.py +0 -346
- griptape_nodes/retained_mode/managers/library_lifecycle/library_fsm.py +0 -439
- griptape_nodes/retained_mode/managers/library_lifecycle/library_provenance/__init__.py +0 -17
- griptape_nodes/retained_mode/managers/library_lifecycle/library_provenance/base.py +0 -82
- griptape_nodes/retained_mode/managers/library_lifecycle/library_provenance/github.py +0 -116
- griptape_nodes/retained_mode/managers/library_lifecycle/library_provenance/local_file.py +0 -367
- griptape_nodes/retained_mode/managers/library_lifecycle/library_provenance/package.py +0 -104
- griptape_nodes/retained_mode/managers/library_lifecycle/library_provenance/sandbox.py +0 -155
- griptape_nodes/retained_mode/managers/library_lifecycle/library_provenance.py +0 -18
- griptape_nodes/retained_mode/managers/library_lifecycle/library_status.py +0 -12
- {griptape_nodes-0.65.6.dist-info → griptape_nodes-0.66.1.dist-info}/WHEEL +0 -0
- {griptape_nodes-0.65.6.dist-info → griptape_nodes-0.66.1.dist-info}/entry_points.txt +0 -0
|
@@ -10,12 +10,13 @@ from typing import TYPE_CHECKING, Any, NamedTuple
|
|
|
10
10
|
|
|
11
11
|
from griptape_nodes.bootstrap.workflow_publishers.subprocess_workflow_publisher import SubprocessWorkflowPublisher
|
|
12
12
|
from griptape_nodes.drivers.storage.storage_backend import StorageBackend
|
|
13
|
+
from griptape_nodes.exe_types import node_types
|
|
13
14
|
from griptape_nodes.exe_types.base_iterative_nodes import (
|
|
14
15
|
BaseIterativeEndNode,
|
|
15
16
|
BaseIterativeStartNode,
|
|
16
17
|
)
|
|
17
18
|
from griptape_nodes.exe_types.core_types import ParameterTypeBuiltin
|
|
18
|
-
from griptape_nodes.exe_types.node_groups import SubflowNodeGroup
|
|
19
|
+
from griptape_nodes.exe_types.node_groups import BaseIterativeNodeGroup, SubflowNodeGroup
|
|
19
20
|
from griptape_nodes.exe_types.node_types import (
|
|
20
21
|
CONTROL_INPUT_PARAMETER,
|
|
21
22
|
LOCAL_EXECUTION,
|
|
@@ -197,6 +198,12 @@ class NodeExecutor:
|
|
|
197
198
|
node: The BaseNode to execute
|
|
198
199
|
library_name: The library that the execute method should come from.
|
|
199
200
|
"""
|
|
201
|
+
# Handle iterative node groups (ForEachGroup, ForLoopGroup, etc.)
|
|
202
|
+
# Check this BEFORE SubflowNodeGroup since BaseIterativeNodeGroup extends SubflowNodeGroup
|
|
203
|
+
if isinstance(node, BaseIterativeNodeGroup):
|
|
204
|
+
await self.handle_iterative_group_execution(node)
|
|
205
|
+
return
|
|
206
|
+
|
|
200
207
|
if isinstance(node, SubflowNodeGroup):
|
|
201
208
|
execution_type = node.get_parameter_value(node.execution_environment.name)
|
|
202
209
|
if execution_type == LOCAL_EXECUTION:
|
|
@@ -814,7 +821,7 @@ class NodeExecutor:
|
|
|
814
821
|
package_result: PackageNodesAsSerializedFlowResultSuccess,
|
|
815
822
|
total_iterations: int,
|
|
816
823
|
parameter_values_per_iteration: dict[int, dict[str, Any]],
|
|
817
|
-
end_loop_node: BaseIterativeEndNode,
|
|
824
|
+
end_loop_node: BaseIterativeEndNode | BaseIterativeNodeGroup,
|
|
818
825
|
) -> tuple[dict[int, Any], list[int], dict[str, Any]]:
|
|
819
826
|
"""Execute loop iterations sequentially by running one flow instance N times.
|
|
820
827
|
|
|
@@ -1143,9 +1150,9 @@ class NodeExecutor:
|
|
|
1143
1150
|
logger.info("No iterations for empty loop from '%s' to '%s'", start_node.name, node.name)
|
|
1144
1151
|
return
|
|
1145
1152
|
|
|
1146
|
-
# Check if we should run in
|
|
1147
|
-
|
|
1148
|
-
if
|
|
1153
|
+
# Check if we should run in order (default is in order / True)
|
|
1154
|
+
run_in_order = start_node.get_parameter_value("run_in_order")
|
|
1155
|
+
if run_in_order:
|
|
1149
1156
|
# Sequential execution - run iterations one at a time in the main execution flow
|
|
1150
1157
|
await self._handle_sequential_loop_execution(start_node, node)
|
|
1151
1158
|
return
|
|
@@ -1234,6 +1241,325 @@ class NodeExecutor:
|
|
|
1234
1241
|
node.name,
|
|
1235
1242
|
)
|
|
1236
1243
|
|
|
1244
|
+
async def handle_iterative_group_execution(self, node: BaseIterativeNodeGroup) -> None:
|
|
1245
|
+
"""Handle execution of an iterative node group by running its child nodes for each iteration.
|
|
1246
|
+
|
|
1247
|
+
This method is similar to handle_loop_execution but simplified for node groups:
|
|
1248
|
+
- Child nodes are already known (node.get_all_nodes())
|
|
1249
|
+
- No need to find/validate start-end node connections
|
|
1250
|
+
- The group itself holds iteration parameters (items, current_item, index, results)
|
|
1251
|
+
|
|
1252
|
+
Args:
|
|
1253
|
+
node: The BaseIterativeNodeGroup to execute
|
|
1254
|
+
"""
|
|
1255
|
+
# Initialize iteration data to determine total iterations
|
|
1256
|
+
node._initialize_iteration_data()
|
|
1257
|
+
|
|
1258
|
+
total_iterations = node._get_total_iterations()
|
|
1259
|
+
if total_iterations == 0:
|
|
1260
|
+
logger.info("No iterations for empty iterative group '%s'", node.name)
|
|
1261
|
+
node._output_results_list()
|
|
1262
|
+
return
|
|
1263
|
+
|
|
1264
|
+
# Get execution environment
|
|
1265
|
+
execution_type = node.get_parameter_value(node.execution_environment.name)
|
|
1266
|
+
|
|
1267
|
+
# Check if we should run in order (default is sequential/True)
|
|
1268
|
+
run_in_order = node.get_parameter_value("run_in_order")
|
|
1269
|
+
|
|
1270
|
+
if run_in_order:
|
|
1271
|
+
# Sequential execution
|
|
1272
|
+
await self._handle_sequential_iterative_group_execution(node, execution_type)
|
|
1273
|
+
return
|
|
1274
|
+
|
|
1275
|
+
# Parallel execution - package and run all iterations concurrently
|
|
1276
|
+
package_result = await self._package_iterative_group_body(node)
|
|
1277
|
+
|
|
1278
|
+
# Handle empty group (no child nodes)
|
|
1279
|
+
if package_result is None:
|
|
1280
|
+
logger.info("Empty iterative group '%s' - no child nodes to execute", node.name)
|
|
1281
|
+
node._output_results_list()
|
|
1282
|
+
return
|
|
1283
|
+
|
|
1284
|
+
# Get parameter values for each iteration
|
|
1285
|
+
parameter_values_to_set_before_run = self._get_merged_parameter_values_for_iterative_group(node, package_result)
|
|
1286
|
+
|
|
1287
|
+
# Execute all iterations based on execution environment
|
|
1288
|
+
match execution_type:
|
|
1289
|
+
case node_types.LOCAL_EXECUTION:
|
|
1290
|
+
(
|
|
1291
|
+
iteration_results,
|
|
1292
|
+
successful_iterations,
|
|
1293
|
+
last_iteration_values,
|
|
1294
|
+
) = await self._execute_loop_iterations_locally(
|
|
1295
|
+
package_result=package_result,
|
|
1296
|
+
total_iterations=total_iterations,
|
|
1297
|
+
parameter_values_per_iteration=parameter_values_to_set_before_run,
|
|
1298
|
+
end_loop_node=node,
|
|
1299
|
+
)
|
|
1300
|
+
case node_types.PRIVATE_EXECUTION:
|
|
1301
|
+
(
|
|
1302
|
+
iteration_results,
|
|
1303
|
+
successful_iterations,
|
|
1304
|
+
last_iteration_values,
|
|
1305
|
+
) = await self._execute_loop_iterations_privately(
|
|
1306
|
+
package_result=package_result,
|
|
1307
|
+
total_iterations=total_iterations,
|
|
1308
|
+
parameter_values_per_iteration=parameter_values_to_set_before_run,
|
|
1309
|
+
end_loop_node=node,
|
|
1310
|
+
)
|
|
1311
|
+
case _:
|
|
1312
|
+
# Cloud publisher execution (Deadline Cloud, etc.)
|
|
1313
|
+
(
|
|
1314
|
+
iteration_results,
|
|
1315
|
+
successful_iterations,
|
|
1316
|
+
last_iteration_values,
|
|
1317
|
+
) = await self._execute_loop_iterations_via_publisher(
|
|
1318
|
+
package_result=package_result,
|
|
1319
|
+
total_iterations=total_iterations,
|
|
1320
|
+
parameter_values_per_iteration=parameter_values_to_set_before_run,
|
|
1321
|
+
end_loop_node=node,
|
|
1322
|
+
execution_type=execution_type,
|
|
1323
|
+
)
|
|
1324
|
+
|
|
1325
|
+
if len(successful_iterations) != total_iterations:
|
|
1326
|
+
failed_count = total_iterations - len(successful_iterations)
|
|
1327
|
+
msg = f"Iterative group execution failed: {failed_count} of {total_iterations} iterations failed"
|
|
1328
|
+
raise RuntimeError(msg)
|
|
1329
|
+
|
|
1330
|
+
logger.info(
|
|
1331
|
+
"Successfully completed parallel execution of %d iterations for iterative group '%s'",
|
|
1332
|
+
total_iterations,
|
|
1333
|
+
node.name,
|
|
1334
|
+
)
|
|
1335
|
+
|
|
1336
|
+
# Build results list in iteration order
|
|
1337
|
+
node._results_list = []
|
|
1338
|
+
for iteration_index in sorted(iteration_results.keys()):
|
|
1339
|
+
value = iteration_results[iteration_index]
|
|
1340
|
+
node._results_list.append(value)
|
|
1341
|
+
|
|
1342
|
+
# Output final results to the results parameter
|
|
1343
|
+
node._output_results_list()
|
|
1344
|
+
|
|
1345
|
+
# Apply last iteration values to the original child nodes in main flow
|
|
1346
|
+
self._apply_last_iteration_to_packaged_nodes(
|
|
1347
|
+
last_iteration_values=last_iteration_values,
|
|
1348
|
+
package_result=package_result,
|
|
1349
|
+
)
|
|
1350
|
+
|
|
1351
|
+
logger.info(
|
|
1352
|
+
"Successfully aggregated %d results for iterative group '%s'",
|
|
1353
|
+
len(iteration_results),
|
|
1354
|
+
node.name,
|
|
1355
|
+
)
|
|
1356
|
+
|
|
1357
|
+
async def _handle_sequential_iterative_group_execution(
|
|
1358
|
+
self, node: BaseIterativeNodeGroup, execution_type: str
|
|
1359
|
+
) -> None:
|
|
1360
|
+
"""Handle sequential execution of an iterative node group.
|
|
1361
|
+
|
|
1362
|
+
Args:
|
|
1363
|
+
node: The BaseIterativeNodeGroup to execute
|
|
1364
|
+
execution_type: The execution environment type
|
|
1365
|
+
"""
|
|
1366
|
+
total_iterations = node._get_total_iterations()
|
|
1367
|
+
logger.info(
|
|
1368
|
+
"Executing iterative group '%s' sequentially for %d iterations",
|
|
1369
|
+
node.name,
|
|
1370
|
+
total_iterations,
|
|
1371
|
+
)
|
|
1372
|
+
|
|
1373
|
+
# Package the group body (child nodes)
|
|
1374
|
+
package_result = await self._package_iterative_group_body(node)
|
|
1375
|
+
|
|
1376
|
+
# Handle empty group (no child nodes)
|
|
1377
|
+
if package_result is None:
|
|
1378
|
+
logger.info("Empty iterative group '%s' - no child nodes to execute", node.name)
|
|
1379
|
+
node._output_results_list()
|
|
1380
|
+
return
|
|
1381
|
+
|
|
1382
|
+
# Get parameter values per iteration
|
|
1383
|
+
parameter_values_per_iteration = self._get_merged_parameter_values_for_iterative_group(node, package_result)
|
|
1384
|
+
|
|
1385
|
+
# Execute iterations sequentially based on execution environment
|
|
1386
|
+
match execution_type:
|
|
1387
|
+
case node_types.LOCAL_EXECUTION:
|
|
1388
|
+
(
|
|
1389
|
+
iteration_results,
|
|
1390
|
+
successful_iterations,
|
|
1391
|
+
last_iteration_values,
|
|
1392
|
+
) = await self._execute_loop_iterations_sequentially(
|
|
1393
|
+
package_result=package_result,
|
|
1394
|
+
total_iterations=total_iterations,
|
|
1395
|
+
parameter_values_per_iteration=parameter_values_per_iteration,
|
|
1396
|
+
end_loop_node=node,
|
|
1397
|
+
)
|
|
1398
|
+
case node_types.PRIVATE_EXECUTION:
|
|
1399
|
+
(
|
|
1400
|
+
iteration_results,
|
|
1401
|
+
successful_iterations,
|
|
1402
|
+
last_iteration_values,
|
|
1403
|
+
) = await self._execute_loop_iterations_sequentially_private(
|
|
1404
|
+
package_result=package_result,
|
|
1405
|
+
total_iterations=total_iterations,
|
|
1406
|
+
parameter_values_per_iteration=parameter_values_per_iteration,
|
|
1407
|
+
end_loop_node=node,
|
|
1408
|
+
)
|
|
1409
|
+
case _:
|
|
1410
|
+
# Cloud publisher execution
|
|
1411
|
+
(
|
|
1412
|
+
iteration_results,
|
|
1413
|
+
successful_iterations,
|
|
1414
|
+
last_iteration_values,
|
|
1415
|
+
) = await self._execute_loop_iterations_sequentially_via_publisher(
|
|
1416
|
+
package_result=package_result,
|
|
1417
|
+
total_iterations=total_iterations,
|
|
1418
|
+
parameter_values_per_iteration=parameter_values_per_iteration,
|
|
1419
|
+
end_loop_node=node,
|
|
1420
|
+
execution_type=execution_type,
|
|
1421
|
+
)
|
|
1422
|
+
|
|
1423
|
+
# Check if execution stopped early due to break (not failure)
|
|
1424
|
+
if len(successful_iterations) < total_iterations:
|
|
1425
|
+
expected_count = len(successful_iterations)
|
|
1426
|
+
actual_count = len(iteration_results)
|
|
1427
|
+
if expected_count != actual_count:
|
|
1428
|
+
failed_count = expected_count - actual_count
|
|
1429
|
+
msg = f"Iterative group execution failed: {failed_count} of {expected_count} iterations failed"
|
|
1430
|
+
raise RuntimeError(msg)
|
|
1431
|
+
logger.info(
|
|
1432
|
+
"Iterative group execution stopped early at %d of %d iterations (break signal)",
|
|
1433
|
+
len(successful_iterations),
|
|
1434
|
+
total_iterations,
|
|
1435
|
+
)
|
|
1436
|
+
|
|
1437
|
+
# Build results list in iteration order
|
|
1438
|
+
node._results_list = []
|
|
1439
|
+
for iteration_index in sorted(iteration_results.keys()):
|
|
1440
|
+
value = iteration_results[iteration_index]
|
|
1441
|
+
node._results_list.append(value)
|
|
1442
|
+
|
|
1443
|
+
logger.info(
|
|
1444
|
+
"Iterative group '%s': Built results list with %d items from sequential iterations",
|
|
1445
|
+
node.name,
|
|
1446
|
+
len(node._results_list),
|
|
1447
|
+
)
|
|
1448
|
+
|
|
1449
|
+
# Output final results
|
|
1450
|
+
node._output_results_list()
|
|
1451
|
+
|
|
1452
|
+
# Apply last iteration values to the original child nodes
|
|
1453
|
+
self._apply_last_iteration_to_packaged_nodes(
|
|
1454
|
+
last_iteration_values=last_iteration_values,
|
|
1455
|
+
package_result=package_result,
|
|
1456
|
+
)
|
|
1457
|
+
|
|
1458
|
+
logger.info(
|
|
1459
|
+
"Completed sequential iterative group execution for '%s' with %d results",
|
|
1460
|
+
node.name,
|
|
1461
|
+
len(iteration_results),
|
|
1462
|
+
)
|
|
1463
|
+
|
|
1464
|
+
async def _package_iterative_group_body(
|
|
1465
|
+
self, node: BaseIterativeNodeGroup
|
|
1466
|
+
) -> PackageNodesAsSerializedFlowResultSuccess | None:
|
|
1467
|
+
"""Package the child nodes of an iterative group into a serialized flow.
|
|
1468
|
+
|
|
1469
|
+
Args:
|
|
1470
|
+
node: The BaseIterativeNodeGroup whose children should be packaged
|
|
1471
|
+
|
|
1472
|
+
Returns:
|
|
1473
|
+
PackageNodesAsSerializedFlowResultSuccess if successful, None if no child nodes
|
|
1474
|
+
"""
|
|
1475
|
+
# Get all child node names
|
|
1476
|
+
all_nodes = node.get_all_nodes()
|
|
1477
|
+
node_names = list(all_nodes.keys())
|
|
1478
|
+
|
|
1479
|
+
if not node_names:
|
|
1480
|
+
return None
|
|
1481
|
+
|
|
1482
|
+
# Get execution type to determine start/end node types
|
|
1483
|
+
execution_type = node.get_parameter_value(node.execution_environment.name)
|
|
1484
|
+
|
|
1485
|
+
# Determine library and node types
|
|
1486
|
+
library = None
|
|
1487
|
+
if execution_type not in (LOCAL_EXECUTION, PRIVATE_EXECUTION):
|
|
1488
|
+
try:
|
|
1489
|
+
library = LibraryRegistry.get_library(name=execution_type)
|
|
1490
|
+
except KeyError:
|
|
1491
|
+
logger.error("Could not find library '%s' for iterative group execution", execution_type)
|
|
1492
|
+
raise
|
|
1493
|
+
|
|
1494
|
+
workflow_start_end_nodes = await self._get_workflow_start_end_nodes(library)
|
|
1495
|
+
|
|
1496
|
+
# Create the packaging request
|
|
1497
|
+
sanitized_node_name = node.name.replace(" ", "_")
|
|
1498
|
+
output_parameter_prefix = f"{sanitized_node_name}_iterative_group_"
|
|
1499
|
+
|
|
1500
|
+
request = PackageNodesAsSerializedFlowRequest(
|
|
1501
|
+
node_names=node_names,
|
|
1502
|
+
start_node_type=workflow_start_end_nodes.start_flow_node_type,
|
|
1503
|
+
end_node_type=workflow_start_end_nodes.end_flow_node_type,
|
|
1504
|
+
start_node_library_name=workflow_start_end_nodes.start_flow_node_library_name,
|
|
1505
|
+
end_node_library_name=workflow_start_end_nodes.end_flow_node_library_name,
|
|
1506
|
+
output_parameter_prefix=output_parameter_prefix,
|
|
1507
|
+
entry_control_node_name=None,
|
|
1508
|
+
entry_control_parameter_name=None,
|
|
1509
|
+
node_group_name=node.name,
|
|
1510
|
+
)
|
|
1511
|
+
|
|
1512
|
+
package_result = GriptapeNodes.handle_request(request)
|
|
1513
|
+
if not isinstance(package_result, PackageNodesAsSerializedFlowResultSuccess):
|
|
1514
|
+
msg = f"Failed to package iterative group '{node.name}'. Error: {package_result.result_details}"
|
|
1515
|
+
raise TypeError(msg)
|
|
1516
|
+
|
|
1517
|
+
logger.info(
|
|
1518
|
+
"Successfully packaged %d nodes for iterative group '%s'",
|
|
1519
|
+
len(node_names),
|
|
1520
|
+
node.name,
|
|
1521
|
+
)
|
|
1522
|
+
|
|
1523
|
+
# Remove packaged nodes from global queue
|
|
1524
|
+
self._remove_packaged_nodes_from_queue(set(node_names))
|
|
1525
|
+
|
|
1526
|
+
return package_result
|
|
1527
|
+
|
|
1528
|
+
def _get_merged_parameter_values_for_iterative_group(
|
|
1529
|
+
self, node: BaseIterativeNodeGroup, package_result: PackageNodesAsSerializedFlowResultSuccess
|
|
1530
|
+
) -> dict[int, dict[str, Any]]:
|
|
1531
|
+
"""Get parameter values for each iteration with resolved upstream values merged in.
|
|
1532
|
+
|
|
1533
|
+
Args:
|
|
1534
|
+
node: The iterative node group
|
|
1535
|
+
package_result: The packaged flow result
|
|
1536
|
+
|
|
1537
|
+
Returns:
|
|
1538
|
+
Dict mapping iteration_index -> {parameter_name: value}
|
|
1539
|
+
"""
|
|
1540
|
+
# Get parameter values that vary per iteration (current_item, index mappings)
|
|
1541
|
+
parameter_values_per_iteration = self.get_parameter_values_per_iteration(node, package_result)
|
|
1542
|
+
|
|
1543
|
+
# Get resolved upstream values (constant across all iterations)
|
|
1544
|
+
resolved_upstream_values = self.get_resolved_upstream_values(
|
|
1545
|
+
packaged_node_names=package_result.packaged_node_names, package_result=package_result
|
|
1546
|
+
)
|
|
1547
|
+
|
|
1548
|
+
# Merge upstream values into each iteration
|
|
1549
|
+
if resolved_upstream_values:
|
|
1550
|
+
for iteration_index in parameter_values_per_iteration:
|
|
1551
|
+
for param_name, param_value in resolved_upstream_values.items():
|
|
1552
|
+
if param_name not in parameter_values_per_iteration[iteration_index]:
|
|
1553
|
+
parameter_values_per_iteration[iteration_index][param_name] = param_value
|
|
1554
|
+
logger.info(
|
|
1555
|
+
"Added %d resolved upstream values to %d iterations for group '%s'",
|
|
1556
|
+
len(resolved_upstream_values),
|
|
1557
|
+
len(parameter_values_per_iteration),
|
|
1558
|
+
node.name,
|
|
1559
|
+
)
|
|
1560
|
+
|
|
1561
|
+
return parameter_values_per_iteration
|
|
1562
|
+
|
|
1237
1563
|
def _get_iteration_value_for_parameter(
|
|
1238
1564
|
self,
|
|
1239
1565
|
source_param_name: str,
|
|
@@ -1263,7 +1589,7 @@ class NodeExecutor:
|
|
|
1263
1589
|
|
|
1264
1590
|
def get_parameter_values_per_iteration( # noqa: C901, Needed to add special handling for node groups.
|
|
1265
1591
|
self,
|
|
1266
|
-
|
|
1592
|
+
iteration_source: BaseIterativeStartNode | BaseIterativeNodeGroup,
|
|
1267
1593
|
package_result: PackageNodesAsSerializedFlowResultSuccess,
|
|
1268
1594
|
) -> dict[int, dict[str, Any]]:
|
|
1269
1595
|
"""Get parameter values for each iteration of the loop.
|
|
@@ -1272,30 +1598,29 @@ class NodeExecutor:
|
|
|
1272
1598
|
Useful for: setting local values, sending as input for cloud publishing, or private workflow execution.
|
|
1273
1599
|
|
|
1274
1600
|
Args:
|
|
1275
|
-
|
|
1276
|
-
|
|
1601
|
+
iteration_source: The node providing iteration values (BaseIterativeStartNode or BaseIterativeNodeGroup)
|
|
1277
1602
|
package_result: PackageNodesAsSerializedFlowResultSuccess containing parameter_name_mappings
|
|
1278
1603
|
|
|
1279
1604
|
Returns:
|
|
1280
1605
|
Dict mapping iteration_index -> {startflow_param_name: value}
|
|
1281
1606
|
"""
|
|
1282
|
-
total_iterations =
|
|
1607
|
+
total_iterations = iteration_source._get_total_iterations()
|
|
1283
1608
|
|
|
1284
1609
|
# Calculate current_item values for ForEach nodes
|
|
1285
|
-
|
|
1286
|
-
iteration_items = start_node._get_iteration_items()
|
|
1610
|
+
iteration_items = iteration_source._get_iteration_items()
|
|
1287
1611
|
current_item_values = list(iteration_items)
|
|
1288
1612
|
|
|
1289
1613
|
# Calculate index values for ForLoop nodes
|
|
1290
1614
|
# For ForLoop, we need actual loop values (start, start+step, start+2*step, ...)
|
|
1291
1615
|
# not just 0-based iteration indices
|
|
1292
|
-
index_values =
|
|
1293
|
-
index_values = start_node.get_all_iteration_values()
|
|
1616
|
+
index_values = iteration_source.get_all_iteration_values()
|
|
1294
1617
|
|
|
1295
|
-
list_connections_request = ListConnectionsForNodeRequest(node_name=
|
|
1618
|
+
list_connections_request = ListConnectionsForNodeRequest(node_name=iteration_source.name)
|
|
1296
1619
|
list_connections_result = GriptapeNodes.handle_request(list_connections_request)
|
|
1297
1620
|
if not isinstance(list_connections_result, ListConnectionsForNodeResultSuccess):
|
|
1298
|
-
msg =
|
|
1621
|
+
msg = (
|
|
1622
|
+
f"Failed to list connections for node {iteration_source.name}: {list_connections_result.result_details}"
|
|
1623
|
+
)
|
|
1299
1624
|
raise RuntimeError(msg) # noqa: TRY004 This should be a runtime error because it happens during execution.
|
|
1300
1625
|
# Build parameter values for each iteration
|
|
1301
1626
|
outgoing_connections = list_connections_result.outgoing_connections
|
|
@@ -1304,7 +1629,7 @@ class NodeExecutor:
|
|
|
1304
1629
|
start_node_mapping = self.get_node_parameter_mappings(package_result, "start")
|
|
1305
1630
|
start_node_param_mappings = start_node_mapping.parameter_mappings
|
|
1306
1631
|
|
|
1307
|
-
# For each outgoing connection from
|
|
1632
|
+
# For each outgoing connection from iteration_source, find the corresponding StartFlow parameter
|
|
1308
1633
|
# The start_node_param_mappings tells us: startflow_param_name -> OriginalNodeParameter(target_node, target_param)
|
|
1309
1634
|
# We need to match the target of each connection to find the right startflow parameter
|
|
1310
1635
|
parameter_val_mappings = {}
|
|
@@ -1312,7 +1637,7 @@ class NodeExecutor:
|
|
|
1312
1637
|
iteration_values = {}
|
|
1313
1638
|
# iteration_values is going to be startflow parameter name -> value to set
|
|
1314
1639
|
|
|
1315
|
-
# For each outgoing data connection from
|
|
1640
|
+
# For each outgoing data connection from iteration_source
|
|
1316
1641
|
for conn in outgoing_connections:
|
|
1317
1642
|
source_param_name = conn.source_parameter_name
|
|
1318
1643
|
target_node_name = conn.target_node_name
|
|
@@ -1324,7 +1649,7 @@ class NodeExecutor:
|
|
|
1324
1649
|
try:
|
|
1325
1650
|
target_node = node_manager.get_node_by_name(target_node_name)
|
|
1326
1651
|
except ValueError:
|
|
1327
|
-
msg = f"Failed to get node {target_node_name} for connection {conn} from
|
|
1652
|
+
msg = f"Failed to get node {target_node_name} for connection {conn} from node {iteration_source.name}. Can't get parameter value iterations."
|
|
1328
1653
|
logger.error(msg)
|
|
1329
1654
|
raise RuntimeError(msg) # noqa: B904
|
|
1330
1655
|
if isinstance(target_node, SubflowNodeGroup):
|
|
@@ -1542,7 +1867,7 @@ class NodeExecutor:
|
|
|
1542
1867
|
|
|
1543
1868
|
def get_parameter_values_from_iterations(
|
|
1544
1869
|
self,
|
|
1545
|
-
end_loop_node: BaseIterativeEndNode,
|
|
1870
|
+
end_loop_node: BaseIterativeEndNode | BaseIterativeNodeGroup,
|
|
1546
1871
|
deserialized_flows: list[tuple[int, str, dict[str, str]]],
|
|
1547
1872
|
package_flow_result_success: PackageNodesAsSerializedFlowResultSuccess,
|
|
1548
1873
|
) -> dict[int, Any]:
|
|
@@ -1698,7 +2023,7 @@ class NodeExecutor:
|
|
|
1698
2023
|
package_result: PackageNodesAsSerializedFlowResultSuccess,
|
|
1699
2024
|
total_iterations: int,
|
|
1700
2025
|
parameter_values_per_iteration: dict[int, dict[str, Any]],
|
|
1701
|
-
end_loop_node: BaseIterativeEndNode,
|
|
2026
|
+
end_loop_node: BaseIterativeEndNode | BaseIterativeNodeGroup,
|
|
1702
2027
|
) -> tuple[dict[int, Any], list[int], dict[str, Any]]:
|
|
1703
2028
|
"""Execute loop iterations locally by deserializing and running flows.
|
|
1704
2029
|
|
|
@@ -1885,7 +2210,7 @@ class NodeExecutor:
|
|
|
1885
2210
|
package_result: PackageNodesAsSerializedFlowResultSuccess,
|
|
1886
2211
|
total_iterations: int,
|
|
1887
2212
|
parameter_values_per_iteration: dict[int, dict[str, Any]],
|
|
1888
|
-
end_loop_node: BaseIterativeEndNode,
|
|
2213
|
+
end_loop_node: BaseIterativeEndNode | BaseIterativeNodeGroup,
|
|
1889
2214
|
workflow_path: Path,
|
|
1890
2215
|
workflow_result: Any, # noqa: ARG002 - Used by wrapper methods for cleanup
|
|
1891
2216
|
file_name_prefix: str,
|
|
@@ -2013,7 +2338,7 @@ class NodeExecutor:
|
|
|
2013
2338
|
package_result: PackageNodesAsSerializedFlowResultSuccess,
|
|
2014
2339
|
total_iterations: int,
|
|
2015
2340
|
parameter_values_per_iteration: dict[int, dict[str, Any]],
|
|
2016
|
-
end_loop_node: BaseIterativeEndNode,
|
|
2341
|
+
end_loop_node: BaseIterativeEndNode | BaseIterativeNodeGroup,
|
|
2017
2342
|
) -> tuple[dict[int, Any], list[int], dict[str, Any]]:
|
|
2018
2343
|
"""Execute loop iterations sequentially in private subprocesses (no cloud publishing)."""
|
|
2019
2344
|
workflow_path, workflow_result = await self._save_workflow_file_for_loop(
|
|
@@ -2049,7 +2374,7 @@ class NodeExecutor:
|
|
|
2049
2374
|
package_result: PackageNodesAsSerializedFlowResultSuccess,
|
|
2050
2375
|
total_iterations: int,
|
|
2051
2376
|
parameter_values_per_iteration: dict[int, dict[str, Any]],
|
|
2052
|
-
end_loop_node: BaseIterativeEndNode,
|
|
2377
|
+
end_loop_node: BaseIterativeEndNode | BaseIterativeNodeGroup,
|
|
2053
2378
|
) -> tuple[dict[int, Any], list[int], dict[str, Any]]:
|
|
2054
2379
|
"""Execute loop iterations in parallel via private subprocesses (no cloud publishing)."""
|
|
2055
2380
|
workflow_path, workflow_result = await self._save_workflow_file_for_loop(
|
|
@@ -2082,7 +2407,7 @@ class NodeExecutor:
|
|
|
2082
2407
|
|
|
2083
2408
|
async def _save_workflow_file_for_loop(
|
|
2084
2409
|
self,
|
|
2085
|
-
end_loop_node: BaseIterativeEndNode,
|
|
2410
|
+
end_loop_node: BaseIterativeEndNode | BaseIterativeNodeGroup,
|
|
2086
2411
|
package_result: PackageNodesAsSerializedFlowResultSuccess,
|
|
2087
2412
|
*,
|
|
2088
2413
|
pickle_control_flow_result: bool,
|
|
@@ -2121,7 +2446,7 @@ class NodeExecutor:
|
|
|
2121
2446
|
self,
|
|
2122
2447
|
iteration_outputs: list[tuple[int, bool, dict[str, Any] | None]],
|
|
2123
2448
|
package_result: PackageNodesAsSerializedFlowResultSuccess,
|
|
2124
|
-
end_loop_node: BaseIterativeEndNode,
|
|
2449
|
+
end_loop_node: BaseIterativeEndNode | BaseIterativeNodeGroup,
|
|
2125
2450
|
) -> tuple[dict[int, Any], list[int], dict[str, Any]]:
|
|
2126
2451
|
"""Extract results from subprocess iteration outputs.
|
|
2127
2452
|
|
|
@@ -2178,7 +2503,7 @@ class NodeExecutor:
|
|
|
2178
2503
|
package_result: PackageNodesAsSerializedFlowResultSuccess,
|
|
2179
2504
|
total_iterations: int,
|
|
2180
2505
|
parameter_values_per_iteration: dict[int, dict[str, Any]],
|
|
2181
|
-
end_loop_node: BaseIterativeEndNode,
|
|
2506
|
+
end_loop_node: BaseIterativeEndNode | BaseIterativeNodeGroup,
|
|
2182
2507
|
execution_type: str,
|
|
2183
2508
|
) -> tuple[dict[int, Any], list[int], dict[str, Any]]:
|
|
2184
2509
|
"""Execute loop iterations sequentially via cloud publisher (Deadline Cloud, etc.)."""
|
|
@@ -2221,7 +2546,7 @@ class NodeExecutor:
|
|
|
2221
2546
|
package_result: PackageNodesAsSerializedFlowResultSuccess,
|
|
2222
2547
|
total_iterations: int,
|
|
2223
2548
|
parameter_values_per_iteration: dict[int, dict[str, Any]],
|
|
2224
|
-
end_loop_node: BaseIterativeEndNode,
|
|
2549
|
+
end_loop_node: BaseIterativeEndNode | BaseIterativeNodeGroup,
|
|
2225
2550
|
execution_type: str,
|
|
2226
2551
|
) -> tuple[dict[int, Any], list[int], dict[str, Any]]:
|
|
2227
2552
|
"""Execute loop iterations in parallel via cloud publisher (Deadline Cloud, etc.)."""
|
|
@@ -5,6 +5,8 @@ from typing import TypedDict
|
|
|
5
5
|
|
|
6
6
|
import httpx
|
|
7
7
|
|
|
8
|
+
from griptape_nodes.retained_mode.events.os_events import ExistingFilePolicy
|
|
9
|
+
|
|
8
10
|
logger = logging.getLogger("griptape_nodes")
|
|
9
11
|
|
|
10
12
|
|
|
@@ -12,6 +14,7 @@ class CreateSignedUploadUrlResponse(TypedDict):
|
|
|
12
14
|
"""Response type for create_signed_upload_url method."""
|
|
13
15
|
|
|
14
16
|
url: str
|
|
17
|
+
file_path: str
|
|
15
18
|
headers: dict
|
|
16
19
|
method: str
|
|
17
20
|
|
|
@@ -28,11 +31,14 @@ class BaseStorageDriver(ABC):
|
|
|
28
31
|
self.workspace_directory = workspace_directory
|
|
29
32
|
|
|
30
33
|
@abstractmethod
|
|
31
|
-
def create_signed_upload_url(
|
|
34
|
+
def create_signed_upload_url(
|
|
35
|
+
self, path: Path, existing_file_policy: ExistingFilePolicy = ExistingFilePolicy.OVERWRITE
|
|
36
|
+
) -> CreateSignedUploadUrlResponse:
|
|
32
37
|
"""Create a signed upload URL for the given path.
|
|
33
38
|
|
|
34
39
|
Args:
|
|
35
40
|
path: The path of the file to create a signed URL for.
|
|
41
|
+
existing_file_policy: How to handle existing files. Defaults to OVERWRITE for backward compatibility.
|
|
36
42
|
|
|
37
43
|
Returns:
|
|
38
44
|
CreateSignedUploadUrlResponse: A dictionary containing the signed URL, headers, and operation type.
|
|
@@ -69,12 +75,15 @@ class BaseStorageDriver(ABC):
|
|
|
69
75
|
"""
|
|
70
76
|
...
|
|
71
77
|
|
|
72
|
-
def upload_file(
|
|
78
|
+
def upload_file(
|
|
79
|
+
self, path: Path, file_content: bytes, existing_file_policy: ExistingFilePolicy = ExistingFilePolicy.OVERWRITE
|
|
80
|
+
) -> str:
|
|
73
81
|
"""Upload a file to storage.
|
|
74
82
|
|
|
75
83
|
Args:
|
|
76
84
|
path: The path of the file to upload.
|
|
77
85
|
file_content: The file content as bytes.
|
|
86
|
+
existing_file_policy: How to handle existing files. Defaults to OVERWRITE for backward compatibility.
|
|
78
87
|
|
|
79
88
|
Returns:
|
|
80
89
|
The URL where the file can be accessed.
|
|
@@ -84,7 +93,7 @@ class BaseStorageDriver(ABC):
|
|
|
84
93
|
"""
|
|
85
94
|
try:
|
|
86
95
|
# Get signed upload URL
|
|
87
|
-
upload_response = self.create_signed_upload_url(path)
|
|
96
|
+
upload_response = self.create_signed_upload_url(path, existing_file_policy)
|
|
88
97
|
|
|
89
98
|
# Upload the file using the signed URL
|
|
90
99
|
response = httpx.request(
|
|
@@ -6,6 +6,7 @@ from urllib.parse import urljoin
|
|
|
6
6
|
import httpx
|
|
7
7
|
|
|
8
8
|
from griptape_nodes.drivers.storage.base_storage_driver import BaseStorageDriver, CreateSignedUploadUrlResponse
|
|
9
|
+
from griptape_nodes.retained_mode.events.os_events import ExistingFilePolicy
|
|
9
10
|
|
|
10
11
|
logger = logging.getLogger("griptape_nodes")
|
|
11
12
|
|
|
@@ -38,7 +39,17 @@ class GriptapeCloudStorageDriver(BaseStorageDriver):
|
|
|
38
39
|
|
|
39
40
|
self.bucket_id = bucket_id
|
|
40
41
|
|
|
41
|
-
def create_signed_upload_url(
|
|
42
|
+
def create_signed_upload_url(
|
|
43
|
+
self, path: Path, existing_file_policy: ExistingFilePolicy = ExistingFilePolicy.OVERWRITE
|
|
44
|
+
) -> CreateSignedUploadUrlResponse:
|
|
45
|
+
if existing_file_policy != ExistingFilePolicy.OVERWRITE:
|
|
46
|
+
logger.warning(
|
|
47
|
+
"Griptape Cloud storage only supports OVERWRITE policy. "
|
|
48
|
+
"Requested policy '%s' will be ignored for file: %s",
|
|
49
|
+
existing_file_policy.value,
|
|
50
|
+
path,
|
|
51
|
+
)
|
|
52
|
+
|
|
42
53
|
self._create_asset(path.as_posix())
|
|
43
54
|
|
|
44
55
|
url = urljoin(self.base_url, f"/api/buckets/{self.bucket_id}/asset-urls/{path.as_posix()}")
|
|
@@ -52,7 +63,12 @@ class GriptapeCloudStorageDriver(BaseStorageDriver):
|
|
|
52
63
|
|
|
53
64
|
response_data = response.json()
|
|
54
65
|
|
|
55
|
-
return {
|
|
66
|
+
return {
|
|
67
|
+
"url": response_data["url"],
|
|
68
|
+
"headers": response_data.get("headers", {}),
|
|
69
|
+
"method": "PUT",
|
|
70
|
+
"file_path": str(path),
|
|
71
|
+
}
|
|
56
72
|
|
|
57
73
|
def create_signed_download_url(self, path: Path) -> str:
|
|
58
74
|
url = urljoin(self.base_url, f"/api/buckets/{self.bucket_id}/asset-urls/{path.as_posix()}")
|