siliconcompiler 0.34.1__py3-none-any.whl → 0.34.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. siliconcompiler/__init__.py +23 -4
  2. siliconcompiler/__main__.py +1 -7
  3. siliconcompiler/_metadata.py +1 -1
  4. siliconcompiler/apps/_common.py +104 -23
  5. siliconcompiler/apps/sc.py +4 -8
  6. siliconcompiler/apps/sc_dashboard.py +6 -4
  7. siliconcompiler/apps/sc_install.py +10 -6
  8. siliconcompiler/apps/sc_issue.py +7 -5
  9. siliconcompiler/apps/sc_remote.py +1 -1
  10. siliconcompiler/apps/sc_server.py +9 -14
  11. siliconcompiler/apps/sc_show.py +7 -6
  12. siliconcompiler/apps/smake.py +130 -94
  13. siliconcompiler/apps/utils/replay.py +4 -7
  14. siliconcompiler/apps/utils/summarize.py +3 -5
  15. siliconcompiler/asic.py +420 -0
  16. siliconcompiler/checklist.py +25 -2
  17. siliconcompiler/cmdlineschema.py +534 -0
  18. siliconcompiler/constraints/__init__.py +17 -0
  19. siliconcompiler/constraints/asic_component.py +378 -0
  20. siliconcompiler/constraints/asic_floorplan.py +449 -0
  21. siliconcompiler/constraints/asic_pins.py +489 -0
  22. siliconcompiler/constraints/asic_timing.py +517 -0
  23. siliconcompiler/core.py +10 -35
  24. siliconcompiler/data/templates/tcl/manifest.tcl.j2 +8 -0
  25. siliconcompiler/dependencyschema.py +96 -202
  26. siliconcompiler/design.py +327 -241
  27. siliconcompiler/filesetschema.py +250 -0
  28. siliconcompiler/flowgraph.py +298 -106
  29. siliconcompiler/fpga.py +124 -1
  30. siliconcompiler/library.py +331 -0
  31. siliconcompiler/metric.py +327 -92
  32. siliconcompiler/metrics/__init__.py +7 -0
  33. siliconcompiler/metrics/asic.py +245 -0
  34. siliconcompiler/metrics/fpga.py +220 -0
  35. siliconcompiler/package/__init__.py +391 -67
  36. siliconcompiler/package/git.py +92 -16
  37. siliconcompiler/package/github.py +114 -22
  38. siliconcompiler/package/https.py +79 -16
  39. siliconcompiler/packageschema.py +341 -16
  40. siliconcompiler/pathschema.py +255 -0
  41. siliconcompiler/pdk.py +566 -1
  42. siliconcompiler/project.py +1460 -0
  43. siliconcompiler/record.py +38 -1
  44. siliconcompiler/remote/__init__.py +5 -2
  45. siliconcompiler/remote/client.py +11 -6
  46. siliconcompiler/remote/schema.py +5 -23
  47. siliconcompiler/remote/server.py +41 -54
  48. siliconcompiler/report/__init__.py +3 -3
  49. siliconcompiler/report/dashboard/__init__.py +48 -14
  50. siliconcompiler/report/dashboard/cli/__init__.py +99 -21
  51. siliconcompiler/report/dashboard/cli/board.py +364 -179
  52. siliconcompiler/report/dashboard/web/__init__.py +90 -12
  53. siliconcompiler/report/dashboard/web/components/__init__.py +219 -240
  54. siliconcompiler/report/dashboard/web/components/flowgraph.py +49 -26
  55. siliconcompiler/report/dashboard/web/components/graph.py +139 -100
  56. siliconcompiler/report/dashboard/web/layouts/__init__.py +29 -1
  57. siliconcompiler/report/dashboard/web/layouts/_common.py +38 -2
  58. siliconcompiler/report/dashboard/web/layouts/vertical_flowgraph.py +39 -26
  59. siliconcompiler/report/dashboard/web/layouts/vertical_flowgraph_node_tab.py +50 -50
  60. siliconcompiler/report/dashboard/web/layouts/vertical_flowgraph_sac_tabs.py +49 -46
  61. siliconcompiler/report/dashboard/web/state.py +141 -14
  62. siliconcompiler/report/dashboard/web/utils/__init__.py +79 -16
  63. siliconcompiler/report/dashboard/web/utils/file_utils.py +74 -11
  64. siliconcompiler/report/dashboard/web/viewer.py +25 -1
  65. siliconcompiler/report/report.py +5 -2
  66. siliconcompiler/report/summary_image.py +29 -11
  67. siliconcompiler/scheduler/__init__.py +9 -1
  68. siliconcompiler/scheduler/docker.py +81 -4
  69. siliconcompiler/scheduler/run_node.py +37 -20
  70. siliconcompiler/scheduler/scheduler.py +211 -36
  71. siliconcompiler/scheduler/schedulernode.py +394 -60
  72. siliconcompiler/scheduler/send_messages.py +77 -29
  73. siliconcompiler/scheduler/slurm.py +76 -12
  74. siliconcompiler/scheduler/taskscheduler.py +142 -21
  75. siliconcompiler/schema/__init__.py +0 -4
  76. siliconcompiler/schema/baseschema.py +338 -59
  77. siliconcompiler/schema/editableschema.py +14 -6
  78. siliconcompiler/schema/journal.py +28 -17
  79. siliconcompiler/schema/namedschema.py +22 -14
  80. siliconcompiler/schema/parameter.py +89 -28
  81. siliconcompiler/schema/parametertype.py +2 -0
  82. siliconcompiler/schema/parametervalue.py +258 -15
  83. siliconcompiler/schema/safeschema.py +25 -2
  84. siliconcompiler/schema/schema_cfg.py +23 -19
  85. siliconcompiler/schema/utils.py +2 -2
  86. siliconcompiler/schema_obj.py +24 -5
  87. siliconcompiler/tool.py +1131 -265
  88. siliconcompiler/tools/bambu/__init__.py +41 -0
  89. siliconcompiler/tools/builtin/concatenate.py +2 -2
  90. siliconcompiler/tools/builtin/minimum.py +2 -1
  91. siliconcompiler/tools/builtin/mux.py +2 -1
  92. siliconcompiler/tools/builtin/nop.py +2 -1
  93. siliconcompiler/tools/builtin/verify.py +2 -1
  94. siliconcompiler/tools/klayout/__init__.py +95 -0
  95. siliconcompiler/tools/openroad/__init__.py +289 -0
  96. siliconcompiler/tools/openroad/scripts/apr/preamble.tcl +3 -0
  97. siliconcompiler/tools/openroad/scripts/apr/sc_detailed_route.tcl +7 -2
  98. siliconcompiler/tools/openroad/scripts/apr/sc_global_route.tcl +8 -4
  99. siliconcompiler/tools/openroad/scripts/apr/sc_init_floorplan.tcl +9 -5
  100. siliconcompiler/tools/openroad/scripts/common/write_images.tcl +5 -1
  101. siliconcompiler/tools/slang/__init__.py +1 -1
  102. siliconcompiler/tools/slang/elaborate.py +2 -1
  103. siliconcompiler/tools/vivado/scripts/sc_run.tcl +1 -1
  104. siliconcompiler/tools/vivado/scripts/sc_syn_fpga.tcl +8 -1
  105. siliconcompiler/tools/vivado/syn_fpga.py +6 -0
  106. siliconcompiler/tools/vivado/vivado.py +35 -2
  107. siliconcompiler/tools/vpr/__init__.py +150 -0
  108. siliconcompiler/tools/yosys/__init__.py +369 -1
  109. siliconcompiler/tools/yosys/scripts/procs.tcl +0 -1
  110. siliconcompiler/toolscripts/_tools.json +5 -10
  111. siliconcompiler/utils/__init__.py +66 -0
  112. siliconcompiler/utils/flowgraph.py +2 -2
  113. siliconcompiler/utils/issue.py +2 -1
  114. siliconcompiler/utils/logging.py +14 -0
  115. siliconcompiler/utils/multiprocessing.py +256 -0
  116. siliconcompiler/utils/showtools.py +10 -0
  117. {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/METADATA +6 -6
  118. {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/RECORD +122 -115
  119. {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/entry_points.txt +3 -0
  120. siliconcompiler/schema/cmdlineschema.py +0 -250
  121. siliconcompiler/schema/packageschema.py +0 -101
  122. siliconcompiler/toolscripts/rhel8/install-slang.sh +0 -40
  123. siliconcompiler/toolscripts/rhel9/install-slang.sh +0 -40
  124. siliconcompiler/toolscripts/ubuntu20/install-slang.sh +0 -47
  125. siliconcompiler/toolscripts/ubuntu22/install-slang.sh +0 -37
  126. siliconcompiler/toolscripts/ubuntu24/install-slang.sh +0 -37
  127. {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/WHEEL +0 -0
  128. {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/licenses/LICENSE +0 -0
  129. {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/top_level.txt +0 -0
@@ -10,7 +10,23 @@ from siliconcompiler import NodeStatus
10
10
 
11
11
 
12
12
  class FlowgraphSchema(NamedSchema):
13
+ '''
14
+ Schema for defining and interacting with a flowgraph.
15
+
16
+ A flowgraph is a directed acyclic graph (DAG) that represents the
17
+ compilation flow. Each node in the graph is a step/index pair that
18
+
19
+ maps to a specific tool task, and edges represent dependencies between
20
+ these tasks.
21
+ '''
22
+
13
23
  def __init__(self, name=None):
24
+ '''
25
+ Initializes a new FlowgraphSchema object.
26
+
27
+ Args:
28
+ name (str, optional): The name of the flowgraph. Defaults to None.
29
+ '''
14
30
  super().__init__()
15
31
  self.set_name(name)
16
32
 
@@ -21,7 +37,9 @@ class FlowgraphSchema(NamedSchema):
21
37
 
22
38
  def __clear_cache(self):
23
39
  '''
24
- Clear the cache of node information
40
+ Clears the internal cache for memoized flowgraph properties.
41
+
42
+ This should be called any time the graph structure is modified.
25
43
  '''
26
44
 
27
45
  self.__cache_nodes = None
@@ -38,27 +56,27 @@ class FlowgraphSchema(NamedSchema):
38
56
  '''
39
57
  Creates a flowgraph node.
40
58
 
41
- Creates a flowgraph node by binding a step to a tool specific task.
42
- A tool can be an external executable or one of the built in functions
43
- in the SiliconCompiler framework). Built in functions include: minimum,
44
- maximum, join, mux, verify.
59
+ Creates a flowgraph node by binding a step to a tool-specific task.
60
+ A tool can be an external executable or one of the built-in functions
61
+ in the SiliconCompiler framework (e.g., minimum, maximum, join).
45
62
 
46
63
  The method modifies the following schema parameters:
47
64
 
48
- * [<step>,<index>,tool,<tool>]
49
- * [<step>,<index>,task,<task>]
50
- * [<step>,<index>,taskmodule,<taskmodule>]
65
+ * `['<step>', '<index>', 'tool']`
66
+ * `['<step>', '<index>', 'task']`
67
+ * `['<step>', '<index>', 'taskmodule']`
51
68
 
52
69
  Args:
53
- step (str): Step name
54
- task (module/str): Task to associate with this node
55
- index (int/str): Step index
70
+ step (str): Step name for the node.
71
+ task (module or str): The task to associate with this node. Can be
72
+ a module object or a string in the format '<tool>.<task>'.
73
+ index (int or str): Index for the step. Defaults to 0.
56
74
 
57
75
  Examples:
58
- >>> import siliconcomiler.tools.openroad.place as place
59
- >>> flow.node('apr_place', place, index=0)
60
- Creates a 'place' task with step='apr_place' and index=0 and binds it to the
61
- 'openroad' tool.
76
+ >>> import siliconcompiler.tools.openroad as openroad
77
+ >>> flow.node('place', openroad.place, index=0)
78
+ # Creates a node for the 'place' task in the 'openroad' tool,
79
+ # identified by step='place' and index=0.
62
80
  '''
63
81
 
64
82
  if step in (Schema.GLOBAL_KEY, 'default', 'sc_collected_files'):
@@ -68,6 +86,11 @@ class FlowgraphSchema(NamedSchema):
68
86
  if index in (Schema.GLOBAL_KEY, 'default'):
69
87
  raise ValueError(f"{index} is a reserved name")
70
88
 
89
+ if '/' in step:
90
+ raise ValueError(f"{step} is not a valid step, it cannot contain '/'")
91
+ if '/' in index:
92
+ raise ValueError(f"{index} is not a valid index, it cannot contain '/'")
93
+
71
94
  # Determine task name and module
72
95
  task_module = None
73
96
  if isinstance(task, str):
@@ -83,11 +106,6 @@ class FlowgraphSchema(NamedSchema):
83
106
  raise ValueError(f"{task} is not a valid task, it must be associated with "
84
107
  "a tool '<tool>.<task>'.")
85
108
 
86
- if '/' in step:
87
- raise ValueError(f"{step} is not a valid step, it cannot contain '/'")
88
- if '/' in index:
89
- raise ValueError(f"{index} is not a valid index, it cannot contain '/'")
90
-
91
109
  tool_name, task_name = task_parts[-2:]
92
110
 
93
111
  # bind tool to node
@@ -104,26 +122,26 @@ class FlowgraphSchema(NamedSchema):
104
122
  Connects the output of a tail node with the input of a head node by
105
123
  setting the 'input' field of the head node in the schema flowgraph.
106
124
 
107
- The method modifies the following parameters:
125
+ The method modifies the following parameter:
108
126
 
109
- [<head>,<head_index>,input]
127
+ * `['<head>', '<head_index>', 'input']`
110
128
 
111
129
  Args:
112
- tail (str): Name of tail node
113
- head (str): Name of head node
114
- tail_index (int/str): Index of tail node to connect
115
- head_index (int/str): Index of head node to connect
130
+ tail (str): Step name of the tail node.
131
+ head (str): Step name of the head node.
132
+ tail_index (int or str): Index of the tail node. Defaults to 0.
133
+ head_index (int or str): Index of the head node. Defaults to 0.
116
134
 
117
135
  Examples:
118
136
  >>> chip.edge('place', 'cts')
119
- Creates a directed edge from place to cts.
137
+ # Creates a directed edge from ('place', '0') to ('cts', '0').
120
138
  '''
121
139
  head_index = str(head_index)
122
140
  tail_index = str(tail_index)
123
141
 
124
142
  for step, index in [(head, head_index), (tail, tail_index)]:
125
143
  if not self.valid(step, index):
126
- raise ValueError(f"{step}/{index} is not a defined node in {self.name()}.")
144
+ raise ValueError(f"{step}/{index} is not a defined node in {self.name}.")
127
145
 
128
146
  tail_node = (tail, tail_index)
129
147
  if tail_node in self.get(head, head_index, 'input'):
@@ -135,15 +153,16 @@ class FlowgraphSchema(NamedSchema):
135
153
 
136
154
  def remove_node(self, step, index=None):
137
155
  '''
138
- Remove a flowgraph node.
156
+ Removes a flowgraph node and reconnects its inputs to its outputs.
139
157
 
140
158
  Args:
141
- step (str): Step name
142
- index (int/str): Step index
159
+ step (str): Step name of the node to remove.
160
+ index (int or str, optional): Index of the node to remove. If None,
161
+ all nodes for the given step are removed. Defaults to None.
143
162
  '''
144
163
 
145
164
  if step not in self.getkeys():
146
- raise ValueError(f'{step} is not a valid step in {self.name()}')
165
+ raise ValueError(f'{step} is not a valid step in {self.name}')
147
166
 
148
167
  if index is None:
149
168
  # Iterate over all indexes
@@ -153,7 +172,7 @@ class FlowgraphSchema(NamedSchema):
153
172
 
154
173
  index = str(index)
155
174
  if index not in self.getkeys(step):
156
- raise ValueError(f'{index} is not a valid index for {step} in {self.name()}')
175
+ raise ValueError(f'{index} is not a valid index for {step} in {self.name}')
157
176
 
158
177
  # Save input edges
159
178
  node = (step, index)
@@ -178,21 +197,24 @@ class FlowgraphSchema(NamedSchema):
178
197
 
179
198
  def insert_node(self, step, task, before_step, index=0, before_index=0):
180
199
  '''
181
- Insert a new node after the specified node
200
+ Inserts a new node in the graph before a specified node.
201
+
202
+ The new node is placed between the `before` node and all of its
203
+ original inputs.
182
204
 
183
205
  Args:
184
- step (str): Step name
185
- index (int/str): Step index
186
- task (module/str): Task to associate with this node
187
- before_step (str): name of step to insert task after
188
- before_index (int/str): index of step to insert task after
206
+ step (str): Step name for the new node.
207
+ task (module or str): Task to associate with the new node.
208
+ before_step (str): Step name of the existing node to insert before.
209
+ index (int or str): Index for the new node. Defaults to 0.
210
+ before_index (int or str): Index of the existing node. Defaults to 0.
189
211
  '''
190
212
 
191
213
  index = str(index)
192
214
  before_index = str(before_index)
193
215
 
194
216
  if (before_step, before_index) not in self.get_nodes():
195
- raise ValueError(f'{before_step}/{before_index} is not a valid node in {self.name()}')
217
+ raise ValueError(f'{before_step}/{before_index} is not a valid node in {self.name}')
196
218
 
197
219
  # add the node
198
220
  self.node(step, task, index=index)
@@ -210,15 +232,16 @@ class FlowgraphSchema(NamedSchema):
210
232
  ###########################################################################
211
233
  def graph(self, subflow, name=None):
212
234
  '''
213
- Instantiates a named flow as a graph in the current flowgraph.
235
+ Instantiates a sub-flowgraph within the current flowgraph.
214
236
 
215
237
  Args:
216
- subflow (str): Name of flow to instantiate
217
- name (str): Name of instance
238
+ subflow (FlowgraphSchema): The flowgraph to instantiate.
239
+ name (str, optional): A prefix to add to the names of the
240
+ instantiated steps to ensure they are unique. Defaults to None.
218
241
 
219
- Examples:
220
- >>> chip.graph(asicflow)
221
- Instantiates a flow named 'asicflow'.
242
+ Raises:
243
+ ValueError: If `subflow` is not a `FlowgraphSchema` object, or if
244
+ a step from the sub-flowgraph already exists in the current graph.
222
245
  '''
223
246
  if not isinstance(subflow, FlowgraphSchema):
224
247
  raise ValueError(f"subflow must a FlowgraphSchema, not: {type(subflow)}")
@@ -252,7 +275,12 @@ class FlowgraphSchema(NamedSchema):
252
275
 
253
276
  def get_nodes(self):
254
277
  '''
255
- Returns all the nodes defined in this flowgraph
278
+ Returns a sorted tuple of all nodes defined in this flowgraph.
279
+
280
+ A node is represented as a `(step, index)` tuple.
281
+
282
+ Returns:
283
+ tuple[tuple(str,str)]: All nodes in the graph.
256
284
  '''
257
285
  if self.__cache_nodes is not None:
258
286
  return self.__cache_nodes
@@ -268,8 +296,12 @@ class FlowgraphSchema(NamedSchema):
268
296
 
269
297
  def get_entry_nodes(self):
270
298
  '''
271
- Collect all step/indices that represent the entry
272
- nodes for the flowgraph
299
+ Collects all nodes that are entry points to the flowgraph.
300
+
301
+ Entry nodes are those with no inputs.
302
+
303
+ Returns:
304
+ tuple[tuple(str,str)]: All entry nodes in the graph.
273
305
  '''
274
306
 
275
307
  if self.__cache_nodes_entry is not None:
@@ -286,8 +318,12 @@ class FlowgraphSchema(NamedSchema):
286
318
 
287
319
  def get_exit_nodes(self):
288
320
  '''
289
- Collect all step/indices that represent the exit
290
- nodes for the flowgraph
321
+ Collects all nodes that are exit points of the flowgraph.
322
+
323
+ Exit nodes are those that are not inputs to any other node.
324
+
325
+ Returns:
326
+ tuple[tuple(str,str)]: All exit nodes in the graph.
291
327
  '''
292
328
 
293
329
  if self.__cache_nodes_exit is not None:
@@ -307,11 +343,15 @@ class FlowgraphSchema(NamedSchema):
307
343
 
308
344
  def get_execution_order(self, reverse=False):
309
345
  '''
310
- Generates a list of nodes in the order they will be executed.
346
+ Generates a topologically sorted list of nodes for execution.
311
347
 
312
348
  Args:
313
- reverse (boolean): if True, the nodes will be ordered from exit nodes
314
- to entry nodes.
349
+ reverse (bool): If True, the order is reversed, from exit nodes
350
+ to entry nodes. Defaults to False.
351
+
352
+ Returns:
353
+ tuple[tuple[tuple(str,str)]]: A tuple of tuples, where each inner
354
+ tuple represents a level of nodes that can be executed in parallel.
315
355
  '''
316
356
 
317
357
  if reverse:
@@ -387,11 +427,14 @@ class FlowgraphSchema(NamedSchema):
387
427
 
388
428
  def get_node_outputs(self, step, index):
389
429
  '''
390
- Returns the nodes the given nodes provides input to.
430
+ Returns the nodes that the given node provides input to.
391
431
 
392
432
  Args:
393
- step (str): step name
394
- index (str/int) index name
433
+ step (str): Step name of the source node.
434
+ index (str or int): Index of the source node.
435
+
436
+ Returns:
437
+ tuple[tuple(str,str)]: A tuple of destination nodes.
395
438
  '''
396
439
 
397
440
  index = str(index)
@@ -423,12 +466,15 @@ class FlowgraphSchema(NamedSchema):
423
466
 
424
467
  def __find_loops(self, step, index, path=None):
425
468
  '''
426
- Search for loops in the graph.
469
+ Internal helper to search for loops in the graph via depth-first search.
427
470
 
428
471
  Args:
429
- step (str): step name to start from
430
- index (str) index name to start from
431
- path (list of nodes): path in graph so far
472
+ step (str): Step name to start from.
473
+ index (str): Index name to start from.
474
+ path (list, optional): The path taken so far. Defaults to None.
475
+
476
+ Returns:
477
+ list: A list of nodes forming a loop, or None if no loop is found.
432
478
  '''
433
479
  if path is None:
434
480
  path = []
@@ -448,17 +494,20 @@ class FlowgraphSchema(NamedSchema):
448
494
 
449
495
  def validate(self, logger=None):
450
496
  '''
451
- Check if flowgraph is valid.
497
+ Checks if the flowgraph is valid.
452
498
 
453
- * Checks if all edges have valid nodes
454
- * Checks that there are no duplicate edges
455
- * Checks if nodes are defined properly
456
- * Checks if there are any loops present in the graph
457
-
458
- Returns True if valid, False otherwise.
499
+ This method performs several checks:
500
+ * All edges must point to and from valid nodes.
501
+ * There should be no duplicate edges.
502
+ * All nodes must have their tool, task, and taskmodule defined.
503
+ * The graph must not contain any loops (it must be a DAG).
459
504
 
460
505
  Args:
461
- logger (logging.Logger): logger to use for reporting
506
+ logger (logging.Logger, optional): A logger to use for reporting
507
+ errors. Defaults to None.
508
+
509
+ Returns:
510
+ bool: True if the graph is valid, False otherwise.
462
511
  '''
463
512
 
464
513
  error = False
@@ -474,14 +523,14 @@ class FlowgraphSchema(NamedSchema):
474
523
  in_step, in_index = node
475
524
  if logger:
476
525
  logger.error(f'Duplicate edge from {in_step}/{in_index} to '
477
- f'{step}/{index} in the {self.name()} flowgraph')
526
+ f'{step}/{index} in the {self.name} flowgraph')
478
527
  error = True
479
528
 
480
529
  diff_nodes = check_nodes.difference(self.get_nodes())
481
530
  if diff_nodes:
482
531
  if logger:
483
532
  for step, index in diff_nodes:
484
- logger.error(f'{step}/{index} is missing in the {self.name()} flowgraph')
533
+ logger.error(f'{step}/{index} is missing in the {self.name} flowgraph')
485
534
  error = True
486
535
 
487
536
  # Detect missing definitions
@@ -490,7 +539,7 @@ class FlowgraphSchema(NamedSchema):
490
539
  if not self.get(step, index, item):
491
540
  if logger:
492
541
  logger.error(f'{step}/{index} is missing a {item} definition in the '
493
- f'{self.name()} flowgraph')
542
+ f'{self.name} flowgraph')
494
543
  error = True
495
544
 
496
545
  # detect loops
@@ -500,49 +549,90 @@ class FlowgraphSchema(NamedSchema):
500
549
  error = True
501
550
  if logger:
502
551
  loop_path = [f"{step}/{index}" for step, index in loop_path]
503
- logger.error(f"{' -> '.join(loop_path)} forms a loop in {self.name()}")
552
+ logger.error(f"{' -> '.join(loop_path)} forms a loop in {self.name}")
504
553
 
505
554
  return not error
506
555
 
556
+ def __get_task_module(self, name):
557
+ '''
558
+ Internal helper to import and cache a task module by name.
559
+ '''
560
+ # Create cache
561
+ if self.__cache_tasks is None:
562
+ self.__cache_tasks = {}
563
+
564
+ if name in self.__cache_tasks:
565
+ return self.__cache_tasks[name]
566
+
567
+ self.__cache_tasks[name] = importlib.import_module(name)
568
+ return self.__cache_tasks[name]
569
+
507
570
  def get_task_module(self, step, index):
508
571
  """
509
- Returns the module for a given task
572
+ Returns the imported Python module for a given task node.
510
573
 
511
574
  Args:
512
- step (str): Step name
513
- index (int/str): Step index
575
+ step (str): Step name of the node.
576
+ index (int or str): Index of the node.
577
+
578
+ Returns:
579
+ module: The imported task module.
514
580
  """
515
581
 
516
582
  index = str(index)
517
583
 
518
584
  if (step, index) not in self.get_nodes():
519
- raise ValueError(f"{step}/{index} is not a valid node in {self.name()}.")
585
+ raise ValueError(f"{step}/{index} is not a valid node in {self.name}.")
520
586
 
521
- taskmodule = self.get(step, index, 'taskmodule')
587
+ return self.__get_task_module(self.get(step, index, 'taskmodule'))
522
588
 
523
- # Create cache
524
- if self.__cache_tasks is None:
525
- self.__cache_tasks = {}
589
+ def get_all_tasks(self):
590
+ '''
591
+ Returns all unique task modules used in this flowgraph.
592
+
593
+ Returns:
594
+ set[module]: A set of all imported task modules.
595
+ '''
596
+ tasks = set()
597
+ for step, index in self.get_nodes():
598
+ tasks.add(self.get_task_module(step, index))
599
+ return tasks
526
600
 
527
- if taskmodule in self.__cache_tasks:
528
- return self.__cache_tasks[taskmodule]
601
+ @classmethod
602
+ def _getdict_type(cls) -> str:
603
+ """
604
+ Returns the metadata type for `getdict` serialization.
605
+ """
529
606
 
530
- self.__cache_tasks[taskmodule] = importlib.import_module(taskmodule)
531
- return self.__cache_tasks[taskmodule]
607
+ return FlowgraphSchema.__name__
532
608
 
533
609
 
534
610
  class RuntimeFlowgraph:
535
611
  '''
536
- Runtime representation of a flowgraph
612
+ A runtime representation of a flowgraph for a specific execution.
537
613
 
538
- Args:
539
- base (:class:`FlowgraphSchema`): base flowgraph for this runtime
540
- args (tuple of step, index): specific node to apply runtime to
541
- from_steps (list of steps): steps to start the runtime from
542
- to_steps (list of steps): step to end the runtime at
543
- prune_nodes (list of nodes): nodes to remove from execution
614
+ This class creates a "view" of a base flowgraph that considers runtime
615
+ options such as the start step (`-from`), end step (`-to`), and nodes to
616
+ exclude (`-prune`). It computes the precise subgraph of nodes that need
617
+ to be executed for a given run.
544
618
  '''
619
+
545
620
  def __init__(self, base, args=None, from_steps=None, to_steps=None, prune_nodes=None):
621
+ '''
622
+ Initializes a new RuntimeFlowgraph.
623
+
624
+ Args:
625
+ base (FlowgraphSchema): The base flowgraph to create a view of.
626
+ args (tuple[str, str], optional): A specific `(step, index)` to run.
627
+ If provided, this overrides `from_steps` and `to_steps`.
628
+ Defaults to None.
629
+ from_steps (list[str], optional): List of step names to start execution
630
+ from. Defaults to the base graph's entry nodes.
631
+ to_steps (list[str], optional): List of step names to end execution at.
632
+ Defaults to the base graph's exit nodes.
633
+ prune_nodes (list[tuple(str,str)], optional): A list of `(step, index)`
634
+ nodes to exclude from the graph. Defaults to None.
635
+ '''
546
636
  if not all([hasattr(base, attr) for attr in dir(FlowgraphSchema)]):
547
637
  raise ValueError(f"base must a FlowgraphSchema, not: {type(base)}")
548
638
 
@@ -594,6 +684,21 @@ class RuntimeFlowgraph:
594
684
  self.__compute_graph()
595
685
 
596
686
  def __walk_graph(self, node, path=None, reverse=True):
687
+ '''
688
+ Internal helper to recursively walk the graph to find all connected nodes.
689
+
690
+ This walk respects the runtime boundaries (`-from`, `-to`, `-prune`).
691
+
692
+ Args:
693
+ node (tuple(str,str)): The node to start the walk from.
694
+ path (list, optional): The path taken so far, used for cycle
695
+ detection. Defaults to None.
696
+ reverse (bool, optional): If True, walks backwards along inputs.
697
+ If False, walks forwards along outputs. Defaults to True.
698
+
699
+ Returns:
700
+ set[tuple(str,str)]: The set of nodes visited during the walk.
701
+ '''
597
702
  if node in self.__prune:
598
703
  return set()
599
704
 
@@ -622,7 +727,10 @@ class RuntimeFlowgraph:
622
727
 
623
728
  def __compute_graph(self):
624
729
  '''
625
- Precompute graph information
730
+ Internal helper to precompute the runtime graph information.
731
+
732
+ This method determines the final set of nodes, entry/exit points, and
733
+ the execution order based on the runtime constraints.
626
734
  '''
627
735
 
628
736
  self.__nodes = set()
@@ -651,35 +759,51 @@ class RuntimeFlowgraph:
651
759
 
652
760
  def get_nodes(self):
653
761
  '''
654
- Returns the nodes available in this graph
762
+ Returns the nodes that are part of this runtime graph.
763
+
764
+ Returns:
765
+ tuple[tuple(str,str)]: A tuple of all nodes in the runtime graph.
655
766
  '''
656
767
  return self.__nodes
657
768
 
658
769
  def get_execution_order(self):
659
770
  '''
660
- Returns the execution order of the nodes
771
+ Returns the execution order of the nodes in this runtime graph.
772
+
773
+ Returns:
774
+ tuple[tuple[tuple(str,str)]]: A tuple of tuples representing
775
+ parallel execution levels.
661
776
  '''
662
777
  return self.__execution_order
663
778
 
664
779
  def get_entry_nodes(self):
665
780
  '''
666
- Returns the entry nodes for this graph
781
+ Returns the entry nodes for this runtime graph.
782
+
783
+ Returns:
784
+ tuple[tuple(str,str)]: A tuple of all entry nodes.
667
785
  '''
668
786
  return self.__from
669
787
 
670
788
  def get_exit_nodes(self):
671
789
  '''
672
- Returns the exit nodes for this graph
790
+ Returns the exit nodes for this runtime graph.
791
+
792
+ Returns:
793
+ tuple[tuple(str,str)]: A tuple of all exit nodes.
673
794
  '''
674
795
  return self.__to
675
796
 
676
797
  def get_nodes_starting_at(self, step, index):
677
798
  '''
678
- Returns all the nodes that the given step, index connect to
799
+ Returns all nodes reachable from a given starting node in this runtime graph.
679
800
 
680
801
  Args:
681
- step (str): step to start from
682
- index (str/int): index to start from
802
+ step (str): The step name of the starting node.
803
+ index (str or int): The index of the starting node.
804
+
805
+ Returns:
806
+ tuple[tuple(str,str)]: A tuple of all reachable nodes.
683
807
  '''
684
808
  index = str(index)
685
809
 
@@ -689,6 +813,21 @@ class RuntimeFlowgraph:
689
813
  return tuple(sorted(self.__walk_graph((step, str(index)), reverse=False)))
690
814
 
691
815
  def get_node_inputs(self, step, index, record=None):
816
+ '''
817
+ Gets the inputs for a specific node in the runtime graph.
818
+
819
+ If a `record` object is provided, this method will traverse through
820
+ any input nodes that were SKIPPED to find the true, non-skipped inputs.
821
+
822
+ Args:
823
+ step (str): Step name of the node.
824
+ index (str): Index of the node.
825
+ record (Schema, optional): A schema object containing run records.
826
+ Used to check the status of input nodes. Defaults to None.
827
+
828
+ Returns:
829
+ list[tuple(str,str)]: A list of input nodes.
830
+ '''
692
831
  if (step, index) not in self.get_nodes():
693
832
  raise ValueError(f"{step}/{index} is not a valid node")
694
833
 
@@ -712,6 +851,16 @@ class RuntimeFlowgraph:
712
851
  return sorted(inputs)
713
852
 
714
853
  def get_completed_nodes(self, record=None):
854
+ '''
855
+ Finds all nodes in this runtime graph that have successfully completed.
856
+
857
+ Args:
858
+ record (Schema, optional): A schema object containing run records
859
+ to check for node status. Defaults to None.
860
+
861
+ Returns:
862
+ list[tuple(str,str)]: A sorted list of successfully completed nodes.
863
+ '''
715
864
  if not record:
716
865
  return []
717
866
 
@@ -724,6 +873,24 @@ class RuntimeFlowgraph:
724
873
 
725
874
  @staticmethod
726
875
  def validate(flow, from_steps=None, to_steps=None, prune_nodes=None, logger=None):
876
+ '''
877
+ Validates runtime options against a flowgraph.
878
+
879
+ Checks for undefined steps and ensures that pruning does not break
880
+ the graph by removing all entry/exit points or creating disjoint paths.
881
+
882
+ Args:
883
+ flow (FlowgraphSchema): The flowgraph to validate against.
884
+ from_steps (list[str], optional): List of start steps. Defaults to None.
885
+ to_steps (list[str], optional): List of end steps. Defaults to None.
886
+ prune_nodes (list[tuple(str,str)], optional): List of nodes to prune.
887
+ Defaults to None.
888
+ logger (logging.Logger, optional): Logger for error reporting.
889
+ Defaults to None.
890
+
891
+ Returns:
892
+ bool: True if the runtime configuration is valid, False otherwise.
893
+ '''
727
894
  all_steps = set([step for step, _ in flow.get_nodes()])
728
895
 
729
896
  if from_steps:
@@ -746,18 +913,18 @@ class RuntimeFlowgraph:
746
913
  # Check for undefined steps
747
914
  for step in sorted(from_steps.difference(all_steps)):
748
915
  if logger:
749
- logger.error(f'From {step} is not defined in the {flow.name()} flowgraph')
916
+ logger.error(f'From {step} is not defined in the {flow.name} flowgraph')
750
917
  error = True
751
918
 
752
919
  for step in sorted(to_steps.difference(all_steps)):
753
920
  if logger:
754
- logger.error(f'To {step} is not defined in the {flow.name()} flowgraph')
921
+ logger.error(f'To {step} is not defined in the {flow.name} flowgraph')
755
922
  error = True
756
923
 
757
924
  # Check for undefined prunes
758
925
  for step, index in sorted(prune_nodes.difference(flow.get_nodes())):
759
926
  if logger:
760
- logger.error(f'{step}/{index} is not defined in the {flow.name()} flowgraph')
927
+ logger.error(f'{step}/{index} is not defined in the {flow.name} flowgraph')
761
928
  error = True
762
929
 
763
930
  if not error:
@@ -776,7 +943,7 @@ class RuntimeFlowgraph:
776
943
  runtime_exits = set([step for step, _ in runtime.get_exit_nodes()])
777
944
  for step in unpruned_exits.difference(runtime_exits):
778
945
  if logger:
779
- logger.error(f'pruning removed all exit nodes for {step} in the {flow.name()} '
946
+ logger.error(f'pruning removed all exit nodes for {step} in the {flow.name} '
780
947
  'flowgraph')
781
948
  error = True
782
949
 
@@ -784,7 +951,7 @@ class RuntimeFlowgraph:
784
951
  runtime_entry = set([step for step, _ in runtime.get_entry_nodes()])
785
952
  for step in unpruned_entry.difference(runtime_entry):
786
953
  if logger:
787
- logger.error(f'pruning removed all entry nodes for {step} in the {flow.name()} '
954
+ logger.error(f'pruning removed all entry nodes for {step} in the {flow.name} '
788
955
  'flowgraph')
789
956
  error = True
790
957
 
@@ -801,7 +968,7 @@ class RuntimeFlowgraph:
801
968
  exits = ",".join([f"{step}/{index}"
802
969
  for step, index in runtime.get_exit_nodes()])
803
970
  missing.append(f'no path from {entrynode[0]}/{entrynode[1]} to {exits} '
804
- f'in the {flow.name()} flowgraph')
971
+ f'in the {flow.name} flowgraph')
805
972
  if found:
806
973
  found_any = True
807
974
  if not found_any:
@@ -814,16 +981,41 @@ class RuntimeFlowgraph:
814
981
 
815
982
 
816
983
  class FlowgraphNodeSchema(BaseSchema):
984
+ '''
985
+ Schema definition for a single node within a flowgraph.
986
+ '''
987
+
817
988
  def __init__(self):
989
+ '''
990
+ Initializes a new FlowgraphNodeSchema.
991
+ '''
818
992
  super().__init__()
819
993
 
820
994
  schema_flowgraph(self)
821
995
 
996
+ @classmethod
997
+ def _getdict_type(cls) -> str:
998
+ """
999
+ Returns the metadata type for `getdict` serialization.
1000
+ """
1001
+
1002
+ return FlowgraphNodeSchema.__name__
1003
+
822
1004
 
823
1005
  ###############################################################################
824
1006
  # Flow Configuration
825
1007
  ###############################################################################
826
1008
  def schema_flowgraph(schema):
1009
+ '''
1010
+ Defines the schema parameters for a flowgraph node.
1011
+
1012
+ This function is called to populate a schema with parameters that
1013
+ define a node's properties, such as its inputs, weights, goals, and the
1014
+ tool/task it executes.
1015
+
1016
+ Args:
1017
+ schema (Schema): The schema object to configure.
1018
+ '''
827
1019
  schema = EditableSchema(schema)
828
1020
 
829
1021
  # flowgraph input