siliconcompiler 0.34.2__py3-none-any.whl → 0.34.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- siliconcompiler/__init__.py +12 -5
- siliconcompiler/__main__.py +1 -7
- siliconcompiler/_metadata.py +1 -1
- siliconcompiler/apps/_common.py +104 -23
- siliconcompiler/apps/sc.py +4 -8
- siliconcompiler/apps/sc_dashboard.py +6 -4
- siliconcompiler/apps/sc_install.py +10 -6
- siliconcompiler/apps/sc_issue.py +7 -5
- siliconcompiler/apps/sc_remote.py +1 -1
- siliconcompiler/apps/sc_server.py +9 -14
- siliconcompiler/apps/sc_show.py +6 -5
- siliconcompiler/apps/smake.py +130 -94
- siliconcompiler/apps/utils/replay.py +4 -7
- siliconcompiler/apps/utils/summarize.py +3 -5
- siliconcompiler/asic.py +420 -0
- siliconcompiler/checklist.py +25 -2
- siliconcompiler/cmdlineschema.py +534 -0
- siliconcompiler/constraints/asic_component.py +2 -2
- siliconcompiler/constraints/asic_pins.py +2 -2
- siliconcompiler/constraints/asic_timing.py +3 -3
- siliconcompiler/core.py +7 -32
- siliconcompiler/data/templates/tcl/manifest.tcl.j2 +8 -0
- siliconcompiler/dependencyschema.py +89 -31
- siliconcompiler/design.py +176 -207
- siliconcompiler/filesetschema.py +250 -0
- siliconcompiler/flowgraph.py +274 -95
- siliconcompiler/fpga.py +124 -1
- siliconcompiler/library.py +218 -20
- siliconcompiler/metric.py +233 -20
- siliconcompiler/package/__init__.py +271 -50
- siliconcompiler/package/git.py +92 -16
- siliconcompiler/package/github.py +108 -12
- siliconcompiler/package/https.py +79 -16
- siliconcompiler/packageschema.py +88 -7
- siliconcompiler/pathschema.py +31 -2
- siliconcompiler/pdk.py +566 -1
- siliconcompiler/project.py +1095 -94
- siliconcompiler/record.py +38 -1
- siliconcompiler/remote/__init__.py +5 -2
- siliconcompiler/remote/client.py +11 -6
- siliconcompiler/remote/schema.py +5 -23
- siliconcompiler/remote/server.py +41 -54
- siliconcompiler/report/__init__.py +3 -3
- siliconcompiler/report/dashboard/__init__.py +48 -14
- siliconcompiler/report/dashboard/cli/__init__.py +99 -21
- siliconcompiler/report/dashboard/cli/board.py +364 -179
- siliconcompiler/report/dashboard/web/__init__.py +90 -12
- siliconcompiler/report/dashboard/web/components/__init__.py +219 -240
- siliconcompiler/report/dashboard/web/components/flowgraph.py +49 -26
- siliconcompiler/report/dashboard/web/components/graph.py +139 -100
- siliconcompiler/report/dashboard/web/layouts/__init__.py +29 -1
- siliconcompiler/report/dashboard/web/layouts/_common.py +38 -2
- siliconcompiler/report/dashboard/web/layouts/vertical_flowgraph.py +39 -26
- siliconcompiler/report/dashboard/web/layouts/vertical_flowgraph_node_tab.py +50 -50
- siliconcompiler/report/dashboard/web/layouts/vertical_flowgraph_sac_tabs.py +49 -46
- siliconcompiler/report/dashboard/web/state.py +141 -14
- siliconcompiler/report/dashboard/web/utils/__init__.py +79 -16
- siliconcompiler/report/dashboard/web/utils/file_utils.py +74 -11
- siliconcompiler/report/dashboard/web/viewer.py +25 -1
- siliconcompiler/report/report.py +5 -2
- siliconcompiler/report/summary_image.py +29 -11
- siliconcompiler/scheduler/__init__.py +9 -1
- siliconcompiler/scheduler/docker.py +79 -1
- siliconcompiler/scheduler/run_node.py +35 -19
- siliconcompiler/scheduler/scheduler.py +208 -24
- siliconcompiler/scheduler/schedulernode.py +372 -46
- siliconcompiler/scheduler/send_messages.py +77 -29
- siliconcompiler/scheduler/slurm.py +76 -12
- siliconcompiler/scheduler/taskscheduler.py +140 -20
- siliconcompiler/schema/__init__.py +0 -2
- siliconcompiler/schema/baseschema.py +194 -38
- siliconcompiler/schema/journal.py +7 -4
- siliconcompiler/schema/namedschema.py +16 -10
- siliconcompiler/schema/parameter.py +55 -9
- siliconcompiler/schema/parametervalue.py +60 -0
- siliconcompiler/schema/safeschema.py +25 -2
- siliconcompiler/schema/schema_cfg.py +5 -5
- siliconcompiler/schema/utils.py +2 -2
- siliconcompiler/schema_obj.py +20 -3
- siliconcompiler/tool.py +979 -302
- siliconcompiler/tools/bambu/__init__.py +41 -0
- siliconcompiler/tools/builtin/concatenate.py +2 -2
- siliconcompiler/tools/builtin/minimum.py +2 -1
- siliconcompiler/tools/builtin/mux.py +2 -1
- siliconcompiler/tools/builtin/nop.py +2 -1
- siliconcompiler/tools/builtin/verify.py +2 -1
- siliconcompiler/tools/klayout/__init__.py +95 -0
- siliconcompiler/tools/openroad/__init__.py +289 -0
- siliconcompiler/tools/openroad/scripts/apr/preamble.tcl +3 -0
- siliconcompiler/tools/openroad/scripts/apr/sc_detailed_route.tcl +7 -2
- siliconcompiler/tools/openroad/scripts/apr/sc_global_route.tcl +8 -4
- siliconcompiler/tools/openroad/scripts/apr/sc_init_floorplan.tcl +9 -5
- siliconcompiler/tools/openroad/scripts/common/write_images.tcl +5 -1
- siliconcompiler/tools/slang/__init__.py +1 -1
- siliconcompiler/tools/slang/elaborate.py +2 -1
- siliconcompiler/tools/vivado/scripts/sc_run.tcl +1 -1
- siliconcompiler/tools/vivado/scripts/sc_syn_fpga.tcl +8 -1
- siliconcompiler/tools/vivado/syn_fpga.py +6 -0
- siliconcompiler/tools/vivado/vivado.py +35 -2
- siliconcompiler/tools/vpr/__init__.py +150 -0
- siliconcompiler/tools/yosys/__init__.py +369 -1
- siliconcompiler/tools/yosys/scripts/procs.tcl +0 -1
- siliconcompiler/toolscripts/_tools.json +5 -10
- siliconcompiler/utils/__init__.py +66 -0
- siliconcompiler/utils/flowgraph.py +2 -2
- siliconcompiler/utils/issue.py +2 -1
- siliconcompiler/utils/logging.py +14 -0
- siliconcompiler/utils/multiprocessing.py +256 -0
- siliconcompiler/utils/showtools.py +10 -0
- {siliconcompiler-0.34.2.dist-info → siliconcompiler-0.34.3.dist-info}/METADATA +5 -5
- {siliconcompiler-0.34.2.dist-info → siliconcompiler-0.34.3.dist-info}/RECORD +115 -118
- {siliconcompiler-0.34.2.dist-info → siliconcompiler-0.34.3.dist-info}/entry_points.txt +3 -0
- siliconcompiler/schema/cmdlineschema.py +0 -250
- siliconcompiler/toolscripts/rhel8/install-slang.sh +0 -40
- siliconcompiler/toolscripts/rhel9/install-slang.sh +0 -40
- siliconcompiler/toolscripts/ubuntu20/install-slang.sh +0 -47
- siliconcompiler/toolscripts/ubuntu22/install-slang.sh +0 -37
- siliconcompiler/toolscripts/ubuntu24/install-slang.sh +0 -37
- {siliconcompiler-0.34.2.dist-info → siliconcompiler-0.34.3.dist-info}/WHEEL +0 -0
- {siliconcompiler-0.34.2.dist-info → siliconcompiler-0.34.3.dist-info}/licenses/LICENSE +0 -0
- {siliconcompiler-0.34.2.dist-info → siliconcompiler-0.34.3.dist-info}/top_level.txt +0 -0
siliconcompiler/flowgraph.py
CHANGED
|
@@ -10,7 +10,23 @@ from siliconcompiler import NodeStatus
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class FlowgraphSchema(NamedSchema):
|
|
13
|
+
'''
|
|
14
|
+
Schema for defining and interacting with a flowgraph.
|
|
15
|
+
|
|
16
|
+
A flowgraph is a directed acyclic graph (DAG) that represents the
|
|
17
|
+
compilation flow. Each node in the graph is a step/index pair that
|
|
18
|
+
|
|
19
|
+
maps to a specific tool task, and edges represent dependencies between
|
|
20
|
+
these tasks.
|
|
21
|
+
'''
|
|
22
|
+
|
|
13
23
|
def __init__(self, name=None):
|
|
24
|
+
'''
|
|
25
|
+
Initializes a new FlowgraphSchema object.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
name (str, optional): The name of the flowgraph. Defaults to None.
|
|
29
|
+
'''
|
|
14
30
|
super().__init__()
|
|
15
31
|
self.set_name(name)
|
|
16
32
|
|
|
@@ -21,7 +37,9 @@ class FlowgraphSchema(NamedSchema):
|
|
|
21
37
|
|
|
22
38
|
def __clear_cache(self):
|
|
23
39
|
'''
|
|
24
|
-
|
|
40
|
+
Clears the internal cache for memoized flowgraph properties.
|
|
41
|
+
|
|
42
|
+
This should be called any time the graph structure is modified.
|
|
25
43
|
'''
|
|
26
44
|
|
|
27
45
|
self.__cache_nodes = None
|
|
@@ -38,27 +56,27 @@ class FlowgraphSchema(NamedSchema):
|
|
|
38
56
|
'''
|
|
39
57
|
Creates a flowgraph node.
|
|
40
58
|
|
|
41
|
-
Creates a flowgraph node by binding a step to a tool
|
|
42
|
-
A tool can be an external executable or one of the built
|
|
43
|
-
in the SiliconCompiler framework
|
|
44
|
-
maximum, join, mux, verify.
|
|
59
|
+
Creates a flowgraph node by binding a step to a tool-specific task.
|
|
60
|
+
A tool can be an external executable or one of the built-in functions
|
|
61
|
+
in the SiliconCompiler framework (e.g., minimum, maximum, join).
|
|
45
62
|
|
|
46
63
|
The method modifies the following schema parameters:
|
|
47
64
|
|
|
48
|
-
* [<step
|
|
49
|
-
* [<step
|
|
50
|
-
* [<step
|
|
65
|
+
* `['<step>', '<index>', 'tool']`
|
|
66
|
+
* `['<step>', '<index>', 'task']`
|
|
67
|
+
* `['<step>', '<index>', 'taskmodule']`
|
|
51
68
|
|
|
52
69
|
Args:
|
|
53
|
-
step (str): Step name
|
|
54
|
-
task (module
|
|
55
|
-
|
|
70
|
+
step (str): Step name for the node.
|
|
71
|
+
task (module or str): The task to associate with this node. Can be
|
|
72
|
+
a module object or a string in the format '<tool>.<task>'.
|
|
73
|
+
index (int or str): Index for the step. Defaults to 0.
|
|
56
74
|
|
|
57
75
|
Examples:
|
|
58
|
-
>>> import
|
|
59
|
-
>>> flow.node('
|
|
60
|
-
Creates a 'place' task
|
|
61
|
-
'
|
|
76
|
+
>>> import siliconcompiler.tools.openroad as openroad
|
|
77
|
+
>>> flow.node('place', openroad.place, index=0)
|
|
78
|
+
# Creates a node for the 'place' task in the 'openroad' tool,
|
|
79
|
+
# identified by step='place' and index=0.
|
|
62
80
|
'''
|
|
63
81
|
|
|
64
82
|
if step in (Schema.GLOBAL_KEY, 'default', 'sc_collected_files'):
|
|
@@ -104,26 +122,26 @@ class FlowgraphSchema(NamedSchema):
|
|
|
104
122
|
Connects the output of a tail node with the input of a head node by
|
|
105
123
|
setting the 'input' field of the head node in the schema flowgraph.
|
|
106
124
|
|
|
107
|
-
The method modifies the following
|
|
125
|
+
The method modifies the following parameter:
|
|
108
126
|
|
|
109
|
-
[<head
|
|
127
|
+
* `['<head>', '<head_index>', 'input']`
|
|
110
128
|
|
|
111
129
|
Args:
|
|
112
|
-
tail (str):
|
|
113
|
-
head (str):
|
|
114
|
-
tail_index (int
|
|
115
|
-
head_index (int
|
|
130
|
+
tail (str): Step name of the tail node.
|
|
131
|
+
head (str): Step name of the head node.
|
|
132
|
+
tail_index (int or str): Index of the tail node. Defaults to 0.
|
|
133
|
+
head_index (int or str): Index of the head node. Defaults to 0.
|
|
116
134
|
|
|
117
135
|
Examples:
|
|
118
136
|
>>> chip.edge('place', 'cts')
|
|
119
|
-
Creates a directed edge from place to cts.
|
|
137
|
+
# Creates a directed edge from ('place', '0') to ('cts', '0').
|
|
120
138
|
'''
|
|
121
139
|
head_index = str(head_index)
|
|
122
140
|
tail_index = str(tail_index)
|
|
123
141
|
|
|
124
142
|
for step, index in [(head, head_index), (tail, tail_index)]:
|
|
125
143
|
if not self.valid(step, index):
|
|
126
|
-
raise ValueError(f"{step}/{index} is not a defined node in {self.name
|
|
144
|
+
raise ValueError(f"{step}/{index} is not a defined node in {self.name}.")
|
|
127
145
|
|
|
128
146
|
tail_node = (tail, tail_index)
|
|
129
147
|
if tail_node in self.get(head, head_index, 'input'):
|
|
@@ -135,15 +153,16 @@ class FlowgraphSchema(NamedSchema):
|
|
|
135
153
|
|
|
136
154
|
def remove_node(self, step, index=None):
|
|
137
155
|
'''
|
|
138
|
-
|
|
156
|
+
Removes a flowgraph node and reconnects its inputs to its outputs.
|
|
139
157
|
|
|
140
158
|
Args:
|
|
141
|
-
step (str): Step name
|
|
142
|
-
index (int
|
|
159
|
+
step (str): Step name of the node to remove.
|
|
160
|
+
index (int or str, optional): Index of the node to remove. If None,
|
|
161
|
+
all nodes for the given step are removed. Defaults to None.
|
|
143
162
|
'''
|
|
144
163
|
|
|
145
164
|
if step not in self.getkeys():
|
|
146
|
-
raise ValueError(f'{step} is not a valid step in {self.name
|
|
165
|
+
raise ValueError(f'{step} is not a valid step in {self.name}')
|
|
147
166
|
|
|
148
167
|
if index is None:
|
|
149
168
|
# Iterate over all indexes
|
|
@@ -153,7 +172,7 @@ class FlowgraphSchema(NamedSchema):
|
|
|
153
172
|
|
|
154
173
|
index = str(index)
|
|
155
174
|
if index not in self.getkeys(step):
|
|
156
|
-
raise ValueError(f'{index} is not a valid index for {step} in {self.name
|
|
175
|
+
raise ValueError(f'{index} is not a valid index for {step} in {self.name}')
|
|
157
176
|
|
|
158
177
|
# Save input edges
|
|
159
178
|
node = (step, index)
|
|
@@ -178,21 +197,24 @@ class FlowgraphSchema(NamedSchema):
|
|
|
178
197
|
|
|
179
198
|
def insert_node(self, step, task, before_step, index=0, before_index=0):
|
|
180
199
|
'''
|
|
181
|
-
|
|
200
|
+
Inserts a new node in the graph before a specified node.
|
|
201
|
+
|
|
202
|
+
The new node is placed between the `before` node and all of its
|
|
203
|
+
original inputs.
|
|
182
204
|
|
|
183
205
|
Args:
|
|
184
|
-
step (str): Step name
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
before_index (int
|
|
206
|
+
step (str): Step name for the new node.
|
|
207
|
+
task (module or str): Task to associate with the new node.
|
|
208
|
+
before_step (str): Step name of the existing node to insert before.
|
|
209
|
+
index (int or str): Index for the new node. Defaults to 0.
|
|
210
|
+
before_index (int or str): Index of the existing node. Defaults to 0.
|
|
189
211
|
'''
|
|
190
212
|
|
|
191
213
|
index = str(index)
|
|
192
214
|
before_index = str(before_index)
|
|
193
215
|
|
|
194
216
|
if (before_step, before_index) not in self.get_nodes():
|
|
195
|
-
raise ValueError(f'{before_step}/{before_index} is not a valid node in {self.name
|
|
217
|
+
raise ValueError(f'{before_step}/{before_index} is not a valid node in {self.name}')
|
|
196
218
|
|
|
197
219
|
# add the node
|
|
198
220
|
self.node(step, task, index=index)
|
|
@@ -210,15 +232,16 @@ class FlowgraphSchema(NamedSchema):
|
|
|
210
232
|
###########################################################################
|
|
211
233
|
def graph(self, subflow, name=None):
|
|
212
234
|
'''
|
|
213
|
-
Instantiates a
|
|
235
|
+
Instantiates a sub-flowgraph within the current flowgraph.
|
|
214
236
|
|
|
215
237
|
Args:
|
|
216
|
-
subflow (
|
|
217
|
-
name (str):
|
|
238
|
+
subflow (FlowgraphSchema): The flowgraph to instantiate.
|
|
239
|
+
name (str, optional): A prefix to add to the names of the
|
|
240
|
+
instantiated steps to ensure they are unique. Defaults to None.
|
|
218
241
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
242
|
+
Raises:
|
|
243
|
+
ValueError: If `subflow` is not a `FlowgraphSchema` object, or if
|
|
244
|
+
a step from the sub-flowgraph already exists in the current graph.
|
|
222
245
|
'''
|
|
223
246
|
if not isinstance(subflow, FlowgraphSchema):
|
|
224
247
|
raise ValueError(f"subflow must a FlowgraphSchema, not: {type(subflow)}")
|
|
@@ -252,7 +275,12 @@ class FlowgraphSchema(NamedSchema):
|
|
|
252
275
|
|
|
253
276
|
def get_nodes(self):
|
|
254
277
|
'''
|
|
255
|
-
Returns all
|
|
278
|
+
Returns a sorted tuple of all nodes defined in this flowgraph.
|
|
279
|
+
|
|
280
|
+
A node is represented as a `(step, index)` tuple.
|
|
281
|
+
|
|
282
|
+
Returns:
|
|
283
|
+
tuple[tuple(str,str)]: All nodes in the graph.
|
|
256
284
|
'''
|
|
257
285
|
if self.__cache_nodes is not None:
|
|
258
286
|
return self.__cache_nodes
|
|
@@ -268,8 +296,12 @@ class FlowgraphSchema(NamedSchema):
|
|
|
268
296
|
|
|
269
297
|
def get_entry_nodes(self):
|
|
270
298
|
'''
|
|
271
|
-
|
|
272
|
-
|
|
299
|
+
Collects all nodes that are entry points to the flowgraph.
|
|
300
|
+
|
|
301
|
+
Entry nodes are those with no inputs.
|
|
302
|
+
|
|
303
|
+
Returns:
|
|
304
|
+
tuple[tuple(str,str)]: All entry nodes in the graph.
|
|
273
305
|
'''
|
|
274
306
|
|
|
275
307
|
if self.__cache_nodes_entry is not None:
|
|
@@ -286,8 +318,12 @@ class FlowgraphSchema(NamedSchema):
|
|
|
286
318
|
|
|
287
319
|
def get_exit_nodes(self):
|
|
288
320
|
'''
|
|
289
|
-
|
|
290
|
-
|
|
321
|
+
Collects all nodes that are exit points of the flowgraph.
|
|
322
|
+
|
|
323
|
+
Exit nodes are those that are not inputs to any other node.
|
|
324
|
+
|
|
325
|
+
Returns:
|
|
326
|
+
tuple[tuple(str,str)]: All exit nodes in the graph.
|
|
291
327
|
'''
|
|
292
328
|
|
|
293
329
|
if self.__cache_nodes_exit is not None:
|
|
@@ -307,11 +343,15 @@ class FlowgraphSchema(NamedSchema):
|
|
|
307
343
|
|
|
308
344
|
def get_execution_order(self, reverse=False):
|
|
309
345
|
'''
|
|
310
|
-
Generates a list of nodes
|
|
346
|
+
Generates a topologically sorted list of nodes for execution.
|
|
311
347
|
|
|
312
348
|
Args:
|
|
313
|
-
reverse (
|
|
314
|
-
to entry nodes.
|
|
349
|
+
reverse (bool): If True, the order is reversed, from exit nodes
|
|
350
|
+
to entry nodes. Defaults to False.
|
|
351
|
+
|
|
352
|
+
Returns:
|
|
353
|
+
tuple[tuple[tuple(str,str)]]: A tuple of tuples, where each inner
|
|
354
|
+
tuple represents a level of nodes that can be executed in parallel.
|
|
315
355
|
'''
|
|
316
356
|
|
|
317
357
|
if reverse:
|
|
@@ -387,11 +427,14 @@ class FlowgraphSchema(NamedSchema):
|
|
|
387
427
|
|
|
388
428
|
def get_node_outputs(self, step, index):
|
|
389
429
|
'''
|
|
390
|
-
Returns the nodes the given
|
|
430
|
+
Returns the nodes that the given node provides input to.
|
|
391
431
|
|
|
392
432
|
Args:
|
|
393
|
-
step (str):
|
|
394
|
-
index (str
|
|
433
|
+
step (str): Step name of the source node.
|
|
434
|
+
index (str or int): Index of the source node.
|
|
435
|
+
|
|
436
|
+
Returns:
|
|
437
|
+
tuple[tuple(str,str)]: A tuple of destination nodes.
|
|
395
438
|
'''
|
|
396
439
|
|
|
397
440
|
index = str(index)
|
|
@@ -423,12 +466,15 @@ class FlowgraphSchema(NamedSchema):
|
|
|
423
466
|
|
|
424
467
|
def __find_loops(self, step, index, path=None):
|
|
425
468
|
'''
|
|
426
|
-
|
|
469
|
+
Internal helper to search for loops in the graph via depth-first search.
|
|
427
470
|
|
|
428
471
|
Args:
|
|
429
|
-
step (str):
|
|
430
|
-
index (str)
|
|
431
|
-
path (list
|
|
472
|
+
step (str): Step name to start from.
|
|
473
|
+
index (str): Index name to start from.
|
|
474
|
+
path (list, optional): The path taken so far. Defaults to None.
|
|
475
|
+
|
|
476
|
+
Returns:
|
|
477
|
+
list: A list of nodes forming a loop, or None if no loop is found.
|
|
432
478
|
'''
|
|
433
479
|
if path is None:
|
|
434
480
|
path = []
|
|
@@ -448,17 +494,20 @@ class FlowgraphSchema(NamedSchema):
|
|
|
448
494
|
|
|
449
495
|
def validate(self, logger=None):
|
|
450
496
|
'''
|
|
451
|
-
|
|
497
|
+
Checks if the flowgraph is valid.
|
|
452
498
|
|
|
453
|
-
|
|
454
|
-
*
|
|
455
|
-
*
|
|
456
|
-
*
|
|
457
|
-
|
|
458
|
-
Returns True if valid, False otherwise.
|
|
499
|
+
This method performs several checks:
|
|
500
|
+
* All edges must point to and from valid nodes.
|
|
501
|
+
* There should be no duplicate edges.
|
|
502
|
+
* All nodes must have their tool, task, and taskmodule defined.
|
|
503
|
+
* The graph must not contain any loops (it must be a DAG).
|
|
459
504
|
|
|
460
505
|
Args:
|
|
461
|
-
logger (logging.Logger): logger to use for reporting
|
|
506
|
+
logger (logging.Logger, optional): A logger to use for reporting
|
|
507
|
+
errors. Defaults to None.
|
|
508
|
+
|
|
509
|
+
Returns:
|
|
510
|
+
bool: True if the graph is valid, False otherwise.
|
|
462
511
|
'''
|
|
463
512
|
|
|
464
513
|
error = False
|
|
@@ -474,14 +523,14 @@ class FlowgraphSchema(NamedSchema):
|
|
|
474
523
|
in_step, in_index = node
|
|
475
524
|
if logger:
|
|
476
525
|
logger.error(f'Duplicate edge from {in_step}/{in_index} to '
|
|
477
|
-
f'{step}/{index} in the {self.name
|
|
526
|
+
f'{step}/{index} in the {self.name} flowgraph')
|
|
478
527
|
error = True
|
|
479
528
|
|
|
480
529
|
diff_nodes = check_nodes.difference(self.get_nodes())
|
|
481
530
|
if diff_nodes:
|
|
482
531
|
if logger:
|
|
483
532
|
for step, index in diff_nodes:
|
|
484
|
-
logger.error(f'{step}/{index} is missing in the {self.name
|
|
533
|
+
logger.error(f'{step}/{index} is missing in the {self.name} flowgraph')
|
|
485
534
|
error = True
|
|
486
535
|
|
|
487
536
|
# Detect missing definitions
|
|
@@ -490,7 +539,7 @@ class FlowgraphSchema(NamedSchema):
|
|
|
490
539
|
if not self.get(step, index, item):
|
|
491
540
|
if logger:
|
|
492
541
|
logger.error(f'{step}/{index} is missing a {item} definition in the '
|
|
493
|
-
f'{self.name
|
|
542
|
+
f'{self.name} flowgraph')
|
|
494
543
|
error = True
|
|
495
544
|
|
|
496
545
|
# detect loops
|
|
@@ -500,11 +549,14 @@ class FlowgraphSchema(NamedSchema):
|
|
|
500
549
|
error = True
|
|
501
550
|
if logger:
|
|
502
551
|
loop_path = [f"{step}/{index}" for step, index in loop_path]
|
|
503
|
-
logger.error(f"{' -> '.join(loop_path)} forms a loop in {self.name
|
|
552
|
+
logger.error(f"{' -> '.join(loop_path)} forms a loop in {self.name}")
|
|
504
553
|
|
|
505
554
|
return not error
|
|
506
555
|
|
|
507
556
|
def __get_task_module(self, name):
|
|
557
|
+
'''
|
|
558
|
+
Internal helper to import and cache a task module by name.
|
|
559
|
+
'''
|
|
508
560
|
# Create cache
|
|
509
561
|
if self.__cache_tasks is None:
|
|
510
562
|
self.__cache_tasks = {}
|
|
@@ -517,45 +569,70 @@ class FlowgraphSchema(NamedSchema):
|
|
|
517
569
|
|
|
518
570
|
def get_task_module(self, step, index):
|
|
519
571
|
"""
|
|
520
|
-
Returns the module for a given task
|
|
572
|
+
Returns the imported Python module for a given task node.
|
|
521
573
|
|
|
522
574
|
Args:
|
|
523
|
-
step (str): Step name
|
|
524
|
-
index (int
|
|
575
|
+
step (str): Step name of the node.
|
|
576
|
+
index (int or str): Index of the node.
|
|
577
|
+
|
|
578
|
+
Returns:
|
|
579
|
+
module: The imported task module.
|
|
525
580
|
"""
|
|
526
581
|
|
|
527
582
|
index = str(index)
|
|
528
583
|
|
|
529
584
|
if (step, index) not in self.get_nodes():
|
|
530
|
-
raise ValueError(f"{step}/{index} is not a valid node in {self.name
|
|
585
|
+
raise ValueError(f"{step}/{index} is not a valid node in {self.name}.")
|
|
531
586
|
|
|
532
587
|
return self.__get_task_module(self.get(step, index, 'taskmodule'))
|
|
533
588
|
|
|
534
589
|
def get_all_tasks(self):
|
|
535
590
|
'''
|
|
536
|
-
Returns all
|
|
591
|
+
Returns all unique task modules used in this flowgraph.
|
|
537
592
|
|
|
538
593
|
Returns:
|
|
539
|
-
set of modules
|
|
594
|
+
set[module]: A set of all imported task modules.
|
|
540
595
|
'''
|
|
541
596
|
tasks = set()
|
|
542
597
|
for step, index in self.get_nodes():
|
|
543
598
|
tasks.add(self.get_task_module(step, index))
|
|
544
599
|
return tasks
|
|
545
600
|
|
|
601
|
+
@classmethod
|
|
602
|
+
def _getdict_type(cls) -> str:
|
|
603
|
+
"""
|
|
604
|
+
Returns the metadata type for `getdict` serialization.
|
|
605
|
+
"""
|
|
606
|
+
|
|
607
|
+
return FlowgraphSchema.__name__
|
|
608
|
+
|
|
546
609
|
|
|
547
610
|
class RuntimeFlowgraph:
|
|
548
611
|
'''
|
|
549
|
-
|
|
612
|
+
A runtime representation of a flowgraph for a specific execution.
|
|
550
613
|
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
to_steps (list of steps): step to end the runtime at
|
|
556
|
-
prune_nodes (list of nodes): nodes to remove from execution
|
|
614
|
+
This class creates a "view" of a base flowgraph that considers runtime
|
|
615
|
+
options such as the start step (`-from`), end step (`-to`), and nodes to
|
|
616
|
+
exclude (`-prune`). It computes the precise subgraph of nodes that need
|
|
617
|
+
to be executed for a given run.
|
|
557
618
|
'''
|
|
619
|
+
|
|
558
620
|
def __init__(self, base, args=None, from_steps=None, to_steps=None, prune_nodes=None):
|
|
621
|
+
'''
|
|
622
|
+
Initializes a new RuntimeFlowgraph.
|
|
623
|
+
|
|
624
|
+
Args:
|
|
625
|
+
base (FlowgraphSchema): The base flowgraph to create a view of.
|
|
626
|
+
args (tuple[str, str], optional): A specific `(step, index)` to run.
|
|
627
|
+
If provided, this overrides `from_steps` and `to_steps`.
|
|
628
|
+
Defaults to None.
|
|
629
|
+
from_steps (list[str], optional): List of step names to start execution
|
|
630
|
+
from. Defaults to the base graph's entry nodes.
|
|
631
|
+
to_steps (list[str], optional): List of step names to end execution at.
|
|
632
|
+
Defaults to the base graph's exit nodes.
|
|
633
|
+
prune_nodes (list[tuple(str,str)], optional): A list of `(step, index)`
|
|
634
|
+
nodes to exclude from the graph. Defaults to None.
|
|
635
|
+
'''
|
|
559
636
|
if not all([hasattr(base, attr) for attr in dir(FlowgraphSchema)]):
|
|
560
637
|
raise ValueError(f"base must a FlowgraphSchema, not: {type(base)}")
|
|
561
638
|
|
|
@@ -607,6 +684,21 @@ class RuntimeFlowgraph:
|
|
|
607
684
|
self.__compute_graph()
|
|
608
685
|
|
|
609
686
|
def __walk_graph(self, node, path=None, reverse=True):
|
|
687
|
+
'''
|
|
688
|
+
Internal helper to recursively walk the graph to find all connected nodes.
|
|
689
|
+
|
|
690
|
+
This walk respects the runtime boundaries (`-from`, `-to`, `-prune`).
|
|
691
|
+
|
|
692
|
+
Args:
|
|
693
|
+
node (tuple(str,str)): The node to start the walk from.
|
|
694
|
+
path (list, optional): The path taken so far, used for cycle
|
|
695
|
+
detection. Defaults to None.
|
|
696
|
+
reverse (bool, optional): If True, walks backwards along inputs.
|
|
697
|
+
If False, walks forwards along outputs. Defaults to True.
|
|
698
|
+
|
|
699
|
+
Returns:
|
|
700
|
+
set[tuple(str,str)]: The set of nodes visited during the walk.
|
|
701
|
+
'''
|
|
610
702
|
if node in self.__prune:
|
|
611
703
|
return set()
|
|
612
704
|
|
|
@@ -635,7 +727,10 @@ class RuntimeFlowgraph:
|
|
|
635
727
|
|
|
636
728
|
def __compute_graph(self):
|
|
637
729
|
'''
|
|
638
|
-
|
|
730
|
+
Internal helper to precompute the runtime graph information.
|
|
731
|
+
|
|
732
|
+
This method determines the final set of nodes, entry/exit points, and
|
|
733
|
+
the execution order based on the runtime constraints.
|
|
639
734
|
'''
|
|
640
735
|
|
|
641
736
|
self.__nodes = set()
|
|
@@ -664,35 +759,51 @@ class RuntimeFlowgraph:
|
|
|
664
759
|
|
|
665
760
|
def get_nodes(self):
|
|
666
761
|
'''
|
|
667
|
-
Returns the nodes
|
|
762
|
+
Returns the nodes that are part of this runtime graph.
|
|
763
|
+
|
|
764
|
+
Returns:
|
|
765
|
+
tuple[tuple(str,str)]: A tuple of all nodes in the runtime graph.
|
|
668
766
|
'''
|
|
669
767
|
return self.__nodes
|
|
670
768
|
|
|
671
769
|
def get_execution_order(self):
|
|
672
770
|
'''
|
|
673
|
-
Returns the execution order of the nodes
|
|
771
|
+
Returns the execution order of the nodes in this runtime graph.
|
|
772
|
+
|
|
773
|
+
Returns:
|
|
774
|
+
tuple[tuple[tuple(str,str)]]: A tuple of tuples representing
|
|
775
|
+
parallel execution levels.
|
|
674
776
|
'''
|
|
675
777
|
return self.__execution_order
|
|
676
778
|
|
|
677
779
|
def get_entry_nodes(self):
|
|
678
780
|
'''
|
|
679
|
-
Returns the entry nodes for this graph
|
|
781
|
+
Returns the entry nodes for this runtime graph.
|
|
782
|
+
|
|
783
|
+
Returns:
|
|
784
|
+
tuple[tuple(str,str)]: A tuple of all entry nodes.
|
|
680
785
|
'''
|
|
681
786
|
return self.__from
|
|
682
787
|
|
|
683
788
|
def get_exit_nodes(self):
|
|
684
789
|
'''
|
|
685
|
-
Returns the exit nodes for this graph
|
|
790
|
+
Returns the exit nodes for this runtime graph.
|
|
791
|
+
|
|
792
|
+
Returns:
|
|
793
|
+
tuple[tuple(str,str)]: A tuple of all exit nodes.
|
|
686
794
|
'''
|
|
687
795
|
return self.__to
|
|
688
796
|
|
|
689
797
|
def get_nodes_starting_at(self, step, index):
|
|
690
798
|
'''
|
|
691
|
-
Returns all
|
|
799
|
+
Returns all nodes reachable from a given starting node in this runtime graph.
|
|
692
800
|
|
|
693
801
|
Args:
|
|
694
|
-
step (str): step
|
|
695
|
-
index (str
|
|
802
|
+
step (str): The step name of the starting node.
|
|
803
|
+
index (str or int): The index of the starting node.
|
|
804
|
+
|
|
805
|
+
Returns:
|
|
806
|
+
tuple[tuple(str,str)]: A tuple of all reachable nodes.
|
|
696
807
|
'''
|
|
697
808
|
index = str(index)
|
|
698
809
|
|
|
@@ -702,6 +813,21 @@ class RuntimeFlowgraph:
|
|
|
702
813
|
return tuple(sorted(self.__walk_graph((step, str(index)), reverse=False)))
|
|
703
814
|
|
|
704
815
|
def get_node_inputs(self, step, index, record=None):
|
|
816
|
+
'''
|
|
817
|
+
Gets the inputs for a specific node in the runtime graph.
|
|
818
|
+
|
|
819
|
+
If a `record` object is provided, this method will traverse through
|
|
820
|
+
any input nodes that were SKIPPED to find the true, non-skipped inputs.
|
|
821
|
+
|
|
822
|
+
Args:
|
|
823
|
+
step (str): Step name of the node.
|
|
824
|
+
index (str): Index of the node.
|
|
825
|
+
record (Schema, optional): A schema object containing run records.
|
|
826
|
+
Used to check the status of input nodes. Defaults to None.
|
|
827
|
+
|
|
828
|
+
Returns:
|
|
829
|
+
list[tuple(str,str)]: A list of input nodes.
|
|
830
|
+
'''
|
|
705
831
|
if (step, index) not in self.get_nodes():
|
|
706
832
|
raise ValueError(f"{step}/{index} is not a valid node")
|
|
707
833
|
|
|
@@ -725,6 +851,16 @@ class RuntimeFlowgraph:
|
|
|
725
851
|
return sorted(inputs)
|
|
726
852
|
|
|
727
853
|
def get_completed_nodes(self, record=None):
|
|
854
|
+
'''
|
|
855
|
+
Finds all nodes in this runtime graph that have successfully completed.
|
|
856
|
+
|
|
857
|
+
Args:
|
|
858
|
+
record (Schema, optional): A schema object containing run records
|
|
859
|
+
to check for node status. Defaults to None.
|
|
860
|
+
|
|
861
|
+
Returns:
|
|
862
|
+
list[tuple(str,str)]: A sorted list of successfully completed nodes.
|
|
863
|
+
'''
|
|
728
864
|
if not record:
|
|
729
865
|
return []
|
|
730
866
|
|
|
@@ -737,6 +873,24 @@ class RuntimeFlowgraph:
|
|
|
737
873
|
|
|
738
874
|
@staticmethod
|
|
739
875
|
def validate(flow, from_steps=None, to_steps=None, prune_nodes=None, logger=None):
|
|
876
|
+
'''
|
|
877
|
+
Validates runtime options against a flowgraph.
|
|
878
|
+
|
|
879
|
+
Checks for undefined steps and ensures that pruning does not break
|
|
880
|
+
the graph by removing all entry/exit points or creating disjoint paths.
|
|
881
|
+
|
|
882
|
+
Args:
|
|
883
|
+
flow (FlowgraphSchema): The flowgraph to validate against.
|
|
884
|
+
from_steps (list[str], optional): List of start steps. Defaults to None.
|
|
885
|
+
to_steps (list[str], optional): List of end steps. Defaults to None.
|
|
886
|
+
prune_nodes (list[tuple(str,str)], optional): List of nodes to prune.
|
|
887
|
+
Defaults to None.
|
|
888
|
+
logger (logging.Logger, optional): Logger for error reporting.
|
|
889
|
+
Defaults to None.
|
|
890
|
+
|
|
891
|
+
Returns:
|
|
892
|
+
bool: True if the runtime configuration is valid, False otherwise.
|
|
893
|
+
'''
|
|
740
894
|
all_steps = set([step for step, _ in flow.get_nodes()])
|
|
741
895
|
|
|
742
896
|
if from_steps:
|
|
@@ -759,18 +913,18 @@ class RuntimeFlowgraph:
|
|
|
759
913
|
# Check for undefined steps
|
|
760
914
|
for step in sorted(from_steps.difference(all_steps)):
|
|
761
915
|
if logger:
|
|
762
|
-
logger.error(f'From {step} is not defined in the {flow.name
|
|
916
|
+
logger.error(f'From {step} is not defined in the {flow.name} flowgraph')
|
|
763
917
|
error = True
|
|
764
918
|
|
|
765
919
|
for step in sorted(to_steps.difference(all_steps)):
|
|
766
920
|
if logger:
|
|
767
|
-
logger.error(f'To {step} is not defined in the {flow.name
|
|
921
|
+
logger.error(f'To {step} is not defined in the {flow.name} flowgraph')
|
|
768
922
|
error = True
|
|
769
923
|
|
|
770
924
|
# Check for undefined prunes
|
|
771
925
|
for step, index in sorted(prune_nodes.difference(flow.get_nodes())):
|
|
772
926
|
if logger:
|
|
773
|
-
logger.error(f'{step}/{index} is not defined in the {flow.name
|
|
927
|
+
logger.error(f'{step}/{index} is not defined in the {flow.name} flowgraph')
|
|
774
928
|
error = True
|
|
775
929
|
|
|
776
930
|
if not error:
|
|
@@ -789,7 +943,7 @@ class RuntimeFlowgraph:
|
|
|
789
943
|
runtime_exits = set([step for step, _ in runtime.get_exit_nodes()])
|
|
790
944
|
for step in unpruned_exits.difference(runtime_exits):
|
|
791
945
|
if logger:
|
|
792
|
-
logger.error(f'pruning removed all exit nodes for {step} in the {flow.name
|
|
946
|
+
logger.error(f'pruning removed all exit nodes for {step} in the {flow.name} '
|
|
793
947
|
'flowgraph')
|
|
794
948
|
error = True
|
|
795
949
|
|
|
@@ -797,7 +951,7 @@ class RuntimeFlowgraph:
|
|
|
797
951
|
runtime_entry = set([step for step, _ in runtime.get_entry_nodes()])
|
|
798
952
|
for step in unpruned_entry.difference(runtime_entry):
|
|
799
953
|
if logger:
|
|
800
|
-
logger.error(f'pruning removed all entry nodes for {step} in the {flow.name
|
|
954
|
+
logger.error(f'pruning removed all entry nodes for {step} in the {flow.name} '
|
|
801
955
|
'flowgraph')
|
|
802
956
|
error = True
|
|
803
957
|
|
|
@@ -814,7 +968,7 @@ class RuntimeFlowgraph:
|
|
|
814
968
|
exits = ",".join([f"{step}/{index}"
|
|
815
969
|
for step, index in runtime.get_exit_nodes()])
|
|
816
970
|
missing.append(f'no path from {entrynode[0]}/{entrynode[1]} to {exits} '
|
|
817
|
-
f'in the {flow.name
|
|
971
|
+
f'in the {flow.name} flowgraph')
|
|
818
972
|
if found:
|
|
819
973
|
found_any = True
|
|
820
974
|
if not found_any:
|
|
@@ -827,16 +981,41 @@ class RuntimeFlowgraph:
|
|
|
827
981
|
|
|
828
982
|
|
|
829
983
|
class FlowgraphNodeSchema(BaseSchema):
|
|
984
|
+
'''
|
|
985
|
+
Schema definition for a single node within a flowgraph.
|
|
986
|
+
'''
|
|
987
|
+
|
|
830
988
|
def __init__(self):
|
|
989
|
+
'''
|
|
990
|
+
Initializes a new FlowgraphNodeSchema.
|
|
991
|
+
'''
|
|
831
992
|
super().__init__()
|
|
832
993
|
|
|
833
994
|
schema_flowgraph(self)
|
|
834
995
|
|
|
996
|
+
@classmethod
|
|
997
|
+
def _getdict_type(cls) -> str:
|
|
998
|
+
"""
|
|
999
|
+
Returns the metadata type for `getdict` serialization.
|
|
1000
|
+
"""
|
|
1001
|
+
|
|
1002
|
+
return FlowgraphNodeSchema.__name__
|
|
1003
|
+
|
|
835
1004
|
|
|
836
1005
|
###############################################################################
|
|
837
1006
|
# Flow Configuration
|
|
838
1007
|
###############################################################################
|
|
839
1008
|
def schema_flowgraph(schema):
|
|
1009
|
+
'''
|
|
1010
|
+
Defines the schema parameters for a flowgraph node.
|
|
1011
|
+
|
|
1012
|
+
This function is called to populate a schema with parameters that
|
|
1013
|
+
define a node's properties, such as its inputs, weights, goals, and the
|
|
1014
|
+
tool/task it executes.
|
|
1015
|
+
|
|
1016
|
+
Args:
|
|
1017
|
+
schema (Schema): The schema object to configure.
|
|
1018
|
+
'''
|
|
840
1019
|
schema = EditableSchema(schema)
|
|
841
1020
|
|
|
842
1021
|
# flowgraph input
|