siliconcompiler 0.34.1__py3-none-any.whl → 0.34.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- siliconcompiler/__init__.py +23 -4
- siliconcompiler/__main__.py +1 -7
- siliconcompiler/_metadata.py +1 -1
- siliconcompiler/apps/_common.py +104 -23
- siliconcompiler/apps/sc.py +4 -8
- siliconcompiler/apps/sc_dashboard.py +6 -4
- siliconcompiler/apps/sc_install.py +10 -6
- siliconcompiler/apps/sc_issue.py +7 -5
- siliconcompiler/apps/sc_remote.py +1 -1
- siliconcompiler/apps/sc_server.py +9 -14
- siliconcompiler/apps/sc_show.py +7 -6
- siliconcompiler/apps/smake.py +130 -94
- siliconcompiler/apps/utils/replay.py +4 -7
- siliconcompiler/apps/utils/summarize.py +3 -5
- siliconcompiler/asic.py +420 -0
- siliconcompiler/checklist.py +25 -2
- siliconcompiler/cmdlineschema.py +534 -0
- siliconcompiler/constraints/__init__.py +17 -0
- siliconcompiler/constraints/asic_component.py +378 -0
- siliconcompiler/constraints/asic_floorplan.py +449 -0
- siliconcompiler/constraints/asic_pins.py +489 -0
- siliconcompiler/constraints/asic_timing.py +517 -0
- siliconcompiler/core.py +10 -35
- siliconcompiler/data/templates/tcl/manifest.tcl.j2 +8 -0
- siliconcompiler/dependencyschema.py +96 -202
- siliconcompiler/design.py +327 -241
- siliconcompiler/filesetschema.py +250 -0
- siliconcompiler/flowgraph.py +298 -106
- siliconcompiler/fpga.py +124 -1
- siliconcompiler/library.py +331 -0
- siliconcompiler/metric.py +327 -92
- siliconcompiler/metrics/__init__.py +7 -0
- siliconcompiler/metrics/asic.py +245 -0
- siliconcompiler/metrics/fpga.py +220 -0
- siliconcompiler/package/__init__.py +391 -67
- siliconcompiler/package/git.py +92 -16
- siliconcompiler/package/github.py +114 -22
- siliconcompiler/package/https.py +79 -16
- siliconcompiler/packageschema.py +341 -16
- siliconcompiler/pathschema.py +255 -0
- siliconcompiler/pdk.py +566 -1
- siliconcompiler/project.py +1460 -0
- siliconcompiler/record.py +38 -1
- siliconcompiler/remote/__init__.py +5 -2
- siliconcompiler/remote/client.py +11 -6
- siliconcompiler/remote/schema.py +5 -23
- siliconcompiler/remote/server.py +41 -54
- siliconcompiler/report/__init__.py +3 -3
- siliconcompiler/report/dashboard/__init__.py +48 -14
- siliconcompiler/report/dashboard/cli/__init__.py +99 -21
- siliconcompiler/report/dashboard/cli/board.py +364 -179
- siliconcompiler/report/dashboard/web/__init__.py +90 -12
- siliconcompiler/report/dashboard/web/components/__init__.py +219 -240
- siliconcompiler/report/dashboard/web/components/flowgraph.py +49 -26
- siliconcompiler/report/dashboard/web/components/graph.py +139 -100
- siliconcompiler/report/dashboard/web/layouts/__init__.py +29 -1
- siliconcompiler/report/dashboard/web/layouts/_common.py +38 -2
- siliconcompiler/report/dashboard/web/layouts/vertical_flowgraph.py +39 -26
- siliconcompiler/report/dashboard/web/layouts/vertical_flowgraph_node_tab.py +50 -50
- siliconcompiler/report/dashboard/web/layouts/vertical_flowgraph_sac_tabs.py +49 -46
- siliconcompiler/report/dashboard/web/state.py +141 -14
- siliconcompiler/report/dashboard/web/utils/__init__.py +79 -16
- siliconcompiler/report/dashboard/web/utils/file_utils.py +74 -11
- siliconcompiler/report/dashboard/web/viewer.py +25 -1
- siliconcompiler/report/report.py +5 -2
- siliconcompiler/report/summary_image.py +29 -11
- siliconcompiler/scheduler/__init__.py +9 -1
- siliconcompiler/scheduler/docker.py +81 -4
- siliconcompiler/scheduler/run_node.py +37 -20
- siliconcompiler/scheduler/scheduler.py +211 -36
- siliconcompiler/scheduler/schedulernode.py +394 -60
- siliconcompiler/scheduler/send_messages.py +77 -29
- siliconcompiler/scheduler/slurm.py +76 -12
- siliconcompiler/scheduler/taskscheduler.py +142 -21
- siliconcompiler/schema/__init__.py +0 -4
- siliconcompiler/schema/baseschema.py +338 -59
- siliconcompiler/schema/editableschema.py +14 -6
- siliconcompiler/schema/journal.py +28 -17
- siliconcompiler/schema/namedschema.py +22 -14
- siliconcompiler/schema/parameter.py +89 -28
- siliconcompiler/schema/parametertype.py +2 -0
- siliconcompiler/schema/parametervalue.py +258 -15
- siliconcompiler/schema/safeschema.py +25 -2
- siliconcompiler/schema/schema_cfg.py +23 -19
- siliconcompiler/schema/utils.py +2 -2
- siliconcompiler/schema_obj.py +24 -5
- siliconcompiler/tool.py +1131 -265
- siliconcompiler/tools/bambu/__init__.py +41 -0
- siliconcompiler/tools/builtin/concatenate.py +2 -2
- siliconcompiler/tools/builtin/minimum.py +2 -1
- siliconcompiler/tools/builtin/mux.py +2 -1
- siliconcompiler/tools/builtin/nop.py +2 -1
- siliconcompiler/tools/builtin/verify.py +2 -1
- siliconcompiler/tools/klayout/__init__.py +95 -0
- siliconcompiler/tools/openroad/__init__.py +289 -0
- siliconcompiler/tools/openroad/scripts/apr/preamble.tcl +3 -0
- siliconcompiler/tools/openroad/scripts/apr/sc_detailed_route.tcl +7 -2
- siliconcompiler/tools/openroad/scripts/apr/sc_global_route.tcl +8 -4
- siliconcompiler/tools/openroad/scripts/apr/sc_init_floorplan.tcl +9 -5
- siliconcompiler/tools/openroad/scripts/common/write_images.tcl +5 -1
- siliconcompiler/tools/slang/__init__.py +1 -1
- siliconcompiler/tools/slang/elaborate.py +2 -1
- siliconcompiler/tools/vivado/scripts/sc_run.tcl +1 -1
- siliconcompiler/tools/vivado/scripts/sc_syn_fpga.tcl +8 -1
- siliconcompiler/tools/vivado/syn_fpga.py +6 -0
- siliconcompiler/tools/vivado/vivado.py +35 -2
- siliconcompiler/tools/vpr/__init__.py +150 -0
- siliconcompiler/tools/yosys/__init__.py +369 -1
- siliconcompiler/tools/yosys/scripts/procs.tcl +0 -1
- siliconcompiler/toolscripts/_tools.json +5 -10
- siliconcompiler/utils/__init__.py +66 -0
- siliconcompiler/utils/flowgraph.py +2 -2
- siliconcompiler/utils/issue.py +2 -1
- siliconcompiler/utils/logging.py +14 -0
- siliconcompiler/utils/multiprocessing.py +256 -0
- siliconcompiler/utils/showtools.py +10 -0
- {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/METADATA +6 -6
- {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/RECORD +122 -115
- {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/entry_points.txt +3 -0
- siliconcompiler/schema/cmdlineschema.py +0 -250
- siliconcompiler/schema/packageschema.py +0 -101
- siliconcompiler/toolscripts/rhel8/install-slang.sh +0 -40
- siliconcompiler/toolscripts/rhel9/install-slang.sh +0 -40
- siliconcompiler/toolscripts/ubuntu20/install-slang.sh +0 -47
- siliconcompiler/toolscripts/ubuntu22/install-slang.sh +0 -37
- siliconcompiler/toolscripts/ubuntu24/install-slang.sh +0 -37
- {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/WHEEL +0 -0
- {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/licenses/LICENSE +0 -0
- {siliconcompiler-0.34.1.dist-info → siliconcompiler-0.34.3.dist-info}/top_level.txt +0 -0
siliconcompiler/flowgraph.py
CHANGED
|
@@ -10,7 +10,23 @@ from siliconcompiler import NodeStatus
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class FlowgraphSchema(NamedSchema):
|
|
13
|
+
'''
|
|
14
|
+
Schema for defining and interacting with a flowgraph.
|
|
15
|
+
|
|
16
|
+
A flowgraph is a directed acyclic graph (DAG) that represents the
|
|
17
|
+
compilation flow. Each node in the graph is a step/index pair that
|
|
18
|
+
|
|
19
|
+
maps to a specific tool task, and edges represent dependencies between
|
|
20
|
+
these tasks.
|
|
21
|
+
'''
|
|
22
|
+
|
|
13
23
|
def __init__(self, name=None):
|
|
24
|
+
'''
|
|
25
|
+
Initializes a new FlowgraphSchema object.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
name (str, optional): The name of the flowgraph. Defaults to None.
|
|
29
|
+
'''
|
|
14
30
|
super().__init__()
|
|
15
31
|
self.set_name(name)
|
|
16
32
|
|
|
@@ -21,7 +37,9 @@ class FlowgraphSchema(NamedSchema):
|
|
|
21
37
|
|
|
22
38
|
def __clear_cache(self):
|
|
23
39
|
'''
|
|
24
|
-
|
|
40
|
+
Clears the internal cache for memoized flowgraph properties.
|
|
41
|
+
|
|
42
|
+
This should be called any time the graph structure is modified.
|
|
25
43
|
'''
|
|
26
44
|
|
|
27
45
|
self.__cache_nodes = None
|
|
@@ -38,27 +56,27 @@ class FlowgraphSchema(NamedSchema):
|
|
|
38
56
|
'''
|
|
39
57
|
Creates a flowgraph node.
|
|
40
58
|
|
|
41
|
-
Creates a flowgraph node by binding a step to a tool
|
|
42
|
-
A tool can be an external executable or one of the built
|
|
43
|
-
in the SiliconCompiler framework
|
|
44
|
-
maximum, join, mux, verify.
|
|
59
|
+
Creates a flowgraph node by binding a step to a tool-specific task.
|
|
60
|
+
A tool can be an external executable or one of the built-in functions
|
|
61
|
+
in the SiliconCompiler framework (e.g., minimum, maximum, join).
|
|
45
62
|
|
|
46
63
|
The method modifies the following schema parameters:
|
|
47
64
|
|
|
48
|
-
* [<step
|
|
49
|
-
* [<step
|
|
50
|
-
* [<step
|
|
65
|
+
* `['<step>', '<index>', 'tool']`
|
|
66
|
+
* `['<step>', '<index>', 'task']`
|
|
67
|
+
* `['<step>', '<index>', 'taskmodule']`
|
|
51
68
|
|
|
52
69
|
Args:
|
|
53
|
-
step (str): Step name
|
|
54
|
-
task (module
|
|
55
|
-
|
|
70
|
+
step (str): Step name for the node.
|
|
71
|
+
task (module or str): The task to associate with this node. Can be
|
|
72
|
+
a module object or a string in the format '<tool>.<task>'.
|
|
73
|
+
index (int or str): Index for the step. Defaults to 0.
|
|
56
74
|
|
|
57
75
|
Examples:
|
|
58
|
-
>>> import
|
|
59
|
-
>>> flow.node('
|
|
60
|
-
Creates a 'place' task
|
|
61
|
-
'
|
|
76
|
+
>>> import siliconcompiler.tools.openroad as openroad
|
|
77
|
+
>>> flow.node('place', openroad.place, index=0)
|
|
78
|
+
# Creates a node for the 'place' task in the 'openroad' tool,
|
|
79
|
+
# identified by step='place' and index=0.
|
|
62
80
|
'''
|
|
63
81
|
|
|
64
82
|
if step in (Schema.GLOBAL_KEY, 'default', 'sc_collected_files'):
|
|
@@ -68,6 +86,11 @@ class FlowgraphSchema(NamedSchema):
|
|
|
68
86
|
if index in (Schema.GLOBAL_KEY, 'default'):
|
|
69
87
|
raise ValueError(f"{index} is a reserved name")
|
|
70
88
|
|
|
89
|
+
if '/' in step:
|
|
90
|
+
raise ValueError(f"{step} is not a valid step, it cannot contain '/'")
|
|
91
|
+
if '/' in index:
|
|
92
|
+
raise ValueError(f"{index} is not a valid index, it cannot contain '/'")
|
|
93
|
+
|
|
71
94
|
# Determine task name and module
|
|
72
95
|
task_module = None
|
|
73
96
|
if isinstance(task, str):
|
|
@@ -83,11 +106,6 @@ class FlowgraphSchema(NamedSchema):
|
|
|
83
106
|
raise ValueError(f"{task} is not a valid task, it must be associated with "
|
|
84
107
|
"a tool '<tool>.<task>'.")
|
|
85
108
|
|
|
86
|
-
if '/' in step:
|
|
87
|
-
raise ValueError(f"{step} is not a valid step, it cannot contain '/'")
|
|
88
|
-
if '/' in index:
|
|
89
|
-
raise ValueError(f"{index} is not a valid index, it cannot contain '/'")
|
|
90
|
-
|
|
91
109
|
tool_name, task_name = task_parts[-2:]
|
|
92
110
|
|
|
93
111
|
# bind tool to node
|
|
@@ -104,26 +122,26 @@ class FlowgraphSchema(NamedSchema):
|
|
|
104
122
|
Connects the output of a tail node with the input of a head node by
|
|
105
123
|
setting the 'input' field of the head node in the schema flowgraph.
|
|
106
124
|
|
|
107
|
-
The method modifies the following
|
|
125
|
+
The method modifies the following parameter:
|
|
108
126
|
|
|
109
|
-
[<head
|
|
127
|
+
* `['<head>', '<head_index>', 'input']`
|
|
110
128
|
|
|
111
129
|
Args:
|
|
112
|
-
tail (str):
|
|
113
|
-
head (str):
|
|
114
|
-
tail_index (int
|
|
115
|
-
head_index (int
|
|
130
|
+
tail (str): Step name of the tail node.
|
|
131
|
+
head (str): Step name of the head node.
|
|
132
|
+
tail_index (int or str): Index of the tail node. Defaults to 0.
|
|
133
|
+
head_index (int or str): Index of the head node. Defaults to 0.
|
|
116
134
|
|
|
117
135
|
Examples:
|
|
118
136
|
>>> chip.edge('place', 'cts')
|
|
119
|
-
Creates a directed edge from place to cts.
|
|
137
|
+
# Creates a directed edge from ('place', '0') to ('cts', '0').
|
|
120
138
|
'''
|
|
121
139
|
head_index = str(head_index)
|
|
122
140
|
tail_index = str(tail_index)
|
|
123
141
|
|
|
124
142
|
for step, index in [(head, head_index), (tail, tail_index)]:
|
|
125
143
|
if not self.valid(step, index):
|
|
126
|
-
raise ValueError(f"{step}/{index} is not a defined node in {self.name
|
|
144
|
+
raise ValueError(f"{step}/{index} is not a defined node in {self.name}.")
|
|
127
145
|
|
|
128
146
|
tail_node = (tail, tail_index)
|
|
129
147
|
if tail_node in self.get(head, head_index, 'input'):
|
|
@@ -135,15 +153,16 @@ class FlowgraphSchema(NamedSchema):
|
|
|
135
153
|
|
|
136
154
|
def remove_node(self, step, index=None):
|
|
137
155
|
'''
|
|
138
|
-
|
|
156
|
+
Removes a flowgraph node and reconnects its inputs to its outputs.
|
|
139
157
|
|
|
140
158
|
Args:
|
|
141
|
-
step (str): Step name
|
|
142
|
-
index (int
|
|
159
|
+
step (str): Step name of the node to remove.
|
|
160
|
+
index (int or str, optional): Index of the node to remove. If None,
|
|
161
|
+
all nodes for the given step are removed. Defaults to None.
|
|
143
162
|
'''
|
|
144
163
|
|
|
145
164
|
if step not in self.getkeys():
|
|
146
|
-
raise ValueError(f'{step} is not a valid step in {self.name
|
|
165
|
+
raise ValueError(f'{step} is not a valid step in {self.name}')
|
|
147
166
|
|
|
148
167
|
if index is None:
|
|
149
168
|
# Iterate over all indexes
|
|
@@ -153,7 +172,7 @@ class FlowgraphSchema(NamedSchema):
|
|
|
153
172
|
|
|
154
173
|
index = str(index)
|
|
155
174
|
if index not in self.getkeys(step):
|
|
156
|
-
raise ValueError(f'{index} is not a valid index for {step} in {self.name
|
|
175
|
+
raise ValueError(f'{index} is not a valid index for {step} in {self.name}')
|
|
157
176
|
|
|
158
177
|
# Save input edges
|
|
159
178
|
node = (step, index)
|
|
@@ -178,21 +197,24 @@ class FlowgraphSchema(NamedSchema):
|
|
|
178
197
|
|
|
179
198
|
def insert_node(self, step, task, before_step, index=0, before_index=0):
|
|
180
199
|
'''
|
|
181
|
-
|
|
200
|
+
Inserts a new node in the graph before a specified node.
|
|
201
|
+
|
|
202
|
+
The new node is placed between the `before` node and all of its
|
|
203
|
+
original inputs.
|
|
182
204
|
|
|
183
205
|
Args:
|
|
184
|
-
step (str): Step name
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
before_index (int
|
|
206
|
+
step (str): Step name for the new node.
|
|
207
|
+
task (module or str): Task to associate with the new node.
|
|
208
|
+
before_step (str): Step name of the existing node to insert before.
|
|
209
|
+
index (int or str): Index for the new node. Defaults to 0.
|
|
210
|
+
before_index (int or str): Index of the existing node. Defaults to 0.
|
|
189
211
|
'''
|
|
190
212
|
|
|
191
213
|
index = str(index)
|
|
192
214
|
before_index = str(before_index)
|
|
193
215
|
|
|
194
216
|
if (before_step, before_index) not in self.get_nodes():
|
|
195
|
-
raise ValueError(f'{before_step}/{before_index} is not a valid node in {self.name
|
|
217
|
+
raise ValueError(f'{before_step}/{before_index} is not a valid node in {self.name}')
|
|
196
218
|
|
|
197
219
|
# add the node
|
|
198
220
|
self.node(step, task, index=index)
|
|
@@ -210,15 +232,16 @@ class FlowgraphSchema(NamedSchema):
|
|
|
210
232
|
###########################################################################
|
|
211
233
|
def graph(self, subflow, name=None):
|
|
212
234
|
'''
|
|
213
|
-
Instantiates a
|
|
235
|
+
Instantiates a sub-flowgraph within the current flowgraph.
|
|
214
236
|
|
|
215
237
|
Args:
|
|
216
|
-
subflow (
|
|
217
|
-
name (str):
|
|
238
|
+
subflow (FlowgraphSchema): The flowgraph to instantiate.
|
|
239
|
+
name (str, optional): A prefix to add to the names of the
|
|
240
|
+
instantiated steps to ensure they are unique. Defaults to None.
|
|
218
241
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
242
|
+
Raises:
|
|
243
|
+
ValueError: If `subflow` is not a `FlowgraphSchema` object, or if
|
|
244
|
+
a step from the sub-flowgraph already exists in the current graph.
|
|
222
245
|
'''
|
|
223
246
|
if not isinstance(subflow, FlowgraphSchema):
|
|
224
247
|
raise ValueError(f"subflow must a FlowgraphSchema, not: {type(subflow)}")
|
|
@@ -252,7 +275,12 @@ class FlowgraphSchema(NamedSchema):
|
|
|
252
275
|
|
|
253
276
|
def get_nodes(self):
|
|
254
277
|
'''
|
|
255
|
-
Returns all
|
|
278
|
+
Returns a sorted tuple of all nodes defined in this flowgraph.
|
|
279
|
+
|
|
280
|
+
A node is represented as a `(step, index)` tuple.
|
|
281
|
+
|
|
282
|
+
Returns:
|
|
283
|
+
tuple[tuple(str,str)]: All nodes in the graph.
|
|
256
284
|
'''
|
|
257
285
|
if self.__cache_nodes is not None:
|
|
258
286
|
return self.__cache_nodes
|
|
@@ -268,8 +296,12 @@ class FlowgraphSchema(NamedSchema):
|
|
|
268
296
|
|
|
269
297
|
def get_entry_nodes(self):
|
|
270
298
|
'''
|
|
271
|
-
|
|
272
|
-
|
|
299
|
+
Collects all nodes that are entry points to the flowgraph.
|
|
300
|
+
|
|
301
|
+
Entry nodes are those with no inputs.
|
|
302
|
+
|
|
303
|
+
Returns:
|
|
304
|
+
tuple[tuple(str,str)]: All entry nodes in the graph.
|
|
273
305
|
'''
|
|
274
306
|
|
|
275
307
|
if self.__cache_nodes_entry is not None:
|
|
@@ -286,8 +318,12 @@ class FlowgraphSchema(NamedSchema):
|
|
|
286
318
|
|
|
287
319
|
def get_exit_nodes(self):
|
|
288
320
|
'''
|
|
289
|
-
|
|
290
|
-
|
|
321
|
+
Collects all nodes that are exit points of the flowgraph.
|
|
322
|
+
|
|
323
|
+
Exit nodes are those that are not inputs to any other node.
|
|
324
|
+
|
|
325
|
+
Returns:
|
|
326
|
+
tuple[tuple(str,str)]: All exit nodes in the graph.
|
|
291
327
|
'''
|
|
292
328
|
|
|
293
329
|
if self.__cache_nodes_exit is not None:
|
|
@@ -307,11 +343,15 @@ class FlowgraphSchema(NamedSchema):
|
|
|
307
343
|
|
|
308
344
|
def get_execution_order(self, reverse=False):
|
|
309
345
|
'''
|
|
310
|
-
Generates a list of nodes
|
|
346
|
+
Generates a topologically sorted list of nodes for execution.
|
|
311
347
|
|
|
312
348
|
Args:
|
|
313
|
-
reverse (
|
|
314
|
-
to entry nodes.
|
|
349
|
+
reverse (bool): If True, the order is reversed, from exit nodes
|
|
350
|
+
to entry nodes. Defaults to False.
|
|
351
|
+
|
|
352
|
+
Returns:
|
|
353
|
+
tuple[tuple[tuple(str,str)]]: A tuple of tuples, where each inner
|
|
354
|
+
tuple represents a level of nodes that can be executed in parallel.
|
|
315
355
|
'''
|
|
316
356
|
|
|
317
357
|
if reverse:
|
|
@@ -387,11 +427,14 @@ class FlowgraphSchema(NamedSchema):
|
|
|
387
427
|
|
|
388
428
|
def get_node_outputs(self, step, index):
|
|
389
429
|
'''
|
|
390
|
-
Returns the nodes the given
|
|
430
|
+
Returns the nodes that the given node provides input to.
|
|
391
431
|
|
|
392
432
|
Args:
|
|
393
|
-
step (str):
|
|
394
|
-
index (str
|
|
433
|
+
step (str): Step name of the source node.
|
|
434
|
+
index (str or int): Index of the source node.
|
|
435
|
+
|
|
436
|
+
Returns:
|
|
437
|
+
tuple[tuple(str,str)]: A tuple of destination nodes.
|
|
395
438
|
'''
|
|
396
439
|
|
|
397
440
|
index = str(index)
|
|
@@ -423,12 +466,15 @@ class FlowgraphSchema(NamedSchema):
|
|
|
423
466
|
|
|
424
467
|
def __find_loops(self, step, index, path=None):
|
|
425
468
|
'''
|
|
426
|
-
|
|
469
|
+
Internal helper to search for loops in the graph via depth-first search.
|
|
427
470
|
|
|
428
471
|
Args:
|
|
429
|
-
step (str):
|
|
430
|
-
index (str)
|
|
431
|
-
path (list
|
|
472
|
+
step (str): Step name to start from.
|
|
473
|
+
index (str): Index name to start from.
|
|
474
|
+
path (list, optional): The path taken so far. Defaults to None.
|
|
475
|
+
|
|
476
|
+
Returns:
|
|
477
|
+
list: A list of nodes forming a loop, or None if no loop is found.
|
|
432
478
|
'''
|
|
433
479
|
if path is None:
|
|
434
480
|
path = []
|
|
@@ -448,17 +494,20 @@ class FlowgraphSchema(NamedSchema):
|
|
|
448
494
|
|
|
449
495
|
def validate(self, logger=None):
|
|
450
496
|
'''
|
|
451
|
-
|
|
497
|
+
Checks if the flowgraph is valid.
|
|
452
498
|
|
|
453
|
-
|
|
454
|
-
*
|
|
455
|
-
*
|
|
456
|
-
*
|
|
457
|
-
|
|
458
|
-
Returns True if valid, False otherwise.
|
|
499
|
+
This method performs several checks:
|
|
500
|
+
* All edges must point to and from valid nodes.
|
|
501
|
+
* There should be no duplicate edges.
|
|
502
|
+
* All nodes must have their tool, task, and taskmodule defined.
|
|
503
|
+
* The graph must not contain any loops (it must be a DAG).
|
|
459
504
|
|
|
460
505
|
Args:
|
|
461
|
-
logger (logging.Logger): logger to use for reporting
|
|
506
|
+
logger (logging.Logger, optional): A logger to use for reporting
|
|
507
|
+
errors. Defaults to None.
|
|
508
|
+
|
|
509
|
+
Returns:
|
|
510
|
+
bool: True if the graph is valid, False otherwise.
|
|
462
511
|
'''
|
|
463
512
|
|
|
464
513
|
error = False
|
|
@@ -474,14 +523,14 @@ class FlowgraphSchema(NamedSchema):
|
|
|
474
523
|
in_step, in_index = node
|
|
475
524
|
if logger:
|
|
476
525
|
logger.error(f'Duplicate edge from {in_step}/{in_index} to '
|
|
477
|
-
f'{step}/{index} in the {self.name
|
|
526
|
+
f'{step}/{index} in the {self.name} flowgraph')
|
|
478
527
|
error = True
|
|
479
528
|
|
|
480
529
|
diff_nodes = check_nodes.difference(self.get_nodes())
|
|
481
530
|
if diff_nodes:
|
|
482
531
|
if logger:
|
|
483
532
|
for step, index in diff_nodes:
|
|
484
|
-
logger.error(f'{step}/{index} is missing in the {self.name
|
|
533
|
+
logger.error(f'{step}/{index} is missing in the {self.name} flowgraph')
|
|
485
534
|
error = True
|
|
486
535
|
|
|
487
536
|
# Detect missing definitions
|
|
@@ -490,7 +539,7 @@ class FlowgraphSchema(NamedSchema):
|
|
|
490
539
|
if not self.get(step, index, item):
|
|
491
540
|
if logger:
|
|
492
541
|
logger.error(f'{step}/{index} is missing a {item} definition in the '
|
|
493
|
-
f'{self.name
|
|
542
|
+
f'{self.name} flowgraph')
|
|
494
543
|
error = True
|
|
495
544
|
|
|
496
545
|
# detect loops
|
|
@@ -500,49 +549,90 @@ class FlowgraphSchema(NamedSchema):
|
|
|
500
549
|
error = True
|
|
501
550
|
if logger:
|
|
502
551
|
loop_path = [f"{step}/{index}" for step, index in loop_path]
|
|
503
|
-
logger.error(f"{' -> '.join(loop_path)} forms a loop in {self.name
|
|
552
|
+
logger.error(f"{' -> '.join(loop_path)} forms a loop in {self.name}")
|
|
504
553
|
|
|
505
554
|
return not error
|
|
506
555
|
|
|
556
|
+
def __get_task_module(self, name):
|
|
557
|
+
'''
|
|
558
|
+
Internal helper to import and cache a task module by name.
|
|
559
|
+
'''
|
|
560
|
+
# Create cache
|
|
561
|
+
if self.__cache_tasks is None:
|
|
562
|
+
self.__cache_tasks = {}
|
|
563
|
+
|
|
564
|
+
if name in self.__cache_tasks:
|
|
565
|
+
return self.__cache_tasks[name]
|
|
566
|
+
|
|
567
|
+
self.__cache_tasks[name] = importlib.import_module(name)
|
|
568
|
+
return self.__cache_tasks[name]
|
|
569
|
+
|
|
507
570
|
def get_task_module(self, step, index):
|
|
508
571
|
"""
|
|
509
|
-
Returns the module for a given task
|
|
572
|
+
Returns the imported Python module for a given task node.
|
|
510
573
|
|
|
511
574
|
Args:
|
|
512
|
-
step (str): Step name
|
|
513
|
-
index (int
|
|
575
|
+
step (str): Step name of the node.
|
|
576
|
+
index (int or str): Index of the node.
|
|
577
|
+
|
|
578
|
+
Returns:
|
|
579
|
+
module: The imported task module.
|
|
514
580
|
"""
|
|
515
581
|
|
|
516
582
|
index = str(index)
|
|
517
583
|
|
|
518
584
|
if (step, index) not in self.get_nodes():
|
|
519
|
-
raise ValueError(f"{step}/{index} is not a valid node in {self.name
|
|
585
|
+
raise ValueError(f"{step}/{index} is not a valid node in {self.name}.")
|
|
520
586
|
|
|
521
|
-
|
|
587
|
+
return self.__get_task_module(self.get(step, index, 'taskmodule'))
|
|
522
588
|
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
589
|
+
def get_all_tasks(self):
|
|
590
|
+
'''
|
|
591
|
+
Returns all unique task modules used in this flowgraph.
|
|
592
|
+
|
|
593
|
+
Returns:
|
|
594
|
+
set[module]: A set of all imported task modules.
|
|
595
|
+
'''
|
|
596
|
+
tasks = set()
|
|
597
|
+
for step, index in self.get_nodes():
|
|
598
|
+
tasks.add(self.get_task_module(step, index))
|
|
599
|
+
return tasks
|
|
526
600
|
|
|
527
|
-
|
|
528
|
-
|
|
601
|
+
@classmethod
|
|
602
|
+
def _getdict_type(cls) -> str:
|
|
603
|
+
"""
|
|
604
|
+
Returns the metadata type for `getdict` serialization.
|
|
605
|
+
"""
|
|
529
606
|
|
|
530
|
-
|
|
531
|
-
return self.__cache_tasks[taskmodule]
|
|
607
|
+
return FlowgraphSchema.__name__
|
|
532
608
|
|
|
533
609
|
|
|
534
610
|
class RuntimeFlowgraph:
|
|
535
611
|
'''
|
|
536
|
-
|
|
612
|
+
A runtime representation of a flowgraph for a specific execution.
|
|
537
613
|
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
to_steps (list of steps): step to end the runtime at
|
|
543
|
-
prune_nodes (list of nodes): nodes to remove from execution
|
|
614
|
+
This class creates a "view" of a base flowgraph that considers runtime
|
|
615
|
+
options such as the start step (`-from`), end step (`-to`), and nodes to
|
|
616
|
+
exclude (`-prune`). It computes the precise subgraph of nodes that need
|
|
617
|
+
to be executed for a given run.
|
|
544
618
|
'''
|
|
619
|
+
|
|
545
620
|
def __init__(self, base, args=None, from_steps=None, to_steps=None, prune_nodes=None):
|
|
621
|
+
'''
|
|
622
|
+
Initializes a new RuntimeFlowgraph.
|
|
623
|
+
|
|
624
|
+
Args:
|
|
625
|
+
base (FlowgraphSchema): The base flowgraph to create a view of.
|
|
626
|
+
args (tuple[str, str], optional): A specific `(step, index)` to run.
|
|
627
|
+
If provided, this overrides `from_steps` and `to_steps`.
|
|
628
|
+
Defaults to None.
|
|
629
|
+
from_steps (list[str], optional): List of step names to start execution
|
|
630
|
+
from. Defaults to the base graph's entry nodes.
|
|
631
|
+
to_steps (list[str], optional): List of step names to end execution at.
|
|
632
|
+
Defaults to the base graph's exit nodes.
|
|
633
|
+
prune_nodes (list[tuple(str,str)], optional): A list of `(step, index)`
|
|
634
|
+
nodes to exclude from the graph. Defaults to None.
|
|
635
|
+
'''
|
|
546
636
|
if not all([hasattr(base, attr) for attr in dir(FlowgraphSchema)]):
|
|
547
637
|
raise ValueError(f"base must a FlowgraphSchema, not: {type(base)}")
|
|
548
638
|
|
|
@@ -594,6 +684,21 @@ class RuntimeFlowgraph:
|
|
|
594
684
|
self.__compute_graph()
|
|
595
685
|
|
|
596
686
|
def __walk_graph(self, node, path=None, reverse=True):
|
|
687
|
+
'''
|
|
688
|
+
Internal helper to recursively walk the graph to find all connected nodes.
|
|
689
|
+
|
|
690
|
+
This walk respects the runtime boundaries (`-from`, `-to`, `-prune`).
|
|
691
|
+
|
|
692
|
+
Args:
|
|
693
|
+
node (tuple(str,str)): The node to start the walk from.
|
|
694
|
+
path (list, optional): The path taken so far, used for cycle
|
|
695
|
+
detection. Defaults to None.
|
|
696
|
+
reverse (bool, optional): If True, walks backwards along inputs.
|
|
697
|
+
If False, walks forwards along outputs. Defaults to True.
|
|
698
|
+
|
|
699
|
+
Returns:
|
|
700
|
+
set[tuple(str,str)]: The set of nodes visited during the walk.
|
|
701
|
+
'''
|
|
597
702
|
if node in self.__prune:
|
|
598
703
|
return set()
|
|
599
704
|
|
|
@@ -622,7 +727,10 @@ class RuntimeFlowgraph:
|
|
|
622
727
|
|
|
623
728
|
def __compute_graph(self):
|
|
624
729
|
'''
|
|
625
|
-
|
|
730
|
+
Internal helper to precompute the runtime graph information.
|
|
731
|
+
|
|
732
|
+
This method determines the final set of nodes, entry/exit points, and
|
|
733
|
+
the execution order based on the runtime constraints.
|
|
626
734
|
'''
|
|
627
735
|
|
|
628
736
|
self.__nodes = set()
|
|
@@ -651,35 +759,51 @@ class RuntimeFlowgraph:
|
|
|
651
759
|
|
|
652
760
|
def get_nodes(self):
|
|
653
761
|
'''
|
|
654
|
-
Returns the nodes
|
|
762
|
+
Returns the nodes that are part of this runtime graph.
|
|
763
|
+
|
|
764
|
+
Returns:
|
|
765
|
+
tuple[tuple(str,str)]: A tuple of all nodes in the runtime graph.
|
|
655
766
|
'''
|
|
656
767
|
return self.__nodes
|
|
657
768
|
|
|
658
769
|
def get_execution_order(self):
|
|
659
770
|
'''
|
|
660
|
-
Returns the execution order of the nodes
|
|
771
|
+
Returns the execution order of the nodes in this runtime graph.
|
|
772
|
+
|
|
773
|
+
Returns:
|
|
774
|
+
tuple[tuple[tuple(str,str)]]: A tuple of tuples representing
|
|
775
|
+
parallel execution levels.
|
|
661
776
|
'''
|
|
662
777
|
return self.__execution_order
|
|
663
778
|
|
|
664
779
|
def get_entry_nodes(self):
|
|
665
780
|
'''
|
|
666
|
-
Returns the entry nodes for this graph
|
|
781
|
+
Returns the entry nodes for this runtime graph.
|
|
782
|
+
|
|
783
|
+
Returns:
|
|
784
|
+
tuple[tuple(str,str)]: A tuple of all entry nodes.
|
|
667
785
|
'''
|
|
668
786
|
return self.__from
|
|
669
787
|
|
|
670
788
|
def get_exit_nodes(self):
|
|
671
789
|
'''
|
|
672
|
-
Returns the exit nodes for this graph
|
|
790
|
+
Returns the exit nodes for this runtime graph.
|
|
791
|
+
|
|
792
|
+
Returns:
|
|
793
|
+
tuple[tuple(str,str)]: A tuple of all exit nodes.
|
|
673
794
|
'''
|
|
674
795
|
return self.__to
|
|
675
796
|
|
|
676
797
|
def get_nodes_starting_at(self, step, index):
|
|
677
798
|
'''
|
|
678
|
-
Returns all
|
|
799
|
+
Returns all nodes reachable from a given starting node in this runtime graph.
|
|
679
800
|
|
|
680
801
|
Args:
|
|
681
|
-
step (str): step
|
|
682
|
-
index (str
|
|
802
|
+
step (str): The step name of the starting node.
|
|
803
|
+
index (str or int): The index of the starting node.
|
|
804
|
+
|
|
805
|
+
Returns:
|
|
806
|
+
tuple[tuple(str,str)]: A tuple of all reachable nodes.
|
|
683
807
|
'''
|
|
684
808
|
index = str(index)
|
|
685
809
|
|
|
@@ -689,6 +813,21 @@ class RuntimeFlowgraph:
|
|
|
689
813
|
return tuple(sorted(self.__walk_graph((step, str(index)), reverse=False)))
|
|
690
814
|
|
|
691
815
|
def get_node_inputs(self, step, index, record=None):
|
|
816
|
+
'''
|
|
817
|
+
Gets the inputs for a specific node in the runtime graph.
|
|
818
|
+
|
|
819
|
+
If a `record` object is provided, this method will traverse through
|
|
820
|
+
any input nodes that were SKIPPED to find the true, non-skipped inputs.
|
|
821
|
+
|
|
822
|
+
Args:
|
|
823
|
+
step (str): Step name of the node.
|
|
824
|
+
index (str): Index of the node.
|
|
825
|
+
record (Schema, optional): A schema object containing run records.
|
|
826
|
+
Used to check the status of input nodes. Defaults to None.
|
|
827
|
+
|
|
828
|
+
Returns:
|
|
829
|
+
list[tuple(str,str)]: A list of input nodes.
|
|
830
|
+
'''
|
|
692
831
|
if (step, index) not in self.get_nodes():
|
|
693
832
|
raise ValueError(f"{step}/{index} is not a valid node")
|
|
694
833
|
|
|
@@ -712,6 +851,16 @@ class RuntimeFlowgraph:
|
|
|
712
851
|
return sorted(inputs)
|
|
713
852
|
|
|
714
853
|
def get_completed_nodes(self, record=None):
|
|
854
|
+
'''
|
|
855
|
+
Finds all nodes in this runtime graph that have successfully completed.
|
|
856
|
+
|
|
857
|
+
Args:
|
|
858
|
+
record (Schema, optional): A schema object containing run records
|
|
859
|
+
to check for node status. Defaults to None.
|
|
860
|
+
|
|
861
|
+
Returns:
|
|
862
|
+
list[tuple(str,str)]: A sorted list of successfully completed nodes.
|
|
863
|
+
'''
|
|
715
864
|
if not record:
|
|
716
865
|
return []
|
|
717
866
|
|
|
@@ -724,6 +873,24 @@ class RuntimeFlowgraph:
|
|
|
724
873
|
|
|
725
874
|
@staticmethod
|
|
726
875
|
def validate(flow, from_steps=None, to_steps=None, prune_nodes=None, logger=None):
|
|
876
|
+
'''
|
|
877
|
+
Validates runtime options against a flowgraph.
|
|
878
|
+
|
|
879
|
+
Checks for undefined steps and ensures that pruning does not break
|
|
880
|
+
the graph by removing all entry/exit points or creating disjoint paths.
|
|
881
|
+
|
|
882
|
+
Args:
|
|
883
|
+
flow (FlowgraphSchema): The flowgraph to validate against.
|
|
884
|
+
from_steps (list[str], optional): List of start steps. Defaults to None.
|
|
885
|
+
to_steps (list[str], optional): List of end steps. Defaults to None.
|
|
886
|
+
prune_nodes (list[tuple(str,str)], optional): List of nodes to prune.
|
|
887
|
+
Defaults to None.
|
|
888
|
+
logger (logging.Logger, optional): Logger for error reporting.
|
|
889
|
+
Defaults to None.
|
|
890
|
+
|
|
891
|
+
Returns:
|
|
892
|
+
bool: True if the runtime configuration is valid, False otherwise.
|
|
893
|
+
'''
|
|
727
894
|
all_steps = set([step for step, _ in flow.get_nodes()])
|
|
728
895
|
|
|
729
896
|
if from_steps:
|
|
@@ -746,18 +913,18 @@ class RuntimeFlowgraph:
|
|
|
746
913
|
# Check for undefined steps
|
|
747
914
|
for step in sorted(from_steps.difference(all_steps)):
|
|
748
915
|
if logger:
|
|
749
|
-
logger.error(f'From {step} is not defined in the {flow.name
|
|
916
|
+
logger.error(f'From {step} is not defined in the {flow.name} flowgraph')
|
|
750
917
|
error = True
|
|
751
918
|
|
|
752
919
|
for step in sorted(to_steps.difference(all_steps)):
|
|
753
920
|
if logger:
|
|
754
|
-
logger.error(f'To {step} is not defined in the {flow.name
|
|
921
|
+
logger.error(f'To {step} is not defined in the {flow.name} flowgraph')
|
|
755
922
|
error = True
|
|
756
923
|
|
|
757
924
|
# Check for undefined prunes
|
|
758
925
|
for step, index in sorted(prune_nodes.difference(flow.get_nodes())):
|
|
759
926
|
if logger:
|
|
760
|
-
logger.error(f'{step}/{index} is not defined in the {flow.name
|
|
927
|
+
logger.error(f'{step}/{index} is not defined in the {flow.name} flowgraph')
|
|
761
928
|
error = True
|
|
762
929
|
|
|
763
930
|
if not error:
|
|
@@ -776,7 +943,7 @@ class RuntimeFlowgraph:
|
|
|
776
943
|
runtime_exits = set([step for step, _ in runtime.get_exit_nodes()])
|
|
777
944
|
for step in unpruned_exits.difference(runtime_exits):
|
|
778
945
|
if logger:
|
|
779
|
-
logger.error(f'pruning removed all exit nodes for {step} in the {flow.name
|
|
946
|
+
logger.error(f'pruning removed all exit nodes for {step} in the {flow.name} '
|
|
780
947
|
'flowgraph')
|
|
781
948
|
error = True
|
|
782
949
|
|
|
@@ -784,7 +951,7 @@ class RuntimeFlowgraph:
|
|
|
784
951
|
runtime_entry = set([step for step, _ in runtime.get_entry_nodes()])
|
|
785
952
|
for step in unpruned_entry.difference(runtime_entry):
|
|
786
953
|
if logger:
|
|
787
|
-
logger.error(f'pruning removed all entry nodes for {step} in the {flow.name
|
|
954
|
+
logger.error(f'pruning removed all entry nodes for {step} in the {flow.name} '
|
|
788
955
|
'flowgraph')
|
|
789
956
|
error = True
|
|
790
957
|
|
|
@@ -801,7 +968,7 @@ class RuntimeFlowgraph:
|
|
|
801
968
|
exits = ",".join([f"{step}/{index}"
|
|
802
969
|
for step, index in runtime.get_exit_nodes()])
|
|
803
970
|
missing.append(f'no path from {entrynode[0]}/{entrynode[1]} to {exits} '
|
|
804
|
-
f'in the {flow.name
|
|
971
|
+
f'in the {flow.name} flowgraph')
|
|
805
972
|
if found:
|
|
806
973
|
found_any = True
|
|
807
974
|
if not found_any:
|
|
@@ -814,16 +981,41 @@ class RuntimeFlowgraph:
|
|
|
814
981
|
|
|
815
982
|
|
|
816
983
|
class FlowgraphNodeSchema(BaseSchema):
|
|
984
|
+
'''
|
|
985
|
+
Schema definition for a single node within a flowgraph.
|
|
986
|
+
'''
|
|
987
|
+
|
|
817
988
|
def __init__(self):
|
|
989
|
+
'''
|
|
990
|
+
Initializes a new FlowgraphNodeSchema.
|
|
991
|
+
'''
|
|
818
992
|
super().__init__()
|
|
819
993
|
|
|
820
994
|
schema_flowgraph(self)
|
|
821
995
|
|
|
996
|
+
@classmethod
|
|
997
|
+
def _getdict_type(cls) -> str:
|
|
998
|
+
"""
|
|
999
|
+
Returns the metadata type for `getdict` serialization.
|
|
1000
|
+
"""
|
|
1001
|
+
|
|
1002
|
+
return FlowgraphNodeSchema.__name__
|
|
1003
|
+
|
|
822
1004
|
|
|
823
1005
|
###############################################################################
|
|
824
1006
|
# Flow Configuration
|
|
825
1007
|
###############################################################################
|
|
826
1008
|
def schema_flowgraph(schema):
|
|
1009
|
+
'''
|
|
1010
|
+
Defines the schema parameters for a flowgraph node.
|
|
1011
|
+
|
|
1012
|
+
This function is called to populate a schema with parameters that
|
|
1013
|
+
define a node's properties, such as its inputs, weights, goals, and the
|
|
1014
|
+
tool/task it executes.
|
|
1015
|
+
|
|
1016
|
+
Args:
|
|
1017
|
+
schema (Schema): The schema object to configure.
|
|
1018
|
+
'''
|
|
827
1019
|
schema = EditableSchema(schema)
|
|
828
1020
|
|
|
829
1021
|
# flowgraph input
|