siliconcompiler 0.35.0__py3-none-any.whl → 0.35.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- siliconcompiler/_metadata.py +1 -1
- siliconcompiler/apps/_common.py +3 -2
- siliconcompiler/apps/sc_dashboard.py +3 -1
- siliconcompiler/apps/sc_install.py +149 -37
- siliconcompiler/apps/smake.py +9 -3
- siliconcompiler/checklist.py +3 -3
- siliconcompiler/data/demo_fpga/z1000_yosys_config.json +24 -0
- siliconcompiler/design.py +51 -45
- siliconcompiler/flowgraph.py +2 -2
- siliconcompiler/library.py +23 -12
- siliconcompiler/package/__init__.py +77 -49
- siliconcompiler/package/git.py +11 -6
- siliconcompiler/package/github.py +11 -6
- siliconcompiler/package/https.py +6 -4
- siliconcompiler/pdk.py +23 -16
- siliconcompiler/scheduler/scheduler.py +30 -22
- siliconcompiler/scheduler/schedulernode.py +60 -50
- siliconcompiler/scheduler/taskscheduler.py +52 -32
- siliconcompiler/schema/baseschema.py +88 -69
- siliconcompiler/schema/docs/schemagen.py +4 -3
- siliconcompiler/schema/editableschema.py +5 -5
- siliconcompiler/schema/journal.py +19 -13
- siliconcompiler/schema/namedschema.py +16 -10
- siliconcompiler/schema/parameter.py +64 -37
- siliconcompiler/schema/parametervalue.py +126 -80
- siliconcompiler/schema/safeschema.py +16 -7
- siliconcompiler/schema/utils.py +3 -1
- siliconcompiler/schema_support/cmdlineschema.py +9 -9
- siliconcompiler/schema_support/dependencyschema.py +12 -7
- siliconcompiler/schema_support/filesetschema.py +15 -10
- siliconcompiler/schema_support/metric.py +29 -17
- siliconcompiler/schema_support/packageschema.py +2 -2
- siliconcompiler/schema_support/pathschema.py +30 -18
- siliconcompiler/schema_support/record.py +30 -23
- siliconcompiler/tool.py +265 -210
- siliconcompiler/tools/opensta/timing.py +13 -0
- siliconcompiler/tools/yosys/syn_fpga.py +3 -2
- siliconcompiler/toolscripts/_tools.json +3 -3
- siliconcompiler/utils/__init__.py +23 -16
- siliconcompiler/utils/curation.py +11 -5
- siliconcompiler/utils/multiprocessing.py +16 -14
- siliconcompiler/utils/paths.py +24 -12
- siliconcompiler/utils/units.py +16 -12
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/METADATA +3 -4
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/RECORD +49 -48
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/entry_points.txt +4 -3
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/WHEEL +0 -0
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/licenses/LICENSE +0 -0
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/top_level.txt +0 -0
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
import logging
|
|
1
2
|
import multiprocessing
|
|
2
3
|
import sys
|
|
3
4
|
import time
|
|
4
5
|
|
|
5
6
|
import os.path
|
|
6
7
|
|
|
8
|
+
from typing import List, Dict, Tuple, Optional, Callable, ClassVar, Any, Literal, TYPE_CHECKING
|
|
9
|
+
|
|
7
10
|
from logging.handlers import QueueListener
|
|
8
11
|
|
|
9
12
|
from siliconcompiler import NodeStatus
|
|
@@ -17,6 +20,12 @@ from siliconcompiler.utils.logging import SCBlankLoggerFormatter, SCBlankColorle
|
|
|
17
20
|
from siliconcompiler.utils.multiprocessing import MPManager
|
|
18
21
|
from siliconcompiler.scheduler import SCRuntimeError
|
|
19
22
|
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from siliconcompiler import Flowgraph
|
|
25
|
+
from siliconcompiler.project import Project
|
|
26
|
+
from siliconcompiler.scheduler import SchedulerNode
|
|
27
|
+
from siliconcompiler.schema_support.record import RecordSchema
|
|
28
|
+
|
|
20
29
|
|
|
21
30
|
class TaskScheduler:
|
|
22
31
|
"""A class for managing the execution of individual tasks in a flowgraph.
|
|
@@ -26,7 +35,7 @@ class TaskScheduler:
|
|
|
26
35
|
dependency checking. It operates on a set of pending tasks defined by the
|
|
27
36
|
main Scheduler and executes them in a loop until the flow is complete.
|
|
28
37
|
"""
|
|
29
|
-
__callbacks = {
|
|
38
|
+
__callbacks: ClassVar[Dict[str, Callable[..., None]]] = {
|
|
30
39
|
"pre_run": lambda project: None,
|
|
31
40
|
"pre_node": lambda project, step, index: None,
|
|
32
41
|
"post_node": lambda project, step, index: None,
|
|
@@ -34,7 +43,8 @@ class TaskScheduler:
|
|
|
34
43
|
}
|
|
35
44
|
|
|
36
45
|
@staticmethod
|
|
37
|
-
def register_callback(hook,
|
|
46
|
+
def register_callback(hook: Literal["pre_run", "pre_node", "post_node", "post_run"],
|
|
47
|
+
func: Callable[..., None]) -> None:
|
|
38
48
|
"""Registers a callback function to be executed at a specific hook point.
|
|
39
49
|
|
|
40
50
|
Valid hooks are 'pre_run', 'pre_node', 'post_node', and 'post_run'.
|
|
@@ -51,7 +61,7 @@ class TaskScheduler:
|
|
|
51
61
|
raise ValueError(f"{hook} is not a valid callback")
|
|
52
62
|
TaskScheduler.__callbacks[hook] = func
|
|
53
63
|
|
|
54
|
-
def __init__(self, project, tasks):
|
|
64
|
+
def __init__(self, project: "Project", tasks: Dict[Tuple[str, str], "SchedulerNode"]):
|
|
55
65
|
"""Initializes the TaskScheduler.
|
|
56
66
|
|
|
57
67
|
Args:
|
|
@@ -63,14 +73,15 @@ class TaskScheduler:
|
|
|
63
73
|
self.__logger = self.__project.logger
|
|
64
74
|
self.__logger_console_handler = self.__project._logger_console
|
|
65
75
|
self.__schema = self.__project
|
|
66
|
-
self.__flow = self.__schema.get("flowgraph",
|
|
67
|
-
|
|
68
|
-
|
|
76
|
+
self.__flow: "Flowgraph" = self.__schema.get("flowgraph",
|
|
77
|
+
self.__project.get('option', 'flow'),
|
|
78
|
+
field="schema")
|
|
79
|
+
self.__record: "RecordSchema" = self.__schema.get("record", field="schema")
|
|
69
80
|
self.__dashboard = project._Project__dashboard
|
|
70
81
|
|
|
71
82
|
self.__max_cores = utils.get_cores()
|
|
72
83
|
self.__max_threads = utils.get_cores()
|
|
73
|
-
self.__max_parallel_run = self.__project.
|
|
84
|
+
self.__max_parallel_run = self.__project.option.scheduler.get_maxnodes()
|
|
74
85
|
if not self.__max_parallel_run:
|
|
75
86
|
self.__max_parallel_run = utils.get_cores()
|
|
76
87
|
# clip max parallel jobs to 1 <= jobs <= max_cores
|
|
@@ -78,19 +89,19 @@ class TaskScheduler:
|
|
|
78
89
|
|
|
79
90
|
self.__runtime_flow = RuntimeFlowgraph(
|
|
80
91
|
self.__flow,
|
|
81
|
-
from_steps=self.__project.
|
|
82
|
-
to_steps=self.__project.
|
|
83
|
-
prune_nodes=self.__project.
|
|
92
|
+
from_steps=self.__project.option.get_from(),
|
|
93
|
+
to_steps=self.__project.option.get_to(),
|
|
94
|
+
prune_nodes=self.__project.option.get_prune())
|
|
84
95
|
|
|
85
96
|
self.__log_queue = MPManager.get_manager().Queue()
|
|
86
97
|
|
|
87
|
-
self.__nodes = {}
|
|
88
|
-
self.__startTimes = {}
|
|
89
|
-
self.__dwellTime = 0.1
|
|
98
|
+
self.__nodes: Dict[Tuple[str, str], Dict[str, Any]] = {}
|
|
99
|
+
self.__startTimes: Dict[Optional[Tuple[str, str]], float] = {}
|
|
100
|
+
self.__dwellTime: float = 0.1
|
|
90
101
|
|
|
91
102
|
self.__create_nodes(tasks)
|
|
92
103
|
|
|
93
|
-
def __create_nodes(self, tasks):
|
|
104
|
+
def __create_nodes(self, tasks: Dict[Tuple[str, str], "SchedulerNode"]) -> None:
|
|
94
105
|
"""
|
|
95
106
|
Private helper to prepare all pending tasks for execution.
|
|
96
107
|
|
|
@@ -104,7 +115,7 @@ class TaskScheduler:
|
|
|
104
115
|
runtime = RuntimeFlowgraph(
|
|
105
116
|
self.__flow,
|
|
106
117
|
from_steps=set([step for step, _ in self.__flow.get_entry_nodes()]),
|
|
107
|
-
prune_nodes=self.__project.
|
|
118
|
+
prune_nodes=self.__project.option.get_prune())
|
|
108
119
|
|
|
109
120
|
init_funcs = set()
|
|
110
121
|
|
|
@@ -137,11 +148,18 @@ class TaskScheduler:
|
|
|
137
148
|
init_funcs.add(task["node"].init)
|
|
138
149
|
self.__nodes[(step, index)] = task
|
|
139
150
|
|
|
151
|
+
# Create ordered list of nodes
|
|
152
|
+
self.__ordered_nodes: List[Tuple[str, str]] = []
|
|
153
|
+
for levelnodes in self.__runtime_flow.get_execution_order():
|
|
154
|
+
for node in levelnodes:
|
|
155
|
+
if node in self.__nodes:
|
|
156
|
+
self.__ordered_nodes.append(node)
|
|
157
|
+
|
|
140
158
|
# Call preprocessing for schedulers
|
|
141
159
|
for init_func in init_funcs:
|
|
142
160
|
init_func(self.__project)
|
|
143
161
|
|
|
144
|
-
def run(self, job_log_handler):
|
|
162
|
+
def run(self, job_log_handler: logging.Handler) -> None:
|
|
145
163
|
"""
|
|
146
164
|
The main entry point for the task scheduling loop.
|
|
147
165
|
|
|
@@ -190,7 +208,7 @@ class TaskScheduler:
|
|
|
190
208
|
job_log_handler.setFormatter(file_formatter)
|
|
191
209
|
self.__logger.addHandler(job_log_handler)
|
|
192
210
|
|
|
193
|
-
def __run_loop(self):
|
|
211
|
+
def __run_loop(self) -> None:
|
|
194
212
|
"""
|
|
195
213
|
The core execution loop of the scheduler.
|
|
196
214
|
|
|
@@ -223,39 +241,41 @@ class TaskScheduler:
|
|
|
223
241
|
# if there are more than 1, join the first with a timeout
|
|
224
242
|
self.__nodes[running_nodes[0]]["proc"].join(timeout=self.__dwellTime)
|
|
225
243
|
|
|
226
|
-
def get_nodes(self):
|
|
227
|
-
"""Gets
|
|
244
|
+
def get_nodes(self) -> List[Tuple[str, str]]:
|
|
245
|
+
"""Gets an ordered list of all nodes managed by this scheduler.
|
|
228
246
|
|
|
229
247
|
Returns:
|
|
230
248
|
list: A list of (step, index) tuples for all nodes.
|
|
231
249
|
"""
|
|
232
|
-
return
|
|
250
|
+
return self.__ordered_nodes
|
|
233
251
|
|
|
234
|
-
def get_running_nodes(self):
|
|
235
|
-
"""Gets
|
|
252
|
+
def get_running_nodes(self) -> List[Tuple[str, str]]:
|
|
253
|
+
"""Gets an ordered list of all nodes that are currently running.
|
|
236
254
|
|
|
237
255
|
Returns:
|
|
238
256
|
list: A list of (step, index) tuples for running nodes.
|
|
239
257
|
"""
|
|
240
258
|
nodes = []
|
|
241
|
-
for node
|
|
259
|
+
for node in self.get_nodes():
|
|
260
|
+
info = self.__nodes[node]
|
|
242
261
|
if info["running"]:
|
|
243
262
|
nodes.append(node)
|
|
244
|
-
return
|
|
263
|
+
return nodes
|
|
245
264
|
|
|
246
|
-
def get_nodes_waiting_to_run(self):
|
|
247
|
-
"""Gets
|
|
265
|
+
def get_nodes_waiting_to_run(self) -> List[Tuple[str, str]]:
|
|
266
|
+
"""Gets an ordered list of all nodes that are pending execution.
|
|
248
267
|
|
|
249
268
|
Returns:
|
|
250
269
|
list: A list of (step, index) tuples for pending nodes.
|
|
251
270
|
"""
|
|
252
271
|
nodes = []
|
|
253
|
-
for node
|
|
272
|
+
for node in self.get_nodes():
|
|
273
|
+
info = self.__nodes[node]
|
|
254
274
|
if not info["running"] and info["proc"]:
|
|
255
275
|
nodes.append(node)
|
|
256
|
-
return
|
|
276
|
+
return nodes
|
|
257
277
|
|
|
258
|
-
def __process_completed_nodes(self):
|
|
278
|
+
def __process_completed_nodes(self) -> bool:
|
|
259
279
|
"""
|
|
260
280
|
Private helper to check for and process completed nodes.
|
|
261
281
|
|
|
@@ -310,7 +330,7 @@ class TaskScheduler:
|
|
|
310
330
|
|
|
311
331
|
return changed
|
|
312
332
|
|
|
313
|
-
def __allow_start(self, node):
|
|
333
|
+
def __allow_start(self, node: Tuple[str, str]) -> bool:
|
|
314
334
|
"""
|
|
315
335
|
Private helper to check if a node is allowed to start based on resources.
|
|
316
336
|
|
|
@@ -344,7 +364,7 @@ class TaskScheduler:
|
|
|
344
364
|
# allow
|
|
345
365
|
return True
|
|
346
366
|
|
|
347
|
-
def __launch_nodes(self):
|
|
367
|
+
def __launch_nodes(self) -> bool:
|
|
348
368
|
"""
|
|
349
369
|
Private helper to launch new nodes whose dependencies are met.
|
|
350
370
|
|
|
@@ -405,7 +425,7 @@ class TaskScheduler:
|
|
|
405
425
|
|
|
406
426
|
return changed
|
|
407
427
|
|
|
408
|
-
def check(self):
|
|
428
|
+
def check(self) -> None:
|
|
409
429
|
"""
|
|
410
430
|
Checks if the flow completed successfully.
|
|
411
431
|
|
|
@@ -25,7 +25,7 @@ except ModuleNotFoundError:
|
|
|
25
25
|
import os.path
|
|
26
26
|
|
|
27
27
|
from functools import cache
|
|
28
|
-
from typing import Dict, Type, Tuple, Union, Set, Callable, List
|
|
28
|
+
from typing import Dict, Type, Tuple, Union, Set, Callable, List, Optional, TextIO, Iterable
|
|
29
29
|
|
|
30
30
|
from .parameter import Parameter, NodeValue
|
|
31
31
|
from .journal import Journal
|
|
@@ -50,7 +50,7 @@ class BaseSchema:
|
|
|
50
50
|
self.__key = None
|
|
51
51
|
|
|
52
52
|
@property
|
|
53
|
-
def _keypath(self):
|
|
53
|
+
def _keypath(self) -> Tuple[str, ...]:
|
|
54
54
|
'''
|
|
55
55
|
Returns the key to the current section of the schema
|
|
56
56
|
'''
|
|
@@ -103,7 +103,7 @@ class BaseSchema:
|
|
|
103
103
|
|
|
104
104
|
@staticmethod
|
|
105
105
|
@cache
|
|
106
|
-
def __load_schema_class(cls_name: str) -> Type["BaseSchema"]:
|
|
106
|
+
def __load_schema_class(cls_name: str) -> Optional[Type["BaseSchema"]]:
|
|
107
107
|
"""
|
|
108
108
|
Load a schema class from a string
|
|
109
109
|
"""
|
|
@@ -125,7 +125,7 @@ class BaseSchema:
|
|
|
125
125
|
return cls
|
|
126
126
|
|
|
127
127
|
@staticmethod
|
|
128
|
-
def __process_meta_section(meta: Dict[str, str]) -> Type["BaseSchema"]:
|
|
128
|
+
def __process_meta_section(meta: Dict[str, str]) -> Optional[Type["BaseSchema"]]:
|
|
129
129
|
"""
|
|
130
130
|
Handle __meta__ section of the schema by loading the appropriate class
|
|
131
131
|
"""
|
|
@@ -143,21 +143,24 @@ class BaseSchema:
|
|
|
143
143
|
return cls
|
|
144
144
|
|
|
145
145
|
@staticmethod
|
|
146
|
-
def __extractversion(manifest: Dict):
|
|
146
|
+
def __extractversion(manifest: Dict) -> Optional[Tuple[int, ...]]:
|
|
147
147
|
schema_version = manifest.get(BaseSchema._version_key, None)
|
|
148
148
|
if schema_version:
|
|
149
149
|
param = Parameter.from_dict(schema_version, [BaseSchema._version_key], None)
|
|
150
150
|
return tuple([int(v) for v in param.get().split('.')])
|
|
151
151
|
return None
|
|
152
152
|
|
|
153
|
-
def _from_dict(self, manifest: Dict,
|
|
153
|
+
def _from_dict(self, manifest: Dict,
|
|
154
|
+
keypath: Union[List[str], Tuple[str, ...]],
|
|
155
|
+
version: Optional[Tuple[int, ...]] = None) \
|
|
156
|
+
-> Tuple[Set[Tuple[str, ...]], Set[Tuple[str, ...]]]:
|
|
154
157
|
'''
|
|
155
158
|
Decodes a dictionary into a schema object
|
|
156
159
|
|
|
157
160
|
Args:
|
|
158
161
|
manifest (dict): Manifest to decide.
|
|
159
162
|
keypath (list of str): Path to the current keypath.
|
|
160
|
-
version (
|
|
163
|
+
version ((int, int, int)): Version of the dictionary schema
|
|
161
164
|
'''
|
|
162
165
|
# find schema version
|
|
163
166
|
if not version:
|
|
@@ -180,7 +183,7 @@ class BaseSchema:
|
|
|
180
183
|
data = manifest.get("default", None)
|
|
181
184
|
if data:
|
|
182
185
|
del manifest["default"]
|
|
183
|
-
self.__default._from_dict(data, keypath + ["default"], version=version)
|
|
186
|
+
self.__default._from_dict(data, list(keypath) + ["default"], version=version)
|
|
184
187
|
handled.add("default")
|
|
185
188
|
|
|
186
189
|
for key, data in manifest.items():
|
|
@@ -190,7 +193,7 @@ class BaseSchema:
|
|
|
190
193
|
cls = BaseSchema.__process_meta_section(data["__meta__"])
|
|
191
194
|
if cls is BaseSchema and self.__default:
|
|
192
195
|
# Use default when BaseSchema is the class
|
|
193
|
-
obj = self.__default.copy(key=keypath + [key])
|
|
196
|
+
obj = self.__default.copy(key=list(keypath) + [key])
|
|
194
197
|
self.__manifest[key] = obj
|
|
195
198
|
elif cls:
|
|
196
199
|
# Create object and connect to schema
|
|
@@ -201,11 +204,11 @@ class BaseSchema:
|
|
|
201
204
|
|
|
202
205
|
# Use default if it is available
|
|
203
206
|
if not obj and self.__default:
|
|
204
|
-
obj = self.__default.copy(key=keypath + [key])
|
|
207
|
+
obj = self.__default.copy(key=list(keypath) + [key])
|
|
205
208
|
self.__manifest[key] = obj
|
|
206
209
|
|
|
207
210
|
if obj:
|
|
208
|
-
obj._from_dict(data, keypath + [key], version=version)
|
|
211
|
+
obj._from_dict(data, list(keypath) + [key], version=version)
|
|
209
212
|
handled.add(key)
|
|
210
213
|
else:
|
|
211
214
|
missing.add(key)
|
|
@@ -214,7 +217,9 @@ class BaseSchema:
|
|
|
214
217
|
|
|
215
218
|
# Manifest methods
|
|
216
219
|
@classmethod
|
|
217
|
-
def from_manifest(cls,
|
|
220
|
+
def from_manifest(cls,
|
|
221
|
+
filepath: Union[None, str] = None,
|
|
222
|
+
cfg: Union[None, Dict] = None) -> "BaseSchema":
|
|
218
223
|
'''
|
|
219
224
|
Create a new schema based on the provided source files.
|
|
220
225
|
|
|
@@ -240,12 +245,12 @@ class BaseSchema:
|
|
|
240
245
|
else:
|
|
241
246
|
schema = cls()
|
|
242
247
|
|
|
243
|
-
schema._from_dict(cfg,
|
|
248
|
+
schema._from_dict(cfg, tuple())
|
|
244
249
|
|
|
245
250
|
return schema
|
|
246
251
|
|
|
247
252
|
@staticmethod
|
|
248
|
-
def __open_file(filepath: str, is_read: bool = True):
|
|
253
|
+
def __open_file(filepath: str, is_read: bool = True) -> TextIO:
|
|
249
254
|
_, ext = os.path.splitext(filepath)
|
|
250
255
|
if ext.lower() == ".gz":
|
|
251
256
|
if not _has_gzip:
|
|
@@ -257,7 +262,7 @@ class BaseSchema:
|
|
|
257
262
|
return f"[{','.join([*self._keypath, *key])}]"
|
|
258
263
|
|
|
259
264
|
@staticmethod
|
|
260
|
-
def _read_manifest(filepath: str):
|
|
265
|
+
def _read_manifest(filepath: str) -> Dict:
|
|
261
266
|
"""
|
|
262
267
|
Reads a manifest from disk and returns dictionary.
|
|
263
268
|
|
|
@@ -273,7 +278,7 @@ class BaseSchema:
|
|
|
273
278
|
|
|
274
279
|
return manifest
|
|
275
280
|
|
|
276
|
-
def read_manifest(self, filepath: str):
|
|
281
|
+
def read_manifest(self, filepath: str) -> None:
|
|
277
282
|
"""
|
|
278
283
|
Reads a manifest from disk and replaces the current data with the data in the file.
|
|
279
284
|
|
|
@@ -287,7 +292,7 @@ class BaseSchema:
|
|
|
287
292
|
|
|
288
293
|
self._from_dict(BaseSchema._read_manifest(filepath), [])
|
|
289
294
|
|
|
290
|
-
def write_manifest(self, filepath: str):
|
|
295
|
+
def write_manifest(self, filepath: str) -> None:
|
|
291
296
|
'''
|
|
292
297
|
Writes the manifest to a file.
|
|
293
298
|
|
|
@@ -301,13 +306,14 @@ class BaseSchema:
|
|
|
301
306
|
|
|
302
307
|
fout = BaseSchema.__open_file(filepath, is_read=False)
|
|
303
308
|
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
309
|
+
try:
|
|
310
|
+
if _has_orjson:
|
|
311
|
+
manifest_str = json.dumps(self.getdict(), option=json.OPT_INDENT_2).decode()
|
|
312
|
+
else:
|
|
313
|
+
manifest_str = json.dumps(self.getdict(), indent=2)
|
|
314
|
+
fout.write(manifest_str)
|
|
315
|
+
finally:
|
|
316
|
+
fout.close()
|
|
311
317
|
|
|
312
318
|
# Accessor methods
|
|
313
319
|
def __search(self,
|
|
@@ -315,7 +321,7 @@ class BaseSchema:
|
|
|
315
321
|
insert_defaults: bool = False,
|
|
316
322
|
use_default: bool = False,
|
|
317
323
|
require_leaf: bool = True,
|
|
318
|
-
complete_path:
|
|
324
|
+
complete_path: Optional[List[str]] = None) -> Union["BaseSchema", Parameter]:
|
|
319
325
|
if len(keypath) == 0:
|
|
320
326
|
if require_leaf:
|
|
321
327
|
raise KeyError
|
|
@@ -356,8 +362,8 @@ class BaseSchema:
|
|
|
356
362
|
raise KeyError
|
|
357
363
|
return key_param
|
|
358
364
|
|
|
359
|
-
def get(self, *keypath: str, field: str = 'value',
|
|
360
|
-
step: str = None, index: Union[int, str] = None):
|
|
365
|
+
def get(self, *keypath: str, field: Optional[str] = 'value',
|
|
366
|
+
step: Optional[str] = None, index: Optional[Union[int, str]] = None):
|
|
361
367
|
"""
|
|
362
368
|
Returns a parameter field from the schema.
|
|
363
369
|
|
|
@@ -413,8 +419,9 @@ class BaseSchema:
|
|
|
413
419
|
e.args = (new_msg, *e.args[1:])
|
|
414
420
|
raise e
|
|
415
421
|
|
|
416
|
-
def set(self, *args
|
|
417
|
-
step: str = None, index: Union[int, str] = None)
|
|
422
|
+
def set(self, *args, field: str = 'value', clobber: bool = True,
|
|
423
|
+
step: Optional[str] = None, index: Optional[Union[int, str]] = None) \
|
|
424
|
+
-> Optional[Union[List[NodeValue], NodeValue]]:
|
|
418
425
|
'''
|
|
419
426
|
Sets a schema parameter field.
|
|
420
427
|
|
|
@@ -442,7 +449,7 @@ class BaseSchema:
|
|
|
442
449
|
*keypath, value = args
|
|
443
450
|
|
|
444
451
|
try:
|
|
445
|
-
param = self.__search(*keypath, insert_defaults=True)
|
|
452
|
+
param: Parameter = self.__search(*keypath, insert_defaults=True)
|
|
446
453
|
except KeyError:
|
|
447
454
|
raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
|
|
448
455
|
|
|
@@ -459,8 +466,9 @@ class BaseSchema:
|
|
|
459
466
|
e.args = (new_msg, *e.args[1:])
|
|
460
467
|
raise e
|
|
461
468
|
|
|
462
|
-
def add(self, *args
|
|
463
|
-
step: str = None, index: Union[int, str] = None)
|
|
469
|
+
def add(self, *args, field: str = 'value',
|
|
470
|
+
step: Optional[str] = None, index: Optional[Union[int, str]] = None) \
|
|
471
|
+
-> Optional[Union[List[NodeValue], NodeValue]]:
|
|
464
472
|
'''
|
|
465
473
|
Adds item(s) to a schema parameter list.
|
|
466
474
|
|
|
@@ -487,7 +495,7 @@ class BaseSchema:
|
|
|
487
495
|
*keypath, value = args
|
|
488
496
|
|
|
489
497
|
try:
|
|
490
|
-
param = self.__search(*keypath, insert_defaults=True)
|
|
498
|
+
param: Parameter = self.__search(*keypath, insert_defaults=True)
|
|
491
499
|
except KeyError:
|
|
492
500
|
raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
|
|
493
501
|
|
|
@@ -503,7 +511,9 @@ class BaseSchema:
|
|
|
503
511
|
e.args = (new_msg, *e.args[1:])
|
|
504
512
|
raise e
|
|
505
513
|
|
|
506
|
-
def unset(self, *keypath: str,
|
|
514
|
+
def unset(self, *keypath: str,
|
|
515
|
+
step: Optional[str] = None,
|
|
516
|
+
index: Optional[Union[int, str]] = None) -> None:
|
|
507
517
|
'''
|
|
508
518
|
Unsets a schema parameter.
|
|
509
519
|
|
|
@@ -610,7 +620,7 @@ class BaseSchema:
|
|
|
610
620
|
return isinstance(param, Parameter)
|
|
611
621
|
return True
|
|
612
622
|
|
|
613
|
-
def getkeys(self, *keypath: str) -> Tuple[str]:
|
|
623
|
+
def getkeys(self, *keypath: str) -> Tuple[str, ...]:
|
|
614
624
|
"""
|
|
615
625
|
Returns a tuple of schema dictionary keys.
|
|
616
626
|
|
|
@@ -640,7 +650,7 @@ class BaseSchema:
|
|
|
640
650
|
|
|
641
651
|
return tuple(sorted(key_param.__manifest.keys()))
|
|
642
652
|
|
|
643
|
-
def allkeys(self, *keypath: str, include_default: bool = True) -> Set[Tuple[str]]:
|
|
653
|
+
def allkeys(self, *keypath: str, include_default: bool = True) -> Set[Tuple[str, ...]]:
|
|
644
654
|
'''
|
|
645
655
|
Returns all keypaths in the schema as a set of tuples.
|
|
646
656
|
|
|
@@ -677,7 +687,7 @@ class BaseSchema:
|
|
|
677
687
|
|
|
678
688
|
return "BaseSchema"
|
|
679
689
|
|
|
680
|
-
def _getdict_meta(self) -> Dict[str, str]:
|
|
690
|
+
def _getdict_meta(self) -> Dict[str, Optional[Union[str, int, float]]]:
|
|
681
691
|
"""
|
|
682
692
|
Returns the meta data for getdict
|
|
683
693
|
"""
|
|
@@ -746,7 +756,7 @@ class BaseSchema:
|
|
|
746
756
|
return manifest
|
|
747
757
|
|
|
748
758
|
# Utility functions
|
|
749
|
-
def copy(self, key: Tuple[str] = None):
|
|
759
|
+
def copy(self, key: Optional[Tuple[str, ...]] = None) -> "BaseSchema":
|
|
750
760
|
"""
|
|
751
761
|
Returns a copy of this schema.
|
|
752
762
|
|
|
@@ -769,20 +779,22 @@ class BaseSchema:
|
|
|
769
779
|
|
|
770
780
|
return schema_copy
|
|
771
781
|
|
|
772
|
-
def _find_files_search_paths(self,
|
|
782
|
+
def _find_files_search_paths(self, key: str,
|
|
783
|
+
step: Optional[str],
|
|
784
|
+
index: Optional[Union[int, str]]) -> List[str]:
|
|
773
785
|
"""
|
|
774
786
|
Returns a list of paths to search during find files.
|
|
775
787
|
|
|
776
788
|
Args:
|
|
777
|
-
|
|
789
|
+
key (str): final component of keypath
|
|
778
790
|
step (str): Step name.
|
|
779
791
|
index (str): Index name.
|
|
780
792
|
"""
|
|
781
793
|
return []
|
|
782
794
|
|
|
783
|
-
def _find_files_dataroot_resolvers(self):
|
|
795
|
+
def _find_files_dataroot_resolvers(self) -> Dict[str, Union[str, Callable]]:
|
|
784
796
|
"""
|
|
785
|
-
Returns a dictionary of path
|
|
797
|
+
Returns a dictionary of path resolvers data directory handling for find_files
|
|
786
798
|
|
|
787
799
|
Returns:
|
|
788
800
|
dictionary of str to resolver mapping
|
|
@@ -792,10 +804,11 @@ class BaseSchema:
|
|
|
792
804
|
return self.__parent._find_files_dataroot_resolvers()
|
|
793
805
|
|
|
794
806
|
def _find_files(self, *keypath: str, missing_ok: bool = False,
|
|
795
|
-
step: str = None, index: Union[int, str] = None,
|
|
796
|
-
dataroots: Dict[str, Union[str, Callable]] = None,
|
|
797
|
-
collection_dir: str = None,
|
|
798
|
-
cwd: str = None)
|
|
807
|
+
step: Optional[str] = None, index: Optional[Union[int, str]] = None,
|
|
808
|
+
dataroots: Optional[Dict[str, Union[str, Callable]]] = None,
|
|
809
|
+
collection_dir: Optional[str] = None,
|
|
810
|
+
cwd: Optional[str] = None) \
|
|
811
|
+
-> Union[Optional[str], List[Optional[str]], Set[Optional[str]]]:
|
|
799
812
|
"""
|
|
800
813
|
Returns absolute paths to files or directories based on the keypath
|
|
801
814
|
provided.
|
|
@@ -836,11 +849,12 @@ class BaseSchema:
|
|
|
836
849
|
cwd=cwd, hash=False)
|
|
837
850
|
|
|
838
851
|
def __find_files_or_hash(self, *keypath: str, missing_ok: bool = False,
|
|
839
|
-
step: str = None, index: Union[int, str] = None,
|
|
840
|
-
dataroots: Dict[str, Union[str, Callable]] = None,
|
|
841
|
-
collection_dir: str = None,
|
|
842
|
-
cwd: str = None,
|
|
843
|
-
hash: bool = False)
|
|
852
|
+
step: Optional[str] = None, index: Optional[Union[int, str]] = None,
|
|
853
|
+
dataroots: Optional[Dict[str, Union[str, Callable]]] = None,
|
|
854
|
+
collection_dir: Optional[str] = None,
|
|
855
|
+
cwd: Optional[str] = None,
|
|
856
|
+
hash: bool = False) \
|
|
857
|
+
-> Union[Optional[str], List[Optional[str]], Set[Optional[str]]]:
|
|
844
858
|
"""
|
|
845
859
|
Returns absolute paths to files or directories based on the keypath
|
|
846
860
|
provided.
|
|
@@ -875,14 +889,15 @@ class BaseSchema:
|
|
|
875
889
|
the schema.
|
|
876
890
|
"""
|
|
877
891
|
|
|
878
|
-
base_schema = self.get(*keypath[0:-1], field="schema")
|
|
892
|
+
base_schema: BaseSchema = self.get(*keypath[0:-1], field="schema")
|
|
879
893
|
|
|
880
|
-
param = base_schema.get(keypath[-1], field=None)
|
|
881
|
-
paramtype = param.get(field='type')
|
|
894
|
+
param: Parameter = base_schema.get(keypath[-1], field=None)
|
|
895
|
+
paramtype: str = param.get(field='type')
|
|
882
896
|
if 'file' not in paramtype and 'dir' not in paramtype:
|
|
883
897
|
raise TypeError(
|
|
884
898
|
f'Cannot find files on {self.__format_key(*keypath)}, must be a path type')
|
|
885
899
|
|
|
900
|
+
hashalgo: Optional[str] = None
|
|
886
901
|
if hash:
|
|
887
902
|
hashalgo = param.get(field="hashalgo")
|
|
888
903
|
|
|
@@ -911,7 +926,7 @@ class BaseSchema:
|
|
|
911
926
|
for path in paths:
|
|
912
927
|
search_paths = root_search_paths.copy()
|
|
913
928
|
|
|
914
|
-
dataroot = path.get(field="dataroot")
|
|
929
|
+
dataroot: Optional[str] = path.get(field="dataroot")
|
|
915
930
|
if dataroot:
|
|
916
931
|
if dataroot not in dataroots:
|
|
917
932
|
raise ValueError(f"Resolver for {dataroot} not provided: "
|
|
@@ -956,11 +971,11 @@ class BaseSchema:
|
|
|
956
971
|
return resolved_paths[0]
|
|
957
972
|
return resolved_paths
|
|
958
973
|
|
|
959
|
-
def _check_filepaths(self, ignore_keys:
|
|
960
|
-
logger: logging.Logger = None,
|
|
961
|
-
dataroots: Dict[str, Union[str, Callable]] = None,
|
|
962
|
-
collection_dir: str = None,
|
|
963
|
-
cwd: str = None) -> bool:
|
|
974
|
+
def _check_filepaths(self, ignore_keys: Optional[Iterable[Tuple[str, ...]]] = None,
|
|
975
|
+
logger: Optional[logging.Logger] = None,
|
|
976
|
+
dataroots: Optional[Dict[str, Union[str, Callable]]] = None,
|
|
977
|
+
collection_dir: Optional[str] = None,
|
|
978
|
+
cwd: Optional[str] = None) -> bool:
|
|
964
979
|
'''
|
|
965
980
|
Verifies that paths to all files in manifest are valid.
|
|
966
981
|
|
|
@@ -990,8 +1005,8 @@ class BaseSchema:
|
|
|
990
1005
|
if keypath in ignore_keys:
|
|
991
1006
|
continue
|
|
992
1007
|
|
|
993
|
-
param = self.get(*keypath, field=None)
|
|
994
|
-
paramtype = param.get(field='type')
|
|
1008
|
+
param: Parameter = self.get(*keypath, field=None)
|
|
1009
|
+
paramtype: str = param.get(field='type')
|
|
995
1010
|
|
|
996
1011
|
if 'file' not in paramtype and 'dir' not in paramtype:
|
|
997
1012
|
continue
|
|
@@ -1026,10 +1041,10 @@ class BaseSchema:
|
|
|
1026
1041
|
return not error
|
|
1027
1042
|
|
|
1028
1043
|
def _hash_files(self, *keypath, missing_ok: bool = False,
|
|
1029
|
-
step: str = None, index: Union[int, str] = None,
|
|
1030
|
-
dataroots: Dict[str, Union[str, Callable]] = None,
|
|
1031
|
-
collection_dir: str = None,
|
|
1032
|
-
cwd: str = None):
|
|
1044
|
+
step: Optional[str] = None, index: Optional[Union[int, str]] = None,
|
|
1045
|
+
dataroots: Optional[Dict[str, Union[str, Callable]]] = None,
|
|
1046
|
+
collection_dir: Optional[str] = None,
|
|
1047
|
+
cwd: Optional[str] = None):
|
|
1033
1048
|
'''Generates hash values for a list of parameter files.
|
|
1034
1049
|
|
|
1035
1050
|
Generates a hash value for each file found in the keypath. If existing
|
|
@@ -1144,7 +1159,11 @@ class BaseSchema:
|
|
|
1144
1159
|
|
|
1145
1160
|
return self.__active.get(field, defvalue)
|
|
1146
1161
|
|
|
1147
|
-
def __process_active(self, param,
|
|
1162
|
+
def __process_active(self, param: Parameter,
|
|
1163
|
+
nodevalues: Optional[Union[List[NodeValue],
|
|
1164
|
+
Set[NodeValue],
|
|
1165
|
+
Tuple[NodeValue, ...],
|
|
1166
|
+
NodeValue]]) -> None:
|
|
1148
1167
|
if not self.__active:
|
|
1149
1168
|
return
|
|
1150
1169
|
|
|
@@ -1173,14 +1192,14 @@ class BaseSchema:
|
|
|
1173
1192
|
|
|
1174
1193
|
def _generate_doc(self, doc,
|
|
1175
1194
|
ref_root: str = "",
|
|
1176
|
-
key_offset: Tuple[str] = None,
|
|
1195
|
+
key_offset: Optional[Tuple[str, ...]] = None,
|
|
1177
1196
|
detailed: bool = True):
|
|
1178
1197
|
from .docs.utils import build_section_with_target, build_schema_value_table, get_key_ref, \
|
|
1179
1198
|
parse_rst
|
|
1180
1199
|
from docutils import nodes
|
|
1181
1200
|
|
|
1182
1201
|
if not key_offset:
|
|
1183
|
-
key_offset =
|
|
1202
|
+
key_offset = tuple()
|
|
1184
1203
|
|
|
1185
1204
|
if detailed:
|
|
1186
1205
|
sections = []
|
|
@@ -136,9 +136,10 @@ class SchemaGen(SphinxDirective):
|
|
|
136
136
|
methods_sec += cls_ref
|
|
137
137
|
schema_sec += methods_sec
|
|
138
138
|
|
|
139
|
-
key_offset =
|
|
140
|
-
|
|
141
|
-
|
|
139
|
+
key_offset = tuple(
|
|
140
|
+
[key_part
|
|
141
|
+
for key_part in self.options.get("key_offset", "").split(",")
|
|
142
|
+
if key_part])
|
|
142
143
|
if not key_offset:
|
|
143
144
|
key_offset = None
|
|
144
145
|
|