siliconcompiler 0.35.0__py3-none-any.whl → 0.35.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. siliconcompiler/_metadata.py +1 -1
  2. siliconcompiler/apps/_common.py +3 -2
  3. siliconcompiler/apps/sc_dashboard.py +3 -1
  4. siliconcompiler/apps/sc_install.py +149 -37
  5. siliconcompiler/apps/smake.py +9 -3
  6. siliconcompiler/checklist.py +3 -3
  7. siliconcompiler/data/demo_fpga/z1000_yosys_config.json +24 -0
  8. siliconcompiler/design.py +51 -45
  9. siliconcompiler/flowgraph.py +2 -2
  10. siliconcompiler/library.py +23 -12
  11. siliconcompiler/package/__init__.py +77 -49
  12. siliconcompiler/package/git.py +11 -6
  13. siliconcompiler/package/github.py +11 -6
  14. siliconcompiler/package/https.py +6 -4
  15. siliconcompiler/pdk.py +23 -16
  16. siliconcompiler/scheduler/scheduler.py +30 -22
  17. siliconcompiler/scheduler/schedulernode.py +60 -50
  18. siliconcompiler/scheduler/taskscheduler.py +52 -32
  19. siliconcompiler/schema/baseschema.py +88 -69
  20. siliconcompiler/schema/docs/schemagen.py +4 -3
  21. siliconcompiler/schema/editableschema.py +5 -5
  22. siliconcompiler/schema/journal.py +19 -13
  23. siliconcompiler/schema/namedschema.py +16 -10
  24. siliconcompiler/schema/parameter.py +64 -37
  25. siliconcompiler/schema/parametervalue.py +126 -80
  26. siliconcompiler/schema/safeschema.py +16 -7
  27. siliconcompiler/schema/utils.py +3 -1
  28. siliconcompiler/schema_support/cmdlineschema.py +9 -9
  29. siliconcompiler/schema_support/dependencyschema.py +12 -7
  30. siliconcompiler/schema_support/filesetschema.py +15 -10
  31. siliconcompiler/schema_support/metric.py +29 -17
  32. siliconcompiler/schema_support/packageschema.py +2 -2
  33. siliconcompiler/schema_support/pathschema.py +30 -18
  34. siliconcompiler/schema_support/record.py +30 -23
  35. siliconcompiler/tool.py +265 -210
  36. siliconcompiler/tools/opensta/timing.py +13 -0
  37. siliconcompiler/tools/yosys/syn_fpga.py +3 -2
  38. siliconcompiler/toolscripts/_tools.json +3 -3
  39. siliconcompiler/utils/__init__.py +23 -16
  40. siliconcompiler/utils/curation.py +11 -5
  41. siliconcompiler/utils/multiprocessing.py +16 -14
  42. siliconcompiler/utils/paths.py +24 -12
  43. siliconcompiler/utils/units.py +16 -12
  44. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/METADATA +3 -4
  45. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/RECORD +49 -48
  46. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/entry_points.txt +4 -3
  47. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/WHEEL +0 -0
  48. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/licenses/LICENSE +0 -0
  49. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/top_level.txt +0 -0
@@ -1,9 +1,12 @@
1
+ import logging
1
2
  import multiprocessing
2
3
  import sys
3
4
  import time
4
5
 
5
6
  import os.path
6
7
 
8
+ from typing import List, Dict, Tuple, Optional, Callable, ClassVar, Any, Literal, TYPE_CHECKING
9
+
7
10
  from logging.handlers import QueueListener
8
11
 
9
12
  from siliconcompiler import NodeStatus
@@ -17,6 +20,12 @@ from siliconcompiler.utils.logging import SCBlankLoggerFormatter, SCBlankColorle
17
20
  from siliconcompiler.utils.multiprocessing import MPManager
18
21
  from siliconcompiler.scheduler import SCRuntimeError
19
22
 
23
+ if TYPE_CHECKING:
24
+ from siliconcompiler import Flowgraph
25
+ from siliconcompiler.project import Project
26
+ from siliconcompiler.scheduler import SchedulerNode
27
+ from siliconcompiler.schema_support.record import RecordSchema
28
+
20
29
 
21
30
  class TaskScheduler:
22
31
  """A class for managing the execution of individual tasks in a flowgraph.
@@ -26,7 +35,7 @@ class TaskScheduler:
26
35
  dependency checking. It operates on a set of pending tasks defined by the
27
36
  main Scheduler and executes them in a loop until the flow is complete.
28
37
  """
29
- __callbacks = {
38
+ __callbacks: ClassVar[Dict[str, Callable[..., None]]] = {
30
39
  "pre_run": lambda project: None,
31
40
  "pre_node": lambda project, step, index: None,
32
41
  "post_node": lambda project, step, index: None,
@@ -34,7 +43,8 @@ class TaskScheduler:
34
43
  }
35
44
 
36
45
  @staticmethod
37
- def register_callback(hook, func):
46
+ def register_callback(hook: Literal["pre_run", "pre_node", "post_node", "post_run"],
47
+ func: Callable[..., None]) -> None:
38
48
  """Registers a callback function to be executed at a specific hook point.
39
49
 
40
50
  Valid hooks are 'pre_run', 'pre_node', 'post_node', and 'post_run'.
@@ -51,7 +61,7 @@ class TaskScheduler:
51
61
  raise ValueError(f"{hook} is not a valid callback")
52
62
  TaskScheduler.__callbacks[hook] = func
53
63
 
54
- def __init__(self, project, tasks):
64
+ def __init__(self, project: "Project", tasks: Dict[Tuple[str, str], "SchedulerNode"]):
55
65
  """Initializes the TaskScheduler.
56
66
 
57
67
  Args:
@@ -63,14 +73,15 @@ class TaskScheduler:
63
73
  self.__logger = self.__project.logger
64
74
  self.__logger_console_handler = self.__project._logger_console
65
75
  self.__schema = self.__project
66
- self.__flow = self.__schema.get("flowgraph", self.__project.get('option', 'flow'),
67
- field="schema")
68
- self.__record = self.__schema.get("record", field="schema")
76
+ self.__flow: "Flowgraph" = self.__schema.get("flowgraph",
77
+ self.__project.get('option', 'flow'),
78
+ field="schema")
79
+ self.__record: "RecordSchema" = self.__schema.get("record", field="schema")
69
80
  self.__dashboard = project._Project__dashboard
70
81
 
71
82
  self.__max_cores = utils.get_cores()
72
83
  self.__max_threads = utils.get_cores()
73
- self.__max_parallel_run = self.__project.get('option', 'scheduler', 'maxnodes')
84
+ self.__max_parallel_run = self.__project.option.scheduler.get_maxnodes()
74
85
  if not self.__max_parallel_run:
75
86
  self.__max_parallel_run = utils.get_cores()
76
87
  # clip max parallel jobs to 1 <= jobs <= max_cores
@@ -78,19 +89,19 @@ class TaskScheduler:
78
89
 
79
90
  self.__runtime_flow = RuntimeFlowgraph(
80
91
  self.__flow,
81
- from_steps=self.__project.get('option', 'from'),
82
- to_steps=self.__project.get('option', 'to'),
83
- prune_nodes=self.__project.get('option', 'prune'))
92
+ from_steps=self.__project.option.get_from(),
93
+ to_steps=self.__project.option.get_to(),
94
+ prune_nodes=self.__project.option.get_prune())
84
95
 
85
96
  self.__log_queue = MPManager.get_manager().Queue()
86
97
 
87
- self.__nodes = {}
88
- self.__startTimes = {}
89
- self.__dwellTime = 0.1
98
+ self.__nodes: Dict[Tuple[str, str], Dict[str, Any]] = {}
99
+ self.__startTimes: Dict[Optional[Tuple[str, str]], float] = {}
100
+ self.__dwellTime: float = 0.1
90
101
 
91
102
  self.__create_nodes(tasks)
92
103
 
93
- def __create_nodes(self, tasks):
104
+ def __create_nodes(self, tasks: Dict[Tuple[str, str], "SchedulerNode"]) -> None:
94
105
  """
95
106
  Private helper to prepare all pending tasks for execution.
96
107
 
@@ -104,7 +115,7 @@ class TaskScheduler:
104
115
  runtime = RuntimeFlowgraph(
105
116
  self.__flow,
106
117
  from_steps=set([step for step, _ in self.__flow.get_entry_nodes()]),
107
- prune_nodes=self.__project.get('option', 'prune'))
118
+ prune_nodes=self.__project.option.get_prune())
108
119
 
109
120
  init_funcs = set()
110
121
 
@@ -137,11 +148,18 @@ class TaskScheduler:
137
148
  init_funcs.add(task["node"].init)
138
149
  self.__nodes[(step, index)] = task
139
150
 
151
+ # Create ordered list of nodes
152
+ self.__ordered_nodes: List[Tuple[str, str]] = []
153
+ for levelnodes in self.__runtime_flow.get_execution_order():
154
+ for node in levelnodes:
155
+ if node in self.__nodes:
156
+ self.__ordered_nodes.append(node)
157
+
140
158
  # Call preprocessing for schedulers
141
159
  for init_func in init_funcs:
142
160
  init_func(self.__project)
143
161
 
144
- def run(self, job_log_handler):
162
+ def run(self, job_log_handler: logging.Handler) -> None:
145
163
  """
146
164
  The main entry point for the task scheduling loop.
147
165
 
@@ -190,7 +208,7 @@ class TaskScheduler:
190
208
  job_log_handler.setFormatter(file_formatter)
191
209
  self.__logger.addHandler(job_log_handler)
192
210
 
193
- def __run_loop(self):
211
+ def __run_loop(self) -> None:
194
212
  """
195
213
  The core execution loop of the scheduler.
196
214
 
@@ -223,39 +241,41 @@ class TaskScheduler:
223
241
  # if there are more than 1, join the first with a timeout
224
242
  self.__nodes[running_nodes[0]]["proc"].join(timeout=self.__dwellTime)
225
243
 
226
- def get_nodes(self):
227
- """Gets a sorted list of all nodes managed by this scheduler.
244
+ def get_nodes(self) -> List[Tuple[str, str]]:
245
+ """Gets an ordered list of all nodes managed by this scheduler.
228
246
 
229
247
  Returns:
230
248
  list: A list of (step, index) tuples for all nodes.
231
249
  """
232
- return sorted(self.__nodes.keys())
250
+ return self.__ordered_nodes
233
251
 
234
- def get_running_nodes(self):
235
- """Gets a sorted list of all nodes that are currently running.
252
+ def get_running_nodes(self) -> List[Tuple[str, str]]:
253
+ """Gets an ordered list of all nodes that are currently running.
236
254
 
237
255
  Returns:
238
256
  list: A list of (step, index) tuples for running nodes.
239
257
  """
240
258
  nodes = []
241
- for node, info in self.__nodes.items():
259
+ for node in self.get_nodes():
260
+ info = self.__nodes[node]
242
261
  if info["running"]:
243
262
  nodes.append(node)
244
- return sorted(nodes)
263
+ return nodes
245
264
 
246
- def get_nodes_waiting_to_run(self):
247
- """Gets a sorted list of all nodes that are pending execution.
265
+ def get_nodes_waiting_to_run(self) -> List[Tuple[str, str]]:
266
+ """Gets an ordered list of all nodes that are pending execution.
248
267
 
249
268
  Returns:
250
269
  list: A list of (step, index) tuples for pending nodes.
251
270
  """
252
271
  nodes = []
253
- for node, info in self.__nodes.items():
272
+ for node in self.get_nodes():
273
+ info = self.__nodes[node]
254
274
  if not info["running"] and info["proc"]:
255
275
  nodes.append(node)
256
- return sorted(nodes)
276
+ return nodes
257
277
 
258
- def __process_completed_nodes(self):
278
+ def __process_completed_nodes(self) -> bool:
259
279
  """
260
280
  Private helper to check for and process completed nodes.
261
281
 
@@ -310,7 +330,7 @@ class TaskScheduler:
310
330
 
311
331
  return changed
312
332
 
313
- def __allow_start(self, node):
333
+ def __allow_start(self, node: Tuple[str, str]) -> bool:
314
334
  """
315
335
  Private helper to check if a node is allowed to start based on resources.
316
336
 
@@ -344,7 +364,7 @@ class TaskScheduler:
344
364
  # allow
345
365
  return True
346
366
 
347
- def __launch_nodes(self):
367
+ def __launch_nodes(self) -> bool:
348
368
  """
349
369
  Private helper to launch new nodes whose dependencies are met.
350
370
 
@@ -405,7 +425,7 @@ class TaskScheduler:
405
425
 
406
426
  return changed
407
427
 
408
- def check(self):
428
+ def check(self) -> None:
409
429
  """
410
430
  Checks if the flow completed successfully.
411
431
 
@@ -25,7 +25,7 @@ except ModuleNotFoundError:
25
25
  import os.path
26
26
 
27
27
  from functools import cache
28
- from typing import Dict, Type, Tuple, Union, Set, Callable, List
28
+ from typing import Dict, Type, Tuple, Union, Set, Callable, List, Optional, TextIO, Iterable
29
29
 
30
30
  from .parameter import Parameter, NodeValue
31
31
  from .journal import Journal
@@ -50,7 +50,7 @@ class BaseSchema:
50
50
  self.__key = None
51
51
 
52
52
  @property
53
- def _keypath(self):
53
+ def _keypath(self) -> Tuple[str, ...]:
54
54
  '''
55
55
  Returns the key to the current section of the schema
56
56
  '''
@@ -103,7 +103,7 @@ class BaseSchema:
103
103
 
104
104
  @staticmethod
105
105
  @cache
106
- def __load_schema_class(cls_name: str) -> Type["BaseSchema"]:
106
+ def __load_schema_class(cls_name: str) -> Optional[Type["BaseSchema"]]:
107
107
  """
108
108
  Load a schema class from a string
109
109
  """
@@ -125,7 +125,7 @@ class BaseSchema:
125
125
  return cls
126
126
 
127
127
  @staticmethod
128
- def __process_meta_section(meta: Dict[str, str]) -> Type["BaseSchema"]:
128
+ def __process_meta_section(meta: Dict[str, str]) -> Optional[Type["BaseSchema"]]:
129
129
  """
130
130
  Handle __meta__ section of the schema by loading the appropriate class
131
131
  """
@@ -143,21 +143,24 @@ class BaseSchema:
143
143
  return cls
144
144
 
145
145
  @staticmethod
146
- def __extractversion(manifest: Dict):
146
+ def __extractversion(manifest: Dict) -> Optional[Tuple[int, ...]]:
147
147
  schema_version = manifest.get(BaseSchema._version_key, None)
148
148
  if schema_version:
149
149
  param = Parameter.from_dict(schema_version, [BaseSchema._version_key], None)
150
150
  return tuple([int(v) for v in param.get().split('.')])
151
151
  return None
152
152
 
153
- def _from_dict(self, manifest: Dict, keypath: Tuple[str], version: str = None):
153
+ def _from_dict(self, manifest: Dict,
154
+ keypath: Union[List[str], Tuple[str, ...]],
155
+ version: Optional[Tuple[int, ...]] = None) \
156
+ -> Tuple[Set[Tuple[str, ...]], Set[Tuple[str, ...]]]:
154
157
  '''
155
158
  Decodes a dictionary into a schema object
156
159
 
157
160
  Args:
158
161
  manifest (dict): Manifest to decide.
159
162
  keypath (list of str): Path to the current keypath.
160
- version (packaging.Version): Version of the dictionary schema
163
+ version ((int, int, int)): Version of the dictionary schema
161
164
  '''
162
165
  # find schema version
163
166
  if not version:
@@ -180,7 +183,7 @@ class BaseSchema:
180
183
  data = manifest.get("default", None)
181
184
  if data:
182
185
  del manifest["default"]
183
- self.__default._from_dict(data, keypath + ["default"], version=version)
186
+ self.__default._from_dict(data, list(keypath) + ["default"], version=version)
184
187
  handled.add("default")
185
188
 
186
189
  for key, data in manifest.items():
@@ -190,7 +193,7 @@ class BaseSchema:
190
193
  cls = BaseSchema.__process_meta_section(data["__meta__"])
191
194
  if cls is BaseSchema and self.__default:
192
195
  # Use default when BaseSchema is the class
193
- obj = self.__default.copy(key=keypath + [key])
196
+ obj = self.__default.copy(key=list(keypath) + [key])
194
197
  self.__manifest[key] = obj
195
198
  elif cls:
196
199
  # Create object and connect to schema
@@ -201,11 +204,11 @@ class BaseSchema:
201
204
 
202
205
  # Use default if it is available
203
206
  if not obj and self.__default:
204
- obj = self.__default.copy(key=keypath + [key])
207
+ obj = self.__default.copy(key=list(keypath) + [key])
205
208
  self.__manifest[key] = obj
206
209
 
207
210
  if obj:
208
- obj._from_dict(data, keypath + [key], version=version)
211
+ obj._from_dict(data, list(keypath) + [key], version=version)
209
212
  handled.add(key)
210
213
  else:
211
214
  missing.add(key)
@@ -214,7 +217,9 @@ class BaseSchema:
214
217
 
215
218
  # Manifest methods
216
219
  @classmethod
217
- def from_manifest(cls, filepath: str = None, cfg: Dict = None) -> "BaseSchema":
220
+ def from_manifest(cls,
221
+ filepath: Union[None, str] = None,
222
+ cfg: Union[None, Dict] = None) -> "BaseSchema":
218
223
  '''
219
224
  Create a new schema based on the provided source files.
220
225
 
@@ -240,12 +245,12 @@ class BaseSchema:
240
245
  else:
241
246
  schema = cls()
242
247
 
243
- schema._from_dict(cfg, [])
248
+ schema._from_dict(cfg, tuple())
244
249
 
245
250
  return schema
246
251
 
247
252
  @staticmethod
248
- def __open_file(filepath: str, is_read: bool = True):
253
+ def __open_file(filepath: str, is_read: bool = True) -> TextIO:
249
254
  _, ext = os.path.splitext(filepath)
250
255
  if ext.lower() == ".gz":
251
256
  if not _has_gzip:
@@ -257,7 +262,7 @@ class BaseSchema:
257
262
  return f"[{','.join([*self._keypath, *key])}]"
258
263
 
259
264
  @staticmethod
260
- def _read_manifest(filepath: str):
265
+ def _read_manifest(filepath: str) -> Dict:
261
266
  """
262
267
  Reads a manifest from disk and returns dictionary.
263
268
 
@@ -273,7 +278,7 @@ class BaseSchema:
273
278
 
274
279
  return manifest
275
280
 
276
- def read_manifest(self, filepath: str):
281
+ def read_manifest(self, filepath: str) -> None:
277
282
  """
278
283
  Reads a manifest from disk and replaces the current data with the data in the file.
279
284
 
@@ -287,7 +292,7 @@ class BaseSchema:
287
292
 
288
293
  self._from_dict(BaseSchema._read_manifest(filepath), [])
289
294
 
290
- def write_manifest(self, filepath: str):
295
+ def write_manifest(self, filepath: str) -> None:
291
296
  '''
292
297
  Writes the manifest to a file.
293
298
 
@@ -301,13 +306,14 @@ class BaseSchema:
301
306
 
302
307
  fout = BaseSchema.__open_file(filepath, is_read=False)
303
308
 
304
- if _has_orjson:
305
- manifest_str = json.dumps(self.getdict(), option=json.OPT_INDENT_2).decode()
306
- else:
307
- manifest_str = json.dumps(self.getdict(), indent=2)
308
- fout.write(manifest_str)
309
-
310
- fout.close()
309
+ try:
310
+ if _has_orjson:
311
+ manifest_str = json.dumps(self.getdict(), option=json.OPT_INDENT_2).decode()
312
+ else:
313
+ manifest_str = json.dumps(self.getdict(), indent=2)
314
+ fout.write(manifest_str)
315
+ finally:
316
+ fout.close()
311
317
 
312
318
  # Accessor methods
313
319
  def __search(self,
@@ -315,7 +321,7 @@ class BaseSchema:
315
321
  insert_defaults: bool = False,
316
322
  use_default: bool = False,
317
323
  require_leaf: bool = True,
318
- complete_path: bool = None) -> Union["BaseSchema", Parameter]:
324
+ complete_path: Optional[List[str]] = None) -> Union["BaseSchema", Parameter]:
319
325
  if len(keypath) == 0:
320
326
  if require_leaf:
321
327
  raise KeyError
@@ -356,8 +362,8 @@ class BaseSchema:
356
362
  raise KeyError
357
363
  return key_param
358
364
 
359
- def get(self, *keypath: str, field: str = 'value',
360
- step: str = None, index: Union[int, str] = None):
365
+ def get(self, *keypath: str, field: Optional[str] = 'value',
366
+ step: Optional[str] = None, index: Optional[Union[int, str]] = None):
361
367
  """
362
368
  Returns a parameter field from the schema.
363
369
 
@@ -413,8 +419,9 @@ class BaseSchema:
413
419
  e.args = (new_msg, *e.args[1:])
414
420
  raise e
415
421
 
416
- def set(self, *args: str, field: str = 'value', clobber: bool = True,
417
- step: str = None, index: Union[int, str] = None):
422
+ def set(self, *args, field: str = 'value', clobber: bool = True,
423
+ step: Optional[str] = None, index: Optional[Union[int, str]] = None) \
424
+ -> Optional[Union[List[NodeValue], NodeValue]]:
418
425
  '''
419
426
  Sets a schema parameter field.
420
427
 
@@ -442,7 +449,7 @@ class BaseSchema:
442
449
  *keypath, value = args
443
450
 
444
451
  try:
445
- param = self.__search(*keypath, insert_defaults=True)
452
+ param: Parameter = self.__search(*keypath, insert_defaults=True)
446
453
  except KeyError:
447
454
  raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
448
455
 
@@ -459,8 +466,9 @@ class BaseSchema:
459
466
  e.args = (new_msg, *e.args[1:])
460
467
  raise e
461
468
 
462
- def add(self, *args: str, field: str = 'value',
463
- step: str = None, index: Union[int, str] = None):
469
+ def add(self, *args, field: str = 'value',
470
+ step: Optional[str] = None, index: Optional[Union[int, str]] = None) \
471
+ -> Optional[Union[List[NodeValue], NodeValue]]:
464
472
  '''
465
473
  Adds item(s) to a schema parameter list.
466
474
 
@@ -487,7 +495,7 @@ class BaseSchema:
487
495
  *keypath, value = args
488
496
 
489
497
  try:
490
- param = self.__search(*keypath, insert_defaults=True)
498
+ param: Parameter = self.__search(*keypath, insert_defaults=True)
491
499
  except KeyError:
492
500
  raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
493
501
 
@@ -503,7 +511,9 @@ class BaseSchema:
503
511
  e.args = (new_msg, *e.args[1:])
504
512
  raise e
505
513
 
506
- def unset(self, *keypath: str, step: str = None, index: Union[int, str] = None):
514
+ def unset(self, *keypath: str,
515
+ step: Optional[str] = None,
516
+ index: Optional[Union[int, str]] = None) -> None:
507
517
  '''
508
518
  Unsets a schema parameter.
509
519
 
@@ -610,7 +620,7 @@ class BaseSchema:
610
620
  return isinstance(param, Parameter)
611
621
  return True
612
622
 
613
- def getkeys(self, *keypath: str) -> Tuple[str]:
623
+ def getkeys(self, *keypath: str) -> Tuple[str, ...]:
614
624
  """
615
625
  Returns a tuple of schema dictionary keys.
616
626
 
@@ -640,7 +650,7 @@ class BaseSchema:
640
650
 
641
651
  return tuple(sorted(key_param.__manifest.keys()))
642
652
 
643
- def allkeys(self, *keypath: str, include_default: bool = True) -> Set[Tuple[str]]:
653
+ def allkeys(self, *keypath: str, include_default: bool = True) -> Set[Tuple[str, ...]]:
644
654
  '''
645
655
  Returns all keypaths in the schema as a set of tuples.
646
656
 
@@ -677,7 +687,7 @@ class BaseSchema:
677
687
 
678
688
  return "BaseSchema"
679
689
 
680
- def _getdict_meta(self) -> Dict[str, str]:
690
+ def _getdict_meta(self) -> Dict[str, Optional[Union[str, int, float]]]:
681
691
  """
682
692
  Returns the meta data for getdict
683
693
  """
@@ -746,7 +756,7 @@ class BaseSchema:
746
756
  return manifest
747
757
 
748
758
  # Utility functions
749
- def copy(self, key: Tuple[str] = None):
759
+ def copy(self, key: Optional[Tuple[str, ...]] = None) -> "BaseSchema":
750
760
  """
751
761
  Returns a copy of this schema.
752
762
 
@@ -769,20 +779,22 @@ class BaseSchema:
769
779
 
770
780
  return schema_copy
771
781
 
772
- def _find_files_search_paths(self, keypath: Tuple[str], step: str, index: str):
782
+ def _find_files_search_paths(self, key: str,
783
+ step: Optional[str],
784
+ index: Optional[Union[int, str]]) -> List[str]:
773
785
  """
774
786
  Returns a list of paths to search during find files.
775
787
 
776
788
  Args:
777
- keypath (str): final component of keypath
789
+ key (str): final component of keypath
778
790
  step (str): Step name.
779
791
  index (str): Index name.
780
792
  """
781
793
  return []
782
794
 
783
- def _find_files_dataroot_resolvers(self):
795
+ def _find_files_dataroot_resolvers(self) -> Dict[str, Union[str, Callable]]:
784
796
  """
785
- Returns a dictionary of path resolevrs data directory handling for find_files
797
+ Returns a dictionary of path resolvers data directory handling for find_files
786
798
 
787
799
  Returns:
788
800
  dictionary of str to resolver mapping
@@ -792,10 +804,11 @@ class BaseSchema:
792
804
  return self.__parent._find_files_dataroot_resolvers()
793
805
 
794
806
  def _find_files(self, *keypath: str, missing_ok: bool = False,
795
- step: str = None, index: Union[int, str] = None,
796
- dataroots: Dict[str, Union[str, Callable]] = None,
797
- collection_dir: str = None,
798
- cwd: str = None) -> Union[str, List[str], Set[str]]:
807
+ step: Optional[str] = None, index: Optional[Union[int, str]] = None,
808
+ dataroots: Optional[Dict[str, Union[str, Callable]]] = None,
809
+ collection_dir: Optional[str] = None,
810
+ cwd: Optional[str] = None) \
811
+ -> Union[Optional[str], List[Optional[str]], Set[Optional[str]]]:
799
812
  """
800
813
  Returns absolute paths to files or directories based on the keypath
801
814
  provided.
@@ -836,11 +849,12 @@ class BaseSchema:
836
849
  cwd=cwd, hash=False)
837
850
 
838
851
  def __find_files_or_hash(self, *keypath: str, missing_ok: bool = False,
839
- step: str = None, index: Union[int, str] = None,
840
- dataroots: Dict[str, Union[str, Callable]] = None,
841
- collection_dir: str = None,
842
- cwd: str = None,
843
- hash: bool = False) -> Union[str, List[str], Set[str]]:
852
+ step: Optional[str] = None, index: Optional[Union[int, str]] = None,
853
+ dataroots: Optional[Dict[str, Union[str, Callable]]] = None,
854
+ collection_dir: Optional[str] = None,
855
+ cwd: Optional[str] = None,
856
+ hash: bool = False) \
857
+ -> Union[Optional[str], List[Optional[str]], Set[Optional[str]]]:
844
858
  """
845
859
  Returns absolute paths to files or directories based on the keypath
846
860
  provided.
@@ -875,14 +889,15 @@ class BaseSchema:
875
889
  the schema.
876
890
  """
877
891
 
878
- base_schema = self.get(*keypath[0:-1], field="schema")
892
+ base_schema: BaseSchema = self.get(*keypath[0:-1], field="schema")
879
893
 
880
- param = base_schema.get(keypath[-1], field=None)
881
- paramtype = param.get(field='type')
894
+ param: Parameter = base_schema.get(keypath[-1], field=None)
895
+ paramtype: str = param.get(field='type')
882
896
  if 'file' not in paramtype and 'dir' not in paramtype:
883
897
  raise TypeError(
884
898
  f'Cannot find files on {self.__format_key(*keypath)}, must be a path type')
885
899
 
900
+ hashalgo: Optional[str] = None
886
901
  if hash:
887
902
  hashalgo = param.get(field="hashalgo")
888
903
 
@@ -911,7 +926,7 @@ class BaseSchema:
911
926
  for path in paths:
912
927
  search_paths = root_search_paths.copy()
913
928
 
914
- dataroot = path.get(field="dataroot")
929
+ dataroot: Optional[str] = path.get(field="dataroot")
915
930
  if dataroot:
916
931
  if dataroot not in dataroots:
917
932
  raise ValueError(f"Resolver for {dataroot} not provided: "
@@ -956,11 +971,11 @@ class BaseSchema:
956
971
  return resolved_paths[0]
957
972
  return resolved_paths
958
973
 
959
- def _check_filepaths(self, ignore_keys: bool = None,
960
- logger: logging.Logger = None,
961
- dataroots: Dict[str, Union[str, Callable]] = None,
962
- collection_dir: str = None,
963
- cwd: str = None) -> bool:
974
+ def _check_filepaths(self, ignore_keys: Optional[Iterable[Tuple[str, ...]]] = None,
975
+ logger: Optional[logging.Logger] = None,
976
+ dataroots: Optional[Dict[str, Union[str, Callable]]] = None,
977
+ collection_dir: Optional[str] = None,
978
+ cwd: Optional[str] = None) -> bool:
964
979
  '''
965
980
  Verifies that paths to all files in manifest are valid.
966
981
 
@@ -990,8 +1005,8 @@ class BaseSchema:
990
1005
  if keypath in ignore_keys:
991
1006
  continue
992
1007
 
993
- param = self.get(*keypath, field=None)
994
- paramtype = param.get(field='type')
1008
+ param: Parameter = self.get(*keypath, field=None)
1009
+ paramtype: str = param.get(field='type')
995
1010
 
996
1011
  if 'file' not in paramtype and 'dir' not in paramtype:
997
1012
  continue
@@ -1026,10 +1041,10 @@ class BaseSchema:
1026
1041
  return not error
1027
1042
 
1028
1043
  def _hash_files(self, *keypath, missing_ok: bool = False,
1029
- step: str = None, index: Union[int, str] = None,
1030
- dataroots: Dict[str, Union[str, Callable]] = None,
1031
- collection_dir: str = None,
1032
- cwd: str = None):
1044
+ step: Optional[str] = None, index: Optional[Union[int, str]] = None,
1045
+ dataroots: Optional[Dict[str, Union[str, Callable]]] = None,
1046
+ collection_dir: Optional[str] = None,
1047
+ cwd: Optional[str] = None):
1033
1048
  '''Generates hash values for a list of parameter files.
1034
1049
 
1035
1050
  Generates a hash value for each file found in the keypath. If existing
@@ -1144,7 +1159,11 @@ class BaseSchema:
1144
1159
 
1145
1160
  return self.__active.get(field, defvalue)
1146
1161
 
1147
- def __process_active(self, param, nodevalues):
1162
+ def __process_active(self, param: Parameter,
1163
+ nodevalues: Optional[Union[List[NodeValue],
1164
+ Set[NodeValue],
1165
+ Tuple[NodeValue, ...],
1166
+ NodeValue]]) -> None:
1148
1167
  if not self.__active:
1149
1168
  return
1150
1169
 
@@ -1173,14 +1192,14 @@ class BaseSchema:
1173
1192
 
1174
1193
  def _generate_doc(self, doc,
1175
1194
  ref_root: str = "",
1176
- key_offset: Tuple[str] = None,
1195
+ key_offset: Optional[Tuple[str, ...]] = None,
1177
1196
  detailed: bool = True):
1178
1197
  from .docs.utils import build_section_with_target, build_schema_value_table, get_key_ref, \
1179
1198
  parse_rst
1180
1199
  from docutils import nodes
1181
1200
 
1182
1201
  if not key_offset:
1183
- key_offset = []
1202
+ key_offset = tuple()
1184
1203
 
1185
1204
  if detailed:
1186
1205
  sections = []
@@ -136,9 +136,10 @@ class SchemaGen(SphinxDirective):
136
136
  methods_sec += cls_ref
137
137
  schema_sec += methods_sec
138
138
 
139
- key_offset = [key_part
140
- for key_part in self.options.get("key_offset", "").split(",")
141
- if key_part]
139
+ key_offset = tuple(
140
+ [key_part
141
+ for key_part in self.options.get("key_offset", "").split(",")
142
+ if key_part])
142
143
  if not key_offset:
143
144
  key_offset = None
144
145