siliconcompiler 0.35.0__py3-none-any.whl → 0.35.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. siliconcompiler/_metadata.py +1 -1
  2. siliconcompiler/apps/_common.py +3 -2
  3. siliconcompiler/apps/sc_dashboard.py +3 -1
  4. siliconcompiler/apps/sc_install.py +149 -37
  5. siliconcompiler/apps/smake.py +9 -3
  6. siliconcompiler/checklist.py +3 -3
  7. siliconcompiler/data/demo_fpga/z1000_yosys_config.json +24 -0
  8. siliconcompiler/design.py +51 -45
  9. siliconcompiler/flowgraph.py +2 -2
  10. siliconcompiler/library.py +23 -12
  11. siliconcompiler/package/__init__.py +77 -49
  12. siliconcompiler/package/git.py +11 -6
  13. siliconcompiler/package/github.py +11 -6
  14. siliconcompiler/package/https.py +6 -4
  15. siliconcompiler/pdk.py +23 -16
  16. siliconcompiler/scheduler/scheduler.py +30 -22
  17. siliconcompiler/scheduler/schedulernode.py +60 -50
  18. siliconcompiler/scheduler/taskscheduler.py +52 -32
  19. siliconcompiler/schema/baseschema.py +88 -69
  20. siliconcompiler/schema/docs/schemagen.py +4 -3
  21. siliconcompiler/schema/editableschema.py +5 -5
  22. siliconcompiler/schema/journal.py +19 -13
  23. siliconcompiler/schema/namedschema.py +16 -10
  24. siliconcompiler/schema/parameter.py +64 -37
  25. siliconcompiler/schema/parametervalue.py +126 -80
  26. siliconcompiler/schema/safeschema.py +16 -7
  27. siliconcompiler/schema/utils.py +3 -1
  28. siliconcompiler/schema_support/cmdlineschema.py +9 -9
  29. siliconcompiler/schema_support/dependencyschema.py +12 -7
  30. siliconcompiler/schema_support/filesetschema.py +15 -10
  31. siliconcompiler/schema_support/metric.py +29 -17
  32. siliconcompiler/schema_support/packageschema.py +2 -2
  33. siliconcompiler/schema_support/pathschema.py +30 -18
  34. siliconcompiler/schema_support/record.py +30 -23
  35. siliconcompiler/tool.py +265 -210
  36. siliconcompiler/tools/opensta/timing.py +13 -0
  37. siliconcompiler/tools/yosys/syn_fpga.py +3 -2
  38. siliconcompiler/toolscripts/_tools.json +3 -3
  39. siliconcompiler/utils/__init__.py +23 -16
  40. siliconcompiler/utils/curation.py +11 -5
  41. siliconcompiler/utils/multiprocessing.py +16 -14
  42. siliconcompiler/utils/paths.py +24 -12
  43. siliconcompiler/utils/units.py +16 -12
  44. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/METADATA +3 -4
  45. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/RECORD +49 -48
  46. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/entry_points.txt +4 -3
  47. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/WHEEL +0 -0
  48. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/licenses/LICENSE +0 -0
  49. {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/top_level.txt +0 -0
siliconcompiler/tool.py CHANGED
@@ -32,7 +32,8 @@ import os.path
32
32
  from packaging.version import Version, InvalidVersion
33
33
  from packaging.specifiers import SpecifierSet, InvalidSpecifier
34
34
 
35
- from typing import List, Dict, Tuple, Union
35
+ from typing import List, Dict, Tuple, Union, Optional, Set, TextIO, Type, TYPE_CHECKING
36
+ from pathlib import Path
36
37
 
37
38
  from siliconcompiler.schema import BaseSchema, NamedSchema, Journal, DocsSchema
38
39
  from siliconcompiler.schema import EditableSchema, Parameter, PerNode, Scope
@@ -48,6 +49,10 @@ from siliconcompiler.schema_support.record import RecordTool, RecordSchema
48
49
  from siliconcompiler.schema_support.metric import MetricSchema
49
50
  from siliconcompiler.flowgraph import RuntimeFlowgraph
50
51
 
52
+ if TYPE_CHECKING:
53
+ from siliconcompiler.scheduler import SchedulerNode
54
+ from siliconcompiler import Project
55
+
51
56
 
52
57
  class TaskError(Exception):
53
58
  '''Error indicating that task execution cannot continue and should be terminated.'''
@@ -114,6 +119,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
114
119
  r"^\s*" + __parse_version_check_str + r"\s*$",
115
120
  re.VERBOSE | re.IGNORECASE)
116
121
 
122
+ __POLL_INTERVAL: float = 0.1
123
+ __MEMORY_WARN_LIMIT: int = 90
124
+
117
125
  def __init__(self):
118
126
  super().__init__()
119
127
 
@@ -126,7 +134,10 @@ class Task(NamedSchema, PathSchema, DocsSchema):
126
134
  """Returns the metadata for getdict."""
127
135
  return Task.__name__
128
136
 
129
- def _from_dict(self, manifest, keypath, version=None):
137
+ def _from_dict(self, manifest: Dict,
138
+ keypath: Union[List[str], Tuple[str, ...]],
139
+ version: Optional[Tuple[int, ...]] = None) \
140
+ -> Tuple[Set[Tuple[str, ...]], Set[Tuple[str, ...]]]:
130
141
  """
131
142
  Populates the schema from a dictionary, dynamically adding 'var'
132
143
  parameters found in the manifest that are not already defined.
@@ -142,7 +153,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
142
153
  edit.insert("var", var,
143
154
  Parameter.from_dict(
144
155
  manifest["var"][var],
145
- keypath=keypath + [var],
156
+ keypath=tuple([*keypath, var]),
146
157
  version=version))
147
158
  del manifest["var"][var]
148
159
 
@@ -152,7 +163,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
152
163
  return super()._from_dict(manifest, keypath, version)
153
164
 
154
165
  @contextlib.contextmanager
155
- def runtime(self, node, step=None, index=None, relpath=None):
166
+ def runtime(self, node: "SchedulerNode",
167
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
168
+ relpath: Optional[str] = None):
156
169
  """
157
170
  A context manager to set the runtime information for a task.
158
171
 
@@ -172,20 +185,24 @@ class Task(NamedSchema, PathSchema, DocsSchema):
172
185
  obj_copy.__set_runtime(node, step=step, index=index, relpath=relpath)
173
186
  yield obj_copy
174
187
 
175
- def __set_runtime(self, node, step=None, index=None, relpath=None):
188
+ def __set_runtime(self, node: Optional["SchedulerNode"],
189
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
190
+ relpath: Optional[str] = None) -> None:
176
191
  """
177
192
  Private helper to set the runtime information for executing a task.
178
193
 
179
194
  Args:
180
195
  node (SchedulerNode): The scheduler node for this runtime.
181
196
  """
182
- self.__node = node
183
- self.__schema_full = None
184
- self.__logger = None
185
- self.__design_name = None
186
- self.__design_top = None
187
- self.__relpath = relpath
188
- self.__jobdir = None
197
+ self.__node: Optional["SchedulerNode"] = node
198
+ self.__schema_full: Optional["Project"] = None
199
+ self.__logger: Optional[logging.Logger] = None
200
+ self.__design_name: Optional[str] = None
201
+ self.__design_top: Optional[str] = None
202
+ self.__relpath: Optional[str] = relpath
203
+ self.__jobdir: Optional[str] = None
204
+ self.__step: Optional[str] = None
205
+ self.__index: Optional[str] = None
189
206
  if node:
190
207
  if step is not None or index is not None:
191
208
  raise RuntimeError("step and index cannot be provided with node")
@@ -200,12 +217,14 @@ class Task(NamedSchema, PathSchema, DocsSchema):
200
217
  self.__index = node.index
201
218
  else:
202
219
  self.__step = step
220
+ if isinstance(index, int):
221
+ index = str(index)
203
222
  self.__index = index
204
223
 
205
- self.__schema_record = None
206
- self.__schema_metric = None
207
- self.__schema_flow = None
208
- self.__schema_flow_runtime = None
224
+ self.__schema_record: Optional[RecordSchema] = None
225
+ self.__schema_metric: Optional[MetricSchema] = None
226
+ self.__schema_flow: Optional[Flowgraph] = None
227
+ self.__schema_flow_runtime: Optional[RuntimeFlowgraph] = None
209
228
  if self.__schema_full:
210
229
  self.__schema_record = self.__schema_full.get("record", field="schema")
211
230
  self.__schema_metric = self.__schema_full.get("metric", field="schema")
@@ -239,7 +258,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
239
258
  return self.__design_top
240
259
 
241
260
  @property
242
- def node(self):
261
+ def node(self) -> "SchedulerNode":
243
262
  """SchedulerNode: The scheduler node for the current runtime."""
244
263
  return self.__node
245
264
 
@@ -288,7 +307,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
288
307
  return self.__schema_metric
289
308
 
290
309
  @property
291
- def project(self):
310
+ def project(self) -> "Project":
292
311
  return self.__schema_full
293
312
 
294
313
  @property
@@ -318,9 +337,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
318
337
  Returns:
319
338
  bool: True if a breakpoint is active, False otherwise.
320
339
  """
321
- return self.project.get("option", "breakpoint", step=self.__step, index=self.__index)
340
+ return self.project.option.get_breakpoint(step=self.__step, index=self.__index)
322
341
 
323
- def get_exe(self) -> str:
342
+ def get_exe(self) -> Optional[str]:
324
343
  """
325
344
  Determines the absolute path for the task's executable.
326
345
 
@@ -331,7 +350,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
331
350
  str: The absolute path to the executable, or None if not specified.
332
351
  """
333
352
 
334
- exe = self.get('exe')
353
+ exe: Optional[str] = self.get('exe')
335
354
 
336
355
  if exe is None:
337
356
  return None
@@ -358,7 +377,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
358
377
  if self.tool() in tools:
359
378
  self.logger.info(f"Missing tool can be installed via: \"sc-install {self.tool()}\"")
360
379
 
361
- def get_exe_version(self) -> str:
380
+ def get_exe_version(self) -> Optional[str]:
362
381
  """
363
382
  Gets the version of the task's executable by running it with a version switch.
364
383
 
@@ -370,7 +389,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
370
389
  str: The parsed version string.
371
390
  """
372
391
 
373
- veropt = self.get('vswitch')
392
+ veropt: Optional[List[str]] = self.get('vswitch')
374
393
  if not veropt:
375
394
  return None
376
395
 
@@ -383,8 +402,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
383
402
  cmdlist = [exe]
384
403
  cmdlist.extend(veropt)
385
404
 
386
- self.__logger.debug(f'Running {self.tool()}/{self.task()} version check: '
387
- f'{" ".join(cmdlist)}')
405
+ self.logger.debug(f'Running {self.tool()}/{self.task()} version check: '
406
+ f'{" ".join(cmdlist)}')
388
407
 
389
408
  proc = subprocess.run(cmdlist,
390
409
  stdin=subprocess.DEVNULL,
@@ -393,8 +412,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
393
412
  universal_newlines=True)
394
413
 
395
414
  if proc.returncode != 0:
396
- self.__logger.warning(f"Version check on '{exe_base}' ended with "
397
- f"code {proc.returncode}")
415
+ self.logger.warning(f"Version check on '{exe_base}' ended with "
416
+ f"code {proc.returncode}")
398
417
 
399
418
  try:
400
419
  version = self.parse_version(proc.stdout)
@@ -402,16 +421,16 @@ class Task(NamedSchema, PathSchema, DocsSchema):
402
421
  raise NotImplementedError(f'{self.tool()}/{self.task()} does not implement '
403
422
  'parse_version()')
404
423
  except Exception as e:
405
- self.__logger.error(f'{self.tool()}/{self.task()} failed to parse version string: '
406
- f'{proc.stdout}')
424
+ self.logger.error(f'{self.tool()}/{self.task()} failed to parse version string: '
425
+ f'{proc.stdout}')
407
426
  raise e from None
408
427
 
409
- self.__logger.info(f"Tool '{exe_base}' found with version '{version}' "
410
- f"in directory '{exe_path}'")
428
+ self.logger.info(f"Tool '{exe_base}' found with version '{version}' "
429
+ f"in directory '{exe_path}'")
411
430
 
412
431
  return version
413
432
 
414
- def check_exe_version(self, reported_version) -> bool:
433
+ def check_exe_version(self, reported_version: str) -> bool:
415
434
  """
416
435
  Checks if the reported version of a tool satisfies the requirements
417
436
  specified in the schema.
@@ -423,7 +442,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
423
442
  bool: True if the version is acceptable, False otherwise.
424
443
  """
425
444
 
426
- spec_sets = self.get('version')
445
+ spec_sets: Optional[List[str]] = self.get('version')
427
446
  if not spec_sets:
428
447
  # No requirement, so always true
429
448
  return True
@@ -434,8 +453,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
434
453
  for spec in split_specs:
435
454
  match = re.match(Task.__parse_version_check, spec)
436
455
  if match is None:
437
- self.__logger.warning(f'Invalid version specifier {spec}. '
438
- f'Defaulting to =={spec}.')
456
+ self.logger.warning(f'Invalid version specifier {spec}. '
457
+ f'Defaulting to =={spec}.')
439
458
  operator = '=='
440
459
  spec_version = spec
441
460
  else:
@@ -446,15 +465,15 @@ class Task(NamedSchema, PathSchema, DocsSchema):
446
465
  try:
447
466
  normalized_version = self.normalize_version(reported_version)
448
467
  except Exception as e:
449
- self.__logger.error(f'Unable to normalize version for {self.tool()}/{self.task()}: '
450
- f'{reported_version}')
468
+ self.logger.error(f'Unable to normalize version for {self.tool()}/{self.task()}: '
469
+ f'{reported_version}')
451
470
  raise e from None
452
471
 
453
472
  try:
454
473
  version = Version(normalized_version)
455
474
  except InvalidVersion:
456
- self.__logger.error(f'Version {normalized_version} reported by '
457
- f'{self.tool()}/{self.task()} does not match standard.')
475
+ self.logger.error(f'Version {normalized_version} reported by '
476
+ f'{self.tool()}/{self.task()} does not match standard.')
458
477
  return False
459
478
 
460
479
  try:
@@ -462,29 +481,30 @@ class Task(NamedSchema, PathSchema, DocsSchema):
462
481
  f'{op}{self.normalize_version(ver)}' for op, ver in specs_list]
463
482
  normalized_specs = ','.join(normalized_spec_list)
464
483
  except Exception as e:
465
- self.__logger.error(f'Unable to normalize versions for '
466
- f'{self.tool()}/{self.task()}: '
467
- f'{",".join([f"{op}{ver}" for op, ver in specs_list])}')
484
+ self.logger.error(f'Unable to normalize versions for '
485
+ f'{self.tool()}/{self.task()}: '
486
+ f'{",".join([f"{op}{ver}" for op, ver in specs_list])}')
468
487
  raise e from None
469
488
 
470
489
  try:
471
490
  spec_set = SpecifierSet(normalized_specs)
472
491
  except InvalidSpecifier:
473
- self.__logger.error(f'Version specifier set {normalized_specs} '
474
- 'does not match standard.')
492
+ self.logger.error(f'Version specifier set {normalized_specs} '
493
+ 'does not match standard.')
475
494
  return False
476
495
 
477
496
  if version in spec_set:
478
497
  return True
479
498
 
480
499
  allowedstr = '; '.join(spec_sets)
481
- self.__logger.error(f"Version check failed for {self.tool()}/{self.task()}. "
482
- "Check installation.")
483
- self.__logger.error(f"Found version {reported_version}, "
484
- f"did not satisfy any version specifier set {allowedstr}.")
500
+ self.logger.error(f"Version check failed for {self.tool()}/{self.task()}. "
501
+ "Check installation.")
502
+ self.logger.error(f"Found version {reported_version}, "
503
+ f"did not satisfy any version specifier set {allowedstr}.")
485
504
  return False
486
505
 
487
- def get_runtime_environmental_variables(self, include_path=True):
506
+ def get_runtime_environmental_variables(self, include_path: bool = True) \
507
+ -> Dict[str, str]:
488
508
  """
489
509
  Determines the environment variables needed for the task.
490
510
 
@@ -496,18 +516,18 @@ class Task(NamedSchema, PathSchema, DocsSchema):
496
516
  """
497
517
 
498
518
  # Add global environmental vars
499
- envvars = {}
519
+ envvars: Dict[str, str] = {}
500
520
  for env in self.__schema_full.getkeys('option', 'env'):
501
521
  envvars[env] = self.__schema_full.get('option', 'env', env)
502
522
 
503
523
  # Add tool-specific license server vars
504
524
  for lic_env in self.getkeys('licenseserver'):
505
- license_file = self.get('licenseserver', lic_env)
525
+ license_file: List[str] = self.get('licenseserver', lic_env)
506
526
  if license_file:
507
527
  envvars[lic_env] = ':'.join(license_file)
508
528
 
509
529
  if include_path:
510
- path = self.find_files("path", missing_ok=True)
530
+ path: Optional[str] = self.find_files("path", missing_ok=True)
511
531
 
512
532
  envvars["PATH"] = os.getenv("PATH", os.defpath)
513
533
 
@@ -526,7 +546,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
526
546
 
527
547
  return envvars
528
548
 
529
- def get_runtime_arguments(self):
549
+ def get_runtime_arguments(self) -> List[str]:
530
550
  """
531
551
  Constructs the command-line arguments needed to run the task.
532
552
 
@@ -555,7 +575,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
555
575
 
556
576
  cmdargs.extend(args)
557
577
  except Exception as e:
558
- self.__logger.error(f'Failed to get runtime options for {self.tool()}/{self.task()}')
578
+ self.logger.error(f'Failed to get runtime options for {self.tool()}/{self.task()}')
559
579
  raise e from None
560
580
 
561
581
  # Cleanup args
@@ -563,7 +583,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
563
583
 
564
584
  return cmdargs
565
585
 
566
- def generate_replay_script(self, filepath, workdir, include_path=True):
586
+ def generate_replay_script(self, filepath: str, workdir: str, include_path: bool = True) \
587
+ -> None:
567
588
  """
568
589
  Generates a shell script to replay the task's execution.
569
590
 
@@ -572,7 +593,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
572
593
  workdir (str): The path to the run's working directory.
573
594
  include_path (bool): If True, includes PATH information.
574
595
  """
575
- replay_opts = {}
596
+ replay_opts: Dict[str, Optional[Union[Dict[str, str], str, int]]] = {}
576
597
  replay_opts["work_dir"] = workdir
577
598
  replay_opts["exports"] = self.get_runtime_environmental_variables(include_path=include_path)
578
599
 
@@ -582,7 +603,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
582
603
  replay_opts["cfg_file"] = f"inputs/{self.__design_name}.pkg.json"
583
604
  replay_opts["node_only"] = 0 if replay_opts["executable"] else 1
584
605
 
585
- vswitch = self.get('vswitch')
606
+ vswitch: Optional[List[str]] = self.get('vswitch')
586
607
  if vswitch:
587
608
  replay_opts["version_flag"] = shlex.join(vswitch)
588
609
 
@@ -591,7 +612,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
591
612
  file_test = re.compile(r'^[/\.]')
592
613
 
593
614
  if replay_opts["executable"]:
594
- format_cmd = [replay_opts["executable"]]
615
+ format_cmd: List[str] = [replay_opts["executable"]]
595
616
 
596
617
  for cmdarg in self.get_runtime_arguments():
597
618
  add_new_line = len(format_cmd) == 1
@@ -618,7 +639,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
618
639
 
619
640
  os.chmod(filepath, 0o755)
620
641
 
621
- def setup_work_directory(self, workdir, remove_exist=True):
642
+ def setup_work_directory(self, workdir: str, remove_exist: bool = True) -> None:
622
643
  """
623
644
  Creates the runtime directories needed to execute a task.
624
645
 
@@ -637,7 +658,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
637
658
  os.makedirs(os.path.join(workdir, 'outputs'), exist_ok=True)
638
659
  os.makedirs(os.path.join(workdir, 'reports'), exist_ok=True)
639
660
 
640
- def __write_yaml_manifest(self, fout, manifest):
661
+ def __write_yaml_manifest(self, fout: TextIO, manifest: BaseSchema) -> None:
641
662
  """Private helper to write a manifest in YAML format."""
642
663
  class YamlIndentDumper(yaml.Dumper):
643
664
  def increase_indent(self, flow=False, indentless=False):
@@ -646,7 +667,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
646
667
  fout.write(yaml.dump(manifest.getdict(), Dumper=YamlIndentDumper,
647
668
  default_flow_style=False))
648
669
 
649
- def get_tcl_variables(self, manifest: BaseSchema = None) -> Dict[str, str]:
670
+ def get_tcl_variables(self, manifest: Optional[BaseSchema] = None) -> Dict[str, str]:
650
671
  """
651
672
  Gets a dictionary of variables to define for the task in a Tcl manifest.
652
673
 
@@ -667,13 +688,14 @@ class Task(NamedSchema, PathSchema, DocsSchema):
667
688
  "sc_designlib": NodeType.to_tcl(self.design_name, "str")
668
689
  }
669
690
 
670
- refdir = manifest.get("tool", self.tool(), "task", self.task(), "refdir", field=None)
691
+ refdir: Parameter = manifest.get("tool", self.tool(), "task", self.task(), "refdir",
692
+ field=None)
671
693
  if refdir.get(step=self.__step, index=self.__index):
672
694
  vars["sc_refdir"] = refdir.gettcl(step=self.__step, index=self.__index)
673
695
 
674
696
  return vars
675
697
 
676
- def __write_tcl_manifest(self, fout, manifest):
698
+ def __write_tcl_manifest(self, fout: TextIO, manifest: BaseSchema):
677
699
  """Private helper to write a manifest in Tcl format."""
678
700
  template = utils.get_file_template('tcl/manifest.tcl.j2')
679
701
  tcl_set_cmds = []
@@ -682,7 +704,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
682
704
  if 'default' in key:
683
705
  continue
684
706
 
685
- param = manifest.get(*key, field=None)
707
+ param: Parameter = manifest.get(*key, field=None)
686
708
 
687
709
  # Create a Tcl dict key string
688
710
  keystr = ' '.join([NodeType.to_tcl(keypart, 'str') for keypart in key])
@@ -709,14 +731,14 @@ class Task(NamedSchema, PathSchema, DocsSchema):
709
731
  fout.write(cmd + '\n')
710
732
  fout.write('\n')
711
733
 
712
- def __write_csv_manifest(self, fout, manifest):
734
+ def __write_csv_manifest(self, fout: TextIO, manifest: BaseSchema) -> None:
713
735
  """Private helper to write a manifest in CSV format."""
714
736
  csvwriter = csv.writer(fout)
715
737
  csvwriter.writerow(['Keypath', 'Value'])
716
738
 
717
739
  for key in sorted(manifest.allkeys()):
718
740
  keypath = ','.join(key)
719
- param = manifest.get(*key, field=None)
741
+ param: Parameter = manifest.get(*key, field=None)
720
742
  if param.get(field="pernode").is_never():
721
743
  value = param.get()
722
744
  else:
@@ -728,7 +750,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
728
750
  else:
729
751
  csvwriter.writerow([keypath, value])
730
752
 
731
- def write_task_manifest(self, directory, backup=True):
753
+ def write_task_manifest(self, directory: str, backup: bool = True) -> None:
732
754
  """
733
755
  Writes the manifest needed for the task in the format specified by the tool.
734
756
 
@@ -752,15 +774,14 @@ class Task(NamedSchema, PathSchema, DocsSchema):
752
774
  if re.search(r'\.json(\.gz)?$', manifest_path):
753
775
  schema.write_manifest(manifest_path)
754
776
  else:
777
+ # Format-specific dumping
778
+ if manifest_path.endswith('.gz'):
779
+ fout = gzip.open(manifest_path, 'wt', encoding='UTF-8')
780
+ elif re.search(r'\.csv$', manifest_path):
781
+ fout = open(manifest_path, 'w', newline='')
782
+ else:
783
+ fout = open(manifest_path, 'w')
755
784
  try:
756
- # Format-specific dumping
757
- if manifest_path.endswith('.gz'):
758
- fout = gzip.open(manifest_path, 'wt', encoding='UTF-8')
759
- elif re.search(r'\.csv$', manifest_path):
760
- fout = open(manifest_path, 'w', newline='')
761
- else:
762
- fout = open(manifest_path, 'w')
763
-
764
785
  if re.search(r'(\.yaml|\.yml)(\.gz)?$', manifest_path):
765
786
  self.__write_yaml_manifest(fout, schema)
766
787
  elif re.search(r'\.tcl(\.gz)?$', manifest_path):
@@ -772,7 +793,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
772
793
  finally:
773
794
  fout.close()
774
795
 
775
- def __abspath_schema(self):
796
+ def __abspath_schema(self) -> "Project":
776
797
  """
777
798
  Private helper to create a copy of the schema with all file/dir paths
778
799
  converted to absolute paths.
@@ -781,7 +802,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
781
802
  schema = root.copy()
782
803
 
783
804
  for keypath in root.allkeys():
784
- paramtype = schema.get(*keypath, field='type')
805
+ paramtype: str = schema.get(*keypath, field='type')
785
806
  if 'file' not in paramtype and 'dir' not in paramtype:
786
807
  continue
787
808
 
@@ -804,7 +825,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
804
825
 
805
826
  return schema
806
827
 
807
- def __get_io_file(self, io_type):
828
+ def __get_io_file(self, io_type: str) -> Tuple[str, bool]:
808
829
  """
809
830
  Private helper to get the runtime destination for stdout or stderr.
810
831
 
@@ -826,7 +847,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
826
847
 
827
848
  return io_file, io_log
828
849
 
829
- def __terminate_exe(self, proc):
850
+ def __terminate_exe(self, proc: subprocess.Popen) -> None:
830
851
  """
831
852
  Private helper to terminate a subprocess and its children.
832
853
 
@@ -834,7 +855,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
834
855
  proc (subprocess.Process): The process to terminate.
835
856
  """
836
857
 
837
- def terminate_process(pid, timeout=3):
858
+ def terminate_process(pid: int, timeout: int = 3) -> None:
838
859
  """Terminates a process and all its (grand+)children.
839
860
  Based on https://psutil.readthedocs.io/en/latest/#psutil.wait_procs and
840
861
  https://psutil.readthedocs.io/en/latest/#kill-process-tree."""
@@ -851,19 +872,24 @@ class Task(NamedSchema, PathSchema, DocsSchema):
851
872
  for p in alive:
852
873
  p.kill()
853
874
 
854
- TERMINATE_TIMEOUT = 5
875
+ timeout = 5
855
876
 
856
- terminate_process(proc.pid, timeout=TERMINATE_TIMEOUT)
857
- self.__logger.info(f'Waiting for {self.tool()}/{self.task()} to exit...')
877
+ terminate_process(proc.pid, timeout=timeout)
878
+ self.logger.info(f'Waiting for {self.tool()}/{self.task()} to exit...')
858
879
  try:
859
- proc.wait(timeout=TERMINATE_TIMEOUT)
880
+ proc.wait(timeout=timeout)
860
881
  except subprocess.TimeoutExpired:
861
882
  if proc.poll() is None:
862
- self.__logger.warning(f'{self.tool()}/{self.task()} did not exit within '
863
- f'{TERMINATE_TIMEOUT} seconds. Terminating...')
864
- terminate_process(proc.pid, timeout=TERMINATE_TIMEOUT)
883
+ self.logger.warning(f'{self.tool()}/{self.task()} did not exit within '
884
+ f'{timeout} seconds. Terminating...')
885
+ terminate_process(proc.pid, timeout=timeout)
865
886
 
866
- def run_task(self, workdir, quiet, breakpoint, nice, timeout):
887
+ def run_task(self,
888
+ workdir: str,
889
+ quiet: bool,
890
+ breakpoint: bool,
891
+ nice: Optional[int],
892
+ timeout: Optional[int]) -> int:
867
893
  """
868
894
  Executes the task's main process.
869
895
 
@@ -895,8 +921,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
895
921
  stdout_file, is_stdout_log = self.__get_io_file("stdout")
896
922
  stderr_file, is_stderr_log = self.__get_io_file("stderr")
897
923
 
898
- stdout_print = self.__logger.info
899
- stderr_print = self.__logger.error
924
+ stdout_print = self.logger.info
925
+ stderr_print = self.logger.error
900
926
 
901
927
  def read_stdio(stdout_reader, stderr_reader):
902
928
  """Helper to read and print stdout/stderr streams."""
@@ -926,8 +952,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
926
952
  contextlib.redirect_stdout(stdout_writer):
927
953
  retcode = self.run()
928
954
  except Exception as e:
929
- self.__logger.error(f'Failed in run() for {self.tool()}/{self.task()}: {e}')
930
- utils.print_traceback(self.__logger, e)
955
+ self.logger.error(f'Failed in run() for {self.tool()}/{self.task()}: {e}')
956
+ utils.print_traceback(self.logger, e)
931
957
  raise e
932
958
  finally:
933
959
  with sc_open(stdout_file) as stdout_reader, \
@@ -948,17 +974,17 @@ class Task(NamedSchema, PathSchema, DocsSchema):
948
974
 
949
975
  # Record tool options
950
976
  self.schema_record.record_tool(
951
- self.__step, self.__index,
977
+ self.step, self.index,
952
978
  cmdlist, RecordTool.ARGS)
953
979
 
954
- self.__logger.info(shlex.join([os.path.basename(exe), *cmdlist]))
980
+ self.logger.info(shlex.join([os.path.basename(exe), *cmdlist]))
955
981
 
956
982
  if not pty and breakpoint:
957
983
  breakpoint = False
958
984
 
959
985
  if breakpoint and sys.platform in ('darwin', 'linux'):
960
986
  # Use pty for interactive breakpoint sessions on POSIX systems
961
- with open(f"{self.__step}.log", 'wb') as log_writer:
987
+ with open(f"{self.step}.log", 'wb') as log_writer:
962
988
  def read(fd):
963
989
  data = os.read(fd, 1024)
964
990
  log_writer.write(data)
@@ -991,8 +1017,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
991
1017
  except Exception as e:
992
1018
  raise TaskError(f"Unable to start {exe}: {str(e)}")
993
1019
 
994
- POLL_INTERVAL = 0.1
995
- MEMORY_WARN_LIMIT = 90
1020
+ memory_warn_limit = Task.__MEMORY_WARN_LIMIT
996
1021
  try:
997
1022
  while proc.poll() is None:
998
1023
  # Monitor subprocess memory usage
@@ -1004,11 +1029,11 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1004
1029
  max_mem_bytes = max(max_mem_bytes, proc_mem_bytes)
1005
1030
 
1006
1031
  memory_usage = psutil.virtual_memory()
1007
- if memory_usage.percent > MEMORY_WARN_LIMIT:
1008
- self.__logger.warning(
1032
+ if memory_usage.percent > memory_warn_limit:
1033
+ self.logger.warning(
1009
1034
  'Current system memory usage is '
1010
1035
  f'{memory_usage.percent:.1f}%')
1011
- MEMORY_WARN_LIMIT = int(memory_usage.percent + 1)
1036
+ memory_warn_limit = int(memory_usage.percent + 1)
1012
1037
  except psutil.Error:
1013
1038
  # Process may have already terminated or been killed.
1014
1039
  # Retain existing memory usage statistics in this case.
@@ -1025,13 +1050,13 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1025
1050
  if timeout is not None and duration > timeout:
1026
1051
  raise TaskTimeout(timeout=duration)
1027
1052
 
1028
- time.sleep(POLL_INTERVAL)
1053
+ time.sleep(Task.__POLL_INTERVAL)
1029
1054
  except KeyboardInterrupt:
1030
- self.__logger.info("Received ctrl-c.")
1055
+ self.logger.info("Received ctrl-c.")
1031
1056
  self.__terminate_exe(proc)
1032
1057
  raise TaskError
1033
1058
  except TaskTimeout as e:
1034
- self.__logger.error(f'Task timed out after {e.timeout:.1f} seconds')
1059
+ self.logger.error(f'Task timed out after {e.timeout:.1f} seconds')
1035
1060
  self.__terminate_exe(proc)
1036
1061
  raise e from None
1037
1062
 
@@ -1042,14 +1067,14 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1042
1067
 
1043
1068
  # Record metrics
1044
1069
  self.schema_record.record_tool(
1045
- self.__step, self.__index,
1070
+ self.step, self.index,
1046
1071
  retcode, RecordTool.EXITCODE)
1047
1072
 
1048
1073
  self.schema_metric.record(
1049
- self.__step, self.__index,
1074
+ self.step, self.index,
1050
1075
  'exetime', time.time() - cpu_start, unit='s')
1051
1076
  self.schema_metric.record(
1052
- self.__step, self.__index,
1077
+ self.step, self.index,
1053
1078
  'memory', max_mem_bytes, unit='B')
1054
1079
 
1055
1080
  return retcode
@@ -1067,11 +1092,11 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1067
1092
  self.__dict__ = state
1068
1093
  self.__set_runtime(None)
1069
1094
 
1070
- def get_output_files(self):
1095
+ def get_output_files(self) -> Set[str]:
1071
1096
  """Gets the set of output files defined for this task."""
1072
1097
  return set(self.get("output"))
1073
1098
 
1074
- def get_files_from_input_nodes(self):
1099
+ def get_files_from_input_nodes(self) -> Dict[str, List[Tuple[str, str]]]:
1075
1100
  """
1076
1101
  Returns a dictionary of files from input nodes, mapped to the node
1077
1102
  they originated from.
@@ -1084,7 +1109,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1084
1109
 
1085
1110
  in_tool = self.schema_flow.get(in_step, in_index, "tool")
1086
1111
  in_task = self.schema_flow.get(in_step, in_index, "task")
1087
- task_obj = self.project.get("tool", in_tool, "task", in_task, field="schema")
1112
+ task_obj: Task = self.project.get("tool", in_tool, "task", in_task, field="schema")
1088
1113
 
1089
1114
  if self.schema_record.get('status', step=in_step, index=in_index) == \
1090
1115
  NodeStatus.SKIPPED:
@@ -1098,7 +1123,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1098
1123
 
1099
1124
  return inputs
1100
1125
 
1101
- def compute_input_file_node_name(self, filename, step, index):
1126
+ def compute_input_file_node_name(self, filename: str, step: str, index: str) -> str:
1102
1127
  """
1103
1128
  Generates a unique name for an input file based on its originating node.
1104
1129
 
@@ -1119,7 +1144,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1119
1144
  else:
1120
1145
  return f'{filename}.{step}{index}'
1121
1146
 
1122
- def add_parameter(self, name, type, help, defvalue=None, **kwargs):
1147
+ def add_parameter(self, name: str, type: str, help: str, defvalue=None, **kwargs) -> Parameter:
1123
1148
  """
1124
1149
  Adds a custom parameter ('var') to the task definition.
1125
1150
 
@@ -1146,7 +1171,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1146
1171
  # Task settings
1147
1172
  ###############################################################
1148
1173
  def add_required_key(self, obj: Union[BaseSchema, str], *key: str,
1149
- step: str = None, index: str = None):
1174
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None):
1150
1175
  '''
1151
1176
  Adds a required keypath to the task driver. If the key is valid relative to the task object
1152
1177
  the key will be assumed as a task key.
@@ -1170,8 +1195,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1170
1195
 
1171
1196
  return self.add("require", ",".join(key), step=step, index=index)
1172
1197
 
1173
- def set_threads(self, max_threads: int = None,
1174
- step: str = None, index: str = None,
1198
+ def set_threads(self, max_threads: Optional[int] = None,
1199
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1175
1200
  clobber: bool = False):
1176
1201
  """
1177
1202
  Sets the requested thread count for the task
@@ -1183,7 +1208,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1183
1208
  clobber (bool): overwrite existing value
1184
1209
  """
1185
1210
  if max_threads is None or max_threads <= 0:
1186
- max_schema_threads = self.project.option.scheduler.get("maxthreads")
1211
+ max_schema_threads: Optional[int] = self.project.option.scheduler.get("maxthreads")
1187
1212
  if max_schema_threads:
1188
1213
  max_threads = max_schema_threads
1189
1214
  else:
@@ -1191,14 +1216,15 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1191
1216
 
1192
1217
  return self.set("threads", max_threads, step=step, index=index, clobber=clobber)
1193
1218
 
1194
- def get_threads(self, step: str = None, index: str = None) -> int:
1219
+ def get_threads(self, step: Optional[str] = None,
1220
+ index: Optional[Union[str, int]] = None) -> int:
1195
1221
  """
1196
1222
  Returns the number of threads requested.
1197
1223
  """
1198
1224
  return self.get("threads", step=step, index=index)
1199
1225
 
1200
1226
  def add_commandline_option(self, option: Union[List[str], str],
1201
- step: str = None, index: str = None,
1227
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1202
1228
  clobber: bool = False):
1203
1229
  """
1204
1230
  Add to the command line options for the task
@@ -1213,14 +1239,17 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1213
1239
  else:
1214
1240
  return self.add("option", option, step=step, index=index)
1215
1241
 
1216
- def get_commandline_options(self, step: str = None, index: str = None) -> List[str]:
1242
+ def get_commandline_options(self,
1243
+ step: Optional[str] = None,
1244
+ index: Optional[Union[str, int]] = None) \
1245
+ -> List[str]:
1217
1246
  """
1218
1247
  Returns the command line options specified
1219
1248
  """
1220
1249
  return self.get("option", step=step, index=index)
1221
1250
 
1222
- def add_input_file(self, file: str = None, ext: str = None,
1223
- step: str = None, index: str = None,
1251
+ def add_input_file(self, file: Optional[str] = None, ext: Optional[str] = None,
1252
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1224
1253
  clobber: bool = False):
1225
1254
  """
1226
1255
  Add a required input file from the previous step in the flow.
@@ -1242,8 +1271,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1242
1271
  else:
1243
1272
  return self.add("input", file, step=step, index=index)
1244
1273
 
1245
- def add_output_file(self, file: str = None, ext: str = None,
1246
- step: str = None, index: str = None,
1274
+ def add_output_file(self, file: Optional[str] = None, ext: Optional[str] = None,
1275
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1247
1276
  clobber: bool = False):
1248
1277
  """
1249
1278
  Add an output file that this task will produce
@@ -1266,7 +1295,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1266
1295
  return self.add("output", file, step=step, index=index)
1267
1296
 
1268
1297
  def set_environmentalvariable(self, name: str, value: str,
1269
- step: str = None, index: str = None,
1298
+ step: Optional[str] = None,
1299
+ index: Optional[Union[str, int]] = None,
1270
1300
  clobber: bool = False):
1271
1301
  '''Sets an environment variable for the tool's execution context.
1272
1302
 
@@ -1288,8 +1318,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1288
1318
  '''
1289
1319
  return self.set("env", name, value, step=step, index=index, clobber=clobber)
1290
1320
 
1291
- def add_prescript(self, script: str, dataroot: str = None,
1292
- step: str = None, index: str = None,
1321
+ def add_prescript(self, script: str, dataroot: Optional[str] = None,
1322
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1293
1323
  clobber: bool = False):
1294
1324
  '''Adds a script to be executed *before* the main tool command.
1295
1325
 
@@ -1316,8 +1346,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1316
1346
  else:
1317
1347
  return self.add("prescript", script, step=step, index=index)
1318
1348
 
1319
- def add_postscript(self, script: str, dataroot: str = None,
1320
- step: str = None, index: str = None,
1349
+ def add_postscript(self, script: str, dataroot: Optional[str] = None,
1350
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1321
1351
  clobber: bool = False):
1322
1352
  '''Adds a script to be executed *after* the main tool command.
1323
1353
 
@@ -1344,7 +1374,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1344
1374
  else:
1345
1375
  return self.add("postscript", script, step=step, index=index)
1346
1376
 
1347
- def has_prescript(self, step: str = None, index: str = None) -> bool:
1377
+ def has_prescript(self,
1378
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None) -> bool:
1348
1379
  '''Checks if any pre-execution scripts are configured for the task.
1349
1380
 
1350
1381
  Args:
@@ -1358,7 +1389,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1358
1389
  return True
1359
1390
  return False
1360
1391
 
1361
- def has_postscript(self, step: str = None, index: str = None) -> bool:
1392
+ def has_postscript(self,
1393
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None) -> bool:
1362
1394
  '''Checks if any post-execution scripts are configured for the task.
1363
1395
 
1364
1396
  Args:
@@ -1372,8 +1404,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1372
1404
  return True
1373
1405
  return False
1374
1406
 
1375
- def set_refdir(self, dir: str, dataroot: str = None,
1376
- step: str = None, index: str = None,
1407
+ def set_refdir(self, dir: str, dataroot: Optional[str] = None,
1408
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1377
1409
  clobber: bool = False):
1378
1410
  '''Sets the reference directory for tool scripts and auxiliary files.
1379
1411
 
@@ -1396,8 +1428,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1396
1428
  with self.active_dataroot(self._get_active_dataroot(dataroot)):
1397
1429
  return self.set("refdir", dir, step=step, index=index, clobber=clobber)
1398
1430
 
1399
- def set_script(self, script: str, dataroot: str = ...,
1400
- step: str = None, index: str = None,
1431
+ def set_script(self, script: str, dataroot: Optional[str] = ...,
1432
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1401
1433
  clobber: bool = False):
1402
1434
  '''Sets the main entry script for a script-based tool (e.g., a TCL script).
1403
1435
 
@@ -1418,7 +1450,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1418
1450
  return self.set("script", script, step=step, index=index, clobber=clobber)
1419
1451
 
1420
1452
  def add_regex(self, type: str, regex: str,
1421
- step: str = None, index: str = None,
1453
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1422
1454
  clobber: bool = False):
1423
1455
  '''Adds a regular expression for parsing the tool's log file.
1424
1456
 
@@ -1443,8 +1475,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1443
1475
  else:
1444
1476
  return self.add("regex", type, regex, step=step, index=index)
1445
1477
 
1446
- def set_logdestination(self, type: str, dest: str, suffix: str = None,
1447
- step: str = None, index: str = None,
1478
+ def set_logdestination(self, type: str, dest: str, suffix: Optional[str] = None,
1479
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1448
1480
  clobber: bool = False):
1449
1481
  '''Configures the destination for log files.
1450
1482
 
@@ -1470,7 +1502,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1470
1502
  rets.append(self.set(type, "suffix", suffix, step=step, index=index, clobber=clobber))
1471
1503
  return rets
1472
1504
 
1473
- def add_warningoff(self, type: str, step: str = None, index: str = None, clobber: bool = False):
1505
+ def add_warningoff(self, type: str,
1506
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1507
+ clobber: bool = False):
1474
1508
  '''Adds a warning message or code to be suppressed during log parsing.
1475
1509
 
1476
1510
  Any warning that matches a regex in this list will be ignored by the
@@ -1496,8 +1530,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1496
1530
  ###############################################################
1497
1531
  # Tool settings
1498
1532
  ###############################################################
1499
- def set_exe(self, exe: str = None, vswitch: List[str] = None, format: str = None,
1500
- step: str = None, index: str = None,
1533
+ def set_exe(self, exe: Optional[str] = None, vswitch: Optional[Union[str, List[str]]] = None,
1534
+ format: Optional[str] = None,
1535
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1501
1536
  clobber: bool = False):
1502
1537
  '''Sets the executable, version switch, and script format for a tool.
1503
1538
 
@@ -1522,18 +1557,18 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1522
1557
  '''
1523
1558
  rets = []
1524
1559
  if exe:
1525
- rets.append(self.set("exe", exe, clobber=clobber))
1560
+ rets.append(self.set("exe", exe, step=step, index=index, clobber=clobber))
1526
1561
  if vswitch:
1527
- switches = self.add_vswitch(vswitch, clobber=clobber)
1562
+ switches = self.add_vswitch(vswitch, step=step, index=index, clobber=clobber)
1528
1563
  if not isinstance(switches, list):
1529
1564
  switches = list(switches)
1530
1565
  rets.extend(switches)
1531
1566
  if format:
1532
- rets.append(self.set("format", format, clobber=clobber))
1567
+ rets.append(self.set("format", format, step=step, index=index, clobber=clobber))
1533
1568
  return rets
1534
1569
 
1535
- def set_path(self, path: str, dataroot: str = None,
1536
- step: str = None, index: str = None,
1570
+ def set_path(self, path: str, dataroot: Optional[str] = None,
1571
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1537
1572
  clobber: bool = False):
1538
1573
  '''Sets the directory path where the tool's executable is located.
1539
1574
 
@@ -1557,7 +1592,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1557
1592
  with self.active_dataroot(self._get_active_dataroot(dataroot)):
1558
1593
  return self.set("path", path, step=step, index=index, clobber=clobber)
1559
1594
 
1560
- def add_version(self, version: str, step: str = None, index: str = None, clobber: bool = False):
1595
+ def add_version(self, version: Union[List[str], str],
1596
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1597
+ clobber: bool = False):
1561
1598
  '''Adds a supported version specifier for the tool.
1562
1599
 
1563
1600
  SiliconCompiler checks the tool's actual version against these
@@ -1581,7 +1618,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1581
1618
  else:
1582
1619
  return self.add("version", version, step=step, index=index)
1583
1620
 
1584
- def add_vswitch(self, switch: str, clobber: bool = False):
1621
+ def add_vswitch(self, switch: Union[List[str], str],
1622
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1623
+ clobber: bool = False):
1585
1624
  '''Adds the command-line switch used to print the tool's version.
1586
1625
 
1587
1626
  This switch is passed to the executable to get its version string
@@ -1596,12 +1635,12 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1596
1635
  The schema key that was set.
1597
1636
  '''
1598
1637
  if clobber:
1599
- return self.set("vswitch", switch)
1638
+ return self.set("vswitch", switch, step=step, index=index)
1600
1639
  else:
1601
- return self.add("vswitch", switch)
1640
+ return self.add("vswitch", switch, step=step, index=index)
1602
1641
 
1603
1642
  def add_licenseserver(self, name: str, server: str,
1604
- step: str = None, index: str = None,
1643
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1605
1644
  clobber: bool = False):
1606
1645
  '''Configures a license server connection for the tool.
1607
1646
 
@@ -1626,7 +1665,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1626
1665
  else:
1627
1666
  return self.add("licenseserver", name, server, step=step, index=index)
1628
1667
 
1629
- def add_sbom(self, version: str, sbom: str, dataroot: str = None, clobber: bool = False):
1668
+ def add_sbom(self, version: str, sbom: Union[str, List[str]], dataroot: Optional[str] = None,
1669
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1670
+ clobber: bool = False):
1630
1671
  '''Adds a Software Bill of Materials (SBOM) file for a tool version.
1631
1672
 
1632
1673
  Associates a specific tool version with its corresponding SBOM file,
@@ -1645,11 +1686,13 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1645
1686
  '''
1646
1687
  with self.active_dataroot(self._get_active_dataroot(dataroot)):
1647
1688
  if clobber:
1648
- return self.set("sbom", version, sbom)
1689
+ return self.set("sbom", version, sbom, step=step, index=index)
1649
1690
  else:
1650
- return self.add("sbom", version, sbom)
1691
+ return self.add("sbom", version, sbom, step=step, index=index)
1651
1692
 
1652
- def record_metric(self, metric, value, source_file=None, source_unit=None, quiet=False):
1693
+ def record_metric(self, metric: str, value: Union[int, float],
1694
+ source_file: Optional[str] = None, source_unit: Optional[str] = None,
1695
+ quiet: bool = False):
1653
1696
  '''
1654
1697
  Records a metric and associates the source file with it.
1655
1698
 
@@ -1671,11 +1714,11 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1671
1714
  self.logger.warning(f"{metric} is not a valid metric")
1672
1715
  return
1673
1716
 
1674
- self.schema_metric.record(self.__step, self.__index, metric, value, unit=source_unit)
1717
+ self.schema_metric.record(self.step, self.index, metric, value, unit=source_unit)
1675
1718
  if source_file:
1676
1719
  self.add("report", metric, source_file)
1677
1720
 
1678
- def get_fileset_file_keys(self, filetype: str) -> List[Tuple[NamedSchema, Tuple[str]]]:
1721
+ def get_fileset_file_keys(self, filetype: str) -> List[Tuple[NamedSchema, Tuple[str, ...]]]:
1679
1722
  """
1680
1723
  Collect a set of keys for a particular filetype.
1681
1724
 
@@ -1698,60 +1741,68 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1698
1741
  ###############################################################
1699
1742
  # Schema
1700
1743
  ###############################################################
1701
- def get(self, *keypath, field='value', step: str = None, index: str = None):
1702
- if not step:
1744
+ def get(self, *keypath: str, field: Optional[str] = 'value',
1745
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None):
1746
+ if step is None:
1703
1747
  step = self.__step
1704
- if not index:
1748
+ if index is None:
1705
1749
  index = self.__index
1706
1750
  return super().get(*keypath, field=field, step=step, index=index)
1707
1751
 
1708
- def set(self, *args, field='value', step: str = None, index: str = None, clobber=True):
1709
- if not step:
1752
+ def set(self, *args, field: str = 'value',
1753
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None,
1754
+ clobber: bool = True):
1755
+ if step is None:
1710
1756
  step = self.__step
1711
- if not index:
1757
+ if index is None:
1712
1758
  index = self.__index
1713
1759
  return super().set(*args, field=field, clobber=clobber, step=step, index=index)
1714
1760
 
1715
- def add(self, *args, field='value', step: str = None, index: str = None):
1716
- if not step:
1761
+ def add(self, *args, field: str = 'value',
1762
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None):
1763
+ if step is None:
1717
1764
  step = self.__step
1718
- if not index:
1765
+ if index is None:
1719
1766
  index = self.__index
1720
1767
  return super().add(*args, field=field, step=step, index=index)
1721
1768
 
1722
- def unset(self, *args, step: str = None, index: str = None):
1723
- if not step:
1769
+ def unset(self, *args: str,
1770
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None):
1771
+ if step is None:
1724
1772
  step = self.__step
1725
- if not index:
1773
+ if index is None:
1726
1774
  index = self.__index
1727
1775
  return super().unset(*args, step=step, index=index)
1728
1776
 
1729
- def find_files(self, *keypath, missing_ok=False, step=None, index=None):
1730
- if not step:
1777
+ def find_files(self, *keypath: str, missing_ok: bool = False,
1778
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None):
1779
+ if step is None:
1731
1780
  step = self.__step
1732
- if not index:
1781
+ if index is None:
1733
1782
  index = self.__index
1734
1783
  return super().find_files(*keypath, missing_ok=missing_ok,
1735
1784
  step=step, index=index)
1736
1785
 
1737
- def _find_files_search_paths(self, keypath, step, index):
1738
- search_paths = super()._find_files_search_paths(keypath, step, index)
1739
- if keypath == "script":
1786
+ def _find_files_search_paths(self, key: str,
1787
+ step: Optional[str],
1788
+ index: Optional[Union[int, str]]) -> List[str]:
1789
+ search_paths = super()._find_files_search_paths(key, step, index)
1790
+ if key == "script":
1740
1791
  search_paths.extend(self.find_files("refdir", step=step, index=index))
1741
- elif keypath == "input":
1792
+ elif key == "input":
1742
1793
  search_paths.append(os.path.join(
1743
1794
  paths.workdir(self._parent(root=True), step=step, index=index), "inputs"))
1744
- elif keypath == "report":
1795
+ elif key == "report":
1745
1796
  search_paths.append(os.path.join(
1746
- paths.workdir(self._parent(root=True), step=step, index=index), "report"))
1747
- elif keypath == "output":
1797
+ paths.workdir(self._parent(root=True), step=step, index=index), "reports"))
1798
+ elif key == "output":
1748
1799
  search_paths.append(os.path.join(
1749
1800
  paths.workdir(self._parent(root=True), step=step, index=index), "outputs"))
1750
1801
  return search_paths
1751
1802
 
1752
1803
  def _generate_doc(self, doc,
1753
1804
  ref_root: str = "",
1754
- key_offset: Tuple[str] = None,
1805
+ key_offset: Optional[Tuple[str, ...]] = None,
1755
1806
  detailed: bool = True):
1756
1807
  from .schema.docs.utils import build_section, strong, KeyPath, code, para, \
1757
1808
  build_table, build_schema_value_table
@@ -1760,7 +1811,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1760
1811
  docs = []
1761
1812
 
1762
1813
  if not key_offset:
1763
- key_offset = []
1814
+ key_offset = tuple()
1764
1815
 
1765
1816
  # Show dataroot
1766
1817
  dataroot = PathSchema._generate_doc(self, doc, ref_root=ref_root, key_offset=key_offset)
@@ -1826,8 +1877,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1826
1877
  params[key] = self.get(*key, field=None)
1827
1878
 
1828
1879
  with KeyPath.fallback(...):
1829
- table = build_schema_value_table(params, "", key_offset + list(self._keypath),
1830
- trim_prefix=key_offset + list(self._keypath))
1880
+ table = build_schema_value_table(params, "", list(key_offset) + list(self._keypath),
1881
+ trim_prefix=list(key_offset) + list(self._keypath))
1831
1882
  setup_info = build_section("Configuration", f"{ref_root}-config")
1832
1883
  setup_info += table
1833
1884
  docs.append(setup_info)
@@ -1854,55 +1905,55 @@ class Task(NamedSchema, PathSchema, DocsSchema):
1854
1905
  node.setup()
1855
1906
  return node.task
1856
1907
 
1857
- def parse_version(self, stdout):
1908
+ def parse_version(self, stdout: str) -> str:
1858
1909
  """
1859
1910
  Parses the tool's version from its stdout. Must be implemented by subclasses.
1860
1911
  """
1861
1912
  raise NotImplementedError("must be implemented by the implementation class")
1862
1913
 
1863
- def normalize_version(self, version):
1914
+ def normalize_version(self, version: str) -> str:
1864
1915
  """
1865
1916
  Normalizes a version string to a standard format. Can be overridden.
1866
1917
  """
1867
1918
  return version
1868
1919
 
1869
- def setup(self):
1920
+ def setup(self) -> None:
1870
1921
  """
1871
1922
  A hook for setting up the task before execution. Can be overridden.
1872
1923
  """
1873
1924
  pass
1874
1925
 
1875
- def select_input_nodes(self):
1926
+ def select_input_nodes(self) -> List[Tuple[str, str]]:
1876
1927
  """
1877
1928
  Determines which preceding nodes are inputs to this task.
1878
1929
  """
1879
1930
  return self.schema_flowruntime.get_node_inputs(
1880
- self.__step, self.__index, record=self.schema_record)
1931
+ self.step, self.index, record=self.schema_record)
1881
1932
 
1882
- def pre_process(self):
1933
+ def pre_process(self) -> None:
1883
1934
  """
1884
1935
  A hook for pre-processing before the main tool execution. Can be overridden.
1885
1936
  """
1886
1937
  pass
1887
1938
 
1888
- def runtime_options(self):
1939
+ def runtime_options(self) -> List[Union[int, str, Path]]:
1889
1940
  """
1890
1941
  Constructs the default runtime options for the task. Can be extended.
1891
1942
  """
1892
- cmdargs = []
1943
+ cmdargs: List[Union[int, str, Path]] = []
1893
1944
  cmdargs.extend(self.get("option"))
1894
- script = self.find_files('script', missing_ok=True)
1945
+ script: List[str] = self.find_files('script', missing_ok=True)
1895
1946
  if script:
1896
1947
  cmdargs.extend(script)
1897
1948
  return cmdargs
1898
1949
 
1899
- def run(self):
1950
+ def run(self) -> int:
1900
1951
  """
1901
1952
  The main execution logic for Python-based tasks. Must be implemented.
1902
1953
  """
1903
1954
  raise NotImplementedError("must be implemented by the implementation class")
1904
1955
 
1905
- def post_process(self):
1956
+ def post_process(self) -> None:
1906
1957
  """
1907
1958
  A hook for post-processing after the main tool execution. Can be overridden.
1908
1959
  """
@@ -1932,7 +1983,7 @@ class ShowTask(Task):
1932
1983
  self.add_parameter("showexit", "bool", "exit after opening", defvalue=False)
1933
1984
 
1934
1985
  @classmethod
1935
- def __check_task(cls, task):
1986
+ def __check_task(cls, task: Optional[Type["ShowTask"]]) -> bool:
1936
1987
  """
1937
1988
  Private helper to validate if a task is a valid ShowTask or ScreenshotTask.
1938
1989
  """
@@ -1940,7 +1991,7 @@ class ShowTask(Task):
1940
1991
  raise TypeError("class must be ShowTask or ScreenshotTask")
1941
1992
 
1942
1993
  if task is None:
1943
- return
1994
+ return False
1944
1995
 
1945
1996
  if cls is ShowTask:
1946
1997
  check, task_filter = ShowTask, ScreenshotTask
@@ -1955,7 +2006,7 @@ class ShowTask(Task):
1955
2006
  return True
1956
2007
 
1957
2008
  @classmethod
1958
- def register_task(cls, task):
2009
+ def register_task(cls, task: Optional[Type["ShowTask"]]) -> None:
1959
2010
  """
1960
2011
  Registers a new show task class for dynamic discovery.
1961
2012
 
@@ -1972,7 +2023,7 @@ class ShowTask(Task):
1972
2023
  cls.__TASKS.setdefault(cls, set()).add(task)
1973
2024
 
1974
2025
  @classmethod
1975
- def __populate_tasks(cls):
2026
+ def __populate_tasks(cls) -> None:
1976
2027
  """
1977
2028
  Private helper to discover and populate all available show/screenshot tasks.
1978
2029
 
@@ -1981,7 +2032,7 @@ class ShowTask(Task):
1981
2032
  """
1982
2033
  cls.__check_task(None)
1983
2034
 
1984
- def recurse(searchcls):
2035
+ def recurse(searchcls: Type["ShowTask"]):
1985
2036
  subclss = set()
1986
2037
  if not cls.__check_task(searchcls):
1987
2038
  return subclss
@@ -2005,7 +2056,7 @@ class ShowTask(Task):
2005
2056
  ShowTask.__TASKS.setdefault(cls, set()).update(classes)
2006
2057
 
2007
2058
  @classmethod
2008
- def get_task(cls, ext):
2059
+ def get_task(cls, ext: Optional[str]) -> Union[Optional["ShowTask"], Set[Type["ShowTask"]]]:
2009
2060
  """
2010
2061
  Retrieves a suitable show task instance for a given file extension.
2011
2062
 
@@ -2040,11 +2091,11 @@ class ShowTask(Task):
2040
2091
 
2041
2092
  return None
2042
2093
 
2043
- def task(self):
2094
+ def task(self) -> str:
2044
2095
  """Returns the name of this task."""
2045
2096
  return "show"
2046
2097
 
2047
- def setup(self):
2098
+ def setup(self) -> None:
2048
2099
  """Sets up the parameters and requirements for the show task."""
2049
2100
  super().setup()
2050
2101
 
@@ -2070,7 +2121,7 @@ class ShowTask(Task):
2070
2121
  raise NotImplementedError(
2071
2122
  "get_supported_show_extentions must be implemented by the child class")
2072
2123
 
2073
- def _set_filetype(self):
2124
+ def _set_filetype(self) -> None:
2074
2125
  """
2075
2126
  Private helper to determine and set the 'showfiletype' parameter based
2076
2127
  on the provided 'showfilepath' or available input files.
@@ -2097,24 +2148,28 @@ class ShowTask(Task):
2097
2148
  ext = utils.get_file_ext(file)
2098
2149
  set_file(file, ext)
2099
2150
 
2100
- def set_showfilepath(self, path: str, step: str = None, index: str = None):
2151
+ def set_showfilepath(self, path: str,
2152
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None):
2101
2153
  """Sets the path to the file to be displayed."""
2102
2154
  return self.set("var", "showfilepath", path, step=step, index=index)
2103
2155
 
2104
- def set_showfiletype(self, file_type: str, step: str = None, index: str = None):
2156
+ def set_showfiletype(self, file_type: str,
2157
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None):
2105
2158
  """Sets the type of the file to be displayed."""
2106
2159
  return self.set("var", "showfiletype", file_type, step=step, index=index)
2107
2160
 
2108
- def set_showexit(self, value: bool, step: str = None, index: str = None):
2161
+ def set_showexit(self, value: bool,
2162
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None):
2109
2163
  """Sets whether the viewer application should exit after opening the file."""
2110
2164
  return self.set("var", "showexit", value, step=step, index=index)
2111
2165
 
2112
- def set_shownode(self, jobname: str = None, nodestep: str = None, nodeindex: str = None,
2113
- step: str = None, index: str = None):
2166
+ def set_shownode(self, jobname: Optional[str] = None,
2167
+ nodestep: Optional[str] = None, nodeindex: Optional[Union[str, int]] = None,
2168
+ step: Optional[str] = None, index: Optional[Union[str, int]] = None):
2114
2169
  """Sets the source node information for the file being displayed."""
2115
2170
  return self.set("var", "shownode", (jobname, nodestep, nodeindex), step=step, index=index)
2116
2171
 
2117
- def get_tcl_variables(self, manifest=None):
2172
+ def get_tcl_variables(self, manifest: Optional[BaseSchema] = None) -> Dict[str, str]:
2118
2173
  """
2119
2174
  Gets Tcl variables for the task, ensuring 'sc_do_screenshot' is false
2120
2175
  for regular show tasks.
@@ -2133,11 +2188,11 @@ class ScreenshotTask(ShowTask):
2133
2188
  sets the 'showexit' parameter to True.
2134
2189
  """
2135
2190
 
2136
- def task(self):
2191
+ def task(self) -> str:
2137
2192
  """Returns the name of this task."""
2138
2193
  return "screenshot"
2139
2194
 
2140
- def setup(self):
2195
+ def setup(self) -> None:
2141
2196
  """
2142
2197
  Sets up the screenshot task, ensuring that the viewer will exit
2143
2198
  after the screenshot is taken.
@@ -2146,7 +2201,7 @@ class ScreenshotTask(ShowTask):
2146
2201
  # Ensure the viewer exits after taking the screenshot
2147
2202
  self.set_showexit(True)
2148
2203
 
2149
- def get_tcl_variables(self, manifest=None):
2204
+ def get_tcl_variables(self, manifest: Optional[BaseSchema] = None) -> Dict[str, str]:
2150
2205
  """
2151
2206
  Gets Tcl variables for the task, setting 'sc_do_screenshot' to true.
2152
2207
  """