siliconcompiler 0.35.0__py3-none-any.whl → 0.35.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- siliconcompiler/_metadata.py +1 -1
- siliconcompiler/apps/_common.py +3 -2
- siliconcompiler/apps/sc_dashboard.py +3 -1
- siliconcompiler/apps/sc_install.py +149 -37
- siliconcompiler/apps/smake.py +9 -3
- siliconcompiler/checklist.py +3 -3
- siliconcompiler/data/demo_fpga/z1000_yosys_config.json +24 -0
- siliconcompiler/design.py +51 -45
- siliconcompiler/flowgraph.py +2 -2
- siliconcompiler/library.py +23 -12
- siliconcompiler/package/__init__.py +77 -49
- siliconcompiler/package/git.py +11 -6
- siliconcompiler/package/github.py +11 -6
- siliconcompiler/package/https.py +6 -4
- siliconcompiler/pdk.py +23 -16
- siliconcompiler/scheduler/scheduler.py +30 -22
- siliconcompiler/scheduler/schedulernode.py +60 -50
- siliconcompiler/scheduler/taskscheduler.py +52 -32
- siliconcompiler/schema/baseschema.py +88 -69
- siliconcompiler/schema/docs/schemagen.py +4 -3
- siliconcompiler/schema/editableschema.py +5 -5
- siliconcompiler/schema/journal.py +19 -13
- siliconcompiler/schema/namedschema.py +16 -10
- siliconcompiler/schema/parameter.py +64 -37
- siliconcompiler/schema/parametervalue.py +126 -80
- siliconcompiler/schema/safeschema.py +16 -7
- siliconcompiler/schema/utils.py +3 -1
- siliconcompiler/schema_support/cmdlineschema.py +9 -9
- siliconcompiler/schema_support/dependencyschema.py +12 -7
- siliconcompiler/schema_support/filesetschema.py +15 -10
- siliconcompiler/schema_support/metric.py +29 -17
- siliconcompiler/schema_support/packageschema.py +2 -2
- siliconcompiler/schema_support/pathschema.py +30 -18
- siliconcompiler/schema_support/record.py +30 -23
- siliconcompiler/tool.py +265 -210
- siliconcompiler/tools/opensta/timing.py +13 -0
- siliconcompiler/tools/yosys/syn_fpga.py +3 -2
- siliconcompiler/toolscripts/_tools.json +3 -3
- siliconcompiler/utils/__init__.py +23 -16
- siliconcompiler/utils/curation.py +11 -5
- siliconcompiler/utils/multiprocessing.py +16 -14
- siliconcompiler/utils/paths.py +24 -12
- siliconcompiler/utils/units.py +16 -12
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/METADATA +3 -4
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/RECORD +49 -48
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/entry_points.txt +4 -3
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/WHEEL +0 -0
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/licenses/LICENSE +0 -0
- {siliconcompiler-0.35.0.dist-info → siliconcompiler-0.35.1.dist-info}/top_level.txt +0 -0
siliconcompiler/tool.py
CHANGED
|
@@ -32,7 +32,8 @@ import os.path
|
|
|
32
32
|
from packaging.version import Version, InvalidVersion
|
|
33
33
|
from packaging.specifiers import SpecifierSet, InvalidSpecifier
|
|
34
34
|
|
|
35
|
-
from typing import List, Dict, Tuple, Union
|
|
35
|
+
from typing import List, Dict, Tuple, Union, Optional, Set, TextIO, Type, TYPE_CHECKING
|
|
36
|
+
from pathlib import Path
|
|
36
37
|
|
|
37
38
|
from siliconcompiler.schema import BaseSchema, NamedSchema, Journal, DocsSchema
|
|
38
39
|
from siliconcompiler.schema import EditableSchema, Parameter, PerNode, Scope
|
|
@@ -48,6 +49,10 @@ from siliconcompiler.schema_support.record import RecordTool, RecordSchema
|
|
|
48
49
|
from siliconcompiler.schema_support.metric import MetricSchema
|
|
49
50
|
from siliconcompiler.flowgraph import RuntimeFlowgraph
|
|
50
51
|
|
|
52
|
+
if TYPE_CHECKING:
|
|
53
|
+
from siliconcompiler.scheduler import SchedulerNode
|
|
54
|
+
from siliconcompiler import Project
|
|
55
|
+
|
|
51
56
|
|
|
52
57
|
class TaskError(Exception):
|
|
53
58
|
'''Error indicating that task execution cannot continue and should be terminated.'''
|
|
@@ -114,6 +119,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
114
119
|
r"^\s*" + __parse_version_check_str + r"\s*$",
|
|
115
120
|
re.VERBOSE | re.IGNORECASE)
|
|
116
121
|
|
|
122
|
+
__POLL_INTERVAL: float = 0.1
|
|
123
|
+
__MEMORY_WARN_LIMIT: int = 90
|
|
124
|
+
|
|
117
125
|
def __init__(self):
|
|
118
126
|
super().__init__()
|
|
119
127
|
|
|
@@ -126,7 +134,10 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
126
134
|
"""Returns the metadata for getdict."""
|
|
127
135
|
return Task.__name__
|
|
128
136
|
|
|
129
|
-
def _from_dict(self, manifest
|
|
137
|
+
def _from_dict(self, manifest: Dict,
|
|
138
|
+
keypath: Union[List[str], Tuple[str, ...]],
|
|
139
|
+
version: Optional[Tuple[int, ...]] = None) \
|
|
140
|
+
-> Tuple[Set[Tuple[str, ...]], Set[Tuple[str, ...]]]:
|
|
130
141
|
"""
|
|
131
142
|
Populates the schema from a dictionary, dynamically adding 'var'
|
|
132
143
|
parameters found in the manifest that are not already defined.
|
|
@@ -142,7 +153,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
142
153
|
edit.insert("var", var,
|
|
143
154
|
Parameter.from_dict(
|
|
144
155
|
manifest["var"][var],
|
|
145
|
-
keypath=keypath
|
|
156
|
+
keypath=tuple([*keypath, var]),
|
|
146
157
|
version=version))
|
|
147
158
|
del manifest["var"][var]
|
|
148
159
|
|
|
@@ -152,7 +163,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
152
163
|
return super()._from_dict(manifest, keypath, version)
|
|
153
164
|
|
|
154
165
|
@contextlib.contextmanager
|
|
155
|
-
def runtime(self, node
|
|
166
|
+
def runtime(self, node: "SchedulerNode",
|
|
167
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
168
|
+
relpath: Optional[str] = None):
|
|
156
169
|
"""
|
|
157
170
|
A context manager to set the runtime information for a task.
|
|
158
171
|
|
|
@@ -172,20 +185,24 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
172
185
|
obj_copy.__set_runtime(node, step=step, index=index, relpath=relpath)
|
|
173
186
|
yield obj_copy
|
|
174
187
|
|
|
175
|
-
def __set_runtime(self, node
|
|
188
|
+
def __set_runtime(self, node: Optional["SchedulerNode"],
|
|
189
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
190
|
+
relpath: Optional[str] = None) -> None:
|
|
176
191
|
"""
|
|
177
192
|
Private helper to set the runtime information for executing a task.
|
|
178
193
|
|
|
179
194
|
Args:
|
|
180
195
|
node (SchedulerNode): The scheduler node for this runtime.
|
|
181
196
|
"""
|
|
182
|
-
self.__node = node
|
|
183
|
-
self.__schema_full = None
|
|
184
|
-
self.__logger = None
|
|
185
|
-
self.__design_name = None
|
|
186
|
-
self.__design_top = None
|
|
187
|
-
self.__relpath = relpath
|
|
188
|
-
self.__jobdir = None
|
|
197
|
+
self.__node: Optional["SchedulerNode"] = node
|
|
198
|
+
self.__schema_full: Optional["Project"] = None
|
|
199
|
+
self.__logger: Optional[logging.Logger] = None
|
|
200
|
+
self.__design_name: Optional[str] = None
|
|
201
|
+
self.__design_top: Optional[str] = None
|
|
202
|
+
self.__relpath: Optional[str] = relpath
|
|
203
|
+
self.__jobdir: Optional[str] = None
|
|
204
|
+
self.__step: Optional[str] = None
|
|
205
|
+
self.__index: Optional[str] = None
|
|
189
206
|
if node:
|
|
190
207
|
if step is not None or index is not None:
|
|
191
208
|
raise RuntimeError("step and index cannot be provided with node")
|
|
@@ -200,12 +217,14 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
200
217
|
self.__index = node.index
|
|
201
218
|
else:
|
|
202
219
|
self.__step = step
|
|
220
|
+
if isinstance(index, int):
|
|
221
|
+
index = str(index)
|
|
203
222
|
self.__index = index
|
|
204
223
|
|
|
205
|
-
self.__schema_record = None
|
|
206
|
-
self.__schema_metric = None
|
|
207
|
-
self.__schema_flow = None
|
|
208
|
-
self.__schema_flow_runtime = None
|
|
224
|
+
self.__schema_record: Optional[RecordSchema] = None
|
|
225
|
+
self.__schema_metric: Optional[MetricSchema] = None
|
|
226
|
+
self.__schema_flow: Optional[Flowgraph] = None
|
|
227
|
+
self.__schema_flow_runtime: Optional[RuntimeFlowgraph] = None
|
|
209
228
|
if self.__schema_full:
|
|
210
229
|
self.__schema_record = self.__schema_full.get("record", field="schema")
|
|
211
230
|
self.__schema_metric = self.__schema_full.get("metric", field="schema")
|
|
@@ -239,7 +258,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
239
258
|
return self.__design_top
|
|
240
259
|
|
|
241
260
|
@property
|
|
242
|
-
def node(self):
|
|
261
|
+
def node(self) -> "SchedulerNode":
|
|
243
262
|
"""SchedulerNode: The scheduler node for the current runtime."""
|
|
244
263
|
return self.__node
|
|
245
264
|
|
|
@@ -288,7 +307,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
288
307
|
return self.__schema_metric
|
|
289
308
|
|
|
290
309
|
@property
|
|
291
|
-
def project(self):
|
|
310
|
+
def project(self) -> "Project":
|
|
292
311
|
return self.__schema_full
|
|
293
312
|
|
|
294
313
|
@property
|
|
@@ -318,9 +337,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
318
337
|
Returns:
|
|
319
338
|
bool: True if a breakpoint is active, False otherwise.
|
|
320
339
|
"""
|
|
321
|
-
return self.project.
|
|
340
|
+
return self.project.option.get_breakpoint(step=self.__step, index=self.__index)
|
|
322
341
|
|
|
323
|
-
def get_exe(self) -> str:
|
|
342
|
+
def get_exe(self) -> Optional[str]:
|
|
324
343
|
"""
|
|
325
344
|
Determines the absolute path for the task's executable.
|
|
326
345
|
|
|
@@ -331,7 +350,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
331
350
|
str: The absolute path to the executable, or None if not specified.
|
|
332
351
|
"""
|
|
333
352
|
|
|
334
|
-
exe = self.get('exe')
|
|
353
|
+
exe: Optional[str] = self.get('exe')
|
|
335
354
|
|
|
336
355
|
if exe is None:
|
|
337
356
|
return None
|
|
@@ -358,7 +377,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
358
377
|
if self.tool() in tools:
|
|
359
378
|
self.logger.info(f"Missing tool can be installed via: \"sc-install {self.tool()}\"")
|
|
360
379
|
|
|
361
|
-
def get_exe_version(self) -> str:
|
|
380
|
+
def get_exe_version(self) -> Optional[str]:
|
|
362
381
|
"""
|
|
363
382
|
Gets the version of the task's executable by running it with a version switch.
|
|
364
383
|
|
|
@@ -370,7 +389,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
370
389
|
str: The parsed version string.
|
|
371
390
|
"""
|
|
372
391
|
|
|
373
|
-
veropt = self.get('vswitch')
|
|
392
|
+
veropt: Optional[List[str]] = self.get('vswitch')
|
|
374
393
|
if not veropt:
|
|
375
394
|
return None
|
|
376
395
|
|
|
@@ -383,8 +402,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
383
402
|
cmdlist = [exe]
|
|
384
403
|
cmdlist.extend(veropt)
|
|
385
404
|
|
|
386
|
-
self.
|
|
387
|
-
|
|
405
|
+
self.logger.debug(f'Running {self.tool()}/{self.task()} version check: '
|
|
406
|
+
f'{" ".join(cmdlist)}')
|
|
388
407
|
|
|
389
408
|
proc = subprocess.run(cmdlist,
|
|
390
409
|
stdin=subprocess.DEVNULL,
|
|
@@ -393,8 +412,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
393
412
|
universal_newlines=True)
|
|
394
413
|
|
|
395
414
|
if proc.returncode != 0:
|
|
396
|
-
self.
|
|
397
|
-
|
|
415
|
+
self.logger.warning(f"Version check on '{exe_base}' ended with "
|
|
416
|
+
f"code {proc.returncode}")
|
|
398
417
|
|
|
399
418
|
try:
|
|
400
419
|
version = self.parse_version(proc.stdout)
|
|
@@ -402,16 +421,16 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
402
421
|
raise NotImplementedError(f'{self.tool()}/{self.task()} does not implement '
|
|
403
422
|
'parse_version()')
|
|
404
423
|
except Exception as e:
|
|
405
|
-
self.
|
|
406
|
-
|
|
424
|
+
self.logger.error(f'{self.tool()}/{self.task()} failed to parse version string: '
|
|
425
|
+
f'{proc.stdout}')
|
|
407
426
|
raise e from None
|
|
408
427
|
|
|
409
|
-
self.
|
|
410
|
-
|
|
428
|
+
self.logger.info(f"Tool '{exe_base}' found with version '{version}' "
|
|
429
|
+
f"in directory '{exe_path}'")
|
|
411
430
|
|
|
412
431
|
return version
|
|
413
432
|
|
|
414
|
-
def check_exe_version(self, reported_version) -> bool:
|
|
433
|
+
def check_exe_version(self, reported_version: str) -> bool:
|
|
415
434
|
"""
|
|
416
435
|
Checks if the reported version of a tool satisfies the requirements
|
|
417
436
|
specified in the schema.
|
|
@@ -423,7 +442,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
423
442
|
bool: True if the version is acceptable, False otherwise.
|
|
424
443
|
"""
|
|
425
444
|
|
|
426
|
-
spec_sets = self.get('version')
|
|
445
|
+
spec_sets: Optional[List[str]] = self.get('version')
|
|
427
446
|
if not spec_sets:
|
|
428
447
|
# No requirement, so always true
|
|
429
448
|
return True
|
|
@@ -434,8 +453,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
434
453
|
for spec in split_specs:
|
|
435
454
|
match = re.match(Task.__parse_version_check, spec)
|
|
436
455
|
if match is None:
|
|
437
|
-
self.
|
|
438
|
-
|
|
456
|
+
self.logger.warning(f'Invalid version specifier {spec}. '
|
|
457
|
+
f'Defaulting to =={spec}.')
|
|
439
458
|
operator = '=='
|
|
440
459
|
spec_version = spec
|
|
441
460
|
else:
|
|
@@ -446,15 +465,15 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
446
465
|
try:
|
|
447
466
|
normalized_version = self.normalize_version(reported_version)
|
|
448
467
|
except Exception as e:
|
|
449
|
-
self.
|
|
450
|
-
|
|
468
|
+
self.logger.error(f'Unable to normalize version for {self.tool()}/{self.task()}: '
|
|
469
|
+
f'{reported_version}')
|
|
451
470
|
raise e from None
|
|
452
471
|
|
|
453
472
|
try:
|
|
454
473
|
version = Version(normalized_version)
|
|
455
474
|
except InvalidVersion:
|
|
456
|
-
self.
|
|
457
|
-
|
|
475
|
+
self.logger.error(f'Version {normalized_version} reported by '
|
|
476
|
+
f'{self.tool()}/{self.task()} does not match standard.')
|
|
458
477
|
return False
|
|
459
478
|
|
|
460
479
|
try:
|
|
@@ -462,29 +481,30 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
462
481
|
f'{op}{self.normalize_version(ver)}' for op, ver in specs_list]
|
|
463
482
|
normalized_specs = ','.join(normalized_spec_list)
|
|
464
483
|
except Exception as e:
|
|
465
|
-
self.
|
|
466
|
-
|
|
467
|
-
|
|
484
|
+
self.logger.error(f'Unable to normalize versions for '
|
|
485
|
+
f'{self.tool()}/{self.task()}: '
|
|
486
|
+
f'{",".join([f"{op}{ver}" for op, ver in specs_list])}')
|
|
468
487
|
raise e from None
|
|
469
488
|
|
|
470
489
|
try:
|
|
471
490
|
spec_set = SpecifierSet(normalized_specs)
|
|
472
491
|
except InvalidSpecifier:
|
|
473
|
-
self.
|
|
474
|
-
|
|
492
|
+
self.logger.error(f'Version specifier set {normalized_specs} '
|
|
493
|
+
'does not match standard.')
|
|
475
494
|
return False
|
|
476
495
|
|
|
477
496
|
if version in spec_set:
|
|
478
497
|
return True
|
|
479
498
|
|
|
480
499
|
allowedstr = '; '.join(spec_sets)
|
|
481
|
-
self.
|
|
482
|
-
|
|
483
|
-
self.
|
|
484
|
-
|
|
500
|
+
self.logger.error(f"Version check failed for {self.tool()}/{self.task()}. "
|
|
501
|
+
"Check installation.")
|
|
502
|
+
self.logger.error(f"Found version {reported_version}, "
|
|
503
|
+
f"did not satisfy any version specifier set {allowedstr}.")
|
|
485
504
|
return False
|
|
486
505
|
|
|
487
|
-
def get_runtime_environmental_variables(self, include_path=True)
|
|
506
|
+
def get_runtime_environmental_variables(self, include_path: bool = True) \
|
|
507
|
+
-> Dict[str, str]:
|
|
488
508
|
"""
|
|
489
509
|
Determines the environment variables needed for the task.
|
|
490
510
|
|
|
@@ -496,18 +516,18 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
496
516
|
"""
|
|
497
517
|
|
|
498
518
|
# Add global environmental vars
|
|
499
|
-
envvars = {}
|
|
519
|
+
envvars: Dict[str, str] = {}
|
|
500
520
|
for env in self.__schema_full.getkeys('option', 'env'):
|
|
501
521
|
envvars[env] = self.__schema_full.get('option', 'env', env)
|
|
502
522
|
|
|
503
523
|
# Add tool-specific license server vars
|
|
504
524
|
for lic_env in self.getkeys('licenseserver'):
|
|
505
|
-
license_file = self.get('licenseserver', lic_env)
|
|
525
|
+
license_file: List[str] = self.get('licenseserver', lic_env)
|
|
506
526
|
if license_file:
|
|
507
527
|
envvars[lic_env] = ':'.join(license_file)
|
|
508
528
|
|
|
509
529
|
if include_path:
|
|
510
|
-
path = self.find_files("path", missing_ok=True)
|
|
530
|
+
path: Optional[str] = self.find_files("path", missing_ok=True)
|
|
511
531
|
|
|
512
532
|
envvars["PATH"] = os.getenv("PATH", os.defpath)
|
|
513
533
|
|
|
@@ -526,7 +546,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
526
546
|
|
|
527
547
|
return envvars
|
|
528
548
|
|
|
529
|
-
def get_runtime_arguments(self):
|
|
549
|
+
def get_runtime_arguments(self) -> List[str]:
|
|
530
550
|
"""
|
|
531
551
|
Constructs the command-line arguments needed to run the task.
|
|
532
552
|
|
|
@@ -555,7 +575,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
555
575
|
|
|
556
576
|
cmdargs.extend(args)
|
|
557
577
|
except Exception as e:
|
|
558
|
-
self.
|
|
578
|
+
self.logger.error(f'Failed to get runtime options for {self.tool()}/{self.task()}')
|
|
559
579
|
raise e from None
|
|
560
580
|
|
|
561
581
|
# Cleanup args
|
|
@@ -563,7 +583,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
563
583
|
|
|
564
584
|
return cmdargs
|
|
565
585
|
|
|
566
|
-
def generate_replay_script(self, filepath, workdir, include_path=True)
|
|
586
|
+
def generate_replay_script(self, filepath: str, workdir: str, include_path: bool = True) \
|
|
587
|
+
-> None:
|
|
567
588
|
"""
|
|
568
589
|
Generates a shell script to replay the task's execution.
|
|
569
590
|
|
|
@@ -572,7 +593,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
572
593
|
workdir (str): The path to the run's working directory.
|
|
573
594
|
include_path (bool): If True, includes PATH information.
|
|
574
595
|
"""
|
|
575
|
-
replay_opts = {}
|
|
596
|
+
replay_opts: Dict[str, Optional[Union[Dict[str, str], str, int]]] = {}
|
|
576
597
|
replay_opts["work_dir"] = workdir
|
|
577
598
|
replay_opts["exports"] = self.get_runtime_environmental_variables(include_path=include_path)
|
|
578
599
|
|
|
@@ -582,7 +603,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
582
603
|
replay_opts["cfg_file"] = f"inputs/{self.__design_name}.pkg.json"
|
|
583
604
|
replay_opts["node_only"] = 0 if replay_opts["executable"] else 1
|
|
584
605
|
|
|
585
|
-
vswitch = self.get('vswitch')
|
|
606
|
+
vswitch: Optional[List[str]] = self.get('vswitch')
|
|
586
607
|
if vswitch:
|
|
587
608
|
replay_opts["version_flag"] = shlex.join(vswitch)
|
|
588
609
|
|
|
@@ -591,7 +612,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
591
612
|
file_test = re.compile(r'^[/\.]')
|
|
592
613
|
|
|
593
614
|
if replay_opts["executable"]:
|
|
594
|
-
format_cmd = [replay_opts["executable"]]
|
|
615
|
+
format_cmd: List[str] = [replay_opts["executable"]]
|
|
595
616
|
|
|
596
617
|
for cmdarg in self.get_runtime_arguments():
|
|
597
618
|
add_new_line = len(format_cmd) == 1
|
|
@@ -618,7 +639,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
618
639
|
|
|
619
640
|
os.chmod(filepath, 0o755)
|
|
620
641
|
|
|
621
|
-
def setup_work_directory(self, workdir, remove_exist=True):
|
|
642
|
+
def setup_work_directory(self, workdir: str, remove_exist: bool = True) -> None:
|
|
622
643
|
"""
|
|
623
644
|
Creates the runtime directories needed to execute a task.
|
|
624
645
|
|
|
@@ -637,7 +658,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
637
658
|
os.makedirs(os.path.join(workdir, 'outputs'), exist_ok=True)
|
|
638
659
|
os.makedirs(os.path.join(workdir, 'reports'), exist_ok=True)
|
|
639
660
|
|
|
640
|
-
def __write_yaml_manifest(self, fout, manifest):
|
|
661
|
+
def __write_yaml_manifest(self, fout: TextIO, manifest: BaseSchema) -> None:
|
|
641
662
|
"""Private helper to write a manifest in YAML format."""
|
|
642
663
|
class YamlIndentDumper(yaml.Dumper):
|
|
643
664
|
def increase_indent(self, flow=False, indentless=False):
|
|
@@ -646,7 +667,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
646
667
|
fout.write(yaml.dump(manifest.getdict(), Dumper=YamlIndentDumper,
|
|
647
668
|
default_flow_style=False))
|
|
648
669
|
|
|
649
|
-
def get_tcl_variables(self, manifest: BaseSchema = None) -> Dict[str, str]:
|
|
670
|
+
def get_tcl_variables(self, manifest: Optional[BaseSchema] = None) -> Dict[str, str]:
|
|
650
671
|
"""
|
|
651
672
|
Gets a dictionary of variables to define for the task in a Tcl manifest.
|
|
652
673
|
|
|
@@ -667,13 +688,14 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
667
688
|
"sc_designlib": NodeType.to_tcl(self.design_name, "str")
|
|
668
689
|
}
|
|
669
690
|
|
|
670
|
-
refdir = manifest.get("tool", self.tool(), "task", self.task(), "refdir",
|
|
691
|
+
refdir: Parameter = manifest.get("tool", self.tool(), "task", self.task(), "refdir",
|
|
692
|
+
field=None)
|
|
671
693
|
if refdir.get(step=self.__step, index=self.__index):
|
|
672
694
|
vars["sc_refdir"] = refdir.gettcl(step=self.__step, index=self.__index)
|
|
673
695
|
|
|
674
696
|
return vars
|
|
675
697
|
|
|
676
|
-
def __write_tcl_manifest(self, fout, manifest):
|
|
698
|
+
def __write_tcl_manifest(self, fout: TextIO, manifest: BaseSchema):
|
|
677
699
|
"""Private helper to write a manifest in Tcl format."""
|
|
678
700
|
template = utils.get_file_template('tcl/manifest.tcl.j2')
|
|
679
701
|
tcl_set_cmds = []
|
|
@@ -682,7 +704,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
682
704
|
if 'default' in key:
|
|
683
705
|
continue
|
|
684
706
|
|
|
685
|
-
param = manifest.get(*key, field=None)
|
|
707
|
+
param: Parameter = manifest.get(*key, field=None)
|
|
686
708
|
|
|
687
709
|
# Create a Tcl dict key string
|
|
688
710
|
keystr = ' '.join([NodeType.to_tcl(keypart, 'str') for keypart in key])
|
|
@@ -709,14 +731,14 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
709
731
|
fout.write(cmd + '\n')
|
|
710
732
|
fout.write('\n')
|
|
711
733
|
|
|
712
|
-
def __write_csv_manifest(self, fout, manifest):
|
|
734
|
+
def __write_csv_manifest(self, fout: TextIO, manifest: BaseSchema) -> None:
|
|
713
735
|
"""Private helper to write a manifest in CSV format."""
|
|
714
736
|
csvwriter = csv.writer(fout)
|
|
715
737
|
csvwriter.writerow(['Keypath', 'Value'])
|
|
716
738
|
|
|
717
739
|
for key in sorted(manifest.allkeys()):
|
|
718
740
|
keypath = ','.join(key)
|
|
719
|
-
param = manifest.get(*key, field=None)
|
|
741
|
+
param: Parameter = manifest.get(*key, field=None)
|
|
720
742
|
if param.get(field="pernode").is_never():
|
|
721
743
|
value = param.get()
|
|
722
744
|
else:
|
|
@@ -728,7 +750,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
728
750
|
else:
|
|
729
751
|
csvwriter.writerow([keypath, value])
|
|
730
752
|
|
|
731
|
-
def write_task_manifest(self, directory, backup=True):
|
|
753
|
+
def write_task_manifest(self, directory: str, backup: bool = True) -> None:
|
|
732
754
|
"""
|
|
733
755
|
Writes the manifest needed for the task in the format specified by the tool.
|
|
734
756
|
|
|
@@ -752,15 +774,14 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
752
774
|
if re.search(r'\.json(\.gz)?$', manifest_path):
|
|
753
775
|
schema.write_manifest(manifest_path)
|
|
754
776
|
else:
|
|
777
|
+
# Format-specific dumping
|
|
778
|
+
if manifest_path.endswith('.gz'):
|
|
779
|
+
fout = gzip.open(manifest_path, 'wt', encoding='UTF-8')
|
|
780
|
+
elif re.search(r'\.csv$', manifest_path):
|
|
781
|
+
fout = open(manifest_path, 'w', newline='')
|
|
782
|
+
else:
|
|
783
|
+
fout = open(manifest_path, 'w')
|
|
755
784
|
try:
|
|
756
|
-
# Format-specific dumping
|
|
757
|
-
if manifest_path.endswith('.gz'):
|
|
758
|
-
fout = gzip.open(manifest_path, 'wt', encoding='UTF-8')
|
|
759
|
-
elif re.search(r'\.csv$', manifest_path):
|
|
760
|
-
fout = open(manifest_path, 'w', newline='')
|
|
761
|
-
else:
|
|
762
|
-
fout = open(manifest_path, 'w')
|
|
763
|
-
|
|
764
785
|
if re.search(r'(\.yaml|\.yml)(\.gz)?$', manifest_path):
|
|
765
786
|
self.__write_yaml_manifest(fout, schema)
|
|
766
787
|
elif re.search(r'\.tcl(\.gz)?$', manifest_path):
|
|
@@ -772,7 +793,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
772
793
|
finally:
|
|
773
794
|
fout.close()
|
|
774
795
|
|
|
775
|
-
def __abspath_schema(self):
|
|
796
|
+
def __abspath_schema(self) -> "Project":
|
|
776
797
|
"""
|
|
777
798
|
Private helper to create a copy of the schema with all file/dir paths
|
|
778
799
|
converted to absolute paths.
|
|
@@ -781,7 +802,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
781
802
|
schema = root.copy()
|
|
782
803
|
|
|
783
804
|
for keypath in root.allkeys():
|
|
784
|
-
paramtype = schema.get(*keypath, field='type')
|
|
805
|
+
paramtype: str = schema.get(*keypath, field='type')
|
|
785
806
|
if 'file' not in paramtype and 'dir' not in paramtype:
|
|
786
807
|
continue
|
|
787
808
|
|
|
@@ -804,7 +825,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
804
825
|
|
|
805
826
|
return schema
|
|
806
827
|
|
|
807
|
-
def __get_io_file(self, io_type):
|
|
828
|
+
def __get_io_file(self, io_type: str) -> Tuple[str, bool]:
|
|
808
829
|
"""
|
|
809
830
|
Private helper to get the runtime destination for stdout or stderr.
|
|
810
831
|
|
|
@@ -826,7 +847,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
826
847
|
|
|
827
848
|
return io_file, io_log
|
|
828
849
|
|
|
829
|
-
def __terminate_exe(self, proc):
|
|
850
|
+
def __terminate_exe(self, proc: subprocess.Popen) -> None:
|
|
830
851
|
"""
|
|
831
852
|
Private helper to terminate a subprocess and its children.
|
|
832
853
|
|
|
@@ -834,7 +855,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
834
855
|
proc (subprocess.Process): The process to terminate.
|
|
835
856
|
"""
|
|
836
857
|
|
|
837
|
-
def terminate_process(pid, timeout=3):
|
|
858
|
+
def terminate_process(pid: int, timeout: int = 3) -> None:
|
|
838
859
|
"""Terminates a process and all its (grand+)children.
|
|
839
860
|
Based on https://psutil.readthedocs.io/en/latest/#psutil.wait_procs and
|
|
840
861
|
https://psutil.readthedocs.io/en/latest/#kill-process-tree."""
|
|
@@ -851,19 +872,24 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
851
872
|
for p in alive:
|
|
852
873
|
p.kill()
|
|
853
874
|
|
|
854
|
-
|
|
875
|
+
timeout = 5
|
|
855
876
|
|
|
856
|
-
terminate_process(proc.pid, timeout=
|
|
857
|
-
self.
|
|
877
|
+
terminate_process(proc.pid, timeout=timeout)
|
|
878
|
+
self.logger.info(f'Waiting for {self.tool()}/{self.task()} to exit...')
|
|
858
879
|
try:
|
|
859
|
-
proc.wait(timeout=
|
|
880
|
+
proc.wait(timeout=timeout)
|
|
860
881
|
except subprocess.TimeoutExpired:
|
|
861
882
|
if proc.poll() is None:
|
|
862
|
-
self.
|
|
863
|
-
|
|
864
|
-
terminate_process(proc.pid, timeout=
|
|
883
|
+
self.logger.warning(f'{self.tool()}/{self.task()} did not exit within '
|
|
884
|
+
f'{timeout} seconds. Terminating...')
|
|
885
|
+
terminate_process(proc.pid, timeout=timeout)
|
|
865
886
|
|
|
866
|
-
def run_task(self,
|
|
887
|
+
def run_task(self,
|
|
888
|
+
workdir: str,
|
|
889
|
+
quiet: bool,
|
|
890
|
+
breakpoint: bool,
|
|
891
|
+
nice: Optional[int],
|
|
892
|
+
timeout: Optional[int]) -> int:
|
|
867
893
|
"""
|
|
868
894
|
Executes the task's main process.
|
|
869
895
|
|
|
@@ -895,8 +921,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
895
921
|
stdout_file, is_stdout_log = self.__get_io_file("stdout")
|
|
896
922
|
stderr_file, is_stderr_log = self.__get_io_file("stderr")
|
|
897
923
|
|
|
898
|
-
stdout_print = self.
|
|
899
|
-
stderr_print = self.
|
|
924
|
+
stdout_print = self.logger.info
|
|
925
|
+
stderr_print = self.logger.error
|
|
900
926
|
|
|
901
927
|
def read_stdio(stdout_reader, stderr_reader):
|
|
902
928
|
"""Helper to read and print stdout/stderr streams."""
|
|
@@ -926,8 +952,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
926
952
|
contextlib.redirect_stdout(stdout_writer):
|
|
927
953
|
retcode = self.run()
|
|
928
954
|
except Exception as e:
|
|
929
|
-
self.
|
|
930
|
-
utils.print_traceback(self.
|
|
955
|
+
self.logger.error(f'Failed in run() for {self.tool()}/{self.task()}: {e}')
|
|
956
|
+
utils.print_traceback(self.logger, e)
|
|
931
957
|
raise e
|
|
932
958
|
finally:
|
|
933
959
|
with sc_open(stdout_file) as stdout_reader, \
|
|
@@ -948,17 +974,17 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
948
974
|
|
|
949
975
|
# Record tool options
|
|
950
976
|
self.schema_record.record_tool(
|
|
951
|
-
self.
|
|
977
|
+
self.step, self.index,
|
|
952
978
|
cmdlist, RecordTool.ARGS)
|
|
953
979
|
|
|
954
|
-
self.
|
|
980
|
+
self.logger.info(shlex.join([os.path.basename(exe), *cmdlist]))
|
|
955
981
|
|
|
956
982
|
if not pty and breakpoint:
|
|
957
983
|
breakpoint = False
|
|
958
984
|
|
|
959
985
|
if breakpoint and sys.platform in ('darwin', 'linux'):
|
|
960
986
|
# Use pty for interactive breakpoint sessions on POSIX systems
|
|
961
|
-
with open(f"{self.
|
|
987
|
+
with open(f"{self.step}.log", 'wb') as log_writer:
|
|
962
988
|
def read(fd):
|
|
963
989
|
data = os.read(fd, 1024)
|
|
964
990
|
log_writer.write(data)
|
|
@@ -991,8 +1017,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
991
1017
|
except Exception as e:
|
|
992
1018
|
raise TaskError(f"Unable to start {exe}: {str(e)}")
|
|
993
1019
|
|
|
994
|
-
|
|
995
|
-
MEMORY_WARN_LIMIT = 90
|
|
1020
|
+
memory_warn_limit = Task.__MEMORY_WARN_LIMIT
|
|
996
1021
|
try:
|
|
997
1022
|
while proc.poll() is None:
|
|
998
1023
|
# Monitor subprocess memory usage
|
|
@@ -1004,11 +1029,11 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1004
1029
|
max_mem_bytes = max(max_mem_bytes, proc_mem_bytes)
|
|
1005
1030
|
|
|
1006
1031
|
memory_usage = psutil.virtual_memory()
|
|
1007
|
-
if memory_usage.percent >
|
|
1008
|
-
self.
|
|
1032
|
+
if memory_usage.percent > memory_warn_limit:
|
|
1033
|
+
self.logger.warning(
|
|
1009
1034
|
'Current system memory usage is '
|
|
1010
1035
|
f'{memory_usage.percent:.1f}%')
|
|
1011
|
-
|
|
1036
|
+
memory_warn_limit = int(memory_usage.percent + 1)
|
|
1012
1037
|
except psutil.Error:
|
|
1013
1038
|
# Process may have already terminated or been killed.
|
|
1014
1039
|
# Retain existing memory usage statistics in this case.
|
|
@@ -1025,13 +1050,13 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1025
1050
|
if timeout is not None and duration > timeout:
|
|
1026
1051
|
raise TaskTimeout(timeout=duration)
|
|
1027
1052
|
|
|
1028
|
-
time.sleep(
|
|
1053
|
+
time.sleep(Task.__POLL_INTERVAL)
|
|
1029
1054
|
except KeyboardInterrupt:
|
|
1030
|
-
self.
|
|
1055
|
+
self.logger.info("Received ctrl-c.")
|
|
1031
1056
|
self.__terminate_exe(proc)
|
|
1032
1057
|
raise TaskError
|
|
1033
1058
|
except TaskTimeout as e:
|
|
1034
|
-
self.
|
|
1059
|
+
self.logger.error(f'Task timed out after {e.timeout:.1f} seconds')
|
|
1035
1060
|
self.__terminate_exe(proc)
|
|
1036
1061
|
raise e from None
|
|
1037
1062
|
|
|
@@ -1042,14 +1067,14 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1042
1067
|
|
|
1043
1068
|
# Record metrics
|
|
1044
1069
|
self.schema_record.record_tool(
|
|
1045
|
-
self.
|
|
1070
|
+
self.step, self.index,
|
|
1046
1071
|
retcode, RecordTool.EXITCODE)
|
|
1047
1072
|
|
|
1048
1073
|
self.schema_metric.record(
|
|
1049
|
-
self.
|
|
1074
|
+
self.step, self.index,
|
|
1050
1075
|
'exetime', time.time() - cpu_start, unit='s')
|
|
1051
1076
|
self.schema_metric.record(
|
|
1052
|
-
self.
|
|
1077
|
+
self.step, self.index,
|
|
1053
1078
|
'memory', max_mem_bytes, unit='B')
|
|
1054
1079
|
|
|
1055
1080
|
return retcode
|
|
@@ -1067,11 +1092,11 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1067
1092
|
self.__dict__ = state
|
|
1068
1093
|
self.__set_runtime(None)
|
|
1069
1094
|
|
|
1070
|
-
def get_output_files(self):
|
|
1095
|
+
def get_output_files(self) -> Set[str]:
|
|
1071
1096
|
"""Gets the set of output files defined for this task."""
|
|
1072
1097
|
return set(self.get("output"))
|
|
1073
1098
|
|
|
1074
|
-
def get_files_from_input_nodes(self):
|
|
1099
|
+
def get_files_from_input_nodes(self) -> Dict[str, List[Tuple[str, str]]]:
|
|
1075
1100
|
"""
|
|
1076
1101
|
Returns a dictionary of files from input nodes, mapped to the node
|
|
1077
1102
|
they originated from.
|
|
@@ -1084,7 +1109,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1084
1109
|
|
|
1085
1110
|
in_tool = self.schema_flow.get(in_step, in_index, "tool")
|
|
1086
1111
|
in_task = self.schema_flow.get(in_step, in_index, "task")
|
|
1087
|
-
task_obj = self.project.get("tool", in_tool, "task", in_task, field="schema")
|
|
1112
|
+
task_obj: Task = self.project.get("tool", in_tool, "task", in_task, field="schema")
|
|
1088
1113
|
|
|
1089
1114
|
if self.schema_record.get('status', step=in_step, index=in_index) == \
|
|
1090
1115
|
NodeStatus.SKIPPED:
|
|
@@ -1098,7 +1123,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1098
1123
|
|
|
1099
1124
|
return inputs
|
|
1100
1125
|
|
|
1101
|
-
def compute_input_file_node_name(self, filename, step, index):
|
|
1126
|
+
def compute_input_file_node_name(self, filename: str, step: str, index: str) -> str:
|
|
1102
1127
|
"""
|
|
1103
1128
|
Generates a unique name for an input file based on its originating node.
|
|
1104
1129
|
|
|
@@ -1119,7 +1144,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1119
1144
|
else:
|
|
1120
1145
|
return f'{filename}.{step}{index}'
|
|
1121
1146
|
|
|
1122
|
-
def add_parameter(self, name, type, help, defvalue=None, **kwargs):
|
|
1147
|
+
def add_parameter(self, name: str, type: str, help: str, defvalue=None, **kwargs) -> Parameter:
|
|
1123
1148
|
"""
|
|
1124
1149
|
Adds a custom parameter ('var') to the task definition.
|
|
1125
1150
|
|
|
@@ -1146,7 +1171,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1146
1171
|
# Task settings
|
|
1147
1172
|
###############################################################
|
|
1148
1173
|
def add_required_key(self, obj: Union[BaseSchema, str], *key: str,
|
|
1149
|
-
step: str = None, index: str = None):
|
|
1174
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None):
|
|
1150
1175
|
'''
|
|
1151
1176
|
Adds a required keypath to the task driver. If the key is valid relative to the task object
|
|
1152
1177
|
the key will be assumed as a task key.
|
|
@@ -1170,8 +1195,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1170
1195
|
|
|
1171
1196
|
return self.add("require", ",".join(key), step=step, index=index)
|
|
1172
1197
|
|
|
1173
|
-
def set_threads(self, max_threads: int = None,
|
|
1174
|
-
step: str = None, index: str = None,
|
|
1198
|
+
def set_threads(self, max_threads: Optional[int] = None,
|
|
1199
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1175
1200
|
clobber: bool = False):
|
|
1176
1201
|
"""
|
|
1177
1202
|
Sets the requested thread count for the task
|
|
@@ -1183,7 +1208,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1183
1208
|
clobber (bool): overwrite existing value
|
|
1184
1209
|
"""
|
|
1185
1210
|
if max_threads is None or max_threads <= 0:
|
|
1186
|
-
max_schema_threads = self.project.option.scheduler.get("maxthreads")
|
|
1211
|
+
max_schema_threads: Optional[int] = self.project.option.scheduler.get("maxthreads")
|
|
1187
1212
|
if max_schema_threads:
|
|
1188
1213
|
max_threads = max_schema_threads
|
|
1189
1214
|
else:
|
|
@@ -1191,14 +1216,15 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1191
1216
|
|
|
1192
1217
|
return self.set("threads", max_threads, step=step, index=index, clobber=clobber)
|
|
1193
1218
|
|
|
1194
|
-
def get_threads(self, step: str = None,
|
|
1219
|
+
def get_threads(self, step: Optional[str] = None,
|
|
1220
|
+
index: Optional[Union[str, int]] = None) -> int:
|
|
1195
1221
|
"""
|
|
1196
1222
|
Returns the number of threads requested.
|
|
1197
1223
|
"""
|
|
1198
1224
|
return self.get("threads", step=step, index=index)
|
|
1199
1225
|
|
|
1200
1226
|
def add_commandline_option(self, option: Union[List[str], str],
|
|
1201
|
-
step: str = None, index: str = None,
|
|
1227
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1202
1228
|
clobber: bool = False):
|
|
1203
1229
|
"""
|
|
1204
1230
|
Add to the command line options for the task
|
|
@@ -1213,14 +1239,17 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1213
1239
|
else:
|
|
1214
1240
|
return self.add("option", option, step=step, index=index)
|
|
1215
1241
|
|
|
1216
|
-
def get_commandline_options(self,
|
|
1242
|
+
def get_commandline_options(self,
|
|
1243
|
+
step: Optional[str] = None,
|
|
1244
|
+
index: Optional[Union[str, int]] = None) \
|
|
1245
|
+
-> List[str]:
|
|
1217
1246
|
"""
|
|
1218
1247
|
Returns the command line options specified
|
|
1219
1248
|
"""
|
|
1220
1249
|
return self.get("option", step=step, index=index)
|
|
1221
1250
|
|
|
1222
|
-
def add_input_file(self, file: str = None, ext: str = None,
|
|
1223
|
-
step: str = None, index: str = None,
|
|
1251
|
+
def add_input_file(self, file: Optional[str] = None, ext: Optional[str] = None,
|
|
1252
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1224
1253
|
clobber: bool = False):
|
|
1225
1254
|
"""
|
|
1226
1255
|
Add a required input file from the previous step in the flow.
|
|
@@ -1242,8 +1271,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1242
1271
|
else:
|
|
1243
1272
|
return self.add("input", file, step=step, index=index)
|
|
1244
1273
|
|
|
1245
|
-
def add_output_file(self, file: str = None, ext: str = None,
|
|
1246
|
-
step: str = None, index: str = None,
|
|
1274
|
+
def add_output_file(self, file: Optional[str] = None, ext: Optional[str] = None,
|
|
1275
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1247
1276
|
clobber: bool = False):
|
|
1248
1277
|
"""
|
|
1249
1278
|
Add an output file that this task will produce
|
|
@@ -1266,7 +1295,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1266
1295
|
return self.add("output", file, step=step, index=index)
|
|
1267
1296
|
|
|
1268
1297
|
def set_environmentalvariable(self, name: str, value: str,
|
|
1269
|
-
step: str = None,
|
|
1298
|
+
step: Optional[str] = None,
|
|
1299
|
+
index: Optional[Union[str, int]] = None,
|
|
1270
1300
|
clobber: bool = False):
|
|
1271
1301
|
'''Sets an environment variable for the tool's execution context.
|
|
1272
1302
|
|
|
@@ -1288,8 +1318,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1288
1318
|
'''
|
|
1289
1319
|
return self.set("env", name, value, step=step, index=index, clobber=clobber)
|
|
1290
1320
|
|
|
1291
|
-
def add_prescript(self, script: str, dataroot: str = None,
|
|
1292
|
-
step: str = None, index: str = None,
|
|
1321
|
+
def add_prescript(self, script: str, dataroot: Optional[str] = None,
|
|
1322
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1293
1323
|
clobber: bool = False):
|
|
1294
1324
|
'''Adds a script to be executed *before* the main tool command.
|
|
1295
1325
|
|
|
@@ -1316,8 +1346,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1316
1346
|
else:
|
|
1317
1347
|
return self.add("prescript", script, step=step, index=index)
|
|
1318
1348
|
|
|
1319
|
-
def add_postscript(self, script: str, dataroot: str = None,
|
|
1320
|
-
step: str = None, index: str = None,
|
|
1349
|
+
def add_postscript(self, script: str, dataroot: Optional[str] = None,
|
|
1350
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1321
1351
|
clobber: bool = False):
|
|
1322
1352
|
'''Adds a script to be executed *after* the main tool command.
|
|
1323
1353
|
|
|
@@ -1344,7 +1374,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1344
1374
|
else:
|
|
1345
1375
|
return self.add("postscript", script, step=step, index=index)
|
|
1346
1376
|
|
|
1347
|
-
def has_prescript(self,
|
|
1377
|
+
def has_prescript(self,
|
|
1378
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None) -> bool:
|
|
1348
1379
|
'''Checks if any pre-execution scripts are configured for the task.
|
|
1349
1380
|
|
|
1350
1381
|
Args:
|
|
@@ -1358,7 +1389,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1358
1389
|
return True
|
|
1359
1390
|
return False
|
|
1360
1391
|
|
|
1361
|
-
def has_postscript(self,
|
|
1392
|
+
def has_postscript(self,
|
|
1393
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None) -> bool:
|
|
1362
1394
|
'''Checks if any post-execution scripts are configured for the task.
|
|
1363
1395
|
|
|
1364
1396
|
Args:
|
|
@@ -1372,8 +1404,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1372
1404
|
return True
|
|
1373
1405
|
return False
|
|
1374
1406
|
|
|
1375
|
-
def set_refdir(self, dir: str, dataroot: str = None,
|
|
1376
|
-
step: str = None, index: str = None,
|
|
1407
|
+
def set_refdir(self, dir: str, dataroot: Optional[str] = None,
|
|
1408
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1377
1409
|
clobber: bool = False):
|
|
1378
1410
|
'''Sets the reference directory for tool scripts and auxiliary files.
|
|
1379
1411
|
|
|
@@ -1396,8 +1428,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1396
1428
|
with self.active_dataroot(self._get_active_dataroot(dataroot)):
|
|
1397
1429
|
return self.set("refdir", dir, step=step, index=index, clobber=clobber)
|
|
1398
1430
|
|
|
1399
|
-
def set_script(self, script: str, dataroot: str = ...,
|
|
1400
|
-
step: str = None, index: str = None,
|
|
1431
|
+
def set_script(self, script: str, dataroot: Optional[str] = ...,
|
|
1432
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1401
1433
|
clobber: bool = False):
|
|
1402
1434
|
'''Sets the main entry script for a script-based tool (e.g., a TCL script).
|
|
1403
1435
|
|
|
@@ -1418,7 +1450,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1418
1450
|
return self.set("script", script, step=step, index=index, clobber=clobber)
|
|
1419
1451
|
|
|
1420
1452
|
def add_regex(self, type: str, regex: str,
|
|
1421
|
-
step: str = None, index: str = None,
|
|
1453
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1422
1454
|
clobber: bool = False):
|
|
1423
1455
|
'''Adds a regular expression for parsing the tool's log file.
|
|
1424
1456
|
|
|
@@ -1443,8 +1475,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1443
1475
|
else:
|
|
1444
1476
|
return self.add("regex", type, regex, step=step, index=index)
|
|
1445
1477
|
|
|
1446
|
-
def set_logdestination(self, type: str, dest: str, suffix: str = None,
|
|
1447
|
-
step: str = None, index: str = None,
|
|
1478
|
+
def set_logdestination(self, type: str, dest: str, suffix: Optional[str] = None,
|
|
1479
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1448
1480
|
clobber: bool = False):
|
|
1449
1481
|
'''Configures the destination for log files.
|
|
1450
1482
|
|
|
@@ -1470,7 +1502,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1470
1502
|
rets.append(self.set(type, "suffix", suffix, step=step, index=index, clobber=clobber))
|
|
1471
1503
|
return rets
|
|
1472
1504
|
|
|
1473
|
-
def add_warningoff(self, type: str,
|
|
1505
|
+
def add_warningoff(self, type: str,
|
|
1506
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1507
|
+
clobber: bool = False):
|
|
1474
1508
|
'''Adds a warning message or code to be suppressed during log parsing.
|
|
1475
1509
|
|
|
1476
1510
|
Any warning that matches a regex in this list will be ignored by the
|
|
@@ -1496,8 +1530,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1496
1530
|
###############################################################
|
|
1497
1531
|
# Tool settings
|
|
1498
1532
|
###############################################################
|
|
1499
|
-
def set_exe(self, exe: str = None, vswitch:
|
|
1500
|
-
|
|
1533
|
+
def set_exe(self, exe: Optional[str] = None, vswitch: Optional[Union[str, List[str]]] = None,
|
|
1534
|
+
format: Optional[str] = None,
|
|
1535
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1501
1536
|
clobber: bool = False):
|
|
1502
1537
|
'''Sets the executable, version switch, and script format for a tool.
|
|
1503
1538
|
|
|
@@ -1522,18 +1557,18 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1522
1557
|
'''
|
|
1523
1558
|
rets = []
|
|
1524
1559
|
if exe:
|
|
1525
|
-
rets.append(self.set("exe", exe, clobber=clobber))
|
|
1560
|
+
rets.append(self.set("exe", exe, step=step, index=index, clobber=clobber))
|
|
1526
1561
|
if vswitch:
|
|
1527
|
-
switches = self.add_vswitch(vswitch, clobber=clobber)
|
|
1562
|
+
switches = self.add_vswitch(vswitch, step=step, index=index, clobber=clobber)
|
|
1528
1563
|
if not isinstance(switches, list):
|
|
1529
1564
|
switches = list(switches)
|
|
1530
1565
|
rets.extend(switches)
|
|
1531
1566
|
if format:
|
|
1532
|
-
rets.append(self.set("format", format, clobber=clobber))
|
|
1567
|
+
rets.append(self.set("format", format, step=step, index=index, clobber=clobber))
|
|
1533
1568
|
return rets
|
|
1534
1569
|
|
|
1535
|
-
def set_path(self, path: str, dataroot: str = None,
|
|
1536
|
-
step: str = None, index: str = None,
|
|
1570
|
+
def set_path(self, path: str, dataroot: Optional[str] = None,
|
|
1571
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1537
1572
|
clobber: bool = False):
|
|
1538
1573
|
'''Sets the directory path where the tool's executable is located.
|
|
1539
1574
|
|
|
@@ -1557,7 +1592,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1557
1592
|
with self.active_dataroot(self._get_active_dataroot(dataroot)):
|
|
1558
1593
|
return self.set("path", path, step=step, index=index, clobber=clobber)
|
|
1559
1594
|
|
|
1560
|
-
def add_version(self, version: str,
|
|
1595
|
+
def add_version(self, version: Union[List[str], str],
|
|
1596
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1597
|
+
clobber: bool = False):
|
|
1561
1598
|
'''Adds a supported version specifier for the tool.
|
|
1562
1599
|
|
|
1563
1600
|
SiliconCompiler checks the tool's actual version against these
|
|
@@ -1581,7 +1618,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1581
1618
|
else:
|
|
1582
1619
|
return self.add("version", version, step=step, index=index)
|
|
1583
1620
|
|
|
1584
|
-
def add_vswitch(self, switch: str,
|
|
1621
|
+
def add_vswitch(self, switch: Union[List[str], str],
|
|
1622
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1623
|
+
clobber: bool = False):
|
|
1585
1624
|
'''Adds the command-line switch used to print the tool's version.
|
|
1586
1625
|
|
|
1587
1626
|
This switch is passed to the executable to get its version string
|
|
@@ -1596,12 +1635,12 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1596
1635
|
The schema key that was set.
|
|
1597
1636
|
'''
|
|
1598
1637
|
if clobber:
|
|
1599
|
-
return self.set("vswitch", switch)
|
|
1638
|
+
return self.set("vswitch", switch, step=step, index=index)
|
|
1600
1639
|
else:
|
|
1601
|
-
return self.add("vswitch", switch)
|
|
1640
|
+
return self.add("vswitch", switch, step=step, index=index)
|
|
1602
1641
|
|
|
1603
1642
|
def add_licenseserver(self, name: str, server: str,
|
|
1604
|
-
step: str = None, index: str = None,
|
|
1643
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1605
1644
|
clobber: bool = False):
|
|
1606
1645
|
'''Configures a license server connection for the tool.
|
|
1607
1646
|
|
|
@@ -1626,7 +1665,9 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1626
1665
|
else:
|
|
1627
1666
|
return self.add("licenseserver", name, server, step=step, index=index)
|
|
1628
1667
|
|
|
1629
|
-
def add_sbom(self, version: str, sbom: str, dataroot: str = None,
|
|
1668
|
+
def add_sbom(self, version: str, sbom: Union[str, List[str]], dataroot: Optional[str] = None,
|
|
1669
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1670
|
+
clobber: bool = False):
|
|
1630
1671
|
'''Adds a Software Bill of Materials (SBOM) file for a tool version.
|
|
1631
1672
|
|
|
1632
1673
|
Associates a specific tool version with its corresponding SBOM file,
|
|
@@ -1645,11 +1686,13 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1645
1686
|
'''
|
|
1646
1687
|
with self.active_dataroot(self._get_active_dataroot(dataroot)):
|
|
1647
1688
|
if clobber:
|
|
1648
|
-
return self.set("sbom", version, sbom)
|
|
1689
|
+
return self.set("sbom", version, sbom, step=step, index=index)
|
|
1649
1690
|
else:
|
|
1650
|
-
return self.add("sbom", version, sbom)
|
|
1691
|
+
return self.add("sbom", version, sbom, step=step, index=index)
|
|
1651
1692
|
|
|
1652
|
-
def record_metric(self, metric, value
|
|
1693
|
+
def record_metric(self, metric: str, value: Union[int, float],
|
|
1694
|
+
source_file: Optional[str] = None, source_unit: Optional[str] = None,
|
|
1695
|
+
quiet: bool = False):
|
|
1653
1696
|
'''
|
|
1654
1697
|
Records a metric and associates the source file with it.
|
|
1655
1698
|
|
|
@@ -1671,11 +1714,11 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1671
1714
|
self.logger.warning(f"{metric} is not a valid metric")
|
|
1672
1715
|
return
|
|
1673
1716
|
|
|
1674
|
-
self.schema_metric.record(self.
|
|
1717
|
+
self.schema_metric.record(self.step, self.index, metric, value, unit=source_unit)
|
|
1675
1718
|
if source_file:
|
|
1676
1719
|
self.add("report", metric, source_file)
|
|
1677
1720
|
|
|
1678
|
-
def get_fileset_file_keys(self, filetype: str) -> List[Tuple[NamedSchema, Tuple[str]]]:
|
|
1721
|
+
def get_fileset_file_keys(self, filetype: str) -> List[Tuple[NamedSchema, Tuple[str, ...]]]:
|
|
1679
1722
|
"""
|
|
1680
1723
|
Collect a set of keys for a particular filetype.
|
|
1681
1724
|
|
|
@@ -1698,60 +1741,68 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1698
1741
|
###############################################################
|
|
1699
1742
|
# Schema
|
|
1700
1743
|
###############################################################
|
|
1701
|
-
def get(self, *keypath
|
|
1702
|
-
|
|
1744
|
+
def get(self, *keypath: str, field: Optional[str] = 'value',
|
|
1745
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None):
|
|
1746
|
+
if step is None:
|
|
1703
1747
|
step = self.__step
|
|
1704
|
-
if
|
|
1748
|
+
if index is None:
|
|
1705
1749
|
index = self.__index
|
|
1706
1750
|
return super().get(*keypath, field=field, step=step, index=index)
|
|
1707
1751
|
|
|
1708
|
-
def set(self, *args, field
|
|
1709
|
-
|
|
1752
|
+
def set(self, *args, field: str = 'value',
|
|
1753
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None,
|
|
1754
|
+
clobber: bool = True):
|
|
1755
|
+
if step is None:
|
|
1710
1756
|
step = self.__step
|
|
1711
|
-
if
|
|
1757
|
+
if index is None:
|
|
1712
1758
|
index = self.__index
|
|
1713
1759
|
return super().set(*args, field=field, clobber=clobber, step=step, index=index)
|
|
1714
1760
|
|
|
1715
|
-
def add(self, *args, field
|
|
1716
|
-
|
|
1761
|
+
def add(self, *args, field: str = 'value',
|
|
1762
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None):
|
|
1763
|
+
if step is None:
|
|
1717
1764
|
step = self.__step
|
|
1718
|
-
if
|
|
1765
|
+
if index is None:
|
|
1719
1766
|
index = self.__index
|
|
1720
1767
|
return super().add(*args, field=field, step=step, index=index)
|
|
1721
1768
|
|
|
1722
|
-
def unset(self, *args
|
|
1723
|
-
|
|
1769
|
+
def unset(self, *args: str,
|
|
1770
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None):
|
|
1771
|
+
if step is None:
|
|
1724
1772
|
step = self.__step
|
|
1725
|
-
if
|
|
1773
|
+
if index is None:
|
|
1726
1774
|
index = self.__index
|
|
1727
1775
|
return super().unset(*args, step=step, index=index)
|
|
1728
1776
|
|
|
1729
|
-
def find_files(self, *keypath, missing_ok=False,
|
|
1730
|
-
|
|
1777
|
+
def find_files(self, *keypath: str, missing_ok: bool = False,
|
|
1778
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None):
|
|
1779
|
+
if step is None:
|
|
1731
1780
|
step = self.__step
|
|
1732
|
-
if
|
|
1781
|
+
if index is None:
|
|
1733
1782
|
index = self.__index
|
|
1734
1783
|
return super().find_files(*keypath, missing_ok=missing_ok,
|
|
1735
1784
|
step=step, index=index)
|
|
1736
1785
|
|
|
1737
|
-
def _find_files_search_paths(self,
|
|
1738
|
-
|
|
1739
|
-
|
|
1786
|
+
def _find_files_search_paths(self, key: str,
|
|
1787
|
+
step: Optional[str],
|
|
1788
|
+
index: Optional[Union[int, str]]) -> List[str]:
|
|
1789
|
+
search_paths = super()._find_files_search_paths(key, step, index)
|
|
1790
|
+
if key == "script":
|
|
1740
1791
|
search_paths.extend(self.find_files("refdir", step=step, index=index))
|
|
1741
|
-
elif
|
|
1792
|
+
elif key == "input":
|
|
1742
1793
|
search_paths.append(os.path.join(
|
|
1743
1794
|
paths.workdir(self._parent(root=True), step=step, index=index), "inputs"))
|
|
1744
|
-
elif
|
|
1795
|
+
elif key == "report":
|
|
1745
1796
|
search_paths.append(os.path.join(
|
|
1746
|
-
paths.workdir(self._parent(root=True), step=step, index=index), "
|
|
1747
|
-
elif
|
|
1797
|
+
paths.workdir(self._parent(root=True), step=step, index=index), "reports"))
|
|
1798
|
+
elif key == "output":
|
|
1748
1799
|
search_paths.append(os.path.join(
|
|
1749
1800
|
paths.workdir(self._parent(root=True), step=step, index=index), "outputs"))
|
|
1750
1801
|
return search_paths
|
|
1751
1802
|
|
|
1752
1803
|
def _generate_doc(self, doc,
|
|
1753
1804
|
ref_root: str = "",
|
|
1754
|
-
key_offset: Tuple[str] = None,
|
|
1805
|
+
key_offset: Optional[Tuple[str, ...]] = None,
|
|
1755
1806
|
detailed: bool = True):
|
|
1756
1807
|
from .schema.docs.utils import build_section, strong, KeyPath, code, para, \
|
|
1757
1808
|
build_table, build_schema_value_table
|
|
@@ -1760,7 +1811,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1760
1811
|
docs = []
|
|
1761
1812
|
|
|
1762
1813
|
if not key_offset:
|
|
1763
|
-
key_offset =
|
|
1814
|
+
key_offset = tuple()
|
|
1764
1815
|
|
|
1765
1816
|
# Show dataroot
|
|
1766
1817
|
dataroot = PathSchema._generate_doc(self, doc, ref_root=ref_root, key_offset=key_offset)
|
|
@@ -1826,8 +1877,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1826
1877
|
params[key] = self.get(*key, field=None)
|
|
1827
1878
|
|
|
1828
1879
|
with KeyPath.fallback(...):
|
|
1829
|
-
table = build_schema_value_table(params, "", key_offset + list(self._keypath),
|
|
1830
|
-
trim_prefix=key_offset + list(self._keypath))
|
|
1880
|
+
table = build_schema_value_table(params, "", list(key_offset) + list(self._keypath),
|
|
1881
|
+
trim_prefix=list(key_offset) + list(self._keypath))
|
|
1831
1882
|
setup_info = build_section("Configuration", f"{ref_root}-config")
|
|
1832
1883
|
setup_info += table
|
|
1833
1884
|
docs.append(setup_info)
|
|
@@ -1854,55 +1905,55 @@ class Task(NamedSchema, PathSchema, DocsSchema):
|
|
|
1854
1905
|
node.setup()
|
|
1855
1906
|
return node.task
|
|
1856
1907
|
|
|
1857
|
-
def parse_version(self, stdout):
|
|
1908
|
+
def parse_version(self, stdout: str) -> str:
|
|
1858
1909
|
"""
|
|
1859
1910
|
Parses the tool's version from its stdout. Must be implemented by subclasses.
|
|
1860
1911
|
"""
|
|
1861
1912
|
raise NotImplementedError("must be implemented by the implementation class")
|
|
1862
1913
|
|
|
1863
|
-
def normalize_version(self, version):
|
|
1914
|
+
def normalize_version(self, version: str) -> str:
|
|
1864
1915
|
"""
|
|
1865
1916
|
Normalizes a version string to a standard format. Can be overridden.
|
|
1866
1917
|
"""
|
|
1867
1918
|
return version
|
|
1868
1919
|
|
|
1869
|
-
def setup(self):
|
|
1920
|
+
def setup(self) -> None:
|
|
1870
1921
|
"""
|
|
1871
1922
|
A hook for setting up the task before execution. Can be overridden.
|
|
1872
1923
|
"""
|
|
1873
1924
|
pass
|
|
1874
1925
|
|
|
1875
|
-
def select_input_nodes(self):
|
|
1926
|
+
def select_input_nodes(self) -> List[Tuple[str, str]]:
|
|
1876
1927
|
"""
|
|
1877
1928
|
Determines which preceding nodes are inputs to this task.
|
|
1878
1929
|
"""
|
|
1879
1930
|
return self.schema_flowruntime.get_node_inputs(
|
|
1880
|
-
self.
|
|
1931
|
+
self.step, self.index, record=self.schema_record)
|
|
1881
1932
|
|
|
1882
|
-
def pre_process(self):
|
|
1933
|
+
def pre_process(self) -> None:
|
|
1883
1934
|
"""
|
|
1884
1935
|
A hook for pre-processing before the main tool execution. Can be overridden.
|
|
1885
1936
|
"""
|
|
1886
1937
|
pass
|
|
1887
1938
|
|
|
1888
|
-
def runtime_options(self):
|
|
1939
|
+
def runtime_options(self) -> List[Union[int, str, Path]]:
|
|
1889
1940
|
"""
|
|
1890
1941
|
Constructs the default runtime options for the task. Can be extended.
|
|
1891
1942
|
"""
|
|
1892
|
-
cmdargs = []
|
|
1943
|
+
cmdargs: List[Union[int, str, Path]] = []
|
|
1893
1944
|
cmdargs.extend(self.get("option"))
|
|
1894
|
-
script = self.find_files('script', missing_ok=True)
|
|
1945
|
+
script: List[str] = self.find_files('script', missing_ok=True)
|
|
1895
1946
|
if script:
|
|
1896
1947
|
cmdargs.extend(script)
|
|
1897
1948
|
return cmdargs
|
|
1898
1949
|
|
|
1899
|
-
def run(self):
|
|
1950
|
+
def run(self) -> int:
|
|
1900
1951
|
"""
|
|
1901
1952
|
The main execution logic for Python-based tasks. Must be implemented.
|
|
1902
1953
|
"""
|
|
1903
1954
|
raise NotImplementedError("must be implemented by the implementation class")
|
|
1904
1955
|
|
|
1905
|
-
def post_process(self):
|
|
1956
|
+
def post_process(self) -> None:
|
|
1906
1957
|
"""
|
|
1907
1958
|
A hook for post-processing after the main tool execution. Can be overridden.
|
|
1908
1959
|
"""
|
|
@@ -1932,7 +1983,7 @@ class ShowTask(Task):
|
|
|
1932
1983
|
self.add_parameter("showexit", "bool", "exit after opening", defvalue=False)
|
|
1933
1984
|
|
|
1934
1985
|
@classmethod
|
|
1935
|
-
def __check_task(cls, task):
|
|
1986
|
+
def __check_task(cls, task: Optional[Type["ShowTask"]]) -> bool:
|
|
1936
1987
|
"""
|
|
1937
1988
|
Private helper to validate if a task is a valid ShowTask or ScreenshotTask.
|
|
1938
1989
|
"""
|
|
@@ -1940,7 +1991,7 @@ class ShowTask(Task):
|
|
|
1940
1991
|
raise TypeError("class must be ShowTask or ScreenshotTask")
|
|
1941
1992
|
|
|
1942
1993
|
if task is None:
|
|
1943
|
-
return
|
|
1994
|
+
return False
|
|
1944
1995
|
|
|
1945
1996
|
if cls is ShowTask:
|
|
1946
1997
|
check, task_filter = ShowTask, ScreenshotTask
|
|
@@ -1955,7 +2006,7 @@ class ShowTask(Task):
|
|
|
1955
2006
|
return True
|
|
1956
2007
|
|
|
1957
2008
|
@classmethod
|
|
1958
|
-
def register_task(cls, task):
|
|
2009
|
+
def register_task(cls, task: Optional[Type["ShowTask"]]) -> None:
|
|
1959
2010
|
"""
|
|
1960
2011
|
Registers a new show task class for dynamic discovery.
|
|
1961
2012
|
|
|
@@ -1972,7 +2023,7 @@ class ShowTask(Task):
|
|
|
1972
2023
|
cls.__TASKS.setdefault(cls, set()).add(task)
|
|
1973
2024
|
|
|
1974
2025
|
@classmethod
|
|
1975
|
-
def __populate_tasks(cls):
|
|
2026
|
+
def __populate_tasks(cls) -> None:
|
|
1976
2027
|
"""
|
|
1977
2028
|
Private helper to discover and populate all available show/screenshot tasks.
|
|
1978
2029
|
|
|
@@ -1981,7 +2032,7 @@ class ShowTask(Task):
|
|
|
1981
2032
|
"""
|
|
1982
2033
|
cls.__check_task(None)
|
|
1983
2034
|
|
|
1984
|
-
def recurse(searchcls):
|
|
2035
|
+
def recurse(searchcls: Type["ShowTask"]):
|
|
1985
2036
|
subclss = set()
|
|
1986
2037
|
if not cls.__check_task(searchcls):
|
|
1987
2038
|
return subclss
|
|
@@ -2005,7 +2056,7 @@ class ShowTask(Task):
|
|
|
2005
2056
|
ShowTask.__TASKS.setdefault(cls, set()).update(classes)
|
|
2006
2057
|
|
|
2007
2058
|
@classmethod
|
|
2008
|
-
def get_task(cls, ext):
|
|
2059
|
+
def get_task(cls, ext: Optional[str]) -> Union[Optional["ShowTask"], Set[Type["ShowTask"]]]:
|
|
2009
2060
|
"""
|
|
2010
2061
|
Retrieves a suitable show task instance for a given file extension.
|
|
2011
2062
|
|
|
@@ -2040,11 +2091,11 @@ class ShowTask(Task):
|
|
|
2040
2091
|
|
|
2041
2092
|
return None
|
|
2042
2093
|
|
|
2043
|
-
def task(self):
|
|
2094
|
+
def task(self) -> str:
|
|
2044
2095
|
"""Returns the name of this task."""
|
|
2045
2096
|
return "show"
|
|
2046
2097
|
|
|
2047
|
-
def setup(self):
|
|
2098
|
+
def setup(self) -> None:
|
|
2048
2099
|
"""Sets up the parameters and requirements for the show task."""
|
|
2049
2100
|
super().setup()
|
|
2050
2101
|
|
|
@@ -2070,7 +2121,7 @@ class ShowTask(Task):
|
|
|
2070
2121
|
raise NotImplementedError(
|
|
2071
2122
|
"get_supported_show_extentions must be implemented by the child class")
|
|
2072
2123
|
|
|
2073
|
-
def _set_filetype(self):
|
|
2124
|
+
def _set_filetype(self) -> None:
|
|
2074
2125
|
"""
|
|
2075
2126
|
Private helper to determine and set the 'showfiletype' parameter based
|
|
2076
2127
|
on the provided 'showfilepath' or available input files.
|
|
@@ -2097,24 +2148,28 @@ class ShowTask(Task):
|
|
|
2097
2148
|
ext = utils.get_file_ext(file)
|
|
2098
2149
|
set_file(file, ext)
|
|
2099
2150
|
|
|
2100
|
-
def set_showfilepath(self, path: str,
|
|
2151
|
+
def set_showfilepath(self, path: str,
|
|
2152
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None):
|
|
2101
2153
|
"""Sets the path to the file to be displayed."""
|
|
2102
2154
|
return self.set("var", "showfilepath", path, step=step, index=index)
|
|
2103
2155
|
|
|
2104
|
-
def set_showfiletype(self, file_type: str,
|
|
2156
|
+
def set_showfiletype(self, file_type: str,
|
|
2157
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None):
|
|
2105
2158
|
"""Sets the type of the file to be displayed."""
|
|
2106
2159
|
return self.set("var", "showfiletype", file_type, step=step, index=index)
|
|
2107
2160
|
|
|
2108
|
-
def set_showexit(self, value: bool,
|
|
2161
|
+
def set_showexit(self, value: bool,
|
|
2162
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None):
|
|
2109
2163
|
"""Sets whether the viewer application should exit after opening the file."""
|
|
2110
2164
|
return self.set("var", "showexit", value, step=step, index=index)
|
|
2111
2165
|
|
|
2112
|
-
def set_shownode(self, jobname: str = None,
|
|
2113
|
-
|
|
2166
|
+
def set_shownode(self, jobname: Optional[str] = None,
|
|
2167
|
+
nodestep: Optional[str] = None, nodeindex: Optional[Union[str, int]] = None,
|
|
2168
|
+
step: Optional[str] = None, index: Optional[Union[str, int]] = None):
|
|
2114
2169
|
"""Sets the source node information for the file being displayed."""
|
|
2115
2170
|
return self.set("var", "shownode", (jobname, nodestep, nodeindex), step=step, index=index)
|
|
2116
2171
|
|
|
2117
|
-
def get_tcl_variables(self, manifest=None):
|
|
2172
|
+
def get_tcl_variables(self, manifest: Optional[BaseSchema] = None) -> Dict[str, str]:
|
|
2118
2173
|
"""
|
|
2119
2174
|
Gets Tcl variables for the task, ensuring 'sc_do_screenshot' is false
|
|
2120
2175
|
for regular show tasks.
|
|
@@ -2133,11 +2188,11 @@ class ScreenshotTask(ShowTask):
|
|
|
2133
2188
|
sets the 'showexit' parameter to True.
|
|
2134
2189
|
"""
|
|
2135
2190
|
|
|
2136
|
-
def task(self):
|
|
2191
|
+
def task(self) -> str:
|
|
2137
2192
|
"""Returns the name of this task."""
|
|
2138
2193
|
return "screenshot"
|
|
2139
2194
|
|
|
2140
|
-
def setup(self):
|
|
2195
|
+
def setup(self) -> None:
|
|
2141
2196
|
"""
|
|
2142
2197
|
Sets up the screenshot task, ensuring that the viewer will exit
|
|
2143
2198
|
after the screenshot is taken.
|
|
@@ -2146,7 +2201,7 @@ class ScreenshotTask(ShowTask):
|
|
|
2146
2201
|
# Ensure the viewer exits after taking the screenshot
|
|
2147
2202
|
self.set_showexit(True)
|
|
2148
2203
|
|
|
2149
|
-
def get_tcl_variables(self, manifest=None):
|
|
2204
|
+
def get_tcl_variables(self, manifest: Optional[BaseSchema] = None) -> Dict[str, str]:
|
|
2150
2205
|
"""
|
|
2151
2206
|
Gets Tcl variables for the task, setting 'sc_do_screenshot' to true.
|
|
2152
2207
|
"""
|