siliconcompiler 0.29.4__py3-none-any.whl → 0.31.0__py3-none-any.whl

siliconcompiler/scheduler/__init__.py

@@ -1,5 +1,4 @@
 import contextlib
-import copy
 import distro
 import getpass
 import multiprocessing
@@ -27,7 +26,7 @@ from siliconcompiler.scheduler import slurm
 from siliconcompiler.scheduler import docker_runner
 from siliconcompiler import NodeStatus, SiliconCompilerError
 from siliconcompiler.flowgraph import _get_flowgraph_nodes, _get_flowgraph_execution_order, \
-    _get_pruned_node_inputs, _get_flowgraph_node_inputs, _get_flowgraph_entry_nodes, \
+    _get_pruned_node_inputs, _get_flowgraph_entry_nodes, \
     _unreachable_steps_to_execute, _nodes_to_execute, \
     get_nodes_from, nodes_to_execute, _check_flowgraph
 from siliconcompiler.tools._common import input_file_node_name
@@ -99,13 +98,6 @@ def run(chip):
     _reset_flow_nodes(chip, flow, nodes_to_execute(chip, flow))
     __record_packages(chip)
 
-    # Save current environment
-    environment = copy.deepcopy(os.environ)
-    # Set env variables
-    for envvar in chip.getkeys('option', 'env'):
-        val = chip.get('option', 'env', envvar)
-        os.environ[envvar] = val
-
     if chip.get('option', 'remote'):
         client = Client(chip)
         client.run()
@@ -113,23 +105,18 @@ def run(chip):
         _local_process(chip, flow)
 
     # Merge cfgs from last executed tasks, and write out a final manifest.
-    _finalize_run(chip, environment)
+    _finalize_run(chip)
 
 
 ###########################################################################
-def _finalize_run(chip, environment):
+def _finalize_run(chip):
     '''
     Helper function to finalize a job run after it completes:
-    * Restore any environment variable changes made during the run.
     * Clear any -arg_step/-arg_index values in case only one node was run.
     * Store this run in the Schema's 'history' field.
     * Write out a final JSON manifest containing the full results and history.
     '''
 
-    # Restore environment
-    os.environ.clear()
-    os.environ.update(environment)
-
     # Clear scratchpad args since these are checked on run() entry
     chip.set('arg', 'step', None, clobber=True)
     chip.set('arg', 'index', None, clobber=True)
@@ -502,10 +489,13 @@ def _haltstep(chip, flow, step, index, log=True):
 def _setupnode(chip, flow, step, index, replay):
     _hash_files(chip, step, index, setup=True)
 
+    # Select the inputs to this node
+    _select_inputs(chip, step, index)
+
     # Write manifest prior to step running into inputs
     chip.write_manifest(f'inputs/{chip.get("design")}.pkg.json')
 
-    _select_inputs(chip, step, index)
+    # Forward data
     _copy_previous_steps_output_data(chip, step, index, replay)
 
     # Check manifest
@@ -541,7 +531,7 @@ def _setup_workdir(chip, step, index, replay):
     return workdir
 
 
-def _select_inputs(chip, step, index):
+def _select_inputs(chip, step, index, trial=False):
 
     flow = chip.get('option', 'flow')
     tool, _ = get_tool_task(chip, step, index, flow)
@@ -551,15 +541,24 @@ def _select_inputs(chip, step, index):
                             '_select_inputs',
                             None)
     if select_inputs:
+        log_handlers = None
+        if trial:
+            log_handlers = chip.logger.handlers.copy()
+            chip.logger.handlers.clear()
         sel_inputs = select_inputs(chip, step, index)
+        if log_handlers:
+            chip.logger.handlers = log_handlers
     else:
-        sel_inputs = _get_flowgraph_node_inputs(chip, flow, (step, index))
+        sel_inputs = _get_pruned_node_inputs(chip, flow, (step, index))
 
     if (step, index) not in _get_flowgraph_entry_nodes(chip, flow) and not sel_inputs:
         chip.logger.error(f'No inputs selected after running {tool}')
         _haltstep(chip, flow, step, index)
 
-    chip.set('record', 'inputnode', sel_inputs, step=step, index=index)
+    if not trial:
+        chip.set('record', 'inputnode', sel_inputs, step=step, index=index)
+
+    return sel_inputs
 
 
 def copy_output_file(chip, outfile, folder='inputs'):
@@ -608,8 +607,14 @@ def _copy_previous_steps_output_data(chip, step, index, replay):
         # configuration into inputs/{design}.pkg.json earlier in _runstep.
         if not replay:
             in_workdir = chip.getworkdir(step=in_step, index=in_index)
+            output_dir = os.path.join(in_workdir, "outputs")
+
+            if not os.path.isdir(output_dir):
+                chip.logger.error(
+                    f'Unable to locate outputs directory for {in_step}{in_index}: {output_dir}')
+                _haltstep(chip, flow, step, index)
 
-            for outfile in os.scandir(f"{in_workdir}/outputs"):
+            for outfile in os.scandir(output_dir):
                 new_name = input_file_node_name(outfile.name, in_step, in_index)
                 if strict:
                     if outfile.name not in in_files and new_name not in in_files:
@@ -649,13 +654,36 @@ def _getexe(chip, tool, step, index):
     syspath = os.getenv('PATH', os.defpath)
     if path:
         # Prepend 'path' schema var to system path
-        syspath = utils._resolve_env_vars(chip, path) + os.pathsep + syspath
+        syspath = utils._resolve_env_vars(chip, path, step, index) + os.pathsep + syspath
 
     fullexe = shutil.which(exe, path=syspath)
 
     return fullexe
 
 
+def _get_run_env_vars(chip, tool, task, step, index, include_path):
+    envvars = utils.get_env_vars(chip, step, index)
+    for item in chip.getkeys('tool', tool, 'licenseserver'):
+        license_file = chip.get('tool', tool, 'licenseserver', item, step=step, index=index)
+        if license_file:
+            envvars[item] = ':'.join(license_file)
+
+    if include_path:
+        path = chip.get('tool', tool, 'path', step=step, index=index)
+        if path:
+            envvars['PATH'] = path + os.pathsep + os.environ['PATH']
+        else:
+            envvars['PATH'] = os.environ['PATH']
+
+        # Forward additional variables
+        for var in ('LD_LIBRARY_PATH',):
+            val = os.getenv(var, None)
+            if val:
+                envvars[var] = val
+
+    return envvars
+
+
 #######################################
 def _makecmd(chip, tool, task, step, index, script_name='replay.sh', include_path=True):
     '''
@@ -707,32 +735,6 @@ def _makecmd(chip, tool, task, step, index, script_name='replay.sh', include_pat
             chip.logger.error(f'Failed to get runtime options for {tool}/{task}')
             raise e
 
-    envvars = {}
-    for key in chip.getkeys('option', 'env'):
-        envvars[key] = chip.get('option', 'env', key)
-    for item in chip.getkeys('tool', tool, 'licenseserver'):
-        license_file = chip.get('tool', tool, 'licenseserver', item, step=step, index=index)
-        if license_file:
-            envvars[item] = ':'.join(license_file)
-
-    if include_path:
-        path = chip.get('tool', tool, 'path', step=step, index=index)
-        if path:
-            envvars['PATH'] = path + os.pathsep + os.environ['PATH']
-        else:
-            envvars['PATH'] = os.environ['PATH']
-
-        # Forward additional variables
-        for var in ('LD_LIBRARY_PATH',):
-            val = os.getenv(var, None)
-            if val:
-                envvars[var] = val
-
-    for key in chip.getkeys('tool', tool, 'task', task, 'env'):
-        val = chip.get('tool', tool, 'task', task, 'env', key, step=step, index=index)
-        if val:
-            envvars[key] = val
-
     # Separate variables to be able to display nice name of executable
     cmd = os.path.basename(cmdlist[0])
     cmd_args = cmdlist[1:]
@@ -747,19 +749,23 @@ def _makecmd(chip, tool, task, step, index, script_name='replay.sh', include_pat
 
     # create replay file
     with open(script_name, 'w') as f:
-        print('#!/usr/bin/env bash', file=f)
-
-        envvar_cmd = 'export'
-        for key, val in envvars.items():
-            print(f'{envvar_cmd} {key}="{val}"', file=f)
-
         # Ensure execution runs from the same directory
+        replay_opts = {}
         work_dir = chip.getworkdir(step=step, index=index)
         if chip._relative_path:
             work_dir = os.path.relpath(work_dir, chip._relative_path)
-        print(f'cd {work_dir}', file=f)
-
-        format_cmd = [chip.get('tool', tool, 'exe')]
+        replay_opts["work_dir"] = work_dir
+        replay_opts["exports"] = _get_run_env_vars(chip,
+                                                   tool, task,
+                                                   step, index,
+                                                   include_path=include_path)
+        replay_opts["executable"] = chip.get('tool', tool, 'exe')
+
+        vswitch = chip.get('tool', tool, 'vswitch')
+        if vswitch:
+            replay_opts["version_flag"] = " ".join(vswitch)
+
+        format_cmd = [replay_opts["executable"]]
         arg_test = re.compile(r'^[-+]')
         file_test = re.compile(r'^[/]')
         for cmdarg in cmd_args:
@@ -775,7 +781,11 @@ def _makecmd(chip, tool, task, step, index, script_name='replay.sh', include_pat
                 format_cmd.append(cmdarg)
             else:
                 format_cmd[-1] += f' {cmdarg}'
-        print(" \\\n    ".join(format_cmd), file=f)
+
+        replay_opts["cmds"] = format_cmd
+
+        f.write(utils.get_file_template("replay/replay.sh.j2").render(replay_opts))
+        f.write("\n")
 
     os.chmod(script_name, 0o755)
 
@@ -830,6 +840,7 @@ def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, ru
 
     # TODO: Currently no memory usage tracking in breakpoints, builtins, or unexpected errors.
     max_mem_bytes = 0
+    cpu_start = time.time()
 
     stdout_file, stderr_file = __get_stdio(chip, tool, task, flow, step, index)
     is_stdout_log = chip.get('tool', tool, 'task', task, 'stdout', 'destination',
@@ -881,7 +892,10 @@ def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, ru
 
         ##################
         # Make record of tool options
-        __record_tool(chip, step, index, version, toolpath, cmd_args)
+        if cmd_args is not None:
+            chip.set('record', 'toolargs',
+                     ' '.join(f'"{arg}"' if ' ' in arg else arg for arg in cmd_args),
+                     step=step, index=index)
 
         chip.logger.info('%s', printable_cmd)
         timeout = chip.get('option', 'timeout', step=step, index=index)
@@ -1003,8 +1017,15 @@ def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, ru
         chip.logger.warning(msg)
         chip._error = True
 
+    # Capture cpu runtime
+    record_metric(chip, step, index, 'exetime', round((time.time() - cpu_start), 2),
+                  source=None,
+                  source_unit='s')
+
     # Capture memory usage
-    record_metric(chip, step, index, 'memory', max_mem_bytes, source=None, source_unit='B')
+    record_metric(chip, step, index, 'memory', max_mem_bytes,
+                  source=None,
+                  source_unit='B')
 
 
 def _post_process(chip, step, index):
@@ -1074,26 +1095,26 @@ def _executenode(chip, step, index, replay):
 
         send_messages.send(chip, "skipped", step, index)
     else:
-        _set_env_vars(chip, step, index)
+        org_env = _set_env_vars(chip, step, index)
 
         run_func = getattr(chip._get_task_module(step, index, flow=flow), 'run', None)
-        (toolpath, version) = _check_tool_version(chip, step, index, run_func)
+        toolpath, version = _check_tool_version(chip, step, index, run_func)
+
+        if version:
+            chip.set('record', 'toolversion', version, step=step, index=index)
+
+        if toolpath:
+            chip.set('record', 'toolpath', toolpath, step=step, index=index)
 
         # Write manifest (tool interface) (Don't move this!)
         _write_task_manifest(chip, tool)
 
         send_messages.send(chip, "begin", step, index)
 
-        # Start CPU Timer
-        chip.logger.debug("Starting executable")
-        cpu_start = time.time()
-
         _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, run_func)
 
-        # Capture cpu runtime
-        cpu_end = time.time()
-        cputime = round((cpu_end - cpu_start), 2)
-        record_metric(chip, step, index, 'exetime', cputime, source=None, source_unit='s')
+        os.environ.clear()
+        os.environ.update(org_env)
 
         _post_process(chip, step, index)
 
@@ -1120,24 +1141,18 @@ def _pre_process(chip, step, index):
 
 
 def _set_env_vars(chip, step, index):
-    flow = chip.get('option', 'flow')
-    tool, task = get_tool_task(chip, step, index, flow)
+    org_env = os.environ.copy()
+
+    tool, task = get_tool_task(chip, step, index)
 
     chip.schema._start_record_access()
-    # License file configuration.
-    for item in chip.getkeys('tool', tool, 'licenseserver'):
-        license_file = chip.get('tool', tool, 'licenseserver', item, step=step, index=index)
-        if license_file:
-            os.environ[item] = ':'.join(license_file)
 
-    # Tool-specific environment variables for this task.
-    for item in chip.getkeys('tool', tool, 'task', task, 'env'):
-        val = chip.get('tool', tool, 'task', task, 'env', item, step=step, index=index)
-        if val:
-            os.environ[item] = val
+    os.environ.update(_get_run_env_vars(chip, tool, task, step, index, include_path=True))
 
     chip.schema._stop_record_access()
 
+    return org_env
+
 
 def _check_tool_version(chip, step, index, run_func=None):
     '''
@@ -1214,8 +1229,7 @@ def _hash_files(chip, step, index, setup=False):
             args = item.split(',')
             sc_type = chip.get(*args, field='type')
             if 'file' in sc_type or 'dir' in sc_type:
-                pernode = chip.get(*args, field='pernode')
-                if pernode == 'never':
+                if chip.get(*args, field='pernode').is_never():
                     if not setup:
                         if chip.get(*args, field='filehash'):
                             continue
@@ -1355,7 +1369,7 @@ def assert_required_accesses(chip, step, index):
                     gets.add(tuple(key))
 
     def get_value(*key):
-        if chip.get(*key, field='pernode') == 'never':
+        if chip.get(*key, field='pernode').is_never():
             return chip.get(*key)
         else:
             return chip.get(*key, step=step, index=index)
@@ -1728,19 +1742,6 @@ def get_record_time(chip, step, index, timetype):
         '%Y-%m-%d %H:%M:%S').timestamp()
 
 
-#######################################
-def __record_tool(chip, step, index, toolversion=None, toolpath=None, cli_args=None):
-    if toolversion:
-        chip.set('record', 'toolversion', toolversion, step=step, index=index)
-
-    if toolpath:
-        chip.set('record', 'toolpath', toolpath, step=step, index=index)
-
-    if cli_args is not None:
-        toolargs = ' '.join(f'"{arg}"' if ' ' in arg else arg for arg in cli_args)
-        chip.set('record', 'toolargs', toolargs, step=step, index=index)
-
-
 #######################################
 def _get_cloud_region():
     # TODO: add logic to figure out if we're running on a remote cluster and
@@ -1910,6 +1911,12 @@ def check_node_inputs(chip, step, index):
     if tool != input_tool or task != input_task:
         return False
 
+    # Check if inputs changed
+    new_inputs = set(_select_inputs(chip, step, index, trial=True))
+    if set(input_chip.get('record', 'inputnode', step=step, index=index)) != new_inputs:
+        chip.logger.warning(f'inputs to {step}{index} has been modified from previous run')
+        return False
+
     # Collect keys to check for changes
     required = chip.get('tool', tool, 'task', task, 'require', step=step, index=index)
     required.extend(input_chip.get('tool', tool, 'task', task, 'require', step=step, index=index))
@@ -1918,7 +1925,7 @@ def check_node_inputs(chip, step, index):
     for key in ('option', 'threads', 'prescript', 'postscript', 'refdir', 'script',):
         required.append(",".join([*tool_task_key, key]))
     for check_chip in (chip, input_chip):
-        for env_key in chip.getkeys(*tool_task_key, 'env'):
+        for env_key in check_chip.getkeys(*tool_task_key, 'env'):
             required.append(",".join([*tool_task_key, 'env', env_key]))
 
     def print_warning(key, extra=None):
@@ -1937,11 +1944,9 @@ def check_node_inputs(chip, step, index):
             print_warning(key)
             return False
 
-        pernode = chip.get(*key, field='pernode')
-
         check_step = step
         check_index = index
-        if pernode == 'never':
+        if chip.get(*key, field='pernode').is_never():
             check_step = None
             check_index = None
 
@@ -2244,7 +2249,7 @@ def _check_manifest_dynamic(chip, step, index):
             error = True
         else:
             paramtype = chip.get(*keypath, field='type')
-            is_perstep = chip.get(*keypath, field='pernode') != 'never'
+            is_perstep = not chip.get(*keypath, field='pernode').is_never()
             if ('file' in paramtype) or ('dir' in paramtype):
                 for val, check_step, check_index in chip.schema._getvals(*keypath):
                     if is_perstep:
@@ -2298,7 +2303,7 @@ def _clear_record(chip, step, index, record, preserve=None):
     if preserve and record in preserve:
         return
 
-    if chip.get('record', record, field='pernode') == 'never':
+    if chip.get('record', record, field='pernode').is_never():
         chip.unset('record', record)
     else:
         chip.unset('record', record, step=step, index=index)

siliconcompiler/scheduler/docker_runner.py

@@ -29,7 +29,7 @@ def get_volumes_directories(chip, cache_dir, workdir, step, index):
             cstep = step
             cindex = index
 
-            if 'never' in chip.get(*key, field='pernode'):
+            if chip.get(*key, field='pernode').is_never():
                 cstep = None
                 cindex = None
 

siliconcompiler/scheduler/send_messages.py

@@ -122,7 +122,7 @@ def send(chip, msg_type, step, index):
         records = {}
         for record in chip.getkeys('record'):
             value = None
-            if chip.get('record', record, field='pernode') == 'never':
+            if chip.get('record', record, field='pernode').is_never():
                 value = chip.get('record', record)
             else:
                 value = chip.get('record', record, step=step, index=index)