siliconcompiler 0.33.1__py3-none-any.whl → 0.34.0__py3-none-any.whl

siliconcompiler/scheduler/docker.py

@@ -0,0 +1,268 @@
+import docker
+import os
+import shlex
+import sys
+
+from pathlib import Path
+
+from siliconcompiler.package import RemoteResolver
+from siliconcompiler.utils import default_email_credentials_file
+from siliconcompiler.scheduler.schedulernode import SchedulerNode
+
+
+def get_image(chip, step, index):
+    from siliconcompiler import __version__
+
+    queue = chip.get('option', 'scheduler', 'queue', step=step, index=index)
+    if queue:
+        return queue
+
+    return os.getenv(
+        'SC_DOCKER_IMAGE',
+        f'ghcr.io/siliconcompiler/sc_runner:v{__version__}')
+
+
+def get_volumes_directories(chip, cache_dir, workdir, step, index):
+    all_dirs = set()
+    # Collect files
+    for key in chip.allkeys():
+        sc_type = chip.get(*key, field='type')
+
+        if 'file' in sc_type or 'dir' in sc_type:
+            cstep = step
+            cindex = index
+
+            if chip.get(*key, field='pernode').is_never():
+                cstep = None
+                cindex = None
+
+            files = chip.find_files(*key, step=cstep, index=cindex, missing_ok=True)
+            if files:
+                if not isinstance(files, list):
+                    files = [files]
+                for path in files:
+                    if path is None:
+                        continue
+                    if 'file' in sc_type:
+                        all_dirs.add(os.path.dirname(path))
+                    else:
+                        all_dirs.add(path)
+
+    # Collect caches
+    for resolver in chip.get('package', field="schema").get_resolvers().values():
+        all_dirs.add(resolver())
+
+    all_dirs = [
+        Path(cache_dir),
+        Path(workdir),
+        Path(chip.scroot),
+        *[Path(path) for path in all_dirs]]
+
+    pruned_dirs = all_dirs.copy()
+    for base_path in all_dirs:
+        if base_path not in pruned_dirs:
+            continue
+
+        new_pruned_dirs = [base_path]
+        for check_path in pruned_dirs:
+            if base_path == check_path:
+                continue
+
+            if base_path not in check_path.parents:
+                new_pruned_dirs.append(check_path)
+        pruned_dirs = new_pruned_dirs
+
+    pruned_dirs = set(pruned_dirs)
+
+    builddir = chip.find_files('option', 'builddir')
+
+    rw_volumes = set()
+
+    for path in pruned_dirs:
+        for rw_allow in (Path(builddir), Path(workdir), Path(cache_dir)):
+            if path == rw_allow or path in rw_allow.parents:
+                rw_volumes.add(path)
+
+    ro_volumes = pruned_dirs.difference(rw_volumes)
+
+    return rw_volumes, ro_volumes
+
+
+class DockerSchedulerNode(SchedulerNode):
+    def __init__(self, chip, step, index, replay=False):
+        super().__init__(chip, step, index, replay=replay)
+
+        self.__queue = get_image(self.chip, self.step, self.index)
+
+    @property
+    def queue(self):
+        return self.__queue
+
+    @staticmethod
+    def init(chip):
+        if sys.platform == 'win32':
+            # this avoids the issue of different file system types
+            chip.logger.error('Setting copy field to true for docker run on Windows')
+            for key in chip.allkeys():
+                if key[0] == 'history':
+                    continue
+                sc_type = chip.get(*key, field='type')
+                if 'dir' in sc_type or 'file' in sc_type:
+                    chip.set(*key, True, field='copy')
+            chip.collect()
+
+    def run(self):
+        try:
+            client = docker.from_env()
+            client.version()
+        except (docker.errors.DockerException, docker.errors.APIError) as e:
+            self.logger.error(f'Unable to connect to docker: {e}')
+            self.halt()
+
+        is_windows = sys.platform == 'win32'
+
+        workdir = self.chip.getworkdir()
+        start_cwd = os.getcwd()
+
+        # Remove handlers from logger
+        for handler in self.logger.handlers.copy():
+            self.logger.removeHandler(handler)
+
+        # Reinit logger
+        self.chip._init_logger(step=self.step, index=self.index, in_run=True)
+
+        # Change working directory since the run may delete this folder
+        os.makedirs(workdir, exist_ok=True)
+        os.chdir(workdir)
+
+        image_name = get_image(self.chip, self.step, self.index)
+
+        # Pull image if needed
+        try:
+            image = client.images.get(image_name)
+        except docker.errors.ImageNotFound:
+            # Needs a lock to avoid downloading a bunch in parallel
+            image_repo, image_tag = image_name.split(':')
+            self.logger.info(f'Pulling docker image {image_name}')
+            try:
+                image = client.images.pull(image_repo, tag=image_tag)
+            except docker.errors.APIError as e:
+                self.logger.error(f'Unable to pull image: {e}')
+                image_src = image_repo.split('/')[0]
+                self.logger.error(f"  if you are logged into {image_src} with expired credentials, "
+                                  f"please use 'docker logout {image_src}'")
+                self.halt()
+
+        email_file = default_email_credentials_file()
+        if is_windows:
+            # Hack to get around manifest merging
+            self.chip.set('option', 'cachedir', None)
+            cache_dir = '/sc_cache'
+            cwd = '/sc_docker'
+            builddir = f'{cwd}/build'
+
+            local_cfg = os.path.join(start_cwd, 'sc_docker.json')
+            job = self.chip.get('option', 'jobname')
+            cfg = f'{builddir}/{self.chip.design}/{job}/{self.step}/{self.index}/sc_docker.json'
+
+            user = None
+
+            volumes = [
+                f"{self.chip.cwd}:{cwd}:rw",
+                f"{RemoteResolver.determine_cache_dir(self.chip)}:{cache_dir}:rw"
+            ]
+            self.logger.debug(f'Volumes: {volumes}')
+
+            env = {}
+
+            if os.path.exists(email_file):
+                env["HOME"] = "/sc_home"
+
+                volumes.append(f'{os.path.dirname(email_file)}:/sc_home/.sc:ro')
+        else:
+            cache_dir = RemoteResolver.determine_cache_dir(self.chip)
+            cwd = self.chip.cwd
+            builddir = self.chip.find_files('option', 'builddir')
+
+            local_cfg = os.path.abspath('sc_docker.json')
+            cfg = local_cfg
+
+            user = os.getuid()
+
+            rw_volumes, ro_volumes = get_volumes_directories(
+                self.chip, cache_dir, workdir, self.step, self.index)
+            volumes = [
+                *[
+                    f'{path}:{path}:rw' for path in rw_volumes
+                ],
+                *[
+                    f'{path}:{path}:ro' for path in ro_volumes
+                ]
+            ]
+            self.logger.debug(f'Read write volumes: {rw_volumes}')
+            self.logger.debug(f'Read only volumes: {ro_volumes}')
+
+            env = {}
+            if os.path.exists(email_file):
+                env["HOME"] = "/sc_home"
+
+                volumes.append(f'{os.path.dirname(email_file)}:/sc_home/.sc:ro')
+
+        container = None
+        try:
+            container = client.containers.run(
+                image.id,
+                volumes=volumes,
+                labels=[
+                    "siliconcompiler",
+                    f"sc_node:{self.chip.design}:{self.step}{self.index}"
+                ],
+                user=user,
+                detach=True,
+                tty=True,
+                auto_remove=True,
+                environment=env)
+
+            # Write manifest to make it available to the docker runner
+            self.chip.write_manifest(local_cfg)
+
+            cachemap = []
+            for package, resolver in self.chip.get(
+                    'package', field="schema").get_resolvers().items():
+                cachemap.append(f'{package}:{resolver()}')
+
+            self.logger.info('Running in docker container: '
+                             f'{container.name} ({container.short_id})')
+            args = [
+                '-cfg', cfg,
+                '-cwd', cwd,
+                '-builddir', str(builddir),
+                '-cachedir', str(cache_dir),
+                '-step', self.step,
+                '-index', self.index,
+                '-unset_scheduler'
+            ]
+            if not is_windows and cachemap:
+                args.append('-cachemap')
+                args.extend(cachemap)
+            cmd = f'python3 -m siliconcompiler.scheduler.run_node {shlex.join(args)}'
+            exec_handle = client.api.exec_create(container.name, cmd)
+            stream = client.api.exec_start(exec_handle, stream=True)
+
+            # Print the log
+            for chunk in stream:
+                for line in chunk.decode().splitlines():
+                    print(line)
+
+            if client.api.exec_inspect(exec_handle['Id']).get('ExitCode') != 0:
+                self.halt()
+        finally:
+            # Ensure we clean up containers
+            if container:
+                try:
+                    container.stop()
+                except docker.errors.APIError:
+                    self.logger.error(f'Failed to stop docker container: {container.name}')
+
+        # Restore working directory
+        os.chdir(start_cwd)

siliconcompiler/scheduler/run_node.py

@@ -4,9 +4,10 @@ import argparse
 import os
 import sys
 import tarfile
+import os.path
+
 from siliconcompiler import Chip, Schema
-from siliconcompiler.package import path as sc_path
-from siliconcompiler.scheduler import _runtask, _executenode
+from siliconcompiler.scheduler.schedulernode import SchedulerNode
 from siliconcompiler import __version__
 
 
@@ -38,16 +39,12 @@ def main():
                                         field='shorthelp'))
     parser.add_argument('-cachedir',
                         metavar='<directory>',
-                        required=True,
                         help=schema.get('option', 'cachedir',
                                         field='shorthelp'))
     parser.add_argument('-cachemap',
                         metavar='<package>:<directory>',
                         nargs='+',
                         help='Map of caches to prepopulate runner with')
-    parser.add_argument('-fetch_cache',
-                        action='store_true',
-                        help='Allow for cache downloads')
     parser.add_argument('-step',
                         required=True,
                         metavar='<step>',
@@ -72,12 +69,15 @@ def main():
     parser.add_argument('-unset_scheduler',
                         action='store_true',
                         help='Unset scheduler to ensure local run')
+    parser.add_argument('-replay',
+                        action='store_true',
+                        help='Running as replay')
     args = parser.parse_args()
 
     # Change to working directory to allow rel path to be build dir
     # this avoids needing to deal with the job hash on the client
     # side
-    os.chdir(args.cwd)
+    os.chdir(os.path.abspath(args.cwd))
 
     # Create the Chip object.
     chip = Chip('<design>')
@@ -86,8 +86,10 @@ def main():
     # setup work directory
     chip.set('arg', 'step', args.step)
     chip.set('arg', 'index', args.index)
-    chip.set('option', 'builddir', args.builddir)
-    chip.set('option', 'cachedir', args.cachedir)
+    chip.set('option', 'builddir', os.path.abspath(args.builddir))
+
+    if args.cachedir:
+        chip.set('option', 'cachedir', os.path.abspath(args.cachedir))
 
     if args.remoteid:
         chip.set('record', 'remoteid', args.remoteid)
@@ -98,27 +100,26 @@ def main():
             chip.unset('option', 'scheduler', 'name', step=step, index=index)
 
     # Init logger to ensure consistent view
-    chip._init_logger(step=chip.get('arg', 'step'),
-                      index=chip.get('arg', 'index'),
+    chip._init_logger(step=args.step,
+                      index=args.index,
                       in_run=True)
 
     if args.cachemap:
         for cachepair in args.cachemap:
             package, path = cachepair.split(':')
-            chip._packages[package] = path
+            chip.get("package", field="schema")._set_cache(package, path)
 
     # Populate cache
-    for package in chip.getkeys('package', 'source'):
-        sc_path(chip, package, fetch=args.fetch_cache)
+    for resolver in chip.get('package', field='schema').get_resolvers().values():
+        resolver()
 
     # Run the task.
     error = True
     try:
-        _runtask(chip,
-                 chip.get('option', 'flow'),
-                 chip.get('arg', 'step'),
-                 chip.get('arg', 'index'),
-                 _executenode)
+        SchedulerNode(chip,
+                      args.step,
+                      args.index,
+                      replay=args.replay).run()
         error = False
 
     finally:

siliconcompiler/scheduler/scheduler.py

@@ -0,0 +1,308 @@
+import os
+import re
+import shutil
+import sys
+
+import os.path
+
+from siliconcompiler import Schema
+from siliconcompiler import NodeStatus
+from siliconcompiler.schema import Journal
+from siliconcompiler.flowgraph import RuntimeFlowgraph
+from siliconcompiler.scheduler.schedulernode import SchedulerNode
+from siliconcompiler.scheduler.slurm import SlurmSchedulerNode
+from siliconcompiler.scheduler.docker import DockerSchedulerNode
+from siliconcompiler.scheduler.taskscheduler import TaskScheduler
+
+from siliconcompiler import utils
+from siliconcompiler.scheduler import send_messages
+
+
+class Scheduler:
+    def __init__(self, chip):
+        self.__chip = chip
+        self.__logger = self.__chip.logger
+
+        flow = self.__chip.get("option", "flow")
+        if not flow:
+            raise ValueError("flow must be specified")
+
+        if flow not in self.__chip.getkeys("flowgraph"):
+            raise ValueError("flow is not defined")
+
+        self.__flow = self.__chip.schema.get("flowgraph", flow, field="schema")
+        from_steps = self.__chip.get('option', 'from')
+        to_steps = self.__chip.get('option', 'to')
+        prune_nodes = self.__chip.get('option', 'prune')
+
+        if not self.__flow.validate(logger=self.__logger):
+            raise ValueError(f"{self.__flow.name()} flowgraph contains errors and cannot be run.")
+        if not RuntimeFlowgraph.validate(
+                self.__flow,
+                from_steps=from_steps,
+                to_steps=to_steps,
+                prune_nodes=prune_nodes,
+                logger=chip.logger):
+            raise ValueError(f"{self.__flow.name()} flowgraph contains errors and cannot be run.")
+
+        self.__flow_runtime = RuntimeFlowgraph(
+            self.__flow,
+            from_steps=from_steps,
+            to_steps=to_steps,
+            prune_nodes=self.__chip.get('option', 'prune'))
+
+        self.__flow_runtime_no_prune = RuntimeFlowgraph(
+            self.__flow,
+            from_steps=from_steps,
+            to_steps=to_steps)
+
+        self.__flow_load_runtime = RuntimeFlowgraph(
+            self.__flow,
+            to_steps=from_steps,
+            prune_nodes=prune_nodes)
+
+        self.__flow_something = RuntimeFlowgraph(
+            self.__flow,
+            from_steps=set([step for step, _ in self.__flow.get_entry_nodes()]),
+            prune_nodes=prune_nodes)
+
+        self.__record = self.__chip.schema.get("record", field="schema")
+        self.__metrics = self.__chip.schema.get("metric", field="schema")
+
+        self.__tasks = {}
+
+    def __print_status(self, header):
+        self.__logger.debug(f"#### {header}")
+        for step, index in self.__flow.get_nodes():
+            self.__logger.debug(f"({step}, {index}) -> "
+                                f"{self.__record.get('status', step=step, index=index)}")
+        self.__logger.debug("####")
+
+    def check_manifest(self):
+        self.__logger.info("Checking manifest before running.")
+        return self.__chip.check_manifest()
+
+    def run_core(self):
+        self.__record.record_python_packages()
+
+        task_scheduler = TaskScheduler(self.__chip, self.__tasks)
+        task_scheduler.run()
+        task_scheduler.check()
+
+    def run(self):
+        self.__run_setup()
+        self.configure_nodes()
+
+        # Check validity of setup
+        if not self.check_manifest():
+            raise RuntimeError("check_manifest() failed")
+
+        self.run_core()
+
+        # Store run in history
+        self.__chip.schema.record_history()
+
+        # Record final manifest
+        filepath = os.path.join(self.__chip.getworkdir(), f"{self.__chip.design}.pkg.json")
+        self.__chip.write_manifest(filepath)
+
+        send_messages.send(self.__chip, 'summary', None, None)
+
+    def __mark_pending(self, step, index):
+        if (step, index) not in self.__flow_runtime.get_nodes():
+            return
+
+        self.__record.set('status', NodeStatus.PENDING, step=step, index=index)
+        for next_step, next_index in self.__flow_runtime.get_nodes_starting_at(step, index):
+            if self.__record.get('status', step=next_step, index=next_index) == NodeStatus.SKIPPED:
+                continue
+
+            # Mark following steps as pending
+            self.__record.set('status', NodeStatus.PENDING, step=next_step, index=next_index)
+
+    def __run_setup(self):
+        self.__check_display()
+
+        org_jobname = self.__chip.get('option', 'jobname')
+        copy_prev_job = self.__increment_job_name()
+
+        # Create tasks
+        copy_from_nodes = set(self.__flow_load_runtime.get_nodes()).difference(
+            self.__flow_runtime.get_entry_nodes())
+        for step, index in self.__flow.get_nodes():
+            node_cls = SchedulerNode
+
+            node_scheduler = self.__chip.get('option', 'scheduler', 'name', step=step, index=index)
+            if node_scheduler == 'slurm':
+                node_cls = SlurmSchedulerNode
+            elif node_scheduler == 'docker':
+                node_cls = DockerSchedulerNode
+            self.__tasks[(step, index)] = node_cls(self.__chip, step, index)
+            if self.__flow.get(step, index, "tool") == "builtin":
+                self.__tasks[(step, index)].set_builtin()
+
+            if copy_prev_job and (step, index) in copy_from_nodes:
+                self.__tasks[(step, index)].copy_from(org_jobname)
+
+        if copy_prev_job:
+            # Copy collection directory
+            copy_from = self.__chip._getcollectdir(jobname=org_jobname)
+            copy_to = self.__chip._getcollectdir()
+            if os.path.exists(copy_from):
+                shutil.copytree(copy_from, copy_to,
+                                dirs_exist_ok=True,
+                                copy_function=utils.link_copy)
+
+        self.__clean_build_dir()
+        self.__reset_flow_nodes()
+
+    def __reset_flow_nodes(self):
+        # Reset record
+        for step, index in self.__flow.get_nodes():
+            self.__record.clear(step, index, keep=['remoteid', 'status', 'pythonpackage'])
+            self.__record.set('status', NodeStatus.PENDING, step=step, index=index)
+
+        # Reset metrics
+        for step, index in self.__flow.get_nodes():
+            self.__metrics.clear(step, index)
+
+    def __clean_build_dir(self):
+        if self.__record.get('remoteid'):
+            return
+
+        if self.__chip.get('option', 'clean') and not self.__chip.get('option', 'from'):
+            # If no step or nodes to start from were specified, the whole flow is being run
+            # start-to-finish. Delete the build dir to clear stale results.
+            cur_job_dir = self.__chip.getworkdir()
+            if os.path.isdir(cur_job_dir):
+                shutil.rmtree(cur_job_dir)
+
+    def configure_nodes(self):
+        from_nodes = []
+        extra_setup_nodes = {}
+
+        journal = Journal.access(self.__chip.schema)
+        journal.start()
+
+        self.__print_status("Start")
+
+        if self.__chip.get('option', 'clean'):
+            if self.__chip.get("option", "from"):
+                from_nodes = self.__flow_runtime.get_entry_nodes()
+            load_nodes = self.__flow.get_nodes()
+        else:
+            if self.__chip.get("option", "from"):
+                from_nodes = self.__flow_runtime.get_entry_nodes()
+            load_nodes = self.__flow_load_runtime.get_nodes()
+
+        # Collect previous run information
+        for step, index in self.__flow.get_nodes():
+            if (step, index) not in load_nodes:
+                # Node not marked for loading
+                continue
+            if (step, index) in from_nodes:
+                # Node will be run so no need to load
+                continue
+
+            manifest = os.path.join(self.__chip.getworkdir(step=step, index=index),
+                                    'outputs',
+                                    f'{self.__chip.design}.pkg.json')
+            if os.path.exists(manifest):
+                # ensure we setup these nodes again
+                try:
+                    extra_setup_nodes[(step, index)] = Schema.from_manifest(filepath=manifest)
+                except Exception:
+                    pass
+
+        # Setup tools for all nodes to run
+        for layer_nodes in self.__flow.get_execution_order():
+            for step, index in layer_nodes:
+                node_kept = self.__tasks[(step, index)].setup()
+                if not node_kept and (step, index) in extra_setup_nodes:
+                    # remove from previous node data
+                    del extra_setup_nodes[(step, index)]
+
+                if (step, index) in extra_setup_nodes:
+                    schema = extra_setup_nodes[(step, index)]
+                    node_status = None
+                    try:
+                        node_status = schema.get('record', 'status', step=step, index=index)
+                    except:  # noqa E722
+                        pass
+                    if node_status:
+                        # Forward old status
+                        self.__record.set('status', node_status, step=step, index=index)
+
+        self.__print_status("After setup")
+
+        # Check for modified information
+        for layer_nodes in self.__flow.get_execution_order():
+            for step, index in layer_nodes:
+                # Only look at successful nodes
+                if self.__record.get("status", step=step, index=index) != NodeStatus.SUCCESS:
+                    continue
+
+                if self.__tasks[(step, index)].requires_run():
+                    # This node must be run
+                    self.__mark_pending(step, index)
+                elif (step, index) in extra_setup_nodes:
+                    # import old information
+                    Journal.access(extra_setup_nodes[(step, index)]).replay(self.__chip.schema)
+
+        self.__print_status("After requires run")
+
+        # Ensure all nodes are marked as pending if needed
+        for layer_nodes in self.__flow_runtime.get_execution_order():
+            for step, index in layer_nodes:
+                status = self.__record.get("status", step=step, index=index)
+                if NodeStatus.is_waiting(status) or NodeStatus.is_error(status):
+                    self.__mark_pending(step, index)
+
+        self.__print_status("After ensure")
+
+        self.__chip.write_manifest(os.path.join(self.__chip.getworkdir(),
+                                                f"{self.__chip.get('design')}.pkg.json"))
+        journal.stop()
+
+        # Clean nodes marked pending
+        for step, index in self.__flow_runtime.get_nodes():
+            if NodeStatus.is_waiting(self.__record.get('status', step=step, index=index)):
+                self.__tasks[(step, index)].clean_directory()
+
+    def __check_display(self):
+        '''
+        Automatically disable display for Linux systems without desktop environment
+        '''
+
+        if not self.__chip.get('option', 'nodisplay') and sys.platform == 'linux' \
+                and 'DISPLAY' not in os.environ and 'WAYLAND_DISPLAY' not in os.environ:
+            self.__logger.warning('Environment variable $DISPLAY or $WAYLAND_DISPLAY not set')
+            self.__logger.warning("Setting [option,nodisplay] to True")
+            self.__chip.set('option', 'nodisplay', True)
+
+    def __increment_job_name(self):
+        '''
+        Auto-update jobname if [option,jobincr] is True
+        Do this before initializing logger so that it picks up correct jobname
+        '''
+
+        if not self.__chip.get('option', 'clean'):
+            return False
+        if not self.__chip.get('option', 'jobincr'):
+            return False
+
+        workdir = self.__chip.getworkdir()
+        if os.path.isdir(workdir):
+            # Strip off digits following jobname, if any
+            stem = self.__chip.get('option', 'jobname').rstrip('0123456789')
+
+            dir_check = re.compile(fr'{stem}(\d+)')
+
+            jobid = 0
+            for job in os.listdir(os.path.dirname(workdir)):
+                m = dir_check.match(job)
+                if m:
+                    jobid = max(jobid, int(m.group(1)))
+            self.__chip.set('option', 'jobname', f'{stem}{jobid + 1}')
+            return True
+        return False