siliconcompiler 0.26.5__py3-none-any.whl

siliconcompiler/scheduler/docker_runner.py

@@ -0,0 +1,253 @@
+import docker
+import os
+from siliconcompiler.package import get_cache_path
+from siliconcompiler.package import _path as sc_path
+from siliconcompiler.utils import default_email_credentials_file
+from pathlib import Path
+import sys
+
+
+def get_image(chip, step, index):
+    from siliconcompiler import __version__
+
+    queue = chip.get('option', 'scheduler', 'queue', step=step, index=index)
+    if queue:
+        return queue
+
+    return os.getenv(
+        'SC_DOCKER_IMAGE',
+        f'ghcr.io/siliconcompiler/sc_runner:v{__version__}')
+
+
+def get_volumes_directories(chip, cache_dir, workdir, step, index):
+    all_dirs = set()
+    # Collect files
+    for key in chip.allkeys():
+        sc_type = chip.get(*key, field='type')
+
+        if 'file' in sc_type or 'dir' in sc_type:
+            cstep = step
+            cindex = index
+
+            if 'never' in chip.get(*key, field='pernode'):
+                cstep = None
+                cindex = None
+
+            files = chip.find_files(*key, step=cstep, index=cindex, missing_ok=True)
+            if files:
+                if not isinstance(files, list):
+                    files = [files]
+                for path in files:
+                    if path is None:
+                        continue
+                    if 'file' in sc_type:
+                        all_dirs.add(os.path.dirname(path))
+                    else:
+                        all_dirs.add(path)
+
+    # Collect caches
+    for package in chip.getkeys('package', 'source'):
+        all_dirs.add(sc_path(chip, package, None))
+
+    all_dirs = [
+        Path(cache_dir),
+        Path(workdir),
+        Path(chip.scroot),
+        *[Path(path) for path in all_dirs]]
+
+    pruned_dirs = all_dirs.copy()
+    for base_path in all_dirs:
+        if base_path not in pruned_dirs:
+            continue
+
+        new_pruned_dirs = [base_path]
+        for check_path in pruned_dirs:
+            if base_path == check_path:
+                continue
+
+            if base_path not in check_path.parents:
+                new_pruned_dirs.append(check_path)
+        pruned_dirs = new_pruned_dirs
+
+    pruned_dirs = set(pruned_dirs)
+
+    builddir = chip.find_files('option', 'builddir')
+
+    rw_volumes = set()
+
+    for path in pruned_dirs:
+        for rw_allow in (Path(builddir), Path(workdir), Path(cache_dir)):
+            if path == rw_allow or path in rw_allow.parents:
+                rw_volumes.add(path)
+
+    ro_volumes = pruned_dirs.difference(rw_volumes)
+
+    return rw_volumes, ro_volumes
+
+
+def init(chip):
+    if sys.platform == 'win32':
+        # this avoids the issue of different file system types
+        chip.logger.error('Setting copy field to true for docker run on Windows')
+        for key in chip.allkeys():
+            if key[0] == 'history':
+                continue
+            sc_type = chip.get(*key, field='type')
+            if 'dir' in sc_type or 'file' in sc_type:
+                chip.set(*key, True, field='copy')
+        chip.collect()
+
+
+def run(chip, step, index, replay):
+    # Import here to avoid circular import
+    from siliconcompiler.scheduler import _haltstep
+
+    try:
+        client = docker.from_env()
+        client.version()
+    except (docker.errors.DockerException, docker.errors.APIError) as e:
+        chip.logger.error(f'Unable to connect to docker: {e}')
+        _haltstep(chip, chip.get('option', 'flow'), step, index)
+
+    is_windows = sys.platform == 'win32'
+
+    workdir = chip.getworkdir()
+    start_cwd = os.getcwd()
+
+    # Remove handlers from logger
+    chip.logger.handlers.clear()
+
+    # Reinit logger
+    chip._init_logger(step=step, index=index, in_run=True)
+
+    # Change working directory since the run may delete this folder
+    os.chdir(workdir)
+
+    image_name = get_image(chip, step, index)
+
+    # Pull image if needed
+    try:
+        image = client.images.get(image_name)
+    except docker.errors.ImageNotFound:
+        # Needs a lock to avoid downloading a bunch in parallel
+        image_repo, image_tag = image_name.split(':')
+        chip.logger.info(f'Pulling docker image {image_name}')
+        try:
+            image = client.images.pull(image_repo, tag=image_tag)
+        except docker.errors.APIError as e:
+            chip.logger.error(f'Unable to pull image: {e}')
+            image_src = image_repo.split('/')[0]
+            chip.logger.error(f"  if you are logged into {image_src} with expired credentials, "
+                              f"please use 'docker logout {image_src}'")
+            _haltstep(chip, chip.get('option', 'flow'), step, index)
+
+    email_file = default_email_credentials_file()
+    if is_windows:
+        # Hack to get around manifest merging
+        chip.set('option', 'cachedir', None)
+        cache_dir = '/sc_cache'
+        cwd = '/sc_docker'
+        builddir = f'{cwd}/build'
+
+        local_cfg = os.path.join(start_cwd, 'sc_docker.json')
+        job = chip.get('option', 'jobname')
+        cfg = f'{builddir}/{chip.design}/{job}/{step}/{index}/sc_docker.json'
+
+        user = None
+
+        volumes = [
+            f"{chip.cwd}:{cwd}:rw",
+            f"{get_cache_path(chip)}:{cache_dir}:rw"
+        ]
+        chip.logger.debug(f'Volumes: {volumes}')
+
+        env = {}
+
+        if os.path.exists(email_file):
+            env["HOME"] = "/sc_home"
+
+            volumes.append(f'{os.path.dirname(email_file)}:/sc_home/.sc:ro')
+    else:
+        cache_dir = get_cache_path(chip)
+        cwd = chip.cwd
+        builddir = chip.find_files('option', 'builddir')
+
+        local_cfg = os.path.abspath('sc_docker.json')
+        cfg = local_cfg
+
+        user = os.getuid()
+
+        rw_volumes, ro_volumes = get_volumes_directories(chip, cache_dir, workdir, step, index)
+        volumes = [
+            *[
+                f'{path}:{path}:rw' for path in rw_volumes
+            ],
+            *[
+                f'{path}:{path}:ro' for path in ro_volumes
+            ]
+        ]
+        chip.logger.debug(f'Read write volumes: {rw_volumes}')
+        chip.logger.debug(f'Read only volumes: {ro_volumes}')
+
+        env = {}
+        if os.path.exists(email_file):
+            env["HOME"] = "/sc_home"
+
+            volumes.append(f'{os.path.dirname(email_file)}:/sc_home/.sc:ro')
+
+    container = None
+    try:
+        container = client.containers.run(
+            image.id,
+            volumes=volumes,
+            labels=[
+                "siliconcompiler",
+                f"sc_node:{chip.design}:{step}{index}"
+            ],
+            user=user,
+            detach=True,
+            tty=True,
+            auto_remove=True,
+            environment=env)
+
+        # Write manifest to make it available to the docker runner
+        chip.write_manifest(local_cfg)
+
+        cachemap = []
+        for package in chip.getkeys('package', 'source'):
+            cachemap.append(f'{package}:{sc_path(chip, package, None)}')
+
+        chip.logger.info(f'Running in docker container: {container.name} ({container.short_id})')
+        args = [
+            '-cfg', cfg,
+            '-cwd', cwd,
+            '-builddir', builddir,
+            '-cachedir', cache_dir,
+            '-step', step,
+            '-index', index,
+            '-unset_scheduler'
+        ]
+        if not is_windows and cachemap:
+            args.append('-cachemap')
+            args.append(' '.join(cachemap))
+        cmd = f'python3 -m siliconcompiler.scheduler.run_node {" ".join(args)}'
+        exec_handle = client.api.exec_create(container.name, cmd)
+        stream = client.api.exec_start(exec_handle, stream=True)
+
+        # Print the log
+        for chunk in stream:
+            for line in chunk.decode().splitlines():
+                print(line)
+
+        if client.api.exec_inspect(exec_handle['Id']).get('ExitCode') != 0:
+            _haltstep(chip, chip.get('option', 'flow'), step, index, log=False)
+    finally:
+        # Ensure we clean up containers
+        if container:
+            try:
+                container.stop()
+            except docker.errors.APIError:
+                chip.logger.error('Failed to stop docker container')
+
+    # Restore working directory
+    os.chdir(start_cwd)

siliconcompiler/scheduler/run_node.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import sys
+import tarfile
+from siliconcompiler import Chip, Schema
+from siliconcompiler.package import _path as sc_path
+from siliconcompiler.scheduler import _runtask, _executenode
+from siliconcompiler import __version__
+
+
+##########################
+def main():
+    schema = Schema()
+
+    # Can't use chip.cmdline because we don't want a bunch of extra logger information
+    parser = argparse.ArgumentParser(prog='run_node',
+                                     formatter_class=argparse.RawDescriptionHelpFormatter,
+                                     description='Script to run a single node in an SC flowgraph')
+
+    parser.add_argument('-version',
+                        action='version',
+                        version=__version__)
+    parser.add_argument('-cfg',
+                        required=True,
+                        metavar='<file>',
+                        help=schema.get('option', 'cfg',
+                                        field='shorthelp'))
+    parser.add_argument('-cwd',
+                        required=True,
+                        metavar='<directory>',
+                        help='Run current working directory')
+    parser.add_argument('-builddir',
+                        metavar='<directory>',
+                        required=True,
+                        help=schema.get('option', 'builddir',
+                                        field='shorthelp'))
+    parser.add_argument('-cachedir',
+                        metavar='<directory>',
+                        required=True,
+                        help=schema.get('option', 'cachedir',
+                                        field='shorthelp'))
+    parser.add_argument('-cachemap',
+                        metavar='<package>:<directory>',
+                        nargs='+',
+                        help='Map of caches to prepopulate runner with')
+    parser.add_argument('-step',
+                        required=True,
+                        metavar='<step>',
+                        help=schema.get('arg', 'step',
+                                        field='shorthelp'))
+    parser.add_argument('-index',
+                        required=True,
+                        metavar='<index>',
+                        help=schema.get('arg', 'index',
+                                        field='shorthelp'))
+    parser.add_argument('-remoteid',
+                        metavar='<id>',
+                        help=schema.get('record', 'remoteid',
+                                        field='shorthelp'))
+    parser.add_argument('-archive',
+                        metavar='<file>',
+                        help='Generate archive')
+    parser.add_argument('-include',
+                        metavar='<path>',
+                        nargs='+',
+                        help='Files to include in archive')
+    parser.add_argument('-unset_scheduler',
+                        action='store_true',
+                        help='Unset scheduler to ensure local run')
+    args = parser.parse_args()
+
+    # Change to working directory to allow rel path to be build dir
+    # this avoids needing to deal with the job hash on the client
+    # side
+    os.chdir(args.cwd)
+
+    # Create the Chip object.
+    chip = Chip('<design>')
+    chip.read_manifest(args.cfg)
+
+    # setup work directory
+    chip.set('arg', 'step', args.step)
+    chip.set('arg', 'index', args.index)
+    chip.set('option', 'builddir', args.builddir)
+    chip.set('option', 'cachedir', args.cachedir)
+
+    if args.remoteid:
+        chip.set('record', 'remoteid', args.remoteid)
+
+    if args.unset_scheduler:
+        for vals, step, index in chip.schema._getvals('option', 'scheduler', 'name'):
+            chip.unset('option', 'scheduler', 'name', step=step, index=index)
+
+    # Init logger to ensure consistent view
+    chip._init_logger(step=chip.get('arg', 'step'),
+                      index=chip.get('arg', 'index'),
+                      in_run=True)
+
+    if args.cachemap:
+        for cachepair in args.cachemap:
+            package, path = cachepair.split(':')
+            chip._packages[package] = path
+
+    # Populate cache without downloading
+    for package in chip.getkeys('package', 'source'):
+        sc_path(chip, package, None)
+
+    # Run the task.
+    error = True
+    try:
+        _runtask(chip,
+                 chip.get('option', 'flow'),
+                 chip.get('arg', 'step'),
+                 chip.get('arg', 'index'),
+                 _executenode)
+        error = False
+
+    finally:
+        if args.archive:
+            # Archive the results.
+            with tarfile.open(args.archive,
+                              mode='w:gz') as tf:
+                chip._archive_node(tf,
+                                   step=args.step,
+                                   index=args.index,
+                                   include=args.include)
+
+    # Return success/fail flag, in case the caller is interested.
+    if error:
+        return 1
+    return 0
+
+
+##########################
+if __name__ == "__main__":
+    sys.exit(main())

siliconcompiler/scheduler/send_messages.py

@@ -0,0 +1,178 @@
+from siliconcompiler.utils import default_email_credentials_file, get_file_template
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.mime.application import MIMEApplication
+import json
+import os
+from siliconcompiler import sc_open
+from siliconcompiler.schema import Schema
+from siliconcompiler.report import utils as report_utils
+import fastjsonschema
+from pathlib import Path
+from siliconcompiler.flowgraph import get_executed_nodes
+import uuid
+
+
+# Compile validation code for API request bodies.
+api_dir = Path(__file__).parent / 'validation'
+
+# 'remote_run': Run a stage of a job using the server's cluster settings.
+with open(api_dir / 'email_credentials.json') as schema:
+    validate_creds = fastjsonschema.compile(json.loads(schema.read()))
+
+
+def __load_config(chip):
+    path = default_email_credentials_file()
+    if not os.path.exists(path):
+        chip.logger.warn(f'Email credentials are not available: {path}')
+        return {}
+
+    with open(path) as f:
+        creds = json.load(f)
+
+    try:
+        return validate_creds(creds)
+    except fastjsonschema.JsonSchemaException as e:
+        chip.logger.error(f'Email credentials failed to validate: {e}')
+        return {}
+
+
+def send(chip, msg_type, step, index):
+    chip_step, chip_index = step, index
+    if step is None:
+        chip_step = Schema.GLOBAL_KEY
+    if index is None:
+        chip_index = Schema.GLOBAL_KEY
+    to = chip.get('option', 'scheduler', 'msgcontact', step=chip_step, index=chip_index)
+    event = chip.get('option', 'scheduler', 'msgevent', step=chip_step, index=chip_index)
+
+    if not to or not event:
+        # nothing to do
+        return
+
+    if 'all' not in event and msg_type not in event:
+        # nothing to do
+        return
+
+    cred = __load_config(chip)
+
+    if not cred:
+        return
+
+    jobname = chip.get("option", "jobname")
+    flow = chip.get("option", "flow")
+
+    msg = MIMEMultipart()
+
+    if step and index:
+        subject = f'SiliconCompiler : {chip.design} | {jobname} | {step}{index} | {msg_type}'
+    else:
+        subject = f'SiliconCompiler : {chip.design} | {jobname} | {msg_type}'
+
+    # Setup email header
+    msg['Subject'] = subject
+
+    if "from" in cred:
+        msg['From'] = cred["from"]
+    else:
+        msg['From'] = to[0]
+    msg['To'] = ", ".join(to)
+    msg['X-Entity-Ref-ID'] = uuid.uuid4().hex  # keep emails from getting grouped
+
+    if msg_type == "summary":
+        layout_img = report_utils._find_summary_image(chip)
+        if layout_img and os.path.isfile(layout_img):
+            with open(layout_img, 'rb') as img_file:
+                img_attach = MIMEApplication(img_file.read())
+                img_attach.add_header('Content-Disposition',
+                                      'attachment',
+                                      filename=os.path.basename(layout_img))
+                msg.attach(img_attach)
+
+        nodes_to_execute = get_executed_nodes(chip, flow)
+        nodes, errors, metrics, metrics_unit, metrics_to_show, _ = \
+            report_utils._collect_data(chip, flow=flow, flowgraph_nodes=nodes_to_execute)
+
+        text_msg = get_file_template('email/summary.j2').render(
+            design=chip.design,
+            nodes=nodes,
+            errors=errors,
+            metrics=metrics,
+            metrics_unit=metrics_unit,
+            metric_keys=metrics_to_show)
+    else:
+        # Attach logs
+        for log in (f'sc_{step}{index}.log', f'{step}.log'):
+            log_file = f'{chip.getworkdir(step=step, index=index)}/{log}'
+            if os.path.exists(log_file):
+                with sc_open(log_file) as f:
+                    log_attach = MIMEApplication(f.read())
+                    log_name, _ = os.path.splitext(log)
+                    # Make attachment a txt file to avoid issues with tools not loading .log
+                    log_attach.add_header('Content-Disposition',
+                                          'attachment',
+                                          filename=f'{log_name}.txt')
+                    msg.attach(log_attach)
+
+        records = {}
+        for record in chip.getkeys('record'):
+            value = None
+            if chip.get('record', record, field='pernode') == 'never':
+                value = chip.get('record', record)
+            else:
+                value = chip.get('record', record, step=step, index=index)
+
+            if value is not None:
+                records[record] = value
+
+        nodes, errors, metrics, metrics_unit, metrics_to_show, _ = \
+            report_utils._collect_data(chip, flow=flow, flowgraph_nodes=[(step, index)])
+
+        status = chip.get('record', 'status', step=step, index=index)
+
+        text_msg = get_file_template('email/general.j2').render(
+            design=chip.design,
+            job=jobname,
+            step=step,
+            index=index,
+            status=status,
+            records=records,
+            nodes=nodes,
+            errors=errors,
+            metrics=metrics,
+            metrics_unit=metrics_unit,
+            metric_keys=metrics_to_show)
+
+    body = MIMEText(text_msg, 'html')
+    msg.attach(body)
+
+    if cred['ssl']:
+        smtp_use = smtplib.SMTP_SSL
+    else:
+        smtp_use = smtplib.SMTP
+
+    with smtp_use(cred["server"], cred["port"]) as smtp_server:
+        do_send = False
+        try:
+            smtp_server.login(cred["username"], cred["password"])
+            do_send = True
+        except smtplib.SMTPAuthenticationError as e:
+            chip.logger.error(f'Unable to authenticate to email server: {e}')
+        except Exception as e:
+            chip.logger.error(f'An error occurred during login to email server: {e}')
+
+        if do_send:
+            try:
+                smtp_server.sendmail(msg['From'], to, msg.as_string())
+            except Exception as e:
+                chip.logger.error(f'An error occurred while sending email: {e}')
+
+
+if __name__ == "__main__":
+    from siliconcompiler import Chip
+    chip = Chip('test')
+    chip.load_target("freepdk45_demo")
+    chip.set('option', 'scheduler', 'msgevent', 'ALL')
+    # chip.set('option', 'scheduler', 'msgcontact', 'fillin')
+    send(chip, "BEGIN", "import", "0")