siliconcompiler 0.33.2__py3-none-any.whl → 0.34.1__py3-none-any.whl

siliconcompiler/scheduler/docker_runner.py

@@ -1,254 +0,0 @@
-import docker
-import os
-from siliconcompiler.package import get_cache_path
-from siliconcompiler.package import path as sc_path
-from siliconcompiler.utils import default_email_credentials_file
-from pathlib import Path
-import sys
-
-
-def get_image(chip, step, index):
-    from siliconcompiler import __version__
-
-    queue = chip.get('option', 'scheduler', 'queue', step=step, index=index)
-    if queue:
-        return queue
-
-    return os.getenv(
-        'SC_DOCKER_IMAGE',
-        f'ghcr.io/siliconcompiler/sc_runner:v{__version__}')
-
-
-def get_volumes_directories(chip, cache_dir, workdir, step, index):
-    all_dirs = set()
-    # Collect files
-    for key in chip.allkeys():
-        sc_type = chip.get(*key, field='type')
-
-        if 'file' in sc_type or 'dir' in sc_type:
-            cstep = step
-            cindex = index
-
-            if chip.get(*key, field='pernode').is_never():
-                cstep = None
-                cindex = None
-
-            files = chip.find_files(*key, step=cstep, index=cindex, missing_ok=True)
-            if files:
-                if not isinstance(files, list):
-                    files = [files]
-                for path in files:
-                    if path is None:
-                        continue
-                    if 'file' in sc_type:
-                        all_dirs.add(os.path.dirname(path))
-                    else:
-                        all_dirs.add(path)
-
-    # Collect caches
-    for package in chip.getkeys('package', 'source'):
-        all_dirs.add(sc_path(chip, package))
-
-    all_dirs = [
-        Path(cache_dir),
-        Path(workdir),
-        Path(chip.scroot),
-        *[Path(path) for path in all_dirs]]
-
-    pruned_dirs = all_dirs.copy()
-    for base_path in all_dirs:
-        if base_path not in pruned_dirs:
-            continue
-
-        new_pruned_dirs = [base_path]
-        for check_path in pruned_dirs:
-            if base_path == check_path:
-                continue
-
-            if base_path not in check_path.parents:
-                new_pruned_dirs.append(check_path)
-        pruned_dirs = new_pruned_dirs
-
-    pruned_dirs = set(pruned_dirs)
-
-    builddir = chip.find_files('option', 'builddir')
-
-    rw_volumes = set()
-
-    for path in pruned_dirs:
-        for rw_allow in (Path(builddir), Path(workdir), Path(cache_dir)):
-            if path == rw_allow or path in rw_allow.parents:
-                rw_volumes.add(path)
-
-    ro_volumes = pruned_dirs.difference(rw_volumes)
-
-    return rw_volumes, ro_volumes
-
-
-def init(chip):
-    if sys.platform == 'win32':
-        # this avoids the issue of different file system types
-        chip.logger.error('Setting copy field to true for docker run on Windows')
-        for key in chip.allkeys():
-            if key[0] == 'history':
-                continue
-            sc_type = chip.get(*key, field='type')
-            if 'dir' in sc_type or 'file' in sc_type:
-                chip.set(*key, True, field='copy')
-        chip.collect()
-
-
-def run(chip, step, index, replay):
-    # Import here to avoid circular import
-    from siliconcompiler.scheduler import _haltstep
-
-    try:
-        client = docker.from_env()
-        client.version()
-    except (docker.errors.DockerException, docker.errors.APIError) as e:
-        chip.logger.error(f'Unable to connect to docker: {e}')
-        _haltstep(chip, chip.get('option', 'flow'), step, index)
-
-    is_windows = sys.platform == 'win32'
-
-    workdir = chip.getworkdir()
-    start_cwd = os.getcwd()
-
-    # Remove handlers from logger
-    for handler in chip.logger.handlers.copy():
-        chip.logger.removeHandler(handler)
-
-    # Reinit logger
-    chip._init_logger(step=step, index=index, in_run=True)
-
-    # Change working directory since the run may delete this folder
-    os.chdir(workdir)
-
-    image_name = get_image(chip, step, index)
-
-    # Pull image if needed
-    try:
-        image = client.images.get(image_name)
-    except docker.errors.ImageNotFound:
-        # Needs a lock to avoid downloading a bunch in parallel
-        image_repo, image_tag = image_name.split(':')
-        chip.logger.info(f'Pulling docker image {image_name}')
-        try:
-            image = client.images.pull(image_repo, tag=image_tag)
-        except docker.errors.APIError as e:
-            chip.logger.error(f'Unable to pull image: {e}')
-            image_src = image_repo.split('/')[0]
-            chip.logger.error(f"  if you are logged into {image_src} with expired credentials, "
-                              f"please use 'docker logout {image_src}'")
-            _haltstep(chip, chip.get('option', 'flow'), step, index)
-
-    email_file = default_email_credentials_file()
-    if is_windows:
-        # Hack to get around manifest merging
-        chip.set('option', 'cachedir', None)
-        cache_dir = '/sc_cache'
-        cwd = '/sc_docker'
-        builddir = f'{cwd}/build'
-
-        local_cfg = os.path.join(start_cwd, 'sc_docker.json')
-        job = chip.get('option', 'jobname')
-        cfg = f'{builddir}/{chip.design}/{job}/{step}/{index}/sc_docker.json'
-
-        user = None
-
-        volumes = [
-            f"{chip.cwd}:{cwd}:rw",
-            f"{get_cache_path(chip)}:{cache_dir}:rw"
-        ]
-        chip.logger.debug(f'Volumes: {volumes}')
-
-        env = {}
-
-        if os.path.exists(email_file):
-            env["HOME"] = "/sc_home"
-
-            volumes.append(f'{os.path.dirname(email_file)}:/sc_home/.sc:ro')
-    else:
-        cache_dir = get_cache_path(chip)
-        cwd = chip.cwd
-        builddir = chip.find_files('option', 'builddir')
-
-        local_cfg = os.path.abspath('sc_docker.json')
-        cfg = local_cfg
-
-        user = os.getuid()
-
-        rw_volumes, ro_volumes = get_volumes_directories(chip, cache_dir, workdir, step, index)
-        volumes = [
-            *[
-                f'{path}:{path}:rw' for path in rw_volumes
-            ],
-            *[
-                f'{path}:{path}:ro' for path in ro_volumes
-            ]
-        ]
-        chip.logger.debug(f'Read write volumes: {rw_volumes}')
-        chip.logger.debug(f'Read only volumes: {ro_volumes}')
-
-        env = {}
-        if os.path.exists(email_file):
-            env["HOME"] = "/sc_home"
-
-            volumes.append(f'{os.path.dirname(email_file)}:/sc_home/.sc:ro')
-
-    container = None
-    try:
-        container = client.containers.run(
-            image.id,
-            volumes=volumes,
-            labels=[
-                "siliconcompiler",
-                f"sc_node:{chip.design}:{step}{index}"
-            ],
-            user=user,
-            detach=True,
-            tty=True,
-            auto_remove=True,
-            environment=env)
-
-        # Write manifest to make it available to the docker runner
-        chip.write_manifest(local_cfg)
-
-        cachemap = []
-        for package in chip.getkeys('package', 'source'):
-            cachemap.append(f'{package}:{sc_path(chip, package)}')
-
-        chip.logger.info(f'Running in docker container: {container.name} ({container.short_id})')
-        args = [
-            '-cfg', cfg,
-            '-cwd', cwd,
-            '-builddir', builddir,
-            '-cachedir', cache_dir,
-            '-step', step,
-            '-index', index,
-            '-unset_scheduler'
-        ]
-        if not is_windows and cachemap:
-            args.append('-cachemap')
-            args.append(' '.join(cachemap))
-        cmd = f'python3 -m siliconcompiler.scheduler.run_node {" ".join(args)}'
-        exec_handle = client.api.exec_create(container.name, cmd)
-        stream = client.api.exec_start(exec_handle, stream=True)
-
-        # Print the log
-        for chunk in stream:
-            for line in chunk.decode().splitlines():
-                print(line)
-
-        if client.api.exec_inspect(exec_handle['Id']).get('ExitCode') != 0:
-            _haltstep(chip, chip.get('option', 'flow'), step, index, log=False)
-    finally:
-        # Ensure we clean up containers
-        if container:
-            try:
-                container.stop()
-            except docker.errors.APIError:
-                chip.logger.error('Failed to stop docker container')
-
-    # Restore working directory
-    os.chdir(start_cwd)

siliconcompiler/schema/journalingschema.py

@@ -1,242 +0,0 @@
-import copy
-import types
-
-from .baseschema import BaseSchema
-from .baseschema import json
-
-
-class JournalingSchema(BaseSchema):
-    """
-    This class provides the ability to record the schema transactions:
-    :meth:`set`, :meth:`add`, :meth:`remove`, and :meth:`unset`.
-
-    Args:
-        schema (:class:`BaseSchema`): schema to track
-        keyprefix (list of str): keypath to prefix on to recorded path
-    """
-
-    def __init__(self, schema, keyprefix=None):
-        if not isinstance(schema, BaseSchema):
-            raise TypeError(f"schema must be of BaseSchema type, not: {type(schema)}")
-        if isinstance(schema, JournalingSchema):
-            raise TypeError("schema must be of cannot be a JournalingSchema")
-
-        self.__schema = schema
-
-        journal_attrs = dir(self)
-
-        # Transfer access to internal schema
-        for param, value in self.__schema.__dict__.items():
-            setattr(self, param, value)
-        for param in dir(type(self.__schema)):
-            if param in journal_attrs:
-                continue
-            method = getattr(type(self.__schema), param)
-            if callable(method):
-                setattr(self, param, types.MethodType(method, self))
-
-        if not keyprefix:
-            self.__keyprefix = tuple()
-        else:
-            self.__keyprefix = tuple(keyprefix)
-
-        self.__record_types = set()
-        self.stop_journal()
-
-        self.__parent = self
-
-    @classmethod
-    def from_manifest(cls, filepath=None, cfg=None):
-        raise RuntimeError("Journal cannot be generated from manifest")
-
-    def get(self, *keypath, field='value', step=None, index=None):
-        get_ret = super().get(*keypath, field=field, step=step, index=index)
-        self.__record_journal("get", keypath, field=field, step=step, index=index)
-        if field == 'schema':
-            child = JournalingSchema(get_ret, keyprefix=[*self.__keyprefix, *keypath])
-            child.__parent = self.__parent
-            return child
-
-        return get_ret
-
-    def set(self, *args, field='value', clobber=True, step=None, index=None):
-        ret = super().set(*args, field=field, clobber=clobber, step=step, index=index)
-        if ret:
-            *keypath, value = args
-            self.__record_journal("set", keypath, value=value, field=field, step=step, index=index)
-        return ret
-
-    def add(self, *args, field='value', step=None, index=None):
-        ret = super().add(*args, field=field, step=step, index=index)
-        if ret:
-            *keypath, value = args
-            self.__record_journal("add", keypath, value=value, field=field, step=step, index=index)
-        return ret
-
-    def remove(self, *keypath):
-        self.__record_journal("remove", keypath)
-        return super().remove(*keypath)
-
-    def unset(self, *keypath, step=None, index=None):
-        self.__record_journal("unset", keypath, step=step, index=index)
-        return super().unset(*keypath, step=step, index=index)
-
-    def _from_dict(self, manifest, keypath, version=None):
-        if "__journal__" in manifest:
-            self.__journal = manifest["__journal__"]
-            del manifest["__journal__"]
-
-        self.__schema._from_dict(manifest, keypath, version=version)
-
-    def getdict(self, *keypath, include_default=True):
-        manifest = super().getdict(*keypath, include_default=include_default)
-
-        if self.__journal:
-            manifest["__journal__"] = self.get_journal()
-
-        return manifest
-
-    def get_journal(self):
-        """
-        Returns a copy of the current journal
-        """
-
-        return copy.deepcopy(self.__journal)
-
-    def is_journaling(self):
-        """
-        Returns true if the schema is currently setup for journaling
-        """
-        return self.__journal is not None
-
-    def get_journaling_types(self):
-        """
-        Returns the current schema accesses that are being recorded
-        """
-
-        return self.__record_types.copy()
-
-    def add_journaling_type(self, value):
-        """
-        Adds a new access type to the journal record.
-
-        Args:
-            value (str): access type
-        """
-
-        if value not in ("set", "add", "remove", "unset", "get"):
-            raise ValueError(f"{value} is not a valid type")
-
-        return self.__record_types.add(value)
-
-    def remove_journaling_type(self, value):
-        """
-        Removes a new access type to the journal record.
-
-        Args:
-            value (str): access type
-        """
-
-        try:
-            self.__record_types.remove(value)
-        except KeyError:
-            pass
-
-    def get_base_schema(self):
-        """
-        Returns the base schema
-        """
-
-        return self.__schema
-
-    def __record_journal(self, record_type, key, value=None, field=None, step=None, index=None):
-        '''
-        Record the schema transaction
-        '''
-
-        if self.__parent.__journal is None:
-            return
-
-        if record_type not in self.__parent.__record_types:
-            return
-
-        self.__parent.__journal.append({
-            "type": record_type,
-            "key": tuple([*self.__keyprefix, *key]),
-            "value": value,
-            "field": field,
-            "step": step,
-            "index": index
-        })
-
-    def start_journal(self):
-        '''
-        Start journaling the schema transactions
-        '''
-        self.__journal = []
-        self.add_journaling_type("set")
-        self.add_journaling_type("add")
-        self.add_journaling_type("remove")
-        self.add_journaling_type("unset")
-
-    def stop_journal(self):
-        '''
-        Stop journaling the schema transactions
-        '''
-        self.__journal = None
-        self.__record_types.clear()
-
-    def read_journal(self, filename):
-        '''
-        Reads a manifest and replays the journal
-        '''
-
-        with open(filename) as f:
-            data = json.loads(f.read())
-
-        self.import_journal(cfg=data)
-
-    def import_journal(self, schema=None, cfg=None):
-        '''
-        Import the journaled transactions from a different schema.
-        Only one argument is supported at a time.
-
-        Args:
-            schema (:class:`JournalingSchema`): schema to replay transactions from
-            cfg (dict): dictionary to replay transactions from
-        '''
-
-        if schema and cfg:
-            raise ValueError("only one argument is supported")
-
-        journal = []
-        if schema:
-            if isinstance(schema, JournalingSchema):
-                journal = schema.__journal
-            else:
-                raise TypeError(f"schema must be a JournalingSchema, not {type(schema)}")
-        elif cfg and "__journal__" in cfg:
-            journal = cfg["__journal__"]
-
-        if journal is None:
-            return
-
-        for action in journal:
-            record_type = action['type']
-            keypath = action['key']
-            value = action['value']
-            field = action['field']
-            step = action['step']
-            index = action['index']
-            if record_type == 'set':
-                self.set(*keypath, value, field=field, step=step, index=index)
-            elif record_type == 'add':
-                self.add(*keypath, value, field=field, step=step, index=index)
-            elif record_type == 'unset':
-                self.unset(*keypath, step=step, index=index)
-            elif record_type == 'remove':
-                self.remove(*keypath)
-            elif record_type == 'get':
-                continue
-            else:
-                raise ValueError(f'Unknown record type {record_type}')

siliconcompiler/tools/yosys/procs.tcl

@@ -1,71 +0,0 @@
-# This file contains a set of procedures that are shared
-# between syn_asic.tcl and syn_fpga.tcl
-
-proc post_techmap { { opt_args "" } } {
-    # perform techmap in case previous techmaps introduced constructs
-    # that need techmapping
-    yosys techmap
-    # Quick optimization
-    yosys opt {*}$opt_args -purge
-}
-
-proc sc_map_memory { lib_file techmap_file do_rom } {
-    set design_mod 0
-
-    if { $lib_file != "" } {
-        yosys memory_libmap -lib $lib_file
-        set design_mod 1
-    }
-
-    if { $do_rom } {
-        yosys memory_map -rom-only
-        set design_mod 1
-    }
-
-    if { $techmap_file != "" } {
-        yosys techmap -map $techmap_file
-        set design_mod 1
-    }
-
-    return $design_mod
-}
-
-proc sc_apply_params { } {
-    global sc_design
-
-    yosys chparam -list $sc_design
-    if { [sc_cfg_exists option param] } {
-        yosys echo off
-        set module_params [yosys tee -q -s result.string chparam -list $sc_design]
-        yosys echo on
-
-        dict for {key value} [sc_cfg_get option param] {
-            if { ![string is integer $value] } {
-                set value [concat \"$value\"]
-            }
-
-            if { [string first $key $module_params] != -1 } {
-                yosys chparam -set $key $value $sc_design
-            } else {
-                puts "Warning: $key is not a defined parameter in $sc_design"
-            }
-        }
-    }
-}
-
-proc sc_get_scratchpad { name } {
-    yosys echo off
-    set value [yosys tee -q -s result.string scratchpad -get $name]
-    yosys echo on
-
-    return $value
-}
-
-proc sc_load_plugin { name } {
-    catch { yosys tee -q -s sc.load.test plugin -i $name }
-    set load_test [sc_get_scratchpad sc.load.test]
-    if { [string first "ERROR" $load_test] == -1 } {
-        return 1
-    }
-    return 0
-}

siliconcompiler/toolscripts/rhel9/install-yosys-parmys.sh

@@ -1,68 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-# Get directory of script
-src_path=$(cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P)/..
-
-USE_SUDO_INSTALL="${USE_SUDO_INSTALL:-yes}"
-if [ "${USE_SUDO_INSTALL:-yes}" = "yes" ]; then
-    SUDO_INSTALL=sudo
-else
-    SUDO_INSTALL=""
-fi
-
-sudo yum install -y git
-
-mkdir -p deps
-cd deps
-
-git clone $(python3 ${src_path}/_tools.py --tool yosys-parmys --field git-url) yosys-parmys
-cd yosys-parmys
-git checkout $(python3 ${src_path}/_tools.py --tool yosys-parmys --field git-commit)
-git submodule update --init --recursive
-
-# apply patch
-cat > build_patch <<EOF
-diff --git a/parmys/parmys-plugin/Makefile b/parmys/parmys-plugin/Makefile
-index dbb3eb11e..cb85631bc 100644
---- a/parmys/parmys-plugin/Makefile
-+++ b/parmys/parmys-plugin/Makefile
-@@ -49,7 +49,7 @@ VTR_INSTALL_DIR ?= /usr/local
- 
- include ../Makefile_plugin.common
- 
--CXXFLAGS += -std=c++14 -Wall -W -Wextra \\
-+CXXFLAGS += -std=c++17 -Wall -W -Wextra \\
-             -Wno-deprecated-declarations \\
-             -Wno-unused-parameter \\
-             -I. \\
-diff --git a/parmys/parmys-plugin/parmys_update.cc b/parmys/parmys-plugin/parmys_update.cc
-index ef55213c5..4e4d6dd15 100644
---- a/parmys/parmys-plugin/parmys_update.cc
-+++ b/parmys/parmys-plugin/parmys_update.cc
-@@ -506,9 +506,9 @@ void define_logical_function_yosys(nnode_t *node, Module *module)
-         lutptr = &cell->parameters.at(ID::LUT);
-         for (int i = 0; i < (1 << node->num_input_pins); i++) {
-             if (i == 3 || i == 5 || i == 6 || i == 7) //"011 1\n101 1\n110 1\n111 1\n"
--                lutptr->bits.at(i) = RTLIL::State::S1;
-+                lutptr->bits().at(i) = RTLIL::State::S1;
-             else
--                lutptr->bits.at(i) = RTLIL::State::S0;
-+                lutptr->bits().at(i) = RTLIL::State::S0;
-         }
-     } else {
-         cell->parameters[ID::A_WIDTH] = RTLIL::Const(int(node->num_input_pins));
-EOF
-
-git apply build_patch
-
-export VTR_INSTALL_DIR=$(dirname $(which vpr))/..
-YOSYS_PLUGIN=$(yosys-config --datdir)/plugins/
-
-cd parmys
-
-make -j$(nproc)
-$SUDO_INSTALL mkdir -p $YOSYS_PLUGIN
-$SUDO_INSTALL cp parmys-plugin/build/parmys.so $YOSYS_PLUGIN
-cd -

siliconcompiler/toolscripts/ubuntu22/install-yosys-parmys.sh

@@ -1,68 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-# Get directory of script
-src_path=$(cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P)/..
-
-USE_SUDO_INSTALL="${USE_SUDO_INSTALL:-yes}"
-if [ "${USE_SUDO_INSTALL:-yes}" = "yes" ]; then
-    SUDO_INSTALL=sudo
-else
-    SUDO_INSTALL=""
-fi
-
-sudo apt-get install -y git
-
-mkdir -p deps
-cd deps
-
-git clone $(python3 ${src_path}/_tools.py --tool yosys-parmys --field git-url) yosys-parmys
-cd yosys-parmys
-git checkout $(python3 ${src_path}/_tools.py --tool yosys-parmys --field git-commit)
-git submodule update --init --recursive
-
-# apply patch
-cat > build_patch <<EOF
-diff --git a/parmys/parmys-plugin/Makefile b/parmys/parmys-plugin/Makefile
-index dbb3eb11e..cb85631bc 100644
---- a/parmys/parmys-plugin/Makefile
-+++ b/parmys/parmys-plugin/Makefile
-@@ -49,7 +49,7 @@ VTR_INSTALL_DIR ?= /usr/local
- 
- include ../Makefile_plugin.common
- 
--CXXFLAGS += -std=c++14 -Wall -W -Wextra \\
-+CXXFLAGS += -std=c++17 -Wall -W -Wextra \\
-             -Wno-deprecated-declarations \\
-             -Wno-unused-parameter \\
-             -I. \\
-diff --git a/parmys/parmys-plugin/parmys_update.cc b/parmys/parmys-plugin/parmys_update.cc
-index ef55213c5..4e4d6dd15 100644
---- a/parmys/parmys-plugin/parmys_update.cc
-+++ b/parmys/parmys-plugin/parmys_update.cc
-@@ -506,9 +506,9 @@ void define_logical_function_yosys(nnode_t *node, Module *module)
-         lutptr = &cell->parameters.at(ID::LUT);
-         for (int i = 0; i < (1 << node->num_input_pins); i++) {
-             if (i == 3 || i == 5 || i == 6 || i == 7) //"011 1\n101 1\n110 1\n111 1\n"
--                lutptr->bits.at(i) = RTLIL::State::S1;
-+                lutptr->bits().at(i) = RTLIL::State::S1;
-             else
--                lutptr->bits.at(i) = RTLIL::State::S0;
-+                lutptr->bits().at(i) = RTLIL::State::S0;
-         }
-     } else {
-         cell->parameters[ID::A_WIDTH] = RTLIL::Const(int(node->num_input_pins));
-EOF
-
-git apply build_patch
-
-export VTR_INSTALL_DIR=$(dirname $(which vpr))/..
-YOSYS_PLUGIN=$(yosys-config --datdir)/plugins/
-
-cd parmys
-
-make -j$(nproc)
-$SUDO_INSTALL mkdir -p $YOSYS_PLUGIN
-$SUDO_INSTALL cp parmys-plugin/build/parmys.so $YOSYS_PLUGIN
-cd -