siliconcompiler 0.35.3__py3-none-any.whl → 0.35.4__py3-none-any.whl

siliconcompiler/toolscripts/_tools.json

@@ -1,7 +1,7 @@
 {
   "openroad": {
     "git-url": "https://github.com/The-OpenROAD-Project/OpenROAD.git",
-    "git-commit": "f22f811d9d338b1dbc6fe854d35b53757596449d",
+    "git-commit": "dd56d50c413ecd215117898437c57bec68b59a87",
     "docker-cmds": [
       "# Remove OR-Tools files",
       "RUN rm -f $SC_PREFIX/Makefile $SC_PREFIX/README.md",
@@ -17,7 +17,7 @@
   },
   "opensta": {
     "git-url": "https://github.com/parallaxsw/OpenSTA.git",
-    "git-commit": "cf903f4db688e6f89d732ef28cce7b8185587201",
+    "git-commit": "bd3efdc322c0677eb8e3d76f22ab297f7a6048b9",
     "auto-update": true
   },
   "netgen": {
@@ -41,7 +41,7 @@
     "auto-update": false
   },
   "klayout": {
-    "version": "0.30.4",
+    "version": "0.30.5",
     "git-url": "https://github.com/KLayout/klayout.git",
     "auto-update": true,
     "run-version": "source version.sh && echo $KLAYOUT_VERSION",
@@ -55,7 +55,7 @@
   },
   "sv2v": {
     "git-url": "https://github.com/zachjs/sv2v.git",
-    "git-commit": "c1ce7d067b0a7edf8d1589dcb5731265c854c490",
+    "git-commit": "d3812098d8feb2d04cc4cb55e034a1f2338db02f",
     "auto-update": true
   },
   "verilator": {
@@ -101,7 +101,7 @@
   },
   "yosys": {
     "git-url": "https://github.com/YosysHQ/yosys.git",
-    "git-commit": "v0.58",
+    "git-commit": "v0.60",
     "version-prefix": "",
     "auto-update": true
   },
@@ -145,7 +145,7 @@
   },
   "yosys-slang": {
     "git-url": "https://github.com/povik/yosys-slang.git",
-    "git-commit": "f44908907726e684965ba71959cc147df50c0357",
+    "git-commit": "875539b8cae5a2ac7c86fee43b1e38a743ee8659",
     "docker-depends": "yosys",
     "auto-update": true
   },

siliconcompiler/utils/__init__.py

@@ -28,6 +28,14 @@ if TYPE_CHECKING:
 
 
 def link_symlink_copy(srcfile, dstfile):
+    """
+    Attempts to link a source file to a destination using hard link, symbolic link,
+    or copy, in that order.
+
+    Args:
+        srcfile (str): Path to the source file.
+        dstfile (str): Path to the destination file.
+    """
     # first try hard linking, then symbolic linking,
     # and finally just copy the file
     for method in [os.link, os.symlink, shutil.copy2]:
@@ -40,6 +48,14 @@ def link_symlink_copy(srcfile, dstfile):
 
 
 def link_copy(srcfile, dstfile):
+    """
+    Attempts to link a source file to a destination using hard link or copy,
+    in that order.
+
+    Args:
+        srcfile (str): Path to the source file.
+        dstfile (str): Path to the destination file.
+    """
     # first try hard linking or just copy the file
     for method in [os.link, shutil.copy2]:
         try:
@@ -51,7 +67,15 @@ def link_copy(srcfile, dstfile):
 
 
 def get_file_ext(filename: str) -> str:
-    '''Get base file extension for a given path, disregarding .gz.'''
+    """
+    Get base file extension for a given path, disregarding .gz.
+
+    Args:
+        filename (str): The filename to extract the extension from.
+
+    Returns:
+        str: The file extension (e.g., 'v', 'py') without the dot.
+    """
     if filename.lower().endswith('.gz'):
         filename = os.path.splitext(filename)[0]
     filetype = os.path.splitext(filename)[1].lower().lstrip('.')
@@ -60,7 +84,10 @@ def get_file_ext(filename: str) -> str:
 
 def get_default_iomap() -> Dict[str, str]:
     """
-    Default input file map for SC with filesets and extensions
+    Returns the default input file map for SC with filesets and extensions.
+
+    Returns:
+        Dict[str, str]: A dictionary mapping file extensions to their SiliconCompiler type.
     """
 
     # Record extensions:
@@ -175,26 +202,71 @@ def get_default_iomap() -> Dict[str, str]:
     return default_iomap
 
 
+def default_sc_dir() -> str:
+    """
+    Returns the default SiliconCompiler configuration directory.
+
+    Returns:
+        str: The absolute path to the .sc directory in the user's home folder.
+    """
+    return os.path.join(Path.home(), '.sc')
+
+
+def default_sc_path(path: str) -> str:
+    """
+    Returns a path relative to the default SiliconCompiler configuration directory.
+
+    Args:
+        path (str): The relative path to append to the default SC directory.
+
+    Returns:
+        str: The absolute path joined with the default SC directory.
+    """
+    return os.path.join(default_sc_dir(), path)
+
+
 def default_credentials_file() -> str:
-    cfg_file = os.path.join(Path.home(), '.sc', 'credentials')
+    """
+    Returns the default path for the SiliconCompiler credentials file.
 
-    return cfg_file
+    Returns:
+        str: The absolute path to the credentials file.
+    """
+    return default_sc_path('credentials')
 
 
 def default_cache_dir() -> str:
-    cfg_file = os.path.join(Path.home(), '.sc', 'cache')
+    """
+    Returns the default path for the SiliconCompiler cache directory.
 
-    return cfg_file
+    Returns:
+        str: The absolute path to the cache directory.
+    """
+    return default_sc_path('cache')
 
 
 def default_email_credentials_file() -> str:
-    cfg_file = os.path.join(Path.home(), '.sc', 'email.json')
+    """
+    Returns the default path for the SiliconCompiler email credentials file.
 
-    return cfg_file
+    Returns:
+        str: The absolute path to the email.json file.
+    """
+    return default_sc_path('email.json')
 
 
 @contextlib.contextmanager
 def sc_open(path: str, *args, **kwargs):
+    """
+    A context manager for opening files with default settings convenient for SC.
+
+    Sets ``errors='ignore'`` and ``newline='\n'`` by default unless overridden.
+
+    Args:
+        path (str): The file path to open.
+        *args: Positional arguments passed to the builtin open().
+        **kwargs: Keyword arguments passed to the builtin open().
+    """
     if 'errors' not in kwargs:
         kwargs['errors'] = 'ignore'
     kwargs["newline"] = "\n"
@@ -212,6 +284,17 @@ def get_file_template(path: str,
                               os.path.dirname(os.path.abspath(__file__))),
                           'data',
                           'templates')) -> Template:
+    """
+    Retrieves a Jinja2 template object for the specified file.
+
+    Args:
+        path (str): The name of the template file or an absolute path to it.
+        root (str, optional): The root directory to search for templates if path
+            is relative. Defaults to the SiliconCompiler data/templates directory.
+
+    Returns:
+        Template: The loaded Jinja2 Template object.
+    """
     if os.path.isabs(path):
         root = os.path.dirname(path)
         path = os.path.basename(path)
@@ -225,6 +308,20 @@ def get_file_template(path: str,
 
 #######################################
 def safecompare(value: Union[int, float], op: str, goal: Union[int, float]) -> bool:
+    """
+    Compares a value against a goal using a string operator.
+
+    Args:
+        value (Union[int, float]): The value to compare.
+        op (str): The comparison operator ('>', '>=', '<', '<=', '==', '!=').
+        goal (Union[int, float]): The target value to compare against.
+
+    Returns:
+        bool: The result of the comparison.
+
+    Raises:
+        ValueError: If the provided operator is not supported.
+    """
     # supported relational operations
     # >, >=, <=, <. ==, !=
     if op == ">":
@@ -244,75 +341,127 @@ def safecompare(value: Union[int, float], op: str, goal: Union[int, float]) -> b
 
 
 ###########################################################################
-def grep(project: "Project", args: str, line: str) -> Union[None, str]:
+def grep(logger: logging.Logger, args: str, line: str) -> Union[None, str]:
     """
     Emulates the Unix grep command on a string.
 
-    Emulates the behavior of the Unix grep command that is etched into
-    our muscle memory. Partially implemented, not all features supported.
-    The function returns None if no match is found.
-
     Args:
-        arg (string): Command line arguments for grep command
-        line (string): Line to process
+        logger (logging.Logger): used for logging errors.
+        args (str): Command line arguments for grep command.
+        line (str): Line to process.
 
     Returns:
-        Result of grep command (string).
-
+        Union[None, str]: Result of grep command (string) or None if no match.
     """
 
     # Quick return if input is None
     if line is None:
         return None
 
-    # Partial list of supported grep options
+    # --- 1. Initialize Options and Parse Arguments ---
     options = {
         '-v': False,  # Invert the sense of matching
-        '-i': False,  # Ignore case distinctions in patterns and data
-        '-E': False,  # Interpret PATTERNS as extended regular expressions.
-        '-e': False,  # Safe interpretation of pattern starting with "-"
-        '-x': False,  # Select only matches that exactly match the whole line.
-        '-o': False,  # Print only the match parts of a matching line
-        '-w': False}  # Select only lines containing matches that form whole words.
+        '-i': False,  # Ignore case distinctions
+        '-E': False,  # Extended regular expressions (Python's 're' module is ERE by default)
+        '-e': False,  # Pattern starts with '-' (simplified logic)
+        '-x': False,  # Exact line match
+        '-o': False,  # Print only the match
+        '-w': False}  # Whole word match
+
+    parts = args.split()
+    pattern = ""
+    pattern_start_index = -1
+
+    # Identify switches and the start of the pattern
+    for i, part in enumerate(parts):
+        # Check for switch starting with '-' and not just '-'
+        if part.startswith('-') and len(part) > 1 and part != '-e':
+            # Handle concatenated switches (e.g., -vi)
+            is_valid_switch_group = True
+            for char in part[1:]:
+                switch = f"-{char}"
+                if switch in options:
+                    options[switch] = True
+                else:
+                    logger.error(f"Unknown switch: {switch}")
+                    is_valid_switch_group = False
+                    break
+            if not is_valid_switch_group:
+                # If an invalid switch was found, the rest must be the pattern
+                pattern_start_index = i
+                break
+        elif part == '-e':
+            # The next part is the pattern, regardless of what it looks like
+            options['-e'] = True
+            if i + 1 < len(parts):
+                pattern_start_index = i + 1
+            break
+        elif not pattern.strip():
+            # First non-switch part is the start of the pattern
+            pattern_start_index = i
+            break
 
-    # Split into repeating switches and everything else
-    match = re.match(r'\s*((?:\-\w\s)*)(.*)', args)
+    # Assemble the pattern from the determined starting index
+    if pattern_start_index != -1:
+        pattern = " ".join(parts[pattern_start_index:])
 
-    if not match:
+    if not pattern:
         return None
 
-    pattern = match.group(2)
+    # --- 2. Prepare Regex Flags and Pattern Modifiers ---
 
-    # Split space separated switch string into list
-    switches = match.group(1).strip().split(' ')
+    regex_flags = 0
+    if options['-i']:
+        regex_flags |= re.IGNORECASE
 
-    # Find special -e switch update the pattern
-    for i in range(len(switches)):
-        if switches[i] == "-e":
-            if i != (len(switches)):
-                pattern = ' '.join(switches[i + 1:]) + " " + pattern
-                switches = switches[0:i + 1]
-                break
-            options["-e"] = True
-        elif switches[i] in options.keys():
-            options[switches[i]] = True
-        elif switches[i] != '':
-            project.logger.error(switches[i])
-
-    # REGEX
-    # TODO: add all the other optinos
-    match = re.search(rf"({pattern})", line)
-    if bool(match) == bool(options["-v"]):
+    # Apply Whole Word (-w)
+    pattern_to_search = pattern
+    if options['-w']:
+        # Apply word boundaries
+        pattern_to_search = rf"\b({pattern})\b"
+
+    # Apply Whole Line (-x)
+    if options['-x']:
+        # Exact line match, using the prepared pattern (which may have \b already)
+        pattern_to_search = rf"^{pattern_to_search}$"
+
+    # --- 3. Perform Search ---
+    try:
+        # re.search is used to find the pattern anywhere in the line
+        match = re.search(pattern_to_search, line, regex_flags)
+    except re.error as e:
+        # Handle cases where the pattern itself is invalid regex
+        logger.error(f"Invalid regex pattern '{pattern}': {e}")
         return None
+
+    # --- 4. Handle Inversion (-v) and Final Return ---
+
+    # Check if a result should be returned: (Match found) XOR (Invert is on)
+    should_return = bool(match) != options["-v"]
+
+    if should_return:
+        if options['-o'] and match:
+            # Print only the match part (which is match.group(0))
+            return match.group(0)
+        else:
+            # Return the whole line (standard behavior)
+            return line
     else:
-        return line
+        # Match found AND inverted, OR Match NOT found AND not inverted
+        return None
 
 
 def get_plugins(system: str, name: Optional[str] = None) -> List[Callable]:
     '''
-    Search for python modules with a specific function
-    '''
+    Search for python modules with a specific function.
 
+    Args:
+        system (str): The system/group to search within (e.g. 'siliconcompiler.metrics').
+        name (str, optional): Specific plugin name to filter by.
+
+    Returns:
+        List[Callable]: A list of loaded plugin functions.
+    '''
     plugins = []
     discovered_plugins = entry_points(group=f'siliconcompiler.{system}')
     for plugin in discovered_plugins:
@@ -326,6 +475,19 @@ def get_plugins(system: str, name: Optional[str] = None) -> List[Callable]:
 
 
 def truncate_text(text: str, width: int) -> str:
+    """
+    Truncates text to a specific width, replacing the middle with '...'.
+
+    Attempts to preserve the end of the string if it appears to be a version number
+    or numeric index.
+
+    Args:
+        text (str): The text to truncate.
+        width (int): The maximum desired width of the text.
+
+    Returns:
+        str: The truncated text.
+    """
     width = max(width, 5)
 
     if len(text) <= width:
@@ -349,7 +511,10 @@ def get_cores(physical: bool = False) -> int:
     Get max number of cores for this machine.
 
     Args:
-        physical (boolean): if true, only count physical cores
+        physical (bool): if true, only count physical cores. Defaults to False.
+
+    Returns:
+        int: The number of available cores. Defaults to 1 if detection fails.
     '''
 
     cores = psutil.cpu_count(logical=not physical)
@@ -367,6 +532,13 @@ def get_cores(physical: bool = False) -> int:
 
 
 def print_traceback(logger: logging.Logger, exception: Exception):
+    """
+    Prints the full traceback of an exception to the provided logger.
+
+    Args:
+        logger (logging.Logger): The logger instance to write the traceback to.
+        exception (Exception): The exception to log.
+    """
     logger.error(f'{exception}')
     trace = StringIO()
     traceback.print_tb(exception.__traceback__, file=trace)
@@ -376,6 +548,13 @@ def print_traceback(logger: logging.Logger, exception: Exception):
 
 
 class FilterDirectories:
+    """
+    A helper class to filter directories and files during file collection.
+
+    This class prevents collecting files from the home directory, the build directory,
+    and hidden files (on Linux, Windows, and macOS). It also enforces a limit on the
+    number of files collected.
+    """
     def __init__(self, project: "Project"):
         self.file_count = 0
         self.directory_file_limit = None
@@ -391,6 +570,19 @@ class FilterDirectories:
         return builddir(self.project)
 
     def filter(self, path: str, files: List[str]) -> List[str]:
+        """
+        Filters a list of files based on the current directory path.
+
+        Args:
+            path (str): The current directory path being traversed.
+            files (List[str]): The list of filenames in that directory.
+
+        Returns:
+            List[str]: A list of filenames that should be excluded (i.e., hidden files)
+                or handled specifically. Note that despite the name returning 'hidden_files',
+                the logic is essentially identifying what to *exclude* from the main processing
+                loop in some contexts, or identifying hidden files to ignore.
+        """
         if pathlib.Path(path) == pathlib.Path.home():
             # refuse to collect home directory
             self.logger.error(f'Cannot collect user home directory: {path}')

siliconcompiler/utils/curation.py

@@ -5,9 +5,10 @@ import os.path
 
 from typing import List, Optional, TYPE_CHECKING
 
+from siliconcompiler.schema import BaseSchema, Parameter
 from siliconcompiler.schema.parametervalue import NodeListValue, NodeSetValue
 from siliconcompiler.utils import FilterDirectories
-from siliconcompiler.utils.paths import collectiondir
+from siliconcompiler.utils.paths import collectiondir, cwdir
 from siliconcompiler.scheduler import SchedulerNode
 from siliconcompiler.flowgraph import RuntimeFlowgraph
 
@@ -43,16 +44,42 @@ def collect(project: "Project",
 
     if not directory:
         directory = collectiondir(project)
+    if not directory:
+        raise ValueError("unable to determine collection directory")
+
     directory = os.path.abspath(directory)
 
-    # Remove existing directory
+    # Move existing directory
+    prev_dir = None
     if os.path.exists(directory):
-        shutil.rmtree(directory)
+        prev_dir = os.path.join(os.path.dirname(directory), "sc_previous_collection")
+        os.rename(directory, prev_dir)
     os.makedirs(directory)
 
     if verbose:
         project.logger.info(f'Collecting files to: {directory}')
 
+    cwd = cwdir(project)
+
+    def find_files(*key, step: Optional[str] = None, index: Optional[str] = None):
+        """
+        Find the files in the filesystem, otherwise look in previous collection
+        """
+        e = None
+        try:
+            return BaseSchema._find_files(project, *key, step=step, index=index,
+                                          cwd=cwd,
+                                          collection_dir=directory)
+        except FileNotFoundError as err:
+            e = err
+        if prev_dir:
+            # Try previous location next
+            return BaseSchema._find_files(project, *key, step=step, index=index,
+                                          cwd=cwd,
+                                          collection_dir=prev_dir)
+        if e:
+            raise e from None
+
     dirs = {}
     files = {}
 
@@ -75,17 +102,18 @@ def collect(project: "Project",
             # skip flow files files from builds
             continue
 
-        leaftype = project.get(*key, field='type')
+        param: Parameter = project.get(*key, field=None)
+        leaftype: str = param.get(field='type')
         is_dir = "dir" in leaftype
         is_file = "file" in leaftype
 
         if not is_dir and not is_file:
             continue
 
-        if not project.get(*key, field='copy'):
+        if not param.get(field='copy'):
             continue
 
-        for values, step, index in project.get(*key, field=None).getvalues(return_values=False):
+        for values, step, index in param.getvalues(return_values=False):
             if not values.has_value:
                 continue
 
@@ -99,73 +127,78 @@ def collect(project: "Project",
             else:
                 files[(key, step, index)] = values
 
-    path_filter = FilterDirectories(project)
-    for key, step, index in sorted(dirs.keys()):
-        abs_paths = project.find_files(*key, step=step, index=index)
+    try:
+        path_filter = FilterDirectories(project)
+        for key, step, index in sorted(dirs.keys()):
+            abs_paths = find_files(*key, step=step, index=index)
 
-        new_paths = set()
+            new_paths = set()
 
-        if not isinstance(abs_paths, (list, tuple, set)):
-            abs_paths = [abs_paths]
+            if not isinstance(abs_paths, (list, tuple, set)):
+                abs_paths = [abs_paths]
 
-        abs_paths = zip(abs_paths, dirs[(key, step, index)])
-        abs_paths = sorted(abs_paths, key=lambda p: p[0])
+            abs_paths = zip(abs_paths, dirs[(key, step, index)])
+            abs_paths = sorted(abs_paths, key=lambda p: p[0])
 
-        for abs_path, value in abs_paths:
-            if not abs_path:
-                raise FileNotFoundError(f"{value.get()} could not be copied")
+            for abs_path, value in abs_paths:
+                if not abs_path:
+                    raise FileNotFoundError(f"{value.get()} could not be copied")
 
-            if abs_path.startswith(directory):
-                # File already imported in directory
-                continue
+                if abs_path.startswith(directory):
+                    # File already imported in directory
+                    continue
 
-            imported = False
-            for new_path in new_paths:
-                if abs_path.startwith(new_path):
-                    imported = True
-                    break
-            if imported:
-                continue
+                imported = False
+                for new_path in new_paths:
+                    if abs_path.startswith(new_path):
+                        imported = True
+                        break
+                if imported:
+                    continue
 
-            new_paths.add(abs_path)
+                new_paths.add(abs_path)
 
-            import_path = os.path.join(directory, value.get_hashed_filename())
-            if os.path.exists(import_path):
-                continue
+                import_path = os.path.join(directory, value.get_hashed_filename())
+                if os.path.exists(import_path):
+                    continue
 
-            if whitelist is not None and abs_path not in whitelist:
-                raise RuntimeError(f'{abs_path} is not on the approved collection list.')
+                if whitelist is not None and abs_path not in whitelist:
+                    raise RuntimeError(f'{abs_path} is not on the approved collection list.')
 
-            if verbose:
-                project.logger.info(f"  Collecting directory: {abs_path}")
-            path_filter.abspath = abs_path
-            shutil.copytree(abs_path, import_path, ignore=path_filter.filter)
-            path_filter.abspath = None
+                if verbose:
+                    project.logger.info(f"  Collecting directory: {abs_path}")
+                path_filter.abspath = abs_path
+                shutil.copytree(abs_path, import_path, ignore=path_filter.filter)
+                path_filter.abspath = None
 
-    for key, step, index in sorted(files.keys()):
-        abs_paths = project.find_files(*key, step=step, index=index)
+        for key, step, index in sorted(files.keys()):
+            abs_paths = find_files(*key, step=step, index=index)
 
-        if not isinstance(abs_paths, (list, tuple, set)):
-            abs_paths = [abs_paths]
+            if not isinstance(abs_paths, (list, tuple, set)):
+                abs_paths = [abs_paths]
 
-        abs_paths = zip(abs_paths, files[(key, step, index)])
-        abs_paths = sorted(abs_paths, key=lambda p: p[0])
+            abs_paths = zip(abs_paths, files[(key, step, index)])
+            abs_paths = sorted(abs_paths, key=lambda p: p[0])
 
-        for abs_path, value in abs_paths:
-            if not abs_path:
-                raise FileNotFoundError(f"{value.get()} could not be copied")
+            for abs_path, value in abs_paths:
+                if not abs_path:
+                    raise FileNotFoundError(f"{value.get()} could not be copied")
 
-            if abs_path.startswith(directory):
-                # File already imported in directory
-                continue
+                if abs_path.startswith(directory):
+                    # File already imported in directory
+                    continue
 
-            import_path = os.path.join(directory, value.get_hashed_filename())
-            if os.path.exists(import_path):
-                continue
+                import_path = os.path.join(directory, value.get_hashed_filename())
+                if os.path.exists(import_path):
+                    continue
 
-            if verbose:
-                project.logger.info(f"  Collecting file: {abs_path}")
-            shutil.copy2(abs_path, import_path)
+                if verbose:
+                    project.logger.info(f"  Collecting file: {abs_path}")
+                shutil.copy2(abs_path, import_path)
+    finally:
+        if prev_dir:
+            # Delete existing directory
+            shutil.rmtree(prev_dir)
 
 
 def archive(project: "Project",