siliconcompiler 0.35.2__py3-none-any.whl → 0.35.4__py3-none-any.whl

siliconcompiler/library.py

@@ -7,7 +7,7 @@ from siliconcompiler.schema_support.filesetschema import FileSetSchema
 from siliconcompiler.schema_support.pathschema import PathSchema
 from siliconcompiler.schema import NamedSchema, BaseSchema
 
-from siliconcompiler.schema import EditableSchema, Parameter, Scope, PerNode
+from siliconcompiler.schema import EditableSchema, Parameter, Scope, PerNode, LazyLoad
 from siliconcompiler.schema.utils import trim
 
 
@@ -99,7 +99,8 @@ class ToolLibrarySchema(LibrarySchema):
 
     def _from_dict(self, manifest: Dict,
                    keypath: Union[List[str], Tuple[str, ...]],
-                   version: Optional[Tuple[int, ...]] = None) \
+                   version: Optional[Tuple[int, ...]] = None,
+                   lazyload: LazyLoad = LazyLoad.ON) \
             -> Tuple[Set[Tuple[str, ...]], Set[Tuple[str, ...]]]:
         """
         Constructs a schema from a dictionary.
@@ -112,7 +113,7 @@ class ToolLibrarySchema(LibrarySchema):
         Returns:
             dict: The constructed dictionary.
         """
-        if "tool" in manifest:
+        if not lazyload.is_enforced and "tool" in manifest:
             # collect tool keys
             tool_keys = self.allkeys("tool")
 
@@ -136,7 +137,7 @@ class ToolLibrarySchema(LibrarySchema):
             if not manifest["tool"]:
                 del manifest["tool"]
 
-        return super()._from_dict(manifest, keypath, version)
+        return super()._from_dict(manifest, keypath, version=version, lazyload=lazyload)
 
     def _generate_doc(self, doc,
                       ref_root: str = "",

siliconcompiler/package/__init__.py

@@ -16,6 +16,7 @@ import logging
 import os
 import random
 import re
+import shutil
 import time
 import threading
 import uuid
@@ -29,7 +30,8 @@ from importlib.metadata import distributions, distribution
 from pathlib import Path
 from urllib import parse as url_parse
 
-from siliconcompiler.utils import get_plugins
+from siliconcompiler.utils import get_plugins, default_cache_dir
+from siliconcompiler.utils.paths import cwdirsafe
 
 if TYPE_CHECKING:
     from siliconcompiler.project import Project
@@ -370,7 +372,7 @@ class RemoteResolver(Resolver):
         Returns:
             Path: The path to the cache directory.
         """
-        default_path = os.path.join(Path.home(), '.sc', 'cache')
+        default_path = default_cache_dir()
         if not root:
             return Path(default_path)
 
@@ -380,8 +382,7 @@ class RemoteResolver(Resolver):
             if path:
                 path = root.find_files('option', 'cachedir', missing_ok=True)
                 if not path:
-                    path = os.path.join(getattr(root, "_Project__cwd", os.getcwd()),
-                                        root.get('option', 'cachedir'))
+                    path = os.path.join(cwdirsafe(root), root.get('option', 'cachedir'))
         if not path:
             path = default_path
 
@@ -543,7 +544,17 @@ class RemoteResolver(Resolver):
             if self.check_cache():
                 return self.cache_path
 
-            self.resolve_remote()
+            try:
+                self.resolve_remote()
+            except BaseException as e:
+                # Exception occurred, so need to cleanup
+                try:
+                    shutil.rmtree(self.cache_path)
+                except BaseException as cleane:
+                    self.logger.error(f"Exception occurred during cleanup: {cleane} "
+                                      f"({cleane.__class__.__name__})")
+                raise e from None
+
             self.set_changed()
             return self.cache_path
 
@@ -560,7 +571,7 @@ class FileResolver(Resolver):
         if source.startswith("file://"):
             source = source[7:]
         if source[0] != "$" and not os.path.isabs(source):
-            source = os.path.join(getattr(root, "_Project__cwd", os.getcwd()), source)
+            source = os.path.join(cwdirsafe(root), source)
 
         super().__init__(name, root, f"file://{source}", None)
 

siliconcompiler/package/https.py

@@ -114,11 +114,16 @@ class HTTPResolver(RemoteResolver):
         except tarfile.ReadError:
             fileobj.seek(0)
             try:
-                with zipfile.ZipFile(fileobj) as zip_ref:
-                    zip_ref.extractall(path=self.cache_path)
-            except zipfile.BadZipFile:
-                raise TypeError(f"Could not extract file from {data_url}. "
-                                "File is not a valid tar.gz or zip archive.")
+                with tarfile.open(fileobj=fileobj, mode='r:bz2') as tar_ref:
+                    tar_ref.extractall(path=self.cache_path)
+            except tarfile.ReadError:
+                fileobj.seek(0)
+                try:
+                    with zipfile.ZipFile(fileobj) as zip_ref:
+                        zip_ref.extractall(path=self.cache_path)
+                except zipfile.BadZipFile:
+                    raise TypeError(f"Could not extract file from {data_url}. "
+                                    "File is not a valid tar.gz or zip archive.")
 
         # --- GitHub-specific directory flattening ---
         # GitHub archives often have a single top-level directory like 'repo-v1.0'.

siliconcompiler/project.py

@@ -5,10 +5,11 @@ import uuid
 
 import os.path
 
-from typing import Union, List, Tuple, TextIO, Optional
+from typing import Union, List, Tuple, TextIO, Optional, Dict, Set
 
 from siliconcompiler.schema import BaseSchema, NamedSchema, EditableSchema, Parameter, Scope, \
-    __version__ as schema_version
+    __version__ as schema_version, \
+    LazyLoad
 
 from siliconcompiler import Design
 from siliconcompiler import Flowgraph
@@ -107,7 +108,7 @@ class Project(PathSchemaBase, CommandLineSchema, BaseSchema):
                 is not intended for external use."""))
 
         schema.insert("checklist", "default", Checklist())
-        schema.insert("library", BaseSchema())
+        schema.insert("library", _ProjectLibrary())
         schema.insert("flowgraph", "default", Flowgraph())
         schema.insert("metric", MetricSchema())
         schema.insert("record", RecordSchema())
@@ -289,26 +290,11 @@ class Project(PathSchemaBase, CommandLineSchema, BaseSchema):
         """
         return Project.__name__
 
-    def __populate_deps(self, obj: DependencySchema = None):
-        """
-        Ensures that all loaded dependencies (like libraries) within the project
-        contain correct internal pointers back to the project's libraries.
-        This is crucial for maintaining a consistent and navigable schema graph.
-
-        Args:
-            obj (DependencySchema, optional): An optional dependency object to
-                reset and populate. If None, all existing library dependencies
-                in the project are processed. Defaults to None.
-        """
-        if obj:
-            obj._reset_deps()
-        dep_map = {name: self.get("library", name, field="schema")
-                   for name in self.getkeys("library")}
-        for obj in dep_map.values():
-            if isinstance(obj, DependencySchema):
-                obj._populate_deps(dep_map)
-
-    def _from_dict(self, manifest, keypath, version=None):
+    def _from_dict(self, manifest: Dict,
+                   keypath: Union[List[str], Tuple[str, ...]],
+                   version: Optional[Tuple[int, ...]] = None,
+                   lazyload: LazyLoad = LazyLoad.ON) \
+            -> Tuple[Set[Tuple[str, ...]], Set[Tuple[str, ...]]]:
         """
         Populates the project's schema from a dictionary representation.
 
@@ -325,15 +311,13 @@ class Project(PathSchemaBase, CommandLineSchema, BaseSchema):
         Returns:
             Any: The result of the superclass's `_from_dict` method.
         """
-        ret = super()._from_dict(manifest, keypath, version)
+        ret = super()._from_dict(manifest, keypath, version=version, lazyload=lazyload)
 
-        # Restore dependencies
-        self.__populate_deps()
-
-        # Preserve logger in history
-        for history in self.getkeys("history"):
-            hist = self.get("history", history, field="schema")
-            hist.__logger = self.__logger
+        if not lazyload.is_enforced:
+            # Preserve logger in history
+            for history in self.getkeys("history"):
+                hist: "Project" = self.get("history", history, field="schema")
+                hist.__logger = self.__logger
 
         return ret
 
@@ -392,7 +376,7 @@ class Project(PathSchemaBase, CommandLineSchema, BaseSchema):
                 self.add_dep(dep)
 
             # Rebuild dependencies to ensure instances are correct
-            self.__populate_deps(obj)
+            self.get("library", field="schema")._populate_deps(obj)
 
     def __import_flow(self, flow: Flowgraph):
         """
@@ -410,13 +394,21 @@ class Project(PathSchemaBase, CommandLineSchema, BaseSchema):
             return
 
         edit_schema = EditableSchema(self)
-        edit_schema.insert("flowgraph", flow.name, flow)
 
         # Instantiate tasks
         for task_cls in flow.get_all_tasks():
             task = task_cls()
             if not self.valid("tool", task.tool(), "task", task.task()):
                 edit_schema.insert("tool", task.tool(), "task", task.task(), task)
+            else:
+                existing_task: Task = self.get("tool", task.tool(), "task", task.task(),
+                                               field="schema")
+                if type(existing_task) is not type(task):
+                    raise TypeError(f"Task {task.tool()}/{task.task()} already exists with "
+                                    f"different type {type(existing_task).__name__}, "
+                                    f"imported type is {type(task).__name__}")
+
+        edit_schema.insert("flowgraph", flow.name, flow)
 
     def check_manifest(self) -> bool:
         """
@@ -664,6 +656,9 @@ class Project(PathSchemaBase, CommandLineSchema, BaseSchema):
         # Pass along manager address
         state["__manager__"] = MPManager._get_manager_address()
 
+        # Pass along logger level
+        state["__loglevel__"] = self.logger.level
+
         return state
 
     def __setstate__(self, state):
@@ -676,6 +671,10 @@ class Project(PathSchemaBase, CommandLineSchema, BaseSchema):
         Args:
             state (dict): The deserialized state of the object.
         """
+        # Retrieve log level
+        loglevel = state["__loglevel__"]
+        del state["__loglevel__"]
+
         # Retrieve manager address
         MPManager._set_manager_address(state["__manager__"])
         del state["__manager__"]
@@ -684,6 +683,7 @@ class Project(PathSchemaBase, CommandLineSchema, BaseSchema):
 
         # Reinitialize logger on restore
         self.__init_logger()
+        self.logger.setLevel(loglevel)
 
         # Restore callbacks
         self.__init_option_callbacks()
@@ -1225,6 +1225,7 @@ class Project(PathSchemaBase, CommandLineSchema, BaseSchema):
         # Setup options:
         for option, value in [
                 ("track", False),
+                ("remote", False),
                 ("hash", False),
                 ("nodisplay", False),
                 ("continue", True),
@@ -1276,3 +1277,61 @@ class Lint(Project):
     @classmethod
     def _getdict_type(cls) -> str:
         return Lint.__name__
+
+
+class _ProjectLibrary(BaseSchema):
+    def _from_dict(self, manifest: Dict,
+                   keypath: Union[List[str], Tuple[str, ...]],
+                   version: Optional[Tuple[int, ...]] = None,
+                   lazyload: LazyLoad = LazyLoad.ON) \
+            -> Tuple[Set[Tuple[str, ...]], Set[Tuple[str, ...]]]:
+        """
+        Populates the project's schema from a dictionary representation.
+
+        This method is typically used during deserialization or when loading
+        a project state from a manifest. After loading the data, it ensures
+        that internal dependencies are correctly re-established.
+
+        Args:
+            manifest (dict): The dictionary containing the schema data.
+            keypath (list): The current keypath being processed (used internally
+                            for recursive loading).
+            version (str, optional): The schema version of the manifest. Defaults to None.
+
+        Returns:
+            Any: The result of the superclass's `_from_dict` method.
+        """
+        ret = super()._from_dict(manifest, keypath, version=version, lazyload=lazyload)
+
+        if not lazyload.is_enforced:
+            # Restore dependencies
+            self._populate_deps(complete=True)
+
+        return ret
+
+    def _populate_deps(self, obj: Optional[DependencySchema] = None, complete: bool = False):
+        """
+        Ensures that all loaded dependencies (like libraries) within the project
+        contain correct internal pointers back to the project's libraries.
+        This is crucial for maintaining a consistent and navigable schema graph.
+
+        Args:
+            obj (DependencySchema, optional): An optional dependency object to
+                reset and populate. If None, all existing library dependencies
+                in the project are processed. Defaults to None.
+            complete (bool, optional): If True, performs a full reset of all
+                DependencySchema objects before populating dependencies. This
+                ensures a clean state during manifest deserialization. Defaults to False.
+        """
+        if obj:
+            obj._reset_deps()
+        dep_map = {name: self.get(name, field="schema") for name in self.getkeys()}
+
+        if complete:
+            for obj in dep_map.values():
+                if isinstance(obj, DependencySchema):
+                    obj._reset_deps()
+
+        for obj in dep_map.values():
+            if isinstance(obj, DependencySchema):
+                obj._populate_deps(dep_map)

siliconcompiler/remote/client.py

@@ -18,7 +18,8 @@ from siliconcompiler import NodeStatus as SCNodeStatus
 from siliconcompiler._metadata import default_server
 from siliconcompiler.flowgraph import RuntimeFlowgraph
 from siliconcompiler.scheduler import Scheduler
-from siliconcompiler.schema import Journal
+from siliconcompiler.schema import Journal, Parameter
+from siliconcompiler.package import PythonPathResolver, FileResolver, KeyPathResolver
 
 from siliconcompiler.utils.logging import get_console_formatter
 from siliconcompiler.utils.curation import collect
@@ -573,19 +574,29 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
 
         # Ensure dataroots with python sources are copied
         for key in self.__project.allkeys():
-            key_type = self.__project.get(*key, field='type')
+            if key[0] == "history":
+                continue
+
+            param: Parameter = self.__project.get(*key, field=None)
+            key_type: str = param.get(field="type")
 
             if 'dir' in key_type or 'file' in key_type:
-                for _, step, index in self.__project.get(*key, field=None).getvalues(
-                        return_defvalue=False):
-                    dataroots = self.__project.get(*key, field='dataroot', step=step, index=index)
+                schema_obj = self.__project.get(*key[:-1], field="schema")
+                dataroot_objs = schema_obj._find_files_dataroot_resolvers(True)
+
+                for value, step, index in param.getvalues():
+                    if not value:
+                        continue
+                    dataroots = param.get(field='dataroot', step=step, index=index)
                     if not isinstance(dataroots, list):
                         dataroots = [dataroots]
                     force_copy = False
                     for dataroot in dataroots:
                         if not dataroot:
                             continue
-                        if dataroot.startswith('python://'):
+                        dataroot_resolver = dataroot_objs.get(dataroot, None)
+                        if isinstance(dataroot_resolver,
+                                      (PythonPathResolver, FileResolver, KeyPathResolver)):
                             force_copy = True
                     if force_copy:
                         self.__project.set(*key, True, field='copy', step=step, index=index)

siliconcompiler/scheduler/docker.py

@@ -3,6 +3,8 @@ import os
 import shlex
 import sys
 
+import docker.errors
+
 from pathlib import Path
 
 import siliconcompiler
@@ -11,10 +13,9 @@ from siliconcompiler.package import RemoteResolver
 from siliconcompiler.utils import default_email_credentials_file
 from siliconcompiler.scheduler import SchedulerNode
 from siliconcompiler.utils.logging import SCBlankLoggerFormatter
-from siliconcompiler.utils.curation import collect
 
 
-def get_image(project, step, index):
+def get_image(project, step, index) -> str:
     """Determines the Docker image to use for a given node.
 
     The image is selected based on the following priority:
@@ -32,7 +33,7 @@ def get_image(project, step, index):
     """
     from siliconcompiler import __version__
 
-    queue = project.get('option', 'scheduler', 'queue', step=step, index=index)
+    queue = project.option.scheduler.get_queue(step=step, index=index)
     if queue:
         return queue
 
@@ -161,24 +162,24 @@ class DockerSchedulerNode(SchedulerNode):
         """
         A static pre-processing hook for the Docker scheduler.
 
-        On Windows, this method forces all file/directory parameters to be
-        copied rather than linked, which avoids issues with differing
-        filesystem types between the host and the Linux-based container.
-        It then triggers :meth:`.collect()` to ensure all files are staged.
-
         Args:
             project (Project): The project object to perform pre-processing on.
         """
-        if sys.platform == 'win32':
-            # this avoids the issue of different file system types
-            project.logger.error('Setting copy field to true for docker run on Windows')
-            for key in project.allkeys():
-                if key[0] == 'history':
-                    continue
-                sc_type = project.get(*key, field='type')
-                if 'dir' in sc_type or 'file' in sc_type:
-                    project.set(*key, True, field='copy')
-            collect(project)
+        try:
+            client = docker.from_env()
+            client.version()
+        except (docker.errors.DockerException, docker.errors.APIError):
+            raise RuntimeError('docker is not available or installed on this machine')
+
+    def mark_copy(self) -> bool:
+        if sys.platform != 'win32':
+            return False
+
+        do_collect = False
+        for key in self.get_required_path_keys():
+            self.project.set(*key, True, field='copy')
+            do_collect = True
+        return do_collect
 
     def run(self):
         """
@@ -196,12 +197,7 @@ class DockerSchedulerNode(SchedulerNode):
         """
         self._init_run_logger()
 
-        try:
-            client = docker.from_env()
-            client.version()
-        except (docker.errors.DockerException, docker.errors.APIError) as e:
-            self.logger.error(f'Unable to connect to docker: {e}')
-            self.halt()
+        client = docker.from_env()
 
         is_windows = sys.platform == 'win32'
 
@@ -233,7 +229,7 @@ class DockerSchedulerNode(SchedulerNode):
         email_file = default_email_credentials_file()
         if is_windows:
             # Hack to get around manifest merging
-            self.project.set('option', 'cachedir', None)
+            self.project.option.set_cachedir(None)
             cache_dir = '/sc_cache'
             cwd = '/sc_docker'
             builddir = f'{cwd}/build'
@@ -347,3 +343,6 @@ class DockerSchedulerNode(SchedulerNode):
 
         # Restore working directory
         os.chdir(start_cwd)
+
+    def check_required_paths(self) -> bool:
+        return True