siliconcompiler 0.33.1__py3-none-any.whl → 0.34.0__py3-none-any.whl

siliconcompiler/packageschema.py

@@ -0,0 +1,260 @@
+import os
+
+import os.path
+
+from siliconcompiler.schema import BaseSchema
+from siliconcompiler.schema import EditableSchema, Parameter, Scope
+from siliconcompiler.schema.utils import trim
+from siliconcompiler.package import Resolver
+
+
+class PackageSchema(BaseSchema):
+    def __init__(self):
+        super().__init__()
+
+        schema_package(self)
+
+        self.__cache = {}
+
+    def register(self, name, path, ref=None, clobber=True):
+        """
+        Registers a package by its name with the source path and reference
+
+        Registered package sources are stored in the package section of the schema.
+
+        Args:
+            name (str): Package name
+            path (str): Path to the sources, can be file, git url, archive url
+            ref (str): Reference of the sources, can be commitid, branch name, tag
+            clobber (bool): Overwrite existing
+
+        Examples:
+            >>> schema.register('siliconcompiler_data',
+                    'git+https://github.com/siliconcompiler/siliconcompiler',
+                    'cd328131bafd361787f9137a6ffed999d64c8c30')
+        """
+
+        # If this is a file, get the directory for this
+        if os.path.isfile(path):
+            path = os.path.dirname(os.path.abspath(path))
+        elif os.path.isdir(path):
+            path = os.path.abspath(path)
+
+        success = False
+        if self.set('source', name, 'path', path, clobber=clobber):
+            success = True
+        if success and ref:
+            success = False
+            if self.set('source', name, 'ref', ref, clobber=clobber):
+                success = True
+        return success
+
+    def get_resolver(self, package):
+        '''
+        Returns a specific resolver
+
+        Args:
+            package (str): name of package
+        '''
+        resolver_cls = Resolver.find_resolver(self.get("source", package, "path"))
+        resolver = resolver_cls(package, self._parent(root=True),
+                                self.get("source", package, "path"),
+                                self.get("source", package, "ref"))
+        resolver.set_cache(self.__cache)
+        return resolver
+
+    def get_resolvers(self):
+        '''
+        Returns a dictionary of packages with their resolver method.
+        '''
+        resolvers = {}
+        for package in self.getkeys("source"):
+            resolvers[package] = self.get_resolver(package).get_path
+
+        return resolvers
+
+    def _set_cache(self, package, path):
+        self.__cache[package] = path
+
+    def get_path_cache(self):
+        return self.__cache.copy()
+
+
+############################################
+# Package information
+############################################
+def schema_package(schema):
+    schema = EditableSchema(schema)
+
+    schema.insert(
+        'version',
+        Parameter(
+            'str',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: version",
+            switch="-package_version <str>",
+            example=[
+                "cli: -package_version 1.0",
+                "api: chip.set('package', 'version', '1.0')"],
+            help=trim("""Package version. Can be a branch, tag, commit hash,
+            or a semver compatible version.""")))
+
+    schema.insert(
+        'description',
+        Parameter(
+            'str',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: description",
+            switch="-package_description <str>",
+            example=[
+                "cli: -package_description 'Yet another cpu'",
+                "api: chip.set('package', 'description', 'Yet another cpu')"],
+            help=trim("""Package short one line description for package
+            managers and summary reports.""")))
+
+    schema.insert(
+        'keyword',
+        Parameter(
+            'str',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: keyword",
+            switch="-package_keyword <str>",
+            example=[
+                "cli: -package_keyword cpu",
+                "api: chip.set('package', 'keyword', 'cpu')"],
+            help=trim("""Package keyword(s) used to characterize package.""")))
+    schema.insert(
+        'doc', 'homepage',
+        Parameter(
+            'str',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: documentation homepage",
+            switch="-package_doc_homepage <str>",
+            example=[
+                "cli: -package_doc_homepage index.html",
+                "api: chip.set('package', 'doc', 'homepage', 'index.html')"],
+            help=trim("""
+            Package documentation homepage. Filepath to design docs homepage.
+            Complex designs can can include a long non standard list of
+            documents dependent. A single html entry point can be used to
+            present an organized documentation dashboard to the designer.""")))
+
+    doctypes = ['datasheet',
+                'reference',
+                'userguide',
+                'quickstart',
+                'releasenotes',
+                'testplan',
+                'signoff',
+                'tutorial']
+
+    for item in doctypes:
+        schema.insert(
+            'doc', item,
+            Parameter(
+                '[file]',
+                scope=Scope.GLOBAL,
+                shorthelp=f"Package: {item} document",
+                switch=f"-package_doc_{item} <file>",
+                example=[
+                    f"cli: -package_doc_{item} {item}.pdf",
+                    f"api: chip.set('package', 'doc', '{item}', '{item}.pdf')"],
+                help=trim(f"""Package list of {item} documents.""")))
+
+    schema.insert(
+        'license',
+        Parameter(
+            '[str]',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: license identifiers",
+            switch="-package_license <str>",
+            example=[
+                "cli: -package_license 'Apache-2.0'",
+                "api: chip.set('package', 'license', 'Apache-2.0')"],
+            help=trim("""Package list of SPDX license identifiers.""")))
+
+    schema.insert(
+        'licensefile',
+        Parameter(
+            '[file]',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: license files",
+            switch="-package_licensefile <file>",
+            example=[
+                "cli: -package_licensefile './LICENSE'",
+                "api: chip.set('package', 'licensefile', './LICENSE')"],
+            help=trim("""Package list of license files for to be
+            applied in cases when a SPDX identifier is not available.
+            (eg. proprietary licenses).""")))
+
+    schema.insert(
+        'organization',
+        Parameter(
+            '[str]',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: sponsoring organization",
+            switch="-package_organization <str>",
+            example=[
+                "cli: -package_organization 'humanity'",
+                "api: chip.set('package', 'organization', 'humanity')"],
+            help=trim("""Package sponsoring organization. The field can be left
+            blank if not applicable.""")))
+
+    record = ['name',
+              'email',
+              'username',
+              'location',
+              'organization',
+              'publickey']
+
+    for item in record:
+        schema.insert(
+            'author', 'default', item,
+            Parameter(
+                'str',
+                scope=Scope.GLOBAL,
+                shorthelp=f"Package: author {item}",
+                switch=f"-package_author_{item} 'userid <str>'",
+                example=[
+                    f"cli: -package_author_{item} 'wiley wiley@acme.com'",
+                    f"api: chip.set('package', 'author', 'wiley', '{item}', 'wiley@acme.com')"],
+                help=trim(f"""Package author {item} provided with full name as key and
+                {item} as value.""")))
+
+    schema.insert(
+        'source', 'default', 'path',
+        Parameter(
+            'str',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: data source path",
+            switch="-package_source_path 'source <str>'",
+            example=[
+                "cli: -package_source_path "
+                "'freepdk45_data ssh://git@github.com/siliconcompiler/freepdk45/'",
+                "api: chip.set('package', 'source', "
+                "'freepdk45_data', 'path', 'ssh://git@github.com/siliconcompiler/freepdk45/')"],
+            help=trim("""
+            Package data source path, allowed paths:
+
+            * /path/on/network/drive
+            * file:///path/on/network/drive
+            * git+https://github.com/xyz/xyz
+            * git://github.com/xyz/xyz
+            * git+ssh://github.com/xyz/xyz
+            * ssh://github.com/xyz/xyz
+            * https://github.com/xyz/xyz/archive
+            * https://zeroasic.com/xyz.tar.gz
+            * python://siliconcompiler
+            """)))
+
+    schema.insert(
+        'source', 'default', 'ref',
+        Parameter(
+            'str',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: data source reference",
+            switch="-package_source_ref 'source <str>'",
+            example=[
+                "cli: -package_source_ref 'freepdk45_data 07ec4aa'",
+                "api: chip.set('package', 'source', 'freepdk45_data', 'ref', '07ec4aa')"],
+            help=trim("""Package data source reference""")))

siliconcompiler/pdk.py

@@ -4,8 +4,8 @@ from siliconcompiler.schema.utils import trim
 
 
 class PDKSchema(NamedSchema, PackageSchema):
-    def __init__(self, name=None, package=None):
-        NamedSchema.__init__(self, name=name)
+    def __init__(self, name, package=None):
+        NamedSchema.__init__(self, name)
         PackageSchema.__init__(self, package=package)
 
         schema_pdk(self)

siliconcompiler/record.py

@@ -30,13 +30,25 @@ class RecordTool(Enum):
 
 
 class RecordSchema(BaseSchema):
-    __TIMEFORMAT = "%Y-%m-%d %H:%M:%S"
+    __TIMEFORMAT = "%Y-%m-%d %H:%M:%S.%f"
 
     def __init__(self):
         super().__init__()
 
         schema_record(self)
 
+    def _from_dict(self, manifest, keypath, version=None):
+        ret = super()._from_dict(manifest, keypath, version)
+
+        # Correct for change specification
+        if version and version < (0, 50, 4):
+            for timekey in RecordTime:
+                start_param = self.get(timekey.value, field=None)
+                for value, step, index in start_param.getvalues():
+                    start_param.set(f"{value}.000000", step=step, index=index)
+
+        return ret
+
     def clear(self, step, index, keep=None):
         '''
         Clear all saved metrics for a given step and index
@@ -275,6 +287,44 @@ class RecordSchema(BaseSchema):
             record_time+"+0000",
             RecordSchema.__TIMEFORMAT+"%z").timestamp()
 
+    def get_earliest_time(self, type):
+        '''
+        Returns the earliest recorded time.
+
+        Args:
+            type (:class:`RecordTime`): type of time to record
+        '''
+        type = RecordTime(type)
+        record_param = self.get(type.value, field=None)
+
+        times = set()
+        for _, step, index in record_param.getvalues():
+            times.add(self.get_recorded_time(step, index, type))
+
+        if not times:
+            return None
+
+        return min(times)
+
+    def get_latest_time(self, type):
+        '''
+        Returns the last recorded time.
+
+        Args:
+            type (:class:`RecordTime`): type of time to record
+        '''
+        type = RecordTime(type)
+        record_param = self.get(type.value, field=None)
+
+        times = set()
+        for _, step, index in record_param.getvalues():
+            times.add(self.get_recorded_time(step, index, type))
+
+        if not times:
+            return None
+
+        return max(times)
+
     def record_tool(self, step, index, info, type):
         '''
         Record information about the tool used during this record.
@@ -320,11 +370,13 @@ def schema_record(schema):
                         'x86_64',
                         '(x86_64, rv64imafdc)'],
                'starttime': ['start time',
-                             '\"2021-09-06 12:20:20\"',
-                             'Time is reported in the ISO 8601 format YYYY-MM-DD HR:MIN:SEC'],
+                             '\"2021-09-06 12:20:20.000000\"',
+                             'Time is recorded with the format YYYY-MM-DD HR:MIN:SEC.MICROSEC for '
+                             'UTC'],
                'endtime': ['end time',
-                           '\"2021-09-06 12:20:20\"',
-                           'Time is reported in the ISO 8601 format YYYY-MM-DD HR:MIN:SEC'],
+                           '\"2021-09-06 12:20:20.000000\"',
+                           'Time is recorded with the format YYYY-MM-DD HR:MIN:SEC.MICROSEC for '
+                           'UTC'],
                'region': ['cloud region',
                           '\"US Gov Boston\"',
                           """Recommended naming methodology:

siliconcompiler/remote/client.py

@@ -10,18 +10,32 @@ import tarfile
 import tempfile
 import multiprocessing
 
-from siliconcompiler import utils, SiliconCompilerError, NodeStatus
+import os.path
+
+from siliconcompiler import utils, SiliconCompilerError
 from siliconcompiler import NodeStatus as SCNodeStatus
 from siliconcompiler._metadata import default_server
-from siliconcompiler.remote import JobStatus
+from siliconcompiler.remote import JobStatus, NodeStatus
 from siliconcompiler.report.dashboard import DashboardType
 from siliconcompiler.flowgraph import RuntimeFlowgraph
-from siliconcompiler.schema import JournalingSchema
+from siliconcompiler.scheduler.scheduler import Scheduler
+from siliconcompiler.schema import Journal
 
 # Step name to use while logging
 remote_step_name = 'remote'
 
 
+class ClientScheduler(Scheduler):
+    def run_core(self):
+        Client(self._Scheduler__chip).run()
+
+    def configure_nodes(self):
+        return
+
+    def check_manifest(self):
+        return True
+
+
 class Client():
     # Step name to use while logging
     STEP_NAME = "remote"
@@ -310,19 +324,29 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
         try:
             # Decode response JSON, if possible.
             job_info = json.loads(info['message'])
-        except json.JSONDecodeError as e:
-            self.__logger.warning(f"Job is still running: {e}")
+            if "null" in job_info:
+                job_info[None] = job_info["null"]
+                del job_info["null"]
+        except json.JSONDecodeError:
+            self.__logger.warning(f"Job is still running: {info['message']}")
             return completed, starttimes, True
 
         nodes_to_log = {}
         for node, node_info in job_info.items():
             status = node_info['status']
-            nodes_to_log.setdefault(status, []).append((node, node_info))
+
+            if status == NodeStatus.UPLOADED:
+                status = SCNodeStatus.PENDING
 
             if SCNodeStatus.is_done(status):
                 # collect completed
                 completed.append(node)
 
+            if not node:
+                continue
+
+            nodes_to_log.setdefault(status, []).append((node, node_info))
+
             if self.__node_information and node in self.__node_information:
                 self.__chip.set('record', 'status', status,
                                 step=self.__node_information[node]["step"],
@@ -580,6 +604,22 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
             raise SiliconCompilerError('Job canceled by user keyboard interrupt')
 
     def __import_run_manifests(self, starttimes):
+        if not self.__setup_information_loaded:
+            if self.__setup_information_fetched:
+                manifest = os.path.join(self.__chip.getworkdir(), f'{self.__chip.design}.pkg.json')
+                if os.path.exists(manifest):
+                    try:
+                        Journal.replay_file(self.__chip.schema, manifest)
+                        self.__setup_information_loaded = True
+                        changed = True
+                    except:  # noqa E722
+                        # Import may fail if file is still getting written
+                        pass
+
+        if not self.__setup_information_loaded:
+            # Dont do anything until this has been loaded
+            return
+
         changed = False
         for _, node_info in self.__node_information.items():
             if node_info["imported"]:
@@ -591,7 +631,7 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
                 f'{self.__chip.design}.pkg.json')
             if os.path.exists(manifest):
                 try:
-                    JournalingSchema(self.__chip.schema).read_journal(manifest)
+                    Journal.replay_file(self.__chip.schema, manifest)
                     node_info["imported"] = True
                     changed = True
                 except:  # noqa E722
@@ -599,7 +639,7 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
                     pass
             elif self.__chip.get('record', 'status',
                                  step=node_info["step"], index=node_info["index"]) \
-                    == NodeStatus.SKIPPED:
+                    == SCNodeStatus.SKIPPED:
                 node_info["imported"] = True
                 changed = True
 
@@ -618,6 +658,9 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
             check_info = self.__check()
             self.__check_interval = check_info['progress_interval']
 
+        self.__setup_information_fetched = False
+        self.__setup_information_loaded = False
+
         self.__node_information = {}
         runtime = RuntimeFlowgraph(
             self.__chip.schema.get("flowgraph", self.__chip.get('option', 'flow'), field='schema'),
@@ -666,6 +709,11 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
                 # Update dashboard if active
                 self.__chip._dash.update_manifest({"starttimes": starttimes})
 
+            if None in completed:
+                completed.remove(None)
+                if not self.__setup_information_fetched:
+                    self.__schedule_fetch_result(None)
+
             nodes_to_fetch = []
             for node in completed:
                 if not self.__node_information[node]["fetched"]:
@@ -681,7 +729,6 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
         for node, node_info in self.__node_information.items():
             if not node_info["fetched"]:
                 self.__schedule_fetch_result(node)
-        self.__schedule_fetch_result(node)
 
         self._finalize_loop()
 
@@ -700,11 +747,12 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
         self.__import_run_manifests({})
 
     def __schedule_fetch_result(self, node):
-        self.__node_information[node]["fetched"] = True
+        if node:
+            self.__node_information[node]["fetched"] = True
+            self.__logger.info(f'    {node}')
+        else:
+            self.__setup_information_fetched = True
         self.__download_pool.apply_async(Client._fetch_result, (self, node))
-        if node is None:
-            node = 'final result'
-        self.__logger.info(f'    {node}')
 
     def _fetch_result(self, node):
         '''