jolt 0.9.123__py3-none-any.whl → 0.9.435__py3-none-any.whl

jolt/plugins/fetch.py

@@ -0,0 +1,141 @@
+import hashlib
+from jolt import BooleanParameter, Parameter
+from jolt.plugins.git import ErrorDict
+from jolt.tasks import Resource, TaskRegistry
+from jolt import filesystem as fs
+from jolt import utils
+from jolt.error import raise_error_if
+from jolt.loader import JoltLoader
+from jolt.tools import SUPPORTED_ARCHIVE_TYPES
+
+
+class Fetch(Resource):
+    """
+    Fetch a file from a URL and extract it.
+
+    By default, the fetched file is extracted if it is a supported archive type.
+    Extraction may be disable by assigning the `extract` parameter to false.
+
+    The fetched content is placed in a build directory named after the resource.
+    The `path` parameter can be used to specify a different location relative to
+    the workspace root.
+
+    The path of the fetched content is made available to consuming tasks through
+    the `fetch` attribute. The `fetch` attribute is a dictionary where the key
+    is the name of the fetched file and the value is a path relative to the
+    workspace root. You can specify an alias for the filename / key through the
+    `alias` parameter.
+
+    The plugin must be loaded before it can be used. This is done by importing
+    the module, or by adding the following line to the configuration file:
+
+    .. code-block:: ini
+
+            [fetch]
+
+
+    Example:
+
+    .. code-block:: python
+
+            from jolt.plugins import fetch
+
+            class Example(Task):
+                requires = ["fetch:alias=zlib,url=https://zlib.net/zlib-1.3.1.tar.gz"]
+
+                def run(self, deps, tools):
+                    self.info("The source tree is located at: {fetch[zlib]}")
+                    with tools.cwd(self.fetch["zlib"]):
+                        tools.run("make")
+
+            """
+
+    name = "fetch"
+    alias = Parameter(required=False, help="Name of the task used when referencing content. Defaults to the filename of the fetched file.")
+    extract = BooleanParameter(default=True, help="Whether to extract the fetched file.")
+    path = Parameter(required=False, help="Destination directory.")
+    url = Parameter(help="URL to fetch from.")
+    md5 = Parameter(required=False, help="Expected MD5 hash of the fetched file.")
+    sha256 = Parameter(required=False, help="Expected SHA256 hash of the fetched file.")
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.joltdir = JoltLoader.get().joltdir
+
+        # Set the path to the extraction directory
+        if self.path.is_unset():
+            self.abspath = self.tools.builddir(utils.canonical(self.short_qualified_name), incremental="always", unique=False)
+            if not self._extract():
+                # Join with filename if not extracting
+                self.abspath = fs.path.join(self.abspath, self._get_filename())
+        else:
+            self.abspath = fs.path.join(self.joltdir, str(self.path) or self._get_name())
+
+        self.relpath = fs.path.relpath(self.abspath, self.tools.wsroot)
+
+    def _extract(self):
+        """ Check if the fetched file should/can be extracted. """
+        filename = self._get_filename()
+
+        if not any([filename.endswith(ext) for ext in SUPPORTED_ARCHIVE_TYPES]):
+            return False
+
+        return bool(self.extract)
+
+    def _acquire_ws(self):
+        # Create the destination directory if it does not exist
+        self.tools.rmtree(self.abspath, ignore_errors=True)
+        self.tools.mkdir(fs.path.dirname(self.abspath), recursively=True)
+
+        if self._extract():
+            with self.tools.tmpdir() as tmpdir, self.tools.cwd(tmpdir):
+                filename = self._get_filename()
+                self.tools.download(self.url, filename)
+                self._verify_sha256(filename)
+                self.tools.extract(filename, self.abspath)
+        else:
+            self.tools.download(self.url, self.abspath)
+            self._verify_sha256(self.abspath)
+
+    def acquire(self, artifact, deps, tools, owner):
+        self._acquire_ws()
+        self._assign_fetch(owner)
+        artifact.worktree = fs.path.relpath(self.abspath, owner.joltdir)
+
+    def _assign_fetch(self, task, none=False):
+        if not hasattr(task, "fetch"):
+            task.fetch = ErrorDict(self)
+        if none:
+            # None means the git repo is not cloned or checked out
+            # and should not be included in the git dictionary
+            # of the consuming task yet. If the consuming task
+            # requires the git repo for its influence collection,
+            # the dict will raise an error. The solution is to
+            # assign hash=true to the git requirement which
+            # will cause the git repo to be cloned and checked out
+            # before the influence collection is performed.
+            task.fetch[self._get_name()] = None
+        else:
+            # Assign the git repo to the consuming task.
+            # The git repo is cloned and checked out before
+            # any influence collection is performed.
+            task.fetch[self._get_name()] = fs.path.relpath(self.abspath, task.joltdir)
+
+    def _get_name(self):
+        return str(self.alias) if self.alias.is_set() else self._get_filename()
+
+    def _get_filename(self):
+        return fs.path.basename(str(self.url))
+
+    def _verify_sha256(self, filepath):
+        if not self.sha256.is_set():
+            return
+        actual_hash = self.tools.checksum_file(filepath, hashfn=hashlib.sha256)
+        expected_hash = str(self.sha256)
+        raise_error_if(
+            actual_hash != expected_hash,
+            f"SHA256 hash mismatch for fetched file '{filepath}': expected {expected_hash}, got {actual_hash}"
+        )
+
+
+TaskRegistry.get().add_task_class(Fetch)

jolt/plugins/gdb.py

@@ -1,7 +1,6 @@
 import click
 import os
 import sys
-import subprocess
 
 from jolt import cache
 from jolt import cli
@@ -9,6 +8,7 @@ from jolt import filesystem as fs
 from jolt import graph
 from jolt import log
 from jolt import scheduler
+from jolt.error import raise_error
 from jolt.error import raise_task_error_if
 from jolt.hooks import TaskHookRegistry
 from jolt.options import JoltOptions
@@ -25,10 +25,11 @@ def stage_artifacts(artifacts, tools):
         tools.sandbox(artifact, incremental=True, reflect=fs.has_symlinks())
 
 
-def get_task_artifacts(task, artifact=None):
-    acache = cache.ArtifactCache.get()
-    artifact = artifact or acache.get_artifact(task)
-    return artifact, [acache.get_artifact(dep) for dep in task.children]
+def get_task_artifacts(task):
+    artifacts = []
+    for dep in task.children:
+        artifacts.extend(dep.artifacts)
+    return task.artifacts, artifacts
 
 
 @cli.cli.command(name="gdb", context_settings={"ignore_unknown_options": True})
@@ -69,36 +70,44 @@ def gdb(ctx, task, default, machine_interface, no_binary, gdb_args):
     if machine_interface:
         log.enable_gdb()
 
-    manifest = ctx.obj["manifest"]
     options = JoltOptions(default=default)
     acache = cache.ArtifactCache.get(options)
     TaskHookRegistry.get(options)
     executors = scheduler.ExecutorRegistry.get(options)
     registry = TaskRegistry.get()
     strategy = scheduler.DownloadStrategy(executors, acache)
-    queue = scheduler.TaskQueue(strategy)
+    queue = scheduler.TaskQueue()
 
     for params in default:
         registry.set_default_parameters(params)
 
-    gb = graph.GraphBuilder(registry, manifest, options, progress=True)
+    gb = graph.GraphBuilder(registry, acache, options, progress=True)
     dag = gb.build([task])
 
     try:
         with log.progress("Progress", dag.number_of_tasks(), " tasks", estimates=False, debug=False) as p:
+            in_progress = set()
+
             while dag.has_tasks() or not queue.empty():
-                leafs = dag.select(lambda graph, task: task.is_ready())
+                leafs = dag.select(lambda graph, task: task.is_ready() and task not in in_progress)
 
                 while leafs:
                     task = leafs.pop()
-                    queue.submit(acache, task)
+                    executor = strategy.create_executor({}, task)
+                    queue.submit(executor)
+                    in_progress.add(task)
 
-                task, error = queue.wait()
+                task, _ = queue.wait()
 
                 # Materialize workspace resources so that
                 # source code is available to the debugger.
                 if isinstance(task.task, WorkspaceResource):
-                    task.task.acquire()
+                    task.task.acquire_ws()
+
+                # Unpack the task if it is not a resource task and has a custom unpack method
+                if not task.is_resource():
+                    if task.is_unpackable():
+                        task.unpack()
 
                 p.update(1)
 
@@ -107,27 +116,37 @@ def gdb(ctx, task, default, machine_interface, no_binary, gdb_args):
         log.warning("Interrupted by user")
         try:
             queue.abort()
+            executors.shutdown()
             sys.exit(1)
         except KeyboardInterrupt:
             print()
             log.warning("Interrupted again, exiting")
             os._exit(1)
+
     except Exception as e:
         log.set_interactive(True)
         raise e
 
+    finally:
+        queue.shutdown()
+
     assert len(dag.goals), "Too many tasks, can only debug one executable at a time"
 
     for goal in dag.goals:
-        artifact, deps = get_task_artifacts(goal)
-        stage_artifacts(deps + [artifact], goal.tools)
+        main, deps = get_task_artifacts(goal)
+        stage_artifacts(main + deps, goal.tools)
 
+        # Find an artifact with an executable
+        main = [artifact for artifact in main if artifact.strings.executable]
         raise_task_error_if(
-            artifact.strings.executable.get_value() is None,
-            goal, "No executable found in task artifact")
+            not main, goal, "No executable found in task artifact")
+        main = main[0]
 
         with acache.get_context(goal):
             gdb = goal.tools.getenv("GDB", "gdb")
+            gdb = goal.tools.which(gdb)
+            if not gdb:
+                raise_error("GDB not found in PATH")
             cmd = [gdb]
             sysroot = goal.tools.getenv("SDKTARGETSYSROOT", goal.tools.getenv("SYSROOT"))
             if sysroot:
@@ -141,13 +160,14 @@ def gdb(ctx, task, default, machine_interface, no_binary, gdb_args):
             cmd += ["-ex", "set print thread-events off"]
             cmd += ["-ex", "handle SIG32 nostop noprint"]
             if not no_binary:
-                cmd += [os.path.join(artifact.path, str(artifact.strings.executable))]
+                cmd += [os.path.join(main.path, str(main.strings.executable))]
             cmd += gdb_args
 
             if isinstance(goal.task, ninja.CXXProject):
-                cwd = goal.tools.builddir("ninja", incremental=True)
+                cwd = goal.tools.wsroot
             else:
                 cwd = goal.task.joltdir
 
             with goal.tools.environ() as env:
-                subprocess.call(cmd, env=env, cwd=cwd)
+                os.chdir(cwd)
+                os.execve(cmd[0], cmd, env)

jolt/plugins/gerrit.py

@@ -7,18 +7,6 @@ from jolt.plugins import git
 log.verbose("[Gerrit] Loaded")
 
 
-class GerritSrc(git.GitSrc):
-    name = "gerrit-src"
-    url = Parameter(help="URL to the Gerrit git repo to be cloned. Required.")
-    sha = Parameter(required=False, help="Specific commit or tag to be checked out. Optional.")
-    path = Parameter(required=False, help="Local path where the repository should be cloned.")
-    _revision = Export(value=lambda self: self._get_revision() or self.git.head())
-
-    def __init__(self, *args, **kwargs):
-        refspec1 = '+refs/changes/*:refs/remotes/origin/changes/*'
-        super(GerritSrc, self).__init__(*args, refspecs=[refspec1], **kwargs)
-
-
 class Gerrit(git.Git):
     name = "gerrit"
     url = Parameter(help="URL to the Gerrit git repo to be cloned. Required.")
@@ -28,8 +16,7 @@ class Gerrit(git.Git):
 
     def __init__(self, *args, **kwargs):
         refspec1 = '+refs/changes/*:refs/remotes/origin/changes/*'
-        super(Gerrit, self).__init__(*args, refspecs=[refspec1], **kwargs)
+        super().__init__(*args, refspecs=[refspec1], **kwargs)
 
 
-TaskRegistry.get().add_task_class(GerritSrc)
 TaskRegistry.get().add_task_class(Gerrit)