jolt 0.9.76__py3-none-any.whl → 0.9.429__py3-none-any.whl

jolt/plugins/logstash.py

@@ -13,9 +13,9 @@ log.verbose("[LogStash] Loaded")
 
 class LogStashHooks(TaskHook):
     def __init__(self):
-        self._uri = config.get("logstash", "http.uri")
+        self._uri = config.get("logstash", "http.uri", "http://logstash")
         self._failed_enabled = config.getboolean("logstash", "failed", False)
-        self._finished_enabled = config.getboolean("logstash", "finished", False)
+        self._finished_enabled = config.getboolean("logstash", "passed", config.getboolean("logstash", "finished", False))
         raise_error_if(not self._uri, "logstash.http.uri not configured")
 
     def _get_uri(self, task):
@@ -25,8 +25,8 @@ class LogStashHooks(TaskHook):
             task.canonical_name)
 
     def _stash_log(self, task, logbuffer):
-        with task.tools.tmpdir("logstash") as t:
-            filepath = fs.path.join(t.path, "log")
+        with task.tools.tmpdir("logstash") as tmp:
+            filepath = fs.path.join(tmp, "log")
             with open(filepath, "w") as f:
                 f.write(logbuffer)
             task.logstash = self._get_uri(task)

jolt/plugins/meson.py

@@ -0,0 +1,61 @@
+from jolt import Task, attributes
+
+
+@attributes.common_metadata()
+class Meson(Task):
+    """ Base class for Meson-based build tasks. """
+
+    abstract = True
+    """ This is an abstract base class that should be inherited by concrete tasks. """
+
+    incremental = True
+    """
+    Whether to use incremental builds.
+    If True, the build directories are preserved between runs.
+    """
+
+    options = []
+    """
+    Additional options to pass to the `meson` command.
+    """
+
+    srcdir = None
+    """
+    Source directory for the Meson project.
+
+    If None, defaults to the task work directory (joltdir).
+    """
+
+    def clean(self, tools):
+        at = tools.meson(incremental=self.incremental)
+        at.clean()
+
+    def run(self, deps, tools):
+        self.deps = deps
+        options = tools.expand(self.options)
+        at = tools.meson(deps, incremental=self.incremental)
+        at.configure(self.srcdir or self.joltdir, *options)
+        at.build()
+        at.install()
+
+    def publish(self, artifact, tools):
+        at = tools.meson(incremental=self.incremental)
+        at.publish(artifact)
+
+
+def requires(meson=True, ninja=True):
+    """ Decorator to add Meson and Ninja requirements to a task. """
+
+    import jolt.pkgs.meson
+    import jolt.pkgs.ninja
+
+    def decorate(cls):
+        if meson:
+            cls = attributes.requires("requires_meson")(cls)
+            cls.requires_meson = ["meson"]
+        if ninja:
+            cls = attributes.requires("requires_ninja")(cls)
+            cls.requires_ninja = ["ninja"]
+        return cls
+
+    return decorate

jolt/plugins/ninja-compdb.py

@@ -30,7 +30,7 @@ def patch(command, attrib, search, replace):
 
 
 class CompDB(object):
-    def __init__(self, path="compile_commands.json", artifact=None):
+    def __init__(self, path="compdb/compile_commands.json", artifact=None):
         self.commands = []
         self.attribs = {}
         if artifact:
@@ -73,6 +73,12 @@ class CompDB(object):
                 utils.call_and_catch(patch, command, "command", "sandboxes/", "sandboxes-reflected/")
             utils.call_and_catch(patch, command, "directory", self.attribs.get("joltdir", joltdir()), joltdir())
 
+    def remove_wrappers(self, task):
+        for command in self.commands:
+            for wrapper in [task.tools.getenv("CXXWRAP", ""), task.tools.getenv("CCWRAP", "")]:
+                if command["command"].startswith(wrapper):
+                    command["command"] = command["command"][len(wrapper):]
+
     def merge(self, db):
         self.commands.extend(db.commands)
 
@@ -83,62 +89,110 @@ def has_incpaths(artifact):
 
 def stage_artifacts(artifacts, tools):
     for artifact in filter(has_incpaths, artifacts):
-        tools.sandbox(artifact, incremental=True, reflect=fs.has_symlinks())
+        tools.sandbox(artifact, incremental=True)
+        if fs.has_symlinks:
+            tools.sandbox(artifact, incremental=True, reflect=True)
+
+
+def get_compdb_artifacts(task):
+    artifacts = []
+
+    def select_compdb(artifacts):
+        return list(filter(lambda a: a.name == "compdb", artifacts))
+
+    for dep in task.children:
+        artifacts.extend(dep.artifacts)
+    return select_compdb(task.artifacts)[0], select_compdb(artifacts)
 
 
-def get_task_artifacts(task, artifact=None):
-    acache = cache.ArtifactCache.get()
-    artifact = artifact or acache.get_artifact(task)
-    return artifact, [acache.get_artifact(dep) for dep in task.children]
+def get_task_artifacts(task):
+    compdb_artifact = task.get_artifact("compdb")
+    artifacts = [a for a in task.artifacts]
+    for dep in task.children:
+        artifacts.extend(dep.artifacts)
+    return compdb_artifact, artifacts
 
 
 class CompDBHooks(TaskHook):
+    def __init__(self):
+        self._depfiles = config.getboolean("ninja-compdb", "depfiles", False)
+
+    def publish_compdb(self, artifact, tools):
+        with tools.cwd(self.outdir):
+            artifact.collect("**/*compile_commands.json", "compdb/")
+
+    def publish_depfiles(self, artifact, tools):
+        with tools.cwd(self.outdir):
+            artifact.collect("**/*.d", "depfiles/")
+
     def task_created(self, task):
         task.task.influence.append(StringInfluence("NinjaCompDB: v3"))
+        if isinstance(task.task, ninja.CXXProject):
+            compdb_artifact = task.cache.get_artifact(task, "compdb")
+            task.add_artifact(compdb_artifact)
+            publish_compdb = getattr(task.task, "publish_compdb", None)
+            if not publish_compdb:
+                setattr(task.task, "publish_compdb", CompDBHooks.publish_compdb.__get__(task.task, task.task.__class__))
+            if self._depfiles:
+                depfile_artifact = task.cache.get_artifact(task, "depfiles")
+                task.add_artifact(depfile_artifact)
+                publish_depfiles = getattr(task.task, "publish_depfiles", None)
+                if not publish_depfiles:
+                    setattr(task.task, "publish_depfiles", CompDBHooks.publish_depfiles.__get__(task.task, task.task.__class__))
 
     def task_postrun(self, task, deps, tools):
         if not isinstance(task.task, ninja.CXXProject):
             return
-        with tools.cwd(task.task.outdir):
-            utils.call_and_catch(tools.run, "ninja -f build.ninja -t compdb > compile_commands.json")
+        with tools.cwd(tools.wsroot):
+            utils.call_and_catch(tools.run, "ninja -f {outdir}/build.ninja -t compdb > {outdir}/compile_commands.json")
 
     def task_postpublish(self, task, artifact, tools):
-        if isinstance(task.task, ninja.CXXProject):
-            with tools.cwd(task.task.outdir):
-                artifact.collect("*compile_commands.json")
+        if artifact.name != "compdb":
+            return
 
         # Add information about the workspace and cachedir roots
         db = CompDB(artifact=artifact)
         db.read()
         db.annotate(task)
+        db.remove_wrappers(task)
         db.write()
 
         if isinstance(task.task, ninja.CXXProject):
             dbpath = fs.path.join(task.task.outdir, "all_compile_commands.json")
-            _, deps = get_task_artifacts(task, artifact)
+            _, deps = get_compdb_artifacts(task)
             db = CompDB(dbpath)
             for dep in [artifact] + deps:
                 depdb = CompDB(artifact=dep)
                 depdb.read()
-                depdb.relocate(task)
+                depdb.relocate(task, sandboxes=fs.has_symlinks())
                 db.merge(depdb)
             db.write()
-            artifact.collect(dbpath, flatten=True)
+            artifact.collect(dbpath, "compdb/", flatten=True)
 
-    def task_finished_execution(self, task):
+    def task_finished_execution(self, task: graph.TaskProxy):
         if task.options.network or task.options.worker:
             return
         if not task.is_goal():
             return
         if isinstance(task.task, ninja.CXXProject):
-            artifact, deps = get_task_artifacts(task)
-            db = CompDB("all_compile_commands.json", artifact)
+            artifact, deps = get_compdb_artifacts(task)
+            db = CompDB("compdb/all_compile_commands.json", artifact)
             db.read()
-            db.relocate(task)
+            db.relocate(task, sandboxes=fs.has_symlinks())
             outdir = task.tools.builddir("compdb", incremental=True)
             dbpath = fs.path.join(outdir, "all_compile_commands.json")
             db.write(dbpath, force=True)
-            stage_artifacts(deps + [artifact], task.tools)
+
+            # Save the compilation database to the configured path
+            if task.is_goal():
+                last_path = config.get("ninja-compdb", "path")
+                if last_path:
+                    last_path = os.path.join(task.tools.wsroot, last_path)
+                    task.tools.mkdirname(last_path)
+                    db.write(last_path, force=True)
+
+            artifact, deps = get_task_artifacts(task)
+            stage_artifacts(deps, task.tools)
 
     def task_finished_download(self, task):
         self.task_finished_execution(task)
@@ -172,54 +226,76 @@ def compdb(ctx, task, default):
 
     """
 
-    manifest = ctx.obj["manifest"]
     options = JoltOptions(default=default)
     acache = cache.ArtifactCache.get(options)
     TaskHookRegistry.get(options)
     executors = scheduler.ExecutorRegistry.get(options)
     registry = TaskRegistry.get()
     strategy = scheduler.DownloadStrategy(executors, acache)
-    queue = scheduler.TaskQueue(strategy)
+    queue = scheduler.TaskQueue()
 
     for params in default:
         registry.set_default_parameters(params)
 
-    gb = graph.GraphBuilder(registry, manifest, options, progress=True)
+    gb = graph.GraphBuilder(registry, acache, options, progress=True)
     dag = gb.build(task)
 
     try:
-        with log.progress("Progress", dag.number_of_tasks(), " tasks", estimates=False, debug=False) as p:
-            while dag.has_tasks():
-                leafs = dag.select(lambda graph, task: task.is_ready())
+        with log.progress("Progress", dag.number_of_tasks(), " tasks", estimates=False, debug=False) as progress:
+            in_progress = set()
+
+            while dag.has_tasks() or not queue.empty():
+                leafs = dag.select(lambda graph, task: task.is_ready() and task not in in_progress)
 
                 # Order the tasks by their weights to improve build times
                 leafs.sort(key=lambda x: x.weight)
 
                 while leafs:
                     task = leafs.pop()
-                    queue.submit(acache, task)
+                    executor = strategy.create_executor({}, task)
+                    queue.submit(executor)
+                    in_progress.add(task)
+
+                task, _ = queue.wait()
+
+                # Materialize workspace resources so that
+                # source code is available to the debugger.
+                if task.is_workspace_resource():
+                    task.task.acquire_ws(force=True)
 
-                task, error = queue.wait()
-                p.update(1)
+                # Unpack the task if it is not a resource task and has a custom unpack method
+                if not task.is_resource():
+                    if task.is_unpackable():
+                        task.unpack()
+
+                progress.update(1)
 
     except KeyboardInterrupt:
         print()
         log.warning("Interrupted by user")
         try:
             queue.abort()
+            executors.shutdown()
             sys.exit(1)
         except KeyboardInterrupt:
             print()
             log.warning("Interrupted again, exiting")
             os._exit(1)
 
+    finally:
+        queue.shutdown()
+
     for goal in dag.goals:
-        artifact, deps = get_task_artifacts(goal)
-        db = CompDB("all_compile_commands.json", artifact)
+        if goal.is_resource():
+            continue
+        if goal.is_alias():
+            continue
+        compdb_artifact, artifacts = get_task_artifacts(goal)
+        db = CompDB("compdb/all_compile_commands.json", compdb_artifact)
         db.read()
         db.relocate(goal, sandboxes=fs.has_symlinks())
         outdir = goal.tools.builddir("compdb", incremental=True)
         dbpath = fs.path.join(outdir, "all_compile_commands.json")
         db.write(dbpath, force=True)
-        stage_artifacts(deps + [artifact], goal.tools)
+        stage_artifacts(artifacts, goal.tools)
         log.info("Compilation DB: {}", dbpath)