jolt 0.9.172__py3-none-any.whl → 0.9.435__py3-none-any.whl

jolt/plugins/scheduler.py

@@ -2,6 +2,7 @@ import click
 import grpc
 import queue
 from threading import Lock
+import time
 
 from google.protobuf.timestamp_pb2 import Timestamp
 
@@ -12,11 +13,10 @@ from jolt import config
 from jolt import hooks
 from jolt import loader
 from jolt import log
-from jolt import manifest
 from jolt import common_pb2 as common_pb
 from jolt import scheduler
 from jolt import utils
-from jolt.error import JoltError, raise_error, raise_error_if, raise_task_error, raise_task_error_if
+from jolt.error import LoggedJoltError, JoltError, raise_error, raise_error_if, raise_task_error, raise_task_error_if
 from jolt.graph import GraphBuilder
 from jolt.scheduler import ExecutorRegistry, JoltEnvironment, NetworkExecutor, NetworkExecutorFactory, WorkerStrategy
 from jolt.tasks import TaskRegistry
@@ -81,7 +81,7 @@ class LogHandler(object):
                 loglines=[
                     common_pb.LogLine(
                         context=self.task.task_id[:8],
-                        level=record.levelno,
+                        level=log.level_to_pb(record.levelno),
                         time=timestamp,
                         message=record.message,
                     ),
@@ -108,12 +108,17 @@ class Queue(object):
     def __next__(self):
         """ Get the next item from the queue. """
         data = self.q.get()
+        if data is None:
+            raise StopIteration
         return data
 
     def push(self, item):
         """ Push an item to the queue. """
         self.q.put(item)
 
+    def close(self):
+        self.q.put(None)
+
 
 class RemoteExecutor(NetworkExecutor):
     """
@@ -139,6 +144,15 @@ class RemoteExecutor(NetworkExecutor):
         self.session = session
         self.task = task
 
+    def schedule(self, env):
+        """
+        Schedule the task for execution.
+
+        The task is marked as in progress before scheduling.
+        """
+        self.task.set_in_progress()
+        return super().schedule(env)
+
     def cancel(self):
         """
         Cancel the build session.
@@ -169,12 +183,17 @@ class RemoteExecutor(NetworkExecutor):
             self.download_session_artifacts(extension)
         if not task.has_artifact():
             return
-        if not task.cache.download_enabled():
+        if not task.cache.download_session_enabled():
             return
         if not task.is_downloadable():
             return
         if not task.download(session_only=True):
             task.warning("Failed to download session artifact")
+        if not task.is_resource():
+            # Tasks also download session artifacts of consumed resources
+            for resource in filter(lambda task: task.is_resource() and not task.is_workspace_resource(), task.children):
+                if not resource.is_available_locally(persistent_only=False):
+                    self.download_session_artifacts(resource)
 
     def download_log(self, task):
         """ Download log and transfer lines into local logging system. """
@@ -184,7 +203,7 @@ class RemoteExecutor(NetworkExecutor):
         for response in self.session.logs.ReadLog(request):
             for line in response.loglines:
                 log.log(
-                    line.level,
+                    log.pb_to_level(line.level),
                     line.message,
                     created=line.time.ToMicroseconds() / 1000000,
                     context=line.context[:7],
@@ -196,10 +215,19 @@ class RemoteExecutor(NetworkExecutor):
 
     def run(self, env):
         """ Run the task. """
+        if self.is_aborted():
+            return
         try:
-            self.run_build(env)
-        except grpc.RpcError as rpc_error:
-            raise_task_error(self.task, "Scheduler error: {}", rpc_error.details())
+            with hooks.task_run([self.task] + self.task.extensions), self.task.run_resources():
+                try:
+                    self.run_build(env)
+                except (grpc.RpcError, grpc._channel._MultiThreadedRendezvous) as rpc_error:
+                    raise_task_error(self.task, rpc_error.details(), type="Scheduler Error")
+        except Exception as e:
+            if not self.task.is_unstable:
+                raise e
+        finally:
+            self.download_session_artifacts(self.task)
 
     @utils.retried.on_exception(grpc.RpcError)
     def run_build(self, env):
@@ -219,10 +247,7 @@ class RemoteExecutor(NetworkExecutor):
             response = self.session.exec.ScheduleTask(request)
 
             self.update_logstash(self.task)
-
-            with hooks.task_run([self.task] + self.task.extensions):
-                self.run_task(env, response)
-
+            self.run_task(env, response)
             self.download_persistent_artifacts(self.task)
 
             self.task.finished_execution(remote=True)
@@ -232,15 +257,23 @@ class RemoteExecutor(NetworkExecutor):
         except TaskCancelledException:
             pass
 
-        except grpc.RpcError as rpc_error:
-            if rpc_error.code() != grpc.StatusCode.UNAVAILABLE:
-                raise_task_error(self.task, "Scheduler error: {}", rpc_error.details())
-            log.warning("Scheduler error: {}", rpc_error.details())
-            self.session.clear_build_request()
+        except (grpc.RpcError, grpc._channel._MultiThreadedRendezvous) as rpc_error:
+            if self.is_aborted():
+                if self.task.is_running():
+                    self.task.failed_execution(remote=True, interrupt=True)
+                    for extension in self.task.extensions:
+                        extension.failed_execution(remote=True, interrupt=True)
+                return
+
+            if rpc_error.code() not in [grpc.StatusCode.NOT_FOUND, grpc.StatusCode.UNAVAILABLE]:
+                raise_task_error(self.task, rpc_error.details(), type="Scheduler Error")
+
+            self.session.clear_build_request(f"Scheduler Error: {rpc_error.details()}")
             raise rpc_error
 
         except Exception as e:
-            log.exception()
+            if not isinstance(e, LoggedJoltError):
+                log.exception()
 
             if self.factory.options.mute:
                 try:
@@ -251,11 +284,8 @@ class RemoteExecutor(NetworkExecutor):
             self.task.failed_execution(remote=True)
             for extension in self.task.extensions:
                 extension.failed_execution(remote=True)
-            if not self.task.is_unstable:
-                raise e
 
-        finally:
-            self.download_session_artifacts(self.task)
+            raise e
 
     def run_task(self, env, response):
         """ Run the task.
@@ -276,18 +306,27 @@ class RemoteExecutor(NetworkExecutor):
         for progress in response:
             for line in progress.loglines:
                 log.log(
-                    line.level,
+                    log.pb_to_level(line.level),
                     line.message,
                     created=line.time.ToMicroseconds() / 1000000,
                     context=line.context[:7],
                     prefix=True)
 
+            if progress.worker:
+                self.task.worker = progress.worker.hostname
+
             if progress.status in [common_pb.TaskStatus.TASK_RUNNING] \
                and progress.status != self.task.status():
                 self.task.running_execution(remote=True)
                 for extension in self.task.extensions:
                     extension.running_execution(remote=True)
 
+            if progress.status in [common_pb.TaskStatus.TASK_QUEUED]:
+                if last_status in [common_pb.TaskStatus.TASK_RUNNING]:
+                    self.task.restarted_execution(remote=True)
+                    for extension in self.task.extensions:
+                        extension.restarted_execution(remote=True)
+
             if progress.status in [
                     common_pb.TaskStatus.TASK_PASSED,
                     common_pb.TaskStatus.TASK_DOWNLOADED,
@@ -296,14 +335,17 @@ class RemoteExecutor(NetworkExecutor):
             ]:
                 break
 
+            if progress.status in [common_pb.TaskStatus.TASK_CANCELLED]:
+                if last_status in [common_pb.TaskStatus.TASK_RUNNING]:
+                    self.task.failed_execution(remote=True, interrupt=True)
+                    for extension in self.task.extensions:
+                        extension.failed_execution(remote=True, interrupt=True)
+                raise TaskCancelledException()
+
             if progress.status in [
-                    common_pb.TaskStatus.TASK_CANCELLED,
                     common_pb.TaskStatus.TASK_FAILED,
                     common_pb.TaskStatus.TASK_UNSTABLE,
             ]:
-                if last_status in [common_pb.TaskStatus.TASK_QUEUED]:
-                    raise TaskCancelledException()
-
                 for error in progress.errors:
                     with self.task.task.report() as report:
                         report.add_error(
@@ -312,14 +354,19 @@ class RemoteExecutor(NetworkExecutor):
                             error.message,
                             error.details,
                         )
-                with self.task.task.report() as report:
-                    for error in report.errors:
-                        raise_error(f"{error.type}: {error.message}")
+                self.task.raise_for_status()
                 raise raise_error("Remote execution failed")
 
             if progress.status in [
                     common_pb.TaskStatus.TASK_ERROR,
             ]:
+                log.log(
+                    log.VERBOSE,
+                    f"Host: {progress.worker.hostname}",
+                    created=time.time(),
+                    context=self.task.identity[:7],
+                    prefix=True)
+
                 for error in progress.errors:
                     with self.task.task.report() as report:
                         report.add_error(
@@ -328,11 +375,7 @@ class RemoteExecutor(NetworkExecutor):
                             error.message,
                             error.details,
                         )
-                with self.task.task.report() as report:
-                    for error in report.errors:
-                        for line in error.details.splitlines():
-                            log.error(line)
-                        raise_error(f"{error.type}: {error.message}")
+                self.task.raise_for_status(log_details=not self.factory.options.mute)
                 raise raise_error("Remote execution failed")
 
             last_status = progress.status
@@ -384,31 +427,16 @@ class RemoteSession(object):
         # Lock to ensure only one build is registered at a time.
         self.lock = Lock()
 
+        # The build environment: client, workspace, etc.
+        self.buildenv = None
+
     def initialize(self, graph):
         """ Initialize the session with the scheduler. """
-
-        registry = ExecutorRegistry.get()
-
-        # Create a list of parameters to send to the scheduler.
-        parameters = []
-        for key, value in registry.get_network_parameters(None).items():
-            parameters.append(common_pb.Property(key=key, value=value))
-
-        # Add parameters from the config / command line (-c params.key).
-        parameters.extend(config.export_params())
-
-        # Create the build environment.
-        self.buildenv = common_pb.BuildEnvironment(
-            client=selfdeploy.get_client(),
-            parameters=parameters,
-            task_default_parameters=scheduler.export_task_default_params(graph.tasks),
-            tasks=scheduler.export_tasks(graph.tasks + graph.pruned),
-            workspace=loader.export_workspace(graph.tasks),
-            loglevel=log.get_level(),
-            config=config.export_config(),
-        )
+        self.tasks = graph.tasks
+        self.pruned = graph.pruned
 
     @locked
+    @utils.retried.on_exception(grpc.RpcError)
     def make_build_request(self):
         """ Create a build request with the scheduler. """
 
@@ -416,6 +444,18 @@ class RemoteSession(object):
         if self.build:
             return
 
+        if not self.buildenv:
+            # Create the build environment.
+            self.buildenv = common_pb.BuildEnvironment(
+                client=selfdeploy.get_client(),
+                parameters=config.export_params(),
+                task_default_parameters=scheduler.export_task_default_params(self.tasks),
+                tasks=scheduler.export_tasks(self.tasks + self.pruned),
+                workspace=loader.export_workspace(self.tasks),
+                loglevel=log.get_level_pb(),
+                config=config.export_config(),
+            )
+
         # Create the build request.
         req = scheduler_pb.BuildRequest(
             environment=self.buildenv,
@@ -439,12 +479,15 @@ class RemoteSession(object):
         log.info(colors.blue("Build registered with scheduler, waiting for worker"))
         return self.build
 
-    def clear_build_request(self):
+    @locked
+    def clear_build_request(self, message=None):
         """ Clear the build request. Called when a build fails. """
 
         # Close grpc server response stream
         if self.build:
             self.build.cancel()
+            if message:
+                log.warning(message)
         self.build = None
         self.build_id = None
 
@@ -510,7 +553,7 @@ class RemoteExecutionFactory(NetworkExecutorFactory):
 log.verbose("[Remote] Loaded")
 
 
-@cli.cli.command()
+@cli.cli.command(hidden=True)
 @click.option("-w", "--worker", required=True, help="Worker identifier.")
 @click.option("-b", "--build", required=True, help="Build identifier to enlist for.")
 @click.argument("request", required=True)
@@ -533,13 +576,19 @@ def executor(ctx, worker, build, request):
 
     # Set log level
     loglevel = request.environment.loglevel
-    log.set_level(loglevel)
+    log.set_level_pb(loglevel)
+
+    # Import workspace
+    loader.import_workspace(request.environment)
 
-    # Import protobuf build description
-    manifest.ManifestExtensionRegistry.import_protobuf(request.environment)
+    # Import configuration snippet
+    config.import_config(request.environment.config)
+
+    # Import configuration parameters (-c params.key)
+    config.import_params({param.key: param.value for param in request.environment.parameters})
 
     options = JoltOptions(
-        network=False,
+        network=True,
         local=False,
         download=config.getboolean("network", "download", True),
         upload=config.getboolean("network", "upload", True),
@@ -569,12 +618,13 @@ def executor(ctx, worker, build, request):
 
     # Build the graph of tasks
     gb = GraphBuilder(registry, acache, options=options, progress=True, buildenv=request.environment)
-    dag = gb.build(request.environment.tasks.keys())
+    task_names = [task.name for task in request.environment.tasks.values()]
+    dag = gb.build(task_names)
 
     # Enlist to execute build tasks from the scheduler
     enlist_msg = scheduler_pb.TaskUpdate(
         request=scheduler_pb.TaskRequest(build_id=build),
-        worker=scheduler_pb.WorkerAllocation(id=worker),
+        worker=scheduler_pb.WorkerAllocation(id=worker, hostname=utils.hostname()),
     )
 
     # A queue to send updates to the scheduler
@@ -586,11 +636,12 @@ def executor(ctx, worker, build, request):
 
         # Subscribe to tasks
         for task in sched.GetTasks(updates):
-            log.set_level(loglevel)
+            log.set_level_pb(loglevel)
 
             log.info("Queuing {}", task.task_id)
             graph_task = dag.get_task_by_identity(task.task_id)
             executor = None
+            status = None
 
             try:
                 session = {}
@@ -600,27 +651,30 @@ def executor(ctx, worker, build, request):
 
                 # Run the task
                 with log.handler(LogHandler(updates, task)):
-                    executor.run(JoltEnvironment(cache=acache))
-
-            except KeyboardInterrupt as interrupt:
-                log.info("Task cancelled")
+                    executor.run(JoltEnvironment(cache=acache, queue=None, worker=True))
 
+            except KeyboardInterrupt:
                 # Send an update to the scheduler
                 update = scheduler_pb.TaskUpdate(
                     request=task,
                     status=common_pb.TaskStatus.TASK_CANCELLED,
                 )
                 updates.push(update)
+                continue
 
-                # Interrupt sent to process, not only the running task
-                raise interrupt
+            except Exception:
+                status = common_pb.TaskStatus.TASK_FAILED
 
-            except Exception as e:
-                log.set_level(log.EXCEPTION)
-                log.exception(e)
+            else:
+                status = graph_task.status()
 
+            finally:
                 errors = []
 
+                # If the task status remains queued, mark it as failed
+                if status in [common_pb.TaskStatus.TASK_QUEUED]:
+                    status = common_pb.TaskStatus.TASK_FAILED
+
                 # Add errors from the task to the update sent to the scheduler
                 with graph_task.task.report() as report:
                     for error in report.errors:
@@ -634,24 +688,19 @@ def executor(ctx, worker, build, request):
                 # Send an update to the scheduler
                 update = scheduler_pb.TaskUpdate(
                     request=task,
-                    status=common_pb.TaskStatus.TASK_FAILED,
+                    status=status,
                     errors=errors,
                 )
                 updates.push(update)
 
-            else:
-                status = graph_task.status()
+                # Release references to cache artifacts
+                acache.release()
 
-                # If the task status remains queued, mark it as failed
-                if status in [common_pb.TaskStatus.TASK_QUEUED]:
-                    status = common_pb.TaskStatus.TASK_FAILED
+    except grpc.RpcError as rpc_error:
+        log.warning("Scheduler Error: {}", rpc_error.details())
 
-                # Send an update to the scheduler
-                update = scheduler_pb.TaskUpdate(
-                    request=task,
-                    status=status,
-                )
-                updates.push(update)
+    except KeyboardInterrupt:
+        log.info("Interrupted, exiting")
 
     except Exception as e:
         log.set_level(log.EXCEPTION)

jolt/plugins/selfdeploy/setup.py

@@ -74,32 +74,32 @@ setup(
         "click>=8.1",
         "colorama",
         "fasteners",
-        "grpcio",
+        "grpcio>=1.62.2",
         "jinja2",
         "keyring",
         "keyrings.alt",
         "importlib_metadata",
         "lxml",
         "multi_key_dict",
-        "ninja-syntax",
+        "ninja",
         "protobuf",
         "psutil",
         "pygit2",
+        "py7zr",
         "requests",
+        "zstandard",
         "tqdm",
-        "unshare",
     ],
     dependency_links=[],
     extras_require={
         "allure": ["allure-python-commons"],
-        "amqp": ["pika"],
-        "conan": ["conan<2.0"],
+        "conan": ["conan>=2.0"],
         "dev": ["check-manifest"],
         "doc": ["sphinx-click", "sphinx-rtd-theme"],
         "test": ["coverage"],
     },
     package_data={
-        "jolt": ["**/*.sh", "**/*.xslt", "**/*.template"],
+        "jolt": ["**/*.sh", "**/*.xslt", "**/*.template", "**/fstree-*-x86_64"],
     },
     entry_points={
         "console_scripts": [

jolt/plugins/selfdeploy.py

@@ -1,4 +1,5 @@
-import pkg_resources
+import importlib_metadata
+import os
 
 from jolt import config
 from jolt import filesystem as fs
@@ -11,12 +12,9 @@ from jolt.cache import ArtifactCache
 from jolt.error import raise_error_if
 from jolt.graph import GraphBuilder
 from jolt.loader import JoltLoader
-from jolt.manifest import JoltManifest
 from jolt.scheduler import JoltEnvironment
 from jolt.scheduler import LocalExecutor
 from jolt.scheduler import LocalExecutorFactory
-from jolt.scheduler import NetworkExecutorExtension
-from jolt.scheduler import NetworkExecutorExtensionFactory
 from jolt.tasks import Task, TaskRegistry
 
 
@@ -90,6 +88,7 @@ class Jolt(Task):
             artifact.collect('jolt/*/*/*.py')
             artifact.collect('jolt/*/*.xslt')
             artifact.collect('jolt/*/*.template')
+            artifact.collect('jolt/bin')
             artifact.collect('jolt/plugins/selfdeploy/README.rst', flatten=True)
             artifact.collect('jolt/plugins/selfdeploy/setup.py', flatten=True)
             for e in self.extra_files:
@@ -97,43 +96,37 @@ class Jolt(Task):
                     artifact.collect(fs.path.basename(e))
 
 
-class SelfDeployExtension(NetworkExecutorExtension):
-    @utils.cached.instance
-    def get_parameters(self, _):
-        client = get_client()
-        params = {}
-        if client.identity:
-            params["jolt_identity"] = client.identity
-        if client.url:
-            params["jolt_url"] = client.url
-        return params
-
-
-@NetworkExecutorExtensionFactory.Register
-class SelfDeployExtensionFactory(NetworkExecutorExtensionFactory):
-    def create(self):
-        return SelfDeployExtension()
-
-
 @utils.cached.method
 def get_dependencies(packages=None):
     reqs = packages or ["jolt"]
     pkgs = {}
+    skip = set()
 
     while reqs:
         req = reqs.pop()
 
-        dist = pkg_resources.working_set.by_key.get(req)
+        try:
+            dist = importlib_metadata.distribution(req)
+        except (ImportError, importlib_metadata.PackageNotFoundError):
+            dist = None
+        except Exception:
+            dist = None
         if dist is None:
-            log.debug("[SelfDeploy] Dependency not found: {}", req)
-            pkgs[req] = req
+            skip.add(req)
             continue
 
-        for dep in dist.requires():
-            if dep.name not in pkgs:
-                reqs.append(dep.name)
+        for dep in dist.requires or []:
+            dep = dep.split(" ", 1)[0].strip()
+            dep = dep.split("[", 1)[0].strip()
+            dep = dep.split(";", 1)[0].strip()
+            dep = dep.split(">", 1)[0].strip()
+            dep = dep.split("=", 1)[0].strip()
+            dep = dep.split("<", 1)[0].strip()
+            dep = dep.split("!", 1)[0].strip()
+            if dep not in pkgs and dep not in skip:
+                reqs.append(dep)
 
-        pkgs[req] = f"{dist.project_name}=={dist.version}"
+        pkgs[req] = f"{dist.name}=={dist.version}"
 
     try:
         del pkgs["jolt"]
@@ -154,8 +147,8 @@ def publish_artifact():
     registry = TaskRegistry()
     registry.add_task_class(Jolt)
     acache = ArtifactCache.get()
-    env = JoltEnvironment(cache=acache)
-    gb = GraphBuilder(registry, acache, JoltManifest())
+    env = JoltEnvironment(cache=acache, queue=None)
+    gb = GraphBuilder(registry, acache)
     dag = gb.build(["jolt"])
     task = dag.select(lambda graph, task: True)
     assert len(task) == 1, "Too many selfdeploy tasks found"
@@ -189,8 +182,33 @@ def get_pinned_version():
     )
 
 
+def get_nix_version():
+    return common_pb.Client(
+        requirements=get_extra_dependencies(),
+        version=version.__version__,
+        nix=True,
+    )
+
+
 def get_client():
+    # Floating version is a special case where we want to deploy the Jolt
+    # source code to a remote cache and use that URL as the client URL
+    # for workers.
     floating = config.getboolean("selfdeploy", "floating", False)
     if floating:
         return get_floating_version()
+
+    # If Nix has been explicitly disabled, we want to pin versions.
+    if not config.getboolean("selfdeploy", "nix", True):
+        return get_pinned_version()
+
+    # If Nix has been explicitly enabled, we want to use the Nix shell.
+    if config.getboolean("selfdeploy", "nix", False):
+        return get_nix_version()
+
+    # If we are in a Nix shell, we want to use the Nix shell.
+    if os.environ.get("IN_NIX_SHELL"):
+        return get_nix_version()
+
+    # Default to pinned version
     return get_pinned_version()