primitive 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

primitive/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Dylan Stein <dylan@primitive.tech>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.2.1"
+__version__ = "0.2.3"

primitive/agent/actions.py

@@ -1,4 +1,3 @@
-import shutil
 import sys
 from time import sleep
 
@@ -7,9 +6,8 @@ from loguru import logger
 from primitive.__about__ import __version__
 from primitive.utils.actions import BaseAction
 
-from ..utils.cache import get_sources_cache
 from ..utils.exceptions import P_CLI_100
-from .runner import AgentRunner
+from .runner import Runner
 from .uploader import Uploader
 
 
@@ -28,9 +26,6 @@ class Agent(BaseAction):
         logger.info(" [*] primitive")
         logger.info(f" [*] Version: {__version__}")
 
-        # Create cache dir if it doesn't exist
-        cache_dir = get_sources_cache()
-
         # Create uploader
         uploader = Uploader(primitive=self.primitive)
 
@@ -51,12 +46,12 @@ class Agent(BaseAction):
             active_reservation_pk = None
 
             while True:
-                logger.debug("Syncing children...")
-                self.primitive.hardware._sync_children()
-
                 logger.debug("Scanning for files to upload...")
                 uploader.scan()
 
+                logger.debug("Syncing children...")
+                self.primitive.hardware._sync_children()
+
                 hardware = self.primitive.hardware.get_own_hardware_details()
 
                 if hardware["activeReservation"]:
@@ -125,29 +120,18 @@ class Agent(BaseAction):
                     logger.debug(f"Job Run ID: {job_run['id']}")
                     logger.debug(f"Job Name: {job_run['job']['name']}")
 
-                    git_repo_full_name = job_run["gitCommit"]["repoFullName"]
-                    git_ref = job_run["gitCommit"]["sha"]
-                    logger.debug(
-                        f"Downloading repository {git_repo_full_name} at ref {git_ref}"
-                    )
-
-                    github_access_token = (
-                        self.primitive.jobs.github_access_token_for_job_run(
-                            job_run["id"]
-                        )
+                    runner = Runner(
+                        primitive=self.primitive,
+                        job_run=job_run,
+                        max_log_size=500 * 1024,
                     )
 
                     try:
-                        downloaded_git_repository_dir = (
-                            self.primitive.git.download_git_repository_at_ref(
-                                git_repo_full_name=git_repo_full_name,
-                                git_ref=git_ref,
-                                github_access_token=github_access_token,
-                                destination=cache_dir,
-                            )
-                        )
+                        runner.setup()
                     except Exception as exception:
-                        logger.error(f"Error downloading source: {exception}")
+                        logger.exception(
+                            f"Exception while initializing runner: {exception}"
+                        )
                         self.primitive.jobs.job_run_update(
                             id=job_run["id"],
                             status="request_completed",
@@ -155,40 +139,21 @@ class Agent(BaseAction):
                         )
                         continue
 
-                    source_dir = downloaded_git_repository_dir.joinpath(
-                        job_run["jobSettings"]["rootDirectory"]
-                    )
-
                     try:
-                        # Initialize Runner
-                        runner = AgentRunner(
-                            primitive=self.primitive,
-                            source_dir=source_dir,
-                            job_run=job_run,
-                            max_log_size=500 * 1024,
-                        )
+                        runner.execute()
                     except Exception as exception:
-                        # Log Error
-                        logger.exception(
-                            f"Error initializing agent runner: {exception}"
+                        logger.exception(f"Exception while executing job: {exception}")
+                        self.primitive.jobs.job_run_update(
+                            id=job_run["id"],
+                            status="request_completed",
+                            conclusion="failure",
                         )
-                    else:
-                        try:
-                            # Execute job
-                            runner.execute()
-                        except Exception as exception:
-                            # Log Error
-                            logger.exception(
-                                f"AgentRunner exception while running executing customer job: {exception}"
-                            )
-                            self.primitive.jobs.job_run_update(
-                                id=job_run["id"],
-                                status="request_completed",
-                                conclusion="failure",
-                            )
                     finally:
-                        # Clean up
-                        shutil.rmtree(path=downloaded_git_repository_dir)
+                        runner.cleanup()
+
+                        # NOTE: also run scan here to force upload of artifacts
+                        # This should probably eventuall be another daemon?
+                        uploader.scan()
 
                 sleep(5)
         except KeyboardInterrupt:

primitive/agent/runner.py

@@ -1,18 +1,16 @@
 import os
-import threading
 import typing
-from enum import IntEnum
+import re
+import shutil
+from typing import Dict, TypedDict, List
+from abc import abstractmethod
+from enum import IntEnum, Enum
 from pathlib import Path, PurePath
-from time import sleep
-from typing import Callable, Dict, Iterable, List, Optional, TypedDict
-
-import yaml
 from loguru import logger
-
-from ..utils.cache import get_artifacts_cache, get_logs_cache
-from ..utils.files import find_files_for_extension
-from .process import Process
-from .provision import ProvisionPython
+import yaml
+import asyncio
+from primitive.utils.shell import env_string_to_dict
+from ..utils.cache import get_artifacts_cache, get_sources_cache, get_logs_cache
 
 try:
     from yaml import CLoader as Loader
@@ -22,23 +20,21 @@ except ImportError:
 if typing.TYPE_CHECKING:
     import primitive.client
 
+ENV_VAR_LOOKUP_START = "_ENV_VAR_LOOKUP_START"
+ENV_VAR_LOOKUP_END = "_ENV_VAR_LOOKUP_END"
 
-class Artifact(TypedDict):
-    name: str
-    extension: str
 
-
-class JobStep(TypedDict):
-    name: str
+class Task(TypedDict):
+    label: str
     workdir: str
-    artifacts: List[Artifact]
+    tags: Dict
     cmd: str
 
 
-class JobDescription(TypedDict):
-    name: str
-    provision: str
-    steps: List[JobStep]
+class JobConfig(TypedDict):
+    requires: List[str]
+    executes: List[Task]
+    stores: List[str]
 
 
 # NOTE This must match FailureLevel subclass in JobSettings model
@@ -47,237 +43,273 @@ class FailureLevel(IntEnum):
     WARNING = 2
 
 
-class AgentRunner:
+class LogLevel(Enum):
+    INFO = "INFO"
+    ERROR = "ERROR"
+    WARNING = "WARNING"
+
+
+class Runner:
     def __init__(
         self,
         primitive: "primitive.client.Primitive",
-        source_dir: Path,
         job_run: Dict,
-        max_log_size: int = 10 * 1024 * 1024,
-        log_to_file: bool = True,
+        max_log_size: int = 10 * 1024 * 1024,  # 10MB
     ) -> None:
         self.primitive = primitive
-        self.source_dir = source_dir
-        self.workdir = "."
+        self.job = job_run["job"]
         self.job_run = job_run
-        self.job_id = job_run["id"]
-        self.job_slug = job_run["job"]["slug"]
-        self.max_log_size = max_log_size
-        self.parse_logs = job_run["jobSettings"]["parseLogs"]
-        self.failure_level = job_run["jobSettings"]["failureLevel"]
-        self.log_to_file = log_to_file
-
-        # Enable and configure logger
+        self.job_settings = job_run["jobSettings"]
+        self.config = None
+        self.source_dir = None
+        self.initial_env = {}
+        self.modified_env = {}
+        self.file_logger = None
+
         logger.enable("primitive")
 
-        if self.log_to_file:
-            log_name = f"{self.job_slug}_{self.job_id}_{{time}}.primitive.log"
-            logger.add(
-                Path(get_logs_cache(self.job_id) / log_name),
-                rotation=self.max_log_size,
-                format=AgentRunner.log_serializer(),
+        # If max_log_size set to <= 0, disable file logging
+        if max_log_size > 0:
+            log_name = f"{self.job['slug']}_{self.job_run['jobRunNumber']}_{{time}}.primitive.log"
+
+            self.file_logger = logger.add(
+                Path(get_logs_cache(self.job_run["id"]) / log_name),
+                rotation=max_log_size,
+                format=Runner.fmt,
                 backtrace=True,
                 diagnose=True,
             )
 
-        logger.info(f"Scanning directory for job file {self.job_slug}")
-        yaml_file = Path(self.source_dir / ".primitive" / f"{self.job_slug}.yaml")
-        yml_file = Path(self.source_dir / ".primitive" / f"{self.job_slug}.yml")
+    def setup(self) -> None:
+        # Attempt to download the job source code
+        git_repo_full_name = self.job_run["gitCommit"]["repoFullName"]
+        git_ref = self.job_run["gitCommit"]["sha"]
+        logger.info(f"Downloading repository {git_repo_full_name} at ref {git_ref}")
+
+        github_access_token = self.primitive.jobs.github_access_token_for_job_run(
+            self.job_run["id"]
+        )
+
+        downloaded_git_repository_dir = (
+            self.primitive.git.download_git_repository_at_ref(
+                git_repo_full_name=git_repo_full_name,
+                git_ref=git_ref,
+                github_access_token=github_access_token,
+                destination=get_sources_cache(),
+            )
+        )
+
+        self.source_dir = downloaded_git_repository_dir.joinpath(
+            self.job_settings["rootDirectory"]
+        )
+
+        # Attempt to parse the job yaml file
+        logger.info(f"Scanning directory for job file {self.job['slug']}")
+        yaml_file = Path(self.source_dir / ".primitive" / f"{self.job['slug']}.yaml")
+        yml_file = Path(self.source_dir / ".primitive" / f"{self.job['slug']}.yml")
 
         if yaml_file.exists() and yml_file.exists():
             logger.error(
-                f"Found two job descriptions with the same slug: {self.job_slug}"
-            )
-            self.primitive.jobs.job_run_update(
-                self.job_id, status="request_completed", conclusion="failure"
+                f"Found two job descriptions with the same slug: {self.job['slug']}"
             )
             raise FileExistsError
 
-        if yaml_file.exists():
-            self.job = yaml.load(open(yaml_file, "r"), Loader=Loader)
-        elif yml_file.exists():
-            self.job = yaml.load(open(yml_file, "r"), Loader=Loader)
+        if yaml_file.exists() or yml_file.exists():
+            logger.info(f"Found job description for {self.job['slug']}")
+            config_file = yaml_file if yaml_file.exists() else yml_file
+            self.config = yaml.load(open(config_file, "r"), Loader=Loader)[
+                self.job["name"]
+            ]
         else:
             logger.error(
-                f"No job description with matching slug '{self.job_slug}' found"
-            )
-            self.primitive.jobs.job_run_update(
-                self.job_id, status="request_completed", conclusion="failure"
+                f"No job description with matching slug '{self.job['slug']}' found"
             )
             raise FileNotFoundError
 
-        logger.info(f"Found job description for {self.job_slug}")
-
-    @staticmethod
-    def log_serializer() -> Callable:
-        def fmt(record):
-            step = ""
-            if "step" in record["extra"]:
-                step = record["extra"]["step"]
-
-            log = (
-                "<green>{time:YYYY-MM-DD HH:mm:ss.SSS!UTC}</green> | "
-                "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | "
-                "<level>{level}</level> | "
-                f"{step} | "
-                "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - "
-                "<level>{message}</level>\n"
-            )
-
-            return log
-
-        return fmt
+        # Setup initial process environment
+        self.initial_env = os.environ
+        self.initial_env["PRIMITIVE_GIT_SHA"] = str(self.job_run["gitCommit"]["sha"])
+        self.initial_env["PRIMITIVE_GIT_BRANCH"] = str(
+            self.job_run["gitCommit"]["branch"]
+        )
+        self.initial_env["PRIMITIVE_GIT_REPO"] = str(
+            self.job_run["gitCommit"]["repoFullName"]
+        )
 
-    def name(self) -> str:
-        return self.job["name"]
+    def execute(self) -> None:
+        logger.info(f"Executing {self.job['slug']} job")
+        self.primitive.jobs.job_run_update(
+            self.job_run["id"], status="request_in_progress"
+        )
+        self.modified_env = {**self.initial_env}
 
-    def steps(self) -> Iterable[JobStep]:
-        for step in self.job["steps"]:
-            yield step
+        task_failed = False
+        for task in self.config["executes"]:
+            with logger.contextualize(label=task["label"]):
+                with asyncio.Runner() as async_runner:
+                    if task_failed := async_runner.run(self.run_task(task)):
+                        break
 
-    def execute(self) -> None:
-        logger.info(f"Executing {self.job_slug} job")
-        self.primitive.jobs.job_run_update(self.job_id, status="request_in_progress")
+        if not task_failed:
+            self.primitive.jobs.job_run_update(
+                self.job_run["id"], status="request_completed", conclusion="success"
+            )
+            logger.success(f"Completed {self.job['slug']} job")
+
+    async def run_task(self, task: Task) -> bool:
+        for cmd in task["cmd"].strip().split("\n"):
+            args = [
+                "/bin/bash",
+                "-c",
+                f"{cmd} && echo '{ENV_VAR_LOOKUP_START}' && env && echo '{ENV_VAR_LOOKUP_END}'",
+            ]
+
+            process = await asyncio.create_subprocess_exec(
+                *args,
+                env=self.modified_env,
+                cwd=str(Path(self.source_dir / task.get("workdir", ""))),
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
 
-        # Initialize the environment with the system
-        environment = os.environ
+            stdout_failed, stderr_failed, cancelled = await asyncio.gather(
+                self.log_cmd(
+                    process=process, stream=process.stdout, tags=task.get("tags", {})
+                ),
+                self.log_cmd(
+                    process=process, stream=process.stderr, tags=task.get("tags", {})
+                ),
+                asyncio.create_task(self.monitor_cmd(process=process)),
+            )
 
-        # Add local variables
-        environment["PRIMITIVE_GIT_SHA"] = str(self.job_run["gitCommit"]["sha"])
+            returncode = await process.wait()
 
-        if "provision" in self.job:
-            logger.info(f"Provisioning for {self.job['provision']} environment")
-            environment = self.provision()
+            if cancelled:
+                return True
 
-            if not environment:
-                self.primitive.jobs.job_run_update(
-                    self.job_id, status="request_completed", conclusion="failure"
-                )
-                logger.error(f"{self.job_slug} concluded with error(s)")
-                return
-
-        fail_level_detected = False
-        total_parsed_levels = {FailureLevel.ERROR: 0, FailureLevel.WARNING: 0}
-        for step in self.steps():
-            logger.info(f"Beginning step {step['name']}")
-
-            with logger.contextualize(step=step["name"]):
-                if "workdir" in step:
-                    self.workdir = step["workdir"]
-
-                proc = Process(
-                    cmd=step["cmd"],
-                    workdir=Path(self.source_dir / self.workdir),
-                    env=environment,
+            if returncode > 0 or stdout_failed or stderr_failed:
+                await self.primitive.jobs.ajob_run_update(
+                    self.job_run["id"], status="request_completed", conclusion="failure"
                 )
 
-                try:
-                    proc.start()
-                except Exception as exception:
-                    logger.exception(
-                        f"Error while attempting to run process {exception}"
+                if returncode > 0:
+                    logger.error(
+                        f"Task {task['label']} failed on '{cmd}' with return code {returncode}"
                     )
-                    self.primitive.jobs.job_run_update(
-                        self.job_id, status="request_completed", conclusion="failure"
-                    )
-                    logger.error(f"{self.job_slug} concluded with error(s)")
-                    return
-
-                def status_check():
-                    while proc.is_running():
-                        # Check job status
-                        status = self.primitive.jobs.get_job_status(self.job_id)
-                        status_value = status.data["jobRun"]["status"]
-
-                        # TODO: Should probably use request_cancelled or something
-                        # once we change it, we'll have to call conclude w/ cancelled status
-                        if status_value == "completed":
-                            logger.warning("Job cancelled by user")
-                            proc.terminate()
-                            return
-
-                        sleep(5)
-
-                status_thread = threading.Thread(target=status_check)
-                status_thread.start()
+                else:
+                    logger.error(f"Task {task['label']} failed on '{cmd}'")
+
+                return True
+
+        return False
+
+    async def log_cmd(self, process, stream, tags: Dict = {}) -> bool:
+        failure_detected = False
+        while line := await stream.readline():
+            raw_data = line.decode()
+
+            # handle env vars
+            if ENV_VAR_LOOKUP_START in raw_data:
+                env_vars_string = ""
+                while env_line := await stream.readline():
+                    if ENV_VAR_LOOKUP_END in env_line.decode():
+                        break
+                    env_vars_string += env_line.decode()
+
+                self.modified_env = env_string_to_dict(env_vars_string)
+                continue
+
+            # Handle logging
+            parse_logs = self.job_settings["parseLogs"]
+            parse_stderr = self.job_settings["parseStderr"]
+
+            level = LogLevel.INFO
+            tag = None
+            if (parse_logs and "error" in raw_data.lower()) or (
+                parse_stderr and stream is process.stderr
+            ):
+                level = LogLevel.ERROR
+            elif parse_logs and "warning" in raw_data.lower():
+                level = LogLevel.WARNING
+
+            failure_detected = (
+                level == LogLevel.ERROR
+                and self.job_settings["failureLevel"] >= FailureLevel.ERROR
+            ) or (
+                level == LogLevel.WARNING
+                and self.job_settings["failureLevel"] >= FailureLevel.WARNING
+            )
 
-                returncode = proc.wait()
+            # Tag on the first matching regex in the list
+            for tag_key, regex in tags.items():
+                pattern = re.compile(regex)
+                if pattern.match(raw_data):
+                    tag = tag_key
+                    break
 
-                logger.debug(
-                    f"Process {step['name']} finished with return code {returncode}"
-                )
+            logger.bind(tag=tag).log(level.value, raw_data.rstrip())
 
-                if proc.errors > 0 and self.failure_level >= FailureLevel.ERROR:
-                    fail_level_detected = True
+        return failure_detected
 
-                if proc.warnings > 0 and self.failure_level >= FailureLevel.WARNING:
-                    fail_level_detected = True
+    async def monitor_cmd(self, process) -> bool:
+        while process.returncode is None:
+            status = await self.primitive.jobs.aget_job_status(self.job_run["id"])
+            status_value = status.data["jobRun"]["status"]
+            conclusion_value = status.data["jobRun"]["conclusion"]
 
-                total_parsed_levels[FailureLevel.ERROR] += proc.errors
-                total_parsed_levels[FailureLevel.WARNING] += proc.warnings
+            if status_value == "completed" and conclusion_value == "cancelled":
+                logger.warning("Job cancelled by user")
+                try:
+                    process.terminate()
+                except ProcessLookupError:
+                    pass
 
-                status_thread.join()
+                return True
 
-            self.collect_artifacts(step)
+            await asyncio.sleep(5)
 
-            if returncode > 0:
-                self.primitive.jobs.job_run_update(
-                    self.job_id, status="request_completed", conclusion="failure"
-                )
-                logger.error(
-                    f"Step {step['name']} failed with return code {returncode}"
-                )
-                return
+        return False
 
-        if fail_level_detected and self.parse_logs:
-            self.primitive.jobs.job_run_update(
-                self.job_id, status="request_completed", conclusion="failure"
-            )
+    def cleanup(self) -> None:
+        logger.remove(self.file_logger)
 
-            logger.error(
-                (
-                    f"{self.job_slug} concluded"
-                    f" with {total_parsed_levels[FailureLevel.ERROR]} error(s)"
-                    f" and {total_parsed_levels[FailureLevel.WARNING]} warning(s)"
-                )
-            )
+        if "stores" not in self.config:
             return
 
-        self.primitive.jobs.job_run_update(
-            self.job_id, status="request_completed", conclusion="success"
-        )
-        logger.success(f"Completed {self.job_slug} job")
-
-    def provision(self) -> Optional[Dict]:
-        match self.job["provision"]:
-            case "python":
-                requirements_glob = self.source_dir.rglob("requirements.txt")
-
-                requirements_path = next(requirements_glob, None)
-
-                if not requirements_path:
-                    logger.error("Unable to locate requirements.txt")
-                    return None
+        for glob in self.config["stores"]:
+            path = Path(glob)
 
-                prov = ProvisionPython(self.source_dir, requirements_path)
-                return prov.create_env()
-
-    def collect_artifacts(self, step: JobStep) -> None:
-        if "artifacts" not in step:
-            return
-
-        for artifact in step["artifacts"]:
-            files = find_files_for_extension(self.source_dir, artifact["extension"])
+            if path.is_dir():
+                files = [str(f) for f in path.rglob("*") if f.is_file()]
+            else:
+                files = [str(f) for f in Path().glob(glob) if f.is_file()]
 
             for file in files:
-                # Find path relative to source_dir
                 relative_path = PurePath(file).relative_to(self.source_dir)
-
-                # Construct destination to preserve directory structure
-                destination = Path(get_artifacts_cache(self.job_id) / relative_path)
-
-                # Create directories if they don't exist
+                destination = Path(
+                    get_artifacts_cache(self.job_run["id"]) / relative_path
+                )
                 destination.parent.mkdir(parents=True, exist_ok=True)
+                Path(file).replace(destination)
+
+        shutil.rmtree(path=self.source_dir)
+
+    @abstractmethod
+    def fmt(record) -> str:
+        extra = record["extra"]
+        # Delimiters with empty space MUST exist for LogQL pattern matching
+        label = extra.get("label", None)
+        tag = extra.get("tag", None)
+        context = f"{label} | " if label else " | "
+        context += f"{tag} | " if tag else " | "
+
+        log = (
+            "<green>{time:YYYY-MM-DD HH:mm:ss.SSS!UTC}</green> | "
+            "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | "
+            "<level>{level}</level> | "
+            f"{context}"
+            "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - "
+            "<level>{message}</level>\n"
+        )
 
-                # Move file
-                file.rename(destination)
+        return log

primitive/cli.py

@@ -48,7 +48,7 @@ from .reservations.commands import cli as reservations_commands
 @click.version_option(__version__)
 @click.pass_context
 def cli(context, host, yes, debug, json, verbose):
-    """primitive - a CLI tool for https://primitive.design"""
+    """primitive - a CLI tool for https://primitive.tech"""
     context.ensure_object(dict)
     context.obj["YES"] = yes
     context.obj["DEBUG"] = debug

primitive/git/actions.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 import shutil
-from subprocess import run, CalledProcessError, DEVNULL
+from subprocess import run, CalledProcessError
 
 from gql import gql
 from loguru import logger
@@ -38,8 +38,17 @@ class Git(BaseAction):
         )
         source_dir = Path(destination).joinpath(git_repo_full_name.split("/")[-1])
 
+        # Clone with throw an exception if the directory already exists
+        if source_dir.exists():
+            shutil.rmtree(path=source_dir)
+
         try:
-            run(["git", "clone", url, source_dir, "--no-checkout"], check=True)
+            run(
+                ["git", "clone", url, source_dir, "--no-checkout"],
+                check=True,
+                # stdout=DEVNULL,
+                # stderr=DEVNULL,
+            )
         except CalledProcessError:
             raise Exception("Failed to download repository")
 
@@ -48,8 +57,8 @@ class Git(BaseAction):
                 ["git", "checkout", git_ref],
                 check=True,
                 cwd=source_dir,
-                stdout=DEVNULL,
-                stderr=DEVNULL,
+                # stdout=DEVNULL,
+                # stderr=DEVNULL,
             )
         except CalledProcessError:
             # Clean up directory if checkout failed

primitive/jobs/actions.py

@@ -128,6 +128,30 @@ class Jobs(BaseAction):
         )
         return result
 
+    @guard
+    async def ajob_run_update(
+        self,
+        id: str,
+        status: str = None,
+        conclusion: str = None,
+        file_ids: Optional[List[str]] = [],
+    ):
+        mutation = gql(job_run_update_mutation)
+        input = {"id": id}
+        if status:
+            input["status"] = status
+        if conclusion:
+            input["conclusion"] = conclusion
+        if file_ids and len(file_ids) > 0:
+            input["files"] = file_ids
+        variables = {"input": input}
+
+        async with self.primitive.session as session:
+            result = await session.execute(
+                mutation, variable_values=variables, get_execution_result=True
+            )
+        return result
+
     @guard
     def github_access_token_for_job_run(self, job_run_id: str):
         query = gql(github_app_token_for_job_run_query)
@@ -160,3 +184,14 @@ class Jobs(BaseAction):
             query, variable_values=variables, get_execution_result=True
         )
         return result
+
+    @guard
+    async def aget_job_status(self, id: str):
+        query = gql(job_run_status_query)
+        variables = {"id": id}
+
+        async with self.primitive.session as session:
+            result = await session.execute(
+                query, variable_values=variables, get_execution_result=True
+            )
+        return result

primitive/jobs/graphql/fragments.py

@@ -13,6 +13,7 @@ job_run_fragment = """
 fragment JobRunFragment on JobRun {
   id
   pk
+  jobRunNumber
   createdAt
   updatedAt
   completedAt
@@ -31,6 +32,7 @@ fragment JobRunFragment on JobRun {
     containerArgs
     rootDirectory
     parseLogs
+    parseStderr
     failureLevel
   }
   gitCommit {
@@ -45,5 +47,6 @@ job_run_status_fragment = """
 fragment JobRunStatusFragment on JobRun {
     id
     status
+    conclusion
 }
 """

{primitive-0.2.1.dist-info → primitive-0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: primitive
-Version: 0.2.1
+Version: 0.2.3
 Project-URL: Documentation, https://github.com//primitivecorp/primitive-cli#readme
 Project-URL: Issues, https://github.com//primitivecorp/primitive-cli/issues
 Project-URL: Source, https://github.com//primitivecorp/primitive-cli

{primitive-0.2.1.dist-info → primitive-0.2.3.dist-info}/RECORD

@@ -1,13 +1,11 @@
-primitive/__about__.py,sha256=35gFZI1DLl6m1EtBovM9NL6lrmA-BCOe4IxZ2XQyJu0,129
+primitive/__about__.py,sha256=8woIYDZ0kangkX7EciKlVF5rI46fZIDsaNCx1uPNmDM,129
 primitive/__init__.py,sha256=bwKdgggKNVssJFVPfKSxqFMz4IxSr54WWbmiZqTMPNI,106
-primitive/cli.py,sha256=CiI60bG3UZyNFuLTpchr0KeJRG5SALj455Ob11CegGE,2412
+primitive/cli.py,sha256=58fn6ayVSC1f4hLKx3FUNT9CkuPLva8dFQg0_YUwpio,2410
 primitive/client.py,sha256=PPyIQRvKKSqCF9RRF5mJJ4Vqqolpzy1YXqffNLKIvAA,2390
 primitive/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/agent/actions.py,sha256=vNe46YvSD5RXZaOa0pXD6b5T9V3pG23TUq-U3c4j_Xg,8487
+primitive/agent/actions.py,sha256=5Lmqxsf24eH6BDEesbJbn1Xj-8ifVlQ_Y3QR8xpkFtM,6869
 primitive/agent/commands.py,sha256=-dVDilELfkGfbZB7qfEPs77Dm1oT62qJj4tsIk4KoxI,254
-primitive/agent/process.py,sha256=32eoj0W1-LG-9xxeHia-jk9jTah1cnmjCYnvczgXYGU,3538
-primitive/agent/provision.py,sha256=rmwnro1K5F8mwtd45XAq7RVQmpDWnbBCQ8X_qgWhm3M,1546
-primitive/agent/runner.py,sha256=nuRtwEHl41Ic2BM8WhAwuJ2bCrtt7GxjIa9JSw9WTF8,9631
+primitive/agent/runner.py,sha256=S4fFBnbezUNKd4ma4lMNgIiqXaDF_ro6ga_GzqBj3no,10655
 primitive/agent/uploader.py,sha256=OkgwXhWKoECOJnW_ZmpzmUS_cpb-orC_uebNcmf5byw,2948
 primitive/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/auth/actions.py,sha256=MPsG9LcKcOPwA7gZ9Ewk0PZJhTQvIrGfODdz4GxSzgA,999
@@ -31,7 +29,7 @@ primitive/files/graphql/fragments.py,sha256=II6WHZjzSqX4IELwdiWokqHTKvDq6mMHF5gp
 primitive/files/graphql/mutations.py,sha256=Da_e6WSp-fsCYVE9A6SGkIQy9WDzjeQycNyHEn7vJqE,935
 primitive/files/graphql/queries.py,sha256=_ky-IRz928sKeSJuqaggTPxV4CGgmho3OyaAFu1z7nw,397
 primitive/git/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/git/actions.py,sha256=7mOhpujYYXGxjtpjsxH-mle-O1WXv1Vvx0bD6LUDGxY,1745
+primitive/git/actions.py,sha256=O17HX3eg8yAKzBX_5cMy49qDopU9cxx_5tuOhUAAjsU,2003
 primitive/git/commands.py,sha256=sCeSjkRgSEjCEsB5seXgB_h6xfk0KpvMvzMKoRfUbRA,1177
 primitive/git/graphql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/git/graphql/queries.py,sha256=I1HGlqBb1lHIAWVSsC8tVY9JdsQ8DJVqs4nqSTcL30M,98
@@ -48,10 +46,10 @@ primitive/hardware/graphql/fragments.py,sha256=kI6qnTNjaEaUr-C6eD55COphtueVYbYOW
 primitive/hardware/graphql/mutations.py,sha256=_4Hkbfik9Ron4T-meulu6T-9FR_BZjyPNwn745MPksU,1484
 primitive/hardware/graphql/queries.py,sha256=I86uLuOSjHSph11Y5MVCYko5Js7hoiEZ-cEoPTc4J-k,1392
 primitive/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/jobs/actions.py,sha256=CtyO-Z9614TgIoXJJX1QGsoll0fgpBIjG9PJH5JwCQs,4901
+primitive/jobs/actions.py,sha256=gnJVlCEiu0O98UG8EGAGlOrGNNQCDWXQBHX8UoSZH84,5959
 primitive/jobs/commands.py,sha256=MxPCkBEYW_eLNqgCRYeyj7ZcLOFAWfpVZlqDR2Y_S0o,830
 primitive/jobs/graphql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/jobs/graphql/fragments.py,sha256=GZ_rVc_pc3MfC8EhCY_X9rjeNUdiwEytdqrknHYWO-E,573
+primitive/jobs/graphql/fragments.py,sha256=1_ZttT7dx36KDC3DClJz9M8LMpsPwXySBygHSiUEcGg,619
 primitive/jobs/graphql/mutations.py,sha256=8ASvCmwQh7cMeeiykOdYaYVryG8FRIuVF6v_J8JJZuw,219
 primitive/jobs/graphql/queries.py,sha256=BrU_GnLjK0bTAmWsLSmGEUea7EM8MqTKxN1Qp6sSjwc,1597
 primitive/organizations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -86,15 +84,12 @@ primitive/utils/cache.py,sha256=FHGmVWYLJFQOazpXXcEwI0YJEZbdkgG39nOLdOv6VNk,1575
 primitive/utils/chunk_size.py,sha256=PAuVuirUTA9oRXyjo1c6MWxo31WVBRkWMuWw-AS58Bw,2914
 primitive/utils/config.py,sha256=DlFM5Nglo22WPtbpZSVtH7NX-PTMaKYlcrUE7GPRG4c,1058
 primitive/utils/exceptions.py,sha256=DrYHTcCAJGC7cCUwOx_FmdlVLWRdpzvDvpLb82heppE,311
-primitive/utils/files.py,sha256=QUa7c4t2PNvKOtyndLAxQMGvDM4cBftSeFh28xprVbM,752
-primitive/utils/git.py,sha256=1qNOu8X-33CavmrD580BmrFhD_WVO9PGWHUUboXJR_g,663
 primitive/utils/memory_size.py,sha256=4xfha21kW82nFvOTtDFx9Jk2ZQoEhkfXii-PGNTpIUk,3058
 primitive/utils/printer.py,sha256=f1XUpqi5dkTL3GWvYRUGlSwtj2IxU1q745T4Fxo7Tn4,370
 primitive/utils/shell.py,sha256=vpjr2Y7UQGYOvPGa6_RYXPPjqScfa9k7kT3tugF9h4Y,1837
 primitive/utils/text.py,sha256=XiESMnlhjQ534xE2hMNf08WehE1SKaYFRNih0MmnK0k,829
-primitive/utils/verible.py,sha256=Zb5NUISvcaIgEvgCDBWr-GCoceMa79Tcwvr5Wl9lfnA,2252
-primitive-0.2.1.dist-info/METADATA,sha256=SkWObx5lg3EFazkPqt6KrYiVItNl7l8IW5FVW5Fkow4,3669
-primitive-0.2.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-primitive-0.2.1.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
-primitive-0.2.1.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
-primitive-0.2.1.dist-info/RECORD,,
+primitive-0.2.3.dist-info/METADATA,sha256=lt9QA5sglw8QZVEiT5OaO3agzpJDYF1qFeMww-5Spp0,3669
+primitive-0.2.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+primitive-0.2.3.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
+primitive-0.2.3.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
+primitive-0.2.3.dist-info/RECORD,,

primitive/agent/process.py

@@ -1,125 +0,0 @@
-from typing import List
-from pathlib import Path
-from subprocess import Popen, PIPE
-import shlex
-import glob
-import selectors
-from loguru import logger
-from abc import abstractmethod
-
-
-class Process:
-    def __init__(
-        self,
-        cmd,
-        env,
-        workdir: str = ".",
-    ):
-        self.cmd = Process.expand_glob_in_cmd(
-            cmd_parts=shlex.split(cmd), workdir=workdir
-        )
-        self.env = env
-        self.workdir = workdir
-        self.process = None
-        self.stdout_thread = None
-        self.stderr_thread = None
-        self._errors = 0
-        self._warnings = 0
-
-    def start(self):
-        # Start the process
-        self.sel = selectors.DefaultSelector()
-        self.process = Popen(
-            self.cmd,
-            env=self.env,
-            cwd=self.workdir,
-            stdout=PIPE,
-            stderr=PIPE,
-            text=True,
-        )
-
-        self.sel.register(self.process.stdout, selectors.EVENT_READ)
-        self.sel.register(self.process.stderr, selectors.EVENT_READ)
-
-    def log(self):
-        for key, _ in self.sel.select():
-            data = key.fileobj.readline()
-            if not data:
-                continue
-
-            if key.fileobj is self.process.stdout:
-                raw_data = data.rstrip()
-                if "error" in raw_data.lower():
-                    logger.error(raw_data)
-                    self._errors += 1
-                elif "warning" in raw_data.lower():
-                    logger.warning(raw_data)
-                    self._warnings += 1
-                else:
-                    logger.info(raw_data)
-            elif key.fileobj is self.process.stderr:
-                logger.error(data.rstrip())
-                self._errors += 1
-
-    def wait(self):
-        while True:
-            self.log()
-            if not self.is_running():
-                break
-
-        return self.finish()
-
-    def run(self):
-        """Start and wait for the process."""
-        self.start()
-        return self.wait()
-
-    def is_running(self):
-        """Check if the process is still running."""
-        return self.process and self.process.poll() is None
-
-    def finish(self):
-        """Make sure that logging finishes"""
-        if self.process:
-            self.sel.unregister(self.process.stdout)
-            self.sel.unregister(self.process.stderr)
-            self.process.stdout.close()
-            self.process.stderr.close()
-
-            return self.process.poll()
-
-    def terminate(self):
-        """Terminate the process."""
-        if self.process:
-            self.process.terminate()
-
-    def kill(self):
-        """Kill the process."""
-        if self.process:
-            self.process.kill()
-
-    @abstractmethod
-    def expand_glob_in_cmd(cmd_parts: List[str], workdir: Path):
-        # Characters that indicate a glob pattern
-        glob_chars = {"*", "?", "[", "]", "{", "}"}
-        expanded_cmd = []
-        for part in cmd_parts:
-            if any(c in part for c in glob_chars):
-                matches = glob.glob(str(workdir / part))
-                if matches:
-                    expanded_cmd.extend(
-                        [str(Path(match).relative_to(workdir)) for match in matches]
-                    )
-                else:
-                    expanded_cmd.append(part)
-            else:
-                expanded_cmd.append(part)
-        return expanded_cmd
-
-    @property
-    def errors(self) -> int:
-        return self._errors
-
-    @property
-    def warnings(self) -> int:
-        return self._warnings

primitive/agent/provision.py

@@ -1,52 +0,0 @@
-import sys
-from subprocess import Popen, PIPE
-from pathlib import Path
-from typing import Dict
-from ..utils.shell import env_string_to_dict
-
-
-class ProvisionPython:
-    def __init__(self, source_dir: Path, requirements_path: Path):
-        self.source_dir = source_dir
-        self.requirements_path = requirements_path
-
-    def create_env(self) -> Dict:
-        cmd = f"{sys.executable} -m ensurepip"
-        proc = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, text=True)
-        proc.wait()
-
-        cmd = f"{sys.executable} -m pip install virtualenv"
-        proc = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, text=True)
-        proc.wait()
-
-        cmd = f"{sys.executable} -m virtualenv venv"
-        proc = Popen(
-            cmd, cwd=self.source_dir, stdout=PIPE, stderr=PIPE, shell=True, text=True
-        )
-        proc.wait()
-
-        cmd = "source venv/bin/activate && env"
-        proc = Popen(
-            cmd, cwd=self.source_dir, stdout=PIPE, stderr=PIPE, shell=True, text=True
-        )
-        proc.wait()
-
-        # Read the output and decode it
-        output, _ = proc.communicate()
-
-        # Split the output into lines and parse it into a dictionary
-        env_vars = env_string_to_dict(output)
-
-        cmd = f"python -m pip install -r {self.requirements_path}"
-        proc = Popen(
-            cmd,
-            cwd=self.source_dir,
-            env=env_vars,
-            stdout=PIPE,
-            stderr=PIPE,
-            shell=True,
-            text=True,
-        )
-        proc.wait()
-
-        return env_vars

primitive/utils/files.py

@@ -1,26 +0,0 @@
-from typing import List, Tuple
-from pathlib import Path
-from loguru import logger
-import os
-
-
-def find_files_for_extension(source: Path, extensions: Tuple[str]) -> List[Path]:
-    matching_files = []
-    logger.debug(f"Looking for files at {source} with extensions {extensions}")
-
-    has_walk = getattr(source, "walk", None)
-
-    if has_walk:
-        files = source.walk()
-    else:
-        files = os.walk(source)
-
-    for dirpath, dirnames, filenames in files:
-        for filename in filenames:
-            if filename.endswith(extensions):
-                matching_files.append(Path(dirpath).joinpath(filename))
-
-    logger.debug(
-        f"Found {len(matching_files)} following files that match: {matching_files}"
-    )
-    return matching_files

primitive/utils/git.py

@@ -1,15 +0,0 @@
-import os
-from loguru import logger
-
-
-def download_source(github_access_token, git_repository, git_ref) -> None:
-    # Download code to current directory
-    logger.debug(f"Downloading source code from {git_repository} {git_ref}")
-    url = f"https://api.github.com/repos/{git_repository}/tarball/{git_ref}"
-    # TODO: switch to supbrocess.run or subprocess.Popen
-    result = os.system(
-        f"curl -s -L -H 'Accept: application/vnd.github+json' -H 'Authorization: Bearer {github_access_token}' -H 'X-GitHub-Api-Version: 2022-11-28' {url} | tar zx --strip-components 1 -C ."
-    )
-
-    if result != 0:
-        raise Exception("Failed to import repository.")

primitive/utils/verible.py

@@ -1,57 +0,0 @@
-import tarfile
-
-import requests
-from loguru import logger
-
-from .cache import get_deps_cache
-from .shell import add_path_to_shell
-
-VERIBLE_MAC_OS_LINK = "https://github.com/chipsalliance/verible/releases/download/v0.0-3752-g8b64887e/verible-v0.0-3752-g8b64887e-macOS.tar.gz"
-VERIBLE_WINDOWS_64_OS_LINK = "https://github.com/chipsalliance/verible/releases/download/v0.0-3752-g8b64887e/verible-v0.0-3752-g8b64887e-win64.zip"
-VERIBLE_LINUX_X86_64_OS_LINK = "https://github.com/chipsalliance/verible/releases/download/v0.0-3752-g8b64887e/verible-v0.0-3752-g8b64887e-linux-static-x86_64.tar.gz"
-VERIBLE_LINUX_ARM64_LINK = "https://github.com/chipsalliance/verible/releases/download/v0.0-3752-g8b64887e/verible-v0.0-3752-g8b64887e-linux-static-arm64.tar.gz"
-
-
-def install_verible(system_info: dict) -> str:
-    url = None
-    if system_info.get("os_family") == "Darwin":
-        url = VERIBLE_MAC_OS_LINK
-    elif system_info.get("os_family") == "Windows":
-        url = VERIBLE_WINDOWS_64_OS_LINK
-    elif system_info.get("processor") == "x86_64":
-        url = VERIBLE_LINUX_X86_64_OS_LINK
-    elif system_info.get("processor") == "arm":
-        url = VERIBLE_LINUX_X86_64_OS_LINK
-
-    deps_cache = get_deps_cache()
-
-    verible_dir_name = url.split("/")[-1].split(".tar.gz")[0]
-    file_download_path = deps_cache / f"{verible_dir_name}.tar.gz"
-
-    logger.debug("Downloading verible")
-    response = requests.get(url, stream=True)
-    if response.status_code == 200:
-        with open(file_download_path, "wb") as file:
-            file.write(response.raw.read())
-    else:
-        raise Exception(
-            f"Failed to download verible. {response.status_code}. {response.text}"
-        )
-
-    logger.debug("Untaring verible")
-    with tarfile.open(file_download_path) as tar:
-        tar.extractall(deps_cache)
-
-    logger.debug("Deleting tar.gz artifact")
-    file_download_path.unlink()
-
-    unpacked_verible_dir_name = verible_dir_name
-    if "linux" in unpacked_verible_dir_name:
-        unpacked_verible_dir_name = unpacked_verible_dir_name.split("-linux")[0]
-
-    verible_bin = deps_cache.joinpath(unpacked_verible_dir_name).joinpath("bin")
-
-    logger.debug("Adding verible to PATH")
-    add_path_to_shell(verible_bin)
-
-    return verible_bin