primitive 0.2.11__py3-none-any.whl → 0.2.13__py3-none-any.whl

primitive/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Dylan Stein <dylan@primitive.tech>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.2.11"
+__version__ = "0.2.13"

primitive/agent/actions.py

@@ -6,10 +6,10 @@ from loguru import logger
 from primitive.__about__ import __version__
 from primitive.utils.actions import BaseAction
 
-from .runner import Runner
-from .uploader import Uploader
 from ..db import sqlite
 from ..db.models import JobRun
+from .runner import Runner
+from .uploader import Uploader
 
 
 class Agent(BaseAction):
@@ -33,137 +33,77 @@ class Agent(BaseAction):
         # Create uploader
         uploader = Uploader(primitive=self.primitive)
 
-        # self.primitive.hardware.update_hardware_system_info()
-        try:
-            # hey stupid:
-            # do not set is_available to True here, it will mess up the reservation logic
-            # only set is_available after we've checked that no active reservation is present
-            # setting is_available of the parent also effects the children,
-            # which may have active reservations as well
-            self.primitive.hardware.check_in_http(is_online=True)
-        except Exception as exception:
-            logger.exception(f"Error checking in hardware: {exception}")
-            sys.exit(1)
-
         try:
-            active_reservation_id = None
-            active_reservation_pk = None
-
             while True:
                 logger.debug("Scanning for files to upload...")
                 uploader.scan()
 
-                hardware = self.primitive.hardware.get_own_hardware_details()
-
-                if hardware["activeReservation"]:
-                    if (
-                        hardware["activeReservation"]["id"] != active_reservation_id
-                        or hardware["activeReservation"]["pk"] != active_reservation_pk
-                    ):
-                        logger.warning("New reservation for this hardware.")
-                        active_reservation_id = hardware["activeReservation"]["id"]
-                        active_reservation_pk = hardware["activeReservation"]["pk"]
-                        logger.debug("Active Reservation:")
-                        logger.debug(f"Node ID: {active_reservation_id}")
-                        logger.debug(f"PK: {active_reservation_pk}")
-
-                        logger.debug("Running pre provisioning steps for reservation.")
-                        self.primitive.provisioning.add_reservation_authorized_keys(
-                            reservation_id=active_reservation_id
-                        )
-                else:
-                    if (
-                        hardware["activeReservation"] is None
-                        and active_reservation_id is not None
-                        # and hardware["isAvailable"] NOTE: this condition was causing the CLI to get into a loop searching for job runs
-                    ):
-                        logger.debug("Previous Reservation is Complete:")
-                        logger.debug(f"Node ID: {active_reservation_id}")
-                        logger.debug(f"PK: {active_reservation_pk}")
-                        logger.debug(
-                            "Running cleanup provisioning steps for reservation."
-                        )
-                        self.primitive.provisioning.remove_reservation_authorized_keys(
-                            reservation_id=active_reservation_id
-                        )
-                        active_reservation_id = None
-                        active_reservation_pk = None
-
-                if not active_reservation_id:
-                    self.primitive.hardware.check_in_http(
-                        is_available=True, is_online=True
-                    )
+                db_job_run = JobRun.objects.first()
+
+                if not db_job_run:
                     sleep_amount = 5
                     logger.debug(
-                        f"No active reservation found... [sleeping {sleep_amount} seconds]"
+                        f"No pending job runs... [sleeping {sleep_amount} seconds]"
                     )
                     sleep(sleep_amount)
                     continue
 
-                job_runs_result = self.primitive.jobs.get_job_runs(
-                    status="pending", first=1, reservation_id=active_reservation_id
+                api_job_run_data = self.primitive.jobs.get_job_run(
+                    id=db_job_run.job_run_id,
                 )
 
-                pending_job_runs = [
-                    edge["node"] for edge in job_runs_result.data["jobRuns"]["edges"]
-                ]
-
-                if not pending_job_runs:
-                    self.primitive.hardware.check_in_http(is_online=True)
-                    sleep_amount = 5
-                    logger.debug(
-                        f"Waiting for Job Runs... [sleeping {sleep_amount} seconds]"
+                if not api_job_run_data or not api_job_run_data.data:
+                    logger.error(
+                        f"Job Run {db_job_run.job_run_id} not found in API, deleting from DB"
                     )
-                    sleep(sleep_amount)
+                    JobRun.objects.filter_by(job_run_id=db_job_run.job_run_id).delete()
                     continue
 
-                for job_run in pending_job_runs:
-                    logger.debug("Found pending Job Run")
-                    logger.debug(f"Job Run ID: {job_run['id']}")
-                    logger.debug(f"Job Name: {job_run['job']['name']}")
+                api_job_run = api_job_run_data.data["jobRun"]
 
-                    JobRun.objects.create(
-                        job_run_id=job_run["id"],
-                        pid=None,
+                logger.debug("Found pending Job Run")
+                logger.debug(f"Job Run ID: {api_job_run.get('id')}")
+                logger.debug(f"Job Name: {api_job_run.get('name')}")
+
+                runner = Runner(
+                    primitive=self.primitive,
+                    job_run=api_job_run,
+                    # max_log_size=500 * 1024,
+                )
+
+                try:
+                    runner.setup()
+                except Exception as exception:
+                    logger.exception(
+                        f"Exception while initializing runner: {exception}"
+                    )
+                    self.primitive.jobs.job_run_update(
+                        id=api_job_run.get("id"),
+                        status="request_completed",
+                        conclusion="failure",
                     )
+                    JobRun.objects.filter_by(job_run_id=api_job_run.get("id")).delete()
+                    continue
 
-                    runner = Runner(
-                        primitive=self.primitive,
-                        job_run=job_run,
-                        max_log_size=500 * 1024,
+                try:
+                    runner.execute()
+                except Exception as exception:
+                    logger.exception(f"Exception while executing job: {exception}")
+                    self.primitive.jobs.job_run_update(
+                        id=api_job_run.get("id"),
+                        status="request_completed",
+                        conclusion="failure",
                     )
+                finally:
+                    runner.cleanup()
+
+                    # NOTE: also run scan here to force upload of artifacts
+                    # This should probably eventually be another daemon?
+                    uploader.scan()
 
-                    try:
-                        runner.setup()
-                    except Exception as exception:
-                        logger.exception(
-                            f"Exception while initializing runner: {exception}"
-                        )
-                        self.primitive.jobs.job_run_update(
-                            id=job_run["id"],
-                            status="request_completed",
-                            conclusion="failure",
-                        )
-                        JobRun.objects.filter_by(job_run_id=job_run["id"]).delete()
-                        continue
-
-                    try:
-                        runner.execute()
-                    except Exception as exception:
-                        logger.exception(f"Exception while executing job: {exception}")
-                        self.primitive.jobs.job_run_update(
-                            id=job_run["id"],
-                            status="request_completed",
-                            conclusion="failure",
-                        )
-                    finally:
-                        runner.cleanup()
-
-                        # NOTE: also run scan here to force upload of artifacts
-                        # This should probably eventually be another daemon?
-                        uploader.scan()
-
-                        JobRun.objects.filter_by(job_run_id=job_run["id"]).delete()
+                    JobRun.objects.filter_by(
+                        job_run_id=api_job_run.get("id"),
+                    ).delete()
 
                 sleep(5)
         except KeyboardInterrupt:

primitive/agent/commands.py

@@ -1,6 +1,7 @@
-import click
 import typing
 
+import click
+
 if typing.TYPE_CHECKING:
     from ..client import Primitive
 

primitive/agent/runner.py

@@ -7,11 +7,11 @@ from abc import abstractmethod
 from enum import Enum, IntEnum
 from pathlib import Path, PurePath
 from typing import Dict, List, TypedDict
-from ..db.models import JobRun
 
 import yaml
 from loguru import logger
 
+from ..db.models import JobRun
 from ..utils.cache import get_artifacts_cache, get_logs_cache, get_sources_cache
 from ..utils.shell import env_to_dict
 
@@ -157,8 +157,11 @@ class Runner:
 
         task_failed = False
         cancelled = False
-        conclusion = "success"
+
         for task in self.config["executes"]:
+            # the get status check here is to ensure that if cancel is called
+            # while one task is running, we do not run any OTHER laebeled tasks
+            # THIS is required for MULTI STEP JOBS
             status = self.primitive.jobs.get_job_status(self.job_run["id"])
             status_value = status.data["jobRun"]["status"]
             conclusion_value = status.data["jobRun"]["conclusion"]
@@ -177,6 +180,14 @@ class Runner:
             f"Produced {number_of_files_produced} files for {self.job['slug']} job"
         )
 
+        # FOR NONE MULTI STEP JOBS
+        # we still have to check that the job was cancelled here as well
+        status = self.primitive.jobs.get_job_status(self.job_run["id"])
+        status_value = status.data["jobRun"]["status"]
+        conclusion_value = status.data["jobRun"]["conclusion"]
+        if status_value == "completed" and conclusion_value == "cancelled":
+            cancelled = True
+
         if cancelled:
             logger.warning("Job cancelled by user")
             self.primitive.jobs.job_run_update(
@@ -185,6 +196,7 @@ class Runner:
             )
             return
 
+        conclusion = "success"
         if task_failed:
             conclusion = "failure"
         else:

primitive/agent/uploader.py

@@ -50,8 +50,8 @@ class Uploader:
                     path=file,
                     key_prefix=str(PurePath(file).relative_to(cache.parent).parent),
                 )
-            except Exception as e:
-                if "is empty" in str(e):
+            except Exception as exception:
+                if "is empty" in str(exception):
                     logger.warning(f"{file} is empty, skipping upload")
                 continue
 

primitive/daemons/actions.py

@@ -1,13 +1,18 @@
 import platform
+import subprocess
 import typing
-from typing import Dict, Optional, List
+from pathlib import Path
+from typing import Dict, List, Optional
 
 if typing.TYPE_CHECKING:
     from ..client import Primitive
 
+from ..utils.daemons import Daemon
 from .launch_agents import LaunchAgent
 from .launch_service import LaunchService
-from ..utils.daemons import Daemon
+
+HOME_DIRECTORY = Path.home()
+PRIMITIVE_BINARY_PATH = Path(HOME_DIRECTORY / ".pyenv" / "shims" / "primitive")
 
 
 class Daemons:
@@ -15,16 +20,42 @@ class Daemons:
         self.primitive: Primitive = primitive
         self.os_family = platform.system()
 
+        found_primitive_binary_path = PRIMITIVE_BINARY_PATH
+        if not PRIMITIVE_BINARY_PATH.exists():
+            result = subprocess.run(["which", "primitive"], capture_output=True)
+            if result.returncode == 0:
+                found_primitive_binary_path = result.stdout.decode().rstrip("\n")
+            else:
+                raise Exception(
+                    f"primitive binary not found at {PRIMITIVE_BINARY_PATH}"
+                )
+
+        base_primitive_command = f'/bin/sh -lc "{found_primitive_binary_path} "'
+
         match self.os_family:
             case "Darwin":
                 self.daemons: Dict[str, Daemon] = {
-                    "agent": LaunchAgent("tech.primitive.agent"),
-                    "monitor": LaunchAgent("tech.primitive.monitor"),
+                    "agent": LaunchAgent(
+                        "tech.primitive.agent",
+                        executable=str(found_primitive_binary_path),
+                        command="agent --debug",
+                    ),
+                    "monitor": LaunchAgent(
+                        "tech.primitive.monitor",
+                        executable=str(found_primitive_binary_path),
+                        command="monitor --debug",
+                    ),
                 }
             case "Linux":
                 self.daemons: Dict[str, Daemon] = {
-                    "agent": LaunchService("tech.primitive.agent"),
-                    "monitor": LaunchService("tech.primitive.monitor"),
+                    "agent": LaunchService(
+                        "tech.primitive.agent",
+                        command=f"{base_primitive_command} agent --debug",
+                    ),
+                    "monitor": LaunchService(
+                        "tech.primitive.monitor",
+                        command=f"{base_primitive_command} monitor --debug",
+                    ),
                 }
             case _:
                 raise NotImplementedError(f"{self.os_family} is not supported.")

primitive/daemons/launch_agents.py

@@ -1,18 +1,21 @@
 import os
-from pathlib import Path
 import subprocess
+from pathlib import Path
+
 from loguru import logger
+
 from ..utils.daemons import Daemon
 
 HOME_DIRECTORY = Path.home()
 CURRENT_USER = str(HOME_DIRECTORY.expanduser()).lstrip("/Users/")
-PRIMITIVE_BINARY_PATH = Path(HOME_DIRECTORY / ".pyenv" / "shims" / "primitive")
 
 
 class LaunchAgent(Daemon):
-    def __init__(self, label: str):
+    def __init__(self, label: str, executable: str, command: str):
         self.label = label
         self.name = label.split(".")[-1]
+        self.executable = executable
+        self.command = command
 
     @property
     def file_path(self) -> Path:
@@ -22,10 +25,6 @@ class LaunchAgent(Daemon):
     def logs(self) -> Path:
         return Path(HOME_DIRECTORY / "Library" / "Logs" / f"{self.label}.log")
 
-    @property
-    def cmd(self) -> str:
-        return self.label.split(".")[-1]
-
     def stop(self, unload: bool = True) -> bool:
         try:
             stop_existing_process = f"launchctl stop {self.label}"
@@ -107,15 +106,6 @@ class LaunchAgent(Daemon):
         self.file_path.parent.mkdir(parents=True, exist_ok=True)
         self.file_path.touch()
 
-        found_primitive_binary_path = PRIMITIVE_BINARY_PATH
-        if not PRIMITIVE_BINARY_PATH.exists():
-            result = subprocess.run(["which", "primitive"], capture_output=True)
-            if result.returncode == 0:
-                found_primitive_binary_path = result.stdout.decode().rstrip("\n")
-            else:
-                logger.error("primitive binary not found")
-                return False
-
         self.file_path.write_text(
             f"""<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
@@ -134,8 +124,8 @@ class LaunchAgent(Daemon):
 	</array>
     <key>ProgramArguments</key>
     <array>
-        <string>{found_primitive_binary_path}</string>
-        <string>{self.cmd}</string>
+        <string>{self.executable}</string>
+        <string>{self.command}</string>
     </array>
     <key>RunAtLoad</key>
     <true/>

primitive/daemons/launch_service.py

@@ -1,18 +1,20 @@
-import os
 import configparser
+import os
 import subprocess
 from pathlib import Path
+
 from loguru import logger
+
 from ..utils.daemons import Daemon
 
 HOME_DIRECTORY = Path.home()
-PRIMITIVE_BINARY_PATH = Path(HOME_DIRECTORY / ".pyenv" / "shims" / "primitive")
 
 
 class LaunchService(Daemon):
-    def __init__(self, label: str):
+    def __init__(self, label: str, command: str):
         self.label = label
         self.name = label.split(".")[-1]
+        self.command = command
 
     @property
     def service_name(self) -> str:
@@ -119,17 +121,8 @@ class LaunchService(Daemon):
             "After": "network.target",
         }
 
-        found_primitive_binary_path = PRIMITIVE_BINARY_PATH
-        if not PRIMITIVE_BINARY_PATH.exists():
-            result = subprocess.run(["which", "primitive"], capture_output=True)
-            if result.returncode == 0:
-                found_primitive_binary_path = result.stdout.decode().rstrip("\n")
-            else:
-                print("primitive binary not found")
-                return False
-
         config["Service"] = {
-            "ExecStart": f'/bin/sh -lc "{found_primitive_binary_path} agent"',
+            "ExecStart": self.command,
             "Restart": "always",
             "StandardError": f"append:{self.logs}",
             "StandardOutput": f"append:{self.logs}",

primitive/db/models.py

@@ -1,8 +1,10 @@
+from typing import Any, Callable, Dict, Generic, List, Optional, Type, TypeVar, Union
+
 from sqlalchemy import Column, Integer, String
-from sqlalchemy.orm import Mapped, mapped_column, Query
-from typing import Optional, Generic, TypeVar, Callable, Type, List, Union, Dict, Any
-from .sqlite import Session
+from sqlalchemy.orm import Mapped, Query, mapped_column
+
 from .base import Base
+from .sqlite import Session
 
 T = TypeVar("T", bound="Base")
 
@@ -25,6 +27,14 @@ class Manager(Generic[T]):
         self.filters = kwargs
         return self
 
+    def exists(self) -> bool:
+        with Session() as session:
+            model = self.model_cls_lambda()
+            query = session.query(model)
+            query.filter_by(**self.filters)
+            self.filters.clear()
+            return query.count() > 0
+
     def all(self) -> List[T]:
         with Session() as session:
             model = self.model_cls_lambda()
@@ -51,7 +61,7 @@ class Manager(Generic[T]):
                 session.commit()
                 return query
             else:
-                raise ValueError(f"{model.__name__} not found")
+                raise ValueError(f"Update failed, {model.__name__} not found")
 
     def delete(self) -> None:
         with Session() as session:
@@ -62,7 +72,7 @@ class Manager(Generic[T]):
                 query.delete()
                 session.commit()
             else:
-                raise ValueError(f"{model.__name__} not found")
+                raise ValueError(f"Delete failed, {model.__name__} not found")
 
 
 class JobRun(Base):

primitive/db/sqlite.py

@@ -1,6 +1,9 @@
 from pathlib import Path
-from sqlalchemy import create_engine, Engine
+
+from loguru import logger
+from sqlalchemy import Engine, create_engine
 from sqlalchemy.orm import Session as SQLAlchemySession
+
 from ..utils.cache import get_cache_dir
 from .base import Base
 
@@ -9,9 +12,13 @@ def init() -> None:
     db_path: Path = get_cache_dir() / "primitive.sqlite3"
 
     # Drop DB existing database if it exists
+    # if db_path.exists():
+    #     logger.warning(f"[*] Deleting existing SQLite database at {db_path}")
+    #     db_path.unlink()
     if db_path.exists():
-        db_path.unlink()
+        return
 
+    logger.info(f"[*] Initializing SQLite database at {db_path}")
     engine = create_engine(f"sqlite:///{db_path}", echo=False)
     Base.metadata.create_all(engine)
 

primitive/hardware/actions.py

@@ -592,12 +592,17 @@ class Hardware(BaseAction):
         pass
 
     @guard
-    def _sync_children(self):
+    def _sync_children(self, hardware: Optional[Dict[str, str]] = None):
         # get the existing children if any from the hardware details
         # get the latest children from the node
         # compare the two and update the node with the latest children
         # remove any children from remote that are not in the latest children
-        hardware = self.primitive.hardware.get_own_hardware_details()
+        if not hardware:
+            hardware = self.primitive.hardware.get_own_hardware_details()
+        if not hardware:
+            logger.error("No hardware found.")
+            return
+
         remote_children = hardware.get("children", [])
         local_children = self.primitive.hardware._list_local_children()
 

primitive/monitor/actions.py

@@ -1,13 +1,15 @@
-from primitive.utils.actions import BaseAction
-from loguru import logger
-from primitive.__about__ import __version__
-from ..utils.exceptions import P_CLI_100
 import sys
-import psutil
-from ..db import sqlite
-from ..db.models import JobRun
 from time import sleep
 
+import psutil
+from loguru import logger
+
+from primitive.__about__ import __version__
+from primitive.db import sqlite
+from primitive.db.models import JobRun
+from primitive.utils.actions import BaseAction
+from primitive.utils.exceptions import P_CLI_100
+
 
 class Monitor(BaseAction):
     def start(self):
@@ -22,9 +24,6 @@ class Monitor(BaseAction):
         logger.info("[*] primitive monitor")
         logger.info(f"[*] Version: {__version__}")
 
-        # Initialize the database
-        sqlite.init()
-
         try:
             # hey stupid:
             # do not set is_available to True here, it will mess up the reservation logic
@@ -36,34 +35,39 @@ class Monitor(BaseAction):
             logger.exception(f"Error checking in hardware: {exception}")
             sys.exit(1)
 
+        # Initialize the database
+        sqlite.init()
+
         try:
-            while True:
-                logger.debug("Syncing children...")
-                self.primitive.hardware._sync_children()
+            active_reservation_id = None
+            active_reservation_pk = None
 
-                # Look for entries in the database
-                procs = JobRun.objects.all()
+            while True:
+                # FIRST, check for jobs in the database that are running
+                db_job_runs = JobRun.objects.all()
+                for job_run in db_job_runs:
+                    if job_run.pid is None:
+                        pid_sleep_amount = 0.1
+                        logger.debug(
+                            f"Job run {job_run.job_run_id} has no PID. Agent has not started."
+                        )
+                        logger.debug(
+                            f"Sleeping {pid_sleep_amount} seconds before checking again..."
+                        )
+                        sleep(pid_sleep_amount)
+                        continue
 
-                # No procs in the database => nothing to monitor
-                if len(procs) == 0:
-                    sleep_amount = 5
                     logger.debug(
-                        f"No active processes found... [sleeping {sleep_amount} seconds]"
+                        f"Checking process PID {job_run.pid} for JobRun {job_run.job_run_id}..."
                     )
-                    sleep(sleep_amount)
-                    continue
 
-                # If there is a process in the database, take over check in from agent
-                try:
-                    self.primitive.hardware.check_in_http(is_online=True)
-                except Exception as exception:
-                    logger.exception(f"Error checking in hardware: {exception}")
-
-                # For each process, check status and kill if cancelled
-                for proc in procs:
-                    logger.debug(f"Checking process {proc.pid}...")
+                    status = self.primitive.jobs.get_job_status(job_run.job_run_id)
+                    if status is None or status.data is None:
+                        logger.error(
+                            f"Error fetching status of <JobRun {job_run.job_run_id}>."
+                        )
+                        continue
 
-                    status = self.primitive.jobs.get_job_status(proc.job_run_id)
                     status_value = status.data["jobRun"]["status"]
                     conclusion_value = status.data["jobRun"]["conclusion"]
 
@@ -71,7 +75,7 @@ class Monitor(BaseAction):
                     logger.debug(f"- Conclusion: {conclusion_value}")
 
                     try:
-                        parent = psutil.Process(proc.pid)
+                        parent = psutil.Process(job_run.pid)
                     except psutil.NoSuchProcess:
                         logger.debug("Process not found")
                         continue
@@ -87,7 +91,100 @@ class Monitor(BaseAction):
                         logger.debug(f"Killing parent process {parent.pid}...")
                         parent.kill()
 
-                sleep(5)
+                    if status != "completed":
+                        sleep(1)
+                        continue
+
+                # Second, check for active reservations
+                hardware = self.primitive.hardware.get_own_hardware_details()
+                if hardware["activeReservation"]:
+                    if (
+                        hardware["activeReservation"]["id"] != active_reservation_id
+                        or hardware["activeReservation"]["pk"] != active_reservation_pk
+                    ):
+                        logger.info("New reservation for this hardware.")
+                        active_reservation_id = hardware["activeReservation"]["id"]
+                        active_reservation_pk = hardware["activeReservation"]["pk"]
+                        logger.debug("Active Reservation:")
+                        logger.debug(f"Node ID: {active_reservation_id}")
+                        logger.debug(f"PK: {active_reservation_pk}")
+
+                        logger.debug("Running pre provisioning steps for reservation.")
+                        self.primitive.provisioning.add_reservation_authorized_keys(
+                            reservation_id=active_reservation_id
+                        )
+
+                if not active_reservation_id:
+                    self.primitive.hardware.check_in_http(
+                        is_available=True, is_online=True
+                    )
+                    logger.debug("Syncing children...")
+                    self.primitive.hardware._sync_children(hardware=hardware)
+
+                    sleep_amount = 5
+                    logger.debug(
+                        f"No active reservation found... [sleeping {sleep_amount} seconds]"
+                    )
+                    sleep(sleep_amount)
+                    continue
+                else:
+                    if (
+                        hardware["activeReservation"] is None
+                        and active_reservation_id is not None
+                        # and hardware["isAvailable"] NOTE: this condition was causing the CLI to get into a loop searching for job runs
+                    ):
+                        logger.debug("Previous Reservation is Complete:")
+                        logger.debug(f"Node ID: {active_reservation_id}")
+                        logger.debug(f"PK: {active_reservation_pk}")
+                        logger.debug(
+                            "Running cleanup provisioning steps for reservation."
+                        )
+                        self.primitive.provisioning.remove_reservation_authorized_keys(
+                            reservation_id=active_reservation_id
+                        )
+                        active_reservation_id = None
+                        active_reservation_pk = None
+
+                # Third, see if the active reservation has any pending job runs
+                job_runs_for_reservation = self.primitive.jobs.get_job_runs(
+                    status="pending", first=1, reservation_id=active_reservation_id
+                )
+
+                if (
+                    job_runs_for_reservation is None
+                    or job_runs_for_reservation.data is None
+                ):
+                    logger.error("Error fetching job runs.")
+                    sleep_amount = 5
+                    logger.debug(
+                        f"Error fetching job runs... [sleeping {sleep_amount} seconds]"
+                    )
+                    sleep(sleep_amount)
+                    continue
+
+                pending_job_runs = [
+                    edge["node"]
+                    for edge in job_runs_for_reservation.data["jobRuns"]["edges"]
+                ]
+
+                if not pending_job_runs:
+                    self.primitive.hardware.check_in_http(
+                        is_available=False, is_online=True
+                    )
+                    sleep_amount = 5
+                    logger.debug(
+                        f"Waiting for Job Runs... [sleeping {sleep_amount} seconds]"
+                    )
+                    sleep(sleep_amount)
+                    continue
+
+                # If we did find a pending job run, check if it exists in the database
+                # and create it if it doesn't.
+                # This will trigger the agent to start the job run.
+                job_run = pending_job_runs[0]
+                if not JobRun.objects.filter_by(job_run_id=job_run["id"]).exists():
+                    JobRun.objects.create(job_run_id=job_run["id"], pid=None)
+                    logger.debug(f"Creating job run in database: {job_run['id']}")
 
         except KeyboardInterrupt:
             logger.info("[*] Stopping primitive monitor...")

primitive/utils/daemons.py

@@ -1,5 +1,5 @@
-from pathlib import Path
 from abc import ABC, abstractmethod
+from pathlib import Path
 
 
 class Daemon(ABC):

{primitive-0.2.11.dist-info → primitive-0.2.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: primitive
-Version: 0.2.11
+Version: 0.2.13
 Project-URL: Documentation, https://github.com//primitivecorp/primitive-cli#readme
 Project-URL: Issues, https://github.com//primitivecorp/primitive-cli/issues
 Project-URL: Source, https://github.com//primitivecorp/primitive-cli

{primitive-0.2.11.dist-info → primitive-0.2.13.dist-info}/RECORD

@@ -1,26 +1,26 @@
-primitive/__about__.py,sha256=51uVt8anKtwpofvSe0Zwd_GncrkYXPbI0DjswyDbXRs,130
+primitive/__about__.py,sha256=bGWt1PkABewYSRyq23zRxqOW2ES-H58OSO6OmpHi7ac,130
 primitive/__init__.py,sha256=bwKdgggKNVssJFVPfKSxqFMz4IxSr54WWbmiZqTMPNI,106
 primitive/cli.py,sha256=g7EtHI9MATAB0qQu5w-WzbXtxz_8zu8z5E7sETmMkKU,2509
 primitive/client.py,sha256=h8WZVnQylVe0vbpuyC8YZHl2JyITSPC-1HbUcmrE5pc,3623
 primitive/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/agent/actions.py,sha256=mmMwc9D8zo1w9aLlaYPBArncQ_qqnG6L9TWadcxvO-g,7018
-primitive/agent/commands.py,sha256=-dVDilELfkGfbZB7qfEPs77Dm1oT62qJj4tsIk4KoxI,254
-primitive/agent/runner.py,sha256=IuY-UMZ1WupYBXuPr-Ne0dcWeLFeV2_O-6F0SiG3VOs,13942
-primitive/agent/uploader.py,sha256=6pjUyb1LyUCpHBE6p13pRpXxy6iDxu14qJGvE3R6cVo,3155
+primitive/agent/actions.py,sha256=PzFOgxuRrhbUGpUygnTZVru58Fv87GI8tgJYkZB1LjI,3773
+primitive/agent/commands.py,sha256=cK7d3OcN5Z65gQWVZFQ-Y9ddw9Pes4f9OVBpeMsj5sE,255
+primitive/agent/runner.py,sha256=CoRyReO3jPV8B7vILVWdszFD4GVop7HsVEUo1hoRXjo,14556
+primitive/agent/uploader.py,sha256=ZzrzsajNBogwEC7mT6Ejy0h2Jd9axMYGzt9pbCvVMlk,3171
 primitive/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/auth/actions.py,sha256=MPsG9LcKcOPwA7gZ9Ewk0PZJhTQvIrGfODdz4GxSzgA,999
 primitive/auth/commands.py,sha256=2z5u5xX64n0yILucx9emtWh3uQXLvs2QQQQIldZGr94,2341
 primitive/auth/graphql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/auth/graphql/queries.py,sha256=jhrr_VFzHIn8vcVprMIzUx7V4kkWYdR6CKMKPoVFv60,180
 primitive/daemons/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/daemons/actions.py,sha256=V4BUCLS8UoQOZoS2vwEkYQpWAUNdZnMPBhQR19RvQXs,2023
+primitive/daemons/actions.py,sha256=jUE2DSNI5GDbbAZiGaauwfh50UQZf_uDLqpZLEwDq6w,3292
 primitive/daemons/commands.py,sha256=Xt4qFymNrDLdHJhRnEH_4Re-2xX6w1OT-chV9k7dFCs,2670
-primitive/daemons/launch_agents.py,sha256=KD7cqQZDtfDmMyNiYrswTRWEktvS9A1QsqQF1jhMDjw,7940
-primitive/daemons/launch_service.py,sha256=IhvKZqU5juA3hKsvUB2275BP8lNBl5XWkVoqVgdwy-o,8013
+primitive/daemons/launch_agents.py,sha256=VQ-c9PVTOr3JVt2jfLxIPDS1glKyBM2oDAzSUsFP9_A,7455
+primitive/daemons/launch_service.py,sha256=JA2kj1obCzu9sGmLFUMdEVKI-aj018b_tZBor0wVJSQ,7531
 primitive/daemons/ui.py,sha256=Af3OJWJ0jdGlb1nfA5yaGYdhBEqqpM8zP2U2vUQdCbw,1236
 primitive/db/base.py,sha256=mH7f2d_jiyxJSSx9Gk53QBXRa3LiKBsBjkFgvmtH1WA,83
-primitive/db/models.py,sha256=UnbRVhYoIG3rjfcjAQaGaJfSi_c7iJVQWfnpKU9TOp4,2543
-primitive/db/sqlite.py,sha256=t942Yrj7Z1wyRla3ZtLdk5Z7rmKDuGUY3fNKc39UbRQ,735
+primitive/db/models.py,sha256=GfnJdAq4Tb68CI4BKAuJDZVqioGavveaAHbCPeLNngw,2840
+primitive/db/sqlite.py,sha256=3V9ZxbgME1ThfJp90MPLUxU8b9imgNZM5CHOnA-WkaQ,953
 primitive/exec/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/exec/actions.py,sha256=4d_TCjNDcVFoZ9Zw7ZuBa6hKMv2Xzm7_UX_8wcX1aSk,4124
 primitive/exec/commands.py,sha256=66LO2kkJC-ynNZQpUCXv4Ol15QoacdSZAHblePDcmLo,510
@@ -42,7 +42,7 @@ primitive/graphql/relay.py,sha256=bmij2AjdpURQ6GGVCxwWhauF-r_SxuAU2oJ4sDbLxpI,72
 primitive/graphql/sdk.py,sha256=KhVWDZms_eMBgt6ftSJitRALguagy-nmrj4IC2taeXY,1535
 primitive/graphql/utility_fragments.py,sha256=uIjwILC4QtWNyO5vu77VjQf_p0jvP3A9q_6zRq91zqs,303
 primitive/hardware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/hardware/actions.py,sha256=8irVoDN4uzMMp58sgA9VOB1aO2NwBBVnJfg_k7i5tmA,25912
+primitive/hardware/actions.py,sha256=d5KwuSsceOhDH9rgOL7YTCpQPhqT2inRTiZnROtiDic,26076
 primitive/hardware/android.py,sha256=tu7pBPxWFrIwb_mm5CEdFFf1_veNDOKjOCQg13i_Lh4,2758
 primitive/hardware/commands.py,sha256=ixMPhDOpsU-eONxmimqKVynus-Eaq2XPKEK017WM_rM,3229
 primitive/hardware/ui.py,sha256=12rucuZ2s-w5R4bKyxON5dEbrdDnVf5sbj3K_nbdo44,2473
@@ -57,7 +57,7 @@ primitive/jobs/graphql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 primitive/jobs/graphql/fragments.py,sha256=1_ZttT7dx36KDC3DClJz9M8LMpsPwXySBygHSiUEcGg,619
 primitive/jobs/graphql/mutations.py,sha256=8ASvCmwQh7cMeeiykOdYaYVryG8FRIuVF6v_J8JJZuw,219
 primitive/jobs/graphql/queries.py,sha256=BrU_GnLjK0bTAmWsLSmGEUea7EM8MqTKxN1Qp6sSjwc,1597
-primitive/monitor/actions.py,sha256=AHPtZRXdvlZNfzpBJd4PkXF5QxTYj0BYwq-qB8PLZZY,3990
+primitive/monitor/actions.py,sha256=GUQrwuan82pOJ5gI2FvQYzgDoP4fs28PdcI_fg_aXRs,8692
 primitive/monitor/commands.py,sha256=dZsD8WKGU4OYO_AlKawfeRNVTMN0xJ-DFRkmKTS464s,258
 primitive/organizations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/organizations/actions.py,sha256=Tgp_rox0jcvfhQ-LmcWc9vkPdeJu5Bk6U1rNuT9oDnw,1088
@@ -90,14 +90,14 @@ primitive/utils/auth.py,sha256=uBIZNPF2CpbaPV2UMi6eWVUKghV6WIm-pG3-UM29bNs,1465
 primitive/utils/cache.py,sha256=FHGmVWYLJFQOazpXXcEwI0YJEZbdkgG39nOLdOv6VNk,1575
 primitive/utils/chunk_size.py,sha256=PAuVuirUTA9oRXyjo1c6MWxo31WVBRkWMuWw-AS58Bw,2914
 primitive/utils/config.py,sha256=DlFM5Nglo22WPtbpZSVtH7NX-PTMaKYlcrUE7GPRG4c,1058
-primitive/utils/daemons.py,sha256=YkG-OcrTxMhGedbNJMKLq_e7CdTy30Ba2oCVUY-09Co,1086
+primitive/utils/daemons.py,sha256=mSoSHitiGfS4KYAEK9sKsiv_YcACHKgY3qISnDpUUIE,1086
 primitive/utils/exceptions.py,sha256=DrYHTcCAJGC7cCUwOx_FmdlVLWRdpzvDvpLb82heppE,311
 primitive/utils/memory_size.py,sha256=4xfha21kW82nFvOTtDFx9Jk2ZQoEhkfXii-PGNTpIUk,3058
 primitive/utils/printer.py,sha256=f1XUpqi5dkTL3GWvYRUGlSwtj2IxU1q745T4Fxo7Tn4,370
 primitive/utils/shell.py,sha256=jWzb7ky7p987dJas6ZvarK3IJNZ5cwBXcryRWb9Uh6U,2072
 primitive/utils/text.py,sha256=XiESMnlhjQ534xE2hMNf08WehE1SKaYFRNih0MmnK0k,829
-primitive-0.2.11.dist-info/METADATA,sha256=MR8wugqoR7n8zbscUyXhhgynbMQnmeXAXMnD2fk4gPs,3733
-primitive-0.2.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-primitive-0.2.11.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
-primitive-0.2.11.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
-primitive-0.2.11.dist-info/RECORD,,
+primitive-0.2.13.dist-info/METADATA,sha256=Zq5jWlNGydETK3dKcPdCw0yUZsnJkwCsxuRq1LC4fnM,3733
+primitive-0.2.13.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+primitive-0.2.13.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
+primitive-0.2.13.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
+primitive-0.2.13.dist-info/RECORD,,