primitive 0.1.52__tar.gz → 0.1.54__tar.gz

{primitive-0.1.52 → primitive-0.1.54}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: primitive
-Version: 0.1.52
+Version: 0.1.54
 Project-URL: Documentation, https://github.com//primitivecorp/primitive-cli#readme
 Project-URL: Issues, https://github.com//primitivecorp/primitive-cli/issues
 Project-URL: Source, https://github.com//primitivecorp/primitive-cli

{primitive-0.1.52 → primitive-0.1.54}/src/primitive/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Dylan Stein <dylan@primitive.tech>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.1.52"
+__version__ = "0.1.54"

{primitive-0.1.52 → primitive-0.1.54}/src/primitive/agent/actions.py

@@ -1,4 +1,5 @@
 import sys
+import shutil
 from time import sleep
 from primitive.utils.actions import BaseAction
 from loguru import logger
@@ -119,7 +120,7 @@ class Agent(BaseAction):
                     )
 
                     runner = AgentRunner(
-                        self.primitive,
+                        primitive=self.primitive,
                         source_dir=source_dir,
                         job_id=job_run["id"],
                         job_slug=job_run["job"]["slug"],
@@ -129,7 +130,7 @@ class Agent(BaseAction):
                     runner.execute()
 
                     # Clean up
-                    # shutil.rmtree(path=downloaded_git_repository_dir)
+                    shutil.rmtree(path=downloaded_git_repository_dir)
 
                 sleep(5)
         except KeyboardInterrupt:

{primitive-0.1.52 → primitive-0.1.54}/src/primitive/agent/process.py

@@ -1,6 +1,6 @@
 from subprocess import Popen, PIPE
-import threading
 import shlex
+import selectors
 from loguru import logger
 
 
@@ -21,8 +21,7 @@ class Process:
 
     def start(self):
         # Start the process
-        logger.info(f"cmd: {self.cmd}")
-        logger.info(f"workdir: {self.workdir}")
+        self.sel = selectors.DefaultSelector()
         self.process = Popen(
             self.cmd,
             env=self.env,
@@ -32,40 +31,33 @@ class Process:
             text=True,
         )
 
-        # Function to read and log output from a pipe
-        def log_output(pipe, level):
-            for line in iter(pipe.readline, ""):
-                if line:
-                    logger.log(level, line.rstrip())
-                    if level == "ERROR":
-                        self._errors += 1
+        self.sel.register(self.process.stdout, selectors.EVENT_READ)
+        self.sel.register(self.process.stderr, selectors.EVENT_READ)
 
-            pipe.close()
+    def probe_logs(self):
+        for key, _ in self.sel.select():
+            data = key.fileobj.readline()
+            if not data:
+                continue
 
-        # Create threads for stdout and stderr
-        self.stdout_thread = threading.Thread(
-            target=log_output, args=(self.process.stdout, "INFO")
-        )
-        self.stderr_thread = threading.Thread(
-            target=log_output, args=(self.process.stderr, "ERROR")
-        )
-
-        # Start the threads
-        self.stdout_thread.start()
-        self.stderr_thread.start()
+            if key.fileobj is self.process.stdout:
+                logger.info(data.rstrip())
+            elif key.fileobj is self.process.stderr:
+                logger.error(data.rstrip())
+                self._errors += 1
 
     def wait(self):
-        if self.process:
-            # Wait for the process to complete
-            self.process.wait()
-            # Wait for the threads to finish reading output
-            self.stdout_thread.join()
-            self.stderr_thread.join()
+        while True:
+            self.probe_logs()
+            if not self.is_running():
+                break
+
+        return self.finish()
 
     def run(self):
         """Start and wait for the process."""
         self.start()
-        self.wait()
+        return self.wait()
 
     def is_running(self):
         """Check if the process is still running."""
@@ -73,10 +65,13 @@ class Process:
 
     def finish(self):
         """Make sure that logging finishes"""
-        self.stderr_thread.join()
-        self.stderr_thread.join()
+        if self.process:
+            self.sel.unregister(self.process.stdout)
+            self.sel.unregister(self.process.stderr)
+            self.process.stdout.close()
+            self.process.stderr.close()
 
-        return self.process.poll()
+            return self.process.poll()
 
     def terminate(self):
         """Terminate the process."""

{primitive-0.1.52 → primitive-0.1.54}/src/primitive/agent/runner.py

@@ -2,6 +2,7 @@ import yaml
 import sys
 import typing
 import os
+import threading
 from time import sleep
 from typing import TypedDict, Iterable, List, Optional, Dict
 from pathlib import Path, PurePath
@@ -111,10 +112,8 @@ class AgentRunner:
             logger.info(f"Beginning step {step['name']}")
 
             # Define step proc
-            logger.info(f"first cmd: {step['cmd']}")
-            logger.info(f"first workdir: {Path(self.source_dir / step['workdir'])}")
             proc = Process(
-                step["cmd"],
+                cmd=step["cmd"],
                 workdir=Path(self.source_dir / step["workdir"]),
                 env=environment,
             )
@@ -127,24 +126,31 @@ class AgentRunner:
                 conclusion = "failure"
                 break
 
-            # Check for updates to status while running
-            while proc.is_running():
-                status = self.primitive.jobs.get_job_status(self.job_id)
+            def status_check():
+                while proc.is_running():
+                    # Check job status
+                    status = self.primitive.jobs.get_job_status(self.job_id)
+                    status_value = status["jobRun"]["status"]
 
-                status_value = status["jobRun"]["status"]
+                    # TODO: Should probably use request_cancelled or something
+                    # once we change it, we'll have to call conclude w/ cancelled status
+                    if status_value == "completed":
+                        logger.warning("Job cancelled by user")
+                        proc.terminate()
+                        return
 
-                # TODO: Should probably use request_cancelled or something
-                # once we change it, we'll have to call conclude w/ cancelled status
-                if status_value == "completed":
-                    logger.warning("Job cancelled by user")
-                    proc.terminate()
-                    return
+                    sleep(5)
 
-                sleep(5)
+            status_thread = threading.Thread(target=status_check)
+            status_thread.start()
 
-            returncode = proc.finish()
+            # Wait for proc to finish
+            returncode = proc.wait()
             total_errors += proc.errors
 
+            # Wait for status check
+            status_thread.join()
+
             # Collect artifacts
             if "artifacts" in step:
                 self.collect_artifacts(step)

primitive-0.1.54/src/primitive/agent/uploader.py

@@ -0,0 +1,61 @@
+import typing
+import shutil
+from loguru import logger
+from pathlib import Path, PurePath
+from ..utils.cache import get_artifacts_cache
+
+if typing.TYPE_CHECKING:
+    import primitive.client
+
+
+class Uploader:
+    def __init__(
+        self,
+        primitive: "primitive.client.Primitive",
+    ):
+        self.primitive = primitive
+
+    def upload_file(self, path: Path, prefix: str) -> str:
+        file_upload_response = self.primitive.files.file_upload(path, key_prefix=prefix)
+        return file_upload_response.json()["data"]["fileUpload"]["id"]
+
+    def scan(self) -> None:
+        # Scan artifacts directory
+        artifacts_dir = get_artifacts_cache()
+
+        subdirs = sorted(
+            [job_cache for job_cache in artifacts_dir.iterdir() if job_cache.is_dir()],
+            key=lambda p: p.stat().st_ctime,
+        )
+
+        for job_cache in subdirs:
+            job_run_id = job_cache.name
+
+            files = sorted(
+                [
+                    w_path / file
+                    for w_path, _, w_files in job_cache.walk()
+                    for file in w_files
+                ],
+                key=lambda p: p.stat().st_size,
+            )
+
+            file_ids = []
+            for file in files:
+                upload_id = self.upload_file(
+                    file,
+                    prefix=str(PurePath(file).relative_to(job_cache.parent).parent),
+                )
+
+                if upload_id:
+                    file_ids.append(upload_id)
+                    continue
+
+                logger.error(f"Unable to upload file {file}")
+
+            # Update job run
+            if len(file_ids) > 0:
+                self.primitive.jobs.job_run_update(id=job_run_id, file_ids=file_ids)
+
+            # Clean up job cache
+            shutil.rmtree(path=job_cache)

primitive-0.1.52/src/primitive/agent/uploader.py

@@ -1,46 +0,0 @@
-import typing
-import shutil
-from pathlib import Path, PurePath
-from ..utils.cache import get_artifacts_cache
-
-if typing.TYPE_CHECKING:
-    import primitive.client
-
-
-class Uploader:
-    def __init__(
-        self,
-        primitive: "primitive.client.Primitive",
-    ):
-        self.primitive = primitive
-
-    def upload_file(self, path: Path, prefix: str) -> str:
-        file_upload_response = self.primitive.files.file_upload(path, key_prefix=prefix)
-        return file_upload_response.json()["data"]["fileUpload"]["id"]
-
-    def scan(self) -> None:
-        # Scan artifacts directory
-        artifacts_dir = get_artifacts_cache()
-
-        subdirs = [
-            job_cache for job_cache in artifacts_dir.iterdir() if job_cache.is_dir()
-        ]
-
-        for job_cache in subdirs:
-            job_run_id = job_cache.name
-            files = [file for file in job_cache.rglob("*") if file.is_file()]
-
-            file_ids = []
-            for file in files:
-                file_ids.append(
-                    self.upload_file(
-                        file,
-                        prefix=str(PurePath(file).relative_to(job_cache.parent).parent),
-                    )
-                )
-
-            # Update job run
-            self.primitive.jobs.job_run_update(id=job_run_id, file_ids=file_ids)
-
-            # Clean up job cache
-            shutil.rmtree(path=job_cache)