primitive 0.1.53__py3-none-any.whl → 0.1.55__py3-none-any.whl

primitive/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Dylan Stein <dylan@primitive.tech>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.1.53"
+__version__ = "0.1.55"

primitive/agent/actions.py

@@ -120,7 +120,7 @@ class Agent(BaseAction):
                     )
 
                     runner = AgentRunner(
-                        self.primitive,
+                        primitive=self.primitive,
                         source_dir=source_dir,
                         job_id=job_run["id"],
                         job_slug=job_run["job"]["slug"],

primitive/agent/process.py

@@ -1,6 +1,6 @@
 from subprocess import Popen, PIPE
-import threading
 import shlex
+import selectors
 from loguru import logger
 
 
@@ -21,8 +21,7 @@ class Process:
 
     def start(self):
         # Start the process
-        logger.info(f"cmd: {self.cmd}")
-        logger.info(f"workdir: {self.workdir}")
+        self.sel = selectors.DefaultSelector()
         self.process = Popen(
             self.cmd,
             env=self.env,
@@ -32,40 +31,33 @@ class Process:
             text=True,
         )
 
-        # Function to read and log output from a pipe
-        def log_output(pipe, level):
-            for line in iter(pipe.readline, ""):
-                if line:
-                    logger.log(level, line.rstrip())
-                    if level == "ERROR":
-                        self._errors += 1
+        self.sel.register(self.process.stdout, selectors.EVENT_READ)
+        self.sel.register(self.process.stderr, selectors.EVENT_READ)
 
-            pipe.close()
+    def probe_logs(self):
+        for key, _ in self.sel.select():
+            data = key.fileobj.readline()
+            if not data:
+                continue
 
-        # Create threads for stdout and stderr
-        self.stdout_thread = threading.Thread(
-            target=log_output, args=(self.process.stdout, "INFO")
-        )
-        self.stderr_thread = threading.Thread(
-            target=log_output, args=(self.process.stderr, "ERROR")
-        )
-
-        # Start the threads
-        self.stdout_thread.start()
-        self.stderr_thread.start()
+            if key.fileobj is self.process.stdout:
+                logger.info(data.rstrip())
+            elif key.fileobj is self.process.stderr:
+                logger.error(data.rstrip())
+                self._errors += 1
 
     def wait(self):
-        if self.process:
-            # Wait for the process to complete
-            self.process.wait()
-            # Wait for the threads to finish reading output
-            self.stdout_thread.join()
-            self.stderr_thread.join()
+        while True:
+            self.probe_logs()
+            if not self.is_running():
+                break
+
+        return self.finish()
 
     def run(self):
         """Start and wait for the process."""
         self.start()
-        self.wait()
+        return self.wait()
 
     def is_running(self):
         """Check if the process is still running."""
@@ -73,10 +65,13 @@ class Process:
 
     def finish(self):
         """Make sure that logging finishes"""
-        self.stderr_thread.join()
-        self.stderr_thread.join()
+        if self.process:
+            self.sel.unregister(self.process.stdout)
+            self.sel.unregister(self.process.stderr)
+            self.process.stdout.close()
+            self.process.stderr.close()
 
-        return self.process.poll()
+            return self.process.poll()
 
     def terminate(self):
         """Terminate the process."""

primitive/agent/runner.py

@@ -2,6 +2,7 @@ import yaml
 import sys
 import typing
 import os
+import threading
 from time import sleep
 from typing import TypedDict, Iterable, List, Optional, Dict
 from pathlib import Path, PurePath
@@ -111,10 +112,8 @@ class AgentRunner:
             logger.info(f"Beginning step {step['name']}")
 
             # Define step proc
-            logger.info(f"first cmd: {step['cmd']}")
-            logger.info(f"first workdir: {Path(self.source_dir / step['workdir'])}")
             proc = Process(
-                step["cmd"],
+                cmd=step["cmd"],
                 workdir=Path(self.source_dir / step["workdir"]),
                 env=environment,
             )
@@ -127,24 +126,31 @@ class AgentRunner:
                 conclusion = "failure"
                 break
 
-            # Check for updates to status while running
-            while proc.is_running():
-                status = self.primitive.jobs.get_job_status(self.job_id)
+            def status_check():
+                while proc.is_running():
+                    # Check job status
+                    status = self.primitive.jobs.get_job_status(self.job_id)
+                    status_value = status["jobRun"]["status"]
 
-                status_value = status["jobRun"]["status"]
+                    # TODO: Should probably use request_cancelled or something
+                    # once we change it, we'll have to call conclude w/ cancelled status
+                    if status_value == "completed":
+                        logger.warning("Job cancelled by user")
+                        proc.terminate()
+                        return
 
-                # TODO: Should probably use request_cancelled or something
-                # once we change it, we'll have to call conclude w/ cancelled status
-                if status_value == "completed":
-                    logger.warning("Job cancelled by user")
-                    proc.terminate()
-                    return
+                    sleep(5)
 
-                sleep(5)
+            status_thread = threading.Thread(target=status_check)
+            status_thread.start()
 
-            returncode = proc.finish()
+            # Wait for proc to finish
+            returncode = proc.wait()
             total_errors += proc.errors
 
+            # Wait for status check
+            status_thread.join()
+
             # Collect artifacts
             if "artifacts" in step:
                 self.collect_artifacts(step)

primitive/agent/uploader.py

@@ -1,5 +1,6 @@
 import typing
 import shutil
+from loguru import logger
 from pathlib import Path, PurePath
 from ..utils.cache import get_artifacts_cache
 
@@ -14,33 +15,50 @@ class Uploader:
     ):
         self.primitive = primitive
 
-    def upload_file(self, path: Path, prefix: str) -> str:
-        file_upload_response = self.primitive.files.file_upload(path, key_prefix=prefix)
+    def upload_file(self, path: Path, prefix: str, job_run_id: str) -> str:
+        file_upload_response = self.primitive.files.file_upload(
+            path, key_prefix=prefix, job_run_id=job_run_id
+        )
         return file_upload_response.json()["data"]["fileUpload"]["id"]
 
     def scan(self) -> None:
         # Scan artifacts directory
         artifacts_dir = get_artifacts_cache()
 
-        subdirs = [
-            job_cache for job_cache in artifacts_dir.iterdir() if job_cache.is_dir()
-        ]
+        subdirs = sorted(
+            [job_cache for job_cache in artifacts_dir.iterdir() if job_cache.is_dir()],
+            key=lambda p: p.stat().st_ctime,
+        )
 
         for job_cache in subdirs:
             job_run_id = job_cache.name
-            files = [file for file in job_cache.rglob("*") if file.is_file()]
+
+            files = sorted(
+                [
+                    w_path / file
+                    for w_path, _, w_files in job_cache.walk()
+                    for file in w_files
+                ],
+                key=lambda p: p.stat().st_size,
+            )
 
             file_ids = []
             for file in files:
-                file_ids.append(
-                    self.upload_file(
-                        file,
-                        prefix=str(PurePath(file).relative_to(job_cache.parent).parent),
-                    )
+                upload_id = self.upload_file(
+                    file,
+                    prefix=str(PurePath(file).relative_to(job_cache.parent).parent),
+                    job_run_id=job_run_id,
                 )
 
+                if upload_id:
+                    file_ids.append(upload_id)
+                    continue
+
+                logger.error(f"Unable to upload file {file}")
+
             # Update job run
-            self.primitive.jobs.job_run_update(id=job_run_id, file_ids=file_ids)
+            if len(file_ids) > 0:
+                self.primitive.jobs.job_run_update(id=job_run_id, file_ids=file_ids)
 
             # Clean up job cache
             shutil.rmtree(path=job_cache)

primitive/files/actions.py

@@ -43,7 +43,13 @@ class Files(BaseAction):
         return result
 
     @guard
-    def file_upload(self, path: Path, is_public: bool = False, key_prefix: str = ""):
+    def file_upload(
+        self,
+        path: Path,
+        is_public: bool = False,
+        key_prefix: str = "",
+        job_run_id: str = "",
+    ):
         file_path = str(path.resolve())
         if path.exists() is False:
             raise FileNotFoundError(f"File not found at {file_path}")
@@ -54,6 +60,8 @@ class Files(BaseAction):
                 + file_path
                 + """\", "keyPrefix": \""""
                 + key_prefix
+                + """\", "jobRunId": \""""
+                + job_run_id
                 + """\" } } }"""
             )  # noqa
 
@@ -63,6 +71,8 @@ class Files(BaseAction):
                 + file_path
                 + """\", "keyPrefix": \""""
                 + key_prefix
+                + """\", "jobRunId": \""""
+                + job_run_id
                 + """\" } } }"""
             )  # noqa
         body = {

{primitive-0.1.53.dist-info → primitive-0.1.55.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: primitive
-Version: 0.1.53
+Version: 0.1.55
 Project-URL: Documentation, https://github.com//primitivecorp/primitive-cli#readme
 Project-URL: Issues, https://github.com//primitivecorp/primitive-cli/issues
 Project-URL: Source, https://github.com//primitivecorp/primitive-cli

{primitive-0.1.53.dist-info → primitive-0.1.55.dist-info}/RECORD

@@ -1,13 +1,13 @@
-primitive/__about__.py,sha256=eX7mKUzZD5kknoBHuD860zr_YILfDsBdXo5NYHeClXI,130
+primitive/__about__.py,sha256=NkPE6ibF3SMDPKeHuwyX-BngPvgNwRJhl1PjxPpV5XQ,130
 primitive/__init__.py,sha256=bwKdgggKNVssJFVPfKSxqFMz4IxSr54WWbmiZqTMPNI,106
 primitive/cli.py,sha256=VQPSewC6ouGdEG9W1gllawGJTydpOY0Lzg7LURXcqQg,2374
 primitive/client.py,sha256=vSJkifx450czuLvu0f2o-viSCC0p2f1UicA-2P5cJAw,2188
-primitive/agent/actions.py,sha256=d_garPm3rl5uMEu-g681yMYEnBqfUDeEPeeKl_VEd64,5489
+primitive/agent/actions.py,sha256=m1FGvtOHdrZRP-YC31bFoUj-ZunXlFFW4NnPH9Bpzc0,5499
 primitive/agent/commands.py,sha256=-dVDilELfkGfbZB7qfEPs77Dm1oT62qJj4tsIk4KoxI,254
-primitive/agent/process.py,sha256=rbRB8MOxhTtMax-182Cwy_KP2I1fPyXLiu9p_GIE15Y,2451
+primitive/agent/process.py,sha256=2ZY3YoJHvoukrsCAZIt-AF2YKY4HEO5_jWji5K3W9fM,2267
 primitive/agent/provision.py,sha256=3EEzOV-ria6zf-pvfNddad1lzzd1QmfKInTIjmwX71Y,1673
-primitive/agent/runner.py,sha256=tYjzQyfd1rEAmkBQ5C8ghryR2weH9J0hTXSPS5ijapM,6490
-primitive/agent/uploader.py,sha256=ngbynmQzxKVNMp7VWkTWW7vZIN_rnnRzYC6OAL21n1k,1378
+primitive/agent/runner.py,sha256=hDh6ip_Jb0EsvygrB7-GWVTOf-jN8E7H0RgCGOdQnPU,6634
+primitive/agent/uploader.py,sha256=Yiaszf44euL-spYe4lluDtgiPRi_0nq5l3eQtrWQ8bM,1903
 primitive/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/auth/actions.py,sha256=2vZEC3LLAXj6eBBmt_2OEDEcBIb3uLkkNjgbZTIaQsY,1095
 primitive/auth/commands.py,sha256=JahUq0E2e7Xa-FX1WEUv7TgM6ieDvNH4VwRRtxAW7HE,2340
@@ -15,7 +15,7 @@ primitive/daemons/actions.py,sha256=Nt3yNtbBhen0jK4sRsH_N7AP3UBuyL48VaUhtC7wYq8,
 primitive/daemons/commands.py,sha256=-Muh-6ib4uAVtPn_67AcMrDwuCwYlCnRQozCi2Xurmk,1726
 primitive/daemons/launch_agents.py,sha256=qovt32gwpjGDd82z_SY5EGCUjaUyNA49pZFajZsw3eE,4796
 primitive/daemons/launch_service.py,sha256=FPB9qKEjhllRfEpct0ng2L9lpIaGJbQwn1JdFT8uBA8,5600
-primitive/files/actions.py,sha256=2X1_rq8A7X1Nf0q9xyAI9ziMkow_VKARPfdnsAoZh9w,2676
+primitive/files/actions.py,sha256=3AOGLEUeplNlkKkyi1QajzxMc93Adh5jRQ8SQHFlJeU,2889
 primitive/files/commands.py,sha256=DDizo3xJnU3KLUBTMeeM72viVpnJinLwxs84tmqKhqo,810
 primitive/git/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/git/actions.py,sha256=OTuGRXfjop9u1LtNaBen87mEU6ROeoa_MDgBbB2l6ig,1428
@@ -46,8 +46,8 @@ primitive/utils/memory_size.py,sha256=4xfha21kW82nFvOTtDFx9Jk2ZQoEhkfXii-PGNTpIU
 primitive/utils/printer.py,sha256=f1XUpqi5dkTL3GWvYRUGlSwtj2IxU1q745T4Fxo7Tn4,370
 primitive/utils/shell.py,sha256=-7UjQaBqSGHzEEyX8pNjeYFFP0P3lVnDV0OkgPz1qHU,1050
 primitive/utils/verible.py,sha256=r7c_hfqvL0UicMmIzK3Cy_BfZI1ZpcfBeLqKEWFWqJo,2252
-primitive-0.1.53.dist-info/METADATA,sha256=4-y4DVEtRwaFuYTycrfi5QPHGQuzdJBzGrIzkRIskoo,3782
-primitive-0.1.53.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-primitive-0.1.53.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
-primitive-0.1.53.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
-primitive-0.1.53.dist-info/RECORD,,
+primitive-0.1.55.dist-info/METADATA,sha256=f8IGnZPje8mcvN8YQ-JhQbfKxaeHAwF5FdU7dsSuxtc,3782
+primitive-0.1.55.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+primitive-0.1.55.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
+primitive-0.1.55.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
+primitive-0.1.55.dist-info/RECORD,,