primitive 0.1.59__py3-none-any.whl → 0.1.60__py3-none-any.whl

primitive/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Dylan Stein <dylan@primitive.tech>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.1.59"
+__version__ = "0.1.60"

primitive/agent/actions.py

@@ -129,6 +129,7 @@ class Agent(BaseAction):
                             source_dir=source_dir,
                             job_id=job_run["id"],
                             job_slug=job_run["job"]["slug"],
+                            max_log_size=500 * 1024,
                         )
                     except Exception as e:
                         # Log Error

primitive/agent/process.py

@@ -34,7 +34,7 @@ class Process:
         self.sel.register(self.process.stdout, selectors.EVENT_READ)
         self.sel.register(self.process.stderr, selectors.EVENT_READ)
 
-    def probe_logs(self):
+    def log(self):
         for key, _ in self.sel.select():
             data = key.fileobj.readline()
             if not data:
@@ -48,7 +48,7 @@ class Process:
 
     def wait(self):
         while True:
-            self.probe_logs()
+            self.log()
             if not self.is_running():
                 break
 

primitive/agent/provision.py

@@ -2,6 +2,7 @@ import sys
 from subprocess import Popen, PIPE
 from pathlib import Path
 from typing import Dict
+from ..utils.shell import env_string_to_dict
 
 
 class ProvisionPython:
@@ -34,14 +35,7 @@ class ProvisionPython:
         output, _ = proc.communicate()
 
         # Split the output into lines and parse it into a dictionary
-        env_vars = {}
-
-        for line in output.splitlines():
-            var_line = line.split("=", 1)
-
-            if len(var_line) == 2:
-                key, value = var_line
-                env_vars[key] = value
+        env_vars = env_string_to_dict(output)
 
         cmd = f"python -m pip install -r {self.requirements_path}"
         proc = Popen(

primitive/agent/runner.py

@@ -7,11 +7,10 @@ from typing import Dict, Iterable, List, Optional, TypedDict
 
 import yaml
 from loguru import logger
-
-from ..utils.cache import get_artifacts_cache
-from ..utils.files import find_files_for_extension
 from .process import Process
 from .provision import ProvisionPython
+from ..utils.cache import get_artifacts_cache, get_logs_cache
+from ..utils.files import find_files_for_extension
 
 try:
     from yaml import CLoader as Loader
@@ -56,12 +55,11 @@ class AgentRunner:
         self.job_slug = job_slug
         self.max_log_size = max_log_size
         self.artifacts_dir = get_artifacts_cache(self.job_id)
+        self.logs_dir = get_logs_cache(self.job_id)
+        self.logger_handle = None
 
         logger.enable("primitive")
-        self.logger_handle = logger.add(
-            Path(self.artifacts_dir / "runner_{time}.log"),
-            rotation=self.max_log_size,  # Rotate when the log file reaches 10MB
-        )
+        self.swap_logs(label="init")
 
         logger.info(f"Scanning directory for job {self.job_slug}")
 
@@ -113,6 +111,9 @@ class AgentRunner:
         conclusion = None
         total_errors = 0
         for step in self.steps():
+            # Swap logger
+            self.swap_logs(label=step["name"])
+
             logger.info(f"Beginning step {step['name']}")
 
             # Update workdir
@@ -138,7 +139,7 @@ class AgentRunner:
                 while proc.is_running():
                     # Check job status
                     status = self.primitive.jobs.get_job_status(self.job_id)
-                    status_value = status["jobRun"]["status"]
+                    status_value = status.data["jobRun"]["status"]
 
                     # TODO: Should probably use request_cancelled or something
                     # once we change it, we'll have to call conclude w/ cancelled status
@@ -184,6 +185,15 @@ class AgentRunner:
         logger.info(f"Completed {self.job_slug} job")
         logger.remove(self.logger_handle)
 
+    def swap_logs(self, label: str):
+        # Remove Handle
+        if self.logger_handle:
+            logger.remove(self.logger_handle)
+
+        self.logger_handle = logger.add(
+            Path(self.logs_dir / f"{label}_{{time}}.primitive.log"), rotation=self.max_log_size
+        )
+
     def provision(self) -> Optional[Dict]:
         match self.job["provision"]:
             case "python":
@@ -199,8 +209,6 @@ class AgentRunner:
                 return prov.create_env()
 
     def collect_artifacts(self, step: JobStep) -> None:
-        # str(PurePath(file_path).relative_to(Path(source))
-
         # Search each artifact type
         for artifact in step["artifacts"]:
             files = find_files_for_extension(self.source_dir, artifact["extension"])

primitive/agent/uploader.py

@@ -1,9 +1,10 @@
 import typing
+from typing import Dict
 import shutil
 import os
 from loguru import logger
 from pathlib import Path, PurePath
-from ..utils.cache import get_artifacts_cache
+from ..utils.cache import get_artifacts_cache, get_logs_cache
 
 if typing.TYPE_CHECKING:
     import primitive.client
@@ -21,57 +22,85 @@ class Uploader:
             path, key_prefix=prefix, job_run_id=job_run_id
         )
         return file_upload_response.json()["data"]["fileUpload"]["id"]
+    
+    def upload_dir(self, cache: Path) -> Dict:
+        file_ids = []
+        job_run_id = cache.name
+
+        files = None
+        has_walk = getattr(cache, "walk", None)
+        if has_walk:
+            files = sorted(
+                [
+                    current_path / file
+                    for current_path, _, current_path_files in cache.walk()
+                    for file in current_path_files
+                ],
+                key=lambda p: p.stat().st_size,
+            )
+        else:
+            files = sorted(
+                [
+                    Path(Path(current_path) / file)
+                    for current_path, _, current_path_files in os.walk(cache)
+                    for file in current_path_files
+                ],
+                key=lambda p: p.stat().st_size,
+            )
+
+        for file in files:
+            upload_id = self.upload_file(
+                file,
+                prefix=str(PurePath(file).relative_to(cache.parent).parent),
+                job_run_id=job_run_id,
+            )
+
+            if upload_id:
+                file_ids.append(upload_id)
+                continue
+
+            logger.error(f"Unable to upload file {file}")
+
+        # Clean up job cache
+        shutil.rmtree(path=cache)
+
+        return {job_run_id: file_ids}
+
 
     def scan(self) -> None:
         # Scan artifacts directory
         artifacts_dir = get_artifacts_cache()
+        logs_dir = get_logs_cache()
 
-        subdirs = sorted(
-            [job_cache for job_cache in artifacts_dir.iterdir() if job_cache.is_dir()],
-            key=lambda p: p.stat().st_ctime,
-        )
+        artifacts = sorted([
+            artifacts_cache
+            for artifacts_cache in artifacts_dir.iterdir()
+            if artifacts_cache.is_dir()
+        ], key=lambda p: p.stat().st_ctime)
+
+        logs = sorted([
+            logs_cache 
+            for logs_cache in logs_dir.iterdir() 
+            if logs_cache.is_dir()
+        ], key=lambda p: p.stat().st_ctime)
+
+        log_files = {
+            job_id: files
+            for log_path in logs
+            for job_id, files in self.upload_dir(log_path).items()
+        }
+
+        artifact_files = {
+            job_id: files
+            for artifact_path in artifacts
+            for job_id, files in self.upload_dir(artifact_path).items()
+        }
+
+        files_by_id = {
+            job_id: log_files.get(job_id, []) + artifact_files.get(job_id, []) 
+            for job_id in log_files.keys() | artifact_files.keys()
+        }
 
-        for job_cache in subdirs:
-            job_run_id = job_cache.name
-
-            files = None
-            has_walk = getattr(job_cache, "walk", None)
-            if has_walk:
-                files = sorted(
-                    [
-                        w_path / file
-                        for w_path, _, w_files in job_cache.walk()
-                        for file in w_files
-                    ],
-                    key=lambda p: p.stat().st_size,
-                )
-            else:
-                files = sorted(
-                    [
-                        Path(Path(w_path) / file)
-                        for w_path, _, w_files in os.walk(job_cache)
-                        for file in w_files
-                    ],
-                    key=lambda p: p.stat().st_size,
-                )
-
-            file_ids = []
-            for file in files:
-                upload_id = self.upload_file(
-                    file,
-                    prefix=str(PurePath(file).relative_to(job_cache.parent).parent),
-                    job_run_id=job_run_id,
-                )
-
-                if upload_id:
-                    file_ids.append(upload_id)
-                    continue
-
-                logger.error(f"Unable to upload file {file}")
-
-            # Update job run
-            if len(file_ids) > 0:
-                self.primitive.jobs.job_run_update(id=job_run_id, file_ids=file_ids)
-
-            # Clean up job cache
-            shutil.rmtree(path=job_cache)
+        # Update job run
+        for job_id, files in files_by_id.items():
+            self.primitive.jobs.job_run_update(id=job_id, file_ids=files)

primitive/utils/cache.py

@@ -44,6 +44,20 @@ def get_artifacts_cache(cache_id: str = None) -> Path:
     return artifacts_dir
 
 
+def get_logs_cache(cache_id: str = None) -> Path:
+    cache_dir = get_cache_dir()
+
+    logs_dir = cache_dir / "logs"
+
+    if cache_id:
+        logs_dir = logs_dir / cache_id
+
+    if not logs_dir.exists():
+        logs_dir.mkdir(parents=True, exist_ok=True)
+
+    return logs_dir
+
+
 def get_deps_cache() -> Path:
     cache_dir = get_cache_dir()
 

primitive/utils/shell.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 import subprocess
+from typing import Dict
 
 
 def add_path_to_shell(path: Path):
@@ -31,3 +32,27 @@ def add_path_to_shell(path: Path):
         file.write(f"export PATH={path}:$PATH\n")
 
     return True
+
+
+def env_string_to_dict(env_str: str) -> Dict:
+    lines = env_str.splitlines()
+
+    current_key = None
+    current_value = []
+    env_dict = {}
+    for line in lines:
+        if "=" in line:
+            if current_key is not None:
+                env_dict[current_key] = "\n".join(current_value)
+
+            key, value = line.split("=", 1)
+
+            current_key = key
+            current_value = [value]
+        else:
+            current_value.append(line)
+
+    if current_key is not None:
+        env_dict[current_key] = "\n".join(current_value)
+
+    return env_dict

{primitive-0.1.59.dist-info → primitive-0.1.60.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: primitive
-Version: 0.1.59
+Version: 0.1.60
 Project-URL: Documentation, https://github.com//primitivecorp/primitive-cli#readme
 Project-URL: Issues, https://github.com//primitivecorp/primitive-cli/issues
 Project-URL: Source, https://github.com//primitivecorp/primitive-cli

{primitive-0.1.59.dist-info → primitive-0.1.60.dist-info}/RECORD

@@ -1,14 +1,14 @@
-primitive/__about__.py,sha256=C9j2a-6ERCVy_C_OHbf9s91MGCcAIm1Dlc-h-vRuSew,130
+primitive/__about__.py,sha256=ltk-zb4IMmLsM4NDisEz2bXWrL5uUzzqsV5yJM95Caw,130
 primitive/__init__.py,sha256=bwKdgggKNVssJFVPfKSxqFMz4IxSr54WWbmiZqTMPNI,106
 primitive/cli.py,sha256=CGmWiqqCLMHtHGOUPuf3tVO6VvChBZ1VdSwCCglnBgA,2582
 primitive/client.py,sha256=p-5z1iGM8ZydIrkYf4R6b7Yna73oszlGdXim9-Zsbyk,2364
 primitive/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/agent/actions.py,sha256=HuKR0bi3VVO8bR5IgcZ1paZw0lGK__8FTgfbWTm68go,5830
+primitive/agent/actions.py,sha256=B7d2oNMjtjUP2RhD-QnNDWNl3jHwjUDk5KLWQ2OnNQ4,5883
 primitive/agent/commands.py,sha256=-dVDilELfkGfbZB7qfEPs77Dm1oT62qJj4tsIk4KoxI,254
-primitive/agent/process.py,sha256=2ZY3YoJHvoukrsCAZIt-AF2YKY4HEO5_jWji5K3W9fM,2267
-primitive/agent/provision.py,sha256=3EEzOV-ria6zf-pvfNddad1lzzd1QmfKInTIjmwX71Y,1673
-primitive/agent/runner.py,sha256=La1XSjH1vVj75DNI2rLR8GSRJvk3BosG_h_x1evGLCE,6878
-primitive/agent/uploader.py,sha256=5ZxonvRlMGRVBYx3hPEaWiPio5lMPi_zhtxz64V5S-A,2379
+primitive/agent/process.py,sha256=LVI-RB4a0YEuXUTYMXKL5Xi9euNwUI2nxj00mv8EFOg,2253
+primitive/agent/provision.py,sha256=rmwnro1K5F8mwtd45XAq7RVQmpDWnbBCQ8X_qgWhm3M,1546
+primitive/agent/runner.py,sha256=pX1KgYNxSxkSIDydsQa60C4zV2EvNaVXA9rQlG17wc8,7129
+primitive/agent/uploader.py,sha256=La33T2csENWKRsNagog3ELYe6110jufMnb7owmrPsIk,3150
 primitive/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/auth/actions.py,sha256=MPsG9LcKcOPwA7gZ9Ewk0PZJhTQvIrGfODdz4GxSzgA,999
 primitive/auth/commands.py,sha256=JahUq0E2e7Xa-FX1WEUv7TgM6ieDvNH4VwRRtxAW7HE,2340
@@ -80,16 +80,16 @@ primitive/sim/commands.py,sha256=8PaOfL1MO6qxTn7mNVRnBU1X2wa3gk_mlbAhBW6MnI0,591
 primitive/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/utils/actions.py,sha256=HOFrmM3-0A_A3NS84MqrZ6JmQEiiPSoDqEeuu6b_qfQ,196
 primitive/utils/auth.py,sha256=TtJKTR6tLmNrtWbOjJI-KJh4ZSJ1uG7ApE9GcY63m00,836
-primitive/utils/cache.py,sha256=hDVpEL2TePrWOH6q7Me_Oi-DH_viFrRxrVta-z4wBhM,1295
+primitive/utils/cache.py,sha256=FHGmVWYLJFQOazpXXcEwI0YJEZbdkgG39nOLdOv6VNk,1575
 primitive/utils/config.py,sha256=DlFM5Nglo22WPtbpZSVtH7NX-PTMaKYlcrUE7GPRG4c,1058
 primitive/utils/files.py,sha256=Yv__bQes3YIlzhOT9kVxtYhoA5CmUjPSvphl9PZ41k4,867
 primitive/utils/git.py,sha256=1qNOu8X-33CavmrD580BmrFhD_WVO9PGWHUUboXJR_g,663
 primitive/utils/memory_size.py,sha256=4xfha21kW82nFvOTtDFx9Jk2ZQoEhkfXii-PGNTpIUk,3058
 primitive/utils/printer.py,sha256=f1XUpqi5dkTL3GWvYRUGlSwtj2IxU1q745T4Fxo7Tn4,370
-primitive/utils/shell.py,sha256=-7UjQaBqSGHzEEyX8pNjeYFFP0P3lVnDV0OkgPz1qHU,1050
+primitive/utils/shell.py,sha256=j7E1YwgNWw57dFHVfEbqRNVcPHX0xDefX2vFSNgeI_8,1648
 primitive/utils/verible.py,sha256=r7c_hfqvL0UicMmIzK3Cy_BfZI1ZpcfBeLqKEWFWqJo,2252
-primitive-0.1.59.dist-info/METADATA,sha256=pkPSFAV49Rz8KgwQyXJX9sjkuSlyc_mzsC_owgi7cto,3782
-primitive-0.1.59.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-primitive-0.1.59.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
-primitive-0.1.59.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
-primitive-0.1.59.dist-info/RECORD,,
+primitive-0.1.60.dist-info/METADATA,sha256=Ehe8DxemJilg_9uewSTTCQE9vobsu-P4EmSIKexXNLo,3782
+primitive-0.1.60.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+primitive-0.1.60.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
+primitive-0.1.60.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
+primitive-0.1.60.dist-info/RECORD,,