primitive 0.2.31__py3-none-any.whl → 0.2.33__py3-none-any.whl

primitive/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Dylan Stein <dylan@primitive.tech>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.2.31"
+__version__ = "0.2.33"

primitive/agent/actions.py

@@ -100,10 +100,6 @@ class Agent(BaseAction):
                     # This should probably eventually be another daemon?
                     uploader.scan()
 
-                    JobRun.objects.filter_by(
-                        job_run_id=api_job_run.get("id"),
-                    ).delete()
-
                 sleep(5)
         except KeyboardInterrupt:
             logger.info("[agent] Stopping primitive agent...")

primitive/agent/runner.py

@@ -25,7 +25,11 @@ if typing.TYPE_CHECKING:
 
 CHUNK_SIZE = 64 * 1024
 ENV_VAR_LOOKUP_START = "_ENV_VAR_LOOKUP_START"
+START_DELIMITER_SIZE = len(bytes(ENV_VAR_LOOKUP_START, encoding="utf-8"))
 ENV_VAR_LOOKUP_END = "_ENV_VAR_LOOKUP_END"
+END_DELIMITER_SIZE = len(bytes(ENV_VAR_LOOKUP_END, encoding="utf-8"))
+
+assert CHUNK_SIZE > START_DELIMITER_SIZE + END_DELIMITER_SIZE
 
 
 class Task(TypedDict):
@@ -132,6 +136,10 @@ class Runner:
 
         # Setup initial process environment
         self.initial_env = os.environ
+        self.initial_env = {
+            **self.initial_env,
+            **self.primitive.jobs.get_job_secrets_for_job_run(self.job_run["id"]),
+        }
         self.initial_env["PRIMITIVE_GIT_SHA"] = str(self.job_run["gitCommit"]["sha"])
         self.initial_env["PRIMITIVE_GIT_BRANCH"] = str(
             self.job_run["gitCommit"]["branch"]
@@ -253,12 +261,7 @@ class Runner:
             args = [
                 "/bin/bash",
                 "-c",
-                (
-                    f"{cmd} "
-                    f"&& stdbuf -oL echo && stdbuf -oL echo '{ENV_VAR_LOOKUP_START}' "
-                    "&& env "
-                    f"&& stdbuf -oL echo && echo '{ENV_VAR_LOOKUP_END}'"
-                ),
+                f"{cmd} && echo -n '{ENV_VAR_LOOKUP_START}' && env && echo -n '{ENV_VAR_LOOKUP_END}'",
             ]
 
             process = await asyncio.create_subprocess_exec(
@@ -302,28 +305,32 @@ class Runner:
     async def log_cmd(self, process, stream, tags: Dict = {}) -> bool:
         failure_detected = False
         parse_environment = False
-        env_lines = []
+        last_chunk_buffer = b""
+        environment_buffer = b""
         while chunk := await stream.read(CHUNK_SIZE):
-            processed_lines = await self.read_chunk(chunk)
-            single_line = len(processed_lines) == 1
-            empty_chunk = len(processed_lines) == 0
-
-            if empty_chunk:
-                # Ignore the empty lines
+            if parse_environment:
+                environment_buffer += chunk
                 continue
-            elif single_line and ENV_VAR_LOOKUP_START in processed_lines[0]:
-                # Next chunk will contain the environent variables
+
+            # First, look for start delimiter in chunk
+            full_chunk = last_chunk_buffer + chunk
+            last_chunk_buffer = b""
+            start_index = full_chunk.find(bytes(ENV_VAR_LOOKUP_START, encoding="utf-8"))
+
+            if start_index != -1:
+                environment_buffer = full_chunk[start_index + START_DELIMITER_SIZE :]
+                processed_lines = await self.read_chunk(full_chunk[:start_index])
                 parse_environment = True
-                continue
-            elif single_line and ENV_VAR_LOOKUP_END in processed_lines[0]:
-                # Done reading environment variables
-                parse_environment = False
-                self.modified_env = env_to_dict(env_lines)
-                continue
+            else:
+                processed_lines = await self.read_chunk(full_chunk)
 
-            if parse_environment:
-                env_lines.extend(processed_lines)
-                continue
+                while (
+                    len(last_chunk_buffer) < START_DELIMITER_SIZE
+                    and len(processed_lines) > 0
+                ):
+                    last_chunk_buffer += bytes(
+                        processed_lines.pop() + "\n", encoding="utf-8"
+                    )
 
             # Handle logging
             parse_logs = self.job_settings["parseLogs"]
@@ -339,13 +346,15 @@ class Runner:
                 elif parse_logs and "warning" in line.lower():
                     level = LogLevel.WARNING
 
-                failure_detected = (
-                    level == LogLevel.ERROR
-                    and self.job_settings["failureLevel"] >= FailureLevel.ERROR
-                ) or (
-                    level == LogLevel.WARNING
-                    and self.job_settings["failureLevel"] >= FailureLevel.WARNING
-                )
+                # If we already detected a failure, skip checking
+                if not failure_detected:
+                    failure_detected = (
+                        level == LogLevel.ERROR
+                        and self.job_settings["failureLevel"] >= FailureLevel.ERROR
+                    ) or (
+                        level == LogLevel.WARNING
+                        and self.job_settings["failureLevel"] >= FailureLevel.WARNING
+                    )
 
                 # Tag on the first matching regex in the list
                 for tag_key, regex in tags.items():
@@ -356,6 +365,16 @@ class Runner:
 
                 logger.bind(tag=tag).log(level.value, line)
 
+        start_index = environment_buffer.find(
+            bytes(ENV_VAR_LOOKUP_END, encoding="utf-8")
+        )
+        if parse_environment and start_index == -1:
+            logger.error("Environment variable buffer did not contain end delimiter")
+            failure_detected = True
+            return failure_detected
+
+        environment_buffer = environment_buffer[:start_index]
+        self.modified_env = env_to_dict(environment_buffer)
         return failure_detected
 
     async def read_chunk(self, chunk: bytes) -> List[str]:

primitive/jobs/actions.py

@@ -11,6 +11,7 @@ from .graphql.queries import (
     job_run_query,
     job_run_status_query,
     job_runs_query,
+    job_secrets_for_job_run_query,
     jobs_query,
 )
 
@@ -163,3 +164,12 @@ class Jobs(BaseAction):
             query, variable_values=variables, get_execution_result=True
         )
         return result
+
+    @guard
+    def get_job_secrets_for_job_run(self, id: str):
+        query = gql(job_secrets_for_job_run_query)
+        variables = {"jobRunId": id}
+        result = self.primitive.session.execute(
+            query, variable_values=variables, get_execution_result=True
+        )
+        return result.data["jobSecretsForJobRun"]

primitive/jobs/graphql/queries.py

@@ -98,3 +98,10 @@ query jobRun($id: GlobalID!) {
 }
 """
 )
+
+
+job_secrets_for_job_run_query = """
+query jobSecretsForJobRun($jobRunId: GlobalID!) {
+    jobSecretsForJobRun(jobRunId: $jobRunId)
+}
+"""

primitive/utils/shell.py

@@ -35,12 +35,14 @@ def add_path_to_shell(path: Path):
     return True
 
 
-def env_to_dict(env_vars: Union[str, List[str]]) -> Dict:
+def env_to_dict(env_vars: Union[bytes, str, List[str]]) -> Dict:
     lines = None
     if isinstance(env_vars, list):
         lines = env_vars
     elif isinstance(env_vars, str):
         lines = env_vars.splitlines()
+    elif isinstance(env_vars, bytes):
+        lines = env_vars.decode("utf-8").splitlines()
     else:
         raise ValueError("Unsupported type. Env_vars must be a list or a string")
 

{primitive-0.2.31.dist-info → primitive-0.2.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: primitive
-Version: 0.2.31
+Version: 0.2.33
 Project-URL: Documentation, https://github.com//primitivecorp/primitive-cli#readme
 Project-URL: Issues, https://github.com//primitivecorp/primitive-cli/issues
 Project-URL: Source, https://github.com//primitivecorp/primitive-cli

{primitive-0.2.31.dist-info → primitive-0.2.33.dist-info}/RECORD

@@ -1,11 +1,11 @@
-primitive/__about__.py,sha256=LH2fP6k1EgAIeykRYn1baGNEf_y6ro_QPuG8jugYJpo,130
+primitive/__about__.py,sha256=VgaNzMy13Zl_7Ph2LzRKGJY9WBIR3X3iXpy-0CtxYTQ,130
 primitive/__init__.py,sha256=bwKdgggKNVssJFVPfKSxqFMz4IxSr54WWbmiZqTMPNI,106
 primitive/cli.py,sha256=g7EtHI9MATAB0qQu5w-WzbXtxz_8zu8z5E7sETmMkKU,2509
 primitive/client.py,sha256=h8WZVnQylVe0vbpuyC8YZHl2JyITSPC-1HbUcmrE5pc,3623
 primitive/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/agent/actions.py,sha256=HDktV5FwtD19Bieg653w-ZWp7dQHTOj7cQx0-Aiae84,3836
+primitive/agent/actions.py,sha256=JLnHCtu0AgeHRZccCc2I39mziLixmj9EKtKeYmvBE7A,3700
 primitive/agent/commands.py,sha256=cK7d3OcN5Z65gQWVZFQ-Y9ddw9Pes4f9OVBpeMsj5sE,255
-primitive/agent/runner.py,sha256=SGI_vcFfey-VUH3gQasVJRgzPeSLMlP6FBjSImJqbOA,14180
+primitive/agent/runner.py,sha256=duF0iQCvf23ZOKRUk5e3yL2wQz9pr-XnpPijFqmb4uc,15143
 primitive/agent/uploader.py,sha256=ZzrzsajNBogwEC7mT6Ejy0h2Jd9axMYGzt9pbCvVMlk,3171
 primitive/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/auth/actions.py,sha256=9NIEXJ1BNJutJs6AMMSjMN_ziONUAUhY_xHwojYJCLA,942
@@ -51,12 +51,12 @@ primitive/hardware/graphql/fragments.py,sha256=kI6qnTNjaEaUr-C6eD55COphtueVYbYOW
 primitive/hardware/graphql/mutations.py,sha256=_4Hkbfik9Ron4T-meulu6T-9FR_BZjyPNwn745MPksU,1484
 primitive/hardware/graphql/queries.py,sha256=I86uLuOSjHSph11Y5MVCYko5Js7hoiEZ-cEoPTc4J-k,1392
 primitive/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/jobs/actions.py,sha256=lOfmjzZ7XvqLyP0f-ilyfYwQQZuNgUb4qxFCpf1d93M,5076
+primitive/jobs/actions.py,sha256=Fx2cPc1x09nRasOVtjhPjNRJ-jNoi3RJhXqC3verD9s,5444
 primitive/jobs/commands.py,sha256=MxPCkBEYW_eLNqgCRYeyj7ZcLOFAWfpVZlqDR2Y_S0o,830
 primitive/jobs/graphql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/jobs/graphql/fragments.py,sha256=pS6jXZ9yxvPKjKx50zpwzARJHYlg7NQLYCm1voiuCzI,642
 primitive/jobs/graphql/mutations.py,sha256=8ASvCmwQh7cMeeiykOdYaYVryG8FRIuVF6v_J8JJZuw,219
-primitive/jobs/graphql/queries.py,sha256=BrU_GnLjK0bTAmWsLSmGEUea7EM8MqTKxN1Qp6sSjwc,1597
+primitive/jobs/graphql/queries.py,sha256=ZxNmm-WovytbggNuKRnwa0kc26T34_0yhqkoqx-2uj0,1736
 primitive/monitor/actions.py,sha256=aYe5OfgCxhapXbcvz7vSlIMAcLOFRcAUWmdBZ8H7UWs,10889
 primitive/monitor/commands.py,sha256=VDlEL_Qpm_ysHxug7VpI0cVAZ0ny6AS91Y58D7F1zkU,409
 primitive/organizations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -94,10 +94,10 @@ primitive/utils/daemons.py,sha256=mSoSHitiGfS4KYAEK9sKsiv_YcACHKgY3qISnDpUUIE,10
 primitive/utils/exceptions.py,sha256=DrYHTcCAJGC7cCUwOx_FmdlVLWRdpzvDvpLb82heppE,311
 primitive/utils/memory_size.py,sha256=4xfha21kW82nFvOTtDFx9Jk2ZQoEhkfXii-PGNTpIUk,3058
 primitive/utils/printer.py,sha256=f1XUpqi5dkTL3GWvYRUGlSwtj2IxU1q745T4Fxo7Tn4,370
-primitive/utils/shell.py,sha256=jWzb7ky7p987dJas6ZvarK3IJNZ5cwBXcryRWb9Uh6U,2072
+primitive/utils/shell.py,sha256=Z4zxmOaSyGCrS0D6I436iQci-ewHLt4UxVg1CD9Serc,2171
 primitive/utils/text.py,sha256=XiESMnlhjQ534xE2hMNf08WehE1SKaYFRNih0MmnK0k,829
-primitive-0.2.31.dist-info/METADATA,sha256=jEXyjNe3Uv2BFG9MZAUt0Em4YMU950c4T_xUq26f7NY,3569
-primitive-0.2.31.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-primitive-0.2.31.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
-primitive-0.2.31.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
-primitive-0.2.31.dist-info/RECORD,,
+primitive-0.2.33.dist-info/METADATA,sha256=d38zBbaz0OFTGWtf-ZmSEUSWJX1NgmWf3aotjjZkMA8,3569
+primitive-0.2.33.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+primitive-0.2.33.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
+primitive-0.2.33.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
+primitive-0.2.33.dist-info/RECORD,,