primitive 0.2.30__py3-none-any.whl → 0.2.32__py3-none-any.whl

primitive/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Dylan Stein <dylan@primitive.tech>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.2.30"
+__version__ = "0.2.32"

primitive/agent/actions.py

@@ -100,10 +100,6 @@ class Agent(BaseAction):
                     # This should probably eventually be another daemon?
                     uploader.scan()
 
-                    JobRun.objects.filter_by(
-                        job_run_id=api_job_run.get("id"),
-                    ).delete()
-
                 sleep(5)
         except KeyboardInterrupt:
             logger.info("[agent] Stopping primitive agent...")

primitive/agent/runner.py

@@ -25,7 +25,11 @@ if typing.TYPE_CHECKING:
 
 CHUNK_SIZE = 64 * 1024
 ENV_VAR_LOOKUP_START = "_ENV_VAR_LOOKUP_START"
+START_DELIMITER_SIZE = len(bytes(ENV_VAR_LOOKUP_START, encoding="utf-8"))
 ENV_VAR_LOOKUP_END = "_ENV_VAR_LOOKUP_END"
+END_DELIMITER_SIZE = len(bytes(ENV_VAR_LOOKUP_END, encoding="utf-8"))
+
+assert CHUNK_SIZE > START_DELIMITER_SIZE + END_DELIMITER_SIZE
 
 
 class Task(TypedDict):
@@ -116,30 +120,26 @@ class Runner:
         )
 
         # Attempt to parse the job yaml file
-        logger.info(f"Scanning directory for job file {self.job['slug']}")
-        yaml_file = Path(self.source_dir / ".primitive" / f"{self.job['slug']}.yaml")
-        yml_file = Path(self.source_dir / ".primitive" / f"{self.job['slug']}.yml")
+        job_filename = self.job_settings["repositoryFilename"]
+        logger.info(f"Scanning directory for job file {job_filename}")
 
-        if yaml_file.exists() and yml_file.exists():
-            logger.error(
-                f"Found two job descriptions with the same slug: {self.job['slug']}"
-            )
-            raise FileExistsError
+        job_config_file = Path(self.source_dir / ".primitive" / job_filename)
 
-        if yaml_file.exists() or yml_file.exists():
+        if job_config_file.exists():
             logger.info(f"Found job description for {self.job['slug']}")
-            config_file = yaml_file if yaml_file.exists() else yml_file
-            self.config = yaml.load(open(config_file, "r"), Loader=Loader)[
-                self.job["name"]
-            ]
+            self.config = yaml.load(open(job_config_file, "r"), Loader=Loader)
         else:
             logger.error(
-                f"No job description with matching slug '{self.job['slug']}' found"
+                f"No job description with matching filename '{job_filename}' found"
             )
             raise FileNotFoundError
 
         # Setup initial process environment
         self.initial_env = os.environ
+        self.initial_env = {
+            **self.initial_env,
+            **self.primitive.jobs.get_job_secrets_for_job_run(self.job_run["id"]),
+        }
         self.initial_env["PRIMITIVE_GIT_SHA"] = str(self.job_run["gitCommit"]["sha"])
         self.initial_env["PRIMITIVE_GIT_BRANCH"] = str(
             self.job_run["gitCommit"]["branch"]
@@ -261,12 +261,7 @@ class Runner:
             args = [
                 "/bin/bash",
                 "-c",
-                (
-                    f"{cmd} "
-                    f"&& stdbuf -oL echo && stdbuf -oL echo '{ENV_VAR_LOOKUP_START}' "
-                    "&& env "
-                    f"&& stdbuf -oL echo && echo '{ENV_VAR_LOOKUP_END}'"
-                ),
+                f"{cmd} && echo -n '{ENV_VAR_LOOKUP_START}' && env && echo -n '{ENV_VAR_LOOKUP_END}'",
             ]
 
             process = await asyncio.create_subprocess_exec(
@@ -310,28 +305,32 @@ class Runner:
     async def log_cmd(self, process, stream, tags: Dict = {}) -> bool:
         failure_detected = False
         parse_environment = False
-        env_lines = []
+        last_chunk_buffer = b""
+        environment_buffer = b""
         while chunk := await stream.read(CHUNK_SIZE):
-            processed_lines = await self.read_chunk(chunk)
-            single_line = len(processed_lines) == 1
-            empty_chunk = len(processed_lines) == 0
-
-            if empty_chunk:
-                # Ignore the empty lines
+            if parse_environment:
+                environment_buffer += chunk
                 continue
-            elif single_line and ENV_VAR_LOOKUP_START in processed_lines[0]:
-                # Next chunk will contain the environent variables
+
+            # First, look for start delimiter in chunk
+            full_chunk = last_chunk_buffer + chunk
+            last_chunk_buffer = b""
+            start_index = full_chunk.find(bytes(ENV_VAR_LOOKUP_START, encoding="utf-8"))
+
+            if start_index != -1:
+                environment_buffer = full_chunk[start_index + START_DELIMITER_SIZE :]
+                processed_lines = await self.read_chunk(full_chunk[:start_index])
                 parse_environment = True
-                continue
-            elif single_line and ENV_VAR_LOOKUP_END in processed_lines[0]:
-                # Done reading environment variables
-                parse_environment = False
-                self.modified_env = env_to_dict(env_lines)
-                continue
+            else:
+                processed_lines = await self.read_chunk(full_chunk)
 
-            if parse_environment:
-                env_lines.extend(processed_lines)
-                continue
+                while (
+                    len(last_chunk_buffer) < START_DELIMITER_SIZE
+                    and len(processed_lines) > 0
+                ):
+                    last_chunk_buffer += bytes(
+                        processed_lines.pop() + "\n", encoding="utf-8"
+                    )
 
             # Handle logging
             parse_logs = self.job_settings["parseLogs"]
@@ -364,6 +363,16 @@ class Runner:
 
                 logger.bind(tag=tag).log(level.value, line)
 
+        start_index = environment_buffer.find(
+            bytes(ENV_VAR_LOOKUP_END, encoding="utf-8")
+        )
+        if parse_environment and start_index == -1:
+            logger.error("Environment variable buffer did not contain end delimiter")
+            failure_detected = True
+            return failure_detected
+
+        environment_buffer = environment_buffer[:start_index]
+        self.modified_env = env_to_dict(environment_buffer)
         return failure_detected
 
     async def read_chunk(self, chunk: bytes) -> List[str]:

primitive/daemons/launch_agents.py

@@ -132,7 +132,7 @@ class LaunchAgent(Daemon):
     <key>ProgramArguments</key>
     <array>
         <string>{self.executable}</string>
-        <string>{self.command}</string>
+        {"".join([f"<string>{arg}</string>" for arg in self.command.split(" ") if arg.strip() != ""])}
     </array>
     <key>RunAtLoad</key>
     <true/>

primitive/jobs/actions.py

@@ -11,6 +11,7 @@ from .graphql.queries import (
     job_run_query,
     job_run_status_query,
     job_runs_query,
+    job_secrets_for_job_run_query,
     jobs_query,
 )
 
@@ -163,3 +164,12 @@ class Jobs(BaseAction):
             query, variable_values=variables, get_execution_result=True
         )
         return result
+
+    @guard
+    def get_job_secrets_for_job_run(self, id: str):
+        query = gql(job_secrets_for_job_run_query)
+        variables = {"jobRunId": id}
+        result = self.primitive.session.execute(
+            query, variable_values=variables, get_execution_result=True
+        )
+        return result.data["jobSecretsForJobRun"]

primitive/jobs/graphql/fragments.py

@@ -34,6 +34,7 @@ fragment JobRunFragment on JobRun {
     parseLogs
     parseStderr
     failureLevel
+    repositoryFilename
   }
   gitCommit {
     sha

primitive/jobs/graphql/queries.py

@@ -98,3 +98,10 @@ query jobRun($id: GlobalID!) {
 }
 """
 )
+
+
+job_secrets_for_job_run_query = """
+query jobSecretsForJobRun($jobRunId: GlobalID!) {
+    jobSecretsForJobRun(jobRunId: $jobRunId)
+}
+"""

primitive/utils/shell.py

@@ -35,12 +35,14 @@ def add_path_to_shell(path: Path):
     return True
 
 
-def env_to_dict(env_vars: Union[str, List[str]]) -> Dict:
+def env_to_dict(env_vars: Union[bytes, str, List[str]]) -> Dict:
     lines = None
     if isinstance(env_vars, list):
         lines = env_vars
     elif isinstance(env_vars, str):
         lines = env_vars.splitlines()
+    elif isinstance(env_vars, bytes):
+        lines = env_vars.decode("utf-8").splitlines()
     else:
         raise ValueError("Unsupported type. Env_vars must be a list or a string")
 

{primitive-0.2.30.dist-info → primitive-0.2.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: primitive
-Version: 0.2.30
+Version: 0.2.32
 Project-URL: Documentation, https://github.com//primitivecorp/primitive-cli#readme
 Project-URL: Issues, https://github.com//primitivecorp/primitive-cli/issues
 Project-URL: Source, https://github.com//primitivecorp/primitive-cli

{primitive-0.2.30.dist-info → primitive-0.2.32.dist-info}/RECORD

@@ -1,11 +1,11 @@
-primitive/__about__.py,sha256=RmCzeHEVe-XHumqPgZrfYW5qjcAtsXdA16gAEzHGmjQ,130
+primitive/__about__.py,sha256=oC0l5fABhgagaC-_r-5QZ1VPn7hy8tdJID7wI0iTSow,130
 primitive/__init__.py,sha256=bwKdgggKNVssJFVPfKSxqFMz4IxSr54WWbmiZqTMPNI,106
 primitive/cli.py,sha256=g7EtHI9MATAB0qQu5w-WzbXtxz_8zu8z5E7sETmMkKU,2509
 primitive/client.py,sha256=h8WZVnQylVe0vbpuyC8YZHl2JyITSPC-1HbUcmrE5pc,3623
 primitive/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/agent/actions.py,sha256=HDktV5FwtD19Bieg653w-ZWp7dQHTOj7cQx0-Aiae84,3836
+primitive/agent/actions.py,sha256=JLnHCtu0AgeHRZccCc2I39mziLixmj9EKtKeYmvBE7A,3700
 primitive/agent/commands.py,sha256=cK7d3OcN5Z65gQWVZFQ-Y9ddw9Pes4f9OVBpeMsj5sE,255
-primitive/agent/runner.py,sha256=CoRyReO3jPV8B7vILVWdszFD4GVop7HsVEUo1hoRXjo,14556
+primitive/agent/runner.py,sha256=e4HBhK16zuhTpKAdKsgoij6G4YjMTn9JQo2YTLRI8i8,15008
 primitive/agent/uploader.py,sha256=ZzrzsajNBogwEC7mT6Ejy0h2Jd9axMYGzt9pbCvVMlk,3171
 primitive/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/auth/actions.py,sha256=9NIEXJ1BNJutJs6AMMSjMN_ziONUAUhY_xHwojYJCLA,942
@@ -15,7 +15,7 @@ primitive/auth/graphql/queries.py,sha256=jhrr_VFzHIn8vcVprMIzUx7V4kkWYdR6CKMKPoV
 primitive/daemons/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 primitive/daemons/actions.py,sha256=dqhxgIDxiJlcN6xvkzgBerN215KvbBKKdWteRpT_ICs,2730
 primitive/daemons/commands.py,sha256=Xt4qFymNrDLdHJhRnEH_4Re-2xX6w1OT-chV9k7dFCs,2670
-primitive/daemons/launch_agents.py,sha256=5MFDacs_IZd6PCm4VRNq9dPEYIDivkydBzqex1ggkFY,7740
+primitive/daemons/launch_agents.py,sha256=cCxsvCBFmGlKOiBINsKi_NxcozLC8yPw3w6pxqnz-qI,7803
 primitive/daemons/launch_service.py,sha256=iuklHeuEqadlf8U1n9xFg4ZG1EKdK2jyaPI-VTlpJ4I,7907
 primitive/daemons/ui.py,sha256=Af3OJWJ0jdGlb1nfA5yaGYdhBEqqpM8zP2U2vUQdCbw,1236
 primitive/db/base.py,sha256=mH7f2d_jiyxJSSx9Gk53QBXRa3LiKBsBjkFgvmtH1WA,83
@@ -51,12 +51,12 @@ primitive/hardware/graphql/fragments.py,sha256=kI6qnTNjaEaUr-C6eD55COphtueVYbYOW
 primitive/hardware/graphql/mutations.py,sha256=_4Hkbfik9Ron4T-meulu6T-9FR_BZjyPNwn745MPksU,1484
 primitive/hardware/graphql/queries.py,sha256=I86uLuOSjHSph11Y5MVCYko5Js7hoiEZ-cEoPTc4J-k,1392
 primitive/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/jobs/actions.py,sha256=lOfmjzZ7XvqLyP0f-ilyfYwQQZuNgUb4qxFCpf1d93M,5076
+primitive/jobs/actions.py,sha256=Fx2cPc1x09nRasOVtjhPjNRJ-jNoi3RJhXqC3verD9s,5444
 primitive/jobs/commands.py,sha256=MxPCkBEYW_eLNqgCRYeyj7ZcLOFAWfpVZlqDR2Y_S0o,830
 primitive/jobs/graphql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-primitive/jobs/graphql/fragments.py,sha256=1_ZttT7dx36KDC3DClJz9M8LMpsPwXySBygHSiUEcGg,619
+primitive/jobs/graphql/fragments.py,sha256=pS6jXZ9yxvPKjKx50zpwzARJHYlg7NQLYCm1voiuCzI,642
 primitive/jobs/graphql/mutations.py,sha256=8ASvCmwQh7cMeeiykOdYaYVryG8FRIuVF6v_J8JJZuw,219
-primitive/jobs/graphql/queries.py,sha256=BrU_GnLjK0bTAmWsLSmGEUea7EM8MqTKxN1Qp6sSjwc,1597
+primitive/jobs/graphql/queries.py,sha256=ZxNmm-WovytbggNuKRnwa0kc26T34_0yhqkoqx-2uj0,1736
 primitive/monitor/actions.py,sha256=aYe5OfgCxhapXbcvz7vSlIMAcLOFRcAUWmdBZ8H7UWs,10889
 primitive/monitor/commands.py,sha256=VDlEL_Qpm_ysHxug7VpI0cVAZ0ny6AS91Y58D7F1zkU,409
 primitive/organizations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -94,10 +94,10 @@ primitive/utils/daemons.py,sha256=mSoSHitiGfS4KYAEK9sKsiv_YcACHKgY3qISnDpUUIE,10
 primitive/utils/exceptions.py,sha256=DrYHTcCAJGC7cCUwOx_FmdlVLWRdpzvDvpLb82heppE,311
 primitive/utils/memory_size.py,sha256=4xfha21kW82nFvOTtDFx9Jk2ZQoEhkfXii-PGNTpIUk,3058
 primitive/utils/printer.py,sha256=f1XUpqi5dkTL3GWvYRUGlSwtj2IxU1q745T4Fxo7Tn4,370
-primitive/utils/shell.py,sha256=jWzb7ky7p987dJas6ZvarK3IJNZ5cwBXcryRWb9Uh6U,2072
+primitive/utils/shell.py,sha256=Z4zxmOaSyGCrS0D6I436iQci-ewHLt4UxVg1CD9Serc,2171
 primitive/utils/text.py,sha256=XiESMnlhjQ534xE2hMNf08WehE1SKaYFRNih0MmnK0k,829
-primitive-0.2.30.dist-info/METADATA,sha256=WAlddgCvqYTWwuqvUNQJIyGHtGZgNFB0cAWKP2PvWiY,3569
-primitive-0.2.30.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-primitive-0.2.30.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
-primitive-0.2.30.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
-primitive-0.2.30.dist-info/RECORD,,
+primitive-0.2.32.dist-info/METADATA,sha256=z-2bverx3L8RAyVNxxYWBv_-3UjdTgfZbk_ety8Swlo,3569
+primitive-0.2.32.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+primitive-0.2.32.dist-info/entry_points.txt,sha256=p1K8DMCWka5FqLlqP1sPek5Uovy9jq8u51gUsP-z334,48
+primitive-0.2.32.dist-info/licenses/LICENSE.txt,sha256=B8kmQMJ2sxYygjCLBk770uacaMci4mPSoJJ8WoDBY_c,1098
+primitive-0.2.32.dist-info/RECORD,,