pipekit-sdk 2.1.2__tar.gz → 7.2.0__tar.gz

{pipekit_sdk-2.1.2 → pipekit_sdk-7.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pipekit-sdk
-Version: 2.1.2
+Version: 7.2.0
 Summary: Pipekit Python SDK
 License: BSD-3-Clause
 License-File: LICENSE
@@ -97,6 +97,25 @@ pipekit.submit(w, "<cluster-name>")
 pipekit.print_logs(pipe_run.uuid)
 ```
 
+## Connecting to Pipekit
+
+`pipekit_url` is the single base URL for the ID, Users, and UI APIs. This is correct when they sit behind one gateway, which is the default (`https://api.pipekit.io`).
+
+When the services are reachable on separate hosts, set `id_url` and `users_url` (or the `PIPEKIT_ID_URL` and `PIPEKIT_USERS_URL` env vars). The SDK logs in against `id_url` and makes every other call against `users_url`.
+
+```py
+pipekit = PipekitService(
+    username="<user>",
+    password="<password>",
+    id_url="http://id.internal:8080",
+    users_url="http://users.internal:8080",
+)
+```
+
+`insecure=True` (or `PIPEKIT_INSECURE=true`) skips TLS verification. Use it only for testing against a cluster with a self-signed certificate, never in production.
+
+`timeout` (or `PIPEKIT_TIMEOUT`) sets the per-request timeout in seconds, default 10. Raise it for slow links or for the cron lifecycle calls, which can take up to the server's notification timeout to return. The log stream is exempt and stays unbounded.
+
 ## Managing CronWorkflows
 
 You can create, update, and delete a `CronWorkflow` from Python. The namespace

{pipekit_sdk-2.1.2 → pipekit_sdk-7.2.0}/README.md

@@ -70,6 +70,25 @@ pipekit.submit(w, "<cluster-name>")
 pipekit.print_logs(pipe_run.uuid)
 ```
 
+## Connecting to Pipekit
+
+`pipekit_url` is the single base URL for the ID, Users, and UI APIs. This is correct when they sit behind one gateway, which is the default (`https://api.pipekit.io`).
+
+When the services are reachable on separate hosts, set `id_url` and `users_url` (or the `PIPEKIT_ID_URL` and `PIPEKIT_USERS_URL` env vars). The SDK logs in against `id_url` and makes every other call against `users_url`.
+
+```py
+pipekit = PipekitService(
+    username="<user>",
+    password="<password>",
+    id_url="http://id.internal:8080",
+    users_url="http://users.internal:8080",
+)
+```
+
+`insecure=True` (or `PIPEKIT_INSECURE=true`) skips TLS verification. Use it only for testing against a cluster with a self-signed certificate, never in production.
+
+`timeout` (or `PIPEKIT_TIMEOUT`) sets the per-request timeout in seconds, default 10. Raise it for slow links or for the cron lifecycle calls, which can take up to the server's notification timeout to return. The log stream is exempt and stays unbounded.
+
 ## Managing CronWorkflows
 
 You can create, update, and delete a `CronWorkflow` from Python. The namespace

{pipekit_sdk-2.1.2 → pipekit_sdk-7.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pipekit-sdk"
-version = "2.1.2"
+version = "7.2.0"
 description = "Pipekit Python SDK"
 authors = ["Pipekit <ci@pipekit.io>"]
 license = "BSD-3-Clause"
@@ -59,4 +59,13 @@ ignore = ["E501"]  # ignore line too long (80 chars)
 "**/__init__.py" = ["F401", "D107"]
 
 [tool.mypy]
-mypy_path = "src"
+mypy_path = "src"
+
+[tool.pytest.ini_options]
+# `make test` runs unit tests only. The integration suite needs a live, in-cluster
+# Pipekit and runs via `make integration`, which passes tests/integration explicitly.
+testpaths = ["tests/test_unit"]
+markers = [
+    "integration: end-to-end test that needs a live, in-cluster Pipekit",
+]
+

{pipekit_sdk-2.1.2 → pipekit_sdk-7.2.0}/src/pipekit_sdk/service.py

@@ -9,6 +9,7 @@ from typing import Any, Generator, List, Optional, cast
 
 import jwt
 import requests
+import urllib3
 from hera.workflows import CronWorkflow, Workflow
 from hera.workflows.models import CronWorkflow as CronWorkflowModel
 from hera.workflows.models import UpdateCronWorkflowRequest, WorkflowCreateRequest
@@ -19,6 +20,23 @@ from pipekit_sdk.models.model import Cluster, Logs, Pipe, PipeRun
 
 _CONTENT_TYPE_JSON = "application/json"
 
+_SSE_DATA_PREFIX = "data: "
+_SSE_ID_PREFIX = "id: "
+_SSE_PING = "ping"
+_SSE_CLOSE = "close"
+
+
+class PipekitAPIError(Exception):
+    """Raised when a Pipekit API request returns a non-2xx response."""
+
+
+def _strip_trailing_slash(url: str) -> str:
+    return url[:-1] if url.endswith("/") else url
+
+
+def _env_is_truthy(name: str) -> bool:
+    return os.getenv(name, "").strip().lower() in {"1", "true", "yes", "on"}
+
 
 class _LoginResponse(BaseModel):
     """LoginResponse is a model for Login responses."""
@@ -48,6 +66,15 @@ class _ValidationResult(Enum):
     EXPIRED_TOKEN = 2
 
 
+class _SSELineKind(Enum):
+    """The meaning of one line in the log event stream."""
+
+    IGNORE = 0
+    ID = 1
+    LOG = 2
+    CLOSE = 3
+
+
 class PipekitService:
     """PipekitService is a wrapper around the Pipekit API. It provides a simple interface to interact with Pipekit."""
 
@@ -59,7 +86,8 @@ class PipekitService:
             with open(pipekit_token_file_path, "r") as f:
                 raw_data = f.read()
                 return _LoginResponse.model_validate_json(raw_data)
-        except ValidationError:
+        except (FileNotFoundError, ValidationError):
+            # No saved token, or it is unreadable. Fall back to username/password login.
             return None
 
     @staticmethod
@@ -76,18 +104,42 @@ class PipekitService:
         password: str = "",
         token: str = "",
         pipekit_url: str = "https://api.pipekit.io",
+        id_url: str = "",
+        users_url: str = "",
+        insecure: bool = False,
+        timeout: float = 10,
     ):
-        """Initialise the service connection with the given username and password."""
+        """Initialise the service connection.
+
+        pipekit_url is the single base URL for the ID, Users, and UI APIs, which is right
+        when they sit behind one gateway (the default, api.pipekit.io). When the services
+        are reachable on separate hosts, set id_url and users_url, or the PIPEKIT_ID_URL
+        and PIPEKIT_USERS_URL env vars, to point at each one.
+
+        insecure, or the PIPEKIT_INSECURE env var, skips TLS verification. Use it only for
+        testing against a cluster with a self-signed certificate.
+
+        timeout, or the PIPEKIT_TIMEOUT env var, sets the per-request timeout in seconds
+        (default 10). Raise it for slow links or for the cron lifecycle calls, which can
+        take up to the server's notification timeout to return. The log stream is exempt
+        and stays unbounded.
+        """
         self.username = username
         self.password = password
 
-        pipekit_url = os.getenv("PIPEKIT_URL", pipekit_url)
-        if pipekit_url.endswith("/"):
-            pipekit_url = pipekit_url[:-1]
+        base_url = _strip_trailing_slash(os.getenv("PIPEKIT_URL", pipekit_url))
+        self.id_url = _strip_trailing_slash(os.getenv("PIPEKIT_ID_URL", id_url) or base_url)
+        self.users_url = _strip_trailing_slash(os.getenv("PIPEKIT_USERS_URL", users_url) or base_url)
+        self.ui_url = base_url
+
+        self.verify = not (insecure or _env_is_truthy("PIPEKIT_INSECURE"))
+        self._session = requests.Session()
+        self._session.verify = self.verify
+        if not self.verify:
+            # The caller opted out of TLS verification, so silence the per-request warning.
+            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
-        self.id_url = pipekit_url
-        self.users_url = pipekit_url
-        self.ui_url = pipekit_url
+        self.timeout = float(os.getenv("PIPEKIT_TIMEOUT") or timeout)
 
         if token != "":
             self.access_token = token
@@ -106,14 +158,14 @@ class PipekitService:
         login_url = f"{self.id_url}/api/id/v1/oauth/token"
         assert self.token_data is not None, "Token data was None, this implies a bug"
         login_request = {"grant_type": "refresh_token", "refresh_token": self.token_data.refresh_token}
-        login_response = requests.post(
+        login_response = self._session.post(
             login_url,
             data=json.dumps(login_request),
             headers={"Content-Type": _CONTENT_TYPE_JSON},
-            timeout=10,
+            timeout=self.timeout,
         )
         if login_response.status_code // 100 != 2:
-            raise Exception(
+            raise PipekitAPIError(
                 "Couldn't refresh access token, obtained non 2xx status code, try logging in again via the cli"
             )
 
@@ -124,7 +176,9 @@ class PipekitService:
     def __validate_token(self) -> _ValidationResult:
         assert self.token_data is not None
         try:
-            decoded_payload = jwt.decode(self.token_data.access_token, options={"verify_signature": False})
+            # We read only the exp claim to decide whether to refresh. The client has no
+            # key to verify the signature, and the server verifies it on every request.
+            decoded_payload = jwt.decode(self.token_data.access_token, options={"verify_signature": False})  # NOSONAR
             acc_tk = _AccessToken.model_validate(decoded_payload)
             exp_time = datetime.fromtimestamp(acc_tk.exp)
             now_time = datetime.now()
@@ -159,15 +213,15 @@ class PipekitService:
             login_request["username"] = ""
 
         login_url = f"{self.id_url}/api/id/v1/oauth/token"
-        login_response = requests.post(
+        login_response = self._session.post(
             login_url,
             data=json.dumps(login_request),
             headers={"Content-Type": _CONTENT_TYPE_JSON},
-            timeout=10,
+            timeout=self.timeout,
         )
 
         if login_response.status_code // 100 != 2:
-            raise Exception(f"Login failed: {login_response.text}")
+            raise PipekitAPIError(f"Login failed: {login_response.text}")
 
         self.access_token = cast(str, login_response.json()["access_token"])
 
@@ -178,14 +232,14 @@ class PipekitService:
     def list_clusters(self) -> list[Cluster]:
         """Get a list of all clusters."""
         clusters_url = f"{self.users_url}/api/users/v1/clusters"
-        clusters_response = requests.get(
+        clusters_response = self._session.get(
             clusters_url,
             headers={"Authorization": f"Bearer {self.access_token}"},
-            timeout=10,
+            timeout=self.timeout,
         )
 
         if clusters_response.status_code // 100 != 2:
-            raise Exception(f"List clusters failed: {clusters_response.text}")
+            raise PipekitAPIError(f"List clusters failed: {clusters_response.text}")
 
         clusters = [Cluster.model_validate(cluster) for cluster in clusters_response.json()]
         return clusters
@@ -205,14 +259,14 @@ class PipekitService:
                 pipes_url += f"/clusters/{cluster_uuid}"
             pipes_url += f"/pipes?page={page}&limit={limit}"
 
-            pipes_response = requests.get(
+            pipes_response = self._session.get(
                 pipes_url,
                 headers={"Authorization": f"Bearer {self.access_token}"},
-                timeout=10,
+                timeout=self.timeout,
             )
 
             if pipes_response.status_code // 100 != 2:
-                raise Exception(f"List pipes failed: {pipes_response.text}")
+                raise PipekitAPIError(f"List pipes failed: {pipes_response.text}")
 
             pipes_page = [Pipe.model_validate(item) for item in pipes_response.json()["items"]]
             if len(pipes_page) == 0:
@@ -233,14 +287,14 @@ class PipekitService:
     ) -> Pipe:
         """Get a single pipe using its uuid."""
         pipe_url = f"{self.users_url}/api/users/v1/pipes/{pipe_uuid}"
-        pipe_response = requests.get(
+        pipe_response = self._session.get(
             pipe_url,
             headers={"Authorization": f"Bearer {self.access_token}"},
-            timeout=10,
+            timeout=self.timeout,
         )
 
         if pipe_response.status_code // 100 != 2:
-            raise Exception(f"Get pipe failed: {pipe_response.text}")
+            raise PipekitAPIError(f"Get pipe failed: {pipe_response.text}")
 
         return Pipe.model_validate(pipe_response.json())
 
@@ -258,14 +312,14 @@ class PipekitService:
         while True:
             runs_url = f"{base_url}?page={page}&limit={limit}&status={status}"
 
-            runs_response = requests.get(
+            runs_response = self._session.get(
                 runs_url,
                 headers={"Authorization": f"Bearer {self.access_token}"},
-                timeout=10,
+                timeout=self.timeout,
             )
 
             if runs_response.status_code // 100 != 2:
-                raise Exception(f"List runs failed: {runs_response.text}")
+                raise PipekitAPIError(f"List runs failed: {runs_response.text}")
 
             runs_page = [PipeRun.model_validate(item) for item in runs_response.json()["items"]]
             if len(runs_page) == 0:
@@ -294,14 +348,14 @@ class PipekitService:
         while True:
             runs_url = f"{base_url}?page={page}&limit={limit}&status={status}"
 
-            runs_response = requests.get(
+            runs_response = self._session.get(
                 runs_url,
                 headers={"Authorization": f"Bearer {self.access_token}"},
-                timeout=10,
+                timeout=self.timeout,
             )
 
             if runs_response.status_code // 100 != 2:
-                raise Exception(f"List runs failed: {runs_response.text}")
+                raise PipekitAPIError(f"List runs failed: {runs_response.text}")
 
             runs_page = [PipeRun.model_validate(item) for item in runs_response.json()["items"]]
             if len(runs_page) == 0:
@@ -322,14 +376,14 @@ class PipekitService:
     ) -> PipeRun:
         """Get a PipeRun by its uuid."""
         run_url = f"{self.users_url}/api/users/v1/runs/{run_uuid}"
-        run_response = requests.get(
+        run_response = self._session.get(
             run_url,
             headers={"Authorization": f"Bearer {self.access_token}"},
-            timeout=10,
+            timeout=self.timeout,
         )
 
         if run_response.status_code // 100 != 2:
-            raise Exception(f"Get run failed: {run_response.text}")
+            raise PipekitAPIError(f"Get run failed: {run_response.text}")
 
         return PipeRun.model_validate(run_response.json())
 
@@ -339,14 +393,14 @@ class PipekitService:
     ) -> Cluster:
         """Get a single Cluster by its uuid."""
         cluster_url = f"{self.users_url}/api/users/v1/clusters/{cluster_uuid}"
-        cluster_response = requests.get(
+        cluster_response = self._session.get(
             cluster_url,
             headers={"Authorization": f"Bearer {self.access_token}"},
-            timeout=10,
+            timeout=self.timeout,
         )
 
         if cluster_response.status_code // 100 != 2:
-            raise Exception(f"Get cluster failed: {cluster_response.text}")
+            raise PipekitAPIError(f"Get cluster failed: {cluster_response.text}")
 
         return Cluster.model_validate(cluster_response.json())
 
@@ -386,7 +440,7 @@ class PipekitService:
         model_workflow = WorkflowCreateRequest(workflow=workflow.build()).workflow  # type: ignore
         assert model_workflow is not None
 
-        submit_response = requests.post(
+        submit_response = self._session.post(
             submit_url,
             data=model_workflow.json(
                 exclude_none=True,
@@ -398,11 +452,11 @@ class PipekitService:
                 "Authorization": f"Bearer {self.access_token}",
                 "Content-Type": _CONTENT_TYPE_JSON,
             },
-            timeout=10,
+            timeout=self.timeout,
         )
 
         if submit_response.status_code // 100 != 2:
-            raise Exception(f"Submit failed: {submit_response.text}")
+            raise PipekitAPIError(f"Submit failed: {submit_response.text}")
 
         return PipeRun.model_validate(submit_response.json())
 
@@ -417,7 +471,7 @@ class PipekitService:
         model_workflow = cron_workflow.build()
         assert model_workflow is not None
 
-        create_response = requests.post(
+        create_response = self._session.post(
             create_url,
             data=model_workflow.json(
                 exclude_none=True,
@@ -429,11 +483,11 @@ class PipekitService:
                 "Authorization": f"Bearer {self.access_token}",
                 "Content-Type": _CONTENT_TYPE_JSON,
             },
-            timeout=10,
+            timeout=self.timeout,
         )
 
         if create_response.status_code // 100 != 2:
-            raise Exception(f"Create failed: {create_response.text}")
+            raise PipekitAPIError(f"Create failed: {create_response.text}")
 
         return PipeRun.model_validate(create_response.json())
 
@@ -460,7 +514,7 @@ class PipekitService:
         if data is not None:
             headers["Content-Type"] = _CONTENT_TYPE_JSON
 
-        response = requests.request(method, url, data=data, headers=headers, timeout=10)
+        response = self._session.request(method, url, data=data, headers=headers, timeout=self.timeout)
         if response.status_code // 100 != 2:
             raise RuntimeError(f"{action} cron failed: {response.text}")
 
@@ -543,7 +597,10 @@ class PipekitService:
         The namespace must match the CronWorkflow manifest namespace.
         """
         url = self.__cron_by_name(cluster, namespace, name) + "/suspend"
-        response = self.__cron_request("PUT", url, "Suspend")
+        # Argo's suspend endpoint expects a request body. Sending it also sets the JSON
+        # Content-Type; a bodyless PUT is rejected with 415 Unsupported Media Type.
+        body = json.dumps({"name": name, "namespace": namespace})
+        response = self.__cron_request("PUT", url, "Suspend", data=body)
         return CronWorkflowModel.parse_raw(response.text)
 
     def resume_cron(
@@ -557,7 +614,10 @@ class PipekitService:
         The namespace must match the CronWorkflow manifest namespace.
         """
         url = self.__cron_by_name(cluster, namespace, name) + "/resume"
-        response = self.__cron_request("PUT", url, "Resume")
+        # Argo's resume endpoint expects a request body. Sending it also sets the JSON
+        # Content-Type; a bodyless PUT is rejected with 415 Unsupported Media Type.
+        body = json.dumps({"name": name, "namespace": namespace})
+        response = self.__cron_request("PUT", url, "Resume", data=body)
         return CronWorkflowModel.parse_raw(response.text)
 
     def __apply_run_action(
@@ -566,17 +626,17 @@ class PipekitService:
         action: str,
     ) -> PipeRun:
         action_url = f"{self.users_url}/api/users/v1/runs/{run_uuid}/workflows/{action}"
-        action_response = requests.put(
+        action_response = self._session.put(
             action_url,
             headers={
                 "Authorization": f"Bearer {self.access_token}",
                 "Content-Type": _CONTENT_TYPE_JSON,
             },
-            timeout=10,
+            timeout=self.timeout,
         )
 
         if action_response.status_code // 100 != 2:
-            raise Exception(f"Action failed: {action_response.text}")
+            raise PipekitAPIError(f"Action failed: {action_response.text}")
 
         return PipeRun.model_validate(action_response.json())
 
@@ -611,17 +671,17 @@ class PipekitService:
 
         # empty "" / [] must be omitted, not sent as ?pod-name= (the API filters to a pod named "")
         params = {k: v for k, v in {"pod-name": pod_name, "container-name": container_name}.items() if v}
-        logs_response = requests.get(
+        logs_response = self._session.get(
             logs_url,
             params=params,
             headers={"Authorization": f"Bearer {self.access_token}"},
-            timeout=10,
+            timeout=self.timeout,
         )
 
         if logs_response.status_code // 100 != 2:
-            raise Exception(f"Get logs failed: {logs_response.text}")
+            raise PipekitAPIError(f"Get logs failed: {logs_response.text}")
 
-        return [Logs.model_validate(log) for log in list(logs_response.json())]
+        return [Logs.model_validate(log) for log in logs_response.json()]
 
     def __get_logs_stream(
         self,
@@ -634,7 +694,7 @@ class PipekitService:
 
         # empty "" / [] must be omitted, not sent as ?pod-name= (the API filters to a pod named "")
         params = {k: v for k, v in {"pod-name": pod_name, "container-name": container_name}.items() if v}
-        logs_response = requests.get(
+        logs_response = self._session.get(
             logs_url,
             params=params,
             stream=True,
@@ -648,7 +708,7 @@ class PipekitService:
         )
 
         if logs_response.status_code // 100 != 2:
-            raise Exception(f"Get logs stream failed: {logs_response.text}")
+            raise PipekitAPIError(f"Get logs stream failed: {logs_response.text}")
 
         return logs_response
 
@@ -661,6 +721,26 @@ class PipekitService:
         """Get logs for a given run."""
         return self.__get_run_container_logs(run_uuid, pod_name, container_name)
 
+    @staticmethod
+    def __parse_sse_line(line: str) -> tuple[_SSELineKind, Any]:
+        # Classify one Server-Sent Events line from the log stream. The value is the
+        # new last-event-id for ID, a parsed Logs for LOG, and None otherwise.
+        line = line.strip()
+        if line == "":
+            return _SSELineKind.IGNORE, None
+        if line.startswith(_SSE_ID_PREFIX):
+            return _SSELineKind.ID, line[len(_SSE_ID_PREFIX) :]
+        if not line.startswith(_SSE_DATA_PREFIX):
+            print(f"unknown message: {line}")
+            return _SSELineKind.IGNORE, None
+
+        payload = line[len(_SSE_DATA_PREFIX) :]
+        if payload == _SSE_PING:
+            return _SSELineKind.IGNORE, None
+        if payload == _SSE_CLOSE:
+            return _SSELineKind.CLOSE, None
+        return _SSELineKind.LOG, Logs.model_validate(json.loads(payload))
+
     def follow_logs(
         self,
         run_uuid: str,
@@ -668,39 +748,19 @@ class PipekitService:
         container_name: List[str] | str = "",
     ) -> Generator[Logs, Any, None]:
         """Return a generator that yields pod logs for a given run."""
-        data_msg_prefix = "data: "
-        id_msg_prefix = "id: "
-        ping_msg = "ping"
-        close_msg = "close"
         last_event_id = ""
 
         while True:
             try:
                 logs_stream = self.__get_logs_stream(run_uuid, pod_name, container_name, last_event_id)
                 for line in logs_stream.iter_lines(delimiter="\n", decode_unicode=True):
-                    line = line.strip()
-
-                    if line == "":
-                        continue
-
-                    if line.startswith(data_msg_prefix):
-                        # strip the prefix
-                        line = line[len(data_msg_prefix) :]
-                    elif line.startswith(id_msg_prefix):
-                        last_event_id = line[len(id_msg_prefix) :]
-                        continue
-                    else:
-                        print(f"unknown message: {line}")
-                        continue
-
-                    if line == ping_msg:
-                        # pong
-                        continue
-                    elif line == close_msg:
-                        # bye
+                    kind, value = self.__parse_sse_line(line)
+                    if kind == _SSELineKind.ID:
+                        last_event_id = value
+                    elif kind == _SSELineKind.LOG:
+                        yield value
+                    elif kind == _SSELineKind.CLOSE:
                         return
-                    else:
-                        yield Logs.model_validate(json.loads(line))
 
             except requests.exceptions.Timeout:
                 pass  # we'll ignore timeout errors and reconnect