feldera 0.69.0__py3-none-any.whl → 0.189.0__py3-none-any.whl

feldera/rest/feldera_client.py

@@ -1,16 +1,20 @@
-import pathlib
-from typing import Any, Dict, Optional
+import json
 import logging
+import pathlib
 import time
-import json
 from decimal import Decimal
-from typing import Generator
+from typing import Any, Dict, Generator, Mapping, Optional
+from urllib.parse import quote
+
+import requests
 
+from feldera.enums import BootstrapPolicy, PipelineFieldSelector, PipelineStatus
+from feldera.rest._helpers import determine_client_version
+from feldera.rest._httprequests import HttpRequests
 from feldera.rest.config import Config
+from feldera.rest.errors import FelderaAPIError, FelderaTimeoutError
 from feldera.rest.feldera_config import FelderaConfig
-from feldera.rest.errors import FelderaTimeoutError
 from feldera.rest.pipeline import Pipeline
-from feldera.rest._httprequests import HttpRequests
 
 
 def _validate_no_none_keys_in_map(data):
@@ -34,35 +38,65 @@ def _prepare_boolean_input(value: bool) -> str:
 
 class FelderaClient:
     """
-    A client for the Feldera HTTP API
+    A client for the Feldera HTTP API.
 
-    A client instance is needed for every Feldera API method to know the location of
-    Feldera and its permissions.
+    The client is initialized with the configuration needed for interacting with the
+    Feldera HTTP API, which it uses in its calls. Its methods are implemented
+    by issuing one or more HTTP requests to the API, and as such can provide higher
+    level operations (e.g., support waiting for the success of asynchronous HTTP API
+    functionality).
     """
 
     def __init__(
         self,
-        url: str,
+        url: Optional[str] = None,
         api_key: Optional[str] = None,
         timeout: Optional[float] = None,
-        requests_verify: bool = True,
+        connection_timeout: Optional[float] = None,
+        requests_verify: Optional[bool | str] = None,
     ) -> None:
         """
-        :param url: The url to Feldera API (ex: https://try.feldera.com)
-        :param api_key: The optional API key for Feldera
-        :param timeout: (optional) The amount of time in seconds that the client will wait for a response before timing
-            out.
-        :param requests_verify: The `verify` parameter passed to the requests
-            library. `True` by default.
+        Constructs a Feldera client.
+
+        :param url: (Optional) URL to the Feldera API.
+            The default is read from the `FELDERA_HOST` environment variable;
+            if the variable is not set, the default is `"http://localhost:8080"`.
+        :param api_key: (Optional) API key to access Feldera (format: `"apikey:..."`).
+            The default is read from the `FELDERA_API_KEY` environment variable;
+            if the variable is not set, the default is `None` (no API key is provided).
+        :param timeout: (Optional) Duration in seconds that the client will wait to receive
+            a response of an HTTP request, after which it times out.
+            The default is `None` (wait indefinitely; no timeout is enforced).
+        :param connection_timeout: (Optional) Duration in seconds that the client will wait
+            to establish the connection of an HTTP request, after which it times out.
+            The default is `None` (wait indefinitely; no timeout is enforced).
+        :param requests_verify: (Optional) The `verify` parameter passed to the `requests` library,
+            which is used internally to perform HTTP requests.
+            See also: https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification .
+            The default is based on the `FELDERA_HTTPS_TLS_CERT` or the `FELDERA_TLS_INSECURE` environment variable.
+            By setting `FELDERA_HTTPS_TLS_CERT` to a path, the default becomes the CA bundle it points to.
+            By setting `FELDERA_TLS_INSECURE` to `"1"`, `"true"` or `"yes"` (all case-insensitive), the default becomes
+            `False` which means to disable TLS verification by default. If both variables are set, the former takes
+            priority over the latter. If neither variable is set, the default is `True`.
         """
 
         self.config = Config(
-            url, api_key, timeout=timeout, requests_verify=requests_verify
+            url=url,
+            api_key=api_key,
+            timeout=timeout,
+            connection_timeout=connection_timeout,
+            requests_verify=requests_verify,
         )
         self.http = HttpRequests(self.config)
 
         try:
-            self.pipelines()
+            client_version = determine_client_version()
+            server_config = self.get_config()
+            if client_version != server_config.version:
+                logging.warning(
+                    f"Feldera client is on version {client_version} while server is at "
+                    f"{server_config.version}. There could be incompatibilities."
+                )
         except Exception as e:
             logging.error(f"Failed to connect to Feldera API: {e}")
             raise e
@@ -75,35 +109,46 @@ class FelderaClient:
 
         return FelderaClient(f"http://127.0.0.1:{port}")
 
-    def get_pipeline(self, pipeline_name) -> Pipeline:
+    def get_pipeline(
+        self, pipeline_name: str, field_selector: PipelineFieldSelector
+    ) -> Pipeline:
         """
         Get a pipeline by name
 
         :param pipeline_name: The name of the pipeline
+        :param field_selector: Choose what pipeline information to refresh; see PipelineFieldSelector enum definition.
         """
 
-        resp = self.http.get(f"/pipelines/{pipeline_name}")
+        resp = self.http.get(
+            f"/pipelines/{pipeline_name}?selector={field_selector.value}"
+        )
 
         return Pipeline.from_dict(resp)
 
-    def get_runtime_config(self, pipeline_name) -> dict:
+    def get_runtime_config(self, pipeline_name) -> Mapping[str, Any]:
         """
         Get the runtime config of a pipeline by name
 
         :param pipeline_name: The name of the pipeline
         """
 
-        resp: dict = self.http.get(f"/pipelines/{pipeline_name}")
+        resp: Mapping[str, Any] = self.http.get(f"/pipelines/{pipeline_name}")
+
+        runtime_config: Mapping[str, Any] | None = resp.get("runtime_config")
+        if runtime_config is None:
+            raise ValueError(f"Pipeline {pipeline_name} has no runtime config")
 
-        return resp.get("runtime_config")
+        return runtime_config
 
-    def pipelines(self) -> list[Pipeline]:
+    def pipelines(
+        self, selector: PipelineFieldSelector = PipelineFieldSelector.STATUS
+    ) -> list[Pipeline]:
         """
         Get all pipelines
         """
 
         resp = self.http.get(
-            path="/pipelines",
+            path=f"/pipelines?selector={selector.value}",
         )
 
         return [Pipeline.from_dict(pipeline) for pipeline in resp]
@@ -112,12 +157,14 @@ class FelderaClient:
         wait = ["Pending", "CompilingSql", "SqlCompiled", "CompilingRust"]
 
         while True:
-            p = self.get_pipeline(name)
+            p = self.get_pipeline(name, PipelineFieldSelector.STATUS)
             status = p.program_status
 
             if status == "Success":
-                return p
+                return self.get_pipeline(name, PipelineFieldSelector.ALL)
             elif status not in wait:
+                p = self.get_pipeline(name, PipelineFieldSelector.ALL)
+
                 # error handling for SQL compilation errors
                 if status == "SqlError":
                     sql_errors = p.program_error["sql_compilation"]["messages"]
@@ -130,17 +177,107 @@ class FelderaClient:
                             err_msg += f"Code snippet:\n{sql_error['snippet']}"
                         raise RuntimeError(err_msg)
 
-                raise RuntimeError(f"The program failed to compile: {status}")
+                error_message = f"The program failed to compile: {status}\n"
+
+                rust_error = p.program_error.get("rust_compilation")
+                if rust_error is not None:
+                    error_message += f"Rust Error: {rust_error}\n"
+
+                system_error = p.program_error.get("system_error")
+                if system_error is not None:
+                    error_message += f"System Error: {system_error}"
+
+                raise RuntimeError(error_message)
 
             logging.debug("still compiling %s, waiting for 100 more milliseconds", name)
             time.sleep(0.1)
 
-    def create_pipeline(self, pipeline: Pipeline) -> Pipeline:
+    def __wait_for_pipeline_state(
+        self,
+        pipeline_name: str,
+        state: str,
+        timeout_s: Optional[float] = None,
+        start: bool = True,
+    ):
+        start_time = time.monotonic()
+
+        while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError(
+                        f"Timed out waiting for pipeline {pipeline_name} to "
+                        f"transition to '{state}' state"
+                    )
+
+            resp = self.get_pipeline(pipeline_name, PipelineFieldSelector.STATUS)
+            status = resp.deployment_status
+
+            if status.lower() == state.lower():
+                break
+            elif (
+                status == "Stopped"
+                and len(resp.deployment_error or {}) > 0
+                and resp.deployment_desired_status == "Stopped"
+            ):
+                err_msg = "Unable to START the pipeline:\n" if start else ""
+                raise RuntimeError(
+                    f"""{err_msg}Unable to transition the pipeline to '{state}'.
+Reason: The pipeline is in a STOPPED state due to the following error:
+{resp.deployment_error.get("message", "")}"""
+                )
+
+            logging.debug(
+                "still starting %s, waiting for 100 more milliseconds", pipeline_name
+            )
+            time.sleep(0.1)
+
+    def __wait_for_pipeline_state_one_of(
+        self,
+        pipeline_name: str,
+        states: list[str],
+        timeout_s: float | None = None,
+        start: bool = True,
+    ) -> PipelineStatus:
+        start_time = time.monotonic()
+        states = [state.lower() for state in states]
+
+        while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError(
+                        f"Timed out waiting for pipeline {pipeline_name} to"
+                        f"transition to one of the states: {states}"
+                    )
+
+            resp = self.get_pipeline(pipeline_name, PipelineFieldSelector.STATUS)
+            status = resp.deployment_status
+
+            if status.lower() in states:
+                return PipelineStatus.from_str(status)
+            elif (
+                status == "Stopped"
+                and len(resp.deployment_error or {}) > 0
+                and resp.deployment_desired_status == "Stopped"
+            ):
+                err_msg = "Unable to START the pipeline:\n" if start else ""
+                raise RuntimeError(
+                    f"""{err_msg}Unable to transition the pipeline to one of the states: {states}.
+Reason: The pipeline is in a STOPPED state due to the following error:
+{resp.deployment_error.get("message", "")}"""
+                )
+            logging.debug(
+                "still starting %s, waiting for 100 more milliseconds", pipeline_name
+            )
+            time.sleep(0.1)
+
+    def create_pipeline(self, pipeline: Pipeline, wait: bool = True) -> Pipeline:
         """
         Create a pipeline if it doesn't exist and wait for it to compile
 
-
-        :name: The name of the pipeline
+        :param pipeline: The pipeline to create
+        :param wait: Whether to wait for the pipeline to compile. True by default
         """
 
         body = {
@@ -158,11 +295,21 @@ class FelderaClient:
             body=body,
         )
 
+        if not wait:
+            return pipeline
+
         return self.__wait_for_compilation(pipeline.name)
 
-    def create_or_update_pipeline(self, pipeline: Pipeline) -> Pipeline:
+    def create_or_update_pipeline(
+        self, pipeline: Pipeline, wait: bool = True
+    ) -> Pipeline:
         """
-        Create a pipeline if it doesn't exist or update a pipeline and wait for it to compile
+        Create a pipeline if it doesn't exist or update a pipeline and wait for
+        it to compile
+
+        :param pipeline: The pipeline to create or update
+        :param wait: Whether to wait for the pipeline to compile. True by default
+        :return: The created or updated pipeline
         """
 
         body = {
@@ -180,21 +327,54 @@ class FelderaClient:
             body=body,
         )
 
+        if not wait:
+            return pipeline
+
         return self.__wait_for_compilation(pipeline.name)
 
-    def patch_pipeline(self, name: str, sql: str):
+    def patch_pipeline(
+        self,
+        name: str,
+        sql: Optional[str] = None,
+        udf_rust: Optional[str] = None,
+        udf_toml: Optional[str] = None,
+        program_config: Optional[Mapping[str, Any]] = None,
+        runtime_config: Optional[Mapping[str, Any]] = None,
+        description: Optional[str] = None,
+    ):
         """
-        Incrementally update the pipeline SQL
+        Incrementally update pipeline
 
         :param name: The name of the pipeline
-        :param sql: The SQL snippet. Replaces the existing SQL code with this one.
         """
 
         self.http.patch(
             path=f"/pipelines/{name}",
-            body={"program_code": sql},
+            body={
+                "program_code": sql,
+                "udf_rust": udf_rust,
+                "udf_toml": udf_toml,
+                "program_config": program_config,
+                "runtime_config": runtime_config,
+                "description": description,
+            },
         )
 
+    def testing_force_update_platform_version(self, name: str, platform_version: str):
+        self.http.post(
+            path=f"/pipelines/{name}/testing",
+            params={"set_platform_version": platform_version},
+        )
+
+    def update_pipeline_runtime(self, name: str):
+        """
+        Recompile a pipeline with the Feldera runtime version included in the currently installed Feldera platform.
+
+        :param name: The name of the pipeline
+        """
+
+        self.http.post(path=f"/pipelines/{name}/update_runtime")
+
     def delete_pipeline(self, name: str):
         """
         Deletes a pipeline by name
@@ -218,173 +398,360 @@ class FelderaClient:
 
         return resp
 
-    def start_pipeline(self, pipeline_name: str, timeout_s: Optional[float] = 300):
+    def get_pipeline_logs(self, pipeline_name: str) -> Generator[str, None, None]:
+        """
+        Get the pipeline logs
+
+        :param name: The name of the pipeline
+        :return: A generator yielding the logs, one line at a time.
+        """
+        chunk: bytes
+        with self.http.get(
+            path=f"/pipelines/{pipeline_name}/logs",
+            stream=True,
+        ) as resp:
+            for chunk in resp.iter_lines(chunk_size=50000000):
+                if chunk:
+                    yield chunk.decode("utf-8")
+
+    def activate_pipeline(
+        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = None
+    ) -> Optional[PipelineStatus]:
+        """
+
+        :param pipeline_name: The name of the pipeline to activate
+        :param wait: Set True to wait for the pipeline to activate. True by
+            default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to activate.
+        """
+
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/activate",
+        )
+
+        if not wait:
+            return None
+
+        return self.__wait_for_pipeline_state_one_of(
+            pipeline_name, ["running", "AwaitingApproval"], timeout_s
+        )
+
+    def _inner_start_pipeline(
+        self,
+        pipeline_name: str,
+        initial: str = "running",
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ) -> Optional[PipelineStatus]:
         """
 
         :param pipeline_name: The name of the pipeline to start
-        :param timeout_s: The amount of time in seconds to wait for the pipeline to start. 300 seconds by default.
+        :param initial: The initial state to start the pipeline in. "running"
+            by default.
+        :param wait: Set True to wait for the pipeline to start. True by default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
+        params = {"initial": initial}
+        if bootstrap_policy is not None:
+            params["bootstrap_policy"] = bootstrap_policy.value
 
         self.http.post(
             path=f"/pipelines/{pipeline_name}/start",
+            params=params,
         )
 
-        start_time = time.monotonic()
+        if not wait:
+            return None
 
-        while True:
-            if timeout_s is not None:
-                elapsed = time.monotonic() - start_time
-                if elapsed > timeout_s:
-                    raise TimeoutError(
-                        f"Timed out waiting for pipeline {pipeline_name} to start"
-                    )
+        return self.__wait_for_pipeline_state_one_of(
+            pipeline_name, [initial, "AwaitingApproval"], timeout_s
+        )
 
-            resp = self.get_pipeline(pipeline_name)
-            status = resp.deployment_status
+    def start_pipeline(
+        self,
+        pipeline_name: str,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ) -> Optional[PipelineStatus]:
+        """
 
-            if status == "Running":
-                break
-            elif status == "Failed":
-                raise RuntimeError(
-                    f"""Unable to START the pipeline.
-Reason: The pipeline is in a FAILED state due to the following error:
-{resp.deployment_error.get("message", "")}"""
-                )
+        :param pipeline_name: The name of the pipeline to start
+        :param wait: Set True to wait for the pipeline to start.
+            True by default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start.
+        """
 
-            logging.debug(
-                "still starting %s, waiting for 100 more milliseconds", pipeline_name
-            )
-            time.sleep(0.1)
+        return self._inner_start_pipeline(
+            pipeline_name, "running", bootstrap_policy, wait, timeout_s
+        )
+
+    def start_pipeline_as_paused(
+        self,
+        pipeline_name: str,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: float | None = None,
+    ) -> Optional[PipelineStatus]:
+        """
+        :param pipeline_name: The name of the pipeline to start as paused.
+        :param wait: Set True to wait for the pipeline to start as pause.
+            True by default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start.
+        """
+
+        return self._inner_start_pipeline(
+            pipeline_name, "paused", bootstrap_policy, wait, timeout_s
+        )
+
+    def start_pipeline_as_standby(
+        self,
+        pipeline_name: str,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        :param pipeline_name: The name of the pipeline to start as standby.
+        :param wait: Set True to wait for the pipeline to start as standby.
+            True by default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start.
+        """
+
+        self._inner_start_pipeline(
+            pipeline_name, "standby", bootstrap_policy, wait, timeout_s
+        )
+
+    def resume_pipeline(
+        self,
+        pipeline_name: str,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Resume a pipeline
+
+        :param pipeline_name: The name of the pipeline to stop
+        :param wait: Set True to wait for the pipeline to pause. True by default
+        :param timeout_s: The amount of time in seconds to wait for the pipeline
+            to pause.
+        """
+
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/resume",
+        )
+
+        if not wait:
+            return
+
+        self.__wait_for_pipeline_state(pipeline_name, "running", timeout_s)
 
     def pause_pipeline(
         self,
         pipeline_name: str,
-        error_message: str = None,
-        timeout_s: Optional[float] = 300,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Pause a pipeline
+
+        :param pipeline_name: The name of the pipeline to stop
+        :param error_message: The error message to show if the pipeline is in
+             STOPPED state due to a failure.
+        :param wait: Set True to wait for the pipeline to pause. True by default
+        :param timeout_s: The amount of time in seconds to wait for the pipeline
+            to pause.
+        """
+
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/pause",
+        )
+
+        if not wait:
+            return
+
+        self.__wait_for_pipeline_state(pipeline_name, "paused", timeout_s)
+
+    def approve_pipeline(
+        self,
+        pipeline_name: str,
+    ):
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/approve",
+        )
+
+    def stop_pipeline(
+        self,
+        pipeline_name: str,
+        force: bool,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
     ):
         """
         Stop a pipeline
 
         :param pipeline_name: The name of the pipeline to stop
-        :param error_message: The error message to show if the pipeline is in FAILED state
-        :param timeout_s: The amount of time in seconds to wait for the pipeline to pause. 300 seconds by default.
+        :param force: Set True to immediately scale compute resources to zero.
+            Set False to automatically checkpoint before stopping.
+        :param wait: Set True to wait for the pipeline to stop. True by default
+        :param timeout_s: The amount of time in seconds to wait for the pipeline
+            to stop.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
+        params = {"force": str(force).lower()}
 
         self.http.post(
-            path=f"/pipelines/{pipeline_name}/pause",
+            path=f"/pipelines/{pipeline_name}/stop",
+            params=params,
         )
 
-        if error_message is None:
-            error_message = "Unable to PAUSE the pipeline.\n"
+        if not wait:
+            return
 
-        start_time = time.monotonic()
+        start = time.monotonic()
 
         while True:
-            if timeout_s is not None:
-                elapsed = time.monotonic() - start_time
-                if elapsed > timeout_s:
-                    raise TimeoutError(
-                        f"Timed out waiting for pipeline {pipeline_name} to pause"
-                    )
+            if timeout_s is not None and time.monotonic() - start > timeout_s:
+                raise FelderaTimeoutError(
+                    f"timeout error: pipeline '{pipeline_name}' did not stop in {timeout_s} seconds"
+                )
 
-            resp = self.get_pipeline(pipeline_name)
-            status = resp.deployment_status
+            status = self.get_pipeline(
+                pipeline_name, PipelineFieldSelector.STATUS
+            ).deployment_status
 
-            if status == "Paused":
-                break
-            elif status == "Failed":
-                raise RuntimeError(
-                    error_message
-                    + f"""Reason: The pipeline is in a FAILED state due to the following error:
-{resp.deployment_error.get("message", "")}"""
-                )
+            if status == "Stopped":
+                return
 
             logging.debug(
-                "still pausing %s, waiting for 100 more milliseconds", pipeline_name
+                "still stopping %s, waiting for 100 more milliseconds",
+                pipeline_name,
             )
             time.sleep(0.1)
 
-    def shutdown_pipeline(self, pipeline_name: str, timeout_s: Optional[float] = 300):
+    def clear_storage(self, pipeline_name: str, timeout_s: Optional[float] = None):
         """
-        Shutdown a pipeline
+        Clears the storage from the pipeline.
+        This operation cannot be canceled.
 
-        :param pipeline_name: The name of the pipeline to shut down
-        :param timeout_s: The amount of time in seconds to wait for the pipeline to shut down. Default is 300 seconds.
+        :param pipeline_name: The name of the pipeline
+        :param timeout_s: The amount of time in seconds to wait for the storage
+            to clear.
         """
-
-        if timeout_s is None:
-            timeout_s = 300
-
         self.http.post(
-            path=f"/pipelines/{pipeline_name}/shutdown",
+            path=f"/pipelines/{pipeline_name}/clear",
         )
 
         start = time.monotonic()
 
-        while time.monotonic() - start < timeout_s:
-            status = self.get_pipeline(pipeline_name).deployment_status
+        while True:
+            if timeout_s is not None and time.monotonic() - start > timeout_s:
+                raise FelderaTimeoutError(
+                    f"timeout error: pipeline '{pipeline_name}' did not clear storage in {timeout_s} seconds"
+                )
+            status = self.get_pipeline(
+                pipeline_name, PipelineFieldSelector.STATUS
+            ).storage_status
 
-            if status == "Shutdown":
+            if status == "Cleared":
                 return
 
             logging.debug(
-                "still shutting down %s, waiting for 100 more milliseconds",
+                "still clearing %s, waiting for 100 more milliseconds",
                 pipeline_name,
             )
             time.sleep(0.1)
 
-        raise FelderaTimeoutError(
-            f"timeout error: pipeline '{pipeline_name}' did not shutdown in {timeout_s} seconds"
+    def start_transaction(self, pipeline_name: str) -> int:
+        """
+        Start a new transaction.
+
+            Transaction ID.
+
+        :param pipeline_name: The name of the pipeline.
+        """
+
+        resp = self.http.post(
+            path=f"/pipelines/{pipeline_name}/start_transaction",
         )
 
-    def suspend_pipeline(self, pipeline_name: str, timeout_s: Optional[float] = 300):
+        return int(resp.get("transaction_id"))
+
+    def commit_transaction(
+        self,
+        pipeline_name: str,
+        transaction_id: Optional[int] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
         """
-        Suspend a pipeline
+        Commits the currently active transaction.
+
+        :param pipeline_name: The name of the pipeline.
+
+        :param transaction_id: If provided, the function verifies that the currently active transaction matches this ID.
+            If the active transaction ID does not match, the function raises an error.
+
+        :param wait: If True, the function blocks until the transaction either commits successfully or the timeout is reached.
+            If False, the function initiates the commit and returns immediately without waiting for completion. The default value is True.
+
+        :param timeout_s: Maximum time (in seconds) to wait for the transaction to commit when `wait` is True.
+            If None, the function will wait indefinitely.
 
-        :param pipeline_name: The name of the pipeline to suspend
-        :param timeout_s: The amount of time in seconds to wait for the pipeline to suspend. Default is 300 seconds.
+        :raises RuntimeError: If there is currently no transaction in progress.
+        :raises ValueError: If the provided `transaction_id` does not match the current transaction.
+        :raises TimeoutError: If the transaction does not commit within the specified timeout (when `wait` is True).
+        :raises FelderaAPIError: If the pipeline fails to commit a transaction.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
+        # TODO: implement this without using /stats when we have a better pipeline status reporting API.
+        stats = self.get_pipeline_stats(pipeline_name)
+        current_transaction_id = stats["global_metrics"]["transaction_id"]
+
+        if current_transaction_id == 0:
+            raise RuntimeError(
+                "Attempting to commit a transaction, but there is no transaction in progress"
+            )
+
+        if transaction_id and current_transaction_id != transaction_id:
+            raise ValueError(
+                f"Specified transaction id {transaction_id} doesn't match current active transaction id {current_transaction_id}"
+            )
+
+        transaction_id = current_transaction_id
 
         self.http.post(
-            path=f"/pipelines/{pipeline_name}/suspend",
+            path=f"/pipelines/{pipeline_name}/commit_transaction",
         )
 
-        start = time.monotonic()
+        start_time = time.monotonic()
 
-        while time.monotonic() - start < timeout_s:
-            resp = self.get_pipeline(pipeline_name)
-            status = resp.deployment_status
+        if not wait:
+            return
 
-            if status == "Suspended":
-                return
-            elif status == "Failed":
-                raise RuntimeError(
-                    f"""Unable to Suspend pipeline '{pipeline_name}'.\nReason: The pipeline is in a FAILED state due to the following error:
-{resp.deployment_error.get("message", "")}"""
-                )
+        while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError("Timed out waiting for transaction to commit")
 
-            logging.debug(
-                "still suspending %s, waiting for 100 more milliseconds",
-                pipeline_name,
-            )
-            time.sleep(0.1)
+            stats = self.get_pipeline_stats(pipeline_name)
+            if stats["global_metrics"]["transaction_id"] != transaction_id:
+                return
 
-        raise FelderaTimeoutError(
-            f"timeout error: pipeline '{pipeline_name}' did not suspend in {timeout_s} seconds"
-        )
+            logging.debug("commit hasn't completed, waiting for 1 more second")
+            time.sleep(1.0)
 
     def checkpoint_pipeline(self, pipeline_name: str) -> int:
         """
-        Checkpoint a fault-tolerant pipeline
+        Checkpoint a pipeline.
 
         :param pipeline_name: The name of the pipeline to checkpoint
         """
@@ -434,13 +801,15 @@ Reason: The pipeline is in a FAILED state due to the following error:
         pipeline_name: str,
         table_name: str,
         format: str,
-        data: list[list | str | dict] | dict,
+        data: list[list | str | dict] | dict | str,
         array: bool = False,
         force: bool = False,
         update_format: str = "raw",
-        json_flavor: str = None,
+        json_flavor: Optional[str] = None,
         serialize: bool = True,
-    ):
+        wait: bool = True,
+        wait_timeout_s: Optional[float] = None,
+    ) -> str:
         """
         Insert data into a pipeline
 
@@ -457,6 +826,11 @@ Reason: The pipeline is in a FAILED state due to the following error:
             "debezium_mysql", "snowflake", "kafka_connect_json_converter", "pandas"
         :param data: The data to insert
         :param serialize: If True, the data will be serialized to JSON. True by default
+        :param wait: If True, blocks until this input has been processed by the pipeline
+        :param wait_timeout_s: The timeout in seconds to wait for this set of
+            inputs to be processed by the pipeline. None by default
+
+        :returns: The completion token to this input.
         """
 
         if format not in ["json", "csv"]:
@@ -490,15 +864,15 @@ Reason: The pipeline is in a FAILED state due to the following error:
                     _validate_no_none_keys_in_map(datum.get("insert", {}))
                     _validate_no_none_keys_in_map(datum.get("delete", {}))
             else:
-                data: dict = data
+                data: Mapping[str, Any] = data
                 _validate_no_none_keys_in_map(data.get("insert", {}))
                 _validate_no_none_keys_in_map(data.get("delete", {}))
         else:
             _validate_no_none_keys_in_map(data)
 
         # python sends `True` which isn't accepted by the backend
-        array = _prepare_boolean_input(array)
-        force = _prepare_boolean_input(force)
+        array: str = _prepare_boolean_input(array)
+        force: str = _prepare_boolean_input(force)
 
         params = {
             "force": force,
@@ -518,14 +892,96 @@ Reason: The pipeline is in a FAILED state due to the following error:
             content_type = "text/csv"
             data = bytes(str(data), "utf-8")
 
-        self.http.post(
-            path=f"/pipelines/{pipeline_name}/ingress/{table_name}",
+        resp = self.http.post(
+            path=f"/pipelines/{quote(pipeline_name, safe='')}/ingress/{quote(table_name, safe='')}",
             params=params,
             content_type=content_type,
             body=data,
             serialize=serialize,
         )
 
+        token = resp.get("token")
+        if token is None:
+            raise FelderaAPIError("response did not contain a completion token", resp)
+
+        if not wait:
+            return token
+
+        self.wait_for_token(pipeline_name, token, timeout_s=wait_timeout_s)
+
+        return token
+
+    def completion_token_processed(self, pipeline_name: str, token: str) -> bool:
+        """
+        Check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+
+        :param pipeline_name: The name of the pipeline
+        :param token: The token to check for completion
+        :return: True if the pipeline has finished processing all inputs represented by the token, False otherwise
+        """
+
+        params = {
+            "token": token,
+        }
+
+        resp = self.http.get(
+            path=f"/pipelines/{quote(pipeline_name, safe='')}/completion_status",
+            params=params,
+        )
+
+        status: Optional[str] = resp.get("status")
+
+        if status is None:
+            raise FelderaAPIError(
+                f"got empty status when checking for completion status for token: {token}",
+                resp,
+            )
+
+        return status.lower() == "complete"
+
+    def wait_for_token(
+        self, pipeline_name: str, token: str, timeout_s: Optional[float] = None
+    ):
+        """
+        Blocks until all records represented by this completion token have
+        been processed.
+
+        :param pipeline_name: The name of the pipeline
+        :param token: The token to check for completion
+        :param timeout_s: The amount of time in seconds to wait for the pipeline
+            to process these records.
+        """
+
+        start = time.monotonic()
+        end = start + timeout_s if timeout_s else None
+        initial_backoff = 0.1
+        max_backoff = 5
+        exponent = 1.2
+        retries = 0
+
+        while True:
+            if end:
+                if time.monotonic() > end:
+                    raise FelderaTimeoutError(
+                        f"timeout error: pipeline '{pipeline_name}' did not"
+                        + f" process records represented by token {token} within"
+                        + f" {timeout_s}"
+                    )
+
+            if self.completion_token_processed(pipeline_name, token):
+                break
+
+            elapsed = time.monotonic() - start
+            logging.debug(
+                f"still waiting for inputs represented by {token} to be processed; elapsed: {elapsed}s"
+            )
+
+            retries += 1
+            backoff = min(max_backoff, initial_backoff * (exponent**retries))
+
+            time.sleep(backoff)
+
     def listen_to_pipeline(
         self,
         pipeline_name: str,
@@ -534,6 +990,7 @@ Reason: The pipeline is in a FAILED state due to the following error:
         backpressure: bool = True,
         array: bool = False,
         timeout: Optional[float] = None,
+        case_sensitive: bool = False,
     ):
         """
         Listen for updates to views for pipeline, yields the chunks of data
@@ -549,6 +1006,7 @@ Reason: The pipeline is in a FAILED state due to the following error:
             "json" format, the default value is False
 
         :param timeout: The amount of time in seconds to listen to the stream for
+        :param case_sensitive: True if the table name is case sensitive or a reserved keyword, False by default
         """
 
         params = {
@@ -559,21 +1017,26 @@ Reason: The pipeline is in a FAILED state due to the following error:
         if format == "json":
             params["array"] = _prepare_boolean_input(array)
 
-        resp = self.http.post(
-            path=f"/pipelines/{pipeline_name}/egress/{table_name}",
+        table_name = f'"{table_name}"' if case_sensitive else table_name
+
+        resp: requests.Response = self.http.post(
+            path=f"/pipelines/{quote(pipeline_name, safe='')}/egress/{quote(table_name, safe='')}",
             params=params,
             stream=True,
         )
 
         end = time.monotonic() + timeout if timeout else None
 
-        # Using the default chunk size below makes `iter_lines` extremely
-        # inefficient when dealing with long lines.
-        for chunk in resp.iter_lines(chunk_size=50000000):
-            if end and time.monotonic() > end:
-                break
-            if chunk:
-                yield json.loads(chunk, parse_float=Decimal)
+        def generator():
+            # Using the default chunk size below makes `iter_lines` extremely
+            # inefficient when dealing with long lines.
+            for chunk in resp.iter_lines(chunk_size=50000000):
+                if end and time.monotonic() > end:
+                    break
+                if chunk:
+                    yield json.loads(chunk, parse_float=Decimal)
+
+        return generator
 
     def query_as_text(
         self, pipeline_name: str, query: str
@@ -602,6 +1065,27 @@ Reason: The pipeline is in a FAILED state due to the following error:
             if chunk:
                 yield chunk.decode("utf-8")
 
+    def query_as_hash(self, pipeline_name: str, query: str) -> str:
+        """
+        Executes an ad-hoc query on the specified pipeline and returns a hash of the result.
+
+        :param pipeline_name: The name of the pipeline to query.
+        :param query: The SQL query to be executed.
+        :return: A string containing the hash of the query result.
+        """
+        params = {
+            "pipeline_name": pipeline_name,
+            "sql": query,
+            "format": "hash",
+        }
+
+        resp = self.http.get(
+            path=f"/pipelines/{pipeline_name}/query",
+            params=params,
+            stream=False,
+        )
+        return resp
+
     def query_as_parquet(self, pipeline_name: str, query: str, path: str):
         """
         Executes an ad-hoc query on the specified pipeline and saves the result to a parquet file.
@@ -640,7 +1124,7 @@ Reason: The pipeline is in a FAILED state due to the following error:
 
     def query_as_json(
         self, pipeline_name: str, query: str
-    ) -> Generator[dict, None, None]:
+    ) -> Generator[Mapping[str, Any], None, None]:
         """
         Executes an ad-hoc query on the specified pipeline and returns the result as a generator that yields
         rows of the query as Python dictionaries.
@@ -662,7 +1146,7 @@ Reason: The pipeline is in a FAILED state due to the following error:
             stream=True,
         )
 
-        for chunk in resp.iter_lines(chunk_size=50000000):
+        for chunk in resp.iter_lines(chunk_size=1024):
             if chunk:
                 yield json.loads(chunk, parse_float=Decimal)
 
@@ -714,9 +1198,66 @@ Reason: The pipeline is in a FAILED state due to the following error:
 
     def get_config(self) -> FelderaConfig:
         """
-        Get general feldera configuration.
+        Retrieves the general Feldera server configuration.
         """
 
         resp = self.http.get(path="/config")
 
         return FelderaConfig(resp)
+
+    def get_pipeline_support_bundle(
+        self, pipeline_name: str, params: Optional[Dict[str, Any]] = None
+    ) -> bytes:
+        """
+        Generate a support bundle containing diagnostic information from a pipeline.
+
+        This endpoint collects various diagnostic data from the pipeline including
+        circuit profile, heap profile, metrics, logs, stats, and connector statistics,
+        and packages them into a single ZIP file for support purposes.
+
+        :param pipeline_name: The name of the pipeline
+        :param params: Optional query parameters to control data collection
+        :return: The support bundle as bytes (ZIP file)
+        :raises FelderaAPIError: If the pipeline does not exist or if there's an error
+        """
+
+        resp = self.http.get(
+            path=f"/pipelines/{pipeline_name}/support_bundle",
+            params=params,
+            stream=True,
+        )
+
+        buffer = b""
+        for chunk in resp.iter_content(chunk_size=1024):
+            if chunk:
+                buffer += chunk
+
+        return buffer
+
+    def generate_completion_token(
+        self, pipeline_name: str, table_name: str, connector_name: str
+    ) -> str:
+        """
+        Generate a completion token that can be passed to :meth:`.FelderaClient.completion_token_processed` to
+        check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+
+        :param pipeline_name: The name of the pipeline
+        :param table_name: The name of the table associated with this connector.
+        :param connector_name: The name of the connector.
+
+        :raises FelderaAPIError: If the connector cannot be found, or if the pipeline is not running.
+        """
+
+        resp = self.http.get(
+            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/completion_token",
+        )
+
+        token: str | None = resp.get("token")
+
+        if token is None:
+            raise ValueError(
+                "got invalid response from feldera when generating completion token"
+            )
+
+        return token