feldera 0.131.0__py3-none-any.whl → 0.192.0__py3-none-any.whl

feldera/rest/feldera_client.py

@@ -1,17 +1,20 @@
-import pathlib
-from typing import Any, Dict, Optional
+import json
 import logging
+import pathlib
 import time
-import json
 from decimal import Decimal
-from typing import Generator, Mapping
+from typing import Any, Dict, Generator, Mapping, Optional
+from urllib.parse import quote
+
+import requests
 
+from feldera.enums import BootstrapPolicy, PipelineFieldSelector, PipelineStatus
+from feldera.rest._helpers import determine_client_version
+from feldera.rest._httprequests import HttpRequests
 from feldera.rest.config import Config
+from feldera.rest.errors import FelderaAPIError, FelderaTimeoutError
 from feldera.rest.feldera_config import FelderaConfig
-from feldera.rest.errors import FelderaTimeoutError, FelderaAPIError
 from feldera.rest.pipeline import Pipeline
-from feldera.rest._httprequests import HttpRequests
-from feldera.rest._helpers import client_version
 
 
 def _validate_no_none_keys_in_map(data):
@@ -35,34 +38,51 @@ def _prepare_boolean_input(value: bool) -> str:
 
 class FelderaClient:
     """
-    A client for the Feldera HTTP API
+    A client for the Feldera HTTP API.
 
-    A client instance is needed for every Feldera API method to know the
-    location of Feldera and its permissions.
+    The client is initialized with the configuration needed for interacting with the
+    Feldera HTTP API, which it uses in its calls. Its methods are implemented
+    by issuing one or more HTTP requests to the API, and as such can provide higher
+    level operations (e.g., support waiting for the success of asynchronous HTTP API
+    functionality).
     """
 
     def __init__(
         self,
-        url: str,
+        url: Optional[str] = None,
         api_key: Optional[str] = None,
         timeout: Optional[float] = None,
         connection_timeout: Optional[float] = None,
-        requests_verify: bool = True,
+        requests_verify: Optional[bool | str] = None,
     ) -> None:
         """
-        :param url: The url to Feldera API (ex: https://try.feldera.com)
-        :param api_key: The optional API key for Feldera
-        :param timeout: (optional) The amount of time in seconds that the
-            client will wait for a response before timing out.
-        :param connection_timeout: (optional) The amount of time in seconds that
-            the client will wait to establish connection before timing out.
-        :param requests_verify: The `verify` parameter passed to the requests
-            library. `True` by default.
+        Constructs a Feldera client.
+
+        :param url: (Optional) URL to the Feldera API.
+            The default is read from the `FELDERA_HOST` environment variable;
+            if the variable is not set, the default is `"http://localhost:8080"`.
+        :param api_key: (Optional) API key to access Feldera (format: `"apikey:..."`).
+            The default is read from the `FELDERA_API_KEY` environment variable;
+            if the variable is not set, the default is `None` (no API key is provided).
+        :param timeout: (Optional) Duration in seconds that the client will wait to receive
+            a response of an HTTP request, after which it times out.
+            The default is `None` (wait indefinitely; no timeout is enforced).
+        :param connection_timeout: (Optional) Duration in seconds that the client will wait
+            to establish the connection of an HTTP request, after which it times out.
+            The default is `None` (wait indefinitely; no timeout is enforced).
+        :param requests_verify: (Optional) The `verify` parameter passed to the `requests` library,
+            which is used internally to perform HTTP requests.
+            See also: https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification .
+            The default is based on the `FELDERA_HTTPS_TLS_CERT` or the `FELDERA_TLS_INSECURE` environment variable.
+            By setting `FELDERA_HTTPS_TLS_CERT` to a path, the default becomes the CA bundle it points to.
+            By setting `FELDERA_TLS_INSECURE` to `"1"`, `"true"` or `"yes"` (all case-insensitive), the default becomes
+            `False` which means to disable TLS verification by default. If both variables are set, the former takes
+            priority over the latter. If neither variable is set, the default is `True`.
         """
 
         self.config = Config(
-            url,
-            api_key,
+            url=url,
+            api_key=api_key,
             timeout=timeout,
             connection_timeout=connection_timeout,
             requests_verify=requests_verify,
@@ -70,12 +90,12 @@ class FelderaClient:
         self.http = HttpRequests(self.config)
 
         try:
-            config = self.get_config()
-            version = client_version()
-            if config.version != version:
+            client_version = determine_client_version()
+            server_config = self.get_config()
+            if client_version != server_config.version:
                 logging.warning(
-                    f"Client is on version {version} while server is at "
-                    f"{config.version}. There could be incompatibilities."
+                    f"Feldera client is on version {client_version} while server is at "
+                    f"{server_config.version}. There could be incompatibilities."
                 )
         except Exception as e:
             logging.error(f"Failed to connect to Feldera API: {e}")
@@ -89,35 +109,46 @@ class FelderaClient:
 
         return FelderaClient(f"http://127.0.0.1:{port}")
 
-    def get_pipeline(self, pipeline_name) -> Pipeline:
+    def get_pipeline(
+        self, pipeline_name: str, field_selector: PipelineFieldSelector
+    ) -> Pipeline:
         """
         Get a pipeline by name
 
         :param pipeline_name: The name of the pipeline
+        :param field_selector: Choose what pipeline information to refresh; see PipelineFieldSelector enum definition.
         """
 
-        resp = self.http.get(f"/pipelines/{pipeline_name}")
+        resp = self.http.get(
+            f"/pipelines/{pipeline_name}?selector={field_selector.value}"
+        )
 
         return Pipeline.from_dict(resp)
 
-    def get_runtime_config(self, pipeline_name) -> dict:
+    def get_runtime_config(self, pipeline_name) -> Mapping[str, Any]:
         """
         Get the runtime config of a pipeline by name
 
         :param pipeline_name: The name of the pipeline
         """
 
-        resp: dict = self.http.get(f"/pipelines/{pipeline_name}")
+        resp: Mapping[str, Any] = self.http.get(f"/pipelines/{pipeline_name}")
 
-        return resp.get("runtime_config")
+        runtime_config: Mapping[str, Any] | None = resp.get("runtime_config")
+        if runtime_config is None:
+            raise ValueError(f"Pipeline {pipeline_name} has no runtime config")
 
-    def pipelines(self) -> list[Pipeline]:
+        return runtime_config
+
+    def pipelines(
+        self, selector: PipelineFieldSelector = PipelineFieldSelector.STATUS
+    ) -> list[Pipeline]:
         """
         Get all pipelines
         """
 
         resp = self.http.get(
-            path="/pipelines",
+            path=f"/pipelines?selector={selector.value}",
         )
 
         return [Pipeline.from_dict(pipeline) for pipeline in resp]
@@ -126,12 +157,14 @@ class FelderaClient:
         wait = ["Pending", "CompilingSql", "SqlCompiled", "CompilingRust"]
 
         while True:
-            p = self.get_pipeline(name)
+            p = self.get_pipeline(name, PipelineFieldSelector.STATUS)
             status = p.program_status
 
             if status == "Success":
-                return p
+                return self.get_pipeline(name, PipelineFieldSelector.ALL)
             elif status not in wait:
+                p = self.get_pipeline(name, PipelineFieldSelector.ALL)
+
                 # error handling for SQL compilation errors
                 if status == "SqlError":
                     sql_errors = p.program_error["sql_compilation"]["messages"]
@@ -159,12 +192,92 @@ class FelderaClient:
             logging.debug("still compiling %s, waiting for 100 more milliseconds", name)
             time.sleep(0.1)
 
-    def create_pipeline(self, pipeline: Pipeline) -> Pipeline:
+    def __wait_for_pipeline_state(
+        self,
+        pipeline_name: str,
+        state: str,
+        timeout_s: Optional[float] = None,
+        start: bool = True,
+    ):
+        start_time = time.monotonic()
+
+        while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError(
+                        f"Timed out waiting for pipeline {pipeline_name} to "
+                        f"transition to '{state}' state"
+                    )
+
+            resp = self.get_pipeline(pipeline_name, PipelineFieldSelector.STATUS)
+            status = resp.deployment_status
+
+            if status.lower() == state.lower():
+                break
+            elif (
+                status == "Stopped"
+                and len(resp.deployment_error or {}) > 0
+                and resp.deployment_desired_status == "Stopped"
+            ):
+                err_msg = "Unable to START the pipeline:\n" if start else ""
+                raise RuntimeError(
+                    f"""{err_msg}Unable to transition the pipeline to '{state}'.
+Reason: The pipeline is in a STOPPED state due to the following error:
+{resp.deployment_error.get("message", "")}"""
+                )
+
+            logging.debug(
+                "still starting %s, waiting for 100 more milliseconds", pipeline_name
+            )
+            time.sleep(0.1)
+
+    def __wait_for_pipeline_state_one_of(
+        self,
+        pipeline_name: str,
+        states: list[str],
+        timeout_s: float | None = None,
+        start: bool = True,
+    ) -> PipelineStatus:
+        start_time = time.monotonic()
+        states = [state.lower() for state in states]
+
+        while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError(
+                        f"Timed out waiting for pipeline {pipeline_name} to"
+                        f"transition to one of the states: {states}"
+                    )
+
+            resp = self.get_pipeline(pipeline_name, PipelineFieldSelector.STATUS)
+            status = resp.deployment_status
+
+            if status.lower() in states:
+                return PipelineStatus.from_str(status)
+            elif (
+                status == "Stopped"
+                and len(resp.deployment_error or {}) > 0
+                and resp.deployment_desired_status == "Stopped"
+            ):
+                err_msg = "Unable to START the pipeline:\n" if start else ""
+                raise RuntimeError(
+                    f"""{err_msg}Unable to transition the pipeline to one of the states: {states}.
+Reason: The pipeline is in a STOPPED state due to the following error:
+{resp.deployment_error.get("message", "")}"""
+                )
+            logging.debug(
+                "still starting %s, waiting for 100 more milliseconds", pipeline_name
+            )
+            time.sleep(0.1)
+
+    def create_pipeline(self, pipeline: Pipeline, wait: bool = True) -> Pipeline:
         """
         Create a pipeline if it doesn't exist and wait for it to compile
 
-
-        :name: The name of the pipeline
+        :param pipeline: The pipeline to create
+        :param wait: Whether to wait for the pipeline to compile. True by default
         """
 
         body = {
@@ -182,12 +295,21 @@ class FelderaClient:
             body=body,
         )
 
+        if not wait:
+            return pipeline
+
         return self.__wait_for_compilation(pipeline.name)
 
-    def create_or_update_pipeline(self, pipeline: Pipeline) -> Pipeline:
+    def create_or_update_pipeline(
+        self, pipeline: Pipeline, wait: bool = True
+    ) -> Pipeline:
         """
         Create a pipeline if it doesn't exist or update a pipeline and wait for
         it to compile
+
+        :param pipeline: The pipeline to create or update
+        :param wait: Whether to wait for the pipeline to compile. True by default
+        :return: The created or updated pipeline
         """
 
         body = {
@@ -205,6 +327,9 @@ class FelderaClient:
             body=body,
         )
 
+        if not wait:
+            return pipeline
+
         return self.__wait_for_compilation(pipeline.name)
 
     def patch_pipeline(
@@ -235,6 +360,21 @@ class FelderaClient:
             },
         )
 
+    def testing_force_update_platform_version(self, name: str, platform_version: str):
+        self.http.post(
+            path=f"/pipelines/{name}/testing",
+            params={"set_platform_version": platform_version},
+        )
+
+    def update_pipeline_runtime(self, name: str):
+        """
+        Recompile a pipeline with the Feldera runtime version included in the currently installed Feldera platform.
+
+        :param name: The name of the pipeline
+        """
+
+        self.http.post(path=f"/pipelines/{name}/update_runtime")
+
     def delete_pipeline(self, name: str):
         """
         Deletes a pipeline by name
@@ -275,130 +415,161 @@ class FelderaClient:
                     yield chunk.decode("utf-8")
 
     def activate_pipeline(
-        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
-    ):
+        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = None
+    ) -> Optional[PipelineStatus]:
         """
 
         :param pipeline_name: The name of the pipeline to activate
-        :param wait: Set True to wait for the pipeline to activate. True by default
-        :param timeout_s: The amount of time in seconds to wait for the pipeline
-            to activate. 300 seconds by default.
+        :param wait: Set True to wait for the pipeline to activate. True by
+            default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to activate.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
-
         self.http.post(
             path=f"/pipelines/{pipeline_name}/activate",
         )
 
         if not wait:
-            return
+            return None
 
-        start_time = time.monotonic()
+        return self.__wait_for_pipeline_state_one_of(
+            pipeline_name, ["running", "AwaitingApproval"], timeout_s
+        )
 
-        while True:
-            if timeout_s is not None:
-                elapsed = time.monotonic() - start_time
-                if elapsed > timeout_s:
-                    raise TimeoutError(
-                        f"Timed out waiting for pipeline {pipeline_name} to activate"
-                    )
+    def _inner_start_pipeline(
+        self,
+        pipeline_name: str,
+        initial: str = "running",
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ) -> Optional[PipelineStatus]:
+        """
 
-            resp = self.get_pipeline(pipeline_name)
-            status = resp.deployment_status
+        :param pipeline_name: The name of the pipeline to start
+        :param initial: The initial state to start the pipeline in. "running"
+            by default.
+        :param wait: Set True to wait for the pipeline to start. True by default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start.
+        """
 
-            if status == "Running":
-                break
-            elif (
-                status == "Stopped"
-                and len(resp.deployment_error or {}) > 0
-                and resp.deployment_desired_status == "Stopped"
-            ):
-                raise RuntimeError(
-                    f"""Unable to ACTIVATE the pipeline.
-Reason: The pipeline is in a STOPPED state due to the following error:
-{resp.deployment_error.get("message", "")}"""
-                )
+        params = {"initial": initial}
+        if bootstrap_policy is not None:
+            params["bootstrap_policy"] = bootstrap_policy.value
 
-            logging.debug(
-                "still starting %s, waiting for 100 more milliseconds", pipeline_name
-            )
-            time.sleep(0.1)
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/start",
+            params=params,
+        )
+
+        if not wait:
+            return None
+
+        return self.__wait_for_pipeline_state_one_of(
+            pipeline_name, [initial, "AwaitingApproval"], timeout_s
+        )
 
     def start_pipeline(
-        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
-    ):
+        self,
+        pipeline_name: str,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ) -> Optional[PipelineStatus]:
         """
 
         :param pipeline_name: The name of the pipeline to start
-        :param wait: Set True to wait for the pipeline to start. True by default
-        :param timeout_s: The amount of time in seconds to wait for the pipeline
-            to start. 300 seconds by default.
+        :param wait: Set True to wait for the pipeline to start.
+            True by default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
+        return self._inner_start_pipeline(
+            pipeline_name, "running", bootstrap_policy, wait, timeout_s
+        )
 
-        self.http.post(
-            path=f"/pipelines/{pipeline_name}/start",
+    def start_pipeline_as_paused(
+        self,
+        pipeline_name: str,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: float | None = None,
+    ) -> Optional[PipelineStatus]:
+        """
+        :param pipeline_name: The name of the pipeline to start as paused.
+        :param wait: Set True to wait for the pipeline to start as pause.
+            True by default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start.
+        """
+
+        return self._inner_start_pipeline(
+            pipeline_name, "paused", bootstrap_policy, wait, timeout_s
         )
 
-        if not wait:
-            return
+    def start_pipeline_as_standby(
+        self,
+        pipeline_name: str,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        :param pipeline_name: The name of the pipeline to start as standby.
+        :param wait: Set True to wait for the pipeline to start as standby.
+            True by default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start.
+        """
 
-        start_time = time.monotonic()
+        self._inner_start_pipeline(
+            pipeline_name, "standby", bootstrap_policy, wait, timeout_s
+        )
 
-        while True:
-            if timeout_s is not None:
-                elapsed = time.monotonic() - start_time
-                if elapsed > timeout_s:
-                    raise TimeoutError(
-                        f"Timed out waiting for pipeline {pipeline_name} to start"
-                    )
+    def resume_pipeline(
+        self,
+        pipeline_name: str,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Resume a pipeline
 
-            resp = self.get_pipeline(pipeline_name)
-            status = resp.deployment_status
+        :param pipeline_name: The name of the pipeline to stop
+        :param wait: Set True to wait for the pipeline to pause. True by default
+        :param timeout_s: The amount of time in seconds to wait for the pipeline
+            to pause.
+        """
 
-            if status == "Running":
-                break
-            elif (
-                status == "Stopped"
-                and len(resp.deployment_error or {}) > 0
-                and resp.deployment_desired_status == "Stopped"
-            ):
-                raise RuntimeError(
-                    f"""Unable to START the pipeline.
-Reason: The pipeline is in a STOPPED state due to the following error:
-{resp.deployment_error.get("message", "")}"""
-                )
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/resume",
+        )
 
-            logging.debug(
-                "still starting %s, waiting for 100 more milliseconds", pipeline_name
-            )
-            time.sleep(0.1)
+        if not wait:
+            return
+
+        self.__wait_for_pipeline_state(pipeline_name, "running", timeout_s)
 
     def pause_pipeline(
         self,
         pipeline_name: str,
-        error_message: str = None,
         wait: bool = True,
-        timeout_s: Optional[float] = 300,
+        timeout_s: Optional[float] = None,
     ):
         """
-        Stop a pipeline
+        Pause a pipeline
 
         :param pipeline_name: The name of the pipeline to stop
         :param error_message: The error message to show if the pipeline is in
              STOPPED state due to a failure.
         :param wait: Set True to wait for the pipeline to pause. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
-            to pause. 300 seconds by default.
+            to pause.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
-
         self.http.post(
             path=f"/pipelines/{pipeline_name}/pause",
         )
@@ -406,46 +577,22 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         if not wait:
             return
 
-        if error_message is None:
-            error_message = "Unable to PAUSE the pipeline.\n"
-
-        start_time = time.monotonic()
-
-        while True:
-            if timeout_s is not None:
-                elapsed = time.monotonic() - start_time
-                if elapsed > timeout_s:
-                    raise TimeoutError(
-                        f"Timed out waiting for pipeline {pipeline_name} to pause"
-                    )
-
-            resp = self.get_pipeline(pipeline_name)
-            status = resp.deployment_status
-
-            if status == "Paused":
-                break
-            elif (
-                status == "Stopped"
-                and len(resp.deployment_error or {}) > 0
-                and resp.deployment_desired_status == "Stopped"
-            ):
-                raise RuntimeError(
-                    error_message
-                    + f"""Reason: The pipeline is in a STOPPED state due to the following error:
-{resp.deployment_error.get("message", "")}"""
-                )
+        self.__wait_for_pipeline_state(pipeline_name, "paused", timeout_s)
 
-            logging.debug(
-                "still pausing %s, waiting for 100 more milliseconds", pipeline_name
-            )
-            time.sleep(0.1)
+    def approve_pipeline(
+        self,
+        pipeline_name: str,
+    ):
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/approve",
+        )
 
     def stop_pipeline(
         self,
         pipeline_name: str,
         force: bool,
         wait: bool = True,
-        timeout_s: Optional[float] = 300,
+        timeout_s: Optional[float] = None,
     ):
         """
         Stop a pipeline
@@ -455,12 +602,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             Set False to automatically checkpoint before stopping.
         :param wait: Set True to wait for the pipeline to stop. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
-            to stop. Default is 300 seconds.
+            to stop.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
-
         params = {"force": str(force).lower()}
 
         self.http.post(
@@ -473,8 +617,15 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
         start = time.monotonic()
 
-        while time.monotonic() - start < timeout_s:
-            status = self.get_pipeline(pipeline_name).deployment_status
+        while True:
+            if timeout_s is not None and time.monotonic() - start > timeout_s:
+                raise FelderaTimeoutError(
+                    f"timeout error: pipeline '{pipeline_name}' did not stop in {timeout_s} seconds"
+                )
+
+            status = self.get_pipeline(
+                pipeline_name, PipelineFieldSelector.STATUS
+            ).deployment_status
 
             if status == "Stopped":
                 return
@@ -485,30 +636,29 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             )
             time.sleep(0.1)
 
-        raise FelderaTimeoutError(
-            f"timeout error: pipeline '{pipeline_name}' did not stop in {timeout_s} seconds"
-        )
-
-    def clear_storage(self, pipeline_name: str, timeout_s: Optional[float] = 300):
+    def clear_storage(self, pipeline_name: str, timeout_s: Optional[float] = None):
         """
         Clears the storage from the pipeline.
         This operation cannot be canceled.
 
         :param pipeline_name: The name of the pipeline
         :param timeout_s: The amount of time in seconds to wait for the storage
-            to clear. Default is 300 seconds.
+            to clear.
         """
-        if timeout_s is None:
-            timeout_s = 300
-
         self.http.post(
             path=f"/pipelines/{pipeline_name}/clear",
         )
 
         start = time.monotonic()
 
-        while time.monotonic() - start < timeout_s:
-            status = self.get_pipeline(pipeline_name).storage_status
+        while True:
+            if timeout_s is not None and time.monotonic() - start > timeout_s:
+                raise FelderaTimeoutError(
+                    f"timeout error: pipeline '{pipeline_name}' did not clear storage in {timeout_s} seconds"
+                )
+            status = self.get_pipeline(
+                pipeline_name, PipelineFieldSelector.STATUS
+            ).storage_status
 
             if status == "Cleared":
                 return
@@ -519,10 +669,6 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             )
             time.sleep(0.1)
 
-        raise FelderaTimeoutError(
-            f"timeout error: pipeline '{pipeline_name}' did not clear storage in {timeout_s} seconds"
-        )
-
     def start_transaction(self, pipeline_name: str) -> int:
         """
         Start a new transaction.
@@ -562,7 +708,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         :raises RuntimeError: If there is currently no transaction in progress.
         :raises ValueError: If the provided `transaction_id` does not match the current transaction.
         :raises TimeoutError: If the transaction does not commit within the specified timeout (when `wait` is True).
-        :raises FelderaAPIError: If the pipeline fails to start a transaction.
+        :raises FelderaAPIError: If the pipeline fails to commit a transaction.
         """
 
         # TODO: implement this without using /stats when we have a better pipeline status reporting API.
@@ -718,15 +864,15 @@ Reason: The pipeline is in a STOPPED state due to the following error:
                     _validate_no_none_keys_in_map(datum.get("insert", {}))
                     _validate_no_none_keys_in_map(datum.get("delete", {}))
             else:
-                data: dict = data
+                data: Mapping[str, Any] = data
                 _validate_no_none_keys_in_map(data.get("insert", {}))
                 _validate_no_none_keys_in_map(data.get("delete", {}))
         else:
             _validate_no_none_keys_in_map(data)
 
         # python sends `True` which isn't accepted by the backend
-        array = _prepare_boolean_input(array)
-        force = _prepare_boolean_input(force)
+        array: str = _prepare_boolean_input(array)
+        force: str = _prepare_boolean_input(force)
 
         params = {
             "force": force,
@@ -747,7 +893,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             data = bytes(str(data), "utf-8")
 
         resp = self.http.post(
-            path=f"/pipelines/{pipeline_name}/ingress/{table_name}",
+            path=f"/pipelines/{quote(pipeline_name, safe='')}/ingress/{quote(table_name, safe='')}",
             params=params,
             content_type=content_type,
             body=data,
@@ -765,8 +911,37 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
         return token
 
+    def completion_token_processed(self, pipeline_name: str, token: str) -> bool:
+        """
+        Check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+
+        :param pipeline_name: The name of the pipeline
+        :param token: The token to check for completion
+        :return: True if the pipeline has finished processing all inputs represented by the token, False otherwise
+        """
+
+        params = {
+            "token": token,
+        }
+
+        resp = self.http.get(
+            path=f"/pipelines/{quote(pipeline_name, safe='')}/completion_status",
+            params=params,
+        )
+
+        status: Optional[str] = resp.get("status")
+
+        if status is None:
+            raise FelderaAPIError(
+                f"got empty status when checking for completion status for token: {token}",
+                resp,
+            )
+
+        return status.lower() == "complete"
+
     def wait_for_token(
-        self, pipeline_name: str, token: str, timeout_s: Optional[float] = 600
+        self, pipeline_name: str, token: str, timeout_s: Optional[float] = None
     ):
         """
         Blocks until all records represented by this completion token have
@@ -775,13 +950,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         :param pipeline_name: The name of the pipeline
         :param token: The token to check for completion
         :param timeout_s: The amount of time in seconds to wait for the pipeline
-            to process these records. Default 600s
+            to process these records.
         """
 
-        params = {
-            "token": token,
-        }
-
         start = time.monotonic()
         end = start + timeout_s if timeout_s else None
         initial_backoff = 0.1
@@ -794,22 +965,11 @@ Reason: The pipeline is in a STOPPED state due to the following error:
                 if time.monotonic() > end:
                     raise FelderaTimeoutError(
                         f"timeout error: pipeline '{pipeline_name}' did not"
-                        f" process records represented by token {token} within"
-                        f" {timeout_s}"
+                        + f" process records represented by token {token} within"
+                        + f" {timeout_s}"
                     )
 
-            resp = self.http.get(
-                path=f"/pipelines/{pipeline_name}/completion_status", params=params
-            )
-
-            status: Optional[str] = resp.get("status")
-            if status is None:
-                raise FelderaAPIError(
-                    f"got empty status when checking for completion status for token: {token}",
-                    resp,
-                )
-
-            if status.lower() == "complete":
+            if self.completion_token_processed(pipeline_name, token):
                 break
 
             elapsed = time.monotonic() - start
@@ -830,6 +990,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         backpressure: bool = True,
         array: bool = False,
         timeout: Optional[float] = None,
+        case_sensitive: bool = False,
     ):
         """
         Listen for updates to views for pipeline, yields the chunks of data
@@ -845,6 +1006,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             "json" format, the default value is False
 
         :param timeout: The amount of time in seconds to listen to the stream for
+        :param case_sensitive: True if the table name is case sensitive or a reserved keyword, False by default
         """
 
         params = {
@@ -855,8 +1017,10 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         if format == "json":
             params["array"] = _prepare_boolean_input(array)
 
-        resp = self.http.post(
-            path=f"/pipelines/{pipeline_name}/egress/{table_name}",
+        table_name = f'"{table_name}"' if case_sensitive else table_name
+
+        resp: requests.Response = self.http.post(
+            path=f"/pipelines/{quote(pipeline_name, safe='')}/egress/{quote(table_name, safe='')}",
             params=params,
             stream=True,
         )
@@ -960,7 +1124,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
     def query_as_json(
         self, pipeline_name: str, query: str
-    ) -> Generator[dict, None, None]:
+    ) -> Generator[Mapping[str, Any], None, None]:
         """
         Executes an ad-hoc query on the specified pipeline and returns the result as a generator that yields
         rows of the query as Python dictionaries.
@@ -982,7 +1146,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             stream=True,
         )
 
-        for chunk in resp.iter_lines(chunk_size=50000000):
+        for chunk in resp.iter_lines(chunk_size=1024):
             if chunk:
                 yield json.loads(chunk, parse_float=Decimal)
 
@@ -1034,7 +1198,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
     def get_config(self) -> FelderaConfig:
         """
-        Get general feldera configuration.
+        Retrieves the general Feldera server configuration.
         """
 
         resp = self.http.get(path="/config")
@@ -1069,3 +1233,31 @@ Reason: The pipeline is in a STOPPED state due to the following error:
                 buffer += chunk
 
         return buffer
+
+    def generate_completion_token(
+        self, pipeline_name: str, table_name: str, connector_name: str
+    ) -> str:
+        """
+        Generate a completion token that can be passed to :meth:`.FelderaClient.completion_token_processed` to
+        check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+
+        :param pipeline_name: The name of the pipeline
+        :param table_name: The name of the table associated with this connector.
+        :param connector_name: The name of the connector.
+
+        :raises FelderaAPIError: If the connector cannot be found, or if the pipeline is not running.
+        """
+
+        resp = self.http.get(
+            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/completion_token",
+        )
+
+        token: str | None = resp.get("token")
+
+        if token is None:
+            raise ValueError(
+                "got invalid response from feldera when generating completion token"
+            )
+
+        return token