feldera 0.162.0__tar.gz → 0.164.0__tar.gz

{feldera-0.162.0 → feldera-0.164.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.162.0
+Version: 0.164.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

feldera-0.164.0/feldera/_callback_runner.py

@@ -0,0 +1,69 @@
+from threading import Thread
+from typing import Callable, Optional
+
+import pandas as pd
+from feldera import FelderaClient
+from feldera._helpers import dataframe_from_response
+from feldera.enums import PipelineFieldSelector
+
+
+class CallbackRunner(Thread):
+    def __init__(
+        self,
+        client: FelderaClient,
+        pipeline_name: str,
+        view_name: str,
+        callback: Callable[[pd.DataFrame, int], None],
+        exception_callback: Callable[[BaseException], None],
+    ):
+        super().__init__()
+        self.daemon = True
+        self.client: FelderaClient = client
+        self.pipeline_name: str = pipeline_name
+        self.view_name: str = view_name
+        self.callback: Callable[[pd.DataFrame, int], None] = callback
+        self.exception_callback: Callable[[BaseException], None] = exception_callback
+        self.schema: Optional[dict] = None
+
+    def run(self):
+        """
+        The main loop of the thread. Listens for data and calls the callback function on each chunk of data received.
+
+        :meta private:
+        """
+
+        try:
+            pipeline = self.client.get_pipeline(
+                self.pipeline_name, PipelineFieldSelector.ALL
+            )
+
+            schemas = pipeline.tables + pipeline.views
+            for schema in schemas:
+                if schema.name == self.view_name:
+                    self.schema = schema
+                    break
+
+            if self.schema is None:
+                raise ValueError(
+                    f"Table or View {self.view_name} not found in the pipeline schema."
+                )
+
+            gen_obj = self.client.listen_to_pipeline(
+                self.pipeline_name,
+                self.view_name,
+                format="json",
+                case_sensitive=self.schema.case_sensitive,
+            )
+
+            iterator = gen_obj()
+
+            for chunk in iterator:
+                chunk: dict = chunk
+                data: Optional[list[dict]] = chunk.get("json_data")
+                seq_no: Optional[int] = chunk.get("sequence_number")
+                if data is not None and seq_no is not None:
+                    self.callback(
+                        dataframe_from_response([data], self.schema.fields), seq_no
+                    )
+        except BaseException as e:
+            self.exception_callback(e)

{feldera-0.162.0 → feldera-0.164.0}/feldera/enums.py

@@ -122,12 +122,13 @@ class DeploymentRuntimeStatus(Enum):
 
     UNAVAILABLE = 0
     STANDBY = 1
-    INITIALIZING = 2
-    BOOTSTRAPPING = 3
-    REPLAYING = 4
-    PAUSED = 5
-    RUNNING = 6
-    SUSPENDED = 7
+    AWAITINGAPPROVAL = 2
+    INITIALIZING = 3
+    BOOTSTRAPPING = 4
+    REPLAYING = 5
+    PAUSED = 6
+    RUNNING = 7
+    SUSPENDED = 8
 
     @staticmethod
     def from_str(value):
@@ -149,13 +150,14 @@ class PipelineStatus(Enum):
     PROVISIONING = 2
     UNAVAILABLE = 3
     STANDBY = 4
-    INITIALIZING = 5
-    BOOTSTRAPPING = 6
-    REPLAYING = 7
-    PAUSED = 8
-    RUNNING = 9
-    SUSPENDED = 10
-    STOPPING = 11
+    AWAITINGAPPROVAL = 5
+    INITIALIZING = 6
+    BOOTSTRAPPING = 7
+    REPLAYING = 8
+    PAUSED = 9
+    RUNNING = 10
+    SUSPENDED = 11
+    STOPPING = 12
 
     @staticmethod
     def from_str(value):
@@ -344,3 +346,9 @@ class PipelineFieldSelector(Enum):
 
     STATUS = "status"
     """Select only the fields required to know the status of a pipeline."""
+
+
+class BootstrapPolicy(Enum):
+    AWAIT_APPROVAL = "await_approval"
+    ALLOW = "allow"
+    REJECT = "reject"

{feldera-0.162.0 → feldera-0.164.0}/feldera/output_handler.py

@@ -1,5 +1,7 @@
 import pandas as pd
 
+from typing import Optional
+
 from feldera import FelderaClient
 from feldera._callback_runner import CallbackRunner
 
@@ -20,15 +22,23 @@ class OutputHandler:
         self.pipeline_name: str = pipeline_name
         self.view_name: str = view_name
         self.buffer: list[pd.DataFrame] = []
+        self.exception: Optional[BaseException] = None
 
         # the callback that is passed to the `CallbackRunner`
         def callback(df: pd.DataFrame, _: int):
             if not df.empty:
                 self.buffer.append(df)
 
+        def exception_callback(exception: BaseException):
+            self.exception = exception
+
         # sets up the callback runner
         self.handler = CallbackRunner(
-            self.client, self.pipeline_name, self.view_name, callback
+            self.client,
+            self.pipeline_name,
+            self.view_name,
+            callback,
+            exception_callback,
         )
 
     def start(self):
@@ -45,6 +55,8 @@ class OutputHandler:
         :param clear_buffer: Whether to clear the buffer after getting the output.
         """
 
+        if self.exception is not None:
+            raise self.exception
         if len(self.buffer) == 0:
             return pd.DataFrame()
         res = pd.concat(self.buffer, ignore_index=True)

{feldera-0.162.0 → feldera-0.164.0}/feldera/pipeline.py

@@ -11,6 +11,7 @@ from collections import deque
 
 from feldera.rest.errors import FelderaAPIError
 from feldera.enums import (
+    BootstrapPolicy,
     PipelineFieldSelector,
     PipelineStatus,
     ProgramStatus,
@@ -71,6 +72,30 @@ class Pipeline:
             else:
                 raise err
 
+    def wait_for_status(
+        self, expected_status: PipelineStatus, timeout: Optional[int] = None
+    ) -> None:
+        """
+        Wait for the pipeline to reach the specified status.
+
+        :param expected_status: The status to wait for
+        :param timeout: Maximum time to wait in seconds. If None, waits forever (default: None)
+        :raises TimeoutError: If the expected status is not reached within the timeout
+        """
+        start_time = time.time()
+
+        while True:
+            current_status = self.status()
+            if current_status == expected_status:
+                return
+
+            if timeout is not None and time.time() - start_time >= timeout:
+                raise TimeoutError(
+                    f"Pipeline did not reach {expected_status.name} status within {timeout} seconds"
+                )
+
+            time.sleep(1)
+
     def stats(self) -> PipelineStatistics:
         """Gets the pipeline metrics and performance counters."""
 
@@ -269,11 +294,13 @@ class Pipeline:
         if self.status() not in [PipelineStatus.RUNNING, PipelineStatus.PAUSED]:
             raise RuntimeError("Pipeline must be running or paused to listen to output")
 
-        handler = CallbackRunner(self.client, self.name, view_name, callback)
+        handler = CallbackRunner(
+            self.client, self.name, view_name, callback, lambda exception: None
+        )
         handler.start()
 
     def wait_for_completion(
-        self, force_stop: bool = False, timeout_s: Optional[float] = None
+        self, force_stop: bool = False, timeout_s: float | None = None
     ):
         """
         Block until the pipeline has completed processing all input records.
@@ -301,6 +328,7 @@ class Pipeline:
             PipelineStatus.RUNNING,
             PipelineStatus.INITIALIZING,
             PipelineStatus.PROVISIONING,
+            PipelineStatus.BOOTSTRAPPING,
         ]:
             raise RuntimeError("Pipeline must be running to wait for completion")
 
@@ -345,25 +373,10 @@ class Pipeline:
 
         return self.stats().global_metrics.pipeline_complete
 
-    def restart(self, timeout_s: Optional[float] = None):
-        """
-        Restarts the pipeline.
-
-        This method forcibly **STOPS** the pipeline regardless of its current
-        state and then starts it again. No checkpoints are made when stopping
-        the pipeline.
-
-        :param timeout_s: The maximum time (in seconds) to wait for the
-            pipeline to restart.
-        """
-
-        self.stop(force=True, timeout_s=timeout_s)
-        self.start(timeout_s=timeout_s)
-
     def wait_for_idle(
         self,
         idle_interval_s: float = 5.0,
-        timeout_s: float = 600.0,
+        timeout_s: float | None = None,
         poll_interval_s: float = 0.2,
     ):
         """
@@ -383,12 +396,12 @@ class Pipeline:
         :raises RuntimeError: If the metrics are missing or the timeout was
             reached.
         """
-        if idle_interval_s > timeout_s:
+        if timeout_s is not None and idle_interval_s > timeout_s:
             raise ValueError(
                 f"idle interval ({idle_interval_s}s) cannot be larger than"
                 f" timeout ({timeout_s}s)"
             )
-        if poll_interval_s > timeout_s:
+        if timeout_s is not None and poll_interval_s > timeout_s:
             raise ValueError(
                 f"poll interval ({poll_interval_s}s) cannot be larger than"
                 f" timeout ({timeout_s}s)"
@@ -434,11 +447,13 @@ metrics"""
                 return
 
             # Timeout
-            if now_s - start_time_s >= timeout_s:
+            if timeout_s is not None and now_s - start_time_s >= timeout_s:
                 raise RuntimeError(f"waiting for idle reached timeout ({timeout_s}s)")
             time.sleep(poll_interval_s)
 
-    def activate(self, wait: bool = True, timeout_s: Optional[float] = None):
+    def activate(
+        self, wait: bool = True, timeout_s: Optional[float] = None
+    ) -> Optional[PipelineStatus]:
         """
         Activates the pipeline when starting from STANDBY mode. Only applicable
         when the pipeline is starting from a checkpoint in object store.
@@ -449,9 +464,14 @@ metrics"""
             pipeline to pause.
         """
 
-        self.client.activate_pipeline(self.name, wait=wait, timeout_s=timeout_s)
+        return self.client.activate_pipeline(self.name, wait=wait, timeout_s=timeout_s)
 
-    def start(self, wait: bool = True, timeout_s: Optional[float] = None):
+    def start(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
         """
         .. _start:
 
@@ -461,6 +481,7 @@ metrics"""
         - If the pipeline is in any other state, an error will be raised.
         - If the pipeline is in PAUSED state, use `.meth:resume` instead.
 
+        :param bootstrap_policy: The bootstrap policy to use.
         :param timeout_s: The maximum time (in seconds) to wait for the
             pipeline to start.
         :param wait: Set True to wait for the pipeline to start. True by default
@@ -468,21 +489,57 @@ metrics"""
         :raises RuntimeError: If the pipeline is not in STOPPED state.
         """
 
-        self.client.start_pipeline(self.name, wait=wait, timeout_s=timeout_s)
+        self.client.start_pipeline(
+            self.name, bootstrap_policy=bootstrap_policy, wait=wait, timeout_s=timeout_s
+        )
 
-    def start_paused(self, wait: bool = True, timeout_s: Optional[float] = None):
+    def start_paused(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
         """
         Starts the pipeline in the paused state.
         """
 
-        self.client.start_pipeline_as_paused(self.name, wait=wait, timeout_s=timeout_s)
+        return self.client.start_pipeline_as_paused(
+            self.name, bootstrap_policy=bootstrap_policy, wait=wait, timeout_s=timeout_s
+        )
 
-    def start_standby(self, wait: bool = True, timeout_s: Optional[float] = None):
+    def start_standby(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
         """
         Starts the pipeline in the standby state.
         """
 
-        self.client.start_pipeline_as_standby(self.name, wait=wait, timeout_s=timeout_s)
+        self.client.start_pipeline_as_standby(
+            self.name, bootstrap_policy=bootstrap_policy, wait=wait, timeout_s=timeout_s
+        )
+
+    def restart(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Restarts the pipeline.
+
+        This method forcibly **STOPS** the pipeline regardless of its current
+        state and then starts it again. No checkpoints are made when stopping
+        the pipeline.
+
+        :param bootstrap_policy: The bootstrap policy to use.
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            pipeline to restart.
+        """
+
+        self.stop(force=True, timeout_s=timeout_s)
+        self.start(bootstrap_policy=bootstrap_policy, timeout_s=timeout_s)
 
     def pause(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
@@ -517,6 +574,18 @@ metrics"""
             self.name, force=force, wait=wait, timeout_s=timeout_s
         )
 
+    def approve(self):
+        """
+        Approves the pipeline to proceed with bootstrapping.
+
+        This method is used when a pipeline has been started with
+        `bootstrap_policy=BootstrapPolicy.AWAIT_APPROVAL` and is currently in the
+        AWAITINGAPPROVAL state. The pipeline will wait for explicit user approval
+        before proceeding with the bootstrapping process.
+        """
+
+        self.client.approve_pipeline(self.name)
+
     def resume(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         Resumes the pipeline from the PAUSED state. If the pipeline is already
@@ -627,7 +696,7 @@ metrics"""
                 err.message = f"Pipeline with name {name} not found"
                 raise err
 
-    def checkpoint(self, wait: bool = False, timeout_s=300) -> int:
+    def checkpoint(self, wait: bool = False, timeout_s: Optional[float] = None) -> int:
         """
         Checkpoints this pipeline.
 
@@ -649,7 +718,7 @@ metrics"""
 
         while True:
             elapsed = time.monotonic() - start
-            if elapsed > timeout_s:
+            if timeout_s is not None and elapsed > timeout_s:
                 raise TimeoutError(
                     f"""timeout ({timeout_s}s) reached while waiting for \
 pipeline '{self.name}' to make checkpoint '{seq}'"""
@@ -685,7 +754,9 @@ pipeline '{self.name}' to make checkpoint '{seq}'"""
         if seq < success:
             return CheckpointStatus.Unknown
 
-    def sync_checkpoint(self, wait: bool = False, timeout_s=300) -> str:
+    def sync_checkpoint(
+        self, wait: bool = False, timeout_s: Optional[float] = None
+    ) -> str:
         """
         Syncs this checkpoint to object store.
 
@@ -707,7 +778,7 @@ pipeline '{self.name}' to make checkpoint '{seq}'"""
 
         while True:
             elapsed = time.monotonic() - start
-            if elapsed > timeout_s:
+            if timeout_s is not None and elapsed > timeout_s:
                 raise TimeoutError(
                     f"""timeout ({timeout_s}s) reached while waiting for \
 pipeline '{self.name}' to sync checkpoint '{uuid}'"""
@@ -1177,6 +1248,14 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         self.refresh(PipelineFieldSelector.STATUS)
         return DeploymentRuntimeStatus.from_str(self._inner.deployment_runtime_status)
 
+    def deployment_runtime_status_details(self) -> Optional[dict]:
+        """
+        Return the deployment runtime status details.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return self._inner.deployment_runtime_status_details
+
     def deployment_error(self) -> Mapping[str, Any]:
         """
         Return the deployment error of the pipeline.

{feldera-0.162.0 → feldera-0.164.0}/feldera/rest/_helpers.py

@@ -20,20 +20,20 @@ def requests_verify_from_env() -> str | bool:
     if env_feldera_tls_insecure is not None and FELDERA_HTTPS_TLS_CERT is not None:
         logging.warning(
             "environment variables FELDERA_HTTPS_TLS_CERT and "
-            "FELDERA_TLS_INSECURE both are set."
-            "\nFELDERA_HTTPS_TLS_CERT takes priority."
+            + "FELDERA_TLS_INSECURE both are set."
+            + "\nFELDERA_HTTPS_TLS_CERT takes priority."
         )
 
     if env_feldera_tls_insecure is None:
-        FELDERA_TLS_INSECURE = False
+        feldera_tls_insecure = False
     else:
-        FELDERA_TLS_INSECURE = env_feldera_tls_insecure.strip().lower() in (
+        feldera_tls_insecure = env_feldera_tls_insecure.strip().lower() in (
             "1",
             "true",
             "yes",
         )
 
-    requests_verify = not FELDERA_TLS_INSECURE
+    requests_verify = not feldera_tls_insecure
     if FELDERA_HTTPS_TLS_CERT is not None:
         requests_verify = FELDERA_HTTPS_TLS_CERT
 

{feldera-0.162.0 → feldera-0.164.0}/feldera/rest/_httprequests.py

@@ -56,7 +56,7 @@ class HttpRequests:
         """
         self.headers["Content-Type"] = content_type
 
-        prev_resp: Optional[requests.Response] = None
+        prev_resp: requests.Response | None = None
 
         try:
             conn_timeout = self.config.connection_timeout
@@ -156,7 +156,7 @@ class HttpRequests:
         body: Optional[
             Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
         ] = None,
-        content_type: Optional[str] = "application/json",
+        content_type: str = "application/json",
         params: Optional[Mapping[str, Any]] = None,
         stream: bool = False,
         serialize: bool = True,
@@ -177,7 +177,7 @@ class HttpRequests:
         body: Optional[
             Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
         ] = None,
-        content_type: Optional[str] = "application/json",
+        content_type: str = "application/json",
         params: Optional[Mapping[str, Any]] = None,
     ) -> Any:
         return self.send_request(requests.patch, path, body, content_type, params)
@@ -188,7 +188,7 @@ class HttpRequests:
         body: Optional[
             Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
         ] = None,
-        content_type: Optional[str] = "application/json",
+        content_type: str = "application/json",
         params: Optional[Mapping[str, Any]] = None,
     ) -> Any:
         return self.send_request(requests.put, path, body, content_type, params)

{feldera-0.162.0 → feldera-0.164.0}/feldera/rest/config.py

@@ -39,7 +39,7 @@ class Config:
         )
         self.url: str = BASE_URL
         self.api_key: Optional[str] = os.environ.get("FELDERA_API_KEY", api_key)
-        self.version: Optional[str] = version or "v0"
+        self.version: str = version or "v0"
         self.timeout: Optional[float] = timeout
         self.connection_timeout: Optional[float] = connection_timeout
         env_verify = requests_verify_from_env()

{feldera-0.162.0 → feldera-0.164.0}/feldera/rest/feldera_client.py

@@ -6,8 +6,9 @@ import json
 from decimal import Decimal
 from typing import Generator, Mapping
 from urllib.parse import quote
+import requests
 
-from feldera.enums import PipelineFieldSelector
+from feldera.enums import BootstrapPolicy, PipelineFieldSelector, PipelineStatus
 from feldera.rest.config import Config
 from feldera.rest.feldera_config import FelderaConfig
 from feldera.rest.errors import FelderaTimeoutError, FelderaAPIError
@@ -110,16 +111,20 @@ class FelderaClient:
 
         return Pipeline.from_dict(resp)
 
-    def get_runtime_config(self, pipeline_name) -> dict:
+    def get_runtime_config(self, pipeline_name) -> Mapping[str, Any]:
         """
         Get the runtime config of a pipeline by name
 
         :param pipeline_name: The name of the pipeline
         """
 
-        resp: dict = self.http.get(f"/pipelines/{pipeline_name}")
+        resp: Mapping[str, Any] = self.http.get(f"/pipelines/{pipeline_name}")
 
-        return resp.get("runtime_config")
+        runtime_config: Mapping[str, Any] | None = resp.get("runtime_config")
+        if runtime_config is None:
+            raise ValueError(f"Pipeline {pipeline_name} has no runtime config")
+
+        return runtime_config
 
     def pipelines(self) -> list[Pipeline]:
         """
@@ -175,7 +180,7 @@ class FelderaClient:
         self,
         pipeline_name: str,
         state: str,
-        timeout_s: float = 300.0,
+        timeout_s: Optional[float] = None,
         start: bool = True,
     ):
         start_time = time.monotonic()
@@ -211,6 +216,46 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             )
             time.sleep(0.1)
 
+    def __wait_for_pipeline_state_one_of(
+        self,
+        pipeline_name: str,
+        states: list[str],
+        timeout_s: float | None = None,
+        start: bool = True,
+    ) -> PipelineStatus:
+        start_time = time.monotonic()
+        states = [state.lower() for state in states]
+
+        while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError(
+                        f"Timed out waiting for pipeline {pipeline_name} to"
+                        f"transition to one of the states: {states}"
+                    )
+
+            resp = self.get_pipeline(pipeline_name, PipelineFieldSelector.STATUS)
+            status = resp.deployment_status
+
+            if status.lower() in states:
+                return PipelineStatus.from_str(status)
+            elif (
+                status == "Stopped"
+                and len(resp.deployment_error or {}) > 0
+                and resp.deployment_desired_status == "Stopped"
+            ):
+                err_msg = "Unable to START the pipeline:\n" if start else ""
+                raise RuntimeError(
+                    f"""{err_msg}Unable to transition the pipeline to one of the states: {states}.
+Reason: The pipeline is in a STOPPED state due to the following error:
+{resp.deployment_error.get("message", "")}"""
+                )
+            logging.debug(
+                "still starting %s, waiting for 100 more milliseconds", pipeline_name
+            )
+            time.sleep(0.1)
+
     def create_pipeline(self, pipeline: Pipeline) -> Pipeline:
         """
         Create a pipeline if it doesn't exist and wait for it to compile
@@ -342,36 +387,36 @@ Reason: The pipeline is in a STOPPED state due to the following error:
                     yield chunk.decode("utf-8")
 
     def activate_pipeline(
-        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
-    ):
+        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = None
+    ) -> Optional[PipelineStatus]:
         """
 
         :param pipeline_name: The name of the pipeline to activate
         :param wait: Set True to wait for the pipeline to activate. True by
             default
         :param timeout_s: The amount of time in seconds to wait for the
-            pipeline to activate. 300 seconds by default.
+            pipeline to activate.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
-
         self.http.post(
             path=f"/pipelines/{pipeline_name}/activate",
         )
 
         if not wait:
-            return
+            return None
 
-        self.__wait_for_pipeline_state(pipeline_name, "running", timeout_s)
+        return self.__wait_for_pipeline_state_one_of(
+            pipeline_name, ["running", "AwaitingApproval"], timeout_s
+        )
 
     def _inner_start_pipeline(
         self,
         pipeline_name: str,
         initial: str = "running",
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
         wait: bool = True,
-        timeout_s: Optional[float] = 300,
-    ):
+        timeout_s: Optional[float] = None,
+    ) -> Optional[PipelineStatus]:
         """
 
         :param pipeline_name: The name of the pipeline to start
@@ -379,67 +424,88 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             by default.
         :param wait: Set True to wait for the pipeline to start. True by default
         :param timeout_s: The amount of time in seconds to wait for the
-            pipeline to start. 300 seconds by default.
+            pipeline to start.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
+        params = {"initial": initial}
+        if bootstrap_policy is not None:
+            params["bootstrap_policy"] = bootstrap_policy.value
 
         self.http.post(
             path=f"/pipelines/{pipeline_name}/start",
-            params={"initial": initial},
+            params=params,
         )
 
         if not wait:
-            return
+            return None
 
-        self.__wait_for_pipeline_state(pipeline_name, initial, timeout_s)
+        return self.__wait_for_pipeline_state_one_of(
+            pipeline_name, [initial, "AwaitingApproval"], timeout_s
+        )
 
     def start_pipeline(
-        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
-    ):
+        self,
+        pipeline_name: str,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ) -> Optional[PipelineStatus]:
         """
 
         :param pipeline_name: The name of the pipeline to start
         :param wait: Set True to wait for the pipeline to start.
             True by default
         :param timeout_s: The amount of time in seconds to wait for the
-            pipeline to start. 300 seconds by default.
+            pipeline to start.
         """
 
-        self._inner_start_pipeline(pipeline_name, "running", wait, timeout_s)
+        return self._inner_start_pipeline(
+            pipeline_name, "running", bootstrap_policy, wait, timeout_s
+        )
 
     def start_pipeline_as_paused(
-        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
-    ):
+        self,
+        pipeline_name: str,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: float | None = None,
+    ) -> Optional[PipelineStatus]:
         """
         :param pipeline_name: The name of the pipeline to start as paused.
         :param wait: Set True to wait for the pipeline to start as pause.
             True by default
         :param timeout_s: The amount of time in seconds to wait for the
-            pipeline to start. 300 seconds by default.
+            pipeline to start.
         """
 
-        self._inner_start_pipeline(pipeline_name, "paused", wait, timeout_s)
+        return self._inner_start_pipeline(
+            pipeline_name, "paused", bootstrap_policy, wait, timeout_s
+        )
 
     def start_pipeline_as_standby(
-        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
+        self,
+        pipeline_name: str,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
     ):
         """
         :param pipeline_name: The name of the pipeline to start as standby.
         :param wait: Set True to wait for the pipeline to start as standby.
             True by default
         :param timeout_s: The amount of time in seconds to wait for the
-            pipeline to start. 300 seconds by default.
+            pipeline to start.
         """
 
-        self._inner_start_pipeline(pipeline_name, "standby", wait, timeout_s)
+        self._inner_start_pipeline(
+            pipeline_name, "standby", bootstrap_policy, wait, timeout_s
+        )
 
     def resume_pipeline(
         self,
         pipeline_name: str,
         wait: bool = True,
-        timeout_s: Optional[float] = 300,
+        timeout_s: Optional[float] = None,
     ):
         """
         Resume a pipeline
@@ -447,12 +513,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         :param pipeline_name: The name of the pipeline to stop
         :param wait: Set True to wait for the pipeline to pause. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
-            to pause. 300 seconds by default.
+            to pause.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
-
         self.http.post(
             path=f"/pipelines/{pipeline_name}/resume",
         )
@@ -466,7 +529,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         self,
         pipeline_name: str,
         wait: bool = True,
-        timeout_s: Optional[float] = 300,
+        timeout_s: Optional[float] = None,
     ):
         """
         Pause a pipeline
@@ -476,12 +539,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
              STOPPED state due to a failure.
         :param wait: Set True to wait for the pipeline to pause. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
-            to pause. 300 seconds by default.
+            to pause.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
-
         self.http.post(
             path=f"/pipelines/{pipeline_name}/pause",
         )
@@ -491,12 +551,20 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
         self.__wait_for_pipeline_state(pipeline_name, "paused", timeout_s)
 
+    def approve_pipeline(
+        self,
+        pipeline_name: str,
+    ):
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/approve",
+        )
+
     def stop_pipeline(
         self,
         pipeline_name: str,
         force: bool,
         wait: bool = True,
-        timeout_s: Optional[float] = 300,
+        timeout_s: Optional[float] = None,
     ):
         """
         Stop a pipeline
@@ -506,12 +574,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             Set False to automatically checkpoint before stopping.
         :param wait: Set True to wait for the pipeline to stop. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
-            to stop. Default is 300 seconds.
+            to stop.
         """
 
-        if timeout_s is None:
-            timeout_s = 300
-
         params = {"force": str(force).lower()}
 
         self.http.post(
@@ -524,7 +589,12 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
         start = time.monotonic()
 
-        while time.monotonic() - start < timeout_s:
+        while True:
+            if timeout_s is not None and time.monotonic() - start > timeout_s:
+                raise FelderaTimeoutError(
+                    f"timeout error: pipeline '{pipeline_name}' did not stop in {timeout_s} seconds"
+                )
+
             status = self.get_pipeline(
                 pipeline_name, PipelineFieldSelector.STATUS
             ).deployment_status
@@ -538,29 +608,26 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             )
             time.sleep(0.1)
 
-        raise FelderaTimeoutError(
-            f"timeout error: pipeline '{pipeline_name}' did not stop in {timeout_s} seconds"
-        )
-
-    def clear_storage(self, pipeline_name: str, timeout_s: Optional[float] = 300):
+    def clear_storage(self, pipeline_name: str, timeout_s: Optional[float] = None):
         """
         Clears the storage from the pipeline.
         This operation cannot be canceled.
 
         :param pipeline_name: The name of the pipeline
         :param timeout_s: The amount of time in seconds to wait for the storage
-            to clear. Default is 300 seconds.
+            to clear.
         """
-        if timeout_s is None:
-            timeout_s = 300
-
         self.http.post(
             path=f"/pipelines/{pipeline_name}/clear",
         )
 
         start = time.monotonic()
 
-        while time.monotonic() - start < timeout_s:
+        while True:
+            if timeout_s is not None and time.monotonic() - start > timeout_s:
+                raise FelderaTimeoutError(
+                    f"timeout error: pipeline '{pipeline_name}' did not clear storage in {timeout_s} seconds"
+                )
             status = self.get_pipeline(
                 pipeline_name, PipelineFieldSelector.STATUS
             ).storage_status
@@ -574,10 +641,6 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             )
             time.sleep(0.1)
 
-        raise FelderaTimeoutError(
-            f"timeout error: pipeline '{pipeline_name}' did not clear storage in {timeout_s} seconds"
-        )
-
     def start_transaction(self, pipeline_name: str) -> int:
         """
         Start a new transaction.
@@ -773,15 +836,15 @@ Reason: The pipeline is in a STOPPED state due to the following error:
                     _validate_no_none_keys_in_map(datum.get("insert", {}))
                     _validate_no_none_keys_in_map(datum.get("delete", {}))
             else:
-                data: dict = data
+                data: Mapping[str, Any] = data
                 _validate_no_none_keys_in_map(data.get("insert", {}))
                 _validate_no_none_keys_in_map(data.get("delete", {}))
         else:
             _validate_no_none_keys_in_map(data)
 
         # python sends `True` which isn't accepted by the backend
-        array = _prepare_boolean_input(array)
-        force = _prepare_boolean_input(force)
+        array: str = _prepare_boolean_input(array)
+        force: str = _prepare_boolean_input(force)
 
         params = {
             "force": force,
@@ -821,7 +884,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         return token
 
     def wait_for_token(
-        self, pipeline_name: str, token: str, timeout_s: Optional[float] = 600
+        self, pipeline_name: str, token: str, timeout_s: Optional[float] = None
     ):
         """
         Blocks until all records represented by this completion token have
@@ -830,7 +893,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         :param pipeline_name: The name of the pipeline
         :param token: The token to check for completion
         :param timeout_s: The amount of time in seconds to wait for the pipeline
-            to process these records. Default 600s
+            to process these records.
         """
 
         params = {
@@ -849,8 +912,8 @@ Reason: The pipeline is in a STOPPED state due to the following error:
                 if time.monotonic() > end:
                     raise FelderaTimeoutError(
                         f"timeout error: pipeline '{pipeline_name}' did not"
-                        f" process records represented by token {token} within"
-                        f" {timeout_s}"
+                        + f" process records represented by token {token} within"
+                        + f" {timeout_s}"
                     )
 
             resp = self.http.get(
@@ -914,7 +977,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
         table_name = f'"{table_name}"' if case_sensitive else table_name
 
-        resp = self.http.post(
+        resp: requests.Response = self.http.post(
             path=f"/pipelines/{quote(pipeline_name, safe='')}/egress/{quote(table_name, safe='')}",
             params=params,
             stream=True,
@@ -1019,7 +1082,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
     def query_as_json(
         self, pipeline_name: str, query: str
-    ) -> Generator[dict, None, None]:
+    ) -> Generator[Mapping[str, Any], None, None]:
         """
         Executes an ad-hoc query on the specified pipeline and returns the result as a generator that yields
         rows of the query as Python dictionaries.

{feldera-0.162.0 → feldera-0.164.0}/feldera/rest/pipeline.py

@@ -69,6 +69,7 @@ class Pipeline:
         self.deployment_runtime_desired_status: Optional[str] = None
         self.deployment_runtime_desired_status_since: Optional[str] = None
         self.deployment_runtime_status: Optional[str] = None
+        self.deployment_runtime_status_details: Optional[dict] = None
         self.deployment_runtime_status_since: Optional[str] = None
 
     @classmethod

{feldera-0.162.0 → feldera-0.164.0}/feldera.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.162.0
+Version: 0.164.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.162.0 → feldera-0.164.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "feldera"
 readme = "README.md"
 description = "The feldera python client"
-version = "0.162.0"
+version = "0.164.0"
 license = { text = "MIT" }
 requires-python = ">=3.10"
 authors = [

feldera-0.162.0/feldera/_callback_runner.py

@@ -1,64 +0,0 @@
-from threading import Thread
-from typing import Callable, Optional
-
-import pandas as pd
-from feldera import FelderaClient
-from feldera._helpers import dataframe_from_response
-from feldera.enums import PipelineFieldSelector
-
-
-class CallbackRunner(Thread):
-    def __init__(
-        self,
-        client: FelderaClient,
-        pipeline_name: str,
-        view_name: str,
-        callback: Callable[[pd.DataFrame, int], None],
-    ):
-        super().__init__()
-        self.daemon = True
-        self.client: FelderaClient = client
-        self.pipeline_name: str = pipeline_name
-        self.view_name: str = view_name
-        self.callback: Callable[[pd.DataFrame, int], None] = callback
-        self.schema: Optional[dict] = None
-
-    def run(self):
-        """
-        The main loop of the thread. Listens for data and calls the callback function on each chunk of data received.
-
-        :meta private:
-        """
-
-        pipeline = self.client.get_pipeline(
-            self.pipeline_name, PipelineFieldSelector.ALL
-        )
-
-        schemas = pipeline.tables + pipeline.views
-        for schema in schemas:
-            if schema.name == self.view_name:
-                self.schema = schema
-                break
-
-        if self.schema is None:
-            raise ValueError(
-                f"Table or View {self.view_name} not found in the pipeline schema."
-            )
-
-        gen_obj = self.client.listen_to_pipeline(
-            self.pipeline_name,
-            self.view_name,
-            format="json",
-            case_sensitive=self.schema.case_sensitive,
-        )
-
-        iterator = gen_obj()
-
-        for chunk in iterator:
-            chunk: dict = chunk
-            data: Optional[list[dict]] = chunk.get("json_data")
-            seq_no: Optional[int] = chunk.get("sequence_number")
-            if data is not None and seq_no is not None:
-                self.callback(
-                    dataframe_from_response([data], self.schema.fields), seq_no
-                )