feldera 0.146.0__tar.gz → 0.148.0__tar.gz

{feldera-0.146.0 → feldera-0.148.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.146.0
+Version: 0.148.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.146.0 → feldera-0.148.0}/feldera/enums.py

@@ -34,131 +34,129 @@ class BuildMode(Enum):
     GET_OR_CREATE = 3
 
 
-class PipelineStatus(Enum):
+class DeploymentDesiredStatus(Enum):
+    """
+    Deployment desired status of the pipeline.
     """
-    Represents the state that this pipeline is currently in.
 
-    .. code-block:: text
+    STOPPED = 0
+    UNAVAILABLE = 1
+    STANDBY = 2
+    PAUSED = 3
+    RUNNING = 4
+    SUSPENDED = 5
 
-                    Stopped ◄─────────── Stopping ◄───── All states can transition
-                       │                    ▲            to Stopping by either:
-      /start or /pause │                    │            (1) user calling /stop?force=true, or;
-                       ▼                    │            (2) pipeline encountering a fatal
-                ⌛Provisioning          Suspending            resource or runtime error,
-                       │                    ▲                having the system call /stop?force=true
-                       ▼                    │ /stop          effectively
-                ⌛Initializing ─────────────┤  ?force=false
-                       │                    │
-             ┌─────────┼────────────────────┴─────┐
-             │         ▼                          │
-             │       Paused  ◄──────► Unavailable │
-             │        │   ▲                ▲      │
-             │ /start │   │  /pause        │      │
-             │        ▼   │                │      │
-             │       Running ◄─────────────┘      │
-             └────────────────────────────────────┘
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentDesiredStatus.__name__}"
+        )
 
-    """
 
-    NOT_FOUND = 0
+class DeploymentResourcesDesiredStatus(Enum):
     """
-    The pipeline has not been created yet.
+    The desired status of deployment resources of the pipeline.
     """
 
-    STOPPED = 1
-    """
-    The pipeline has not (yet) been started or has been stopped either
-    manually by the user or automatically by the system due to a
-    resource or runtime error.
+    STOPPED = 0
+    PROVISIONED = 1
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentResourcesDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentResourcesDesiredStatus.__name__}"
+        )
 
-        1. The user starts it via `/start` or `/pause`, transitioning to `PROVISIONING`.
-        2. Early start fails (e.g., compilation failure), transitioning to `STOPPING`.
-    """
 
-    PROVISIONING = 2
+class DeploymentResourcesStatus(Enum):
     """
-    Compute (and optionally storage) resources needed for running the pipeline
-    are being provisioned.
-
-    The pipeline remains in this state until:
-
-        1. Resources are provisioned successfully, transitioning to `INITIALIZING`.
-        2. Provisioning fails or times out, transitioning to `STOPPING`.
-        3. The user cancels the pipeline via `/stop`, transitioning to `STOPPING`.
+    The desired status of deployment resources of the pipeline.
     """
 
-    INITIALIZING = 3
-    """
-    The pipeline is initializing its internal state and connectors.
+    STOPPED = 0
+    PROVISIONING = 1
+    PROVISIONED = 2
+    STOPPING = 3
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentResourcesStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentResourcesStatus.__name__}"
+        )
 
-        1. Initialization succeeds, transitioning to `PAUSED`.
-        2. Initialization fails or times out, transitioning to `STOPPING`.
-        3. The user suspends the pipeline via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops the pipeline via `/stop`, transitioning to `STOPPING`.
-    """
 
-    PAUSED = 4
+class DeploymentRuntimeDesiredStatus(Enum):
     """
-    The pipeline is initialized but data processing is paused.
-
-    The pipeline remains in this state until:
-
-        1. The user starts it via `/start`, transitioning to `RUNNING`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-        3. The user suspends it via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops it via `/stop`, transitioning to `STOPPING`.
+    Deployment runtime desired status of the pipeline.
     """
 
-    RUNNING = 5
-    """
-    The pipeline is processing data.
+    UNAVAILABLE = 0
+    STANDBY = 1
+    PAUSED = 2
+    RUNNING = 3
+    SUSPENDED = 4
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentRuntimeDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentRuntimeDesiredStatus.__name__}"
+        )
 
-        1. The user pauses it via `/pause`, transitioning to `PAUSED`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-        3. The user suspends it via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops it via `/stop`, transitioning to `STOPPING`.
-    """
 
-    UNAVAILABLE = 6
+class DeploymentRuntimeStatus(Enum):
     """
-    The pipeline was initialized at least once but is currently unreachable
-    or not ready.
-
-    The pipeline remains in this state until:
-
-        1. A successful status check transitions it back to `PAUSED` or `RUNNING`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-        3. The user suspends it via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops it via `/stop`, transitioning to `STOPPING`.
-
-    Note: While in this state, `/start` or `/pause` express desired state but
-    are only applied once the pipeline becomes reachable.
+    Deployment runtime status of the pipeline.
     """
 
-    SUSPENDING = 7
-    """
-    The pipeline is being suspended to storage.
+    UNAVAILABLE = 0
+    STANDBY = 1
+    INITIALIZING = 2
+    BOOTSTRAPPING = 3
+    REPLAYING = 4
+    PAUSED = 5
+    RUNNING = 6
+    SUSPENDED = 7
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentRuntimeStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentRuntimeStatus.__name__}"
+        )
 
-        1. Suspension succeeds, transitioning to `STOPPING`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-    """
 
-    STOPPING = 8
+class PipelineStatus(Enum):
     """
-    The pipeline's compute resources are being scaled down to zero.
-
-    The pipeline remains in this state until deallocation completes,
-    transitioning to `STOPPED`.
+    Represents the state that this pipeline is currently in.
     """
 
+    NOT_FOUND = 0
+    STOPPED = 1
+    PROVISIONING = 2
+    UNAVAILABLE = 3
+    STANDBY = 4
+    INITIALIZING = 5
+    BOOTSTRAPPING = 6
+    REPLAYING = 7
+    PAUSED = 8
+    RUNNING = 9
+    SUSPENDED = 10
+    STOPPING = 11
+
     @staticmethod
     def from_str(value):
         for member in PipelineStatus:

{feldera-0.146.0 → feldera-0.148.0}/feldera/pipeline.py

@@ -16,8 +16,13 @@ from feldera.enums import (
     ProgramStatus,
     CheckpointStatus,
     TransactionStatus,
+    StorageStatus,
+    DeploymentDesiredStatus,
+    DeploymentResourcesDesiredStatus,
+    DeploymentResourcesStatus,
+    DeploymentRuntimeDesiredStatus,
+    DeploymentRuntimeStatus,
 )
-from feldera.enums import StorageStatus
 from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.rest.feldera_client import FelderaClient
 from feldera._callback_runner import _CallbackRunnerInstruction, CallbackRunner
@@ -393,9 +398,7 @@ method or use `Pipeline.resume()` to resume a paused pipeline."""
 
             return
 
-        self.client.pause_pipeline(
-            self.name, "Unable to START the pipeline.\n", wait=wait, timeout_s=timeout_s
-        )
+        self.client.start_pipeline_as_paused(self.name, wait=wait, timeout_s=timeout_s)
         self.__setup_output_listeners()
         self.resume(timeout_s=timeout_s)
 
@@ -505,6 +508,20 @@ metrics"""
 
         self.client.activate_pipeline(self.name, wait=wait, timeout_s=timeout_s)
 
+    def start_paused(self, wait: bool = True, timeout_s: Optional[float] = None):
+        """
+        Starts the pipeline in the paused state.
+        """
+
+        self.client.start_pipeline_as_paused(self.name, wait=wait, timeout_s=timeout_s)
+
+    def start_standby(self, wait: bool = True, timeout_s: Optional[float] = None):
+        """
+        Starts the pipeline in the standby state.
+        """
+
+        self.client.start_pipeline_as_standby(self.name, wait=wait, timeout_s=timeout_s)
+
     def pause(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         Pause the pipeline.
@@ -562,7 +579,7 @@ metrics"""
             pipeline to resume.
         """
 
-        self.client.start_pipeline(self.name, wait=wait, timeout_s=timeout_s)
+        self.client.resume_pipeline(self.name, wait=wait, timeout_s=timeout_s)
 
     def start_transaction(self) -> int:
         """
@@ -1116,14 +1133,52 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         self.refresh()
         return self._inner.deployment_config
 
-    def deployment_desired_status(self) -> PipelineStatus:
+    def deployment_desired_status(self) -> DeploymentDesiredStatus:
         """
         Return the desired deployment status of the pipeline.
         This is the next state that the pipeline should transition to.
         """
 
         self.refresh()
-        return PipelineStatus.from_str(self._inner.deployment_desired_status)
+        return DeploymentDesiredStatus.from_str(self._inner.deployment_desired_status)
+
+    def deployment_resources_desired_status(self) -> DeploymentResourcesDesiredStatus:
+        """
+        Return the desired status of the the deployment resources.
+        """
+
+        self.refresh()
+        return DeploymentResourcesDesiredStatus.from_str(
+            self._inner.deployment_resources_desired_status
+        )
+
+    def deployment_resources_status(self) -> DeploymentResourcesStatus:
+        """
+        Return the status of the deployment resources.
+        """
+
+        self.refresh()
+        return DeploymentResourcesStatus.from_str(
+            self._inner.deployment_resources_status
+        )
+
+    def deployment_runtime_desired_status(self) -> DeploymentRuntimeDesiredStatus:
+        """
+        Return the deployment runtime desired status.
+        """
+
+        self.refresh()
+        return DeploymentRuntimeDesiredStatus.from_str(
+            self._inner.deployment_runtime_desired_status
+        )
+
+    def deployment_runtime_status(self) -> DeploymentRuntimeStatus:
+        """
+        Return the deployment runtime status.
+        """
+
+        self.refresh()
+        return DeploymentRuntimeStatus.from_str(self._inner.deployment_runtime_status)
 
     def deployment_error(self) -> Mapping[str, Any]:
         """

{feldera-0.146.0 → feldera-0.148.0}/feldera/rest/_httprequests.py

@@ -206,6 +206,11 @@ class HttpRequests:
         try:
             request.raise_for_status()
 
+            if request is None:
+                # This shouldn't ever be the case, but we've seen it happen
+                return FelderaCommunicationError(
+                    "Failed to Communicate with Feldera Received None as Response",
+                )
             if stream:
                 return request
             if request.headers.get("content-type") == "text/plain":

{feldera-0.146.0 → feldera-0.148.0}/feldera/rest/feldera_client.py

@@ -163,6 +163,46 @@ class FelderaClient:
             logging.debug("still compiling %s, waiting for 100 more milliseconds", name)
             time.sleep(0.1)
 
+    def __wait_for_pipeline_state(
+        self,
+        pipeline_name: str,
+        state: str,
+        timeout_s: float = 300.0,
+        start: bool = True,
+    ):
+        start_time = time.monotonic()
+
+        while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError(
+                        f"Timed out waiting for pipeline {pipeline_name} to"
+                        f"transition to '{state}' state"
+                    )
+
+            resp = self.get_pipeline(pipeline_name)
+            status = resp.deployment_status
+
+            if status.lower() == state.lower():
+                break
+            elif (
+                status == "Stopped"
+                and len(resp.deployment_error or {}) > 0
+                and resp.deployment_desired_status == "Stopped"
+            ):
+                err_msg = "Unable to START the pipeline:\n" if start else ""
+                raise RuntimeError(
+                    f"""{err_msg}Unable to transition the pipeline to '{state}'.
+Reason: The pipeline is in a STOPPED state due to the following error:
+{resp.deployment_error.get("message", "")}"""
+                )
+
+            logging.debug(
+                "still starting %s, waiting for 100 more milliseconds", pipeline_name
+            )
+            time.sleep(0.1)
+
     def create_pipeline(self, pipeline: Pipeline) -> Pipeline:
         """
         Create a pipeline if it doesn't exist and wait for it to compile
@@ -284,9 +324,10 @@ class FelderaClient:
         """
 
         :param pipeline_name: The name of the pipeline to activate
-        :param wait: Set True to wait for the pipeline to activate. True by default
-        :param timeout_s: The amount of time in seconds to wait for the pipeline
-            to activate. 300 seconds by default.
+        :param wait: Set True to wait for the pipeline to activate. True by
+            default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to activate. 300 seconds by default.
         """
 
         if timeout_s is None:
@@ -299,46 +340,23 @@ class FelderaClient:
         if not wait:
             return
 
-        start_time = time.monotonic()
+        self.__wait_for_pipeline_state(pipeline_name, "running", timeout_s)
 
-        while True:
-            if timeout_s is not None:
-                elapsed = time.monotonic() - start_time
-                if elapsed > timeout_s:
-                    raise TimeoutError(
-                        f"Timed out waiting for pipeline {pipeline_name} to activate"
-                    )
-
-            resp = self.get_pipeline(pipeline_name)
-            status = resp.deployment_status
-
-            if status == "Running":
-                break
-            elif (
-                status == "Stopped"
-                and len(resp.deployment_error or {}) > 0
-                and resp.deployment_desired_status == "Stopped"
-            ):
-                raise RuntimeError(
-                    f"""Unable to ACTIVATE the pipeline.
-Reason: The pipeline is in a STOPPED state due to the following error:
-{resp.deployment_error.get("message", "")}"""
-                )
-
-            logging.debug(
-                "still starting %s, waiting for 100 more milliseconds", pipeline_name
-            )
-            time.sleep(0.1)
-
-    def start_pipeline(
-        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
+    def _inner_start_pipeline(
+        self,
+        pipeline_name: str,
+        initial: str = "running",
+        wait: bool = True,
+        timeout_s: Optional[float] = 300,
     ):
         """
 
         :param pipeline_name: The name of the pipeline to start
+        :param initial: The initial state to start the pipeline in. "running"
+            by default.
         :param wait: Set True to wait for the pipeline to start. True by default
-        :param timeout_s: The amount of time in seconds to wait for the pipeline
-            to start. 300 seconds by default.
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start. 300 seconds by default.
         """
 
         if timeout_s is None:
@@ -346,55 +364,64 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
         self.http.post(
             path=f"/pipelines/{pipeline_name}/start",
+            params={"initial": initial},
         )
 
         if not wait:
             return
 
-        start_time = time.monotonic()
+        self.__wait_for_pipeline_state(pipeline_name, initial, timeout_s)
 
-        while True:
-            if timeout_s is not None:
-                elapsed = time.monotonic() - start_time
-                if elapsed > timeout_s:
-                    raise TimeoutError(
-                        f"Timed out waiting for pipeline {pipeline_name} to start"
-                    )
+    def start_pipeline(
+        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
+    ):
+        """
 
-            resp = self.get_pipeline(pipeline_name)
-            status = resp.deployment_status
+        :param pipeline_name: The name of the pipeline to start
+        :param wait: Set True to wait for the pipeline to start.
+            True by default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start. 300 seconds by default.
+        """
 
-            if status == "Running":
-                break
-            elif (
-                status == "Stopped"
-                and len(resp.deployment_error or {}) > 0
-                and resp.deployment_desired_status == "Stopped"
-            ):
-                raise RuntimeError(
-                    f"""Unable to START the pipeline.
-Reason: The pipeline is in a STOPPED state due to the following error:
-{resp.deployment_error.get("message", "")}"""
-                )
+        self._inner_start_pipeline(pipeline_name, "running", wait, timeout_s)
 
-            logging.debug(
-                "still starting %s, waiting for 100 more milliseconds", pipeline_name
-            )
-            time.sleep(0.1)
+    def start_pipeline_as_paused(
+        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
+    ):
+        """
+        :param pipeline_name: The name of the pipeline to start as paused.
+        :param wait: Set True to wait for the pipeline to start as pause.
+            True by default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start. 300 seconds by default.
+        """
 
-    def pause_pipeline(
+        self._inner_start_pipeline(pipeline_name, "paused", wait, timeout_s)
+
+    def start_pipeline_as_standby(
+        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
+    ):
+        """
+        :param pipeline_name: The name of the pipeline to start as standby.
+        :param wait: Set True to wait for the pipeline to start as standby.
+            True by default
+        :param timeout_s: The amount of time in seconds to wait for the
+            pipeline to start. 300 seconds by default.
+        """
+
+        self._inner_start_pipeline(pipeline_name, "paused", wait, timeout_s)
+
+    def resume_pipeline(
         self,
         pipeline_name: str,
-        error_message: str = None,
         wait: bool = True,
         timeout_s: Optional[float] = 300,
     ):
         """
-        Stop a pipeline
+        Resume a pipeline
 
         :param pipeline_name: The name of the pipeline to stop
-        :param error_message: The error message to show if the pipeline is in
-             STOPPED state due to a failure.
         :param wait: Set True to wait for the pipeline to pause. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
             to pause. 300 seconds by default.
@@ -404,45 +431,42 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             timeout_s = 300
 
         self.http.post(
-            path=f"/pipelines/{pipeline_name}/pause",
+            path=f"/pipelines/{pipeline_name}/resume",
         )
 
         if not wait:
             return
 
-        if error_message is None:
-            error_message = "Unable to PAUSE the pipeline.\n"
+        self.__wait_for_pipeline_state(pipeline_name, "running", timeout_s)
 
-        start_time = time.monotonic()
+    def pause_pipeline(
+        self,
+        pipeline_name: str,
+        wait: bool = True,
+        timeout_s: Optional[float] = 300,
+    ):
+        """
+        Pause a pipeline
 
-        while True:
-            if timeout_s is not None:
-                elapsed = time.monotonic() - start_time
-                if elapsed > timeout_s:
-                    raise TimeoutError(
-                        f"Timed out waiting for pipeline {pipeline_name} to pause"
-                    )
+        :param pipeline_name: The name of the pipeline to stop
+        :param error_message: The error message to show if the pipeline is in
+             STOPPED state due to a failure.
+        :param wait: Set True to wait for the pipeline to pause. True by default
+        :param timeout_s: The amount of time in seconds to wait for the pipeline
+            to pause. 300 seconds by default.
+        """
 
-            resp = self.get_pipeline(pipeline_name)
-            status = resp.deployment_status
+        if timeout_s is None:
+            timeout_s = 300
 
-            if status == "Paused":
-                break
-            elif (
-                status == "Stopped"
-                and len(resp.deployment_error or {}) > 0
-                and resp.deployment_desired_status == "Stopped"
-            ):
-                raise RuntimeError(
-                    error_message
-                    + f"""Reason: The pipeline is in a STOPPED state due to the following error:
-{resp.deployment_error.get("message", "")}"""
-                )
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/pause",
+        )
 
-            logging.debug(
-                "still pausing %s, waiting for 100 more milliseconds", pipeline_name
-            )
-            time.sleep(0.1)
+        if not wait:
+            return
+
+        self.__wait_for_pipeline_state(pipeline_name, "paused", timeout_s)
 
     def stop_pipeline(
         self,

{feldera-0.146.0 → feldera-0.148.0}/feldera/rest/pipeline.py

@@ -47,6 +47,9 @@ class Pipeline:
         self.program_version: Optional[int] = None
         self.deployment_config: Optional[dict] = None
         self.deployment_desired_status: Optional[str] = None
+        self.deployment_desired_status_since: Optional[str] = None
+        self.deployment_id: Optional[str] = None
+        self.deployment_initial: Optional[str] = None
         self.deployment_error: Optional[dict] = None
         self.deployment_location: Optional[str] = None
         self.program_info: Optional[dict] = (
@@ -57,6 +60,16 @@ class Pipeline:
         self.program_error: Optional[dict] = None
         self.storage_status: Optional[str] = None
 
+        self.deployment_resources_desired_status: Optional[str] = None
+        self.deployment_resources_desired_status_since: Optional[str] = None
+        self.deployment_resources_status: Optional[str] = None
+        self.deployment_resources_status_since: Optional[str] = None
+
+        self.deployment_runtime_desired_status: Optional[str] = None
+        self.deployment_runtime_desired_status_since: Optional[str] = None
+        self.deployment_runtime_status: Optional[str] = None
+        self.deployment_runtime_status_since: Optional[str] = None
+
     @classmethod
     def from_dict(cls, d: Mapping[str, Any]):
         pipeline = cls("", "", "", "", {}, {})

{feldera-0.146.0 → feldera-0.148.0}/feldera.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.146.0
+Version: 0.148.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.146.0 → feldera-0.148.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "feldera"
 readme = "README.md"
 description = "The feldera python client"
-version = "0.146.0"
+version = "0.148.0"
 license = { text = "MIT" }
 requires-python = ">=3.10"
 authors = [