feldera 0.131.0__py3-none-any.whl → 0.192.0__py3-none-any.whl

feldera/pipeline.py

@@ -7,15 +7,28 @@ import pandas
 from uuid import UUID
 
 from typing import List, Dict, Callable, Optional, Generator, Mapping, Any
+from threading import Event
 from collections import deque
-from queue import Queue
 
 from feldera.rest.errors import FelderaAPIError
-from feldera.enums import PipelineStatus, ProgramStatus, CheckpointStatus
-from feldera.enums import StorageStatus
+from feldera.enums import (
+    BootstrapPolicy,
+    CompletionTokenStatus,
+    PipelineFieldSelector,
+    PipelineStatus,
+    ProgramStatus,
+    CheckpointStatus,
+    TransactionStatus,
+    StorageStatus,
+    DeploymentDesiredStatus,
+    DeploymentResourcesDesiredStatus,
+    DeploymentResourcesStatus,
+    DeploymentRuntimeDesiredStatus,
+    DeploymentRuntimeStatus,
+)
 from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.rest.feldera_client import FelderaClient
-from feldera._callback_runner import _CallbackRunnerInstruction, CallbackRunner
+from feldera._callback_runner import CallbackRunner
 from feldera.output_handler import OutputHandler
 from feldera._helpers import ensure_dataframe_has_columns, chunk_dataframe
 from feldera.rest.sql_table import SQLTable
@@ -28,7 +41,6 @@ class Pipeline:
     def __init__(self, client: FelderaClient):
         self.client: FelderaClient = client
         self._inner: InnerPipeline | None = None
-        self.views_tx: List[Dict[str, Queue]] = []
 
     @staticmethod
     def _from_inner(inner: InnerPipeline, client: FelderaClient) -> "Pipeline":
@@ -36,28 +48,16 @@ class Pipeline:
         pipeline._inner = inner
         return pipeline
 
-    def __setup_output_listeners(self):
-        """
-        Internal function used to set up the output listeners.
-
-        :meta private:
-        """
-
-        for view_queue in self.views_tx:
-            for view_name, queue in view_queue.items():
-                # sends a message to the callback runner to start listening
-                queue.put(_CallbackRunnerInstruction.PipelineStarted)
-                # block until the callback runner is ready
-                queue.join()
-
-    def refresh(self):
+    def refresh(self, field_selector: PipelineFieldSelector):
         """
         Calls the backend to get the updated, latest version of the pipeline.
 
+        :param field_selector: Choose what pipeline information to refresh; see PipelineFieldSelector enum definition.
+
         :raises FelderaConnectionError: If there is an issue connecting to the backend.
         """
 
-        self._inner = self.client.get_pipeline(self.name)
+        self._inner = self.client.get_pipeline(self.name, field_selector)
 
     def status(self) -> PipelineStatus:
         """
@@ -65,7 +65,7 @@ class Pipeline:
         """
 
         try:
-            self.refresh()
+            self.refresh(PipelineFieldSelector.STATUS)
             return PipelineStatus.from_str(self._inner.deployment_status)
 
         except FelderaAPIError as err:
@@ -74,6 +74,30 @@ class Pipeline:
             else:
                 raise err
 
+    def wait_for_status(
+        self, expected_status: PipelineStatus, timeout: Optional[int] = None
+    ) -> None:
+        """
+        Wait for the pipeline to reach the specified status.
+
+        :param expected_status: The status to wait for
+        :param timeout: Maximum time to wait in seconds. If None, waits forever (default: None)
+        :raises TimeoutError: If the expected status is not reached within the timeout
+        """
+        start_time = time.time()
+
+        while True:
+            current_status = self.status()
+            if current_status == expected_status:
+                return
+
+            if timeout is not None and time.time() - start_time >= timeout:
+                raise TimeoutError(
+                    f"Pipeline did not reach {expected_status.name} status within {timeout} seconds"
+                )
+
+            time.sleep(1)
+
     def stats(self) -> PipelineStatistics:
         """Gets the pipeline metrics and performance counters."""
 
@@ -114,7 +138,7 @@ class Pipeline:
 
         ensure_dataframe_has_columns(df)
 
-        pipeline = self.client.get_pipeline(self.name)
+        pipeline = self.client.get_pipeline(self.name, PipelineFieldSelector.ALL)
         if table_name.lower() != "now" and table_name.lower() not in [
             tbl.name.lower() for tbl in pipeline.tables
         ]:
@@ -224,23 +248,21 @@ class Pipeline:
     def listen(self, view_name: str) -> OutputHandler:
         """
         Follow the change stream (i.e., the output) of the provided view.
-        Returns an output handler to read the changes.
+        Returns an output handle to read the changes.
 
-        When the pipeline is stopped, these listeners are dropped.
+        When the pipeline is stopped, the handle is dropped.
 
-        You must call this method before starting the pipeline to get the entire output of the view.
-        If this method is called once the pipeline has started, you will only get the output from that point onwards.
+        The handle will only receive changes from the point in time when the listener is created.
+        In order to receive all changes since the pipeline started, you can create the pipeline in the `PAUSED` state
+        using :meth:`start_paused`, attach listeners and unpause the pipeline using :meth:`resume`.
 
         :param view_name: The name of the view to listen to.
         """
 
-        queue: Optional[Queue] = None
-
         if self.status() not in [PipelineStatus.PAUSED, PipelineStatus.RUNNING]:
-            queue = Queue(maxsize=1)
-            self.views_tx.append({view_name: queue})
+            raise RuntimeError("Pipeline must be running or paused to listen to output")
 
-        handler = OutputHandler(self.client, self.name, view_name, queue)
+        handler = OutputHandler(self.client, self.name, view_name)
         handler.start()
 
         return handler
@@ -251,8 +273,9 @@ class Pipeline:
         """
         Run the given callback on each chunk of the output of the specified view.
 
-        You must call this method before starting the pipeline to operate on the entire output.
-        You can call this method after the pipeline has started, but you will only get the output from that point onwards.
+        The callback will only receive changes from the point in time when the listener is created.
+        In order to receive all changes since the pipeline started, you can create the pipeline in the `PAUSED` state
+        using :meth:`start_paused`, attach listeners and unpause the pipeline using :meth:`resume`.
 
         :param view_name: The name of the view.
         :param callback: The callback to run on each chunk. The callback should take two arguments:
@@ -270,17 +293,18 @@ class Pipeline:
 
         """
 
-        queue: Optional[Queue] = None
-
         if self.status() not in [PipelineStatus.RUNNING, PipelineStatus.PAUSED]:
-            queue = Queue(maxsize=1)
-            self.views_tx.append({view_name: queue})
+            raise RuntimeError("Pipeline must be running or paused to listen to output")
 
-        handler = CallbackRunner(self.client, self.name, view_name, callback, queue)
+        event = Event()
+        handler = CallbackRunner(
+            self.client, self.name, view_name, callback, lambda exception: None, event
+        )
         handler.start()
+        event.wait()
 
     def wait_for_completion(
-        self, force_stop: bool = False, timeout_s: Optional[float] = None
+        self, force_stop: bool = False, timeout_s: float | None = None
     ):
         """
         Block until the pipeline has completed processing all input records.
@@ -308,6 +332,7 @@ class Pipeline:
             PipelineStatus.RUNNING,
             PipelineStatus.INITIALIZING,
             PipelineStatus.PROVISIONING,
+            PipelineStatus.BOOTSTRAPPING,
         ]:
             raise RuntimeError("Pipeline must be running to wait for completion")
 
@@ -326,7 +351,7 @@ class Pipeline:
                     f" time {elapsed}s, timeout: {timeout_s}s"
                 )
 
-            pipeline_complete: bool = self.stats().global_metrics.pipeline_complete
+            pipeline_complete: bool = self.is_complete()
             if pipeline_complete is None:
                 raise RuntimeError(
                     "received unknown metrics from the pipeline, pipeline_complete is None"
@@ -339,67 +364,23 @@ class Pipeline:
         if force_stop:
             self.stop(force=True)
 
-    def start(self, wait: bool = True, timeout_s: Optional[float] = None):
+    def is_complete(self) -> bool:
         """
-        .. _start:
-
-        Starts this pipeline.
-
-        - The pipeline must be in STOPPED state to start.
-        - If the pipeline is in any other state, an error will be raised.
-        - If the pipeline is in PAUSED state, use `.meth:resume` instead.
+        Check if the pipeline has completed processing all input records.
 
-        :param timeout_s: The maximum time (in seconds) to wait for the
-            pipeline to start.
-        :param wait: Set True to wait for the pipeline to start. True by default
-
-        :raises RuntimeError: If the pipeline is not in STOPPED state.
-        """
-
-        status = self.status()
-        if status != PipelineStatus.STOPPED:
-            raise RuntimeError(
-                f"""Cannot start pipeline '{self.name}' in state \
-'{str(status.name)}'. The pipeline must be in STOPPED state before it can be \
-started. You can either stop the pipeline using the `Pipeline.stop()` \
-method or use `Pipeline.resume()` to resume a paused pipeline."""
-            )
-
-        if not wait:
-            if len(self.views_tx) > 0:
-                raise ValueError(
-                    "cannot start with 'wait=False' when output listeners are configured. Try setting 'wait=True'."
-                )
-
-            self.client.start_pipeline(self.name, wait=wait)
-
-            return
-
-        self.client.pause_pipeline(
-            self.name, "Unable to START the pipeline.\n", wait=wait, timeout_s=timeout_s
-        )
-        self.__setup_output_listeners()
-        self.resume(timeout_s=timeout_s)
-
-    def restart(self, timeout_s: Optional[float] = None):
-        """
-        Restarts the pipeline.
-
-        This method forcibly **STOPS** the pipeline regardless of its current
-        state and then starts it again. No checkpoints are made when stopping
-        the pipeline.
-
-        :param timeout_s: The maximum time (in seconds) to wait for the
-            pipeline to restart.
+        Returns True if (1) all input connectors attached to the
+        pipeline have finished reading their input data sources and issued
+        end-of-input notifications to the pipeline, and (2) all inputs received
+        from these connectors have been fully processed and corresponding
+        outputs have been sent out through the output connectors.
         """
 
-        self.stop(force=True, timeout_s=timeout_s)
-        self.start(timeout_s=timeout_s)
+        return self.stats().global_metrics.pipeline_complete
 
     def wait_for_idle(
         self,
         idle_interval_s: float = 5.0,
-        timeout_s: float = 600.0,
+        timeout_s: float | None = None,
         poll_interval_s: float = 0.2,
     ):
         """
@@ -419,12 +400,12 @@ method or use `Pipeline.resume()` to resume a paused pipeline."""
         :raises RuntimeError: If the metrics are missing or the timeout was
             reached.
         """
-        if idle_interval_s > timeout_s:
+        if timeout_s is not None and idle_interval_s > timeout_s:
             raise ValueError(
                 f"idle interval ({idle_interval_s}s) cannot be larger than"
                 f" timeout ({timeout_s}s)"
             )
-        if poll_interval_s > timeout_s:
+        if timeout_s is not None and poll_interval_s > timeout_s:
             raise ValueError(
                 f"poll interval ({poll_interval_s}s) cannot be larger than"
                 f" timeout ({timeout_s}s)"
@@ -470,11 +451,13 @@ metrics"""
                 return
 
             # Timeout
-            if now_s - start_time_s >= timeout_s:
+            if timeout_s is not None and now_s - start_time_s >= timeout_s:
                 raise RuntimeError(f"waiting for idle reached timeout ({timeout_s}s)")
             time.sleep(poll_interval_s)
 
-    def activate(self, wait: bool = True, timeout_s: Optional[float] = None):
+    def activate(
+        self, wait: bool = True, timeout_s: Optional[float] = None
+    ) -> Optional[PipelineStatus]:
         """
         Activates the pipeline when starting from STANDBY mode. Only applicable
         when the pipeline is starting from a checkpoint in object store.
@@ -485,7 +468,82 @@ metrics"""
             pipeline to pause.
         """
 
-        self.client.activate_pipeline(self.name, wait=wait, timeout_s=timeout_s)
+        return self.client.activate_pipeline(self.name, wait=wait, timeout_s=timeout_s)
+
+    def start(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        .. _start:
+
+        Starts this pipeline.
+
+        - The pipeline must be in STOPPED state to start.
+        - If the pipeline is in any other state, an error will be raised.
+        - If the pipeline is in PAUSED state, use `.meth:resume` instead.
+
+        :param bootstrap_policy: The bootstrap policy to use.
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            pipeline to start.
+        :param wait: Set True to wait for the pipeline to start. True by default
+
+        :raises RuntimeError: If the pipeline is not in STOPPED state.
+        """
+
+        self.client.start_pipeline(
+            self.name, bootstrap_policy=bootstrap_policy, wait=wait, timeout_s=timeout_s
+        )
+
+    def start_paused(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Starts the pipeline in the paused state.
+        """
+
+        return self.client.start_pipeline_as_paused(
+            self.name, bootstrap_policy=bootstrap_policy, wait=wait, timeout_s=timeout_s
+        )
+
+    def start_standby(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Starts the pipeline in the standby state.
+        """
+
+        self.client.start_pipeline_as_standby(
+            self.name, bootstrap_policy=bootstrap_policy, wait=wait, timeout_s=timeout_s
+        )
+
+    def restart(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Restarts the pipeline.
+
+        This method forcibly **STOPS** the pipeline regardless of its current
+        state and then starts it again. No checkpoints are made when stopping
+        the pipeline.
+
+        :param bootstrap_policy: The bootstrap policy to use.
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            pipeline to restart.
+        """
+
+        self.stop(force=True, timeout_s=timeout_s)
+        self.start(bootstrap_policy=bootstrap_policy, timeout_s=timeout_s)
 
     def pause(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
@@ -516,24 +574,22 @@ metrics"""
             pipeline to stop.
         """
 
-        if wait:
-            for view_queue in self.views_tx:
-                for _, queue in view_queue.items():
-                    # sends a message to the callback runner to stop listening
-                    queue.put(_CallbackRunnerInstruction.RanToCompletion)
-
-            if len(self.views_tx) > 0:
-                while self.views_tx:
-                    view = self.views_tx.pop()
-                    for view_name, queue in view.items():
-                        # block until the callback runner has been stopped
-                        queue.join()
-
-        time.sleep(3)
         self.client.stop_pipeline(
             self.name, force=force, wait=wait, timeout_s=timeout_s
         )
 
+    def approve(self):
+        """
+        Approves the pipeline to proceed with bootstrapping.
+
+        This method is used when a pipeline has been started with
+        `bootstrap_policy=BootstrapPolicy.AWAIT_APPROVAL` and is currently in the
+        AWAITINGAPPROVAL state. The pipeline will wait for explicit user approval
+        before proceeding with the bootstrapping process.
+        """
+
+        self.client.approve_pipeline(self.name)
+
     def resume(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         Resumes the pipeline from the PAUSED state. If the pipeline is already
@@ -544,17 +600,19 @@ metrics"""
             pipeline to resume.
         """
 
-        self.client.start_pipeline(self.name, wait=wait, timeout_s=timeout_s)
+        self.client.resume_pipeline(self.name, wait=wait, timeout_s=timeout_s)
 
-    def start_transaction(self):
+    def start_transaction(self) -> int:
         """
         Start a new transaction.
 
-        Returns:
-            Transaction ID.
+        :return: Transaction ID.
+
+        :raises FelderaAPIError: If the pipeline fails to start a transaction, e.g., if the pipeline is not running or
+            there is already an active transaction.
         """
 
-        self.client.start_transaction(self.name)
+        return self.client.start_transaction(self.name)
 
     def commit_transaction(
         self,
@@ -563,7 +621,7 @@ metrics"""
         timeout_s: Optional[float] = None,
     ):
         """
-        Commits the currently active transaction.
+        Commit the currently active transaction.
 
         :param transaction_id: If provided, the function verifies that the currently active transaction matches this ID.
             If the active transaction ID does not match, the function raises an error.
@@ -577,11 +635,36 @@ metrics"""
         :raises RuntimeError: If there is currently no transaction in progress.
         :raises ValueError: If the provided `transaction_id` does not match the current transaction.
         :raises TimeoutError: If the transaction does not commit within the specified timeout (when `wait` is True).
-        :raises FelderaAPIError: If the pipeline fails to start a transaction.
+        :raises FelderaAPIError: If the pipeline fails to commit a transaction.
         """
 
         self.client.commit_transaction(self.name, transaction_id, wait, timeout_s)
 
+    def transaction_status(self) -> TransactionStatus:
+        """
+        Get pipeline's transaction handling status.
+
+        :return: Current transaction handling status of the pipeline.
+
+        :raises FelderaAPIError: If pipeline's status couldn't be read, e.g., because the pipeline is not currently running.
+        """
+
+        return self.stats().global_metrics.transaction_status
+
+    def transaction_id(self) -> Optional[int]:
+        """
+        Gets the ID of the currently active transaction or None if there is no active transaction.
+
+        :return: The ID of the transaction.
+        """
+
+        transaction_id = self.stats().global_metrics.transaction_id
+
+        if transaction_id == 0:
+            return None
+        else:
+            return transaction_id
+
     def delete(self, clear_storage: bool = False):
         """
         Deletes the pipeline.
@@ -610,14 +693,25 @@ metrics"""
         """
 
         try:
-            inner = client.get_pipeline(name)
+            inner = client.get_pipeline(name, PipelineFieldSelector.ALL)
             return Pipeline._from_inner(inner, client)
         except FelderaAPIError as err:
             if err.status_code == 404:
                 err.message = f"Pipeline with name {name} not found"
                 raise err
 
-    def checkpoint(self, wait: bool = False, timeout_s=300) -> int:
+    @staticmethod
+    def all(client: FelderaClient) -> List["Pipeline"]:
+        """
+        Get all pipelines.
+
+        :param client: The FelderaClient instance.
+        :return: A list of Pipeline objects.
+        """
+
+        return [Pipeline._from_inner(p, client) for p in client.pipelines()]
+
+    def checkpoint(self, wait: bool = False, timeout_s: Optional[float] = None) -> int:
         """
         Checkpoints this pipeline.
 
@@ -625,6 +719,8 @@ metrics"""
         :param timeout_s: The maximum time (in seconds) to wait for the
             checkpoint to complete.
 
+        :return: The checkpoint sequence number.
+
         :raises FelderaAPIError: If enterprise features are not enabled.
         """
 
@@ -637,7 +733,7 @@ metrics"""
 
         while True:
             elapsed = time.monotonic() - start
-            if elapsed > timeout_s:
+            if timeout_s is not None and elapsed > timeout_s:
                 raise TimeoutError(
                     f"""timeout ({timeout_s}s) reached while waiting for \
 pipeline '{self.name}' to make checkpoint '{seq}'"""
@@ -647,9 +743,7 @@ pipeline '{self.name}' to make checkpoint '{seq}'"""
                 time.sleep(0.1)
                 continue
 
-            return status
-
-        return seq
+            return seq
 
     def checkpoint_status(self, seq: int) -> CheckpointStatus:
         """
@@ -675,7 +769,9 @@ pipeline '{self.name}' to make checkpoint '{seq}'"""
         if seq < success:
             return CheckpointStatus.Unknown
 
-    def sync_checkpoint(self, wait: bool = False, timeout_s=300) -> str:
+    def sync_checkpoint(
+        self, wait: bool = False, timeout_s: Optional[float] = None
+    ) -> str:
         """
         Syncs this checkpoint to object store.
 
@@ -685,6 +781,7 @@ pipeline '{self.name}' to make checkpoint '{seq}'"""
             checkpoint to complete syncing.
 
         :raises FelderaAPIError: If no checkpoints have been made.
+        :raises RuntimeError: If syncing the checkpoint fails.
         """
 
         uuid = self.client.sync_checkpoint(self.name)
@@ -696,12 +793,17 @@ pipeline '{self.name}' to make checkpoint '{seq}'"""
 
         while True:
             elapsed = time.monotonic() - start
-            if elapsed > timeout_s:
+            if timeout_s is not None and elapsed > timeout_s:
                 raise TimeoutError(
                     f"""timeout ({timeout_s}s) reached while waiting for \
 pipeline '{self.name}' to sync checkpoint '{uuid}'"""
                 )
             status = self.sync_checkpoint_status(uuid)
+            if status == CheckpointStatus.Failure:
+                raise RuntimeError(
+                    f"failed to sync checkpoint '{uuid}': ", status.get_error()
+                )
+
             if status in [CheckpointStatus.InProgress, CheckpointStatus.Unknown]:
                 time.sleep(0.1)
                 continue
@@ -716,6 +818,9 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         If the checkpoint is currently being synchronized, returns
         `CheckpointStatus.Unknown`.
 
+        Failures are not raised as runtime errors and must be explicitly
+        checked.
+
         :param uuid: The checkpoint uuid.
         """
 
@@ -731,6 +836,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         if uuid == fail.get("uuid"):
             failure = CheckpointStatus.Failure
             failure.error = fail.get("error", "")
+            logging.error(f"failed to sync checkpoint '{uuid}': {failure.error}")
             return failure
 
         if (success is None) or UUID(uuid) > UUID(success):
@@ -876,15 +982,76 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the program SQL code of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_code
 
+    def modify(
+        self,
+        sql: Optional[str] = None,
+        udf_rust: Optional[str] = None,
+        udf_toml: Optional[str] = None,
+        program_config: Optional[Mapping[str, Any]] = None,
+        runtime_config: Optional[Mapping[str, Any]] = None,
+        description: Optional[str] = None,
+    ):
+        """
+        Modify the pipeline.
+
+        Modify the values of pipeline attributes: SQL code, UDF Rust code,
+        UDF Rust dependencies (TOML), program config, runtime config, and
+        description. Only the provided attributes will be modified. Other
+        attributes will remain unchanged.
+
+        The pipeline must be in the STOPPED state to be modified.
+
+        :raises FelderaAPIError: If the pipeline is not in a STOPPED state.
+        """
+
+        self.client.patch_pipeline(
+            name=self._inner.name,
+            sql=sql,
+            udf_rust=udf_rust,
+            udf_toml=udf_toml,
+            program_config=program_config,
+            runtime_config=runtime_config,
+            description=description,
+        )
+
+    def update_runtime(self):
+        """
+        Recompile a pipeline with the Feldera runtime version included in the
+        currently installed Feldera platform.
+
+        Use this endpoint after upgrading Feldera to rebuild pipelines that were
+        compiled with older platform versions. In most cases, recompilation is not
+        required—pipelines compiled with older versions will continue to run on the
+        upgraded platform.
+
+        Situations where recompilation may be necessary:
+        - To benefit from the latest bug fixes and performance optimizations.
+        - When backward-incompatible changes are introduced in Feldera. In this case,
+        attempting to start a pipeline compiled with an unsupported version will
+        result in an error.
+
+        If the pipeline is already compiled with the current platform version,
+        this operation is a no-op.
+
+        Note that recompiling the pipeline with a new platform version may change its
+        query plan. If the modified pipeline is started from an existing checkpoint,
+        it may require bootstrapping parts of its state from scratch.  See Feldera
+        documentation for details on the bootstrapping process.
+
+        :raises FelderaAPIError: If the pipeline is not in a STOPPED state.
+        """
+
+        self.client.update_pipeline_runtime(self._inner.name)
+
     def storage_status(self) -> StorageStatus:
         """
         Return the storage status of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return StorageStatus.from_str(self._inner.storage_status)
 
     def program_status(self) -> ProgramStatus:
@@ -896,23 +1063,43 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Rust code to a binary.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return ProgramStatus.from_value(self._inner.program_status)
 
+    def testing_force_update_platform_version(self, platform_version: str):
+        """
+        Used to simulate a pipeline compiled with a different platform version than the one currently in use.
+        This is useful for testing platform upgrade behavior without actually upgrading Feldera.
+
+        This method is only available when Feldera runs with the "testing" unstable feature enabled.
+        """
+
+        self.client.testing_force_update_platform_version(
+            name=self._inner.name, platform_version=platform_version
+        )
+
     def program_status_since(self) -> datetime:
         """
         Return the timestamp when the current program status was set.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return datetime.fromisoformat(self._inner.program_status_since)
 
+    def platform_version(self) -> str:
+        """
+        Return the Feldera platform witch which the program was compiled.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return self._inner.platform_version
+
     def udf_rust(self) -> str:
         """
         Return the Rust code for UDFs.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.udf_rust
 
     def udf_toml(self) -> str:
@@ -920,7 +1107,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the Rust dependencies required by UDFs (in the TOML format).
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.udf_toml
 
     def program_config(self) -> Mapping[str, Any]:
@@ -928,7 +1115,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the program config of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_config
 
     def runtime_config(self) -> RuntimeConfig:
@@ -936,7 +1123,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the runtime config of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return RuntimeConfig.from_dict(self._inner.runtime_config)
 
     def set_runtime_config(self, runtime_config: RuntimeConfig):
@@ -961,7 +1148,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the ID of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.id
 
     def description(self) -> str:
@@ -969,7 +1156,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the description of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.description
 
     def tables(self) -> List[SQLTable]:
@@ -977,7 +1164,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the tables of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.tables
 
     def views(self) -> List[SQLView]:
@@ -985,7 +1172,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the views of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.views
 
     def created_at(self) -> datetime:
@@ -993,7 +1180,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the creation time of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return datetime.fromisoformat(self._inner.created_at)
 
     def version(self) -> int:
@@ -1001,7 +1188,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the version of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.version
 
     def program_version(self) -> int:
@@ -1009,7 +1196,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the program version of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.program_version
 
     def deployment_status_since(self) -> datetime:
@@ -1018,7 +1205,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         was set.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return datetime.fromisoformat(self._inner.deployment_status_since)
 
     def deployment_config(self) -> Mapping[str, Any]:
@@ -1026,17 +1213,63 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the deployment config of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.deployment_config
 
-    def deployment_desired_status(self) -> PipelineStatus:
+    def deployment_desired_status(self) -> DeploymentDesiredStatus:
         """
         Return the desired deployment status of the pipeline.
         This is the next state that the pipeline should transition to.
         """
 
-        self.refresh()
-        return PipelineStatus.from_str(self._inner.deployment_desired_status)
+        self.refresh(PipelineFieldSelector.STATUS)
+        return DeploymentDesiredStatus.from_str(self._inner.deployment_desired_status)
+
+    def deployment_resources_desired_status(self) -> DeploymentResourcesDesiredStatus:
+        """
+        Return the desired status of the the deployment resources.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return DeploymentResourcesDesiredStatus.from_str(
+            self._inner.deployment_resources_desired_status
+        )
+
+    def deployment_resources_status(self) -> DeploymentResourcesStatus:
+        """
+        Return the status of the deployment resources.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return DeploymentResourcesStatus.from_str(
+            self._inner.deployment_resources_status
+        )
+
+    def deployment_runtime_desired_status(self) -> DeploymentRuntimeDesiredStatus:
+        """
+        Return the deployment runtime desired status.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return DeploymentRuntimeDesiredStatus.from_str(
+            self._inner.deployment_runtime_desired_status
+        )
+
+    def deployment_runtime_status(self) -> DeploymentRuntimeStatus:
+        """
+        Return the deployment runtime status.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return DeploymentRuntimeStatus.from_str(self._inner.deployment_runtime_status)
+
+    def deployment_runtime_status_details(self) -> Optional[dict]:
+        """
+        Return the deployment runtime status details.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return self._inner.deployment_runtime_status_details
 
     def deployment_error(self) -> Mapping[str, Any]:
         """
@@ -1044,7 +1277,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Returns an empty string if there is no error.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.deployment_error
 
     def deployment_location(self) -> str:
@@ -1054,7 +1287,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         at runtime (a TCP port number or a URI).
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.deployment_location
 
     def program_info(self) -> Mapping[str, Any]:
@@ -1065,7 +1298,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         and the SQL program schema.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_info
 
     def program_error(self) -> Mapping[str, Any]:
@@ -1075,7 +1308,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         `sql_compilation` and `rust_compilation` will be 0.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_error
 
     def errors(self) -> List[Mapping[str, Any]]:
@@ -1159,3 +1392,31 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
             print(f"Support bundle written to {path}")
 
         return support_bundle_bytes
+
+    def generate_completion_token(self, table_name: str, connector_name: str) -> str:
+        """
+        Returns a completion token that can be passed to :meth:`.Pipeline.completion_token_status` to
+        check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+        """
+
+        return self.client.generate_completion_token(
+            self.name, table_name, connector_name
+        )
+
+    def completion_token_status(self, token: str) -> CompletionTokenStatus:
+        """
+        Returns the status of the completion token.
+        """
+
+        if self.client.completion_token_processed(self.name, token):
+            return CompletionTokenStatus.COMPLETE
+        else:
+            return CompletionTokenStatus.IN_PROGRESS
+
+    def wait_for_token(self, token: str):
+        """
+        Blocks until the pipeline processes all inputs represented by the completion token.
+        """
+
+        self.client.wait_for_token(self.name, token)