feldera 0.69.0__py3-none-any.whl → 0.189.0__py3-none-any.whl

feldera/pipeline.py

@@ -1,29 +1,46 @@
 import logging
 import time
 from datetime import datetime
+import pathlib
 
 import pandas
+from uuid import UUID
 
 from typing import List, Dict, Callable, Optional, Generator, Mapping, Any
+from threading import Event
 from collections import deque
-from queue import Queue
 
 from feldera.rest.errors import FelderaAPIError
-from feldera.enums import PipelineStatus, ProgramStatus, CheckpointStatus
+from feldera.enums import (
+    BootstrapPolicy,
+    CompletionTokenStatus,
+    PipelineFieldSelector,
+    PipelineStatus,
+    ProgramStatus,
+    CheckpointStatus,
+    TransactionStatus,
+    StorageStatus,
+    DeploymentDesiredStatus,
+    DeploymentResourcesDesiredStatus,
+    DeploymentResourcesStatus,
+    DeploymentRuntimeDesiredStatus,
+    DeploymentRuntimeStatus,
+)
 from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.rest.feldera_client import FelderaClient
-from feldera._callback_runner import _CallbackRunnerInstruction, CallbackRunner
+from feldera._callback_runner import CallbackRunner
 from feldera.output_handler import OutputHandler
 from feldera._helpers import ensure_dataframe_has_columns, chunk_dataframe
 from feldera.rest.sql_table import SQLTable
 from feldera.rest.sql_view import SQLView
+from feldera.runtime_config import RuntimeConfig
+from feldera.stats import PipelineStatistics
 
 
 class Pipeline:
     def __init__(self, client: FelderaClient):
         self.client: FelderaClient = client
         self._inner: InnerPipeline | None = None
-        self.views_tx: List[Dict[str, Queue]] = []
 
     @staticmethod
     def _from_inner(inner: InnerPipeline, client: FelderaClient) -> "Pipeline":
@@ -31,28 +48,16 @@ class Pipeline:
         pipeline._inner = inner
         return pipeline
 
-    def __setup_output_listeners(self):
-        """
-        Internal function used to set up the output listeners.
-
-        :meta private:
-        """
-
-        for view_queue in self.views_tx:
-            for view_name, queue in view_queue.items():
-                # sends a message to the callback runner to start listening
-                queue.put(_CallbackRunnerInstruction.PipelineStarted)
-                # block until the callback runner is ready
-                queue.join()
-
-    def refresh(self):
+    def refresh(self, field_selector: PipelineFieldSelector):
         """
         Calls the backend to get the updated, latest version of the pipeline.
 
+        :param field_selector: Choose what pipeline information to refresh; see PipelineFieldSelector enum definition.
+
         :raises FelderaConnectionError: If there is an issue connecting to the backend.
         """
 
-        self._inner = self.client.get_pipeline(self.name)
+        self._inner = self.client.get_pipeline(self.name, field_selector)
 
     def status(self) -> PipelineStatus:
         """
@@ -60,7 +65,7 @@ class Pipeline:
         """
 
         try:
-            self.refresh()
+            self.refresh(PipelineFieldSelector.STATUS)
             return PipelineStatus.from_str(self._inner.deployment_status)
 
         except FelderaAPIError as err:
@@ -69,6 +74,40 @@ class Pipeline:
             else:
                 raise err
 
+    def wait_for_status(
+        self, expected_status: PipelineStatus, timeout: Optional[int] = None
+    ) -> None:
+        """
+        Wait for the pipeline to reach the specified status.
+
+        :param expected_status: The status to wait for
+        :param timeout: Maximum time to wait in seconds. If None, waits forever (default: None)
+        :raises TimeoutError: If the expected status is not reached within the timeout
+        """
+        start_time = time.time()
+
+        while True:
+            current_status = self.status()
+            if current_status == expected_status:
+                return
+
+            if timeout is not None and time.time() - start_time >= timeout:
+                raise TimeoutError(
+                    f"Pipeline did not reach {expected_status.name} status within {timeout} seconds"
+                )
+
+            time.sleep(1)
+
+    def stats(self) -> PipelineStatistics:
+        """Gets the pipeline metrics and performance counters."""
+
+        return PipelineStatistics.from_dict(self.client.get_pipeline_stats(self.name))
+
+    def logs(self) -> Generator[str, None, None]:
+        """Gets the pipeline logs."""
+
+        return self.client.get_pipeline_logs(self.name)
+
     def input_pandas(self, table_name: str, df: pandas.DataFrame, force: bool = False):
         """
         Push all rows in a pandas DataFrame to the pipeline.
@@ -99,14 +138,13 @@ class Pipeline:
 
         ensure_dataframe_has_columns(df)
 
-        pipeline = self.client.get_pipeline(self.name)
+        pipeline = self.client.get_pipeline(self.name, PipelineFieldSelector.ALL)
         if table_name.lower() != "now" and table_name.lower() not in [
             tbl.name.lower() for tbl in pipeline.tables
         ]:
             raise ValueError(
-                f"Cannot push to table '{
-                    table_name
-                }': table with this name does not exist in the '{self.name}' pipeline"
+                f"Cannot push to table '{table_name}': table with this name"
+                f" does not exist in the '{self.name}' pipeline"
             )
         else:
             # consider validating the schema here
@@ -129,6 +167,7 @@ class Pipeline:
         data: Dict | list,
         update_format: str = "raw",
         force: bool = False,
+        wait: bool = True,
     ):
         """
         Push this JSON data to the specified table of the pipeline.
@@ -140,8 +179,9 @@ class Pipeline:
         :param data: The JSON encoded data to be pushed to the pipeline. The data should be in the form:
             `{'col1': 'val1', 'col2': 'val2'}` or `[{'col1': 'val1', 'col2': 'val2'}, {'col1': 'val1', 'col2': 'val2'}]`
         :param update_format: The update format of the JSON data to be pushed to the pipeline. Must be one of:
-            "raw", "insert_delete". <https://docs.feldera.com/formats/json#the-insertdelete-format>
+            "raw", "insert_delete". https://docs.feldera.com/formats/json#the-insertdelete-format
         :param force: `True` to push data even if the pipeline is paused. `False` by default.
+        :param wait: If True, blocks until this input has been processed by the pipeline
 
         :raises ValueError: If the update format is invalid.
         :raises FelderaAPIError: If the pipeline is not in a valid state to push data.
@@ -164,6 +204,7 @@ class Pipeline:
             update_format=update_format,
             array=array,
             force=force,
+            wait=wait,
         )
 
     def pause_connector(self, table_name: str, connector_name: str):
@@ -175,7 +216,7 @@ class Pipeline:
         All connectors are RUNNING by default.
 
         Refer to the connector documentation for more information:
-        <https://docs.feldera.com/connectors/#input-connector-orchestration>
+            https://docs.feldera.com/connectors/#input-connector-orchestration
 
         :param table_name: The name of the table that the connector is attached to.
         :param connector_name: The name of the connector to pause.
@@ -194,7 +235,7 @@ class Pipeline:
         All connectors are RUNNING by default.
 
         Refer to the connector documentation for more information:
-        <https://docs.feldera.com/connectors/#input-connector-orchestration>
+            https://docs.feldera.com/connectors/#input-connector-orchestration
 
         :param table_name: The name of the table that the connector is attached to.
         :param connector_name: The name of the connector to resume.
@@ -207,23 +248,21 @@ class Pipeline:
     def listen(self, view_name: str) -> OutputHandler:
         """
         Follow the change stream (i.e., the output) of the provided view.
-        Returns an output handler to read the changes.
+        Returns an output handle to read the changes.
 
-        When the pipeline is shutdown, these listeners are dropped.
+        When the pipeline is stopped, the handle is dropped.
 
-        You must call this method before starting the pipeline to get the entire output of the view.
-        If this method is called once the pipeline has started, you will only get the output from that point onwards.
+        The handle will only receive changes from the point in time when the listener is created.
+        In order to receive all changes since the pipeline started, you can create the pipeline in the `PAUSED` state
+        using :meth:`start_paused`, attach listeners and unpause the pipeline using :meth:`resume`.
 
         :param view_name: The name of the view to listen to.
         """
 
-        queue: Optional[Queue] = None
-
         if self.status() not in [PipelineStatus.PAUSED, PipelineStatus.RUNNING]:
-            queue = Queue(maxsize=1)
-            self.views_tx.append({view_name: queue})
+            raise RuntimeError("Pipeline must be running or paused to listen to output")
 
-        handler = OutputHandler(self.client, self.name, view_name, queue)
+        handler = OutputHandler(self.client, self.name, view_name)
         handler.start()
 
         return handler
@@ -234,8 +273,9 @@ class Pipeline:
         """
         Run the given callback on each chunk of the output of the specified view.
 
-        You must call this method before starting the pipeline to operate on the entire output.
-        You can call this method after the pipeline has started, but you will only get the output from that point onwards.
+        The callback will only receive changes from the point in time when the listener is created.
+        In order to receive all changes since the pipeline started, you can create the pipeline in the `PAUSED` state
+        using :meth:`start_paused`, attach listeners and unpause the pipeline using :meth:`resume`.
 
         :param view_name: The name of the view.
         :param callback: The callback to run on each chunk. The callback should take two arguments:
@@ -253,34 +293,37 @@ class Pipeline:
 
         """
 
-        queue: Optional[Queue] = None
-
         if self.status() not in [PipelineStatus.RUNNING, PipelineStatus.PAUSED]:
-            queue = Queue(maxsize=1)
-            self.views_tx.append({view_name: queue})
+            raise RuntimeError("Pipeline must be running or paused to listen to output")
 
-        handler = CallbackRunner(self.client, self.name, view_name, callback, queue)
+        event = Event()
+        handler = CallbackRunner(
+            self.client, self.name, view_name, callback, lambda exception: None, event
+        )
         handler.start()
+        event.wait()
 
     def wait_for_completion(
-        self, shutdown: bool = False, timeout_s: Optional[float] = None
+        self, force_stop: bool = False, timeout_s: float | None = None
     ):
         """
         Block until the pipeline has completed processing all input records.
 
-        This method blocks until (1) all input connectors attached to the pipeline
-        have finished reading their input data sources and issued end-of-input
-        notifications to the pipeline, and (2) all inputs received from these
-        connectors have been fully processed and corresponding outputs have been
-        sent out through the output connectors.
+        This method blocks until (1) all input connectors attached to the
+        pipeline have finished reading their input data sources and issued
+        end-of-input notifications to the pipeline, and (2) all inputs received
+        from these connectors have been fully processed and corresponding
+        outputs have been sent out through the output connectors.
 
         This method will block indefinitely if at least one of the input
         connectors attached to the pipeline is a streaming connector, such as
         Kafka, that does not issue the end-of-input notification.
 
-        :param shutdown: If True, the pipeline will be shutdown after completion. False by default.
-        :param timeout_s: Optional. The maximum time (in seconds) to wait for the pipeline to complete.
-            The default is None, which means wait indefinitely.
+        :param force_stop: If True, the pipeline will be forcibly stopped after
+            completion. False by default. No checkpoints will be made.
+        :param timeout_s: Optional. The maximum time (in seconds) to wait for
+            the pipeline to complete. The default is None, which means wait
+            indefinitely.
 
         :raises RuntimeError: If the pipeline returns unknown metrics.
         """
@@ -289,6 +332,7 @@ class Pipeline:
             PipelineStatus.RUNNING,
             PipelineStatus.INITIALIZING,
             PipelineStatus.PROVISIONING,
+            PipelineStatus.BOOTSTRAPPING,
         ]:
             raise RuntimeError("Pipeline must be running to wait for completion")
 
@@ -299,99 +343,44 @@ class Pipeline:
                 elapsed = time.monotonic() - start_time
                 if elapsed > timeout_s:
                     raise TimeoutError(
-                        f"timeout ({timeout_s}s) reached while waiting for pipeline '{
-                            self.name
-                        }' to complete"
+                        f"timeout ({timeout_s}s) reached while waiting for"
+                        f" pipeline '{self.name}' to complete"
                     )
                 logging.debug(
-                    f"waiting for pipeline {self.name} to complete: elapsed time {
-                        elapsed
-                    }s, timeout: {timeout_s}s"
+                    f"waiting for pipeline {self.name} to complete: elapsed"
+                    f" time {elapsed}s, timeout: {timeout_s}s"
                 )
 
-            metrics: dict = self.client.get_pipeline_stats(self.name).get(
-                "global_metrics"
-            )
-            pipeline_complete: bool = metrics.get("pipeline_complete")
-
+            pipeline_complete: bool = self.is_complete()
             if pipeline_complete is None:
                 raise RuntimeError(
                     "received unknown metrics from the pipeline, pipeline_complete is None"
                 )
-
-            if pipeline_complete:
+            elif pipeline_complete:
                 break
 
             time.sleep(1)
 
-        if shutdown:
-            self.shutdown()
-
-    def __failed_check(self, next):
-        """
-        Checks if the pipeline is in FAILED state and raises an error if it is.
-        :meta private:
-        """
-        status = self.status()
-        if status == PipelineStatus.FAILED:
-            deployment_error = self.client.get_pipeline(self.name).deployment_error
-            error_msg = deployment_error.get("message", "")
-            raise RuntimeError(
-                f"""Cannot {next} pipeline '{self.name}' in FAILED state.
-The pipeline must be in SHUTDOWN state before it can be started, but it is currently in FAILED state.
-Use `Pipeline.shutdown()` method to shut down the pipeline.
-Error Message:
-{error_msg}"""
-            )
-
-    def start(self, timeout_s: Optional[float] = None):
-        """
-        .. _start:
-
-        Starts this pipeline.
-
-        The pipeline must be in SHUTDOWN state to start.
-        If the pipeline is in any other state, an error will be raised.
-        If the pipeline is in PAUSED state, use `.meth:resume` instead.
-        If the pipeline is in FAILED state, it must be shutdown before starting it again.
-
-        :param timeout_s: The maximum time (in seconds) to wait for the pipeline to start.
-
-        :raises RuntimeError: If the pipeline is not in SHUTDOWN state.
-        """
-
-        self.__failed_check("start")
-        status = self.status()
-        if status != PipelineStatus.SHUTDOWN:
-            raise RuntimeError(
-                f"""Cannot start pipeline '{self.name}' in state '{str(status.name)}'.
-The pipeline must be in SHUTDOWN state before it can be started.
-You can either shut down the pipeline using the `Pipeline.shutdown()` method or use `Pipeline.resume()` to \
-resume a paused pipeline."""
-            )
-
-        self.client.pause_pipeline(
-            self.name, "Unable to START the pipeline.", timeout_s
-        )
-        self.__setup_output_listeners()
-        self.resume(timeout_s)
+        if force_stop:
+            self.stop(force=True)
 
-    def restart(self, timeout_s: Optional[float] = None):
+    def is_complete(self) -> bool:
         """
-        Restarts the pipeline.
-
-        This method **SHUTS DOWN** the pipeline regardless of its current state and then starts it again.
+        Check if the pipeline has completed processing all input records.
 
-        :param timeout_s: The maximum time (in seconds) to wait for the pipeline to restart.
+        Returns True if (1) all input connectors attached to the
+        pipeline have finished reading their input data sources and issued
+        end-of-input notifications to the pipeline, and (2) all inputs received
+        from these connectors have been fully processed and corresponding
+        outputs have been sent out through the output connectors.
         """
 
-        self.shutdown(timeout_s)
-        self.start(timeout_s)
+        return self.stats().global_metrics.pipeline_complete
 
     def wait_for_idle(
         self,
         idle_interval_s: float = 5.0,
-        timeout_s: float = 600.0,
+        timeout_s: float | None = None,
         poll_interval_s: float = 0.2,
     ):
         """
@@ -411,17 +400,15 @@ resume a paused pipeline."""
         :raises RuntimeError: If the metrics are missing or the timeout was
             reached.
         """
-        if idle_interval_s > timeout_s:
+        if timeout_s is not None and idle_interval_s > timeout_s:
             raise ValueError(
-                f"idle interval ({idle_interval_s}s) cannot be larger than timeout ({
-                    timeout_s
-                }s)"
+                f"idle interval ({idle_interval_s}s) cannot be larger than"
+                f" timeout ({timeout_s}s)"
             )
-        if poll_interval_s > timeout_s:
+        if timeout_s is not None and poll_interval_s > timeout_s:
             raise ValueError(
-                f"poll interval ({poll_interval_s}s) cannot be larger than timeout ({
-                    timeout_s
-                }s)"
+                f"poll interval ({poll_interval_s}s) cannot be larger than"
+                f" timeout ({timeout_s}s)"
             )
         if poll_interval_s > idle_interval_s:
             raise ValueError(
@@ -436,18 +423,17 @@ resume a paused pipeline."""
             now_s = time.monotonic()
 
             # Metrics retrieval
-            metrics: dict = self.client.get_pipeline_stats(self.name).get(
-                "global_metrics"
-            )
-            total_input_records: int | None = metrics.get("total_input_records")
-            total_processed_records: int | None = metrics.get("total_processed_records")
-            if total_input_records is None:
+            metrics = self.stats().global_metrics
+            total_input_records = metrics.total_input_records
+            total_processed_records = metrics.total_processed_records
+            if metrics.total_input_records is None:
                 raise RuntimeError(
                     "total_input_records is missing from the pipeline metrics"
                 )
-            if total_processed_records is None:
+            if metrics.total_processed_records is None:
                 raise RuntimeError(
-                    "total_processed_records is missing from the pipeline metrics"
+                    """total_processed_records is missing from the pipeline \
+metrics"""
                 )
 
             # Idle check
@@ -465,80 +451,236 @@ resume a paused pipeline."""
                 return
 
             # Timeout
-            if now_s - start_time_s >= timeout_s:
+            if timeout_s is not None and now_s - start_time_s >= timeout_s:
                 raise RuntimeError(f"waiting for idle reached timeout ({timeout_s}s)")
             time.sleep(poll_interval_s)
 
-    def pause(self, timeout_s: Optional[float] = None):
+    def activate(
+        self, wait: bool = True, timeout_s: Optional[float] = None
+    ) -> Optional[PipelineStatus]:
+        """
+        Activates the pipeline when starting from STANDBY mode. Only applicable
+        when the pipeline is starting from a checkpoint in object store.
+
+        :param wait: Set True to wait for the pipeline to activate. True by
+            default
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            pipeline to pause.
+        """
+
+        return self.client.activate_pipeline(self.name, wait=wait, timeout_s=timeout_s)
+
+    def start(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        .. _start:
+
+        Starts this pipeline.
+
+        - The pipeline must be in STOPPED state to start.
+        - If the pipeline is in any other state, an error will be raised.
+        - If the pipeline is in PAUSED state, use `.meth:resume` instead.
+
+        :param bootstrap_policy: The bootstrap policy to use.
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            pipeline to start.
+        :param wait: Set True to wait for the pipeline to start. True by default
+
+        :raises RuntimeError: If the pipeline is not in STOPPED state.
+        """
+
+        self.client.start_pipeline(
+            self.name, bootstrap_policy=bootstrap_policy, wait=wait, timeout_s=timeout_s
+        )
+
+    def start_paused(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Starts the pipeline in the paused state.
+        """
+
+        return self.client.start_pipeline_as_paused(
+            self.name, bootstrap_policy=bootstrap_policy, wait=wait, timeout_s=timeout_s
+        )
+
+    def start_standby(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Starts the pipeline in the standby state.
+        """
+
+        self.client.start_pipeline_as_standby(
+            self.name, bootstrap_policy=bootstrap_policy, wait=wait, timeout_s=timeout_s
+        )
+
+    def restart(
+        self,
+        bootstrap_policy: Optional[BootstrapPolicy] = None,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Restarts the pipeline.
+
+        This method forcibly **STOPS** the pipeline regardless of its current
+        state and then starts it again. No checkpoints are made when stopping
+        the pipeline.
+
+        :param bootstrap_policy: The bootstrap policy to use.
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            pipeline to restart.
+        """
+
+        self.stop(force=True, timeout_s=timeout_s)
+        self.start(bootstrap_policy=bootstrap_policy, timeout_s=timeout_s)
+
+    def pause(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         Pause the pipeline.
 
-        The pipeline can only transition to the PAUSED state from the RUNNING state.
-        If the pipeline is already paused, it will remain in the PAUSED state.
+        The pipeline can only transition to the PAUSED state from the RUNNING
+        state. If the pipeline is already paused, it will remain in the PAUSED
+        state.
+
+        :param wait: Set True to wait for the pipeline to pause. True by default
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            pipeline to pause.
+        """
 
-        :param timeout_s: The maximum time (in seconds) to wait for the pipeline to pause.
+        self.client.pause_pipeline(self.name, wait=wait, timeout_s=timeout_s)
 
-        :raises FelderaAPIError: If the pipeline is in FAILED state.
+    def stop(self, force: bool, wait: bool = True, timeout_s: Optional[float] = None):
         """
+        Stops the pipeline.
 
-        self.__failed_check("pause")
-        self.client.pause_pipeline(self.name, timeout_s=timeout_s)
+        Stops the pipeline regardless of its current state.
 
-    def shutdown(self, timeout_s: Optional[float] = None):
+        :param force: Set True to immediately scale compute resources to zero.
+            Set False to automatically checkpoint before stopping.
+        :param wait: Set True to gracefully shutdown listeners and wait for the
+            pipeline to stop. True by default.
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            pipeline to stop.
         """
-        Shut down the pipeline.
 
-        Shuts down the pipeline regardless of its current state.
+        self.client.stop_pipeline(
+            self.name, force=force, wait=wait, timeout_s=timeout_s
+        )
+
+    def approve(self):
+        """
+        Approves the pipeline to proceed with bootstrapping.
 
-        :param timeout_s: The maximum time (in seconds) to wait for the pipeline to shut down.
+        This method is used when a pipeline has been started with
+        `bootstrap_policy=BootstrapPolicy.AWAIT_APPROVAL` and is currently in the
+        AWAITINGAPPROVAL state. The pipeline will wait for explicit user approval
+        before proceeding with the bootstrapping process.
         """
 
-        if len(self.views_tx) > 0:
-            for _, queue in self.views_tx.pop().items():
-                # sends a message to the callback runner to stop listening
-                queue.put(_CallbackRunnerInstruction.RanToCompletion)
-                # block until the callback runner has been stopped
-                queue.join()
+        self.client.approve_pipeline(self.name)
+
+    def resume(self, wait: bool = True, timeout_s: Optional[float] = None):
+        """
+        Resumes the pipeline from the PAUSED state. If the pipeline is already
+        running, it will remain in the RUNNING state.
+
+        :param wait: Set True to wait for the pipeline to resume. True by default
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            pipeline to resume.
+        """
+
+        self.client.resume_pipeline(self.name, wait=wait, timeout_s=timeout_s)
+
+    def start_transaction(self) -> int:
+        """
+        Start a new transaction.
+
+        :return: Transaction ID.
+
+        :raises FelderaAPIError: If the pipeline fails to start a transaction, e.g., if the pipeline is not running or
+            there is already an active transaction.
+        """
 
-        self.client.shutdown_pipeline(self.name, timeout_s=timeout_s)
+        return self.client.start_transaction(self.name)
 
-    def suspend(self, timeout_s: Optional[float] = None):
+    def commit_transaction(
+        self,
+        transaction_id: Optional[int] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
         """
-        Suspends the pipeline to storage.
+        Commit the currently active transaction.
+
+        :param transaction_id: If provided, the function verifies that the currently active transaction matches this ID.
+            If the active transaction ID does not match, the function raises an error.
 
-        :param timeout_s: The maximum time (in seconds) to wait for the pipeline to suspend.
+        :param wait: If True, the function blocks until the transaction either commits successfully or the timeout is reached.
+            If False, the function initiates the commit and returns immediately without waiting for completion. The default value is True.
+
+        :param timeout_s: Maximum time (in seconds) to wait for the transaction to commit when `wait` is True.
+            If None, the function will wait indefinitely.
+
+        :raises RuntimeError: If there is currently no transaction in progress.
+        :raises ValueError: If the provided `transaction_id` does not match the current transaction.
+        :raises TimeoutError: If the transaction does not commit within the specified timeout (when `wait` is True).
+        :raises FelderaAPIError: If the pipeline fails to commit a transaction.
         """
 
-        if len(self.views_tx) > 0:
-            for _, queue in self.views_tx.pop().items():
-                # sends a message to the callback runner to stop listening
-                queue.put(_CallbackRunnerInstruction.RanToCompletion)
-                # block until the callback runner has been stopped
-                queue.join()
+        self.client.commit_transaction(self.name, transaction_id, wait, timeout_s)
+
+    def transaction_status(self) -> TransactionStatus:
+        """
+        Get pipeline's transaction handling status.
 
-        self.client.suspend_pipeline(self.name, timeout_s=timeout_s)
+        :return: Current transaction handling status of the pipeline.
 
-    def resume(self, timeout_s: Optional[float] = None):
+        :raises FelderaAPIError: If pipeline's status couldn't be read, e.g., because the pipeline is not currently running.
         """
-        Resumes the pipeline from the PAUSED state. If the pipeline is already running, it will remain in the RUNNING state.
 
-        :param timeout_s: The maximum time (in seconds) to wait for the pipeline to shut down.
+        return self.stats().global_metrics.transaction_status
 
-        :raises FelderaAPIError: If the pipeline is in FAILED state.
+    def transaction_id(self) -> Optional[int]:
         """
+        Gets the ID of the currently active transaction or None if there is no active transaction.
 
-        self.__failed_check("resume")
-        self.client.start_pipeline(self.name, timeout_s=timeout_s)
+        :return: The ID of the transaction.
+        """
+
+        transaction_id = self.stats().global_metrics.transaction_id
+
+        if transaction_id == 0:
+            return None
+        else:
+            return transaction_id
 
-    def delete(self):
+    def delete(self, clear_storage: bool = False):
         """
         Deletes the pipeline.
 
-        The pipeline must be shutdown before it can be deleted.
+        The pipeline must be stopped, and the storage cleared before it can be
+        deleted.
 
-        :raises FelderaAPIError: If the pipeline is not in SHUTDOWN state.
+        :param clear_storage: True if the storage should be cleared before
+            deletion. False by default
+
+        :raises FelderaAPIError: If the pipeline is not in STOPPED state or the
+            storage is still bound.
         """
 
+        if clear_storage:
+            self.clear_storage()
         self.client.delete_pipeline(self.name)
 
     @staticmethod
@@ -551,21 +693,35 @@ resume a paused pipeline."""
         """
 
         try:
-            inner = client.get_pipeline(name)
+            inner = client.get_pipeline(name, PipelineFieldSelector.ALL)
             return Pipeline._from_inner(inner, client)
         except FelderaAPIError as err:
             if err.status_code == 404:
-                raise RuntimeError(f"Pipeline with name {name} not found")
+                err.message = f"Pipeline with name {name} not found"
+                raise err
+
+    @staticmethod
+    def all(client: FelderaClient) -> List["Pipeline"]:
+        """
+        Get all pipelines.
+
+        :param client: The FelderaClient instance.
+        :return: A list of Pipeline objects.
+        """
+
+        return [Pipeline._from_inner(p, client) for p in client.pipelines()]
 
-    def checkpoint(self, wait: bool = False, timeout_s=300) -> int:
+    def checkpoint(self, wait: bool = False, timeout_s: Optional[float] = None) -> int:
         """
-        Checkpoints this pipeline, if fault-tolerance is enabled.
-        Fault Tolerance in Feldera: <https://docs.feldera.com/pipelines/fault-tolerance/>
+        Checkpoints this pipeline.
 
         :param wait: If true, will block until the checkpoint completes.
-        :param timeout_s: The maximum time (in seconds) to wait for the checkpoint to complete.
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            checkpoint to complete.
 
-        :raises FelderaAPIError: If checkpointing is not enabled.
+        :return: The checkpoint sequence number.
+
+        :raises FelderaAPIError: If enterprise features are not enabled.
         """
 
         seq = self.client.checkpoint_pipeline(self.name)
@@ -577,20 +733,17 @@ resume a paused pipeline."""
 
         while True:
             elapsed = time.monotonic() - start
-            if elapsed > timeout_s:
+            if timeout_s is not None and elapsed > timeout_s:
                 raise TimeoutError(
-                    f"timeout ({timeout_s}s) reached while waiting for pipeline '{
-                        self.name
-                    }' to make checkpoint '{seq}'"
+                    f"""timeout ({timeout_s}s) reached while waiting for \
+pipeline '{self.name}' to make checkpoint '{seq}'"""
                 )
             status = self.checkpoint_status(seq)
             if status == CheckpointStatus.InProgress:
                 time.sleep(0.1)
                 continue
 
-            return status
-
-        return seq
+            return seq
 
     def checkpoint_status(self, seq: int) -> CheckpointStatus:
         """
@@ -616,14 +769,19 @@ resume a paused pipeline."""
         if seq < success:
             return CheckpointStatus.Unknown
 
-    def sync_checkpoint(self, wait: bool = False, timeout_s=300) -> str:
+    def sync_checkpoint(
+        self, wait: bool = False, timeout_s: Optional[float] = None
+    ) -> str:
         """
         Syncs this checkpoint to object store.
 
-        :param wait: If true, will block until the checkpoint sync opeartion completes.
-        :param timeout_s: The maximum time (in seconds) to wait for the checkpoint to complete syncing.
+        :param wait: If true, will block until the checkpoint sync operation
+            completes.
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            checkpoint to complete syncing.
 
         :raises FelderaAPIError: If no checkpoints have been made.
+        :raises RuntimeError: If syncing the checkpoint fails.
         """
 
         uuid = self.client.sync_checkpoint(self.name)
@@ -635,18 +793,22 @@ resume a paused pipeline."""
 
         while True:
             elapsed = time.monotonic() - start
-            if elapsed > timeout_s:
+            if timeout_s is not None and elapsed > timeout_s:
                 raise TimeoutError(
-                    f"timeout ({timeout_s}s) reached while waiting for pipeline '{
-                        self.name
-                    }' to sync checkpoint '{uuid}'"
+                    f"""timeout ({timeout_s}s) reached while waiting for \
+pipeline '{self.name}' to sync checkpoint '{uuid}'"""
                 )
             status = self.sync_checkpoint_status(uuid)
+            if status == CheckpointStatus.Failure:
+                raise RuntimeError(
+                    f"failed to sync checkpoint '{uuid}': ", status.get_error()
+                )
+
             if status in [CheckpointStatus.InProgress, CheckpointStatus.Unknown]:
                 time.sleep(0.1)
                 continue
 
-            return status
+            break
 
         return uuid
 
@@ -656,12 +818,17 @@ resume a paused pipeline."""
         If the checkpoint is currently being synchronized, returns
         `CheckpointStatus.Unknown`.
 
+        Failures are not raised as runtime errors and must be explicitly
+        checked.
+
         :param uuid: The checkpoint uuid.
         """
 
         resp = self.client.sync_checkpoint_status(self.name)
         success = resp.get("success")
 
+        fail = resp.get("failure") or {}
+
         if uuid == success:
             return CheckpointStatus.Success
 
@@ -669,26 +836,35 @@ resume a paused pipeline."""
         if uuid == fail.get("uuid"):
             failure = CheckpointStatus.Failure
             failure.error = fail.get("error", "")
+            logging.error(f"failed to sync checkpoint '{uuid}': {failure.error}")
             return failure
 
+        if (success is None) or UUID(uuid) > UUID(success):
+            return CheckpointStatus.InProgress
+
         return CheckpointStatus.Unknown
 
     def query(self, query: str) -> Generator[Mapping[str, Any], None, None]:
         """
-        Executes an ad-hoc SQL query on this pipeline and returns a generator that yields the rows of the result as Python dictionaries.
-        For ``INSERT`` and ``DELETE`` queries, consider using :meth:`.execute` instead.
-        All floating-point numbers are deserialized as Decimal objects to avoid precision loss.
+        Executes an ad-hoc SQL query on this pipeline and returns a generator
+        that yields the rows of the result as Python dictionaries. For
+        ``INSERT`` and ``DELETE`` queries, consider using :meth:`.execute`
+        instead. All floating-point numbers are deserialized as Decimal objects
+        to avoid precision loss.
 
         Note:
             You can only ``SELECT`` from materialized tables and views.
 
         Important:
-            This method is lazy. It returns a generator and is not evaluated until you consume the result.
+            This method is lazy. It returns a generator and is not evaluated
+            until you consume the result.
 
         :param query: The SQL query to be executed.
-        :return: A generator that yields the rows of the result as Python dictionaries.
+        :return: A generator that yields the rows of the result as Python
+            dictionaries.
 
-        :raises FelderaAPIError: If the pipeline is not in a RUNNING or PAUSED state.
+        :raises FelderaAPIError: If the pipeline is not in a RUNNING or PAUSED
+            state.
         :raises FelderaAPIError: If querying a non materialized table or view.
         :raises FelderaAPIError: If the query is invalid.
         """
@@ -697,8 +873,9 @@ resume a paused pipeline."""
 
     def query_parquet(self, query: str, path: str):
         """
-        Executes an ad-hoc SQL query on this pipeline and saves the result to the specified path as a parquet file.
-        If the extension isn't `parquet`, it will be automatically appended to `path`.
+        Executes an ad-hoc SQL query on this pipeline and saves the result to
+        the specified path as a parquet file. If the extension isn't `parquet`,
+        it will be automatically appended to `path`.
 
         Note:
             You can only ``SELECT`` from materialized tables and views.
@@ -706,7 +883,8 @@ resume a paused pipeline."""
         :param query: The SQL query to be executed.
         :param path: The path of the parquet file.
 
-        :raises FelderaAPIError: If the pipeline is not in a RUNNING or PAUSED state.
+        :raises FelderaAPIError: If the pipeline is not in a RUNNING or PAUSED
+            state.
         :raises FelderaAPIError: If querying a non materialized table or view.
         :raises FelderaAPIError: If the query is invalid.
         """
@@ -715,37 +893,62 @@ resume a paused pipeline."""
 
     def query_tabular(self, query: str) -> Generator[str, None, None]:
         """
-        Executes a SQL query on this pipeline and returns the result as a formatted string.
+        Executes a SQL query on this pipeline and returns the result as a
+        formatted string.
 
         Note:
             You can only ``SELECT`` from materialized tables and views.
 
         Important:
-            This method is lazy. It returns a generator and is not evaluated until you consume the result.
+            This method is lazy. It returns a generator and is not evaluated
+            until you consume the result.
 
         :param query: The SQL query to be executed.
-        :return: A generator that yields a string representing the query result in a human-readable, tabular format.
+        :return: A generator that yields a string representing the query result
+            in a human-readable, tabular format.
 
-        :raises FelderaAPIError: If the pipeline is not in a RUNNING or PAUSED state.
+        :raises FelderaAPIError: If the pipeline is not in a RUNNING or PAUSED
+            state.
         :raises FelderaAPIError: If querying a non materialized table or view.
         :raises FelderaAPIError: If the query is invalid.
         """
 
         return self.client.query_as_text(self.name, query)
 
+    def query_hash(self, query: str):
+        """
+        Executes an ad-hoc SQL query on this pipeline and returns the result
+        as a hash of the result set. This is useful for quickly checking
+        if the result set has changed without retrieving the entire result.
+
+        Note:
+            For a stable hash, the query must be deterministic which means
+            it should be sorted.
+
+        :param query: The SQL query to be executed.
+
+        :raises FelderaAPIError: If the pipeline is not in a RUNNING or PAUSED
+            state.
+        :raises FelderaAPIError: If querying a non materialized table or view.
+        :raises FelderaAPIError: If the query is invalid.
+        """
+        return self.client.query_as_hash(self.name, query)
+
     def execute(self, query: str):
         """
-        Executes an ad-hoc SQL query on the current pipeline, discarding its result.
-        Unlike the :meth:`.query` method which returns a generator for retrieving query results lazily,
-        this method processes the query eagerly and fully before returning.
+        Executes an ad-hoc SQL query on the current pipeline, discarding its
+        result. Unlike the :meth:`.query` method which returns a generator for
+        retrieving query results lazily, this method processes the query
+        eagerly and fully before returning.
 
-        This method is suitable for SQL operations like ``INSERT`` and ``DELETE``, where the user needs
-        confirmation of successful query execution, but does not require the query result.
-        If the query fails, an exception will be raised.
+        This method is suitable for SQL operations like ``INSERT`` and
+        ``DELETE``, where the user needs confirmation of successful query
+        execution, but does not require the query result. If the query fails,
+        an exception will be raised.
 
         Important:
-            If you try to ``INSERT`` or ``DELETE`` data from a table while the pipeline is paused,
-            it will block until the pipeline is resumed.
+            If you try to ``INSERT`` or ``DELETE`` data from a table while the
+            pipeline is paused, it will block until the pipeline is resumed.
 
         :param query: The SQL query to be executed.
 
@@ -756,6 +959,16 @@ resume a paused pipeline."""
         gen = self.query_tabular(query)
         deque(gen, maxlen=0)
 
+    def clear_storage(self):
+        """
+        Clears the storage of the pipeline if it is currently in use.
+        This action cannot be canceled, and will delete all the pipeline
+        storage.
+        """
+
+        if self.storage_status() == StorageStatus.INUSE:
+            self.client.clear_storage(self.name)
+
     @property
     def name(self) -> str:
         """
@@ -769,34 +982,124 @@ resume a paused pipeline."""
         Return the program SQL code of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_code
 
+    def modify(
+        self,
+        sql: Optional[str] = None,
+        udf_rust: Optional[str] = None,
+        udf_toml: Optional[str] = None,
+        program_config: Optional[Mapping[str, Any]] = None,
+        runtime_config: Optional[Mapping[str, Any]] = None,
+        description: Optional[str] = None,
+    ):
+        """
+        Modify the pipeline.
+
+        Modify the values of pipeline attributes: SQL code, UDF Rust code,
+        UDF Rust dependencies (TOML), program config, runtime config, and
+        description. Only the provided attributes will be modified. Other
+        attributes will remain unchanged.
+
+        The pipeline must be in the STOPPED state to be modified.
+
+        :raises FelderaAPIError: If the pipeline is not in a STOPPED state.
+        """
+
+        self.client.patch_pipeline(
+            name=self._inner.name,
+            sql=sql,
+            udf_rust=udf_rust,
+            udf_toml=udf_toml,
+            program_config=program_config,
+            runtime_config=runtime_config,
+            description=description,
+        )
+
+    def update_runtime(self):
+        """
+        Recompile a pipeline with the Feldera runtime version included in the
+        currently installed Feldera platform.
+
+        Use this endpoint after upgrading Feldera to rebuild pipelines that were
+        compiled with older platform versions. In most cases, recompilation is not
+        required—pipelines compiled with older versions will continue to run on the
+        upgraded platform.
+
+        Situations where recompilation may be necessary:
+        - To benefit from the latest bug fixes and performance optimizations.
+        - When backward-incompatible changes are introduced in Feldera. In this case,
+        attempting to start a pipeline compiled with an unsupported version will
+        result in an error.
+
+        If the pipeline is already compiled with the current platform version,
+        this operation is a no-op.
+
+        Note that recompiling the pipeline with a new platform version may change its
+        query plan. If the modified pipeline is started from an existing checkpoint,
+        it may require bootstrapping parts of its state from scratch.  See Feldera
+        documentation for details on the bootstrapping process.
+
+        :raises FelderaAPIError: If the pipeline is not in a STOPPED state.
+        """
+
+        self.client.update_pipeline_runtime(self._inner.name)
+
+    def storage_status(self) -> StorageStatus:
+        """
+        Return the storage status of the pipeline.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return StorageStatus.from_str(self._inner.storage_status)
+
     def program_status(self) -> ProgramStatus:
         """
         Return the program status of the pipeline.
 
         Program status is the status of compilation of this SQL program.
-        We first compile the SQL program to Rust code, and then compile the Rust code to a binary.
+        We first compile the SQL program to Rust code, and then compile the
+        Rust code to a binary.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return ProgramStatus.from_value(self._inner.program_status)
 
+    def testing_force_update_platform_version(self, platform_version: str):
+        """
+        Used to simulate a pipeline compiled with a different platform version than the one currently in use.
+        This is useful for testing platform upgrade behavior without actually upgrading Feldera.
+
+        This method is only available when Feldera runs with the "testing" unstable feature enabled.
+        """
+
+        self.client.testing_force_update_platform_version(
+            name=self._inner.name, platform_version=platform_version
+        )
+
     def program_status_since(self) -> datetime:
         """
         Return the timestamp when the current program status was set.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return datetime.fromisoformat(self._inner.program_status_since)
 
+    def platform_version(self) -> str:
+        """
+        Return the Feldera platform witch which the program was compiled.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return self._inner.platform_version
+
     def udf_rust(self) -> str:
         """
         Return the Rust code for UDFs.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.udf_rust
 
     def udf_toml(self) -> str:
@@ -804,7 +1107,7 @@ resume a paused pipeline."""
         Return the Rust dependencies required by UDFs (in the TOML format).
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.udf_toml
 
     def program_config(self) -> Mapping[str, Any]:
@@ -812,23 +1115,40 @@ resume a paused pipeline."""
         Return the program config of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_config
 
-    def runtime_config(self) -> Mapping[str, Any]:
+    def runtime_config(self) -> RuntimeConfig:
         """
         Return the runtime config of the pipeline.
         """
 
-        self.refresh()
-        return self._inner.runtime_config
+        self.refresh(PipelineFieldSelector.ALL)
+        return RuntimeConfig.from_dict(self._inner.runtime_config)
+
+    def set_runtime_config(self, runtime_config: RuntimeConfig):
+        """Updates the runtime config of the pipeline.  The pipeline
+        must be stopped.  Changing some pipeline configuration, such
+        as the number of workers, requires storage to be cleared.
+
+        For example, to set 'min_batch_size_records' on a pipeline::
+
+            runtime_config = pipeline.runtime_config()
+            runtime_config.min_batch_size_records = 500
+            pipeline.set_runtime_config(runtime_config)
+
+        """
+
+        self.client.patch_pipeline(
+            name=self._inner.name, runtime_config=runtime_config.to_dict()
+        )
 
     def id(self) -> str:
         """
         Return the ID of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.id
 
     def description(self) -> str:
@@ -836,7 +1156,7 @@ resume a paused pipeline."""
         Return the description of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.description
 
     def tables(self) -> List[SQLTable]:
@@ -844,7 +1164,7 @@ resume a paused pipeline."""
         Return the tables of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.tables
 
     def views(self) -> List[SQLView]:
@@ -852,7 +1172,7 @@ resume a paused pipeline."""
         Return the views of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.views
 
     def created_at(self) -> datetime:
@@ -860,7 +1180,7 @@ resume a paused pipeline."""
         Return the creation time of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return datetime.fromisoformat(self._inner.created_at)
 
     def version(self) -> int:
@@ -868,7 +1188,7 @@ resume a paused pipeline."""
         Return the version of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.version
 
     def program_version(self) -> int:
@@ -876,15 +1196,16 @@ resume a paused pipeline."""
         Return the program version of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.program_version
 
     def deployment_status_since(self) -> datetime:
         """
-        Return the timestamp when the current deployment status of the pipeline was set.
+        Return the timestamp when the current deployment status of the pipeline
+        was set.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return datetime.fromisoformat(self._inner.deployment_status_since)
 
     def deployment_config(self) -> Mapping[str, Any]:
@@ -892,17 +1213,63 @@ resume a paused pipeline."""
         Return the deployment config of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.deployment_config
 
-    def deployment_desired_status(self) -> PipelineStatus:
+    def deployment_desired_status(self) -> DeploymentDesiredStatus:
         """
         Return the desired deployment status of the pipeline.
         This is the next state that the pipeline should transition to.
         """
 
-        self.refresh()
-        return PipelineStatus.from_str(self._inner.deployment_desired_status)
+        self.refresh(PipelineFieldSelector.STATUS)
+        return DeploymentDesiredStatus.from_str(self._inner.deployment_desired_status)
+
+    def deployment_resources_desired_status(self) -> DeploymentResourcesDesiredStatus:
+        """
+        Return the desired status of the the deployment resources.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return DeploymentResourcesDesiredStatus.from_str(
+            self._inner.deployment_resources_desired_status
+        )
+
+    def deployment_resources_status(self) -> DeploymentResourcesStatus:
+        """
+        Return the status of the deployment resources.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return DeploymentResourcesStatus.from_str(
+            self._inner.deployment_resources_status
+        )
+
+    def deployment_runtime_desired_status(self) -> DeploymentRuntimeDesiredStatus:
+        """
+        Return the deployment runtime desired status.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return DeploymentRuntimeDesiredStatus.from_str(
+            self._inner.deployment_runtime_desired_status
+        )
+
+    def deployment_runtime_status(self) -> DeploymentRuntimeStatus:
+        """
+        Return the deployment runtime status.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return DeploymentRuntimeStatus.from_str(self._inner.deployment_runtime_status)
+
+    def deployment_runtime_status_details(self) -> Optional[dict]:
+        """
+        Return the deployment runtime status details.
+        """
+
+        self.refresh(PipelineFieldSelector.STATUS)
+        return self._inner.deployment_runtime_status_details
 
     def deployment_error(self) -> Mapping[str, Any]:
         """
@@ -910,43 +1277,38 @@ resume a paused pipeline."""
         Returns an empty string if there is no error.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.deployment_error
 
     def deployment_location(self) -> str:
         """
         Return the deployment location of the pipeline.
-        Deployment location is the location where the pipeline can be reached at runtime (a TCP port number or a URI).
+        Deployment location is the location where the pipeline can be reached
+        at runtime (a TCP port number or a URI).
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.deployment_location
 
-    def program_binary_url(self) -> str:
-        """
-        Return the program binary URL of the pipeline.
-        This is the URL where the compiled program binary can be downloaded from.
-        """
-
-        self.refresh()
-        return self._inner.program_binary_url
-
     def program_info(self) -> Mapping[str, Any]:
         """
         Return the program info of the pipeline.
-        This is the output returned by the SQL compiler, including: the list of input and output connectors, the generated Rust code for the pipeline, and the SQL program schema.
+        This is the output returned by the SQL compiler, including: the list of
+        input and output connectors, the generated Rust code for the pipeline,
+        and the SQL program schema.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_info
 
     def program_error(self) -> Mapping[str, Any]:
         """
         Return the program error of the pipeline.
-        If there are no errors, the `exit_code` field inside both `sql_compilation` and `rust_compilation` will be 0.
+        If there are no errors, the `exit_code` field inside both
+        `sql_compilation` and `rust_compilation` will be 0.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_error
 
     def errors(self) -> List[Mapping[str, Any]]:
@@ -963,3 +1325,98 @@ resume a paused pipeline."""
         if derr:
             errors.append(derr)
         return errors
+
+    def support_bundle(
+        self,
+        output_path: Optional[str] = None,
+        *,
+        circuit_profile: bool = True,
+        heap_profile: bool = True,
+        metrics: bool = True,
+        logs: bool = True,
+        stats: bool = True,
+        pipeline_config: bool = True,
+        system_config: bool = True,
+    ) -> bytes:
+        """
+        Generate a support bundle containing diagnostic information from this pipeline.
+
+        This method collects various diagnostic data from the pipeline including
+        circuit profile, heap profile, metrics, logs, stats, and connector statistics,
+        and packages them into a single ZIP file for support purposes.
+
+        :param output_path: Optional path to save the support bundle file. If None,
+            the support bundle is only returned as bytes.
+        :param circuit_profile: Whether to collect circuit profile data (default: True)
+        :param heap_profile: Whether to collect heap profile data (default: True)
+        :param metrics: Whether to collect metrics data (default: True)
+        :param logs: Whether to collect logs data (default: True)
+        :param stats: Whether to collect stats data (default: True)
+        :param pipeline_config: Whether to collect pipeline configuration data (default: True)
+        :param system_config: Whether to collect system configuration data (default: True)
+        :return: The support bundle as bytes (ZIP archive)
+        :raises FelderaAPIError: If the pipeline does not exist or if there's an error
+        """
+
+        # Build query parameters
+        params = {}
+        if not circuit_profile:
+            params["circuit_profile"] = "false"
+        if not heap_profile:
+            params["heap_profile"] = "false"
+        if not metrics:
+            params["metrics"] = "false"
+        if not logs:
+            params["logs"] = "false"
+        if not stats:
+            params["stats"] = "false"
+        if not pipeline_config:
+            params["pipeline_config"] = "false"
+        if not system_config:
+            params["system_config"] = "false"
+
+        support_bundle_bytes = self.client.get_pipeline_support_bundle(
+            self.name, params=params
+        )
+
+        if output_path is not None:
+            path = pathlib.Path(output_path)
+
+            # Ensure the file has .zip extension
+            if path.suffix != ".zip":
+                path = path.with_suffix(".zip")
+
+            with open(path, "wb") as f:
+                f.write(support_bundle_bytes)
+
+            print(f"Support bundle written to {path}")
+
+        return support_bundle_bytes
+
+    def generate_completion_token(self, table_name: str, connector_name: str) -> str:
+        """
+        Returns a completion token that can be passed to :meth:`.Pipeline.completion_token_status` to
+        check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+        """
+
+        return self.client.generate_completion_token(
+            self.name, table_name, connector_name
+        )
+
+    def completion_token_status(self, token: str) -> CompletionTokenStatus:
+        """
+        Returns the status of the completion token.
+        """
+
+        if self.client.completion_token_processed(self.name, token):
+            return CompletionTokenStatus.COMPLETE
+        else:
+            return CompletionTokenStatus.IN_PROGRESS
+
+    def wait_for_token(self, token: str):
+        """
+        Blocks until the pipeline processes all inputs represented by the completion token.
+        """
+
+        self.client.wait_for_token(self.name, token)