feldera 0.32.0__py3-none-any.whl → 0.33.0__py3-none-any.whl

feldera/enums.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import Optional
 
 
 class CompilationProfile(Enum):
@@ -190,3 +191,44 @@ class PipelineStatus(Enum):
 
     def __eq__(self, other):
         return self.value == other.value
+
+
+class ProgramStatus(Enum):
+    Pending = 1
+    CompilingSql = 2
+    SqlCompiled = 3
+    CompilingRust = 4
+    Success = 5
+    SqlError = 6
+    RustError = 7
+    SystemError = 8
+
+    def __init__(self, value):
+        self.error: Optional[dict] = None
+        self._value_ = value
+
+    @staticmethod
+    def from_value(value):
+        error = None
+        if isinstance(value, dict):
+            error = value
+            value = list(value.keys())[0]
+
+        for member in ProgramStatus:
+            if member.name.lower() == value.lower():
+                member.error = error
+                return member
+        raise ValueError(f"Unknown value '{value}' for enum {ProgramStatus.__name__}")
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __str__(self):
+        return self.name + (f": ({self.error})" if self.error else "")
+
+    def get_error(self) -> Optional[dict]:
+        """
+        Returns the compilation error, if any.
+        """
+
+        return self.error

feldera/pipeline.py

@@ -1,4 +1,7 @@
+import logging
 import time
+from datetime import datetime
+
 import pandas
 
 from typing import List, Dict, Callable, Optional, Generator, Mapping, Any
@@ -6,21 +9,28 @@ from collections import deque
 from queue import Queue
 
 from feldera.rest.errors import FelderaAPIError
-from feldera.enums import PipelineStatus
+from feldera.enums import PipelineStatus, ProgramStatus
 from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.rest.feldera_client import FelderaClient
 from feldera._callback_runner import _CallbackRunnerInstruction, CallbackRunner
 from feldera.output_handler import OutputHandler
 from feldera._helpers import ensure_dataframe_has_columns, chunk_dataframe
+from feldera.rest.sql_table import SQLTable
+from feldera.rest.sql_view import SQLView
 
 
 class Pipeline:
-    def __init__(self, name: str, client: FelderaClient):
-        self.name = name
+    def __init__(self, client: FelderaClient):
         self.client: FelderaClient = client
         self._inner: InnerPipeline | None = None
         self.views_tx: List[Dict[str, Queue]] = []
 
+    @staticmethod
+    def _from_inner(inner: InnerPipeline, client: FelderaClient) -> "Pipeline":
+        pipeline = Pipeline(client)
+        pipeline._inner = inner
+        return pipeline
+
     def __setup_output_listeners(self):
         """
         Internal function used to set up the output listeners.
@@ -35,15 +45,23 @@ class Pipeline:
                 # block until the callback runner is ready
                 queue.join()
 
+    def refresh(self):
+        """
+        Calls the backend to get the updated, latest version of the pipeline.
+
+        :raises FelderaConnectionError: If there is an issue connecting to the backend.
+        """
+
+        self._inner = self.client.get_pipeline(self.name)
+
     def status(self) -> PipelineStatus:
         """
         Return the current status of the pipeline.
         """
 
         try:
-            inner = self.client.get_pipeline(self.name)
-            self._inner = inner
-            return PipelineStatus.from_str(inner.deployment_status)
+            self.refresh()
+            return PipelineStatus.from_str(self._inner.deployment_status)
 
         except FelderaAPIError as err:
             if err.status_code == 404:
@@ -55,9 +73,18 @@ class Pipeline:
         """
         Push all rows in a pandas DataFrame to the pipeline.
 
+        The pipeline must either be in RUNNING or PAUSED states to push data.
+        An error will be raised if the pipeline is in any other state.
+
+        The dataframe must have the same columns as the table in the pipeline.
+
         :param table_name: The name of the table to insert data into.
         :param df: The pandas DataFrame to be pushed to the pipeline.
         :param force: `True` to push data even if the pipeline is paused. `False` by default.
+
+        :raises ValueError: If the table does not exist in the pipeline.
+        :raises RuntimeError: If the pipeline is not in a valid state to push data.
+        :raises RuntimeError: If the pipeline is paused and force is not set to `True`.
         """
 
         status = self.status()
@@ -77,7 +104,7 @@ class Pipeline:
             tbl.name.lower() for tbl in pipeline.tables
         ]:
             raise ValueError(
-                f"Cannot push to table '{table_name}' as it is not registered yet"
+                f"Cannot push to table '{table_name}': table with this name does not exist in the '{self.name}' pipeline"
             )
         else:
             # consider validating the schema here
@@ -104,14 +131,25 @@ class Pipeline:
         """
         Push this JSON data to the specified table of the pipeline.
 
+        The pipeline must either be in RUNNING or PAUSED states to push data.
+        An error will be raised if the pipeline is in any other state.
+
         :param table_name: The name of the table to push data into.
         :param data: The JSON encoded data to be pushed to the pipeline. The data should be in the form:
             `{'col1': 'val1', 'col2': 'val2'}` or `[{'col1': 'val1', 'col2': 'val2'}, {'col1': 'val1', 'col2': 'val2'}]`
         :param update_format: The update format of the JSON data to be pushed to the pipeline. Must be one of:
             "raw", "insert_delete". <https://docs.feldera.com/formats/json#the-insertdelete-format>
         :param force: `True` to push data even if the pipeline is paused. `False` by default.
+
+        :raises ValueError: If the update format is invalid.
+        :raises FelderaAPIError: If the pipeline is not in a valid state to push data.
+        :raises RuntimeError: If the pipeline is paused and `force` is not set to `True`.
         """
 
+        status = self.status()
+        if not force and status == PipelineStatus.PAUSED:
+            raise RuntimeError("Pipeline is paused, set force=True to push data")
+
         if update_format not in ["raw", "insert_delete"]:
             ValueError("update_format must be one of raw or insert_delete")
 
@@ -126,11 +164,54 @@ class Pipeline:
             force=force,
         )
 
+    def pause_connector(self, table_name: str, connector_name: str):
+        """
+        Pause the specified input connector.
+
+        Connectors allow feldera to fetch data from a source or write data to a sink.
+        This method allows users to **PAUSE** a specific **INPUT** connector.
+        All connectors are RUNNING by default.
+
+        Refer to the connector documentation for more information:
+        <https://docs.feldera.com/connectors/#input-connector-orchestration>
+
+        :param table_name: The name of the table that the connector is attached to.
+        :param connector_name: The name of the connector to pause.
+
+        :raises FelderaAPIError: If the connector is not found, or if the pipeline is not running.
+        """
+
+        self.client.pause_connector(self.name, table_name, connector_name)
+
+    def resume_connector(self, table_name: str, connector_name: str):
+        """
+        Resume the specified connector.
+
+        Connectors allow feldera to fetch data from a source or write data to a sink.
+        This method allows users to **RESUME / START** a specific **INPUT** connector.
+        All connectors are RUNNING by default.
+
+        Refer to the connector documentation for more information:
+        <https://docs.feldera.com/connectors/#input-connector-orchestration>
+
+        :param table_name: The name of the table that the connector is attached to.
+        :param connector_name: The name of the connector to resume.
+
+        :raises FelderaAPIError: If the connector is not found, or if the pipeline is not running.
+        """
+
+        self.client.resume_connector(self.name, table_name, connector_name)
+
     def listen(self, view_name: str) -> OutputHandler:
         """
-        Listen to the output of the provided view so that it is available in the notebook / python code.
+        Follow the change stream (i.e., the output) of the provided view.
+        Returns an output handler to read the changes.
+
         When the pipeline is shutdown, these listeners are dropped.
 
+        You must call this method before starting the pipeline to get the entire output of the view.
+        If this method is called once the pipeline has started, you will only get the output from that point onwards.
+
         :param view_name: The name of the view to listen to.
         """
 
@@ -151,6 +232,9 @@ class Pipeline:
         """
         Run the given callback on each chunk of the output of the specified view.
 
+        You must call this method before starting the pipeline to operate on the entire output.
+        You can call this method after the pipeline has started, but you will only get the output from that point onwards.
+
         :param view_name: The name of the view.
         :param callback: The callback to run on each chunk. The callback should take two arguments:
 
@@ -176,7 +260,9 @@ class Pipeline:
         handler = CallbackRunner(self.client, self.name, view_name, callback, queue)
         handler.start()
 
-    def wait_for_completion(self, shutdown: bool = False):
+    def wait_for_completion(
+        self, shutdown: bool = False, timeout_s: Optional[float] = None
+    ):
         """
         Block until the pipeline has completed processing all input records.
 
@@ -191,6 +277,8 @@ class Pipeline:
         Kafka, that does not issue the end-of-input notification.
 
         :param shutdown: If True, the pipeline will be shutdown after completion. False by default.
+        :param timeout_s: Optional. The maximum time (in seconds) to wait for the pipeline to complete.
+            The default is None, which means wait indefinitely.
 
         :raises RuntimeError: If the pipeline returns unknown metrics.
         """
@@ -202,7 +290,19 @@ class Pipeline:
         ]:
             raise RuntimeError("Pipeline must be running to wait for completion")
 
+        start_time = time.monotonic()
+
         while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError(
+                        f"timeout ({timeout_s}s) reached while waiting for pipeline '{self.name}' to complete"
+                    )
+                logging.debug(
+                    f"waiting for pipeline {self.name} to complete: elapsed time {elapsed}s, timeout: {timeout_s}s"
+                )
+
             metrics: dict = self.client.get_pipeline_stats(self.name).get(
                 "global_metrics"
             )
@@ -221,35 +321,66 @@ class Pipeline:
         if shutdown:
             self.shutdown()
 
-    def start(self):
+    def __failed_check(self, next):
+        """
+        Checks if the pipeline is in FAILED state and raises an error if it is.
+        :meta private:
+        """
+        status = self.status()
+        if status == PipelineStatus.FAILED:
+            deployment_error = self.client.get_pipeline(self.name).deployment_error
+            error_msg = deployment_error.get("message", "")
+            raise RuntimeError(
+                f"""Cannot {next} pipeline '{self.name}' in FAILED state.
+The pipeline must be in SHUTDOWN state before it can be started, but it is currently in FAILED state.
+Use `Pipeline.shutdown()` method to shut down the pipeline.
+Error Message:
+{error_msg}"""
+            )
+
+    def start(self, timeout_s: Optional[float] = None):
         """
         .. _start:
 
         Starts this pipeline.
 
-        :raises RuntimeError: If the pipeline returns unknown metrics.
+        The pipeline must be in SHUTDOWN state to start.
+        If the pipeline is in any other state, an error will be raised.
+        If the pipeline is in PAUSED state, use `.meth:resume` instead.
+        If the pipeline is in FAILED state, it must be shutdown before starting it again.
+
+        :param timeout_s: The maximum time (in seconds) to wait for the pipeline to start.
+
+        :raises RuntimeError: If the pipeline is not in SHUTDOWN state.
         """
 
+        self.__failed_check("start")
         status = self.status()
         if status != PipelineStatus.SHUTDOWN:
             raise RuntimeError(
-                f"pipeline {self.name} in state {str(status.name)} cannot be started\n"
-                + self.client.get_pipeline(self.name).deployment_error.get(
-                    "message", ""
-                )
+                f"""Cannot start pipeline '{self.name}' in state '{str(status.name)}'.
+The pipeline must be in SHUTDOWN state before it can be started.
+You can either shut down the pipeline using the `Pipeline.shutdown()` method or use `Pipeline.resume()` to \
+resume a paused pipeline."""
             )
 
-        self.client.pause_pipeline(self.name, "Unable to START the pipeline.")
+        self.client.pause_pipeline(
+            self.name, "Unable to START the pipeline.", timeout_s
+        )
         self.__setup_output_listeners()
-        self.resume()
+        self.resume(timeout_s)
 
-    def restart(self):
+    def restart(self, timeout_s: Optional[float] = None):
         """
         Restarts the pipeline.
+
+        This method **SHUTS DOWN** the pipeline regardless of its current state and then starts it again.
+
+        :param timeout_s: The maximum time (in seconds) to wait for the pipeline to restart.
         """
 
-        self.shutdown()
-        self.start()
+        self.shutdown(timeout_s)
+        self.start(timeout_s)
 
     def wait_for_idle(
         self,
@@ -328,16 +459,28 @@ class Pipeline:
                 raise RuntimeError(f"waiting for idle reached timeout ({timeout_s}s)")
             time.sleep(poll_interval_s)
 
-    def pause(self):
+    def pause(self, timeout_s: Optional[float] = None):
         """
         Pause the pipeline.
+
+        The pipeline can only transition to the PAUSED state from the RUNNING state.
+        If the pipeline is already paused, it will remain in the PAUSED state.
+
+        :param timeout_s: The maximum time (in seconds) to wait for the pipeline to pause.
+
+        :raises FelderaAPIError: If the pipeline is in FAILED state.
         """
 
-        self.client.pause_pipeline(self.name)
+        self.__failed_check("pause")
+        self.client.pause_pipeline(self.name, timeout_s=timeout_s)
 
-    def shutdown(self):
+    def shutdown(self, timeout_s: Optional[float] = None):
         """
         Shut down the pipeline.
+
+        Shuts down the pipeline regardless of its current state.
+
+        :param timeout_s: The maximum time (in seconds) to wait for the pipeline to shut down.
         """
 
         if len(self.views_tx) > 0:
@@ -347,18 +490,27 @@ class Pipeline:
                 # block until the callback runner has been stopped
                 queue.join()
 
-        self.client.shutdown_pipeline(self.name)
+        self.client.shutdown_pipeline(self.name, timeout_s=timeout_s)
 
-    def resume(self):
+    def resume(self, timeout_s: Optional[float] = None):
         """
-        Resumes the pipeline.
+        Resumes the pipeline from the PAUSED state. If the pipeline is already running, it will remain in the RUNNING state.
+
+        :param timeout_s: The maximum time (in seconds) to wait for the pipeline to shut down.
+
+        :raises FelderaAPIError: If the pipeline is in FAILED state.
         """
 
-        self.client.start_pipeline(self.name)
+        self.__failed_check("resume")
+        self.client.start_pipeline(self.name, timeout_s=timeout_s)
 
     def delete(self):
         """
         Deletes the pipeline.
+
+        The pipeline must be shutdown before it can be deleted.
+
+        :raises FelderaAPIError: If the pipeline is not in SHUTDOWN state.
         """
 
         self.client.delete_pipeline(self.name)
@@ -374,23 +526,38 @@ class Pipeline:
 
         try:
             inner = client.get_pipeline(name)
-            pipeline = Pipeline(inner.name, client)
-            pipeline.__inner = inner
-            return pipeline
+            return Pipeline._from_inner(inner, client)
         except FelderaAPIError as err:
             if err.status_code == 404:
                 raise RuntimeError(f"Pipeline with name {name} not found")
 
+    def checkpoint(self):
+        """
+        Checkpoints this pipeline, if fault-tolerance is enabled.
+        Fault Tolerance in Feldera: <https://docs.feldera.com/fault-tolerance/>
+
+        :raises FelderaAPIError: If checkpointing is not enabled.
+        """
+
+        self.client.checkpoint_pipeline(self.name)
+
     def query(self, query: str) -> Generator[Mapping[str, Any], None, None]:
         """
-        Executes an ad-hoc SQL query on this pipeline and returns the result in the specified format.
+        Executes an ad-hoc SQL query on this pipeline and returns a generator that yields the rows of the result as Python dictionaries.
         For ``INSERT`` and ``DELETE`` queries, consider using :meth:`.execute` instead.
 
+        Note:
+            You can only ``SELECT`` from materialized tables and views.
+
         Important:
             This method is lazy. It returns a generator and is not evaluated until you consume the result.
 
         :param query: The SQL query to be executed.
         :return: A generator that yields the rows of the result as Python dictionaries.
+
+        :raises FelderaAPIError: If the pipeline is not in a RUNNING or PAUSED state.
+        :raises FelderaAPIError: If querying a non materialized table or view.
+        :raises FelderaAPIError: If the query is invalid.
         """
 
         return self.client.query_as_json(self.name, query)
@@ -400,8 +567,15 @@ class Pipeline:
         Executes an ad-hoc SQL query on this pipeline and saves the result to the specified path as a parquet file.
         If the extension isn't `parquet`, it will be automatically appended to `path`.
 
+        Note:
+            You can only ``SELECT`` from materialized tables and views.
+
         :param query: The SQL query to be executed.
         :param path: The path of the parquet file.
+
+        :raises FelderaAPIError: If the pipeline is not in a RUNNING or PAUSED state.
+        :raises FelderaAPIError: If querying a non materialized table or view.
+        :raises FelderaAPIError: If the query is invalid.
         """
 
         self.client.query_as_parquet(self.name, query, path)
@@ -410,11 +584,18 @@ class Pipeline:
         """
         Executes a SQL query on this pipeline and returns the result as a formatted string.
 
+        Note:
+            You can only ``SELECT`` from materialized tables and views.
+
         Important:
             This method is lazy. It returns a generator and is not evaluated until you consume the result.
 
         :param query: The SQL query to be executed.
         :return: A generator that yields a string representing the query result in a human-readable, tabular format.
+
+        :raises FelderaAPIError: If the pipeline is not in a RUNNING or PAUSED state.
+        :raises FelderaAPIError: If querying a non materialized table or view.
+        :raises FelderaAPIError: If the query is invalid.
         """
 
         return self.client.query_as_text(self.name, query)
@@ -429,8 +610,199 @@ class Pipeline:
         confirmation of successful query execution, but does not require the query result.
         If the query fails, an exception will be raised.
 
+        Important:
+            If you try to ``INSERT`` or ``DELETE`` data from a table while the pipeline is paused,
+            it will block until the pipeline is resumed.
+
         :param query: The SQL query to be executed.
+
+        :raises FelderaAPIError: If the pipeline is not in a RUNNING state.
+        :raises FelderaAPIError: If the query is invalid.
         """
 
         gen = self.query_tabular(query)
         deque(gen, maxlen=0)
+
+    @property
+    def name(self) -> str:
+        """
+        Return the name of the pipeline.
+        """
+
+        return self._inner.name
+
+    def program_code(self) -> str:
+        """
+        Return the program SQL code of the pipeline.
+        """
+
+        self.refresh()
+        return self._inner.program_code
+
+    def program_status(self) -> ProgramStatus:
+        """
+        Return the program status of the pipeline.
+
+        Program status is the status of compilation of this SQL program.
+        We first compile the SQL program to Rust code, and then compile the Rust code to a binary.
+        """
+
+        self.refresh()
+        return ProgramStatus.from_value(self._inner.program_status)
+
+    def program_status_since(self) -> datetime:
+        """
+        Return the timestamp when the current program status was set.
+        """
+
+        self.refresh()
+        return datetime.fromisoformat(self._inner.program_status_since)
+
+    def udf_rust(self) -> str:
+        """
+        Return the Rust code for UDFs.
+        """
+
+        self.refresh()
+        return self._inner.udf_rust
+
+    def udf_toml(self) -> str:
+        """
+        Return the Rust dependencies required by UDFs (in the TOML format).
+        """
+
+        self.refresh()
+        return self._inner.udf_toml
+
+    def program_config(self) -> Mapping[str, Any]:
+        """
+        Return the program config of the pipeline.
+        """
+
+        self.refresh()
+        return self._inner.program_config
+
+    def runtime_config(self) -> Mapping[str, Any]:
+        """
+        Return the runtime config of the pipeline.
+        """
+
+        self.refresh()
+        return self._inner.runtime_config
+
+    def id(self) -> str:
+        """
+        Return the ID of the pipeline.
+        """
+
+        self.refresh()
+        return self._inner.id
+
+    def description(self) -> str:
+        """
+        Return the description of the pipeline.
+        """
+
+        self.refresh()
+        return self._inner.description
+
+    def tables(self) -> List[SQLTable]:
+        """
+        Return the tables of the pipeline.
+        """
+
+        self.refresh()
+        return self._inner.tables
+
+    def views(self) -> List[SQLView]:
+        """
+        Return the views of the pipeline.
+        """
+
+        self.refresh()
+        return self._inner.views
+
+    def created_at(self) -> datetime:
+        """
+        Return the creation time of the pipeline.
+        """
+
+        self.refresh()
+        return datetime.fromisoformat(self._inner.created_at)
+
+    def version(self) -> int:
+        """
+        Return the version of the pipeline.
+        """
+
+        self.refresh()
+        return self._inner.version
+
+    def program_version(self) -> int:
+        """
+        Return the program version of the pipeline.
+        """
+
+        self.refresh()
+        return self._inner.program_version
+
+    def deployment_status_since(self) -> datetime:
+        """
+        Return the timestamp when the current deployment status of the pipeline was set.
+        """
+
+        self.refresh()
+        return datetime.fromisoformat(self._inner.deployment_status_since)
+
+    def deployment_config(self) -> Mapping[str, Any]:
+        """
+        Return the deployment config of the pipeline.
+        """
+
+        self.refresh()
+        return self._inner.deployment_config
+
+    def deployment_desired_status(self) -> PipelineStatus:
+        """
+        Return the desired deployment status of the pipeline.
+        This is the next state that the pipeline should transition to.
+        """
+
+        self.refresh()
+        return PipelineStatus.from_str(self._inner.deployment_desired_status)
+
+    def deployment_error(self) -> Mapping[str, Any]:
+        """
+        Return the deployment error of the pipeline.
+        Returns an empty string if there is no error.
+        """
+
+        self.refresh()
+        return self._inner.deployment_error
+
+    def deployment_location(self) -> str:
+        """
+        Return the deployment location of the pipeline.
+        Deployment location is the location where the pipeline can be reached at runtime (a TCP port number or a URI).
+        """
+
+        self.refresh()
+        return self._inner.deployment_location
+
+    def program_binary_url(self) -> str:
+        """
+        Return the program binary URL of the pipeline.
+        This is the URL where the compiled program binary can be downloaded from.
+        """
+
+        self.refresh()
+        return self._inner.program_binary_url
+
+    def program_info(self) -> Mapping[str, Any]:
+        """
+        Return the program info of the pipeline.
+        This is the output returned by the SQL compiler, including: the list of input and output connectors, the generated Rust code for the pipeline, and the SQL program schema.
+        """
+
+        self.refresh()
+        return self._inner.program_info

feldera/pipeline_builder.py

@@ -66,7 +66,7 @@ class PipelineBuilder:
         )
 
         inner = self.client.create_pipeline(inner)
-        pipeline = Pipeline(inner.name, self.client)
+        pipeline = Pipeline(self.client)
         pipeline._inner = inner
 
         return pipeline
@@ -103,7 +103,7 @@ class PipelineBuilder:
         )
 
         inner = self.client.create_or_update_pipeline(inner)
-        pipeline = Pipeline(inner.name, self.client)
+        pipeline = Pipeline(self.client)
         pipeline._inner = inner
 
         return pipeline

feldera/rest/feldera_client.py

@@ -7,6 +7,7 @@ from decimal import Decimal
 from typing import Generator
 
 from feldera.rest.config import Config
+from feldera.rest.errors import FelderaTimeoutError
 from feldera.rest.pipeline import Pipeline
 from feldera.rest._httprequests import HttpRequests
 
@@ -27,7 +28,7 @@ class FelderaClient:
         self,
         url: str,
         api_key: Optional[str] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         """
         :param url: The url to Feldera API (ex: https://try.feldera.com)
@@ -36,7 +37,7 @@ class FelderaClient:
             out.
         """
 
-        self.config = Config(url, api_key, timeout)
+        self.config = Config(url, api_key, timeout=timeout)
         self.http = HttpRequests(self.config)
 
         try:
@@ -45,6 +46,14 @@ class FelderaClient:
             logging.error(f"Failed to connect to Feldera API: {e}")
             raise e
 
+    @staticmethod
+    def localhost(port: int = 8080) -> "FelderaClient":
+        """
+        Create a FelderaClient that connects to the local Feldera instance
+        """
+
+        return FelderaClient(f"http://localhost:{port}")
+
     def get_pipeline(self, pipeline_name) -> Pipeline:
         """
         Get a pipeline by name
@@ -188,18 +197,30 @@ class FelderaClient:
 
         return resp
 
-    def start_pipeline(self, pipeline_name: str):
+    def start_pipeline(self, pipeline_name: str, timeout_s: Optional[float] = 300):
         """
-        Start a pipeline
 
         :param pipeline_name: The name of the pipeline to start
+        :param timeout_s: The amount of time in seconds to wait for the pipeline to start. 300 seconds by default.
         """
 
+        if timeout_s is None:
+            timeout_s = 300
+
         self.http.post(
             path=f"/pipelines/{pipeline_name}/start",
         )
 
+        start_time = time.monotonic()
+
         while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError(
+                        f"Timed out waiting for pipeline {pipeline_name} to start"
+                    )
+
             resp = self.get_pipeline(pipeline_name)
             status = resp.deployment_status
 
@@ -217,13 +238,23 @@ Reason: The pipeline is in a FAILED state due to the following error:
             )
             time.sleep(0.1)
 
-    def pause_pipeline(self, pipeline_name: str, error_message: str = None):
+    def pause_pipeline(
+        self,
+        pipeline_name: str,
+        error_message: str = None,
+        timeout_s: Optional[float] = 300,
+    ):
         """
         Stop a pipeline
 
         :param pipeline_name: The name of the pipeline to stop
         :param error_message: The error message to show if the pipeline is in FAILED state
+        :param timeout_s: The amount of time in seconds to wait for the pipeline to pause. 300 seconds by default.
         """
+
+        if timeout_s is None:
+            timeout_s = 300
+
         self.http.post(
             path=f"/pipelines/{pipeline_name}/pause",
         )
@@ -231,7 +262,16 @@ Reason: The pipeline is in a FAILED state due to the following error:
         if error_message is None:
             error_message = "Unable to PAUSE the pipeline.\n"
 
+        start_time = time.monotonic()
+
         while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError(
+                        f"Timed out waiting for pipeline {pipeline_name} to pause"
+                    )
+
             resp = self.get_pipeline(pipeline_name)
             status = resp.deployment_status
 
@@ -249,44 +289,24 @@ Reason: The pipeline is in a FAILED state due to the following error:
             )
             time.sleep(0.1)
 
-    def shutdown_pipeline(self, pipeline_name: str):
+    def shutdown_pipeline(self, pipeline_name: str, timeout_s: Optional[float] = 300):
         """
         Shutdown a pipeline
 
         :param pipeline_name: The name of the pipeline to shut down
+        :param timeout_s: The amount of time in seconds to wait for the pipeline to shut down. Default is 15 seconds.
         """
 
-        self.http.post(
-            path=f"/pipelines/{pipeline_name}/shutdown",
-        )
-
-        start = time.time()
-        timeout = 15
-
-        while time.time() - start < timeout:
-            status = self.get_pipeline(pipeline_name).deployment_status
+        if timeout_s is None:
+            timeout_s = 300
 
-            if status == "Shutdown":
-                return
-
-            logging.debug(
-                "still shutting down %s, waiting for 100 more milliseconds",
-                pipeline_name,
-            )
-            time.sleep(0.1)
-
-        # retry sending shutdown request as the pipline hasn't shutdown yet
-        logging.debug(
-            "pipeline %s hasn't shutdown after %s s, retrying", pipeline_name, timeout
-        )
         self.http.post(
             path=f"/pipelines/{pipeline_name}/shutdown",
         )
 
-        start = time.time()
-        timeout = 5
+        start = time.monotonic()
 
-        while time.time() - start < timeout:
+        while time.monotonic() - start < timeout_s:
             status = self.get_pipeline(pipeline_name).deployment_status
 
             if status == "Shutdown":
@@ -298,7 +318,9 @@ Reason: The pipeline is in a FAILED state due to the following error:
             )
             time.sleep(0.1)
 
-        raise RuntimeError(f"Failed to shutdown pipeline {pipeline_name}")
+        raise FelderaTimeoutError(
+            f"timeout error: pipeline '{pipeline_name}' did not shutdown in {timeout_s} seconds"
+        )
 
     def checkpoint_pipeline(self, pipeline_name: str):
         """
@@ -435,12 +457,12 @@ Reason: The pipeline is in a FAILED state due to the following error:
             stream=True,
         )
 
-        end = time.time() + timeout if timeout else None
+        end = time.monotonic() + timeout if timeout else None
 
         # Using the default chunk size below makes `iter_lines` extremely
         # inefficient when dealing with long lines.
         for chunk in resp.iter_lines(chunk_size=50000000):
-            if end and time.time() > end:
+            if end and time.monotonic() > end:
                 break
             if chunk:
                 yield json.loads(chunk, parse_float=Decimal)
@@ -534,3 +556,49 @@ Reason: The pipeline is in a FAILED state due to the following error:
         for chunk in resp.iter_lines(chunk_size=50000000):
             if chunk:
                 yield json.loads(chunk, parse_float=Decimal)
+
+    def pause_connector(self, pipeline_name, table_name, connector_name):
+        """
+        Pause the specified input connector.
+
+        Connectors allow feldera to fetch data from a source or write data to a sink.
+        This method allows users to **PAUSE** a specific **INPUT** connector.
+        All connectors are RUNNING by default.
+
+        Refer to the connector documentation for more information:
+        <https://docs.feldera.com/connectors/#input-connector-orchestration>
+
+        :param pipeline_name: The name of the pipeline.
+        :param table_name: The name of the table associated with this connector.
+        :param connector_name: The name of the connector.
+
+        :raises FelderaAPIError: If the connector cannot be found, or if the pipeline is not running.
+        """
+
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/pause",
+        )
+
+    def resume_connector(
+        self, pipeline_name: str, table_name: str, connector_name: str
+    ):
+        """
+        Resume the specified connector.
+
+        Connectors allow feldera to fetch data from a source or write data to a sink.
+        This method allows users to **RESUME / START** a specific **INPUT** connector.
+        All connectors are RUNNING by default.
+
+        Refer to the connector documentation for more information:
+        <https://docs.feldera.com/connectors/#input-connector-orchestration>
+
+        :param pipeline_name: The name of the pipeline.
+        :param table_name: The name of the table associated with this connector.
+        :param connector_name: The name of the connector.
+
+        :raises FelderaAPIError: If the connector cannot be found, or if the pipeline is not running.
+        """
+
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/start",
+        )

feldera/rest/pipeline.py

@@ -37,7 +37,7 @@ class Pipeline:
         self.description: Optional[str] = description
         self.program_config: Mapping[str, Any] = program_config
         self.runtime_config: Mapping[str, Any] = runtime_config
-        self.id: Optional[str] = id
+        self.id: Optional[str] = None
         self.tables: list[SQLTable] = []
         self.views: list[SQLView] = []
         self.deployment_status: Optional[str] = None

{feldera-0.32.0.dist-info → feldera-0.33.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: feldera
-Version: 0.32.0
+Version: 0.33.0
 Summary: The feldera python client
 Author-email: Abhinav <abhinav.gyawali@feldera.com>
 License: MIT

{feldera-0.32.0.dist-info → feldera-0.33.0.dist-info}/RECORD

@@ -1,20 +1,20 @@
 feldera/__init__.py,sha256=PxkgCtEAuFwo4u8NGEDio-bF3M-GnbeV45tAQVoBbqE,297
 feldera/_callback_runner.py,sha256=Tdf6BXN4zppyoy8t_y-Ooa3B0wEfvyezMHU9jxY2ZhA,4713
 feldera/_helpers.py,sha256=TuaJPQdAnRV9K5bG7-DCAr45b2JxsZyrwkZBJf1806M,2684
-feldera/enums.py,sha256=pdidN0KsYDruQRIDusdcklVsx2NW9sMXYfo9-kUqtak,6247
+feldera/enums.py,sha256=tI48tTF65AU5ZLem_IDnC5ycPVMKMv591lW2T__U4C8,7281
 feldera/output_handler.py,sha256=64J3ljhOaKIhxdjOKYi-BUz_HnMwROfmN8eE-btYygU,1930
-feldera/pipeline.py,sha256=-ReBWeDGed-o9V0uG5RKk21RtD_TBaehomrBiMape4E,16127
-feldera/pipeline_builder.py,sha256=FzpoBlMGPmN76uxLQ768ISI6f3N5jkGriune8jZMkJA,3688
+feldera/pipeline.py,sha256=zKUlqH9Vb9Ut5qeWThEZXJoL6b6mDl0Z1JbR2hCqAHw,29609
+feldera/pipeline_builder.py,sha256=4rmklRZ0-otvTUb-HTESfNsJopEK-E2jxpJXiYlKpps,3664
 feldera/runtime_config.py,sha256=PfYXsrLrs5Duty-7x3dGDf2uvp5hwp3Yb5n3bRQtLVk,2898
 feldera/rest/__init__.py,sha256=Eg-EKUU3RSTDcdxTR_7wNDnCly8VpXEzsZCQUmf-y2M,308
 feldera/rest/_httprequests.py,sha256=y3RxFn4BCTKbUztO1LN2CWXgGA93dIIV5VLdyiWQWuQ,6181
 feldera/rest/config.py,sha256=84Lj2QX6SYNZJdBfrCHPMh29Nj4MY7nRB-uddytx_ok,795
 feldera/rest/errors.py,sha256=b4i2JjrbSmej7jdko_FL8UeXklLKenSipwMT80jowaM,1720
-feldera/rest/feldera_client.py,sha256=nBAJHCEbBAPGsZjGvYDZJHIlEz8AuTLHPli-oJ9Q3ek,17549
-feldera/rest/pipeline.py,sha256=W5Oo_bwVduae-alF3g69RyDwQcuSUsWh-rL6cfvvunQ,2786
+feldera/rest/feldera_client.py,sha256=oqdXhdQMG_CsPcmVkalJag7Chj7bc5SW5F1o4fn7KwE,20473
+feldera/rest/pipeline.py,sha256=o6BFLL3DuurvAhneX1LH7mLjbvX3dn4lCXziYRciUI4,2788
 feldera/rest/sql_table.py,sha256=qrw-YwMzx5T81zDefNO1KOx7EyypFz1vPwGBzSUB7kc,652
 feldera/rest/sql_view.py,sha256=hN12mPM0mvwLCIPYywpb12s9Hd2Ws31IlTMXPriMisw,644
-feldera-0.32.0.dist-info/METADATA,sha256=h_zxiUnfUuDLcI0fSeTNDU6PO-i5La_OY7-WjeybcIc,2582
-feldera-0.32.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-feldera-0.32.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
-feldera-0.32.0.dist-info/RECORD,,
+feldera-0.33.0.dist-info/METADATA,sha256=77PLYhYRhgf3v2hBqbC3Xwv5cpHfk47fw2iCnGhPF2M,2582
+feldera-0.33.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+feldera-0.33.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
+feldera-0.33.0.dist-info/RECORD,,