feldera 0.131.0__py3-none-any.whl → 0.189.0__py3-none-any.whl

feldera/__init__.py

@@ -1,9 +1,9 @@
 from feldera.rest.feldera_client import FelderaClient as FelderaClient
 from feldera.pipeline import Pipeline as Pipeline
 from feldera.pipeline_builder import PipelineBuilder as PipelineBuilder
-from feldera.rest._helpers import client_version
+from feldera.rest._helpers import determine_client_version
 
-__version__ = client_version()
+__version__ = determine_client_version()
 
 import pretty_errors
 

feldera/_callback_runner.py

@@ -1,16 +1,13 @@
-from enum import Enum
-from threading import Thread
-from typing import Callable, Optional
-from queue import Queue, Empty
+from threading import Thread, Event
+from typing import Callable, List, Optional, Mapping, Any
 
 import pandas as pd
 from feldera import FelderaClient
 from feldera._helpers import dataframe_from_response
-
-
-class _CallbackRunnerInstruction(Enum):
-    PipelineStarted = 1
-    RanToCompletion = 2
+from feldera.enums import PipelineFieldSelector
+from feldera.rest.sql_table import SQLTable
+from feldera.rest.sql_view import SQLView
+from feldera.rest.pipeline import Pipeline
 
 
 class CallbackRunner(Thread):
@@ -20,16 +17,51 @@ class CallbackRunner(Thread):
         pipeline_name: str,
         view_name: str,
         callback: Callable[[pd.DataFrame, int], None],
-        queue: Optional[Queue],
+        exception_callback: Callable[[BaseException], None],
+        event: Event,
     ):
+        """
+        :param client: The :class:`.FelderaClient` to use.
+        :param pipeline_name: The name of the current pipeline.
+        :param view_name: The name of the view we are listening to.
+        :param callback: The callback function to call on the data we receive.
+        :param exception_callback: The callback function to call when an exception occurs.
+        :param event: The event to wait for before starting the callback runner.
+        """
+
         super().__init__()
         self.daemon = True
         self.client: FelderaClient = client
         self.pipeline_name: str = pipeline_name
         self.view_name: str = view_name
         self.callback: Callable[[pd.DataFrame, int], None] = callback
-        self.queue: Optional[Queue] = queue
-        self.schema: Optional[dict] = None
+        self.exception_callback: Callable[[BaseException], None] = exception_callback
+        self.event: Event = event
+
+        self.pipeline: Pipeline = self.client.get_pipeline(
+            self.pipeline_name, PipelineFieldSelector.ALL
+        )
+
+        view_schema = None
+
+        schemas: List[SQLTable | SQLView] = self.pipeline.tables + self.pipeline.views
+        for schema in schemas:
+            if schema.name == self.view_name:
+                view_schema = schema
+                break
+
+        if view_schema is None:
+            raise ValueError(
+                f"Table or View {self.view_name} not found in the pipeline schema."
+            )
+
+        self.schema: SQLTable | SQLView = view_schema
+
+    def to_callback(self, chunk: Mapping[str, Any]):
+        data: Optional[list[Mapping[str, Any]]] = chunk.get("json_data")
+        seq_no: Optional[int] = chunk.get("sequence_number")
+        if data is not None and seq_no is not None:
+            self.callback(dataframe_from_response([data], self.schema.fields), seq_no)
 
     def run(self):
         """
@@ -38,82 +70,26 @@ class CallbackRunner(Thread):
         :meta private:
         """
 
-        pipeline = self.client.get_pipeline(self.pipeline_name)
-        schema = pipeline.program_info["schema"]
+        try:
+            gen_obj = self.client.listen_to_pipeline(
+                self.pipeline_name,
+                self.view_name,
+                format="json",
+                case_sensitive=self.schema.case_sensitive,
+            )
 
-        if schema:
-            schemas = [relation for relation in schema["inputs"] + schema["outputs"]]
-            for schema in schemas:
-                if schema["name"] == self.view_name:
-                    self.schema = schema
-                    break
+            iterator = gen_obj()
 
-        if self.schema is None:
-            raise ValueError(
-                f"Table or View {self.view_name} not found in the pipeline schema."
-            )
+            # Trigger the HTTP call
+            chunk = next(iterator)
+
+            # Unblock the main thread
+            self.event.set()
+
+            self.to_callback(chunk)
+
+            for chunk in iterator:
+                self.to_callback(chunk)
 
-        # by default, we assume that the pipeline has been started
-        ack = _CallbackRunnerInstruction.PipelineStarted
-
-        # if there is Queue, we wait for the instruction to start the pipeline
-        # this means that we are listening to the pipeline before running it, therefore, all data should be received
-        if self.queue:
-            ack = self.queue.get()
-
-        match ack:
-            # if the pipeline has actually been started, we start a listener
-            case _CallbackRunnerInstruction.PipelineStarted:
-                # listen to the pipeline
-                gen_obj = self.client.listen_to_pipeline(
-                    self.pipeline_name, self.view_name, format="json"
-                )
-
-                # if there is a queue set up, inform the main thread that the listener has been started, and it can
-                # proceed with starting the pipeline
-                if self.queue:
-                    # stop blocking the main thread on `join` for the previous message
-                    self.queue.task_done()
-
-                iterator = gen_obj()
-
-                for chunk in iterator:
-                    chunk: dict = chunk
-                    data: Optional[list[dict]] = chunk.get("json_data")
-                    seq_no: Optional[int] = chunk.get("sequence_number")
-                    if data is not None and seq_no is not None:
-                        self.callback(
-                            dataframe_from_response([data], self.schema), seq_no
-                        )
-
-                    if self.queue:
-                        try:
-                            # if a non-blocking way, check if the queue has received further instructions
-                            # this should be a RanToCompletion instruction, which means that the pipeline has been
-                            # completed
-                            again_ack = self.queue.get_nowait()
-
-                            # if the queue has received a message
-                            if again_ack:
-                                match again_ack:
-                                    case _CallbackRunnerInstruction.RanToCompletion:
-                                        # stop blocking the main thread on `join` and return from this thread
-                                        self.queue.task_done()
-
-                                        return
-
-                                    case _CallbackRunnerInstruction.PipelineStarted:
-                                        # if the pipeline has been started again, which shouldn't happen,
-                                        # ignore it and continue listening, call `task_done` to avoid blocking the main
-                                        # thread on `join`
-                                        self.queue.task_done()
-
-                                        continue
-                        except Empty:
-                            # if the queue is empty, continue listening
-                            continue
-
-            case _CallbackRunnerInstruction.RanToCompletion:
-                if self.queue:
-                    self.queue.task_done()
-                return
+        except BaseException as e:
+            self.exception_callback(e)

feldera/_helpers.py

@@ -2,6 +2,7 @@ import uuid
 
 import pandas as pd
 from decimal import Decimal
+from typing import Mapping, Any
 
 
 def sql_type_to_pandas_type(sql_type: str):
@@ -60,9 +61,14 @@ def ensure_dataframe_has_columns(df: pd.DataFrame):
         )
 
 
-def dataframe_from_response(buffer: list[list[dict]], schema: dict):
+def dataframe_from_response(
+    buffer: list[list[Mapping[str, Any]]], fields: list[Mapping[str, Any]]
+):
     """
     Converts the response from Feldera to a pandas DataFrame.
+
+    :param buffer: A buffer of a list of JSON formatted output of the view you are listening to.
+    :param fields: The schema (list of fields) of the view you are listening to.
     """
 
     pd_schema = {}
@@ -70,7 +76,7 @@ def dataframe_from_response(buffer: list[list[dict]], schema: dict):
     decimal_col = []
     uuid_col = []
 
-    for column in schema["fields"]:
+    for column in fields:
         column_name = column["name"]
         if not column["case_sensitive"]:
             column_name = column_name.lower()

feldera/enums.py

@@ -34,137 +34,164 @@ class BuildMode(Enum):
     GET_OR_CREATE = 3
 
 
-class PipelineStatus(Enum):
+class DeploymentDesiredStatus(Enum):
+    """
+    Deployment desired status of the pipeline.
     """
-    Represents the state that this pipeline is currently in.
 
-    .. code-block:: text
+    STOPPED = 0
+    UNAVAILABLE = 1
+    STANDBY = 2
+    PAUSED = 3
+    RUNNING = 4
+    SUSPENDED = 5
 
-                    Stopped ◄─────────── Stopping ◄───── All states can transition
-                       │                    ▲            to Stopping by either:
-      /start or /pause │                    │            (1) user calling /stop?force=true, or;
-                       ▼                    │            (2) pipeline encountering a fatal
-                ⌛Provisioning          Suspending            resource or runtime error,
-                       │                    ▲                having the system call /stop?force=true
-                       ▼                    │ /stop          effectively
-                ⌛Initializing ─────────────┤  ?force=false
-                       │                    │
-             ┌─────────┼────────────────────┴─────┐
-             │         ▼                          │
-             │       Paused  ◄──────► Unavailable │
-             │        │   ▲                ▲      │
-             │ /start │   │  /pause        │      │
-             │        ▼   │                │      │
-             │       Running ◄─────────────┘      │
-             └────────────────────────────────────┘
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentDesiredStatus.__name__}"
+        )
 
-    """
 
-    NOT_FOUND = 0
+class DeploymentResourcesDesiredStatus(Enum):
     """
-    The pipeline has not been created yet.
+    The desired status of deployment resources of the pipeline.
     """
 
-    STOPPED = 1
-    """
-    The pipeline has not (yet) been started or has been stopped either
-    manually by the user or automatically by the system due to a
-    resource or runtime error.
+    STOPPED = 0
+    PROVISIONED = 1
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentResourcesDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentResourcesDesiredStatus.__name__}"
+        )
 
-        1. The user starts it via `/start` or `/pause`, transitioning to `PROVISIONING`.
-        2. Early start fails (e.g., compilation failure), transitioning to `STOPPING`.
-    """
 
-    PROVISIONING = 2
+class DeploymentResourcesStatus(Enum):
     """
-    Compute (and optionally storage) resources needed for running the pipeline
-    are being provisioned.
-
-    The pipeline remains in this state until:
-
-        1. Resources are provisioned successfully, transitioning to `INITIALIZING`.
-        2. Provisioning fails or times out, transitioning to `STOPPING`.
-        3. The user cancels the pipeline via `/stop`, transitioning to `STOPPING`.
+    The desired status of deployment resources of the pipeline.
     """
 
-    INITIALIZING = 3
-    """
-    The pipeline is initializing its internal state and connectors.
+    STOPPED = 0
+    PROVISIONING = 1
+    PROVISIONED = 2
+    STOPPING = 3
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentResourcesStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentResourcesStatus.__name__}"
+        )
 
-        1. Initialization succeeds, transitioning to `PAUSED`.
-        2. Initialization fails or times out, transitioning to `STOPPING`.
-        3. The user suspends the pipeline via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops the pipeline via `/stop`, transitioning to `STOPPING`.
-    """
 
-    PAUSED = 4
+class DeploymentRuntimeDesiredStatus(Enum):
     """
-    The pipeline is initialized but data processing is paused.
-
-    The pipeline remains in this state until:
-
-        1. The user starts it via `/start`, transitioning to `RUNNING`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-        3. The user suspends it via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops it via `/stop`, transitioning to `STOPPING`.
+    Deployment runtime desired status of the pipeline.
     """
 
-    RUNNING = 5
-    """
-    The pipeline is processing data.
+    UNAVAILABLE = 0
+    STANDBY = 1
+    PAUSED = 2
+    RUNNING = 3
+    SUSPENDED = 4
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentRuntimeDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentRuntimeDesiredStatus.__name__}"
+        )
 
-        1. The user pauses it via `/pause`, transitioning to `PAUSED`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-        3. The user suspends it via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops it via `/stop`, transitioning to `STOPPING`.
-    """
 
-    UNAVAILABLE = 6
+class DeploymentRuntimeStatus(Enum):
     """
-    The pipeline was initialized at least once but is currently unreachable
-    or not ready.
+    Deployment runtime status of the pipeline.
+    """
+
+    UNAVAILABLE = 0
+    STANDBY = 1
+    AWAITINGAPPROVAL = 2
+    INITIALIZING = 3
+    BOOTSTRAPPING = 4
+    REPLAYING = 5
+    PAUSED = 6
+    RUNNING = 7
+    SUSPENDED = 8
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentRuntimeStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentRuntimeStatus.__name__}"
+        )
 
-        1. A successful status check transitions it back to `PAUSED` or `RUNNING`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-        3. The user suspends it via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops it via `/stop`, transitioning to `STOPPING`.
 
-    Note: While in this state, `/start` or `/pause` express desired state but
-    are only applied once the pipeline becomes reachable.
+class PipelineStatus(Enum):
     """
-
-    SUSPENDING = 7
+    Represents the state that this pipeline is currently in.
     """
-    The pipeline is being suspended to storage.
 
-    The pipeline remains in this state until:
+    NOT_FOUND = 0
+    STOPPED = 1
+    PROVISIONING = 2
+    UNAVAILABLE = 3
+    STANDBY = 4
+    AWAITINGAPPROVAL = 5
+    INITIALIZING = 6
+    BOOTSTRAPPING = 7
+    REPLAYING = 8
+    PAUSED = 9
+    RUNNING = 10
+    SUSPENDED = 11
+    STOPPING = 12
 
-        1. Suspension succeeds, transitioning to `STOPPING`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-    """
+    @staticmethod
+    def from_str(value):
+        for member in PipelineStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(f"Unknown value '{value}' for enum {PipelineStatus.__name__}")
+
+    def __eq__(self, other):
+        return self.value == other.value
 
-    STOPPING = 8
-    """
-    The pipeline's compute resources are being scaled down to zero.
 
-    The pipeline remains in this state until deallocation completes,
-    transitioning to `STOPPED`.
+class TransactionStatus(Enum):
+    """
+    Represents the transaction handling status of a pipeline.
     """
 
+    NoTransaction = 1
+    """There is currently no active transaction."""
+
+    TransactionInProgress = 2
+    """There is an active transaction in progress."""
+
+    CommitInProgress = 3
+    """A commit is currently in progress."""
+
     @staticmethod
     def from_str(value):
-        for member in PipelineStatus:
+        for member in TransactionStatus:
             if member.name.lower() == value.lower():
                 return member
-        raise ValueError(f"Unknown value '{value}' for enum {PipelineStatus.__name__}")
+        raise ValueError(
+            f"Unknown value '{value}' for enum {TransactionStatus.__name__}"
+        )
 
     def __eq__(self, other):
         return self.value == other.value
@@ -311,3 +338,29 @@ class FaultToleranceModel(Enum):
         raise ValueError(
             f"Unknown value '{value}' for enum {FaultToleranceModel.__name__}"
         )
+
+
+class PipelineFieldSelector(Enum):
+    ALL = "all"
+    """Select all fields of a pipeline."""
+
+    STATUS = "status"
+    """Select only the fields required to know the status of a pipeline."""
+
+
+class BootstrapPolicy(Enum):
+    AWAIT_APPROVAL = "await_approval"
+    ALLOW = "allow"
+    REJECT = "reject"
+
+
+class CompletionTokenStatus(Enum):
+    COMPLETE = "complete"
+    """
+    Feldera has completed processing all inputs represented by this token.
+    """
+
+    IN_PROGRESS = "inprogress"
+    """
+    Feldera is still processing the inputs represented by this token.
+    """

feldera/output_handler.py

@@ -1,7 +1,8 @@
 import pandas as pd
+
 from typing import Optional
+from threading import Event
 
-from queue import Queue
 from feldera import FelderaClient
 from feldera._callback_runner import CallbackRunner
 
@@ -12,7 +13,6 @@ class OutputHandler:
         client: FelderaClient,
         pipeline_name: str,
         view_name: str,
-        queue: Optional[Queue],
     ):
         """
         Initializes the output handler, but doesn't start it.
@@ -22,17 +22,26 @@ class OutputHandler:
         self.client: FelderaClient = client
         self.pipeline_name: str = pipeline_name
         self.view_name: str = view_name
-        self.queue: Optional[Queue] = queue
         self.buffer: list[pd.DataFrame] = []
+        self.exception: Optional[BaseException] = None
+        self.event = Event()
 
         # the callback that is passed to the `CallbackRunner`
         def callback(df: pd.DataFrame, _: int):
             if not df.empty:
                 self.buffer.append(df)
 
+        def exception_callback(exception: BaseException):
+            self.exception = exception
+
         # sets up the callback runner
         self.handler = CallbackRunner(
-            self.client, self.pipeline_name, self.view_name, callback, queue
+            self.client,
+            self.pipeline_name,
+            self.view_name,
+            callback,
+            exception_callback,
+            self.event,
         )
 
     def start(self):
@@ -41,6 +50,7 @@ class OutputHandler:
         """
 
         self.handler.start()
+        _ = self.event.wait()
 
     def to_pandas(self, clear_buffer: bool = True):
         """
@@ -49,6 +59,8 @@ class OutputHandler:
         :param clear_buffer: Whether to clear the buffer after getting the output.
         """
 
+        if self.exception is not None:
+            raise self.exception
         if len(self.buffer) == 0:
             return pd.DataFrame()
         res = pd.concat(self.buffer, ignore_index=True)