feldera 0.69.0__py3-none-any.whl → 0.189.0__py3-none-any.whl

feldera/__init__.py

@@ -1,6 +1,9 @@
 from feldera.rest.feldera_client import FelderaClient as FelderaClient
 from feldera.pipeline import Pipeline as Pipeline
 from feldera.pipeline_builder import PipelineBuilder as PipelineBuilder
+from feldera.rest._helpers import determine_client_version
+
+__version__ = determine_client_version()
 
 import pretty_errors
 

feldera/_callback_runner.py

@@ -1,16 +1,13 @@
-from enum import Enum
-from threading import Thread
-from typing import Callable, Optional
-from queue import Queue, Empty
+from threading import Thread, Event
+from typing import Callable, List, Optional, Mapping, Any
 
 import pandas as pd
 from feldera import FelderaClient
 from feldera._helpers import dataframe_from_response
-
-
-class _CallbackRunnerInstruction(Enum):
-    PipelineStarted = 1
-    RanToCompletion = 2
+from feldera.enums import PipelineFieldSelector
+from feldera.rest.sql_table import SQLTable
+from feldera.rest.sql_view import SQLView
+from feldera.rest.pipeline import Pipeline
 
 
 class CallbackRunner(Thread):
@@ -20,16 +17,51 @@ class CallbackRunner(Thread):
         pipeline_name: str,
         view_name: str,
         callback: Callable[[pd.DataFrame, int], None],
-        queue: Optional[Queue],
+        exception_callback: Callable[[BaseException], None],
+        event: Event,
     ):
+        """
+        :param client: The :class:`.FelderaClient` to use.
+        :param pipeline_name: The name of the current pipeline.
+        :param view_name: The name of the view we are listening to.
+        :param callback: The callback function to call on the data we receive.
+        :param exception_callback: The callback function to call when an exception occurs.
+        :param event: The event to wait for before starting the callback runner.
+        """
+
         super().__init__()
         self.daemon = True
         self.client: FelderaClient = client
         self.pipeline_name: str = pipeline_name
         self.view_name: str = view_name
         self.callback: Callable[[pd.DataFrame, int], None] = callback
-        self.queue: Optional[Queue] = queue
-        self.schema: Optional[dict] = None
+        self.exception_callback: Callable[[BaseException], None] = exception_callback
+        self.event: Event = event
+
+        self.pipeline: Pipeline = self.client.get_pipeline(
+            self.pipeline_name, PipelineFieldSelector.ALL
+        )
+
+        view_schema = None
+
+        schemas: List[SQLTable | SQLView] = self.pipeline.tables + self.pipeline.views
+        for schema in schemas:
+            if schema.name == self.view_name:
+                view_schema = schema
+                break
+
+        if view_schema is None:
+            raise ValueError(
+                f"Table or View {self.view_name} not found in the pipeline schema."
+            )
+
+        self.schema: SQLTable | SQLView = view_schema
+
+    def to_callback(self, chunk: Mapping[str, Any]):
+        data: Optional[list[Mapping[str, Any]]] = chunk.get("json_data")
+        seq_no: Optional[int] = chunk.get("sequence_number")
+        if data is not None and seq_no is not None:
+            self.callback(dataframe_from_response([data], self.schema.fields), seq_no)
 
     def run(self):
         """
@@ -38,79 +70,26 @@ class CallbackRunner(Thread):
         :meta private:
         """
 
-        pipeline = self.client.get_pipeline(self.pipeline_name)
-        schema = pipeline.program_info["schema"]
+        try:
+            gen_obj = self.client.listen_to_pipeline(
+                self.pipeline_name,
+                self.view_name,
+                format="json",
+                case_sensitive=self.schema.case_sensitive,
+            )
 
-        if schema:
-            schemas = [relation for relation in schema["inputs"] + schema["outputs"]]
-            for schema in schemas:
-                if schema["name"] == self.view_name:
-                    self.schema = schema
-                    break
+            iterator = gen_obj()
 
-        if self.schema is None:
-            raise ValueError(
-                f"Table or View {self.view_name} not found in the pipeline schema."
-            )
+            # Trigger the HTTP call
+            chunk = next(iterator)
+
+            # Unblock the main thread
+            self.event.set()
+
+            self.to_callback(chunk)
+
+            for chunk in iterator:
+                self.to_callback(chunk)
 
-        # by default, we assume that the pipeline has been started
-        ack: _CallbackRunnerInstruction = _CallbackRunnerInstruction.PipelineStarted
-
-        # if there is Queue, we wait for the instruction to start the pipeline
-        # this means that we are listening to the pipeline before running it, therefore, all data should be received
-        if self.queue:
-            ack: _CallbackRunnerInstruction = self.queue.get()
-
-        match ack:
-            # if the pipeline has actually been started, we start a listener
-            case _CallbackRunnerInstruction.PipelineStarted:
-                # listen to the pipeline
-                gen_obj = self.client.listen_to_pipeline(
-                    self.pipeline_name, self.view_name, format="json"
-                )
-
-                # if there is a queue set up, inform the main thread that the listener has been started, and it can
-                # proceed with starting the pipeline
-                if self.queue:
-                    # stop blocking the main thread on `join` for the previous message
-                    self.queue.task_done()
-
-                for chunk in gen_obj:
-                    chunk: dict = chunk
-                    data: list[dict] = chunk.get("json_data")
-                    seq_no: int = chunk.get("sequence_number")
-
-                    if data is not None:
-                        self.callback(dataframe_from_response([data], schema), seq_no)
-
-                    if self.queue:
-                        try:
-                            # if a non-blocking way, check if the queue has received further instructions
-                            # this should be a RanToCompletion instruction, which means that the pipeline has been
-                            # completed
-                            again_ack = self.queue.get_nowait()
-
-                            # if the queue has received a message
-                            if again_ack:
-                                match again_ack:
-                                    case _CallbackRunnerInstruction.RanToCompletion:
-                                        # stop blocking the main thread on `join` and return from this thread
-                                        self.queue.task_done()
-
-                                        return
-
-                                    case _CallbackRunnerInstruction.PipelineStarted:
-                                        # if the pipeline has been started again, which shouldn't happen,
-                                        # ignore it and continue listening, call `task_done` to avoid blocking the main
-                                        # thread on `join`
-                                        self.queue.task_done()
-
-                                        continue
-                        except Empty:
-                            # if the queue is empty, continue listening
-                            continue
-
-            case _CallbackRunnerInstruction.RanToCompletion:
-                if self.queue:
-                    self.queue.task_done()
-                return
+        except BaseException as e:
+            self.exception_callback(e)

feldera/_helpers.py

@@ -2,6 +2,7 @@ import uuid
 
 import pandas as pd
 from decimal import Decimal
+from typing import Mapping, Any
 
 
 def sql_type_to_pandas_type(sql_type: str):
@@ -60,9 +61,14 @@ def ensure_dataframe_has_columns(df: pd.DataFrame):
         )
 
 
-def dataframe_from_response(buffer: list[list[dict]], schema: dict):
+def dataframe_from_response(
+    buffer: list[list[Mapping[str, Any]]], fields: list[Mapping[str, Any]]
+):
     """
     Converts the response from Feldera to a pandas DataFrame.
+
+    :param buffer: A buffer of a list of JSON formatted output of the view you are listening to.
+    :param fields: The schema (list of fields) of the view you are listening to.
     """
 
     pd_schema = {}
@@ -70,7 +76,7 @@ def dataframe_from_response(buffer: list[list[dict]], schema: dict):
     decimal_col = []
     uuid_col = []
 
-    for column in schema["fields"]:
+    for column in fields:
         column_name = column["name"]
         if not column["case_sensitive"]:
             column_name = column_name.lower()

feldera/enums.py

@@ -34,165 +34,164 @@ class BuildMode(Enum):
     GET_OR_CREATE = 3
 
 
-class PipelineStatus(Enum):
+class DeploymentDesiredStatus(Enum):
     """
-    Represents the state that this pipeline is currently in.
-
-    .. code-block:: text
-
-        Shutdown     ◄────┐
-        │         │
-        /deploy   │       │
-        │   ⌛ShuttingDown
-        ▼         ▲
-        ⌛Provisioning    │
-        │         │
-        Provisioned        │
-        ▼         │/shutdown
-        ⌛Initializing     │
-        │        │
-        ┌────────┴─────────┴─┐
-        │        ▼           │
-        │      Paused        │
-        │      │    ▲        │
-        │/start│    │/pause  │
-        │      ▼    │        │
-        │     Running        │
-        └──────────┬─────────┘
-                   │
-                   ▼
-                Failed
+    Deployment desired status of the pipeline.
     """
 
-    NOT_FOUND = 1
-    """
-    The pipeline has not been created yet.
-    """
+    STOPPED = 0
+    UNAVAILABLE = 1
+    STANDBY = 2
+    PAUSED = 3
+    RUNNING = 4
+    SUSPENDED = 5
 
-    SHUTDOWN = 2
-    """
-    Pipeline has not been started or has been shut down.
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentDesiredStatus.__name__}"
+        )
 
-    The pipeline remains in this state until the user triggers
-    a deployment by invoking the `/deploy` endpoint.
-    """
 
-    PROVISIONING = 3
+class DeploymentResourcesDesiredStatus(Enum):
+    """
+    The desired status of deployment resources of the pipeline.
     """
-    The runner triggered a deployment of the pipeline and is
-    waiting for the pipeline HTTP server to come up.
 
-    In this state, the runner provisions a runtime for the pipeline,
-    starts the pipeline within this runtime and waits for it to start accepting HTTP requests.
+    STOPPED = 0
+    PROVISIONED = 1
 
-    The user is unable to communicate with the pipeline during this
-    time.  The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentResourcesDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentResourcesDesiredStatus.__name__}"
+        )
 
-        1. Its HTTP server is up and running; the pipeline transitions to the
-           `PipelineStatus.INITIALIZING` state.
-        2. A pre-defined timeout has passed.  The runner performs forced
-           shutdown of the pipeline; returns to the `PipelineStatus.SHUTDOWN` state.
-        3. The user cancels the pipeline by invoking the `/shutdown` endpoint.
-           The manager performs forced shutdown of the pipeline, returns to the
-           `PipelineStatus.SHUTDOWN` state.
 
+class DeploymentResourcesStatus(Enum):
     """
-
-    INITIALIZING = 4
+    The desired status of deployment resources of the pipeline.
     """
-    The pipeline is initializing its internal state and connectors.
 
-    This state is part of the pipeline's deployment process.  In this state,
-    the pipeline's HTTP server is up and running, but its query engine
-    and input and output connectors are still initializing.
+    STOPPED = 0
+    PROVISIONING = 1
+    PROVISIONED = 2
+    STOPPING = 3
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentResourcesStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentResourcesStatus.__name__}"
+        )
 
-        1.  Initialization completes successfully; the pipeline transitions to the
-            `PipelineStatus.PAUSED` state.
-        2.  Initialization fails; transitions to the `PipelineStatus.FAILED` state.
-        3.  A pre-defined timeout has passed.  The runner performs forced
-            shutdown of the pipeline; returns to the `PipelineStatus.SHUTDOWN` state.
-        4.  The user cancels the pipeline by invoking the `/shutdown` endpoint.
-            The manager performs forced shutdown of the pipeline; returns to the
-            `PipelineStatus.SHUTDOWN` state.
 
+class DeploymentRuntimeDesiredStatus(Enum):
     """
-
-    PAUSED = 5
+    Deployment runtime desired status of the pipeline.
     """
-    The pipeline is fully initialized, but data processing has been paused.
 
-    The pipeline remains in this state until:
+    UNAVAILABLE = 0
+    STANDBY = 1
+    PAUSED = 2
+    RUNNING = 3
+    SUSPENDED = 4
+
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentRuntimeDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentRuntimeDesiredStatus.__name__}"
+        )
 
-        1.  The user starts the pipeline by invoking the `/start` endpoint. The
-            manager passes the request to the pipeline; transitions to the
-            `PipelineStatus.RUNNING` state.
-        2.  The user cancels the pipeline by invoking the `/shutdown` endpoint.
-            The manager passes the shutdown request to the pipeline to perform a
-            graceful shutdown; transitions to the `PipelineStatus.SHUTTING_DOWN` state.
-        3.  An unexpected runtime error renders the pipeline `PipelineStatus.FAILED`.
 
+class DeploymentRuntimeStatus(Enum):
     """
-
-    RUNNING = 6
+    Deployment runtime status of the pipeline.
     """
-    The pipeline is processing data.
 
-    The pipeline remains in this state until:
+    UNAVAILABLE = 0
+    STANDBY = 1
+    AWAITINGAPPROVAL = 2
+    INITIALIZING = 3
+    BOOTSTRAPPING = 4
+    REPLAYING = 5
+    PAUSED = 6
+    RUNNING = 7
+    SUSPENDED = 8
 
-        1. The user pauses the pipeline by invoking the `/pause` endpoint. The
-           manager passes the request to the pipeline; transitions to the
-           `PipelineStatus.PAUSED` state.
-        2. The user cancels the pipeline by invoking the `/shutdown` endpoint.
-           The runner passes the shutdown request to the pipeline to perform a
-           graceful shutdown; transitions to the
-           `PipelineStatus.SHUTTING_DOWN` state.
-        3. An unexpected runtime error renders the pipeline
-           `PipelineStatus.FAILED`.
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentRuntimeStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentRuntimeStatus.__name__}"
+        )
 
-    """
 
-    SHUTTING_DOWN = 7
+class PipelineStatus(Enum):
+    """
+    Represents the state that this pipeline is currently in.
     """
-    Graceful shutdown in progress.
 
-    In this state, the pipeline finishes any ongoing data processing,
-    produces final outputs, shuts down input/output connectors and
-    terminates.
+    NOT_FOUND = 0
+    STOPPED = 1
+    PROVISIONING = 2
+    UNAVAILABLE = 3
+    STANDBY = 4
+    AWAITINGAPPROVAL = 5
+    INITIALIZING = 6
+    BOOTSTRAPPING = 7
+    REPLAYING = 8
+    PAUSED = 9
+    RUNNING = 10
+    SUSPENDED = 11
+    STOPPING = 12
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in PipelineStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(f"Unknown value '{value}' for enum {PipelineStatus.__name__}")
 
-        1. Shutdown completes successfully; transitions to the `PipelineStatus.SHUTDOWN` state.
-        2. A pre-defined timeout has passed. The manager performs forced shutdown of the pipeline; returns to the
-           `PipelineStatus.SHUTDOWN` state.
+    def __eq__(self, other):
+        return self.value == other.value
 
-    """
 
-    FAILED = 8
+class TransactionStatus(Enum):
     """
-    The pipeline remains in this state until the users acknowledge the failure
-    by issuing a call to shutdown the pipeline; transitions to the
-    `PipelineStatus.SHUTDOWN` state.
+    Represents the transaction handling status of a pipeline.
     """
 
-    UNAVAILABLE = 9
-    """
-    The pipeline was at least once initialized, but in the most recent status check either
-    could not be reached or returned it is not yet ready.
-    """
+    NoTransaction = 1
+    """There is currently no active transaction."""
 
-    SUSPENDED = 10
-    """
-    The pipeline was successfully suspended to storage.
-    """
+    TransactionInProgress = 2
+    """There is an active transaction in progress."""
+
+    CommitInProgress = 3
+    """A commit is currently in progress."""
 
     @staticmethod
     def from_str(value):
-        for member in PipelineStatus:
+        for member in TransactionStatus:
             if member.name.lower() == value.lower():
                 return member
-        raise ValueError(f"Unknown value '{value}' for enum {PipelineStatus.__name__}")
+        raise ValueError(
+            f"Unknown value '{value}' for enum {TransactionStatus.__name__}"
+        )
 
     def __eq__(self, other):
         return self.value == other.value
@@ -258,3 +257,110 @@ class CheckpointStatus(Enum):
         """
 
         return self.error
+
+
+class StorageStatus(Enum):
+    """
+    Represents the current storage usage status of the pipeline.
+    """
+
+    CLEARED = 0
+    """
+    The pipeline has not been started before, or the user has cleared storage.
+
+    In this state, the pipeline has no storage resources bound to it.
+    """
+
+    INUSE = 1
+    """
+    The pipeline was (attempted to be) started before, transitioning from `STOPPED`
+    to `PROVISIONING`, which caused the storage status to become `INUSE`.
+
+    Being in the `INUSE` state restricts certain edits while the pipeline is `STOPPED`.
+
+    The pipeline remains in this state until the user invokes `/clear`, transitioning
+    it to `CLEARING`.
+    """
+
+    CLEARING = 2
+    """
+    The pipeline is in the process of becoming unbound from its storage resources.
+
+    If storage resources are configured to be deleted upon clearing, their deletion
+    occurs before transitioning to `CLEARED`. Otherwise, no actual work is required,
+    and the transition happens immediately.
+
+    If storage is not deleted during clearing, the responsibility to manage or delete
+    those resources lies with the user.
+    """
+
+    @staticmethod
+    def from_str(value):
+        for member in StorageStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(f"Unknown value '{value}' for enum {StorageStatus.__name__}")
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+
+class FaultToleranceModel(Enum):
+    """
+    The fault tolerance model.
+    """
+
+    AtLeastOnce = 1
+    """
+    Each record is output at least once.  Crashes may duplicate output, but
+    no input or output is dropped.
+    """
+
+    ExactlyOnce = 2
+    """
+    Each record is output exactly once.  Crashes do not drop or duplicate
+    input or output.
+    """
+
+    def __str__(self) -> str:
+        match self:
+            case FaultToleranceModel.AtLeastOnce:
+                return "at_least_once"
+            case FaultToleranceModel.ExactlyOnce:
+                return "exactly_once"
+
+    @staticmethod
+    def from_str(value):
+        for member in FaultToleranceModel:
+            if str(member) == value.lower():
+                return member
+
+        raise ValueError(
+            f"Unknown value '{value}' for enum {FaultToleranceModel.__name__}"
+        )
+
+
+class PipelineFieldSelector(Enum):
+    ALL = "all"
+    """Select all fields of a pipeline."""
+
+    STATUS = "status"
+    """Select only the fields required to know the status of a pipeline."""
+
+
+class BootstrapPolicy(Enum):
+    AWAIT_APPROVAL = "await_approval"
+    ALLOW = "allow"
+    REJECT = "reject"
+
+
+class CompletionTokenStatus(Enum):
+    COMPLETE = "complete"
+    """
+    Feldera has completed processing all inputs represented by this token.
+    """
+
+    IN_PROGRESS = "inprogress"
+    """
+    Feldera is still processing the inputs represented by this token.
+    """

feldera/output_handler.py

@@ -1,7 +1,8 @@
 import pandas as pd
+
 from typing import Optional
+from threading import Event
 
-from queue import Queue
 from feldera import FelderaClient
 from feldera._callback_runner import CallbackRunner
 
@@ -12,7 +13,6 @@ class OutputHandler:
         client: FelderaClient,
         pipeline_name: str,
         view_name: str,
-        queue: Optional[Queue],
     ):
         """
         Initializes the output handler, but doesn't start it.
@@ -22,17 +22,26 @@ class OutputHandler:
         self.client: FelderaClient = client
         self.pipeline_name: str = pipeline_name
         self.view_name: str = view_name
-        self.queue: Optional[Queue] = queue
         self.buffer: list[pd.DataFrame] = []
+        self.exception: Optional[BaseException] = None
+        self.event = Event()
 
         # the callback that is passed to the `CallbackRunner`
         def callback(df: pd.DataFrame, _: int):
             if not df.empty:
                 self.buffer.append(df)
 
+        def exception_callback(exception: BaseException):
+            self.exception = exception
+
         # sets up the callback runner
         self.handler = CallbackRunner(
-            self.client, self.pipeline_name, self.view_name, callback, queue
+            self.client,
+            self.pipeline_name,
+            self.view_name,
+            callback,
+            exception_callback,
+            self.event,
         )
 
     def start(self):
@@ -41,6 +50,7 @@ class OutputHandler:
         """
 
         self.handler.start()
+        _ = self.event.wait()
 
     def to_pandas(self, clear_buffer: bool = True):
         """
@@ -49,6 +59,8 @@ class OutputHandler:
         :param clear_buffer: Whether to clear the buffer after getting the output.
         """
 
+        if self.exception is not None:
+            raise self.exception
         if len(self.buffer) == 0:
             return pd.DataFrame()
         res = pd.concat(self.buffer, ignore_index=True)