feldera 0.118.0__tar.gz → 0.179.0__tar.gz

feldera-0.179.0/PKG-INFO

@@ -0,0 +1,97 @@
+Metadata-Version: 2.4
+Name: feldera
+Version: 0.179.0
+Summary: The feldera python client
+Author-email: Feldera Team <dev@feldera.com>
+License: MIT
+Project-URL: Homepage, https://www.feldera.com
+Project-URL: Documentation, https://docs.feldera.com/python
+Project-URL: Repository, https://github.com/feldera/feldera
+Project-URL: Issues, https://github.com/feldera/feldera/issues
+Keywords: feldera,python
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: requests
+Requires-Dist: pandas>=2.1.2
+Requires-Dist: typing-extensions
+Requires-Dist: numpy>=2.2.4
+Requires-Dist: pretty-errors
+Requires-Dist: ruff>=0.6.9
+Requires-Dist: PyJWT>=2.8.0
+
+# Feldera Python SDK
+
+Feldera Python is the Feldera SDK for Python developers.
+
+## Installation
+
+```bash
+uv pip install feldera
+```
+
+### Installing from Github
+
+```bash
+uv pip install git+https://github.com/feldera/feldera#subdirectory=python
+```
+
+Similarly, to install from a specific branch:
+
+```bash
+uv pip install git+https://github.com/feldera/feldera@{BRANCH_NAME}#subdirectory=python
+```
+
+Replace `{BRANCH_NAME}` with the name of the branch you want to install from.
+
+### Installing from Local Directory
+
+If you have cloned the Feldera repo, you can install the python SDK as follows:
+
+```bash
+# the Feldera Python SDK is present inside the python/ directory
+cd python
+# If you don't have a virtual environment, create one
+uv venv
+source .venv/activate
+# Install the SDK in editable mode
+uv pip install .
+```
+
+## Documentation
+
+The Python SDK documentation is available at
+[Feldera Python SDK Docs](https://docs.feldera.com/python).
+
+To build the html documentation run:
+
+Ensure that you have sphinx installed. If not, install it using `uv pip install sphinx`.
+
+Then run the following commands:
+
+```bash
+cd docs
+sphinx-apidoc -o . ../feldera
+make html
+```
+
+To clean the build, run `make clean`.
+
+## Linting and formatting
+
+Use [Ruff] to run the lint checks that will be executed by the
+precommit hook when a PR is submitted:
+
+```bash
+ruff check python/
+```
+
+To reformat the code in the same way as the precommit hook:
+
+```bash
+ruff format
+```
+
+[Ruff]: https://github.com/astral-sh/ruff

feldera-0.179.0/README.md

@@ -0,0 +1,73 @@
+# Feldera Python SDK
+
+Feldera Python is the Feldera SDK for Python developers.
+
+## Installation
+
+```bash
+uv pip install feldera
+```
+
+### Installing from Github
+
+```bash
+uv pip install git+https://github.com/feldera/feldera#subdirectory=python
+```
+
+Similarly, to install from a specific branch:
+
+```bash
+uv pip install git+https://github.com/feldera/feldera@{BRANCH_NAME}#subdirectory=python
+```
+
+Replace `{BRANCH_NAME}` with the name of the branch you want to install from.
+
+### Installing from Local Directory
+
+If you have cloned the Feldera repo, you can install the python SDK as follows:
+
+```bash
+# the Feldera Python SDK is present inside the python/ directory
+cd python
+# If you don't have a virtual environment, create one
+uv venv
+source .venv/activate
+# Install the SDK in editable mode
+uv pip install .
+```
+
+## Documentation
+
+The Python SDK documentation is available at
+[Feldera Python SDK Docs](https://docs.feldera.com/python).
+
+To build the html documentation run:
+
+Ensure that you have sphinx installed. If not, install it using `uv pip install sphinx`.
+
+Then run the following commands:
+
+```bash
+cd docs
+sphinx-apidoc -o . ../feldera
+make html
+```
+
+To clean the build, run `make clean`.
+
+## Linting and formatting
+
+Use [Ruff] to run the lint checks that will be executed by the
+precommit hook when a PR is submitted:
+
+```bash
+ruff check python/
+```
+
+To reformat the code in the same way as the precommit hook:
+
+```bash
+ruff format
+```
+
+[Ruff]: https://github.com/astral-sh/ruff

feldera-0.179.0/feldera/_callback_runner.py

@@ -0,0 +1,95 @@
+from threading import Thread, Event
+from typing import Callable, List, Optional, Mapping, Any
+
+import pandas as pd
+from feldera import FelderaClient
+from feldera._helpers import dataframe_from_response
+from feldera.enums import PipelineFieldSelector
+from feldera.rest.sql_table import SQLTable
+from feldera.rest.sql_view import SQLView
+from feldera.rest.pipeline import Pipeline
+
+
+class CallbackRunner(Thread):
+    def __init__(
+        self,
+        client: FelderaClient,
+        pipeline_name: str,
+        view_name: str,
+        callback: Callable[[pd.DataFrame, int], None],
+        exception_callback: Callable[[BaseException], None],
+        event: Event,
+    ):
+        """
+        :param client: The :class:`.FelderaClient` to use.
+        :param pipeline_name: The name of the current pipeline.
+        :param view_name: The name of the view we are listening to.
+        :param callback: The callback function to call on the data we receive.
+        :param exception_callback: The callback function to call when an exception occurs.
+        :param event: The event to wait for before starting the callback runner.
+        """
+
+        super().__init__()
+        self.daemon = True
+        self.client: FelderaClient = client
+        self.pipeline_name: str = pipeline_name
+        self.view_name: str = view_name
+        self.callback: Callable[[pd.DataFrame, int], None] = callback
+        self.exception_callback: Callable[[BaseException], None] = exception_callback
+        self.event: Event = event
+
+        self.pipeline: Pipeline = self.client.get_pipeline(
+            self.pipeline_name, PipelineFieldSelector.ALL
+        )
+
+        view_schema = None
+
+        schemas: List[SQLTable | SQLView] = self.pipeline.tables + self.pipeline.views
+        for schema in schemas:
+            if schema.name == self.view_name:
+                view_schema = schema
+                break
+
+        if view_schema is None:
+            raise ValueError(
+                f"Table or View {self.view_name} not found in the pipeline schema."
+            )
+
+        self.schema: SQLTable | SQLView = view_schema
+
+    def to_callback(self, chunk: Mapping[str, Any]):
+        data: Optional[list[Mapping[str, Any]]] = chunk.get("json_data")
+        seq_no: Optional[int] = chunk.get("sequence_number")
+        if data is not None and seq_no is not None:
+            self.callback(dataframe_from_response([data], self.schema.fields), seq_no)
+
+    def run(self):
+        """
+        The main loop of the thread. Listens for data and calls the callback function on each chunk of data received.
+
+        :meta private:
+        """
+
+        try:
+            gen_obj = self.client.listen_to_pipeline(
+                self.pipeline_name,
+                self.view_name,
+                format="json",
+                case_sensitive=self.schema.case_sensitive,
+            )
+
+            iterator = gen_obj()
+
+            # Trigger the HTTP call
+            chunk = next(iterator)
+
+            # Unblock the main thread
+            self.event.set()
+
+            self.to_callback(chunk)
+
+            for chunk in iterator:
+                self.to_callback(chunk)
+
+        except BaseException as e:
+            self.exception_callback(e)

{feldera-0.118.0 → feldera-0.179.0}/feldera/_helpers.py

@@ -2,6 +2,7 @@ import uuid
 
 import pandas as pd
 from decimal import Decimal
+from typing import Mapping, Any
 
 
 def sql_type_to_pandas_type(sql_type: str):
@@ -60,9 +61,14 @@ def ensure_dataframe_has_columns(df: pd.DataFrame):
         )
 
 
-def dataframe_from_response(buffer: list[list[dict]], schema: dict):
+def dataframe_from_response(
+    buffer: list[list[Mapping[str, Any]]], fields: list[Mapping[str, Any]]
+):
     """
     Converts the response from Feldera to a pandas DataFrame.
+
+    :param buffer: A buffer of a list of JSON formatted output of the view you are listening to.
+    :param fields: The schema (list of fields) of the view you are listening to.
     """
 
     pd_schema = {}
@@ -70,7 +76,7 @@ def dataframe_from_response(buffer: list[list[dict]], schema: dict):
     decimal_col = []
     uuid_col = []
 
-    for column in schema["fields"]:
+    for column in fields:
         column_name = column["name"]
         if not column["case_sensitive"]:
             column_name = column_name.lower()

{feldera-0.118.0 → feldera-0.179.0}/feldera/enums.py

@@ -34,137 +34,164 @@ class BuildMode(Enum):
     GET_OR_CREATE = 3
 
 
-class PipelineStatus(Enum):
+class DeploymentDesiredStatus(Enum):
+    """
+    Deployment desired status of the pipeline.
     """
-    Represents the state that this pipeline is currently in.
 
-    .. code-block:: text
+    STOPPED = 0
+    UNAVAILABLE = 1
+    STANDBY = 2
+    PAUSED = 3
+    RUNNING = 4
+    SUSPENDED = 5
 
-                    Stopped ◄─────────── Stopping ◄───── All states can transition
-                       │                    ▲            to Stopping by either:
-      /start or /pause │                    │            (1) user calling /stop?force=true, or;
-                       ▼                    │            (2) pipeline encountering a fatal
-                ⌛Provisioning          Suspending            resource or runtime error,
-                       │                    ▲                having the system call /stop?force=true
-                       ▼                    │ /stop          effectively
-                ⌛Initializing ─────────────┤  ?force=false
-                       │                    │
-             ┌─────────┼────────────────────┴─────┐
-             │         ▼                          │
-             │       Paused  ◄──────► Unavailable │
-             │        │   ▲                ▲      │
-             │ /start │   │  /pause        │      │
-             │        ▼   │                │      │
-             │       Running ◄─────────────┘      │
-             └────────────────────────────────────┘
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentDesiredStatus.__name__}"
+        )
 
-    """
 
-    NOT_FOUND = 0
+class DeploymentResourcesDesiredStatus(Enum):
     """
-    The pipeline has not been created yet.
+    The desired status of deployment resources of the pipeline.
     """
 
-    STOPPED = 1
-    """
-    The pipeline has not (yet) been started or has been stopped either
-    manually by the user or automatically by the system due to a
-    resource or runtime error.
+    STOPPED = 0
+    PROVISIONED = 1
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentResourcesDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentResourcesDesiredStatus.__name__}"
+        )
 
-        1. The user starts it via `/start` or `/pause`, transitioning to `PROVISIONING`.
-        2. Early start fails (e.g., compilation failure), transitioning to `STOPPING`.
-    """
 
-    PROVISIONING = 2
+class DeploymentResourcesStatus(Enum):
     """
-    Compute (and optionally storage) resources needed for running the pipeline
-    are being provisioned.
-
-    The pipeline remains in this state until:
-
-        1. Resources are provisioned successfully, transitioning to `INITIALIZING`.
-        2. Provisioning fails or times out, transitioning to `STOPPING`.
-        3. The user cancels the pipeline via `/stop`, transitioning to `STOPPING`.
+    The desired status of deployment resources of the pipeline.
     """
 
-    INITIALIZING = 3
-    """
-    The pipeline is initializing its internal state and connectors.
+    STOPPED = 0
+    PROVISIONING = 1
+    PROVISIONED = 2
+    STOPPING = 3
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentResourcesStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentResourcesStatus.__name__}"
+        )
 
-        1. Initialization succeeds, transitioning to `PAUSED`.
-        2. Initialization fails or times out, transitioning to `STOPPING`.
-        3. The user suspends the pipeline via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops the pipeline via `/stop`, transitioning to `STOPPING`.
-    """
 
-    PAUSED = 4
+class DeploymentRuntimeDesiredStatus(Enum):
     """
-    The pipeline is initialized but data processing is paused.
-
-    The pipeline remains in this state until:
-
-        1. The user starts it via `/start`, transitioning to `RUNNING`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-        3. The user suspends it via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops it via `/stop`, transitioning to `STOPPING`.
+    Deployment runtime desired status of the pipeline.
     """
 
-    RUNNING = 5
-    """
-    The pipeline is processing data.
+    UNAVAILABLE = 0
+    STANDBY = 1
+    PAUSED = 2
+    RUNNING = 3
+    SUSPENDED = 4
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentRuntimeDesiredStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentRuntimeDesiredStatus.__name__}"
+        )
 
-        1. The user pauses it via `/pause`, transitioning to `PAUSED`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-        3. The user suspends it via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops it via `/stop`, transitioning to `STOPPING`.
-    """
 
-    UNAVAILABLE = 6
+class DeploymentRuntimeStatus(Enum):
     """
-    The pipeline was initialized at least once but is currently unreachable
-    or not ready.
+    Deployment runtime status of the pipeline.
+    """
+
+    UNAVAILABLE = 0
+    STANDBY = 1
+    AWAITINGAPPROVAL = 2
+    INITIALIZING = 3
+    BOOTSTRAPPING = 4
+    REPLAYING = 5
+    PAUSED = 6
+    RUNNING = 7
+    SUSPENDED = 8
 
-    The pipeline remains in this state until:
+    @staticmethod
+    def from_str(value):
+        for member in DeploymentRuntimeStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(
+            f"Unknown value '{value}' for enum {DeploymentRuntimeStatus.__name__}"
+        )
 
-        1. A successful status check transitions it back to `PAUSED` or `RUNNING`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-        3. The user suspends it via `/suspend`, transitioning to `SUSPENDING`.
-        4. The user stops it via `/stop`, transitioning to `STOPPING`.
 
-    Note: While in this state, `/start` or `/pause` express desired state but
-    are only applied once the pipeline becomes reachable.
+class PipelineStatus(Enum):
     """
-
-    SUSPENDING = 7
+    Represents the state that this pipeline is currently in.
     """
-    The pipeline is being suspended to storage.
 
-    The pipeline remains in this state until:
+    NOT_FOUND = 0
+    STOPPED = 1
+    PROVISIONING = 2
+    UNAVAILABLE = 3
+    STANDBY = 4
+    AWAITINGAPPROVAL = 5
+    INITIALIZING = 6
+    BOOTSTRAPPING = 7
+    REPLAYING = 8
+    PAUSED = 9
+    RUNNING = 10
+    SUSPENDED = 11
+    STOPPING = 12
 
-        1. Suspension succeeds, transitioning to `STOPPING`.
-        2. A runtime error occurs, transitioning to `STOPPING`.
-    """
+    @staticmethod
+    def from_str(value):
+        for member in PipelineStatus:
+            if member.name.lower() == value.lower():
+                return member
+        raise ValueError(f"Unknown value '{value}' for enum {PipelineStatus.__name__}")
+
+    def __eq__(self, other):
+        return self.value == other.value
 
-    STOPPING = 8
-    """
-    The pipeline's compute resources are being scaled down to zero.
 
-    The pipeline remains in this state until deallocation completes,
-    transitioning to `STOPPED`.
+class TransactionStatus(Enum):
+    """
+    Represents the transaction handling status of a pipeline.
     """
 
+    NoTransaction = 1
+    """There is currently no active transaction."""
+
+    TransactionInProgress = 2
+    """There is an active transaction in progress."""
+
+    CommitInProgress = 3
+    """A commit is currently in progress."""
+
     @staticmethod
     def from_str(value):
-        for member in PipelineStatus:
+        for member in TransactionStatus:
             if member.name.lower() == value.lower():
                 return member
-        raise ValueError(f"Unknown value '{value}' for enum {PipelineStatus.__name__}")
+        raise ValueError(
+            f"Unknown value '{value}' for enum {TransactionStatus.__name__}"
+        )
 
     def __eq__(self, other):
         return self.value == other.value
@@ -311,3 +338,29 @@ class FaultToleranceModel(Enum):
         raise ValueError(
             f"Unknown value '{value}' for enum {FaultToleranceModel.__name__}"
         )
+
+
+class PipelineFieldSelector(Enum):
+    ALL = "all"
+    """Select all fields of a pipeline."""
+
+    STATUS = "status"
+    """Select only the fields required to know the status of a pipeline."""
+
+
+class BootstrapPolicy(Enum):
+    AWAIT_APPROVAL = "await_approval"
+    ALLOW = "allow"
+    REJECT = "reject"
+
+
+class CompletionTokenStatus(Enum):
+    COMPLETE = "complete"
+    """
+    Feldera has completed processing all inputs represented by this token.
+    """
+
+    IN_PROGRESS = "inprogress"
+    """
+    Feldera is still processing the inputs represented by this token.
+    """

{feldera-0.118.0 → feldera-0.179.0}/feldera/output_handler.py

@@ -1,7 +1,8 @@
 import pandas as pd
+
 from typing import Optional
+from threading import Event
 
-from queue import Queue
 from feldera import FelderaClient
 from feldera._callback_runner import CallbackRunner
 
@@ -12,7 +13,6 @@ class OutputHandler:
         client: FelderaClient,
         pipeline_name: str,
         view_name: str,
-        queue: Optional[Queue],
     ):
         """
         Initializes the output handler, but doesn't start it.
@@ -22,17 +22,26 @@ class OutputHandler:
         self.client: FelderaClient = client
         self.pipeline_name: str = pipeline_name
         self.view_name: str = view_name
-        self.queue: Optional[Queue] = queue
         self.buffer: list[pd.DataFrame] = []
+        self.exception: Optional[BaseException] = None
+        self.event = Event()
 
         # the callback that is passed to the `CallbackRunner`
         def callback(df: pd.DataFrame, _: int):
             if not df.empty:
                 self.buffer.append(df)
 
+        def exception_callback(exception: BaseException):
+            self.exception = exception
+
         # sets up the callback runner
         self.handler = CallbackRunner(
-            self.client, self.pipeline_name, self.view_name, callback, queue
+            self.client,
+            self.pipeline_name,
+            self.view_name,
+            callback,
+            exception_callback,
+            self.event,
         )
 
     def start(self):
@@ -41,6 +50,7 @@ class OutputHandler:
         """
 
         self.handler.start()
+        _ = self.event.wait()
 
     def to_pandas(self, clear_buffer: bool = True):
         """
@@ -49,6 +59,8 @@ class OutputHandler:
         :param clear_buffer: Whether to clear the buffer after getting the output.
         """
 
+        if self.exception is not None:
+            raise self.exception
         if len(self.buffer) == 0:
             return pd.DataFrame()
         res = pd.concat(self.buffer, ignore_index=True)