feldera 0.153.0__tar.gz → 0.186.0__tar.gz

feldera-0.186.0/PKG-INFO

@@ -0,0 +1,163 @@
+Metadata-Version: 2.4
+Name: feldera
+Version: 0.186.0
+Summary: The feldera python client
+Author-email: Feldera Team <dev@feldera.com>
+License: MIT
+Project-URL: Homepage, https://www.feldera.com
+Project-URL: Documentation, https://docs.feldera.com/python
+Project-URL: Repository, https://github.com/feldera/feldera
+Project-URL: Issues, https://github.com/feldera/feldera/issues
+Keywords: feldera,python
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: requests
+Requires-Dist: pandas>=2.1.2
+Requires-Dist: typing-extensions
+Requires-Dist: numpy>=2.2.4
+Requires-Dist: pretty-errors
+Requires-Dist: ruff>=0.6.9
+Requires-Dist: PyJWT>=2.8.0
+
+# Feldera Python SDK
+
+The `feldera` Python package is the Python client for the Feldera HTTP API.
+
+The Python SDK documentation is available at: https://docs.feldera.com/python
+
+## Getting started
+
+### Installation
+
+```bash
+uv pip install feldera
+```
+
+### Example usage
+
+The Python client interacts with the API server of the Feldera instance.
+
+```python
+# File: example.py
+from feldera import FelderaClient, PipelineBuilder, Pipeline
+
+# Instantiate client
+client = FelderaClient()  # Default: http://localhost:8080 without authentication
+# client = FelderaClient(url="https://localhost:8080", api_key="apikey:...", requests_verify="/path/to/tls.crt")
+
+# (Re)create pipeline
+name = "example"
+sql = """
+CREATE TABLE t1 (i1 INT) WITH ('materialized' = 'true');
+CREATE MATERIALIZED VIEW v1 AS SELECT * FROM t1;
+"""
+print("(Re)creating pipeline...")
+pipeline = PipelineBuilder(client, name, sql).create_or_replace()
+pipeline.start()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.pause()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.stop(force=True)
+
+# Find existing pipeline
+pipeline = Pipeline.get(name, client)
+pipeline.start()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.stop(force=True)
+pipeline.clear_storage()
+```
+
+Run using:
+```bash
+uv run python example.py
+```
+
+### Environment variables
+
+Some default parameter values in the Python SDK can be overridden via environment variables.
+
+**Environment variables for `FelderaClient(...)`**
+
+```bash
+export FELDERA_HOST="https://localhost:8080"  # Overrides default for `url`
+export FELDERA_API_KEY="apikey:..."  # Overrides default for `api_key`
+
+# The following together override default for `requests_verify`
+# export FELDERA_TLS_INSECURE="false"  # If set to "1", "true" or "yes" (all case-insensitive), disables TLS certificate verification
+# export FELDERA_HTTPS_TLS_CERT="/path/to/tls.crt"  # Custom TLS certificate
+```
+
+**Environment variables for `PipelineBuilder(...)`**
+
+```bash
+export FELDERA_RUNTIME_VERSION="..."  # Overrides default for `runtime_version`
+```
+
+## Development
+
+Development assumes you have cloned the Feldera code repository.
+
+### Installation
+
+```bash
+cd python
+# Optional: create and activate virtual environment if you don't have one
+uv venv
+source .venv/bin/activate
+# Install in editable mode
+uv pip install -e .
+```
+
+### Formatting
+
+Formatting requires the `ruff` package: `uv pip install ruff`
+
+```bash
+cd python
+ruff check
+ruff format
+```
+
+### Tests
+
+Running the test requires the `pytest` package: `uv pip install pytest`
+
+```bash
+# All tests
+cd python
+uv run python -m pytest tests/
+
+# Specific tests directory
+uv run python -m pytest tests/platform/
+
+# Specific test file
+uv run python -m pytest tests/platform/test_pipeline_crud.py
+```
+
+For further information about the tests, please see `tests/README.md`.
+
+### Documentation
+
+Building documentation requires the `sphinx` package: `uv pip install sphinx`
+
+```bash
+cd python/docs
+sphinx-apidoc -o . ../feldera
+make html
+make clean  # Cleanup afterwards
+```
+
+### Installation from GitHub
+
+Latest `main` branch:
+```bash
+uv pip install git+https://github.com/feldera/feldera#subdirectory=python
+```
+
+Different branch (replace `BRANCH_NAME`):
+```bash
+uv pip install git+https://github.com/feldera/feldera@BRANCH_NAME#subdirectory=python
+```

feldera-0.186.0/README.md

@@ -0,0 +1,139 @@
+# Feldera Python SDK
+
+The `feldera` Python package is the Python client for the Feldera HTTP API.
+
+The Python SDK documentation is available at: https://docs.feldera.com/python
+
+## Getting started
+
+### Installation
+
+```bash
+uv pip install feldera
+```
+
+### Example usage
+
+The Python client interacts with the API server of the Feldera instance.
+
+```python
+# File: example.py
+from feldera import FelderaClient, PipelineBuilder, Pipeline
+
+# Instantiate client
+client = FelderaClient()  # Default: http://localhost:8080 without authentication
+# client = FelderaClient(url="https://localhost:8080", api_key="apikey:...", requests_verify="/path/to/tls.crt")
+
+# (Re)create pipeline
+name = "example"
+sql = """
+CREATE TABLE t1 (i1 INT) WITH ('materialized' = 'true');
+CREATE MATERIALIZED VIEW v1 AS SELECT * FROM t1;
+"""
+print("(Re)creating pipeline...")
+pipeline = PipelineBuilder(client, name, sql).create_or_replace()
+pipeline.start()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.pause()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.stop(force=True)
+
+# Find existing pipeline
+pipeline = Pipeline.get(name, client)
+pipeline.start()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.stop(force=True)
+pipeline.clear_storage()
+```
+
+Run using:
+```bash
+uv run python example.py
+```
+
+### Environment variables
+
+Some default parameter values in the Python SDK can be overridden via environment variables.
+
+**Environment variables for `FelderaClient(...)`**
+
+```bash
+export FELDERA_HOST="https://localhost:8080"  # Overrides default for `url`
+export FELDERA_API_KEY="apikey:..."  # Overrides default for `api_key`
+
+# The following together override default for `requests_verify`
+# export FELDERA_TLS_INSECURE="false"  # If set to "1", "true" or "yes" (all case-insensitive), disables TLS certificate verification
+# export FELDERA_HTTPS_TLS_CERT="/path/to/tls.crt"  # Custom TLS certificate
+```
+
+**Environment variables for `PipelineBuilder(...)`**
+
+```bash
+export FELDERA_RUNTIME_VERSION="..."  # Overrides default for `runtime_version`
+```
+
+## Development
+
+Development assumes you have cloned the Feldera code repository.
+
+### Installation
+
+```bash
+cd python
+# Optional: create and activate virtual environment if you don't have one
+uv venv
+source .venv/bin/activate
+# Install in editable mode
+uv pip install -e .
+```
+
+### Formatting
+
+Formatting requires the `ruff` package: `uv pip install ruff`
+
+```bash
+cd python
+ruff check
+ruff format
+```
+
+### Tests
+
+Running the test requires the `pytest` package: `uv pip install pytest`
+
+```bash
+# All tests
+cd python
+uv run python -m pytest tests/
+
+# Specific tests directory
+uv run python -m pytest tests/platform/
+
+# Specific test file
+uv run python -m pytest tests/platform/test_pipeline_crud.py
+```
+
+For further information about the tests, please see `tests/README.md`.
+
+### Documentation
+
+Building documentation requires the `sphinx` package: `uv pip install sphinx`
+
+```bash
+cd python/docs
+sphinx-apidoc -o . ../feldera
+make html
+make clean  # Cleanup afterwards
+```
+
+### Installation from GitHub
+
+Latest `main` branch:
+```bash
+uv pip install git+https://github.com/feldera/feldera#subdirectory=python
+```
+
+Different branch (replace `BRANCH_NAME`):
+```bash
+uv pip install git+https://github.com/feldera/feldera@BRANCH_NAME#subdirectory=python
+```

{feldera-0.153.0 → feldera-0.186.0}/feldera/__init__.py

@@ -1,9 +1,9 @@
 from feldera.rest.feldera_client import FelderaClient as FelderaClient
 from feldera.pipeline import Pipeline as Pipeline
 from feldera.pipeline_builder import PipelineBuilder as PipelineBuilder
-from feldera.rest._helpers import client_version
+from feldera.rest._helpers import determine_client_version
 
-__version__ = client_version()
+__version__ = determine_client_version()
 
 import pretty_errors
 

feldera-0.186.0/feldera/_callback_runner.py

@@ -0,0 +1,95 @@
+from threading import Thread, Event
+from typing import Callable, List, Optional, Mapping, Any
+
+import pandas as pd
+from feldera import FelderaClient
+from feldera._helpers import dataframe_from_response
+from feldera.enums import PipelineFieldSelector
+from feldera.rest.sql_table import SQLTable
+from feldera.rest.sql_view import SQLView
+from feldera.rest.pipeline import Pipeline
+
+
+class CallbackRunner(Thread):
+    def __init__(
+        self,
+        client: FelderaClient,
+        pipeline_name: str,
+        view_name: str,
+        callback: Callable[[pd.DataFrame, int], None],
+        exception_callback: Callable[[BaseException], None],
+        event: Event,
+    ):
+        """
+        :param client: The :class:`.FelderaClient` to use.
+        :param pipeline_name: The name of the current pipeline.
+        :param view_name: The name of the view we are listening to.
+        :param callback: The callback function to call on the data we receive.
+        :param exception_callback: The callback function to call when an exception occurs.
+        :param event: The event to wait for before starting the callback runner.
+        """
+
+        super().__init__()
+        self.daemon = True
+        self.client: FelderaClient = client
+        self.pipeline_name: str = pipeline_name
+        self.view_name: str = view_name
+        self.callback: Callable[[pd.DataFrame, int], None] = callback
+        self.exception_callback: Callable[[BaseException], None] = exception_callback
+        self.event: Event = event
+
+        self.pipeline: Pipeline = self.client.get_pipeline(
+            self.pipeline_name, PipelineFieldSelector.ALL
+        )
+
+        view_schema = None
+
+        schemas: List[SQLTable | SQLView] = self.pipeline.tables + self.pipeline.views
+        for schema in schemas:
+            if schema.name == self.view_name:
+                view_schema = schema
+                break
+
+        if view_schema is None:
+            raise ValueError(
+                f"Table or View {self.view_name} not found in the pipeline schema."
+            )
+
+        self.schema: SQLTable | SQLView = view_schema
+
+    def to_callback(self, chunk: Mapping[str, Any]):
+        data: Optional[list[Mapping[str, Any]]] = chunk.get("json_data")
+        seq_no: Optional[int] = chunk.get("sequence_number")
+        if data is not None and seq_no is not None:
+            self.callback(dataframe_from_response([data], self.schema.fields), seq_no)
+
+    def run(self):
+        """
+        The main loop of the thread. Listens for data and calls the callback function on each chunk of data received.
+
+        :meta private:
+        """
+
+        try:
+            gen_obj = self.client.listen_to_pipeline(
+                self.pipeline_name,
+                self.view_name,
+                format="json",
+                case_sensitive=self.schema.case_sensitive,
+            )
+
+            iterator = gen_obj()
+
+            # Trigger the HTTP call
+            chunk = next(iterator)
+
+            # Unblock the main thread
+            self.event.set()
+
+            self.to_callback(chunk)
+
+            for chunk in iterator:
+                self.to_callback(chunk)
+
+        except BaseException as e:
+            self.exception_callback(e)

{feldera-0.153.0 → feldera-0.186.0}/feldera/_helpers.py

@@ -2,6 +2,7 @@ import uuid
 
 import pandas as pd
 from decimal import Decimal
+from typing import Mapping, Any
 
 
 def sql_type_to_pandas_type(sql_type: str):
@@ -60,9 +61,14 @@ def ensure_dataframe_has_columns(df: pd.DataFrame):
         )
 
 
-def dataframe_from_response(buffer: list[list[dict]], fields: list[dict]):
+def dataframe_from_response(
+    buffer: list[list[Mapping[str, Any]]], fields: list[Mapping[str, Any]]
+):
     """
     Converts the response from Feldera to a pandas DataFrame.
+
+    :param buffer: A buffer of a list of JSON formatted output of the view you are listening to.
+    :param fields: The schema (list of fields) of the view you are listening to.
     """
 
     pd_schema = {}

{feldera-0.153.0 → feldera-0.186.0}/feldera/enums.py

@@ -122,12 +122,13 @@ class DeploymentRuntimeStatus(Enum):
 
     UNAVAILABLE = 0
     STANDBY = 1
-    INITIALIZING = 2
-    BOOTSTRAPPING = 3
-    REPLAYING = 4
-    PAUSED = 5
-    RUNNING = 6
-    SUSPENDED = 7
+    AWAITINGAPPROVAL = 2
+    INITIALIZING = 3
+    BOOTSTRAPPING = 4
+    REPLAYING = 5
+    PAUSED = 6
+    RUNNING = 7
+    SUSPENDED = 8
 
     @staticmethod
     def from_str(value):
@@ -149,13 +150,14 @@ class PipelineStatus(Enum):
     PROVISIONING = 2
     UNAVAILABLE = 3
     STANDBY = 4
-    INITIALIZING = 5
-    BOOTSTRAPPING = 6
-    REPLAYING = 7
-    PAUSED = 8
-    RUNNING = 9
-    SUSPENDED = 10
-    STOPPING = 11
+    AWAITINGAPPROVAL = 5
+    INITIALIZING = 6
+    BOOTSTRAPPING = 7
+    REPLAYING = 8
+    PAUSED = 9
+    RUNNING = 10
+    SUSPENDED = 11
+    STOPPING = 12
 
     @staticmethod
     def from_str(value):
@@ -344,3 +346,21 @@ class PipelineFieldSelector(Enum):
 
     STATUS = "status"
     """Select only the fields required to know the status of a pipeline."""
+
+
+class BootstrapPolicy(Enum):
+    AWAIT_APPROVAL = "await_approval"
+    ALLOW = "allow"
+    REJECT = "reject"
+
+
+class CompletionTokenStatus(Enum):
+    COMPLETE = "complete"
+    """
+    Feldera has completed processing all inputs represented by this token.
+    """
+
+    IN_PROGRESS = "inprogress"
+    """
+    Feldera is still processing the inputs represented by this token.
+    """

{feldera-0.153.0 → feldera-0.186.0}/feldera/output_handler.py

@@ -1,7 +1,8 @@
 import pandas as pd
+
 from typing import Optional
+from threading import Event
 
-from queue import Queue
 from feldera import FelderaClient
 from feldera._callback_runner import CallbackRunner
 
@@ -12,7 +13,6 @@ class OutputHandler:
         client: FelderaClient,
         pipeline_name: str,
         view_name: str,
-        queue: Optional[Queue],
     ):
         """
         Initializes the output handler, but doesn't start it.
@@ -22,17 +22,26 @@ class OutputHandler:
         self.client: FelderaClient = client
         self.pipeline_name: str = pipeline_name
         self.view_name: str = view_name
-        self.queue: Optional[Queue] = queue
         self.buffer: list[pd.DataFrame] = []
+        self.exception: Optional[BaseException] = None
+        self.event = Event()
 
         # the callback that is passed to the `CallbackRunner`
         def callback(df: pd.DataFrame, _: int):
             if not df.empty:
                 self.buffer.append(df)
 
+        def exception_callback(exception: BaseException):
+            self.exception = exception
+
         # sets up the callback runner
         self.handler = CallbackRunner(
-            self.client, self.pipeline_name, self.view_name, callback, queue
+            self.client,
+            self.pipeline_name,
+            self.view_name,
+            callback,
+            exception_callback,
+            self.event,
         )
 
     def start(self):
@@ -41,6 +50,7 @@ class OutputHandler:
         """
 
         self.handler.start()
+        _ = self.event.wait()
 
     def to_pandas(self, clear_buffer: bool = True):
         """
@@ -49,6 +59,8 @@ class OutputHandler:
         :param clear_buffer: Whether to clear the buffer after getting the output.
         """
 
+        if self.exception is not None:
+            raise self.exception
         if len(self.buffer) == 0:
             return pd.DataFrame()
         res = pd.concat(self.buffer, ignore_index=True)