feldera 0.169.0__tar.gz → 0.170.0__tar.gz

{feldera-0.169.0 → feldera-0.170.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.169.0
+Version: 0.170.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

feldera-0.170.0/feldera/_callback_runner.py

@@ -0,0 +1,95 @@
+from threading import Thread, Event
+from typing import Callable, List, Optional, Mapping, Any
+
+import pandas as pd
+from feldera import FelderaClient
+from feldera._helpers import dataframe_from_response
+from feldera.enums import PipelineFieldSelector
+from feldera.rest.sql_table import SQLTable
+from feldera.rest.sql_view import SQLView
+from feldera.rest.pipeline import Pipeline
+
+
+class CallbackRunner(Thread):
+    def __init__(
+        self,
+        client: FelderaClient,
+        pipeline_name: str,
+        view_name: str,
+        callback: Callable[[pd.DataFrame, int], None],
+        exception_callback: Callable[[BaseException], None],
+        event: Event,
+    ):
+        """
+        :param client: The :class:`.FelderaClient` to use.
+        :param pipeline_name: The name of the current pipeline.
+        :param view_name: The name of the view we are listening to.
+        :param callback: The callback function to call on the data we receive.
+        :param exception_callback: The callback function to call when an exception occurs.
+        :param event: The event to wait for before starting the callback runner.
+        """
+
+        super().__init__()
+        self.daemon = True
+        self.client: FelderaClient = client
+        self.pipeline_name: str = pipeline_name
+        self.view_name: str = view_name
+        self.callback: Callable[[pd.DataFrame, int], None] = callback
+        self.exception_callback: Callable[[BaseException], None] = exception_callback
+        self.event: Event = event
+
+        self.pipeline: Pipeline = self.client.get_pipeline(
+            self.pipeline_name, PipelineFieldSelector.ALL
+        )
+
+        view_schema = None
+
+        schemas: List[SQLTable | SQLView] = self.pipeline.tables + self.pipeline.views
+        for schema in schemas:
+            if schema.name == self.view_name:
+                view_schema = schema
+                break
+
+        if view_schema is None:
+            raise ValueError(
+                f"Table or View {self.view_name} not found in the pipeline schema."
+            )
+
+        self.schema: SQLTable | SQLView = view_schema
+
+    def to_callback(self, chunk: Mapping[str, Any]):
+        data: Optional[list[Mapping[str, Any]]] = chunk.get("json_data")
+        seq_no: Optional[int] = chunk.get("sequence_number")
+        if data is not None and seq_no is not None:
+            self.callback(dataframe_from_response([data], self.schema.fields), seq_no)
+
+    def run(self):
+        """
+        The main loop of the thread. Listens for data and calls the callback function on each chunk of data received.
+
+        :meta private:
+        """
+
+        try:
+            gen_obj = self.client.listen_to_pipeline(
+                self.pipeline_name,
+                self.view_name,
+                format="json",
+                case_sensitive=self.schema.case_sensitive,
+            )
+
+            iterator = gen_obj()
+
+            # Trigger the HTTP call
+            chunk = next(iterator)
+
+            # Unblock the main thread
+            self.event.set()
+
+            self.to_callback(chunk)
+
+            for chunk in iterator:
+                self.to_callback(chunk)
+
+        except BaseException as e:
+            self.exception_callback(e)

{feldera-0.169.0 → feldera-0.170.0}/feldera/_helpers.py

@@ -2,6 +2,7 @@ import uuid
 
 import pandas as pd
 from decimal import Decimal
+from typing import Mapping, Any
 
 
 def sql_type_to_pandas_type(sql_type: str):
@@ -60,9 +61,14 @@ def ensure_dataframe_has_columns(df: pd.DataFrame):
         )
 
 
-def dataframe_from_response(buffer: list[list[dict]], fields: list[dict]):
+def dataframe_from_response(
+    buffer: list[list[Mapping[str, Any]]], fields: list[Mapping[str, Any]]
+):
     """
     Converts the response from Feldera to a pandas DataFrame.
+
+    :param buffer: A buffer of a list of JSON formatted output of the view you are listening to.
+    :param fields: The schema (list of fields) of the view you are listening to.
     """
 
     pd_schema = {}

{feldera-0.169.0 → feldera-0.170.0}/feldera/enums.py

@@ -352,3 +352,15 @@ class BootstrapPolicy(Enum):
     AWAIT_APPROVAL = "await_approval"
     ALLOW = "allow"
     REJECT = "reject"
+
+
+class CompletionTokenStatus(Enum):
+    COMPLETE = "complete"
+    """
+    Feldera has completed processing all inputs represented by this token.
+    """
+
+    IN_PROGRESS = "inprogress"
+    """
+    Feldera is still processing the inputs represented by this token.
+    """

{feldera-0.169.0 → feldera-0.170.0}/feldera/output_handler.py

@@ -1,6 +1,7 @@
 import pandas as pd
 
 from typing import Optional
+from threading import Event
 
 from feldera import FelderaClient
 from feldera._callback_runner import CallbackRunner
@@ -23,6 +24,7 @@ class OutputHandler:
         self.view_name: str = view_name
         self.buffer: list[pd.DataFrame] = []
         self.exception: Optional[BaseException] = None
+        self.event = Event()
 
         # the callback that is passed to the `CallbackRunner`
         def callback(df: pd.DataFrame, _: int):
@@ -39,6 +41,7 @@ class OutputHandler:
             self.view_name,
             callback,
             exception_callback,
+            self.event,
         )
 
     def start(self):
@@ -47,6 +50,7 @@ class OutputHandler:
         """
 
         self.handler.start()
+        _ = self.event.wait()
 
     def to_pandas(self, clear_buffer: bool = True):
         """

{feldera-0.169.0 → feldera-0.170.0}/feldera/pipeline.py

@@ -7,11 +7,13 @@ import pandas
 from uuid import UUID
 
 from typing import List, Dict, Callable, Optional, Generator, Mapping, Any
+from threading import Event
 from collections import deque
 
 from feldera.rest.errors import FelderaAPIError
 from feldera.enums import (
     BootstrapPolicy,
+    CompletionTokenStatus,
     PipelineFieldSelector,
     PipelineStatus,
     ProgramStatus,
@@ -294,10 +296,12 @@ class Pipeline:
         if self.status() not in [PipelineStatus.RUNNING, PipelineStatus.PAUSED]:
             raise RuntimeError("Pipeline must be running or paused to listen to output")
 
+        event = Event()
         handler = CallbackRunner(
-            self.client, self.name, view_name, callback, lambda exception: None
+            self.client, self.name, view_name, callback, lambda exception: None, event
         )
         handler.start()
+        event.wait()
 
     def wait_for_completion(
         self, force_stop: bool = False, timeout_s: float | None = None
@@ -696,6 +700,17 @@ metrics"""
                 err.message = f"Pipeline with name {name} not found"
                 raise err
 
+    @staticmethod
+    def all(client: FelderaClient) -> List["Pipeline"]:
+        """
+        Get all pipelines.
+
+        :param client: The FelderaClient instance.
+        :return: A list of Pipeline objects.
+        """
+
+        return [Pipeline._from_inner(p, client) for p in client.pipelines()]
+
     def checkpoint(self, wait: bool = False, timeout_s: Optional[float] = None) -> int:
         """
         Checkpoints this pipeline.
@@ -1377,3 +1392,31 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
             print(f"Support bundle written to {path}")
 
         return support_bundle_bytes
+
+    def generate_completion_token(self, table_name: str, connector_name: str) -> str:
+        """
+        Returns a completion token that can be passed to :meth:`.Pipeline.completion_token_status` to
+        check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+        """
+
+        return self.client.generate_completion_token(
+            self.name, table_name, connector_name
+        )
+
+    def completion_token_status(self, token: str) -> CompletionTokenStatus:
+        """
+        Returns the status of the completion token.
+        """
+
+        if self.client.completion_token_processed(self.name, token):
+            return CompletionTokenStatus.COMPLETE
+        else:
+            return CompletionTokenStatus.IN_PROGRESS
+
+    def wait_for_token(self, token: str):
+        """
+        Blocks until the pipeline processes all inputs represented by the completion token.
+        """
+
+        self.client.wait_for_token(self.name, token)

{feldera-0.169.0 → feldera-0.170.0}/feldera/rest/feldera_client.py

@@ -126,13 +126,15 @@ class FelderaClient:
 
         return runtime_config
 
-    def pipelines(self) -> list[Pipeline]:
+    def pipelines(
+        self, selector: PipelineFieldSelector = PipelineFieldSelector.STATUS
+    ) -> list[Pipeline]:
         """
         Get all pipelines
         """
 
         resp = self.http.get(
-            path="/pipelines",
+            path=f"/pipelines?selector={selector.value}",
         )
 
         return [Pipeline.from_dict(pipeline) for pipeline in resp]
@@ -883,6 +885,35 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
         return token
 
+    def completion_token_processed(self, pipeline_name: str, token: str) -> bool:
+        """
+        Check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+
+        :param pipeline_name: The name of the pipeline
+        :param token: The token to check for completion
+        :return: True if the pipeline has finished processing all inputs represented by the token, False otherwise
+        """
+
+        params = {
+            "token": token,
+        }
+
+        resp = self.http.get(
+            path=f"/pipelines/{quote(pipeline_name, safe='')}/completion_status",
+            params=params,
+        )
+
+        status: Optional[str] = resp.get("status")
+
+        if status is None:
+            raise FelderaAPIError(
+                f"got empty status when checking for completion status for token: {token}",
+                resp,
+            )
+
+        return status.lower() == "complete"
+
     def wait_for_token(
         self, pipeline_name: str, token: str, timeout_s: Optional[float] = None
     ):
@@ -896,10 +927,6 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             to process these records.
         """
 
-        params = {
-            "token": token,
-        }
-
         start = time.monotonic()
         end = start + timeout_s if timeout_s else None
         initial_backoff = 0.1
@@ -916,18 +943,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
                         + f" {timeout_s}"
                     )
 
-            resp = self.http.get(
-                path=f"/pipelines/{pipeline_name}/completion_status", params=params
-            )
-
-            status: Optional[str] = resp.get("status")
-            if status is None:
-                raise FelderaAPIError(
-                    f"got empty status when checking for completion status for token: {token}",
-                    resp,
-                )
-
-            if status.lower() == "complete":
+            if self.completion_token_processed(pipeline_name, token):
                 break
 
             elapsed = time.monotonic() - start
@@ -1191,3 +1207,31 @@ Reason: The pipeline is in a STOPPED state due to the following error:
                 buffer += chunk
 
         return buffer
+
+    def generate_completion_token(
+        self, pipeline_name: str, table_name: str, connector_name: str
+    ) -> str:
+        """
+        Generate a completion token that can be passed to :meth:`.FelderaClient.completion_token_processed` to
+        check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+
+        :param pipeline_name: The name of the pipeline
+        :param table_name: The name of the table associated with this connector.
+        :param connector_name: The name of the connector.
+
+        :raises FelderaAPIError: If the connector cannot be found, or if the pipeline is not running.
+        """
+
+        resp = self.http.get(
+            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/completion_token",
+        )
+
+        token: str | None = resp.get("token")
+
+        if token is None:
+            raise ValueError(
+                "got invalid response from feldera when generating completion token"
+            )
+
+        return token

{feldera-0.169.0 → feldera-0.170.0}/feldera.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.169.0
+Version: 0.170.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.169.0 → feldera-0.170.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "feldera"
 readme = "README.md"
 description = "The feldera python client"
-version = "0.169.0"
+version = "0.170.0"
 license = { text = "MIT" }
 requires-python = ">=3.10"
 authors = [

feldera-0.169.0/feldera/_callback_runner.py

@@ -1,69 +0,0 @@
-from threading import Thread
-from typing import Callable, Optional
-
-import pandas as pd
-from feldera import FelderaClient
-from feldera._helpers import dataframe_from_response
-from feldera.enums import PipelineFieldSelector
-
-
-class CallbackRunner(Thread):
-    def __init__(
-        self,
-        client: FelderaClient,
-        pipeline_name: str,
-        view_name: str,
-        callback: Callable[[pd.DataFrame, int], None],
-        exception_callback: Callable[[BaseException], None],
-    ):
-        super().__init__()
-        self.daemon = True
-        self.client: FelderaClient = client
-        self.pipeline_name: str = pipeline_name
-        self.view_name: str = view_name
-        self.callback: Callable[[pd.DataFrame, int], None] = callback
-        self.exception_callback: Callable[[BaseException], None] = exception_callback
-        self.schema: Optional[dict] = None
-
-    def run(self):
-        """
-        The main loop of the thread. Listens for data and calls the callback function on each chunk of data received.
-
-        :meta private:
-        """
-
-        try:
-            pipeline = self.client.get_pipeline(
-                self.pipeline_name, PipelineFieldSelector.ALL
-            )
-
-            schemas = pipeline.tables + pipeline.views
-            for schema in schemas:
-                if schema.name == self.view_name:
-                    self.schema = schema
-                    break
-
-            if self.schema is None:
-                raise ValueError(
-                    f"Table or View {self.view_name} not found in the pipeline schema."
-                )
-
-            gen_obj = self.client.listen_to_pipeline(
-                self.pipeline_name,
-                self.view_name,
-                format="json",
-                case_sensitive=self.schema.case_sensitive,
-            )
-
-            iterator = gen_obj()
-
-            for chunk in iterator:
-                chunk: dict = chunk
-                data: Optional[list[dict]] = chunk.get("json_data")
-                seq_no: Optional[int] = chunk.get("sequence_number")
-                if data is not None and seq_no is not None:
-                    self.callback(
-                        dataframe_from_response([data], self.schema.fields), seq_no
-                    )
-        except BaseException as e:
-            self.exception_callback(e)