feldera 0.155.0__tar.gz → 0.157.0__tar.gz

{feldera-0.155.0 → feldera-0.157.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.155.0
+Version: 0.157.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT
@@ -60,12 +60,6 @@ source .venv/activate
 uv pip install .
 ```
 
-You also have to install the `pytest` module:
-
-```bash
-python3 -m pip install pytest
-```
-
 ## Documentation
 
 The Python SDK documentation is available at

{feldera-0.155.0 → feldera-0.157.0}/README.md

@@ -36,12 +36,6 @@ source .venv/activate
 uv pip install .
 ```
 
-You also have to install the `pytest` module:
-
-```bash
-python3 -m pip install pytest
-```
-
 ## Documentation
 
 The Python SDK documentation is available at

feldera-0.157.0/feldera/_callback_runner.py

@@ -0,0 +1,64 @@
+from threading import Thread
+from typing import Callable, Optional
+
+import pandas as pd
+from feldera import FelderaClient
+from feldera._helpers import dataframe_from_response
+from feldera.enums import PipelineFieldSelector
+
+
+class CallbackRunner(Thread):
+    def __init__(
+        self,
+        client: FelderaClient,
+        pipeline_name: str,
+        view_name: str,
+        callback: Callable[[pd.DataFrame, int], None],
+    ):
+        super().__init__()
+        self.daemon = True
+        self.client: FelderaClient = client
+        self.pipeline_name: str = pipeline_name
+        self.view_name: str = view_name
+        self.callback: Callable[[pd.DataFrame, int], None] = callback
+        self.schema: Optional[dict] = None
+
+    def run(self):
+        """
+        The main loop of the thread. Listens for data and calls the callback function on each chunk of data received.
+
+        :meta private:
+        """
+
+        pipeline = self.client.get_pipeline(
+            self.pipeline_name, PipelineFieldSelector.ALL
+        )
+
+        schemas = pipeline.tables + pipeline.views
+        for schema in schemas:
+            if schema.name == self.view_name:
+                self.schema = schema
+                break
+
+        if self.schema is None:
+            raise ValueError(
+                f"Table or View {self.view_name} not found in the pipeline schema."
+            )
+
+        gen_obj = self.client.listen_to_pipeline(
+            self.pipeline_name,
+            self.view_name,
+            format="json",
+            case_sensitive=self.schema.case_sensitive,
+        )
+
+        iterator = gen_obj()
+
+        for chunk in iterator:
+            chunk: dict = chunk
+            data: Optional[list[dict]] = chunk.get("json_data")
+            seq_no: Optional[int] = chunk.get("sequence_number")
+            if data is not None and seq_no is not None:
+                self.callback(
+                    dataframe_from_response([data], self.schema.fields), seq_no
+                )

{feldera-0.155.0 → feldera-0.157.0}/feldera/output_handler.py

@@ -1,7 +1,5 @@
 import pandas as pd
-from typing import Optional
 
-from queue import Queue
 from feldera import FelderaClient
 from feldera._callback_runner import CallbackRunner
 
@@ -12,7 +10,6 @@ class OutputHandler:
         client: FelderaClient,
         pipeline_name: str,
         view_name: str,
-        queue: Optional[Queue],
     ):
         """
         Initializes the output handler, but doesn't start it.
@@ -22,7 +19,6 @@ class OutputHandler:
         self.client: FelderaClient = client
         self.pipeline_name: str = pipeline_name
         self.view_name: str = view_name
-        self.queue: Optional[Queue] = queue
         self.buffer: list[pd.DataFrame] = []
 
         # the callback that is passed to the `CallbackRunner`
@@ -32,7 +28,7 @@ class OutputHandler:
 
         # sets up the callback runner
         self.handler = CallbackRunner(
-            self.client, self.pipeline_name, self.view_name, callback, queue
+            self.client, self.pipeline_name, self.view_name, callback
         )
 
     def start(self):

{feldera-0.155.0 → feldera-0.157.0}/feldera/pipeline.py

@@ -8,7 +8,6 @@ from uuid import UUID
 
 from typing import List, Dict, Callable, Optional, Generator, Mapping, Any
 from collections import deque
-from queue import Queue
 
 from feldera.rest.errors import FelderaAPIError
 from feldera.enums import (
@@ -26,7 +25,7 @@ from feldera.enums import (
 )
 from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.rest.feldera_client import FelderaClient
-from feldera._callback_runner import _CallbackRunnerInstruction, CallbackRunner
+from feldera._callback_runner import CallbackRunner
 from feldera.output_handler import OutputHandler
 from feldera._helpers import ensure_dataframe_has_columns, chunk_dataframe
 from feldera.rest.sql_table import SQLTable
@@ -39,7 +38,6 @@ class Pipeline:
     def __init__(self, client: FelderaClient):
         self.client: FelderaClient = client
         self._inner: InnerPipeline | None = None
-        self.views_tx: List[Dict[str, Queue]] = []
 
     @staticmethod
     def _from_inner(inner: InnerPipeline, client: FelderaClient) -> "Pipeline":
@@ -47,20 +45,6 @@ class Pipeline:
         pipeline._inner = inner
         return pipeline
 
-    def __setup_output_listeners(self):
-        """
-        Internal function used to set up the output listeners.
-
-        :meta private:
-        """
-
-        for view_queue in self.views_tx:
-            for view_name, queue in view_queue.items():
-                # sends a message to the callback runner to start listening
-                queue.put(_CallbackRunnerInstruction.PipelineStarted)
-                # block until the callback runner is ready
-                queue.join()
-
     def refresh(self, field_selector: PipelineFieldSelector):
         """
         Calls the backend to get the updated, latest version of the pipeline.
@@ -237,23 +221,21 @@ class Pipeline:
     def listen(self, view_name: str) -> OutputHandler:
         """
         Follow the change stream (i.e., the output) of the provided view.
-        Returns an output handler to read the changes.
+        Returns an output handle to read the changes.
 
-        When the pipeline is stopped, these listeners are dropped.
+        When the pipeline is stopped, the handle is dropped.
 
-        You must call this method before starting the pipeline to get the entire output of the view.
-        If this method is called once the pipeline has started, you will only get the output from that point onwards.
+        The handle will only receive changes from the point in time when the listener is created.
+        In order to receive all changes since the pipeline started, you can create the pipeline in the `PAUSED` state
+        using :meth:`start_paused`, attach listeners and unpause the pipeline using :meth:`resume`.
 
         :param view_name: The name of the view to listen to.
         """
 
-        queue: Optional[Queue] = None
-
         if self.status() not in [PipelineStatus.PAUSED, PipelineStatus.RUNNING]:
-            queue = Queue(maxsize=1)
-            self.views_tx.append({view_name: queue})
+            raise RuntimeError("Pipeline must be running or paused to listen to output")
 
-        handler = OutputHandler(self.client, self.name, view_name, queue)
+        handler = OutputHandler(self.client, self.name, view_name)
         handler.start()
 
         return handler
@@ -264,8 +246,9 @@ class Pipeline:
         """
         Run the given callback on each chunk of the output of the specified view.
 
-        You must call this method before starting the pipeline to operate on the entire output.
-        You can call this method after the pipeline has started, but you will only get the output from that point onwards.
+        The callback will only receive changes from the point in time when the listener is created.
+        In order to receive all changes since the pipeline started, you can create the pipeline in the `PAUSED` state
+        using :meth:`start_paused`, attach listeners and unpause the pipeline using :meth:`resume`.
 
         :param view_name: The name of the view.
         :param callback: The callback to run on each chunk. The callback should take two arguments:
@@ -283,13 +266,10 @@ class Pipeline:
 
         """
 
-        queue: Optional[Queue] = None
-
         if self.status() not in [PipelineStatus.RUNNING, PipelineStatus.PAUSED]:
-            queue = Queue(maxsize=1)
-            self.views_tx.append({view_name: queue})
+            raise RuntimeError("Pipeline must be running or paused to listen to output")
 
-        handler = CallbackRunner(self.client, self.name, view_name, callback, queue)
+        handler = CallbackRunner(self.client, self.name, view_name, callback)
         handler.start()
 
     def wait_for_completion(
@@ -365,46 +345,6 @@ class Pipeline:
 
         return self.stats().global_metrics.pipeline_complete
 
-    def start(self, wait: bool = True, timeout_s: Optional[float] = None):
-        """
-        .. _start:
-
-        Starts this pipeline.
-
-        - The pipeline must be in STOPPED state to start.
-        - If the pipeline is in any other state, an error will be raised.
-        - If the pipeline is in PAUSED state, use `.meth:resume` instead.
-
-        :param timeout_s: The maximum time (in seconds) to wait for the
-            pipeline to start.
-        :param wait: Set True to wait for the pipeline to start. True by default
-
-        :raises RuntimeError: If the pipeline is not in STOPPED state.
-        """
-
-        status = self.status()
-        if status != PipelineStatus.STOPPED:
-            raise RuntimeError(
-                f"""Cannot start pipeline '{self.name}' in state \
-'{str(status.name)}'. The pipeline must be in STOPPED state before it can be \
-started. You can either stop the pipeline using the `Pipeline.stop()` \
-method or use `Pipeline.resume()` to resume a paused pipeline."""
-            )
-
-        if not wait:
-            if len(self.views_tx) > 0:
-                raise ValueError(
-                    "cannot start with 'wait=False' when output listeners are configured. Try setting 'wait=True'."
-                )
-
-            self.client.start_pipeline(self.name, wait=wait)
-
-            return
-
-        self.client.start_pipeline_as_paused(self.name, wait=wait, timeout_s=timeout_s)
-        self.__setup_output_listeners()
-        self.resume(timeout_s=timeout_s)
-
     def restart(self, timeout_s: Optional[float] = None):
         """
         Restarts the pipeline.
@@ -511,6 +451,25 @@ metrics"""
 
         self.client.activate_pipeline(self.name, wait=wait, timeout_s=timeout_s)
 
+    def start(self, wait: bool = True, timeout_s: Optional[float] = None):
+        """
+        .. _start:
+
+        Starts this pipeline.
+
+        - The pipeline must be in STOPPED state to start.
+        - If the pipeline is in any other state, an error will be raised.
+        - If the pipeline is in PAUSED state, use `.meth:resume` instead.
+
+        :param timeout_s: The maximum time (in seconds) to wait for the
+            pipeline to start.
+        :param wait: Set True to wait for the pipeline to start. True by default
+
+        :raises RuntimeError: If the pipeline is not in STOPPED state.
+        """
+
+        self.client.start_pipeline(self.name, wait=wait, timeout_s=timeout_s)
+
     def start_paused(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         Starts the pipeline in the paused state.
@@ -554,20 +513,6 @@ metrics"""
             pipeline to stop.
         """
 
-        if wait:
-            for view_queue in self.views_tx:
-                for _, queue in view_queue.items():
-                    # sends a message to the callback runner to stop listening
-                    queue.put(_CallbackRunnerInstruction.RanToCompletion)
-
-            if len(self.views_tx) > 0:
-                while self.views_tx:
-                    view = self.views_tx.pop()
-                    for view_name, queue in view.items():
-                        # block until the callback runner has been stopped
-                        queue.join()
-
-        time.sleep(3)
         self.client.stop_pipeline(
             self.name, force=force, wait=wait, timeout_s=timeout_s
         )

{feldera-0.155.0 → feldera-0.157.0}/feldera.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.155.0
+Version: 0.157.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT
@@ -60,12 +60,6 @@ source .venv/activate
 uv pip install .
 ```
 
-You also have to install the `pytest` module:
-
-```bash
-python3 -m pip install pytest
-```
-
 ## Documentation
 
 The Python SDK documentation is available at

{feldera-0.155.0 → feldera-0.157.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "feldera"
 readme = "README.md"
 description = "The feldera python client"
-version = "0.155.0"
+version = "0.157.0"
 license = { text = "MIT" }
 requires-python = ">=3.10"
 authors = [

feldera-0.155.0/feldera/_callback_runner.py

@@ -1,123 +0,0 @@
-from enum import Enum
-from threading import Thread
-from typing import Callable, Optional
-from queue import Queue, Empty
-
-import pandas as pd
-from feldera import FelderaClient
-from feldera._helpers import dataframe_from_response
-from feldera.enums import PipelineFieldSelector
-
-
-class _CallbackRunnerInstruction(Enum):
-    PipelineStarted = 1
-    RanToCompletion = 2
-
-
-class CallbackRunner(Thread):
-    def __init__(
-        self,
-        client: FelderaClient,
-        pipeline_name: str,
-        view_name: str,
-        callback: Callable[[pd.DataFrame, int], None],
-        queue: Optional[Queue],
-    ):
-        super().__init__()
-        self.daemon = True
-        self.client: FelderaClient = client
-        self.pipeline_name: str = pipeline_name
-        self.view_name: str = view_name
-        self.callback: Callable[[pd.DataFrame, int], None] = callback
-        self.queue: Optional[Queue] = queue
-        self.schema: Optional[dict] = None
-
-    def run(self):
-        """
-        The main loop of the thread. Listens for data and calls the callback function on each chunk of data received.
-
-        :meta private:
-        """
-
-        pipeline = self.client.get_pipeline(
-            self.pipeline_name, PipelineFieldSelector.ALL
-        )
-
-        schemas = pipeline.tables + pipeline.views
-        for schema in schemas:
-            if schema.name == self.view_name:
-                self.schema = schema
-                break
-
-        if self.schema is None:
-            raise ValueError(
-                f"Table or View {self.view_name} not found in the pipeline schema."
-            )
-
-        # by default, we assume that the pipeline has been started
-        ack = _CallbackRunnerInstruction.PipelineStarted
-
-        # if there is Queue, we wait for the instruction to start the pipeline
-        # this means that we are listening to the pipeline before running it, therefore, all data should be received
-        if self.queue:
-            ack = self.queue.get()
-
-        match ack:
-            # if the pipeline has actually been started, we start a listener
-            case _CallbackRunnerInstruction.PipelineStarted:
-                # listen to the pipeline
-                gen_obj = self.client.listen_to_pipeline(
-                    self.pipeline_name,
-                    self.view_name,
-                    format="json",
-                    case_sensitive=self.schema.case_sensitive,
-                )
-
-                # if there is a queue set up, inform the main thread that the listener has been started, and it can
-                # proceed with starting the pipeline
-                if self.queue:
-                    # stop blocking the main thread on `join` for the previous message
-                    self.queue.task_done()
-
-                iterator = gen_obj()
-
-                for chunk in iterator:
-                    chunk: dict = chunk
-                    data: Optional[list[dict]] = chunk.get("json_data")
-                    seq_no: Optional[int] = chunk.get("sequence_number")
-                    if data is not None and seq_no is not None:
-                        self.callback(
-                            dataframe_from_response([data], self.schema.fields), seq_no
-                        )
-
-                    if self.queue:
-                        try:
-                            # if a non-blocking way, check if the queue has received further instructions
-                            # this should be a RanToCompletion instruction, which means that the pipeline has been
-                            # completed
-                            again_ack = self.queue.get_nowait()
-
-                            # if the queue has received a message
-                            if again_ack:
-                                match again_ack:
-                                    case _CallbackRunnerInstruction.RanToCompletion:
-                                        # stop blocking the main thread on `join` and return from this thread
-                                        self.queue.task_done()
-
-                                        return
-
-                                    case _CallbackRunnerInstruction.PipelineStarted:
-                                        # if the pipeline has been started again, which shouldn't happen,
-                                        # ignore it and continue listening, call `task_done` to avoid blocking the main
-                                        # thread on `join`
-                                        self.queue.task_done()
-
-                                        continue
-                        except Empty:
-                            # if the queue is empty, continue listening
-                            continue
-
-            case _CallbackRunnerInstruction.RanToCompletion:
-                if self.queue:
-                    self.queue.task_done()
-                return