feldera 0.166.0__py3-none-any.whl → 0.177.0__py3-none-any.whl

feldera/_callback_runner.py

@@ -1,10 +1,13 @@
-from threading import Thread
-from typing import Callable, Optional
+from threading import Thread, Event
+from typing import Callable, List, Optional, Mapping, Any
 
 import pandas as pd
 from feldera import FelderaClient
 from feldera._helpers import dataframe_from_response
 from feldera.enums import PipelineFieldSelector
+from feldera.rest.sql_table import SQLTable
+from feldera.rest.sql_view import SQLView
+from feldera.rest.pipeline import Pipeline
 
 
 class CallbackRunner(Thread):
@@ -15,7 +18,17 @@ class CallbackRunner(Thread):
         view_name: str,
         callback: Callable[[pd.DataFrame, int], None],
         exception_callback: Callable[[BaseException], None],
+        event: Event,
     ):
+        """
+        :param client: The :class:`.FelderaClient` to use.
+        :param pipeline_name: The name of the current pipeline.
+        :param view_name: The name of the view we are listening to.
+        :param callback: The callback function to call on the data we receive.
+        :param exception_callback: The callback function to call when an exception occurs.
+        :param event: The event to wait for before starting the callback runner.
+        """
+
         super().__init__()
         self.daemon = True
         self.client: FelderaClient = client
@@ -23,7 +36,32 @@ class CallbackRunner(Thread):
         self.view_name: str = view_name
         self.callback: Callable[[pd.DataFrame, int], None] = callback
         self.exception_callback: Callable[[BaseException], None] = exception_callback
-        self.schema: Optional[dict] = None
+        self.event: Event = event
+
+        self.pipeline: Pipeline = self.client.get_pipeline(
+            self.pipeline_name, PipelineFieldSelector.ALL
+        )
+
+        view_schema = None
+
+        schemas: List[SQLTable | SQLView] = self.pipeline.tables + self.pipeline.views
+        for schema in schemas:
+            if schema.name == self.view_name:
+                view_schema = schema
+                break
+
+        if view_schema is None:
+            raise ValueError(
+                f"Table or View {self.view_name} not found in the pipeline schema."
+            )
+
+        self.schema: SQLTable | SQLView = view_schema
+
+    def to_callback(self, chunk: Mapping[str, Any]):
+        data: Optional[list[Mapping[str, Any]]] = chunk.get("json_data")
+        seq_no: Optional[int] = chunk.get("sequence_number")
+        if data is not None and seq_no is not None:
+            self.callback(dataframe_from_response([data], self.schema.fields), seq_no)
 
     def run(self):
         """
@@ -33,21 +71,6 @@ class CallbackRunner(Thread):
         """
 
         try:
-            pipeline = self.client.get_pipeline(
-                self.pipeline_name, PipelineFieldSelector.ALL
-            )
-
-            schemas = pipeline.tables + pipeline.views
-            for schema in schemas:
-                if schema.name == self.view_name:
-                    self.schema = schema
-                    break
-
-            if self.schema is None:
-                raise ValueError(
-                    f"Table or View {self.view_name} not found in the pipeline schema."
-                )
-
             gen_obj = self.client.listen_to_pipeline(
                 self.pipeline_name,
                 self.view_name,
@@ -57,13 +80,16 @@ class CallbackRunner(Thread):
 
             iterator = gen_obj()
 
+            # Trigger the HTTP call
+            chunk = next(iterator)
+
+            # Unblock the main thread
+            self.event.set()
+
+            self.to_callback(chunk)
+
             for chunk in iterator:
-                chunk: dict = chunk
-                data: Optional[list[dict]] = chunk.get("json_data")
-                seq_no: Optional[int] = chunk.get("sequence_number")
-                if data is not None and seq_no is not None:
-                    self.callback(
-                        dataframe_from_response([data], self.schema.fields), seq_no
-                    )
+                self.to_callback(chunk)
+
         except BaseException as e:
             self.exception_callback(e)

feldera/_helpers.py

@@ -2,6 +2,7 @@ import uuid
 
 import pandas as pd
 from decimal import Decimal
+from typing import Mapping, Any
 
 
 def sql_type_to_pandas_type(sql_type: str):
@@ -60,9 +61,14 @@ def ensure_dataframe_has_columns(df: pd.DataFrame):
         )
 
 
-def dataframe_from_response(buffer: list[list[dict]], fields: list[dict]):
+def dataframe_from_response(
+    buffer: list[list[Mapping[str, Any]]], fields: list[Mapping[str, Any]]
+):
     """
     Converts the response from Feldera to a pandas DataFrame.
+
+    :param buffer: A buffer of a list of JSON formatted output of the view you are listening to.
+    :param fields: The schema (list of fields) of the view you are listening to.
     """
 
     pd_schema = {}

feldera/enums.py

@@ -352,3 +352,15 @@ class BootstrapPolicy(Enum):
     AWAIT_APPROVAL = "await_approval"
     ALLOW = "allow"
     REJECT = "reject"
+
+
+class CompletionTokenStatus(Enum):
+    COMPLETE = "complete"
+    """
+    Feldera has completed processing all inputs represented by this token.
+    """
+
+    IN_PROGRESS = "inprogress"
+    """
+    Feldera is still processing the inputs represented by this token.
+    """

feldera/output_handler.py

@@ -1,6 +1,7 @@
 import pandas as pd
 
 from typing import Optional
+from threading import Event
 
 from feldera import FelderaClient
 from feldera._callback_runner import CallbackRunner
@@ -23,6 +24,7 @@ class OutputHandler:
         self.view_name: str = view_name
         self.buffer: list[pd.DataFrame] = []
         self.exception: Optional[BaseException] = None
+        self.event = Event()
 
         # the callback that is passed to the `CallbackRunner`
         def callback(df: pd.DataFrame, _: int):
@@ -39,6 +41,7 @@ class OutputHandler:
             self.view_name,
             callback,
             exception_callback,
+            self.event,
         )
 
     def start(self):
@@ -47,6 +50,7 @@ class OutputHandler:
         """
 
         self.handler.start()
+        _ = self.event.wait()
 
     def to_pandas(self, clear_buffer: bool = True):
         """

feldera/pipeline.py

@@ -7,11 +7,13 @@ import pandas
 from uuid import UUID
 
 from typing import List, Dict, Callable, Optional, Generator, Mapping, Any
+from threading import Event
 from collections import deque
 
 from feldera.rest.errors import FelderaAPIError
 from feldera.enums import (
     BootstrapPolicy,
+    CompletionTokenStatus,
     PipelineFieldSelector,
     PipelineStatus,
     ProgramStatus,
@@ -294,10 +296,12 @@ class Pipeline:
         if self.status() not in [PipelineStatus.RUNNING, PipelineStatus.PAUSED]:
             raise RuntimeError("Pipeline must be running or paused to listen to output")
 
+        event = Event()
         handler = CallbackRunner(
-            self.client, self.name, view_name, callback, lambda exception: None
+            self.client, self.name, view_name, callback, lambda exception: None, event
         )
         handler.start()
+        event.wait()
 
     def wait_for_completion(
         self, force_stop: bool = False, timeout_s: float | None = None
@@ -696,6 +700,17 @@ metrics"""
                 err.message = f"Pipeline with name {name} not found"
                 raise err
 
+    @staticmethod
+    def all(client: FelderaClient) -> List["Pipeline"]:
+        """
+        Get all pipelines.
+
+        :param client: The FelderaClient instance.
+        :return: A list of Pipeline objects.
+        """
+
+        return [Pipeline._from_inner(p, client) for p in client.pipelines()]
+
     def checkpoint(self, wait: bool = False, timeout_s: Optional[float] = None) -> int:
         """
         Checkpoints this pipeline.
@@ -1377,3 +1392,31 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
             print(f"Support bundle written to {path}")
 
         return support_bundle_bytes
+
+    def generate_completion_token(self, table_name: str, connector_name: str) -> str:
+        """
+        Returns a completion token that can be passed to :meth:`.Pipeline.completion_token_status` to
+        check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+        """
+
+        return self.client.generate_completion_token(
+            self.name, table_name, connector_name
+        )
+
+    def completion_token_status(self, token: str) -> CompletionTokenStatus:
+        """
+        Returns the status of the completion token.
+        """
+
+        if self.client.completion_token_processed(self.name, token):
+            return CompletionTokenStatus.COMPLETE
+        else:
+            return CompletionTokenStatus.IN_PROGRESS
+
+    def wait_for_token(self, token: str):
+        """
+        Blocks until the pipeline processes all inputs represented by the completion token.
+        """
+
+        self.client.wait_for_token(self.name, token)

feldera/pipeline_builder.py

@@ -1,12 +1,12 @@
 import os
 from typing import Optional
 
+from feldera.enums import CompilationProfile, PipelineFieldSelector
+from feldera.pipeline import Pipeline
+from feldera.rest.errors import FelderaAPIError
 from feldera.rest.feldera_client import FelderaClient
 from feldera.rest.pipeline import Pipeline as InnerPipeline
-from feldera.pipeline import Pipeline
-from feldera.enums import CompilationProfile, PipelineFieldSelector
 from feldera.runtime_config import RuntimeConfig
-from feldera.rest.errors import FelderaAPIError
 
 
 class PipelineBuilder:
@@ -49,10 +49,11 @@ class PipelineBuilder:
             "FELDERA_RUNTIME_VERSION", runtime_version
         )
 
-    def create(self) -> Pipeline:
+    def create(self, wait: bool = True) -> Pipeline:
         """
         Create the pipeline if it does not exist.
 
+        :param wait: Whether to wait for the pipeline to be created. True by default
         :return: The created pipeline
         """
 
@@ -82,17 +83,20 @@ class PipelineBuilder:
             runtime_config=self.runtime_config.to_dict(),
         )
 
-        inner = self.client.create_pipeline(inner)
+        inner = self.client.create_pipeline(inner, wait=wait)
         pipeline = Pipeline(self.client)
         pipeline._inner = inner
 
         return pipeline
 
-    def create_or_replace(self) -> Pipeline:
+    def create_or_replace(self, wait: bool = True) -> Pipeline:
         """
         Creates a pipeline if it does not exist and replaces it if it exists.
 
         If the pipeline exists and is running, it will be stopped and replaced.
+
+        :param wait: Whether to wait for the pipeline to be created. True by default
+        :return: The created pipeline
         """
 
         if self.name is None or self.sql is None:
@@ -121,7 +125,7 @@ class PipelineBuilder:
             runtime_config=self.runtime_config.to_dict(),
         )
 
-        inner = self.client.create_or_update_pipeline(inner)
+        inner = self.client.create_or_update_pipeline(inner, wait=wait)
         pipeline = Pipeline(self.client)
         pipeline._inner = inner
 

feldera/rest/_httprequests.py

@@ -1,19 +1,18 @@
+import json
 import logging
+import time
+from typing import Any, Callable, List, Mapping, Optional, Sequence, Union
 
-from feldera.rest.config import Config
+import requests
+from requests.packages import urllib3
 
+from feldera.rest.config import Config
 from feldera.rest.errors import (
     FelderaAPIError,
-    FelderaTimeoutError,
     FelderaCommunicationError,
+    FelderaTimeoutError,
 )
 
-import json
-import requests
-from requests.packages import urllib3
-from typing import Callable, Optional, Any, Union, Mapping, Sequence, List
-import time
-
 
 def json_serialize(body: Any) -> str:
     # serialize as string if this object cannot be serialized (e.g. UUID)
@@ -111,11 +110,12 @@ class HttpRequests:
                     logging.debug("got response: %s", str(resp))
                     return resp
                 except FelderaAPIError as err:
-                    # Only retry on 503
-                    if err.status_code == 503:
+                    # Only retry on 503 and 408
+                    if err.status_code in [503, 408]:
                         if attempt < max_retries:
                             logging.warning(
-                                "HTTP 503 received for %s, retrying (%d/%d)...",
+                                "HTTP %d received for %s, retrying (%d/%d)...",
+                                err.status_code,
                                 path,
                                 attempt + 1,
                                 max_retries,

feldera/rest/feldera_client.py

@@ -1,20 +1,20 @@
-import pathlib
-from typing import Any, Dict, Optional
+import json
 import logging
+import pathlib
 import time
-import json
 from decimal import Decimal
-from typing import Generator, Mapping
+from typing import Any, Dict, Generator, Mapping, Optional
 from urllib.parse import quote
+
 import requests
 
 from feldera.enums import BootstrapPolicy, PipelineFieldSelector, PipelineStatus
+from feldera.rest._helpers import client_version
+from feldera.rest._httprequests import HttpRequests
 from feldera.rest.config import Config
+from feldera.rest.errors import FelderaAPIError, FelderaTimeoutError
 from feldera.rest.feldera_config import FelderaConfig
-from feldera.rest.errors import FelderaTimeoutError, FelderaAPIError
 from feldera.rest.pipeline import Pipeline
-from feldera.rest._httprequests import HttpRequests
-from feldera.rest._helpers import client_version
 
 
 def _validate_no_none_keys_in_map(data):
@@ -126,13 +126,15 @@ class FelderaClient:
 
         return runtime_config
 
-    def pipelines(self) -> list[Pipeline]:
+    def pipelines(
+        self, selector: PipelineFieldSelector = PipelineFieldSelector.STATUS
+    ) -> list[Pipeline]:
         """
         Get all pipelines
         """
 
         resp = self.http.get(
-            path="/pipelines",
+            path=f"/pipelines?selector={selector.value}",
         )
 
         return [Pipeline.from_dict(pipeline) for pipeline in resp]
@@ -256,12 +258,12 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             )
             time.sleep(0.1)
 
-    def create_pipeline(self, pipeline: Pipeline) -> Pipeline:
+    def create_pipeline(self, pipeline: Pipeline, wait: bool = True) -> Pipeline:
         """
         Create a pipeline if it doesn't exist and wait for it to compile
 
-
-        :name: The name of the pipeline
+        :param pipeline: The pipeline to create
+        :param wait: Whether to wait for the pipeline to compile. True by default
         """
 
         body = {
@@ -279,12 +281,21 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             body=body,
         )
 
+        if not wait:
+            return pipeline
+
         return self.__wait_for_compilation(pipeline.name)
 
-    def create_or_update_pipeline(self, pipeline: Pipeline) -> Pipeline:
+    def create_or_update_pipeline(
+        self, pipeline: Pipeline, wait: bool = True
+    ) -> Pipeline:
         """
         Create a pipeline if it doesn't exist or update a pipeline and wait for
         it to compile
+
+        :param pipeline: The pipeline to create or update
+        :param wait: Whether to wait for the pipeline to compile. True by default
+        :return: The created or updated pipeline
         """
 
         body = {
@@ -302,6 +313,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             body=body,
         )
 
+        if not wait:
+            return pipeline
+
         return self.__wait_for_compilation(pipeline.name)
 
     def patch_pipeline(
@@ -883,6 +897,35 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
         return token
 
+    def completion_token_processed(self, pipeline_name: str, token: str) -> bool:
+        """
+        Check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+
+        :param pipeline_name: The name of the pipeline
+        :param token: The token to check for completion
+        :return: True if the pipeline has finished processing all inputs represented by the token, False otherwise
+        """
+
+        params = {
+            "token": token,
+        }
+
+        resp = self.http.get(
+            path=f"/pipelines/{quote(pipeline_name, safe='')}/completion_status",
+            params=params,
+        )
+
+        status: Optional[str] = resp.get("status")
+
+        if status is None:
+            raise FelderaAPIError(
+                f"got empty status when checking for completion status for token: {token}",
+                resp,
+            )
+
+        return status.lower() == "complete"
+
     def wait_for_token(
         self, pipeline_name: str, token: str, timeout_s: Optional[float] = None
     ):
@@ -896,10 +939,6 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             to process these records.
         """
 
-        params = {
-            "token": token,
-        }
-
         start = time.monotonic()
         end = start + timeout_s if timeout_s else None
         initial_backoff = 0.1
@@ -916,18 +955,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
                         + f" {timeout_s}"
                     )
 
-            resp = self.http.get(
-                path=f"/pipelines/{pipeline_name}/completion_status", params=params
-            )
-
-            status: Optional[str] = resp.get("status")
-            if status is None:
-                raise FelderaAPIError(
-                    f"got empty status when checking for completion status for token: {token}",
-                    resp,
-                )
-
-            if status.lower() == "complete":
+            if self.completion_token_processed(pipeline_name, token):
                 break
 
             elapsed = time.monotonic() - start
@@ -1191,3 +1219,31 @@ Reason: The pipeline is in a STOPPED state due to the following error:
                 buffer += chunk
 
         return buffer
+
+    def generate_completion_token(
+        self, pipeline_name: str, table_name: str, connector_name: str
+    ) -> str:
+        """
+        Generate a completion token that can be passed to :meth:`.FelderaClient.completion_token_processed` to
+        check whether the pipeline has finished processing all inputs received from the connector before
+        the token was generated.
+
+        :param pipeline_name: The name of the pipeline
+        :param table_name: The name of the table associated with this connector.
+        :param connector_name: The name of the connector.
+
+        :raises FelderaAPIError: If the connector cannot be found, or if the pipeline is not running.
+        """
+
+        resp = self.http.get(
+            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/completion_token",
+        )
+
+        token: str | None = resp.get("token")
+
+        if token is None:
+            raise ValueError(
+                "got invalid response from feldera when generating completion token"
+            )
+
+        return token

feldera/testutils.py

@@ -256,7 +256,6 @@ def build_pipeline(
         compilation_profile=CompilationProfile.OPTIMIZED,
         runtime_config=RuntimeConfig(
             provisioning_timeout_secs=60,
-            dev_tweaks={"backfill_avoidance": True},
             resources=resources,
         ),
     ).create_or_replace()

{feldera-0.166.0.dist-info → feldera-0.177.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.166.0
+Version: 0.177.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.166.0.dist-info → feldera-0.177.0.dist-info}/RECORD

@@ -1,26 +1,26 @@
 feldera/__init__.py,sha256=EiY3bTj_mnfNhCGrZo6J__brfovIJ-YYAdy77PyaEoo,378
-feldera/_callback_runner.py,sha256=aa-CVfIxZMqfFDs1q7MGEL77org2n0BrFgUp_ye_ohY,2333
-feldera/_helpers.py,sha256=TQnDQW19fpljD19ppd5dASy1gUC4y8GNnnJjXxbaUmM,3019
-feldera/enums.py,sha256=Ro6rDgoQKCDxxX5VKJ4ndyIaRK2o98n0XeEPfvecQpk,8563
-feldera/output_handler.py,sha256=J3Z3w3JG_hDGO-42FMHk34HrI20zwBI9upNAN04pi40,2118
-feldera/pipeline.py,sha256=3FVj2Sp_N44ZSNZO_FKC9oZciCH1h7uC5C081H0hTjM,50428
-feldera/pipeline_builder.py,sha256=nOKoMyB47YxwrrvByKGn4CagO-dMQNFip3c5PK1DDhQ,4351
+feldera/_callback_runner.py,sha256=cTYbUxSILZDcTrjsWWPiTFdMDlQ9FPoWiMqPfnvQGDg,3226
+feldera/_helpers.py,sha256=wQX2d-C4an7xGXTg6VKo9VMBROXN7TUrS90LITFKQEw,3262
+feldera/enums.py,sha256=p11z4PgdcL4UQLnaPPnZBB_dHTBpNaJLqsT7qJ-R0QA,8835
+feldera/output_handler.py,sha256=VvDkPSfhllH3nEwg3nPkPk-uXGAXs0fdrZ0mrMqijJI,2229
+feldera/pipeline.py,sha256=dDKu-RPjFx0PMpZet0_vaZh4FfuGNoIKP4xB3eM59mo,51855
+feldera/pipeline_builder.py,sha256=qrXaQz0HGGNMo0wPJJSH-gooTVSlm1XODaK9643eoL8,4620
 feldera/runtime_config.py,sha256=w6rPkZyijca9jY1G8PKeqP8txXjnn5MPJYmM7B8iE3U,4602
 feldera/stats.py,sha256=YeDQwE_CixTMb2DjBCgt5jTaJAZRsrHtG-3pYuuII-8,5256
-feldera/testutils.py,sha256=z-efiWz8k6f1skAQ4fw21GJQzELaLrX1zYS7t1ffCVY,12624
+feldera/testutils.py,sha256=ITIqXTRD44j4AhjF-41GOWKOYh3_BAV-GQzQ7HOkgDw,12571
 feldera/testutils_oidc.py,sha256=hv0IQAcVOt4uysWW3iE-5p46ZGfL1X7Vt7PjRefkXz8,12552
 feldera/rest/__init__.py,sha256=Eg-EKUU3RSTDcdxTR_7wNDnCly8VpXEzsZCQUmf-y2M,308
 feldera/rest/_helpers.py,sha256=51oJV2pF4U1NBpjIEZuc9nIEcS0qlbUNJoefhiY-PmM,1129
-feldera/rest/_httprequests.py,sha256=rpoq4_700hP1flEnMbGIFIyc7xeGLXs3Yh4nY387_Qg,8579
+feldera/rest/_httprequests.py,sha256=vMnWJEiXB5xd3zMdLE3y5w2Zg1lvTDGOE9Hqw5D6ByU,8641
 feldera/rest/config.py,sha256=1_D22PQ1x50_aNGo-WwI-q0gBQID4qJGDJr4MURyYB4,1992
 feldera/rest/errors.py,sha256=wKWwlL5WI3SVXC8rQTy3rS3wQ9Hjjn_EY7iRHXZFIVE,2341
-feldera/rest/feldera_client.py,sha256=BDPvQV6J239i7VTG3Cplvm5C2c6l4fKOqLxhlhJ8Kh0,42168
+feldera/rest/feldera_client.py,sha256=AA6wgmiHtzaj-71cyN7HZPS4TFsUPe8Hehfizrz6g6o,44350
 feldera/rest/feldera_config.py,sha256=1pnGbLFMSLvp7Qh_OlPLALSKCSHIktNWKvx6gYU00U4,1374
 feldera/rest/pipeline.py,sha256=Q6TM44a7-SYRhLWbBGQMriCOcMRDk4WVKjfxd8vumys,3696
 feldera/rest/sql_table.py,sha256=qrw-YwMzx5T81zDefNO1KOx7EyypFz1vPwGBzSUB7kc,652
 feldera/rest/sql_view.py,sha256=hN12mPM0mvwLCIPYywpb12s9Hd2Ws31IlTMXPriMisw,644
 feldera/tests/test_datafusionize.py,sha256=NGriTaTWf_WnXFud1wmpFwLFa_-XGjfCh6La3dWc3QA,1337
-feldera-0.166.0.dist-info/METADATA,sha256=pJX4S_egCCqle-hmabLkJq0ynF3BCMT48wJjGUtQpmo,2306
-feldera-0.166.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-feldera-0.166.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
-feldera-0.166.0.dist-info/RECORD,,
+feldera-0.177.0.dist-info/METADATA,sha256=QexyupSHzqF4FDRhGB_n18odxKrXPmsIygzNcBi3JbE,2306
+feldera-0.177.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+feldera-0.177.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
+feldera-0.177.0.dist-info/RECORD,,