feldera 0.117.0__py3-none-any.whl → 0.119.0__py3-none-any.whl

feldera/pipeline.py

@@ -142,6 +142,7 @@ class Pipeline:
         data: Dict | list,
         update_format: str = "raw",
         force: bool = False,
+        wait: bool = True,
     ):
         """
         Push this JSON data to the specified table of the pipeline.
@@ -155,6 +156,7 @@ class Pipeline:
         :param update_format: The update format of the JSON data to be pushed to the pipeline. Must be one of:
             "raw", "insert_delete". https://docs.feldera.com/formats/json#the-insertdelete-format
         :param force: `True` to push data even if the pipeline is paused. `False` by default.
+        :param wait: If True, blocks until this input has been processed by the pipeline
 
         :raises ValueError: If the update format is invalid.
         :raises FelderaAPIError: If the pipeline is not in a valid state to push data.
@@ -177,6 +179,7 @@ class Pipeline:
             update_format=update_format,
             array=array,
             force=force,
+            wait=wait,
         )
 
     def pause_connector(self, table_name: str, connector_name: str):
@@ -372,7 +375,7 @@ method or use `Pipeline.resume()` to resume a paused pipeline."""
             return
 
         self.client.pause_pipeline(
-            self.name, "Unable to START the pipeline.\n", timeout_s
+            self.name, "Unable to START the pipeline.\n", wait=wait, timeout_s=timeout_s
         )
         self.__setup_output_listeners()
         self.resume(timeout_s=timeout_s)
@@ -506,9 +509,11 @@ metrics"""
                     queue.put(_CallbackRunnerInstruction.RanToCompletion)
 
             if len(self.views_tx) > 0:
-                for view_name, queue in self.views_tx.pop().items():
-                    # block until the callback runner has been stopped
-                    queue.join()
+                while self.views_tx:
+                    view = self.views_tx.pop()
+                    for view_name, queue in view.items():
+                        # block until the callback runner has been stopped
+                        queue.join()
 
         time.sleep(3)
         self.client.stop_pipeline(

feldera/pipeline_builder.py

@@ -45,8 +45,8 @@ class PipelineBuilder:
         self.udf_toml: str = udf_toml
         self.compilation_profile: CompilationProfile = compilation_profile
         self.runtime_config: RuntimeConfig = runtime_config
-        self.runtime_version: Optional[str] = runtime_version or os.environ.get(
-            "FELDERA_RUNTIME_VERSION"
+        self.runtime_version: Optional[str] = os.environ.get(
+            "FELDERA_RUNTIME_VERSION", runtime_version
         )
 
     def create(self) -> Pipeline:
@@ -113,6 +113,7 @@ class PipelineBuilder:
             udf_toml=self.udf_toml,
             program_config={
                 "profile": self.compilation_profile.value,
+                "runtime_version": self.runtime_version,
             },
             runtime_config=self.runtime_config.to_dict(),
         )

feldera/rest/feldera_client.py

@@ -8,7 +8,7 @@ from typing import Generator, Mapping
 
 from feldera.rest.config import Config
 from feldera.rest.feldera_config import FelderaConfig
-from feldera.rest.errors import FelderaTimeoutError
+from feldera.rest.errors import FelderaTimeoutError, FelderaAPIError
 from feldera.rest.pipeline import Pipeline
 from feldera.rest._httprequests import HttpRequests
 from feldera.rest._helpers import client_version
@@ -67,7 +67,7 @@ class FelderaClient:
             config = self.get_config()
             version = client_version()
             if config.version != version:
-                logging.warn(
+                logging.warning(
                     f"Client is on version {version} while server is at "
                     f"{config.version}. There could be incompatibilities."
                 )
@@ -593,7 +593,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         update_format: str = "raw",
         json_flavor: Optional[str] = None,
         serialize: bool = True,
-    ):
+        wait: bool = True,
+        wait_timeout_s: Optional[float] = None,
+    ) -> str:
         """
         Insert data into a pipeline
 
@@ -610,6 +612,11 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             "debezium_mysql", "snowflake", "kafka_connect_json_converter", "pandas"
         :param data: The data to insert
         :param serialize: If True, the data will be serialized to JSON. True by default
+        :param wait: If True, blocks until this input has been processed by the pipeline
+        :param wait_timeout_s: The timeout in seconds to wait for this set of
+            inputs to be processed by the pipeline. None by default
+
+        :returns: The completion token to this input.
         """
 
         if format not in ["json", "csv"]:
@@ -671,7 +678,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             content_type = "text/csv"
             data = bytes(str(data), "utf-8")
 
-        self.http.post(
+        resp = self.http.post(
             path=f"/pipelines/{pipeline_name}/ingress/{table_name}",
             params=params,
             content_type=content_type,
@@ -679,6 +686,74 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             serialize=serialize,
         )
 
+        token = resp.get("token")
+        if token is None:
+            raise FelderaAPIError("response did not contain a completion token", resp)
+
+        if not wait:
+            return token
+
+        self.wait_for_token(pipeline_name, token, timeout_s=wait_timeout_s)
+
+        return token
+
+    def wait_for_token(
+        self, pipeline_name: str, token: str, timeout_s: Optional[float] = 600
+    ):
+        """
+        Blocks until all records represented by this completion token have
+        been processed.
+
+        :param pipeline_name: The name of the pipeline
+        :param token: The token to check for completion
+        :param timeout_s: The amount of time in seconds to wait for the pipeline
+            to process these records. Default 600s
+        """
+
+        params = {
+            "token": token,
+        }
+
+        start = time.monotonic()
+        end = start + timeout_s if timeout_s else None
+        initial_backoff = 0.1
+        max_backoff = 5
+        exponent = 1.2
+        retries = 0
+
+        while True:
+            if end:
+                if time.monotonic() > end:
+                    raise FelderaTimeoutError(
+                        f"timeout error: pipeline '{pipeline_name}' did not"
+                        f" process records represented by token {token} within"
+                        f" {timeout_s}"
+                    )
+
+            resp = self.http.get(
+                path=f"/pipelines/{pipeline_name}/completion_status", params=params
+            )
+
+            status: Optional[str] = resp.get("status")
+            if status is None:
+                raise FelderaAPIError(
+                    f"got empty status when checking for completion status for token: {token}",
+                    resp,
+                )
+
+            if status.lower() == "complete":
+                break
+
+            elapsed = time.monotonic() - start
+            logging.debug(
+                f"still waiting for inputs represented by {token} to be processed; elapsed: {elapsed}s"
+            )
+
+            retries += 1
+            backoff = min(max_backoff, initial_backoff * (exponent**retries))
+
+            time.sleep(backoff)
+
     def listen_to_pipeline(
         self,
         pipeline_name: str,

feldera/stats.py

@@ -16,7 +16,7 @@ class PipelineStatistics:
 
         self.global_metrics: GlobalPipelineMetrics = GlobalPipelineMetrics()
         self.suspend_error: Optional[Any] = None
-        self.inputs: Mapping[List[InputEndpointStatus()]] = {}
+        self.inputs: Mapping[List[InputEndpointStatus]] = {}
         self.outputs: Mapping[List[OutputEndpointStatus]] = {}
 
     @classmethod

{feldera-0.117.0.dist-info → feldera-0.119.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.117.0
+Version: 0.119.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT
@@ -84,7 +84,7 @@ cd python && python3 -m pytest tests/
 - This will detect and run all test files that match the pattern `test_*.py` or
   `*_test.py`.
 - By default, the tests expect a running Feldera instance at `http://localhost:8080`.
-  To override the default endpoint, set the `FELDERA_BASE_URL` environment variable.
+  To override the default endpoint, set the `FELDERA_HOST` environment variable.
 
 To run tests from a specific file:
 

{feldera-0.117.0.dist-info → feldera-0.119.0.dist-info}/RECORD

@@ -3,21 +3,21 @@ feldera/_callback_runner.py,sha256=-2hYG70cEkvz4BiOfUTARaH-2Rlv0Qcz-ilvodgyK10,4
 feldera/_helpers.py,sha256=rN0WuGSCCQlXWFMimZUQrgs-LJAfUo074d79sLElncQ,3023
 feldera/enums.py,sha256=MTHBojVANsdRnjbrzCyIOniDIUaH8nTYRfxB7QvajEE,9570
 feldera/output_handler.py,sha256=64J3ljhOaKIhxdjOKYi-BUz_HnMwROfmN8eE-btYygU,1930
-feldera/pipeline.py,sha256=85goeiKa4sX-Os9edYv2S_vBsqVEfZ7qMAOU1qrWKPg,37580
-feldera/pipeline_builder.py,sha256=25tncJd-qiuHWZOezU34oGfDJqFAdjBEMd9QipNfswc,4195
+feldera/pipeline.py,sha256=ZkmIU0nxpXplCl7iLh6XtdiCZtkR5GGKW33J4gdt1ig,37821
+feldera/pipeline_builder.py,sha256=a750hp5SgTmlyrobTHFh1fTaK9Ed4A5qnXaYRctRM-8,4250
 feldera/runtime_config.py,sha256=MuYJPd5G_hnu_eDz4ge4BfYvSBSOvOEtv4NYh5sEwqU,4452
-feldera/stats.py,sha256=XBhkRsV7FXErwWuPP0i3q9W77mzkMo-oThPVEZy5y3U,5028
+feldera/stats.py,sha256=eZxq51bUV3mlo6BW43DUHwmG1wpLs04rVECxOGojqxU,5026
 feldera/rest/__init__.py,sha256=Eg-EKUU3RSTDcdxTR_7wNDnCly8VpXEzsZCQUmf-y2M,308
 feldera/rest/_helpers.py,sha256=q7jWInKp9IiIli8N5o31lDG3hNUbcsJqufZXYHG04ps,222
 feldera/rest/_httprequests.py,sha256=w8tD-_3spAf4vgalJQceIHQ7qw1uvxprDFM2oz3P5QU,7559
 feldera/rest/config.py,sha256=DYzZKngDEhouTEwqVFd-rDrBN9tWqsU07Jl_BTT4mXs,1008
 feldera/rest/errors.py,sha256=b4i2JjrbSmej7jdko_FL8UeXklLKenSipwMT80jowaM,1720
-feldera/rest/feldera_client.py,sha256=Kz4_JLblaJtxtXcbBhoTpjiq123HvRPioNyGB8Jiu2o,30218
+feldera/rest/feldera_client.py,sha256=tU99wyMDNkyHerom1QKeg1WZmno7bLNrIzbU2f98-7U,32855
 feldera/rest/feldera_config.py,sha256=1pnGbLFMSLvp7Qh_OlPLALSKCSHIktNWKvx6gYU00U4,1374
 feldera/rest/pipeline.py,sha256=Rmbflbwjvd86iZ5aSJ5b_bTSs6vgvEKQFwMZDtm0nxE,2835
 feldera/rest/sql_table.py,sha256=qrw-YwMzx5T81zDefNO1KOx7EyypFz1vPwGBzSUB7kc,652
 feldera/rest/sql_view.py,sha256=hN12mPM0mvwLCIPYywpb12s9Hd2Ws31IlTMXPriMisw,644
-feldera-0.117.0.dist-info/METADATA,sha256=WIxaKR_8gNIXNle4oOJ6aUY4X7oYPzebL9Jixzm-o8I,4057
-feldera-0.117.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-feldera-0.117.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
-feldera-0.117.0.dist-info/RECORD,,
+feldera-0.119.0.dist-info/METADATA,sha256=a-EKosH2QXmtKRGM2XhxxM-Q8G8kYUWd5j8xIaOq-Wc,4053
+feldera-0.119.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+feldera-0.119.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
+feldera-0.119.0.dist-info/RECORD,,