feldera 0.115.0__py3-none-any.whl → 0.117.0__py3-none-any.whl

feldera/pipeline.py

@@ -78,6 +78,11 @@ class Pipeline:
 
         return PipelineStatistics.from_dict(self.client.get_pipeline_stats(self.name))
 
+    def logs(self) -> Generator[str, None, None]:
+        """Gets the pipeline logs."""
+
+        return self.client.get_pipeline_logs(self.name)
+
     def input_pandas(self, table_name: str, df: pandas.DataFrame, force: bool = False):
         """
         Push all rows in a pandas DataFrame to the pipeline.
@@ -113,7 +118,8 @@ class Pipeline:
             tbl.name.lower() for tbl in pipeline.tables
         ]:
             raise ValueError(
-                f"Cannot push to table '{table_name}': table with this name does not exist in the '{self.name}' pipeline"
+                f"Cannot push to table '{table_name}': table with this name"
+                f" does not exist in the '{self.name}' pipeline"
             )
         else:
             # consider validating the schema here
@@ -308,10 +314,12 @@ class Pipeline:
                 elapsed = time.monotonic() - start_time
                 if elapsed > timeout_s:
                     raise TimeoutError(
-                        f"timeout ({timeout_s}s) reached while waiting for pipeline '{self.name}' to complete"
+                        f"timeout ({timeout_s}s) reached while waiting for"
+                        f" pipeline '{self.name}' to complete"
                     )
                 logging.debug(
-                    f"waiting for pipeline {self.name} to complete: elapsed time {elapsed}s, timeout: {timeout_s}s"
+                    f"waiting for pipeline {self.name} to complete: elapsed"
+                    f" time {elapsed}s, timeout: {timeout_s}s"
                 )
 
             pipeline_complete: bool = self.stats().global_metrics.pipeline_complete
@@ -327,7 +335,7 @@ class Pipeline:
         if force_stop:
             self.stop(force=True)
 
-    def start(self, timeout_s: Optional[float] = None):
+    def start(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         .. _start:
 
@@ -339,6 +347,7 @@ class Pipeline:
 
         :param timeout_s: The maximum time (in seconds) to wait for the
             pipeline to start.
+        :param wait: Set True to wait for the pipeline to start. True by default
 
         :raises RuntimeError: If the pipeline is not in STOPPED state.
         """
@@ -352,11 +361,21 @@ started. You can either stop the pipeline using the `Pipeline.stop()` \
 method or use `Pipeline.resume()` to resume a paused pipeline."""
             )
 
+        if not wait:
+            if len(self.views_tx) > 0:
+                raise ValueError(
+                    "cannot start with 'wait=False' when output listeners are configured. Try setting 'wait=True'."
+                )
+
+            self.client.start_pipeline(self.name, wait=wait)
+
+            return
+
         self.client.pause_pipeline(
             self.name, "Unable to START the pipeline.\n", timeout_s
         )
         self.__setup_output_listeners()
-        self.resume(timeout_s)
+        self.resume(timeout_s=timeout_s)
 
     def restart(self, timeout_s: Optional[float] = None):
         """
@@ -371,7 +390,7 @@ method or use `Pipeline.resume()` to resume a paused pipeline."""
         """
 
         self.stop(force=True, timeout_s=timeout_s)
-        self.start(timeout_s)
+        self.start(timeout_s=timeout_s)
 
     def wait_for_idle(
         self,
@@ -398,11 +417,13 @@ method or use `Pipeline.resume()` to resume a paused pipeline."""
         """
         if idle_interval_s > timeout_s:
             raise ValueError(
-                f"idle interval ({idle_interval_s}s) cannot be larger than timeout ({timeout_s}s)"
+                f"idle interval ({idle_interval_s}s) cannot be larger than"
+                f" timeout ({timeout_s}s)"
             )
         if poll_interval_s > timeout_s:
             raise ValueError(
-                f"poll interval ({poll_interval_s}s) cannot be larger than timeout ({timeout_s}s)"
+                f"poll interval ({poll_interval_s}s) cannot be larger than"
+                f" timeout ({timeout_s}s)"
             )
         if poll_interval_s > idle_interval_s:
             raise ValueError(
@@ -449,7 +470,7 @@ metrics"""
                 raise RuntimeError(f"waiting for idle reached timeout ({timeout_s}s)")
             time.sleep(poll_interval_s)
 
-    def pause(self, timeout_s: Optional[float] = None):
+    def pause(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         Pause the pipeline.
 
@@ -457,13 +478,14 @@ metrics"""
         state. If the pipeline is already paused, it will remain in the PAUSED
         state.
 
+        :param wait: Set True to wait for the pipeline to pause. True by default
         :param timeout_s: The maximum time (in seconds) to wait for the
             pipeline to pause.
         """
 
-        self.client.pause_pipeline(self.name, timeout_s=timeout_s)
+        self.client.pause_pipeline(self.name, wait=wait, timeout_s=timeout_s)
 
-    def stop(self, force: bool, timeout_s: Optional[float] = None):
+    def stop(self, force: bool, wait: bool = True, timeout_s: Optional[float] = None):
         """
         Stops the pipeline.
 
@@ -471,34 +493,75 @@ metrics"""
 
         :param force: Set True to immediately scale compute resources to zero.
             Set False to automatically checkpoint before stopping.
+        :param wait: Set True to gracefully shutdown listeners and wait for the
+            pipeline to stop. True by default.
         :param timeout_s: The maximum time (in seconds) to wait for the
             pipeline to stop.
         """
 
-        for view_queue in self.views_tx:
-            for _, queue in view_queue.items():
-                # sends a message to the callback runner to stop listening
-                queue.put(_CallbackRunnerInstruction.RanToCompletion)
+        if wait:
+            for view_queue in self.views_tx:
+                for _, queue in view_queue.items():
+                    # sends a message to the callback runner to stop listening
+                    queue.put(_CallbackRunnerInstruction.RanToCompletion)
 
-        if len(self.views_tx) > 0:
-            for view_name, queue in self.views_tx.pop().items():
-                # block until the callback runner has been stopped
-                queue.join()
-        import time
+            if len(self.views_tx) > 0:
+                for view_name, queue in self.views_tx.pop().items():
+                    # block until the callback runner has been stopped
+                    queue.join()
 
         time.sleep(3)
-        self.client.stop_pipeline(self.name, force=force, timeout_s=timeout_s)
+        self.client.stop_pipeline(
+            self.name, force=force, wait=wait, timeout_s=timeout_s
+        )
 
-    def resume(self, timeout_s: Optional[float] = None):
+    def resume(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         Resumes the pipeline from the PAUSED state. If the pipeline is already
         running, it will remain in the RUNNING state.
 
+        :param wait: Set True to wait for the pipeline to resume. True by default
         :param timeout_s: The maximum time (in seconds) to wait for the
             pipeline to resume.
         """
 
-        self.client.start_pipeline(self.name, timeout_s=timeout_s)
+        self.client.start_pipeline(self.name, wait=wait, timeout_s=timeout_s)
+
+    def start_transaction(self):
+        """
+        Start a new transaction.
+
+        Returns:
+            Transaction ID.
+        """
+
+        self.client.start_transaction(self.name)
+
+    def commit_transaction(
+        self,
+        transaction_id: Optional[int] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Commits the currently active transaction.
+
+        :param transaction_id: If provided, the function verifies that the currently active transaction matches this ID.
+            If the active transaction ID does not match, the function raises an error.
+
+        :param wait: If True, the function blocks until the transaction either commits successfully or the timeout is reached.
+            If False, the function initiates the commit and returns immediately without waiting for completion. The default value is True.
+
+        :param timeout_s: Maximum time (in seconds) to wait for the transaction to commit when `wait` is True.
+            If None, the function will wait indefinitely.
+
+        :raises RuntimeError: If there is currently no transaction in progress.
+        :raises ValueError: If the provided `transaction_id` does not match the current transaction.
+        :raises TimeoutError: If the transaction does not commit within the specified timeout (when `wait` is True).
+        :raises FelderaAPIError: If the pipeline fails to start a transaction.
+        """
+
+        self.client.commit_transaction(self.name, transaction_id, wait, timeout_s)
 
     def delete(self, clear_storage: bool = False):
         """
@@ -597,7 +660,7 @@ pipeline '{self.name}' to make checkpoint '{seq}'"""
         """
         Syncs this checkpoint to object store.
 
-        :param wait: If true, will block until the checkpoint sync opeartion
+        :param wait: If true, will block until the checkpoint sync operation
             completes.
         :param timeout_s: The maximum time (in seconds) to wait for the
             checkpoint to complete syncing.
@@ -956,16 +1019,6 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         self.refresh()
         return self._inner.deployment_location
 
-    def program_binary_url(self) -> str:
-        """
-        Return the program binary URL of the pipeline.
-        This is the URL where the compiled program binary can be downloaded
-        from.
-        """
-
-        self.refresh()
-        return self._inner.program_binary_url
-
     def program_info(self) -> Mapping[str, Any]:
         """
         Return the program info of the pipeline.

feldera/rest/feldera_client.py

@@ -242,10 +242,29 @@ class FelderaClient:
 
         return resp
 
-    def start_pipeline(self, pipeline_name: str, timeout_s: Optional[float] = 300):
+    def get_pipeline_logs(self, pipeline_name: str) -> Generator[str, None, None]:
+        """
+        Get the pipeline logs
+
+        :param name: The name of the pipeline
+        :return: A generator yielding the logs, one line at a time.
+        """
+        chunk: bytes
+        with self.http.get(
+            path=f"/pipelines/{pipeline_name}/logs",
+            stream=True,
+        ) as resp:
+            for chunk in resp.iter_lines(chunk_size=50000000):
+                if chunk:
+                    yield chunk.decode("utf-8")
+
+    def start_pipeline(
+        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
+    ):
         """
 
         :param pipeline_name: The name of the pipeline to start
+        :param wait: Set True to wait for the pipeline to start. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
             to start. 300 seconds by default.
         """
@@ -257,6 +276,9 @@ class FelderaClient:
             path=f"/pipelines/{pipeline_name}/start",
         )
 
+        if not wait:
+            return
+
         start_time = time.monotonic()
 
         while True:
@@ -292,6 +314,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         self,
         pipeline_name: str,
         error_message: str = None,
+        wait: bool = True,
         timeout_s: Optional[float] = 300,
     ):
         """
@@ -300,6 +323,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         :param pipeline_name: The name of the pipeline to stop
         :param error_message: The error message to show if the pipeline is in
              STOPPED state due to a failure.
+        :param wait: Set True to wait for the pipeline to pause. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
             to pause. 300 seconds by default.
         """
@@ -311,6 +335,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             path=f"/pipelines/{pipeline_name}/pause",
         )
 
+        if not wait:
+            return
+
         if error_message is None:
             error_message = "Unable to PAUSE the pipeline.\n"
 
@@ -346,7 +373,11 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             time.sleep(0.1)
 
     def stop_pipeline(
-        self, pipeline_name: str, force: bool, timeout_s: Optional[float] = 300
+        self,
+        pipeline_name: str,
+        force: bool,
+        wait: bool = True,
+        timeout_s: Optional[float] = 300,
     ):
         """
         Stop a pipeline
@@ -354,6 +385,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         :param pipeline_name: The name of the pipeline to stop
         :param force: Set True to immediately scale compute resources to zero.
             Set False to automatically checkpoint before stopping.
+        :param wait: Set True to wait for the pipeline to stop. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
             to stop. Default is 300 seconds.
         """
@@ -368,6 +400,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             params=params,
         )
 
+        if not wait:
+            return
+
         start = time.monotonic()
 
         while time.monotonic() - start < timeout_s:
@@ -420,6 +455,86 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             f"timeout error: pipeline '{pipeline_name}' did not clear storage in {timeout_s} seconds"
         )
 
+    def start_transaction(self, pipeline_name: str) -> int:
+        """
+        Start a new transaction.
+
+            Transaction ID.
+
+        :param pipeline_name: The name of the pipeline.
+        """
+
+        resp = self.http.post(
+            path=f"/pipelines/{pipeline_name}/start_transaction",
+        )
+
+        return int(resp.get("transaction_id"))
+
+    def commit_transaction(
+        self,
+        pipeline_name: str,
+        transaction_id: Optional[int] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Commits the currently active transaction.
+
+        :param pipeline_name: The name of the pipeline.
+
+        :param transaction_id: If provided, the function verifies that the currently active transaction matches this ID.
+            If the active transaction ID does not match, the function raises an error.
+
+        :param wait: If True, the function blocks until the transaction either commits successfully or the timeout is reached.
+            If False, the function initiates the commit and returns immediately without waiting for completion. The default value is True.
+
+        :param timeout_s: Maximum time (in seconds) to wait for the transaction to commit when `wait` is True.
+            If None, the function will wait indefinitely.
+
+        :raises RuntimeError: If there is currently no transaction in progress.
+        :raises ValueError: If the provided `transaction_id` does not match the current transaction.
+        :raises TimeoutError: If the transaction does not commit within the specified timeout (when `wait` is True).
+        :raises FelderaAPIError: If the pipeline fails to start a transaction.
+        """
+
+        # TODO: implement this without using /stats when we have a better pipeline status reporting API.
+        stats = self.get_pipeline_stats(pipeline_name)
+        current_transaction_id = stats["global_metrics"]["transaction_id"]
+
+        if current_transaction_id == 0:
+            raise RuntimeError(
+                "Attempting to commit a transaction, but there is no transaction in progress"
+            )
+
+        if transaction_id and current_transaction_id != transaction_id:
+            raise ValueError(
+                f"Specified transaction id {transaction_id} doesn't match current active transaction id {current_transaction_id}"
+            )
+
+        transaction_id = current_transaction_id
+
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/commit_transaction",
+        )
+
+        start_time = time.monotonic()
+
+        if not wait:
+            return
+
+        while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError("Timed out waiting for transaction to commit")
+
+            stats = self.get_pipeline_stats(pipeline_name)
+            if stats["global_metrics"]["transaction_id"] != transaction_id:
+                return
+
+            logging.debug("commit hasn't completed, waiting for 1 more second")
+            time.sleep(1.0)
+
     def checkpoint_pipeline(self, pipeline_name: str) -> int:
         """
         Checkpoint a pipeline.

feldera/rest/pipeline.py

@@ -49,7 +49,6 @@ class Pipeline:
         self.deployment_desired_status: Optional[str] = None
         self.deployment_error: Optional[dict] = None
         self.deployment_location: Optional[str] = None
-        self.program_binary_url: Optional[str] = None
         self.program_info: Optional[dict] = (
             None  # info about input & output connectors and the schema
         )

{feldera-0.115.0.dist-info → feldera-0.117.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.115.0
+Version: 0.117.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.115.0.dist-info → feldera-0.117.0.dist-info}/RECORD

@@ -3,7 +3,7 @@ feldera/_callback_runner.py,sha256=-2hYG70cEkvz4BiOfUTARaH-2Rlv0Qcz-ilvodgyK10,4
 feldera/_helpers.py,sha256=rN0WuGSCCQlXWFMimZUQrgs-LJAfUo074d79sLElncQ,3023
 feldera/enums.py,sha256=MTHBojVANsdRnjbrzCyIOniDIUaH8nTYRfxB7QvajEE,9570
 feldera/output_handler.py,sha256=64J3ljhOaKIhxdjOKYi-BUz_HnMwROfmN8eE-btYygU,1930
-feldera/pipeline.py,sha256=1XNjqvD9MTXLuz6lomuySTF4WZndTVQFv_4OmLkKzq8,35206
+feldera/pipeline.py,sha256=85goeiKa4sX-Os9edYv2S_vBsqVEfZ7qMAOU1qrWKPg,37580
 feldera/pipeline_builder.py,sha256=25tncJd-qiuHWZOezU34oGfDJqFAdjBEMd9QipNfswc,4195
 feldera/runtime_config.py,sha256=MuYJPd5G_hnu_eDz4ge4BfYvSBSOvOEtv4NYh5sEwqU,4452
 feldera/stats.py,sha256=XBhkRsV7FXErwWuPP0i3q9W77mzkMo-oThPVEZy5y3U,5028
@@ -12,12 +12,12 @@ feldera/rest/_helpers.py,sha256=q7jWInKp9IiIli8N5o31lDG3hNUbcsJqufZXYHG04ps,222
 feldera/rest/_httprequests.py,sha256=w8tD-_3spAf4vgalJQceIHQ7qw1uvxprDFM2oz3P5QU,7559
 feldera/rest/config.py,sha256=DYzZKngDEhouTEwqVFd-rDrBN9tWqsU07Jl_BTT4mXs,1008
 feldera/rest/errors.py,sha256=b4i2JjrbSmej7jdko_FL8UeXklLKenSipwMT80jowaM,1720
-feldera/rest/feldera_client.py,sha256=21Izb8DVv06LxdxvnByR_Mhm-lOLmKtnAc00PW7bTmc,25977
+feldera/rest/feldera_client.py,sha256=Kz4_JLblaJtxtXcbBhoTpjiq123HvRPioNyGB8Jiu2o,30218
 feldera/rest/feldera_config.py,sha256=1pnGbLFMSLvp7Qh_OlPLALSKCSHIktNWKvx6gYU00U4,1374
-feldera/rest/pipeline.py,sha256=-dGGUdtHMABKrQEclaeuwGI_FOCrQOk6p2aCFV0FdU8,2890
+feldera/rest/pipeline.py,sha256=Rmbflbwjvd86iZ5aSJ5b_bTSs6vgvEKQFwMZDtm0nxE,2835
 feldera/rest/sql_table.py,sha256=qrw-YwMzx5T81zDefNO1KOx7EyypFz1vPwGBzSUB7kc,652
 feldera/rest/sql_view.py,sha256=hN12mPM0mvwLCIPYywpb12s9Hd2Ws31IlTMXPriMisw,644
-feldera-0.115.0.dist-info/METADATA,sha256=BGEuAOTjBXnKDe4Jme3v-PHmXXTcTI2fIVq38PvtAp0,4057
-feldera-0.115.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-feldera-0.115.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
-feldera-0.115.0.dist-info/RECORD,,
+feldera-0.117.0.dist-info/METADATA,sha256=WIxaKR_8gNIXNle4oOJ6aUY4X7oYPzebL9Jixzm-o8I,4057
+feldera-0.117.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+feldera-0.117.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
+feldera-0.117.0.dist-info/RECORD,,