feldera 0.115.0__tar.gz → 0.117.0__tar.gz

{feldera-0.115.0 → feldera-0.117.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.115.0
+Version: 0.117.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.115.0 → feldera-0.117.0}/feldera/pipeline.py

@@ -78,6 +78,11 @@ class Pipeline:
 
         return PipelineStatistics.from_dict(self.client.get_pipeline_stats(self.name))
 
+    def logs(self) -> Generator[str, None, None]:
+        """Gets the pipeline logs."""
+
+        return self.client.get_pipeline_logs(self.name)
+
     def input_pandas(self, table_name: str, df: pandas.DataFrame, force: bool = False):
         """
         Push all rows in a pandas DataFrame to the pipeline.
@@ -113,7 +118,8 @@ class Pipeline:
             tbl.name.lower() for tbl in pipeline.tables
         ]:
             raise ValueError(
-                f"Cannot push to table '{table_name}': table with this name does not exist in the '{self.name}' pipeline"
+                f"Cannot push to table '{table_name}': table with this name"
+                f" does not exist in the '{self.name}' pipeline"
             )
         else:
             # consider validating the schema here
@@ -308,10 +314,12 @@ class Pipeline:
                 elapsed = time.monotonic() - start_time
                 if elapsed > timeout_s:
                     raise TimeoutError(
-                        f"timeout ({timeout_s}s) reached while waiting for pipeline '{self.name}' to complete"
+                        f"timeout ({timeout_s}s) reached while waiting for"
+                        f" pipeline '{self.name}' to complete"
                     )
                 logging.debug(
-                    f"waiting for pipeline {self.name} to complete: elapsed time {elapsed}s, timeout: {timeout_s}s"
+                    f"waiting for pipeline {self.name} to complete: elapsed"
+                    f" time {elapsed}s, timeout: {timeout_s}s"
                 )
 
             pipeline_complete: bool = self.stats().global_metrics.pipeline_complete
@@ -327,7 +335,7 @@ class Pipeline:
         if force_stop:
             self.stop(force=True)
 
-    def start(self, timeout_s: Optional[float] = None):
+    def start(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         .. _start:
 
@@ -339,6 +347,7 @@ class Pipeline:
 
         :param timeout_s: The maximum time (in seconds) to wait for the
             pipeline to start.
+        :param wait: Set True to wait for the pipeline to start. True by default
 
         :raises RuntimeError: If the pipeline is not in STOPPED state.
         """
@@ -352,11 +361,21 @@ started. You can either stop the pipeline using the `Pipeline.stop()` \
 method or use `Pipeline.resume()` to resume a paused pipeline."""
             )
 
+        if not wait:
+            if len(self.views_tx) > 0:
+                raise ValueError(
+                    "cannot start with 'wait=False' when output listeners are configured. Try setting 'wait=True'."
+                )
+
+            self.client.start_pipeline(self.name, wait=wait)
+
+            return
+
         self.client.pause_pipeline(
             self.name, "Unable to START the pipeline.\n", timeout_s
         )
         self.__setup_output_listeners()
-        self.resume(timeout_s)
+        self.resume(timeout_s=timeout_s)
 
     def restart(self, timeout_s: Optional[float] = None):
         """
@@ -371,7 +390,7 @@ method or use `Pipeline.resume()` to resume a paused pipeline."""
         """
 
         self.stop(force=True, timeout_s=timeout_s)
-        self.start(timeout_s)
+        self.start(timeout_s=timeout_s)
 
     def wait_for_idle(
         self,
@@ -398,11 +417,13 @@ method or use `Pipeline.resume()` to resume a paused pipeline."""
         """
         if idle_interval_s > timeout_s:
             raise ValueError(
-                f"idle interval ({idle_interval_s}s) cannot be larger than timeout ({timeout_s}s)"
+                f"idle interval ({idle_interval_s}s) cannot be larger than"
+                f" timeout ({timeout_s}s)"
             )
         if poll_interval_s > timeout_s:
             raise ValueError(
-                f"poll interval ({poll_interval_s}s) cannot be larger than timeout ({timeout_s}s)"
+                f"poll interval ({poll_interval_s}s) cannot be larger than"
+                f" timeout ({timeout_s}s)"
             )
         if poll_interval_s > idle_interval_s:
             raise ValueError(
@@ -449,7 +470,7 @@ metrics"""
                 raise RuntimeError(f"waiting for idle reached timeout ({timeout_s}s)")
             time.sleep(poll_interval_s)
 
-    def pause(self, timeout_s: Optional[float] = None):
+    def pause(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         Pause the pipeline.
 
@@ -457,13 +478,14 @@ metrics"""
         state. If the pipeline is already paused, it will remain in the PAUSED
         state.
 
+        :param wait: Set True to wait for the pipeline to pause. True by default
         :param timeout_s: The maximum time (in seconds) to wait for the
             pipeline to pause.
         """
 
-        self.client.pause_pipeline(self.name, timeout_s=timeout_s)
+        self.client.pause_pipeline(self.name, wait=wait, timeout_s=timeout_s)
 
-    def stop(self, force: bool, timeout_s: Optional[float] = None):
+    def stop(self, force: bool, wait: bool = True, timeout_s: Optional[float] = None):
         """
         Stops the pipeline.
 
@@ -471,34 +493,75 @@ metrics"""
 
         :param force: Set True to immediately scale compute resources to zero.
             Set False to automatically checkpoint before stopping.
+        :param wait: Set True to gracefully shutdown listeners and wait for the
+            pipeline to stop. True by default.
         :param timeout_s: The maximum time (in seconds) to wait for the
             pipeline to stop.
         """
 
-        for view_queue in self.views_tx:
-            for _, queue in view_queue.items():
-                # sends a message to the callback runner to stop listening
-                queue.put(_CallbackRunnerInstruction.RanToCompletion)
+        if wait:
+            for view_queue in self.views_tx:
+                for _, queue in view_queue.items():
+                    # sends a message to the callback runner to stop listening
+                    queue.put(_CallbackRunnerInstruction.RanToCompletion)
 
-        if len(self.views_tx) > 0:
-            for view_name, queue in self.views_tx.pop().items():
-                # block until the callback runner has been stopped
-                queue.join()
-        import time
+            if len(self.views_tx) > 0:
+                for view_name, queue in self.views_tx.pop().items():
+                    # block until the callback runner has been stopped
+                    queue.join()
 
         time.sleep(3)
-        self.client.stop_pipeline(self.name, force=force, timeout_s=timeout_s)
+        self.client.stop_pipeline(
+            self.name, force=force, wait=wait, timeout_s=timeout_s
+        )
 
-    def resume(self, timeout_s: Optional[float] = None):
+    def resume(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         Resumes the pipeline from the PAUSED state. If the pipeline is already
         running, it will remain in the RUNNING state.
 
+        :param wait: Set True to wait for the pipeline to resume. True by default
         :param timeout_s: The maximum time (in seconds) to wait for the
             pipeline to resume.
         """
 
-        self.client.start_pipeline(self.name, timeout_s=timeout_s)
+        self.client.start_pipeline(self.name, wait=wait, timeout_s=timeout_s)
+
+    def start_transaction(self):
+        """
+        Start a new transaction.
+
+        Returns:
+            Transaction ID.
+        """
+
+        self.client.start_transaction(self.name)
+
+    def commit_transaction(
+        self,
+        transaction_id: Optional[int] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Commits the currently active transaction.
+
+        :param transaction_id: If provided, the function verifies that the currently active transaction matches this ID.
+            If the active transaction ID does not match, the function raises an error.
+
+        :param wait: If True, the function blocks until the transaction either commits successfully or the timeout is reached.
+            If False, the function initiates the commit and returns immediately without waiting for completion. The default value is True.
+
+        :param timeout_s: Maximum time (in seconds) to wait for the transaction to commit when `wait` is True.
+            If None, the function will wait indefinitely.
+
+        :raises RuntimeError: If there is currently no transaction in progress.
+        :raises ValueError: If the provided `transaction_id` does not match the current transaction.
+        :raises TimeoutError: If the transaction does not commit within the specified timeout (when `wait` is True).
+        :raises FelderaAPIError: If the pipeline fails to start a transaction.
+        """
+
+        self.client.commit_transaction(self.name, transaction_id, wait, timeout_s)
 
     def delete(self, clear_storage: bool = False):
         """
@@ -597,7 +660,7 @@ pipeline '{self.name}' to make checkpoint '{seq}'"""
         """
         Syncs this checkpoint to object store.
 
-        :param wait: If true, will block until the checkpoint sync opeartion
+        :param wait: If true, will block until the checkpoint sync operation
             completes.
         :param timeout_s: The maximum time (in seconds) to wait for the
             checkpoint to complete syncing.
@@ -956,16 +1019,6 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         self.refresh()
         return self._inner.deployment_location
 
-    def program_binary_url(self) -> str:
-        """
-        Return the program binary URL of the pipeline.
-        This is the URL where the compiled program binary can be downloaded
-        from.
-        """
-
-        self.refresh()
-        return self._inner.program_binary_url
-
     def program_info(self) -> Mapping[str, Any]:
         """
         Return the program info of the pipeline.

{feldera-0.115.0 → feldera-0.117.0}/feldera/rest/feldera_client.py

@@ -242,10 +242,29 @@ class FelderaClient:
 
         return resp
 
-    def start_pipeline(self, pipeline_name: str, timeout_s: Optional[float] = 300):
+    def get_pipeline_logs(self, pipeline_name: str) -> Generator[str, None, None]:
+        """
+        Get the pipeline logs
+
+        :param name: The name of the pipeline
+        :return: A generator yielding the logs, one line at a time.
+        """
+        chunk: bytes
+        with self.http.get(
+            path=f"/pipelines/{pipeline_name}/logs",
+            stream=True,
+        ) as resp:
+            for chunk in resp.iter_lines(chunk_size=50000000):
+                if chunk:
+                    yield chunk.decode("utf-8")
+
+    def start_pipeline(
+        self, pipeline_name: str, wait: bool = True, timeout_s: Optional[float] = 300
+    ):
         """
 
         :param pipeline_name: The name of the pipeline to start
+        :param wait: Set True to wait for the pipeline to start. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
             to start. 300 seconds by default.
         """
@@ -257,6 +276,9 @@ class FelderaClient:
             path=f"/pipelines/{pipeline_name}/start",
         )
 
+        if not wait:
+            return
+
         start_time = time.monotonic()
 
         while True:
@@ -292,6 +314,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         self,
         pipeline_name: str,
         error_message: str = None,
+        wait: bool = True,
         timeout_s: Optional[float] = 300,
     ):
         """
@@ -300,6 +323,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         :param pipeline_name: The name of the pipeline to stop
         :param error_message: The error message to show if the pipeline is in
              STOPPED state due to a failure.
+        :param wait: Set True to wait for the pipeline to pause. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
             to pause. 300 seconds by default.
         """
@@ -311,6 +335,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             path=f"/pipelines/{pipeline_name}/pause",
         )
 
+        if not wait:
+            return
+
         if error_message is None:
             error_message = "Unable to PAUSE the pipeline.\n"
 
@@ -346,7 +373,11 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             time.sleep(0.1)
 
     def stop_pipeline(
-        self, pipeline_name: str, force: bool, timeout_s: Optional[float] = 300
+        self,
+        pipeline_name: str,
+        force: bool,
+        wait: bool = True,
+        timeout_s: Optional[float] = 300,
     ):
         """
         Stop a pipeline
@@ -354,6 +385,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         :param pipeline_name: The name of the pipeline to stop
         :param force: Set True to immediately scale compute resources to zero.
             Set False to automatically checkpoint before stopping.
+        :param wait: Set True to wait for the pipeline to stop. True by default
         :param timeout_s: The amount of time in seconds to wait for the pipeline
             to stop. Default is 300 seconds.
         """
@@ -368,6 +400,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             params=params,
         )
 
+        if not wait:
+            return
+
         start = time.monotonic()
 
         while time.monotonic() - start < timeout_s:
@@ -420,6 +455,86 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             f"timeout error: pipeline '{pipeline_name}' did not clear storage in {timeout_s} seconds"
         )
 
+    def start_transaction(self, pipeline_name: str) -> int:
+        """
+        Start a new transaction.
+
+            Transaction ID.
+
+        :param pipeline_name: The name of the pipeline.
+        """
+
+        resp = self.http.post(
+            path=f"/pipelines/{pipeline_name}/start_transaction",
+        )
+
+        return int(resp.get("transaction_id"))
+
+    def commit_transaction(
+        self,
+        pipeline_name: str,
+        transaction_id: Optional[int] = None,
+        wait: bool = True,
+        timeout_s: Optional[float] = None,
+    ):
+        """
+        Commits the currently active transaction.
+
+        :param pipeline_name: The name of the pipeline.
+
+        :param transaction_id: If provided, the function verifies that the currently active transaction matches this ID.
+            If the active transaction ID does not match, the function raises an error.
+
+        :param wait: If True, the function blocks until the transaction either commits successfully or the timeout is reached.
+            If False, the function initiates the commit and returns immediately without waiting for completion. The default value is True.
+
+        :param timeout_s: Maximum time (in seconds) to wait for the transaction to commit when `wait` is True.
+            If None, the function will wait indefinitely.
+
+        :raises RuntimeError: If there is currently no transaction in progress.
+        :raises ValueError: If the provided `transaction_id` does not match the current transaction.
+        :raises TimeoutError: If the transaction does not commit within the specified timeout (when `wait` is True).
+        :raises FelderaAPIError: If the pipeline fails to start a transaction.
+        """
+
+        # TODO: implement this without using /stats when we have a better pipeline status reporting API.
+        stats = self.get_pipeline_stats(pipeline_name)
+        current_transaction_id = stats["global_metrics"]["transaction_id"]
+
+        if current_transaction_id == 0:
+            raise RuntimeError(
+                "Attempting to commit a transaction, but there is no transaction in progress"
+            )
+
+        if transaction_id and current_transaction_id != transaction_id:
+            raise ValueError(
+                f"Specified transaction id {transaction_id} doesn't match current active transaction id {current_transaction_id}"
+            )
+
+        transaction_id = current_transaction_id
+
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/commit_transaction",
+        )
+
+        start_time = time.monotonic()
+
+        if not wait:
+            return
+
+        while True:
+            if timeout_s is not None:
+                elapsed = time.monotonic() - start_time
+                if elapsed > timeout_s:
+                    raise TimeoutError("Timed out waiting for transaction to commit")
+
+            stats = self.get_pipeline_stats(pipeline_name)
+            if stats["global_metrics"]["transaction_id"] != transaction_id:
+                return
+
+            logging.debug("commit hasn't completed, waiting for 1 more second")
+            time.sleep(1.0)
+
     def checkpoint_pipeline(self, pipeline_name: str) -> int:
         """
         Checkpoint a pipeline.

{feldera-0.115.0 → feldera-0.117.0}/feldera/rest/pipeline.py

@@ -49,7 +49,6 @@ class Pipeline:
         self.deployment_desired_status: Optional[str] = None
         self.deployment_error: Optional[dict] = None
         self.deployment_location: Optional[str] = None
-        self.program_binary_url: Optional[str] = None
         self.program_info: Optional[dict] = (
             None  # info about input & output connectors and the schema
         )

{feldera-0.115.0 → feldera-0.117.0}/feldera.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.115.0
+Version: 0.117.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.115.0 → feldera-0.117.0}/feldera.egg-info/SOURCES.txt

@@ -27,4 +27,5 @@ feldera/rest/sql_view.py
 tests/test_pipeline_builder.py
 tests/test_shared_pipeline0.py
 tests/test_shared_pipeline1.py
+tests/test_shared_pipeline_stress.py
 tests/test_udf.py

{feldera-0.115.0 → feldera-0.117.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "feldera"
 readme = "README.md"
 description = "The feldera python client"
-version = "0.115.0"
+version = "0.117.0"
 license = { text = "MIT" }
 requires-python = ">=3.10"
 authors = [

{feldera-0.115.0 → feldera-0.117.0}/tests/test_shared_pipeline1.py

@@ -1,11 +1,11 @@
-import random
-from uuid import uuid4
-import time
-import os
-from typing import Optional
-from feldera.runtime_config import RuntimeConfig, Storage
-from tests import enterprise_only
 from tests.shared_test_pipeline import SharedTestPipeline
+from tests import enterprise_only
+from feldera.runtime_config import RuntimeConfig, Storage
+from typing import Optional
+import os
+import time
+from uuid import uuid4
+import random
 
 
 DEFAULT_ENDPOINT = os.environ.get(
@@ -19,6 +19,7 @@ SECRET_KEY = "miniopasswd"
 def storage_cfg(
     endpoint: Optional[str] = None,
     start_from_checkpoint: Optional[str] = None,
+    strict: bool = False,
     auth_err: bool = False,
 ) -> dict:
     return {
@@ -32,6 +33,7 @@ def storage_cfg(
                     "provider": "Minio",
                     "endpoint": endpoint or DEFAULT_ENDPOINT,
                     "start_from_checkpoint": start_from_checkpoint,
+                    "fail_if_no_checkpoint": strict,
                 }
             },
         }
@@ -46,6 +48,8 @@ class TestCheckpointSync(SharedTestPipeline):
         random_uuid: bool = False,
         clear_storage: bool = True,
         auth_err: bool = False,
+        strict: bool = False,
+        expect_empty: bool = False,
     ):
         """
         CREATE TABLE t0 (c0 INT, c1 VARCHAR);
@@ -75,12 +79,17 @@ class TestCheckpointSync(SharedTestPipeline):
 
         # Restart pipeline from checkpoint
         storage_config = storage_cfg(
-            start_from_checkpoint=uuid if from_uuid else "latest", auth_err=auth_err
+            start_from_checkpoint=uuid if from_uuid else "latest",
+            auth_err=auth_err,
+            strict=strict,
         )
         self.set_runtime_config(RuntimeConfig(storage=Storage(config=storage_config)))
         self.pipeline.start()
         got_after = list(self.pipeline.query("SELECT * FROM v0"))
 
+        if expect_empty:
+            got_before = []
+
         self.assertCountEqual(got_before, got_after)
 
         self.pipeline.stop(force=True)
@@ -89,19 +98,27 @@ class TestCheckpointSync(SharedTestPipeline):
             self.pipeline.clear_storage()
 
     @enterprise_only
-    def test_checkpoint_sync_from_uuid(self):
+    def test_from_uuid(self):
         self.test_checkpoint_sync(from_uuid=True)
 
     @enterprise_only
-    def test_checkpoint_sync_without_clearing_storage(self):
+    def test_without_clearing_storage(self):
         self.test_checkpoint_sync(clear_storage=False)
 
     @enterprise_only
-    def test_checkpoint_sync_err(self):
+    def test_autherr_fail(self):
         with self.assertRaisesRegex(RuntimeError, "SignatureDoesNotMatch"):
-            self.test_checkpoint_sync(auth_err=True)
+            self.test_checkpoint_sync(auth_err=True, strict=True)
 
     @enterprise_only
-    def test_checkpoint_sync_err_nonexistent_checkpoint(self):
+    def test_autherr(self):
+        self.test_checkpoint_sync(auth_err=True, strict=False, expect_empty=True)
+
+    @enterprise_only
+    def test_nonexistent_checkpoint_fail(self):
         with self.assertRaisesRegex(RuntimeError, "were not found in source"):
-            self.test_checkpoint_sync(random_uuid=True, from_uuid=True)
+            self.test_checkpoint_sync(random_uuid=True, from_uuid=True, strict=True)
+
+    @enterprise_only
+    def test_nonexistent_checkpoint(self):
+        self.test_checkpoint_sync(random_uuid=True, from_uuid=True, expect_empty=True)

feldera-0.117.0/tests/test_shared_pipeline_stress.py

@@ -0,0 +1,26 @@
+import unittest
+
+from tests.shared_test_pipeline import SharedTestPipeline
+
+
+class TestPipeline(SharedTestPipeline):
+    def test_create_pipeline(self):
+        """
+        CREATE TABLE tbl(id INT) WITH ('materialized' = 'true');
+        CREATE MATERIALIZED VIEW v0 AS SELECT * FROM tbl;
+        """
+        pass
+
+    def test_get_pipeline_logs(self):
+        self.pipeline.start()
+        # regression test for https://github.com/feldera/feldera/issues/4394
+        for _ in range(200):
+            logs = self.pipeline.logs()
+            start = next(logs)
+            assert "Fresh start of pipeline log" in start
+        self.pipeline.pause()
+        self.pipeline.stop(force=True)
+
+
+if __name__ == "__main__":
+    unittest.main()