feldera 0.67.0__tar.gz → 0.69.0__tar.gz

{feldera-0.67.0 → feldera-0.69.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.67.0
+Version: 0.69.0
 Summary: The feldera python client
 Author-email: Abhinav <abhinav.gyawali@feldera.com>
 License: MIT

{feldera-0.67.0 → feldera-0.69.0}/feldera/enums.py

@@ -237,3 +237,24 @@ class ProgramStatus(Enum):
         """
 
         return self.error
+
+
+class CheckpointStatus(Enum):
+    Success = 1
+    Failure = 2
+    InProgress = 3
+    Unknown = 4
+
+    def __init__(self, value):
+        self.error: Optional[str] = None
+        self._value_ = value
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def get_error(self) -> Optional[str]:
+        """
+        Returns the error, if any.
+        """
+
+        return self.error

{feldera-0.67.0 → feldera-0.69.0}/feldera/pipeline.py

@@ -9,7 +9,7 @@ from collections import deque
 from queue import Queue
 
 from feldera.rest.errors import FelderaAPIError
-from feldera.enums import PipelineStatus, ProgramStatus
+from feldera.enums import PipelineStatus, ProgramStatus, CheckpointStatus
 from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.rest.feldera_client import FelderaClient
 from feldera._callback_runner import _CallbackRunnerInstruction, CallbackRunner
@@ -104,7 +104,9 @@ class Pipeline:
             tbl.name.lower() for tbl in pipeline.tables
         ]:
             raise ValueError(
-                f"Cannot push to table '{table_name}': table with this name does not exist in the '{self.name}' pipeline"
+                f"Cannot push to table '{
+                    table_name
+                }': table with this name does not exist in the '{self.name}' pipeline"
             )
         else:
             # consider validating the schema here
@@ -297,10 +299,14 @@ class Pipeline:
                 elapsed = time.monotonic() - start_time
                 if elapsed > timeout_s:
                     raise TimeoutError(
-                        f"timeout ({timeout_s}s) reached while waiting for pipeline '{self.name}' to complete"
+                        f"timeout ({timeout_s}s) reached while waiting for pipeline '{
+                            self.name
+                        }' to complete"
                     )
                 logging.debug(
-                    f"waiting for pipeline {self.name} to complete: elapsed time {elapsed}s, timeout: {timeout_s}s"
+                    f"waiting for pipeline {self.name} to complete: elapsed time {
+                        elapsed
+                    }s, timeout: {timeout_s}s"
                 )
 
             metrics: dict = self.client.get_pipeline_stats(self.name).get(
@@ -407,11 +413,15 @@ resume a paused pipeline."""
         """
         if idle_interval_s > timeout_s:
             raise ValueError(
-                f"idle interval ({idle_interval_s}s) cannot be larger than timeout ({timeout_s}s)"
+                f"idle interval ({idle_interval_s}s) cannot be larger than timeout ({
+                    timeout_s
+                }s)"
             )
         if poll_interval_s > timeout_s:
             raise ValueError(
-                f"poll interval ({poll_interval_s}s) cannot be larger than timeout ({timeout_s}s)"
+                f"poll interval ({poll_interval_s}s) cannot be larger than timeout ({
+                    timeout_s
+                }s)"
             )
         if poll_interval_s > idle_interval_s:
             raise ValueError(
@@ -547,15 +557,121 @@ resume a paused pipeline."""
             if err.status_code == 404:
                 raise RuntimeError(f"Pipeline with name {name} not found")
 
-    def checkpoint(self):
+    def checkpoint(self, wait: bool = False, timeout_s=300) -> int:
         """
         Checkpoints this pipeline, if fault-tolerance is enabled.
         Fault Tolerance in Feldera: <https://docs.feldera.com/pipelines/fault-tolerance/>
 
+        :param wait: If true, will block until the checkpoint completes.
+        :param timeout_s: The maximum time (in seconds) to wait for the checkpoint to complete.
+
         :raises FelderaAPIError: If checkpointing is not enabled.
         """
 
-        self.client.checkpoint_pipeline(self.name)
+        seq = self.client.checkpoint_pipeline(self.name)
+
+        if not wait:
+            return seq
+
+        start = time.time()
+
+        while True:
+            elapsed = time.monotonic() - start
+            if elapsed > timeout_s:
+                raise TimeoutError(
+                    f"timeout ({timeout_s}s) reached while waiting for pipeline '{
+                        self.name
+                    }' to make checkpoint '{seq}'"
+                )
+            status = self.checkpoint_status(seq)
+            if status == CheckpointStatus.InProgress:
+                time.sleep(0.1)
+                continue
+
+            return status
+
+        return seq
+
+    def checkpoint_status(self, seq: int) -> CheckpointStatus:
+        """
+        Checks the status of the given checkpoint.
+
+        :param seq: The checkpoint sequence number.
+        """
+
+        resp = self.client.checkpoint_pipeline_status(self.name)
+        success = resp.get("success")
+        if seq == success:
+            return CheckpointStatus.Success
+
+        fail = resp.get("failure") or {}
+        if seq == fail.get("sequence_number"):
+            failure = CheckpointStatus.Failure
+            failure.error = fail.get("error", "")
+            return failure
+
+        if (success is None) or seq > success:
+            return CheckpointStatus.InProgress
+
+        if seq < success:
+            return CheckpointStatus.Unknown
+
+    def sync_checkpoint(self, wait: bool = False, timeout_s=300) -> str:
+        """
+        Syncs this checkpoint to object store.
+
+        :param wait: If true, will block until the checkpoint sync opeartion completes.
+        :param timeout_s: The maximum time (in seconds) to wait for the checkpoint to complete syncing.
+
+        :raises FelderaAPIError: If no checkpoints have been made.
+        """
+
+        uuid = self.client.sync_checkpoint(self.name)
+
+        if not wait:
+            return uuid
+
+        start = time.time()
+
+        while True:
+            elapsed = time.monotonic() - start
+            if elapsed > timeout_s:
+                raise TimeoutError(
+                    f"timeout ({timeout_s}s) reached while waiting for pipeline '{
+                        self.name
+                    }' to sync checkpoint '{uuid}'"
+                )
+            status = self.sync_checkpoint_status(uuid)
+            if status in [CheckpointStatus.InProgress, CheckpointStatus.Unknown]:
+                time.sleep(0.1)
+                continue
+
+            return status
+
+        return uuid
+
+    def sync_checkpoint_status(self, uuid: str) -> CheckpointStatus:
+        """
+        Checks the status of the given checkpoint sync operation.
+        If the checkpoint is currently being synchronized, returns
+        `CheckpointStatus.Unknown`.
+
+        :param uuid: The checkpoint uuid.
+        """
+
+        resp = self.client.sync_checkpoint_status(self.name)
+        success = resp.get("success")
+
+        if uuid == success:
+            return CheckpointStatus.Success
+
+        fail = resp.get("failure") or {}
+        if uuid == fail.get("uuid"):
+            failure = CheckpointStatus.Failure
+            failure.error = fail.get("error", "")
+            return failure
+
+        return CheckpointStatus.Unknown
 
     def query(self, query: str) -> Generator[Mapping[str, Any], None, None]:
         """

{feldera-0.67.0 → feldera-0.69.0}/feldera/rest/feldera_client.py

@@ -7,6 +7,7 @@ from decimal import Decimal
 from typing import Generator
 
 from feldera.rest.config import Config
+from feldera.rest.feldera_config import FelderaConfig
 from feldera.rest.errors import FelderaTimeoutError
 from feldera.rest.pipeline import Pipeline
 from feldera.rest._httprequests import HttpRequests
@@ -381,17 +382,53 @@ Reason: The pipeline is in a FAILED state due to the following error:
             f"timeout error: pipeline '{pipeline_name}' did not suspend in {timeout_s} seconds"
         )
 
-    def checkpoint_pipeline(self, pipeline_name: str):
+    def checkpoint_pipeline(self, pipeline_name: str) -> int:
         """
         Checkpoint a fault-tolerant pipeline
 
         :param pipeline_name: The name of the pipeline to checkpoint
         """
 
-        self.http.post(
+        resp = self.http.post(
             path=f"/pipelines/{pipeline_name}/checkpoint",
         )
 
+        return int(resp.get("checkpoint_sequence_number"))
+
+    def checkpoint_pipeline_status(self, pipeline_name: str) -> dict:
+        """
+        Gets the checkpoint status
+
+        :param pipeline_name: The name of the pipeline to check the checkpoint status of.
+        """
+
+        return self.http.get(path=f"/pipelines/{pipeline_name}/checkpoint_status")
+
+    def sync_checkpoint(self, pipeline_name: str) -> str:
+        """
+        Triggers a checkpoint synchronization for the specified pipeline.
+        Check the status by calling `pipeline_sync_checkpoint_status`.
+
+        :param pipeline_name: Name of the pipeline whose checkpoint should be synchronized.
+        """
+
+        resp = self.http.post(
+            path=f"/pipelines/{pipeline_name}/checkpoint/sync",
+        )
+
+        return resp.get("checkpoint_uuid")
+
+    def sync_checkpoint_status(self, pipeline_name: str) -> dict:
+        """
+        Gets the checkpoint sync status of the pipeline
+
+        :param pipeline_name: The name of the pipeline to check the checkpoint synchronization status of.
+        """
+
+        return self.http.get(
+            path=f"/pipelines/{pipeline_name}/checkpoint/sync_status",
+        )
+
     def push_to_pipeline(
         self,
         pipeline_name: str,
@@ -674,3 +711,12 @@ Reason: The pipeline is in a FAILED state due to the following error:
         self.http.post(
             path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/start",
         )
+
+    def get_config(self) -> FelderaConfig:
+        """
+        Get general feldera configuration.
+        """
+
+        resp = self.http.get(path="/config")
+
+        return FelderaConfig(resp)

feldera-0.69.0/feldera/rest/feldera_config.py

@@ -0,0 +1,49 @@
+from enum import Enum
+
+
+class FelderaEdition(Enum):
+    """
+    The compilation profile to use when compiling the program.
+    Represents the Feldera edition between Enterprise and Open source.
+    """
+
+    ENTERPRISE = "Enterprise"
+    """
+    The Enterprise version of Feldera.
+    """
+
+    OPEN_SOURCE = "Open source"
+    """
+    The open source version of Feldera.
+    """
+
+    @staticmethod
+    def from_value(value):
+        error = None
+        if isinstance(value, dict):
+            error = value
+            value = list(value.keys())[0]
+
+        for member in FelderaEdition:
+            if member.value.lower() == value.lower():
+                member.error = error
+                return member
+        raise ValueError(f"Unknown value '{value}' for enum {FelderaEdition.__name__}")
+
+    def is_enterprise(self):
+        return self == FelderaEdition.ENTERPRISE
+
+
+class FelderaConfig:
+    """
+    General configuration of the current Feldera instance.
+    """
+
+    def __init__(self, cfg: dict):
+        self.changelog_url = cfg.get("changelog_url")
+        self.edition = FelderaEdition.from_value(cfg.get("edition"))
+        self.license_validity = cfg.get("license_validity")
+        self.revision = cfg.get("revision")
+        self.telemetry = cfg.get("telemetry")
+        self.update_info = cfg.get("update_info")
+        self.version = cfg.get("version")

{feldera-0.67.0 → feldera-0.69.0}/feldera/runtime_config.py

@@ -63,7 +63,7 @@ class RuntimeConfig:
     def __init__(
         self,
         workers: Optional[int] = None,
-        storage: Optional[Storage] = None,
+        storage: Optional[Storage | bool] = None,
         tracing: Optional[bool] = False,
         tracing_endpoint_jaeger: Optional[str] = "",
         cpu_profiler: bool = True,
@@ -74,7 +74,6 @@ class RuntimeConfig:
         resources: Optional[Resources] = None,
     ):
         self.workers = workers
-        self.storage = storage
         self.tracing = tracing
         self.tracing_endpoint_jaeger = tracing_endpoint_jaeger
         self.cpu_profiler = cpu_profiler
@@ -84,6 +83,10 @@ class RuntimeConfig:
         self.provisioning_timeout_secs = provisioning_timeout_secs
         if resources is not None:
             self.resources = resources.__dict__
+        if isinstance(storage, bool):
+            self.storage = storage
+        if isinstance(storage, Storage):
+            self.storage = storage.__dict__
 
     @classmethod
     def from_dict(cls, d: Mapping[str, Any]):

{feldera-0.67.0 → feldera-0.69.0}/feldera.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.67.0
+Version: 0.69.0
 Summary: The feldera python client
 Author-email: Abhinav <abhinav.gyawali@feldera.com>
 License: MIT

{feldera-0.67.0 → feldera-0.69.0}/feldera.egg-info/SOURCES.txt

@@ -18,6 +18,7 @@ feldera/rest/_httprequests.py
 feldera/rest/config.py
 feldera/rest/errors.py
 feldera/rest/feldera_client.py
+feldera/rest/feldera_config.py
 feldera/rest/pipeline.py
 feldera/rest/sql_table.py
 feldera/rest/sql_view.py

{feldera-0.67.0 → feldera-0.69.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "feldera"
 readme = "README.md"
 description = "The feldera python client"
-version = "0.67.0"
+version = "0.69.0"
 license = { text = "MIT" }
 requires-python = ">=3.10"
 authors = [

{feldera-0.67.0 → feldera-0.69.0}/tests/test_pipeline_builder.py

@@ -706,12 +706,8 @@ Code snippet:
 
         pipeline.delete()
 
-    # doesn't run in CI
     def test_suspend(self):
-        in_ci = os.environ.get("IN_CI")
-
-        if in_ci == "1":
-            # if running in CI, skip the test
+        if not TEST_CLIENT.get_config().edition.is_enterprise():
             return
 
         TBL_NAME = "items"
@@ -1031,13 +1027,13 @@ Code snippet:
             pipeline.pause()
 
         got_err: str = err.exception.args[0].strip()
-        assert "attempt to add with overflow" in got_err
+        assert "causes overflow" in got_err
 
         with self.assertRaises(RuntimeError) as err:
             pipeline.start()
 
         got_err: str = err.exception.args[0].strip()
-        assert "attempt to add with overflow" in got_err
+        assert "causes overflow" in got_err
 
         pipeline.shutdown()
         pipeline.delete()