feldera 0.68.0__py3-none-any.whl → 0.69.0__py3-none-any.whl

feldera/enums.py

@@ -237,3 +237,24 @@ class ProgramStatus(Enum):
         """
 
         return self.error
+
+
+class CheckpointStatus(Enum):
+    Success = 1
+    Failure = 2
+    InProgress = 3
+    Unknown = 4
+
+    def __init__(self, value):
+        self.error: Optional[str] = None
+        self._value_ = value
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def get_error(self) -> Optional[str]:
+        """
+        Returns the error, if any.
+        """
+
+        return self.error

feldera/pipeline.py

@@ -9,7 +9,7 @@ from collections import deque
 from queue import Queue
 
 from feldera.rest.errors import FelderaAPIError
-from feldera.enums import PipelineStatus, ProgramStatus
+from feldera.enums import PipelineStatus, ProgramStatus, CheckpointStatus
 from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.rest.feldera_client import FelderaClient
 from feldera._callback_runner import _CallbackRunnerInstruction, CallbackRunner
@@ -104,7 +104,9 @@ class Pipeline:
             tbl.name.lower() for tbl in pipeline.tables
         ]:
             raise ValueError(
-                f"Cannot push to table '{table_name}': table with this name does not exist in the '{self.name}' pipeline"
+                f"Cannot push to table '{
+                    table_name
+                }': table with this name does not exist in the '{self.name}' pipeline"
             )
         else:
             # consider validating the schema here
@@ -297,10 +299,14 @@ class Pipeline:
                 elapsed = time.monotonic() - start_time
                 if elapsed > timeout_s:
                     raise TimeoutError(
-                        f"timeout ({timeout_s}s) reached while waiting for pipeline '{self.name}' to complete"
+                        f"timeout ({timeout_s}s) reached while waiting for pipeline '{
+                            self.name
+                        }' to complete"
                     )
                 logging.debug(
-                    f"waiting for pipeline {self.name} to complete: elapsed time {elapsed}s, timeout: {timeout_s}s"
+                    f"waiting for pipeline {self.name} to complete: elapsed time {
+                        elapsed
+                    }s, timeout: {timeout_s}s"
                 )
 
             metrics: dict = self.client.get_pipeline_stats(self.name).get(
@@ -407,11 +413,15 @@ resume a paused pipeline."""
         """
         if idle_interval_s > timeout_s:
             raise ValueError(
-                f"idle interval ({idle_interval_s}s) cannot be larger than timeout ({timeout_s}s)"
+                f"idle interval ({idle_interval_s}s) cannot be larger than timeout ({
+                    timeout_s
+                }s)"
             )
         if poll_interval_s > timeout_s:
             raise ValueError(
-                f"poll interval ({poll_interval_s}s) cannot be larger than timeout ({timeout_s}s)"
+                f"poll interval ({poll_interval_s}s) cannot be larger than timeout ({
+                    timeout_s
+                }s)"
             )
         if poll_interval_s > idle_interval_s:
             raise ValueError(
@@ -547,15 +557,121 @@ resume a paused pipeline."""
             if err.status_code == 404:
                 raise RuntimeError(f"Pipeline with name {name} not found")
 
-    def checkpoint(self):
+    def checkpoint(self, wait: bool = False, timeout_s=300) -> int:
         """
         Checkpoints this pipeline, if fault-tolerance is enabled.
         Fault Tolerance in Feldera: <https://docs.feldera.com/pipelines/fault-tolerance/>
 
+        :param wait: If true, will block until the checkpoint completes.
+        :param timeout_s: The maximum time (in seconds) to wait for the checkpoint to complete.
+
         :raises FelderaAPIError: If checkpointing is not enabled.
         """
 
-        self.client.checkpoint_pipeline(self.name)
+        seq = self.client.checkpoint_pipeline(self.name)
+
+        if not wait:
+            return seq
+
+        start = time.time()
+
+        while True:
+            elapsed = time.monotonic() - start
+            if elapsed > timeout_s:
+                raise TimeoutError(
+                    f"timeout ({timeout_s}s) reached while waiting for pipeline '{
+                        self.name
+                    }' to make checkpoint '{seq}'"
+                )
+            status = self.checkpoint_status(seq)
+            if status == CheckpointStatus.InProgress:
+                time.sleep(0.1)
+                continue
+
+            return status
+
+        return seq
+
+    def checkpoint_status(self, seq: int) -> CheckpointStatus:
+        """
+        Checks the status of the given checkpoint.
+
+        :param seq: The checkpoint sequence number.
+        """
+
+        resp = self.client.checkpoint_pipeline_status(self.name)
+        success = resp.get("success")
+        if seq == success:
+            return CheckpointStatus.Success
+
+        fail = resp.get("failure") or {}
+        if seq == fail.get("sequence_number"):
+            failure = CheckpointStatus.Failure
+            failure.error = fail.get("error", "")
+            return failure
+
+        if (success is None) or seq > success:
+            return CheckpointStatus.InProgress
+
+        if seq < success:
+            return CheckpointStatus.Unknown
+
+    def sync_checkpoint(self, wait: bool = False, timeout_s=300) -> str:
+        """
+        Syncs this checkpoint to object store.
+
+        :param wait: If true, will block until the checkpoint sync opeartion completes.
+        :param timeout_s: The maximum time (in seconds) to wait for the checkpoint to complete syncing.
+
+        :raises FelderaAPIError: If no checkpoints have been made.
+        """
+
+        uuid = self.client.sync_checkpoint(self.name)
+
+        if not wait:
+            return uuid
+
+        start = time.time()
+
+        while True:
+            elapsed = time.monotonic() - start
+            if elapsed > timeout_s:
+                raise TimeoutError(
+                    f"timeout ({timeout_s}s) reached while waiting for pipeline '{
+                        self.name
+                    }' to sync checkpoint '{uuid}'"
+                )
+            status = self.sync_checkpoint_status(uuid)
+            if status in [CheckpointStatus.InProgress, CheckpointStatus.Unknown]:
+                time.sleep(0.1)
+                continue
+
+            return status
+
+        return uuid
+
+    def sync_checkpoint_status(self, uuid: str) -> CheckpointStatus:
+        """
+        Checks the status of the given checkpoint sync operation.
+        If the checkpoint is currently being synchronized, returns
+        `CheckpointStatus.Unknown`.
+
+        :param uuid: The checkpoint uuid.
+        """
+
+        resp = self.client.sync_checkpoint_status(self.name)
+        success = resp.get("success")
+
+        if uuid == success:
+            return CheckpointStatus.Success
+
+        fail = resp.get("failure") or {}
+        if uuid == fail.get("uuid"):
+            failure = CheckpointStatus.Failure
+            failure.error = fail.get("error", "")
+            return failure
+
+        return CheckpointStatus.Unknown
 
     def query(self, query: str) -> Generator[Mapping[str, Any], None, None]:
         """

feldera/rest/feldera_client.py

@@ -7,6 +7,7 @@ from decimal import Decimal
 from typing import Generator
 
 from feldera.rest.config import Config
+from feldera.rest.feldera_config import FelderaConfig
 from feldera.rest.errors import FelderaTimeoutError
 from feldera.rest.pipeline import Pipeline
 from feldera.rest._httprequests import HttpRequests
@@ -381,17 +382,53 @@ Reason: The pipeline is in a FAILED state due to the following error:
             f"timeout error: pipeline '{pipeline_name}' did not suspend in {timeout_s} seconds"
         )
 
-    def checkpoint_pipeline(self, pipeline_name: str):
+    def checkpoint_pipeline(self, pipeline_name: str) -> int:
         """
         Checkpoint a fault-tolerant pipeline
 
         :param pipeline_name: The name of the pipeline to checkpoint
         """
 
-        self.http.post(
+        resp = self.http.post(
             path=f"/pipelines/{pipeline_name}/checkpoint",
         )
 
+        return int(resp.get("checkpoint_sequence_number"))
+
+    def checkpoint_pipeline_status(self, pipeline_name: str) -> dict:
+        """
+        Gets the checkpoint status
+
+        :param pipeline_name: The name of the pipeline to check the checkpoint status of.
+        """
+
+        return self.http.get(path=f"/pipelines/{pipeline_name}/checkpoint_status")
+
+    def sync_checkpoint(self, pipeline_name: str) -> str:
+        """
+        Triggers a checkpoint synchronization for the specified pipeline.
+        Check the status by calling `pipeline_sync_checkpoint_status`.
+
+        :param pipeline_name: Name of the pipeline whose checkpoint should be synchronized.
+        """
+
+        resp = self.http.post(
+            path=f"/pipelines/{pipeline_name}/checkpoint/sync",
+        )
+
+        return resp.get("checkpoint_uuid")
+
+    def sync_checkpoint_status(self, pipeline_name: str) -> dict:
+        """
+        Gets the checkpoint sync status of the pipeline
+
+        :param pipeline_name: The name of the pipeline to check the checkpoint synchronization status of.
+        """
+
+        return self.http.get(
+            path=f"/pipelines/{pipeline_name}/checkpoint/sync_status",
+        )
+
     def push_to_pipeline(
         self,
         pipeline_name: str,
@@ -674,3 +711,12 @@ Reason: The pipeline is in a FAILED state due to the following error:
         self.http.post(
             path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/start",
         )
+
+    def get_config(self) -> FelderaConfig:
+        """
+        Get general feldera configuration.
+        """
+
+        resp = self.http.get(path="/config")
+
+        return FelderaConfig(resp)

feldera/rest/feldera_config.py

@@ -0,0 +1,49 @@
+from enum import Enum
+
+
+class FelderaEdition(Enum):
+    """
+    The compilation profile to use when compiling the program.
+    Represents the Feldera edition between Enterprise and Open source.
+    """
+
+    ENTERPRISE = "Enterprise"
+    """
+    The Enterprise version of Feldera.
+    """
+
+    OPEN_SOURCE = "Open source"
+    """
+    The open source version of Feldera.
+    """
+
+    @staticmethod
+    def from_value(value):
+        error = None
+        if isinstance(value, dict):
+            error = value
+            value = list(value.keys())[0]
+
+        for member in FelderaEdition:
+            if member.value.lower() == value.lower():
+                member.error = error
+                return member
+        raise ValueError(f"Unknown value '{value}' for enum {FelderaEdition.__name__}")
+
+    def is_enterprise(self):
+        return self == FelderaEdition.ENTERPRISE
+
+
+class FelderaConfig:
+    """
+    General configuration of the current Feldera instance.
+    """
+
+    def __init__(self, cfg: dict):
+        self.changelog_url = cfg.get("changelog_url")
+        self.edition = FelderaEdition.from_value(cfg.get("edition"))
+        self.license_validity = cfg.get("license_validity")
+        self.revision = cfg.get("revision")
+        self.telemetry = cfg.get("telemetry")
+        self.update_info = cfg.get("update_info")
+        self.version = cfg.get("version")

feldera/runtime_config.py

@@ -63,7 +63,7 @@ class RuntimeConfig:
     def __init__(
         self,
         workers: Optional[int] = None,
-        storage: Optional[Storage] = None,
+        storage: Optional[Storage | bool] = None,
         tracing: Optional[bool] = False,
         tracing_endpoint_jaeger: Optional[str] = "",
         cpu_profiler: bool = True,
@@ -74,7 +74,6 @@ class RuntimeConfig:
         resources: Optional[Resources] = None,
     ):
         self.workers = workers
-        self.storage = storage
         self.tracing = tracing
         self.tracing_endpoint_jaeger = tracing_endpoint_jaeger
         self.cpu_profiler = cpu_profiler
@@ -84,6 +83,10 @@ class RuntimeConfig:
         self.provisioning_timeout_secs = provisioning_timeout_secs
         if resources is not None:
             self.resources = resources.__dict__
+        if isinstance(storage, bool):
+            self.storage = storage
+        if isinstance(storage, Storage):
+            self.storage = storage.__dict__
 
     @classmethod
     def from_dict(cls, d: Mapping[str, Any]):

{feldera-0.68.0.dist-info → feldera-0.69.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.68.0
+Version: 0.69.0
 Summary: The feldera python client
 Author-email: Abhinav <abhinav.gyawali@feldera.com>
 License: MIT

{feldera-0.68.0.dist-info → feldera-0.69.0.dist-info}/RECORD

@@ -1,20 +1,21 @@
 feldera/__init__.py,sha256=PxkgCtEAuFwo4u8NGEDio-bF3M-GnbeV45tAQVoBbqE,297
 feldera/_callback_runner.py,sha256=Tdf6BXN4zppyoy8t_y-Ooa3B0wEfvyezMHU9jxY2ZhA,4713
 feldera/_helpers.py,sha256=rN0WuGSCCQlXWFMimZUQrgs-LJAfUo074d79sLElncQ,3023
-feldera/enums.py,sha256=lgGTn0ahOj-IE2M8k-uszq6Dl98kSX2EeqKDixJcWOo,7373
+feldera/enums.py,sha256=ydzCQHno6as8rXUuoRYw4QE1QXCfpADJwGQZNjMS58A,7774
 feldera/output_handler.py,sha256=64J3ljhOaKIhxdjOKYi-BUz_HnMwROfmN8eE-btYygU,1930
-feldera/pipeline.py,sha256=gDadQHgUHi4bdrMJ9GdIshmFU80pcTNvDiveC5S-GZ4,31100
+feldera/pipeline.py,sha256=SRUT9p6_-00zCljPNNRkA7eLKUqSJszGovEnFhssnk4,34731
 feldera/pipeline_builder.py,sha256=4rmklRZ0-otvTUb-HTESfNsJopEK-E2jxpJXiYlKpps,3664
-feldera/runtime_config.py,sha256=DDJTSzG6LCTH0lVnuUjpATAf1STwAYJOB38xyZh_BJI,3367
+feldera/runtime_config.py,sha256=EDvnakyaRTHUS8GLd6TqLZ_jsGp7_4fbcPnSfQCw1k0,3501
 feldera/rest/__init__.py,sha256=Eg-EKUU3RSTDcdxTR_7wNDnCly8VpXEzsZCQUmf-y2M,308
 feldera/rest/_httprequests.py,sha256=e22YbpzOzy7MGo7hk9MOU7ZRTj3F314grY0Ygr-_goI,6636
 feldera/rest/config.py,sha256=DYzZKngDEhouTEwqVFd-rDrBN9tWqsU07Jl_BTT4mXs,1008
 feldera/rest/errors.py,sha256=b4i2JjrbSmej7jdko_FL8UeXklLKenSipwMT80jowaM,1720
-feldera/rest/feldera_client.py,sha256=ELXZbTfy1UdeIY-ZbFv0kPIdq9qSF5WuoQpdhxmK2pg,23184
+feldera/rest/feldera_client.py,sha256=03XZ8gjgviy3uVvu_IWSoz61yVN8zeH1-ugNHdBMldc,24693
+feldera/rest/feldera_config.py,sha256=1pnGbLFMSLvp7Qh_OlPLALSKCSHIktNWKvx6gYU00U4,1374
 feldera/rest/pipeline.py,sha256=a1lx-64SYak5mHX5yKElVijdfaAt5sDYVhVIXCJ97QQ,2839
 feldera/rest/sql_table.py,sha256=qrw-YwMzx5T81zDefNO1KOx7EyypFz1vPwGBzSUB7kc,652
 feldera/rest/sql_view.py,sha256=hN12mPM0mvwLCIPYywpb12s9Hd2Ws31IlTMXPriMisw,644
-feldera-0.68.0.dist-info/METADATA,sha256=eeHpjt5upQ0AGe3L5bEcIwgQmuVQVXo0qaFINpSyyUc,2594
-feldera-0.68.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-feldera-0.68.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
-feldera-0.68.0.dist-info/RECORD,,
+feldera-0.69.0.dist-info/METADATA,sha256=qxtXNShHN2kByrsGjPiQjPKPnvwRs3hAJZ1ojxRDDAc,2594
+feldera-0.69.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+feldera-0.69.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
+feldera-0.69.0.dist-info/RECORD,,