feldera 0.96.0__py3-none-any.whl → 0.98.0__py3-none-any.whl

feldera/rest/feldera_client.py

@@ -36,8 +36,8 @@ class FelderaClient:
     """
     A client for the Feldera HTTP API
 
-    A client instance is needed for every Feldera API method to know the location of
-    Feldera and its permissions.
+    A client instance is needed for every Feldera API method to know the
+    location of Feldera and its permissions.
     """
 
     def __init__(
@@ -50,7 +50,8 @@ class FelderaClient:
         """
         :param url: The url to Feldera API (ex: https://try.feldera.com)
         :param api_key: The optional API key for Feldera
-        :param timeout: (optional) The amount of time in seconds that the client will wait for a response before timing
+        :param timeout: (optional) The amount of time in seconds that the client
+            will wait for a response before timing
             out.
         :param requests_verify: The `verify` parameter passed to the requests
             library. `True` by default.
@@ -162,7 +163,8 @@ class FelderaClient:
 
     def create_or_update_pipeline(self, pipeline: Pipeline) -> Pipeline:
         """
-        Create a pipeline if it doesn't exist or update a pipeline and wait for it to compile
+        Create a pipeline if it doesn't exist or update a pipeline and wait for
+        it to compile
         """
 
         body = {
@@ -187,7 +189,8 @@ class FelderaClient:
         Incrementally update the pipeline SQL
 
         :param name: The name of the pipeline
-        :param sql: The SQL snippet. Replaces the existing SQL code with this one.
+        :param sql: The SQL snippet. Replaces the existing SQL code with this
+            one.
         """
 
         self.http.patch(
@@ -222,7 +225,8 @@ class FelderaClient:
         """
 
         :param pipeline_name: The name of the pipeline to start
-        :param timeout_s: The amount of time in seconds to wait for the pipeline to start. 300 seconds by default.
+        :param timeout_s: The amount of time in seconds to wait for the pipeline
+            to start. 300 seconds by default.
         """
 
         if timeout_s is None:
@@ -250,7 +254,7 @@ class FelderaClient:
             elif status == "Failed":
                 raise RuntimeError(
                     f"""Unable to START the pipeline.
-Reason: The pipeline is in a FAILED state due to the following error:
+Reason: The pipeline is in a STOPPED state due to the following error:
 {resp.deployment_error.get("message", "")}"""
                 )
 
@@ -269,8 +273,10 @@ Reason: The pipeline is in a FAILED state due to the following error:
         Stop a pipeline
 
         :param pipeline_name: The name of the pipeline to stop
-        :param error_message: The error message to show if the pipeline is in FAILED state
-        :param timeout_s: The amount of time in seconds to wait for the pipeline to pause. 300 seconds by default.
+        :param error_message: The error message to show if the pipeline is in
+             STOPPED state due to a failure.
+        :param timeout_s: The amount of time in seconds to wait for the pipeline
+            to pause. 300 seconds by default.
         """
 
         if timeout_s is None:
@@ -298,10 +304,14 @@ Reason: The pipeline is in a FAILED state due to the following error:
 
             if status == "Paused":
                 break
-            elif status == "Failed":
+            elif (
+                status == "Stopped"
+                and len(resp.deployment_error or {}) > 0
+                and resp.deployment_desired_status == "Stopped"
+            ):
                 raise RuntimeError(
                     error_message
-                    + f"""Reason: The pipeline is in a FAILED state due to the following error:
+                    + f"""Reason: The pipeline is in a STOPPED state due to the following error:
 {resp.deployment_error.get("message", "")}"""
                 )
 
@@ -310,19 +320,27 @@ Reason: The pipeline is in a FAILED state due to the following error:
             )
             time.sleep(0.1)
 
-    def shutdown_pipeline(self, pipeline_name: str, timeout_s: Optional[float] = 300):
+    def stop_pipeline(
+        self, pipeline_name: str, force: bool, timeout_s: Optional[float] = 300
+    ):
         """
-        Shutdown a pipeline
+        Stop a pipeline
 
-        :param pipeline_name: The name of the pipeline to shut down
-        :param timeout_s: The amount of time in seconds to wait for the pipeline to shut down. Default is 300 seconds.
+        :param pipeline_name: The name of the pipeline to stop
+        :param force: Set True to immediately scale compute resources to zero.
+            Set False to automatically checkpoint before stopping.
+        :param timeout_s: The amount of time in seconds to wait for the pipeline
+            to stop. Default is 300 seconds.
         """
 
         if timeout_s is None:
             timeout_s = 300
 
+        params = {"force": str(force).lower()}
+
         self.http.post(
-            path=f"/pipelines/{pipeline_name}/shutdown",
+            path=f"/pipelines/{pipeline_name}/stop",
+            params=params,
         )
 
         start = time.monotonic()
@@ -330,56 +348,55 @@ Reason: The pipeline is in a FAILED state due to the following error:
         while time.monotonic() - start < timeout_s:
             status = self.get_pipeline(pipeline_name).deployment_status
 
-            if status == "Shutdown":
+            if status == "Stopped":
                 return
 
             logging.debug(
-                "still shutting down %s, waiting for 100 more milliseconds",
+                "still stopping %s, waiting for 100 more milliseconds",
                 pipeline_name,
             )
             time.sleep(0.1)
 
         raise FelderaTimeoutError(
-            f"timeout error: pipeline '{pipeline_name}' did not shutdown in {timeout_s} seconds"
+            f"timeout error: pipeline '{pipeline_name}' did not stop in {
+                timeout_s
+            } seconds"
         )
 
-    def suspend_pipeline(self, pipeline_name: str, timeout_s: Optional[float] = 300):
+    def clear_storage(self, pipeline_name: str, timeout_s: Optional[float] = 300):
         """
-        Suspend a pipeline
+        Clears the storage from the pipeline.
+        This operation cannot be canceled.
 
-        :param pipeline_name: The name of the pipeline to suspend
-        :param timeout_s: The amount of time in seconds to wait for the pipeline to suspend. Default is 300 seconds.
+        :param pipeline_name: The name of the pipeline
+        :param timeout_s: The amount of time in seconds to wait for the storage
+            to clear. Default is 300 seconds.
         """
-
         if timeout_s is None:
             timeout_s = 300
 
         self.http.post(
-            path=f"/pipelines/{pipeline_name}/suspend",
+            path=f"/pipelines/{pipeline_name}/clear",
         )
 
         start = time.monotonic()
 
         while time.monotonic() - start < timeout_s:
-            resp = self.get_pipeline(pipeline_name)
-            status = resp.deployment_status
+            status = self.get_pipeline(pipeline_name).storage_status
 
-            if status == "Suspended":
+            if status == "Cleared":
                 return
-            elif status == "Failed":
-                raise RuntimeError(
-                    f"""Unable to Suspend pipeline '{pipeline_name}'.\nReason: The pipeline is in a FAILED state due to the following error:
-{resp.deployment_error.get("message", "")}"""
-                )
 
             logging.debug(
-                "still suspending %s, waiting for 100 more milliseconds",
+                "still clearing %s, waiting for 100 more milliseconds",
                 pipeline_name,
             )
             time.sleep(0.1)
 
         raise FelderaTimeoutError(
-            f"timeout error: pipeline '{pipeline_name}' did not suspend in {timeout_s} seconds"
+            f"timeout error: pipeline '{pipeline_name}' did not clear storage in {
+                timeout_s
+            } seconds"
         )
 
     def checkpoint_pipeline(self, pipeline_name: str) -> int:
@@ -685,7 +702,9 @@ Reason: The pipeline is in a FAILED state due to the following error:
         """
 
         self.http.post(
-            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/pause",
+            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{
+                connector_name
+            }/pause",
         )
 
     def resume_connector(
@@ -709,7 +728,9 @@ Reason: The pipeline is in a FAILED state due to the following error:
         """
 
         self.http.post(
-            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/start",
+            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{
+                connector_name
+            }/start",
         )
 
     def get_config(self) -> FelderaConfig:

feldera/rest/pipeline.py

@@ -56,6 +56,7 @@ class Pipeline:
         self.program_status: Optional[str] = None
         self.program_status_since: Optional[str] = None
         self.program_error: Optional[dict] = None
+        self.storage_status: Optional[str] = None
 
     @classmethod
     def from_dict(cls, d: Mapping[str, Any]):

feldera/stats.py

@@ -0,0 +1,149 @@
+from typing import Mapping, Any, Optional, List
+from feldera.enums import PipelineStatus
+from datetime import datetime
+import uuid
+
+
+class PipelineStatistics:
+    """
+    Represents statistics reported by a pipeline's "/stats" endpoint.
+    """
+
+    def __init__(self):
+        """
+        Initializes as an empty set of statistics.
+        """
+
+        self.global_metrics: GlobalPipelineMetrics = GlobalPipelineMetrics()
+        self.suspend_error: Optional[Any] = None
+        self.inputs: Mapping[List[InputEndpointStatus()]] = {}
+        self.outputs: Mapping[List[OutputEndpointStatus]] = {}
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        pipeline = cls()
+        pipeline.global_metrics = GlobalPipelineMetrics.from_dict(d["global_metrics"])
+        pipeline.inputs = [
+            InputEndpointStatus.from_dict(input) for input in d["inputs"]
+        ]
+        pipeline.inputs = [
+            OutputEndpointStatus().from_dict(output) for output in d["outputs"]
+        ]
+        return pipeline
+
+
+class GlobalPipelineMetrics:
+    """Represents the "global_metrics" object within the pipeline's
+    "/stats" endpoint reply.
+    """
+
+    def __init__(self):
+        """
+        Initializes as an empty set of metrics.
+        """
+        self.state: Optional[PipelineStatus] = None
+        self.bootstrap_in_progress: Optional[bool] = None
+        self.rss_bytes: Optional[int] = None
+        self.cpu_msecs: Optional[int] = None
+        self.start_time: Optional[datetime] = None
+        self.incarnation_uuid: Optional[uuid] = None
+        self.storage_bytes: Optional[int] = None
+        self.storage_mb_secs: Optional[int] = None
+        self.runtime_elapsed_msecs: Optional[int] = None
+        self.buffered_input_records: Optional[int] = None
+        self.total_input_records: Optional[int] = None
+        self.total_processed_records: Optional[int] = None
+        self.total_completed_records: Optional[int] = None
+        self.pipeline_complete: Optional[bool] = None
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        metrics = cls()
+        metrics.__dict__.update(d)
+        metrics.state = PipelineStatus.from_str(d["state"])
+        metrics.incarnation_uuid = uuid.UUID(d["incarnation_uuid"])
+        metrics.start_time = datetime.fromtimestamp(d["start_time"])
+        return metrics
+
+
+class InputEndpointStatus:
+    """Represents one member of the "inputs" array within the
+    pipeline's "/stats" endpoint reply.
+    """
+
+    def __init__(self):
+        """Initializes an empty status."""
+        self.endpoint_name: Optional[str] = None
+        self.config: Optional[Mapping] = None
+        self.metrics: Optional[InputEndpointMetrics] = None
+        self.fatal_error: Optional[str] = None
+        self.paused: Optional[bool] = None
+        self.barrier: Optional[bool] = None
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        status = cls()
+        status.__dict__.update(d)
+        status.metrics = InputEndpointMetrics.from_dict(d["metrics"])
+        return status
+
+
+class InputEndpointMetrics:
+    """Represents the "metrics" member within an input endpoint status
+    in the pipeline's "/stats" endpoint reply.
+    """
+
+    def __init__(self):
+        self.total_bytes: Optional[int] = None
+        self.total_records: Optional[int] = None
+        self.buffered_records: Optional[int] = None
+        self.num_transport_errors: Optional[int] = None
+        self.num_parse_errors: Optional[int] = None
+        self.end_of_input: Optional[bool] = None
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        metrics = cls()
+        metrics.__dict__.update(d)
+        return metrics
+
+
+class OutputEndpointStatus:
+    """Represents one member of the "outputs" array within the
+    pipeline's "/stats" endpoint reply.
+    """
+
+    def __init__(self):
+        """Initializes an empty status."""
+        self.endpoint_name: Optional[str] = None
+        self.config: Optional[Mapping] = None
+        self.metrics: Optional[OutputEndpointMetrics] = None
+        self.fatal_error: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        status = cls()
+        status.__dict__.update(d)
+        status.metrics = OutputEndpointMetrics.from_dict(d["metrics"])
+        return status
+
+
+class OutputEndpointMetrics:
+    """Represents the "metrics" member within an output endpoint status
+    in the pipeline's "/stats" endpoint reply.
+    """
+
+    def __init__(self):
+        self.transmitted_records: Optional[int] = None
+        self.transmitted_bytes: Optional[int] = None
+        self.queued_records: Optional[int] = None
+        self.queued_batches: Optional[int] = None
+        self.num_encode_errors: Optional[int] = None
+        self.num_transport_errors: Optional[int] = None
+        self.total_processed_input_records: Optional[int] = None
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        metrics = cls()
+        metrics.__dict__.update(d)
+        return metrics

{feldera-0.96.0.dist-info → feldera-0.98.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.96.0
+Version: 0.98.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT
@@ -10,10 +10,9 @@ Project-URL: Repository, https://github.com/feldera/feldera
 Project-URL: Issues, https://github.com/feldera/feldera/issues
 Keywords: feldera,python
 Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: requests
 Requires-Dist: pandas>=2.1.2
@@ -81,6 +80,8 @@ To run unit tests:
 (cd python && python3 -m unittest)
 ```
 
+> ⚠️ Running the unit tests will **delete all existing pipelines**.
+
 The following command runs end-to-end tests.  You'll need a pipeline
 manager running at `http://localhost:8080`.  For the pipeline builder
 tests, you'll also need a broker available at `localhost:9092` and
@@ -88,13 +89,13 @@ tests, you'll also need a broker available at `localhost:9092` and
 set the environment variables listed in `python/tests/__init__.py`.)
 
 ```bash
-(cd python/tests && python3 -m pytest .)
+(cd python && python3 -m pytest tests)
 ```
 
 To run tests from a specific file:
 
 ```bash
-(cd python/tests && python3 -m unittest ./tests/path-to-file.py)
+(cd python && python3 -m unittest ./tests/path-to-file.py)
 ```
 
 To run the aggregate tests use:
@@ -103,3 +104,20 @@ To run the aggregate tests use:
 cd python
 PYTHONPATH=`pwd` python3 ./tests/aggregate_tests/main.py
 ```
+
+## Linting and formatting
+
+Use [Ruff] to run the lint checks that will be executed by the
+precommit hook when a PR is submitted:
+
+```bash
+ruff check python/
+```
+
+To reformat the code in the same way as the precommit hook:
+
+```bash
+ruff format
+```
+
+[Ruff]: https://github.com/astral-sh/ruff

{feldera-0.96.0.dist-info → feldera-0.98.0.dist-info}/RECORD

@@ -1,21 +1,22 @@
 feldera/__init__.py,sha256=PxkgCtEAuFwo4u8NGEDio-bF3M-GnbeV45tAQVoBbqE,297
 feldera/_callback_runner.py,sha256=Tdf6BXN4zppyoy8t_y-Ooa3B0wEfvyezMHU9jxY2ZhA,4713
 feldera/_helpers.py,sha256=rN0WuGSCCQlXWFMimZUQrgs-LJAfUo074d79sLElncQ,3023
-feldera/enums.py,sha256=ydzCQHno6as8rXUuoRYw4QE1QXCfpADJwGQZNjMS58A,7774
+feldera/enums.py,sha256=CWPvFIdxMuadS548IAiYjPjBrf3fZe7DXi2VCJGjlhc,8671
 feldera/output_handler.py,sha256=64J3ljhOaKIhxdjOKYi-BUz_HnMwROfmN8eE-btYygU,1930
-feldera/pipeline.py,sha256=SRUT9p6_-00zCljPNNRkA7eLKUqSJszGovEnFhssnk4,34731
-feldera/pipeline_builder.py,sha256=4rmklRZ0-otvTUb-HTESfNsJopEK-E2jxpJXiYlKpps,3664
+feldera/pipeline.py,sha256=OClrsP1Fgg-FjDLCh1doORq6-dQT4lIX1yUDDZtQR0o,34523
+feldera/pipeline_builder.py,sha256=2nu88M6W-a2tBTTwBdUG5Bcqe8y5Gsnvlc5j6BW4OfU,3726
 feldera/runtime_config.py,sha256=EDvnakyaRTHUS8GLd6TqLZ_jsGp7_4fbcPnSfQCw1k0,3501
+feldera/stats.py,sha256=XBhkRsV7FXErwWuPP0i3q9W77mzkMo-oThPVEZy5y3U,5028
 feldera/rest/__init__.py,sha256=Eg-EKUU3RSTDcdxTR_7wNDnCly8VpXEzsZCQUmf-y2M,308
 feldera/rest/_httprequests.py,sha256=e22YbpzOzy7MGo7hk9MOU7ZRTj3F314grY0Ygr-_goI,6636
 feldera/rest/config.py,sha256=DYzZKngDEhouTEwqVFd-rDrBN9tWqsU07Jl_BTT4mXs,1008
 feldera/rest/errors.py,sha256=b4i2JjrbSmej7jdko_FL8UeXklLKenSipwMT80jowaM,1720
-feldera/rest/feldera_client.py,sha256=03XZ8gjgviy3uVvu_IWSoz61yVN8zeH1-ugNHdBMldc,24693
+feldera/rest/feldera_client.py,sha256=Vax97r4ZP5ZQ0MsRYFjAxmIXXlCaLNH7EiMN5SfxtJg,25057
 feldera/rest/feldera_config.py,sha256=1pnGbLFMSLvp7Qh_OlPLALSKCSHIktNWKvx6gYU00U4,1374
-feldera/rest/pipeline.py,sha256=a1lx-64SYak5mHX5yKElVijdfaAt5sDYVhVIXCJ97QQ,2839
+feldera/rest/pipeline.py,sha256=-dGGUdtHMABKrQEclaeuwGI_FOCrQOk6p2aCFV0FdU8,2890
 feldera/rest/sql_table.py,sha256=qrw-YwMzx5T81zDefNO1KOx7EyypFz1vPwGBzSUB7kc,652
 feldera/rest/sql_view.py,sha256=hN12mPM0mvwLCIPYywpb12s9Hd2Ws31IlTMXPriMisw,644
-feldera-0.96.0.dist-info/METADATA,sha256=_CW7wPtCfGj0Lv7uqIO68xM5aKA1KKsJ560CvDPMlqg,2587
-feldera-0.96.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-feldera-0.96.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
-feldera-0.96.0.dist-info/RECORD,,
+feldera-0.98.0.dist-info/METADATA,sha256=-fCOuWRvRjySTJpooV4_TDu0IqXSbDLGZN1Eay_boXE,2892
+feldera-0.98.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+feldera-0.98.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
+feldera-0.98.0.dist-info/RECORD,,